diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 000000000..bb314a3d0 --- /dev/null +++ b/.dockerignore @@ -0,0 +1,6 @@ +** +!.dockerignore +!Dockerfile +!target/ +!target/release/ +!target/release/nockchain diff --git a/.github/workflows/jam-consistency.yml b/.github/workflows/jam-consistency.yml new file mode 100644 index 000000000..a2cd00ec6 --- /dev/null +++ b/.github/workflows/jam-consistency.yml @@ -0,0 +1,53 @@ +name: "Check kernel jams are not stale relative to hoon sources" + +# hoonc output bytes are not reproducible across runs (HashMap iteration order +# leaks into the jam), so a byte-for-byte rebuild-and-diff cannot pass +# reliably. Instead, use git history: if any hoon/ file was modified in a +# commit newer than the most recent commit touching assets/*.jam, the jams +# are stale and must be rebuilt. This is what we actually want to catch — +# the trap that hid the duplicate +lock-merkle-proof-full arm and let the +# pre-Aletheia kernel ship. + +on: + push: + branches: + - master + - "nockpool/**" + pull_request: + branches: + - master + - "nockpool/**" + +jobs: + jam-staleness: + name: hoon vs assets git-log staleness + runs-on: ubuntu-latest + timeout-minutes: 5 + steps: + - uses: actions/checkout@v4 + with: + # Need full history so `git log -- assets/*.jam` can find the + # commit that last touched the jams. + fetch-depth: 0 + + - name: Fail if hoon/ has changed since assets/*.jam was last rebuilt + run: | + set -euo pipefail + LAST_JAM_COMMIT=$(git log -1 --format=%H -- 'assets/*.jam') + if [ -z "$LAST_JAM_COMMIT" ]; then + echo "::error::No commit found that touches assets/*.jam — repo state is unexpected." + exit 1 + fi + echo "Last commit touching assets/*.jam: $LAST_JAM_COMMIT" + STALE_HOON=$(git log --name-only --pretty=format: "${LAST_JAM_COMMIT}..HEAD" -- 'hoon/**' | sort -u | sed '/^$/d') + if [ -n "$STALE_HOON" ]; then + echo "::error::These hoon files were modified after assets/*.jam was last rebuilt:" + echo "$STALE_HOON" + echo + echo "::error::Rebuild the jams and commit them:" + echo " make install-hoonc" + echo " make build-hoon-all" + echo " git add assets/*.jam && git commit -m 'bump jams'" + exit 1 + fi + echo "OK: no hoon changes since last jam rebuild." diff --git a/.github/workflows/sync-to-nockchain.yml b/.github/workflows/sync-to-nockchain.yml index 8e3cd698a..83f206b0f 100644 --- a/.github/workflows/sync-to-nockchain.yml +++ b/.github/workflows/sync-to-nockchain.yml @@ -4,6 +4,19 @@ on: push: branches: - master + workflow_dispatch: + inputs: + force_sync: + description: Force sync even if master heads already match + required: false + default: false + type: boolean + schedule: + - cron: "*/30 * * * *" + +concurrency: + group: sync-to-nockchain-master + cancel-in-progress: false jobs: sync: @@ -14,7 +27,55 @@ jobs: fetch-depth: 0 persist-credentials: false + - name: Plan sync + id: plan + env: + EVENT_NAME: ${{ github.event_name }} + FORCE_SYNC: ${{ inputs.force_sync || 'false' }} + run: | + set -euo pipefail + staging_sha="$(git rev-parse HEAD)" + nockchain_sha="$(git ls-remote https://github.com/nockchain/nockchain.git refs/heads/master | awk '{print $1}')" + + if [ -z "$nockchain_sha" ]; then + echo "failed to resolve nockchain/nockchain master SHA" >&2 + exit 1 + fi + + should_sync=false + reason="already-in-sync" + + if [ "$EVENT_NAME" = "push" ]; then + should_sync=true + reason="push-to-master" + elif [ "$FORCE_SYNC" = "true" ]; then + should_sync=true + reason="manual-force" + elif [ "$staging_sha" != "$nockchain_sha" ]; then + should_sync=true + reason="heads-diverged" + fi + + echo "staging_sha=$staging_sha" >> "$GITHUB_OUTPUT" + echo "nockchain_sha=$nockchain_sha" >> "$GITHUB_OUTPUT" + echo "should_sync=$should_sync" >> "$GITHUB_OUTPUT" + echo "reason=$reason" >> "$GITHUB_OUTPUT" + + - name: Sync plan summary + run: | + echo "event=${{ github.event_name }}" + echo "reason=${{ steps.plan.outputs.reason }}" + echo "staging master=${{ steps.plan.outputs.staging_sha }}" + echo "nockchain master=${{ steps.plan.outputs.nockchain_sha }}" + echo "should_sync=${{ steps.plan.outputs.should_sync }}" + - name: Push to nockchain/nockchain + if: steps.plan.outputs.should_sync == 'true' run: | git remote add nockchain https://bitemyapp:${{ secrets.NOCKCHAIN_SYNC_TOKEN }}@github.com/nockchain/nockchain.git git push nockchain master:master + + - name: Skip sync + if: steps.plan.outputs.should_sync != 'true' + run: | + echo "Skipping sync: staging and nockchain master are already aligned" diff --git a/.gitignore b/.gitignore index cebe984f2..6313bcae9 100644 --- a/.gitignore +++ b/.gitignore @@ -8,3 +8,5 @@ test-follower target !crates/nockapp/test-jams keys.export +replay-checkpoints-51094 +nockchain-api-checkpoints-backup diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 99dbfb451..99b9c906c 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -51,10 +51,18 @@ docs-check: build-kernels: stage: kernels + # Kernel build + cargo-test gate the merge request into `open` (the + # required pre-merge check). Re-running them on the post-merge `open` + # push only delays sync-to-staging by ~2h, so skip them there. + rules: + - if: '$CI_COMMIT_BRANCH == "open"' + when: never + - when: on_success script: - | nix develop -c bash -c ' set -euo pipefail + export HOONC_FLAGS="--ephemeral" make update-hoonc make assets/dumb.jam make assets/miner.jam @@ -72,6 +80,11 @@ cargo-test: needs: - job: build-kernels artifacts: true + # See build-kernels: validated on the MR into `open`, skipped on push. + rules: + - if: '$CI_COMMIT_BRANCH == "open"' + when: never + - when: on_success script: - | nix develop -c bash -c ' @@ -85,8 +98,10 @@ cargo-test: sync-to-staging: stage: sync - needs: - - job: cargo-test + # Build/test validation already ran as a required check on the MR into + # `open`; the post-merge `open` pipeline only publishes, so don't gate on + # (or wait ~2h for) the build/test jobs. + needs: [] rules: - if: '$CI_COMMIT_BRANCH == "open" && $CI_PIPELINE_SOURCE == "push"' - if: '$CI_COMMIT_BRANCH == "open" && $FORCE_SYNC_TO_STAGING == "1"' diff --git a/Cargo.lock b/Cargo.lock index c0a9d84ae..d2c90c6d3 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -14,7 +14,7 @@ version = "0.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d122413f284cf2d62fb1b7db97e02edb8cda96d769b16e443a4f6195e35662b0" dependencies = [ - "crypto-common", + "crypto-common 0.1.7", "generic-array", ] @@ -26,7 +26,7 @@ checksum = "b169f7a6d4742236a0a00c541b845991d0ac43e546831af1249753ab4c3aa3a0" dependencies = [ "cfg-if", "cipher", - "cpufeatures", + "cpufeatures 0.2.17", ] [[package]] @@ -62,9 +62,9 @@ checksum = "683d7910e743518b0e34f1186f92494becacb047c7b6bf616c96772180fef923" [[package]] name = "alloy" -version = "1.7.3" +version = "1.8.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4973038846323e4e69a433916522195dce2947770076c03078fc21c80ea0f1c4" +checksum = "f4f7f312b60857fb4f8e0c723a17ae9c687c0e83a3c6b7940a30d4d5a26af091" dependencies = [ "alloy-consensus", "alloy-contract", @@ -87,9 +87,9 @@ dependencies = [ [[package]] name = "alloy-chains" -version = "0.2.30" +version = "0.2.34" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "90f374d3c6d729268bbe2d0e0ff992bb97898b2df756691a62ee1d5f0506bc39" +checksum = "84e0378e959aa6a885897522080a990e80eb317f1e9a222a604492ea50e13096" dependencies = [ "alloy-primitives", "num_enum", @@ -98,9 +98,9 @@ dependencies = [ [[package]] name = "alloy-consensus" -version = "1.7.3" +version = "1.8.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b0c0dc44157867da82c469c13186015b86abef209bf0e41625e4b68bac61d728" +checksum = "7f16daaf7e1f95f62c6c3bf8a3fc3d78b08ae9777810c0bb5e94966c7cd57ef0" dependencies = [ "alloy-eips", "alloy-primitives", @@ -115,8 +115,8 @@ dependencies = [ "either", "k256", "once_cell", - "rand 0.8.5", - "secp256k1", + "rand 0.8.6", + "secp256k1 0.30.0", "serde", "serde_json", "serde_with", @@ -125,9 +125,9 @@ dependencies = [ [[package]] name = "alloy-consensus-any" -version = "1.7.3" +version = "1.8.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ba4cdb42df3871cd6b346d6a938ec2ba69a9a0f49d1f82714bc5c48349268434" +checksum = "118998d9015332ab1b4720ae1f1e3009491966a0349938a1f43ff45a8a4c6299" dependencies = [ "alloy-consensus", "alloy-eips", @@ -139,9 +139,9 @@ dependencies = [ [[package]] name = "alloy-contract" -version = "1.7.3" +version = "1.8.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ca63b7125a981415898ffe2a2a696c83696c9c6bdb1671c8a912946bbd8e49e7" +checksum = "7ac9e0c34dc6bce643b182049cdfcca1b8ce7d9c260cbdd561f511873b7e26cd" dependencies = [ "alloy-consensus", "alloy-dyn-abi", @@ -158,13 +158,14 @@ dependencies = [ "futures-util", "serde_json", "thiserror 2.0.18", + "tracing", ] [[package]] name = "alloy-core" -version = "1.5.7" +version = "1.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "23e8604b0c092fabc80d075ede181c9b9e596249c70b99253082d7e689836529" +checksum = "62ddde5968de6044d67af107ad835bc0069a7ca245870b94c5958a7d8712b184" dependencies = [ "alloy-dyn-abi", "alloy-json-abi", @@ -175,9 +176,9 @@ dependencies = [ [[package]] name = "alloy-dyn-abi" -version = "1.5.7" +version = "1.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cc2db5c583aaef0255aa63a4fe827f826090142528bba48d1bf4119b62780cad" +checksum = "a475bb02d9cef2dbb99065c1664ab3fe1f9352e21d6d5ed3f02cdbfc06ed1abc" dependencies = [ "alloy-json-abi", "alloy-primitives", @@ -186,7 +187,7 @@ dependencies = [ "itoa", "serde", "serde_json", - "winnow", + "winnow 1.0.3", ] [[package]] @@ -230,21 +231,23 @@ dependencies = [ [[package]] name = "alloy-eip7928" -version = "0.3.2" +version = "0.3.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d3231de68d5d6e75332b7489cfcc7f4dfabeba94d990a10e4b923af0e6623540" +checksum = "6b827a6d7784fe3eb3489d40699407a4cdcce74271421a01bdffe60cf573bb16" dependencies = [ "alloy-primitives", "alloy-rlp", "borsh", + "once_cell", "serde", + "thiserror 2.0.18", ] [[package]] name = "alloy-eips" -version = "1.7.3" +version = "1.8.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b9f7ef09f21bd1e9cb8a686f168cb4a206646804567f0889eadb8dcc4c9288c8" +checksum = "e6ef28c9fdad22d4eec52d894f5f2673a0895f1e5ef196734568e68c0f6caca8" dependencies = [ "alloy-eip2124", "alloy-eip2930", @@ -263,14 +266,13 @@ dependencies = [ "serde", "serde_with", "sha2", - "thiserror 2.0.18", ] [[package]] name = "alloy-genesis" -version = "1.7.3" +version = "1.8.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7c9cf3b99f46615fbf7dc1add0c96553abb7bf88fc9ec70dfbe7ad0b47ba7fe8" +checksum = "bbf9480307b09d22876efb67d30cadd9013134c21f3a17ec9f93fd7536d38024" dependencies = [ "alloy-eips", "alloy-primitives", @@ -283,9 +285,9 @@ dependencies = [ [[package]] name = "alloy-json-abi" -version = "1.5.7" +version = "1.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e9dbe713da0c737d9e5e387b0ba790eb98b14dd207fe53eef50e19a5a8ec3dac" +checksum = "7c36c9d7f9021601b04bfef14a4b64849f6d73116a4e91e071d7fbfe10247901" dependencies = [ "alloy-primitives", "alloy-sol-type-parser", @@ -295,9 +297,9 @@ dependencies = [ [[package]] name = "alloy-json-rpc" -version = "1.7.3" +version = "1.8.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ff42cd777eea61f370c0b10f2648a1c81e0b783066cd7269228aa993afd487f7" +checksum = "422d110f1c40f1f8d0e5562b0b649c35f345fccb7093d9f02729943dcd1eef71" dependencies = [ "alloy-primitives", "alloy-sol-types", @@ -310,9 +312,9 @@ dependencies = [ [[package]] name = "alloy-network" -version = "1.7.3" +version = "1.8.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8cbca04f9b410fdc51aaaf88433cbac761213905a65fe832058bcf6690585762" +checksum = "7197a66d94c4de1591cdc16a9bcea5f8cccd0da81b865b49aef97b1b4016e0fa" dependencies = [ "alloy-consensus", "alloy-consensus-any", @@ -336,9 +338,9 @@ dependencies = [ [[package]] name = "alloy-network-primitives" -version = "1.7.3" +version = "1.8.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "42d6d15e069a8b11f56bef2eccbad2a873c6dd4d4c81d04dda29710f5ea52f04" +checksum = "eb82711d59a43fdfd79727c99f270b974c784ec4eb5728a0d0d22f26716c87ef" dependencies = [ "alloy-consensus", "alloy-eips", @@ -349,9 +351,9 @@ dependencies = [ [[package]] name = "alloy-primitives" -version = "1.5.7" +version = "1.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "de3b431b4e72cd8bd0ec7a50b4be18e73dab74de0dba180eef171055e5d5926e" +checksum = "4885c1409b6936c4898e646ef58baf6ec54edaf6d8179f79df805a7b85b7cf3e" dependencies = [ "alloy-rlp", "bytes", @@ -359,26 +361,27 @@ dependencies = [ "const-hex", "derive_more", "foldhash 0.2.0", - "hashbrown 0.16.1", - "indexmap 2.13.0", + "hashbrown 0.17.1", + "indexmap 2.14.0", "itoa", "k256", "keccak-asm", "paste", "proptest", - "rand 0.9.2", + "rand 0.9.4", "rapidhash", "ruint", - "rustc-hash 2.1.1", + "rustc-hash 2.1.2", + "secp256k1 0.31.1", "serde", "sha3", ] [[package]] name = "alloy-provider" -version = "1.7.3" +version = "1.8.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d181c8cc7cf4805d7e589bf4074d56d55064fa1a979f005a45a62b047616d870" +checksum = "bf6b18b929ef1d078b834c3631e9c925177f3b23ddc6fa08a722d13047205876" dependencies = [ "alloy-chains", "alloy-consensus", @@ -389,7 +392,9 @@ dependencies = [ "alloy-primitives", "alloy-pubsub", "alloy-rpc-client", + "alloy-rpc-types-debug", "alloy-rpc-types-eth", + "alloy-rpc-types-trace", "alloy-signer", "alloy-sol-types", "alloy-transport", @@ -402,10 +407,10 @@ dependencies = [ "either", "futures", "futures-utils-wasm", - "lru 0.16.3", + "lru 0.16.4", "parking_lot", "pin-project", - "reqwest", + "reqwest 0.13.3", "serde", "serde_json", "thiserror 2.0.18", @@ -417,9 +422,9 @@ dependencies = [ [[package]] name = "alloy-pubsub" -version = "1.7.3" +version = "1.8.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e8bd82953194dec221aa4cbbbb0b1e2df46066fe9d0333ac25b43a311e122d13" +checksum = "5ad54073131e7292d4e03e1aa2287730f737280eb160d8b579fb31939f558c11" dependencies = [ "alloy-json-rpc", "alloy-primitives", @@ -439,9 +444,9 @@ dependencies = [ [[package]] name = "alloy-rlp" -version = "0.3.13" +version = "0.3.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e93e50f64a77ad9c5470bf2ad0ca02f228da70c792a8f06634801e202579f35e" +checksum = "dc90b1e703d3c03f4ff7f48e82dd0bc1c8211ab7d079cd836a06fcfeb06651cb" dependencies = [ "alloy-rlp-derive", "arrayvec", @@ -450,9 +455,9 @@ dependencies = [ [[package]] name = "alloy-rlp-derive" -version = "0.3.13" +version = "0.3.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ce8849c74c9ca0f5a03da1c865e3eb6f768df816e67dd3721a398a8a7e398011" +checksum = "f36834a5c0a2fa56e171bf256c34d70fca07d0c0031583edea1c4946b7889c9e" dependencies = [ "proc-macro2", "quote", @@ -461,9 +466,9 @@ dependencies = [ [[package]] name = "alloy-rpc-client" -version = "1.7.3" +version = "1.8.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f2792758a93ae32a32e9047c843d536e1448044f78422d71bf7d7c05149e103f" +checksum = "94fcc9604042ca80bd37aa5e232ea1cd851f337e31e2babbbb345bc0b1c30de3" dependencies = [ "alloy-json-rpc", "alloy-primitives", @@ -473,7 +478,7 @@ dependencies = [ "alloy-transport-ws", "futures", "pin-project", - "reqwest", + "reqwest 0.13.3", "serde", "serde_json", "tokio", @@ -486,9 +491,9 @@ dependencies = [ [[package]] name = "alloy-rpc-types" -version = "1.7.3" +version = "1.8.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7bdcbf9dfd5eea8bfeb078b1d906da8cd3a39c4d4dbe7a628025648e323611f6" +checksum = "4faad925d3a669ffc15f43b3deec7fbdf2adeb28a4d6f9cf4bc661698c0f8f4b" dependencies = [ "alloy-primitives", "alloy-rpc-types-eth", @@ -498,20 +503,32 @@ dependencies = [ [[package]] name = "alloy-rpc-types-any" -version = "1.7.3" +version = "1.8.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dd720b63f82b457610f2eaaf1f32edf44efffe03ae25d537632e7d23e7929e1a" +checksum = "3823026d1ed239a40f12364fac50726c8daf1b6ab8077a97212c5123910429ed" dependencies = [ "alloy-consensus-any", "alloy-rpc-types-eth", "alloy-serde", ] +[[package]] +name = "alloy-rpc-types-debug" +version = "1.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2145138f3214928f08cd13da3cb51ef7482b5920d8ac5a02ecd4e38d1a8f6d1e" +dependencies = [ + "alloy-primitives", + "derive_more", + "serde", + "serde_with", +] + [[package]] name = "alloy-rpc-types-engine" -version = "1.7.3" +version = "1.8.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e4ac61f03f1edabccde1c687b5b25fff28f183afee64eaa2e767def3929e4457" +checksum = "bb9b97b6e7965679ad22df297dda809b11cebc13405c1b537e5cffecc95834fa" dependencies = [ "alloy-consensus", "alloy-eips", @@ -521,16 +538,16 @@ dependencies = [ "derive_more", "ethereum_ssz", "ethereum_ssz_derive", - "rand 0.8.5", + "rand 0.8.6", "serde", "strum 0.27.2", ] [[package]] name = "alloy-rpc-types-eth" -version = "1.7.3" +version = "1.8.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9b2dc411f13092f237d2bf6918caf80977fc2f51485f9b90cb2a2f956912c8c9" +checksum = "59c095f92c4e1ff4981d89e9aa02d5f98c762a1980ab66bec49c44be11349da2" dependencies = [ "alloy-consensus", "alloy-consensus-any", @@ -547,11 +564,25 @@ dependencies = [ "thiserror 2.0.18", ] +[[package]] +name = "alloy-rpc-types-trace" +version = "1.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2e5a4d010f86cd4e01e5205ec273911e538e1738e76d8bafe9ecd245910ea5a3" +dependencies = [ + "alloy-primitives", + "alloy-rpc-types-eth", + "alloy-serde", + "serde", + "serde_json", + "thiserror 2.0.18", +] + [[package]] name = "alloy-serde" -version = "1.7.3" +version = "1.8.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e2ce1e0dbf7720eee747700e300c99aac01b1a95bb93f493a01e78ee28bb1a37" +checksum = "11ece63b89294b8614ab3f483560c08d016930f842bf36da56bf0b764a15c11e" dependencies = [ "alloy-primitives", "serde", @@ -560,9 +591,9 @@ dependencies = [ [[package]] name = "alloy-signer" -version = "1.7.3" +version = "1.8.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2425c6f314522c78e8198979c8cbf6769362be4da381d4152ea8eefce383535d" +checksum = "43f447aefab0f1c0649f71edc33f590992d4e122bc35fb9cdbbf67d4421ace85" dependencies = [ "alloy-primitives", "async-trait", @@ -575,9 +606,9 @@ dependencies = [ [[package]] name = "alloy-signer-local" -version = "1.7.3" +version = "1.8.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c3ecb71ee53d8d9c3fa7bac17542c8116ebc7a9726c91b1bf333ec3d04f5a789" +checksum = "f721f4bf2e4812e5505aaf5de16ef3065a8e26b9139ac885862d00b5a55a659a" dependencies = [ "alloy-consensus", "alloy-network", @@ -585,15 +616,15 @@ dependencies = [ "alloy-signer", "async-trait", "k256", - "rand 0.8.5", + "rand 0.8.6", "thiserror 2.0.18", ] [[package]] name = "alloy-sol-macro" -version = "1.5.7" +version = "1.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ab81bab693da9bb79f7a95b64b394718259fdd7e41dceeced4cad57cb71c4f6a" +checksum = "840128ed2b2971d6d4668a553fe403a82683d3acc646c73e75887e7157408033" dependencies = [ "alloy-sol-macro-expander", "alloy-sol-macro-input", @@ -605,15 +636,15 @@ dependencies = [ [[package]] name = "alloy-sol-macro-expander" -version = "1.5.7" +version = "1.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "489f1620bb7e2483fb5819ed01ab6edc1d2f93939dce35a5695085a1afd1d699" +checksum = "63ec265e5d65d725175f6ca7711c970824c90ef9c0d1f1973711d4150ee612dd" dependencies = [ "alloy-json-abi", "alloy-sol-macro-input", "const-hex", "heck", - "indexmap 2.13.0", + "indexmap 2.14.0", "proc-macro-error2", "proc-macro2", "quote", @@ -624,9 +655,9 @@ dependencies = [ [[package]] name = "alloy-sol-macro-input" -version = "1.5.7" +version = "1.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "56cef806ad22d4392c5fc83cf8f2089f988eb99c7067b4e0c6f1971fc1cca318" +checksum = "89bf01077f18650876cfa682eb1f949967b5cde03f1a51c955c469d2c9b4aa67" dependencies = [ "alloy-json-abi", "const-hex", @@ -642,19 +673,19 @@ dependencies = [ [[package]] name = "alloy-sol-type-parser" -version = "1.5.7" +version = "1.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a6df77fea9d6a2a75c0ef8d2acbdfd92286cc599983d3175ccdc170d3433d249" +checksum = "857b470ecdd2ed38beaf82ad1a38c516a8ff75266750f38b9eeed001d575241b" dependencies = [ "serde", - "winnow", + "winnow 1.0.3", ] [[package]] name = "alloy-sol-types" -version = "1.5.7" +version = "1.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "64612d29379782a5dde6f4b6570d9c756d734d760c0c94c254d361e678a6591f" +checksum = "384cf252de0db2dec52821eac037a7f57e2aa33fe5b900ce6fe39973402341f1" dependencies = [ "alloy-json-abi", "alloy-primitives", @@ -664,9 +695,9 @@ dependencies = [ [[package]] name = "alloy-transport" -version = "1.7.3" +version = "1.8.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fa186e560d523d196580c48bf00f1bf62e63041f28ecf276acc22f8b27bb9f53" +checksum = "8098f965442a9feb620965ba4b4be5e2b320f4ec5a3fff6bfa9e1ff7ef42bed1" dependencies = [ "alloy-json-rpc", "auto_impl", @@ -687,14 +718,14 @@ dependencies = [ [[package]] name = "alloy-transport-http" -version = "1.7.3" +version = "1.8.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "aa501ad58dd20acddbfebc65b52e60f05ebf97c52fa40d1b35e91f5e2da0ad0e" +checksum = "e8597d36d546e1dab822345ad563243ec3920e199322cb554ce56c8ef1a1e2e7" dependencies = [ "alloy-json-rpc", "alloy-transport", "itertools 0.14.0", - "reqwest", + "reqwest 0.13.3", "serde_json", "tower", "tracing", @@ -703,30 +734,31 @@ dependencies = [ [[package]] name = "alloy-transport-ws" -version = "1.7.3" +version = "1.8.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b9f00445db69d63298e2b00a0ea1d859f00e6424a3144ffc5eba9c31da995e16" +checksum = "ec3ab7a72b180992881acc112628b7668337a19ce15293ee974600ea7b693691" dependencies = [ "alloy-pubsub", "alloy-transport", "futures", "http", + "rustls", "serde_json", "tokio", "tokio-tungstenite", "tracing", + "url", "ws_stream_wasm", ] [[package]] name = "alloy-trie" -version = "0.9.4" +version = "0.9.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4d7fd448ab0a017de542de1dcca7a58e7019fe0e7a34ed3f9543ebddf6aceffa" +checksum = "3f14b5d9b2c2173980202c6ff470d96e7c5e202c65a9f67884ad565226df7fbb" dependencies = [ "alloy-primitives", "alloy-rlp", - "arrayvec", "derive_more", "nybbles", "serde", @@ -737,11 +769,11 @@ dependencies = [ [[package]] name = "alloy-tx-macros" -version = "1.7.3" +version = "1.8.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6fa0c53e8c1e1ef4d01066b01c737fb62fc9397ab52c6e7bb5669f97d281b9bc" +checksum = "d69722eddcdf1ce096c3ab66cf8116999363f734eb36fe94a148f4f71c85da84" dependencies = [ - "darling 0.21.3", + "darling 0.23.0", "proc-macro2", "quote", "syn 2.0.117", @@ -764,9 +796,9 @@ checksum = "4b46cbb362ab8752921c97e041f5e366ee6297bd428a31275b9fcf1e380f7299" [[package]] name = "anstream" -version = "0.6.21" +version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "43d5b281e737544384e969a5ccad3f1cdd24b48086a0fc1b2a5262a26b8f4f4a" +checksum = "824a212faf96e9acacdbd09febd34438f8f711fb84e09a8916013cd7815ca28d" dependencies = [ "anstyle", "anstyle-parse", @@ -779,15 +811,15 @@ dependencies = [ [[package]] name = "anstyle" -version = "1.0.13" +version = "1.0.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5192cca8006f1fd4f7237516f40fa183bb07f8fbdfedaa0036de5ea9b0b45e78" +checksum = "940b3a0ca603d1eade50a4846a2afffd5ef57a9feac2c0e2ec2e14f9ead76000" [[package]] name = "anstyle-parse" -version = "0.2.7" +version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4e7644824f0aa2c7b9384579234ef10eb7efb6a0deb83f9630a49594dd9c15c2" +checksum = "52ce7f38b242319f7cabaa6813055467063ecdc9d355bbb4ce0c68908cd8130e" dependencies = [ "utf8parse", ] @@ -818,6 +850,22 @@ version = "1.0.102" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7f202df86484c868dbad7eaa557ef785d5c66295e41b460ef922eca0723b842c" +[[package]] +name = "approver" +version = "0.1.0" +dependencies = [ + "bytes", + "clap", + "kernels", + "nockapp", + "nockapp-grpc", + "nockvm", + "tokio", + "tracing", + "tracing-subscriber", + "zkvm-jetpack", +] + [[package]] name = "arboard" version = "3.6.1" @@ -840,9 +888,9 @@ dependencies = [ [[package]] name = "arc-swap" -version = "1.8.2" +version = "1.9.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f9f3647c145568cec02c42054e07bdf9a5a698e15b466fb2341bfc393cd24aa5" +checksum = "6a3a1fd6f75306b68087b831f025c712524bcb19aad54e557b1129cfa0a2b207" dependencies = [ "rustversion", ] @@ -855,7 +903,7 @@ checksum = "3c3610892ee6e0cbce8ae2700349fcf8f98adb0dbfbee85aec3c9179d29cc072" dependencies = [ "base64ct", "blake2", - "cpufeatures", + "cpufeatures 0.2.17", "password-hash", ] @@ -1025,7 +1073,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1df2c09229cbc5a028b1d70e00fdb2acee28b1055dfb5ca73eea49c5a25c4e7c" dependencies = [ "num-traits", - "rand 0.8.5", + "rand 0.8.6", ] [[package]] @@ -1035,7 +1083,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "94893f1e0c6eeab764ade8dc4c0db24caf4fe7cbbaafc0eba0a9030f447b5185" dependencies = [ "num-traits", - "rand 0.8.5", + "rand 0.8.6", ] [[package]] @@ -1045,7 +1093,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "246a225cc6131e9ee4f24619af0f19d67761fff15d7ccc22e42b80846e69449a" dependencies = [ "num-traits", - "rand 0.8.5", + "rand 0.8.6", ] [[package]] @@ -1065,15 +1113,12 @@ name = "arrayvec" version = "0.7.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7c02d123df017efcdfbd739ef81735b36c5ba83ec3c59c80a9d7ecc718f92e50" -dependencies = [ - "serde", -] [[package]] name = "asn1-rs" -version = "0.7.1" +version = "0.7.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "56624a96882bb8c26d61312ae18cb45868e5a9992ea73c58e45c3101e56a1e60" +checksum = "b7f43a50ac4fdca5df8e885c21b835997f0a1cdee65494a6847694a98652d9d8" dependencies = [ "asn1-rs-derive", "asn1-rs-impl", @@ -1110,9 +1155,9 @@ dependencies = [ [[package]] name = "assert_cmd" -version = "2.1.2" +version = "2.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9c5bcfa8749ac45dd12cb11055aeeb6b27a3895560d60d71e3c23bf979e60514" +checksum = "2aa3a22042e45de04255c7bf3626e239f450200fd0493c1e382263544b20aea6" dependencies = [ "anstyle", "bstr", @@ -1255,9 +1300,9 @@ dependencies = [ [[package]] name = "aws-lc-rs" -version = "1.16.0" +version = "1.17.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d9a7b350e3bb1767102698302bc37256cbd48422809984b98d292c40e2579aa9" +checksum = "5ec2f1fc3ec205783a5da9a7e6c1509cc69dedf09a1949e412c1e18469326d00" dependencies = [ "aws-lc-sys", "zeroize", @@ -1265,9 +1310,9 @@ dependencies = [ [[package]] name = "aws-lc-sys" -version = "0.37.1" +version = "0.41.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b092fe214090261288111db7a2b2c2118e5a7f30dc2569f1732c4069a6840549" +checksum = "1a2f9779ce85b93ab6170dd940ad0169b5766ff848247aff13bb788b832fe3f4" dependencies = [ "cc", "cmake", @@ -1277,9 +1322,9 @@ dependencies = [ [[package]] name = "axum" -version = "0.8.8" +version = "0.8.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8b52af3cb4058c895d37317bb27508dccc8e5f2d39454016b297bf4a400597b8" +checksum = "31b698c5f9a010f6573133b09e0de5408834d0c82f8d7475a89fc1867a71cd90" dependencies = [ "axum-core", "bytes", @@ -1426,7 +1471,7 @@ version = "0.69.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "271383c67ccabffb7381723dea0672a673f292304fcb45c01cc648c7a8d58088" dependencies = [ - "bitflags 2.11.0", + "bitflags 2.11.1", "cexpr", "clang-sys", "itertools 0.12.1", @@ -1449,7 +1494,7 @@ version = "0.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "08807e080ed7f9d5433fa9b275196cfc35414f66a0c79d864dc51a0d825231a3" dependencies = [ - "bit-vec", + "bit-vec 0.8.0", ] [[package]] @@ -1458,6 +1503,15 @@ version = "0.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5e764a1d40d510daf35e07be9eb06e75770908c27d411ee6c92109c9840eaaf7" +[[package]] +name = "bit-vec" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b71798fca2c1fe1086445a7258a4bc81e6e49dcd24c8d0dd9a1e57395b603f51" +dependencies = [ + "serde", +] + [[package]] name = "bitcoin-io" version = "0.1.4" @@ -1482,9 +1536,9 @@ checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" [[package]] name = "bitflags" -version = "2.11.0" +version = "2.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "843867be96c8daad0d758b57df9392b6d8d271134fce549de6ce169ff98a92af" +checksum = "c4512299f36f043ab09a583e57bceb5a5aab7a73db1805848e8fef3c9e8c78b3" dependencies = [ "serde_core", ] @@ -1512,16 +1566,16 @@ dependencies = [ [[package]] name = "blake3" -version = "1.8.3" +version = "1.8.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2468ef7d57b3fb7e16b576e8377cdbde2320c60e1491e961d11da40fc4f02a2d" +checksum = "0aa83c34e62843d924f905e0f5c866eb1dd6545fc4d719e803d9ba6030371fce" dependencies = [ "arrayref", "arrayvec", "cc", "cfg-if", "constant_time_eq", - "cpufeatures", + "cpufeatures 0.3.0", "serde", ] @@ -1534,6 +1588,15 @@ dependencies = [ "generic-array", ] +[[package]] +name = "block-buffer" +version = "0.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cdd35008169921d80bc60d3d0ab416eecb028c4cd653352907921d95084790be" +dependencies = [ + "hybrid-array", +] + [[package]] name = "blst" version = "0.3.16" @@ -1548,19 +1611,20 @@ dependencies = [ [[package]] name = "borsh" -version = "1.6.0" +version = "1.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d1da5ab77c1437701eeff7c88d968729e7766172279eab0676857b3d63af7a6f" +checksum = "cfd1e3f8955a5d7de9fab72fc8373fade9fb8a703968cb200ae3dc6cf08e185a" dependencies = [ "borsh-derive", + "bytes", "cfg_aliases", ] [[package]] name = "borsh-derive" -version = "1.6.0" +version = "1.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0686c856aa6aac0c4498f936d7d6a02df690f614c03e4d906d1018062b5c5e2c" +checksum = "bfcfdc083699101d5a7965e49925975f2f55060f94f9a05e7187be95d530ca59" dependencies = [ "once_cell", "proc-macro-crate", @@ -1615,8 +1679,8 @@ dependencies = [ "tikv-jemallocator", "tiny-keccak", "tokio", - "toml", - "tonic 0.14.5", + "toml 0.9.12+spec-1.1.0", + "tonic 0.14.6", "tonic-prost", "tonic-prost-build", "tonic-reflection", @@ -1710,9 +1774,9 @@ dependencies = [ [[package]] name = "c-kzg" -version = "2.1.6" +version = "2.1.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1a0f582957c24870b7bfd12bf562c40b4734b533cafbaf8ded31d6d85f462c01" +checksum = "6648ed1e4ea8e8a1a4a2c78e1cda29a3fd500bc622899c340d8525ea9a76b24a" dependencies = [ "blst", "cc", @@ -1723,6 +1787,38 @@ dependencies = [ "serde", ] +[[package]] +name = "camino" +version = "1.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e629a66d692cb9ff1a1c664e41771b3dcaf961985a9774c0eb0bd1b51cf60a48" +dependencies = [ + "serde_core", +] + +[[package]] +name = "cargo-platform" +version = "0.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e35af189006b9c0f00a064685c727031e3ed2d8020f7ba284d78cc2671bd36ea" +dependencies = [ + "serde", +] + +[[package]] +name = "cargo_metadata" +version = "0.18.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2d886547e41f740c616ae73108f6eb70afe6d940c7bc697cb30f13daec073037" +dependencies = [ + "camino", + "cargo-platform", + "semver 1.0.28", + "serde", + "serde_json", + "thiserror 1.0.69", +] + [[package]] name = "cassowary" version = "0.3.0" @@ -1764,9 +1860,9 @@ dependencies = [ [[package]] name = "cc" -version = "1.2.56" +version = "1.2.62" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "aebf35691d1bfb0ac386a69bac2fde4dd276fb618cf8bf4f5318fe285e821bb2" +checksum = "a1dce859f0832a7d088c4f1119888ab94ef4b5d6795d1ce05afb7fe159d79f98" dependencies = [ "find-msvc-tools", "jobserver", @@ -1861,7 +1957,7 @@ version = "0.4.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "773f3b9af64447d2ce9850330c473515014aa235e6a783b02db81ff39e4a3dad" dependencies = [ - "crypto-common", + "crypto-common 0.1.7", "inout", ] @@ -1878,9 +1974,9 @@ dependencies = [ [[package]] name = "clap" -version = "4.5.60" +version = "4.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2797f34da339ce31042b27d23607e051786132987f595b02ba4f6a6dffb7030a" +checksum = "1ddb117e43bbf7dacf0a4190fef4d345b9bad68dfc649cb349e7d17d28428e51" dependencies = [ "clap_builder", "clap_derive", @@ -1888,9 +1984,9 @@ dependencies = [ [[package]] name = "clap_builder" -version = "4.5.60" +version = "4.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "24a241312cea5059b13574bb9b3861cabf758b879c15190b37b6d6fd63ab6876" +checksum = "714a53001bf66416adb0e2ef5ac857140e7dc3a0c48fb28b2f10762fc4b5069f" dependencies = [ "anstream", "anstyle", @@ -1900,9 +1996,9 @@ dependencies = [ [[package]] name = "clap_derive" -version = "4.5.55" +version = "4.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a92793da1a46a5f2a02a6f4c46c6496b28c43638adea8306fcb0caa1634f24e5" +checksum = "f2ce8604710f6733aa641a2b3731eaa1e8b3d9973d5e3565da11800813f997a9" dependencies = [ "heck", "proc-macro2", @@ -1912,9 +2008,9 @@ dependencies = [ [[package]] name = "clap_lex" -version = "1.0.0" +version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3a822ea5bc7590f9d40f1ba12c0dc3c2760f3482c6984db1573ad11031420831" +checksum = "c8d4a3bb8b1e0c1050499d1815f5ab16d04f0959b233085fb31653fbfc9d98f9" [[package]] name = "clipboard-win" @@ -1938,18 +2034,18 @@ dependencies = [ [[package]] name = "cmake" -version = "0.1.57" +version = "0.1.58" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "75443c44cd6b379beb8c5b45d85d0773baf31cce901fe7bb252f4eff3008ef7d" +checksum = "c0f78a02292a74a88ac736019ab962ece0bc380e3f977bf72e376c5d78ff0678" dependencies = [ "cc", ] [[package]] name = "colorchoice" -version = "1.0.4" +version = "1.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b05b61dc5112cbb17e4b6cd61790d9845d13888356391624cbe7e41efeac1e75" +checksum = "1d07550c9036bf2ae0c684c4297d503f838287c83c53686d05370d0e139ae570" [[package]] name = "colored" @@ -1961,6 +2057,16 @@ dependencies = [ "windows-sys 0.59.0", ] +[[package]] +name = "combine" +version = "4.6.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ba5a308b75df32fe02788e748662718f03fde005016435c444eea572398219fd" +dependencies = [ + "bytes", + "memchr", +] + [[package]] name = "compact_str" version = "0.8.1" @@ -1986,9 +2092,9 @@ dependencies = [ [[package]] name = "config" -version = "0.15.19" +version = "0.15.23" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b30fa8254caad766fc03cb0ccae691e14bf3bd72bfff27f72802ce729551b3d6" +checksum = "f316c6237b2d38be61949ecd15268a4c6ca32570079394a2444d9ce2c72a72d8" dependencies = [ "async-trait", "convert_case 0.6.0", @@ -1999,19 +2105,19 @@ dependencies = [ "serde-untagged", "serde_core", "serde_json", - "toml", - "winnow", + "toml 1.1.2+spec-1.1.0", + "winnow 1.0.3", "yaml-rust2", ] [[package]] name = "const-hex" -version = "1.18.0" +version = "1.19.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "af9a108e542ddf1de36743a6126e94d6659dccda38fc8a77e80b915102ac784a" +checksum = "20d9a563d167a9cce0f94153382b33cb6eded6dfabff03c69ad65a28ea1514e0" dependencies = [ "cfg-if", - "cpufeatures", + "cpufeatures 0.2.17", "proptest", "serde_core", ] @@ -2050,11 +2156,12 @@ checksum = "2f421161cb492475f1661ddc9815a745a1c894592070661180fdec3d4872e9c3" [[package]] name = "const_format" -version = "0.2.35" +version = "0.2.36" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7faa7469a93a566e9ccc1c73fe783b4a65c274c5ace346038dca9c39fe0030ad" +checksum = "4481a617ad9a412be3b97c5d403fef8ed023103368908b9c50af598ff467cc1e" dependencies = [ "const_format_proc_macros", + "konst", ] [[package]] @@ -2128,19 +2235,19 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "773648b94d0e5d620f64f280777445740e61fe701025087ec8b57f45c791888b" [[package]] -name = "core2" -version = "0.4.0" +name = "cpufeatures" +version = "0.2.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b49ba7ef1ad6107f8824dbe97de947cbaac53c44e7f9756a1fba0d37c1eec505" +checksum = "59ed5838eebb26a2bb2e58f6d5b5316989ae9d08bab10e0e6d103e656d1b0280" dependencies = [ - "memchr", + "libc", ] [[package]] name = "cpufeatures" -version = "0.2.17" +version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "59ed5838eebb26a2bb2e58f6d5b5316989ae9d08bab10e0e6d103e656d1b0280" +checksum = "8b2a41393f66f16b0823bb79094d54ac5fbd34ab292ddafb9a0456ac9f87d201" dependencies = [ "libc", ] @@ -2156,9 +2263,9 @@ dependencies = [ [[package]] name = "crc-catalog" -version = "2.4.0" +version = "2.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "19d374276b40fb8bbdee95aef7c7fa6b5316ec764510eb64b8dd0e2ed0d7e7f5" +checksum = "217698eaf96b4a3f0bc4f3662aaa55bdf913cd54d7204591faa790070c6d0853" [[package]] name = "crc32fast" @@ -2291,9 +2398,9 @@ version = "0.28.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "829d955a0bb380ef178a640b91779e3987da38c9aea133b20614cfed8cdea9c6" dependencies = [ - "bitflags 2.11.0", + "bitflags 2.11.1", "crossterm_winapi", - "mio 1.1.1", + "mio 1.2.0", "parking_lot", "rustix 0.38.44", "signal-hook", @@ -2307,11 +2414,11 @@ version = "0.29.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d8b9f2e4c67f833b660cdb0a3523065869fb35570177239812ed4c905aeff87b" dependencies = [ - "bitflags 2.11.0", + "bitflags 2.11.1", "crossterm_winapi", "derive_more", "document-features", - "mio 1.1.1", + "mio 1.2.0", "parking_lot", "rustix 1.1.4", "signal-hook", @@ -2357,6 +2464,15 @@ dependencies = [ "typenum", ] +[[package]] +name = "crypto-common" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ce6e4c961d6cd6c9a86db418387425e8bdeaf05b3c8bc1411e6dca4c252f1453" +dependencies = [ + "hybrid-array", +] + [[package]] name = "ctr" version = "0.9.2" @@ -2373,7 +2489,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "97fb8b7c4503de7d6ae7b42ab72a5a59857b4c937ec27a3d4539dba95b5ab2be" dependencies = [ "cfg-if", - "cpufeatures", + "cpufeatures 0.2.17", "curve25519-dalek-derive", "digest 0.10.7", "fiat-crypto", @@ -2447,7 +2563,6 @@ dependencies = [ "ident_case", "proc-macro2", "quote", - "serde", "strsim", "syn 2.0.117", ] @@ -2461,6 +2576,7 @@ dependencies = [ "ident_case", "proc-macro2", "quote", + "serde", "strsim", "syn 2.0.117", ] @@ -2500,9 +2616,9 @@ dependencies = [ [[package]] name = "dashmap" -version = "6.1.0" +version = "6.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5041cc499144891f3790297212f32a74fb938e5136a14943f338ef9e0ae276cf" +checksum = "e6361d5c062261c78a176addb82d4c821ae42bed6089de0e12603cd25de2059c" dependencies = [ "cfg-if", "crossbeam-utils", @@ -2514,15 +2630,15 @@ dependencies = [ [[package]] name = "data-encoding" -version = "2.10.0" +version = "2.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d7a1e2f27636f116493b8b860f5546edb47c8d8f8ea73e1d2a20be88e28d1fea" +checksum = "a4ae5f15dda3c708c0ade84bfee31ccab44a3da4f88015ed22f63732abe300c8" [[package]] name = "data-encoding-macro" -version = "0.1.19" +version = "0.1.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8142a83c17aa9461d637e649271eae18bf2edd00e91f2e105df36c3c16355bdb" +checksum = "3259c913752a86488b501ed8680446a5ed2d5aeac6e596cb23ba3800768ea32c" dependencies = [ "data-encoding", "data-encoding-macro-internal", @@ -2530,9 +2646,9 @@ dependencies = [ [[package]] name = "data-encoding-macro-internal" -version = "0.1.17" +version = "0.1.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7ab67060fc6b8ef687992d439ca0fa36e7ed17e9a0b16b25b601e8757df720de" +checksum = "ccc2776f0c61eca1ca32528f85548abd1a4be8fb53d1b21c013e4f18da1e7090" dependencies = [ "data-encoding", "syn 2.0.117", @@ -2689,9 +2805,9 @@ dependencies = [ [[package]] name = "diesel" -version = "2.3.6" +version = "2.3.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d9b6c2fc184a6fb6ebcf5f9a5e3bbfa84d8fd268cdfcce4ed508979a6259494d" +checksum = "9940fb8467a0a06312218ed384185cb8536aa10d8ec017d0ce7fad2c1bd882d5" dependencies = [ "diesel_derives", "downcast-rs", @@ -2702,9 +2818,9 @@ dependencies = [ [[package]] name = "diesel_derives" -version = "2.3.7" +version = "2.3.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "47618bf0fac06bb670c036e48404c26a865e6a71af4114dfd97dfe89936e404e" +checksum = "d1817b7f4279b947fc4cafddec12b0e5f8727141706561ce3ac94a60bddd1cf5" dependencies = [ "diesel_table_macro_syntax", "dsl_auto_type", @@ -2713,6 +2829,17 @@ dependencies = [ "syn 2.0.117", ] +[[package]] +name = "diesel_migrations" +version = "2.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "28d0f4a98124ba6d4ca75da535f65984badec16a003b6e2f94a01e31a79490b8" +dependencies = [ + "diesel", + "migrations_internals", + "migrations_macros", +] + [[package]] name = "diesel_table_macro_syntax" version = "0.3.0" @@ -2743,12 +2870,22 @@ version = "0.10.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292" dependencies = [ - "block-buffer", + "block-buffer 0.10.4", "const-oid", - "crypto-common", + "crypto-common 0.1.7", "subtle", ] +[[package]] +name = "digest" +version = "0.11.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f1dd6dbb5841937940781866fa1281a1ff7bd3bf827091440879f9994983d5c2" +dependencies = [ + "block-buffer 0.12.0", + "crypto-common 0.2.2", +] + [[package]] name = "dirs" version = "6.0.0" @@ -2776,7 +2913,7 @@ version = "0.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1e0e367e4e7da84520dedcac1901e4da967309406d1e51017ae1abfb97adbd38" dependencies = [ - "bitflags 2.11.0", + "bitflags 2.11.1", "objc2", ] @@ -2863,7 +3000,7 @@ version = "3.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7f7d4c414c94bc830797115b8e5f434d58e7e80cb42ba88508c14bc6ea270625" dependencies = [ - "bitflags 2.11.0", + "bitflags 2.11.1", "byteorder", "lazy_static", "proc-macro-error2", @@ -2937,9 +3074,9 @@ dependencies = [ [[package]] name = "either" -version = "1.15.0" +version = "1.16.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "48c757948c5ede0e46177b7add2e67155f70e33c07fea8284df6576da70b3719" +checksum = "91622ff5e7162018101f2fea40d6ebf4a78bbe5a49736a2020649edf9693679e" dependencies = [ "serde", ] @@ -3007,25 +3144,19 @@ dependencies = [ [[package]] name = "env_filter" -version = "1.0.0" +version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7a1c3cc8e57274ec99de65301228b537f1e4eedc1b8e0f9411c6caac8ae7308f" +checksum = "32e90c2accc4b07a8456ea0debdc2e7587bdd890680d71173a15d4ae604f6eef" dependencies = [ "log", "regex", ] -[[package]] -name = "env_home" -version = "0.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c7f84e12ccf0a7ddc17a6c41c93326024c42920d7ee630d04950e6926645c0fe" - [[package]] name = "env_logger" -version = "0.11.9" +version = "0.11.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b2daee4ea451f429a58296525ddf28b45a3b64f1acf6587e2067437bb11e218d" +checksum = "0621c04f2196ac3f488dd583365b9c09be011a4ab8b9f37248ffcc8f6198b56a" dependencies = [ "env_filter", "log", @@ -3055,14 +3186,14 @@ name = "equix-latency" version = "0.1.0" dependencies = [ "equix", - "rand 0.9.2", + "rand 0.9.4", ] [[package]] name = "erased-serde" -version = "0.4.9" +version = "0.4.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "89e8918065695684b2b0702da20382d5ae6065cf3327bc2d6436bd49a71ce9f3" +checksum = "d2add8a07dd6a8d93ff627029c51de145e12686fbc36ecb298ac22e74cf02dec" dependencies = [ "serde", "serde_core", @@ -3125,11 +3256,23 @@ dependencies = [ "syn 2.0.117", ] +[[package]] +name = "fallible-iterator" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2acce4a10f12dc2fb14a218589d4f1f62ef011b2d0cc4b3cb1bba8e94da14649" + +[[package]] +name = "fallible-streaming-iterator" +version = "0.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7360491ce676a36bf9bb3c56c1aa791658183a54d2744120f27285738d90465a" + [[package]] name = "fastrand" -version = "2.3.0" +version = "2.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "37909eebbb50d72f9059c3b6d82c0463f2ff062c9e95845c43a6c9c0355411be" +checksum = "9f1f227452a390804cdb637b74a86990f2a7d7ba4b7d5693aac9b4dd6defd8d6" [[package]] name = "fastrlp" @@ -3155,23 +3298,9 @@ dependencies = [ [[package]] name = "fax" -version = "0.2.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f05de7d48f37cd6730705cbca900770cab77a89f413d23e100ad7fad7795a0ab" -dependencies = [ - "fax_derive", -] - -[[package]] -name = "fax_derive" -version = "0.2.0" +version = "0.2.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a0aca10fb742cb43f9e7bb8467c91aa9bcb8e3ffbc6a6f7389bb93ffc920577d" -dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.117", -] +checksum = "caf1079563223d5d59d83c85886a56e586cfd5c1a26292e971a0fa266531ac5a" [[package]] name = "fdeflate" @@ -3200,13 +3329,12 @@ checksum = "28dea519a9695b9977216879a3ebfddf92f1c08c05d984f8996aecd6ecdc811d" [[package]] name = "filetime" -version = "0.2.27" +version = "0.2.29" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f98844151eee8917efc50bd9e8318cb963ae8b297431495d3f758616ea5c57db" +checksum = "5c287a33c7f0a620c38e641e7f60827713987b3c0f26e8ddc9462cc69cf75759" dependencies = [ "cfg-if", "libc", - "libredox", ] [[package]] @@ -3228,7 +3356,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "835c052cb0c08c1acf6ffd71c022172e18723949c8282f2b9f27efbc51e64534" dependencies = [ "byteorder", - "rand 0.8.5", + "rand 0.8.6", "rustc-hex", "static_assertions", ] @@ -3420,9 +3548,9 @@ checksum = "037711b3d59c33004d3856fbdc83b99d4ff37a24768fa1be9ce3538a1cde4393" [[package]] name = "futures-timer" -version = "3.0.3" +version = "3.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f288b0a4f20f9a56b5d1da57e2227c661b7b16168e2f72365f57b63326e29b24" +checksum = "af43fadb8a98512d547e37b4e92e0ced13e205c061b87b4623eff01d918d6968" [[package]] name = "futures-util" @@ -3505,21 +3633,21 @@ dependencies = [ "cfg-if", "js-sys", "libc", - "r-efi", + "r-efi 5.3.0", "wasip2", "wasm-bindgen", ] [[package]] name = "getrandom" -version = "0.4.1" +version = "0.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "139ef39800118c7683f2fd3c98c1b23c09ae076556b435f8e9064ae108aaeeec" +checksum = "0de51e6874e94e7bf76d726fc5d13ba782deca734ff60d5bb2fb2607c7406555" dependencies = [ "cfg-if", "libc", - "r-efi", - "rand_core 0.10.0", + "r-efi 6.0.0", + "rand_core 0.10.1", "wasip2", "wasip3", ] @@ -3575,7 +3703,7 @@ dependencies = [ "parking_lot", "portable-atomic", "quanta", - "rand 0.9.2", + "rand 0.9.4", "smallvec", "spinning_top", "web-time", @@ -3594,9 +3722,9 @@ dependencies = [ [[package]] name = "h2" -version = "0.4.13" +version = "0.4.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2f44da3a8150a6703ed5d34e164b875fd14c2cdab9af1252a9a1020bde2bdc54" +checksum = "171fefbc92fe4a4de27e0698d6a5b392d6a0e333506bc49133760b3bcf948733" dependencies = [ "atomic-waker", "bytes", @@ -3604,7 +3732,7 @@ dependencies = [ "futures-core", "futures-sink", "http", - "indexmap 2.13.0", + "indexmap 2.14.0", "slab", "tokio", "tokio-util", @@ -3638,9 +3766,9 @@ dependencies = [ [[package]] name = "handlebars" -version = "6.4.0" +version = "6.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9b3f9296c208515b87bd915a2f5d1163d4b3f863ba83337d7713cf478055948e" +checksum = "d43ccdfe15a81ab0a8af639e90254227c9a46afd9c5f5b6ec7efaa345c4b0f00" dependencies = [ "derive_builder", "log", @@ -3684,6 +3812,15 @@ dependencies = [ "allocator-api2", "equivalent", "foldhash 0.2.0", +] + +[[package]] +name = "hashbrown" +version = "0.17.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed5909b6e89a2db4456e54cd5f673791d7eca6732202bbf2a9cc504fe2f9b84a" +dependencies = [ + "foldhash 0.2.0", "serde", "serde_core", ] @@ -3697,6 +3834,15 @@ dependencies = [ "hashbrown 0.15.5", ] +[[package]] +name = "hashlink" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ea0b22561a9c04a7cb1a302c013e0259cd3b4bb619f145b32f72b8b4bcbed230" +dependencies = [ + "hashbrown 0.16.1", +] + [[package]] name = "hashx" version = "0.3.4" @@ -3762,7 +3908,7 @@ dependencies = [ "idna", "ipnet", "once_cell", - "rand 0.9.2", + "rand 0.9.4", "socket2 0.5.10", "thiserror 2.0.18", "tinyvec", @@ -3784,7 +3930,7 @@ dependencies = [ "moka", "once_cell", "parking_lot", - "rand 0.9.2", + "rand 0.9.4", "resolv-conf", "smallvec", "thiserror 2.0.18", @@ -3840,6 +3986,7 @@ dependencies = [ "bincode", "blake3", "bytes", + "chaff", "clap", "dirs", "futures", @@ -3906,11 +4053,20 @@ version = "1.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "df3b46402a9d5adb4c86a0cf463f42e19994e3ee891101b1841f30a545cb49a9" +[[package]] +name = "hybrid-array" +version = "0.4.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9155a582abd142abc056962c29e3ce5ff2ad5469f4246b537ed42c5deba857da" +dependencies = [ + "typenum", +] + [[package]] name = "hyper" -version = "1.8.1" +version = "1.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2ab2d4f250c3d7b1c9fcdff1cece94ea4e2dfbec68614f7b87cb205f24ca9d11" +checksum = "6299f016b246a94207e63da54dbe807655bf9e00044f73ded42c3ac5305fbcca" dependencies = [ "atomic-waker", "bytes", @@ -3923,7 +4079,6 @@ dependencies = [ "httpdate", "itoa", "pin-project-lite", - "pin-utils", "smallvec", "tokio", "want", @@ -3931,20 +4086,19 @@ dependencies = [ [[package]] name = "hyper-rustls" -version = "0.27.7" +version = "0.27.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e3c93eb611681b207e1fe55d5a71ecf91572ec8a6705cdb6857f7d8d5242cf58" +checksum = "33ca68d021ef39cf6463ab54c1d0f5daf03377b70561305bb89a8f83aab66e0f" dependencies = [ "http", "hyper", "hyper-util", "rustls", "rustls-native-certs", - "rustls-pki-types", "tokio", "tokio-rustls", "tower-service", - "webpki-roots 1.0.6", + "webpki-roots 1.0.7", ] [[package]] @@ -3977,7 +4131,7 @@ dependencies = [ "libc", "percent-encoding", "pin-project-lite", - "socket2 0.6.2", + "socket2 0.6.3", "tokio", "tower-service", "tracing", @@ -4014,19 +4168,20 @@ dependencies = [ "cfg-if", "criterion", "num-traits", - "rand 0.9.2", + "rand 0.9.4", "serde", "static_assertions", ] [[package]] name = "icu_collections" -version = "2.1.1" +version = "2.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4c6b649701667bbe825c3b7e6388cb521c23d88644678e83c0c4d0a621a34b43" +checksum = "2984d1cd16c883d7935b9e07e44071dca8d917fd52ecc02c04d5fa0b5a3f191c" dependencies = [ "displaydoc", "potential_utf", + "utf8_iter", "yoke", "zerofrom", "zerovec", @@ -4034,9 +4189,9 @@ dependencies = [ [[package]] name = "icu_locale_core" -version = "2.1.1" +version = "2.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "edba7861004dd3714265b4db54a3c390e880ab658fec5f7db895fae2046b5bb6" +checksum = "92219b62b3e2b4d88ac5119f8904c10f8f61bf7e95b640d25ba3075e6cac2c29" dependencies = [ "displaydoc", "litemap", @@ -4047,9 +4202,9 @@ dependencies = [ [[package]] name = "icu_normalizer" -version = "2.1.1" +version = "2.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5f6c8828b67bf8908d82127b2054ea1b4427ff0230ee9141c54251934ab1b599" +checksum = "c56e5ee99d6e3d33bd91c5d85458b6005a22140021cc324cea84dd0e72cff3b4" dependencies = [ "icu_collections", "icu_normalizer_data", @@ -4061,15 +4216,15 @@ dependencies = [ [[package]] name = "icu_normalizer_data" -version = "2.1.1" +version = "2.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7aedcccd01fc5fe81e6b489c15b247b8b0690feb23304303a9e560f37efc560a" +checksum = "da3be0ae77ea334f4da67c12f149704f19f81d1adf7c51cf482943e84a2bad38" [[package]] name = "icu_properties" -version = "2.1.2" +version = "2.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "020bfc02fe870ec3a66d93e677ccca0562506e5872c650f893269e08615d74ec" +checksum = "bee3b67d0ea5c2cca5003417989af8996f8604e34fb9ddf96208a033901e70de" dependencies = [ "icu_collections", "icu_locale_core", @@ -4081,15 +4236,15 @@ dependencies = [ [[package]] name = "icu_properties_data" -version = "2.1.2" +version = "2.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "616c294cf8d725c6afcd8f55abc17c56464ef6211f9ed59cccffe534129c77af" +checksum = "8e2bbb201e0c04f7b4b3e14382af113e17ba4f63e2c9d2ee626b720cbce54a14" [[package]] name = "icu_provider" -version = "2.1.1" +version = "2.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "85962cf0ce02e1e0a629cc34e7ca3e373ce20dda4c4d7294bbd0bf1fdb59e614" +checksum = "139c4cf31c8b5f33d7e199446eff9c1e02decfc2f0eec2c8d71f65befa45b421" dependencies = [ "displaydoc", "icu_locale_core", @@ -4125,9 +4280,9 @@ dependencies = [ [[package]] name = "idna_adapter" -version = "1.2.1" +version = "1.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3acae9609540aa318d1bc588455225fb2085b9ed0c4f6bd0d9d5bcd86f1a0344" +checksum = "cb68373c0d6620ef8105e855e7745e18b0d00d3bdb07fb532e434244cdb9a714" dependencies = [ "icu_normalizer", "icu_properties", @@ -4135,19 +4290,19 @@ dependencies = [ [[package]] name = "if-addrs" -version = "0.10.2" +version = "0.15.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cabb0019d51a643781ff15c9c8a3e5dedc365c47211270f4e8f82812fedd8f0a" +checksum = "c0a05c691e1fae256cf7013d99dad472dc52d5543322761f83ec8d47eab40d2b" dependencies = [ "libc", - "windows-sys 0.48.0", + "windows-sys 0.61.2", ] [[package]] name = "if-watch" -version = "3.2.1" +version = "3.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cdf9d64cfcf380606e64f9a0bcf493616b65331199f984151a6fa11a7b3cde38" +checksum = "71c02a5161c313f0cbdbadc511611893584a10a7b6153cb554bdf83ddce99ec2" dependencies = [ "async-io", "core-foundation 0.9.4", @@ -4163,7 +4318,7 @@ dependencies = [ "rtnetlink", "system-configuration", "tokio", - "windows 0.53.0", + "windows 0.62.2", ] [[package]] @@ -4181,7 +4336,7 @@ dependencies = [ "hyper", "hyper-util", "log", - "rand 0.9.2", + "rand 0.9.4", "tokio", "url", "xmltree", @@ -4189,9 +4344,9 @@ dependencies = [ [[package]] name = "image" -version = "0.25.9" +version = "0.25.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e6506c6c10786659413faa717ceebcb8f70731c0a60cbae39795fdf114519c1a" +checksum = "85ab80394333c02fe689eaf900ab500fbd0c2213da414687ebf995a65d5a6104" dependencies = [ "bytemuck", "byteorder-lite", @@ -4234,12 +4389,12 @@ dependencies = [ [[package]] name = "indexmap" -version = "2.13.0" +version = "2.14.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7714e70437a7dc3ac8eb7e6f8df75fd8eb422675fc7678aff7364301092b1017" +checksum = "d466e9454f08e4a911e14806c24e16fba1b4c121d1ea474396f396069cf949d9" dependencies = [ "equivalent", - "hashbrown 0.16.1", + "hashbrown 0.17.1", "serde", "serde_core", ] @@ -4284,9 +4439,9 @@ dependencies = [ [[package]] name = "instability" -version = "0.3.11" +version = "0.3.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "357b7205c6cd18dd2c86ed312d1e70add149aea98e7ef72b9fdf0270e555c11d" +checksum = "5eb2d60ef19920a3a9193c3e371f726ec1dafc045dac788d0fb3704272458971" dependencies = [ "darling 0.23.0", "indoc", @@ -4325,31 +4480,22 @@ checksum = "a2e611826a1868311677fdcdfbec9e8621d104c732d080f546a854530232f0ee" [[package]] name = "ipconfig" -version = "0.3.2" +version = "0.3.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b58db92f96b720de98181bbbe63c831e87005ab460c1bf306eb2622b4707997f" +checksum = "4d40460c0ce33d6ce4b0630ad68ff63d6661961c48b6dba35e5a4d81cfb48222" dependencies = [ - "socket2 0.5.10", + "socket2 0.6.3", "widestring", - "windows-sys 0.48.0", - "winreg", + "windows-registry", + "windows-result 0.4.1", + "windows-sys 0.61.2", ] [[package]] name = "ipnet" -version = "2.11.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "469fb0b9cefa57e3ef31275ee7cacb78f2fdca44e4765491884a2b119d4eb130" - -[[package]] -name = "iri-string" -version = "0.7.10" +version = "2.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c91338f0783edbd6195decb37bae672fd3b165faffb89bf7b9e6942f8b1a731a" -dependencies = [ - "memchr", - "serde", -] +checksum = "d98f6fed1fde3f8c21bc40a1abb88dd75e67924f9cffc3ef95607bad8017f8e2" [[package]] name = "is-terminal" @@ -4406,9 +4552,58 @@ dependencies = [ [[package]] name = "itoa" -version = "1.0.17" +version = "1.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682" + +[[package]] +name = "jni" +version = "0.22.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5efd9a482cf3a427f00d6b35f14332adc7902ce91efb778580e180ff90fa3498" +dependencies = [ + "cfg-if", + "combine", + "jni-macros", + "jni-sys", + "log", + "simd_cesu8", + "thiserror 2.0.18", + "walkdir", + "windows-link", +] + +[[package]] +name = "jni-macros" +version = "0.22.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a00109accc170f0bdb141fed3e393c565b6f5e072365c3bd58f5b062591560a3" +dependencies = [ + "proc-macro2", + "quote", + "rustc_version 0.4.1", + "simd_cesu8", + "syn 2.0.117", +] + +[[package]] +name = "jni-sys" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c6377a88cb3910bee9b0fa88d4f42e1d2da8e79915598f65fb0c7ee14c878af2" +dependencies = [ + "jni-sys-macros", +] + +[[package]] +name = "jni-sys-macros" +version = "0.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "92ecc6618181def0457392ccd0ee51198e065e016d1d527a7ac1b6dc7c1f09d2" +checksum = "38c0b942f458fe50cdac086d2f946512305e5631e720728f2a61aabcd47a6264" +dependencies = [ + "quote", + "syn 2.0.117", +] [[package]] name = "jobserver" @@ -4422,10 +4617,12 @@ dependencies = [ [[package]] name = "js-sys" -version = "0.3.90" +version = "0.3.98" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "14dc6f6450b3f6d4ed5b16327f38fed626d375a886159ca555bd7822c0c3a5a6" +checksum = "67df7112613f8bfd9150013a0314e196f4800d3201ae742489d999db2f979f08" dependencies = [ + "cfg-if", + "futures-util", "once_cell", "wasm-bindgen", ] @@ -4457,23 +4654,32 @@ dependencies = [ [[package]] name = "keccak" -version = "0.1.6" +version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cb26cec98cce3a3d96cbb7bced3c4b16e3d13f27ec56dbd62cbc8f39cfb9d653" +checksum = "9e24a010dd405bd7ed803e5253182815b41bf2e6a80cc3bfc066658e03a198aa" dependencies = [ - "cpufeatures", + "cfg-if", + "cpufeatures 0.3.0", ] [[package]] name = "keccak-asm" -version = "0.1.5" +version = "0.1.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b646a74e746cd25045aa0fd42f4f7f78aa6d119380182c7e63a5593c4ab8df6f" +checksum = "1766b89733097006f3a1388a02849865d6bc98c89273cb622e29fdd209922183" dependencies = [ "digest 0.10.7", "sha3-asm", ] +[[package]] +name = "kernels" +version = "0.1.0" + +[[package]] +name = "kernels-open-approver" +version = "0.1.0" + [[package]] name = "kernels-open-bridge" version = "0.1.0" @@ -4498,6 +4704,21 @@ version = "0.1.0" name = "kernels-open-wallet" version = "0.1.0" +[[package]] +name = "konst" +version = "0.2.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "128133ed7824fcd73d6e7b17957c5eb7bacb885649bd8c69708b2331a10bcefb" +dependencies = [ + "konst_macro_rules", +] + +[[package]] +name = "konst_macro_rules" +version = "0.2.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a4933f3f57a8e9d9da04db23fb153356ecaf00cbd14aee46279c33dc80925c37" + [[package]] name = "kqueue" version = "1.1.1" @@ -4510,11 +4731,11 @@ dependencies = [ [[package]] name = "kqueue-sys" -version = "1.0.4" +version = "1.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ed9625ffda8729b85e45cf04090035ac368927b8cebc34898e7c120f52e4838b" +checksum = "07293a4e297ac234359b510362495713f75ea345d5307140414f20c69ffeb087" dependencies = [ - "bitflags 1.3.2", + "bitflags 2.11.1", "libc", ] @@ -4561,9 +4782,9 @@ checksum = "09edd9e8b54e49e587e4f6295a7d29c3ea94d469cb40ab8ca70b288248a81db2" [[package]] name = "libc" -version = "0.2.182" +version = "0.2.186" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6800badb6cb2082ffd7b6a67e6125bb39f18782f793520caee8cb8846be06112" +checksum = "68ab91017fe16c622486840e4c83c9a37afeff978bd239b5293d61ece587de66" [[package]] name = "libloading" @@ -4651,7 +4872,7 @@ dependencies = [ "parking_lot", "pin-project", "quick-protobuf", - "rand 0.8.5", + "rand 0.8.6", "rw-stream-sink", "thiserror 2.0.18", "tracing", @@ -4705,7 +4926,7 @@ dependencies = [ "hkdf", "multihash", "quick-protobuf", - "rand 0.8.5", + "rand 0.8.6", "sha2", "thiserror 2.0.18", "tracing", @@ -4729,7 +4950,7 @@ dependencies = [ "libp2p-swarm", "quick-protobuf", "quick-protobuf-codec", - "rand 0.8.5", + "rand 0.8.6", "sha2", "smallvec", "thiserror 2.0.18", @@ -4749,7 +4970,7 @@ dependencies = [ "libp2p-core", "libp2p-identity", "libp2p-swarm", - "rand 0.8.5", + "rand 0.8.6", "smallvec", "socket2 0.5.10", "tokio", @@ -4806,7 +5027,7 @@ dependencies = [ "libp2p-core", "libp2p-identity", "libp2p-swarm", - "rand 0.8.5", + "rand 0.8.6", "tracing", "web-time", ] @@ -4823,7 +5044,7 @@ dependencies = [ "libp2p-identity", "libp2p-tls", "quinn", - "rand 0.8.5", + "rand 0.8.6", "ring", "rustls", "socket2 0.5.10", @@ -4844,7 +5065,7 @@ dependencies = [ "libp2p-core", "libp2p-identity", "libp2p-swarm", - "rand 0.8.5", + "rand 0.8.6", "serde", "smallvec", "tracing", @@ -4864,7 +5085,7 @@ dependencies = [ "libp2p-swarm-derive", "lru 0.12.5", "multistream-select", - "rand 0.8.5", + "rand 0.8.6", "smallvec", "tokio", "tracing", @@ -4911,7 +5132,7 @@ dependencies = [ "rustls-webpki", "thiserror 2.0.18", "x509-parser 0.17.0", - "yasna", + "yasna 0.5.2", ] [[package]] @@ -4930,13 +5151,11 @@ dependencies = [ [[package]] name = "libredox" -version = "0.1.12" +version = "0.1.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3d0b95e02c851351f877147b7deea7b1afb1df71b63aa5f8270716e0c5720616" +checksum = "e02f3bb43d335493c96bf3fd3a321600bf6bd07ed34bc64118e9293bdffea46c" dependencies = [ - "bitflags 2.11.0", "libc", - "redox_syscall 0.7.2", ] [[package]] @@ -4964,9 +5183,9 @@ checksum = "32a66949e030da00e8c7d4434b251670a91556f4144941d37452769c25d58a53" [[package]] name = "litemap" -version = "0.8.1" +version = "0.8.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6373607a59f0be73a39b6fe456b8192fcc3585f602af20751600e974dd455e77" +checksum = "92daf443525c4cce67b150400bc2316076100ce0b3686209eb8cf3c31612e6f0" [[package]] name = "litrs" @@ -5013,9 +5232,9 @@ dependencies = [ [[package]] name = "lru" -version = "0.16.3" +version = "0.16.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a1dc47f592c06f33f8e3aea9591776ec7c9f9e4124778ff8a3c3b87159f7e593" +checksum = "7f66e8d5d03f609abc3a39e6f08e4164ebf1447a732906d39eb9b99b7919ef39" dependencies = [ "hashbrown 0.16.1", ] @@ -5028,9 +5247,9 @@ checksum = "112b39cec0b298b6c1999fee3e31427f74f676e4cb9879ed1a121b43661a4154" [[package]] name = "macro-string" -version = "0.1.4" +version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1b27834086c65ec3f9387b096d66e99f221cf081c2b738042aa252bcd41204e3" +checksum = "59a9dbbfc75d2688ed057456ce8a3ee3f48d12eec09229f560f3643b9f275653" dependencies = [ "proc-macro2", "quote", @@ -5094,6 +5313,27 @@ dependencies = [ "windows-sys 0.52.0", ] +[[package]] +name = "migrations_internals" +version = "2.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "36c791ecdf977c99f45f23280405d7723727470f6689a5e6dbf513ac547ae10d" +dependencies = [ + "serde", + "toml 0.9.12+spec-1.1.0", +] + +[[package]] +name = "migrations_macros" +version = "2.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "36fc5ac76be324cfd2d3f2cf0fdf5d5d3c4f14ed8aaebadb09e304ba42282703" +dependencies = [ + "migrations_internals", + "proc-macro2", + "quote", +] + [[package]] name = "mime" version = "0.3.17" @@ -5149,9 +5389,9 @@ dependencies = [ [[package]] name = "mio" -version = "1.1.1" +version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a69bcab0ad47271a0234d9422b131806bf3968021e5dc9328caf2d4cd58557fc" +checksum = "50b7e5b27aa02a74bac8c3f23f448f8d87ff11f92d3aac1a6ed369ee08cc56c1" dependencies = [ "libc", "log", @@ -5161,9 +5401,9 @@ dependencies = [ [[package]] name = "moka" -version = "0.12.13" +version = "0.12.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b4ac832c50ced444ef6be0767a008b02c106a909ba79d1d830501e94b96f6b7e" +checksum = "957228ad12042ee839f93c8f257b62b4c0ab5eaae1d4fa60de53b27c9d7c5046" dependencies = [ "crossbeam-channel", "crossbeam-epoch", @@ -5178,9 +5418,9 @@ dependencies = [ [[package]] name = "moxcms" -version = "0.7.11" +version = "0.8.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ac9557c559cd6fc9867e122e20d2cbefc9ca29d80d027a8e39310920ed2f0a97" +checksum = "bb85c154ba489f01b25c0d36ae69a87e4a1c73a72631fc6c0eb6dde34a73e44b" dependencies = [ "num-traits", "pxfm", @@ -5219,11 +5459,10 @@ dependencies = [ [[package]] name = "multihash" -version = "0.19.3" +version = "0.19.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6b430e7953c29dd6a09afc29ff0bb69c6e306329ee6794700aee27b76a1aea8d" +checksum = "577c63b00ad74d57e8c9aa870b5fccebf2fd64a308a5aee9f1bb88e4aea19447" dependencies = [ - "core2", "unsigned-varint", ] @@ -5284,46 +5523,30 @@ dependencies = [ [[package]] name = "netlink-packet-core" -version = "0.7.0" +version = "0.8.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "72724faf704479d67b388da142b186f916188505e7e0b26719019c525882eda4" +checksum = "3463cbb78394cb0141e2c926b93fc2197e473394b761986eca3b9da2c63ae0f4" dependencies = [ - "anyhow", - "byteorder", - "netlink-packet-utils", + "paste", ] [[package]] name = "netlink-packet-route" -version = "0.17.1" +version = "0.28.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "053998cea5a306971f88580d0829e90f270f940befd7cf928da179d4187a5a66" +checksum = "4ce3636fa715e988114552619582b530481fd5ef176a1e5c1bf024077c2c9445" dependencies = [ - "anyhow", - "bitflags 1.3.2", - "byteorder", + "bitflags 2.11.1", "libc", + "log", "netlink-packet-core", - "netlink-packet-utils", -] - -[[package]] -name = "netlink-packet-utils" -version = "0.5.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0ede8a08c71ad5a95cdd0e4e52facd37190977039a4704eb82a283f713747d34" -dependencies = [ - "anyhow", - "byteorder", - "paste", - "thiserror 1.0.69", ] [[package]] name = "netlink-proto" -version = "0.11.5" +version = "0.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "72452e012c2f8d612410d89eea01e2d9b56205274abb35d53f60200b2ec41d60" +checksum = "b65d130ee111430e47eed7896ea43ca693c387f097dd97376bffafbf25812128" dependencies = [ "bytes", "futures", @@ -5348,12 +5571,13 @@ dependencies = [ [[package]] name = "nix" -version = "0.26.4" +version = "0.30.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "598beaf3cc6fdd9a5dfb1630c2800c7acd31df7aaf0f565796fba2b53ca1af1b" +checksum = "74523f3a35e05aba87a1d978330aef40f67b0304ac79c1c00b294c9830543db6" dependencies = [ - "bitflags 1.3.2", + "bitflags 2.11.1", "cfg-if", + "cfg_aliases", "libc", ] @@ -5379,22 +5603,26 @@ dependencies = [ "chrono", "clap", "config", + "diesel", + "diesel_migrations", "dirs", "either", "futures", - "getrandom 0.3.4", "gnort", "ibig", "instant-acme", "intmap", + "libc", + "libsqlite3-sys", "nockvm", "nockvm_macros", "noun-serde", "opentelemetry", "opentelemetry-otlp", "opentelemetry_sdk", - "rand 0.9.2", - "rcgen 0.14.7", + "rand 0.9.4", + "rcgen 0.14.8", + "rusqlite", "rustls", "rustls-pki-types", "serde", @@ -5407,14 +5635,15 @@ dependencies = [ "tokio", "tokio-rustls", "tokio-util", - "tonic 0.14.5", + "tonic 0.14.6", "tower-http", "tracing", "tracing-opentelemetry", "tracing-subscriber", "tracing-test", "tracing-tracy", - "webpki-roots 1.0.6", + "uuid", + "webpki-roots 1.0.7", "x509-parser 0.17.0", "yaque", ] @@ -5448,7 +5677,7 @@ dependencies = [ "thiserror 2.0.18", "tokio", "tokio-stream", - "tonic 0.14.5", + "tonic 0.14.6", "tonic-build", "tonic-health", "tonic-prost", @@ -5471,7 +5700,7 @@ dependencies = [ "prost-build", "prost-types", "thiserror 2.0.18", - "tonic 0.14.5", + "tonic 0.14.6", "tonic-build", "tonic-prost", "tonic-prost-build", @@ -5481,6 +5710,7 @@ dependencies = [ name = "nockchain" version = "0.1.0" dependencies = [ + "blake3", "bs58", "chaff", "clap", @@ -5489,7 +5719,9 @@ dependencies = [ "ibig", "kernels-open-dumb", "kernels-open-miner", + "libc", "libp2p", + "memmap2", "nockapp", "nockapp-grpc", "nockchain-libp2p-io", @@ -5499,13 +5731,14 @@ dependencies = [ "nockvm_macros", "noun-serde", "num_cpus", - "rand 0.9.2", + "rand 0.9.4", "tempfile", "termcolor", "tikv-jemallocator", "tokio", "tracing", "tracy-client", + "vergen", "zkvm-jetpack", ] @@ -5525,7 +5758,7 @@ dependencies = [ "tempfile", "tikv-jemallocator", "tokio", - "tonic 0.14.5", + "tonic 0.14.6", "tracy-client", "zkvm-jetpack", ] @@ -5545,7 +5778,7 @@ dependencies = [ "ratatui 0.28.1", "rustls", "tokio", - "tonic 0.14.5", + "tonic 0.14.6", "tracing", "tracing-subscriber", "tracing-tracy", @@ -5564,6 +5797,7 @@ dependencies = [ "equix", "futures", "gnort", + "hex", "hickory-proto", "hickory-resolver", "ibig", @@ -5574,10 +5808,11 @@ dependencies = [ "noun-serde", "num-bigint", "quickcheck", - "rand 0.9.2", + "rand 0.9.4", "serde", "serde_bytes", "serde_cbor", + "serde_json", "tokio", "tracing", ] @@ -5668,9 +5903,10 @@ dependencies = [ "termimad", "thiserror 2.0.18", "tokio", - "tonic 0.14.5", + "tonic 0.14.6", "tracing", "tracing-subscriber", + "wallet-tx-builder", "zkvm-jetpack", ] @@ -5692,8 +5928,8 @@ dependencies = [ "openssl-sys", "predicates", "proptest", - "reqwest", - "semver 1.0.27", + "reqwest 0.12.28", + "semver 1.0.28", "serde", "serde_json", "sha1", @@ -5701,8 +5937,8 @@ dependencies = [ "tempfile", "thiserror 2.0.18", "tokio", - "toml", - "which 8.0.0", + "toml 0.9.12+spec-1.1.0", + "which 8.0.2", ] [[package]] @@ -5726,10 +5962,13 @@ dependencies = [ "nockvm_macros", "num-derive", "num-traits", - "rand 0.9.2", + "proptest", + "rand 0.9.4", "serde", "slotmap", + "smallvec", "static_assertions", + "tempfile", "thiserror 2.0.18", "tracing", "tracing-core", @@ -5853,9 +6092,9 @@ dependencies = [ [[package]] name = "num-conv" -version = "0.2.0" +version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cf97ec579c3c42f953ef76dbf8d55ac91fb219dde70e49aa4a6b7d74e9919050" +checksum = "521739c6d2bac4aa25192232afe6841231376b2b26d4d9fae5ecf8ca5772e441" [[package]] name = "num-derive" @@ -5914,9 +6153,9 @@ dependencies = [ [[package]] name = "num_enum" -version = "0.7.5" +version = "0.7.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b1207a7e20ad57b847bbddc6776b968420d38292bbfe2089accff5e19e82454c" +checksum = "5d0bca838442ec211fa11de3a8b0e0e8f3a4522575b5c4c06ed722e005036f26" dependencies = [ "num_enum_derive", "rustversion", @@ -5924,15 +6163,24 @@ dependencies = [ [[package]] name = "num_enum_derive" -version = "0.7.5" +version = "0.7.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ff32365de1b6743cb203b710788263c44a03de03802daf96092f2da4fe6ba4d7" +checksum = "680998035259dcfcafe653688bf2aa6d3e2dc05e98be6ab46afb089dc84f1df8" dependencies = [ "proc-macro2", "quote", "syn 2.0.117", ] +[[package]] +name = "num_threads" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c7398b9c8b70908f6371f47ed36737907c87c52af34c268fed0bf0ceb92ead9" +dependencies = [ + "libc", +] + [[package]] name = "nybbles" version = "0.4.8" @@ -5962,7 +6210,7 @@ version = "0.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d49e936b501e5c5bf01fda3a9452ff86dc3ea98ad5f283e1455153142d97518c" dependencies = [ - "bitflags 2.11.0", + "bitflags 2.11.1", "objc2", "objc2-core-graphics", "objc2-foundation", @@ -5974,7 +6222,7 @@ version = "0.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2a180dd8642fa45cdb7dd721cd4c11b1cadd4929ce112ebd8b9f5803cc79d536" dependencies = [ - "bitflags 2.11.0", + "bitflags 2.11.1", "dispatch2", "objc2", ] @@ -5985,7 +6233,7 @@ version = "0.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e022c9d066895efa1345f8e33e584b9f958da2fd4cd116792e15e07e4720a807" dependencies = [ - "bitflags 2.11.0", + "bitflags 2.11.1", "dispatch2", "objc2", "objc2-core-foundation", @@ -6004,7 +6252,7 @@ version = "0.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e3e0adef53c21f888deb4fa59fc59f7eb17404926ee8a6f59f5df0fd7f9f3272" dependencies = [ - "bitflags 2.11.0", + "bitflags 2.11.1", "objc2", "objc2-core-foundation", ] @@ -6015,7 +6263,7 @@ version = "0.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "180788110936d59bab6bd83b6060ffdfffb3b922ba1396b312ae795e1de9d81d" dependencies = [ - "bitflags 2.11.0", + "bitflags 2.11.1", "objc2", "objc2-core-foundation", ] @@ -6031,9 +6279,9 @@ dependencies = [ [[package]] name = "once_cell" -version = "1.21.3" +version = "1.21.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "42f5e15c9953c5e4ccceeb2e7382a716482c34515315f7b03532b8b4e8393d2d" +checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" [[package]] name = "once_cell_polyfill" @@ -6063,8 +6311,7 @@ dependencies = [ [[package]] name = "op-alloy-consensus" version = "0.23.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "736381a95471d23e267263cfcee9e1d96d30b9754a94a2819148f83379de8a86" +source = "git+https://github.com/ethereum-optimism/optimism?rev=0c6a03eac768122819c05b71f233bba1e90cf426#0c6a03eac768122819c05b71f233bba1e90cf426" dependencies = [ "alloy-consensus", "alloy-eips", @@ -6075,21 +6322,20 @@ dependencies = [ "alloy-serde", "derive_more", "serde", + "serde_with", "thiserror 2.0.18", ] [[package]] name = "op-alloy-network" version = "0.23.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4034183dca6bff6632e7c24c92e75ff5f0eabb58144edb4d8241814851334d47" +source = "git+https://github.com/ethereum-optimism/optimism?rev=0c6a03eac768122819c05b71f233bba1e90cf426#0c6a03eac768122819c05b71f233bba1e90cf426" dependencies = [ "alloy-consensus", "alloy-network", "alloy-primitives", "alloy-provider", "alloy-rpc-types-eth", - "alloy-signer", "op-alloy-consensus", "op-alloy-rpc-types", ] @@ -6112,8 +6358,7 @@ dependencies = [ [[package]] name = "op-alloy-rpc-types" version = "0.23.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ddd87c6b9e5b6eee8d6b76f41b04368dca0e9f38d83338e5b00e730c282098a4" +source = "git+https://github.com/ethereum-optimism/optimism?rev=0c6a03eac768122819c05b71f233bba1e90cf426#0c6a03eac768122819c05b71f233bba1e90cf426" dependencies = [ "alloy-consensus", "alloy-eips", @@ -6158,18 +6403,18 @@ checksum = "7c87def4c32ab89d880effc9e097653c8da5d6ef28e6b539d313baaacfbafcbe" [[package]] name = "openssl-src" -version = "300.5.5+3.5.5" +version = "300.6.0+3.6.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3f1787d533e03597a7934fd0a765f0d28e94ecc5fb7789f8053b1e699a56f709" +checksum = "a8e8cbfd3a4a8c8f089147fd7aaa33cf8c7450c4d09f8f80698a0cf093abeff4" dependencies = [ "cc", ] [[package]] name = "openssl-sys" -version = "0.9.111" +version = "0.9.116" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "82cab2d520aa75e3c58898289429321eb788c3106963d0dc886ec7a5f4adc321" +checksum = "f28a22dc7140cda5f096e5e7724a6962ca81a7f8bfd2979f9b18c11af56318c4" dependencies = [ "cc", "libc", @@ -6202,7 +6447,7 @@ dependencies = [ "bytes", "http", "opentelemetry", - "reqwest", + "reqwest 0.12.28", ] [[package]] @@ -6217,7 +6462,7 @@ dependencies = [ "opentelemetry-proto", "opentelemetry_sdk", "prost 0.13.5", - "reqwest", + "reqwest 0.12.28", "thiserror 2.0.18", "tokio", "tonic 0.13.1", @@ -6247,7 +6492,7 @@ dependencies = [ "futures-util", "opentelemetry", "percent-encoding", - "rand 0.9.2", + "rand 0.9.4", "serde_json", "thiserror 2.0.18", "tokio", @@ -6322,7 +6567,7 @@ checksum = "2621685985a2ebf1c516881c026032ac7deafcda1a2c9b7850dc81e3dfcb64c1" dependencies = [ "cfg-if", "libc", - "redox_syscall 0.5.18", + "redox_syscall", "smallvec", "windows-link", ] @@ -6417,7 +6662,7 @@ checksum = "8701b58ea97060d5e5b155d383a69952a60943f0e6dfe30b04c287beb0b27455" dependencies = [ "fixedbitset", "hashbrown 0.15.5", - "indexmap 2.13.0", + "indexmap 2.14.0", ] [[package]] @@ -6432,18 +6677,18 @@ dependencies = [ [[package]] name = "pin-project" -version = "1.1.10" +version = "1.1.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "677f1add503faace112b9f1373e43e9e054bfdd22ff1a63c1bc485eaec6a6a8a" +checksum = "2466b2336ed02bcdca6b294417127b90ec92038d1d5c4fbeac971a922e0e0924" dependencies = [ "pin-project-internal", ] [[package]] name = "pin-project-internal" -version = "1.1.10" +version = "1.1.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6e918e4ff8c4549eb882f14b3a4bc8c8bc93de829416eacf579f1207a8fbf861" +checksum = "c96395f0a926bc13b1c17622aaddda1ecb55d49c8f1bf9777e4d877800a43f8b" dependencies = [ "proc-macro2", "quote", @@ -6452,9 +6697,9 @@ dependencies = [ [[package]] name = "pin-project-lite" -version = "0.2.16" +version = "0.2.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3b3cff922bd51709b605d9ead9aa71031d81447142d828eb4a6eba76fe619f9b" +checksum = "a89322df9ebe1c1578d689c92318e070967d1042b512afbe49518723f4e6d5cd" [[package]] name = "pin-utils" @@ -6474,9 +6719,9 @@ dependencies = [ [[package]] name = "pkg-config" -version = "0.3.32" +version = "0.3.33" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7edddbd0b52d732b21ad9a5fab5c704c14cd949e5e9a1ec5929a24fded1b904c" +checksum = "19f132c84eca552bf34cab8ec81f1c1dcc229b811638f9d283dceabe58c5569e" [[package]] name = "plotters" @@ -6512,7 +6757,7 @@ version = "0.18.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "60769b8b31b2a9f263dae2776c37b1b28ae246943cf719eb6946a1db05128a61" dependencies = [ - "bitflags 2.11.0", + "bitflags 2.11.1", "crc32fast", "fdeflate", "flate2", @@ -6541,9 +6786,9 @@ checksum = "c33a9471896f1c69cecef8d20cbe2f7accd12527ce60845ff44c153bb2a21b49" [[package]] name = "potential_utf" -version = "0.1.4" +version = "0.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b73949432f5e2a09657003c25bca5e19a0e9c84f8058ca374f49e0ebe605af77" +checksum = "0103b1cef7ec0cf76490e969665504990193874ea05c85ff9bab8b911d0a0564" dependencies = [ "zerovec", ] @@ -6616,9 +6861,9 @@ dependencies = [ [[package]] name = "proc-macro-crate" -version = "3.4.0" +version = "3.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "219cb19e96be00ab2e37d6e299658a0cfa83e52429179969b0f0121b4ac46983" +checksum = "e67ba7e9b2b56446f1d419b1d807906278ffa1a658a8a5d8a39dcb1f5a78614f" dependencies = [ "toml_edit", ] @@ -6679,15 +6924,15 @@ dependencies = [ [[package]] name = "proptest" -version = "1.10.0" +version = "1.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "37566cb3fdacef14c0737f9546df7cfeadbfbc9fef10991038bf5015d0c80532" +checksum = "4b45fcc2344c680f5025fe57779faef368840d0bd1f42f216291f0dc4ace4744" dependencies = [ "bit-set", - "bit-vec", - "bitflags 2.11.0", + "bit-vec 0.8.0", + "bitflags 2.11.1", "num-traits", - "rand 0.9.2", + "rand 0.9.4", "rand_chacha 0.9.0", "rand_xorshift", "regex-syntax", @@ -6794,11 +7039,11 @@ dependencies = [ [[package]] name = "pulldown-cmark" -version = "0.13.1" +version = "0.13.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "83c41efbf8f90ac44de7f3a868f0867851d261b56291732d0cbf7cceaaeb55a6" +checksum = "e9f068eba8e7071c5f9511831b44f32c740d5adf574e990f946ddb53db2f314e" dependencies = [ - "bitflags 2.11.0", + "bitflags 2.11.1", "memchr", "unicase", ] @@ -6814,12 +7059,9 @@ dependencies = [ [[package]] name = "pxfm" -version = "0.1.27" +version = "0.1.29" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7186d3822593aa4393561d186d1393b3923e9d6163d3fbfd6e825e3e6cf3e6a8" -dependencies = [ - "num-traits", -] +checksum = "e0c5ccf5294c6ccd63a74f1565028353830a9c2f5eb0c682c355c471726a6e3f" [[package]] name = "quanta" @@ -6877,7 +7119,7 @@ checksum = "95c589f335db0f6aaa168a7cd27b1fc6920f5e1470c804f814d9cd6e62a0f70b" dependencies = [ "env_logger", "log", - "rand 0.10.0", + "rand 0.10.1", ] [[package]] @@ -6892,9 +7134,9 @@ dependencies = [ "pin-project-lite", "quinn-proto", "quinn-udp", - "rustc-hash 2.1.1", + "rustc-hash 2.1.2", "rustls", - "socket2 0.6.2", + "socket2 0.6.3", "thiserror 2.0.18", "tokio", "tracing", @@ -6903,16 +7145,17 @@ dependencies = [ [[package]] name = "quinn-proto" -version = "0.11.13" +version = "0.11.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f1906b49b0c3bc04b5fe5d86a77925ae6524a19b816ae38ce1e426255f1d8a31" +checksum = "434b42fec591c96ef50e21e886936e66d3cc3f737104fdb9b737c40ffb94c098" dependencies = [ + "aws-lc-rs", "bytes", "getrandom 0.3.4", "lru-slab", - "rand 0.9.2", + "rand 0.9.4", "ring", - "rustc-hash 2.1.1", + "rustc-hash 2.1.2", "rustls", "rustls-pki-types", "slab", @@ -6931,16 +7174,16 @@ dependencies = [ "cfg_aliases", "libc", "once_cell", - "socket2 0.6.2", + "socket2 0.6.3", "tracing", "windows-sys 0.60.2", ] [[package]] name = "quote" -version = "1.0.44" +version = "1.0.45" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "21b2ebcf727b7760c461f091f9f0f539b77b8e87f2fd88131e7f1b433b3cece4" +checksum = "41f2619966050689382d2b44f664f4bc593e129785a36d6ee376ddf37259b924" dependencies = [ "proc-macro2", ] @@ -6951,6 +7194,12 @@ version = "5.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "69cdb34c158ceb288df11e18b4bd39de994f6657d83847bdffdbd7f346754b0f" +[[package]] +name = "r-efi" +version = "6.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8dcc9c7d52a811697d2151c701e0d08956f92b0e24136cf4cf27b57a6a0d9bf" + [[package]] name = "radium" version = "0.7.0" @@ -6968,9 +7217,9 @@ dependencies = [ [[package]] name = "rand" -version = "0.8.5" +version = "0.8.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "34af8d1a0e25924bc5b7c43c079c942339d8f0a8b57c39049bef581b46327404" +checksum = "5ca0ecfa931c29007047d1bc58e623ab12e5590e8c7cc53200d5202b69266d8a" dependencies = [ "libc", "rand_chacha 0.3.1", @@ -6980,9 +7229,9 @@ dependencies = [ [[package]] name = "rand" -version = "0.9.2" +version = "0.9.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6db2770f06117d490610c7488547d543617b21bfa07796d7a12f6f1bd53850d1" +checksum = "44c5af06bb1b7d3216d91932aed5265164bf384dc89cd6ba05cf59a35f5f76ea" dependencies = [ "rand_chacha 0.9.0", "rand_core 0.9.5", @@ -6991,12 +7240,12 @@ dependencies = [ [[package]] name = "rand" -version = "0.10.0" +version = "0.10.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bc266eb313df6c5c09c1c7b1fbe2510961e5bcd3add930c1e31f7ed9da0feff8" +checksum = "d2e8e8bcc7961af1fdac401278c6a831614941f6164ee3bf4ce61b7edb162207" dependencies = [ - "getrandom 0.4.1", - "rand_core 0.10.0", + "getrandom 0.4.2", + "rand_core 0.10.1", ] [[package]] @@ -7040,9 +7289,9 @@ dependencies = [ [[package]] name = "rand_core" -version = "0.10.0" +version = "0.10.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0c8d0fd677905edcbeedbf2edb6494d676f0e98d54d5cf9bda0b061cb8fb8aba" +checksum = "63b8176103e19a2643978565ca18b50549f6101881c443590420e4dc998a3c69" [[package]] name = "rand_xorshift" @@ -7068,7 +7317,7 @@ version = "0.28.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fdef7f9be5c0122f890d58bdf4d964349ba6a6161f705907526d891efabba57d" dependencies = [ - "bitflags 2.11.0", + "bitflags 2.11.1", "cassowary", "compact_str", "crossterm 0.28.1", @@ -7089,7 +7338,7 @@ version = "0.29.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "eabd94c2f37801c20583fc49dd5cd6b0ba68c716787c2dd6ed18571e1e63117b" dependencies = [ - "bitflags 2.11.0", + "bitflags 2.11.1", "cassowary", "compact_str", "crossterm 0.28.1", @@ -7110,7 +7359,7 @@ version = "11.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "498cd0dc59d73224351ee52a95fee0f1a617a2eae0e7d9d720cc622c73a54186" dependencies = [ - "bitflags 2.11.0", + "bitflags 2.11.1", ] [[package]] @@ -7129,9 +7378,9 @@ dependencies = [ [[package]] name = "rayon" -version = "1.11.0" +version = "1.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "368f01d005bf8fd9b1206fb6fa653e6c4a81ceb1466406b81792d87c5677a58f" +checksum = "fb39b166781f92d482534ef4b4b1b2568f42613b53e5b6c160e24cfbfa30926d" dependencies = [ "either", "rayon-core", @@ -7157,21 +7406,21 @@ dependencies = [ "ring", "rustls-pki-types", "time", - "yasna", + "yasna 0.5.2", ] [[package]] name = "rcgen" -version = "0.14.7" +version = "0.14.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "10b99e0098aa4082912d4c649628623db6aba77335e4f4569ff5083a6448b32e" +checksum = "57f6d249aad744e274e682777a50283a225a32705394ee6d5fcc01efa25e4055" dependencies = [ "pem", "ring", "rustls-pki-types", "time", "x509-parser 0.18.1", - "yasna", + "yasna 0.6.0", ] [[package]] @@ -7180,16 +7429,7 @@ version = "0.5.18" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ed2bf2547551a7053d6fdfafda3f938979645c44812fbfcda098faae3f1a362d" dependencies = [ - "bitflags 2.11.0", -] - -[[package]] -name = "redox_syscall" -version = "0.7.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6d94dd2f7cd932d4dc02cc8b2b50dfd38bd079a4e5d79198b99743d7fcf9a4b4" -dependencies = [ - "bitflags 2.11.0", + "bitflags 2.11.1", ] [[package]] @@ -7301,7 +7541,44 @@ dependencies = [ "wasm-bindgen", "wasm-bindgen-futures", "web-sys", - "webpki-roots 1.0.6", + "webpki-roots 1.0.7", +] + +[[package]] +name = "reqwest" +version = "0.13.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "62e0021ea2c22aed41653bc7e1419abb2c97e038ff2c33d0e1309e49a97deec0" +dependencies = [ + "base64", + "bytes", + "futures-core", + "http", + "http-body", + "http-body-util", + "hyper", + "hyper-rustls", + "hyper-util", + "js-sys", + "log", + "percent-encoding", + "pin-project-lite", + "quinn", + "rustls", + "rustls-pki-types", + "rustls-platform-verifier", + "serde", + "serde_json", + "sync_wrapper", + "tokio", + "tokio-rustls", + "tower", + "tower-http", + "tower-service", + "url", + "wasm-bindgen", + "wasm-bindgen-futures", + "web-sys", ] [[package]] @@ -7316,7 +7593,7 @@ version = "2.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1cab9bd343c737660e523ee69f788018f3db686d537d2fd0f99c9f747c1bda4f" dependencies = [ - "rand 0.9.2", + "rand 0.9.4", ] [[package]] @@ -7345,14 +7622,14 @@ dependencies = [ [[package]] name = "rkyv" -version = "0.8.15" +version = "0.8.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1a30e631b7f4a03dee9056b8ef6982e8ba371dd5bedb74d3ec86df4499132c70" +checksum = "73389e0c99e664f919275ab5b5b0471391fe9a8de61e1dff9b1eaf56a90f16e3" dependencies = [ "bytecheck", "bytes", - "hashbrown 0.16.1", - "indexmap 2.13.0", + "hashbrown 0.17.1", + "indexmap 2.14.0", "munge", "ptr_meta", "rancor", @@ -7364,9 +7641,9 @@ dependencies = [ [[package]] name = "rkyv_derive" -version = "0.8.15" +version = "0.8.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8100bb34c0a1d0f907143db3149e6b4eea3c33b9ee8b189720168e818303986f" +checksum = "5d2ed0b54125315fb36bd021e82d314d1c126548f871634b483f46b31d13cac6" dependencies = [ "proc-macro2", "quote", @@ -7385,11 +7662,11 @@ dependencies = [ [[package]] name = "ron" -version = "0.12.0" +version = "0.12.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fd490c5b18261893f14449cbd28cb9c0b637aebf161cd77900bfdedaff21ec32" +checksum = "4147b952f3f819eca0e99527022f7d6a8d05f111aeb0a62960c74eb283bec8fc" dependencies = [ - "bitflags 2.11.0", + "bitflags 2.11.1", "once_cell", "serde", "serde_derive", @@ -7399,9 +7676,9 @@ dependencies = [ [[package]] name = "rsqlite-vfs" -version = "0.1.0" +version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a8a1f2315036ef6b1fbacd1972e8ee7688030b0a2121edfc2a6550febd41574d" +checksum = "c51c9ae4df8a7fba42103df5c621fa3c37eccf3a3c650879e90fc48b11cc192c" dependencies = [ "hashbrown 0.16.1", "thiserror 2.0.18", @@ -7409,15 +7686,15 @@ dependencies = [ [[package]] name = "rtnetlink" -version = "0.13.1" +version = "0.20.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7a552eb82d19f38c3beed3f786bd23aa434ceb9ac43ab44419ca6d67a7e186c0" +checksum = "4b960d5d873a75b5be9761b1e73b146f52dddcd27bac75263f40fba686d4d7b5" dependencies = [ - "futures", + "futures-channel", + "futures-util", "log", "netlink-packet-core", "netlink-packet-route", - "netlink-packet-utils", "netlink-proto", "netlink-sys", "nix", @@ -7427,9 +7704,9 @@ dependencies = [ [[package]] name = "ruint" -version = "1.17.2" +version = "1.18.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c141e807189ad38a07276942c6623032d3753c8859c146104ac2e4d68865945a" +checksum = "0298da754d1395046b0afdc2f20ee76d29a8ae310cd30ffa84ed42acba9cb12a" dependencies = [ "alloy-rlp", "ark-ff 0.3.0", @@ -7444,8 +7721,8 @@ dependencies = [ "parity-scale-codec", "primitive-types", "proptest", - "rand 0.8.5", - "rand 0.9.2", + "rand 0.8.6", + "rand 0.9.4", "rlp", "ruint-macro", "serde_core", @@ -7459,6 +7736,20 @@ version = "1.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "48fd7bd8a6377e15ad9d42a8ec25371b94ddc67abe7c8b9127bec79bebaaae18" +[[package]] +name = "rusqlite" +version = "0.37.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "165ca6e57b20e1351573e3729b958bc62f0e48025386970b6e4d29e7a7e71f3f" +dependencies = [ + "bitflags 2.11.1", + "fallible-iterator", + "fallible-streaming-iterator", + "hashlink 0.10.0", + "libsqlite3-sys", + "smallvec", +] + [[package]] name = "rust-ini" version = "0.21.3" @@ -7477,9 +7768,9 @@ checksum = "08d43f7aa6b08d49f382cde6a7982047c3426db949b1424bc4b7ec9ae12c6ce2" [[package]] name = "rustc-hash" -version = "2.1.1" +version = "2.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "357703d41365b4b27c590e3ed91eabb1b663f07c4c084095e60cbed4362dff0d" +checksum = "94300abf3f1ae2e2b8ffb7b58043de3d399c73fa6f4b73826402a5c457614dbe" [[package]] name = "rustc-hex" @@ -7502,7 +7793,7 @@ version = "0.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cfcb3a22ef46e85b45de6ee7e79d063319ebb6594faafcf1c225ea92ab6e9b92" dependencies = [ - "semver 1.0.27", + "semver 1.0.28", ] [[package]] @@ -7520,7 +7811,7 @@ version = "0.38.44" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fdb5bc1ae2baa591800df16c9ca78619bf65c0488b41b96ccec5d11220d8c154" dependencies = [ - "bitflags 2.11.0", + "bitflags 2.11.1", "errno", "libc", "linux-raw-sys 0.4.15", @@ -7533,7 +7824,7 @@ version = "1.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b6fe4565b9518b83ef4f91bb47ce29620ca828bd32cb7e408f0062e9930ba190" dependencies = [ - "bitflags 2.11.0", + "bitflags 2.11.1", "errno", "libc", "linux-raw-sys 0.12.1", @@ -7542,9 +7833,9 @@ dependencies = [ [[package]] name = "rustls" -version = "0.23.37" +version = "0.23.40" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "758025cb5fccfd3bc2fd74708fd4682be41d99e5dff73c377c0646c6012c73a4" +checksum = "ef86cd5876211988985292b91c96a8f2d298df24e75989a43a3c73f2d4d8168b" dependencies = [ "aws-lc-rs", "log", @@ -7570,19 +7861,46 @@ dependencies = [ [[package]] name = "rustls-pki-types" -version = "1.14.0" +version = "1.14.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "be040f8b0a225e40375822a563fa9524378b9d63112f53e19ffff34df5d33fdd" +checksum = "30a7197ae7eb376e574fe940d068c30fe0462554a3ddbe4eca7838e049c937a9" dependencies = [ "web-time", "zeroize", ] +[[package]] +name = "rustls-platform-verifier" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "26d1e2536ce4f35f4846aa13bff16bd0ff40157cdb14cc056c7b14ba41233ba0" +dependencies = [ + "core-foundation 0.10.1", + "core-foundation-sys", + "jni", + "log", + "once_cell", + "rustls", + "rustls-native-certs", + "rustls-platform-verifier-android", + "rustls-webpki", + "security-framework", + "security-framework-sys", + "webpki-root-certs", + "windows-sys 0.61.2", +] + +[[package]] +name = "rustls-platform-verifier-android" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f87165f0995f63a9fbeea62b64d10b4d9d8e78ec6d7d51fb2125fda7bb36788f" + [[package]] name = "rustls-webpki" -version = "0.103.9" +version = "0.103.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d7df23109aa6c1567d1c575b9952556388da57401e4ace1d15f79eedad0d8f53" +checksum = "61c429a8649f110dddef65e2a5ad240f747e85f7758a6bccc7e5777bd33f756e" dependencies = [ "aws-lc-rs", "ring", @@ -7635,9 +7953,9 @@ dependencies = [ [[package]] name = "schannel" -version = "0.1.28" +version = "0.1.29" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "891d81b926048e76efe18581bf793546b4c0eaf8448d72be8de2bbee5fd166e1" +checksum = "91c1b7e4904c873ef0710c1f407dde2e6287de2bebc1bbbf7d430bb7cbffd939" dependencies = [ "windows-sys 0.61.2", ] @@ -7700,11 +8018,22 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b50c5943d326858130af85e049f2661ba3c78b26589b8ab98e65e80ae44a1252" dependencies = [ "bitcoin_hashes", - "rand 0.8.5", - "secp256k1-sys", + "rand 0.8.6", + "secp256k1-sys 0.10.1", "serde", ] +[[package]] +name = "secp256k1" +version = "0.31.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2c3c81b43dc2d8877c216a3fccf76677ee1ebccd429566d3e67447290d0c42b2" +dependencies = [ + "bitcoin_hashes", + "rand 0.9.4", + "secp256k1-sys 0.11.0", +] + [[package]] name = "secp256k1-sys" version = "0.10.1" @@ -7714,13 +8043,22 @@ dependencies = [ "cc", ] +[[package]] +name = "secp256k1-sys" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dcb913707158fadaf0d8702c2db0e857de66eb003ccfdda5924b5f5ac98efb38" +dependencies = [ + "cc", +] + [[package]] name = "security-framework" version = "3.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b7f4bc775c73d9a02cde8bf7b2ec4c9d12743edf609006c7facc23998404cd1d" dependencies = [ - "bitflags 2.11.0", + "bitflags 2.11.1", "core-foundation 0.10.1", "core-foundation-sys", "libc", @@ -7748,9 +8086,13 @@ dependencies = [ [[package]] name = "semver" -version = "1.0.27" +version = "1.0.28" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d767eb0aabc880b29956c35734170f26ed551a859dbd361d140cdbeca61ab1e2" +checksum = "8a7852d02fc848982e0c167ef163aaff9cd91dc640ba85e263cb1ce46fae51cd" +dependencies = [ + "serde", + "serde_core", +] [[package]] name = "semver-parser" @@ -7831,9 +8173,9 @@ dependencies = [ [[package]] name = "serde_json" -version = "1.0.149" +version = "1.0.150" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "83fc039473c5595ace860d8c4fafa220ff474b3fc6bfdb4293327f1a37e94d86" +checksum = "e8014e44b4736ed0538adeecded0fce2a272f22dc9578a7eb6b2d9993c74cfb9" dependencies = [ "itoa", "memchr", @@ -7855,9 +8197,9 @@ dependencies = [ [[package]] name = "serde_spanned" -version = "1.0.4" +version = "1.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f8bbf91e5a4d6315eee45e704372590b30e260ee83af6639d64557f51b067776" +checksum = "6662b5879511e06e8999a8a235d848113e942c9124f211511b16466ee2995f26" dependencies = [ "serde_core", ] @@ -7876,15 +8218,16 @@ dependencies = [ [[package]] name = "serde_with" -version = "3.17.0" +version = "3.20.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "381b283ce7bc6b476d903296fb59d0d36633652b633b27f64db4fb46dcbfc3b9" +checksum = "e72c1c2cb7b223fafb600a619537a871c2818583d619401b785e7c0b746ccde2" dependencies = [ "base64", + "bs58", "chrono", "hex", "indexmap 1.9.3", - "indexmap 2.13.0", + "indexmap 2.14.0", "schemars 0.9.0", "schemars 1.2.1", "serde_core", @@ -7895,11 +8238,11 @@ dependencies = [ [[package]] name = "serde_with_macros" -version = "3.17.0" +version = "3.20.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a6d4e30573c8cb306ed6ab1dca8423eec9a463ea0e155f45399455e0368b27e0" +checksum = "b90c488738ecb4fb0262f41f43bc40efc5868d9fb744319ddf5f5317f417bfac" dependencies = [ - "darling 0.21.3", + "darling 0.23.0", "proc-macro2", "quote", "syn 2.0.117", @@ -7922,7 +8265,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e3bf829a2d51ab4a5ddf1352d8470c140cadc8301b2ae1789db023f01cedd6ba" dependencies = [ "cfg-if", - "cpufeatures", + "cpufeatures 0.2.17", "digest 0.10.7", ] @@ -7933,25 +8276,25 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a7507d819769d01a365ab707794a4084392c824f54a7a6a7862f8c3d0892b283" dependencies = [ "cfg-if", - "cpufeatures", + "cpufeatures 0.2.17", "digest 0.10.7", ] [[package]] name = "sha3" -version = "0.10.8" +version = "0.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "75872d278a8f37ef87fa0ddbda7802605cb18344497949862c0d4dcb291eba60" +checksum = "be176f1a57ce4e3d31c1a166222d9768de5954f811601fb7ca06fc8203905ce1" dependencies = [ - "digest 0.10.7", + "digest 0.11.3", "keccak", ] [[package]] name = "sha3-asm" -version = "0.1.5" +version = "0.1.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b31139435f327c93c6038ed350ae4588e2c70a13d50599509fee6349967ba35a" +checksum = "9f3f15d4e239ebe08413eed880e0f9b5af4b40ee0472543320efa91d488e96a7" dependencies = [ "cc", "cfg-if", @@ -7989,7 +8332,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b75a19a7a740b25bc7944bdee6172368f988763b744e3d4dfe753f6b4ece40cc" dependencies = [ "libc", - "mio 1.1.1", + "mio 1.2.0", "signal-hook", ] @@ -8027,9 +8370,19 @@ dependencies = [ [[package]] name = "simd-adler32" -version = "0.3.8" +version = "0.3.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e320a6c5ad31d271ad523dcf3ad13e2767ad8b1cb8f047f75a8aeaf8da139da2" +checksum = "703d5c7ef118737c72f1af64ad2f6f8c5e1921f818cdcb97b8fe6fc69bf66214" + +[[package]] +name = "simd_cesu8" +version = "1.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "94f90157bb87cddf702797c5dadfa0be7d266cdf49e22da2fcaa32eff75b2c33" +dependencies = [ + "rustc_version 0.4.1", + "simdutf8", +] [[package]] name = "simdutf8" @@ -8095,12 +8448,12 @@ dependencies = [ [[package]] name = "socket2" -version = "0.6.2" +version = "0.6.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "86f4aa3ad99f2088c990dfa82d367e19cb29268ed67c574d10d0a4bfe71f07e0" +checksum = "3a766e1110788c36f4fa1c2b71b387a7815aa65f88ce0229841826633d93723e" dependencies = [ "libc", - "windows-sys 0.60.2", + "windows-sys 0.61.2", ] [[package]] @@ -8124,9 +8477,9 @@ dependencies = [ [[package]] name = "sqlite-wasm-rs" -version = "0.5.2" +version = "0.5.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2f4206ed3a67690b9c29b77d728f6acc3ce78f16bf846d83c94f76400320181b" +checksum = "cdd578e94101503d97e2b286bbf8db2135035ca24b2ce4cbf3f9e2fb2bbf1eee" dependencies = [ "cc", "js-sys", @@ -8207,6 +8560,12 @@ version = "2.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "13c2bddecc57b384dee18652358fb23172facb8a2c51ccc10d74c157bdea3292" +[[package]] +name = "symlink" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a7973cce6668464ea31f176d85b13c7ab3bba2cb3b77a2ed26abd7801688010a" + [[package]] name = "syn" version = "1.0.109" @@ -8231,9 +8590,9 @@ dependencies = [ [[package]] name = "syn-solidity" -version = "1.5.7" +version = "1.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "53f425ae0b12e2f5ae65542e00898d500d4d318b4baf09f40fd0d410454e9947" +checksum = "ec005042c7d952febc1a3ef5b0f6674e9054aa836877a31c90b20e25b3d31744" dependencies = [ "paste", "proc-macro2", @@ -8275,6 +8634,20 @@ dependencies = [ "winapi", ] +[[package]] +name = "sysinfo" +version = "0.30.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0a5b4ddaee55fb2bea2bf0e5000747e5f5c0de765e5a5ff87f4cd106439f4bb3" +dependencies = [ + "cfg-if", + "core-foundation-sys", + "libc", + "ntapi", + "once_cell", + "windows 0.52.0", +] + [[package]] name = "sysinfo" version = "0.33.1" @@ -8291,11 +8664,11 @@ dependencies = [ [[package]] name = "system-configuration" -version = "0.6.1" +version = "0.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3c879d448e9d986b661742763247d3693ed13609438cf3d006f51f5368a5ba6b" +checksum = "a13f3d0daba03132c0aa9767f98351b3488edc2c100cda2d2ec2b04f3d8d3c8b" dependencies = [ - "bitflags 2.11.0", + "bitflags 2.11.1", "core-foundation 0.9.4", "system-configuration-sys", ] @@ -8324,9 +8697,9 @@ checksum = "55937e1799185b12863d447f42597ed69d9928686b8d88a1df17376a097d8369" [[package]] name = "tar" -version = "0.4.44" +version = "0.4.46" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1d863878d212c87a19c1a610eb53bb01fe12951c0501cf5a0d65f724914a667a" +checksum = "3f6221d9a6003c78398e3b239969f352578258df48c8eb051caadae0015bc840" dependencies = [ "filetime", "libc", @@ -8335,12 +8708,12 @@ dependencies = [ [[package]] name = "tempfile" -version = "3.26.0" +version = "3.27.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "82a72c767771b47409d2345987fda8628641887d5466101319899796367354a0" +checksum = "32497e9a4c7b38532efcdebeef879707aa9f794296a4f0244f6f69e9bc8574bd" dependencies = [ "fastrand", - "getrandom 0.4.1", + "getrandom 0.4.2", "once_cell", "rustix 1.1.4", "windows-sys 0.61.2", @@ -8437,9 +8810,9 @@ dependencies = [ [[package]] name = "tiff" -version = "0.10.3" +version = "0.11.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "af9605de7fee8d9551863fd692cce7637f548dbd9db9180fcc07ccc6d26c336f" +checksum = "b63feaf3343d35b6ca4d50483f94843803b0f51634937cc2ec519fc32232bc52" dependencies = [ "fax", "flate2", @@ -8477,7 +8850,9 @@ checksum = "743bd48c283afc0388f9b8827b976905fb217ad9e647fae3a379a9283c4def2c" dependencies = [ "deranged", "itoa", + "libc", "num-conv", + "num_threads", "powerfmt", "serde_core", "time-core", @@ -8511,9 +8886,9 @@ dependencies = [ [[package]] name = "tinystr" -version = "0.8.2" +version = "0.8.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "42d3e9c45c09de15d06dd8acf5f4e0e399e85927b7f00711024eb7ae10fa4869" +checksum = "c8323304221c2a851516f22236c5722a72eaa19749016521d6dff0824447d96d" dependencies = [ "displaydoc", "zerovec", @@ -8531,9 +8906,9 @@ dependencies = [ [[package]] name = "tinyvec" -version = "1.10.0" +version = "1.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bfa5fdc3bce6191a1dbc8c02d5c8bffcf557bafa17c124c5264a458f1b0613fa" +checksum = "3e61e67053d25a4e82c844e8424039d9745781b3fc4f32b8d55ed50f5f667ef3" dependencies = [ "tinyvec_macros", ] @@ -8546,17 +8921,17 @@ checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" [[package]] name = "tokio" -version = "1.49.0" +version = "1.52.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "72a2903cd7736441aac9df9d7688bd0ce48edccaadf181c3b90be801e81d3d86" +checksum = "8fc7f01b389ac15039e4dc9531aa973a135d7a4135281b12d7c1bc79fd57fffe" dependencies = [ "bytes", "libc", - "mio 1.1.1", + "mio 1.2.0", "parking_lot", "pin-project-lite", "signal-hook-registry", - "socket2 0.6.2", + "socket2 0.6.3", "tokio-macros", "tracing", "windows-sys 0.61.2", @@ -8564,9 +8939,9 @@ dependencies = [ [[package]] name = "tokio-macros" -version = "2.6.0" +version = "2.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "af407857209536a95c8e56f8231ef2c2e2aff839b22e07a1ffcbc617e9db9fa5" +checksum = "385a6cb71ab9ab790c5fe8d67f1645e6c450a7ce006a33de03daa956cf70a496" dependencies = [ "proc-macro2", "quote", @@ -8597,9 +8972,9 @@ dependencies = [ [[package]] name = "tokio-tungstenite" -version = "0.26.2" +version = "0.28.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7a9daff607c6d2bf6c16fd681ccb7eecc83e4e2cdc1ca067ffaadfca5de7f084" +checksum = "d25a406cddcc431a75d3d9afc6a7c0f7428d4891dd973e4d54c56b46127bf857" dependencies = [ "futures-util", "log", @@ -8631,13 +9006,26 @@ version = "0.9.12+spec-1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cf92845e79fc2e2def6a5d828f0801e29a2f8acc037becc5ab08595c7d5e9863" dependencies = [ - "indexmap 2.13.0", + "indexmap 2.14.0", "serde_core", "serde_spanned", - "toml_datetime", + "toml_datetime 0.7.5+spec-1.1.0", "toml_parser", "toml_writer", - "winnow", + "winnow 0.7.15", +] + +[[package]] +name = "toml" +version = "1.1.2+spec-1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "81f3d15e84cbcd896376e6730314d59fb5a87f31e4b038454184435cd57defee" +dependencies = [ + "serde_core", + "serde_spanned", + "toml_datetime 1.1.1+spec-1.1.0", + "toml_parser", + "winnow 1.0.3", ] [[package]] @@ -8649,32 +9037,41 @@ dependencies = [ "serde_core", ] +[[package]] +name = "toml_datetime" +version = "1.1.1+spec-1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3165f65f62e28e0115a00b2ebdd37eb6f3b641855f9d636d3cd4103767159ad7" +dependencies = [ + "serde_core", +] + [[package]] name = "toml_edit" -version = "0.23.10+spec-1.0.0" +version = "0.25.11+spec-1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "84c8b9f757e028cee9fa244aea147aab2a9ec09d5325a9b01e0a49730c2b5269" +checksum = "0b59c4d22ed448339746c59b905d24568fcbb3ab65a500494f7b8c3e97739f2b" dependencies = [ - "indexmap 2.13.0", - "toml_datetime", + "indexmap 2.14.0", + "toml_datetime 1.1.1+spec-1.1.0", "toml_parser", - "winnow", + "winnow 1.0.3", ] [[package]] name = "toml_parser" -version = "1.0.9+spec-1.1.0" +version = "1.1.2+spec-1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "702d4415e08923e7e1ef96cd5727c0dfed80b4d2fa25db9647fe5eb6f7c5a4c4" +checksum = "a2abe9b86193656635d2411dc43050282ca48aa31c2451210f4202550afb7526" dependencies = [ - "winnow", + "winnow 1.0.3", ] [[package]] name = "toml_writer" -version = "1.0.6+spec-1.1.0" +version = "1.1.1+spec-1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ab16f14aed21ee8bfd8ec22513f7287cd4a91aa92e44edfe2c17ddd004e92607" +checksum = "756daf9b1013ebe47a8776667b466417e2d4c5679d441c26230efd9ef78692db" [[package]] name = "tonic" @@ -8704,9 +9101,9 @@ dependencies = [ [[package]] name = "tonic" -version = "0.14.5" +version = "0.14.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fec7c61a0695dc1887c1b53952990f3ad2e3a31453e1f49f10e75424943a93ec" +checksum = "ac2a5518c70fa84342385732db33fb3f44bc4cc748936eb5833d2df34d6445ef" dependencies = [ "async-trait", "axum", @@ -8721,7 +9118,7 @@ dependencies = [ "hyper-util", "percent-encoding", "pin-project", - "socket2 0.6.2", + "socket2 0.6.3", "sync_wrapper", "tokio", "tokio-rustls", @@ -8730,14 +9127,14 @@ dependencies = [ "tower-layer", "tower-service", "tracing", - "webpki-roots 1.0.6", + "webpki-roots 1.0.7", ] [[package]] name = "tonic-build" -version = "0.14.5" +version = "0.14.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1882ac3bf5ef12877d7ed57aad87e75154c11931c2ba7e6cde5e22d63522c734" +checksum = "c68f61875ac5293cf72e6c8cf0158086428c82c37229e98c840878f1706b0322" dependencies = [ "prettyplease", "proc-macro2", @@ -8747,33 +9144,33 @@ dependencies = [ [[package]] name = "tonic-health" -version = "0.14.5" +version = "0.14.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f4ff0636fef47afb3ec02818f5bceb4377b8abb9d6a386aeade18bd6212f8eb7" +checksum = "fcfab99db777fba2802f0dfa861d1628d1ae916fb199d29819941f139ae85082" dependencies = [ "prost 0.14.3", "tokio", "tokio-stream", - "tonic 0.14.5", + "tonic 0.14.6", "tonic-prost", ] [[package]] name = "tonic-prost" -version = "0.14.5" +version = "0.14.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a55376a0bbaa4975a3f10d009ad763d8f4108f067c7c2e74f3001fb49778d309" +checksum = "50849f68853be452acf590cde0b146665b8d507b3b8af17261df47e02c209ea0" dependencies = [ "bytes", "prost 0.14.3", - "tonic 0.14.5", + "tonic 0.14.6", ] [[package]] name = "tonic-prost-build" -version = "0.14.5" +version = "0.14.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f3144df636917574672e93d0f56d7edec49f90305749c668df5101751bb8f95a" +checksum = "654e5643eff75d7f8c99197ce1440ed19a3474eada74c12bbac488b2cafdae27" dependencies = [ "prettyplease", "proc-macro2", @@ -8787,15 +9184,15 @@ dependencies = [ [[package]] name = "tonic-reflection" -version = "0.14.5" +version = "0.14.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "aaf0685a51e6d02b502ba0764002e766b7f3042aed13d9234925b6ffbfa3fca7" +checksum = "acccd136a4bf19810a1fde9c74edc6129b42a66b44d0c1c8aaa67aeb49a146a7" dependencies = [ "prost 0.14.3", "prost-types", "tokio", "tokio-stream", - "tonic 0.14.5", + "tonic 0.14.6", "tonic-prost", ] @@ -8807,7 +9204,7 @@ checksum = "ebe5ef63511595f1344e2d5cfa636d973292adc0eec1f0ad45fae9f0851ab1d4" dependencies = [ "futures-core", "futures-util", - "indexmap 2.13.0", + "indexmap 2.14.0", "pin-project-lite", "slab", "sync_wrapper", @@ -8820,11 +9217,11 @@ dependencies = [ [[package]] name = "tower-http" -version = "0.6.8" +version = "0.6.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d4e6559d53cc268e5031cd8429d05415bc4cb4aefc4aa5d6cc35fbf5b924a1f8" +checksum = "4cfcf7e2740e6fc6d4d688b4ef00650406bb94adf4731e43c096c3a19fe40840" dependencies = [ - "bitflags 2.11.0", + "bitflags 2.11.1", "bytes", "futures-core", "futures-util", @@ -8833,7 +9230,6 @@ dependencies = [ "http-body-util", "http-range-header", "httpdate", - "iri-string", "mime", "mime_guess", "percent-encoding", @@ -8843,7 +9239,7 @@ dependencies = [ "tower", "tower-layer", "tower-service", - "tracing", + "url", ] [[package]] @@ -8872,11 +9268,12 @@ dependencies = [ [[package]] name = "tracing-appender" -version = "0.2.4" +version = "0.2.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "786d480bce6247ab75f005b14ae1624ad978d3029d9113f0a22fa1ac773faeaf" +checksum = "050686193eb999b4bb3bc2acfa891a13da00f79734704c4b8b4ef1a10b368a3c" dependencies = [ "crossbeam-channel", + "symlink", "thiserror 2.0.18", "time", "tracing-subscriber", @@ -8934,9 +9331,9 @@ dependencies = [ [[package]] name = "tracing-subscriber" -version = "0.3.22" +version = "0.3.23" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2f30143827ddab0d256fd843b7a66d164e9f271cfa0dde49142c5ca0ca291f1e" +checksum = "cb7f578e5945fb242538965c2d0b04418d38ec25c79d160cd279bf0731c8d319" dependencies = [ "matchers", "nu-ansi-term", @@ -9011,16 +9408,16 @@ checksum = "e421abadd41a4225275504ea4d6566923418b7f05506fbc9c0fe86ba7396114b" [[package]] name = "tungstenite" -version = "0.26.2" +version = "0.28.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4793cb5e56680ecbb1d843515b23b6de9a75eb04b66643e256a396d43be33c13" +checksum = "8628dcc84e5a09eb3d8423d6cb682965dea9133204e8fb3efee74c2a0c259442" dependencies = [ "bytes", "data-encoding", "http", "httparse", "log", - "rand 0.9.2", + "rand 0.9.4", "rustls", "rustls-pki-types", "sha1", @@ -9036,9 +9433,9 @@ checksum = "bc7d623258602320d5c55d1bc22793b57daff0ec7efc270ea7d55ce1d5f5471c" [[package]] name = "typenum" -version = "1.19.0" +version = "1.20.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "562d481066bde0658276a35467c4af00bdc6ee726305698a55b86e61d7ad82bb" +checksum = "40ce102ab67701b8526c123c1bab5cbe42d7040ccfd0f64af1a385808d2f43de" [[package]] name = "ucd-trie" @@ -9090,9 +9487,9 @@ checksum = "e6e4313cd5fcd3dad5cafa179702e2b244f760991f45397d14d4ebf38247da75" [[package]] name = "unicode-segmentation" -version = "1.12.0" +version = "1.13.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f6ccf251212114b54433ec949fd6a7841275f9ada20dddd2f29e9ceea4501493" +checksum = "9629274872b2bfaf8d66f5f15725007f635594914870f65218920345aa11aa8c" [[package]] name = "unicode-truncate" @@ -9174,11 +9571,11 @@ checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821" [[package]] name = "uuid" -version = "1.21.0" +version = "1.23.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b672338555252d43fd2240c714dc444b8c6fb0a5c5335e65a07bba7742735ddb" +checksum = "ddd74a9687298c6858e9b88ec8935ec45d22e8fd5e6394fa1bd4e99a87789c76" dependencies = [ - "getrandom 0.4.1", + "getrandom 0.4.2", "js-sys", "wasm-bindgen", ] @@ -9195,6 +9592,22 @@ version = "0.2.15" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "accd4ea62f7bb7a82fe23066fb0957d48ef677f6eeb8215f372f52e48bb32426" +[[package]] +name = "vergen" +version = "8.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2990d9ea5967266ea0ccf413a4aa5c42a93dbcfda9cb49a97de6931726b12566" +dependencies = [ + "anyhow", + "cargo_metadata", + "cfg-if", + "regex", + "rustc_version 0.4.1", + "rustversion", + "sysinfo 0.30.13", + "time", +] + [[package]] name = "version_check" version = "0.9.5" @@ -9237,6 +9650,20 @@ dependencies = [ "winapi-util", ] +[[package]] +name = "wallet-tx-builder" +version = "0.1.0" +dependencies = [ + "bytes", + "nockapp", + "nockchain-math", + "nockchain-types", + "nockvm", + "noun-serde", + "thiserror 2.0.18", + "tracing", +] + [[package]] name = "want" version = "0.3.1" @@ -9254,11 +9681,11 @@ checksum = "ccf3ec651a847eb01de73ccad15eb7d99f80485de043efb2f370cd654f4ea44b" [[package]] name = "wasip2" -version = "1.0.2+wasi-0.2.9" +version = "1.0.3+wasi-0.2.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9517f9239f02c069db75e65f174b3da828fe5f5b945c4dd26bd25d89c03ebcf5" +checksum = "20064672db26d7cdc89c7798c48a0fdfac8213434a1186e5ef29fd560ae223d6" dependencies = [ - "wit-bindgen", + "wit-bindgen 0.57.1", ] [[package]] @@ -9267,14 +9694,14 @@ version = "0.4.0+wasi-0.3.0-rc-2026-01-06" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5428f8bf88ea5ddc08faddef2ac4a67e390b88186c703ce6dbd955e1c145aca5" dependencies = [ - "wit-bindgen", + "wit-bindgen 0.51.0", ] [[package]] name = "wasm-bindgen" -version = "0.2.113" +version = "0.2.121" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "60722a937f594b7fde9adb894d7c092fc1bb6612897c46368d18e7a20208eff2" +checksum = "49ace1d07c165b0864824eee619580c4689389afa9dc9ed3a4c75040d82e6790" dependencies = [ "cfg-if", "once_cell", @@ -9285,23 +9712,19 @@ dependencies = [ [[package]] name = "wasm-bindgen-futures" -version = "0.4.63" +version = "0.4.71" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8a89f4650b770e4521aa6573724e2aed4704372151bd0de9d16a3bbabb87441a" +checksum = "96492d0d3ffba25305a7dc88720d250b1401d7edca02cc3bcd50633b424673b8" dependencies = [ - "cfg-if", - "futures-util", "js-sys", - "once_cell", "wasm-bindgen", - "web-sys", ] [[package]] name = "wasm-bindgen-macro" -version = "0.2.113" +version = "0.2.121" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0fac8c6395094b6b91c4af293f4c79371c163f9a6f56184d2c9a85f5a95f3950" +checksum = "8e68e6f4afd367a562002c05637acb8578ff2dea1943df76afb9e83d177c8578" dependencies = [ "quote", "wasm-bindgen-macro-support", @@ -9309,9 +9732,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro-support" -version = "0.2.113" +version = "0.2.121" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ab3fabce6159dc20728033842636887e4877688ae94382766e00b180abac9d60" +checksum = "d95a9ec35c64b2a7cb35d3fead40c4238d0940c86d107136999567a4703259f2" dependencies = [ "bumpalo", "proc-macro2", @@ -9322,9 +9745,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-shared" -version = "0.2.113" +version = "0.2.121" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "de0e091bdb824da87dc01d967388880d017a0a9bc4f3bdc0d86ee9f9336e3bb5" +checksum = "c4e0100b01e9f0d03189a92b96772a1fb998639d981193d7dbab487302513441" dependencies = [ "unicode-ident", ] @@ -9346,7 +9769,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bb0e353e6a2fbdc176932bbaab493762eb1255a7900fe0fea1a2f96c296cc909" dependencies = [ "anyhow", - "indexmap 2.13.0", + "indexmap 2.14.0", "wasm-encoder", "wasmparser", ] @@ -9357,10 +9780,10 @@ version = "0.244.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "47b807c72e1bac69382b3a6fb3dbe8ea4c0ed87ff5629b8685ae6b9a611028fe" dependencies = [ - "bitflags 2.11.0", + "bitflags 2.11.1", "hashbrown 0.15.5", - "indexmap 2.13.0", - "semver 1.0.27", + "indexmap 2.14.0", + "semver 1.0.28", ] [[package]] @@ -9379,9 +9802,9 @@ dependencies = [ [[package]] name = "web-sys" -version = "0.3.90" +version = "0.3.98" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "705eceb4ce901230f8625bd1d665128056ccbe4b7408faa625eec1ba80f59a97" +checksum = "4b572dff8bcf38bad0fa19729c89bb5748b2b9b1d8be70cf90df697e3a8f32aa" dependencies = [ "js-sys", "wasm-bindgen", @@ -9397,20 +9820,29 @@ dependencies = [ "wasm-bindgen", ] +[[package]] +name = "webpki-root-certs" +version = "1.0.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f31141ce3fc3e300ae89b78c0dd67f9708061d1d2eda54b8209346fd6be9a92c" +dependencies = [ + "rustls-pki-types", +] + [[package]] name = "webpki-roots" version = "0.26.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "521bc38abb08001b01866da9f51eb7c5d647a19260e00054a8c7fd5f9e57f7a9" dependencies = [ - "webpki-roots 1.0.6", + "webpki-roots 1.0.7", ] [[package]] name = "webpki-roots" -version = "1.0.6" +version = "1.0.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "22cfaf3c063993ff62e73cb4311efde4db1efb31ab78a3e5c457939ad5cc0bed" +checksum = "52f5ee44c96cf55f1b349600768e3ece3a8f26010c05265ab73f945bb1a2eb9d" dependencies = [ "rustls-pki-types", ] @@ -9435,13 +9867,11 @@ dependencies = [ [[package]] name = "which" -version = "8.0.0" +version = "8.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d3fabb953106c3c8eea8306e4393700d7657561cb43122571b172bbfb7c7ba1d" +checksum = "81995fafaaaf6ae47a7d0cc83c67caf92aeb7e5331650ae6ff856f7c0c60c459" dependencies = [ - "env_home", - "rustix 1.1.4", - "winsafe", + "libc", ] [[package]] @@ -9483,11 +9913,11 @@ checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" [[package]] name = "windows" -version = "0.53.0" +version = "0.52.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "efc5cf48f83140dcaab716eeaea345f9e93d0018fb81162753a3f76c3397b538" +checksum = "e48a53791691ab099e5e2ad123536d0fff50652600abaf43bbf952894110d0be" dependencies = [ - "windows-core 0.53.0", + "windows-core 0.52.0", "windows-targets 0.52.6", ] @@ -9501,13 +9931,33 @@ dependencies = [ "windows-targets 0.52.6", ] +[[package]] +name = "windows" +version = "0.62.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "527fadee13e0c05939a6a05d5bd6eec6cd2e3dbd648b9f8e447c6518133d8580" +dependencies = [ + "windows-collections", + "windows-core 0.62.2", + "windows-future", + "windows-numerics", +] + +[[package]] +name = "windows-collections" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "23b2d95af1a8a14a3c7367e1ed4fc9c20e0a26e79551b1454d72583c97cc6610" +dependencies = [ + "windows-core 0.62.2", +] + [[package]] name = "windows-core" -version = "0.53.0" +version = "0.52.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9dcc5b895a6377f1ab9fa55acedab1fd5ac0db66ad1e6c7f47e28a22e446a5dd" +checksum = "33ab640c8d7e35bf8ba19b884ba838ceb4fba93a4e8c65a9059d08afcfc683d9" dependencies = [ - "windows-result 0.1.2", "windows-targets 0.52.6", ] @@ -9536,6 +9986,17 @@ dependencies = [ "windows-strings", ] +[[package]] +name = "windows-future" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e1d6f90251fe18a279739e78025bd6ddc52a7e22f921070ccdc67dde84c605cb" +dependencies = [ + "windows-core 0.62.2", + "windows-link", + "windows-threading", +] + [[package]] name = "windows-implement" version = "0.57.0" @@ -9586,6 +10047,27 @@ version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5" +[[package]] +name = "windows-numerics" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6e2e40844ac143cdb44aead537bbf727de9b044e107a0f1220392177d15b0f26" +dependencies = [ + "windows-core 0.62.2", + "windows-link", +] + +[[package]] +name = "windows-registry" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "02752bf7fbdcce7f2a27a742f798510f3e5ad88dbe84871e5168e2120c3d5720" +dependencies = [ + "windows-link", + "windows-result 0.4.1", + "windows-strings", +] + [[package]] name = "windows-result" version = "0.1.2" @@ -9730,6 +10212,15 @@ dependencies = [ "windows_x86_64_msvc 0.53.1", ] +[[package]] +name = "windows-threading" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3949bd5b99cafdf1c7ca86b43ca564028dfe27d66958f2470940f73d86d75b37" +dependencies = [ + "windows-link", +] + [[package]] name = "windows_aarch64_gnullvm" version = "0.42.2" @@ -9912,29 +10403,19 @@ checksum = "d6bbff5f0aada427a1e5a6da5f1f98158182f26556f345ac9e04d36d0ebed650" [[package]] name = "winnow" -version = "0.7.14" +version = "0.7.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5a5364e9d77fcdeeaa6062ced926ee3381faa2ee02d3eb83a5c27a8825540829" -dependencies = [ - "memchr", -] +checksum = "df79d97927682d2fd8adb29682d1140b343be4ac0f08fd68b7765d9c059d3945" [[package]] -name = "winreg" -version = "0.50.0" +name = "winnow" +version = "1.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "524e57b2c537c0f9b1e69f1965311ec12182b4122e45035b1508cd24d2adadb1" +checksum = "0592e1c9d151f854e6fd382574c3a0855250e1d9b2f99d9281c6e6391af352f1" dependencies = [ - "cfg-if", - "windows-sys 0.48.0", + "memchr", ] -[[package]] -name = "winsafe" -version = "0.0.19" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d135d17ab770252ad95e9a872d365cf3090e3be864a34ab46f48555993efc904" - [[package]] name = "wit-bindgen" version = "0.51.0" @@ -9944,6 +10425,12 @@ dependencies = [ "wit-bindgen-rust-macro", ] +[[package]] +name = "wit-bindgen" +version = "0.57.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ebf944e87a7c253233ad6766e082e3cd714b5d03812acc24c318f549614536e" + [[package]] name = "wit-bindgen-core" version = "0.51.0" @@ -9963,7 +10450,7 @@ checksum = "b7c566e0f4b284dd6561c786d9cb0142da491f46a9fbed79ea69cdad5db17f21" dependencies = [ "anyhow", "heck", - "indexmap 2.13.0", + "indexmap 2.14.0", "prettyplease", "syn 2.0.117", "wasm-metadata", @@ -9993,8 +10480,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9d66ea20e9553b30172b5e831994e35fbde2d165325bec84fc43dbf6f4eb9cb2" dependencies = [ "anyhow", - "bitflags 2.11.0", - "indexmap 2.13.0", + "bitflags 2.11.1", + "indexmap 2.14.0", "log", "serde", "serde_derive", @@ -10013,9 +10500,9 @@ checksum = "ecc8ac4bc1dc3381b7f59c34f00b67e18f910c2c0f50015669dde7def656a736" dependencies = [ "anyhow", "id-arena", - "indexmap 2.13.0", + "indexmap 2.14.0", "log", - "semver 1.0.27", + "semver 1.0.28", "serde", "serde_derive", "serde_json", @@ -10025,9 +10512,9 @@ dependencies = [ [[package]] name = "writeable" -version = "0.6.2" +version = "0.6.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9edde0db4769d2dc68579893f2306b26c6ecfbe0ef499b013d731b7b9247e0b9" +checksum = "1ffae5123b2d3fc086436f8834ae3ab053a283cfac8fe0a0b8eaae044768a4c4" [[package]] name = "ws_stream_wasm" @@ -10146,13 +10633,13 @@ dependencies = [ [[package]] name = "yaml-rust2" -version = "0.10.4" +version = "0.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2462ea039c445496d8793d052e13787f2b90e750b833afee748e601c17621ed9" +checksum = "631a50d867fafb7093e709d75aaee9e0e0d5deb934021fcea25ac2fe09edc51e" dependencies = [ "arraydeque", "encoding_rs", - "hashlink", + "hashlink 0.11.0", ] [[package]] @@ -10165,8 +10652,8 @@ dependencies = [ "lazy_static", "log", "notify", - "rand 0.8.5", - "semver 1.0.27", + "rand 0.8.6", + "semver 1.0.28", "sysinfo 0.28.4", ] @@ -10179,11 +10666,21 @@ dependencies = [ "time", ] +[[package]] +name = "yasna" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b5f6765e852b9b4dc8e2a76843e4d64d1cea8e79bcde0b6901aea8e7c7f08282" +dependencies = [ + "bit-vec 0.9.1", + "time", +] + [[package]] name = "yoke" -version = "0.8.1" +version = "0.8.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "72d6e5c6afb84d73944e5cedb052c4680d5657337201555f9f2a16b7406d4954" +checksum = "abe8c5fda708d9ca3df187cae8bfb9ceda00dd96231bed36e445a1a48e66f9ca" dependencies = [ "stable_deref_trait", "yoke-derive", @@ -10192,9 +10689,9 @@ dependencies = [ [[package]] name = "yoke-derive" -version = "0.8.1" +version = "0.8.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b659052874eb698efe5b9e8cf382204678a0086ebf46982b79d6ca3182927e5d" +checksum = "de844c262c8848816172cef550288e7dc6c7b7814b4ee56b3e1553f275f1858e" dependencies = [ "proc-macro2", "quote", @@ -10204,18 +10701,18 @@ dependencies = [ [[package]] name = "zerocopy" -version = "0.8.40" +version = "0.8.48" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a789c6e490b576db9f7e6b6d661bcc9799f7c0ac8352f56ea20193b2681532e5" +checksum = "eed437bf9d6692032087e337407a86f04cd8d6a16a37199ed57949d415bd68e9" dependencies = [ "zerocopy-derive", ] [[package]] name = "zerocopy-derive" -version = "0.8.40" +version = "0.8.48" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f65c489a7071a749c849713807783f70672b28094011623e200cb86dcb835953" +checksum = "70e3cd084b1788766f53af483dd21f93881ff30d7320490ec3ef7526d203bad4" dependencies = [ "proc-macro2", "quote", @@ -10224,18 +10721,18 @@ dependencies = [ [[package]] name = "zerofrom" -version = "0.1.6" +version = "0.1.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "50cc42e0333e05660c3587f3bf9d0478688e15d870fab3346451ce7f8c9fbea5" +checksum = "0ec05a11813ea801ff6d75110ad09cd0824ddba17dfe17128ea0d5f68e6c5272" dependencies = [ "zerofrom-derive", ] [[package]] name = "zerofrom-derive" -version = "0.1.6" +version = "0.1.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d71e5d6e06ab090c67b5e44993ec16b72dcbaabc526db883a360057678b48502" +checksum = "11532158c46691caf0f2593ea8358fed6bbf68a0315e80aae9bd41fbade684a1" dependencies = [ "proc-macro2", "quote", @@ -10265,9 +10762,9 @@ dependencies = [ [[package]] name = "zerotrie" -version = "0.2.3" +version = "0.2.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2a59c17a5562d507e4b54960e8569ebee33bee890c70aa3fe7b97e85a9fd7851" +checksum = "0f9152d31db0792fa83f70fb2f83148effb5c1f5b8c7686c3459e361d9bc20bf" dependencies = [ "displaydoc", "yoke", @@ -10276,9 +10773,9 @@ dependencies = [ [[package]] name = "zerovec" -version = "0.11.5" +version = "0.11.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6c28719294829477f525be0186d13efa9a3c602f7ec202ca9e353d310fb9a002" +checksum = "90f911cbc359ab6af17377d242225f4d75119aec87ea711a880987b18cd7b239" dependencies = [ "yoke", "zerofrom", @@ -10287,9 +10784,9 @@ dependencies = [ [[package]] name = "zerovec-derive" -version = "0.11.2" +version = "0.11.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eadce39539ca5cb3985590102671f2567e659fca9666581ad3411d59207951f3" +checksum = "625dc425cab0dca6dc3c3319506e6593dcb08a9f387ea3b284dbd52a92c40555" dependencies = [ "proc-macro2", "quote", @@ -10308,6 +10805,7 @@ dependencies = [ "hex-literal", "ibig", "nockchain-math", + "nockchain-types", "nockvm", "nockvm_macros", "noun-serde", @@ -10327,15 +10825,15 @@ checksum = "b8848ee67ecc8aedbaf3e4122217aff892639231befc6a1b58d29fff4c2cabaa" [[package]] name = "zune-core" -version = "0.4.12" +version = "0.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3f423a2c17029964870cfaabb1f13dfab7d092a62a29a89264f4d36990ca414a" +checksum = "cb8a0807f7c01457d0379ba880ba6322660448ddebc890ce29bb64da71fb40f9" [[package]] name = "zune-jpeg" -version = "0.4.21" +version = "0.5.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "29ce2c8a9384ad323cf564b67da86e21d3cfdff87908bc1223ed5c99bc792713" +checksum = "27bc9d5b815bc103f142aa054f561d9187d191692ec7c2d1e2b4737f8dbd7296" dependencies = [ "zune-core", ] diff --git a/Cargo.toml b/Cargo.toml index 4ad5363df..566d68835 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,7 +1,6 @@ -resolver = "2" - [workspace] -members = ["crates/bridge", "crates/equix-latency", "crates/habit", "crates/hoon", "crates/hoonc", "crates/kernels/bridge", "crates/kernels/dumb", "crates/kernels/miner", "crates/kernels/nockchain-peek", "crates/kernels/verifier", "crates/kernels/wallet", "crates/nockapp", "crates/nockapp-grpc", "crates/nockapp-grpc-proto", "crates/nockchain", "crates/nockchain-api", "crates/nockchain-explorer-tui", "crates/nockchain-libp2p-io", "crates/nockchain-math", "crates/nockchain-peek", "crates/nockchain-types", "crates/nockchain-wallet", "crates/nockup", "crates/nockvm/rust/ibig", "crates/nockvm/rust/murmur3", "crates/nockvm/rust/nockvm", "crates/nockvm/rust/nockvm_macros", "crates/noun-serde", "crates/noun-serde-derive", "crates/raw-tx-checker", "crates/zkvm-jetpack"] +members = ["crates/approver", "crates/bridge", "crates/equix-latency", "crates/habit", "crates/hoon", "crates/hoonc", "crates/kernels", "crates/kernels/approver", "crates/kernels/bridge", "crates/kernels/dumb", "crates/kernels/miner", "crates/kernels/nockchain-peek", "crates/kernels/verifier", "crates/kernels/wallet", "crates/nockapp", "crates/nockapp-grpc", "crates/nockapp-grpc-proto", "crates/nockchain", "crates/nockchain-api", "crates/nockchain-explorer-tui", "crates/nockchain-libp2p-io", "crates/nockchain-math", "crates/nockchain-peek", "crates/nockchain-types", "crates/nockchain-wallet", "crates/nockup", "crates/nockvm/rust/ibig", "crates/nockvm/rust/murmur3", "crates/nockvm/rust/nockvm", "crates/nockvm/rust/nockvm_macros", "crates/noun-serde", "crates/noun-serde-derive", "crates/raw-tx-checker", "crates/wallet-tx-builder", "crates/zkvm-jetpack"] +resolver = "2" [workspace.package] version = "0.1.0" @@ -10,10 +9,11 @@ edition = "2021" [workspace.dependencies] aes = { version = "0.8.3", default-features = false } aes-siv = { version = "0.7.0", default-features = false, features = ["alloc"] } -alloy = "1.1.2" +alloy = "=1.8.2" anyhow = "1.0" argon2 = "0.5.3" arrayref = "0.3.7" +askama = "0.12.1" assert_cmd = "2.0" async-trait = "0.1" axum = "0.8.1" @@ -22,7 +22,7 @@ backon = { version = "1.6", features = ["tokio-sleep"] } bardecoder = "0.5.0" bincode = "2.0.0-rc.3" bitcoincore-rpc = "0.19.0" -bitvec = "1.0.1" +bitvec = { version = "1.0.1", default-features = true } blake3 = { version = "1.5.1", features = ["serde"] } bs58 = "0.5.1" byteorder = "1.5.0" @@ -42,6 +42,7 @@ curve25519-dalek = { version = "4.1.1", default-features = false } dashmap = "6.1.0" deadpool-diesel = { version = "0.6.1", features = ["rt_tokio_1", "sqlite"] } diesel = { version = "2.1.4", features = ["sqlite"] } +diesel_migrations = { version = "2.3.2", features = ["sqlite"] } dirs = "6.0.0" divan = "0.1.21" ed25519-dalek = { version = "2.1.0", default-features = false } @@ -70,10 +71,10 @@ memmap2 = "^0.9.5" nu-ansi-term = "0.50" num-bigint = "0.4.6" num-derive = "0.4.2" -num-traits = "0.2" +num-traits = { version = "0.2", default-features = true } num_cpus = "1.16.0" once_cell = "1.21.3" -op-alloy = "0.23.1" +op-alloy = "=0.23.1" openssl-sys = "0.9" opentelemetry = { version = "0.30.0", features = [ "trace", @@ -99,7 +100,7 @@ qrcode = "0.14.1" quickcheck = "1.0.3" quickcheck_macros = "1.0" quote = "1.0.37" -rand = { version = "0.9.2", features = ["std"] } +rand = { version = "0.9.2", default-features = true, features = ["std"] } ratatui = "0.29.0" rayon = "1.8.0" rcgen = "0.14.3" @@ -110,8 +111,11 @@ reqwest = { version = "0.12", default-features = false, features = [ "json", "blocking", ] } -rkyv = "0.8.10" +rkyv = "0.8.13" +rusqlite = { version = "0.37.0", features = ["bundled"] } +rustix = { version = "0.38", features = ["fs", "mm"] } rustls = "0.23.0" +rustls-pemfile = "2.0.0" rustls-pki-types = "1.13.1" serde = "1.0.217" serde_bytes = { version = "0.11.15", features = ["alloc"] } @@ -171,6 +175,7 @@ tracing-test = "0.2.5" tracing-tracy = { version = "0.11.4" } tracy-client = { version = "0.18.2" } unroll = "0.1.5" +uuid = { version = "1.23.1", features = ["v4"] } vergen = "8.3.2" void = "1.0.2" walkdir = "2.5.0" @@ -197,6 +202,12 @@ path = "crates/hoonc" [workspace.dependencies.ibig] path = "crates/nockvm/rust/ibig" +[workspace.dependencies.kernels] +path = "crates/kernels" + +[workspace.dependencies.kernels-open-approver] +path = "crates/kernels/approver" + [workspace.dependencies.kernels-open-bridge] path = "crates/kernels/bridge" @@ -223,6 +234,7 @@ path = "crates/nockapp" [workspace.dependencies.nockapp-grpc] path = "crates/nockapp-grpc" +default-features = true [workspace.dependencies.nockapp-grpc-proto] path = "crates/nockapp-grpc-proto" @@ -260,6 +272,17 @@ path = "crates/noun-serde-derive" [workspace.dependencies.zkvm-jetpack] path = "crates/zkvm-jetpack" +[patch] + +[patch.crates-io] +op-alloy-consensus = { git = "https://github.com/ethereum-optimism/optimism", rev = "0c6a03eac768122819c05b71f233bba1e90cf426", package = "op-alloy-consensus" } +op-alloy-network = { git = "https://github.com/ethereum-optimism/optimism", rev = "0c6a03eac768122819c05b71f233bba1e90cf426", package = "op-alloy-network" } +op-alloy-rpc-types = { git = "https://github.com/ethereum-optimism/optimism", rev = "0c6a03eac768122819c05b71f233bba1e90cf426", package = "op-alloy-rpc-types" } + +# [lints.clippy] +# pedantic = { level = "warn", priority = -1 } +# restriction = { level = "warn", priority = -2 } + # Comment out this block if gdb is lacking info [profile.dev] # default settings, no opt-level, full build: 22 seconds, incremental: 1 second diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 000000000..f6162d1aa --- /dev/null +++ b/Dockerfile @@ -0,0 +1,14 @@ +FROM ubuntu:24.04 + +RUN apt-get update \ + && apt-get install -y --no-install-recommends \ + ca-certificates \ + libssl3 \ + && rm -rf /var/lib/apt/lists/* + +COPY target/release/nockchain /usr/local/bin/nockchain + +ENV RUST_LOG=info,nockchain=info,nockchain_libp2p_io=info,libp2p=info,libp2p_quic=info +ENV MINIMAL_LOG_FORMAT=true + +ENTRYPOINT ["/usr/local/bin/nockchain"] diff --git a/Makefile b/Makefile index 797e5d704..44b791ff1 100644 --- a/Makefile +++ b/Makefile @@ -11,6 +11,31 @@ export MINIMAL_LOG_FORMAT ?= true export MINING_PKH ?= 9yPePjfWAdUnzaQKyxcRXKRa5PpUzKKEwtpECBZsUYt9Jd7egSDEWoV export +ZIGBUILD_TARGET ?= x86_64-unknown-linux-gnu.2.39 +DOCKER_IMAGE ?= nockchain-local +DOCKER_MEM ?= 32g +# DOCKER_MEM_SWAP ?= 32g +DOCKER_P2P_PORT ?= 30000 +DOCKER_DATA_DIR ?= $(CURDIR)/.data.nockchain +DOCKER_NOCKCHAIN_ENVS ?= +DOCKER_NOCKCHAIN_ARGS ?= +DOCKER_METRICS_COMPOSE ?= docker-compose.metrics.yml +DOCKER_METRICS_NETWORK ?= nockchain-metrics +INFLUXDB_VERSION ?= 2.7 +TELEGRAF_VERSION ?= 1.30 +STATSD_HOST ?= telegraf +STATSD_PORT ?= 8125 +GENESIS_SYNC_RUST_LOG ?= info,nockapp::kernel::form=info,nockapp::kernel::boot=info,nockapp::utils::durability=info +GENESIS_SYNC_MINIMAL_LOG_FORMAT ?= true +GENESIS_SYNC_PEER ?= +GENESIS_SYNC_COMMON_ARGS ?= +GENESIS_SYNC_EXTRA_ARGS ?= +GENESIS_SYNC_NODE_CMD ?= cargo run --release --bin nockchain -- +GENESIS_SYNC_DATA_DIR_ON ?= $(CURDIR)/.data.nockchain-sync-fsync-on +GENESIS_SYNC_DATA_DIR_OFF ?= $(CURDIR)/.data.nockchain-sync-fsync-off +GENESIS_SYNC_BIND_PORT_ON ?= 31000 +GENESIS_SYNC_BIND_PORT_OFF ?= 31001 + .PHONY: build build: build-hoon-all build-rust $(call show_env_vars) @@ -20,15 +45,97 @@ build: build-hoon-all build-rust build-rust: cargo build --release +.PHONY: contracts-deps +contracts-deps: ## Install Solidity dependencies for bridge crate + $(MAKE) -C crates/bridge/contracts deps + +.PHONY: install-cargo-zigbuild +install-cargo-zigbuild: + cargo install --locked cargo-zigbuild + +.PHONY: zig-build-bridge +zig-build-bridge: + cargo zigbuild --release --target $(ZIGBUILD_TARGET) --bin bridge + .PHONY: build-nockchain-jemalloc build-nockchain-jemalloc: cargo build --release --features jemalloc --bin nockchain +.PHONY: build-nockchain-bridge-tui +build-nockchain-bridge-tui: + cargo build --release --bin nockchain-bridge-tui + ## Run all tests .PHONY: test test: cargo test --release +.PHONY: bench-nockchain-kernel +bench-nockchain-kernel: + cargo run --release -p nockchain --bin bench_nockchain_kernel -- --skip-mining + +.PHONY: bench-nockchain-checkpoint-block +bench-nockchain-checkpoint-block: + cargo run --release -p nockchain --bin bench_nockchain_checkpoint_block -- + +.PHONY: docker-nockchain +docker-nockchain: docker-nockchain-build docker-nockchain-run + +.PHONY: docker-nockchain-pma-persist +docker-nockchain-pma-persist: docker-nockchain-build + $(MAKE) docker-nockchain-run \ + DOCKER_NOCKCHAIN_ENVS="-e NOCK_PMA_PERSIST=1" \ + DOCKER_NOCKCHAIN_ARGS="--pma-persist $(DOCKER_NOCKCHAIN_ARGS)" + +.PHONY: docker-nockchain-build +docker-nockchain-build: + docker build -t $(DOCKER_IMAGE) . +# --checkpoint-mode stream \ +.PHONY: docker-nockchain-run +docker-nockchain-run: + mkdir -p $(DOCKER_DATA_DIR) + @docker network inspect $(DOCKER_METRICS_NETWORK) >/dev/null 2>&1 || docker network create $(DOCKER_METRICS_NETWORK) + docker run --rm -it --name nockchain \ + --network $(DOCKER_METRICS_NETWORK) \ + --memory $(DOCKER_MEM) \ + -e RUST_BACKTRACE=1 \ + -e NOCK_PMA_TIMING=1 \ + -e NOCK_PMA_TIMING_DETAIL=1 \ + -e NOCK_STACK_TIMING_DETAIL=1 \ + -e STATSD_HOST=$(STATSD_HOST) \ + -e STATSD_PORT=$(STATSD_PORT) \ + $(DOCKER_NOCKCHAIN_ENVS) \ + -p $(DOCKER_P2P_PORT):$(DOCKER_P2P_PORT)/udp \ + -v $(DOCKER_DATA_DIR):/data/.data.nockchain \ + $(DOCKER_IMAGE) \ + --fast-sync --num-threads 0 \ + --save-interval 300000 \ + --gc-interval 900 \ + --data-dir /data/.data.nockchain \ + --identity-path /data/.data.nockchain/.nockchain_identity \ + --bind /ip4/0.0.0.0/udp/$(DOCKER_P2P_PORT)/quic-v1 \ + $(DOCKER_NOCKCHAIN_ARGS) + +.PHONY: docker-metrics +docker-metrics: + @docker network inspect $(DOCKER_METRICS_NETWORK) >/dev/null 2>&1 || docker network create $(DOCKER_METRICS_NETWORK) + docker compose -f $(DOCKER_METRICS_COMPOSE) up -d + +.PHONY: run-genesis-sync-fsync-on +run-genesis-sync-fsync-on: + NOCK_PMA_TIMING=1 NOCK_PMA_TIMING_DETAIL=1 RUST_LOG="$(GENESIS_SYNC_RUST_LOG)" MINIMAL_LOG_FORMAT="$(GENESIS_SYNC_MINIMAL_LOG_FORMAT)" $(GENESIS_SYNC_NODE_CMD) \ + --data-dir "$(GENESIS_SYNC_DATA_DIR_ON)" \ + --identity-path "$(GENESIS_SYNC_DATA_DIR_ON)/.nockchain_identity" \ + --bind "/ip4/0.0.0.0/udp/$(GENESIS_SYNC_BIND_PORT_ON)/quic-v1" $(if $(GENESIS_SYNC_PEER),--peer "$(GENESIS_SYNC_PEER)") $(GENESIS_SYNC_COMMON_ARGS) $(GENESIS_SYNC_EXTRA_ARGS) + +.PHONY: run-genesis-sync-fsync-off +run-genesis-sync-fsync-off: + RUST_LOG="$(GENESIS_SYNC_RUST_LOG)" MINIMAL_LOG_FORMAT="$(GENESIS_SYNC_MINIMAL_LOG_FORMAT)" $(GENESIS_SYNC_NODE_CMD) \ + --disable-fsync \ + --data-dir "$(GENESIS_SYNC_DATA_DIR_OFF)" \ + --identity-path "$(GENESIS_SYNC_DATA_DIR_OFF)/.nockchain_identity" \ + --bind "/ip4/0.0.0.0/udp/$(GENESIS_SYNC_BIND_PORT_OFF)/quic-v1" $(if $(GENESIS_SYNC_PEER),--peer "$(GENESIS_SYNC_PEER)") $(GENESIS_SYNC_COMMON_ARGS) $(GENESIS_SYNC_EXTRA_ARGS) + .PHONY: fmt fmt: cargo fmt @@ -78,6 +185,10 @@ install-nockchain-peek: assets/peek.jam $(call show_env_vars) cargo install --locked --force --path crates/nockchain-peek --bin nockchain-peek + +HOONC ?= hoonc +HOONC_FLAGS ?= + .PHONY: ensure-dirs ensure-dirs: mkdir -p hoon @@ -87,9 +198,9 @@ ensure-dirs: build-trivial: ensure-dirs $(call show_env_vars) echo '%trivial' > hoon/trivial.hoon - hoonc --arbitrary hoon/trivial.hoon + $(HOONC) $(HOONC_FLAGS) --arbitrary hoon/trivial.hoon -HOON_TARGETS=assets/dumb.jam assets/wal.jam assets/miner.jam assets/peek.jam assets/bridge.jam assets/verifier.jam +HOON_TARGETS=assets/dumb.jam assets/wal.jam assets/miner.jam assets/peek.jam assets/bridge.jam assets/verifier.jam assets/approver.jam .PHONY: nuke-hoonc-data nuke-hoonc-data: @@ -112,6 +223,10 @@ build-hoon: ensure-dirs update-hoonc $(HOON_TARGETS) build-verifier: ensure-dirs update-hoonc assets/verifier.jam $(call show_env_vars) +.PHONY: build-approver +build-approver: ensure-dirs update-hoonc assets/approver.jam + $(call show_env_vars) + .PHONY: build-assets build-assets: ensure-dirs $(HOON_TARGETS) $(call show_env_vars) @@ -122,35 +237,35 @@ HOON_SRCS := $(find hoon -type file -name '*.hoon') assets/dumb.jam: ensure-dirs hoon/apps/dumbnet/outer.hoon $(HOON_SRCS) $(call show_env_vars) rm -f assets/dumb.jam - hoonc hoon/apps/dumbnet/outer.hoon hoon + $(HOONC) $(HOONC_FLAGS) hoon/apps/dumbnet/outer.hoon hoon mv out.jam assets/dumb.jam ## Build wal.jam with hoonc assets/wal.jam: ensure-dirs hoon/apps/wallet/wallet.hoon $(HOON_SRCS) $(call show_env_vars) rm -f assets/wal.jam - hoonc hoon/apps/wallet/wallet.hoon hoon + $(HOONC) $(HOONC_FLAGS) hoon/apps/wallet/wallet.hoon hoon mv out.jam assets/wal.jam ## Build mining.jam with hoonc assets/miner.jam: ensure-dirs hoon/apps/dumbnet/miner.hoon $(HOON_SRCS) $(call show_env_vars) rm -f assets/miner.jam - hoonc hoon/apps/dumbnet/miner.hoon hoon + $(HOONC) $(HOONC_FLAGS) hoon/apps/dumbnet/miner.hoon hoon mv out.jam assets/miner.jam ## Build peek.jam with hoonc assets/peek.jam: ensure-dirs hoon/apps/peek/peek.hoon $(HOON_SRCS) $(call show_env_vars) rm -f assets/peek.jam - hoonc hoon/apps/peek/peek.hoon hoon + $(HOONC) $(HOONC_FLAGS) hoon/apps/peek/peek.hoon hoon mv out.jam assets/peek.jam ## Build bridge.jam assets/bridge.jam: ensure-dirs hoon/apps/bridge/bridge.hoon $(HOON_SRCS) $(call show_env_vars) rm -f assets/bridge.jam - hoonc hoon/apps/bridge/bridge.hoon hoon + $(HOONC) $(HOONC_FLAGS) hoon/apps/bridge/bridge.hoon hoon mv out.jam assets/bridge.jam ## Build verifier.jam @@ -159,3 +274,10 @@ assets/verifier.jam: ensure-dirs hoon/apps/verifier/verifier.hoon $(HOON_SRCS) rm -f assets/verifier.jam hoonc hoon/apps/verifier/verifier.hoon hoon mv out.jam assets/verifier.jam + +## Build approver.jam +assets/approver.jam: ensure-dirs hoon/apps/approver/approver.hoon $(HOON_SRCS) + $(call show_env_vars) + rm -f assets/approver.jam + hoonc hoon/apps/approver/approver.hoon hoon + mv out.jam assets/approver.jam diff --git a/PMA-FAQ.md b/PMA-FAQ.md new file mode 100644 index 000000000..318e84b80 --- /dev/null +++ b/PMA-FAQ.md @@ -0,0 +1,146 @@ +# PMA FAQ + +This FAQ is for node operators running a PMA-enabled Nockchain release. + +PMA means Persistent Memory Arena. It changes how Nockchain stores and reloads the NockApp kernel state. It is not a protocol feature, not a git artifact, and not a replacement for your keys. + +## What changes for operators? + +In a PMA-enabled release, Nockchain stores the live kernel state in a file-backed persistent memory arena under the node data directory. The main operator-facing result is lower steady-state resident memory for the NockApp/Serf state and faster persistence after accepted events. + +You should not need to change the command or systemd unit that runs `nockchain`. On first boot, the PMA version should discover the existing data directory and migrate from the latest checkpoint if PMA state is not present yet. + +## What happens the first time I run the PMA version? + +If the data directory has legacy checkpoints but no usable PMA or PMA event log, boot uses the latest checkpoint as a bootstrap boundary. + +The checkpoint jam is treated like a state jam: + +- The kernel state is imported. +- The checkpoint cold jet cache is ignored. +- A fresh runtime cold cache is created. +- PMA state and metadata are written. +- An epoch snapshot is created. +- The SQLite event log starts recording accepted events after the checkpoint boundary. + +That last point matters. The node is not replaying every historical event into SQLite. The checkpoint establishes the trusted starting state, and the PMA event log records new accepted events after that point. During this bootstrap window, an empty event log plus PMA metadata at the checkpoint event is expected. + +## Will first boot use more memory than normal PMA boot? + +Yes. First-time migration from a checkpoint still has to cue and hydrate the checkpoint state before it can copy that state into PMA. Do not shrink a server's memory allocation before its first successful PMA boot. + +After the PMA state has been created, later boots should use the PMA fast path and avoid the large checkpoint hydration peak. Let the node boot, build PMA state, and run for a bit before lowering memory limits. + +## What should I see in the logs? + +On first migration from a checkpoint, look for a line like: + +```text +Boot source: checkpoint ... empty_event_log_bootstrap=true checkpoint_cold_state=empty +``` + +On later PMA boots, look for a line like: + +```text +Boot source: PMA ... +``` + +During normal event processing, PMA timing logs include fields such as `event_eval_ms`, `preserve_ms`, `durable_append_ms`, and `cleanup_total_ms`. These split event execution from PMA/event-log durability work. + +## Can I revert to the previous non-PMA version? + +The PMA migration does not delete existing checkpoint jams. If you start a PMA release and hit a problem, you should be able to stop it, revert to the previous non-PMA binary, and continue from the existing checkpoint data while reporting the PMA issue. + +Treat PMA as a forward migration path once you are satisfied it is healthy. The rollback story is for operational escape hatches, not for bouncing back and forth indefinitely. + +## Is PMA a cache? + +No, not in the disposable sense. PMA is the node's durable local kernel-state store after migration. + +Some runtime structures are still caches. PMA does not persist or restore derived runtime caches such as `hot`, `warm`, `cold`, jet-test HAMTs, or memo tables. Those are rebuilt on boot. The durable PMA state is the kernel state itself. + +## Why does PMA rebuild cold jet state on boot? + +The cold jet state is an optimization cache built out of HAMTs and linked runtime structures containing process-local pointers. Those pointers are not a stable disk format. + +PMA stores durable Nouns in offset form so they can be reloaded after the file is mapped again. The runtime cold cache is rebuilt from empty state and repopulated as the kernel runs. This is why checkpoint bootstrap imports checkpoint jams as state-only data and ignores serialized checkpoint cold state. + +## Can I copy the PMA directory from somebody else? + +No. Do not import or copy raw PMA files from third parties. + +Raw PMA files are local runtime artifacts. They are not consensus artifacts, release artifacts, or safe community bootstrap files. If you need to bootstrap a node from somebody else, use a trusted state jam or other supported bootstrap artifact instead. + +## Can I copy PMA files between my own machines? + +Only if you control both machines and know exactly what you are doing. Even then, the safer approach is to copy recovery artifacts rather than the live operative PMA. + +Prefer copying: + +- The SQLite event log and its sidecars. +- Verified snapshots and their manifests. +- Checkpoints, if you are still using a pre-PMA bootstrap path. + +Avoid copying just: + +- `pma/*.pma` +- `pma/*.meta` + +The operative PMA fast path trusts sidecar/trailer consistency and event-number agreement. Snapshots have stronger integrity metadata and are a better transfer boundary. + +## Should PMA files be checked into git? + +No. PMA files, checkpoints, snapshots, and event logs are large node-state artifacts. They do not belong in the git repository. + +The source tree contains code and small fixtures. Chain state belongs in the node data directory. + +## What files are in the PMA data directory? + +The exact directory depends on your `--data-dir`, but the important pieces are: + +- `pma/`: operative PMA slabs and PMA metadata sidecars. +- `event-log.sqlite3`: SQLite accepted-event log for events after the current recovery boundary. +- `event-log.sqlite3-wal` and `event-log.sqlite3-shm`: SQLite WAL sidecars when WAL mode is active. +- `checkpoints/`: legacy checkpoint jams, useful for first PMA bootstrap and rollback. +- snapshot PMA files and manifests under the PMA-managed snapshot paths. + +Do not manually edit these files. If you are troubleshooting, stop the node first and preserve copies before deleting anything. + +## What happens after a crash or power loss? + +PMA commits accepted events in a strict order: + +1. The accepted event is committed to SQLite. +2. The new kernel-state frontier is copied into PMA. +3. The PMA file is synced. +4. The PMA `.meta` sidecar is written last. + +On boot, Nockchain checks whether PMA and SQLite agree on the event boundary. If they agree, PMA can use the fast path. The first checkpoint bootstrap into an empty event log is the special case: PMA may be at the checkpoint event while SQLite has not recorded any post-checkpoint events yet. Outside that bootstrap case, if PMA is missing, invalid, behind SQLite, or ahead of SQLite, boot enters recovery and uses verified snapshots, checkpoints, and event-log replay as appropriate. + +## Does PMA make every workload use less memory? + +No. PMA reduces the memory pressure from the NockApp/Serf kernel state. It does not automatically reduce memory used by unrelated process caches. + +For example, public gRPC block explorer caches can still use substantial memory. PMA helps the node's kernel-state storage, but separate API caches need their own storage strategy. + +## Does PMA make syncing faster? + +It can, especially for workloads that spend a lot of time preserving or checkpointing large state. PMA avoids repeatedly jamming the whole subject for ordinary durability and avoids retaining the whole durable subject in anonymous heap memory during normal operation. + +It is not a magic protocol shortcut. Network, peer quality, block validation, disk speed, and API workload still matter. + +## Can I disable PMA? + +In a PMA-enabled Nockchain release, PMA is the normal operating mode for NockApps and Nockchain peers. It is not intended to be an optional cache layer that operators toggle on and off. + +Development and testing tools may expose lower-level knobs, but production operators should treat PMA as the persistence mode for that release. + +## Is it safe to delete checkpoints after PMA bootstraps? + +Do not rush to delete them. Existing checkpoints are useful as a rollback and recovery aid while you are validating a PMA upgrade. + +Once you are confident the node has booted from PMA repeatedly and has healthy snapshots/event logs, checkpoint retention becomes an operator storage-policy decision. If in doubt, keep the latest known-good checkpoint until the PMA release has aged on your machine. + +## Where are the deeper PMA docs? + +See [`docs/pma/`](./docs/pma/) for PMA durability, design, memory-attribution, and noun-provenance docs. diff --git a/PROTOCOL.md b/PROTOCOL.md index ca83a6724..b49f06aa1 100644 --- a/PROTOCOL.md +++ b/PROTOCOL.md @@ -28,6 +28,7 @@ Legend: | Seq | Codename | Version | Status | Activation Height | Activation Target | Spec | | --- | ---------------------------------------- | ------- | ---------- | ----------------- | ----------------- | ----------------------------------------------------------------------------------------------------- | +| 014 | Aletheia | 0.1.14 | draft | 65500 | - | [`014-aletheia.md`](./changelog/protocol/014-aletheia.md) | | 012 | Bythos | 0.1.11 | final | 54000 | 2026-03-01 | [`012-bythos.md`](./changelog/protocol/012-bythos.md) | | 011 | Legacy 011 - LMP Axis Hotfix | 0.1.10 | activated | 0 | - | [`011-legacy-lmp-axis-hotfix.md`](./changelog/protocol/011-legacy-lmp-axis-hotfix.md) | | 010 | Legacy 010 - V1 Phase 39000 | 0.1.9 | activated | 39000 | - | [`010-legacy-v1-phase-39000.md`](./changelog/protocol/010-legacy-v1-phase-39000.md) | diff --git a/README.md b/README.md index 94a12fc0f..1011687e3 100644 --- a/README.md +++ b/README.md @@ -16,7 +16,22 @@ Nockchain is a ZK-Proof of Work blockchain that combines sound money incentives > For docs read order, trust policy, and canonical sources, start with [`START_HERE.md`](./START_HERE.md). > Consensus/protocol authority is indexed in [`PROTOCOL.md`](./PROTOCOL.md), with canonical upgrade source files in [`changelog/protocol/`](./changelog/protocol/). > This README remains a quickstart for setup and operations. - +> Operators upgrading to a PMA-enabled release should also read [`PMA-FAQ.md`](./PMA-FAQ.md). + +## This is the first release of the persistent memory arena + +The PMA is a major re-architecting of `nockvm`, the runtime for Nockchain, and it significantly reduces the steady-state memory requirements of Nockchain peers and NockApps. We have tested it and run it ourselves but please report any bugs or problems. + +High-level take-aways for PMA: +- Steady-state RAM (RSS) usage has dropped from ~20 GiB to ~1.8 GiB +- Maximum RAM usage has dropped from ~40 GiB to ~20 GiB. This is much less frequent in the PMA version, but it will happen temporarily with the checkpoint bootstrap on the first post-PMA boot and to a lesser extent garbage collection. During the first boot you will see the RSS be elevated initially (~12-16 GiB) until your first garbage collection, then you should see ~1.8 GiB RSS typically. +- You should still provision a swap partition of at least 32 GiB as the PMA enables paging out without swapping for the Arvo (persistent state, which includes the entirety of the chain state) but not `nockvm`'s NockStack itself, this is by design to conserve predictable and efficient execution performance in the NockStack. +- Transaction throughput is improved, 600-700 milliseconds of per-event overhead has been traded down to 2-4 milliseconds of per-event durability overhead. This has a particularly noticeable impact for cheap interstitial events like the timer, duplicate blocks, etc. When we tested baseline against PMA the PMA instance managed to get more than 1,000 blocks ahead of the baseline in a few hours of syncing blocks in the 52k-62k range. This improvement is because the PMA moves the Arvo out of the NockStack and spares the NockStack compacting the entirety of the chain state after every event. +- Write-through burden on SSD disks is markedly lower, about 2x, than the pre-PMA baseline's behavior. This is based on procfs statistics and not our own measurements so the numbers should be sound. This will vary depending on how you configure the snapshot and garbage collection intervals. +- No more 2-3 minute checkpoints bookending one after another. Incremental, efficient persistence to disk through the PMA slab. +- Garbage collection causes a ~5-10 second spike in RSS but it's a lower peak than checkpointing and shouldn't be nearly as susceptible to OOMs as checkpointing was. +- NockStack gets `madvise` trimmed after every event with a high watermark higher than 512 MiB to align the pages allocated for the Nockchain/NockApp process with what's actually needed/used in practice. +- PMA includes the PMA slab and .meta sidecar itself, a sqlite3 event log, and snapshots of the slabs aligned to event log positions. Durability is considerably more robust and efficient now. `fsync`/`fdatasync` ordering during post-event the durability cycle is strictly and carefully designed to make the detection of corrupted PMA slabs due to `SIGKILL`, power cuts, etc. trivially cheap and highly reliable. ## Setup diff --git a/assets/approver.jam b/assets/approver.jam new file mode 100644 index 000000000..d440d0fa3 Binary files /dev/null and b/assets/approver.jam differ diff --git a/assets/bridge.jam b/assets/bridge.jam new file mode 100644 index 000000000..f38229024 Binary files /dev/null and b/assets/bridge.jam differ diff --git a/assets/dumb.jam b/assets/dumb.jam index 6e7f233f7..3dbf5ba62 100644 Binary files a/assets/dumb.jam and b/assets/dumb.jam differ diff --git a/assets/miner.jam b/assets/miner.jam index 02606a855..6a3845ca5 100644 Binary files a/assets/miner.jam and b/assets/miner.jam differ diff --git a/assets/peek.jam b/assets/peek.jam new file mode 100644 index 000000000..eb9725013 Binary files /dev/null and b/assets/peek.jam differ diff --git a/assets/verifier.jam b/assets/verifier.jam index 177b8f849..51d8ef2fd 100644 Binary files a/assets/verifier.jam and b/assets/verifier.jam differ diff --git a/assets/wal.jam b/assets/wal.jam index 214ce3152..69e1d865f 100644 Binary files a/assets/wal.jam and b/assets/wal.jam differ diff --git a/changelog/protocol/013-nous.md b/changelog/protocol/013-nous.md new file mode 100644 index 000000000..74fcc7673 --- /dev/null +++ b/changelog/protocol/013-nous.md @@ -0,0 +1,609 @@ ++++ +version = "0.1.12" +status = "draft" +consensus_critical = false + +activation_height = 0 +published = "2026-02-25" +activation_target = "" + +authors = ["@nockchain-core"] +reviewers = ["@nockchain-core"] + +supersedes = "0.1.11" +superseded_by = "" ++++ + +# Nous + +LibP2P request-response generation 2 (`req-res gen2`) adds batched transport requests, batched transport responses, and protocol-order fallback to generation 1. + +## Summary + +Nous is a networking upgrade for the libp2p request-response path. It reduces round-trip overhead by carrying many request items in one libp2p request-response exchange (one request message and one response message). During rollout, nodes interoperate by negotiating `gen1` or `gen2` from outbound protocol ordering. + +## Motivation + +Gen1 request-response is singleton-oriented and pays round-trip overhead per item. During sync or missing-data recovery, this amplifies latency and redundant outbound request traffic. + +Nous addresses this by adding batched transport while preserving rollout safety in mixed networks. The design keeps kernel/Hoon interfaces stable, keeps gen1 wire compatibility intact, and makes fallback behavior explicit so operators can upgrade incrementally. + +## Technical Specification + +### Scope and Invariants + +In scope: +- Add `gen2` protocol ID and dual registration (`gen1` + `gen2`). +- Add batch request/response transport message schema. +- Add minimal per-item response envelope metadata for routing and dedupe keys. +- Add batch limits, queue pressure controls, and per-peer inflight caps. +- Add protocol-order-based send routing and fallback behavior. +- Add observability and a validation matrix to support rollout decisions. + +Out of scope: +- Any kernel/Hoon interface change. +- Any batching semantic change in kernel execution (kernel continues to process singleton events). +- Any data migration. +- Any hard cutover that requires all peers to upgrade at once. +- Any PoW retuning or PoW-based abuse-defense redesign. + +Normative invariants: +- Kernel and Hoon interfaces are unchanged in this generation. +- Batch handling is implemented only in Rust networking/driver code. +- A batch is executed as an ordered sequence of singleton kernel calls. +- If one item fails, previously applied items are not rolled back. +- Gen2 is additive. Gen1 protocol IDs, constants, and encodings remain byte-for-byte unchanged. + +### Current Network Behavior (Gen1) + +Current protocol ID: +- `/nockchain-1-req-res` + +Current transport behavior: +- One request item per request-response exchange. +- Per-request EquiX PoW solve and verify. +- Request timeout defaults to 30 seconds. +- Randomized peer fanout for requests. +- No explicit per-peer transport generation state in driver routing. +- Kernel/network effects consumed by the driver are singleton-oriented. + +Current transport message schema (simplified): + +```rust +enum NockchainRequest { + Request { pow: [u8; 16], nonce: u64, message: ByteBuf }, + Gossip { message: ByteBuf }, +} + +enum NockchainResponse { + Result { message: ByteBuf }, + Ack { acked: bool }, +} +``` + +### Target Network Behavior (Gen2) + +### Protocol IDs, Compatibility, and Send Routing + +Protocol IDs: +- `gen1`: `/nockchain-1-req-res` +- `gen2`: `/nockchain-2-req-res` + +Compatibility invariants: +- Existing `gen1` protocol constant names and bytes MUST NOT change. +- Existing `gen1` CBOR encoding/decoding behavior MUST remain byte-for-byte stable. +- Gen2 implementation MUST ship compatibility tests that fail on any gen1 protocol ID or gen1 encoding drift. + +Node behavior requirements: +- A Nous-capable node MUST register both IDs. +- If `req_res_gen2_send_enabled=false`, outbound protocol ordering MUST prefer `gen1`. +- If `req_res_gen2_send_enabled=true`, outbound protocol ordering MUST prefer `[gen2, gen1]`. +- If outbound `gen2` is unsupported for a request, sender MUST retry that request via `gen1` when available in the protocol family. +- `UnsupportedProtocols` outcomes SHOULD increment fallback metrics/logs. + +Implementation constraint: +- In current libp2p request-response API, `send_request` does not take an explicit protocol argument. +- Generation choice depends on outbound protocol ordering plus remote support. +- This generation uses one request-response behavior with deterministic protocol ordering. + +Libp2p integration requirements: +- Implementations SHOULD use one request-response behavior that advertises both protocol IDs as one protocol family. +- When `req_res_gen2_send_enabled=true`, outbound protocol ordering MUST prefer `[gen2, gen1]`. +- When `req_res_gen2_send_enabled=false`, implementation SHOULD either: + - prefer outbound ordering `[gen1, gen2]`, or + - configure gen2 as inbound-only via protocol support mode. + +### Gen2 Transport Message Schema + +`gen2` adds batched transport containers. + +```rust +enum NockchainRequest { + // Existing gen1 + Request { pow: [u8; 16], nonce: u64, message: ByteBuf }, + Gossip { message: ByteBuf }, + + // New gen2 + BatchRequest { + pow: [u8; 16], + nonce: u64, + items: Vec, + }, +} + +struct BatchRequestItem { + item_id: u32, + message: ByteBuf, +} + +enum NockchainResponse { + // Existing gen1 + Result { message: ByteBuf }, + Ack { acked: bool }, + + // New gen2 + BatchResult { + results: Vec, + }, +} + +struct BatchResultItem { + item_id: u32, + status: BatchResultStatus, + error: Option, + envelope: Option, +} + +enum BatchResultStatus { + Result, + Ack, + NotFound, + Error, +} + +enum BatchErrorClass { + Decode, + Backpressure, + TooLarge, + InvalidPow, + Internal, +} + +struct ResponseEnvelope { + kind: EnvelopeKind, // HeardBlock | HeardTx | HeardElders + block_id: Option, // present when kind == HeardBlock + tx_id: Option, // present when kind == HeardTx + message: ByteBuf, +} +``` + +Envelope strictness matrix: +- `HeardBlock`: `block_id` required, `tx_id` absent. +- `HeardTx`: `tx_id` required, `block_id` absent. +- `HeardElders`: both IDs absent. +- `height` is not part of the envelope and MUST NOT be included. + +Normative requirements: +- `item_id` MUST be unique within each batch. +- Batch response correlation MUST use `item_id`. +- If `status=Error`, `error` MUST be populated. +- If `status != Error`, `error` MUST be `None`. +- Envelope metadata is routing/dedupe metadata only and MUST NOT replace payload validation. +- Unknown variants MUST fail without panic. + +Response payload granularity (preserved from existing behavior): +- A successful response item represents one requested object/fact payload, not a transitive bundle. +- A block request item returns a block/page fact payload (`HeardBlock`) and MUST NOT implicitly include all raw transaction blobs for that block. +- A raw transaction blob is returned only for explicit raw-transaction request items (`HeardTx` path). +- Batching changes transport shape only; callers that need block data plus transaction blobs SHOULD include both block and tx request items in the same batch. + +### Batch Processing and Control Flow + +Inbound control flow: +1. Receive `BatchRequest`. +2. Perform top-level checks: decode container, verify PoW, enforce hard size/count limits. +3. Decode and vet each item independently. +4. Execute vetted items in wire order, one-by-one, through existing singleton kernel call paths. +5. Collect per-item result statuses and envelopes. +6. Aggregate kernel/network effects generated during execution. +7. Suppress duplicate outbound network requests implied by items within the same batch before enqueueing to the wire. +8. Emit one `BatchResult` with per-item outcomes. + +Execution semantics: +- Partial success is valid (`Result`/`Ack`/`NotFound`/`Error` mixed in one batch). +- Per-item decode errors do not abort sibling items. +- No batch rollback is attempted after prior item success. + +Effect deduplication requirements: +- Dedup applies only to outbound network requests keyed by object identity: + - `BlockRequest(block_id)` + - `TxRequest(tx_id)` +- Driver MUST dedup requests implied by other items in the same batch. +- Dedup against queued or in-flight requests MAY be implemented as a non-normative optimization in a future generation. +- Driver MUST preserve non-network side effects and item-local state transitions. +- Suppression is only for redundant wire requests, not for semantic effects. + +### Driver-Side Outbound Coalescing + +Batch sources: +- Driver-side coalescing of singleton outbound requests (required). +- Optional pre-batched emitters from higher layers are deferred. +- Gossip traffic remains singleton unless a future extension defines batching. + +Coalescing policy: +- Flush at `gen2_batch_max_items`. +- Flush at `gen2_batch_coalesce_window_ms`. +- Flush when payload bytes approach `gen2_batch_max_bytes`. +- Sender SHOULD adapt effective batch size below `gen2_batch_max_items` based on observed timeout and backpressure rates. + +Determinism requirements: +- Preserve request ordering in per-peer queue. +- Generate stable `item_id` sequence within each batch. + +Primary insertion points: +- `open/crates/nockchain-libp2p-io/src/behaviour.rs` (`NockchainBehaviour::pre_new`) for dual protocol registration. +- `open/crates/nockchain-libp2p-io/src/config.rs` for protocol IDs and batch knobs. +- `open/crates/nockchain-libp2p-io/src/driver.rs`: + - swarm loop send branch for outbound coalescing/send, + - `handle_effect` for staging outbound item flow, + - `handle_request_response` for inbound batch execution and `BatchResult` construction. +- `open/crates/nockchain-libp2p-io/src/p2p_state.rs` for per-peer inflight accounting. + +### Backpressure and Flow Control + +### Receiver Behavior + +Admission outcomes: +- **Accept full batch**: top-level checks pass and execution slot is available. +- **Process partially with per-item backpressure**: execution starts, then downstream queue pressure occurs; already-processed items keep their outcomes, remaining items return `Error(Backpressure)`. +- **Reject wholesale**: + - malformed top-level batch, + - invalid PoW, + - hard limit violation, + - no execution slot available before processing begins. + +Receiver response contract: +- If wholesale rejection occurs before any item executes, receiver MAY close the request without per-item results. +- If execution has started, receiver MUST return `BatchResult` with outcomes for processed items and `Error(Backpressure)` for any unprocessed tail items. + +Yield semantics: +- For request items that can return multiple units, requested count `N` is an upper bound, not a guarantee. +- Responders MAY yield fewer units `M` than requested (`M <= N`) for any valid operational reason, including payload byte limits, compute budget, or queue pressure. +- When at least one unit is available, responders MUST support fractional yield down to the minimum quantum of one valid unit for that request kind. +- Senders MUST treat fractional yield as valid behavior and continue via additional requests until completion criteria are met. + +Queue pressure policy: +- Queueing is bounded. +- Receiver MUST NOT block indefinitely waiting for queue room. +- Queue-full at admission is immediate reject, not bounded wait/retry inside receiver. +- Request-response stream-level concurrency limits are necessary but not sufficient; implementation MUST preserve explicit driver-level queue admission semantics. + +Per-peer inflight policy: +- `gen2_max_inflight_per_peer` is a hard cap on outstanding request-response work per peer. +- Hitting the cap causes immediate batch rejection with backpressure classification (when a batch response is available) or early request failure. + +### Sender-Visible Semantics + +Sender-observed outcomes: +- Transport-level failure before any `BatchResult` (timeout, decode failure, unsupported protocol, early reject): sender treats entire batch as failed and retries with bounded backoff. +- `BatchResult` with per-item `Error(Backpressure)`: sender retries only those failed items, preserving successful items. +- `BatchResult` with per-item `Error(Decode|TooLarge|InvalidPow)`: sender MUST NOT retry those items unchanged. + +Retry/backoff requirements: +- Retries MUST use bounded budgets. +- Backpressure retries SHOULD use exponential backoff with jitter. +- Sender MUST NOT spin-retry immediately on repeated backpressure. +- Sender SHOULD reduce retry batch size after repeated transport failures or repeated `Error(Backpressure)` outcomes. + +### Limits and Benchmark Tuning Strategy + +Gen2 limits are configuration defaults, not protocol constants. + +Configured limits: +- `gen2_batch_max_items` +- `gen2_batch_max_bytes` +- `gen2_item_max_bytes` +- `gen2_max_inflight_per_peer` + +Tuning requirements: +- Defaults MUST be finalized by benchmark data before `status = final`. +- `gen2_batch_max_items` and `gen2_batch_max_bytes` MUST be tuned together. +- Benchmark suite MUST include mixed request types, cache hit/miss mixes, and queue pressure cases. +- Safety upper bounds MUST remain in place even if benchmark results suggest larger values. +- Request-response CBOR codec max request/response sizes MUST be configured in lockstep with `gen2_batch_max_bytes` so transport and application limits agree. + +Responder sizing prerequisites: +- Implementation MUST include a weight index or heuristic (for example b-tree index, range heuristic, or precomputed historical sizing data) so responders can avoid overfetching units that will not fit in the response. +- Implementation MUST include a forward-size heuristic for unknown future block sizes so responders can stop before doing expensive serialization work for units that cannot fit. + +Target tuning ranges (starting envelope): +- `gen2_batch_max_items`: 32 to 256 +- `gen2_batch_max_bytes`: 256 KiB to 2 MiB +- `gen2_item_max_bytes`: 64 KiB to 256 KiB + +Initial hard limits in this draft: +- `gen2_batch_max_items = 128` +- `gen2_batch_max_bytes = 1_048_576` +- `gen2_item_max_bytes = 131_072` + +### PoW and Abuse Controls + +PoW policy in this generation: +- PoW mechanism and tuning remain unchanged from existing request-response policy. +- Gen2 does not introduce count-weighted or account-weighted PoW. +- Current PoW is known to be under-tuned and MUST NOT be treated as sufficient DoS protection. + +Abuse resistance for this generation relies on: +- strict hard limits, +- bounded queues, +- per-peer inflight caps, +- dedupe suppression of redundant outbound requests. + +PoW retuning and stronger abuse controls are explicitly deferred to a future generation. + +### PoW Input for Batched Payloads + +Because batch payload bytes differ from singleton payload bytes, verifier input serialization is defined for interoperability: + +Gen2 PoW preimage MUST include: +- fixed domain-separation bytes: ASCII `nockchain:req-res:gen2:pow:v1` +- `nonce` +- sender peer bytes +- receiver peer bytes +- canonical serialized batch item bytes + +Canonical batch item bytes definition: +- Use batch item order exactly as transmitted in `BatchRequest.items` (do not sort). +- Encode item list as: + - `item_count_le_u32` + - for each item: + - `item_id_le_u32` + - `message_len_le_u32` + - `message_bytes` + +Reference preimage layout: +- `domain_sep || nonce_le_u64 || sender_peer_bytes || receiver_peer_bytes || canonical_batch_item_bytes` + +### Configuration + +Recommended key names: + +| Key | Shipped Default | Purpose | +| ----------------------------------- | --------------- | --------------------------------------------------------- | +| `req_res_gen2_accept_enabled` | `true` | Accept gen2 requests | +| `req_res_gen2_send_enabled` | `false` | Enable gen2 send only after rollout gate is satisfied | +| `gen2_batch_max_items` | `128` | Hard item cap (benchmark-tuned before final) | +| `gen2_batch_max_bytes` | `1048576` | Hard batch byte cap (co-tuned with item cap) | +| `gen2_item_max_bytes` | `131072` | Hard per-item byte cap | +| `gen2_batch_coalesce_window_ms` | `10` | Coalescing window | +| `gen2_max_inflight_per_peer` | `32` | Per-peer inflight req-res cap | +| `gen2_swarm_action_queue_capacity` | `1000` | Bounded driver queue size for swarm actions | + +### Failure Handling + +Required handling paths: +- Unsupported protocol on outbound gen2 MUST: + - trigger only on `OutboundFailure::UnsupportedProtocols`, + - retry via `gen1` for that request when `gen1` is available in the outbound protocol family, + - avoid persistent per-peer downgrade state in this generation, + - emit fallback metrics/logs. +- Decode failure of one item: mark that item `Error(Decode)` and continue siblings when decoding context remains valid. +- Top-level batch decode failure: reject entire batch and count failure. +- Invalid PoW: reject entire batch and count failure. +- Over-limit batch: reject entire batch and count failure (`TooLarge` classification where representable). +- Backpressure before item execution: reject entire batch. +- Backpressure during execution: keep prior item outcomes, mark remaining items `Error(Backpressure)`. +- Timeouts remain observable per generation. + +Deterministic per-item error classification: +- `Error(Decode)`: malformed item payload; sender MUST NOT retry unchanged item. +- `Error(TooLarge)`: item or container exceeds configured limits; sender MUST NOT retry unchanged item. +- `Error(InvalidPow)`: invalid proof classification where representable; sender MUST NOT retry unchanged item. +- `Error(Backpressure)`: transient saturation; sender MAY retry with bounded backoff. +- `Error(Internal)`: implementation/internal failure; sender MAY retry with bounded backoff. + +### Observability Requirements + +Required transport metrics: +- `gen2_batch_requests_sent` +- `gen2_batch_requests_received` +- `gen2_batch_item_count_histogram` +- `gen2_batch_rejected_total{reason=*}` +- `gen2_batch_item_errors_total{class=*}` +- `gen2_fallback_total{from,to,reason=*}` +- `req_res_failures_total{generation=*}` +- `req_res_timeouts_total{generation=*}` +- cache hit/miss counters in request and response paths +- `req_res_inflight_per_peer` gauge +- `req_res_swarm_action_queue_depth` gauge +- `req_res_swarm_action_queue_full_total` +- `req_res_kernel_backpressure_total` +- `req_res_effect_dedup_suppressed_total{reason=in_batch}` + +Required logs: +- generation selected per peer +- fallback decisions +- batch reject reason +- per-item failure class counts +- queue saturation decisions (reject/defer) +- dedupe suppression decisions (request key and reason) + +Gate condition: +- A node MUST NOT ship with `req_res_gen2_send_enabled=true` unless above metrics/logs, interoperability tests, and PMA RSS validation are in place, and required performance gates pass (two-peer speed-of-light benchmark, offline payload-size fit tests, requester compute profiling). + +### Rollout Model + +Dual-stack operating model: +- Nous-capable nodes register both protocol IDs and accept both generations. +- Gen1 fallback remains available during migration. + +Compatibility matrix: +- Old node <-> old node: `gen1` only. +- New node <-> old node: new node prefers `gen2`; when unsupported for a request, retry `gen1`. +- Old node <-> new node: old node speaks `gen1`; new node accepts `gen1`. +- New node <-> new node: prefer `gen2`; negotiate `gen1` when peer configuration or support requires. + +Independent node rollout: +- Nodes can be upgraded independently. +- Mixed-generation networks are supported by protocol-order negotiation and fallback. +- Shipped defaults keep gen2 send disabled until rollout gate conditions are met; operators may then enable `req_res_gen2_send_enabled=true`. + +## Activation + +- **Height**: TBD (`activation_height = 0` in frontmatter). +- **Coordination**: staged dual-stack rollout; nodes advertise and accept both generations, prefer gen2 when negotiated, and retain gen1 fallback during migration. + +## Migration + +### Operator Steps + +1. Upgrade to Nous-capable node build. +2. Verify node advertises both protocol IDs. +3. Keep `req_res_gen2_accept_enabled=true`. +4. Run staged validation in representative traffic environments. +5. Confirm rollout gate metrics/tests are passing. +6. Enable `req_res_gen2_send_enabled=true` gradually. +7. Monitor generation, fallback, dedupe, and backpressure metrics. + +### Rollback Steps + +1. Set `req_res_gen2_send_enabled=false`. +2. Keep `req_res_gen2_accept_enabled=true` unless incident response requires full disable. +3. Continue on `gen1` transport path. + +### Data Migration + +- None. + +## Backward Compatibility + +This upgrade is transport-level and additive: +- Existing gen1 protocol IDs and encoding are preserved byte-for-byte. +- New nodes can communicate with old nodes via gen1 fallback. +- Old nodes continue operating on gen1 without protocol-level crashes from gen2 rollout. + +This upgrade does not change transaction formats or consensus semantics, but it is liveness-critical transport: +- Transactions created by old software remain valid. +- Transactions created by new software remain valid. + +## Security Considerations + +Security-relevant points in this generation: +- PoW is unchanged and explicitly not treated as sufficient DoS protection. +- Abuse resistance relies on hard message/item limits, bounded queues, and per-peer inflight caps. +- Envelope metadata is advisory for routing/dedupe and MUST NOT be trusted over payload validation. +- Dedupe suppresses only redundant outbound wire requests, and MUST NOT suppress semantic side effects. + +## Operational Impact + +Operator-facing impact: +- Lower round-trip overhead for high-item request workloads when peers are gen2-capable. +- More explicit flow-control outcomes (admission reject vs partial item backpressure). +- Additional metrics/logs are required for rollout safety (generation selection, fallback rates, dedupe suppressions, queue pressure). + +Rollout risk and mitigation: +- Mixed-version networks remain supported through gen1 fallback. +- Operators can disable gen2 send (`req_res_gen2_send_enabled=false`) as a rollback lever while still accepting gen2 if needed. + +## Testing and Validation + +### A. Serialization and Compatibility +- Gen1 round-trip unchanged. +- Gen1 protocol ID constants unchanged. +- Gen1 vector bytes unchanged against golden files. +- Gen2 batch round-trip for mixed item types. +- Malformed and truncated batch decode rejection without panic. +- Extend `open/crates/nockchain-libp2p-io/src/cbor_tests.rs` for `BatchRequest`/`BatchResult` round-trip and malformed input coverage. +- Maintain machine-readable conformance vectors at `open/crates/nockchain-libp2p-io/testdata/req_res_gen1_cbor_vectors.json`. +- Keep vector-driven tests executable in `open/crates/nockchain-libp2p-io/src/cbor_tests.rs`. + +### B. Interop +- gen1 <-> gen1 +- gen2 <-> gen2 +- gen2 sender with gen1-only peer fallback +- mixed-network reconnect and protocol renegotiation churn +- integration tests around request-response behavior in `open/crates/nockchain-libp2p-io/src/driver.rs` + +### C. Correctness +- per-item correlation by `item_id` +- mixed cache hit/miss batch behavior +- in-batch dedupe usage by item type +- deterministic per-peer queue ordering under fallback retries +- suppression of redundant outbound block/tx requests while preserving non-network side effects + +### D. Performance +- 100-item fetch workload improves materially over gen1 baseline +- no unacceptable increase in timeout/error rate +- benchmark-derived defaults for batch item and byte caps +- add/update benchmark coverage to measure transport generation behavior (existing `peek_refresh` benchmark is not sufficient by itself) +- include PMA RSS harness scenarios for syncing peers (gen1 vs gen2, with and without native peek optimizations) +- include a two-peer speed-of-light benchmark (fat responder with full PMA data vs requester) and gate rollout on the result +- include offline payload-size tests using current checkpoint data (serialize `Vec` and determine practical fit under max byte limits, including the nominal 128-block target) +- include requester compute-time profiling for poke + verification of a full 128-block payload + +### E. Abuse and Pressure Behavior +- over-limit batch rejection +- bad-PoW rejection +- per-item malformed payload isolation +- queue saturation/backpressure behavior without panic or deadlock +- per-peer inflight cap enforcement +- sender retry/backoff behavior under repeated backpressure + +## Implementation File Map + +Primary implementation files: +- `open/crates/nockchain-libp2p-io/src/messages.rs` +- `open/crates/nockchain-libp2p-io/src/driver.rs` +- `open/crates/nockchain-libp2p-io/src/behaviour.rs` +- `open/crates/nockchain-libp2p-io/src/config.rs` +- `open/crates/nockchain-libp2p-io/src/p2p_state.rs` +- `open/crates/nockchain-libp2p-io/src/metrics.rs` +- `open/crates/nockchain-libp2p-io/src/cbor_tests.rs` +- `open/crates/nockchain-api/benches/peek_refresh.rs` (extend or add dedicated transport benchmark) + +Optional future batching-emit producers: +- `open/hoon/apps/dumbnet/lib/types.hoon` +- `open/hoon/apps/dumbnet/inner.hoon` + +Notes: +- Initial rollout does not require Hoon-side batch effect changes. +- Any future Hoon-side batching is a transport optimization and must preserve gen1 compatibility paths. + +## Resolved Design Decisions + +1. PoW policy for large batches +- Decision: keep current fixed-cost PoW policy and strict transport caps in this generation. +- Decision: include explicit PoW preimage domain/version separator (`nockchain:req-res:gen2:pow:v1`). +- Deferred: count-weighted/account-weighted PoW redesign. + +2. Default send flag in shipped config +- Decision: `req_res_gen2_send_enabled=false` in shipped defaults until rollout gate is satisfied. + +3. Envelope strictness +- Decision: minimal envelope is `kind + relevant ID + message`. +- `height` removed. + +4. Future batching source +- Decision: Rust driver coalescing is required and sufficient for initial rollout. +- Deferred: Hoon-side batch emitters. + +5. Request-response behavior architecture +- Decision: single request-response behavior with deterministic protocol ordering and no persistent per-peer capability pinning state in this generation. + +6. Backpressure response contract +- Decision: if execution has started, return partial per-item `Error(Backpressure)` for unprocessed tail items. +- Decision: if admission fails before execution, reject whole batch. + +7. Queue pressure policy +- Decision: bounded queues, no unbounded wait, immediate reject on full admission queue. + +8. Per-peer inflight default and enforcement mode +- Decision: enforce hard cap (`gen2_max_inflight_per_peer`) with immediate reject at cap. + +9. Capability caching policy +- Decision: no persistent per-peer capability downgrade TTL in this generation. +- Deferred: explicit capability state caching heuristics if future measurements show material benefit. + +## Reference Implementation + +TODO diff --git a/changelog/protocol/014-aletheia-audit.md b/changelog/protocol/014-aletheia-audit.md new file mode 100644 index 000000000..fff96f98b --- /dev/null +++ b/changelog/protocol/014-aletheia-audit.md @@ -0,0 +1,543 @@ +# 014-aletheia — ASERT audit vs. da-asert.pdf + +Audit of the Aletheia (aserti3-2d) implementation and spec on branch +`la/asert` against: + +- Mark Lundeberg, *DA-ASERT v.2* (`da-asert.pdf`, July 31 2020). +- Jonathan Toomim's aserti3-2d as shipped on Bitcoin Cash in 2020 + (`bitcoincashorg/bitcoincash.org/spec/2020-11-15-asert.md`, + `upgradespecs.bitcoincashnode.org/2020-11-15-asert/`, BCHN QA vectors). + +Scope: the changelog (`014-aletheia.md`), the library +(`open/hoon/apps/dumbnet/lib/asert.hoon`), its consensus wrapper +(`open/hoon/apps/dumbnet/lib/consensus.hoon`), and the unit / integration +tests (`closed/hoon/tests/dumb/mod/unit/asert.hoon`, +`closed/hoon/tests/dumb/mod/integration/asert-activation.hoon`). + +This is a paper-audit only. No behaviour was executed; no fix is proposed. +Each flag is for reviewer triage. + +## Summary + +| # | Item | Kind | Severity | +| - | ------------------------------------------------------------- | ----------- | -------- | +| 1 | Changelog formula has `(height-diff+1)`, code doesn't *(resolved)* | Discrepancy | HIGH | +| 2 | Anchor timestamp convention ≠ BCH's (parent-of-anchor) *(resolved)* | Discrepancy | MEDIUM | +| 3 | Polynomial missing `+2^47` rounding term *(resolved)* | Discrepancy | LOW | +| 4 | `(need ...)` crashes accept/validate on minority-fork blocks *(resolved — Phase 1)* | Bug | HIGH | +| 5 | Set-once anchor capture is not reorg-safe at anchor-height *(resolved — Phase 1)* | Bug | MEDIUM | +| 6 | `current-min-timestamp` semantics inconsistent lib vs wrapper *(resolved)* | Bug | MEDIUM | +| 7 | `+poly-factor` does not validate `frac < radix` *(resolved)* | Bug | LOW | +| 8 | `anchor-target == 0` is silently rewritten to 1 *(resolved)* | Bug | LOW | +| 9 | No production-semantic activation target test *(resolved)* | Test gap | HIGH | +| 10 | No BCH-parity test for polynomial + shift stages *(resolved)* | Test gap | MEDIUM | +| 11 | No reorg-safety tests for anchor capture *(resolved)* | Test gap | HIGH | +| 12 | Memorylessness / reversibility (PDF §1.2) untested *(resolved — polynomial; end-to-end redundant with on-schedule identity)* | Test gap | MEDIUM | +| 13 | Wrapper-level negative-`time_diff` path untested *(resolved — library seam)* | Test gap | MEDIUM | +| 14 | `+decompose-exponent` corner cases not in isolation *(resolved)* | Test gap | LOW | +| 15 | Test names in changelog ≠ test names in code *(resolved)* | Docs drift | LOW | + +--- + +## Discrepancies + +### 1 — Changelog formula ≠ implementation *(HIGH; consensus-critical — RESOLVED)* + +Changelog `014-aletheia.md:115`: + +``` +exponent = ((time-diff - ideal * (height-diff + 1)) * radix) / half-life +``` + +Implementation `asert.hoon:47` (`+compute-exponent`): + +``` +=/ ideal-total (mul ideal blocks-since-anchor) :: no +1 +``` + +`blocks-since-anchor = current-height - anchor-height` +(`asert.hoon:97`), so the code computes + +``` +exponent = (time-diff - ideal * height-diff) * radix / half-life +``` + +— differing from the changelog by exactly one `ideal / half-life` term. +With mainnet parameters (`ideal=150`, `half-life=43 200`) that is a +`~0.24 %` constant target bias versus what the spec text describes. The +changelog is marked `consensus_critical = true`; spec and code must be in +lockstep for anyone re-implementing from the doc. One of them is wrong. + +**Resolution:** fix applied as the nockchain-convention variant (item 2 +Option b). `+compute-exponent` now uses `ideal * (blocks-since-anchor - 1)` +at `asert.hoon:55`, matching PDF Eq. (2) under §1.3 Option 2 (anchor's +own median-of-11). Changelog formula at `014-aletheia.md:115` updated +to `ideal * (height-diff - 1)` with the convention spelled out in the +adjacent paragraph. See item 2 resolution for the convention rationale. + +### 2 — Anchor timestamp convention differs from BCH aserti3-2d *(MEDIUM — RESOLVED)* + +BCH aserti3-2d (confirmed against `upgradespecs.bitcoincashnode.org` and +the official spec at +`bitcoincashorg/bitcoincash.org/spec/2020-11-15-asert.md`) defines: + +``` +time_diff = parent.time - anchor_parent.time :: ANCHOR'S PARENT +height_diff = parent.height - anchor.height +exponent = (time_diff - ideal*(height_diff+1)) / half_life +``` + +The `+1` in BCH's formula exactly compensates for `time_diff` spanning +(anchor_parent → parent), which is `height_diff + 1` ideal intervals. + +Nockchain captures the **anchor block's own** median-of-11, not its +parent's: + +``` +consensus.hoon:580 +=/ anchor-min-ts=@ + (~(got z-by min-timestamps.c) ~(digest get:page:t cur)) +c(asert-anchor-min-timestamp `anchor-min-ts) +``` + +Under that convention, to match PDF Eq. (2) the subtracted term should +be `ideal * (height_diff - 1)` (where `height_diff = child_height - +anchor_height`), not `ideal * height_diff` (implementation) or +`ideal * (height_diff + 1)` (changelog). + +So **neither** the changelog nor the implementation is right for the +anchor-ts convention the code actually uses. The impl is off by +`-ideal/half_life`; the spec text is off by `-2·ideal/half_life`. Practical +impact at mainnet: a systematic `-0.24 %` (impl) or `-0.48 %` (spec) target +bias that is silently absorbed into the chosen `2^291` anchor target. +Worth either (a) switching to capturing the anchor's parent min-ts and +adopting the BCH formula verbatim, or (b) keeping the current convention +but correcting the `+1` accounting and documenting the choice. + +**Resolution:** took option (b). Nockchain's pre-activation DAA is +Bitcoin-style epoch retarget, not relative ASERT, so the BCH rationale +for anchor-parent-ts (continuity with Eq. (1) at activation per PDF +§1.3) does not apply. Keeping anchor's own median-of-11 as the reference +is sanctioned by PDF §1.3 Option 2 and avoids walking one extra parent +during capture. The formula is corrected at `asert.hoon:55` and the +precondition tightened at `asert.hoon:104` to `current-height > +anchor-height`. Unit-test helpers (`+on-schedule`, `+drift`, +`+ref-compute-target`) now feed production semantics, and +`+bch-vector-check` shifts BCH's t_{M-1} → t_M by `+bch-ideal` at the +helper boundary so the 12 BCH QA vectors verify verbatim under Option 2. +Commit `de56b4167`; all 30 asert arms pass. + +### 3 — Polynomial missing `+2^47` rounding term *(LOW; BCH-divergence — RESOLVED)* + +Canonical BCH `aserti32d.cpp`: + +```c +factor = 65536 + (( + 195766423245049ull*e + 971821376ull*e*e + 5127ull*e*e*e + + (1ull<<47) +) >> 48); +``` + +`asert.hoon:32-33`: + +```hoon +=/ num :(add (mul 195.766.423.245.049 frac) (mul 971.821.376 f2) (mul 5.127 f3)) +(add asert-radix (rsh [0 48] num)) +``` + +`rsh [0 48]` is floor, not round-to-nearest. Missing +`(add num (bex 47))` inside the shift. At most 1 LSB of `factor` +difference per evaluation, but it breaks parity with BCH's published +vectors at the polynomial stage. The changelog already caveats that BCH +vectors aren't reusable *because of median-of-11*; this is the other +reason they aren't reusable, which should be noted or fixed. + +**Resolution:** fix applied at `asert.hoon:36` — +`(rsh [0 48] (add num (bex 47)))` now matches BCH canonical +round-to-nearest. BCH-vector parity in the polynomial + shift pipeline +was validated; see item 10. + +--- + +## Potential bugs + +### 4 — `(need asert-anchor-min-timestamp.c)` can crash accept/validate *(HIGH — RESOLVED, Phase 1)* + +`consensus.hoon:215`: + +```hoon +=/ anchor-min-ts=@ (need asert-anchor-min-timestamp.c) +``` + +`+capture-asert-anchor-if-needed` (`consensus.hoon:566-582`) only runs +from `+update-heaviest` and only writes when the **heaviest** chain first +reaches `asert-anchor-height`. The comment on `consensus.hoon:213-214` +asserts the invariant: + +> by the time any caller asks for an aserti3-2d target +> (child-height >= asert-phase), heaviest has already advanced past the +> anchor, so (need ...) succeeds. + +But `+compute-target-asert` is called from: + +1. `+accept-page` target write, `consensus.hoon:323` — for **every** + accepted post-activation block, heaviest or not. +2. `+validate-page-without-txs`, `consensus.hoon:407` — for every + validated post-activation block. + +Failure scenario: fork A is heaviest at height < `asert-anchor-height`. +Fork B has been received with blocks through `asert-phase` but is not +heaviest (e.g., it came in while the node was still catching up, or its +cumulative work is less than A's). When B's block at `asert-phase` +hits accept/validate, `(need ...)` on the still-empty unit panics. This +is a remote-triggerable crash. + +Mitigation candidates: (a) fire capture on *any* accepted block at +`asert-anchor-height` keyed by its digest, and store per-digest so reorgs +select the right one; (b) inside `+compute-target-asert`, if the unit is +`~`, walk from `parent-digest` back to `anchor-height` and read +`min-timestamps` there — no consensus-state write, just a local +computation; (c) if capture hasn't happened, reject the block with +`%page-target-invalid` (or a fresh reason) rather than panicking. + +**Resolution (Phase 1):** adopted option (b), and removed the scalar +cache entirely. `+compute-target-asert` now derives `anchor-min-ts` +on every call by invoking a new `+find-anchor-min-ts` helper in +`consensus.hoon` that walks `parent-digest` back through `.blocks` +to the ancestor at `asert-anchor-height` and reads its median-of-11 +from `.min-timestamps`. No consensus-state field is introduced — +the state-version-7 bump and the `+capture-asert-anchor-if-needed` +arm were reverted. `miner.hoon` now calls the same helper against +heaviest-block. Because there is no shared mutable cache, the path +is fork-correct by construction (every caller walks its own +ancestry). Tests in +`closed/hoon/tests/dumb/mod/integration/asert-activation.hoon` +exercise the 1-step and 2-step walk cases end-to-end. Phase 2 +(below) replaces the walk with a hardcoded constant + checkpoint at +anchor-height, at which point `+find-anchor-min-ts` is deleted. + +### 5 — Set-once anchor capture is not reorg-safe at anchor-height *(MEDIUM — RESOLVED, Phase 1)* + +`consensus.hoon:565-568`: + +```hoon +:: subsequent calls are silent no-ops: set-once, even across reorgs +:: through anchor-height. +++ capture-asert-anchor-if-needed + ^- consensus-state:dk + ?^ asert-anchor-min-timestamp.c c :: already set - silent no-op +``` + +The changelog (`014-aletheia.md:336-340`) defends this: + +> Pre-activation blocks (including the anchor) are shared history across +> any post-activation reorg, so the captured `asert-anchor-min-timestamp` +> remains valid under reorg. + +That claim is a convention, not an invariant. Pre-activation chains can +fork, and two forks can both have blocks at `anchor-height` with +different median-of-11 timestamps (different ancestors populate the +median window differently). If two nodes capture from different +initially-heaviest forks, then later converge on the same post-activation +heaviest chain, their captured `asert-anchor-min-timestamp` values stay +divergent — producing different ASERT targets for the same child and a +consensus split. + +The risk is small on a quiet chain but non-zero around activation and is +a fundamental property of the set-once design. Options: capture keyed by +digest rather than as a scalar; require the anchor-height block to be +buried by N confirmations before capture; update capture when a reorg +replaces the block at `anchor-height`. + +**Resolution (Phase 1):** fully resolved by removing the scalar +entirely. `+compute-target-asert` and the miner candidate-target path +now derive `anchor-min-ts` per-call via `+find-anchor-min-ts`, which +walks `.blocks` from the caller's own `parent-digest` back to the +ancestor at `asert-anchor-height`. There is no shared mutable state, +so two nodes that observed different pre-activation forks at +anchor-height cannot diverge each other's target computation — each +target compute resolves against the specific fork the block being +validated lives on, which is the correct behaviour. Phase 2 +(post-67000) will additionally checkpoint the anchor-height digest, +making the block at that height unique network-wide, and replace +the walk with a `blockchain-constants` value; see "Phase 2 — +post-anchor cutover" in `014-aletheia.md`. + +### 6 — `current-min-timestamp` semantics inconsistent library vs wrapper *(MEDIUM — RESOLVED)* + +The wrapper passes **parent's** min-ts together with **child's** height +(`consensus.hoon:215-227`): + +```hoon +=/ parent-min-ts=@ (~(got z-by min-timestamps.c) parent-digest) +... +%- compute-target:asert +:* asert-anchor-target-atom.blockchain-constants + anchor-min-ts + asert-anchor-height.blockchain-constants + parent-min-ts :: passed as current-min-timestamp + child-height :: passed as current-height + ... +== +``` + +The unit tests (`closed/hoon/tests/dumb/mod/unit/asert.hoon:26-38`) feed +them as a same-block pair: + +```hoon +++ on-schedule + |= [blocks-since=@ anchor=@] + %- compute-target:asert + :* anchor + anchor-ts + anchor-h + (add anchor-ts (mul blocks-since ideal)) :: child's timestamp + (add anchor-h blocks-since) :: child's height + ... +``` + +On the unit-test semantics, perfect schedule produces `exponent = 0` and +`target = anchor_target` — `test-asert-on-schedule-approx-identity` passes. +On the production semantics (parent-ts, child-height), perfect schedule +produces `exponent = -ideal/half_life` and +`target ≈ 0.9976 * anchor_target`. The tests cannot catch this because +they never instantiate the wrapper-style argument pairing. + +This is the root cause that makes Discrepancies 1 and 2 invisible to CI. + +**Resolution:** subsumed by the fix for items 1 and 2. The library now +documents the anchor-own-ts / parent-ts / child-height convention +explicitly on `+compute-exponent` and `+compute-target` +(`asert.hoon:38-46`, `asert.hoon:83-92`), matching the wrapper. Unit-test +helpers were rewritten to feed production semantics: `+on-schedule` and +`+drift` now produce parent timestamps via `(mul (dec blocks-since) +ideal)`, `+ref-compute-target` uses the same `(dec bsa)` factor with a +tightened `lte` precondition, and `test-asert-anchor-identity` pins the +activation case (child at `anchor+1`, parent = anchor) to +`anchor-target` exactly. Commit `de56b4167`. + +### 7 — `+poly-factor` does not validate `frac < radix` *(LOW — RESOLVED)* + +`asert.hoon:27-33`. `+decompose-exponent` always produces +`frac ∈ [0, radix)` today (`asert.hoon:64-73`), so no in-tree caller can +misuse it. But `+poly-factor` is publicly exported by the `asert` core, +is called from tests, and will be callable from any future +re-implementation of the exponent pipeline. A defensive +`?> (lth frac asert-radix)` fails loudly on misuse vs silently returning +a nonsense factor. + +**Resolution:** guard added at the head of `+poly-factor` in +`open/hoon/apps/dumbnet/lib/asert.hoon` (just after the `^- @` +line). Docstring already stated the `[0, radix)` precondition; the guard +now enforces it. New unit test +`test-asert-poly-factor-rejects-oversized-frac` in +`closed/hoon/tests/dumb/mod/unit/asert.hoon` pins crash behaviour at +`frac = radix` and `frac = radix + 1` via `expect-fail`. + +### 8 — `anchor-target == 0` silently rewrites to 1 *(LOW — RESOLVED)* + +`asert.hoon:108`: `unshifted = anchor-target * factor`. If +`anchor-target = 0`, `unshifted = 0`, `(met 0 unshifted) = 0`, any +negative-shift path returns 0 which is then clamped to 1 +(`asert.hoon:123`). Similarly, any positive-shift path shifts zero → +zero → 1. A misconfigured `blockchain-constants` with +`asert-anchor-target-atom = 0` would turn the DA into "every post-activation +block targets hash = 1" — effectively a frozen chain — silently. An +early `?> !=(0 anchor-target)` would surface this on first call. + +**Resolution:** guard added at the head of `+compute-target` in +`open/hoon/apps/dumbnet/lib/asert.hoon`, immediately after the existing +`current-height > anchor-height` precondition (`?< =(0 anchor-target)`). +A misconfigured `asert-anchor-target-atom = 0` now crashes on the first +post-activation target compute rather than silently freezing the chain +at target = 1. New unit test `test-asert-rejects-zero-anchor-target` in +`closed/hoon/tests/dumb/mod/unit/asert.hoon` pins the crash via +`expect-fail`. + +--- + +## Test gaps + +### 9 — Production-semantic activation target test *(HIGH — RESOLVED)* + +The integration file `asert-activation.hoon` only compares the wrapper +against a *direct library call with the same arguments* — any mis-mapping +between wrapper and library (Discrepancy 1 and/or 2) matches itself. + +Proposed test: at activation (`child-height = anchor-height + 1`, parent += anchor, so `parent-min-ts = anchor-min-ts`), assert the target equals +the value prescribed by Eq. (2) under the nockchain anchor convention +— either `anchor-target` exactly (if the `+1` / anchor-parent question +is resolved to match PDF Eq. (2)), or +`anchor-target * 2^(-ideal/half_life)` (if the current code is deemed +correct). Either way, the test pins a number that is *derived +externally*, not by self-consistency. + +**Resolution:** `test-asert-wrapper-activation-identity` in +`closed/hoon/tests/dumb/mod/integration/asert-activation.hoon` builds +the chain up to the anchor and calls `+compute-target-asert` at +`child-height = anchor-height + 1`. Under the anchor-own-ts +convention (item 2 Option b) the exponent at activation is exactly 0, +so the emitted target must equal `asert-anchor-target-atom.bc`. The +expected value is the bc constant itself — derived externally from +PDF Eq. (2), not by self-consistency with a library call on the same +inputs. If any wrapper-to-library mis-mapping were introduced, this +test would emit a different number at activation and fail. + +### 10 — BCH-parity test for polynomial + shift stages *(MEDIUM — RESOLVED)* + +The changelog (`014-aletheia.md:158-160`) justifies dropping BCH vectors +because "BCH's published aserti3-2d test vectors (which use raw +timestamps) are not directly reusable". That argument only covers the +time-diff stage. The polynomial, shift decomposition, and saturating +shift arithmetic are timestamp-agnostic and can be cross-checked against +BCH's vectors by feeding the BCH exponent directly into the downstream +pipeline and comparing `factor` and `next_target` shifts. Such a test +would catch Discrepancy 3 and any future drift in `+poly-factor`. + +**Resolution:** all 12 BCH aserti3-2d QA runs (runs 01–12, 143 +iter vectors total) are embedded as unit tests in +`closed/hoon/tests/dumb/mod/unit/asert.hoon`. The suite reuses +`+compute-target:asert` directly with `(ideal=600, half-life=172.800, +max-target=0xffff<<208)` — the BCH mainnet parameters — and asserts +per-iter `nBits` parity via `+bch-vector-check`. Two helper sanity +tests (`test-asert-bch-nbits-expansion`, +`test-asert-bch-nbits-roundtrip`) pin the compact-nBits codec that the +check relies on. All 30 asert arms pass under `roswell test-dumb` +(187 OK, 0 FAIL, post-`+2^47` fix). + +### 11 — Reorg-safety of anchor capture *(HIGH — RESOLVED, under Phase 1 walk)* + +`test-asert-anchor-captured-at-activation` builds one chain and checks +the happy path. Missing: + +- Two forks both reach `asert-anchor-height` with **different** + min-of-11 values. Which wins? Is the outcome independent of the + order in which blocks were received? +- Pre-activation reorg that replaces the block at `anchor-height` on + the heaviest chain. Expected behaviour needs to be specified + (currently: no-op, see bug 5) and asserted. +- Post-activation reorg through `anchor-height`. Captured value must + not change (currently true by no-op) and should be asserted so a + future refactor can't break it. + +**Resolution:** under Phase 1 there is no capture — `+find-anchor-min-ts` +walks from the caller-supplied parent-digest per-call, so the three +sub-items collapse to a single property: the walk must trace the +caller's own ancestry and be indifferent to the heaviest pointer. +`test-asert-find-anchor-walk-fork-safety` in +`closed/hoon/tests/dumb/mod/integration/asert-activation.hoon` pins +this property end-to-end: + +- Builds chain A to height 5 as heaviest. +- Builds a sibling chain B = G → A1 → B2 → B3 → B4, where B2 uses a + bumped timestamp so B4's median-of-11 diverges from A4's by + construction. +- Accepts B2..B4 via `+accept-page` but never promotes them via + `+update-heaviest`, so heaviest-block stays on A5. +- Asserts that `+find-anchor-min-ts(a5-digest)` returns A4's median, + `+find-anchor-min-ts(b4-digest)` returns B4's own (different) + median, and that the heaviest pointer never moved off chain A. + +This covers the "two forks with different min-of-11" sub-item +verbatim, the "pre-activation reorg replacing anchor-height" case by +showing two distinct anchor-height blocks coexist in `.blocks` and +each gets its own correct walk result, and the "post-activation reorg +through anchor-height" case by showing heaviest-movement is irrelevant +to the walk. The older `test-asert-wrapper-past-anchor` additionally +pins that the walk works on a descendant two steps past the anchor. + +### 12 — Memorylessness / reversibility (PDF §1.2) *(MEDIUM — RESOLVED)* + +The PDF puts reversibility at the centre of ASERT's design: a drift of +`+x` followed by `-x` returns to the exact pre-perturbation target +(modulo approximation error). Two concrete unit tests would pin this: + +- **Polynomial reversibility**: pick `y ∈ (0,1)` and check that + `poly-factor(y*radix) * poly-factor((1-y)*radix)` is within tolerance + of `2 * radix^2` (since `2^y · 2^{1-y} = 2`). +- **End-to-end reversibility**: chain of (on-schedule → fast by Δ → slow + by Δ → on-schedule) blocks returns to `anchor-target` within + polynomial tolerance. + +**Resolution:** the polynomial-level reversibility check is now +pinned by `test-asert-poly-factor-reversible` in +`closed/hoon/tests/dumb/mod/unit/asert.hoon` — it asserts +`poly-factor(f) * poly-factor(radix - f)` lands within 0.33% of +`2 * radix^2` across `f ∈ {1, radix/4, radix/2, 3*radix/4, radix-1}`. +End-to-end chain reversibility is mathematically equivalent to the +on-schedule identity under ASERT's stateless design — the target at +any post-anchor block depends only on `parent_ts - anchor_ts` and +`child.height - anchor.height`, so a chain whose net drift is zero +lands at `anchor-target` by the same code path exercised by +`test-asert-on-schedule-approx-identity`. No additional chain-level +test adds coverage that the polynomial test and the on-schedule test +do not already cover. + +### 13 — Wrapper-level negative-`time_diff` path *(MEDIUM — RESOLVED, library seam)* + +`+compute-exponent` has a third branch for negative `time_diff` +(`asert.hoon:54-55`). The cross-check test covers it in-library but no +wrapper-or-integration test drives `parent-min-ts < anchor-min-ts`. +Median-of-11 can step backwards across a reorg involving +pre-activation blocks (different ancestors → different medians), so the +branch is reachable in production. + +**Resolution:** covered at the library seam rather than the wrapper +seam. `test-asert-compute-exponent-negative-time` in +`closed/hoon/tests/dumb/mod/unit/asert.hoon` directly pins that the +third branch emits `sign = %.n` for `time-diff-sign = %.n`; the +existing `test-asert-ref-cross-check` and `test-asert-halflife-halves` +already exercise the branch end-to-end through `+compute-target`; and +the wrapper cross-checks (`test-asert-wrapper-matches-library`, +`test-asert-wrapper-past-anchor`) prove the wrapper forwards +`parent-min-ts` and `anchor-min-ts` to the library unchanged — so any +library-seam coverage of negative time-diff transfers to the wrapper. +Synthesising a median-of-11 regression on a real chain is deferred to +item 11 where fork helpers are introduced; the library-seam test is +load-bearing for the branch itself. + +### 14 — `+decompose-exponent` corner cases *(LOW — RESOLVED)* + +Currently only exercised end-to-end via `+ref-compute-target`. Direct +unit tests would shorten the debugging chain when something further +downstream changes: + +- `exp = 0` → `(%.y, 0, 0)` +- `exp = radix - 1` positive → `(%.y, 0, radix-1)` +- `exp = -1` → `(%.n, 1, radix-1)` (round-up path) +- `exp = -radix` exact → `(%.n, 1, 0)` +- `exp = -radix - 1` → `(%.n, 2, radix-1)` + +**Resolution:** `test-asert-decompose-exponent-corners` in +`closed/hoon/tests/dumb/mod/unit/asert.hoon` pins all five cases +verbatim via a single vase-level `expect-eq` on a list of inputs and +expected outputs. If any future rounding refactor perturbs the +boundary behaviour, this test surfaces it before anything downstream. + +### 15 — Test names in changelog ≠ in code *(LOW; docs drift — RESOLVED)* + +Changelog (`014-aletheia.md:374-392`) lists: + +- `test-asert-anchor-returns-identity` +- `test-asert-poly-factor-known-values` +- `test-asert-halflife-scaling` +- `test-asert-reference-impl` + +Code (`closed/hoon/tests/dumb/mod/unit/asert.hoon`) defines: + +- `test-asert-anchor-identity` +- `test-asert-poly-factor-zero` / `-monotonic` / `-near-one` +- `test-asert-halflife-doubles`, `test-asert-halflife-halves` +- `test-asert-ref-cross-check` +- plus `test-asert-bn-wrapper` (not listed in changelog at all) + +Purely a documentation/naming sync issue, but will confuse reviewers +doing a "check that claimed tests exist" pass. + +**Resolution:** the "Unit tests" section of `014-aletheia.md` has +been rewritten to mirror the actual test file, grouped by purpose +(identity & monotonicity / polynomial / exponent / halflife & clamping +/ cross-checks). Every arm in `closed/hoon/tests/dumb/mod/unit/asert.hoon` +is now listed under the correct heading, including the BCH QA runs +and the defensive-guard tests added for audit items 7 and 8. diff --git a/changelog/protocol/014-aletheia.md b/changelog/protocol/014-aletheia.md new file mode 100644 index 000000000..43f30b665 --- /dev/null +++ b/changelog/protocol/014-aletheia.md @@ -0,0 +1,1065 @@ ++++ +version = "0.1.14" +status = "activated" +consensus_critical = true + +activation_height = 65500 +published = "2026-04-27" +activation_target = "2026-05-07" + +authors = ["@nockchain-core"] +reviewers = ["@nockchain-core"] + +supersedes = "0.1.13" +superseded_by = "" ++++ + +# Aletheia + +ASERT per-block difficulty adjustment, block-time reduction from 600s to +150s, and a unified emissions curve with a smooth-decay 9.5-year body and +a 64-NOCK floor sustained for ~68 years to a 2³² hard cap, with every +post-activation reward split 80/20 between the miner and a time-locked +protocol fund. + +## Summary + +Aletheia bundles three consensus changes onto a single activation block at +height 65,500: + +1. **ASERT difficulty adjustment.** Replaces nockchain's Bitcoin-style + 2,016-block epoch retarget with aserti3-2d (Absolutely Scheduled + Exponentially Rising Targets), the per-block difficulty algorithm + formalized by Jonathan Toomim and shipped on Bitcoin Cash in 2020. + +2. **Block-time reduction from 600s to 150s.** The ideal inter-block + interval drops by 4×, with ASERT's half-life scaled proportionally + to preserve its block-count-based stability properties. + +3. **Unified emissions curve.** Replaces the 16-halving schedule with a + single 14-row table that runs from genesis through a 2³² hard cap. + Eons 0–2 (the historical bootstrapping phase) are preserved as-is + but eon 2 is truncated from block 78,895 to block 65,500. Post- + activation, reward decays through nine smaller steps (alternating + 25% / 33% drops, no halvings) over 9.5 years, then settles at a + 64-NOCK floor for ~68 years until the cap is met exactly. Every + post-activation reward is split 80% miner / 20% protocol fund; + both outputs are coinbase outputs subject to the existing standard + coinbase timelock — no separate unlock schedule is layered on the + fund's share. + +The block-time change *is* the immediate halving the new emissions +curve calls for at activation: 16,380 NOCK per 600s pre-activation +(1,638 NOCK/min) becomes 2,048 NOCK per 150s post-activation +(819.2 NOCK/min) — exactly half the per-time issuance rate. Bundling +these into one consensus event avoids two separate hard-fork +coordinations. + +## Motivation + +### Epoch retarget is sampling-sensitive + +The current algorithm recomputes difficulty only at epoch boundaries +(every 2,016 blocks, ~14 days at 600s cadence) by comparing the actual +epoch duration against the target epoch duration and clamping the ratio +to a 4×/¼× bucket. Within an epoch the target is fixed. This has two +well-known failure modes: + +- **Hashrate swings cause long off-target epochs.** A 50% hashrate drop + midway through an epoch leaves the network running at half speed for + up to a week before the next retarget can respond. +- **Sampling attacks.** Because the algorithm samples a single + `(epoch-start, epoch-end)` pair, miners can influence the next-epoch + target by timestamp placement around those two boundary blocks, + independent of real-time difficulty behavior in the interior. This + class of attack is documented against Bitcoin-style retarget since + 2014 and is what motivated BCH's move to aserti3-2d. + +### ASERT responds every block and is anchor-based + +aserti3-2d recomputes the target at every block from a fixed anchor +`(height, target, timestamp)` captured at fork activation. The response +to drift is a smooth exponential rather than a clamped ratio, and the +computation depends only on the current block's parent chain (not on a +per-epoch "window boundary"), which eliminates the boundary-sampling +attack surface entirely. + +### Block time reduction + +This upgrade also drops the ideal inter-block interval from 600s to +150s. The block-time change is bundled with ASERT activation because: + +- ASERT parameters (`asert-anchor-target-atom`, + `asert-ideal-block-time`, `asert-half-life`) directly encode the new + cadence. Introducing the new DAA on the old cadence and then moving + the cadence later would require two separate anchor pinnings and two + separate consensus events. +- ASERT's half-life is measured in real-time seconds but its stability + properties are measured in block counts. BCH's 172,800s half-life + corresponds to 288 blocks at 600s. Preserving the 288-block stability + window at the new 150s cadence gives a 43,200s (12h) half-life. +- The anchor target (`2^291`) is chosen to approximate a ~4× target + increase over the pre-activation mainnet target, so expected blocks + per second at constant hashrate rise by ~4× (600s → 150s). The + closest power of two to 4× the pre-activation mainnet target + (~2^291.38) yields ~3.2× faster blocks (~187s expected) under + unchanged hashrate — slightly conservative vs the ideal 150s target. + ASERT converges to the ideal 150s from that starting point. + +### Why bundle emissions with the difficulty / block-time change + +The original schedule front-loads ~99% of emissions into the chain's +first thirty years, leaving no meaningful long-run mining subsidy and +forcing the network onto fee revenue earlier than the application +ecosystem can sustain. A revised schedule has to (a) cut per-time +emissions to extend the chain's revenue tail, and (b) avoid the +familiar 50%-revenue-cliff dynamics that traditional halvings impose +on miners. + +The block-time drop from 600s to 150s already cuts per-time emissions +by 75% on its own (4× more blocks at the same per-block reward would +quadruple issuance, so a 4× block-time drop demands a per-block reward +that is at least 1/4 the prior value to keep issuance flat — and we go +further, dropping to ~1/8 the prior per-block reward, for a clean 1/2 +per-time cut). Doing the per-block reward change at the same height as +the block-time change means the activation experience is a single +clean per-time halving, not two staggered economic events. + +Beyond the activation halving, the new schedule's smooth-decay curve +(alternating 25% / 33% drops over 9 era boundaries) and 64-NOCK floor +require their own table-lookup `block_reward(height)`. That table is +a strict superset of the activation-height behaviour, so it slots in +cleanly at the same `asert-phase` boundary and reuses no logic from +the legacy halving schedule beyond the eon-0/1/2 head. + +The 80/20 miner/fund split falls out of the same activation: +pre-activation the chain has no fund and a single-output coinbase. +Post-activation it pays a second output to a well-known fund hash, +subject to the same standard coinbase timelock as the miner's share. +Bundling means the consensus rule "a v1 coinbase has either one +output (pre-activation) or two (post-activation)" can be stated +against a single height boundary. + +## Technical Specification + +### Blockchain-Constants + +Five new fields are added to `blockchain-constants` in +`open/hoon/common/tx-engine-1.hoon`: + +```hoon +asert-phase=65.500 :: activation height +asert-anchor-height=65.499 :: asert-phase - 1 +asert-anchor-target-atom=^~((bex 291)) :: 2^291 +asert-ideal-block-time=150 :: seconds, new block time +asert-half-life=^~((mul 12 ^~((mul 60 60)))) :: 43,200s = 12h +``` + +`rbits` is **not** a `blockchain-constants` field. The polynomial +coefficients in `lib/asert.hoon` are tied to `rbits=16` and cannot +vary without replacing the polynomial, which is itself a hard fork. +`rbits` is therefore a compile-time constant (`++ asert-rbits 16`) +rather than a per-network parameter. + +The emissions changes do **not** add any new `blockchain-constants` +fields. The schedule's era boundaries (`asert-phase + 105.000` for +the end of eon 3, `+ 210.000` per subsequent eon, tail-end at +`16.144.876`), the floor (64 NOCK), the 80/20 split (`1/5` to fund), +are all hardcoded as bare integer literals in the relevant +`++ schedule` and `++ new:coinbase` arms. +The single piece of consensus-known data the emissions layer needs — +the fund recipient hash — is a hardcoded `++ fund-address` arm in +`open/hoon/common/tx-engine-1.hoon`. Setting the real fund-multisig +hash before mainnet activation is therefore a code change, not a +constants-overlay change. (The implementation ships with a +placeholder `*hash` zero value for review and testing.) + +The reversion-to-100%-miner trigger described elsewhere is *not* +part of this upgrade. It is a **future** protocol upgrade that will +ship the new fully useful PoW puzzle and a corresponding +consensus-layer mechanism for the split to revert. Until that +ships, every post-activation block pays the 80/20 split +unconditionally. + +The anchor's median-of-11 timestamp is **not** a constant — see +"Consensus State: Anchor Timestamp" below. + +`blocks-per-epoch` is not changed. Legacy `epoch-counter` and +`epoch-start` accounting continue to be written and validated for +pre-activation blocks; post-activation blocks continue writing them +(inert but preserved for header compatibility) but they no longer +gate target computation. + +### ASERT Algorithm + +Added as a new pure-math library at +`open/hoon/apps/dumbnet/lib/asert.hoon`. The core arm is +`+compute-target`: + +``` +time-diff = current-min-timestamp - anchor-min-timestamp (signed) +height-diff = current-height - anchor-height (>= 1) +exponent = ((time-diff - ideal * (height-diff - 1)) * radix) / half-life + (signed) +shifts = exponent >> rbits (arithmetic) +frac = exponent - (shifts << rbits) (in [0, radix)) +factor = radix + ( 195_766_423_245_049 * frac + + 971_821_376 * frac^2 + + 5_127 * frac^3 + + 2^47 ) >> 48 +next-target = (anchor-target * factor) << shifts (or >> -shifts) +next-target = next-target >> rbits +clamp to [1, max-target] +``` + +`current-min-timestamp` is the *parent* block's median-of-11 and +`current-height` is the *child* block's height (the block whose target we +are computing). `anchor-min-timestamp` is the anchor block's *own* +median-of-11 — this is PDF Eq. (2) §1.3 Option 2 ("nearly identical but +distinct" from BCH's anchor-parent convention). Under a perfect schedule, +`current-min-timestamp - anchor-min-timestamp` spans `height-diff - 1` +ideal intervals (parent is `height-diff - 1` blocks past the anchor), so +the `(height-diff - 1)` term leaves a zero exponent at schedule. + +Where `rbits = 16`, `radix = 2^16`. The 3rd-order polynomial +approximates `2^x` on `[0, 1)` with max error under 0.13%. Polynomial +coefficients are tied to `rbits`; changing `rbits` would itself be a +hard fork. The `+ 2^47` term is round-to-nearest on the fixed-point +divide by `2^48` and is required for bit-for-bit parity with BCH's +canonical `aserti3-2d` polynomial. + +Sign handling: `compute-target` branches early on the sign of +`current-min-timestamp - anchor-min-timestamp` and keeps all +intermediate arithmetic on unsigned atoms, tracking the exponent sign +in a `?` and inverting shift direction at the end. This avoids `+si` +ceremony and mirrors how `compute-target-raw` keeps math in plain +atoms today. + +Shift magnitude is capped at `max(max-target-bits) + rbits + 2` for +positive shifts and at the bit-length of the intermediate for negative +shifts, so the noun representation of intermediates stays bounded even +under pathological exponents. + +A thin bignum wrapper `+compute-target-bn` converts at the boundary +so callers on `bignum` don't reach into atom math. + +### Timestamp Source: Median-of-11 + +ASERT reads timestamps from the existing `min-timestamps` median-of-11 +map, not raw `pag.timestamp`. Specifically, `current-min-timestamp` is +`min-timestamps[pag.parent]` — the parent's median-of-11, already +computed, stored, and validated at the point we need it. This +sidesteps the "miner sets a bogus timestamp on the next block" attack +surface that raw-timestamp ASERT implementations are vulnerable to. + +*Consequence*: BCH's published aserti3-2d test vectors (which use +raw timestamps) are **not** directly reusable. The polynomial piece +itself is timestamp-independent and is cross-checked against hand- +computed values; end-to-end correctness is verified against a +reference Hoon implementation in the test file. + +### Anchor timestamp resolution + +The anchor's median-of-11 timestamp is not known at compile time — it +is only well-defined once block `asert-anchor-height` has been +processed and its median-of-11 computed. Phase 1 resolves it at +target-compute time by walking `.blocks` from the requested +`parent-digest` back to its ancestor at `asert-anchor-height` and +reading that ancestor's entry from `.min-timestamps`. No consensus- +state field is introduced; `consensus-state` remains at version 6. + +Fork-correctness falls out for free: every caller walks its own +ancestry, so competing forks never read or write a shared scalar and +cannot diverge each other's target computation. Phase 2 (post-65500) +replaces the walk with a hardcoded protocol constant and retires the +helper. + +#### `+find-anchor-min-ts` + +In `open/hoon/apps/dumbnet/lib/consensus.hoon`: + +```hoon +++ find-anchor-min-ts + |= bid=block-id:t + ^- @ + =/ anchor-height=@ asert-anchor-height.blockchain-constants + =/ cur=page:t (to-page:local-page:t (~(got z-by blocks.c) bid)) + |- + ?: =(~(height get:page:t cur) anchor-height) + (~(got z-by min-timestamps.c) ~(digest get:page:t cur)) + $(cur (to-page:local-page:t (~(got z-by blocks.c) ~(parent get:page:t cur)))) +``` + +Callers (`+compute-target-asert` in `consensus.hoon` and the miner +candidate-target path in `miner.hoon`) must guarantee the input +digest's block is in `.blocks` with height ≥ `asert-anchor-height`. +This invariant holds at every ASERT call site because ASERT activates +at `asert-phase = asert-anchor-height + 1` and all such callers pass +a parent whose block is already in `.blocks`. + +### Consensus Wrapper + +`+compute-target-asert` in +`open/hoon/apps/dumbnet/lib/consensus.hoon` (new arm): + +```hoon +++ compute-target-asert + |= [child-height=@ parent-digest=block-id:t] + ^- bignum:bignum:t + =/ parent-min-ts=@ (~(got z-by min-timestamps.c) parent-digest) + =/ anchor-min-ts=@ (find-anchor-min-ts parent-digest) + %- chunk:bignum:t + %- compute-target:asert + :* asert-anchor-target-atom.blockchain-constants + anchor-min-ts + asert-anchor-height.blockchain-constants + parent-min-ts + child-height + asert-ideal-block-time.blockchain-constants + asert-half-life.blockchain-constants + max-target-atom:t + == +``` + +### Activation Gating + +Two call sites in `consensus.hoon` are gated on +`(gte height asert-phase)`: + +1. **Target write in `+accept-page`** — if post-activation, + `targets[pag.digest] := compute-target-asert(child-height, parent)`. + Else retain existing epoch/genesis/carry-forward logic. +2. **Target check in `+validate-page-without-txs`** — if + post-activation, compare `pag.target` against + `compute-target-asert(child-height, parent)`. Else retain the + existing `pag.target == targets[parent]` lookup. + +Mirrors the existing activation-height patterns +(`height-to-proof-version`, `v1-phase`, `bythos-phase`). + +### Miner + +`open/hoon/apps/dumbnet/lib/miner.hoon`'s `heard-new-block` path +derives `anchor-min-ts` at candidate-target time by calling +`+find-anchor-min-ts` from `consensus.hoon` against heaviest-block. + +### Emissions Schedule + +`open/hoon/common/schedule.hoon`'s `++ schedule` is replaced with a +height-dispatched function that covers all heights from genesis through +the 2³² hard cap in a single 14-row table. The function is pure and +constant-time after a small chain of inequality checks. + +| Eon | Block range | Blocks | Reward (NOCK) | Block time | Duration | Emission (NOCK) | Drop | Cum. % | +|------|--------------------------|------------:|--------------:|------------|----------|----------------:|-------|-------:| +| 0 | 1 – 13,150 | 13,150 | 65,536 | 10 min | ~3.0 mo | 861,798,400 | — | 20.06% | +| 1 | 13,151 – 39,448 | 26,298 | 32,768 | 10 min | ~6.0 mo | 861,732,864 | −50% | 40.12% | +| 2 | 39,449 – 65,500 | 26,052 | 16,384 | 10 min | ~5.9 mo | 426,835,968 | −50% | 50.07% | +| 3 | 65,501 – 170,500 | 105,000 | 2,048 | 2.5 min | 6 mo | 215,040,000 | −50%* | 55.08% | +| 4 | 170,501 – 380,500 | 210,000 | 1,536 | 2.5 min | 1 yr | 322,560,000 | −25% | 62.58% | +| 5 | 380,501 – 590,500 | 210,000 | 1,024 | 2.5 min | 1 yr | 215,040,000 | −33% | 67.59% | +| 6 | 590,501 – 800,500 | 210,000 | 768 | 2.5 min | 1 yr | 161,280,000 | −25% | 71.35% | +| 7 | 800,501 – 1,010,500 | 210,000 | 512 | 2.5 min | 1 yr | 107,520,000 | −33% | 73.85% | +| 8 | 1,010,501 – 1,220,500 | 210,000 | 384 | 2.5 min | 1 yr | 80,640,000 | −25% | 75.73% | +| 9 | 1,220,501 – 1,430,500 | 210,000 | 256 | 2.5 min | 1 yr | 53,760,000 | −33% | 76.98% | +| 10 | 1,430,501 – 1,640,500 | 210,000 | 192 | 2.5 min | 1 yr | 40,320,000 | −25% | 77.92% | +| 11 | 1,640,501 – 1,850,500 | 210,000 | 128 | 2.5 min | 1 yr | 26,880,000 | −33% | 78.54% | +| 12 | 1,850,501 – 2,060,500 | 210,000 | 96 | 2.5 min | 1 yr | 20,160,000 | −25% | 79.01% | +| Tail | 2,060,501 – 16,144,876 | 14,084,376 | 64 | 2.5 min | ~67 yrs | 901,400,064 | −33% | 100.00%| + +\* The −50% drop at the eon 2 → 3 boundary is **per unit time**, not +per block. The per-block reward falls 87.5% (16,384 → 2,048) but the +block time also drops 4× (600s → 150s), so NOCK-per-minute exactly +halves: 1,638.4 NOCK/min → 819.2 NOCK/min. + +The cumulative emission at block 16,144,876 is exactly +`4,294,967,296 = 2³²` NOCK; subsequent blocks emit zero. At 64 +NOCK/block, the tail block count is exactly `901,400,064 ÷ 64 = +14,084,376` and the cap is hit exactly with **no per-block dust +carve-out** required. + +#### Why this shape + +- **Immediate halving on a per-time basis.** The per-time issuance rate + exactly halves at activation, satisfying the canonical "halving" + contract without imposing a 50% per-block revenue drop on miners. + The per-block reward drops by 87.5% but is paired with a 4× increase + in blocks per minute, so a miner running fixed hardware sees their + NOCK-per-minute revenue cut by exactly 50% — the same rate-of-change + a Bitcoin-style halving produces, but with the rest of the per-block + schedule decoupled from this one shock. + +- **Smooth decay, no halvings, max −33% step.** After the activation + era, reward decays through a deliberately smooth sequence: + `2048 → 1536 → 1024 → 768 → 512 → 384 → 256 → 192 → 128 → 96 → 64`. + Each transition is either a 25% drop (a 3/4 reduction) or a 33% drop + (a 2/3 reduction); the drops alternate predictably. No transition is + a full halving. A miner whose hardware becomes unprofitable at a + given era boundary was already close to marginal profitability — + roughly the bottom 25% or 33% of the fleet by efficiency, not the + bottom 50%. + +- **Multiples of 32.** Every reward is a multiple of 32, which keeps + the arithmetic exact in any fixed-point system and ensures the + 20% fund share (= reward / 5) is also exactly representable + (since both `2^16 = atoms-per-nock` and the reward sequence are + divisible by 5 after multiplication; the proportional-allocation + arithmetic in `++ new:coinbase-split` is exact for any reward that + is a clean multiple of 5, which all post- + activation rewards are). + +- **A 64-NOCK floor for ~68 years.** The tail spans approximately + two to three human generations. Setting a fixed floor (rather than + decaying to an asymptotic trickle as the original schedule did) + preserves a meaningful real per-block subsidy through the tail + phase and pins exact emission of `2³²` NOCK at a known terminal + block. + +#### Cap accounting + +Cumulative supply through end-of-decay (block 2,060,500) totals +`3,393,567,232` NOCK: + +- Eons 0–2 (powers of two, the actual on-chain history): + `861,798,400 + 861,732,864 + 426,835,968 = 2,150,367,232 NOCK`. +- Eon 3: `105,000 × 2,048 = 215,040,000 NOCK`. +- Eons 4–12 (smooth decay): `1,028,160,000 NOCK`. + +Remaining budget to the `2³² = 4,294,967,296` cap: +`901,400,064 NOCK`. At 64 NOCK/block this is exactly `14,084,376` +tail blocks, so the tail spans `2,060,501..=16,144,876`, every tail +block emits 64 NOCK, and the cap is hit exactly with no boundary +case for "dust" handling. `++ schedule` returns 0 for any height +past 16,144,876. + +#### Reference pseudocode + +``` +const DECAY_REWARDS: [u64; 9] = + [1536, 1024, 768, 512, 384, 256, 192, 128, 96]; + +fn block_reward(height: u64) -> u64 { + if height == 0 { return 0; } + if height > 16_144_876 { return 0; } + if height <= 13_150 { return 65_536; } + if height <= 39_448 { return 32_768; } + if height <= 65_500 { return 16_384; } + if height <= 170_500 { return 2_048; } + if height <= 2_060_500 { + let era_idx = (height - 170_501) / 210_000; + return DECAY_REWARDS[era_idx as usize]; + } + 64 // tail +} +``` + +The hoon implementation (in `open/hoon/common/schedule.hoon`) follows +this branch order. Returned values are in units of NOCK; the schedule +arm multiplies by `atoms-per-nock = 2^16` internally to produce on- +chain atom-denominated coin amounts. + +### Reward Distribution + +Pre-activation, every coinbase pays one output: 100% to the block +miner, with the existing relative timelock (`coinbase-timelock-min = +100`). Post-activation, every coinbase pays two outputs — 80% to the +miner and 20% to the consensus-known protocol fund address — with +both outputs subject to the same standard coinbase timelock. There +is no separate unlock schedule layered on the fund's share: it is a +normal coinbase output, spendable as soon as the standard relative +lock matures. + +#### 80/20 split + +Post-activation, the miner builds the v1 `coinbase-split` via a +dedicated arm `++ new-with-fund-share:coinbase-split` (in +`tx-engine-1.hoon`) which takes the block's `emission`, `fees`, and +the miner-side `shares` map separately: + +``` +fund-coins = (div emission 5) :: subsidy only +miner-pool = (sub emission fund-coins) + fees :: 80% + all fees +miner-split = (++new miner-pool shares) :: legacy proportional arm +output = miner-split + { (fund-address, fund-coins) } +``` + +`shares` is the standard miner-side recipient map (1 or 2 PKHs +under the existing `max-coinbase-split = 2` cap), and is shared +with the pre-activation builder. Partner mode keeps working +post-activation: a 2-PKH `shares` produces a 3-output coinbase +(2 miner + fund), which the relaxed `++ based:coinbase-split:v1` +admits up to `max-coinbase-split + 1 = 3` entries. + +The fund value is the integer floor `(div emission 5)`. Any per- +block atom remainder accrues to the miner pool (the higher-share +side), and within the miner pool the legacy `++new` arm assigns +the residual to whichever miner key sorts first in z-map order. +This rounding is exact at the NOCK level for every post-activation +reward (the schedule table values are all multiples of 5). + +**Fees never enter the fund slot.** Computing the fund from +`(emission + fees)` would produce a coinbase that consensus rejects +as `%improper-fund-split`, since `+check-fund-split` keys on +`emission` alone — see "Review fixes" below. + +The 80/20 split applies uniformly to every reward value from the +2,048 NOCK activation reward through the 64-NOCK tail floor. The +underlying emission curve is unchanged by the distribution layer: +total NOCK issued per block, per era, and across the full schedule +is identical to the values in the schedule table above. Only the +set of recipients differs. + +The split is gated by a single phase boundary: + +- `height < asert-phase` — single-output coinbase, 100% miner. (Pre- + activation behaviour, unchanged.) +- `height ≥ asert-phase` — two-output coinbase, 80/20. + +A future protocol upgrade is expected to flip the chain back to a +100%-miner coinbase once the new fully useful PoW puzzle ships on +the upgraded Nock ZKVM. That reversion is **not** part of this +upgrade; it will be specified and audited in a separate consensus +event when the puzzle is ready. + +#### Effect on sell-flow + +Under the standard assumption of ~100% miner sell-through to cover +electricity and hardware costs, post-activation sell-flow from +miners is `80%` of emissions rather than `100%`. The remaining `20%` +flows to the consensus-known fund address and is not automatically +sold; the fund's disposition is governed by a multisig, and its on- +chain behaviour is observable — any output moving from the fund to a +known exchange address can be tracked in real time, allowing +sophisticated participants to price in the fund's actual behaviour +rather than worst-case assumptions about it. + +Combined with the per-time halving from the block-time change, the +effective post-activation miner sell-flow is `40%` of the pre- +activation baseline (`50%` from the per-time halving × `80%` from +the split-to-fund), with another `10%` of the pre-activation rate +flowing to the fund. + +#### Validation + +`open/hoon/apps/dumbnet/lib/consensus.hoon`'s coinbase enforcement adds +one new check, `+check-fund-split`, that runs when +`height ≥ asert-phase`: + +- The coinbase is v1. +- The fund-address slot exists in the `coinbase-split` (a hardcoded + `++ fund-address` arm in `tx-engine-1.hoon`). +- That slot's coin value equals `(div emission 5)` (the integer + floor of 20% in atoms). + +The miner-side allocation is implicitly verified: the existing +total-split-equals-(emission+fees) check at the top of +`+validate-page-with-txs` already pins the sum, and `+based: +coinbase-split` caps total entries at `max-coinbase-split + 1 = 3`, +so the miner side equals `(emission - fund-coins) + fees` partitioned +across one or two miner outputs. Both coinbase outputs use the same +standard coinbase timelock, enforced by the existing +`+based:coinbase-split` check and the per-output `timelock-intent` +field on each nnote. + +#### Review fixes + +Two issues caught in review of the initial implementation, both +addressed in this branch: + +- **Fee handling.** The earlier `++ new-with-fund-share` took a + single `assets` parameter that callers populated with + `emission + fees`, so the fund slot was computed as + `(div (emission + fees) 5)` — but consensus expects + `(div emission 5)`. Any post-activation block with non-zero fees + was self-rejected by the same miner that built it. The builder + now takes `emission` and `fees` separately, routing fees entirely + to the miner pool. +- **Multi-miner crash.** The earlier builder asserted exactly one + `shares` key and discarded share weights, which crashed candidate + construction whenever a closed-miner partner-mode configured two + payout PKHs and the chain tip reached `asert-phase`. The builder + now distributes the miner pool across `shares` via the same + proportional arm as `++ new`, supporting 1- or 2-recipient + miner-side configurations. `+check-fund-split` and + `+based:coinbase-split:v1` were relaxed to admit the resulting + 3-entry coinbases (2 miners + fund) while still pinning the + fund slot to `(div emission 5)`. + +## Activation + +- **Height**: 65,500 (`asert-phase`). All three changes — ASERT + difficulty, 150s block time, new emissions schedule with 80/20 + miner/fund split — activate atomically at this single height. +- **Anchor block height**: 65,499 (`asert-phase - 1`). +- **Eon-2 truncation**: under the unified schedule, eon 2 ends at + block 65,500 instead of its original 78,895. The 13,395 blocks of + emission that would have been issued under the old curve at 16,380 + NOCK each (~219.4M NOCK) are absorbed into the new post-activation + budget rather than minted at the legacy rate. +- **Coordination**: all nodes must upgrade before block 65,500. The + anchor's median-of-11 is derived at target-compute time from + `.blocks` and `.min-timestamps`, so there is no runtime capture + step for operators to coordinate. +- **Fund address**: a real fund-multisig hash must be committed to + `blockchain-constants.fund-address` before mainnet rollout. The + reference implementation ships with a placeholder zero hash that + must be replaced before activation; activating with the placeholder + would direct fund-share outputs to an unspendable address. + +## Migration + +### Requirements + +- Software version: 0.1.14+ +- All nodes must upgrade before `asert-phase = 65,500`. + +### Configuration + +No mandatory configuration changes. + +### Data Migration + +No kernel-state version bump for this upgrade. The ASERT schema +extension to `blockchain-constants:v1` is already covered by the +existing `++ state-6-to-7` arm in `open/hoon/apps/dumbnet/inner.hoon` +(Aletheia's pre-existing kernel-state-7 bump). The emissions changes +are pure-logic edits to the `++ schedule` and `++ new:coinbase` +arms; the noun layout of `blockchain-constants` is unchanged. + +The anchor's median-of-11 is still derived at target-compute time +from existing `.blocks` and `.min-timestamps` indices. Phase 2 +(post-65500) will add an `asert-anchor-min-timestamp` constant, +which is read-only config and does not touch consensus state. + +### Steps + +1. Stop the node. +2. Update to version 0.1.14 or later before block 65,500. +3. Restart the node. State auto-upgrades through v7 on load. + +### Rollback + +Rollback to a pre-Aletheia binary is safe only before `asert-phase`. +After activation, downgrading will reject valid blocks (their targets +are computed by ASERT, their coinbases pay two outputs, and the +post-activation rewards are not on the legacy halving schedule). + +## Backward Compatibility + +### Breaking Changes + +This is a **consensus-critical** upgrade. After activation: + +- Nodes running pre-0.1.14 software will reject blocks with ASERT + targets, and will compute expected targets from the old epoch + algorithm. +- The `blockchain-constants` noun structure gains 6 ASERT fields, + which causes decoding failures on pre-0.1.14 software. +- Pre-0.1.14 nodes will reject post-activation coinbases that pay two + outputs (their validators expect the legacy single-output coinbase). +- The legacy `++ schedule:emission` arm produces the same NOCK/block + values as the new schedule for blocks 1..=65,500, so a pre-0.1.14 + node that somehow sees a post-activation block will additionally + reject it on emission validation (`emission-and-fees != total-split`). + +### Network Partition Risk + +Any node that does not upgrade before block 65,500 will: +- fork onto an incompatible chain, +- reject valid post-activation blocks, and +- have mined blocks rejected by upgraded nodes. + +**All node operators must upgrade before block 65,500.** + +### Transaction Compatibility + +This upgrade does not change transaction formats. Transactions created +with pre-Aletheia software remain structurally valid. + +## Security Considerations + +- **Timestamp manipulation resistance.** ASERT reads median-of-11 + timestamps (via the existing `min-timestamps` map), not raw + `pag.timestamp`. A single miner setting a bogus timestamp on one + block cannot move the target more than the median-of-11 absorbs. +- **No sampling attack surface.** Unlike epoch retarget, ASERT + recomputes at every block from a fixed anchor. There is no "next + retarget boundary" a miner can influence by placing timestamps. +- **Anchor derivation is fork-correct.** The anchor's median-of-11 + is derived per-call by `+find-anchor-min-ts` walking `.blocks` + from the caller's own `parent-digest` back to the ancestor at + `asert-anchor-height`. No shared mutable scalar exists, so + competing pre-activation forks at `asert-anchor-height` with + different median-of-11 values produce divergent *ancestries*, + not divergent *state*, and their post-activation descendants + compute independently. Phase 2 (post-65500) replaces the walk + with a hardcoded protocol constant paired with a checkpoint at + anchor-height, at which point only one block at that height is + admissible anywhere on the network. +- **Shift-magnitude bound.** Intermediate noun size is explicitly + capped by clamping shift magnitudes before `lsh`/`rsh`, so + pathological inputs cannot produce a state explosion. +- **No new cryptographic primitives.** The polynomial is a pure + arithmetic approximation and introduces no new trust assumptions. +- **Hard cap is exact.** Cumulative emission summed across all + blocks `1..=16,144,876` equals exactly `2^32` NOCK. Every tail + block emits the same 64 NOCK, with no boundary carve-outs; a + unit test pins the cumulative total to `2^32 × atoms-per-nock`, + and `++ schedule` returns 0 for any height past 16,144,876. + There is no asymptotic-tail trickle and no path to overemission. +- **80/20 split cannot be diverted.** `++ check-fund-split` in + `consensus.hoon` rejects any post-activation coinbase whose split + does not have exactly two entries with one entry's key equal to + the consensus-known `fund-address` and that entry's coin value + equal to `(div emission 5)`. A miner who attempts to redirect the + fund's 20% to a different address produces a block that fails this + check. + +## Operational Impact + +- **Block time drops from 600s to 150s.** Expect ~4× more blocks per + unit time after activation. Mempool churn, block propagation + requirements, and fee estimation tooling should be validated + against the new cadence. +- **Per-block difficulty is lower by ~3.2–4× at activation.** Anchor + target is `2^291`, chosen slightly conservative vs the ideal 150s + target. ASERT converges to the ideal 150s from there under + unchanged hashrate. +- **Per-block reward drops to 2,048 NOCK.** Per-time NOCK issuance + exactly halves at activation (1,638 NOCK/min → 819.2 NOCK/min). A + miner running fixed hardware sees their NOCK-per-minute revenue cut + by 50%; the per-block reward cut is larger (87.5%) only because the + block time also drops. +- **Reorg depth expectations shift.** Confirmation counts measured in + blocks should be reinterpreted in wall-clock terms. Six blocks is + now ~15 minutes of work, not ~1 hour. +- **Half-life is 12 hours.** A block taking `half-life` seconds + longer than scheduled halves the difficulty (doubles the target); + a block `half-life` seconds earlier than scheduled doubles the + difficulty. +- **Effective-revenue cadence is annual after the activation era.** + After the 6-month activation era at 2,048 NOCK, every era boundary + is exactly one calendar year apart and produces either a 25% or a + 33% revenue drop (alternating). Mining-fleet planning horizons + should be aligned to this annual cadence. +- **Coinbase outputs have changed.** Pre-activation coinbases pay one + output (100% miner). Post-activation coinbases pay two: an 80% + miner output and a 20% fund output. Both use the same standard + coinbase timelock; there is no separate per-era unlock schedule. + Wallets, block explorers, and accounting tooling should expect + two coinbase outputs per block from height 65,500 onward. +- **Monitoring.** Operators should watch for: + - first post-activation block at height 65,500 matching expected + ASERT target, + - target drift tracking ASERT convergence toward 150s over the + first few half-lives, + - block 65,501 emitting `2,048` NOCK total (eon 3 starts), + - block 65,501's coinbase split producing exactly two outputs, + with the fund's atom value `= 2,048 × 2^16 / 5` (integer + floor; the `coinbase-split` proportional-allocation arm sends + the per-block atom remainder to the higher-share recipient, + so the miner output is the residual `total − fund`). + +## Testing and Validation + +### Unit tests — `closed/hoon/tests/dumb/mod/unit/asert.hoon` + +Identity and monotonicity: +- `test-asert-anchor-identity`: at `blocks-since-anchor=1` with + `current-min-timestamp = anchor-min-timestamp`, the computed target + equals `anchor-target` exactly. +- `test-asert-on-schedule-approx-identity`: N blocks on schedule keeps + the target at `anchor-target` (exponent = 0, factor = radix). +- `test-asert-monotonic-timestamp`: holding other inputs constant, a + larger `current-min-timestamp` yields a strictly larger target. +- `test-asert-monotonic-height`: holding other inputs constant, a + larger `current-height` yields a strictly smaller target. + +Polynomial approximation (`+poly-factor`): +- `test-asert-poly-factor-zero`: polynomial equals `radix` exactly at + `frac = 0`. +- `test-asert-poly-factor-monotonic`: polynomial is non-decreasing + across `frac` samples spanning `[0, radix)`. +- `test-asert-poly-factor-near-one`: at `frac = radix - 1` the + polynomial is within 200 of `2 * radix` (well inside the 0.13% + bound). +- `test-asert-poly-factor-reversible`: PDF §1.2 reversibility — + `poly-factor(f) * poly-factor(radix - f)` is within 0.33% of + `2 * radix^2` for a range of `f` values. +- `test-asert-poly-factor-rejects-oversized-frac`: `+poly-factor` + guard fails loudly at `frac = radix` and `frac = radix + 1`. + +Exponent decomposition and branches (`+compute-exponent`, +`+decompose-exponent`): +- `test-asert-decompose-exponent-corners`: direct pins for the + `exp = 0, radix-1, -1, -radix, -(radix+1)` corner inputs. +- `test-asert-compute-exponent-negative-time`: canary for the + negative-time-diff branch (parent median below anchor median). + +Halflife and clamping: +- `test-asert-halflife-doubles`: a block `half-life` seconds late + approximately doubles the target (within 0.01%). +- `test-asert-halflife-halves`: a block `half-life` seconds early + approximately halves the target. +- `test-asert-clamps-max-target`: extreme positive exponent saturates + at `max-target-atom`. +- `test-asert-clamps-min`: extreme negative exponent saturates at 1 + (never 0). +- `test-asert-rejects-zero-anchor-target`: `+compute-target` guard + fails loudly on a misconfigured `anchor-target = 0`. + +Cross-checks against independent implementations: +- `test-asert-ref-cross-check`: `+compute-target` agreement with a + separate reference implementation in the test file, across a grid + of `(blocks-since, drift, anchor-target)` inputs including + negative-time-diff cases. +- `test-asert-bn-wrapper`: `+compute-target-bn` matches + `+compute-target` after bignum ↔ atom conversion. +- `test-asert-bch-nbits-expansion`, `test-asert-bch-nbits-roundtrip`: + codec pins for the BCH compact-`nBits` encoding used by the run + vectors. +- `test-asert-bch-run01` … `test-asert-bch-run12`: all 12 published + BCH aserti3-2d QA vectors (143 iter rows total) reproduced via + `+compute-target:asert` under BCH mainnet parameters. + +### Integration tests — `closed/hoon/tests/dumb/mod/integration/asert-activation.hoon` + +- `test-asert-wrapper-matches-library`: the consensus wrapper + `+compute-target-asert` matches a direct call to + `compute-target:asert` with the same inputs at the first + post-anchor height. The test resolves `anchor-min-ts` via + `+find-anchor-min-ts`, exercising the parent-walk against the + trivial one-step case where `parent == anchor`. +- `test-asert-wrapper-past-anchor`: same, one block further past + the anchor, exercising the case where `parent-min-ts` diverges + from `anchor-min-ts`. Drives a two-step `+find-anchor-min-ts` + walk so the fallthrough branch runs end-to-end. + +### Emissions — `closed/hoon/tests/dumb/mod/unit/emissions.hoon` + +Schedule pin tests: +- `test-eon-2-truncation`: `++ schedule` at height 65,500 returns + the eon-2 reward (16,384 NOCK in atoms); at 65,501 returns the + eon-3 reward (2,048 NOCK in atoms). +- `test-decay-table`: pin every era boundary — heights 170,501 + (1,536 NOCK), 380,501 (1,024), 590,501 (768), 800,501 (512), + 1,010,501 (384), 1,220,501 (256), 1,430,501 (192), 1,640,501 + (128), 1,850,501 (96), 2,060,501 (64). +- `test-tail-floor`: a sample of tail-phase heights all return 64 + NOCK (e.g. 2,060,501, 5,000,000, 10,000,000, 16,144,876). +- `test-tail-final-block`: schedule(16,144,876) returns 64 NOCK + (last non-zero block); schedule(16,144,877) returns 0. +- `test-post-cap`: schedule(16,144,877) returns 0; schedule(20,000,000) + returns 0. +- `test-supply-totals-to-cap`: cumulative `++ schedule` over + `1..=16,144,876` equals exactly `(bex 32) * atoms-per-nock` = + `2^32 NOCK` worth of atoms. This is the load-bearing cap-accounting + invariant. + +### Coinbase split — builder unit tests + +`closed/hoon/tests/dumb/mod/unit/coinbase-split.hoon` covers +`++ new-with-fund-share` directly: + +- `test-split-pre-activation`: at `height < asert-phase`, miner + passes `shares = {(miner-hash, 1)}`; the resulting v1 coinbase- + split has one entry, full emission to the miner. +- `test-split-post-activation`: zero-fee post-activation builder + produces fund = `(div emission 5)` and miner-coins the residual. +- `test-split-post-activation-rounding`: pins per-block atom- + remainder behaviour at emission = 134,217,728 atoms. +- `test-split-post-activation-with-fees`: pins fee handling — fund + stays at `(div emission 5)` regardless of fees; the miner pool + is `(emission - fund) + fees`. **Regression test for the bug + where the builder used `(emission + fees) / 5` for the fund slot.** +- `test-split-post-activation-two-miners` / + `…-two-miners-with-fees`: pin partner-mode multi-miner support + (two miner PKHs + fund), at zero fees and with fees. +- `test-split-rejects-fund-in-shares`: builder crashes if `shares` + aliases the fund-address, preventing the honest miner from + paying the fund slot twice by mistake. +- `test-split-fund-address-is-zero-hash`: pins the placeholder so + shipping with the zero hash on mainnet is obvious. + +### Coinbase split — consensus rejection tests + +`closed/hoon/tests/dumb/mod/integration/fund-split.hoon` drives +`validate-page-with-txs` directly with hand-crafted post-activation +coinbases at `bc-fund-split` (v1-phase = asert-phase = 5): + +- `test-fund-split-accepts-honest-block`: building 5 blocks reaches + the first post-activation block; the candidate produced by + `++ new-candidate` validates. +- `test-fund-split-accepts-two-miners-plus-fund`: 3-entry coinbase + with 2 miners + fund-address at the canonical share is accepted + (pins multi-miner support). +- `test-fund-split-rejects-no-fund-slot`: 2-entry coinbase summing + to emission with no fund-address slot — `%improper-fund-split`. +- `test-fund-split-rejects-three-entries-no-fund-slot`: 3-entry + coinbase summing to emission with no fund-address slot — + `%improper-fund-split`. (The legacy `=(2 ~(wyt z-by +.cb))` pin + has been relaxed to enable multi-miner partner mode; the rule + now binds on the fund slot, not the entry count.) +- `test-fund-split-rejects-wrong-fund-share`: 2-entry coinbase with + the fund-address slot present but with a non-canonical amount + (`(div emission 5) + 1`). +- `test-fund-split-rejects-wrong-fund-address`: 2-entry coinbase + totalling emission with the canonical fund amount paid to a + non-fund-address hash. + +### Kernel-state upgrade + +No new kernel-state upgrade arm is required for the emissions +changes. Aletheia's pre-existing `state-6-to-7` upgrade +(commit `3aecbbf1d`) already covers the only schema extension to +`blockchain-constants:v1` shipping in this version, namely the six +ASERT fields. The emissions logic is layered on top of the existing +constants without changing the noun layout, so a saved v6 state on a +node loading 0.1.14 software auto-upgrades to v7 the same way it +would have under 0.1.13. + +### Anchor-pinning sanity + +Once mainnet block 65,500 is produced, run a fixture chain from +genesis through activation and verify: +- the derived `anchor-min-ts` when computing the target for block + 65,500 equals the observed mainnet median-of-11 at height **65,499** + (the anchor block), not 65,500 — `+find-anchor-min-ts` walks from + the parent (block 65,499) and stops immediately because + `parent-height == asert-anchor-height`, +- first ASERT target at height 65,500 matches an offline + reference-implementation computation against real anchor inputs, +- no off-by-one in anchor-height vs anchor-min-ts vs parent-min-ts + at heights 65,499, 65,500 and 65,501. + +## Phase 2 — post-anchor cutover + +Phase 2 was activated once block 65,500 was mined on the canonical +chain (2026-05-07 15:11:01 EDT). The runtime parent-walk that +derived the anchor's median-of-11 timestamp has been replaced with +a hardcoded `asert-anchor-min-timestamp` field on +`blockchain-constants`, and both the anchor (height 65,499) and the +first ASERT block (height 65,500) are pinned in +`checkpointed-digests`. `+find-anchor-min-ts` is deleted, removing +the per-block ancestry walk from the hot path; any attempt to mine +a competing block at either height is now rejected network-wide via +the consensus checkpoint check in `+validate-page-without-txs`. + +### Pinned values + +The three canonical mainnet values, observed at phase-2 cutover and +baked into the source: + +| Field | Value | +|-------------------------------|-----------------------------------------------------------------------| +| `asert-anchor-digest` | `vYekzUpi6o95oA6qHfvcq9kVRzFMZLuUw33YxXQRqNCvBHwU7wys73` (height 65,499) | +| `asert-anchor-min-timestamp` | `9.223.372.093.639.027.842` (urbit-seconds = 2026-05-07 14:50:42 EDT) | +| `asert-activation-digest` | `4dr8f3hWcQfgSMUrKRcNb1Z4nwzECbbUuqDYUp8G4WF6G5ocFXzPp2` (height 65,500) | + +`asert-anchor-min-timestamp` equals exactly what Phase 1's +`+find-anchor-min-ts` returned for the canonical anchor, preserving +bit-for-bit continuity across the cutover. The activation digest +is checkpoint-only and is not used in target computation. + +A reproducible fetcher for the three values from a public gRPC +endpoint ships at `scripts/aletheia_phase2_params.rs`; its +base58-encoding path was cross-checked against the existing +height-16,128 checkpoint. + +### Code changes + +- `open/hoon/common/tx-engine-1.hoon`: appended + `asert-anchor-min-timestamp=@` to `blockchain-constants:v1` and + pinned the canonical mainnet value as the realnet bunt. +- `open/hoon/apps/dumbnet/lib/types.hoon`: froze the previous + blockchain-constants shape as + `+$ blockchain-constants-v1-phase-1` and introduced + `+$ kernel-state-8` carrying the full post-phase-2 shape; + `+$ kernel-state-7` now references the frozen phase-1 snapshot + so old saved v7 states still decode. +- `open/hoon/apps/dumbnet/inner.hoon`: added `++ state-7-to-8`, + renamed the upgrade loop to `state-n-to-8`. The 7-to-8 arm + discards the old constants noun and lets `+update-constants` + reseed from `*blockchain-constants:t` on mainnet. +- `open/hoon/apps/dumbnet/lib/consensus.hoon`: + `+compute-target-asert` now reads + `asert-anchor-min-timestamp.blockchain-constants` directly; + `+find-anchor-min-ts` is deleted; the realnet + `checkpointed-digests` map gains `[65.499 asert-anchor-digest]` + and `[65.500 asert-activation-digest]`. +- `open/hoon/apps/dumbnet/lib/miner.hoon`: candidate-target path + reads the new constant directly; the `dcon` import is dropped + along with the deleted helper. +- `open/hoon/apps/wallet/lib/types.hoon`, + `open/hoon/apps/wallet/wallet.hoon`: parallel `state-8` bump and + `++ state-7-8` upgrade arm so old persisted wallet states still + decode against the extended `blockchain-constants:v1`. + +### Tests + +- `test-asert-wrapper-matches-library` and + `test-asert-wrapper-past-anchor` switched from + "`(find-anchor-min-ts …)`" to + "`asert-anchor-min-timestamp.bc`" — same value, different + source. +- `test-asert-anchor-min-ts-matches-observed` (new) verifies the + bc constant equals `min-timestamps[anchor-digest]` after the + test chain runs (the load-bearing continuity property). +- `test-asert-find-anchor-walk-fork-safety` deleted: the walk it + exercised is gone, so the fork-safety property no longer exists + in any form (the checkpoint at the anchor makes anchor-time + fork-divergence impossible network-wide). +- `test-phase-2-checkpoint-65499-pinned`, + `test-phase-2-checkpoint-65500-pinned`, + `test-phase-2-checkpoint-anchor-and-activation-differ`, + `test-phase-2-asert-anchor-min-timestamp-pinned` (new, in + `closed/hoon/tests/dumb/mod/unit/review-fixes.hoon`) pin the + three baked values directly so reverting any of them fails CI. +- `test-phase-2-kernel-state-7-constants-shape-phase-1` (new) + pins the noun-shape divergence between `kernel-state-7` and + `kernel-state-8`, mirroring the existing `test-fix3a` pin for + `kernel-state-6` ↔ `kernel-state-7`. +- `test-phase-2-state-7-bc-shape-differs-from-state-8`, + `test-phase-2-state-8-bc-matches-transact-default`, + `test-phase-2-upgrade-7-to-8-produces-state-8` (new, in + `closed/hoon/tests/wallet/mod/state-upgrade.hoon`) cover the + parallel wallet bump. +- The test bc-* helpers + (`bc-asert`, `bc-fund-split`, `bc-pre-activation-v1`, `bc-fix`) + now set + `asert-anchor-min-timestamp (add (time-in-secs:page:txe *@da) 1.200)` + — the median-of-11 those test chains actually produce at + anchor-height=4 with 600s/block spacing from genesis `*@da`. + +## Reference Implementation + +- ASERT branch: `la/asert` +- Emissions branch: `la/emissions` (built atop `la/asert`) +- Key commits (ASERT): + - `fe017f962` — ASERT initial implementation + - `8026c2f4b` — aserti3-2d anchor and timing params + - `e4cd48cb0` — capture anchor min-timestamp into consensus state + - `3aecbbf1d` — kernel-state v7 bump for ASERT constants schema + +Primary files (ASERT): +- `open/hoon/apps/dumbnet/lib/asert.hoon` (new) +- `open/hoon/apps/dumbnet/lib/consensus.hoon` (modified: wrapper + + `+find-anchor-min-ts` helper + activation gating) +- `open/hoon/apps/dumbnet/lib/miner.hoon` (modified: derive anchor + via `+find-anchor-min-ts`) +- `open/hoon/common/tx-engine-1.hoon` (modified: blockchain-constants + fields) +- `closed/hoon/tests/dumb/mod/unit/asert.hoon` (new) +- `closed/hoon/tests/dumb/mod/integration/asert-activation.hoon` (new) + +Primary files (emissions): +- `open/hoon/common/schedule.hoon` (rewritten: 14-row unified table, + exact-cap accounting via 47-block tail extension) +- `open/hoon/common/tx-engine-1.hoon` (modified: `++ fund-address` + arm) +- `open/hoon/apps/dumbnet/lib/miner.hoon` (modified: 80/20 share + construction post-activation) +- `open/hoon/apps/dumbnet/lib/consensus.hoon` (modified: + `+check-fund-split`) +- `closed/hoon/tests/dumb/mod/unit/emissions.hoon` (new) +- `closed/hoon/tests/dumb/mod/unit/coinbase-split.hoon` (new) diff --git a/crates/approver/Cargo.toml b/crates/approver/Cargo.toml new file mode 100644 index 000000000..479a05d2f --- /dev/null +++ b/crates/approver/Cargo.toml @@ -0,0 +1,16 @@ +[package] +name = "approver" +version.workspace = true +edition.workspace = true + +[dependencies] +kernels = { workspace = true, features = ["approver"] } +nockapp = { workspace = true } +nockvm = { workspace = true } +clap = { workspace = true, features = ["derive", "cargo", "color", "env"] } +tokio = { workspace = true, features = ["full"] } +zkvm-jetpack = { workspace = true } +nockapp-grpc = { workspace = true } +tracing = { workspace = true } +tracing-subscriber = { workspace = true } +bytes = { workspace = true } diff --git a/crates/approver/src/main.rs b/crates/approver/src/main.rs new file mode 100644 index 000000000..9710c4e6e --- /dev/null +++ b/crates/approver/src/main.rs @@ -0,0 +1,82 @@ +use std::error::Error; +use std::fs; +use std::path::PathBuf; + +use bytes::Bytes; +use clap::Parser; +use kernels::approver::KERNEL; +use nockapp::kernel::boot; +use nockapp::noun::slab::NounSlab; +use nockapp::{exit_driver, file_driver, one_punch_driver, NockApp}; +use nockapp::driver::Operation; +use zkvm_jetpack::hot::produce_prover_hot_state; + +use nockvm::noun::T; +use nockapp::utils::make_tas; + +#[derive(Parser)] +struct Cli { + #[command(flatten)] + boot: boot::Cli, + + /// File to read + #[arg(long, required = true)] + file: String, + + /// Extended key + #[arg(long, required = true)] + extended_key: String, +} + +#[tokio::main] +async fn main() -> Result<(), Box> { + nockvm::check_endian(); + + // Minimal/default boot CLI + let cli = Cli::parse(); + boot::init_default_tracing(&cli.boot); + + let tx_content = fs::read(&cli.file)?; + let tx_bytes = Bytes::from(tx_content); + + let prover_hot_state = produce_prover_hot_state(); + + // Create the app with default data dir and no extra drivers; approver kernel handles behavior + let mut app: NockApp = boot::setup( + KERNEL, + cli.boot, + prover_hot_state.as_slice(), + "approver", + None, + ) + .await?; + + let mut slab: NounSlab = NounSlab::new(); + let sign = make_tas(&mut slab, "sign").as_noun(); + let raw_tx = slab.cue_into(tx_bytes)?; + let extended_key = make_tas(&mut slab, &cli.extended_key).as_noun(); + let signed_path = PathBuf::from(&cli.file).with_extension("signed"); + let save_file = make_tas(&mut slab, &signed_path.to_string_lossy()).as_noun(); + let cause = T(&mut slab, &[sign, raw_tx, extended_key, save_file]); + + slab.set_root(cause); + + app.add_io_driver(one_punch_driver(slab.clone(), Operation::Poke)).await; + + // for writing the tx + app.add_io_driver(file_driver()).await; + + app.add_io_driver(exit_driver()).await; + + match app.run().await { + Ok(_) => { + tracing::info!("Command executed successfully"); + Ok(()) + } + Err(e) => { + tracing::error!("Command failed: {}", e); + Err(e.into()) + } + + } +} diff --git a/crates/bridge/.gitignore b/crates/bridge/.gitignore index c43c50bd2..972ab5702 100644 --- a/crates/bridge/.gitignore +++ b/crates/bridge/.gitignore @@ -36,6 +36,8 @@ contracts/txs/ # Deployment outputs (contain addresses, regenerated on deploy) contracts/deployments/ contracts/out +contracts/cache +contracts/broadcast/ # Contract dependencies (must be built locally) contracts/lib diff --git a/crates/bridge/Cargo.toml b/crates/bridge/Cargo.toml index 21d5955de..e64a04e59 100644 --- a/crates/bridge/Cargo.toml +++ b/crates/bridge/Cargo.toml @@ -36,7 +36,7 @@ ibig.workspace = true kernels-open-bridge = { workspace = true } libsqlite3-sys = { workspace = true } nockapp.workspace = true -nockapp-grpc = { workspace = true, default-features = false, features = [ +nockapp-grpc = { path = "../nockapp-grpc", default-features = false, features = [ "client", ] } nockchain-libp2p-io.workspace = true diff --git a/crates/bridge/README.md b/crates/bridge/README.md new file mode 100644 index 000000000..da820062b --- /dev/null +++ b/crates/bridge/README.md @@ -0,0 +1,76 @@ +# bridge runtime quickstart + +This README is for bridge operators only. If you are not sure whether you are +a bridge operator, you are probably not. + +## prerequisites +- Rust toolchain via `rustup` (uses the workspace `rust-toolchain.toml`). +- Foundry via official foundryup: + ```bash + curl -L https://foundry.paradigm.xyz | bash + ~/.foundry/bin/foundryup + ``` +- Config: copy `bridge-conf.example.toml` to + `~/.nockapp/bridge/bridge-conf.toml` (or pass `--config-path`). + +## quick run setup (from repo root) +From the repository root, install/update contract dependencies and build the +bridge runtime binary: + +```bash +make assets/bridge.jam +make -C crates/bridge/contracts install # installs or updates foundry and downloads contract dependencies +cargo build --profile release --bin bridge +``` + +## cross-compile for x86_64 linux + +From the `open/` repository root: + +```bash +make install-cargo-zigbuild +make zig-build-bridge +``` + +Default target is `x86_64-unknown-linux-gnu.2.39`. +Set `ZIGBUILD_TARGET` to a value compatible with your deployment host's glibc +version, for example: + +```bash +make zig-build-bridge ZIGBUILD_TARGET=x86_64-unknown-linux-gnu.2.39 +``` + +Output binary: + +```bash +target/x86_64-unknown-linux-gnu/release/bridge +``` + +## nockchain bridge tui (operators only) +The TUI client can be launched from your local machine (for example, your +laptop or operator workstation); it does not need to run on the bridge node. +Your local machine must be connected to Tailscale so it can reach the bridge +node ingress gRPC endpoint. + +From the `open/` repository root, build the TUI: + +```bash +make build-nockchain-bridge-tui +``` + +Run the TUI: + +```bash +# Uses default server 127.0.0.1:8001 +./target/release/nockchain-bridge-tui + +# Connect to a bridge node over Tailscale +./target/release/nockchain-bridge-tui --server :8001 +``` + +## running the node +```bash +./bridge -c bridge_config_path +``` +- `--start` will unfreeze stopped nodes +- `--new` wipes the on-disk kernel state; omit it for restarts. diff --git a/crates/bridge/build.rs b/crates/bridge/build.rs index f805dc8c2..2c3aa90ab 100644 --- a/crates/bridge/build.rs +++ b/crates/bridge/build.rs @@ -23,6 +23,7 @@ fn ensure_protoc() -> Result<(), Box> { } fn main() -> Result<(), Box> { + println!("cargo:rerun-if-env-changed=BRIDGE_MANIFEST_DIR"); println!("cargo:rerun-if-changed=contracts/out/MessageInbox.sol/MessageInbox.json"); println!("cargo:rerun-if-changed=contracts/out/Nock.sol/Nock.json"); println!("cargo:rerun-if-changed=contracts/MessageInbox.sol"); @@ -34,78 +35,21 @@ fn main() -> Result<(), Box> { ensure_protoc()?; - let manifest_dir = Path::new(env!("CARGO_MANIFEST_DIR")); + let manifest_dir = bridge_manifest_dir()?; - // Helper to expand ${pwd} in paths from environment variables - let expand_path = |path_str: &str| -> PathBuf { - if path_str.contains("${pwd}") { - let pwd = env::current_dir().unwrap_or_else(|_| PathBuf::from(".")); - PathBuf::from(path_str.replace("${pwd}", &pwd.to_string_lossy())) - } else { - PathBuf::from(path_str) - } - }; - - // Try to get contract file paths from rustc_env first (they contain ${pwd} which we expand) - // If not available, search for them relative to CARGO_MANIFEST_DIR - let mut inbox_src = if let Some(path) = env::var_os("MESSAGE_INBOX_SOL") { + // Use explicit runtime paths when provided; otherwise fall back to the crate-local contracts/. + let inbox_src = if let Some(path) = env::var_os("MESSAGE_INBOX_SOL") { expand_path(&path.to_string_lossy()) } else { manifest_dir.join("contracts/MessageInbox.sol") }; - let mut nock_src = if let Some(path) = env::var_os("NOCK_SOL") { + let nock_src = if let Some(path) = env::var_os("NOCK_SOL") { expand_path(&path.to_string_lossy()) } else { manifest_dir.join("contracts/Nock.sol") }; - // If files don't exist at expected location, search more broadly - // compile_data files might be at different locations in the sandbox - if !inbox_src.exists() { - // Try searching from current directory (might be output directory) - if let Ok(current_dir) = env::current_dir() { - for ancestor in current_dir.ancestors() { - let candidate = ancestor.join("open/crates/bridge/contracts/MessageInbox.sol"); - if candidate.exists() { - inbox_src = candidate; - break; - } - } - } - // Also try from manifest_dir - if !inbox_src.exists() { - for ancestor in manifest_dir.ancestors() { - let candidate = ancestor.join("open/crates/bridge/contracts/MessageInbox.sol"); - if candidate.exists() { - inbox_src = candidate; - break; - } - } - } - } - - if !nock_src.exists() { - if let Ok(current_dir) = env::current_dir() { - for ancestor in current_dir.ancestors() { - let candidate = ancestor.join("open/crates/bridge/contracts/Nock.sol"); - if candidate.exists() { - nock_src = candidate; - break; - } - } - } - if !nock_src.exists() { - for ancestor in manifest_dir.ancestors() { - let candidate = ancestor.join("open/crates/bridge/contracts/Nock.sol"); - if candidate.exists() { - nock_src = candidate; - break; - } - } - } - } - // Determine contracts_root from where we found the source files let contracts_root = if let Some(parent) = inbox_src.parent() { parent.to_path_buf() @@ -155,7 +99,7 @@ fn main() -> Result<(), Box> { "forge binary not found at {:?} (FORGE_BIN={:?}, CARGO_MANIFEST_DIR={:?})", forge, env::var_os("FORGE_BIN"), - env!("CARGO_MANIFEST_DIR") + manifest_dir ) .into()); } @@ -277,7 +221,7 @@ fn resolve_forge_bin() -> Result> { // In Bazel sandbox, compile_data files are available relative to CARGO_MANIFEST_DIR // Try to find forge relative to manifest dir - let manifest_dir = Path::new(env!("CARGO_MANIFEST_DIR")); + let manifest_dir = bridge_manifest_dir()?; for ancestor in manifest_dir.ancestors() { let candidate = ancestor.join("tools/bin/forge_bin"); // genrule output name if candidate.is_file() { @@ -297,6 +241,38 @@ fn resolve_forge_bin() -> Result> { .into()) } +fn bridge_manifest_dir() -> Result> { + if let Some(path) = dir_from_env("BRIDGE_MANIFEST_DIR")? { + return Ok(path); + } + + if let Some(path) = dir_from_env("CARGO_MANIFEST_DIR")? { + return Ok(path); + } + + Err("BRIDGE_MANIFEST_DIR and CARGO_MANIFEST_DIR are not set".into()) +} + +fn dir_from_env(key: &str) -> Result, Box> { + let Some(raw) = env::var_os(key) else { + return Ok(None); + }; + let path = expand_path(&raw.to_string_lossy()); + if path.is_dir() { + return Ok(Some(path)); + } + Err(format!("{key} is set but not a directory: {}", path.display()).into()) +} + +fn expand_path(path_str: &str) -> PathBuf { + if path_str.contains("${pwd}") { + let pwd = env::current_dir().unwrap_or_else(|_| PathBuf::from(".")); + PathBuf::from(path_str.replace("${pwd}", &pwd.to_string_lossy())) + } else { + PathBuf::from(path_str) + } +} + fn create_unique_temp_dir(prefix: &str) -> std::io::Result { use std::io; use std::time::{SystemTime, UNIX_EPOCH}; diff --git a/crates/bridge/contracts/.env.template b/crates/bridge/contracts/.env.template index 81b6cab63..d0d9edd5f 100644 --- a/crates/bridge/contracts/.env.template +++ b/crates/bridge/contracts/.env.template @@ -6,8 +6,9 @@ # REQUIRED FOR DEPLOYMENT # ============================================================================= -# Tenderly RPC endpoint (fork or devnet) -# Get this from: tenderly devnet spawn --template +# Tenderly RPC endpoint (fork or virtual testnet) +# If using ../scripts/tenderly-vnet-deploy.sh, this is written to: +# ../scripts/environments/virtual-testnet.generated.env TENDERLY_RPC_URL= # Deployer private key (with 0x prefix) diff --git a/crates/bridge/contracts/DEPLOYMENT.md b/crates/bridge/contracts/DEPLOYMENT.md index 5de87d69f..72ad5b869 100644 --- a/crates/bridge/contracts/DEPLOYMENT.md +++ b/crates/bridge/contracts/DEPLOYMENT.md @@ -11,25 +11,18 @@ networks (devnets, simulations, or proxied mainnets). ## Quick Start -For a quick deployment to a new Tenderly devnet: +For a quick deployment to a new Tenderly Base Sepolia virtual testnet: ```bash cd crates/bridge/contracts -# 1. Install dependencies -make install - -# 2. Configure environment -cp .env.template .env -# Edit .env with your values - -# 3. Spawn Tenderly devnet -tenderly devnet spawn --network base-sepolia --project bridge-contracts - -# 4. Update .env with RPC URL from above +# Required environment variables: +# TENDERLY_ACCESS_KEY, TENDERLY_ACCOUNT_ID, TENDERLY_PROJECT_SLUG, +# TENDERLY_PRIVATE_KEY, BRIDGE_NODE_0..BRIDGE_NODE_4 +./scripts/tenderly-vnet-deploy.sh --cleanup-old --cleanup-prefix bridge-vnet --cleanup-keep 3 -# 5. Deploy -make deploy +# Load generated RPC + contract addresses for bridge runtime scripts +source scripts/environments/virtual-testnet.generated.env ``` ## Table of Contents @@ -122,21 +115,30 @@ export BRIDGE_NODE_0="0x..." ## 3. Deployment -### Spawn or Select Tenderly Network +### Provision Tenderly Network (Recommended) -For a devnet fork: +Use the bridge provisioning script from the repository root: ```bash -tenderly devnet spawn --network base-sepolia --project bridge-contracts +cd open/crates/bridge +./scripts/tenderly-vnet-deploy.sh ``` -Copy the returned `rpc_url`. For live networks proxied through Tenderly, use -the RPC URL shown in the Tenderly dashboard. +This script: + +- Creates a fresh Base Sepolia virtual testnet via Tenderly API +- Funds deployer + bridge node accounts with `tenderly_setBalance` +- Deploys `MessageInbox` and `Nock` via `contracts/scripts/deploy_tenderly.sh` +- Writes `scripts/environments/virtual-testnet.generated.env` -### Fund Test Accounts (Devnets Only) +The generated env file includes `TENDERLY_RPC_URL`, `BASE_WS_URL`, +`INBOX_CONTRACT_ADDRESS`, and `NOCK_CONTRACT_ADDRESS` for downstream bridge +scripts. -Tenderly devnets start with zero balances. Before running integration tests, -fund the bridge node and test accounts using the `tenderly_setBalance` RPC: +### Manual Network/Funding Path (Fallback) + +If you are bypassing `tenderly-vnet-deploy.sh`, you can provision and fund +manually: ```bash # Fund bridge node 0 and test account with 10 ETH each @@ -153,10 +155,8 @@ curl -X POST "$TENDERLY_RPC_URL" \ }' ``` -The hex value `0x8AC7230489E80000` equals 10 ETH (10 × 10^18 wei). You can -fund multiple addresses in a single call by adding them to the array. - -**Note:** This is only needed for devnets. Mainnet forks inherit real balances. +The hex value `0x8AC7230489E80000` equals 10 ETH (10 × 10^18 wei). Mainnet +forks inherit real balances and usually do not require this step. ### Preview Deployment (Dry Run) diff --git a/crates/bridge/contracts/scripts/deploy_tenderly.sh b/crates/bridge/contracts/scripts/deploy_tenderly.sh new file mode 100755 index 000000000..18b6760e2 --- /dev/null +++ b/crates/bridge/contracts/scripts/deploy_tenderly.sh @@ -0,0 +1,145 @@ +#!/usr/bin/env bash +set -euo pipefail + +# Deploy bridge contracts to Tenderly network +# Usage: ./scripts/deploy_tenderly.sh [--dry-run] [EXTRA_FLAGS] + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +CONTRACTS_DIR="$(cd "$SCRIPT_DIR/.." && pwd)" +cd "$CONTRACTS_DIR" + +# Parse arguments +DRY_RUN=false +EXTRA_ARGS=() +for arg in "$@"; do + if [ "$arg" = "--dry-run" ]; then + DRY_RUN=true + else + EXTRA_ARGS+=("$arg") + fi +done + +# Validate required environment variables +REQUIRED_VARS=( + "TENDERLY_RPC_URL" + "TENDERLY_PRIVATE_KEY" + "NOCK_NAME" + "NOCK_SYMBOL" + "BRIDGE_NODE_0" + "BRIDGE_NODE_1" + "BRIDGE_NODE_2" + "BRIDGE_NODE_3" + "BRIDGE_NODE_4" +) + +MISSING_VARS=() +for var in "${REQUIRED_VARS[@]}"; do + if [ -z "${!var:-}" ]; then + MISSING_VARS+=("$var") + fi +done + +if [ ${#MISSING_VARS[@]} -ne 0 ]; then + echo "Error: Missing required environment variables:" >&2 + printf " - %s\n" "${MISSING_VARS[@]}" >&2 + echo "" >&2 + echo "See .env.template for required variables." >&2 + exit 1 +fi + +# Set defaults for optional variables +export DEPLOY_TARGET_NETWORK="${DEPLOY_TARGET_NETWORK:-tenderly-devnet}" +export DEPLOYER_ADDRESS="${DEPLOYER_ADDRESS:-0x0000000000000000000000000000000000000000}" + +# Determine deployment path +if [ -z "${DEPLOYMENTS_PATH:-}" ]; then + DEPLOYMENTS_DIR="$CONTRACTS_DIR/deployments" + mkdir -p "$DEPLOYMENTS_DIR" + export DEPLOYMENTS_PATH="$DEPLOYMENTS_DIR/${DEPLOY_TARGET_NETWORK}.json" +fi + +# Dry run mode - show what would happen +if [ "$DRY_RUN" = true ]; then + echo "=== DRY RUN MODE ===" + echo "" + echo "Configuration:" + echo " Network: $DEPLOY_TARGET_NETWORK" + echo " RPC URL: ${TENDERLY_RPC_URL:0:50}..." + echo " Deployer: $DEPLOYER_ADDRESS" + echo " Deployments: $DEPLOYMENTS_PATH" + echo "" + echo "Token Configuration:" + echo " Name: $NOCK_NAME" + echo " Symbol: $NOCK_SYMBOL" + echo "" + echo "Bridge Nodes:" + echo " Node 0: $BRIDGE_NODE_0" + echo " Node 1: $BRIDGE_NODE_1" + echo " Node 2: $BRIDGE_NODE_2" + echo " Node 3: $BRIDGE_NODE_3" + echo " Node 4: $BRIDGE_NODE_4" + echo "" + if [ -f "$DEPLOYMENTS_PATH" ]; then + echo "WARNING: Existing deployment at $DEPLOYMENTS_PATH will be backed up" + fi + echo "" + echo "Actions that would be performed:" + echo " 1. Build contracts with forge" + echo " 2. Deploy Nock token" + echo " 3. Deploy MessageInbox implementation" + echo " 4. Deploy ERC1967Proxy with MessageInbox" + echo " 5. Link Nock to MessageInbox proxy" + echo " 6. Write deployment to $DEPLOYMENTS_PATH" + if [ -n "${TENDERLY_ACCESS_KEY:-}" ]; then + echo " 7. Verify contracts on Tenderly (using forge verify-contract)" + else + echo " 7. Skip verification (TENDERLY_ACCESS_KEY not set)" + fi + echo "" + echo "Run without --dry-run to execute." + exit 0 +fi + +# Backup existing deployment if it exists +if [ -f "$DEPLOYMENTS_PATH" ]; then + HISTORY_DIR="$CONTRACTS_DIR/deployments/history/${DEPLOY_TARGET_NETWORK}" + mkdir -p "$HISTORY_DIR" + TIMESTAMP=$(date +%Y%m%d-%H%M%S) + BACKUP_PATH="$HISTORY_DIR/${TIMESTAMP}.json" + cp "$DEPLOYMENTS_PATH" "$BACKUP_PATH" + echo "Backed up existing deployment to: $BACKUP_PATH" +fi + +forge build --force || { + echo "Error: Build failed" >&2 + exit 1 +} +FORGE_CMD=( + forge script forge/Deploy.s.sol:Deploy + --rpc-url "$TENDERLY_RPC_URL" + --private-key "$TENDERLY_PRIVATE_KEY" + --broadcast + --slow +) +if [ ${#EXTRA_ARGS[@]} -gt 0 ]; then + FORGE_CMD+=("${EXTRA_ARGS[@]}") +fi +"${FORGE_CMD[@]}" + +if [ ! -f "$DEPLOYMENTS_PATH" ]; then + echo "Error: Deployment file not created at $DEPLOYMENTS_PATH" >&2 + exit 1 +fi + +# Verify contracts if access key is available +if [ -n "${TENDERLY_ACCESS_KEY:-}" ]; then + echo "" + echo "Verifying contracts on Tenderly..." + "$SCRIPT_DIR/verify_contracts.sh" "$DEPLOYMENTS_PATH" || { + echo "Warning: Contract verification failed. Run 'make verify' to retry." >&2 + } +else + echo "" + echo "Skipping verification: TENDERLY_ACCESS_KEY not set" + echo "To verify later, set TENDERLY_ACCESS_KEY and run: make verify" +fi diff --git a/crates/bridge/contracts/scripts/verify_contracts.sh b/crates/bridge/contracts/scripts/verify_contracts.sh new file mode 100755 index 000000000..bf9e49374 --- /dev/null +++ b/crates/bridge/contracts/scripts/verify_contracts.sh @@ -0,0 +1,86 @@ +#!/usr/bin/env bash +set -euo pipefail + +# Verify deployed contracts on Tenderly +# Usage: ./scripts/verify_contracts.sh [DEPLOYMENTS_PATH] + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +CONTRACTS_DIR="$(cd "$SCRIPT_DIR/.." && pwd)" +cd "$CONTRACTS_DIR" + +# Determine deployment path +DEPLOYMENTS_PATH="${1:-${DEPLOYMENTS_PATH:-}}" +if [ -z "$DEPLOYMENTS_PATH" ]; then + DEPLOY_TARGET_NETWORK="${DEPLOY_TARGET_NETWORK:-tenderly-devnet}" + DEPLOYMENTS_PATH="$CONTRACTS_DIR/deployments/${DEPLOY_TARGET_NETWORK}.json" +fi + +if [ ! -f "$DEPLOYMENTS_PATH" ]; then + echo "Error: Deployment file not found: $DEPLOYMENTS_PATH" >&2 + exit 1 +fi + +# Validate required environment variables +if [ -z "${TENDERLY_RPC_URL:-}" ]; then + echo "Error: TENDERLY_RPC_URL not set" >&2 + exit 1 +fi + +if [ -z "${TENDERLY_ACCESS_KEY:-}" ]; then + echo "Error: TENDERLY_ACCESS_KEY not set" >&2 + echo "Get your access key from: https://dashboard.tenderly.co/account/authorization" >&2 + exit 1 +fi + +# Read addresses from deployment file +NOCK_ADDRESS=$(jq -r '.nock' "$DEPLOYMENTS_PATH") +INBOX_IMPL_ADDRESS=$(jq -r '.messageInboxImplementation' "$DEPLOYMENTS_PATH") +INBOX_PROXY_ADDRESS=$(jq -r '.messageInboxProxy' "$DEPLOYMENTS_PATH") + +echo "Verifying contracts from: $DEPLOYMENTS_PATH" +echo "" +echo "Addresses:" +echo " Nock: $NOCK_ADDRESS" +echo " MessageInbox (impl): $INBOX_IMPL_ADDRESS" +echo " MessageInbox (proxy): $INBOX_PROXY_ADDRESS" +echo "" + +# Build first to ensure artifacts are current +echo "Building contracts..." +forge build --force + +# Construct verifier URL from RPC URL +VERIFIER_URL="${TENDERLY_RPC_URL}/verify/etherscan" + +echo "" +echo "Using verifier URL: $VERIFIER_URL" +echo "" + +# Verify Nock token +echo "=== Verifying Nock token ===" +forge verify-contract "$NOCK_ADDRESS" Nock.sol:Nock \ + --verifier-url "$VERIFIER_URL" \ + --etherscan-api-key "$TENDERLY_ACCESS_KEY" \ + --watch || echo "Warning: Nock verification failed or already verified" + +echo "" + +# Verify MessageInbox implementation +echo "=== Verifying MessageInbox implementation ===" +forge verify-contract "$INBOX_IMPL_ADDRESS" MessageInbox.sol:MessageInbox \ + --verifier-url "$VERIFIER_URL" \ + --etherscan-api-key "$TENDERLY_ACCESS_KEY" \ + --watch || echo "Warning: MessageInbox implementation verification failed or already verified" + +echo "" + +# Note: ERC1967Proxy is from OpenZeppelin and may need special handling +echo "=== Verifying ERC1967Proxy ===" +forge verify-contract "$INBOX_PROXY_ADDRESS" \ + lib/openzeppelin-contracts/contracts/proxy/ERC1967/ERC1967Proxy.sol:ERC1967Proxy \ + --verifier-url "$VERIFIER_URL" \ + --etherscan-api-key "$TENDERLY_ACCESS_KEY" \ + --watch || echo "Warning: Proxy verification failed or already verified" + +echo "" +echo "Verification complete. Check Tenderly dashboard for results." diff --git a/crates/bridge/docs/README.md b/crates/bridge/docs/README.md new file mode 100644 index 000000000..cc80d3858 --- /dev/null +++ b/crates/bridge/docs/README.md @@ -0,0 +1,37 @@ +# Bridge Documentation Index + +Status: Active +Owner: Nockchain Maintainers +Last Reviewed: 2026-03-12 +Canonical/Legacy: Legacy (bridge subsystem index; canonical docs spine starts at [`START_HERE.md`](../../../START_HERE.md)) + +Use this as the bridge docs landing page. Start with architecture, then drill +into setup, operations, and governance guides. + +1. [`architecture.md`](./architecture.md) – Component map, event/effect flow, + and critical pathways between Nockchain, Base, and the Hoon kernel. +2. [`sans-io-architecture.md`](./sans-io-architecture.md) – Canonical sans-IO + execution architecture (planners, ports, loop shells, deterministic + testability). +3. [`signatures.md`](./signatures.md) – Canonical signature formats, hashing, + verification rules, and recommended validation steps. +4. [`node-runbook.md`](./node-runbook.md) – Operational playbook for bridge node + provisioning, monitoring, logging, and incident response. +5. [`governance.md`](./governance.md) – Upgrade procedures for contracts, the + kernel jam, and the Rust runtime, plus multisig responsibilities. +6. [`../OPERATOR-SETUP.md`](../OPERATOR-SETUP.md) – Provisioning and initial + host/bootstrap setup for new bridge operators. +7. [`../QUICKSTART.md`](../QUICKSTART.md) – Build/test/run quickstart for local + bridge development and first boot. +8. [`bridge-withdrawals.md`](./bridge-withdrawals.md) – Canonical bridge + withdrawal protocol and implementation spec. +9. [`withdrawal-dapp.md`](./withdrawal-dapp.md) – DApp/frontend-oriented + withdrawal product and implementation guide derived from the canonical + withdrawal spec. +10. [`../draft/README.md`](../draft/README.md) – Withdrawal milestone plans + and dated progress log. +11. [`../draft/specs/test-harness.md`](../draft/specs/test-harness.md) – + Deterministic test harness design for bridge runtime/kernel tests and the + kernel-to-signing seam. + +When adding new docs, update this index so we have a single place to point future readers. diff --git a/crates/bridge/docs/architecture.md b/crates/bridge/docs/architecture.md new file mode 100644 index 000000000..1f1a27e3c --- /dev/null +++ b/crates/bridge/docs/architecture.md @@ -0,0 +1,140 @@ +# Bridge Architecture Overview + +## Scope + +This document describes how the `crates/bridge` runtime composes: + +- Base contracts (`MessageInbox.sol`, `Nock.sol`) +- The Hoon bridge kernel (`assets/bridge.jam`) +- Rust observers, gRPC services, and posting/signature loops + +It is an implementation map for this crate, not chain-level protocol authority. + +## Authority Boundaries + +- Protocol activation and chain-level authority are external: + [`PROTOCOL.md`](../../../PROTOCOL.md) and + [`changelog/protocol/`](../../../changelog/protocol/). +- This document only describes bridge runtime behavior in this repository. + +## Component Map + +| Layer | Component | Responsibility | Source | +| ---------- | -------------------------------------------------------------- | --------------------------------------------------------------------------------------------------------------------------------------------- | -------- | +| On-chain | `contracts/MessageInbox.sol` | UUPS-upgradeable inbox that mints wrapped nock after seeing a 3-of-5 bridge signature set. Tracks node roster, burns, and prevents replay. | Solidity | +| On-chain | `contracts/Nock.sol` | Non-upgradeable ERC-20 that lets users burn to exit back to Nockchain and forwards those burns to the inbox. | Solidity | +| Kernel | `open/hoon/apps/bridge/*.hoon` -> `open/assets/bridge.jam` | Deterministic state machine for cross-chain hashchain state; emits `%create-withdrawal-txs`, `%commit-nock-deposits`, `%grpc`, `%stop`. | Hoon | +| Runtime | `src/runtime.rs` | Asynchronous event router and kernel poke/peek wrapper that feeds causes into the kernel. | Rust | +| Observers | `src/ethereum.rs`, `src/nockchain.rs` | Pull data from Base (via `alloy` WS) and from the private nockchain gRPC API, turn them into `BridgeEvent`s, and hand them to the runtime. | Rust | +| Interfaces | `src/ingress.rs`, `nockapp_grpc::driver::grpc_listener_driver` | gRPC entry points for peer coordination plus a listener driver for kernel `%grpc` effects. | Rust | +| Signing | `src/signing.rs`, `main.rs::run_signing_cursor_loop` | Computes proposal hashes from the deposit log, signs them locally, and gossips signatures to peers. | Rust | + +## Runtime Inbound Pipeline + +`BridgeRuntime` currently accepts only chain events (`BridgeEvent::Chain`): + +1. `BaseBridge::stream_base_events()` emits `ChainEvent::Base` batches. +2. `NockchainWatcher::run()` emits `ChainEvent::Nock` blocks. +3. `KernelCauseBuilder` converts those events into kernel pokes: + - `base-blocks` + - `nockchain-block` +4. `BridgeRuntime` sends those pokes to the installed nockapp driver. + +Other kernel pokes are injected directly (not via `BridgeRuntime` events): + +- `cfg-load` and `set-constants` at boot (`main.rs`) +- `%start` when `--start` is passed +- `%stop` from stop handling logic + +## Kernel Effect Sinks + +Kernel effects are decoded by IO drivers in `main.rs`: +| Cause tag | Trigger | Payload | Status | +| ------------------- | ------------------------------------------------ | ------------------------------------------- | ----------- | +| `base-blocks` | Batch of Base deposits/withdrawals/settlements | `Vec` | Implemented | +| `nockchain-block` | New nockchain page | `nockchain_types::tx_engine::common::Page` | Implemented | +| `proposed-base-call`| Peer-delivered deposit proposal payload | `ProposedBaseCallData` | Implemented | +| `base-call-sig` | Peer signature routed through ingress | `EthSignatureParts` + calldata | Implemented | +| `cfg-load` | Startup configuration | `NodeConfig` parsed from `bridge-conf.toml` | Implemented | +| `set-constants` | Runtime/operator constants update | `BridgeConstants` | Implemented | +| `stop` / `start` | Operator or fault-triggered state transition | `StopLastBlocks` / null tag | Implemented | + +| Effect tag | Runtime handling | +| ------------------------------------------------------------------------- | --------------------------------------------------------------------------------------------------------------------- | +| `commit-nock-deposits` | Implemented by `create_commit_nock_deposits_driver`: persists requests to `deposit-queue.sqlite`. | +| `stop` | Implemented by `create_stop_driver`: transitions local stop state and broadcasts stop to peers. | +| `grpc` | Implemented by `grpc_listener_driver`: executes `%grpc` effect calls. | +| `base-call`, `assemble-base-call`, `nockchain-tx`, `propose-nockchain-tx` | Type-level support exists (`types.rs`), but no dedicated bridge IO driver is wired for these tags in `main.rs` today. | + +## Signature and Posting Pipeline + +Deposit submission is currently driven by deposit-log + cache loops: + +1. Kernel emits `commit-nock-deposits`. +2. Commit driver persists requests to `deposit-queue.sqlite`. +3. `run_signing_cursor_loop` polls: + - on-chain `lastDepositNonce` + - local deposit queue +4. It signs candidates, inserts signatures into `ProposalCache`, and gossips via `BridgeIngress/BroadcastSignature`. +5. Ingress validates and stores peer signatures in the same cache. +6. `run_posting_loop` selects ready proposals, enforces proposer/failover logic, calls `BaseBridge::submit_deposit`, and then broadcasts `BroadcastConfirmation`. + +This means signature collection and Base posting are coordinated by Rust loops around `ProposalCache`, not by a direct ingress-to-runtime event path. +| Effect tag | Purpose | Consumer | Status | +| ------------------------ | ---------------------------------------------------------------------------------- | ----------------------------------------------- | ----------- | +| `create-withdrawal-txs` | Emit withdrawal requests derived from Base burn events (`nock-withdrawal-request`). | **TODO** withdrawal tx proposal/submission path | **TODO** | +| `commit-nock-deposits` | Emit structured deposit requests for runtime persistence and signing. | `create_commit_nock_deposits_driver` in `main.rs` | Implemented | +| `grpc` | Make gRPC calls: `peek` for queries, `call` for RPC invocations. | gRPC listener driver | Implemented | +| `stop` | Freeze processing in kernel and propagate stop to peers. | `create_stop_driver` in `main.rs` | Implemented | + +Note: when nodes encounter STOP, the kernel transitions to a STOPPED state and no longer processes +new pokes. Drivers in the Rust runtime will spin. If the process is restarted, the +kernel will still be STOPPED, however, the drivers will be restarted. + +Effect handling is implemented directly by IO drivers registered in `main.rs`. +Right now we ship drivers for `%grpc`, `%commit-nock-deposits`, `%stop`, markdown +(debug UI), and exit handling. Deposit submission to `MessageInbox` is driven by +the runtime signing/posting loops after `%commit-nock-deposits` has been +persisted to the local deposit log. + +## Critical Flows + +### Deposit (Nockchain -> Base) + +1. `NockchainWatcher` notices a deposit on the heavy chain and emits a + `nockchain-block` cause. +2. The kernel updates its dual-hashchain state and emits a `commit-nock-deposits` + effect containing a list of `nock-deposit-request` structures with fields: + tx-id, name, recipient, amount, block-height, as-of. +3. The `create_commit_nock_deposits_driver` appends the requests to the local + deposit log so nonce assignment is deterministic across restarts. +4. The signing cursor loop reads from the deposit log, assigns nonces, signs + proposals, and gossips signatures to peers. +5. The proposal cache aggregates signatures per `DepositId`; once threshold is + reached, the posting loop constructs `DepositSubmission` and calls + `BaseBridge::submit_deposit`. +6. MessageInbox validates signatures and nonce ordering, then emits + `DepositProcessed`, which the Base observer feeds back into kernel state. + +### Withdrawal (Base -> Nockchain) + +1. Users burn wrapped tokens through `Nock.sol::burn`, which emits + `BurnForWithdrawal` and calls `MessageInbox.notifyBurn`. +2. `BaseBridge::stream_base_events` captures the burn event, packages it as + `BaseWithdrawalEvent`, and emits a `base-blocks` cause so the kernel can + queue settlement work. +3. The kernel emits `create-withdrawal-txs` with `nock-withdrawal-request` + entries (`base_event_id`, lock-root recipient, amount, batch-end, as-of). +4. Runtime-side conversion of these requests into finalized nockchain + settlement transactions is still in progress. + +## Reference Files + +- `crates/bridge/src/main.rs` +- `crates/bridge/src/runtime.rs` +- `crates/bridge/src/ethereum.rs` +- `crates/bridge/src/nockchain.rs` +- `crates/bridge/src/ingress.rs` +- `crates/bridge/src/deposit_log.rs` +- `crates/bridge/src/stop.rs` +- `crates/bridge/src/types.rs` diff --git a/crates/bridge/docs/bridge-withdrawals.md b/crates/bridge/docs/bridge-withdrawals.md new file mode 100644 index 000000000..a44680469 --- /dev/null +++ b/crates/bridge/docs/bridge-withdrawals.md @@ -0,0 +1,708 @@ +# Bridge Withdrawals + +Status: Active +Owner: Nockchain Maintainers +Last Reviewed: 2026-03-13 +Canonical/Legacy: Canonical bridge withdrawal protocol and implementation spec + +## Scope + +This document specifies the implementation work required to enable bridge withdrawals (Base -> Nockchain) across: + +1. Rust bridge crate (`open/crates/bridge`) +2. Hoon bridge kernel (`open/hoon/apps/bridge/bridge.hoon`, plus `base.hoon`, `nock.hoon`, `types.hoon`) + +This spec focuses on: + +1. What is currently unimplemented +2. What must be changed +3. What the withdrawal coordination model must be +4. How to validate correctness + +## Current Implementation Status + +### Kernel status (Hoon) + +1. Nock-side withdrawal tx detection is now wired (no intentional hard-stop): + - `open/hoon/apps/bridge/nock.hoon` + - arm: `++ process-nock-txs` + - branch: `is-bridge-withdrawal-tx` + - current behavior: detects packed `%bridge-w` note-data, parses withdrawal metadata, and builds `withdrawal-settlement` entries (non-matching/malformed outputs are skipped, not fatal) + +2. Nock-side settlement processing no longer intentionally rejects withdrawals: + - `open/hoon/apps/bridge/nock.hoon` + - arm: `++ nockchain-process-withdrawal-settlements` + - current behavior: reconciles settlements against tracked withdrawals, emits hold when referenced `as_of` base hash is unknown, stops on irreconcilable counterpart issues, and clears matched unsettled entries + +3. Base-side withdrawal proposal arm is now implemented: + - `open/hoon/apps/bridge/base.hoon` + - arm: `++ base-propose-withdrawals` + - current behavior: returns `(list nock-withdrawal-request)` and is emitted as `%create-withdrawal-txs` + +4. Withdrawal data/effect surface was updated: + - `open/hoon/apps/bridge/types.hoon` + - `withdrawal` now uses `dest=nock-lock-root` and no fee field + - `withdrawal-settlement` now includes `base-batch-end` (including hashable encoding) and no `nock-tx-fee` + - effects now include `%create-withdrawal-txs` and the confirmed-only + `%withdrawal-terminal` alongside the existing deposit / stop surface + - `%create-withdrawal-tx` exists as the dedicated poke/cause for asking the + bridge app tx-builder to construct a withdrawal proposal from explicit + inputs / withdrawal data + - `%sign-tx` exists as the dedicated poke/cause for asking Rust-side + withdrawal coordination to sign the transaction inside a + `withdrawal-proposal` + - current behavior: still stubbed as a guarded no-op; the intended + implementation remains milestone 2, but specifically as a split builder + seam before live submission is enabled + +### Runtime status (Rust) + +1. Core runtime and loops are deposit-centric: + - `open/crates/bridge/src/main.rs` + - `open/crates/bridge/src/runtime.rs` + - active loops: signing cursor + posting loop for Base deposit submissions + +2. Rust effect surface now decodes the milestone 1 withdrawal kernels effects: + - `open/crates/bridge/src/types.rs` + - withdrawal-related variant: `BridgeEffectVariant::CreateWithdrawalTxs(Vec)` + - terminal variant: `BridgeEffectVariant::WithdrawalTerminal(WithdrawalTerminalEffect)` + +3. Legacy runtime local effect queue was removed: + - `open/crates/bridge/src/runtime.rs` + - `BridgeRuntime::process_effect` and `send_effect` no longer exist + - effect consumption is now driver-specific (e.g., stop/deposit drivers) + +4. Live effect handling remains deposit-only, with an explicit withdrawal + guard: + - `open/crates/bridge/src/deposit_log.rs` + - `create_commit_nock_deposits_driver` consumes only `CommitNockDeposits` + - `open/crates/bridge/src/withdrawal_guard.rs` + - `open/crates/bridge/src/main.rs` registers a milestone-1 guard driver that + logs and ignores `%create-withdrawal-txs` and `%withdrawal-terminal` + +5. Ingress and proposal cache are deposit-specific: + - `open/crates/bridge/proto/bridge_ingress.proto` + - `open/crates/bridge/src/ingress.rs` + - `open/crates/bridge/src/proposal_cache.rs` + - all keyed by `DepositId` and Ethereum signature flow + +5. Withdrawal proposal validation and tracking belong to Rust, but are not + fully implemented yet: + - there is no kernel proposal-validation cause in the intended design + - Rust must own proposal-envelope validation against withdrawal identity, + snapshot/selected-note inputs, epoch legality, replay/equivocation rules, + and durable per-withdrawal proposal tracking +6. Base burn events are observed, but settlement execution path is missing: + - `open/crates/bridge/src/ethereum.rs` + - `process_nock_log` decodes `BurnForWithdrawal` + - generated local `Withdrawal` currently has `dest: None` + - TODO: base-side kernel processing must enforce the withdrawal minimum so burns at or below the configured minimum do not materialize into withdrawal state + +7. Withdrawal-specific coordination state is partially implemented: + - a durable sequencer-owned record of prepared / peer-canonical / + authorized / submitted / confirmed withdrawals exists in Rust + - append-only withdrawal lifecycle storage and local input-note reservation + projections now exist + - the ingress-side withdrawal proposal transport remains in the main bridge + process, while the separate sequencer gRPC surface is now hosted by the + dedicated `bridge-sequencer` binary + - the Rust submission/release driver layer exists; sequencer-side + submission and confirmed-settlement recording now live in the separate + sequencer binary, while terminal release stays in the main bridge process + - confirmed Nock blocks now drive bridge-owned note snapshot refresh in the + main bridge process plus sequencer-side `tx_confirmed` recording in the + separate sequencer binary + - `%create-withdrawal-tx` and `%sign-tx` are now real kernel execution + seams rather than stubs + +### Contracts and dependencies + +1. Contracts support burn-side initiation: + - `open/crates/bridge/contracts/Nock.sol`: `burn(amount, lockRoot)` emits `BurnForWithdrawal` + - `open/crates/bridge/contracts/MessageInbox.sol`: `notifyBurn` + `withdrawalsEnabled` gate + +2. No immediate `Cargo.toml` dependency gap identified for baseline implementation: + - `open/crates/bridge/Cargo.toml` + - nockapp gRPC client dependencies already present + +## Target Withdrawal Flow + +1. User burns wrapped NOCK on Base (`Nock.sol::burn`), event emitted with `lockRoot`. +2. Rust Base observer ingests burn, emits `%base-blocks` cause to kernel. +3. Kernel stores unsettled withdrawals keyed by `(as_of base hash, base_event_id)`. + Base-side kernel processing must enforce the withdrawal minimum first; only burns strictly above the configured minimum become withdrawals. We are targetting a minimum of 10,000 NOCKS. +4. Kernel emits `%create-withdrawal-txs` with `nock-withdrawal-request` payloads describing withdrawal intent. +5. Runtime treats each `nock-withdrawal-request` as a single-withdrawal coordination unit. `base-batch-end` remains part of withdrawal metadata, but settlement coordination is per-withdrawal rather than multi-withdrawal batching. +6. For a given withdrawal id `(as_of, base_event_id)`, a deterministic epoch + leader selects a pinned balance snapshot `{height, block_id}` and an + explicit selected input-note set. +7. The leader pokes `%create-withdrawal-tx` with the withdrawal id, epoch, + pinned snapshot metadata, withdrawal metadata, and selected input note + names. +8. The kernel bridge tx-builder emits `%withdrawal-proposal-built`, carrying + the full `withdrawal-proposal`, including the exact wallet `transaction` + and its unique `name`. +9. Rust persists and broadcasts that proposal envelope to peers. +10. Peers validate the proposal against kernel state and local bridge-owned note + / tx-builder state, durably persist the exact envelope, and commit to at + most one proposal hash for that `(withdrawal_id, epoch)`. +11. Once a threshold of peers has committed to the same exact proposal hash, + that proposal becomes the peer-canonical candidate for that withdrawal and + epoch. +12. Peer canonicalization alone is not enough to make a withdrawal submit-ready. + The sequencer gRPC service durably records the canonical candidate as the + only authorized next action for that withdrawal id. +13. Only the sequencer gRPC service may finalize and submit a withdrawal tx. + The sequencer service maintains the authoritative submitted / in-flight + withdrawal set and must not authorize the same withdrawal twice. +14. If the scheduled assembler is offline and no proposal becomes canonical + before the assembly timeout, the next epoch leader may assemble a new + candidate tx for the same withdrawal id. +15. The sequencer durably records the authorization / submission / + confirmation lifecycle and owns the global in-flight gate for withdrawals. + Only one withdrawal may be + sequencer-authorized / submitted / unconfirmed at a time. +16. If the sequencer is unavailable, withdrawals pause. There is no automatic + submission failover. +17. Authoritative progress is chain-observable, not gossip-observable. The + bridge should distinguish: + - diagnostic accepted state via `tx-accepted` + - confirmed inclusion via observed settlement in a confirmed block + A local "submitted" event is advisory only; sequencer authorization is the + required precondition for submission, not a consensus fact on its own. +18. The sequencer service and Rust Nock watcher observe confirmed settlement in + block txs. The sequencer durably records confirmation for the authorized + withdrawal before clearing its authoritative in-flight record. +19. Kernel reconciles settlement with counterpart withdrawal on the confirmed + Nock block stream, clears unsettled state, and emits a terminal withdrawal + effect for confirmed outcomes. +20. Rust and the sequencer consume that terminal withdrawal effect, release the + matching local reservations, and confirm that the withdrawal has been + removed from the sequencer's submitted / in-flight set. +21. If settlement references an unknown `as_of` base hash, hold logic blocks + advancement until that base hash is ingested. If `as_of` is known but + counterpart data is inconsistent/missing, processing stops. + +## Coordination Model + +1. The coordination unit is one withdrawal, keyed by `(as_of, base_event_id)`. Once admitted, a withdrawal remains live until confirmed. Withdrawal settlement is not coordinated as a multi-withdrawal batch. +2. There is one failover stage: + - assembly failover: if no proposal reaches canonicalization in the current epoch, the next epoch leader may assemble a new tx +3. A proposal becomes peer-canonical only after a threshold of bridge nodes has: + - validated the same exact proposal hash + - durably persisted the proposal envelope locally + - committed to that proposal for the given `(withdrawal_id, epoch)` +4. Peer canonicalization is not enough to create a submit-ready withdrawal. A withdrawal may only advance past peer-canonical state when the sequencer durably authorizes it. +5. "Submitted" is not a consensus fact. It is a sequencer-owned local event recorded only after sequencer authorization. The protocol must not require all nodes to agree on whether a submit RPC was attempted. +6. Proposal assembly, peer canonicalization, submission, timeout, and supersession tracking live in runtime attempt machinery. They are not kernel withdrawal states. +7. There is one sequencer gRPC service for withdrawals. That sequencer service + owns the authoritative submitted / in-flight withdrawal set and the durable + confirmation record for authorized withdrawals. +8. Only one withdrawal may be sequencer-authorized / submitted / unconfirmed at a time. +9. If the sequencer is unavailable, withdrawals stop. There is no automatic submission failover. +10. Because there is no chain-enforced withdrawal nullifier today, peer threshold agreement alone is not a sufficient duplicate-withdrawal safety mechanism. The sequencer is part of the authorization boundary for withdrawals. + +## Bridge-Owned Note Snapshot + +1. The withdrawal pipeline does not depend on the standalone `nockchain-wallet` application. +2. The bridge runtime uses the Rust note selector and bridge-owned tx-builder flow, with bridge-owned note state fetched from the private nockchain API. +3. The bridge maintains a confirmed note snapshot for the bridge-controlled spend authority / note pool. +4. This confirmed snapshot should refresh whenever the bridge observes a newly confirmed nockchain block. +5. In practice, snapshot freshness is bounded by: + - nockchain block production cadence + - configured nockchain confirmation depth + - nock watcher poll interval +6. A cached confirmed snapshot may be reused across multiple withdrawal assemblies, provided the planner always subtracts currently reserved input notes before selecting new inputs. +7. The runtime may also trigger an on-demand snapshot refresh before assembly if its cached confirmed snapshot is stale. + +## Reservation Lifecycle + +1. Input note reservations are per-node and must be persisted durably. +2. A proposal that is assembled locally but not yet canonical creates a provisional reservation on the assembling node only. +3. A proposal that becomes peer-canonical creates a canonical reservation on every node that accepts and persists that canonical proposal. +4. Provisional reservations prevent the assembling node from reusing selected inputs after restart while canonicalization is still pending. +5. Canonical reservations prevent all honest nodes from reusing inputs belonging to a sequencer-authorized in-flight withdrawal tx. +6. If a proposal does not become canonical, the rollback plan is: + - mark the proposal attempt terminal + - release its provisional reservations + - clear the assembly/canonicalization lock + - advance to the next epoch for the same withdrawal +7. If a sequencer-authorized tx fails to confirm, the withdrawal remains live. Recovery and replacement-attempt policy belongs to the sequencer-owned runtime attempt machinery for that same still-unconfirmed withdrawal. +8. Reservations are released or updated only when: + - the kernel emits a terminal withdrawal effect after confirmed settlement reconciliation + - the proposal attempt is rejected, expired, or superseded before canonicalization + - sequencer-controlled local runtime recovery transfers or replaces canonical reservations for the same still-unconfirmed withdrawal without creating a kernel withdrawal outcome +9. A local submit RPC attempt is not enough to release or modify reservations. +10. The kernel should not emit raw note names for release. It should emit withdrawal identity plus confirmed-settlement information, and Rust should resolve that to the locally reserved note set on each node. + +## Persistence and Tables + +### Source of Truth + +1. The source of truth is append-only. +2. Reserved input notes must be recorded in the append-only log, not only in mutable current-state tables. +3. Mutable tables are projections for fast lookup and operational convenience only. +4. On startup, the bridge rebuilds current attempt state and the current reserved-input set from the append-only log, then reconciles against observed chain state. +5. For withdrawals, the sequencer also owns an authoritative durable projection + of which withdrawal ids are peer-canonical, authorized, submitted, + confirmed, and still in-flight. + +### Tables + +1. `withdrawal_submission_events` + - append-only event header table + - one row per lifecycle event + - minimum columns: + - `event_id` + - `created_at` + - `withdrawal_id_as_of` + - `withdrawal_id_base_event_id` + - `epoch` + - `proposal_hash` + - `transaction_name` + - `event_type` + - `transaction_jam` nullable + - `snapshot_height` nullable + - `snapshot_block_id` nullable +2. `withdrawal_submission_event_inputs` + - append-only child table + - one row per input note referenced by an event + - minimum columns: + - `event_id` + - `note_name_first` + - `note_name_last` +3. `withdrawal_attempts` + - mutable/materialized table + - one row per attempt, approximately keyed by `(withdrawal_id, epoch, proposal_hash)` + - minimum columns: + - `withdrawal_id_as_of` + - `withdrawal_id_base_event_id` + - `epoch` + - `proposal_hash` + - `transaction_name` + - `state` + - `reservation_kind` + - `transaction_jam` + - `snapshot_height` + - `snapshot_block_id` + - `last_seen_accepted_at` nullable + - `confirmed_height` nullable + - `confirmed_block_id` nullable + - `superseded_by_epoch` nullable + - `created_at` + - `updated_at` +4. `current_reserved_inputs` + - mutable/materialized table + - pure current live exclusion set used by the planner + - one row per note that is reserved right now + - minimum columns: + - `note_name_first` + - `note_name_last` + - `withdrawal_id_as_of` + - `withdrawal_id_base_event_id` + - `epoch` + - `proposal_hash` + - `reservation_kind` + - `created_at` +5. `withdrawal_terminal_summaries` + - optional future compaction/checkpoint table + - one row per terminal withdrawal kept after detailed events are compacted + - minimum columns: + - `withdrawal_id_as_of` + - `withdrawal_id_base_event_id` + - `final_state` + - `final_tx_id` nullable + - `winning_epoch` nullable + - `winning_proposal_hash` nullable + - `confirmed_height` nullable + - `confirmed_block_id` nullable + - `terminal_at` + - `covered_through_event_id` +6. `sequenced_withdrawals` + - authoritative mutable table owned by the sequencer + - one row per withdrawal admitted to sequencer control + - minimum columns: + - `withdrawal_id_as_of` + - `withdrawal_id_base_event_id` + - `current_epoch` + - `peer_canonical_proposal_hash` nullable + - `authorized_proposal_hash` nullable + - `authorized_transaction_name` nullable + - `state` + - `created_at` + - `updated_at` +7. `staged_withdrawal_requests` + - mutable staging table owned by the local withdrawal assembly service + - one row per kernel-emitted withdrawal request admitted to local planning + - minimum columns: + - `withdrawal_id_as_of` + - `withdrawal_id_base_event_id` + - `recipient` + - `amount` + - `base_batch_end` + - `state` + - `active_epoch` nullable + - `created_at` + - `updated_at` + +### Event Types + +1. Expected event types include: + - `proposal_prepared` + - `proposal_canonicalized` + - `proposal_authorized` + - `proposal_rejected` + - `proposal_expired` + - `proposal_superseded` + - `tx_submitted` + - `tx_seen_accepted` + - `tx_confirmed` + - `reservation_released` + +### Append Log Schematics + +1. Request staging: + - upsert one `staged_withdrawal_requests` row for each + `%create-withdrawal-txs` request + - mark the chosen request as holding the local assembly lock before note + selection / `%create-withdrawal-tx` +2. Local assembly attempt: + - append `proposal_prepared` + - append one `withdrawal_submission_event_inputs` row per selected input note + - update or insert one `withdrawal_attempts` row + - insert the selected notes into `current_reserved_inputs` as provisional reservations + - update the matching `staged_withdrawal_requests` row to + `proposal_prepared` +3. Canonicalization: + - append `proposal_canonicalized` + - append one `withdrawal_submission_event_inputs` row per canonical reserved note + - update or insert one `withdrawal_attempts` row with canonical state + - ensure the selected notes exist in `current_reserved_inputs` as canonical reservations + - on the sequencer, update `sequenced_withdrawals` with the peer-canonical candidate +4. Sequencer authorization: + - append `proposal_authorized` + - update the sequencer-owned `sequenced_withdrawals` row with the only authorized proposal hash / transaction name for that withdrawal + - do not authorize a second proposal for the same withdrawal while any authorized or submitted state remains live +5. Local rollback before canonicalization: + - append `proposal_rejected`, `proposal_expired`, or `proposal_superseded` + - append `reservation_released` + - append one `withdrawal_submission_event_inputs` row per released note + - update the matching `withdrawal_attempts` row to a terminal or superseded state + - remove the released notes from `current_reserved_inputs` +6. Submission: + - append `tx_submitted` + - on the sequencer, transition `sequenced_withdrawals.state` to submitted / in-flight + - update the matching `withdrawal_attempts` row + - do not release reservations +7. Chain-observed progress: + - append `tx_seen_accepted` as observed for diagnostics + - update the matching `withdrawal_attempts` row + - do not release reservations +8. Confirmed observation: + - when confirmed settlement is observed for an authorized withdrawal, + append `tx_confirmed` + - update the matching `withdrawal_attempts` row to confirmed + - do not release reservations yet + - do not clear the sequencer-owned in-flight row yet +9. Terminal release: + - after the kernel emits a terminal withdrawal effect for confirmed + settlement, append `reservation_released` + - append one `withdrawal_submission_event_inputs` row per released note + - on the sequencer, remove or terminalize the `sequenced_withdrawals` row + - remove the released notes from `current_reserved_inputs` + +### Modification Rules + +1. `withdrawal_attempts` must never be edited on its own. +2. `withdrawal_attempts` may only be inserted or updated in the same transaction that appends a new row to `withdrawal_submission_events`. +3. `current_reserved_inputs` must never be edited on its own during normal runtime. +4. `current_reserved_inputs` may only be inserted into or deleted from in the same transaction that appends the corresponding reservation event rows. +5. The only exception is startup rebuild, where mutable tables may be reconstructed by replaying the append-only log. +6. `current_reserved_inputs` is a pure "currently blocking note names" table. +7. `current_reserved_inputs` must not be treated as a history or audit table. +8. History, audit, restart recovery, and reservation provenance must come from the append-only event tables. + +### Reservation Queries, Deletion, and Truncation + +1. The planner should treat the live spendable set as: + `spendable_notes = confirmed_snapshot - current_reserved_inputs` +2. When a note stops blocking planning, it should be removed from `current_reserved_inputs`. +3. Removing a note from `current_reserved_inputs` does not delete its reservation history. +4. Reservation history remains in the append-only event log forever unless it is safely compacted. +5. Safe truncation requires a terminal summary / checkpoint for a withdrawal before detailed event rows are deleted. +6. Detailed event rows must not be truncated for any withdrawal that: + - is not terminal + - still owns live reservations + - may still be resubmitted + - has not been checkpointed into a terminal summary +7. A practical future compaction path is: + - write a compact terminal summary row for a finished withdrawal + - confirm that no reservations or live attempts remain + - archive or delete the detailed hot-path event rows for that withdrawal + - optionally run SQLite `VACUUM` +8. Correctness is more important than aggressive truncation. It is acceptable to keep the full append-only withdrawal history and defer compaction. + +## Required Changes + +### A) Kernel changes (Hoon) + +1. Keep withdrawal proposal handling out of the kernel. + - do not introduce a kernel withdrawal proposal-validation cause + - do not persist `withdrawal-proposals` in kernel state + - do not validate proposal identity, epoch legality, replay, or + equivocation in the kernel + - do not validate proposal envelope fields against tracked withdrawal + metadata in the kernel + - the kernel should remain withdrawal-level only: qualifying burns create + live/unconfirmed withdrawals, and confirmed settlements clear them +2. Harden Base-side proposal generation in `++ base-propose-withdrawals` (already emitting `nock-withdrawal-request`) with queue semantics suitable for single-flight assembly/canonicalization. +3. Harden withdrawal parsing path in `++ process-nock-txs` (already creating `withdrawal-settlement`) and close remaining schema/validation gaps. +4. Finalize `++ nockchain-process-withdrawal-settlements` behavior in `open/hoon/apps/bridge/nock.hoon`: + - reconcile settlement against tracked unsettled withdrawals + - apply hold/stop semantics for out-of-order or inconsistent settlement + - tolerate sequencer retries of the same authorized tx without treating them as a second withdrawal +5. Add a dedicated create-withdrawal-tx poke/cause for bridge-side tx building. + - this remains milestone 2 work, but it is a split builder seam rather than + part of live submission enablement + - widen the poke/cause so it includes `epoch` and pinned + `withdrawal-snapshot` in addition to explicit withdrawal metadata plus + selected inputs + - it should let Rust ask the bridge app tx-builder to build one full + `withdrawal-proposal`, not just a raw tx + - successful construction should come back out via + `%withdrawal-proposal-built`, carrying that full proposal + - it should remain separate from `%create-withdrawal-txs`, which is the + kernel effect describing withdrawal intent from Base burns +6. Add a dedicated `%sign-tx` poke/cause for transaction signing. + - input should be the full `withdrawal-proposal` + - Rust uses it after proposal construction / authorization, not as part of + kernel withdrawal-state transitions + - signing remains distinct from tx construction and from confirmed terminal + effects +7. Extend the kernel effect surface with a terminal withdrawal outcome effect: + - emit on confirmed settlement reconciliation from the confirmed Nock block stream + - payload should identify the confirmed withdrawal and any confirmed-settlement data needed by Rust + - payload must not include local note names +8. Normalize and finalize withdrawal metadata encoding: + - new packed key path: `%bridge-w` + - includes `base-block-hash`, `beid`, `base-batch-end`, `lock-root` +9. Finalize amount semantics in kernel models (`amount-burned` vs `settled-amount`) now that dedicated withdrawal fee fields were removed from molds. + +### B) Rust bridge crate changes + +1. Add withdrawal proposal transport and validation: + - new ingress RPC and message types for nockchain tx proposal broadcast + - proposal envelope must include withdrawal id, epoch, pinned snapshot metadata, selected input note names, and the built `transaction` + - decode and route directly to Rust withdrawal coordination logic rather + than via a kernel cause + - validate the proposal envelope in Rust by checking: + - whether the withdrawal exists + - whether the proposal matches the tracked withdrawal + - same-epoch replay vs equivocation + - contiguous epoch legality + - also validate pinned snapshot, selected notes, and transaction identity + as part of the Rust-owned proposal envelope checks +2. Add the split withdrawal tx-builder seam: + - Rust selects candidate inputs and pinned snapshot metadata, then pokes the + widened `%create-withdrawal-tx` + - decode `%withdrawal-proposal-built`, carrying the constructed full + `withdrawal-proposal` + - keep this builder seam implementable before durable submission is enabled +3. Add the signing seam: + - Rust pokes `%sign-tx` with the full `withdrawal-proposal` + - successful signing should come back out via `%withdrawal-tx-signed`, + carrying the proposal envelope with signed transaction data + - signing is driven from Rust-side coordination and remains separate from + kernel withdrawal-state transitions +4. Add the withdrawal sequencing service: + - implement the durable sequencing service core and schema first + - append-only local log of proposal/canonicalization/submission/confirmation lifecycle + - sequencer-owned authoritative tracking of peer-canonical / authorized / + submitted withdrawals + - sequencer-owned confirmation watcher that records confirmed withdrawals + and clears the in-flight set + - reserve input notes by note `Name` before submit + - drive local tx-status reconciliation using `tx-accepted` as a diagnostic + mempool/accepted signal and confirmed settlement observations as the + release signal + - consume the kernel's terminal withdrawal effect to release local canonical + reservations by withdrawal id + - make the sequencer the only submission authority for withdrawals + - add the separate sequencer gRPC wrapper / RPC surface in the later + proposal transport step +5. Add withdrawal execution driver: + - consume `BridgeEffectVariant::CreateWithdrawalTxs` + - allow at most one withdrawal in assembly/canonicalization at a time for the bridge-controlled spend authority / note pool + - make the sequencer gRPC service the only component that finalizes and submits an authorized raw tx using nockapp public nockchain gRPC client (`wallet_send_transaction`) + - include separate assembly timeout and sequencer submission timeout behavior + - if the sequencer is unavailable, stop withdrawal progress rather than failing over to peer submission +6. Add proposal broadcast/signature workflow keyed off per-withdrawal epochs rather than deposit ids or withdrawal batches, with sequencer authorization required before a peer-canonical candidate becomes submit-ready. +7. Extend runtime/main wiring: + - register new IO drivers in `open/crates/bridge/src/main.rs` +8. Extend bridge note snapshot handling: + - use a normalized bridge-note balance snapshot for private-node sync as well as public sync + - refresh the confirmed bridge-note snapshot whenever a newly confirmed nockchain block is observed + - support on-demand refresh before assembly when the cached snapshot is stale + - compute `spendable_notes = confirmed_snapshot - current_reserved_inputs` + - prevent reuse of notes that belong to prepared/submitted but unconfirmed canonical txs +9. Add durable withdrawal storage schema and rebuild logic: + - implement append-only event tables for withdrawal lifecycle and reserved inputs + - implement mutable projection tables for attempts and current reserved inputs + - rebuild projections from the append-only log on startup + - reserve/release inputs transactionally with the corresponding event append +10. Optional but recommended: + - enrich `process_nock_log` output in `open/crates/bridge/src/ethereum.rs` to propagate destination/lock-root mapping into runtime observability structures, or explicitly document that kernel state is canonical and Rust-side `dest` remains advisory-only. + +### C) Type/protocol updates + +1. `open/crates/bridge/proto/bridge_ingress.proto` + - add withdrawal proposal request/response + - proposal payload must be a full withdrawal proposal envelope, not just a bare transaction body + - include withdrawal id, epoch, proposal hash / transaction name, pinned snapshot metadata, selected input note names, and typed payload + - add sequencer service RPCs for authorize / submit / status / confirmed-record updates +2. `open/crates/bridge/src/types.rs` + - keep `NockWithdrawalRequestKernelData` as the kernel-emitted withdrawal intent payload + - add `%withdrawal-proposal-built`, carrying a full `withdrawal-proposal` + - add `%withdrawal-tx-signed`, carrying a full signed `withdrawal-proposal` + - add a separate withdrawal proposal envelope type for peer coordination + - add a terminal withdrawal effect variant carrying withdrawal identity plus confirmed-settlement information + - ensure serialized field order matches Hoon `nock-withdrawal-request` (`base_event_id`, `recipient`, `amount`, `base_batch_end`, `as_of`) +3. `open/hoon/apps/bridge/types.hoon` + - keep withdrawal molds aligned with new shapes (`nock-lock-root`, `base-batch-end`, no dedicated withdrawal fee field) + - widen `%create-withdrawal-tx` so bridge-side tx building includes + `epoch` and pinned `withdrawal-snapshot`, not just withdrawal metadata and + selected inputs + - add a `%withdrawal-proposal-built` effect mold carrying a full + `withdrawal-proposal` + - add a `%withdrawal-tx-signed` effect mold carrying a full signed + `withdrawal-proposal` + - add a terminal withdrawal effect mold carrying withdrawal identity plus confirmed-settlement information + - do not add a withdrawal proposal-validation cause to the kernel molds + +## Invariants and Validation Rules + +1. Withdrawal identity must be deterministic and replay-safe: + - key by counterpart `(as_of, base_event_id)` with canonical encoding + - this identity is the coordination key for proposal epochs as well as settlement reconciliation +2. Canonicalization identity must be deterministic: + - proposal hash is computed over the exact proposal envelope / exact typed `transaction` + - transaction identity inside that envelope comes from `transaction.name`, not a separate `tx_id` + - honest peers must commit to at most one proposal hash per `(withdrawal_id, epoch)` +3. Settlement must match counterpart withdrawal on: + - destination lock root / recipient + - amount semantics (pre-fee vs post-fee exactly defined) +4. Authoritative protocol facts are chain-visible, not submit-attempt-visible: + - peer-canonicalization is not sufficient for submission + - sequencer authorization is required before a withdrawal may be submitted + - `tx-accepted` is a diagnostic signal that the tx reached node-accepted / + mempool-visible state, not that it was included in a block + - confirmed settlement in a block is the confirmation signal for inclusion + - the kernel's terminal withdrawal effect is the trigger for releasing + canonical local reservations + - local "submitted" events never make a tx canonical + - the sequencer owns the authoritative submitted / in-flight withdrawal set + - the sequencer durably records confirmation for authorized withdrawals +5. Input notes reserved by a canonical or in-flight tx must not be reused: + - reserve by note `Name` + - local spendable set is `confirmed normalized snapshot - locally reserved inputs` + - reservations are only released on confirmed success or by local runtime attempt rollback/replacement rules that do not change withdrawal state +6. There is no automatic submission failover: + - after sequencer authorization, only the sequencer may submit or retry the same tx + - if the sequencer is unavailable, withdrawals pause +7. Unknown counterpart policy: + - if `as_of` base hash is unknown: set hold and wait for counterpart chain progress + - if `as_of` is known but counterpart event/state is missing: stop +8. Any irreconcilable mismatch is a stop condition. +9. No silent divergence: + - kernel withdrawal-state failures remain explicitly one of ignore, hold, or stop + - Rust proposal-validation failures must be durably recorded and surfaced, + not hidden as kernel state transitions +10. At most one withdrawal may hold the assembly/canonicalization lock at a time for the bridge-controlled spend authority / note pool. +11. At most one withdrawal may be sequencer-authorized / submitted / unconfirmed at a time. + +## Testing Plan + +### Kernel tests + +1. Burn event enters `unsettled-withdrawals`. +2. Valid settlement clears corresponding unsettled entry and emits the terminal withdrawal effect. +3. Settlement-before-counterpart sets hold and later resolves. +4. Settlement with known `as_of` but missing counterpart triggers stop. +5. Settlement mismatch triggers stop. +6. Duplicate/replay settlement does not corrupt state. + +### Rust tests + +1. Ingress decodes withdrawal proposal and hands it directly to Rust + withdrawal coordination logic. +2. Rust proposal validation rejects unknown withdrawals, mismatched withdrawal + metadata, illegal epochs, replay with mismatched envelope, and same-epoch + equivocation. +3. Rust proposal broadcast driver emits full withdrawal proposal envelopes to peers correctly. +4. Canonicalization is reached only when threshold peers persist and commit to the same proposal hash. +5. Nock tx submission driver handles: + - success ack + - retryable errors + - non-retryable errors +6. Only the sequencer may authorize and submit a peer-canonical withdrawal candidate. +7. Input notes used by prepared/submitted canonical txs are filtered from later planning snapshots. +8. Canonical reservations are released on confirmed settlement via the kernel terminal withdrawal effect, not via a withdrawal-level abandon/fail outcome. +9. Sequencer confirmation tracking records confirmed withdrawals and clears the authoritative in-flight set without double-recording the same withdrawal. +10. Sequencer restart preserves the authoritative in-flight withdrawal set and confirmed-withdrawal record, and does not re-authorize the same withdrawal twice. +11. Sequencer loss pauses withdrawal progress rather than failing over submission to peers. +12. End-to-end wiring test: + - kernel emits `%create-withdrawal-txs` + - peers converge on one canonical candidate + - sequencer authorizes and submits that candidate + - sequencer records confirmation for that authorized withdrawal + - kernel later emits the terminal withdrawal effect and Rust releases the matching local reservations + - no panic/regression in existing deposit path + +### Multi-node integration + +1. 5-node single-withdrawal proposal and threshold signature convergence with sequencer authorization. +2. Scheduled assembler offline before proposal timeout; next epoch leader assembles a new candidate tx. +3. After peer canonicalization, only the sequencer authorizes and submits the candidate tx. +4. Sequencer restart mid-flight preserves reservations, the in-flight withdrawal set, and the confirmed-withdrawal record, then resumes safely. +5. Sequencer unavailability pauses withdrawal progress rather than failing over submission to peers. +6. Conflicting later proposal for an already authorized withdrawal is treated as an invariant violation / stop. +7. Simulated out-of-order Base/Nock arrival with hold release. + +## Open Decisions (Must Resolve Before Full Implementation) + +1. Peer identity and signature domain for withdrawal tx proposals: + - `nock-pubkey`, `nock-pkh`, or node id +2. Withdrawal authorization encoding: + - the final withdrawal authorization path must require sequencer participation + - peer threshold agreement alone must not produce a submit-ready withdrawal +3. Fee model: + - exact formula for withdrawal fee deduction and where it is applied +4. Hold metadata requirements: + - whether base block height must be embedded in tx metadata for deterministic unblock behavior +5. Whether single-flight policy should remain permanent, or only the current intended design + +## Phased Delivery Plan + +1. Phase 1: Kernel correctness skeleton + - implement non-crashing withdrawal parse + settlement reconcile logic + - implement hold/stop behavior + - keep transport disabled behind clear guard if needed +2. Phase 2: Rust transport and execution + - ingress proposal broadcast + separate sequencer-owned gRPC withdrawal service + - single-flight assembly/canonicalization workflow + - assemble, persist, authorize, submit, and record confirmation for one sequencer-owned withdrawal at a time +3. Phase 3: Multi-node convergence and failover hardening + - assembly failover, sequencer restart recovery, reservation release +4. Phase 4: Production hardening + - metrics, alerts, operational docs, replay abuse tests + +## Acceptance Criteria + +1. No `TODO`/hard-stop paths remain for withdrawal causes/effects in kernel. +2. Bridge nodes can process Base burns into canonicalized, submitted, and finalized per-withdrawal Nock settlements. +3. If the scheduled assembler is offline before canonicalization, the next epoch leader can assemble a replacement candidate without ambiguity. +4. If a tx becomes peer-canonical, only the sequencer may authorize and submit it. +5. Out-of-order chain arrival is handled via deterministic hold behavior. +6. Submitted-but-unconfirmed withdrawal inputs are not reused for later withdrawal planning. +7. Existing deposit pipeline remains unchanged and green. +8. Integration tests cover nominal, sequencer-stop, restart, and note-reservation scenarios. diff --git a/crates/bridge/docs/governance.md b/crates/bridge/docs/governance.md new file mode 100644 index 000000000..0d2a14b20 --- /dev/null +++ b/crates/bridge/docs/governance.md @@ -0,0 +1,96 @@ +# Bridge Upgrade & Governance Process + +This document codifies how we upgrade the bridge stack—contracts, kernel, and +node software—and who is authorized to operate bridge infrastructure. +It is not a standalone source of chain-level protocol authority. + +Protocol authority routing: + +- Chain-level protocol index and activation authority: [`PROTOCOL.md`](../../../PROTOCOL.md) +- Canonical protocol upgrade sources: [`changelog/protocol/`](../../../changelog/protocol/) +- This document governs bridge subsystem operations and cannot override indexed protocol specs. + +## Actors + +- **Bridge multisig (Base)** – Owns `MessageInbox.sol` (UUPS proxy + owner) and + `Nock.sol`. Minimum requirement: 3-of-5 signatures from the bridge node set or + a dedicated Gnosis Safe controlled by the same operators. +- **Bridge node operators** – Five parties running the Rust runtime + Hoon + kernel. They hold the Ethereum keys referenced in `bridge-conf.toml` and the + Schnorr keys for nockchain signatures. +- **Bridge maintainers** – Engineers permitted to push new Hoon kernels, + update the Rust crate, or publish contract releases. + +## Contract Upgrade Process + +1. **Author & review** + - Implement changes under `crates/bridge/contracts`. + - Update or add Foundry tests (`contracts/test/*.t.sol`). + - Run `forge test` and capture gas snapshots. +2. **Deploy implementation** + - Use Foundry scripts under `contracts/forge/*.s.sol` (and helper shell + scripts under `contracts/scripts/` as needed) to deploy a new logic + contract (do not upgrade yet). + - Update `contracts/deployments.json` if operators rely on deployment-file + address discovery. +3. **Multisig approval** + - Prepare a `upgradeTo(newImplementation)` transaction for the UUPS proxy + (`MessageInbox`). + - Collect signatures from the bridge multisig signers. + - Once executed, emit a memo referencing the git SHA and release notes. +4. **Post-upgrade verification** + - Call `proxiableUUID()` to ensure the slot matches expectations. + - Run a canary deposit using the devnet to verify signature validation still + succeeds. + - Broadcast the new implementation address to every node operator so they can + update monitoring and config files. + +`Nock.sol` is **not** upgradeable; governance is limited to `updateInbox` and +ownership transfers. Treat `updateInbox` as an emergency-only lever (for +example, if the inbox proxy is compromised) and require the same multisig quorum +before invoking it. + +## Kernel & Runtime Releases + +1. **Hoon kernel** + - Modify `hoon/apps/bridge/bridge.hoon` and `hoon/apps/bridge/types.hoon`. + - Regenerate `assets/bridge.jam` (`make assets/bridge.jam`). + - Build and release a bridge binary from that commit (the runtime embeds + `assets/bridge.jam` at build time). + - Tag the release with git SHA and circulate the release artifact or commit. + - Rolling-restart operators on the new build; use `--new` only when state + reset is intentionally required. +2. **Rust runtime** + - Land changes under `crates/bridge/src`. + - Run `cargo test -p bridge --lib` and any integration suites touching the + Base watcher or ingress server. + - Produce a signed release binary (or instruct operators to build from the + tagged commit). + - Rolling restart the five nodes one at a time to maintain quorum. + +## Emergency Procedures + +- **Bridge node rotation** – If a node key leaks, use `MessageInbox.updateBridgeNode` + to swap the compromised address, then distribute a fresh `bridge-conf.toml` to + every operator so proposer selection stays deterministic. +- **Pause submissions** – There is no on-chain pause switch yet; coordinate a + social halt by having all nodes stop calling `submitDeposit`. Because the + kernel persists unsigned proposals, restarting later resumes the queue. +- **Signature threshold adjustment** – Changing the 3-of-5 constant requires a + contract upgrade; follow the full process above and clearly document the new + quorum. + +## Change Logging + +- Record every upgrade (contract logic, kernel jam, runtime release) in the + bridge ops log with: datetime, git SHA / contract address, signer list, and + validation steps performed. + +## Reference + +- `contracts/MessageInbox.sol` – upgradeable inbox (UUPS + Ownable). +- `contracts/Nock.sol` – non-upgradeable ERC-20 with `updateInbox`. +- `crates/bridge/src/main.rs` – ties together observers, ingress, kernel, + and signing driver. +- `crates/bridge/docs/architecture.md` – holistic overview if you need + to understand the blast radius of a change. diff --git a/crates/bridge/docs/node-runbook.md b/crates/bridge/docs/node-runbook.md new file mode 100644 index 000000000..e0b421ba1 --- /dev/null +++ b/crates/bridge/docs/node-runbook.md @@ -0,0 +1,135 @@ +# Bridge Node Runbook (Runtime and Incident Operations) + +Use this runbook after a node is already provisioned. +This is the canonical runtime and incident-response document for bridge operators. + +## Scope + +- In scope: start/restart procedures, runtime health, monitoring, and incident response. +- Out of scope: initial provisioning and key onboarding, which are documented in [`../OPERATOR-SETUP.md`](../OPERATOR-SETUP.md). + +## Entry Conditions + +- Node config is already provisioned via [`../OPERATOR-SETUP.md`](../OPERATOR-SETUP.md). +- If you are using script-driven environments, profile selection is done via [`../scripts/environments/README.md`](../scripts/environments/README.md). + +## 1. Boot and Restart Procedure + +1. `cd crates/bridge`. +2. Launch: + + ```bash + cargo run -p bridge -- --config-path /abs/path/to/bridge-conf.toml + ``` + +3. Optional flags: + - `--new` wipes on-disk kernel state, use only when you intend to reset state. + - `--start` sends `%start` on boot if the kernel is STOPPED. +4. Confirm startup logs include: + - `bridge nockapp started` + - `loaded config from ...` + - `Base bridge and signer initialized successfully` + - `connected to nockchain gRPC endpoint` + - `starting bridge ingress gRPC server` + +## 2. Routine Operations + +### Watchers + +- Base watcher (`src/ethereum.rs`) polls confirmed windows every 30s over the Base WebSocket provider. If logs show `base reorg detected` or repeated `failed to get block number after retries`, pause submissions and investigate chain health. +- Nock watcher (`src/nockchain.rs`) polls private nockapp (`heavy`, `heavy-n`) every 10s by default. Repeated `failed to fetch tip height` or `failed to fetch block at height` usually indicates private nockapp availability or decode-path issues. + +### Runtime and Kernel + +- `BridgeRuntime` throttles pending events to 1024. `dropping oldest pending event` indicates backlog pressure. +- `%commit-nock-deposits` effects are persisted to `deposit-queue.sqlite` by the commit driver. +- Signature gossip is handled by the signing cursor loop. Monitor `bridge.cursor` for `broadcast signature to peer`, `failed to broadcast signature to peer`, and nonce divergence alerts. + +### Ingress + +- gRPC ingress (from `ingress_listen_address`) serves: + - `bridge.ingress.v1.BridgeIngress` + - `bridge.status.v1.BridgeStatus` + - `bridge.tui.v1.BridgeTui` +- For bridge-to-bridge operation, ingress handles `BroadcastSignature`, `BroadcastConfirmation`, and `BroadcastStop` (plus `HealthCheck` and `GetProposalStatus`). + +### Status gRPC + +- Status endpoint: `bridge.status.v1.BridgeStatus/GetStatus` on ingress gRPC. +- Replace `` with your configured `ingress_listen_address` (for example `127.0.0.1:8001` or `127.0.0.1:8002`). +- Inspect API: + + ```bash + grpcurl -plaintext list + grpcurl -plaintext describe bridge.status.v1.BridgeStatus + ``` + +- Query: + + ```bash + grpcurl -plaintext -format json -d '{}' bridge.status.v1.BridgeStatus/GetStatus + ``` + +- `lastSubmittedDeposit` appears only after this process successfully submits a deposit and resets on restart. + +## 3. Common Operational Tasks + +| Task | Steps | +| ---------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | +| Rotate a bridge node address | Use MessageInbox owner (Gnosis multisig) to call `updateBridgeNode(index, newNode)`, then update every operator config with the new `eth_pubkey`. | +| Update contract addresses | Deploy new contracts, update `contracts/deployments.json` (if you rely on deployment lookup), and/or set explicit `inbox_contract_address` + `nock_contract_address` in each operator config before restart. | +| Rebuild the kernel | Regenerate `assets/bridge.jam`, rebuild/release bridge binaries, then rolling-restart nodes. Use `--new` only when a state reset is intentionally required. | +| Verify deposit submission | Tail for `Deposit processed on MessageInbox` and cross-check tx hash in Base explorer. | + +## 4. Incident Response + +1. Base RPC degradation: repeated `failed to get block number after retries` or submission errors (`Transaction failed`, `Transaction reverted`) indicate provider/outage or contract-path issues. Verify provider health first, then restart only if the process does not recover. +2. Submission stalls: posting is strict on `lastDepositNonce + 1`. If chain nonce does not advance, inspect posting-loop errors and confirm no nonce epoch mismatch alerts. +3. Nonce divergence: alerts like `Nonce Divergence Suspected` (ingress/cursor) indicate peers disagree on proposal hash for the same deposit ID. Pause submissions and reconcile nonce-epoch config and deposit-log state across operators. +4. Signature starvation: if proposals stay in `collecting`, verify peer health (`HealthCheck`), ingress reachability, and that peer signatures are being broadcast/accepted. +5. STOP state: if the kernel enters STOPPED state, restart without `--new` and pass `--start` only after confirming it is safe to clear stop state. + +## 5. Shutdown and Recovery + +1. Send SIGINT and allow graceful shutdown. +2. Confirm Base watcher, nock watcher, runtime, ingress, and app tasks exit cleanly. +3. Restart without `--new` to resume from persisted noun state; add `--start` only when recovering from STOPPED kernel state. + +## 6. Logging and Monitoring + +Logs persist under `{data_dir}/logs/` (typically `~/.nockapp/bridge/logs/`). + +Override log directory: + +```bash +cargo run -p bridge -- --log-dir /var/log/bridge +``` + +Set verbosity with `RUST_LOG`: + +```bash +RUST_LOG=info cargo run -p bridge +RUST_LOG=bridge::runtime=debug,bridge::ethereum=trace cargo run -p bridge +RUST_LOG=debug cargo run -p bridge +``` + +Useful queries: + +```bash +tail -f ~/.nockapp/bridge/logs/bridge.log +grep -r "ERROR\\|WARN" ~/.nockapp/bridge/logs/ +grep "bridge.base.observer\\|bridge.posting\\|bridge.cursor" ~/.nockapp/bridge/logs/bridge.log +grep "Deposit processed on MessageInbox\\|posting proposal to BASE\\|failed to post deposit" ~/.nockapp/bridge/logs/bridge.log +``` + +Alert on: +- Base observer and posting-loop errors (`bridge.base.observer`, `bridge.posting`) +- Nock watcher connectivity/decode warnings (`bridge.nock-watcher`) +- Runtime dropped-event warnings (`bridge.runtime`) +- Ingress broadcast validation failures (`bridge.ingress`) + +## Handoff + +- Need initial provisioning or key/node assignment work: [`../OPERATOR-SETUP.md`](../OPERATOR-SETUP.md) +- Need to switch script environment profiles or update env vars: [`../scripts/environments/README.md`](../scripts/environments/README.md) +- Need local first boot workflow: [`../QUICKSTART.md`](../QUICKSTART.md) diff --git a/crates/bridge/docs/sans-io-architecture.md b/crates/bridge/docs/sans-io-architecture.md new file mode 100644 index 000000000..2c966386e --- /dev/null +++ b/crates/bridge/docs/sans-io-architecture.md @@ -0,0 +1,145 @@ +# Bridge Sans-IO Architecture + +## What this doc is for +A readable description of the bridge's sans-io architecture. + +- We separate **thinking** from **doing**. +- “Thinking” means deciding what should happen next. +- “Doing” means talking to the network, contracts, kernel, or disk. + +That split makes behavior easier to reason about, easier to test, and less +fragile when we change integrations. + +## The main idea + +Most bridge loops follow the same pattern: + +1. Read current state (tip heights, next nonce, queued work, etc.) +2. Decide what action makes sense +3. Execute that action +4. Repeat + +In this architecture: + +- step 2 is a small, predictable decision function +- step 3 is handled by shell/integration code + +So we can test decisions without needing real RPC endpoints, timers, or sleeps. + +## A quick example + +This is roughly what a loop does: + +```rust +// Pseudocode +let input = gather_input(); +let action = planner(input); +execute(action).await; +``` + +And for signing/posting, we also expose a single “tick” call: + +```rust +// Pseudocode +let outcome = signing_tick_once(&context, &mut state, input).await; +``` + +That lets tests drive one step at a time and assert exactly what happened. + +## How the bridge is split up + +### Decision code (the “thinking” side) + +- `src/core/base_observer.rs` +- `src/core/nock_observer.rs` +- `src/core/signing.rs` +- `src/core/posting.rs` + +These files decide things like: + +- “wait for more confirmations” +- “fetch this block range” +- “skip, not my turn” +- “submit now” + +### Integration code (the “doing” side) + +- `src/ethereum.rs` +- `src/nockchain.rs` +- `src/runtime.rs` +- `src/main.rs` + +These files do the real work: + +- call RPC/gRPC +- send/receive runtime events +- submit contract transactions +- run timers/retries/stop handling + +## Why this helps + +### Easier to test + +We can test decisions in tight loops without flaky timing behavior. + +### Safer changes + +We can replace transport details (RPC client, watcher wiring, etc.) without +rewriting core behavior. + +### Clearer failures + +When something goes wrong, it is easier to tell whether: + +- the decision was wrong, or +- the network call failed. + +## What “single-tick” gives us + +For signing and posting, we have bounded tick APIs that do one unit of work. +This is useful in tests and debugging. + +Examples in code: + +- `signing_tick_once(...)` +- `posting_tick_once(...)` + +Related tests: + +- `tests/loop_tick_tests.rs` +- `tests/loop_policy_shell_tests.rs` +- `tests/core_runner_integration.rs` + +## What stays the same + +This architecture does **not** change bridge protocol semantics by itself. +The kernel protocol and end-to-end flow are still the same: + +- observers feed runtime +- runtime talks to kernel +- kernel emits effects +- drivers execute effects + +The change is mostly about making Rust-side behavior easier to understand and +verify. + +## Current state + +Already in place: + +- planner/executor split across observer/signing/posting paths +- policy-based loop wrappers for cadence/retry/stop behavior +- deterministic tick tests for core branches + +Still planned: + +- deeper runtime/kernel harness replay tooling for even stronger transcript-style + validation +- final parity soak closure in staging + +## Where to read next + +- `open/crates/bridge/docs/architecture.md` for full system flow +- `open/crates/bridge/src/core/` for decision logic +- `open/crates/bridge/src/runtime.rs` for loop orchestration +- `open/crates/bridge/tests/loop_tick_tests.rs` for deterministic examples diff --git a/crates/bridge/docs/signatures.md b/crates/bridge/docs/signatures.md new file mode 100644 index 000000000..166607653 --- /dev/null +++ b/crates/bridge/docs/signatures.md @@ -0,0 +1,176 @@ +# Bridge Signature Format & Verification + +Status: Active +Owner: Nockchain Maintainers +Last Reviewed: 2026-02-20 +Canonical/Legacy: Legacy (bridge subsystem signing reference; canonical docs spine starts at [`START_HERE.md`](../../../START_HERE.md)) + +This guide describes how proposal hashes and Ethereum signatures are computed, +validated, gossiped, and submitted in the bridge runtime. + +## Data Model + +| Type | Location | Purpose | +| -------------------------- | --------------------- | ------------------------------------------------------------------------------------------------------------------------------ | +| `SignatureSet` | `src/types.rs` | Bundles `eth_signatures: Vec` and `nock_signatures: Vec` for submissions to Base or nockchain. | +| `EthSignatureParts` | `src/types.rs` | Canonical `(r, s, v)` with validation helpers and noun encoding so the kernel can reason about signatures. | +| `BaseCallSigData` | `src/types.rs` | Legacy cause payload that couples an `EthSignatureParts` with signed calldata for peer verification paths. | +| `NockDepositRequestData` | `src/types.rs` | Structured signature request: `[tx-id, name, recipient, amount, block-height, as-of]` matching Hoon `nock-deposit-request`. Here `name` is a full `nockchain_types::v1::Name` (two Tip5 hashes: first & last). | + +Signature validation and proposal hashing are driven from these shared types, +with `%commit-nock-deposits` as the kernel effect that feeds the runtime +signing pipeline. + +## Message Hash Construction + +`MessageInbox.submitDeposit` verifies signatures over: + +```solidity +keccak256(abi.encodePacked( + _encodeTip5(txId), // 40 bytes + _encodeTip5(nameFirst), // 40 bytes + _encodeTip5(nameLast), // 40 bytes + recipient, // 20 bytes + amount, // 32-byte uint256 + blockHeight, // 32-byte uint256 + _encodeTip5(asOf), // 40 bytes + depositNonce // 32-byte uint256 +)); +``` + +In Rust (`compute_proposal_hash` in `src/types.rs`), the payload is built as: + +1. Tip5 limbs encoded as **big-endian** `u64` bytes (`to_be_bytes`), matching Solidity `abi.encodePacked(uint64, ...)`. +2. `amount` converted from nicks to NOCK base units (`NOCK_BASE_PER_NICK`) before encoding as 32-byte `U256`. +3. `block_height` and `nonce` encoded as 32-byte `U256`. + +Total packed payload length is 276 bytes before keccak256. + +## Signing Flow + +1. Kernel emits `%commit-nock-deposits` with nonce-free requests. +2. Runtime assigns nonce deterministically and builds `NockDepositRequestData`. +3. Runtime computes proposal hash and signs with `EthereumSigner` (`src/signing.rs`). +4. `EthereumSigner` uses `alloy::signers::local::PrivateKeySigner::sign_message`, so signatures are EIP-191 prefixed. +5. Runtime gossips signatures and the proposal cache aggregates signatures until the threshold is reached, at + which point the posting loop calls `submitDeposit` on Base. + +## Ingress Wire Format + +Bridge signature gossip is handled by `bridge.ingress.v1.BridgeIngress` (`proto/bridge_ingress.proto`): + +- `BroadcastSignature(SignatureBroadcast)`: + - `deposit_id`: 120 bytes (`DepositId`: `as_of || name.first || name.last`) + - `proposal_hash`: 32 bytes + - `signature`: 65 bytes (`r || s || v`) + - `signer_address`: 20 bytes +- `GetProposalStatus(ProposalStatusRequest)` for cache state. +- `BroadcastConfirmation(ConfirmationBroadcast)` after successful Base posting. + +`ProposalRequest` / `SignatureRequest` messages exist in the proto but are not the active ingress RPC path. + +## Current Kernel Effect Surface + +The kernel now emits only: + +- `%create-withdrawal-txs reqs=(list nock-withdrawal-request)` +- `%commit-nock-deposits reqs=(list nock-deposit-request)` +- `%grpc grpc-effect` +- `%stop reason=cord last=stop-info` + +Deposit signature flow in this document is tied to `%commit-nock-deposits`. +`%create-withdrawal-txs` currently carries withdrawal intent metadata and is not +yet part of the ETH multisig deposit-signature path. + +## Serialization Format + +### Hoon Types + +The `nock-deposit-request` type in `hoon/apps/bridge/types.hoon`: +```hoon ++$ nock-deposit-request + [tx-id=tx-id:t name=nname:t recipient=base-addr amount=@ block-height=@ as-of=nock-hash] +``` + +The `commit-nock-deposits` effect variant: +```hoon +[%commit-nock-deposits reqs=(list nock-deposit-request)] +``` + +### Rust Types + +`NockDepositRequestData` in `src/types.rs` matches the Hoon structure: +- `tx_id: Tip5Hash` - Encoded as 5 uint64 limbs (40 bytes) via `Tip5Hash::to_array()` +- `name: Name` - Includes `first` and `last`, each encoded like `tx_id` (5 limbs => 40 bytes each, total 80 bytes) +- `recipient: EthAddress` - Exactly 20 bytes +- `amount: u64` - Amount in nocks +- `block_height: u64` - Block height +- `as_of: Tip5Hash` - Same encoding as tx_id/name + +The `compute_proposal_hash` function in `src/types.rs` encodes each Tip5Hash +field as its 5 uint64 limbs packed in little-endian order, matching the Solidity +`_encodeTip5` function. + +### gRPC Wire Format + +When broadcasting via `ProposalRequest`: +- `kind`: `ProposalKind::Base` +- `payload`: Protobuf-encoded `EthSignatureRequest` message +- `proposer`: Node identifier string +- `request_id`: Unique request identifier + +When responding via `SignatureRequest`: +- `proposal_hash`: 32-byte keccak256 hash (computed from eth_signature_request fields) +- `signature`: 65-byte ECDSA signature (r || s || v) +- `signer`: Node identifier string +- `request_id`: Matches the original proposal request_id +- `eth_signature_request`: Optional structured fields for validation + +The runtime never trusts an inbound signature blindly: `EthSignatureParts` +exposes `validate()` so we can reject zeroed limbs or malformed `v` before the +kernel even sees the data. + +## On-Chain Verification + +`MessageInbox` enforces: + +1. At least `THRESHOLD` (= 3) unique bridge-node signatures. +2. Canonical signature constraints (`s <= secp256k1_n/2`, `v` in `{27, 28}`). +3. Replay protection via `processedDeposits[txIdHash]`. +4. Monotonic ordering via `depositNonce > lastDepositNonce`. + +A failed signer recovery or non-node signer causes the submission to revert. + +## Operational Checklist + +Before gossiping/submitting: + +- Validate signature length (65 bytes) and signer-address length (20 bytes). +- Ensure proposal hash matches the in-cache request for the same `deposit_id`. +- Ensure signer address is in configured bridge-node set. +- Keep signatures raw (`r || s || v`) when building `bytes[] ethSigs` for contract submission. + +## Testing Pointers + +- Collect at least three validated signatures and keep their ordering stable. +- Build `DepositSubmission` from canonical proposal fields and the assigned nonce + (runtime path), then call `BaseBridge::submit_deposit`. +- Attach the signatures as `bytes[] ethSigs` exactly as returned by peers (no + ABI re-encoding is needed; each entry is the raw 65-byte concatenation of + `r || s || v`). + +## Testing + +- `src/signing.rs` ships an async test that covers the happy path for + `EthereumSigner::sign_proposal`. +- Contract-level tests in `contracts/test/MessageInboxDepositTest.t.sol` (add + more as we expand coverage) should cover malleability, threshold edges, and + replay protection. +- When adding new signing backends, integrate them into the `commit-nock-deposits` + driver and reuse the existing hash computation and noun encoding helpers. + +### Test Catalog +- `src/signing.rs`: signer unit tests. +- `src/ingress.rs`: signature broadcast validation and cache tests. +- `src/proposal_cache.rs`: threshold and signature-aggregation behavior. +- `contracts/test/MessageInboxDepositTest.t.sol`: contract-side signature checks and replay/threshold behavior. diff --git a/crates/bridge/docs/withdrawal-dapp.md b/crates/bridge/docs/withdrawal-dapp.md new file mode 100644 index 000000000..478521db1 --- /dev/null +++ b/crates/bridge/docs/withdrawal-dapp.md @@ -0,0 +1,467 @@ +# Withdrawal DApp Spec + +Status: Draft +Owner: Nockchain Maintainers +Last Reviewed: 2026-03-16 +Canonical/Legacy: DApp-facing product/implementation spec derived from `bridge-withdrawals.md` + +Source of truth: +- `open/crates/bridge/docs/bridge-withdrawals.md` + +## Why This Doc Exists + +`bridge-withdrawals.md` is the canonical protocol and systems spec. It is +written for people implementing the bridge kernel, runtime, sequencer, and +storage layers. + +This document is the DApp/UI translation of that spec. It is meant to answer: + +1. What is the withdrawal DApp actually doing for the user? +2. What backend states matter to the UI? +3. What should the UI show, and what should it never pretend to control? +4. What assumptions should you make before wiring screens and + API calls? + +## One-Sentence Product Definition + +The withdrawal DApp lets a user burn wrapped NOCK on Base to request a +withdrawal to Nockchain, then shows a small, honest set of public statuses +until that withdrawal is confirmed on Nockchain. + +## Core Mental Model + +The most important thing to understand is: + +1. The DApp does not build, authorize, or submit withdrawal transactions by + itself. +2. The user starts a withdrawal by burning wrapped NOCK on Base. +3. That burn creates a withdrawal inside the bridge system if and only if the + amount is strictly above the configured minimum. +4. From that point on, the bridge backend owns the process. +5. The UI is mainly responsible for: + - collecting the withdrawal destination and amount + - helping the user send the Base burn transaction + - displaying accurate coarse-grained withdrawal state + - surfacing delays or support-worthy problems honestly + +For the frontend, a withdrawal is a single object keyed by: +- `withdrawal_id = (as_of, base_event_id)` + +Where: + +1. `as_of` is the Base block hash the bridge uses as the counterpart reference + point for the withdrawal +2. `base_event_id` is the unique identifier of the burn event within that + Base-side history + +This is mostly a backend/internal identifier. You may not want to show it +directly in the public UI, but the current product direction is to show both +the internal `withdrawal_id` and the more human-friendly references like the +Base burn tx hash and timestamps. + +Do not think of the UI as operating on "attempts" or "tx proposals." Those +exist in backend coordination, but the user-facing object is the withdrawal. + +## What Counts As A Withdrawal + +A Base burn becomes a withdrawal only if: + +1. the burn event is observed by the bridge +2. the kernel admits it into withdrawal state +3. the burned amount is strictly above the withdrawal minimum + +Important: + +1. Burns at or below the minimum should not be represented in the UI as active + withdrawals. +2. The current target minimum in the canonical spec is `10,000 NOCKS`. +3. For now, the DApp should hardcode that minimum rather than fetch it from a + backend API. +4. Do not assume the amount burned on Base is the same as the amount received + on Nockchain. The withdrawal fee is deducted, so the received amount is + lower than the burned amount. + +## What The User Does + +At a high level: + +1. User connects Base wallet. +2. User enters: + - withdrawal amount + - Nockchain destination lock root +3. DApp validates the request client-side as much as it can. +4. User signs and submits the Base burn transaction. +5. DApp tracks the burn transaction on Base. +6. After the burn is confirmed enough on Base, the DApp transitions to a coarse + "withdrawal pending" state. +7. Keep the DApp in that pending state until the withdrawal is confirmed on + Nockchain or is delayed long enough that you should point the user to + support/status guidance. + +## What The UI Is Not Responsible For + +The DApp must not pretend to own these steps: + +1. proposal construction +2. peer canonicalization +3. sequencer authorization +4. withdrawal tx submission to Nockchain +5. withdrawal confirmation reconciliation +6. note reservation logic +7. replay/equivocation detection + +Those belong to the bridge runtime, sequencer gRPC service, and kernel. + +Do not expose those internal stages in the public DApp unless there is a real, +reliable product requirement for them. + +## Public User-Facing Lifecycle + +The public withdrawal DApp should use a very small number of states. The user +probably will not have reliable visibility into bridge-internal progress +between the Base burn and the final Nockchain confirmation. + +Recommended user-facing stages: + +1. `draft` + - user is filling out the form +2. `awaiting_wallet_confirmation` + - wallet popup is open for the Base burn tx +3. `awaiting_base_confirmation` + - Base burn transaction has been sent but is not yet confirmed enough from + the user’s point of view +4. `withdrawal_pending` + - the burn is done and the user is now waiting for the bridge withdrawal to + complete on Nockchain +5. `confirmed` + - confirmed settlement observed and reconciled +6. `delayed` + - the withdrawal is taking longer than expected and you should point the + user to support or system status guidance + +The public UI should not expose backend-internal states like: + +1. proposal built +2. peer-canonical +3. sequencer authorized +4. sequencer submitted +5. hold +6. stop + +Those may exist in admin/operator tooling, but keep the public DApp coarse +unless the backend exposes a deliberate user-safe product surface for them. + +## Internal Backend States + +The canonical spec implies these important backend facts: + +1. A withdrawal is either live/unconfirmed or confirmed at the kernel level. +2. The sequencer gRPC service owns: + - the authoritative authorized withdrawal record + - the submitted / in-flight withdrawal record + - the durable confirmation record for authorized withdrawals +3. Only one withdrawal may be sequencer-authorized / submitted / unconfirmed + at a time. +4. If the sequencer is unavailable, withdrawals pause. +5. Unknown counterpart chain state can produce a hold. +6. Irreconcilable mismatch can produce a stop. + +These are useful for backend and operator tooling, but not necessarily for the +public DApp. + +If an internal/admin view exists, it may show: + +1. sequencing +2. authorized +3. submitted +4. held +5. stopped +6. sequencer unavailable + +## Recommended DApp Screens + +### 1. Withdrawal Form + +Purpose: +- create a new withdrawal request by helping the user submit the Base burn tx + +Fields: + +1. amount +2. destination +3. wallet/network status + +Behavior: + +1. Validate amount is present and positive. +2. Treat the destination as a lock root, not as a generic address field. +3. Provide a helper that can derive the simple-PKH lock root from a v1 address + for the common case. +4. Still allow direct lock-root entry, since lock root is the canonical input. +5. Validate destination shape before wallet submission. +6. Warn if amount is below or near the hardcoded minimum. +7. Show the exact Base network the transaction will be submitted to. +8. Make it clear that the user is burning wrapped NOCK on Base to receive + NOCK on Nockchain. +9. Make it clear that the user receives the post-fee amount on Nockchain, not + the full burned amount. + +### 2. Burn Submission Status + +Purpose: +- track the user’s Base burn transaction before bridge-side processing starts + +Show: + +1. wallet-submitted tx hash +2. pending / confirmed status on Base +3. failure/revert if the burn tx fails + +Important: + +1. A successful wallet submission is not the same thing as a successful + withdrawal. +2. After Base confirmation, the DApp can move into a coarse + `withdrawal_pending` state even if it does not have bridge-internal + visibility. + +### 3. Withdrawal Status View + +Purpose: +- show the backend-owned lifecycle once the burn has become a bridge withdrawal + +Show: + +1. withdrawal id +2. burned amount +3. expected received amount if the backend exposes it +4. destination +5. Base burn tx reference +6. current stage +7. timestamps for major stage changes when available +8. delay/support guidance if the withdrawal is taking longer than expected +9. human-friendly references like burn tx hash and timestamps alongside the + internal `withdrawal_id` + +### 4. Withdrawal History + +Purpose: +- let the user see prior withdrawals and their final status + +Show: + +1. recent withdrawals +2. withdrawal id +3. burned amount +4. received amount if available +5. destination +6. created time +7. confirmed time if complete +8. current/final state + +## UX Rules + +### Rule 1: Never imply that backend-internal progress is public truth + +Do not show: +- "confirmed" because peers agreed +- "complete" because a proposal was built +- "done" because the sequencer submitted a tx +- "authorized" or "submitted" unless the product deliberately exposes internal + operator-level states + +Show confirmed only after confirmed settlement has been observed and reconciled. + +### Rule 2: Separate Base burn progress from Nockchain withdrawal progress + +The user performs one Base transaction, but the actual withdrawal completes +later on the bridge/Nockchain side. + +The UI should visually separate: + +1. Base burn submitted/confirmed +2. waiting for bridge completion +3. Nockchain confirmed settlement + +### Rule 3: Public UI should prefer delay messaging over internal hold/stop jargon + +The words `hold` and `stop` are useful for operators, but may not be good +public-user concepts. + +For most users, the better public states are: + +1. pending +2. delayed +3. confirmed + +### Rule 4: Stop is a real operator-visible problem + +If the product exposes an actual user-visible terminal problem: + +1. show the stop reason clearly +2. say bridge processing halted for this withdrawal +3. point the user to support/operator guidance if available + +### Rule 5: Sequencer pause is usually internal/backend state + +Because there is one sequencer-owned in-flight withdrawal at a time, the UI +may need internal/operator visibility for things like: + +1. waiting for sequencer +2. sequencer unavailable +3. withdrawal paused pending sequencer recovery + +But you should usually translate this into a coarse pending/delayed state +rather than expose sequencer internals directly. + +### Rule 6: Delayed withdrawals need explicit support guidance + +For public users: + +1. say that a small delay is expected +2. if no withdrawal has shown up after 24 hours, direct the user to reach out + on Telegram +3. do not imply the user should retry the burn or create a duplicate + withdrawal request on their own + +## Suggested Data Model For The Frontend + +This is not the canonical backend schema. It is a frontend view model. + +```ts +type WithdrawalUiStatus = + | "draft" + | "awaiting_wallet_confirmation" + | "awaiting_base_confirmation" + | "withdrawal_pending" + | "confirmed" + | "delayed"; + +type WithdrawalUiRecord = { + withdrawalId: string; + baseTxHash?: string; + amount: string; + destination: string; + createdAt?: string; + updatedAt?: string; + status: WithdrawalUiStatus; + nockTxName?: string; + nockConfirmationRef?: string; +}; +``` + +The backend may expose more detail, but you should normalize it into a model +like this. + +## API Expectations + +This doc does not define the final backend API, but the DApp probably needs +something close to: + +1. `create burn transaction` or contract-write integration +2. `get withdrawal by id` +3. `list withdrawals for account` +4. `stream or poll withdrawal status updates` +5. `get bridge/sequencer status` for internal/admin tools if needed + +Minimum useful payload for a withdrawal status API: + +1. `withdrawal_id` +2. burned amount +3. expected or actual received amount if available +4. destination +5. base burn tx hash +6. current user-facing status +7. optional machine state +8. optional delay/support hint +9. timestamps +10. optional Nockchain tx name/reference + +One unresolved product/API question is where public Nockchain-side visibility +should come from. The current expectation is that another operator may already +have an API for viewing incoming Nockchain transactions that the DApp can lean +on, because the team is reluctant to expose bridge-operated services publicly. + +## Internal/Admin Sequencer UI Needs + +Because the sequencer is now a separate gRPC service in the canonical design, +an internal/admin surface should assume backend status may come from more than +one subsystem: + +1. bridge runtime +2. kernel-derived withdrawal state +3. sequencer service status + +At minimum, an internal/admin surface should be able to show: + +1. whether the sequencer is healthy +2. whether the withdrawal is waiting on sequencer authorization +3. whether the withdrawal was submitted by the sequencer +4. whether the sequencer has recorded confirmation for the withdrawal + +The DApp does not need direct gRPC access. It just needs a backend/API layer +that exposes sequencer status in a frontend-safe way. + +## Recommended Engineering Order For The DApp + +If you are getting started, this is the right order: + +1. Build a static withdrawal form and status screen. +2. Add wallet/network connection state. +3. Add Base burn submission flow. +4. Add polling/streaming for backend withdrawal status. +5. Implement the UI state machine for: + - awaiting base confirmation + - withdrawal pending + - confirmed + - delayed +6. Add withdrawal history. +7. If needed, add a separate internal/operator status view. +8. Add better delay/support messaging and empty/error states. + +## Non-Goals For The First UI Iteration + +Do not block initial UI work on: + +1. exposing every backend attempt/epoch detail +2. displaying full proposal-envelope internals +3. exposing raw note-selection or reservation data to the user +4. implementing recovery controls in the UI +5. multi-withdrawal batch UX + +The first useful UI should focus on: + +1. initiating withdrawals +2. showing accurate status +3. explaining delays honestly + +## Remaining Open Question + +This still needs a backend/product decision: + +1. What exact status API shape will expose coarse public states vs internal + operator states? + - There may already be an operator-owned API for viewing incoming + Nockchain transactions to be able to flag withdrawals as complete that the DApp can reuse. + - The bridge team is reluctant to expose bridge-operated services publicly, + so this likely should not depend on making the bridge or sequencer + directly public. + +## Bottom Line + +If you remember only a few things, remember these: + +1. The user starts a withdrawal by burning on Base. +2. The burn only becomes a withdrawal if it clears the withdrawal minimum and + is admitted by the bridge. +3. The UI is not the sequencer and not the bridge runtime. +4. The public UI should only show coarse states it can actually know. +5. Burned amount and received amount are not the same thing; the fee is + deducted before the user receives NOCK on Nockchain. +6. Confirmed means confirmed settlement on Nockchain, not just "submitted" or + "peer agreed." +7. The destination should be a lock root, with a helper for deriving the + simple-PKH lock root from a v1 address. +8. Show both the internal `withdrawal_id` and the human-friendly references. +9. A small delay is expected; after 24 hours without a withdrawal showing up, + tell the user to reach out on Telegram. diff --git a/crates/bridge/scripts/create-bridge-spend.sh b/crates/bridge/scripts/create-bridge-spend.sh new file mode 100755 index 000000000..6ba85f988 --- /dev/null +++ b/crates/bridge/scripts/create-bridge-spend.sh @@ -0,0 +1,178 @@ +#!/bin/bash +set -euo pipefail + +# Create and submit a bridge-deposit spend from the local fakenet wallet. +# +# This script is intentionally barebones and hardcoded. +# +# Workflow: +# 1) boot/reset wallet with deterministic seed +# 2) parse list-notes output and select enough notes for amount + fee +# 3) create bridge-deposit tx +# 4) submit tx to the local node + +die() { + echo "Error: $*" >&2 + exit 1 +} + +is_uint() { + [[ "$1" =~ ^[0-9]+$ ]] +} + +run_wallet() { + "$WALLET_SH" --color never "$@" +} + +# Hardcoded transfer parameters and local server. +RECIPIENT="0x15A3DF65662B0235CdF27B3A8dD0f35D41E8A5BE" +AMOUNT="7000000000" +FEE="3457024" +MIN_BRIDGE_DEPOSIT="6553600000" +WALLET_SH="./wallet.sh" + +[[ -x "$WALLET_SH" ]] || die "wallet script not executable: $WALLET_SH" +is_uint "$AMOUNT" || die "amount must be an unsigned integer" +is_uint "$FEE" || die "fee must be an unsigned integer" +is_uint "$MIN_BRIDGE_DEPOSIT" || die "min bridge deposit must be an unsigned integer" + +RECIPIENT_HEX="${RECIPIENT#0x}" +[[ "$RECIPIENT_HEX" =~ ^[0-9a-fA-F]{40}$ ]] || die "recipient must be a 20-byte hex address" +RECIPIENT="0x${RECIPIENT_HEX}" + +if (( AMOUNT < MIN_BRIDGE_DEPOSIT )); then + die "amount ${AMOUNT} is below minimum bridge deposit ${MIN_BRIDGE_DEPOSIT}" +fi + +TOTAL_REQUIRED=$((AMOUNT + FEE)) + +echo "== Bridge Spend Parameters ==" +echo "Recipient: ${RECIPIENT}" +echo "Amount: ${AMOUNT} nicks" +echo "Fee: ${FEE} nicks" +echo "Required: ${TOTAL_REQUIRED} nicks (amount + fee)" +echo "Wallet mode: wallet.sh defaults" +echo "" + +echo "Resetting wallet state via --new..." +run_wallet --new show-balance >/dev/null + +echo "Reading spendable notes..." +LIST_NOTES_OUTPUT="$(run_wallet list-notes)" +printf '%s\n' "$LIST_NOTES_OUTPUT" + +SANITIZED_NOTES="$( + printf '%s\n' "$LIST_NOTES_OUTPUT" \ + | sed -E 's/\x1B\[[0-9;]*[A-Za-z]//g' \ + | tr -cd '\11\12\15\40-\176' +)" + +NOTE_ROWS_RAW="$( + printf '%s\n' "$SANITIZED_NOTES" \ + | awk ' + BEGIN { + collecting_name = 0 + have_name = 0 + name = "" + } + /^- Name: \[/ { + line = $0 + sub(/^- Name: \[/, "", line) + gsub(/^[[:space:]]+|[[:space:]]+$/, "", line) + name = line + if (line ~ /\]$/) { + sub(/\]$/, "", name) + have_name = 1 + collecting_name = 0 + } else { + collecting_name = 1 + } + next + } + collecting_name { + line = $0 + gsub(/^[[:space:]]+|[[:space:]]+$/, "", line) + if (line ~ /\]$/) { + sub(/\]$/, "", line) + name = name " " line + collecting_name = 0 + have_name = 1 + } else { + name = name " " line + } + next + } + /^- Assets \(nicks\): / { + if (!have_name) { + next + } + assets = $0 + sub(/^- Assets \(nicks\): /, "", assets) + gsub(/[^0-9]/, "", assets) + if (assets != "") { + gsub(/[[:space:]]+/, " ", name) + gsub(/^[[:space:]]+|[[:space:]]+$/, "", name) + printf "%s|[%s]\n", assets, name + } + name = "" + have_name = 0 + } + ' \ + | sort -t'|' -k1,1n +)" + +[[ -n "$NOTE_ROWS_RAW" ]] || die "no spendable notes parsed from list-notes output" + +SELECTED_NAMES="" +SELECTED_TOTAL=0 +SELECTED_COUNT=0 + +while IFS= read -r row; do + [[ -n "$row" ]] || continue + assets="${row%%|*}" + note_name="${row#*|}" + is_uint "$assets" || continue + if [[ -n "$SELECTED_NAMES" ]]; then + SELECTED_NAMES+="," + fi + SELECTED_NAMES+="$note_name" + SELECTED_TOTAL=$((SELECTED_TOTAL + assets)) + SELECTED_COUNT=$((SELECTED_COUNT + 1)) + if (( SELECTED_TOTAL >= TOTAL_REQUIRED )); then + break + fi +done <<< "$NOTE_ROWS_RAW" + +if (( SELECTED_TOTAL < TOTAL_REQUIRED )); then + die "failed to select enough notes: selected ${SELECTED_TOTAL}, need ${TOTAL_REQUIRED}" +fi + +echo "Selected ${SELECTED_COUNT} note(s) totaling ${SELECTED_TOTAL} nicks" +echo "Selected names: ${SELECTED_NAMES}" + +RECIPIENT_JSON="$(printf '{"kind":"bridge-deposit","evm-address":"%s","amount":%s}' "$RECIPIENT" "$AMOUNT")" + +echo "Creating bridge-deposit transaction..." +CREATE_OUTPUT="$(run_wallet create-tx --names "$SELECTED_NAMES" --fee "$FEE" --recipient "$RECIPIENT_JSON")" +printf '%s\n' "$CREATE_OUTPUT" + +TX_FILE="$(printf '%s\n' "$CREATE_OUTPUT" | grep -Eo '\./txs/[A-Za-z0-9]+\.tx' | tail -n1 || true)" +if [[ -z "$TX_FILE" ]]; then + TX_NAME="$(printf '%s\n' "$CREATE_OUTPUT" | sed -n 's/^- Name: //p' | head -n1)" + if [[ -n "$TX_NAME" ]]; then + TX_FILE="./txs/${TX_NAME}.tx" + fi +fi +[[ -n "$TX_FILE" ]] || die "failed to parse tx file path from create-tx output" +[[ -f "$TX_FILE" ]] || die "tx file does not exist: $TX_FILE" + +echo "Submitting transaction..." +SEND_OUTPUT="$(run_wallet send-tx "$TX_FILE")" +printf '%s\n' "$SEND_OUTPUT" + +TX_ID="$(printf '%s\n' "$SEND_OUTPUT" | sed -n 's/.*Validation for TX \([A-Za-z0-9]\+\) passed.*/\1/p' | head -n1)" +if [[ -n "$TX_ID" ]]; then + echo "Submitted TX ID: ${TX_ID}" +fi + +echo "Done. Transaction file: ${TX_FILE}" diff --git a/crates/bridge/scripts/environments/.gitignore b/crates/bridge/scripts/environments/.gitignore new file mode 100644 index 000000000..9d626c590 --- /dev/null +++ b/crates/bridge/scripts/environments/.gitignore @@ -0,0 +1,5 @@ +# Ignore actual environment files (contain secrets) +*.env + +# Keep examples +!*.env.example diff --git a/crates/bridge/scripts/environments/README.md b/crates/bridge/scripts/environments/README.md new file mode 100644 index 000000000..ff66aa0e6 --- /dev/null +++ b/crates/bridge/scripts/environments/README.md @@ -0,0 +1,122 @@ +# Bridge Script Environments + +Status: Active +Owner: Nockchain Maintainers +Last Reviewed: 2026-02-20 +Canonical/Legacy: Legacy (profile and variable map for `crates/bridge/scripts/*`) + +Use this guide to select and load environment profiles consumed by bridge helper scripts. + +## Scope + +- In scope: profile files under `scripts/environments`, variable wiring used by script entrypoints. +- Out of scope: contract deployment internals, full operator provisioning, and incident response. + +## Profile Setup + +```bash +cd crates/bridge/scripts + +# Virtual Testnet profile +cp environments/virtual-testnet.env.example environments/virtual-testnet.env +source environments/virtual-testnet.env + +# Base Sepolia profile +cp environments/base-sepolia.env.example environments/base-sepolia.env +source environments/base-sepolia.env +``` + +Load one profile per shell session to keep script behavior deterministic. + +## Profile Files + +| File | Purpose | +| ------------------------------------------- | ---------------------------------------------------------------------- | +| `environments/virtual-testnet.env.example` | Single-node helper defaults for virtual testnet runs | +| `environments/base-sepolia.env.example` | Base Sepolia script profile (expects `BASE_SEPOLIA_*` values in shell) | +| `environments/test-bridge-keys.env.example` | Deterministic local key/address bundle for test workflows | + +Security warning: resolved `.env` files can contain secrets; do not commit them. + +## Variables By Script + +### `run-node-and-bridge.sh` + +Reads: +- `BRIDGE_ENV` +- `BASE_WS_URL` +- `INBOX_CONTRACT_ADDRESS` +- `NOCK_CONTRACT_ADDRESS` +- `BRIDGE_ETH_KEY` +- `BRIDGE_NOCK_KEY` + +### `run-bridge-only.sh` + +Reads: +- `BRIDGE_ENV` +- `BASE_WS_URL` +- `INBOX_CONTRACT_ADDRESS` +- `NOCK_CONTRACT_ADDRESS` +- `BRIDGE_ETH_KEY` +- `BRIDGE_NOCK_KEY` + +The script also requires a bridge ETH address in generated config and currently uses its built-in default when unset. + +### `multi-bridge.sh` + +`virtual-testnet` mode reads: +- `BRIDGE_ENV` (set to `virtual-testnet`) +- `BASE_WS_URL` +- `INBOX_CONTRACT_ADDRESS` +- `NOCK_CONTRACT_ADDRESS` +- `BRIDGE_NODE_KEY_0..4` (optional overrides; script has defaults) + +`base-sepolia` mode reads: +- `BRIDGE_ENV` (set to `base-sepolia`) +- `BASE_SEPOLIA_WS_URL` +- `BASE_SEPOLIA_INBOX_PROXY` +- `BASE_SEPOLIA_NOCK` +- `BASE_SEPOLIA_BRIDGE_NODE_KEY_0..4` +- `BASE_SEPOLIA_BRIDGE_NODE_ADDR_0..4` + +### `fund-bridge-nodes.sh` + +Requires: +- `BASE_SEPOLIA_RPC_URL` +- `BASE_SEPOLIA_DEPLOYER_KEY` + +Optional overrides: +- `BASE_SEPOLIA_BRIDGE_NODE_ADDR_0..4` +- `AMOUNT` (wei sent per node) + +## Base Sepolia Live Reference + +| Contract | Address | +| -------------------- | -------------------------------------------- | +| MessageInbox (Proxy) | `0x9b1becA13c39b9Be10dB616F1bE10C3CeF9Dfb36` | +| Nock Token | `0xA9cd4087D9B050D8B35727AAf810296CA957c7B3` | + +- Basescan: https://sepolia.basescan.org/address/0x9b1becA13c39b9Be10dB616F1bE10C3CeF9Dfb36 +- Deployment detail: [`../../contracts/environments/base-sepolia-testnet-accounts.md`](../../contracts/environments/base-sepolia-testnet-accounts.md) + +## URL Reference + +Virtual testnet: + +```text +wss://virtual.base-sepolia..rpc.tenderly.co/ +``` + +Base Sepolia node RPC: + +```text +wss://base-sepolia.gateway.tenderly.co/ +``` + +Access key source: Tenderly Dashboard -> Node RPCs. + +## Handoff + +- Next for bootstrap flow: [`../../QUICKSTART.md`](../../QUICKSTART.md) +- Next for operator provisioning: [`../../OPERATOR-SETUP.md`](../../OPERATOR-SETUP.md) +- Next for runtime operations/incidents: [`../../docs/node-runbook.md`](../../docs/node-runbook.md) diff --git a/crates/bridge/scripts/environments/base-sepolia.env.example b/crates/bridge/scripts/environments/base-sepolia.env.example new file mode 100644 index 000000000..0e4b44fe8 --- /dev/null +++ b/crates/bridge/scripts/environments/base-sepolia.env.example @@ -0,0 +1,21 @@ +# Real Base Sepolia Environment (unlimited blocks, real network) +# Usage: source environments/base-sepolia.env && ./run-bridge-only.sh +# +# Set the referenced BASE_SEPOLIA_* values in your shell before sourcing. + +# Network identifier (used for deployment tracking) +export BRIDGE_ENV="base-sepolia" + +# Tenderly Node RPC WebSocket URL. +export BASE_WS_URL="${BASE_SEPOLIA_WS_URL}" + +# Contract addresses (deployed 2025-12-17, NOCK ticker) +export INBOX_CONTRACT_ADDRESS="0x9b1becA13c39b9Be10dB616F1bE10C3CeF9Dfb36" +export NOCK_CONTRACT_ADDRESS="0xA9cd4087D9B050D8B35727AAf810296CA957c7B3" + +# Bridge node addresses. +export BRIDGE_NODE_0="${BASE_SEPOLIA_BRIDGE_NODE_ADDR_0}" +export BRIDGE_NODE_1="${BASE_SEPOLIA_BRIDGE_NODE_ADDR_1}" +export BRIDGE_NODE_2="${BASE_SEPOLIA_BRIDGE_NODE_ADDR_2}" +export BRIDGE_NODE_3="${BASE_SEPOLIA_BRIDGE_NODE_ADDR_3}" +export BRIDGE_NODE_4="${BASE_SEPOLIA_BRIDGE_NODE_ADDR_4}" diff --git a/crates/bridge/scripts/environments/test-bridge-keys.env.example b/crates/bridge/scripts/environments/test-bridge-keys.env.example new file mode 100644 index 000000000..4458bfb35 --- /dev/null +++ b/crates/bridge/scripts/environments/test-bridge-keys.env.example @@ -0,0 +1,43 @@ +# Test bridge node keys + derived addresses (LOCAL DEV ONLY) +# +# Copy to `test-bridge-keys.env` to enable (recommended): +# cp environments/test-bridge-keys.env.example environments/test-bridge-keys.env +# +# Then run bridge scripts normally; they will prefer `test-bridge-keys.env` if present. +# +# NOTE: These are deterministic development keys used for local testing and E2E +# pipelines. Do NOT use them for anything real. + +# Ethereum private keys (bridge nodes 0-4) +export BRIDGE_ETH_KEY_0="0x4c0883a69102937d6231471b5dbb6204fe5129617082792ae468d01a3f362318" +export BRIDGE_ETH_KEY_1="0x5c0883a69102937d6231471b5dbb6204fe5129617082792ae468d01a3f362319" +export BRIDGE_ETH_KEY_2="0x6c0883a69102937d6231471b5dbb6204fe5129617082792ae468d01a3f36231a" +export BRIDGE_ETH_KEY_3="0x7c0883a69102937d6231471b5dbb6204fe5129617082792ae468d01a3f36231b" +export BRIDGE_ETH_KEY_4="0x8c0883a69102937d6231471b5dbb6204fe5129617082792ae468d01a3f36231c" + +# Derived Ethereum addresses (used by MessageInbox.bridgeNodes) +export BRIDGE_NODE_0="0x2c7536E3605D9C16a7a3D7b1898e529396a65c23" +export BRIDGE_NODE_1="0x0EE156f080d9cB3BaA3C0DB53D07f13D69CEf4C9" +export BRIDGE_NODE_2="0x274BD645de480C325D618c60c661F11275eB77F1" +export BRIDGE_NODE_3="0x6dc59eb20f7928935c47A391e35545a2CEC51013" +export BRIDGE_NODE_4="0xcaB10dA05fC0aDBb7e91Eadc30f224bcDF601375" + +# Nockchain schnorr private keys (bridge nodes 0-4) (local test defaults) +export BRIDGE_NOCK_KEY_0="5KZuFKrctV5iUburT54Z9fhpf3V3hv2sPf9GRQnjFR8T" +export BRIDGE_NOCK_KEY_1="5KZuFKrctV5iUburT54Z9fhpf3V3hv2sPf9GRQnjFR8U" +export BRIDGE_NOCK_KEY_2="5KZuFKrctV5iUburT54Z9fhpf3V3hv2sPf9GRQnjFR8V" +export BRIDGE_NOCK_KEY_3="5KZuFKrctV5iUburT54Z9fhpf3V3hv2sPf9GRQnjFR8W" +export BRIDGE_NOCK_KEY_4="5KZuFKrctV5iUburT54Z9fhpf3V3hv2sPf9GRQnjFR8X" + +# Optional sanity check (requires Foundry's `cast`) +if command -v cast >/dev/null 2>&1; then + for i in 0 1 2 3 4; do + key_var="BRIDGE_ETH_KEY_${i}" + addr_var="BRIDGE_NODE_${i}" + derived=$(cast wallet address --private-key "${!key_var}" 2>/dev/null || true) + if [ -n "$derived" ] && [ "${derived}" != "${!addr_var}" ]; then + echo "Error: ${key_var} derives ${derived} but ${addr_var} is ${!addr_var}" >&2 + return 1 2>/dev/null || exit 1 + fi + done +fi diff --git a/crates/bridge/scripts/environments/virtual-testnet.env.example b/crates/bridge/scripts/environments/virtual-testnet.env.example new file mode 100644 index 000000000..b44be40cc --- /dev/null +++ b/crates/bridge/scripts/environments/virtual-testnet.env.example @@ -0,0 +1,18 @@ +# Virtual Testnet Environment (50 block limit, state manipulation available) +# Copy to virtual-testnet.env and fill in values +# Usage: source environments/virtual-testnet.env && ./run-bridge-only.sh + +# Network identifier (used for deployment tracking) +export BRIDGE_ENV="virtual-testnet" + +# Tenderly Virtual Testnet WebSocket URL +# Get from: Tenderly Dashboard -> Virtual Testnets -> Your Testnet -> RPC +export BASE_WS_URL="wss://virtual.base-sepolia.us-east.rpc.tenderly.co/YOUR_VIRTUAL_TESTNET_ID" + +# Contract addresses (deployed to your Virtual Testnet) +export INBOX_CONTRACT_ADDRESS="0x68487BbaE6210a7A5d92c1D3D620e7AC9f7Ae29E" +export NOCK_CONTRACT_ADDRESS="0x9f5D8976f0F89eC92241850A585B313F31Ab8b62" + +# Bridge node keys (for testing) +export BRIDGE_ETH_KEY="0x4c0883a69102937d6231471b5dbb6204fe5129617082792ae468d01a3f362318" +export BRIDGE_NOCK_KEY="5KZuFKrctV5iUburT54Z9fhpf3V3hv2sPf9GRQnjFR8T" diff --git a/crates/bridge/scripts/fund-bridge-nodes.sh b/crates/bridge/scripts/fund-bridge-nodes.sh new file mode 100755 index 000000000..6f6c427dd --- /dev/null +++ b/crates/bridge/scripts/fund-bridge-nodes.sh @@ -0,0 +1,81 @@ +#!/bin/bash +# Fund bridge nodes on Base Sepolia from the deployer account +set -e + +# Required env vars +: "${BASE_SEPOLIA_RPC_URL:?BASE_SEPOLIA_RPC_URL not set}" +: "${BASE_SEPOLIA_DEPLOYER_KEY:?BASE_SEPOLIA_DEPLOYER_KEY not set}" + +# Bridge node addresses. +ADDRS=( + "${BASE_SEPOLIA_BRIDGE_NODE_ADDR_0:?BASE_SEPOLIA_BRIDGE_NODE_ADDR_0 not set}" + "${BASE_SEPOLIA_BRIDGE_NODE_ADDR_1:?BASE_SEPOLIA_BRIDGE_NODE_ADDR_1 not set}" + "${BASE_SEPOLIA_BRIDGE_NODE_ADDR_2:?BASE_SEPOLIA_BRIDGE_NODE_ADDR_2 not set}" + "${BASE_SEPOLIA_BRIDGE_NODE_ADDR_3:?BASE_SEPOLIA_BRIDGE_NODE_ADDR_3 not set}" + "${BASE_SEPOLIA_BRIDGE_NODE_ADDR_4:?BASE_SEPOLIA_BRIDGE_NODE_ADDR_4 not set}" +) + +# Amount per node (in wei) - 0.0005 ETH = 500000000000000 wei +AMOUNT="${AMOUNT:-500000000000000}" +AMOUNT_ETH=$(echo "scale=6; $AMOUNT / 1000000000000000000" | bc) + +echo "=== Base Sepolia Bridge Node Funder ===" +echo "" + +# Check deployer balance +DEPLOYER_ADDR=$(cast wallet address --private-key "$BASE_SEPOLIA_DEPLOYER_KEY") +DEPLOYER_BAL=$(cast balance "$DEPLOYER_ADDR" --rpc-url "$BASE_SEPOLIA_RPC_URL") +DEPLOYER_BAL_ETH=$(echo "scale=6; $DEPLOYER_BAL / 1000000000000000000" | bc) + +echo "Deployer: $DEPLOYER_ADDR" +echo "Balance: $DEPLOYER_BAL wei ($DEPLOYER_BAL_ETH ETH)" +echo "" + +TOTAL_NEEDED=$((AMOUNT * 5)) +if [ "$DEPLOYER_BAL" -lt "$TOTAL_NEEDED" ]; then + echo "WARNING: Deployer balance ($DEPLOYER_BAL) < total needed ($TOTAL_NEEDED)" + echo "Will send what we can..." +fi + +echo "Sending $AMOUNT wei ($AMOUNT_ETH ETH) to each node..." +echo "" + +for i in "${!ADDRS[@]}"; do + addr="${ADDRS[$i]}" + echo "[$((i+1))/5] Funding node $i: $addr" + + # Check current balance + bal=$(cast balance "$addr" --rpc-url "$BASE_SEPOLIA_RPC_URL") + echo " Current balance: $bal wei" + + # Send funds + tx=$(cast send "$addr" --value "$AMOUNT" \ + --private-key "$BASE_SEPOLIA_DEPLOYER_KEY" \ + --rpc-url "$BASE_SEPOLIA_RPC_URL" \ + --json 2>&1) || { + echo " ERROR: Failed to send" + echo " $tx" + continue + } + + txhash=$(echo "$tx" | jq -r '.transactionHash') + echo " Sent: $txhash" + + # Check new balance + new_bal=$(cast balance "$addr" --rpc-url "$BASE_SEPOLIA_RPC_URL") + echo " New balance: $new_bal wei" + echo "" +done + +echo "=== Final Balances ===" +for i in "${!ADDRS[@]}"; do + addr="${ADDRS[$i]}" + bal=$(cast balance "$addr" --rpc-url "$BASE_SEPOLIA_RPC_URL") + bal_eth=$(echo "scale=6; $bal / 1000000000000000000" | bc) + echo "Node $i ($addr): $bal wei ($bal_eth ETH)" +done + +echo "" +DEPLOYER_BAL=$(cast balance "$DEPLOYER_ADDR" --rpc-url "$BASE_SEPOLIA_RPC_URL") +DEPLOYER_BAL_ETH=$(echo "scale=6; $DEPLOYER_BAL / 1000000000000000000" | bc) +echo "Deployer ($DEPLOYER_ADDR): $DEPLOYER_BAL wei ($DEPLOYER_BAL_ETH ETH)" diff --git a/crates/bridge/scripts/lib/layout.sh b/crates/bridge/scripts/lib/layout.sh new file mode 100644 index 000000000..8b3b4385d --- /dev/null +++ b/crates/bridge/scripts/lib/layout.sh @@ -0,0 +1,27 @@ +#!/usr/bin/env bash + +# Shared path discovery for bridge helper scripts. +# +# The monorepo layout is: +# /open/crates/bridge +# +# The public repository layout is: +# /crates/bridge +# +# Keep all script entrypoints in terms of these resolved paths instead of +# spelling either layout directly. + +bridge_resolve_layout() { + BRIDGE_SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[1]}")" && pwd)" + BRIDGE_DIR="$(cd "$BRIDGE_SCRIPT_DIR/.." && pwd)" + BRIDGE_CRATES_DIR="$(cd "$BRIDGE_DIR/.." && pwd)" + BRIDGE_SOURCE_ROOT="$(cd "$BRIDGE_CRATES_DIR/.." && pwd)" + + if [[ "$(basename "$BRIDGE_SOURCE_ROOT")" == "open" && -f "$(dirname "$BRIDGE_SOURCE_ROOT")/Cargo.toml" ]]; then + BRIDGE_WORKSPACE_ROOT="$(cd "$BRIDGE_SOURCE_ROOT/.." && pwd)" + else + BRIDGE_WORKSPACE_ROOT="$BRIDGE_SOURCE_ROOT" + fi + + BRIDGE_BIN_DIR="$BRIDGE_WORKSPACE_ROOT/target/release" +} diff --git a/crates/bridge/scripts/multi-bridge.sh b/crates/bridge/scripts/multi-bridge.sh new file mode 100755 index 000000000..4ff77594d --- /dev/null +++ b/crates/bridge/scripts/multi-bridge.sh @@ -0,0 +1,448 @@ +#!/bin/bash +set -e + +# Spawn 5 bridge nodes in zellij panes for proposal signing tests +# Usage: ./multi-bridge.sh [--new] [--start] [--base-start-height N] [--nockchain-start-height N] +# [--nonce-epoch-base N] [--nonce-epoch-start-height N] +# [--nonce-epoch-start-tx-id BASE58] +# +# Options: +# --new Start with fresh bridge state for all nodes +# --start Send a %start poke to clear kernel stop state +# --base-start-height N Override Base chain start height (default: 33387036) +# --nockchain-start-height N Override Nockchain start height (default: 1) +# --nonce-epoch-base N Override nonce epoch base (optional) +# --nonce-epoch-start-height N +# Override nonce epoch start height (optional) +# --nonce-epoch-start-tx-id BASE58 +# Override nonce epoch start tx id (base58, optional) +# +# Prerequisites: +# - zellij installed +# - Bridge binary built: cargo build --release -p bridge +# - Node running (or use run-node-only.sh first) +# - Environment sourced (optional): source environments/virtual-testnet.generated.env +# +# Each bridge gets: +# - Unique node_id (0-4) +# - Unique ingress port (8002-8006) +# - Unique data directory + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +# shellcheck source=lib/layout.sh +source "$SCRIPT_DIR/lib/layout.sh" +bridge_resolve_layout + +BIN_DIR="$BRIDGE_BIN_DIR" +TEST_DATA_DIR="${BRIDGE_DIR}/test_run_data" + +NODE_PRIVATE_GRPC_PORT="${NODE_PRIVATE_GRPC_PORT:-5002}" + +to_ws_url() { + local url="$1" + url="${url/https:\/\//wss://}" + url="${url/http:\/\//ws://}" + echo "$url" +} + +# Environment configuration +# Supports two environments: +# - virtual-testnet: Tenderly VNet (default, uses TENDERLY_* env vars) +# - base-sepolia: Real Base Sepolia testnet (uses BASE_SEPOLIA_* env vars) +BRIDGE_ENV="${BRIDGE_ENV:-virtual-testnet}" +AUTO_LOADED_ENV_FILE="" + +# In virtual-testnet mode, auto-load generated env from tenderly-vnet-deploy.sh +# when no explicit BASE_WS_URL is provided. +if [ "$BRIDGE_ENV" = "virtual-testnet" ] && [ -z "${BASE_WS_URL:-}" ]; then + GENERATED_ENV_FILE="${SCRIPT_DIR}/environments/virtual-testnet.generated.env" + if [ -f "$GENERATED_ENV_FILE" ]; then + # shellcheck disable=SC1090 + source "$GENERATED_ENV_FILE" + AUTO_LOADED_ENV_FILE="$GENERATED_ENV_FILE" + fi +fi + +# If only BASE_RPC_URL is present, derive WS URL from it. +if [ -z "${BASE_WS_URL:-}" ] && [ -n "${BASE_RPC_URL:-}" ]; then + BASE_WS_URL="$(to_ws_url "$BASE_RPC_URL")" +fi + +# Guard against accidental HTTP URL being passed as BASE_WS_URL. +case "${BASE_WS_URL:-}" in + http://*|https://*) + echo "WARN: BASE_WS_URL uses HTTP scheme; converting to websocket URL" + BASE_WS_URL="$(to_ws_url "$BASE_WS_URL")" + ;; +esac + +if [ "$BRIDGE_ENV" = "base-sepolia" ]; then + : "${BASE_SEPOLIA_WS_URL:?BASE_SEPOLIA_WS_URL must be set for base-sepolia mode.}" + : "${BASE_SEPOLIA_INBOX_PROXY:?BASE_SEPOLIA_INBOX_PROXY must be set for base-sepolia mode.}" + : "${BASE_SEPOLIA_NOCK:?BASE_SEPOLIA_NOCK must be set for base-sepolia mode.}" + BASE_WS_URL="$BASE_SEPOLIA_WS_URL" + INBOX_CONTRACT_ADDRESS="$BASE_SEPOLIA_INBOX_PROXY" + NOCK_CONTRACT_ADDRESS="$BASE_SEPOLIA_NOCK" + + BRIDGE_ETH_KEYS=( + "${BASE_SEPOLIA_BRIDGE_NODE_KEY_0:?BASE_SEPOLIA_BRIDGE_NODE_KEY_0 must be set for base-sepolia mode.}" + "${BASE_SEPOLIA_BRIDGE_NODE_KEY_1:?BASE_SEPOLIA_BRIDGE_NODE_KEY_1 must be set for base-sepolia mode.}" + "${BASE_SEPOLIA_BRIDGE_NODE_KEY_2:?BASE_SEPOLIA_BRIDGE_NODE_KEY_2 must be set for base-sepolia mode.}" + "${BASE_SEPOLIA_BRIDGE_NODE_KEY_3:?BASE_SEPOLIA_BRIDGE_NODE_KEY_3 must be set for base-sepolia mode.}" + "${BASE_SEPOLIA_BRIDGE_NODE_KEY_4:?BASE_SEPOLIA_BRIDGE_NODE_KEY_4 must be set for base-sepolia mode.}" + ) + + # Nockchain keys - same for both environments (fakenet keys) + BRIDGE_NOCK_KEYS=( + "5KZuFKrctV5iUburT54Z9fhpf3V3hv2sPf9GRQnjFR8T" # node 0 + "5KZuFKrctV5iUburT54Z9fhpf3V3hv2sPf9GRQnjFR8U" # node 1 + "5KZuFKrctV5iUburT54Z9fhpf3V3hv2sPf9GRQnjFR8V" # node 2 + "5KZuFKrctV5iUburT54Z9fhpf3V3hv2sPf9GRQnjFR8W" # node 3 + "5KZuFKrctV5iUburT54Z9fhpf3V3hv2sPf9GRQnjFR8X" # node 4 + ) + + # Default start height for Base Sepolia (set to recent block to avoid long sync) + # Current block ~35M as of Dec 2026 + BASE_START_HEIGHT="${BASE_START_HEIGHT:-40982896}" +else + # Tenderly Virtual Testnet (default) + : "${BASE_WS_URL:?BASE_WS_URL must be set; source scripts/environments/virtual-testnet.generated.env or an environment profile.}" + : "${INBOX_CONTRACT_ADDRESS:?INBOX_CONTRACT_ADDRESS must be set.}" + : "${NOCK_CONTRACT_ADDRESS:?NOCK_CONTRACT_ADDRESS must be set.}" + + # Keys for Tenderly VNet (deterministic test keys) + BRIDGE_ETH_KEYS=( + "${BRIDGE_NODE_KEY_0:-0x4c0883a69102937d6231471b5dbb6204fe5129617082792ae468d01a3f362318}" + "${BRIDGE_NODE_KEY_1:-0x5c0883a69102937d6231471b5dbb6204fe5129617082792ae468d01a3f362319}" + "${BRIDGE_NODE_KEY_2:-0x6c0883a69102937d6231471b5dbb6204fe5129617082792ae468d01a3f36231a}" + "${BRIDGE_NODE_KEY_3:-0x7c0883a69102937d6231471b5dbb6204fe5129617082792ae468d01a3f36231b}" + "${BRIDGE_NODE_KEY_4:-0x8c0883a69102937d6231471b5dbb6204fe5129617082792ae468d01a3f36231c}" + ) + + # Nockchain keys (fakenet keys) + BRIDGE_NOCK_KEYS=( + "5KZuFKrctV5iUburT54Z9fhpf3V3hv2sPf9GRQnjFR8T" # node 0 + "5KZuFKrctV5iUburT54Z9fhpf3V3hv2sPf9GRQnjFR8U" # node 1 + "5KZuFKrctV5iUburT54Z9fhpf3V3hv2sPf9GRQnjFR8V" # node 2 + "5KZuFKrctV5iUburT54Z9fhpf3V3hv2sPf9GRQnjFR8W" # node 3 + "5KZuFKrctV5iUburT54Z9fhpf3V3hv2sPf9GRQnjFR8X" # node 4 + ) + + # Default start height for Tenderly VNet + BASE_START_HEIGHT="${BASE_START_HEIGHT:-36417335}" +fi + +# Ingress ports for each bridge +INGRESS_PORTS=(8002 8003 8004 8005 8006) + +# Configurable start heights (can be overridden via CLI) +BASE_START_HEIGHT="${BASE_START_HEIGHT:-40982896}" +NOCKCHAIN_START_HEIGHT="${NOCKCHAIN_START_HEIGHT:-1}" + +# Driver-side finality configuration (confirmation depths) +BASE_CONFIRMATION_DEPTH="${BASE_CONFIRMATION_DEPTH:-1}" +NOCKCHAIN_CONFIRMATION_DEPTH="${NOCKCHAIN_CONFIRMATION_DEPTH:-1}" +NONCE_EPOCH_BASE="${NONCE_EPOCH_BASE:-}" +NONCE_EPOCH_START_HEIGHT="${NONCE_EPOCH_START_HEIGHT:-}" +NONCE_EPOCH_START_TX_ID_BASE58="${NONCE_EPOCH_START_TX_ID_BASE58:-}" + +NEW_FLAG="" +START_FLAG="" +while [[ $# -gt 0 ]]; do + case "$1" in + --new) + NEW_FLAG="--new" + echo "Cleaning up old bridge data..." + for i in {0..4}; do + rm -rf "${TEST_DATA_DIR}/bridge-${i}" + done + shift + ;; + --start) + START_FLAG="--start" + shift + ;; + --base-start-height) + BASE_START_HEIGHT="$2" + shift 2 + ;; + --nockchain-start-height) + NOCKCHAIN_START_HEIGHT="$2" + shift 2 + ;; + --nonce-epoch-base) + NONCE_EPOCH_BASE="$2" + shift 2 + ;; + --nonce-epoch-start-height) + NONCE_EPOCH_START_HEIGHT="$2" + shift 2 + ;; + --nonce-epoch-start-tx-id) + NONCE_EPOCH_START_TX_ID_BASE58="$2" + shift 2 + ;; + *) + echo "Unknown option: $1" + echo "Usage: ./multi-bridge.sh [--new] [--start] [--base-start-height N] [--nockchain-start-height N] [--nonce-epoch-base N] [--nonce-epoch-start-height N] [--nonce-epoch-start-tx-id BASE58]" + exit 1 + ;; + esac +done + +# Check prerequisites +if [ ! -f "$BIN_DIR/bridge" ]; then + echo "Error: bridge binary not found. Run: cargo build --release -p bridge" + exit 1 +fi + +if ! command -v zellij &> /dev/null; then + echo "Error: zellij not found. Install with: cargo install zellij" + exit 1 +fi + +echo "============================================" +echo "Spawning 5 Bridge Nodes in Zellij" +echo "Environment: $BRIDGE_ENV" +if [ -n "$AUTO_LOADED_ENV_FILE" ]; then + echo "Loaded env: $AUTO_LOADED_ENV_FILE" +fi +echo "Base WS URL: ${BASE_WS_URL:0:60}..." +echo "Inbox: $INBOX_CONTRACT_ADDRESS" +echo "Nock: $NOCK_CONTRACT_ADDRESS" +echo "Base Start: $BASE_START_HEIGHT" +echo "Nock Start: $NOCKCHAIN_START_HEIGHT" +if [ -n "$NONCE_EPOCH_BASE" ]; then + echo "Epoch Base: $NONCE_EPOCH_BASE" +fi +if [ -n "$NONCE_EPOCH_START_HEIGHT" ]; then + echo "Epoch Start: $NONCE_EPOCH_START_HEIGHT" +fi +if [ -n "$NONCE_EPOCH_START_TX_ID_BASE58" ]; then + echo "Epoch TxId: $NONCE_EPOCH_START_TX_ID_BASE58" +fi +echo "============================================" +echo "" + +# ETH addresses for each environment (derived from keys) +if [ "$BRIDGE_ENV" = "base-sepolia" ]; then + BRIDGE_ETH_ADDRS=( + "${BASE_SEPOLIA_BRIDGE_NODE_ADDR_0:?BASE_SEPOLIA_BRIDGE_NODE_ADDR_0 must be set for base-sepolia mode.}" + "${BASE_SEPOLIA_BRIDGE_NODE_ADDR_1:?BASE_SEPOLIA_BRIDGE_NODE_ADDR_1 must be set for base-sepolia mode.}" + "${BASE_SEPOLIA_BRIDGE_NODE_ADDR_2:?BASE_SEPOLIA_BRIDGE_NODE_ADDR_2 must be set for base-sepolia mode.}" + "${BASE_SEPOLIA_BRIDGE_NODE_ADDR_3:?BASE_SEPOLIA_BRIDGE_NODE_ADDR_3 must be set for base-sepolia mode.}" + "${BASE_SEPOLIA_BRIDGE_NODE_ADDR_4:?BASE_SEPOLIA_BRIDGE_NODE_ADDR_4 must be set for base-sepolia mode.}" + ) +else + # Tenderly VNet addresses (derived from deterministic test keys) + BRIDGE_ETH_ADDRS=( + "0x2c7536E3605D9C16a7a3D7b1898e529396a65c23" + "0x0EE156f080d9cB3BaA3C0DB53D07f13D69CEf4C9" + "0x274BD645de480C325D618c60c661F11275eB77F1" + "0x6dc59eb20f7928935c47A391e35545a2CEC51013" + "0xcaB10dA05fC0aDBb7e91Eadc30f224bcDF601375" + ) +fi + +# Nock PKHs (public key hashes, ~52 chars base58) +# Fake test PKHs (valid format placeholders for local testing) +BRIDGE_NOCK_PKHS=( + "2222222222222222222222222222222222222222222222222222" # test node 0 + "3333333333333333333333333333333333333333333333333333" # test node 1 + "4444444444444444444444444444444444444444444444444444" # test node 2 + "5555555555555555555555555555555555555555555555555555" # test node 3 + "6666666666666666666666666666666666666666666666666666" # test node 4 +) + +# Generate config for each bridge +generate_bridge_config() { + local node_id=$1 + local ingress_port=$2 + local data_dir=$3 + local eth_key=$4 + local nock_key=$5 + + mkdir -p "$data_dir" + local config_file="${data_dir}/bridge-conf.toml" + + cat > "$config_file" << EOF +node_id = ${node_id} +# Environment: ${BRIDGE_ENV} +base_ws_url = "${BASE_WS_URL}" +inbox_contract_address = "${INBOX_CONTRACT_ADDRESS}" +nock_contract_address = "${NOCK_CONTRACT_ADDRESS}" +my_eth_key = "${eth_key}" +my_nock_key = "${nock_key}" +grpc_address = "http://127.0.0.1:${NODE_PRIVATE_GRPC_PORT}" +base_confirmation_depth = ${BASE_CONFIRMATION_DEPTH} +nockchain_confirmation_depth = ${NOCKCHAIN_CONFIRMATION_DEPTH} +ingress_listen_address = "127.0.0.1:${ingress_port}" + +[[nodes]] +ip = "127.0.0.1:8002" +eth_pubkey = "${BRIDGE_ETH_ADDRS[0]}" +nock_pkh = "${BRIDGE_NOCK_PKHS[0]}" + +[[nodes]] +ip = "127.0.0.1:8003" +eth_pubkey = "${BRIDGE_ETH_ADDRS[1]}" +nock_pkh = "${BRIDGE_NOCK_PKHS[1]}" + +[[nodes]] +ip = "127.0.0.1:8004" +eth_pubkey = "${BRIDGE_ETH_ADDRS[2]}" +nock_pkh = "${BRIDGE_NOCK_PKHS[2]}" + +[[nodes]] +ip = "127.0.0.1:8005" +eth_pubkey = "${BRIDGE_ETH_ADDRS[3]}" +nock_pkh = "${BRIDGE_NOCK_PKHS[3]}" + +[[nodes]] +ip = "127.0.0.1:8006" +eth_pubkey = "${BRIDGE_ETH_ADDRS[4]}" +nock_pkh = "${BRIDGE_NOCK_PKHS[4]}" + +# Bridge constants for local testing +[constants] +min_signers = 3 +total_signers = 5 +minimum_event_nocks = 1000 # Lower for testing (prod: 1_000_000) +nicks_fee_per_nock = 195 +base_blocks_chunk = 1 +base_start_height = ${BASE_START_HEIGHT} +nockchain_start_height = ${NOCKCHAIN_START_HEIGHT} +EOF + + if [ -n "$NONCE_EPOCH_BASE" ]; then + cat >> "$config_file" << EOF +nonce_epoch_base = ${NONCE_EPOCH_BASE} +EOF + fi + if [ -n "$NONCE_EPOCH_START_HEIGHT" ]; then + cat >> "$config_file" << EOF +nonce_epoch_start_height = ${NONCE_EPOCH_START_HEIGHT} +EOF + fi + if [ -n "$NONCE_EPOCH_START_TX_ID_BASE58" ]; then + cat >> "$config_file" << EOF +nonce_epoch_start_tx_id_base58 = "${NONCE_EPOCH_START_TX_ID_BASE58}" +EOF + fi + + echo "$config_file" +} + +# Generate all configs first +echo "Generating bridge configs..." +CONFIG_FILES=() +for i in {0..4}; do + data_dir="${TEST_DATA_DIR}/bridge-${i}" + config_file=$(generate_bridge_config $i ${INGRESS_PORTS[$i]} "$data_dir" "${BRIDGE_ETH_KEYS[$i]}" "${BRIDGE_NOCK_KEYS[$i]}") + CONFIG_FILES+=("$config_file") + echo " Bridge $i: $config_file (port ${INGRESS_PORTS[$i]})" +done +echo "" + +# Create a temporary script for each bridge that zellij will run +create_bridge_runner() { + local node_id=$1 + local config_file=$2 + local data_dir=$3 + local runner_script="${data_dir}/run.sh" + + cat > "$runner_script" << EOF +#!/bin/bash + +echo "============================================" +echo "Bridge Node $node_id" +echo "Config: $config_file" +echo "Data: $data_dir" +echo "============================================" +echo "" + +export RUST_LOG=debug,h2=warn,hyper=warn,tower=warn,tonic=info +exec "$BIN_DIR/bridge" \\ + $NEW_FLAG \\ + $START_FLAG \\ + --config-path "$config_file" \\ + --data-dir "$data_dir" +EOF + chmod +x "$runner_script" + echo "$runner_script" +} + +# Create runner scripts +echo "Creating runner scripts..." +RUNNER_SCRIPTS=() +for i in {0..4}; do + data_dir="${TEST_DATA_DIR}/bridge-${i}" + runner=$(create_bridge_runner $i "${CONFIG_FILES[$i]}" "$data_dir") + RUNNER_SCRIPTS+=("$runner") +done +echo "" + +# Create zellij layout file +LAYOUT_FILE="${TEST_DATA_DIR}/bridges.kdl" +cat > "$LAYOUT_FILE" << 'EOF' +layout { + pane split_direction="vertical" { + pane split_direction="horizontal" { + pane { + name "Bridge 0" + command "bash" + args "-c" "RUNNER_SCRIPT_0" + } + pane { + name "Bridge 1" + command "bash" + args "-c" "RUNNER_SCRIPT_1" + } + } + pane split_direction="horizontal" { + pane { + name "Bridge 2" + command "bash" + args "-c" "RUNNER_SCRIPT_2" + } + pane { + name "Bridge 3" + command "bash" + args "-c" "RUNNER_SCRIPT_3" + } + pane { + name "Bridge 4" + command "bash" + args "-c" "RUNNER_SCRIPT_4" + } + } + } +} +EOF + +# Replace placeholders with actual script paths +for i in {0..4}; do + sed -i.bak "s|RUNNER_SCRIPT_${i}|${RUNNER_SCRIPTS[$i]}|g" "$LAYOUT_FILE" +done +rm -f "${LAYOUT_FILE}.bak" + +echo "Starting zellij with 5 bridge panes..." +echo "Layout file: $LAYOUT_FILE" +echo "" +echo "============================================" +echo "Bridge Ports:" +for i in {0..4}; do + echo " Bridge $i: http://127.0.0.1:${INGRESS_PORTS[$i]}" +done +echo "" +echo "All bridges connect to node gRPC: http://127.0.0.1:$NODE_PRIVATE_GRPC_PORT" +echo "============================================" +echo "" +echo "TUI (separate terminal, pick any bridge):" +echo " $BIN_DIR/nockchain-bridge-tui --server \"http://127.0.0.1:${INGRESS_PORTS[0]}\"" +echo "" +echo "Press Ctrl+Q to exit zellij (all bridges will stop)" +echo "" + +# Start zellij with the layout +exec zellij --layout "$LAYOUT_FILE" diff --git a/crates/bridge/scripts/run-bridge-only.sh b/crates/bridge/scripts/run-bridge-only.sh new file mode 100755 index 000000000..04d596881 --- /dev/null +++ b/crates/bridge/scripts/run-bridge-only.sh @@ -0,0 +1,208 @@ +#!/bin/bash +set -e + +# Run just the bridge (assumes node is already running) +# Usage: ./run-bridge-only.sh [--new] [--base-start-height N] [--nockchain-start-height N] +# +# Options: +# --new Start with fresh bridge state +# --base-start-height N Override Base chain start height (default: 33387036) +# --nockchain-start-height N Override Nockchain start height (default: 1) +# +# Environment setup: +# source environments/virtual-testnet.env # For Virtual Testnet (50 block limit) +# source environments/base-sepolia.env # For real Base Sepolia (unlimited) +# +# Before you start, run `make install` and `make deps` in the bridge contracts +# directory. Also make sure your wallet, bridge, and nockchain binaries are up to date. +# +# To run the TUI client, use: +# $BIN_DIR/nockchain-bridge-tui --server "http://$BRIDGE_INGRESS" + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +# shellcheck source=lib/layout.sh +source "$SCRIPT_DIR/lib/layout.sh" +bridge_resolve_layout + +BIN_DIR="$BRIDGE_BIN_DIR" +TEST_DATA_DIR="${BRIDGE_DIR}/test_run_data" +BRIDGE_DATA_DIR="${TEST_DATA_DIR}/bridge" + +NODE_PRIVATE_GRPC_PORT="${NODE_PRIVATE_GRPC_PORT:-5002}" +BRIDGE_INGRESS="127.0.0.1:8002" + +# Environment configuration (set via environment variables or defaults to virtual testnet) +BRIDGE_ENV="${BRIDGE_ENV:-virtual-testnet}" +AUTO_LOADED_ENV_FILE="" +if [ "$BRIDGE_ENV" = "virtual-testnet" ] && [ -z "${BASE_WS_URL:-}" ]; then + GENERATED_ENV_FILE="${SCRIPT_DIR}/environments/virtual-testnet.generated.env" + if [ -f "$GENERATED_ENV_FILE" ]; then + # shellcheck disable=SC1090 + source "$GENERATED_ENV_FILE" + AUTO_LOADED_ENV_FILE="$GENERATED_ENV_FILE" + fi +fi +: "${BASE_WS_URL:?BASE_WS_URL must be set; source scripts/environments/virtual-testnet.generated.env or an environment profile.}" +: "${INBOX_CONTRACT_ADDRESS:?INBOX_CONTRACT_ADDRESS must be set.}" +: "${NOCK_CONTRACT_ADDRESS:?NOCK_CONTRACT_ADDRESS must be set.}" +BRIDGE_ETH_KEY="${BRIDGE_ETH_KEY:-0x4c0883a69102937d6231471b5dbb6204fe5129617082792ae468d01a3f362318}" +BRIDGE_ETH_ADDR="${BRIDGE_ETH_ADDR:-0x2c7536E3605D9C16a7a3D7b1898e529396a65c23}" +BRIDGE_NOCK_KEY="${BRIDGE_NOCK_KEY:-5KZuFKrctV5iUburT54Z9fhpf3V3hv2sPf9GRQnjFR8T}" + +# Checkpoint save interval (ms). Lower values make restarts pick up faster. +BRIDGE_SAVE_INTERVAL_MS="${BRIDGE_SAVE_INTERVAL_MS:-5000}" + +# Configurable start heights (can be overridden via CLI) +BASE_START_HEIGHT="${BASE_START_HEIGHT:-36417335}" +NOCKCHAIN_START_HEIGHT="${NOCKCHAIN_START_HEIGHT:-1}" + +# Driver-side finality configuration (confirmation depths) +BASE_CONFIRMATION_DEPTH="${BASE_CONFIRMATION_DEPTH:-100}" +NOCKCHAIN_CONFIRMATION_DEPTH="${NOCKCHAIN_CONFIRMATION_DEPTH:-1}" + +BRIDGE_ETH_ADDR="${BRIDGE_ETH_ADDR:-}" +if [ -z "$BRIDGE_ETH_ADDR" ]; then + echo "Error: BRIDGE_ETH_ADDR must be set (the Ethereum address for BRIDGE_ETH_KEY)" >&2 + exit 1 +fi + +echo "============================================" +echo "Environment: $BRIDGE_ENV" +if [ -n "$AUTO_LOADED_ENV_FILE" ]; then + echo "Loaded env: $AUTO_LOADED_ENV_FILE" +fi +echo "Base WS URL: $BASE_WS_URL" +echo "Inbox: $INBOX_CONTRACT_ADDRESS" +echo "Nock: $NOCK_CONTRACT_ADDRESS" +echo "Base Start: $BASE_START_HEIGHT" +echo "Nock Start: $NOCKCHAIN_START_HEIGHT" +echo "Base Conf: $BASE_CONFIRMATION_DEPTH" +echo "Nock Conf: $NOCKCHAIN_CONFIRMATION_DEPTH" +echo "============================================" +echo "" + +cleanup() { + echo "Cleaning up..." + [ -n "$BRIDGE_PID" ] && kill $BRIDGE_PID 2>/dev/null || true + wait 2>/dev/null || true + echo "Done." +} + +trap cleanup EXIT INT TERM + +NEW_FLAG="" +while [[ $# -gt 0 ]]; do + case "$1" in + --new) + NEW_FLAG="--new" + rm -rf "$BRIDGE_DATA_DIR" + shift + ;; + --base-start-height) + BASE_START_HEIGHT="$2" + shift 2 + ;; + --nockchain-start-height) + NOCKCHAIN_START_HEIGHT="$2" + shift 2 + ;; + *) + echo "Unknown option: $1" + echo "Usage: ./run-bridge-only.sh [--new] [--base-start-height N] [--nockchain-start-height N]" + exit 1 + ;; + esac +done + +mkdir -p "$BRIDGE_DATA_DIR" + +if [ ! -f "$BIN_DIR/bridge" ]; then + echo "Error: bridge binary not found. Run: cargo build --release -p bridge" + exit 1 +fi + +BRIDGE_CONFIG="${BRIDGE_DATA_DIR}/bridge-conf.toml" +cat > "$BRIDGE_CONFIG" << EOF +node_id = 0 +# Environment: ${BRIDGE_ENV} +base_ws_url = "${BASE_WS_URL}" +inbox_contract_address = "${INBOX_CONTRACT_ADDRESS}" +nock_contract_address = "${NOCK_CONTRACT_ADDRESS}" +my_eth_key = "${BRIDGE_ETH_KEY}" +my_nock_key = "${BRIDGE_NOCK_KEY}" +grpc_address = "http://127.0.0.1:${NODE_PRIVATE_GRPC_PORT}" +base_confirmation_depth = ${BASE_CONFIRMATION_DEPTH} +nockchain_confirmation_depth = ${NOCKCHAIN_CONFIRMATION_DEPTH} +ingress_listen_address = "127.0.0.1:8001" + +# Fake test data (node 0 address derived from BRIDGE_ETH_KEY) +[[nodes]] +ip = "localhost:8001" +eth_pubkey = "${BRIDGE_ETH_ADDR}" +nock_pkh = "2222222222222222222222222222222222222222222222222222" + +[[nodes]] +ip = "127.0.0.1:8002" +eth_pubkey = "0x2222222222222222222222222222222222222222" +nock_pkh = "3333333333333333333333333333333333333333333333333333" + +[[nodes]] +ip = "localhost:8003" +eth_pubkey = "0x3333333333333333333333333333333333333333" +nock_pkh = "4444444444444444444444444444444444444444444444444444" + +[[nodes]] +ip = "localhost:8004" +eth_pubkey = "0x4444444444444444444444444444444444444444" +nock_pkh = "5555555555555555555555555555555555555555555555555555" + +[[nodes]] +ip = "localhost:8005" +eth_pubkey = "0x5555555555555555555555555555555555555555" +nock_pkh = "6666666666666666666666666666666666666666666666666666" + +# Bridge constants for local testing +[constants] +min_signers = 3 +total_signers = 5 +minimum_event_nocks = 1000 # Lower for testing (prod: 1_000_000) +nicks_fee_per_nock = 195 +base_blocks_chunk = 100 +base_start_height = ${BASE_START_HEIGHT} +nockchain_start_height = ${NOCKCHAIN_START_HEIGHT} +EOF + +echo "Bridge config written to $BRIDGE_CONFIG" + +echo "Starting bridge..." +# Filter noisy h2/hyper/tonic internal modules +RUST_LOG=debug,h2=warn,hyper=warn,tower=warn,tonic=info \ +"$BIN_DIR/bridge" \ + $NEW_FLAG \ + --save-interval "$BRIDGE_SAVE_INTERVAL_MS" \ + --config-path "$BRIDGE_CONFIG" \ + --data-dir "$BRIDGE_DATA_DIR" \ + 2>&1 | sed 's/^/[BRIDGE] /' & +BRIDGE_PID=$! + +echo "Bridge started with PID $BRIDGE_PID" + +echo "" +echo "============================================" +echo "Bridge running! [$BRIDGE_ENV]" +echo "============================================" +echo "Bridge: PID=$BRIDGE_PID" +echo " Ingress: http://$BRIDGE_INGRESS" +echo " Config: $BRIDGE_CONFIG" +echo " Node gRPC: http://127.0.0.1:$NODE_PRIVATE_GRPC_PORT" +echo " Base WS: ${BASE_WS_URL:0:50}..." +echo "" +echo "Data directory: $BRIDGE_DATA_DIR" +echo "" +echo "TUI (separate terminal):" +echo " $BIN_DIR/nockchain-bridge-tui --server \"http://$BRIDGE_INGRESS\"" +echo "" +echo "Press Ctrl+C to stop" +echo "============================================" + +wait $BRIDGE_PID diff --git a/crates/bridge/scripts/run-node-and-bridge.sh b/crates/bridge/scripts/run-node-and-bridge.sh new file mode 100755 index 000000000..d04d18d9a --- /dev/null +++ b/crates/bridge/scripts/run-node-and-bridge.sh @@ -0,0 +1,288 @@ +#!/bin/bash +set -e + +# Simple script to start a nockchain node and bridge together +# Usage: ./run-node-and-bridge.sh [--clean] [--base-start-height N] [--nockchain-start-height N] +# +# Options: +# --clean Remove existing test data before starting +# --base-start-height N Override Base chain start height (default: 33387036) +# --nockchain-start-height N Override Nockchain start height (default: 1) +# +# Environment setup: +# source environments/virtual-testnet.env # For Virtual Testnet (50 block limit) +# source environments/base-sepolia.env # For real Base Sepolia (unlimited) +# +# Before you start, run `make install` and `make deps` in the bridge contracts +# directory. Also make sure your wallet, bridge, and nockchain binaries are up to date. + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +# shellcheck source=lib/layout.sh +source "$SCRIPT_DIR/lib/layout.sh" +bridge_resolve_layout + +BIN_DIR="$BRIDGE_BIN_DIR" +TEST_DATA_DIR="${BRIDGE_DIR}/test_run_data" + +NODE_DIR="${TEST_DATA_DIR}/node" +BRIDGE_DATA_DIR="${TEST_DATA_DIR}/bridge" +WALLET_DIR="${TEST_DATA_DIR}/wallet" + +GENESIS_JAM="${GENESIS_JAM_PATH:-${BRIDGE_SOURCE_ROOT}/crates/nockchain/jams/fakenet-genesis-pow-64-bex-2.jam}" + +# Node config +NODE_BIND="/ip4/0.0.0.0/udp/3005/quic-v1" +NODE_PUBLIC_GRPC="127.0.0.1:5001" +NODE_PRIVATE_GRPC_PORT="5002" + +# Bridge config +BRIDGE_INGRESS="127.0.0.1:8002" + +# Environment configuration (set via environment variables or defaults to virtual testnet) +BRIDGE_ENV="${BRIDGE_ENV:-virtual-testnet}" +AUTO_LOADED_ENV_FILE="" +if [ "$BRIDGE_ENV" = "virtual-testnet" ] && [ -z "${BASE_WS_URL:-}" ]; then + GENERATED_ENV_FILE="${SCRIPT_DIR}/environments/virtual-testnet.generated.env" + if [ -f "$GENERATED_ENV_FILE" ]; then + # shellcheck disable=SC1090 + source "$GENERATED_ENV_FILE" + AUTO_LOADED_ENV_FILE="$GENERATED_ENV_FILE" + fi +fi +: "${BASE_WS_URL:?BASE_WS_URL must be set; source scripts/environments/virtual-testnet.generated.env or an environment profile.}" +: "${INBOX_CONTRACT_ADDRESS:?INBOX_CONTRACT_ADDRESS must be set.}" +: "${NOCK_CONTRACT_ADDRESS:?NOCK_CONTRACT_ADDRESS must be set.}" +BRIDGE_ETH_KEY="${BRIDGE_ETH_KEY:-0x4c0883a69102937d6231471b5dbb6204fe5129617082792ae468d01a3f362318}" +BRIDGE_NOCK_KEY="${BRIDGE_NOCK_KEY:-5KZuFKrctV5iUburT54Z9fhpf3V3hv2sPf9GRQnjFR8T}" + +echo "============================================" +echo "Environment: $BRIDGE_ENV" +if [ -n "$AUTO_LOADED_ENV_FILE" ]; then + echo "Loaded env: $AUTO_LOADED_ENV_FILE" +fi +echo "Base WS URL: ${BASE_WS_URL:0:60}..." +echo "Inbox: $INBOX_CONTRACT_ADDRESS" +echo "Nock: $NOCK_CONTRACT_ADDRESS" +echo "Base Start: $BASE_START_HEIGHT" +echo "Nock Start: $NOCKCHAIN_START_HEIGHT" +echo "============================================" +echo "" + +# Cleanup function +cleanup() { + echo "Cleaning up..." + [ -n "$NODE_PID" ] && kill $NODE_PID 2>/dev/null || true + [ -n "$BRIDGE_PID" ] && kill $BRIDGE_PID 2>/dev/null || true + wait 2>/dev/null || true + echo "Done." +} + +trap cleanup EXIT INT TERM + +# Configurable start heights (can be overridden via CLI) +BASE_START_HEIGHT="${BASE_START_HEIGHT:-33387036}" +NOCKCHAIN_START_HEIGHT="${NOCKCHAIN_START_HEIGHT:-1}" + +# Driver-side finality configuration (confirmation depths) +BASE_CONFIRMATION_DEPTH="${BASE_CONFIRMATION_DEPTH:-300}" +NOCKCHAIN_CONFIRMATION_DEPTH="${NOCKCHAIN_CONFIRMATION_DEPTH:-100}" + +# Parse args +while [[ $# -gt 0 ]]; do + case "$1" in + --clean) + echo "Cleaning test data directories..." + rm -rf "$TEST_DATA_DIR" + shift + ;; + --base-start-height) + BASE_START_HEIGHT="$2" + shift 2 + ;; + --nockchain-start-height) + NOCKCHAIN_START_HEIGHT="$2" + shift 2 + ;; + *) + echo "Unknown option: $1" + echo "Usage: ./run-node-and-bridge.sh [--clean] [--base-start-height N] [--nockchain-start-height N]" + exit 1 + ;; + esac +done + +# Create directories +mkdir -p "$NODE_DIR" "$BRIDGE_DATA_DIR" "$WALLET_DIR" + +# Check binaries exist +if [ ! -f "$BIN_DIR/nockchain" ]; then + echo "Error: nockchain not found at $BIN_DIR/nockchain" + echo "Run: cargo build --release -p nockchain" + exit 1 +fi + +if [ ! -f "$BIN_DIR/bridge" ]; then + echo "Error: bridge not found at $BIN_DIR/bridge" + echo "Run: cargo build --release -p bridge" + exit 1 +fi + +if [ ! -f "$GENESIS_JAM" ]; then + echo "Error: genesis jam not found at $GENESIS_JAM" + echo "Set GENESIS_JAM_PATH to override the default." + exit 1 +fi + +# Initialize wallet to get mining address +echo "Initializing wallet..." +FAKENET_SEED="route run sing warrior light swamp clog flower agent ugly wasp fresh tube snow motion salt salon village raccoon chair demise neutral school confirm" + +NOCK_DATA_DIR="$WALLET_DIR" "$BIN_DIR/nockchain-wallet" import-keys \ + --seedphrase "$FAKENET_SEED" \ + --version 1 2>/dev/null || true + +MINING_ADDR=$( NOCK_DATA_DIR="$WALLET_DIR" "$BIN_DIR/nockchain-wallet" list-active-addresses 2>/dev/null | LC_ALL=C sed -n 's/.*- Address: //p' | head -1 ) + +if [ -z "$MINING_ADDR" ]; then + echo "Warning: Could not get mining address, using placeholder" + MINING_ADDR="placeholder" +fi + +echo "Mining address: $MINING_ADDR" + +# Start node +echo "Starting nockchain node..." +cd "$NODE_DIR" + +echo "Running command:" +echo "$BIN_DIR/nockchain \\" +echo " --new \\" +echo " --fakenet \\" +echo " --fakenet-genesis-jam-path $GENESIS_JAM \\" +echo " --fakenet-pow-len 64 \\" +echo " --fakenet-log-difficulty 2 \\" +echo " --mine \\" +echo " --mining-pkh $MINING_ADDR \\" +echo " --bind $NODE_BIND \\" +echo " --bind-public-grpc-addr $NODE_PUBLIC_GRPC \\" +echo " --bind-private-grpc-port $NODE_PRIVATE_GRPC_PORT" +echo "" + +"$BIN_DIR/nockchain" \ + --new \ + --fakenet \ + --fakenet-genesis-jam-path "$GENESIS_JAM" \ + --fakenet-pow-len 64 \ + --fakenet-log-difficulty 2 \ + --mine \ + --mining-pkh "$MINING_ADDR" \ + --bind "$NODE_BIND" \ + --bind-public-grpc-addr "$NODE_PUBLIC_GRPC" \ + --bind-private-grpc-port "$NODE_PRIVATE_GRPC_PORT" \ + 2>&1 | sed 's/^/[NODE] /' & +NODE_PID=$! + +echo "Node started with PID $NODE_PID" +echo "Waiting for node to initialize..." +sleep 3 + +# Check node is running +if ! kill -0 $NODE_PID 2>/dev/null; then + echo "Error: Node failed to start" + exit 1 +fi + +# Generate bridge config +BRIDGE_CONFIG="${BRIDGE_DATA_DIR}/bridge-conf.toml" +cat > "$BRIDGE_CONFIG" << EOF +node_id = 1 +# Environment: ${BRIDGE_ENV} +base_ws_url = "${BASE_WS_URL}" +inbox_contract_address = "${INBOX_CONTRACT_ADDRESS}" +nock_contract_address = "${NOCK_CONTRACT_ADDRESS}" +my_eth_key = "${BRIDGE_ETH_KEY}" +my_nock_key = "${BRIDGE_NOCK_KEY}" +grpc_address = "http://127.0.0.1:${NODE_PRIVATE_GRPC_PORT}" +base_confirmation_depth = ${BASE_CONFIRMATION_DEPTH} +nockchain_confirmation_depth = ${NOCKCHAIN_CONFIRMATION_DEPTH} +ingress_listen_address = "127.0.0.1:8002" + +# Fake test data (valid format placeholders for local testing) +[[nodes]] +ip = "localhost:8001" +eth_pubkey = "0x1111111111111111111111111111111111111111" +nock_pkh = "2222222222222222222222222222222222222222222222222222" + +[[nodes]] +ip = "127.0.0.1:8002" +eth_pubkey = "0x2222222222222222222222222222222222222222" +nock_pkh = "3333333333333333333333333333333333333333333333333333" + +[[nodes]] +ip = "localhost:8003" +eth_pubkey = "0x3333333333333333333333333333333333333333" +nock_pkh = "4444444444444444444444444444444444444444444444444444" + +[[nodes]] +ip = "localhost:8004" +eth_pubkey = "0x4444444444444444444444444444444444444444" +nock_pkh = "5555555555555555555555555555555555555555555555555555" + +[[nodes]] +ip = "localhost:8005" +eth_pubkey = "0x5555555555555555555555555555555555555555" +nock_pkh = "6666666666666666666666666666666666666666666666666666" + +# Bridge constants for local testing +[constants] +min_signers = 3 +total_signers = 5 +minimum_event_nocks = 1000 # Lower for testing (prod: 1_000_000) +nicks_fee_per_nock = 195 +base_blocks_chunk = 100 +base_start_height = ${BASE_START_HEIGHT} +nockchain_start_height = ${NOCKCHAIN_START_HEIGHT} +EOF + +echo "Bridge config written to $BRIDGE_CONFIG" + +# Start bridge +echo "Starting bridge..." +RUST_LOG=debug,connect=warn \ +"$BIN_DIR/bridge" \ + --new \ + --config-path "$BRIDGE_CONFIG" \ + --data-dir "$BRIDGE_DATA_DIR" \ + 2>&1 | sed 's/^/[BRIDGE] /' & +BRIDGE_PID=$! + +echo "Bridge started with PID $BRIDGE_PID" + +echo "" +echo "============================================" +echo "Node and Bridge running! [$BRIDGE_ENV]" +echo "============================================" +echo "Node: PID=$NODE_PID" +echo " Public gRPC: http://$NODE_PUBLIC_GRPC" +echo " Private gRPC: http://127.0.0.1:$NODE_PRIVATE_GRPC_PORT" +echo "" +echo "Bridge: PID=$BRIDGE_PID" +echo " Ingress: http://$BRIDGE_INGRESS" +echo " Config: $BRIDGE_CONFIG" +echo " Base WS: ${BASE_WS_URL:0:50}..." +echo "" +echo "TUI (separate terminal):" +echo " $BIN_DIR/nockchain-bridge-tui --server \"http://$BRIDGE_INGRESS\"" +echo "" +echo "Data directories:" +echo " Node: $NODE_DIR" +echo " Bridge: $BRIDGE_DATA_DIR" +echo " Wallet: $WALLET_DIR" +echo "" +echo "Press Ctrl+C to stop both processes" +echo "============================================" + +# Wait for either process to exit +wait -n $NODE_PID $BRIDGE_PID 2>/dev/null || true + +echo "A process exited, shutting down..." diff --git a/crates/bridge/scripts/run-node-only.sh b/crates/bridge/scripts/run-node-only.sh new file mode 100755 index 000000000..2f26900ef --- /dev/null +++ b/crates/bridge/scripts/run-node-only.sh @@ -0,0 +1,179 @@ +#!/bin/bash +set -e + +# Run just a nockchain node (useful for testing bridge connections separately) +# Usage: ./run-node-only.sh [--clean] [--new] [--v1-phase N] [--pow-len N] [--log-difficulty N] [--genesis-jam-path PATH] + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +# shellcheck source=lib/layout.sh +source "$SCRIPT_DIR/lib/layout.sh" +bridge_resolve_layout + +BIN_DIR="$BRIDGE_BIN_DIR" +TEST_DATA_DIR="${BRIDGE_DIR}/test_run_data" +NODE_DIR="${TEST_DATA_DIR}/node" +WALLET_DIR="${TEST_DATA_DIR}/wallet" + +GENESIS_JAM="${GENESIS_JAM_PATH:-${BRIDGE_SOURCE_ROOT}/crates/nockchain/jams/fakenet-genesis-pow-2-bex-1.jam}" + +NODE_BIND="/ip4/0.0.0.0/udp/3005/quic-v1" +NODE_PUBLIC_GRPC="127.0.0.1:5001" +NODE_PRIVATE_GRPC_PORT="5002" + +CLEAN_FLAG="false" +NEW_FLAG="false" +V1_PHASE="${V1_PHASE:-20}" +POW_LEN="" +LOG_DIFFICULTY="" +GENESIS_JAM_PATH_OVERRIDE="" + +cleanup() { + echo "Cleaning up..." + [ -n "$NODE_PID" ] && kill $NODE_PID 2>/dev/null || true + wait 2>/dev/null || true + echo "Done." +} + +trap cleanup EXIT INT TERM + +while [[ $# -gt 0 ]]; do + case "$1" in + --clean) + CLEAN_FLAG="true" + shift + ;; + --new) + NEW_FLAG="true" + shift + ;; + --v1-phase) + V1_PHASE="$2" + shift 2 + ;; + --v1-phase=*) + V1_PHASE="${1#*=}" + shift + ;; + --pow-len) + POW_LEN="$2" + shift 2 + ;; + --log-difficulty) + LOG_DIFFICULTY="$2" + shift 2 + ;; + --genesis-jam-path) + GENESIS_JAM_PATH_OVERRIDE="$2" + shift 2 + ;; + *) + echo "Unknown option: $1" + echo "Usage: ./run-node-only.sh [--clean] [--new] [--v1-phase N] [--pow-len N] [--log-difficulty N] [--genesis-jam-path PATH]" + exit 1 + ;; + esac +done + +if [ -n "$GENESIS_JAM_PATH_OVERRIDE" ]; then + GENESIS_JAM="$GENESIS_JAM_PATH_OVERRIDE" +fi + +if [ "$CLEAN_FLAG" = "true" ]; then + rm -rf "$TEST_DATA_DIR" +elif [ "$NEW_FLAG" = "true" ]; then + rm -rf "$NODE_DIR" +fi + +mkdir -p "$NODE_DIR" "$WALLET_DIR" + +if [ ! -f "$BIN_DIR/nockchain" ]; then + echo "Error: nockchain not found. Run: cargo build --release -p nockchain" + exit 1 +fi +if [ ! -f "$GENESIS_JAM" ]; then + echo "Error: genesis jam not found at $GENESIS_JAM" + echo "Set GENESIS_JAM_PATH to override the default." + exit 1 +fi + +# Get mining address +FAKENET_SEED="route run sing warrior light swamp clog flower agent ugly wasp fresh tube snow motion salt salon village raccoon chair demise neutral school confirm" +NOCK_DATA_DIR="$WALLET_DIR" "$BIN_DIR/nockchain-wallet" --fakenet import-keys \ + --seedphrase "$FAKENET_SEED" --version 1 2>/dev/null || true + +MINING_ADDR=$( NOCK_DATA_DIR="$WALLET_DIR" "$BIN_DIR/nockchain-wallet" --fakenet list-active-addresses 2>/dev/null | LC_ALL=C sed -n 's/.*- Address: //p' | head -1 ) +[ -z "$MINING_ADDR" ] && MINING_ADDR="placeholder" + +echo "Mining address: $MINING_ADDR" + +echo "Starting nockchain node..." +cd "$NODE_DIR" + +echo "Running command:" +echo "$BIN_DIR/nockchain \\" +echo " --fakenet \\" +echo " --fakenet-v1-phase $V1_PHASE \\" +echo " --fakenet-genesis-jam-path $GENESIS_JAM \\" +if [ "$NEW_FLAG" = "true" ]; then + echo " --new \\" +fi +if [ -n "$POW_LEN" ]; then + echo " --fakenet-pow-len $POW_LEN \\" +fi +if [ -n "$LOG_DIFFICULTY" ]; then + echo " --fakenet-log-difficulty $LOG_DIFFICULTY \\" +fi +echo " --mine \\" +echo " --mining-pkh $MINING_ADDR \\" +echo " --bind $NODE_BIND \\" +echo " --bind-public-grpc-addr $NODE_PUBLIC_GRPC \\" +echo " --bind-private-grpc-port $NODE_PRIVATE_GRPC_PORT" +echo "" + +EXTRA_ARGS=() +if [ "$NEW_FLAG" = "true" ]; then + EXTRA_ARGS+=(--new) +fi +if [ -n "$POW_LEN" ]; then + EXTRA_ARGS+=(--fakenet-pow-len "$POW_LEN") +fi +if [ -n "$LOG_DIFFICULTY" ]; then + EXTRA_ARGS+=(--fakenet-log-difficulty "$LOG_DIFFICULTY") +fi + +"$BIN_DIR/nockchain" \ + --fakenet \ + --fakenet-v1-phase "$V1_PHASE" \ + --fakenet-genesis-jam-path "$GENESIS_JAM" \ + "${EXTRA_ARGS[@]}" \ + --mine \ + --mining-pkh "$MINING_ADDR" \ + --bind "$NODE_BIND" \ + --bind-public-grpc-addr "$NODE_PUBLIC_GRPC" \ + --bind-private-grpc-port "$NODE_PRIVATE_GRPC_PORT" \ + 2>&1 | sed 's/^/[NODE] /' & +NODE_PID=$! + +echo "Node started with PID $NODE_PID" +echo "Waiting for node to initialize..." +sleep 3 + +if ! kill -0 $NODE_PID 2>/dev/null; then + echo "Error: Node failed to start" + exit 1 +fi + +echo "" +echo "============================================" +echo "Node running!" +echo "============================================" +echo "Node: PID=$NODE_PID" +echo " Public gRPC: http://$NODE_PUBLIC_GRPC" +echo " Private gRPC: http://127.0.0.1:$NODE_PRIVATE_GRPC_PORT" +echo "" +echo "Data directory: $NODE_DIR" +echo "" +echo "Press Ctrl+C to stop" +echo "============================================" + +wait $NODE_PID diff --git a/crates/bridge/scripts/stop.sh b/crates/bridge/scripts/stop.sh new file mode 100755 index 000000000..09badf445 --- /dev/null +++ b/crates/bridge/scripts/stop.sh @@ -0,0 +1,38 @@ +#!/usr/bin/env bash +set -euo pipefail + +ADDR="${1:-127.0.0.1:8002}" # multi-bridge node 0; others: 8003..8006 + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +# shellcheck source=lib/layout.sh +source "$SCRIPT_DIR/lib/layout.sh" +bridge_resolve_layout +PROTO_DIR="$BRIDGE_DIR/proto" + +if [[ ! -f "$PROTO_DIR/bridge_ingress.proto" ]]; then + echo "error: can't find $PROTO_DIR/bridge_ingress.proto" >&2 + exit 1 +fi + +BASE_HASH="$(python3 - <<'PY' | base64 +import sys; sys.stdout.buffer.write(b'\x11'*40) +PY +)" +NOCK_HASH="$(python3 - <<'PY' | base64 +import sys; sys.stdout.buffer.write(b'\x22'*40) +PY +)" + +grpcurl -plaintext \ + -import-path "$PROTO_DIR" \ + -proto bridge_ingress.proto \ + -d "{ + \"sender_node_id\": 1, + \"reason\": \"smoke stop\", + \"last_base_hash\": \"${BASE_HASH}\", + \"last_base_height\": 123, + \"last_nock_hash\": \"${NOCK_HASH}\", + \"last_nock_height\": 456, + \"timestamp\": $(date +%s) + }" \ + "${ADDR}" bridge.ingress.v1.BridgeIngress/BroadcastStop diff --git a/crates/bridge/scripts/tenderly-advance-blocks.sh b/crates/bridge/scripts/tenderly-advance-blocks.sh new file mode 100755 index 000000000..94863aa49 --- /dev/null +++ b/crates/bridge/scripts/tenderly-advance-blocks.sh @@ -0,0 +1,72 @@ +#!/usr/bin/env bash +# Advance a Tenderly Virtual TestNet by N blocks +# +# Usage: ./tenderly-advance-blocks.sh [rpc_url] +# +# Examples: +# ./tenderly-advance-blocks.sh 100 +# ./tenderly-advance-blocks.sh 84 https://virtual.base.rpc.tenderly.co/your-vnet-id + +set -euo pipefail + +NUM_BLOCKS="${1:?Usage: $0 [rpc_url]}" +RPC_URL="${2:-${TENDERLY_RPC_URL:-${TENDERLY_VIRTUAL_TESTNET_RPC_URL:-${BASE_WS_URL:-${BASE_RPC_URL:-}}}}}" + +if [[ -z "$RPC_URL" ]]; then + echo "Error: No RPC URL provided." + echo "Set TENDERLY_RPC_URL, TENDERLY_VIRTUAL_TESTNET_RPC_URL, or pass as second argument." + exit 1 +fi + +# Convert ws:// to https:// if needed +RPC_URL="${RPC_URL/wss:\/\//https://}" +RPC_URL="${RPC_URL/ws:\/\//http://}" + +echo "Advancing $NUM_BLOCKS blocks on $RPC_URL" + +# Get current block number +CURRENT_BLOCK=$(curl -s -X POST "$RPC_URL" \ + -H "Content-Type: application/json" \ + -d '{"jsonrpc":"2.0","method":"eth_blockNumber","params":[],"id":1}' \ + | jq -r '.result' | xargs printf "%d") + +echo "Current block: $CURRENT_BLOCK" + +# Tenderly uses evm_increaseBlocks to advance multiple blocks at once +# This is much faster than mining blocks one at a time +RESULT=$(curl -s -X POST "$RPC_URL" \ + -H "Content-Type: application/json" \ + -d "{\"jsonrpc\":\"2.0\",\"method\":\"evm_increaseBlocks\",\"params\":[\"0x$(printf '%x' $NUM_BLOCKS)\"],\"id\":1}") + +ERROR=$(echo "$RESULT" | jq -r '.error // empty') +if [[ -n "$ERROR" ]]; then + echo "Error from RPC: $ERROR" + echo "Trying alternative method: evm_mine in a loop..." + + # Fallback: mine blocks one at a time (slower but more compatible) + for ((i=1; i<=NUM_BLOCKS; i++)); do + curl -s -X POST "$RPC_URL" \ + -H "Content-Type: application/json" \ + -d '{"jsonrpc":"2.0","method":"evm_mine","params":[],"id":1}' > /dev/null + + if ((i % 10 == 0)); then + echo " Mined $i / $NUM_BLOCKS blocks..." + fi + done +fi + +# Get new block number +NEW_BLOCK=$(curl -s -X POST "$RPC_URL" \ + -H "Content-Type: application/json" \ + -d '{"jsonrpc":"2.0","method":"eth_blockNumber","params":[],"id":1}' \ + | jq -r '.result' | xargs printf "%d") + +ADVANCED=$((NEW_BLOCK - CURRENT_BLOCK)) +echo "New block: $NEW_BLOCK (advanced $ADVANCED blocks)" + +if [[ $ADVANCED -lt $NUM_BLOCKS ]]; then + echo "Warning: Only advanced $ADVANCED blocks, expected $NUM_BLOCKS" + exit 1 +fi + +echo "Done!" diff --git a/crates/bridge/scripts/tenderly-vnet-deploy.sh b/crates/bridge/scripts/tenderly-vnet-deploy.sh new file mode 100755 index 000000000..b5e80b0fc --- /dev/null +++ b/crates/bridge/scripts/tenderly-vnet-deploy.sh @@ -0,0 +1,792 @@ +#!/usr/bin/env bash +set -euo pipefail + +# Create a Tenderly Base Sepolia virtual testnet, fund addresses via tenderly_setBalance, +# deploy bridge contracts (MessageInbox + Nock), and optionally clean up old vnets. +# +# Required env vars: +# TENDERLY_ACCESS_KEY +# TENDERLY_ACCOUNT_ID +# TENDERLY_PROJECT_SLUG +# +# Required for deploy/fund: +# TENDERLY_PRIVATE_KEY (or TENDERLY_TEST_PRIVATE_KEY for disposable VNet runs) +# TENDERLY_PUBLIC_ADDRESS (optional; derived from TENDERLY_PRIVATE_KEY when unset) +# BRIDGE_NODE_0..BRIDGE_NODE_4 (or BRIDGE_NODE_KEY_0..BRIDGE_NODE_KEY_4) +# If BRIDGE_NODE_* are unset, defaults are used from test-bridge-keys.env.example +# +# Optional: +# NOCK_NAME (default: Nock) +# NOCK_SYMBOL (default: NOCK) +# +# Quickstart (from the bridge scripts directory): +# export TENDERLY_ACCESS_KEY="..." +# export TENDERLY_ACCOUNT_ID="..." +# export TENDERLY_PROJECT_SLUG="..." +# export TENDERLY_PRIVATE_KEY="0x..." # or rely on TENDERLY_TEST_PRIVATE_KEY for disposable VNet runs +# export BRIDGE_NODE_0="0x..." +# export BRIDGE_NODE_1="0x..." +# export BRIDGE_NODE_2="0x..." +# export BRIDGE_NODE_3="0x..." +# export BRIDGE_NODE_4="0x..." +# ./tenderly-vnet-deploy.sh --dry-run +# ./tenderly-vnet-deploy.sh --cleanup-old --cleanup-prefix bridge-vnet --cleanup-keep 3 +# source environments/virtual-testnet.generated.env +# +# Examples: +# ./tenderly-vnet-deploy.sh +# ./tenderly-vnet-deploy.sh --name bridge-e2e --fund-eth 25 +# ./tenderly-vnet-deploy.sh --cleanup-old --cleanup-prefix bridge-vnet --cleanup-keep 2 +# ./tenderly-vnet-deploy.sh --cleanup-only --cleanup-prefix bridge-vnet --cleanup-mode delete + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +BRIDGE_DIR="$(cd "$SCRIPT_DIR/.." && pwd)" +CONTRACTS_DIR="$BRIDGE_DIR/contracts" +DEPLOY_SCRIPT="$CONTRACTS_DIR/scripts/deploy_tenderly.sh" +API_BASE_URL="https://api.tenderly.co/api/v1" + +VNET_NAME_PREFIX="bridge-vnet" +VNET_NAME="" +FORK_NETWORK_ID="84532" # Base Sepolia +CHAIN_ID="84532" # Base Sepolia +FORK_BLOCK_NUMBER="latest" +STATE_SYNC="true" +PUBLIC_EXPLORER="true" +VERIFICATION_VISIBILITY="src" + +FUND_AMOUNT_ETH="10" +DO_FUND="true" +DO_DEPLOY="true" +INSTALL_DEPS="true" + +CLEANUP_OLD="false" +CLEANUP_ONLY="false" +CLEANUP_PREFIX="" +CLEANUP_KEEP="3" +CLEANUP_MODE="delete" # delete | stop + +OUTPUT_ENV_PATH="$SCRIPT_DIR/environments/virtual-testnet.generated.env" +DEPLOY_TARGET_NETWORK="" +DEPLOYMENTS_PATH="" + +DRY_RUN="false" + +EXTRA_FUND_ADDRS=() +EXTRA_DEPLOY_ARGS=() + +VNET_ID="" +ADMIN_RPC_URL="" +PUBLIC_RPC_URL="" +PUBLIC_WS_URL="" +VNET_BASE_START_HEIGHT="" +DEPLOYER_ADDRESS="" +INBOX_CONTRACT_ADDRESS="" +NOCK_CONTRACT_ADDRESS="" + +DEFAULT_BRIDGE_NODE_0="0x2c7536E3605D9C16a7a3D7b1898e529396a65c23" +DEFAULT_BRIDGE_NODE_1="0x0EE156f080d9cB3BaA3C0DB53D07f13D69CEf4C9" +DEFAULT_BRIDGE_NODE_2="0x274BD645de480C325D618c60c661F11275eB77F1" +DEFAULT_BRIDGE_NODE_3="0x6dc59eb20f7928935c47A391e35545a2CEC51013" +DEFAULT_BRIDGE_NODE_4="0xcaB10dA05fC0aDBb7e91Eadc30f224bcDF601375" +DEFAULT_TENDERLY_TEST_PRIVATE_KEY="0xf9dca69398d5030c8f57a92cb69e4930caf448aefc493357dcafda87747f6098" + +usage() { + cat <<'EOF' +Usage: tenderly-vnet-deploy.sh [options] + +Core options: + --name NAME Display name for the new vnet. + --prefix PREFIX Prefix for generated vnet name/cleanup matching (default: bridge-vnet). + --fork-network-id ID Tenderly fork network id (default: 84532 for Base Sepolia). + --chain-id ID Chain id for the virtual network (default: 84532). + --fork-block N|latest Fork block number (default: latest). + --no-fund Skip tenderly_setBalance funding. + --fund-eth AMOUNT ETH amount per funded address (default: 10). + --fund-address ADDRESS Extra address to fund (repeatable). + --no-deploy Skip contract deploy. + --no-install-deps Do not auto-run `make deps` in contracts/ if libs missing. + --deploy-target-network NAME DEPLOY_TARGET_NETWORK override. + --deployments-path PATH DEPLOYMENTS_PATH override. + --deploy-arg ARG Extra arg forwarded to deploy_tenderly.sh (repeatable). + --output-env PATH Generated env file path (default: scripts/environments/virtual-testnet.generated.env). + +Cleanup options: + --cleanup-old Cleanup old vnets after provisioning. + --cleanup-only Only cleanup old vnets (skip create/fund/deploy). + --cleanup-prefix PREFIX Prefix used to match old vnets (default: --prefix value). + --cleanup-keep N Keep newest N matching vnets (default: 3). + --cleanup-mode delete|stop Delete vnets or stop them (default: delete). + +Other: + --dry-run Print planned actions without mutating remote state. + -h, --help Show this help. +EOF +} + +log() { printf '[tenderly-vnet] %s\n' "$*"; } +warn() { printf '[tenderly-vnet] WARN: %s\n' "$*" >&2; } +die() { printf '[tenderly-vnet] ERROR: %s\n' "$*" >&2; exit 1; } + +require_cmd() { + command -v "$1" >/dev/null 2>&1 || die "Missing required command: $1" +} + +get_var() { + eval "printf '%s' \"\${$1:-}\"" +} + +set_var() { + local key="$1" + local value="$2" + eval "$key=\"\$value\"" + export "$key" +} + +require_env() { + local name="$1" + [[ -n "$(get_var "$name")" ]] || die "$name is required" +} + +is_true() { + [[ "$1" == "true" || "$1" == "1" || "$1" == "yes" ]] +} + +sanitize_slug() { + local s="$1" + s="$(echo "$s" | tr '[:upper:]' '[:lower:]')" + s="$(echo "$s" | sed -E 's/[^a-z0-9-]+/-/g; s/^-+//; s/-+$//; s/-+/-/g')" + printf '%s' "$s" +} + +to_ws_url() { + local url="$1" + url="${url/https:\/\//wss://}" + url="${url/http:\/\//ws://}" + printf '%s' "$url" +} + +api_request() { + local method="$1" + local path="$2" + local body="${3:-}" + local url="$API_BASE_URL/$path" + local resp code payload + + if [[ -n "$body" ]]; then + resp="$(curl -sS -X "$method" "$url" \ + -H "Accept: application/json" \ + -H "Content-Type: application/json" \ + -H "X-Access-Key: $TENDERLY_ACCESS_KEY" \ + -d "$body" \ + -w $'\n%{http_code}')" + else + resp="$(curl -sS -X "$method" "$url" \ + -H "Accept: application/json" \ + -H "X-Access-Key: $TENDERLY_ACCESS_KEY" \ + -w $'\n%{http_code}')" + fi + + code="${resp##*$'\n'}" + payload="${resp%$'\n'*}" + + if (( code < 200 || code >= 300 )); then + echo "$payload" >&2 + return 1 + fi + + echo "$payload" +} + +rpc_request() { + local rpc_url="$1" + local payload="$2" + curl -sS -X POST "$rpc_url" \ + -H "Content-Type: application/json" \ + -d "$payload" +} + +resolve_vnet_start_height() { + local resp="$1" + local block_resp block_hex parsed_height + + VNET_BASE_START_HEIGHT="$( + echo "$resp" | jq -r ' + [ + .fork_config.block_number?, + .forkConfig.blockNumber?, + .virtual_network.fork_config.block_number?, + .virtual_network.forkConfig.blockNumber?, + .fork.block_number?, + .fork.blockNumber?, + .virtual_network.fork.block_number?, + .virtual_network.fork.blockNumber?, + .forked_from.block_number?, + .forked_from.blockNumber?, + .virtual_network.forked_from.block_number?, + .virtual_network.forked_from.blockNumber? + ] + | map( + if type == "number" then tostring + elif type == "string" then . + else "" + end + ) + | map(select(test("^[0-9]+$"))) + | .[0] // empty + ' + )" + + if [[ -z "$VNET_BASE_START_HEIGHT" && "$FORK_BLOCK_NUMBER" =~ ^[0-9]+$ ]]; then + VNET_BASE_START_HEIGHT="$FORK_BLOCK_NUMBER" + fi + + if [[ -z "$VNET_BASE_START_HEIGHT" && -n "$PUBLIC_RPC_URL" ]]; then + if block_resp="$( + rpc_request "$PUBLIC_RPC_URL" '{"jsonrpc":"2.0","method":"eth_blockNumber","params":[],"id":1}' + )"; then + block_hex="$(echo "$block_resp" | jq -r '.result // empty')" + if [[ "$block_hex" =~ ^0x[0-9a-fA-F]+$ ]]; then + parsed_height="$((16#${block_hex#0x}))" + VNET_BASE_START_HEIGHT="$parsed_height" + else + warn "Unable to parse eth_blockNumber result when resolving VNet start height: $block_hex" + fi + else + warn "eth_blockNumber request failed while resolving VNet start height" + fi + fi + + [[ -n "$VNET_BASE_START_HEIGHT" ]] || die "Unable to resolve VNet base start height from Tenderly response or RPC" +} + +resolve_bridge_nodes() { + local i addr key_var key + require_cmd cast + + for i in 0 1 2 3 4; do + addr="$(get_var "BRIDGE_NODE_${i}")" + if [[ -z "$addr" ]]; then + key_var="BRIDGE_NODE_KEY_${i}" + key="$(get_var "$key_var")" + if [[ -n "$key" ]]; then + addr="$(cast wallet address --private-key "$key")" + set_var "BRIDGE_NODE_${i}" "$addr" + fi + fi + if [[ -z "$addr" ]]; then + addr="$(get_var "DEFAULT_BRIDGE_NODE_${i}")" + set_var "BRIDGE_NODE_${i}" "$addr" + fi + done +} + +ensure_contract_deps() { + if [[ -d "$CONTRACTS_DIR/lib/forge-std" && -d "$CONTRACTS_DIR/lib/openzeppelin-contracts" ]]; then + return + fi + if ! is_true "$INSTALL_DEPS"; then + die "Contract deps missing in $CONTRACTS_DIR/lib. Re-run with default install behavior." + fi + log "Installing contract dependencies (make -C contracts deps)..." + make -C "$CONTRACTS_DIR" deps +} + +create_vnet() { + local now name slug payload resp + now="$(date +%Y%m%d-%H%M%S)" + + if [[ -z "$VNET_NAME" ]]; then + name="${VNET_NAME_PREFIX}-${now}" + else + name="$VNET_NAME" + fi + VNET_NAME="$name" + slug="$(sanitize_slug "$name")-$(date +%s)" + + payload="$(jq -n \ + --arg slug "$slug" \ + --arg display_name "$name" \ + --argjson network_id "$FORK_NETWORK_ID" \ + --arg block_number "$FORK_BLOCK_NUMBER" \ + --argjson chain_id "$CHAIN_ID" \ + --argjson sync_enabled "$STATE_SYNC" \ + --argjson explorer_enabled "$PUBLIC_EXPLORER" \ + --arg verification_visibility "$VERIFICATION_VISIBILITY" \ + '{ + slug: $slug, + display_name: $display_name, + fork_config: { + network_id: $network_id, + block_number: $block_number + }, + virtual_network_config: { + chain_config: { + chain_id: $chain_id + } + }, + sync_state_config: { + enabled: $sync_enabled + }, + explorer_page_config: { + enabled: $explorer_enabled, + verification_visibility: $verification_visibility + } + }')" + + if is_true "$DRY_RUN"; then + log "[dry-run] Would create vnet: $name" + log "[dry-run] Payload: $payload" + return + fi + + log "Creating Tenderly vnet '$name'..." + resp="$(api_request POST "account/$TENDERLY_ACCOUNT_ID/project/$TENDERLY_PROJECT_SLUG/vnets" "$payload")" \ + || die "Failed to create vnet via Tenderly API" + + VNET_ID="$(echo "$resp" | jq -r '.id // .virtual_network.id // empty')" + [[ -n "$VNET_ID" ]] || die "Tenderly API response missing vnet id: $resp" + + ADMIN_RPC_URL="$( + echo "$resp" | jq -r ' + def rpc_entries: ([.rpcs[]?] + [.virtual_network.rpcs[]?]); + ( + [rpc_entries[] | select(((.name // "") | ascii_downcase | test("admin"))) | .url] + + [.admin_rpc_url?, .adminRpcUrl?, .virtual_network.admin_rpc_url?, .virtual_network.adminRpcUrl?] + ) + | map(select(type == "string" and . != "" and (startswith("https://") or startswith("http://")))) + | .[0] // empty + ' + )" + if [[ -z "$ADMIN_RPC_URL" ]]; then + ADMIN_RPC_URL="$(echo "$resp" | jq -r '.rpcs[0].url // empty')" + fi + + PUBLIC_RPC_URL="$( + echo "$resp" | jq -r ' + def rpc_entries: ([.rpcs[]?] + [.virtual_network.rpcs[]?]); + ( + [rpc_entries[] | select(((.name // "") | ascii_downcase | test("public"))) | .url] + + [.public_rpc_url?, .publicRpcUrl?, .virtual_network.public_rpc_url?, .virtual_network.publicRpcUrl?] + ) + | map(select(type == "string" and . != "" and (startswith("https://") or startswith("http://")))) + | .[0] // empty + ' + )" + if [[ -z "$PUBLIC_RPC_URL" ]]; then + PUBLIC_RPC_URL="$(echo "$resp" | jq -r '[.rpcs[]?.url, .virtual_network.rpcs[]?.url] | map(select(type == "string" and . != "" and (startswith("https://") or startswith("http://")))) | .[0] // empty')" + fi + + [[ -n "$ADMIN_RPC_URL" ]] || die "Unable to determine admin RPC URL from Tenderly response" + [[ -n "$PUBLIC_RPC_URL" ]] || die "Unable to determine public RPC URL from Tenderly response" + + PUBLIC_WS_URL="$( + echo "$resp" | jq -r ' + def rpc_entries: ([.rpcs[]?] + [.virtual_network.rpcs[]?]); + ( + [rpc_entries[] + | select(((.name // "") | ascii_downcase | test("public"))) + | .ws_url?, .wsUrl?, .websocket_url?, .websocketUrl?, .websocket_rpc_url?, .websocketRpcUrl?, .url? + ] + + [ + .public_ws_url?, .publicWsUrl?, .public_websocket_url?, .publicWebsocketUrl?, + .public_websocket_rpc_url?, .publicWebsocketRpcUrl?, .public_rpc_ws_url?, .publicRpcWsUrl?, + .ws_rpc_url?, .wsRpcUrl?, .websocket_url?, .websocketUrl?, + .virtual_network.public_ws_url?, .virtual_network.publicWsUrl?, + .virtual_network.public_websocket_url?, .virtual_network.publicWebsocketUrl?, + .virtual_network.public_websocket_rpc_url?, .virtual_network.publicWebsocketRpcUrl?, + .virtual_network.ws_rpc_url?, .virtual_network.wsRpcUrl?, + .virtual_network.websocket_url?, .virtual_network.websocketUrl? + ] + ) + | map(select(type == "string" and . != "" and (startswith("wss://") or startswith("ws://")))) + | .[0] // empty + ' + )" + if [[ -z "$PUBLIC_WS_URL" ]]; then + PUBLIC_WS_URL="$(to_ws_url "$PUBLIC_RPC_URL")" + fi + resolve_vnet_start_height "$resp" + + log "Created vnet id: $VNET_ID" + log "Admin RPC: $ADMIN_RPC_URL" + log "Public RPC: $PUBLIC_RPC_URL" + log "Public WS: $PUBLIC_WS_URL" + log "Base start height: $VNET_BASE_START_HEIGHT" +} + +fund_vnet() { + local payload resp err amount_wei amount_hex addr_json + local addrs=() + local i extra_addr + + require_cmd cast + + DEPLOYER_ADDRESS="$(cast wallet address --private-key "$TENDERLY_PRIVATE_KEY")" + addrs+=("$DEPLOYER_ADDRESS") + for i in 0 1 2 3 4; do + addrs+=("$(get_var "BRIDGE_NODE_${i}")") + done + # Use :- form to avoid nounset errors on older bash when optional arrays are empty. + for extra_addr in "${EXTRA_FUND_ADDRS[@]:-}"; do + [[ -z "$extra_addr" ]] && continue + addrs+=("$extra_addr") + done + + addr_json="$( + printf '%s\n' "${addrs[@]}" \ + | awk 'NF && !seen[tolower($0)]++' \ + | jq -Rsc 'split("\n") | map(select(length > 0))' + )" + + amount_wei="$(cast to-wei "$FUND_AMOUNT_ETH" ether)" + amount_hex="$(cast to-hex "$amount_wei")" + + payload="$(jq -n \ + --argjson addrs "$addr_json" \ + --arg amount "$amount_hex" \ + '{jsonrpc:"2.0", method:"tenderly_setBalance", params:[$addrs, $amount], id:1}')" + + if is_true "$DRY_RUN"; then + log "[dry-run] Would fund addresses with tenderly_setBalance amount=${FUND_AMOUNT_ETH} ETH" + log "[dry-run] Addresses: $(echo "$addr_json" | jq -c '.')" + return + fi + + log "Funding deployer/bridge accounts with ${FUND_AMOUNT_ETH} ETH each via tenderly_setBalance..." + resp="$(rpc_request "$ADMIN_RPC_URL" "$payload")" + err="$(echo "$resp" | jq -r '.error.message // empty')" + if [[ -n "$err" ]]; then + die "Funding failed: $err" + fi + + log "Funding complete." +} + +deploy_contracts() { + local network_name deploy_path + local deploy_cmd + local extra_arg + + require_cmd cast + require_cmd forge + [[ -x "$DEPLOY_SCRIPT" ]] || die "Missing deploy script: $DEPLOY_SCRIPT" + + DEPLOYER_ADDRESS="$(cast wallet address --private-key "$TENDERLY_PRIVATE_KEY")" + + export NOCK_NAME="${NOCK_NAME:-Nock}" + export NOCK_SYMBOL="${NOCK_SYMBOL:-NOCK}" + + if [[ -z "$DEPLOY_TARGET_NETWORK" ]]; then + if [[ -n "$VNET_ID" ]]; then + network_name="tenderly-vnet-${VNET_ID}" + else + network_name="tenderly-vnet-$(date +%Y%m%d-%H%M%S)" + fi + else + network_name="$DEPLOY_TARGET_NETWORK" + fi + + if [[ -z "$DEPLOYMENTS_PATH" ]]; then + deploy_path="$CONTRACTS_DIR/deployments/${network_name}.json" + else + deploy_path="$DEPLOYMENTS_PATH" + fi + + export TENDERLY_RPC_URL="$ADMIN_RPC_URL" + export DEPLOY_TARGET_NETWORK="$network_name" + export DEPLOYMENTS_PATH="$deploy_path" + export DEPLOYER_ADDRESS="$DEPLOYER_ADDRESS" + + if is_true "$DRY_RUN"; then + log "[dry-run] Would deploy contracts via $DEPLOY_SCRIPT" + log "[dry-run] DEPLOY_TARGET_NETWORK=$DEPLOY_TARGET_NETWORK" + log "[dry-run] DEPLOYMENTS_PATH=$DEPLOYMENTS_PATH" + return + fi + + ensure_contract_deps + + log "Deploying MessageInbox + Nock..." + deploy_cmd=("$DEPLOY_SCRIPT") + # Use :- form to avoid nounset errors on older bash when optional arrays are empty. + for extra_arg in "${EXTRA_DEPLOY_ARGS[@]:-}"; do + [[ -z "$extra_arg" ]] && continue + deploy_cmd+=("$extra_arg") + done + "${deploy_cmd[@]}" + + [[ -f "$DEPLOYMENTS_PATH" ]] || die "Deploy did not create deployment file: $DEPLOYMENTS_PATH" + + INBOX_CONTRACT_ADDRESS="$(jq -r '.messageInboxProxy // empty' "$DEPLOYMENTS_PATH")" + NOCK_CONTRACT_ADDRESS="$(jq -r '.nock // empty' "$DEPLOYMENTS_PATH")" + [[ -n "$INBOX_CONTRACT_ADDRESS" ]] || die "Deployment file missing messageInboxProxy" + [[ -n "$NOCK_CONTRACT_ADDRESS" ]] || die "Deployment file missing nock" + + log "Deployed MessageInbox: $INBOX_CONTRACT_ADDRESS" + log "Deployed Nock: $NOCK_CONTRACT_ADDRESS" +} + +cleanup_vnet_by_id() { + local id="$1" + local path="account/$TENDERLY_ACCOUNT_ID/project/$TENDERLY_PROJECT_SLUG/vnets/$id" + + if is_true "$DRY_RUN"; then + log "[dry-run] Would $CLEANUP_MODE vnet: $id" + return + fi + + if [[ "$CLEANUP_MODE" == "delete" ]]; then + if api_request DELETE "$path" >/dev/null 2>&1; then + log "Deleted old vnet: $id" + return + fi + warn "Delete failed for $id, trying stop fallback" + fi + + if api_request PATCH "$path" '{"status":"stopped"}' >/dev/null 2>&1; then + log "Stopped old vnet: $id" + return + fi + + warn "Failed to cleanup vnet: $id" +} + +cleanup_old_vnets() { + local prefix list_json ids + prefix="$CLEANUP_PREFIX" + if [[ -z "$prefix" ]]; then + prefix="$VNET_NAME_PREFIX" + fi + + log "Cleaning old vnets with prefix '$prefix' (keep newest $CLEANUP_KEEP, mode=$CLEANUP_MODE)..." + list_json="$(api_request GET "account/$TENDERLY_ACCOUNT_ID/project/$TENDERLY_PROJECT_SLUG/vnets")" \ + || die "Failed to list vnets" + + ids="$( + echo "$list_json" | jq -r \ + --arg prefix "$prefix" \ + --argjson keep "$CLEANUP_KEEP" ' + def items: + if type == "array" then . + elif (.virtual_networks? | type) == "array" then .virtual_networks + elif (.results? | type) == "array" then .results + elif (.vnets? | type) == "array" then .vnets + elif (.data? | type) == "array" then .data + else [] end; + + [items[] + | { + id: (.id // .vnet_id // .virtual_network_id // empty), + name: (.display_name // .name // .slug // ""), + epoch: ( + (.created_at // .createdAt // .created // .inserted_at // .updated_at // 0) + | if type == "string" then (fromdateiso8601? // tonumber? // 0) + elif type == "number" then . + else 0 end + ) + } + | select(.id != "" and (.name | startswith($prefix))) + ] + | sort_by(.epoch) + | reverse + | .[$keep:][]?.id + ' + )" + + if [[ -z "$ids" ]]; then + log "No old matching vnets to cleanup." + return + fi + + while IFS= read -r id; do + [[ -z "$id" ]] && continue + cleanup_vnet_by_id "$id" + done <<< "$ids" +} + +write_env_file() { + local out="$OUTPUT_ENV_PATH" + + if [[ -z "$PUBLIC_WS_URL" ]]; then + PUBLIC_WS_URL="$(to_ws_url "$PUBLIC_RPC_URL")" + fi + [[ -n "$VNET_BASE_START_HEIGHT" ]] || die "VNet base start height missing; cannot write env file" + + if is_true "$DRY_RUN"; then + log "[dry-run] Would write env file: $out" + return + fi + + mkdir -p "$(dirname "$out")" + cat > "$out" < # Submit a transaction to the node +# ./wallet.sh --public-api list-notes # Sync via public API instead of private +# ./wallet.sh --new # Reset wallet state, re-import seed, then run +# ./wallet.sh # Pass through to wallet +# +# ============================================================================ +# BRIDGE DEPOSIT WORKFLOW +# ============================================================================ +# +# 1. Check your balance and find spendable notes: +# +# ./wallet.sh show-balance +# ./wallet.sh list-notes +# +# 2. Create a bridge deposit transaction: +# - Pick notes from list-notes output (need enough to cover amount + fee) +# - The --names arg takes a Hoon list: "[ ]" +# +# ./wallet.sh create-tx \ +# --names "[2naNCqw9F1VxLse3PBxZjTrzTSrEF8HbjgDogcrpXbGhvN6TwNsJAgV 4JjFAYZNzdZqpqZWqQpwaP4BSvXkVGmj6X6bsvJtKB38jDAkzkgub5w]" \ +# --fee 2654208 \ +# --recipient '{"kind":"bridge-deposit", "evm-address": "0x1111111111111111111111111111111111111111", "amount": 4000000000}' +# +# This saves the transaction to ./txs/.tx +# +# 3. Submit the transaction to the nockchain node: +# +# ./wallet.sh send-tx ./txs/.tx --public-grpc-server-addr "http://127.0.0.1:50052" +# +# On success you'll see: "Validation for TX passed. TX has been submitted to node." +# +# 4. The bridge will detect the deposit when it scans the block containing the tx. +# +# ============================================================================ + +set -e + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +# shellcheck source=lib/layout.sh +source "$SCRIPT_DIR/lib/layout.sh" +bridge_resolve_layout + +BIN_DIR="$BRIDGE_BIN_DIR" +TEST_DATA_DIR="${BRIDGE_DIR}/test_run_data" +WALLET_DIR="${TEST_DATA_DIR}/wallet" + +NODE_PRIVATE_GRPC_PORT="${NODE_PRIVATE_GRPC_PORT:-5002}" +NODE_PUBLIC_GRPC_SERVER_ADDR="${NODE_PUBLIC_GRPC_SERVER_ADDR:-http://127.0.0.1:5001}" +COMMON_WALLET_ARGS=(--fakenet) +CLIENT_MODE="private" + +# Same v1 seed as run-node-only.sh, plus the legacy v0 seed that matches +# the node's hardcoded pre-v1 coinbase pubkey. +FAKENET_V1_SEED="route run sing warrior light swamp clog flower agent ugly wasp fresh tube snow motion salt salon village raccoon chair demise neutral school confirm" +FAKENET_V0_SEED="farm step rhythm surprise math august panther pulse protect remain anger depend adjust sting enable poet describe stone essay blast click horse hair practice" +FAKENET_V1_ACTIVE_MASTER="9phXGACnW4238oqgvn2gpwaUjG3RAqcxq2Ash2vaKp8KjzSd3MQ56Jt" + +NEW_WALLET=false +# Script-only flags. +PASSTHRU_ARGS=() +while [[ $# -gt 0 ]]; do + case "$1" in + --new) + NEW_WALLET=true + shift + ;; + --public-api) + CLIENT_MODE="public" + shift + ;; + --private-api) + CLIENT_MODE="private" + shift + ;; + --public-grpc-server-addr) + if [[ $# -lt 2 ]]; then + echo "Error: --public-grpc-server-addr requires a value" + exit 1 + fi + NODE_PUBLIC_GRPC_SERVER_ADDR="$2" + shift 2 + ;; + --public-grpc-server-addr=*) + NODE_PUBLIC_GRPC_SERVER_ADDR="${1#*=}" + shift + ;; + --private-grpc-server-port) + if [[ $# -lt 2 ]]; then + echo "Error: --private-grpc-server-port requires a value" + exit 1 + fi + NODE_PRIVATE_GRPC_PORT="$2" + shift 2 + ;; + --private-grpc-server-port=*) + NODE_PRIVATE_GRPC_PORT="${1#*=}" + shift + ;; + *) + PASSTHRU_ARGS+=("$1") + shift + ;; + esac +done + +if [[ "$CLIENT_MODE" == "public" ]]; then + CLIENT_ARGS=( + --client public + --public-grpc-server-addr "$NODE_PUBLIC_GRPC_SERVER_ADDR" + ) +else + CLIENT_ARGS=( + --client private + --private-grpc-server-port "$NODE_PRIVATE_GRPC_PORT" + ) +fi + +if [ ! -f "$BIN_DIR/nockchain-wallet" ]; then + echo "Error: nockchain-wallet not found. Run: cargo build --release -p nockchain-wallet" + exit 1 +fi + +mkdir -p "$WALLET_DIR" + +if [ "$NEW_WALLET" = true ]; then + # Reset wallet state and import deterministic fakenet keys. + # Keep the v1 bridge address active by default after also importing the + # legacy v0 key needed to spend pre-v1 coinbase notes. + NOCK_DATA_DIR="$WALLET_DIR" "$BIN_DIR/nockchain-wallet" --new \ + "${COMMON_WALLET_ARGS[@]}" import-keys \ + --seedphrase "$FAKENET_V1_SEED" --version 1 + NOCK_DATA_DIR="$WALLET_DIR" "$BIN_DIR/nockchain-wallet" \ + "${COMMON_WALLET_ARGS[@]}" import-keys \ + --seedphrase "$FAKENET_V0_SEED" --version 0 + NOCK_DATA_DIR="$WALLET_DIR" "$BIN_DIR/nockchain-wallet" \ + "${COMMON_WALLET_ARGS[@]}" set-active-master-address "$FAKENET_V1_ACTIVE_MASTER" +fi + +# Run the wallet command +export NOCK_DATA_DIR="$WALLET_DIR" +exec "$BIN_DIR/nockchain-wallet" \ + "${CLIENT_ARGS[@]}" \ + "${COMMON_WALLET_ARGS[@]}" \ + "${PASSTHRU_ARGS[@]}" diff --git a/crates/bridge/src/core/base_observer.rs b/crates/bridge/src/core/base_observer.rs new file mode 100644 index 000000000..883ded507 --- /dev/null +++ b/crates/bridge/src/core/base_observer.rs @@ -0,0 +1,246 @@ +use crate::errors::BridgeError; +use crate::ports::{BaseSourcePort, KernelStatePort}; +use crate::runtime::ChainEvent; + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct BaseObserverCore { + pub batch_size: u64, + pub confirmation_depth: u64, +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct BasePlanInput { + pub state: BasePlanState, + pub batch_size: u64, + pub confirmation_depth: u64, +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum BasePlanState { + HoldActive, + Active { + chain_tip: u64, + next_needed_height: Option, + }, +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum BasePlanError { + ZeroBatchSize, + ZeroConfirmationDepth, + HeightOverflow, +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum BasePlanAction { + HoldActive, + NoPendingHeight { + chain_tip: u64, + }, + InvalidConfig(BasePlanError), + NotYetConfirmed { + chain_tip: u64, + confirmed_height: u64, + next_needed_height: u64, + needed_confirmed_height: u64, + blocks_until_ready: u64, + }, + FetchWindow { + chain_tip: u64, + confirmed_height: u64, + start: u64, + end: u64, + }, +} + +pub fn plan_base_tick(input: BasePlanInput) -> BasePlanAction { + if input.batch_size == 0 { + return BasePlanAction::InvalidConfig(BasePlanError::ZeroBatchSize); + } + if input.confirmation_depth == 0 { + return BasePlanAction::InvalidConfig(BasePlanError::ZeroConfirmationDepth); + } + + let (chain_tip, next_needed_height) = match input.state { + BasePlanState::HoldActive => return BasePlanAction::HoldActive, + BasePlanState::Active { + chain_tip, + next_needed_height, + } => (chain_tip, next_needed_height), + }; + + let Some(next_needed_height) = next_needed_height else { + return BasePlanAction::NoPendingHeight { chain_tip }; + }; + + let confirmed_height = chain_tip.saturating_sub(input.confirmation_depth); + let Some(batch_end) = next_needed_height.checked_add(input.batch_size - 1) else { + return BasePlanAction::InvalidConfig(BasePlanError::HeightOverflow); + }; + + if batch_end > confirmed_height { + return BasePlanAction::NotYetConfirmed { + chain_tip, + confirmed_height, + next_needed_height, + needed_confirmed_height: batch_end, + blocks_until_ready: batch_end.saturating_sub(confirmed_height), + }; + } + + BasePlanAction::FetchWindow { + chain_tip, + confirmed_height, + start: next_needed_height, + end: batch_end, + } +} + +impl BaseObserverCore { + pub fn plan_tick(&self, state: BasePlanState) -> BasePlanAction { + plan_base_tick(BasePlanInput { + state, + batch_size: self.batch_size, + confirmation_depth: self.confirmation_depth, + }) + } +} + +pub struct BaseObserverRunner { + pub core: BaseObserverCore, + pub source: S, + pub kernel: K, +} + +impl BaseObserverRunner +where + S: BaseSourcePort, + K: KernelStatePort, +{ + pub async fn tick_once(&self) -> Result { + let state = if self.kernel.peek_base_hold().await? { + BasePlanState::HoldActive + } else { + BasePlanState::Active { + chain_tip: self.source.chain_tip_height().await?, + next_needed_height: self.kernel.peek_base_next_height().await?, + } + }; + let action = self.core.plan_tick(state); + + if let BasePlanAction::FetchWindow { start, end, .. } = action { + let batch = self.source.fetch_batch(start, end).await?; + let _ = self + .kernel + .emit_chain_event(ChainEvent::Base(batch)) + .await?; + } + + Ok(action) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn no_pending_height_when_kernel_has_no_work() { + let action = plan_base_tick(BasePlanInput { + state: BasePlanState::Active { + chain_tip: 100, + next_needed_height: None, + }, + batch_size: 100, + confirmation_depth: 5, + }); + assert_eq!(action, BasePlanAction::NoPendingHeight { chain_tip: 100 }); + } + + #[test] + fn hold_active_short_circuits_planner() { + let action = plan_base_tick(BasePlanInput { + state: BasePlanState::HoldActive, + batch_size: 100, + confirmation_depth: 5, + }); + assert_eq!(action, BasePlanAction::HoldActive); + } + + #[test] + fn invalid_when_batch_size_zero() { + let action = plan_base_tick(BasePlanInput { + state: BasePlanState::Active { + chain_tip: 100, + next_needed_height: Some(1), + }, + batch_size: 0, + confirmation_depth: 5, + }); + assert_eq!( + action, + BasePlanAction::InvalidConfig(BasePlanError::ZeroBatchSize) + ); + } + + #[test] + fn not_yet_confirmed_for_next_window() { + let action = plan_base_tick(BasePlanInput { + state: BasePlanState::Active { + chain_tip: 2800, + next_needed_height: Some(2001), + }, + batch_size: 1000, + confirmation_depth: 300, + }); + + assert!(matches!( + action, + BasePlanAction::NotYetConfirmed { + confirmed_height: 2500, + next_needed_height: 2001, + needed_confirmed_height: 3000, + .. + } + )); + } + + #[test] + fn fetches_exact_confirmed_window() { + let action = plan_base_tick(BasePlanInput { + state: BasePlanState::Active { + chain_tip: 2800, + next_needed_height: Some(1001), + }, + batch_size: 1000, + confirmation_depth: 300, + }); + + assert_eq!( + action, + BasePlanAction::FetchWindow { + chain_tip: 2800, + confirmed_height: 2500, + start: 1001, + end: 2000, + } + ); + } + + #[test] + fn detects_height_overflow() { + let action = plan_base_tick(BasePlanInput { + state: BasePlanState::Active { + chain_tip: u64::MAX, + next_needed_height: Some(u64::MAX), + }, + batch_size: 2, + confirmation_depth: 1, + }); + + assert_eq!( + action, + BasePlanAction::InvalidConfig(BasePlanError::HeightOverflow) + ); + } +} diff --git a/crates/bridge/src/core/mod.rs b/crates/bridge/src/core/mod.rs new file mode 100644 index 000000000..23e322f9b --- /dev/null +++ b/crates/bridge/src/core/mod.rs @@ -0,0 +1,4 @@ +pub mod base_observer; +pub mod nock_observer; +pub mod posting; +pub mod signing; diff --git a/crates/bridge/src/core/nock_observer.rs b/crates/bridge/src/core/nock_observer.rs new file mode 100644 index 000000000..975c3398c --- /dev/null +++ b/crates/bridge/src/core/nock_observer.rs @@ -0,0 +1,208 @@ +use crate::errors::BridgeError; +use crate::ports::{KernelStatePort, NockSourcePort}; +use crate::runtime::ChainEvent; + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct NockObserverCore { + pub confirmation_depth: u64, +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct NockPlanInput { + pub tip_height: Option, + pub next_needed_height: Option, + pub confirmation_depth: u64, +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum NockPlanError { + ZeroConfirmationDepth, +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum NockPlanAction { + NoTipAvailable, + NoPendingHeight { + tip_height: u64, + confirmed_target: u64, + }, + InvalidConfig(NockPlanError), + BootstrapUnconfirmed { + tip_height: u64, + confirmation_depth: u64, + }, + NotYetConfirmed { + tip_height: u64, + confirmed_target: u64, + next_needed_height: u64, + }, + FetchHeight { + tip_height: u64, + confirmed_target: u64, + height: u64, + }, +} + +pub fn plan_nock_tick(input: NockPlanInput) -> NockPlanAction { + if input.confirmation_depth == 0 { + return NockPlanAction::InvalidConfig(NockPlanError::ZeroConfirmationDepth); + } + + let Some(tip_height) = input.tip_height else { + return NockPlanAction::NoTipAvailable; + }; + + let confirmed_target = tip_height.saturating_sub(input.confirmation_depth); + if confirmed_target == 0 { + return NockPlanAction::BootstrapUnconfirmed { + tip_height, + confirmation_depth: input.confirmation_depth, + }; + } + + let Some(next_needed_height) = input.next_needed_height else { + return NockPlanAction::NoPendingHeight { + tip_height, + confirmed_target, + }; + }; + + if confirmed_target < next_needed_height { + return NockPlanAction::NotYetConfirmed { + tip_height, + confirmed_target, + next_needed_height, + }; + } + + NockPlanAction::FetchHeight { + tip_height, + confirmed_target, + height: next_needed_height, + } +} + +impl NockObserverCore { + pub fn plan_tick( + &self, + tip_height: Option, + next_needed_height: Option, + ) -> NockPlanAction { + plan_nock_tick(NockPlanInput { + tip_height, + next_needed_height, + confirmation_depth: self.confirmation_depth, + }) + } +} + +pub struct NockObserverRunner { + pub core: NockObserverCore, + pub source: S, + pub kernel: K, +} + +impl NockObserverRunner +where + S: NockSourcePort, + K: KernelStatePort, +{ + pub async fn tick_once(&mut self) -> Result { + let tip_info = self.source.tip_info().await?; + let tip_height = tip_info.as_ref().map(|tip| tip.height); + let next_needed_height = self.kernel.peek_nock_next_height().await?; + let action = self.core.plan_tick(tip_height, next_needed_height); + + if let NockPlanAction::FetchHeight { height, .. } = action { + if let Some(block) = self.source.fetch_block_at_height(height).await? { + let _ = self + .kernel + .emit_chain_event(ChainEvent::Nock(block)) + .await?; + } + } + + Ok(action) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn no_tip_when_heaviest_missing() { + let action = plan_nock_tick(NockPlanInput { + tip_height: None, + next_needed_height: Some(1), + confirmation_depth: 10, + }); + assert_eq!(action, NockPlanAction::NoTipAvailable); + } + + #[test] + fn no_pending_height_when_kernel_idle() { + let action = plan_nock_tick(NockPlanInput { + tip_height: Some(200), + next_needed_height: None, + confirmation_depth: 100, + }); + assert_eq!( + action, + NockPlanAction::NoPendingHeight { + tip_height: 200, + confirmed_target: 100, + } + ); + } + + #[test] + fn bootstrap_until_confirmed_target_nonzero() { + let action = plan_nock_tick(NockPlanInput { + tip_height: Some(100), + next_needed_height: Some(1), + confirmation_depth: 100, + }); + assert_eq!( + action, + NockPlanAction::BootstrapUnconfirmed { + tip_height: 100, + confirmation_depth: 100, + } + ); + } + + #[test] + fn not_yet_confirmed_when_target_behind_next_needed() { + let action = plan_nock_tick(NockPlanInput { + tip_height: Some(250), + next_needed_height: Some(175), + confirmation_depth: 100, + }); + assert_eq!( + action, + NockPlanAction::NotYetConfirmed { + tip_height: 250, + confirmed_target: 150, + next_needed_height: 175, + } + ); + } + + #[test] + fn fetches_when_target_confirmed() { + let action = plan_nock_tick(NockPlanInput { + tip_height: Some(250), + next_needed_height: Some(150), + confirmation_depth: 100, + }); + assert_eq!( + action, + NockPlanAction::FetchHeight { + tip_height: 250, + confirmed_target: 150, + height: 150, + } + ); + } +} diff --git a/crates/bridge/src/core/posting.rs b/crates/bridge/src/core/posting.rs new file mode 100644 index 000000000..eb483fd36 --- /dev/null +++ b/crates/bridge/src/core/posting.rs @@ -0,0 +1,366 @@ +use async_trait::async_trait; + +use crate::errors::BridgeError; + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct PostingCandidateDecisionInput { + pub proposal_nonce: u64, + pub next_nonce: u64, + pub my_node_id: usize, + pub current_proposer: usize, + pub num_nodes: usize, + pub ready_at: Option, + pub now_secs: u64, + pub failover_backoff_secs: u64, +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct PostingTickPlanInput { + pub next_nonce: u64, + pub my_node_id: usize, + pub current_proposer: usize, + pub num_nodes: usize, + pub now_secs: u64, + pub failover_backoff_secs: u64, +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct PostingReadyProposal { + pub nonce: u64, + pub ready_at: Option, +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum PostingCandidateDecision { + MarkConfirmedOnChain, + WaitForEarlierNonce, + NotMyTurn, + Submit { is_proposer: bool }, +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct PostingTickRunnerInput { + pub my_node_id: usize, + pub current_proposer: usize, + pub num_nodes: usize, + pub now_secs: u64, + pub failover_backoff_secs: u64, +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct PostingTickPlannedCandidate { + pub proposal: PostingReadyProposal, + pub decision: PostingCandidateDecision, +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub enum PostingTickRunnerAction { + NoReadyProposals, + Planned { + next_nonce: u64, + candidates: Vec, + }, +} + +#[async_trait] +pub trait PostingTickSource: Send + Sync { + async fn ready_proposals(&self) -> Result, BridgeError>; + async fn next_nonce(&self) -> Result; +} + +pub struct PostingTickRunner { + pub source: S, +} + +impl PostingTickRunner +where + S: PostingTickSource, +{ + pub async fn tick_once( + &self, + input: PostingTickRunnerInput, + ) -> Result { + PostingPlanner::tick_once(&self.source, input).await + } +} + +pub struct PostingPlanner; + +impl PostingPlanner { + pub async fn tick_once( + source: &S, + input: PostingTickRunnerInput, + ) -> Result + where + S: PostingTickSource, + { + let proposals = source.ready_proposals().await?; + if proposals.is_empty() { + return Ok(PostingTickRunnerAction::NoReadyProposals); + } + + let next_nonce = source.next_nonce().await?; + let decisions = Self::plan_tick( + PostingTickPlanInput { + next_nonce, + my_node_id: input.my_node_id, + current_proposer: input.current_proposer, + num_nodes: input.num_nodes, + now_secs: input.now_secs, + failover_backoff_secs: input.failover_backoff_secs, + }, + &proposals, + ); + let candidates = proposals + .into_iter() + .zip(decisions) + .map(|(proposal, decision)| PostingTickPlannedCandidate { proposal, decision }) + .collect(); + + Ok(PostingTickRunnerAction::Planned { + next_nonce, + candidates, + }) + } + + pub fn plan_tick( + input: PostingTickPlanInput, + proposals: &[PostingReadyProposal], + ) -> Vec { + proposals + .iter() + .map(|proposal| { + Self::plan_candidate(PostingCandidateDecisionInput { + proposal_nonce: proposal.nonce, + next_nonce: input.next_nonce, + my_node_id: input.my_node_id, + current_proposer: input.current_proposer, + num_nodes: input.num_nodes, + ready_at: proposal.ready_at, + now_secs: input.now_secs, + failover_backoff_secs: input.failover_backoff_secs, + }) + }) + .collect() + } + + pub fn plan_candidate(input: PostingCandidateDecisionInput) -> PostingCandidateDecision { + if input.proposal_nonce < input.next_nonce { + return PostingCandidateDecision::MarkConfirmedOnChain; + } + + if input.proposal_nonce > input.next_nonce { + return PostingCandidateDecision::WaitForEarlierNonce; + } + + if input.my_node_id == input.current_proposer { + return PostingCandidateDecision::Submit { is_proposer: true }; + } + + let Some(ready_at) = input.ready_at else { + return PostingCandidateDecision::NotMyTurn; + }; + + let failover_slot = if input.my_node_id > input.current_proposer { + input.my_node_id - input.current_proposer + } else { + input.num_nodes - input.current_proposer + input.my_node_id + }; + let required_wait = input.failover_backoff_secs * failover_slot as u64; + let elapsed = input.now_secs.saturating_sub(ready_at); + if elapsed >= required_wait { + PostingCandidateDecision::Submit { is_proposer: false } + } else { + PostingCandidateDecision::NotMyTurn + } + } +} + +#[cfg(test)] +mod tests { + use std::sync::Arc; + + use tokio::sync::Mutex; + + use super::*; + + #[test] + fn marks_confirmed_when_nonce_is_behind_chain() { + let decision = PostingPlanner::plan_candidate(PostingCandidateDecisionInput { + proposal_nonce: 4, + next_nonce: 5, + my_node_id: 0, + current_proposer: 0, + num_nodes: 5, + ready_at: Some(100), + now_secs: 200, + failover_backoff_secs: 120, + }); + assert_eq!(decision, PostingCandidateDecision::MarkConfirmedOnChain); + } + + #[test] + fn waits_when_nonce_is_ahead_of_chain() { + let decision = PostingPlanner::plan_candidate(PostingCandidateDecisionInput { + proposal_nonce: 6, + next_nonce: 5, + my_node_id: 0, + current_proposer: 0, + num_nodes: 5, + ready_at: Some(100), + now_secs: 200, + failover_backoff_secs: 120, + }); + assert_eq!(decision, PostingCandidateDecision::WaitForEarlierNonce); + } + + #[test] + fn proposer_submits_immediately() { + let decision = PostingPlanner::plan_candidate(PostingCandidateDecisionInput { + proposal_nonce: 5, + next_nonce: 5, + my_node_id: 2, + current_proposer: 2, + num_nodes: 5, + ready_at: Some(100), + now_secs: 101, + failover_backoff_secs: 120, + }); + assert_eq!( + decision, + PostingCandidateDecision::Submit { is_proposer: true } + ); + } + + #[test] + fn failover_node_waits_until_backoff_elapsed() { + let input = PostingCandidateDecisionInput { + proposal_nonce: 5, + next_nonce: 5, + my_node_id: 3, + current_proposer: 2, + num_nodes: 5, + ready_at: Some(100), + now_secs: 200, + failover_backoff_secs: 120, + }; + + let decision_wait = PostingPlanner::plan_candidate(input); + assert_eq!(decision_wait, PostingCandidateDecision::NotMyTurn); + + let decision_submit = PostingPlanner::plan_candidate(PostingCandidateDecisionInput { + now_secs: 220, + ..input + }); + assert_eq!( + decision_submit, + PostingCandidateDecision::Submit { is_proposer: false } + ); + } + + #[test] + fn plan_tick_returns_one_decision_per_proposal_in_order() { + let decisions = PostingPlanner::plan_tick( + PostingTickPlanInput { + next_nonce: 5, + my_node_id: 1, + current_proposer: 0, + num_nodes: 3, + now_secs: 200, + failover_backoff_secs: 120, + }, + &[ + PostingReadyProposal { + nonce: 4, + ready_at: Some(100), + }, + PostingReadyProposal { + nonce: 6, + ready_at: Some(100), + }, + PostingReadyProposal { + nonce: 5, + ready_at: Some(100), + }, + ], + ); + + assert_eq!( + decisions, + vec![ + PostingCandidateDecision::MarkConfirmedOnChain, + PostingCandidateDecision::WaitForEarlierNonce, + PostingCandidateDecision::NotMyTurn, + ] + ); + } + + #[derive(Clone)] + struct FakePostingTickSource { + proposals: Arc>>, + next_nonce: u64, + } + + #[async_trait] + impl PostingTickSource for FakePostingTickSource { + async fn ready_proposals(&self) -> Result, BridgeError> { + Ok(self.proposals.lock().await.clone()) + } + + async fn next_nonce(&self) -> Result { + Ok(self.next_nonce) + } + } + + #[tokio::test] + async fn tick_runner_queries_source_and_returns_planned_actions() { + let source = FakePostingTickSource { + proposals: Arc::new(Mutex::new(vec![ + PostingReadyProposal { + nonce: 4, + ready_at: Some(100), + }, + PostingReadyProposal { + nonce: 5, + ready_at: Some(100), + }, + ])), + next_nonce: 5, + }; + let runner = PostingTickRunner { source }; + + let action = runner + .tick_once(PostingTickRunnerInput { + my_node_id: 1, + current_proposer: 0, + num_nodes: 3, + now_secs: 200, + failover_backoff_secs: 120, + }) + .await + .expect("tick succeeds"); + + assert_eq!( + action, + PostingTickRunnerAction::Planned { + next_nonce: 5, + candidates: vec![ + PostingTickPlannedCandidate { + proposal: PostingReadyProposal { + nonce: 4, + ready_at: Some(100), + }, + decision: PostingCandidateDecision::MarkConfirmedOnChain + }, + PostingTickPlannedCandidate { + proposal: PostingReadyProposal { + nonce: 5, + ready_at: Some(100), + }, + decision: PostingCandidateDecision::NotMyTurn + }, + ], + } + ); + } +} diff --git a/crates/bridge/src/core/signing.rs b/crates/bridge/src/core/signing.rs new file mode 100644 index 000000000..ef24503f3 --- /dev/null +++ b/crates/bridge/src/core/signing.rs @@ -0,0 +1,444 @@ +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct SigningTipDecisionInput { + pub tip_height: Option, + pub nonce_epoch_start_height: u64, + pub logged_epoch_ready: bool, +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum SigningTipDecision { + WaitForTip, + WaitForEpochStart { tip_height: u64 }, + EpochReachedFirstTime { tip_height: u64 }, + Ready { tip_height: u64 }, +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct SigningNonceBaseDecisionInput { + pub last_chain_nonce: u64, + pub nonce_epoch_base: u64, +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum SigningNonceBaseDecision { + StopNonceEpochMismatch, + Continue, +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct SigningLogProgressDecisionInput { + pub last_chain_nonce: u64, + pub first_epoch_nonce: u64, + pub log_len: u64, +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum SigningLogProgressDecision { + WaitForLogCatchup { spent_epoch_nonces: u64 }, + Continue { next_nonce: u64 }, +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct SigningEpochBoundsDecisionInput { + pub is_before_start_key: bool, +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum SigningEpochBoundsDecision { + StopRecordBeforeStart, + Continue, +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct SigningCandidatePrecheckInput { + pub is_confirmed: bool, + pub has_my_signature: bool, +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum SigningCandidatePrecheckDecision { + SkipConfirmed, + SkipAlreadySigned, + CheckProcessedOnChain, +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct SigningProcessedDecisionInput { + pub processed_on_chain: bool, +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum SigningProcessedDecision { + SkipProcessed, + ContinueSign, +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct SigningTickPlanInput { + pub tip_height: Option, + pub nonce_epoch_start_height: u64, + pub logged_epoch_ready: bool, + pub last_chain_nonce: Option, + pub nonce_epoch_base: u64, + pub first_epoch_nonce: u64, + pub log_len: Option, +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum SigningTickPlanAction { + WaitForTip, + WaitForEpochStart { + tip_height: u64, + }, + NeedLastChainNonce { + tip_height: u64, + reached_epoch_start: bool, + }, + StopNonceEpochMismatch { + tip_height: u64, + reached_epoch_start: bool, + last_chain_nonce: u64, + nonce_epoch_base: u64, + }, + NeedLogLen { + tip_height: u64, + reached_epoch_start: bool, + last_chain_nonce: u64, + }, + WaitForLogCatchup { + tip_height: u64, + reached_epoch_start: bool, + last_chain_nonce: u64, + nonce_epoch_base: u64, + log_len: u64, + spent_epoch_nonces: u64, + }, + Continue { + tip_height: u64, + reached_epoch_start: bool, + last_chain_nonce: u64, + next_nonce: u64, + }, +} + +pub struct SigningPlanner; + +impl SigningPlanner { + pub fn plan_tick(input: SigningTickPlanInput) -> SigningTickPlanAction { + let (tip_height, reached_epoch_start) = match Self::plan_tip(SigningTipDecisionInput { + tip_height: input.tip_height, + nonce_epoch_start_height: input.nonce_epoch_start_height, + logged_epoch_ready: input.logged_epoch_ready, + }) { + SigningTipDecision::WaitForTip => return SigningTickPlanAction::WaitForTip, + SigningTipDecision::WaitForEpochStart { tip_height } => { + return SigningTickPlanAction::WaitForEpochStart { tip_height }; + } + SigningTipDecision::EpochReachedFirstTime { tip_height } => (tip_height, true), + SigningTipDecision::Ready { tip_height } => (tip_height, false), + }; + + let Some(last_chain_nonce) = input.last_chain_nonce else { + return SigningTickPlanAction::NeedLastChainNonce { + tip_height, + reached_epoch_start, + }; + }; + + if matches!( + Self::plan_nonce_base(SigningNonceBaseDecisionInput { + last_chain_nonce, + nonce_epoch_base: input.nonce_epoch_base, + }), + SigningNonceBaseDecision::StopNonceEpochMismatch + ) { + return SigningTickPlanAction::StopNonceEpochMismatch { + tip_height, + reached_epoch_start, + last_chain_nonce, + nonce_epoch_base: input.nonce_epoch_base, + }; + } + + let Some(log_len) = input.log_len else { + return SigningTickPlanAction::NeedLogLen { + tip_height, + reached_epoch_start, + last_chain_nonce, + }; + }; + + match Self::plan_log_progress(SigningLogProgressDecisionInput { + last_chain_nonce, + first_epoch_nonce: input.first_epoch_nonce, + log_len, + }) { + SigningLogProgressDecision::WaitForLogCatchup { spent_epoch_nonces } => { + SigningTickPlanAction::WaitForLogCatchup { + tip_height, + reached_epoch_start, + last_chain_nonce, + nonce_epoch_base: input.nonce_epoch_base, + log_len, + spent_epoch_nonces, + } + } + SigningLogProgressDecision::Continue { next_nonce } => { + SigningTickPlanAction::Continue { + tip_height, + reached_epoch_start, + last_chain_nonce, + next_nonce, + } + } + } + } + + pub fn plan_tip(input: SigningTipDecisionInput) -> SigningTipDecision { + let Some(tip_height) = input.tip_height else { + return SigningTipDecision::WaitForTip; + }; + + if tip_height < input.nonce_epoch_start_height { + return SigningTipDecision::WaitForEpochStart { tip_height }; + } + + if !input.logged_epoch_ready { + return SigningTipDecision::EpochReachedFirstTime { tip_height }; + } + + SigningTipDecision::Ready { tip_height } + } + + pub fn plan_nonce_base(input: SigningNonceBaseDecisionInput) -> SigningNonceBaseDecision { + if input.last_chain_nonce < input.nonce_epoch_base { + SigningNonceBaseDecision::StopNonceEpochMismatch + } else { + SigningNonceBaseDecision::Continue + } + } + + pub fn plan_log_progress(input: SigningLogProgressDecisionInput) -> SigningLogProgressDecision { + let spent_epoch_nonces = if input.last_chain_nonce < input.first_epoch_nonce { + 0 + } else { + input.last_chain_nonce - input.first_epoch_nonce + 1 + }; + + if spent_epoch_nonces > input.log_len { + SigningLogProgressDecision::WaitForLogCatchup { spent_epoch_nonces } + } else { + SigningLogProgressDecision::Continue { + next_nonce: input.last_chain_nonce + 1, + } + } + } + + pub fn plan_candidate_precheck( + input: SigningCandidatePrecheckInput, + ) -> SigningCandidatePrecheckDecision { + if input.is_confirmed { + SigningCandidatePrecheckDecision::SkipConfirmed + } else if input.has_my_signature { + SigningCandidatePrecheckDecision::SkipAlreadySigned + } else { + SigningCandidatePrecheckDecision::CheckProcessedOnChain + } + } + + pub fn plan_epoch_bounds(input: SigningEpochBoundsDecisionInput) -> SigningEpochBoundsDecision { + if input.is_before_start_key { + SigningEpochBoundsDecision::StopRecordBeforeStart + } else { + SigningEpochBoundsDecision::Continue + } + } + + pub fn plan_processed(input: SigningProcessedDecisionInput) -> SigningProcessedDecision { + if !input.processed_on_chain { + return SigningProcessedDecision::ContinueSign; + } + SigningProcessedDecision::SkipProcessed + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn waits_when_tip_unavailable() { + let decision = SigningPlanner::plan_tip(SigningTipDecisionInput { + tip_height: None, + nonce_epoch_start_height: 100, + logged_epoch_ready: false, + }); + assert_eq!(decision, SigningTipDecision::WaitForTip); + } + + #[test] + fn first_epoch_ready_tick_is_distinct() { + let decision = SigningPlanner::plan_tip(SigningTipDecisionInput { + tip_height: Some(100), + nonce_epoch_start_height: 100, + logged_epoch_ready: false, + }); + assert_eq!( + decision, + SigningTipDecision::EpochReachedFirstTime { tip_height: 100 } + ); + } + + #[test] + fn detects_nonce_epoch_mismatch() { + let decision = SigningPlanner::plan_nonce_base(SigningNonceBaseDecisionInput { + last_chain_nonce: 5, + nonce_epoch_base: 6, + }); + assert_eq!(decision, SigningNonceBaseDecision::StopNonceEpochMismatch); + } + + #[test] + fn waits_for_log_when_chain_prefix_exceeds_log() { + let decision = SigningPlanner::plan_log_progress(SigningLogProgressDecisionInput { + last_chain_nonce: 10, + first_epoch_nonce: 1, + log_len: 5, + }); + assert_eq!( + decision, + SigningLogProgressDecision::WaitForLogCatchup { + spent_epoch_nonces: 10 + } + ); + } + + #[test] + fn continues_with_next_nonce_when_log_caught_up() { + let decision = SigningPlanner::plan_log_progress(SigningLogProgressDecisionInput { + last_chain_nonce: 10, + first_epoch_nonce: 1, + log_len: 10, + }); + assert_eq!( + decision, + SigningLogProgressDecision::Continue { next_nonce: 11 } + ); + } + + #[test] + fn candidate_precheck_short_circuits_confirmed_and_signed() { + let confirmed = SigningPlanner::plan_candidate_precheck(SigningCandidatePrecheckInput { + is_confirmed: true, + has_my_signature: false, + }); + assert_eq!(confirmed, SigningCandidatePrecheckDecision::SkipConfirmed); + + let signed = SigningPlanner::plan_candidate_precheck(SigningCandidatePrecheckInput { + is_confirmed: false, + has_my_signature: true, + }); + assert_eq!(signed, SigningCandidatePrecheckDecision::SkipAlreadySigned); + + let check = SigningPlanner::plan_candidate_precheck(SigningCandidatePrecheckInput { + is_confirmed: false, + has_my_signature: false, + }); + assert_eq!( + check, + SigningCandidatePrecheckDecision::CheckProcessedOnChain + ); + } + + #[test] + fn epoch_bounds_decision_detects_before_start_key() { + let stop = SigningPlanner::plan_epoch_bounds(SigningEpochBoundsDecisionInput { + is_before_start_key: true, + }); + assert_eq!(stop, SigningEpochBoundsDecision::StopRecordBeforeStart); + + let cont = SigningPlanner::plan_epoch_bounds(SigningEpochBoundsDecisionInput { + is_before_start_key: false, + }); + assert_eq!(cont, SigningEpochBoundsDecision::Continue); + } + + #[test] + fn processed_decision_skips_any_processed_candidate() { + let skip = SigningPlanner::plan_processed(SigningProcessedDecisionInput { + processed_on_chain: true, + }); + assert_eq!(skip, SigningProcessedDecision::SkipProcessed); + + let cont = SigningPlanner::plan_processed(SigningProcessedDecisionInput { + processed_on_chain: false, + }); + assert_eq!(cont, SigningProcessedDecision::ContinueSign); + } + + #[test] + fn tick_plan_needs_chain_nonce_after_tip_gate() { + let action = SigningPlanner::plan_tick(SigningTickPlanInput { + tip_height: Some(100), + nonce_epoch_start_height: 100, + logged_epoch_ready: false, + last_chain_nonce: None, + nonce_epoch_base: 0, + first_epoch_nonce: 1, + log_len: None, + }); + + assert_eq!( + action, + SigningTickPlanAction::NeedLastChainNonce { + tip_height: 100, + reached_epoch_start: true + } + ); + } + + #[test] + fn tick_plan_requests_log_len_after_nonce_checks() { + let action = SigningPlanner::plan_tick(SigningTickPlanInput { + tip_height: Some(100), + nonce_epoch_start_height: 100, + logged_epoch_ready: true, + last_chain_nonce: Some(5), + nonce_epoch_base: 0, + first_epoch_nonce: 1, + log_len: None, + }); + + assert_eq!( + action, + SigningTickPlanAction::NeedLogLen { + tip_height: 100, + reached_epoch_start: false, + last_chain_nonce: 5 + } + ); + } + + #[test] + fn tick_plan_continues_with_next_nonce_when_ready() { + let action = SigningPlanner::plan_tick(SigningTickPlanInput { + tip_height: Some(100), + nonce_epoch_start_height: 100, + logged_epoch_ready: true, + last_chain_nonce: Some(5), + nonce_epoch_base: 0, + first_epoch_nonce: 1, + log_len: Some(10), + }); + + assert_eq!( + action, + SigningTickPlanAction::Continue { + tip_height: 100, + reached_epoch_start: false, + last_chain_nonce: 5, + next_nonce: 6 + } + ); + } +} diff --git a/crates/bridge/src/deposit_log.rs b/crates/bridge/src/deposit_log.rs index 1045c4597..27631a5ba 100644 --- a/crates/bridge/src/deposit_log.rs +++ b/crates/bridge/src/deposit_log.rs @@ -9,13 +9,14 @@ use diesel::prelude::*; use diesel::sqlite::SqliteConnection; use diesel::OptionalExtension; use nockchain_types::v1::Name; +use nockvm::noun::NounAllocator; use tracing::{info, warn}; use crate::bridge_status::BridgeStatus; use crate::config::NonceEpochConfig; use crate::errors::BridgeError; -use crate::ethereum::BaseBridge; use crate::metrics; +use crate::ports::BaseContractPort; use crate::schema::deposit_log; use crate::stop::StopHandle; use crate::tui::state::TuiStatus; @@ -262,6 +263,18 @@ impl DepositLog { }) } + /// Return the maximum nonce present in the log at/after the epoch start key. + pub async fn max_nonce_in_epoch( + &self, + epoch: &NonceEpochConfig, + ) -> Result, BridgeError> { + let count = self.number_of_deposits_in_epoch(epoch).await?; + if count == 0 { + return Ok(None); + } + Ok(Some(epoch.first_epoch_nonce().saturating_add(count - 1))) + } + /// Fetch a single entry by nonce, if it exists in the epoch window. /// Returns None if the nonce is before the epoch or beyond the log length. pub async fn get_by_nonce( @@ -664,11 +677,13 @@ pub async fn sync_deposit_log_from_hashchain( "deposit log sync from nock hashchain complete" ); + metrics::advance_deposit_log_max_nonce(inserted, nonce_epoch.first_epoch_nonce()); + Ok(inserted) } -pub async fn validate_deposit_log_against_chain_nonce_prefix( - base_bridge: Arc, +pub async fn validate_deposit_log_against_chain_nonce_prefix( + base_bridge: Arc, deposit_log: Arc, nonce_epoch: NonceEpochConfig, ) -> Result<(), BridgeError> { @@ -806,6 +821,8 @@ pub async fn persist_commit_nock_deposits_requests( } } + metrics::advance_deposit_log_max_nonce(inserted, nonce_epoch.first_epoch_nonce()); + Ok(inserted) } @@ -848,11 +865,14 @@ pub fn create_commit_nock_deposits_driver( }; let root = unsafe { effect.root() }; - let bridge_effect = match BridgeEffect::from_noun(root) { - Ok(effect) => effect, - Err(err) => { - warn!("Failed to decode effect: {}", err); - continue; + let bridge_effect = { + let space = effect.noun_space(); + match BridgeEffect::from_noun(root, &space) { + Ok(effect) => effect, + Err(err) => { + warn!("Failed to decode effect: {}", err); + continue; + } } }; @@ -1386,6 +1406,7 @@ mod tests { }; assert_eq!(log.number_of_deposits_in_epoch(&epoch).await.unwrap(), 2); + assert_eq!(log.max_nonce_in_epoch(&epoch).await.unwrap(), Some(51)); assert_eq!(log.max_block_height(&epoch).await.unwrap(), Some(11)); let rows = log.records_from_nonce(50, 10, &epoch).await.unwrap(); @@ -1394,6 +1415,21 @@ mod tests { assert_eq!(rows[1].1.tx_id, next.tx_id); } + #[tokio::test] + async fn max_nonce_in_epoch_is_none_for_empty_epoch() { + let dir = TempDir::new().unwrap(); + let path = dir.path().join("deposit-log.sqlite"); + let log = DepositLog::open(path).await.unwrap(); + + let epoch = NonceEpochConfig { + base: 50, + start_height: 10, + start_tx_id: None, + }; + + assert_eq!(log.max_nonce_in_epoch(&epoch).await.unwrap(), None); + } + #[tokio::test] async fn records_from_nonce_below_epoch_returns_empty() { let dir = TempDir::new().unwrap(); diff --git a/crates/bridge/src/ethereum.rs b/crates/bridge/src/ethereum.rs index 366794389..36dbcbfe5 100644 --- a/crates/bridge/src/ethereum.rs +++ b/crates/bridge/src/ethereum.rs @@ -14,6 +14,7 @@ use alloy::signers::local::PrivateKeySigner; use alloy::sol_types::SolEvent; use alloy::transports::ws::WsConnect; use alloy::transports::TransportError; +use async_trait::async_trait; use backon::{ExponentialBuilder, Retryable}; use hex::encode as hex_encode; use op_alloy::network::Optimism; @@ -21,7 +22,11 @@ use tokio::time::{sleep, Duration}; use tracing::{debug, error, info, trace, warn}; use crate::bridge_status::BridgeStatus; +use crate::core::base_observer::{plan_base_tick, BasePlanAction, BasePlanInput, BasePlanState}; use crate::errors::BridgeError; +use crate::loop_policy::BaseObserverLoopPolicy; +use crate::metrics; +use crate::ports::{BaseContractPort, BaseSourcePort}; use crate::runtime::{BaseBlockBatch, BridgeEvent, BridgeRuntimeHandle, ChainEvent}; use crate::stop::StopHandle; use crate::tui::types::{BridgeTx, TxDirection, TxStatus}; @@ -116,6 +121,7 @@ fn confirmed_batch(chain_tip: u64, batch_size: u64, confirmation_depth: u64) -> /// /// Returns `Some((start, end))` if a full batch is confirmed, `None` otherwise. /// The batch is always exactly `batch_size` blocks, starting at `next_needed_height`. +#[cfg(test)] fn next_confirmed_window( next_needed_height: u64, confirmed_height: u64, @@ -131,17 +137,30 @@ fn next_confirmed_window( } #[allow(dead_code)] -pub struct BaseBridge { +struct BaseBridgeDeps { provider: DynProvider, wallet: EthereumWallet, + runtime_handle: Arc, + stop: StopHandle, +} + +struct BaseBridgeContracts { inbox_contract_address: Address, nock_contract_address: Address, - runtime_handle: Arc, +} + +struct BaseBridgeConfig { /// Batch size for fetching base blocks (must match Hoon kernel's base-blocks-chunk) batch_size: u64, /// Number of confirmations required before emitting a batch to the kernel. confirmation_depth: u64, - stop: StopHandle, +} + +#[allow(dead_code)] +pub struct BaseBridge { + deps: BaseBridgeDeps, + contracts: BaseBridgeContracts, + config: BaseBridgeConfig, } impl BaseBridge { @@ -219,14 +238,20 @@ impl BaseBridge { }; Ok(Self { - provider, - wallet, - inbox_contract_address, - nock_contract_address, - runtime_handle, - batch_size, - confirmation_depth, - stop, + deps: BaseBridgeDeps { + provider, + wallet, + runtime_handle, + stop, + }, + contracts: BaseBridgeContracts { + inbox_contract_address, + nock_contract_address, + }, + config: BaseBridgeConfig { + batch_size, + confirmation_depth, + }, }) } @@ -256,7 +281,10 @@ impl BaseBridge { .map(|sig| Bytes::from(sig.into_vec())) .collect::>(); - let inbox = MessageInbox::new(self.inbox_contract_address, self.provider.clone()); + let inbox = MessageInbox::new( + self.contracts.inbox_contract_address, + self.deps.provider.clone(), + ); let tx_id_sol = MessageInbox::Tip5Hash { limbs: submission.tx_id.to_array(), @@ -280,7 +308,7 @@ impl BaseBridge { as_of_sol, deposit_nonce, eth_sigs, ) .from(NetworkWallet::::default_signer_address( - &self.wallet, + &self.deps.wallet, )) .send() .await @@ -325,7 +353,10 @@ impl BaseBridge { /// This is the source of truth for which nonce to submit next. /// The next valid deposit must have nonce == lastDepositNonce + 1. pub async fn get_last_deposit_nonce(&self) -> Result { - let inbox = MessageInbox::new(self.inbox_contract_address, self.provider.clone()); + let inbox = MessageInbox::new( + self.contracts.inbox_contract_address, + self.deps.provider.clone(), + ); let nonce = inbox.lastDepositNonce().call().await.map_err(|e| { BridgeError::BaseBridgeQuery(format!("Failed to query lastDepositNonce: {}", e)) @@ -341,7 +372,10 @@ impl BaseBridge { use alloy::primitives::B256; use tiny_keccak::{Hasher, Keccak}; - let inbox = MessageInbox::new(self.inbox_contract_address, self.provider.clone()); + let inbox = MessageInbox::new( + self.contracts.inbox_contract_address, + self.deps.provider.clone(), + ); let be40 = tx_id.to_be_limb_bytes(); let mut hasher = Keccak::v256(); @@ -367,13 +401,13 @@ impl BaseBridge { ) -> Result, BridgeError> { let tx_hash = keccak256(tx_id.to_be_limb_bytes()); let filter = Filter::new() - .address(self.inbox_contract_address) + .address(self.contracts.inbox_contract_address) .event_signature(MessageInbox::DepositProcessed::SIGNATURE_HASH) .topic1(tx_hash) .from_block(from_block) .to_block(BlockNumberOrTag::Latest); - let logs = self.provider.get_logs(&filter).await.map_err(|e| { + let logs = self.deps.provider.get_logs(&filter).await.map_err(|e| { BridgeError::BaseBridgeQuery(format!("Failed to query DepositProcessed logs: {e}")) })?; @@ -434,30 +468,44 @@ impl BaseBridge { pub async fn stream_base_events( &self, bridge_status: Option, + ) -> Result<(), BridgeError> { + self.stream_base_events_with_policy(bridge_status, BaseObserverLoopPolicy::default()) + .await + } + + pub async fn stream_base_events_with_policy( + &self, + bridge_status: Option, + policy: BaseObserverLoopPolicy, ) -> Result<(), BridgeError> { info!( "starting base bridge event stream (confirmation_depth={}, batch_size={})", - self.confirmation_depth, self.batch_size + self.config.confirmation_depth, self.config.batch_size ); - // Base blocks are ~2s, but we need 300 confirmations (~10 min), so 30s polling is fine. - const BASE_POLL_INTERVAL: Duration = Duration::from_secs(30); - let rpc_backoff = || { - ExponentialBuilder::default() - .with_min_delay(Duration::from_secs(5)) - .with_max_delay(Duration::from_secs(120)) - .with_jitter() - .with_max_times(10) - }; + let poll_interval = policy.poll_interval; + let rpc_retry = policy.rpc_retry; + let rpc_backoff = || rpc_retry.exponential_builder(); loop { - if self.stop.is_stopped() { - sleep(BASE_POLL_INTERVAL).await; + if self.deps.stop.is_stopped() { + sleep(poll_interval).await; continue; } - sleep(BASE_POLL_INTERVAL).await; + sleep(poll_interval).await; - let chain_tip: u64 = match (|| async { self.provider.get_block_number().await }) + let base_hold_active = match self.deps.runtime_handle.peek_base_hold().await { + Ok(active) => active, + Err(err) => { + warn!( + target: "bridge.base.observer", + error=%err, + "failed to peek base hold state" + ); + continue; + } + }; + let chain_tip = match (|| async { self.deps.provider.get_block_number().await }) .retry(rpc_backoff()) .when(is_rate_limit_error) .notify(|err, dur| { @@ -470,31 +518,59 @@ impl BaseBridge { }) .await { - Ok(tip) => tip, + Ok(tip) => { + metrics::update_base_tip_height(Some(tip)); + Some(tip) + } Err(e) => { + metrics::update_base_tip_height(None); error!( target: "bridge.base.observer", error=%e, "failed to get block number after retries" ); + None + } + }; + let state = if base_hold_active { + BasePlanState::HoldActive + } else { + let Some(chain_tip) = chain_tip else { continue; + }; + let next_needed_height = + match self.deps.runtime_handle.peek_base_next_height().await { + Ok(height) => height, + Err(err) => { + warn!( + target: "bridge.base.observer", + error=%err, + "failed to peek base next height" + ); + continue; + } + }; + BasePlanState::Active { + chain_tip, + next_needed_height, } }; - let confirmed_height = chain_tip.saturating_sub(self.confirmation_depth); - if confirmed_height < self.batch_size { - debug!( - target: "bridge.base.observer", - chain_tip, - confirmed_height, - "no confirmed batch yet (bootstrap)" - ); - continue; - } + let action = plan_base_tick(BasePlanInput { + state, + batch_size: self.config.batch_size, + confirmation_depth: self.config.confirmation_depth, + }); - let next_needed_height = match self.runtime_handle.peek_base_next_height().await { - Ok(Some(height)) => height, - Ok(None) => { + let (chain_tip, batch_start, batch_end) = match action { + BasePlanAction::HoldActive => { + debug!( + target: "bridge.base.observer", + "base hold active, skipping base batch fetch" + ); + continue; + } + BasePlanAction::NoPendingHeight { chain_tip } => { debug!( target: "bridge.base.observer", chain_tip, @@ -502,39 +578,52 @@ impl BaseBridge { ); continue; } - Err(err) => { - warn!( + BasePlanAction::InvalidConfig(err) => { + error!( target: "bridge.base.observer", - error=%err, - "failed to peek base next height" + error=?err, + batch_size=self.config.batch_size, + confirmation_depth=self.config.confirmation_depth, + "invalid observer planner configuration" ); continue; } - }; - - debug!( - target: "bridge.base.observer", - next_needed_height, - "kernel reports base-hashchain-next-height" - ); - - let Some((batch_start, batch_end)) = - next_confirmed_window(next_needed_height, confirmed_height, self.batch_size) - else { - // Calculate what we're waiting for to help debugging - let needed_confirmed = next_needed_height + self.batch_size - 1; - let blocks_until_ready = needed_confirmed.saturating_sub(confirmed_height); - debug!( - target: "bridge.base.observer", + BasePlanAction::NotYetConfirmed { chain_tip, confirmed_height, next_needed_height, - batch_size = self.batch_size, - needed_confirmed_height = needed_confirmed, + needed_confirmed_height, blocks_until_ready, - "batch not yet confirmed for kernel need" - ); - continue; + .. + } => { + debug!( + target: "bridge.base.observer", + chain_tip, + confirmed_height, + next_needed_height, + batch_size = self.config.batch_size, + needed_confirmed_height, + blocks_until_ready, + "batch not yet confirmed for kernel need" + ); + continue; + } + BasePlanAction::FetchWindow { + chain_tip, + start, + end, + confirmed_height, + .. + } => { + debug!( + target: "bridge.base.observer", + chain_tip, + confirmed_height, + next_needed_height = start, + "kernel reports base-hashchain-next-height" + ); + (chain_tip, start, end) + } }; debug!( @@ -563,7 +652,8 @@ impl BaseBridge { .await { Ok(batch) => { - self.runtime_handle + self.deps + .runtime_handle .send_event(BridgeEvent::Chain(Box::new(ChainEvent::Base(batch)))) .await?; info!( @@ -601,13 +691,14 @@ impl BaseBridge { ]; let filter = Filter::new() .address(vec![ - self.inbox_contract_address, self.nock_contract_address, + self.contracts.inbox_contract_address, self.contracts.nock_contract_address, ]) .event_signature(event_signatures) .from_block(batch_start) .to_block(batch_end); let logs = self + .deps .provider .get_logs(&filter) .await @@ -627,7 +718,7 @@ impl BaseBridge { let heights: Vec = (batch_start..=batch_end).collect(); for chunk in heights.chunks(20) { - let mut batch = BatchRequest::new(self.provider.client()); + let mut batch = BatchRequest::new(self.deps.provider.client()); let mut futures = Vec::new(); for &height in chunk { @@ -695,7 +786,7 @@ impl BaseBridge { if let Some(last_block) = blocks.last() { let tip_hash = format!("0x{}", hex_encode(last_block.block_id.as_slice())); - self.runtime_handle.set_base_tip_hash(tip_hash); + self.deps.runtime_handle.set_base_tip_hash(tip_hash); } let mut withdrawals = Vec::new(); @@ -720,7 +811,7 @@ impl BaseBridge { data: log.data().data.clone(), }; - if log.address() == self.inbox_contract_address { + if log.address() == self.contracts.inbox_contract_address { if let Some((event, settlement)) = self.process_inbox_log(&raw, &tx_hash, log_index)? { @@ -765,7 +856,7 @@ impl BaseBridge { } } } - } else if log.address() == self.nock_contract_address { + } else if log.address() == self.contracts.nock_contract_address { if let Some((event, withdrawal)) = self.process_nock_log(&raw, &tx_hash, log_index)? { @@ -1021,6 +1112,39 @@ impl BaseBridge { } } +#[async_trait] +impl BaseContractPort for BaseBridge { + async fn submit_deposit( + &self, + submission: DepositSubmission, + ) -> Result { + BaseBridge::submit_deposit(self, submission).await + } + + async fn get_last_deposit_nonce(&self) -> Result { + BaseBridge::get_last_deposit_nonce(self).await + } + + async fn is_deposit_processed(&self, tx_id: &Tip5Hash) -> Result { + BaseBridge::is_deposit_processed(self, tx_id).await + } +} + +#[async_trait] +impl BaseSourcePort for BaseBridge { + async fn chain_tip_height(&self) -> Result { + self.deps + .provider + .get_block_number() + .await + .map_err(|e| BridgeError::BaseBridgeMonitoring(e.to_string())) + } + + async fn fetch_batch(&self, start: u64, end: u64) -> Result { + BaseBridge::fetch_batch(self, start, end, None).await + } +} + fn atom_bytes_from_b256(value: B256) -> AtomBytes { AtomBytes(value.as_slice().to_vec()) } diff --git a/crates/bridge/src/ingress.rs b/crates/bridge/src/ingress.rs index 316ee61fa..23d49dfaa 100644 --- a/crates/bridge/src/ingress.rs +++ b/crates/bridge/src/ingress.rs @@ -76,20 +76,36 @@ pub fn spawn_broadcast_stop_to_peers( } } -pub struct IngressService { +struct IngressRuntimeDeps { runtime: Arc, - node_id: u64, - start_time: Instant, /// Signer for creating Ethereum signatures on proposals signer: Arc, /// Cache for aggregating signatures from multiple bridge nodes proposal_cache: Arc, +} + +struct IngressNodeState { + node_id: u64, + start_time: Instant, +} + +struct IngressControl { /// Shared TUI state for updating proposal display on peer broadcasts bridge_status: BridgeStatus, - /// Mapping from Ethereum address to node ID for TUI signature display - address_to_node_id: std::collections::HashMap, stop_controller: crate::stop::StopController, - peers: Vec, +} + +struct IngressPeerState { + /// Mapping from Ethereum address to node ID for TUI signature display + address_to_node_id: Arc>, + peers: Arc>, +} + +pub struct IngressService { + deps: IngressRuntimeDeps, + node: IngressNodeState, + control: IngressControl, + peers: IngressPeerState, } impl IngressService { @@ -105,20 +121,28 @@ impl IngressService { peers: Vec, ) -> Self { Self { - runtime, - node_id, - start_time: Instant::now(), - signer, - proposal_cache, - bridge_status, - address_to_node_id, - stop_controller, - peers, + deps: IngressRuntimeDeps { + runtime, + signer, + proposal_cache, + }, + node: IngressNodeState { + node_id, + start_time: Instant::now(), + }, + control: IngressControl { + bridge_status, + stop_controller, + }, + peers: IngressPeerState { + address_to_node_id: Arc::new(address_to_node_id), + peers: Arc::new(peers), + }, } } fn uptime_millis(&self) -> u64 { - self.start_time.elapsed().as_millis() as u64 + self.node.start_time.elapsed().as_millis() as u64 } async fn trigger_stop( @@ -137,7 +161,7 @@ impl IngressService { let resolved_last = match last { Some(last) => Some(last), - None => match self.runtime.peek_stop_info().await { + None => match self.deps.runtime.peek_stop_info().await { Ok(last) => last, Err(err) => { warn!( @@ -157,11 +181,11 @@ impl IngressService { at: SystemTime::now(), }; - if !self.stop_controller.trigger(info) { + if !self.control.stop_controller.trigger(info) { return; } - self.bridge_status.push_alert( + self.control.bridge_status.push_alert( AlertSeverity::Error, "Bridge Stopped".to_string(), reason.clone(), @@ -173,7 +197,7 @@ impl IngressService { ); if let Some(last) = resolved_last.clone() { - if let Err(err) = self.runtime.send_stop(last).await { + if let Err(err) = self.deps.runtime.send_stop(last).await { warn!( target: "bridge.ingress", error=%err, @@ -209,7 +233,7 @@ impl IngressService { }; let msg = StopBroadcast { - sender_node_id: self.node_id, + sender_node_id: self.node.node_id, reason: reason.clone(), last_base_hash, last_base_height, @@ -218,7 +242,7 @@ impl IngressService { timestamp, }; - spawn_broadcast_stop_to_peers(&self.peers, msg, "bridge.ingress"); + spawn_broadcast_stop_to_peers(self.peers.peers.as_ref(), msg, "bridge.ingress"); } /// Verify an Ethereum signature and recover the signer address. @@ -295,7 +319,7 @@ impl BridgeIngress for IngressService { .map(|d| d.as_millis() as u64) .unwrap_or_default(); let response = HealthCheckResponse { - responder_node_id: self.node_id, + responder_node_id: self.node.node_id, uptime_millis: self.uptime_millis(), status: "healthy".into(), timestamp_millis, @@ -379,7 +403,7 @@ impl BridgeIngress for IngressService { let mut proposal_hash = [0u8; 32]; proposal_hash.copy_from_slice(&req.proposal_hash); - if signer_address == self.signer.address() { + if signer_address == self.deps.signer.address() { metrics.ingress_broadcast_signature_ignored_self.increment(); tracing::debug!( target: "bridge.ingress", @@ -392,7 +416,7 @@ impl BridgeIngress for IngressService { })); } - let signer_is_known = self.address_to_node_id.contains_key(&signer_address); + let signer_is_known = self.peers.address_to_node_id.contains_key(&signer_address); if signer_is_known { metrics.ingress_broadcast_signature_known_signer.increment(); } else { @@ -411,7 +435,7 @@ impl BridgeIngress for IngressService { "received signature broadcast" ); - let existing_state = match self.proposal_cache.get_state(&deposit_id) { + let existing_state = match self.deps.proposal_cache.get_state(&deposit_id) { Ok(state) => state, Err(err) => { warn!( @@ -446,7 +470,7 @@ impl BridgeIngress for IngressService { received_hash = %received_hex, "peer signature proposal hash mismatch, possible nonce divergence" ); - self.bridge_status.push_alert( + self.control.bridge_status.push_alert( AlertSeverity::Error, "Nonce Divergence Suspected".to_string(), format!( @@ -474,7 +498,7 @@ impl BridgeIngress for IngressService { // Add signature to cache (or queue if we haven't processed this deposit yet) // The verify_fn will check that signature recovers to claimed signer - let result = self.proposal_cache.add_signature( + let result = self.deps.proposal_cache.add_signature( &deposit_id, crate::proposal_cache::SignatureData { signer_address, @@ -499,10 +523,13 @@ impl BridgeIngress for IngressService { "signature added to cache" ); - if let Ok(Some(state)) = self.proposal_cache.get_state(&deposit_id) { - self.bridge_status.sync_proposal_signatures_from_cache( - &proposal_hash_hex, &state, &self.address_to_node_id, self.node_id, - ); + if let Ok(Some(state)) = self.deps.proposal_cache.get_state(&deposit_id) { + self.control + .bridge_status + .sync_proposal_signatures_from_cache( + &proposal_hash_hex, &state, &self.peers.address_to_node_id, + self.node.node_id, + ); } Ok(Response::new(SignatureBroadcastResponse { @@ -521,10 +548,13 @@ impl BridgeIngress for IngressService { "signature added - threshold reached!" ); - if let Ok(Some(state)) = self.proposal_cache.get_state(&deposit_id) { - self.bridge_status.sync_proposal_signatures_from_cache( - &proposal_hash_hex, &state, &self.address_to_node_id, self.node_id, - ); + if let Ok(Some(state)) = self.deps.proposal_cache.get_state(&deposit_id) { + self.control + .bridge_status + .sync_proposal_signatures_from_cache( + &proposal_hash_hex, &state, &self.peers.address_to_node_id, + self.node.node_id, + ); } Ok(Response::new(SignatureBroadcastResponse { @@ -616,6 +646,7 @@ impl BridgeIngress for IngressService { // Get proposal state from cache let state = self + .deps .proposal_cache .get_state(&deposit_id) .map_err(|e| Status::internal(format!("failed to get proposal state: {}", e)))?; @@ -636,9 +667,9 @@ impl BridgeIngress for IngressService { as u32; // Collect signer addresses - let mut signers = Vec::new(); + let mut signers: Vec> = Vec::new(); if state.my_signature.is_some() { - signers.push(self.signer.address().to_vec()); + signers.push(self.deps.signer.address().to_vec()); } for addr in state.peer_signatures.keys() { signers.push(addr.to_vec()); @@ -712,7 +743,7 @@ impl BridgeIngress for IngressService { ); // Mark the proposal as confirmed in our cache - match self.proposal_cache.mark_confirmed(&deposit_id) { + match self.deps.proposal_cache.mark_confirmed(&deposit_id) { Ok(()) => { info!( target: "bridge.ingress", @@ -723,11 +754,13 @@ impl BridgeIngress for IngressService { // Update TUI to show Executed status // Try to find existing proposal and update it - if let Some(mut tui_proposal) = self.bridge_status.find_proposal(&proposal_hash) { + if let Some(mut tui_proposal) = + self.control.bridge_status.find_proposal(&proposal_hash) + { tui_proposal.status = ProposalStatus::Executed; tui_proposal.tx_hash = Some(tx_hash.clone()); tui_proposal.executed_at_block = Some(req.block_number); - self.bridge_status.update_proposal(tui_proposal); + self.control.bridge_status.update_proposal(tui_proposal); info!( target: "bridge.ingress", proposal_hash = %proposal_hash, @@ -760,7 +793,7 @@ impl BridgeIngress for IngressService { is_my_turn: false, time_until_takeover: None, }; - self.bridge_status.update_proposal(placeholder); + self.control.bridge_status.update_proposal(placeholder); info!( target: "bridge.ingress", proposal_hash = %proposal_hash, diff --git a/crates/bridge/src/lib.rs b/crates/bridge/src/lib.rs index c87d3ad42..c02703fda 100644 --- a/crates/bridge/src/lib.rs +++ b/crates/bridge/src/lib.rs @@ -6,14 +6,17 @@ compile_error!("features `snmalloc` and `malloc` are mutually exclusive"); pub mod bridge_status; pub mod config; +pub mod core; pub mod deposit_log; pub mod errors; pub mod ethereum; pub mod grpc; pub mod health; pub mod ingress; +pub mod loop_policy; pub mod metrics; pub mod nockchain; +pub mod ports; pub mod proposal_cache; pub mod proposer; pub mod runtime; diff --git a/crates/bridge/src/loop_policy.rs b/crates/bridge/src/loop_policy.rs new file mode 100644 index 000000000..7123fd877 --- /dev/null +++ b/crates/bridge/src/loop_policy.rs @@ -0,0 +1,140 @@ +use std::time::Duration; + +use backon::ExponentialBuilder; + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct RetryPolicy { + pub min_delay: Duration, + pub max_delay: Duration, + pub max_times: Option, + pub jitter: bool, +} + +impl RetryPolicy { + pub fn exponential_builder(self) -> ExponentialBuilder { + let builder = ExponentialBuilder::default() + .with_min_delay(self.min_delay) + .with_max_delay(self.max_delay); + let builder = if self.jitter { + builder.with_jitter() + } else { + builder + }; + match self.max_times { + Some(max_times) => builder.with_max_times(max_times), + None => builder, + } + } +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct BaseObserverLoopPolicy { + pub poll_interval: Duration, + pub rpc_retry: RetryPolicy, +} + +impl Default for BaseObserverLoopPolicy { + fn default() -> Self { + Self { + poll_interval: Duration::from_secs(30), + rpc_retry: RetryPolicy { + min_delay: Duration::from_secs(5), + max_delay: Duration::from_secs(120), + max_times: Some(10), + jitter: true, + }, + } + } +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct NockObserverLoopPolicy { + pub poll_interval: Duration, + pub connect_retry: RetryPolicy, + pub connect_failure_sleep: Duration, +} + +impl Default for NockObserverLoopPolicy { + fn default() -> Self { + Self { + poll_interval: Duration::from_secs(10), + connect_retry: RetryPolicy { + min_delay: Duration::from_secs(1), + max_delay: Duration::from_secs(300), + max_times: None, + jitter: true, + }, + connect_failure_sleep: Duration::from_secs(1), + } + } +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct SigningLoopPolicy { + pub poll_interval: Duration, + pub pipeline_depth: usize, + pub regossip_interval: Duration, +} + +impl Default for SigningLoopPolicy { + fn default() -> Self { + Self { + poll_interval: Duration::from_secs(15), + pipeline_depth: 4, + regossip_interval: Duration::from_secs(90), + } + } +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct PostingLoopPolicy { + pub tick_interval: Duration, + pub failover_backoff_secs: u64, +} + +impl Default for PostingLoopPolicy { + fn default() -> Self { + Self { + tick_interval: Duration::from_secs(1), + failover_backoff_secs: 120, + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn default_base_observer_policy_matches_existing_behavior() { + let policy = BaseObserverLoopPolicy::default(); + assert_eq!(policy.poll_interval, Duration::from_secs(30)); + assert_eq!(policy.rpc_retry.min_delay, Duration::from_secs(5)); + assert_eq!(policy.rpc_retry.max_delay, Duration::from_secs(120)); + assert_eq!(policy.rpc_retry.max_times, Some(10)); + assert!(policy.rpc_retry.jitter); + } + + #[test] + fn default_nock_observer_policy_matches_existing_behavior() { + let policy = NockObserverLoopPolicy::default(); + assert_eq!(policy.poll_interval, Duration::from_secs(10)); + assert_eq!(policy.connect_retry.min_delay, Duration::from_secs(1)); + assert_eq!(policy.connect_retry.max_delay, Duration::from_secs(300)); + assert_eq!(policy.connect_retry.max_times, None); + assert!(policy.connect_retry.jitter); + assert_eq!(policy.connect_failure_sleep, Duration::from_secs(1)); + } + + #[test] + fn default_signing_and_posting_policies_match_existing_behavior() { + let signing = SigningLoopPolicy::default(); + assert_eq!(signing.poll_interval, Duration::from_secs(15)); + assert_eq!(signing.pipeline_depth, 4); + assert_eq!(signing.regossip_interval, Duration::from_secs(90)); + + let posting = PostingLoopPolicy::default(); + assert_eq!(posting.tick_interval, Duration::from_secs(1)); + assert_eq!(posting.failover_backoff_secs, 120); + } +} diff --git a/crates/bridge/src/main.rs b/crates/bridge/src/main.rs index 104e77755..171c6bf32 100644 --- a/crates/bridge/src/main.rs +++ b/crates/bridge/src/main.rs @@ -339,6 +339,9 @@ async fn main() -> Result<(), BridgeError> { let deposit_log = Arc::new(bridge::deposit_log::DepositLog::open(deposit_log_path.clone()).await?); info!("Using deposit queue log at {}", deposit_log_path.display()); + bridge::metrics::update_deposit_log_max_nonce( + deposit_log.max_nonce_in_epoch(&nonce_epoch).await?, + ); // Deposit log sync happens after the kernel action loop is running. @@ -683,6 +686,7 @@ mod tests { use nockapp::kernel::boot; use nockchain_math::belt::Belt; use nockchain_types::v1::Name; + use nockvm::noun::NounAllocator; use tempfile::TempDir; use super::*; @@ -743,7 +747,7 @@ mod tests { if std::env::var("RUST_LOG").is_err() { std::env::set_var("RUST_LOG", "debug"); } - let cli = boot::default_boot_cli(true); + let cli = boot::ephemeral_test_boot_cli(true); let temp_log_dir = std::env::temp_dir().join("bridge-test-logs"); let _guard = init_bridge_tracing(&cli, Some(tui::new_log_buffer()), temp_log_dir, 7) .expect("failed to init tracing for tests"); @@ -844,9 +848,11 @@ mod tests { let noun = unsafe { let mut ia = nockvm::noun::IndirectAtom::new_raw_bytes(&mut slab, 32, proposal_hash.as_ptr()); - ia.normalize_as_atom() + let space = slab.noun_space(); + ia.normalize_as_atom(&space) }; - let signature = bridge_signer.sign_proposal(noun.as_noun()).await?; + let space = slab.noun_space(); + let signature = bridge_signer.sign_proposal(noun.as_noun(), &space).await?; assert!(!signature.r().is_zero(), "Expected valid r component"); assert!(!signature.s().is_zero(), "Expected valid s component"); diff --git a/crates/bridge/src/metrics.rs b/crates/bridge/src/metrics.rs index 99dbd37b3..7c2d46cff 100644 --- a/crates/bridge/src/metrics.rs +++ b/crates/bridge/src/metrics.rs @@ -8,11 +8,16 @@ use crate::tui::types::NetworkState; metrics_struct![ BridgeHealthMetrics, (running_status, "bridge.health.running_status", Gauge), + (stop_local_requests, "bridge.stop.local.requests", Count), + (stop_local_triggered, "bridge.stop.local.triggered", Count), + (stop_local_duplicate, "bridge.stop.local.duplicate", Count), (base_hold_height, "bridge.health.base_hold_height", Gauge), (nock_hold_height, "bridge.health.nock_hold_height", Gauge), + (base_tip_height, "bridge.health.base_tip_height", Gauge), (base_block_height, "bridge.health.base_block_height", Gauge), (nock_block_height, "bridge.health.nock_block_height", Gauge), (last_deposit_nonce, "bridge.health.last_deposit_nonce", Gauge), + (deposit_log_max_nonce, "bridge.health.deposit_log_max_nonce", Gauge), (ingress_broadcast_signature_requests, "bridge.ingress.broadcast_signature.requests", Count), ( ingress_broadcast_signature_invalid_deposit_id_len, @@ -320,7 +325,29 @@ pub fn init_metrics() -> Arc { pub fn update_bridge_metrics(network: &NetworkState, last_deposit_nonce: Option) { let metrics = init_metrics(); + update_bridge_metrics_with(&metrics, network, last_deposit_nonce); +} + +pub fn update_base_tip_height(tip_height: Option) { + let metrics = init_metrics(); + update_base_tip_height_with(&metrics, tip_height); +} + +pub fn update_deposit_log_max_nonce(max_nonce: Option) { + let metrics = init_metrics(); + update_deposit_log_max_nonce_with(&metrics, max_nonce); +} + +pub fn advance_deposit_log_max_nonce(inserted: u64, first_epoch_nonce: u64) { + let metrics = init_metrics(); + advance_deposit_log_max_nonce_with(&metrics, inserted, first_epoch_nonce); +} +fn update_bridge_metrics_with( + metrics: &BridgeHealthMetrics, + network: &NetworkState, + last_deposit_nonce: Option, +) { let running_metric = if network.kernel_stopped { RunningStatusMetric::Stop } else { @@ -355,3 +382,85 @@ pub fn update_bridge_metrics(network: &NetworkState, last_deposit_nonce: Option< metrics.nock_block_height.swap(nock_height as f64); metrics.last_deposit_nonce.swap(last_deposit_nonce as f64); } + +fn update_base_tip_height_with(metrics: &BridgeHealthMetrics, tip_height: Option) { + metrics + .base_tip_height + .swap(tip_height.unwrap_or_default() as f64); +} + +fn update_deposit_log_max_nonce_with(metrics: &BridgeHealthMetrics, max_nonce: Option) { + metrics + .deposit_log_max_nonce + .swap(max_nonce.unwrap_or_default() as f64); +} + +fn advance_deposit_log_max_nonce_with( + metrics: &BridgeHealthMetrics, + inserted: u64, + first_epoch_nonce: u64, +) { + if inserted == 0 { + return; + } + + let current = metrics.deposit_log_max_nonce.load() as u64; + let next = if current == 0 { + first_epoch_nonce.saturating_add(inserted - 1) + } else { + current.saturating_add(inserted) + }; + metrics.deposit_log_max_nonce.swap(next as f64); +} + +#[cfg(test)] +mod tests { + use gnort::{MetricsRegistry, RegistryConfig}; + + use super::{ + advance_deposit_log_max_nonce_with, update_base_tip_height_with, + update_deposit_log_max_nonce_with, BridgeHealthMetrics, + }; + + #[test] + fn update_base_tip_height_reports_latest_tip() { + let metrics = + BridgeHealthMetrics::register(&MetricsRegistry::new(RegistryConfig::default())) + .expect("metrics should register"); + + update_base_tip_height_with(&metrics, Some(113)); + assert_eq!(metrics.base_tip_height.load(), 113.0); + + update_base_tip_height_with(&metrics, None); + assert_eq!(metrics.base_tip_height.load(), 0.0); + } + + #[test] + fn update_deposit_log_max_nonce_reports_local_log_head() { + let metrics = + BridgeHealthMetrics::register(&MetricsRegistry::new(RegistryConfig::default())) + .expect("metrics should register"); + + update_deposit_log_max_nonce_with(&metrics, Some(113)); + assert_eq!(metrics.deposit_log_max_nonce.load(), 113.0); + + update_deposit_log_max_nonce_with(&metrics, None); + assert_eq!(metrics.deposit_log_max_nonce.load(), 0.0); + } + + #[test] + fn advance_deposit_log_max_nonce_avoids_requerying_the_log() { + let metrics = + BridgeHealthMetrics::register(&MetricsRegistry::new(RegistryConfig::default())) + .expect("metrics should register"); + + advance_deposit_log_max_nonce_with(&metrics, 2, 101); + assert_eq!(metrics.deposit_log_max_nonce.load(), 102.0); + + advance_deposit_log_max_nonce_with(&metrics, 3, 101); + assert_eq!(metrics.deposit_log_max_nonce.load(), 105.0); + + advance_deposit_log_max_nonce_with(&metrics, 0, 101); + assert_eq!(metrics.deposit_log_max_nonce.load(), 105.0); + } +} diff --git a/crates/bridge/src/nockchain.rs b/crates/bridge/src/nockchain.rs index c4cc0e919..1dfc55b72 100644 --- a/crates/bridge/src/nockchain.rs +++ b/crates/bridge/src/nockchain.rs @@ -1,24 +1,28 @@ use std::sync::Arc; use std::time::{Duration, SystemTime}; -use backon::{ExponentialBuilder, Retryable}; +use async_trait::async_trait; +use backon::Retryable; use nockapp::noun::slab::{NockJammer, NounSlab}; use nockapp::{Bytes, ToBytes}; use nockapp_grpc::services::private_nockapp::client::PrivateNockAppGrpcClient; use nockchain_types::tx_engine::common::{BlockId, Heavy, Page, TxId}; +use nockvm::noun::NounAllocator; use noun_serde::prelude::*; use tokio::time::sleep; use tracing::{debug, info, warn}; use crate::bridge_status::BridgeStatus; +use crate::core::nock_observer::{plan_nock_tick, NockPlanAction, NockPlanInput}; use crate::errors::BridgeError; +use crate::loop_policy::NockObserverLoopPolicy; use crate::metrics; +use crate::ports::{NockSourcePort, NockTipInfo}; use crate::runtime::{BridgeEvent, BridgeRuntimeHandle, ChainEvent, NockBlockEvent}; use crate::stop::StopHandle; use crate::tui::types::{AlertSeverity, ChainState, NetworkState, NockchainApiStatus}; use crate::types::Tx; -const DEFAULT_POLL_INTERVAL: Duration = Duration::from_secs(10); const CLIENT_PID: i32 = 1; /// Default nockchain confirmation depth used by the driver if not specified in config. @@ -26,6 +30,7 @@ const CLIENT_PID: i32 = 1; /// The bridge kernel assumes blocks it receives are final; this is enforced by the Rust driver. pub const DEFAULT_NOCKCHAIN_CONFIRMATION_DEPTH: u64 = 100; +#[cfg(test)] fn confirmed_height(chain_tip: u64, confirmation_depth: u64) -> Option { if confirmation_depth == 0 { return None; @@ -38,23 +43,159 @@ fn confirmed_height(chain_tip: u64, confirmation_depth: u64) -> Option { } } -struct NockBlockObservation { - block: Page, - page_slab: NounSlab, - page_noun: nockapp::Noun, - txs: Vec<(TxId, Tx)>, +pub struct NockGrpcSource { + client: PrivateNockAppGrpcClient, } -pub struct NockchainWatcher { +impl NockGrpcSource { + pub async fn connect(endpoint: String) -> Result { + let client = PrivateNockAppGrpcClient::connect(endpoint) + .await + .map_err(|err| BridgeError::EventMonitoring(err.to_string()))?; + Ok(Self { client }) + } + + pub fn from_client(client: PrivateNockAppGrpcClient) -> Self { + Self { client } + } +} + +#[async_trait] +impl NockSourcePort for NockGrpcSource { + async fn tip_info(&mut self) -> Result, BridgeError> { + let info = Self::fetch_tip_info_from_client(&mut self.client).await?; + Ok(info.map(|(height, tip_hash)| NockTipInfo { height, tip_hash })) + } + + async fn fetch_block_at_height( + &mut self, + height: u64, + ) -> Result, BridgeError> { + Self::fetch_block_at_height_from_client(&mut self.client, height).await + } +} + +impl NockGrpcSource { + async fn fetch_tip_info_from_client( + client: &mut PrivateNockAppGrpcClient, + ) -> Result, BridgeError> { + let heavy_path = vec![Bytes::from("heavy")]; + let heavy_bytes = jam_path(&heavy_path)?; + let response = client + .peek(CLIENT_PID, heavy_bytes) + .await + .map_err(|err| BridgeError::EventMonitoring(err.to_string()))?; + let (heavy_slab, heavy_noun) = cue_response(response)?; + let heavy: Heavy = { + let heavy_space = heavy_slab.noun_space(); + heavy_noun.decode(&heavy_space).map_err(|err| { + BridgeError::EventMonitoring(format!("failed to decode heavy response: {}", err)) + })? + }; + let Some(block_id_base58) = heavy.to_base58() else { + return Ok(None); + }; + let tip_hash = block_id_base58.clone(); + + let block_path = vec![Bytes::from("block"), Bytes::from(block_id_base58)]; + let block_bytes = jam_path(&block_path)?; + let response = client + .peek(CLIENT_PID, block_bytes) + .await + .map_err(|err| BridgeError::EventMonitoring(err.to_string()))?; + let (page_slab, block_noun) = cue_response(response)?; + let (page, _page_noun) = { + let page_space = page_slab.noun_space(); + decode_page_from_peek(&block_noun, &page_space)? + }; + Ok(Some((page.height, tip_hash))) + } + + async fn fetch_block_at_height_from_client( + client: &mut PrivateNockAppGrpcClient, + height: u64, + ) -> Result, BridgeError> { + let heavy_n_path = vec![Bytes::from("heavy-n"), Bytes::from(height.to_bytes()?)]; + let heavy_n_bytes = jam_path(&heavy_n_path)?; + let response = client + .peek(CLIENT_PID, heavy_n_bytes) + .await + .map_err(|err| BridgeError::EventMonitoring(err.to_string()))?; + let (page_slab, block_noun) = cue_response(response)?; + let (page, page_noun) = { + let page_space = page_slab.noun_space(); + match decode_page_from_peek(&block_noun, &page_space) { + Ok(result) => result, + Err(_) => return Ok(None), + } + }; + + let txs = Self::fetch_transactions_from_client(client, &page.digest, &page.tx_ids).await?; + + Ok(Some(NockBlockEvent { + block: page, + page_slab, + page_noun, + txs, + })) + } + + async fn fetch_transactions_from_client( + client: &mut PrivateNockAppGrpcClient, + block_id: &BlockId, + tx_ids: &[TxId], + ) -> Result, BridgeError> { + let block_id_base58 = block_id.to_base58(); + let mut txs = Vec::with_capacity(tx_ids.len()); + for tx_id in tx_ids { + let tx_id_base58 = tx_id.to_base58(); + let tx_path = vec![ + Bytes::from("block-transaction"), + Bytes::from(block_id_base58.clone()), + Bytes::from(tx_id_base58), + ]; + let tx_bytes = jam_path(&tx_path)?; + let response = client + .peek(CLIENT_PID, tx_bytes) + .await + .map_err(|err| BridgeError::EventMonitoring(err.to_string()))?; + let (tx_slab, tx_noun) = cue_response(response)?; + let tx = { + let tx_space = tx_slab.noun_space(); + decode_tx_from_peek(&tx_noun, &tx_space)? + }; + txs.push((tx_id.clone(), tx)); + } + Ok(txs) + } +} + +struct NockWatcherConnection { endpoint: String, - poll_interval: Duration, + policy: NockObserverLoopPolicy, +} + +struct NockWatcherDeps { runtime: Arc, - confirmation_depth: u64, stop: StopHandle, +} + +struct NockWatcherConfig { + confirmation_depth: u64, +} + +struct NockWatcherUi { /// Optional bridge_status for connection status + alert updates. bridge_status: Option, } +pub struct NockchainWatcher { + connection: NockWatcherConnection, + deps: NockWatcherDeps, + config: NockWatcherConfig, + ui: NockWatcherUi, +} + impl NockchainWatcher { pub fn new( endpoint: String, @@ -62,14 +203,13 @@ impl NockchainWatcher { confirmation_depth: u64, stop: StopHandle, ) -> Self { - Self { + Self::with_policy( endpoint, - poll_interval: DEFAULT_POLL_INTERVAL, runtime, confirmation_depth, stop, - bridge_status: None, - } + NockObserverLoopPolicy::default(), + ) } pub fn with_poll_interval( @@ -79,39 +219,53 @@ impl NockchainWatcher { confirmation_depth: u64, stop: StopHandle, ) -> Self { - Self { - endpoint, + let policy = NockObserverLoopPolicy { poll_interval, - runtime, - confirmation_depth, - stop, - bridge_status: None, + ..NockObserverLoopPolicy::default() + }; + Self::with_policy(endpoint, runtime, confirmation_depth, stop, policy) + } + + pub fn with_policy( + endpoint: String, + runtime: Arc, + confirmation_depth: u64, + stop: StopHandle, + policy: NockObserverLoopPolicy, + ) -> Self { + Self { + connection: NockWatcherConnection { endpoint, policy }, + deps: NockWatcherDeps { runtime, stop }, + config: NockWatcherConfig { confirmation_depth }, + ui: NockWatcherUi { + bridge_status: None, + }, } } /// Set the TUI state for connection status updates. pub fn with_bridge_status(mut self, bridge_status: BridgeStatus) -> Self { - self.bridge_status = Some(bridge_status); + self.ui.bridge_status = Some(bridge_status); self } /// Update the nockchain API connection status in the TUI. fn update_status(&self, status: NockchainApiStatus) { - if let Some(ref bridge_status) = self.bridge_status { + if let Some(ref bridge_status) = self.ui.bridge_status { bridge_status.update_nockchain_api_status(status); } } /// Update the nockchain tip hash in the TUI. fn update_tip_hash(&self, tip_hash: String) { - if let Some(ref bridge_status) = self.bridge_status { + if let Some(ref bridge_status) = self.ui.bridge_status { bridge_status.update_nockchain_tip_hash(tip_hash); } } /// Push an alert to the TUI. fn push_alert(&self, severity: AlertSeverity, title: String, message: String) { - if let Some(ref bridge_status) = self.bridge_status { + if let Some(ref bridge_status) = self.ui.bridge_status { bridge_status.push_alert(severity, title, message, "nock-watcher".to_string()); } } @@ -119,31 +273,26 @@ impl NockchainWatcher { pub async fn run(self) -> Result<(), BridgeError> { let mut was_connected = false; - // Unlimited retries with exponential backoff - let backoff = || { - ExponentialBuilder::default() - .with_min_delay(Duration::from_secs(1)) - .with_max_delay(Duration::from_secs(300)) - .with_jitter() - }; + // Unlimited retries with exponential backoff by default. + let connect_backoff = self.connection.policy.connect_retry; self.update_status(NockchainApiStatus::connecting(0, None)); loop { - if self.stop.is_stopped() { - sleep(self.poll_interval).await; + if self.deps.stop.is_stopped() { + sleep(self.connection.policy.poll_interval).await; continue; } let attempt_count = std::sync::Arc::new(std::sync::atomic::AtomicU32::new(0)); let attempt_count_notify = attempt_count.clone(); - let endpoint = self.endpoint.clone(); + let endpoint = self.connection.endpoint.clone(); let connect = || async { PrivateNockAppGrpcClient::connect(endpoint.clone()).await }; self.update_status(NockchainApiStatus::connecting(1, None)); let connect_result = connect - .retry(backoff()) + .retry(connect_backoff.exponential_builder()) .notify(|err, dur| { let attempt = attempt_count_notify.fetch_add(1, std::sync::atomic::Ordering::SeqCst) + 1; @@ -156,7 +305,7 @@ impl NockchainWatcher { warn!( target: "bridge.nock-watcher", - endpoint=%self.endpoint, + endpoint=%self.connection.endpoint, error=%error_msg, attempt=attempt, backoff_secs=dur.as_secs(), @@ -178,30 +327,31 @@ impl NockchainWatcher { .await; match connect_result { - Ok(mut client) => { + Ok(client) => { self.update_status(NockchainApiStatus::connected()); if was_connected { info!( target: "bridge.nock-watcher", - endpoint=%self.endpoint, + endpoint=%self.connection.endpoint, "reconnected to nockchain gRPC endpoint" ); self.push_alert( AlertSeverity::Info, "Nockchain API Reconnected".to_string(), - format!("Reconnected to {}", self.endpoint), + format!("Reconnected to {}", self.connection.endpoint), ); } else { info!( target: "bridge.nock-watcher", - endpoint=%self.endpoint, + endpoint=%self.connection.endpoint, "connected to nockchain gRPC endpoint" ); } was_connected = true; - if let Err(err) = self.stream_events(&mut client).await { + let mut source = NockGrpcSource::from_client(client); + if let Err(err) = self.stream_events_with_source(&mut source).await { let error_msg = err.to_string(); warn!( target: "bridge.nock-watcher", @@ -221,110 +371,140 @@ impl NockchainWatcher { let error_msg = err.to_string(); warn!( target: "bridge.nock-watcher", - endpoint=%self.endpoint, + endpoint=%self.connection.endpoint, error=%error_msg, "connect failed unexpectedly" ); self.update_status(NockchainApiStatus::disconnected(error_msg)); - sleep(Duration::from_secs(1)).await; + sleep(self.connection.policy.connect_failure_sleep).await; } } } } - async fn stream_events( - &self, - client: &mut PrivateNockAppGrpcClient, - ) -> Result<(), BridgeError> { + async fn stream_events_with_source(&self, source: &mut S) -> Result<(), BridgeError> + where + S: NockSourcePort, + { + let poll_interval = self.connection.policy.poll_interval; info!( target: "bridge.nock-watcher", - confirmation_depth = self.confirmation_depth, + confirmation_depth = self.config.confirmation_depth, "starting nock observer with confirmation depth" ); loop { - if self.stop.is_stopped() { - sleep(self.poll_interval).await; + if self.deps.stop.is_stopped() { + sleep(poll_interval).await; continue; } - let (tip_height, tip_hash) = match self.fetch_tip_info(client).await { - Ok(Some(info)) => info, - Ok(None) => { - debug!( + let tip_info = match source.tip_info().await { + Ok(info) => info, + Err(err) => { + warn!( target: "bridge.nock-watcher", - "no heaviest block available from private nockapp" + error=%err, + "failed to fetch tip height" ); - sleep(self.poll_interval).await; + sleep(poll_interval).await; continue; } + }; + + if let Some(info) = &tip_info { + self.update_tip_hash(info.tip_hash.clone()); + } + + let next_needed_height = match self.deps.runtime.peek_nock_next_height().await { + Ok(height) => height, Err(err) => { warn!( target: "bridge.nock-watcher", error=%err, - "failed to fetch tip height" + "failed to peek nock next height" ); - sleep(self.poll_interval).await; + sleep(poll_interval).await; continue; } }; - self.update_tip_hash(tip_hash); - let Some(confirmed_target) = confirmed_height(tip_height, self.confirmation_depth) - else { - debug!( - target: "bridge.nock-watcher", - tip_height, - "no confirmed block yet (bootstrap)" - ); - sleep(self.poll_interval).await; - continue; - }; + let action = plan_nock_tick(NockPlanInput { + tip_height: tip_info.as_ref().map(|info| info.height), + next_needed_height, + confirmation_depth: self.config.confirmation_depth, + }); - let next_needed_height = match self.runtime.peek_nock_next_height().await { - Ok(Some(height)) => height, - Ok(None) => { + let (tip_height, target_height) = match action { + NockPlanAction::NoTipAvailable => { + debug!( + target: "bridge.nock-watcher", + "no heaviest block available from private nockapp" + ); + sleep(poll_interval).await; + continue; + } + NockPlanAction::NoPendingHeight { + tip_height, + confirmed_target, + } => { debug!( target: "bridge.nock-watcher", tip_height, + confirmed_target, "kernel has no pending nock block" ); - sleep(self.poll_interval).await; + sleep(poll_interval).await; continue; } - Err(err) => { + NockPlanAction::InvalidConfig(err) => { warn!( target: "bridge.nock-watcher", - error=%err, - "failed to peek nock next height" + error=?err, + confirmation_depth=self.config.confirmation_depth, + "invalid nock observer planner configuration" ); - sleep(self.poll_interval).await; + sleep(poll_interval).await; continue; } - }; - - if confirmed_target < next_needed_height { - debug!( - target: "bridge.nock-watcher", + NockPlanAction::BootstrapUnconfirmed { + tip_height, + confirmation_depth, + } => { + debug!( + target: "bridge.nock-watcher", + tip_height, + confirmation_depth, + "no confirmed block yet (bootstrap)" + ); + sleep(poll_interval).await; + continue; + } + NockPlanAction::NotYetConfirmed { tip_height, confirmed_target, next_needed_height, - "target height not yet confirmed for kernel need" - ); - sleep(self.poll_interval).await; - continue; - } + } => { + debug!( + target: "bridge.nock-watcher", + tip_height, + confirmed_target, + next_needed_height, + "target height not yet confirmed for kernel need" + ); + sleep(poll_interval).await; + continue; + } + NockPlanAction::FetchHeight { + tip_height, height, .. + } => (tip_height, height), + }; - match self.fetch_block_at_height(client, next_needed_height).await { - Ok(Some(observation)) => { - let height = observation.block.height; - let block_hash = observation.block.digest.to_base58(); - let txs_count = observation.txs.len(); - let event = NockBlockEvent { - block: observation.block, - page_slab: observation.page_slab, - page_noun: observation.page_noun, - txs: observation.txs, - }; - self.runtime + match source.fetch_block_at_height(target_height).await { + Ok(Some(event)) => { + let height = event.block.height; + let block_hash = event.block.digest.to_base58(); + let txs_count = event.txs.len(); + self.deps + .runtime .send_event(BridgeEvent::Chain(Box::new(ChainEvent::Nock(event)))) .await?; info!( @@ -340,109 +520,21 @@ impl NockchainWatcher { Ok(None) => { debug!( target: "bridge.nock-watcher", - target = next_needed_height, + target = target_height, "block at target height not found" ); } Err(err) => { warn!( target: "bridge.nock-watcher", - target = next_needed_height, + target = target_height, error=%err, "failed to fetch block at height" ); } } - sleep(self.poll_interval).await; - } - } - - async fn fetch_tip_info( - &self, - client: &mut PrivateNockAppGrpcClient, - ) -> Result, BridgeError> { - let heavy_path = vec![Bytes::from("heavy")]; - let heavy_bytes = jam_path(&heavy_path)?; - let response = client - .peek(CLIENT_PID, heavy_bytes) - .await - .map_err(|err| BridgeError::EventMonitoring(err.to_string()))?; - let (_heavy_slab, heavy_noun) = cue_response(response)?; - let heavy: Heavy = heavy_noun.decode().map_err(|err| { - BridgeError::EventMonitoring(format!("failed to decode heavy response: {}", err)) - })?; - let Some(block_id_base58) = heavy.to_base58() else { - return Ok(None); - }; - let tip_hash = block_id_base58.clone(); - - let block_path = vec![Bytes::from("block"), Bytes::from(block_id_base58.clone())]; - let block_bytes = jam_path(&block_path)?; - let response = client - .peek(CLIENT_PID, block_bytes) - .await - .map_err(|err| BridgeError::EventMonitoring(err.to_string()))?; - let (_page_slab, block_noun) = cue_response(response)?; - - let (page, _page_noun) = decode_page_from_peek(&block_noun)?; - Ok(Some((page.height, tip_hash))) - } - - async fn fetch_block_at_height( - &self, - client: &mut PrivateNockAppGrpcClient, - height: u64, - ) -> Result, BridgeError> { - let heavy_n_path = vec![Bytes::from("heavy-n"), Bytes::from(height.to_bytes()?)]; - let heavy_n_bytes = jam_path(&heavy_n_path)?; - let response = client - .peek(CLIENT_PID, heavy_n_bytes) - .await - .map_err(|err| BridgeError::EventMonitoring(err.to_string()))?; - let (page_slab, block_noun) = cue_response(response)?; - - let (page, page_noun) = match decode_page_from_peek(&block_noun) { - Ok(result) => result, - Err(_) => return Ok(None), - }; - - let txs = self - .fetch_transactions(client, &page.digest, &page.tx_ids) - .await?; - - Ok(Some(NockBlockObservation { - block: page, - page_slab, - page_noun, - txs, - })) - } - - async fn fetch_transactions( - &self, - client: &mut PrivateNockAppGrpcClient, - block_id: &BlockId, - tx_ids: &[TxId], - ) -> Result, BridgeError> { - let block_id_base58 = block_id.to_base58(); - let mut txs = Vec::with_capacity(tx_ids.len()); - for tx_id in tx_ids { - let tx_id_base58 = tx_id.to_base58(); - let tx_path = vec![ - Bytes::from("block-transaction"), - Bytes::from(block_id_base58.clone()), - Bytes::from(tx_id_base58), - ]; - let tx_bytes = jam_path(&tx_path)?; - let response = client - .peek(CLIENT_PID, tx_bytes) - .await - .map_err(|err| BridgeError::EventMonitoring(err.to_string()))?; - let (_tx_slab, tx_noun) = cue_response(response)?; - let tx = decode_tx_from_peek(&tx_noun)?; - txs.push((tx_id.clone(), tx)); + sleep(poll_interval).await; } - Ok(txs) } } @@ -599,7 +691,8 @@ fn jam_path(path: &[Bytes]) -> Result, BridgeError> { segment.len(), segment.as_ptr(), ); - ia.normalize_as_atom().as_noun() + let space = slab.noun_space(); + ia.normalize_as_atom(&space).as_noun() }; list = nockvm::noun::T(&mut slab, &[atom, list]); } @@ -615,15 +708,19 @@ fn cue_response(bytes: Vec) -> Result<(NounSlab, nockapp::Noun), Ok((slab, noun)) } -fn decode_page_from_peek(noun: &nockapp::Noun) -> Result<(Page, nockapp::Noun), BridgeError> { +fn decode_page_from_peek( + noun: &nockapp::Noun, + space: &nockvm::noun::NounSpace, +) -> Result<(Page, nockapp::Noun), BridgeError> { let outer_cell = noun + .in_space(space) .as_cell() .map_err(|_| BridgeError::EventMonitoring("peek response expected to be cell".into()))?; - let outer_head = outer_cell.head(); - let outer_tail = outer_cell.tail(); + let outer_head = outer_cell.head().noun(); + let outer_tail = outer_cell.tail().noun(); - let outer_tag = outer_head.as_atom().map_err(|_| { + let outer_tag = outer_head.in_space(space).as_atom().map_err(|_| { BridgeError::EventMonitoring("peek response outer unit tag expected to be atom".into()) })?; @@ -646,18 +743,18 @@ fn decode_page_from_peek(noun: &nockapp::Noun) -> Result<(Page, nockapp::Noun), )); } - let inner_cell = inner.as_cell().map_err(|_| { + let inner_cell = inner.in_space(space).as_cell().map_err(|_| { BridgeError::EventMonitoring("peek response inner expected to be cell".into()) })?; - let inner_head = inner_cell.head(); - let inner_tail = inner_cell.tail(); + let inner_head = inner_cell.head().noun(); + let inner_tail = inner_cell.tail().noun(); - if let Ok(tag_atom) = inner_head.as_atom() { + if let Ok(tag_atom) = inner_head.in_space(space).as_atom() { if let Ok(tag_val) = tag_atom.as_u64() { if tag_val == 0 { if inner_tail.is_atom() { - if let Ok(atom) = inner_tail.as_atom() { + if let Ok(atom) = inner_tail.in_space(space).as_atom() { if let Ok(val) = atom.as_u64() { if val == 0 { return Err(BridgeError::EventMonitoring( @@ -671,7 +768,7 @@ fn decode_page_from_peek(noun: &nockapp::Noun) -> Result<(Page, nockapp::Noun), )); } - let page = Page::from_noun(&inner_tail).map_err(|err| { + let page = Page::from_noun(&inner_tail, space).map_err(|err| { BridgeError::EventMonitoring(format!( "failed to decode Page (after inner unwrap): {}", err @@ -682,14 +779,18 @@ fn decode_page_from_peek(noun: &nockapp::Noun) -> Result<(Page, nockapp::Noun), } } - let page = Page::from_noun(&inner) + let page = Page::from_noun(&inner, space) .map_err(|err| BridgeError::EventMonitoring(format!("failed to decode Page: {}", err)))?; Ok((page, inner)) } -fn decode_tx_from_peek(noun: &nockapp::Noun) -> Result { +fn decode_tx_from_peek( + noun: &nockapp::Noun, + space: &nockvm::noun::NounSpace, +) -> Result { // peek returns (unit (unit tx:t)), need to unwrap both layers let outer_cell = noun + .in_space(space) .as_cell() .map_err(|_| BridgeError::EventMonitoring("peek response expected to be cell".into()))?; @@ -707,9 +808,14 @@ fn decode_tx_from_peek(noun: &nockapp::Noun) -> Result { )); } - let inner_cell = outer_cell.tail().as_cell().map_err(|_| { - BridgeError::EventMonitoring("peek response inner unit expected to be cell".into()) - })?; + let inner_cell = outer_cell + .tail() + .noun() + .in_space(space) + .as_cell() + .map_err(|_| { + BridgeError::EventMonitoring("peek response inner unit expected to be cell".into()) + })?; let inner_tag = inner_cell.head().as_atom().map_err(|_| { BridgeError::EventMonitoring("peek response inner unit tag expected to be atom".into()) @@ -725,7 +831,8 @@ fn decode_tx_from_peek(noun: &nockapp::Noun) -> Result { )); } - Tx::from_noun(&inner_cell.tail()) + let tx_noun = inner_cell.tail().noun(); + Tx::from_noun(&tx_noun, space) .map_err(|err| BridgeError::EventMonitoring(format!("failed to decode Tx: {}", err))) } @@ -733,7 +840,7 @@ fn decode_tx_from_peek(noun: &nockapp::Noun) -> Result { mod tests { use super::*; - fn atom_to_string(atom: nockvm::noun::Atom) -> String { + fn atom_to_string(atom: nockvm::noun::AtomHandle<'_>) -> String { let mut bytes = atom.to_be_bytes(); while bytes.first() == Some(&0) { bytes.remove(0); @@ -750,12 +857,13 @@ mod tests { let mut current = slab .cue_into(Bytes::from(jammed.clone())) .expect("cue jammed path"); + let space = slab.noun_space(); for segment in path { - let cell = current.as_cell().expect("cell"); + let cell = current.in_space(&space).as_cell().expect("cell"); let atom = cell.head().as_atom().expect("atom"); let decoded = atom_to_string(atom); assert_eq!(decoded, segment); - current = cell.tail(); + current = cell.tail().noun(); } } @@ -840,7 +948,8 @@ mod tests { let inner_unit = nockvm::noun::T(&mut slab, &[nockvm::noun::D(0), page_noun]); let peek_noun = nockvm::noun::T(&mut slab, &[nockvm::noun::D(0), inner_unit]); - let (page, _) = decode_page_from_peek(&peek_noun).expect("decode tagged-bn page"); + let space = slab.noun_space(); + let (page, _) = decode_page_from_peek(&peek_noun, &space).expect("decode tagged-bn page"); assert_eq!(page.digest, digest, "digest should decode correctly"); assert_eq!(page.parent, parent, "parent should decode correctly"); diff --git a/crates/bridge/src/ports.rs b/crates/bridge/src/ports.rs new file mode 100644 index 000000000..c38501e9e --- /dev/null +++ b/crates/bridge/src/ports.rs @@ -0,0 +1,54 @@ +use async_trait::async_trait; + +use crate::errors::BridgeError; +use crate::ethereum::DepositSubmissionResult; +use crate::runtime::{BaseBlockBatch, ChainEvent, EventId, NockBlockEvent}; +use crate::types::{DepositSubmission, Tip5Hash}; + +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct NockTipInfo { + pub height: u64, + pub tip_hash: String, +} + +#[async_trait] +pub trait BaseContractPort: Send + Sync { + async fn submit_deposit( + &self, + submission: DepositSubmission, + ) -> Result; + + async fn get_last_deposit_nonce(&self) -> Result; + + async fn is_deposit_processed(&self, tx_id: &Tip5Hash) -> Result; +} + +#[async_trait] +pub trait BaseSourcePort: Send + Sync { + async fn chain_tip_height(&self) -> Result; + + async fn fetch_batch(&self, start: u64, end: u64) -> Result; +} + +#[async_trait] +pub trait NockSourcePort: Send { + async fn tip_info(&mut self) -> Result, BridgeError>; + + async fn fetch_block_at_height( + &mut self, + height: u64, + ) -> Result, BridgeError>; +} + +#[async_trait] +pub trait KernelStatePort: Send + Sync { + async fn peek_base_hold(&self) -> Result; + + async fn peek_base_next_height(&self) -> Result, BridgeError>; + + async fn peek_nock_next_height(&self) -> Result, BridgeError>; + + async fn emit_chain_event(&self, event: ChainEvent) -> Result; + + fn set_base_tip_hash(&self, tip_hash: String); +} diff --git a/crates/bridge/src/runtime.rs b/crates/bridge/src/runtime.rs index 15804017c..91b9e3af7 100644 --- a/crates/bridge/src/runtime.rs +++ b/crates/bridge/src/runtime.rs @@ -3,6 +3,7 @@ use std::sync::{Arc, RwLock}; use std::time::{Duration, Instant, SystemTime, UNIX_EPOCH}; use alloy::primitives::Address; +use async_trait::async_trait; use hex::encode; use nockapp::driver::{make_driver, NockAppHandle}; use nockapp::nockapp::wire::WireRepr; @@ -10,6 +11,7 @@ use nockapp::noun::slab::{NockJammer, NounSlab}; use nockapp::one_punch::OnePunchWire; use nockapp::wire::Wire; use nockapp::Bytes; +use nockvm::noun::NounAllocator; use noun_serde::{NounDecode, NounEncode}; use tokio::sync::mpsc::{Receiver, Sender}; use tokio::sync::{mpsc, oneshot}; @@ -17,10 +19,19 @@ use tracing::{debug, error, info, warn}; use crate::bridge_status::BridgeStatus; use crate::config::NonceEpochConfig; +use crate::core::posting::{ + PostingCandidateDecision, PostingPlanner, PostingReadyProposal, PostingTickPlanInput, +}; +use crate::core::signing::{ + SigningCandidatePrecheckDecision, SigningCandidatePrecheckInput, SigningEpochBoundsDecision, + SigningEpochBoundsDecisionInput, SigningPlanner, SigningProcessedDecision, + SigningProcessedDecisionInput, SigningTickPlanAction, SigningTickPlanInput, +}; use crate::errors::BridgeError; -use crate::ethereum::BaseBridge; use crate::health::PeerEndpoint; +use crate::loop_policy::{PostingLoopPolicy, SigningLoopPolicy}; use crate::metrics; +use crate::ports::{BaseContractPort, KernelStatePort}; use crate::proposal_cache::ProposalCache; use crate::signing::BridgeSigner; use crate::types::{ @@ -299,26 +310,37 @@ impl NockchainTxEffect { } #[derive(Clone)] -pub struct BridgeRuntimeHandle { +struct BridgeRuntimeHandleChannels { inbound_tx: Sender>, effect_tx: Sender>, peek_tx: Sender, poke_tx: Sender, +} + +#[derive(Clone)] +struct BridgeRuntimeHandleState { base_tip_hash: Arc>>, } +#[derive(Clone)] +pub struct BridgeRuntimeHandle { + channels: BridgeRuntimeHandleChannels, + state: BridgeRuntimeHandleState, +} + impl BridgeRuntimeHandle { pub fn set_base_tip_hash(&self, tip_hash: String) { if tip_hash.is_empty() { return; } - if let Ok(mut guard) = self.base_tip_hash.write() { + if let Ok(mut guard) = self.state.base_tip_hash.write() { *guard = Some(tip_hash); } } pub fn get_base_tip_hash(&self) -> Option { - self.base_tip_hash + self.state + .base_tip_hash .read() .ok() .and_then(|guard| guard.clone()) @@ -327,20 +349,34 @@ impl BridgeRuntimeHandle { pub async fn send_event(&self, event: BridgeEvent) -> Result { let id = make_event_id(event.kind(), &event.identity_material()); let envelope = EventEnvelope { id, payload: event }; - self.inbound_tx + self.channels + .inbound_tx .send(envelope) .await .map_err(|e| BridgeError::Runtime(format!("inbound channel closed: {}", e)))?; Ok(id) } + /// Typed helper for harnesses/tests to inject a Base batch event. + pub async fn inject_base_batch(&self, batch: BaseBlockBatch) -> Result { + self.send_event(BridgeEvent::Chain(Box::new(ChainEvent::Base(batch)))) + .await + } + + /// Typed helper for harnesses/tests to inject a nock block event. + pub async fn inject_nock_block(&self, block: NockBlockEvent) -> Result { + self.send_event(BridgeEvent::Chain(Box::new(ChainEvent::Nock(block)))) + .await + } + pub async fn send_effect(&self, effect: BridgeEffect) -> Result { let id = make_effect_id(effect.kind(), &effect.identity_material()); let envelope = EventEnvelope { id, payload: effect, }; - self.effect_tx + self.channels + .effect_tx .send(envelope) .await .map_err(|e| BridgeError::Runtime(format!("effect channel closed: {}", e)))?; @@ -501,7 +537,8 @@ impl BridgeRuntimeHandle { let slab = cue_bytes(bytes)?; let noun = unsafe { slab.root() }; - let peek = StopInfoPeek::from_noun(noun).map_err(|err| { + let space = slab.noun_space(); + let peek = StopInfoPeek::from_noun(noun, &space).map_err(|err| { BridgeError::Runtime(format!("failed to decode peek stop-info: {}", err)) })?; Ok(peek.inner.flatten()) @@ -609,7 +646,8 @@ impl BridgeRuntimeHandle { }; let slab = cue_bytes(bytes)?; let noun = unsafe { slab.root() }; - let peek = CountPeek::from_noun(noun) + let space = slab.noun_space(); + let peek = CountPeek::from_noun(noun, &space) .map_err(|err| BridgeError::Runtime(format!("failed to decode peek count: {}", err)))?; Ok(peek.inner.flatten().unwrap_or(0)) } @@ -625,7 +663,8 @@ impl BridgeRuntimeHandle { }; let slab = cue_bytes(bytes)?; let noun = unsafe { slab.root() }; - let peek = BoolPeek::from_noun(noun) + let space = slab.noun_space(); + let peek = BoolPeek::from_noun(noun, &space) .map_err(|err| BridgeError::Runtime(format!("failed to decode peek bool: {}", err)))?; Ok(peek.inner.flatten().unwrap_or(false)) } @@ -641,7 +680,8 @@ impl BridgeRuntimeHandle { }; let slab = cue_bytes(bytes)?; let noun = unsafe { slab.root() }; - let peek = HeightPeek::from_noun(noun).map_err(|err| { + let space = slab.noun_space(); + let peek = HeightPeek::from_noun(noun, &space).map_err(|err| { BridgeError::Runtime(format!("failed to decode peek height: {}", err)) })?; match peek.inner { @@ -661,7 +701,8 @@ impl BridgeRuntimeHandle { }; let slab = cue_bytes(bytes)?; let noun = unsafe { slab.root() }; - let peek = HoldPeek::from_noun(noun) + let space = slab.noun_space(); + let peek = HoldPeek::from_noun(noun, &space) .map_err(|err| BridgeError::Runtime(format!("failed to decode peek hold: {}", err)))?; Ok(peek.inner.flatten()) } @@ -681,7 +722,8 @@ impl BridgeRuntimeHandle { let slab = cue_bytes(bytes)?; let noun = unsafe { slab.root() }; - let decoded = T::from_noun(noun).map_err(|err| { + let space = slab.noun_space(); + let decoded = T::from_noun(noun, &space).map_err(|err| { BridgeError::Runtime(format!("failed to decode typed peek response: {}", err)) })?; Ok(Some(decoded)) @@ -692,7 +734,8 @@ impl BridgeRuntimeHandle { path_slab: NounSlab, ) -> Result>, BridgeError> { let (respond_to, response) = oneshot::channel(); - self.peek_tx + self.channels + .peek_tx .send(PeekRequest { path_slab, respond_to, @@ -708,7 +751,8 @@ impl BridgeRuntimeHandle { /// This is used by the ingress service to poke the kernel with proposed-base-call /// when validating incoming proposals from peers. pub async fn send_poke(&self, poke: BridgePoke) -> Result<(), BridgeError> { - self.poke_tx + self.channels + .poke_tx .send(poke) .await .map_err(|e| BridgeError::Runtime(format!("poke channel closed: {}", e))) @@ -733,6 +777,29 @@ impl BridgeRuntimeHandle { } } +#[async_trait] +impl KernelStatePort for BridgeRuntimeHandle { + async fn peek_base_hold(&self) -> Result { + BridgeRuntimeHandle::peek_base_hold(self).await + } + + async fn peek_base_next_height(&self) -> Result, BridgeError> { + BridgeRuntimeHandle::peek_base_next_height(self).await + } + + async fn peek_nock_next_height(&self) -> Result, BridgeError> { + BridgeRuntimeHandle::peek_nock_next_height(self).await + } + + async fn emit_chain_event(&self, event: ChainEvent) -> Result { + self.send_event(BridgeEvent::Chain(Box::new(event))).await + } + + fn set_base_tip_hash(&self, tip_hash: String) { + BridgeRuntimeHandle::set_base_tip_hash(self, tip_hash); + } +} + pub trait CauseBuilder: Send + Sync { fn build_poke( &self, @@ -793,7 +860,8 @@ impl CauseBuilder for KernelCauseBuilder { "building nockchain-block cause from block" ); let mut poke_slab = NounSlab::new(); - let page_noun = poke_slab.copy_into(nock_block.page_noun); + let page_noun = + poke_slab.copy_into(nock_block.page_noun, &nock_block.page_slab.noun_space()); let tag = String::from("nockchain-block").to_noun(&mut poke_slab); let txs = NockchainTxsMap(nock_block.txs.clone()).to_noun(&mut poke_slab); let cause = @@ -826,16 +894,29 @@ struct PeekRequest { respond_to: oneshot::Sender>, BridgeError>>, } -pub struct BridgeRuntime { +struct BridgeRuntimeDeps { cause_builder: Arc, +} + +struct BridgeRuntimeChannels { inbound_rx: Receiver>, effect_rx: Receiver>, poke_tx: Sender, poke_rx: Option>, peek_rx: Option>, +} + +#[derive(Default)] +struct BridgeRuntimeState { pending_events: VecDeque>, } +pub struct BridgeRuntime { + deps: BridgeRuntimeDeps, + channels: BridgeRuntimeChannels, + state: BridgeRuntimeState, +} + impl BridgeRuntime { pub fn new(cause_builder: Arc) -> (Self, BridgeRuntimeHandle) { let (inbound_tx, inbound_rx) = mpsc::channel(256); @@ -845,20 +926,24 @@ impl BridgeRuntime { let base_tip_hash = Arc::new(RwLock::new(None)); let handle_poke_tx = poke_tx.clone(); let runtime = BridgeRuntime { - cause_builder, - inbound_rx, - effect_rx, - poke_tx, - poke_rx: Some(poke_rx), - peek_rx: Some(peek_rx), - pending_events: VecDeque::new(), + deps: BridgeRuntimeDeps { cause_builder }, + channels: BridgeRuntimeChannels { + inbound_rx, + effect_rx, + poke_tx, + poke_rx: Some(poke_rx), + peek_rx: Some(peek_rx), + }, + state: BridgeRuntimeState::default(), }; let handle = BridgeRuntimeHandle { - inbound_tx, - effect_tx, - peek_tx, - poke_tx: handle_poke_tx, - base_tip_hash, + channels: BridgeRuntimeHandleChannels { + inbound_tx, + effect_tx, + peek_tx, + poke_tx: handle_poke_tx, + }, + state: BridgeRuntimeHandleState { base_tip_hash }, }; (runtime, handle) } @@ -868,10 +953,12 @@ impl BridgeRuntime { app: &mut nockapp::NockApp, ) -> Result<(), BridgeError> { let poke_rx = self + .channels .poke_rx .take() .ok_or_else(|| BridgeError::Runtime("driver already installed".into()))?; let peek_rx = self + .channels .peek_rx .take() .ok_or_else(|| BridgeError::Runtime("driver already installed".into()))?; @@ -914,13 +1001,13 @@ impl BridgeRuntime { // Use biased to prioritize channel messages over timer biased; - event = self.inbound_rx.recv() => { + event = self.channels.inbound_rx.recv() => { match event { Some(e) => self.process_event(e).await?, None => break, // Channel closed, shutdown } } - effect = self.effect_rx.recv() => { + effect = self.channels.effect_rx.recv() => { match effect { Some(e) => self.process_effect(e).await?, None => break, // Channel closed, shutdown @@ -935,10 +1022,11 @@ impl BridgeRuntime { &mut self, event: EventEnvelope, ) -> Result<(), BridgeError> { - let outcome = self.cause_builder.build_poke(&event)?; + let outcome = self.deps.cause_builder.build_poke(&event)?; match outcome { CauseBuildOutcome::Emit(poke) => { - self.poke_tx + self.channels + .poke_tx .send(poke) .await .map_err(|e| BridgeError::Runtime(format!("failed to enqueue poke: {}", e)))?; @@ -952,7 +1040,7 @@ impl BridgeRuntime { kind=%kind, digest=%digest, reason=%reason, - pending=self.pending_events.len(), + pending=self.state.pending_events.len(), "event deferred" ); } @@ -990,8 +1078,8 @@ impl BridgeRuntime { } fn enqueue_pending(&mut self, event: EventEnvelope) { - if self.pending_events.len() >= MAX_PENDING_EVENTS { - if let Some(oldest) = self.pending_events.pop_front() { + if self.state.pending_events.len() >= MAX_PENDING_EVENTS { + if let Some(oldest) = self.state.pending_events.pop_front() { warn!( target: "bridge.runtime", kind=%oldest.id.kind.as_str(), @@ -1000,7 +1088,7 @@ impl BridgeRuntime { ); } } - self.pending_events.push_back(event); + self.state.pending_events.push_back(event); } } @@ -1063,296 +1151,754 @@ impl SignatureBroadcastReason { } } -/// Background loop that deterministically selects deposits to sign from shared history + chain tip. -/// -/// This decouples signing from `%commit-nock-deposits` effects so nodes can restart at different -/// nock heights and still converge on the same `lastDepositNonce + 1` deposit for signing. -#[allow(clippy::too_many_arguments)] -pub async fn run_signing_cursor_loop( - runtime: Arc, - base_bridge: Arc, - deposit_log: Arc, - nonce_epoch: &NonceEpochConfig, - proposal_cache: Arc, - signer: Arc, - valid_addresses: HashSet
, - peers: Vec, - self_node_id: u64, - bridge_status: BridgeStatus, - address_to_node_id: std::collections::HashMap, - stop_controller: crate::stop::StopController, - stop: crate::stop::StopHandle, -) { - use std::time::Instant; +fn system_time_secs(now: SystemTime) -> u64 { + now.duration_since(UNIX_EPOCH).unwrap_or_default().as_secs() +} - use tokio::time::{interval, MissedTickBehavior}; - use tracing::{debug, error, info, warn}; +fn spawn_signature_broadcast( + peers: &[PeerEndpoint], + msg: &crate::ingress::proto::SignatureBroadcast, + prop_id: &str, + reason: SignatureBroadcastReason, +) { + use tracing::{debug, warn}; use crate::ingress::proto::bridge_ingress_client::BridgeIngressClient; - use crate::ingress::proto::SignatureBroadcast; - use crate::signing::verify_bridge_signature; - use crate::stop::trigger_local_stop; - use crate::types::{DepositId, NockDepositRequestData}; - - const POLL_INTERVAL: Duration = Duration::from_secs(15); - const PIPELINE_DEPTH: usize = 4; - const REGOSSIP_INTERVAL: Duration = Duration::from_secs(90); - - let my_eth_address = signer.address(); - - // Fire-and-forget broadcast of a signature to all peers. - fn spawn_signature_broadcast( - peers: &[PeerEndpoint], - msg: &SignatureBroadcast, - prop_id: &str, - reason: SignatureBroadcastReason, - ) { - for peer in peers { - let msg = msg.clone(); - let addr = peer.address.clone(); - let peer_id = peer.node_id; - let prop_id = prop_id.to_string(); - - tokio::spawn(async move { - match BridgeIngressClient::connect(addr.clone()).await { - Ok(mut client) => match client.broadcast_signature(msg).await { - Ok(_) => { - debug!( - target: "bridge.cursor", - peer_node_id=peer_id, - proposal_hash=%prop_id, - reason=reason.as_str(), - "broadcast signature to peer" - ); - } - Err(e) => { - warn!( - target: "bridge.cursor", - peer_node_id=peer_id, - error=%e, - reason=reason.as_str(), - "failed to broadcast signature to peer" - ); - } - }, + + for peer in peers { + let msg = msg.clone(); + let addr = peer.address.clone(); + let peer_id = peer.node_id; + let prop_id = prop_id.to_string(); + + tokio::spawn(async move { + match BridgeIngressClient::connect(addr.clone()).await { + Ok(mut client) => match client.broadcast_signature(msg).await { + Ok(_) => { + debug!( + target: "bridge.cursor", + peer_node_id=peer_id, + proposal_hash=%prop_id, + reason=reason.as_str(), + "broadcast signature to peer" + ); + } Err(e) => { warn!( target: "bridge.cursor", peer_node_id=peer_id, - peer_address=%addr, error=%e, reason=reason.as_str(), - "failed to connect to peer for signature broadcast" + "failed to broadcast signature to peer" ); } + }, + Err(e) => { + warn!( + target: "bridge.cursor", + peer_node_id=peer_id, + peer_address=%addr, + error=%e, + reason=reason.as_str(), + "failed to connect to peer for signature broadcast" + ); } - }); + } + }); + } +} + +#[derive(Clone, Debug)] +pub struct SigningTickState { + pub logged_epoch_ready: bool, + pub last_regossip_at: SystemTime, +} + +impl SigningTickState { + pub fn new(now: SystemTime) -> Self { + Self { + logged_epoch_ready: false, + last_regossip_at: now, } } +} - info!( - target: "bridge.cursor", - poll_interval_secs=POLL_INTERVAL.as_secs(), - pipeline_depth=PIPELINE_DEPTH, - regossip_interval_secs=REGOSSIP_INTERVAL.as_secs(), - "starting signing cursor loop" - ); +impl Default for SigningTickState { + fn default() -> Self { + Self::new(UNIX_EPOCH) + } +} - let mut ticker = interval(POLL_INTERVAL); - ticker.set_missed_tick_behavior(MissedTickBehavior::Skip); - let mut logged_epoch_ready = false; - let mut last_regossip = Instant::now(); +#[derive(Clone, Copy, Debug)] +pub struct SigningTickInput { + pub now: SystemTime, + pub tip_height: Option, +} - loop { - ticker.tick().await; +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] +pub struct SigningTickOutcome { + pub regossip_broadcasts: usize, + pub initial_broadcasts: usize, +} - if stop.is_stopped() { - continue; +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +enum SigningCandidateExecutionResult { + Broadcasted, + SignFailed, + DuplicateSignature, + ProposalStale, + InvalidOwnSignature, + CacheUpdateFailed, +} + +#[derive(Clone)] +pub struct SigningTickPorts { + runtime: Arc, + base_bridge: Arc, + deposit_log: Arc, + proposal_cache: Arc, +} + +impl SigningTickPorts { + pub fn new( + runtime: Arc, + base_bridge: Arc, + deposit_log: Arc, + proposal_cache: Arc, + ) -> Self { + Self { + runtime, + base_bridge, + deposit_log, + proposal_cache, } + } +} - // Periodically re-gossip our own signatures for deposits still collecting. - if last_regossip.elapsed() >= REGOSSIP_INTERVAL { - match proposal_cache.collecting_with_my_sig() { - Ok(pending) => { - for (deposit_id, state) in pending { - let Some(sig) = state.my_signature.clone() else { - continue; - }; - - let broadcast_msg = SignatureBroadcast { - deposit_id: deposit_id.to_bytes(), - proposal_hash: state.proposal_hash.to_vec(), - signature: sig, - signer_address: my_eth_address.as_slice().to_vec(), - timestamp: SystemTime::now() - .duration_since(UNIX_EPOCH) - .unwrap_or_default() - .as_secs(), - }; - - let prop_id = hex::encode(state.proposal_hash); - spawn_signature_broadcast( - &peers, - &broadcast_msg, - &prop_id, - SignatureBroadcastReason::Regossip, - ); - } - } - Err(err) => { - warn!( - target: "bridge.cursor", - error=%err, - "failed to gather proposals for signature re-gossip" - ); - } - } +#[derive(Clone)] +pub struct SigningTickNodeState { + signer: Arc, + valid_addresses: Arc>, + peers: Arc>, + self_node_id: u64, + address_to_node_id: Arc>, +} - last_regossip = Instant::now(); +impl SigningTickNodeState { + pub fn new( + signer: Arc, + valid_addresses: HashSet
, + peers: Vec, + self_node_id: u64, + address_to_node_id: std::collections::HashMap, + ) -> Self { + Self { + signer, + valid_addresses: Arc::new(valid_addresses), + peers: Arc::new(peers), + self_node_id, + address_to_node_id: Arc::new(address_to_node_id), } + } +} - let tip_height = match runtime.nock_hashchain_tip().await { - Ok(height) => height, - Err(err) => { - warn!( - target: "bridge.cursor", - error=%err, - "failed to peek nock hashchain tip height" - ); - continue; - } - }; - let Some(tip_height) = tip_height else { - debug!( - target: "bridge.cursor", - "no nock hashchain tip yet, waiting before signing" - ); - continue; - }; - if tip_height < nonce_epoch.start_height { - debug!( - target: "bridge.cursor", - tip_height, - nonce_epoch_start_height = nonce_epoch.start_height, - "hashchain behind nonce epoch start height, waiting to sign" - ); - continue; +#[derive(Clone)] +pub struct SigningTickControl { + bridge_status: BridgeStatus, + stop_controller: crate::stop::StopController, + stop: crate::stop::StopHandle, + local_stop_mode: SigningLocalStopMode, +} + +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] +pub enum SigningLocalStopMode { + #[default] + RuntimeProbeAndBroadcast, + LocalTriggerOnly, +} + +impl SigningTickControl { + pub fn new( + bridge_status: BridgeStatus, + stop_controller: crate::stop::StopController, + stop: crate::stop::StopHandle, + ) -> Self { + Self { + bridge_status, + stop_controller, + stop, + local_stop_mode: SigningLocalStopMode::RuntimeProbeAndBroadcast, } - if !logged_epoch_ready { - logged_epoch_ready = true; - info!( - target: "bridge.cursor", - tip_height, - nonce_epoch_start_height = nonce_epoch.start_height, - "hashchain reached nonce epoch start height, signing enabled" - ); + } + + pub fn with_local_stop_mode(mut self, local_stop_mode: SigningLocalStopMode) -> Self { + self.local_stop_mode = local_stop_mode; + self + } +} + +#[derive(Clone)] +pub struct SigningTickConfig { + nonce_epoch: NonceEpochConfig, + policy: SigningLoopPolicy, +} + +impl SigningTickConfig { + pub fn new(nonce_epoch: &NonceEpochConfig, policy: SigningLoopPolicy) -> Self { + Self { + nonce_epoch: nonce_epoch.clone(), + policy, } + } +} - // Query the chain for the last confirmed deposit nonce. - // This is the source of truth for the signing cursor. - let last_chain_nonce = match base_bridge.get_last_deposit_nonce().await { - Ok(n) => n, +#[derive(Clone)] +pub struct SigningTickContext { + ports: SigningTickPorts, + node: SigningTickNodeState, + control: SigningTickControl, + config: SigningTickConfig, +} + +impl SigningTickContext { + pub fn new( + ports: SigningTickPorts, + node: SigningTickNodeState, + control: SigningTickControl, + config: SigningTickConfig, + ) -> Self { + Self { + ports, + node, + control, + config, + } + } + + async fn execute_signing_candidate( + &self, + req: &crate::types::NockDepositRequestData, + deposit_id: &crate::types::DepositId, + now: SystemTime, + now_secs: u64, + ) -> SigningCandidateExecutionResult { + use tracing::{debug, error, info, warn}; + + use crate::ingress::proto::SignatureBroadcast; + use crate::signing::verify_bridge_signature; + + let my_eth_address = self.node.signer.address(); + let proposal_hash = req.compute_proposal_hash(); + let proposal_id = hex::encode(proposal_hash); + + // Ensure the proposal is visible in the TUI even if no kernel `%commit-nock-deposits` + // effect fires on this node (e.g. after restart). + self.control + .bridge_status + .update_proposal(crate::tui::types::Proposal { + id: proposal_id.clone(), + proposal_type: "deposit".to_string(), + description: format!( + "Deposit {} wei to {} (nonce {})", + req.amount, + hex::encode(req.recipient.0), + req.nonce + ), + signatures_collected: 0, + signatures_required: crate::proposal_cache::SIGNATURE_THRESHOLD as u8, + signers: vec![], + created_at: now, + status: crate::tui::types::ProposalStatus::Pending, + data_hash: proposal_id.clone(), + submitted_at_block: None, + submitted_at: None, + tx_hash: None, + time_to_submit_ms: None, + executed_at_block: None, + source_block: Some(req.block_height), + amount: Some(req.amount as u128), + recipient: Some(format!("0x{}", hex::encode(req.recipient.0))), + nonce: Some(req.nonce), + source_tx_id: Some(format_source_tx_id(&req.tx_id)), + current_proposer: None, + is_my_turn: false, + time_until_takeover: None, + }); + + // Step 1: Sign the proposal locally. + let signature = match self.node.signer.sign_hash(&proposal_hash).await { + Ok(sig) => sig.as_bytes().to_vec(), Err(e) => { - warn!( + error!( target: "bridge.cursor", error=%e, - "failed to query lastDepositNonce from chain" + proposal_hash=%proposal_id, + "failed to sign proposal" ); - continue; + return SigningCandidateExecutionResult::SignFailed; } }; - bridge_status.update_last_deposit_nonce(last_chain_nonce); - let next_nonce = last_chain_nonce + 1; - // Select the next deposit(s) to sign from the local epoch log. - let nonce_epoch_base = nonce_epoch.base; - if last_chain_nonce < nonce_epoch_base { - let reason = format!( - "nonce epoch mismatch: nonce_epoch_base ({nonce_epoch_base}) is greater than on-chain lastDepositNonce ({last_chain_nonce}); check config" - ); - trigger_local_stop( - runtime.clone(), - stop_controller.clone(), - bridge_status.clone(), - reason, - ) - .await; - continue; - } + // Step 2: Add own signature to cache. + let add_result = self.ports.proposal_cache.add_signature( + deposit_id, + crate::proposal_cache::SignatureData { + signer_address: my_eth_address, + signature: signature.clone(), + proposal_hash, + is_mine: true, + }, + Some(req.clone()), + |hash, sig| verify_bridge_signature(hash, sig, &self.node.valid_addresses), + ); - let first_epoch_nonce = nonce_epoch.first_epoch_nonce(); - let spent_epoch_nonces = if last_chain_nonce < first_epoch_nonce { - 0 - } else { - last_chain_nonce - first_epoch_nonce + 1 - }; - let log_len = match deposit_log.number_of_deposits_in_epoch(nonce_epoch).await { - Ok(v) => v, - Err(err) => { - warn!( + if let Ok(report) = self + .ports + .proposal_cache + .apply_pending_signatures(deposit_id, |hash, sig| { + verify_bridge_signature(hash, sig, &self.node.valid_addresses) + }) + { + if report.applied > 0 { + debug!( target: "bridge.cursor", - error=%err, - "failed to count deposits in sqlite" + proposal_hash=%proposal_id, + applied_count=report.applied, + "applied pending signatures from peers" ); - let reason = format!("failed to count deposits in sqlite: {err}"); - trigger_local_stop( - runtime.clone(), - stop_controller.clone(), - bridge_status.clone(), - reason, - ) - .await; - continue; } - }; - if spent_epoch_nonces > log_len { - debug!( - target: "bridge.cursor", - log_len, - spent_epoch_nonces, - nonce_epoch_base, - "deposit log behind chain prefix, waiting for log to catch up" - ); - continue; - } - - let candidates = match deposit_log - .records_from_nonce(next_nonce, PIPELINE_DEPTH, nonce_epoch) - .await - { - Ok(v) => v, - Err(err) => { + if let Some(first) = report.mismatched.first() { + let deposit_id_hex = hex::encode(deposit_id.to_bytes()); + let expected_hex = hex::encode(first.expected_hash); + let received_hex = hex::encode(first.received_hash); warn!( target: "bridge.cursor", - error=%err, - "failed to query candidate deposits from sqlite" + deposit_id=%deposit_id_hex, + expected_hash=%expected_hex, + received_hash=%received_hex, + signer=%first.signer_address, + mismatch_count=report.mismatched.len(), + "peer signature proposal hash mismatch, possible nonce divergence" + ); + self.control.bridge_status.push_alert( + crate::tui::types::AlertSeverity::Error, + "Nonce Divergence Suspected".to_string(), + format!( + "Deposit {} has {} peer signature(s) for a different proposal hash. expected={}, received={}, signer={}", + deposit_id_hex, + report.mismatched.len(), + expected_hex, + received_hex, + first.signer_address + ), + "nonce-divergence".to_string(), ); - continue; } - }; + } - if candidates.is_empty() { - continue; + if let Ok(Some(proposal_state)) = self.ports.proposal_cache.get_state(deposit_id) { + self.control + .bridge_status + .sync_proposal_signatures_from_cache( + &proposal_id, &proposal_state, &self.node.address_to_node_id, + self.node.self_node_id, + ); } - // Sign and gossip signatures for the tip candidate (and optional pipeline). - for (nonce, record) in candidates { - if stop.is_stopped() { - break; + match add_result { + Ok(crate::proposal_cache::SignatureAddResult::Added) + | Ok(crate::proposal_cache::SignatureAddResult::ThresholdReached) => {} + Ok(crate::proposal_cache::SignatureAddResult::Duplicate) => { + debug!( + target: "bridge.cursor", + proposal_hash=%proposal_id, + "duplicate signature, skipping broadcast" + ); + return SigningCandidateExecutionResult::DuplicateSignature; + } + Ok(crate::proposal_cache::SignatureAddResult::Stale) => { + info!( + target: "bridge.cursor", + proposal_hash=%proposal_id, + "proposal already confirmed, skipping broadcast" + ); + return SigningCandidateExecutionResult::ProposalStale; } + Ok(crate::proposal_cache::SignatureAddResult::Invalid(msg)) => { + warn!( + target: "bridge.cursor", + proposal_hash=%proposal_id, + error=%msg, + "own signature invalid, skipping broadcast" + ); + return SigningCandidateExecutionResult::InvalidOwnSignature; + } + Err(e) => { + warn!( + target: "bridge.cursor", + proposal_hash=%proposal_id, + error=%e, + "failed to add own signature to cache, skipping broadcast" + ); + return SigningCandidateExecutionResult::CacheUpdateFailed; + } + } - if nonce_epoch.is_before_start_key(record.block_height, &record.tx_id) { - let reason = format!( - "deposit log contains record below nonce_epoch start key (record_height={} < start_height={}); refusing to sign", - record.block_height, nonce_epoch.start_height + // Step 3: Broadcast signature to all peers (fire-and-forget, concurrent). + let broadcast_msg = SignatureBroadcast { + deposit_id: deposit_id.to_bytes(), + proposal_hash: proposal_hash.to_vec(), + signature, + signer_address: my_eth_address.as_slice().to_vec(), + timestamp: now_secs, + }; + + spawn_signature_broadcast( + self.node.peers.as_ref(), + &broadcast_msg, + &proposal_id, + SignatureBroadcastReason::Initial, + ); + SigningCandidateExecutionResult::Broadcasted + } + + fn execute_regossip(&self, now_secs: u64) -> usize { + use tracing::{debug, warn}; + + use crate::ingress::proto::SignatureBroadcast; + + let my_eth_address = self.node.signer.address(); + let mut broadcasts = 0; + match self.ports.proposal_cache.collecting_with_my_sig() { + Ok(pending) => { + for (deposit_id, proposal_state) in pending { + let Some(sig) = proposal_state.my_signature.clone() else { + continue; + }; + + let broadcast_msg = SignatureBroadcast { + deposit_id: deposit_id.to_bytes(), + proposal_hash: proposal_state.proposal_hash.to_vec(), + signature: sig, + signer_address: my_eth_address.as_slice().to_vec(), + timestamp: now_secs, + }; + + let prop_id = hex::encode(proposal_state.proposal_hash); + spawn_signature_broadcast( + self.node.peers.as_ref(), + &broadcast_msg, + &prop_id, + SignatureBroadcastReason::Regossip, + ); + broadcasts += 1; + } + } + Err(err) => { + warn!( + target: "bridge.cursor", + error=%err, + "failed to gather proposals for signature re-gossip" ); + } + } + if broadcasts > 0 { + debug!( + target: "bridge.cursor", + regossip_broadcasts=broadcasts, + "re-gossiped collecting signatures" + ); + } + broadcasts + } +} + +impl SigningTickContext { + async fn trigger_local_stop(&self, reason: String) { + use std::time::SystemTime; + + use tracing::info; + + use crate::stop::{trigger_local_stop, StopInfo, StopSource}; + use crate::tui::types::AlertSeverity; + + info!( + target: "bridge.cursor", + mode = ?self.control.local_stop_mode, + reason=%reason, + "signing requested local stop" + ); + + match self.control.local_stop_mode { + SigningLocalStopMode::RuntimeProbeAndBroadcast => { trigger_local_stop( - runtime.clone(), - stop_controller.clone(), - bridge_status.clone(), + self.ports.runtime.clone(), + self.control.stop_controller.clone(), + self.control.bridge_status.clone(), reason, ) .await; + } + SigningLocalStopMode::LocalTriggerOnly => { + let metrics = crate::metrics::init_metrics(); + metrics.stop_local_requests.increment(); + + let info = StopInfo { + reason: reason.clone(), + last: None, + source: StopSource::Local, + at: SystemTime::now(), + }; + if !self.control.stop_controller.trigger(info) { + metrics.stop_local_duplicate.increment(); + info!( + target: "bridge.cursor", + reason=%reason, + "local stop already active, skipping duplicate local-only trigger" + ); + return; + } + metrics.stop_local_triggered.increment(); + info!( + target: "bridge.cursor", + reason=%reason, + "local-only stop activated" + ); + self.control.bridge_status.push_alert( + AlertSeverity::Error, + "Bridge Stopped".to_string(), + reason, + "local-stop".to_string(), + ); + } + } + } + + pub async fn tick_once( + &self, + state: &mut SigningTickState, + input: SigningTickInput, + ) -> SigningTickOutcome { + use tracing::{debug, info, warn}; + + use crate::types::{DepositId, NockDepositRequestData}; + + let context = self; + let mut outcome = SigningTickOutcome::default(); + let now = input.now; + let now_secs = system_time_secs(now); + + // Periodically re-gossip our own signatures for deposits still collecting. + if now + .duration_since(state.last_regossip_at) + .unwrap_or_default() + >= context.config.policy.regossip_interval + { + outcome.regossip_broadcasts += context.execute_regossip(now_secs); + state.last_regossip_at = now; + } + + // Always poll chain nonce for health/TUI visibility, even before tip/epoch gates. + let last_chain_nonce = match context.ports.base_bridge.get_last_deposit_nonce().await { + Ok(nonce) => { + context + .control + .bridge_status + .update_last_deposit_nonce(nonce); + Some(nonce) + } + Err(e) => { + warn!( + target: "bridge.cursor", + error=%e, + "failed to query lastDepositNonce from chain" + ); + None + } + }; + + let nonce_epoch_base = context.config.nonce_epoch.base; + let first_epoch_nonce = context.config.nonce_epoch.first_epoch_nonce(); + let mut epoch_ready_logged = false; + let mut maybe_mark_epoch_ready = |tip_height: u64, state: &mut SigningTickState| { + if epoch_ready_logged { + return; + } + state.logged_epoch_ready = true; + epoch_ready_logged = true; + info!( + target: "bridge.cursor", + tip_height, + nonce_epoch_start_height = context.config.nonce_epoch.start_height, + "hashchain reached nonce epoch start height, signing enabled" + ); + }; + let mut plan = SigningPlanner::plan_tick(SigningTickPlanInput { + tip_height: input.tip_height, + nonce_epoch_start_height: context.config.nonce_epoch.start_height, + logged_epoch_ready: state.logged_epoch_ready, + last_chain_nonce, + nonce_epoch_base, + first_epoch_nonce, + log_len: None, + }); + let next_nonce = loop { + match plan { + SigningTickPlanAction::WaitForTip => { + debug!( + target: "bridge.cursor", + "no nock hashchain tip yet, waiting before signing" + ); + return outcome; + } + SigningTickPlanAction::WaitForEpochStart { tip_height } => { + debug!( + target: "bridge.cursor", + tip_height, + nonce_epoch_start_height = context.config.nonce_epoch.start_height, + "hashchain behind nonce epoch start height, waiting to sign" + ); + return outcome; + } + SigningTickPlanAction::NeedLastChainNonce { + tip_height, + reached_epoch_start, + } => { + if reached_epoch_start { + maybe_mark_epoch_ready(tip_height, state); + } + return outcome; + } + SigningTickPlanAction::StopNonceEpochMismatch { + tip_height, + reached_epoch_start, + last_chain_nonce, + nonce_epoch_base, + } => { + if reached_epoch_start { + maybe_mark_epoch_ready(tip_height, state); + } + let reason = format!( + "nonce epoch mismatch: nonce_epoch_base ({nonce_epoch_base}) is greater than on-chain lastDepositNonce ({last_chain_nonce}); check config" + ); + context.trigger_local_stop(reason).await; + return outcome; + } + SigningTickPlanAction::NeedLogLen { + tip_height, + reached_epoch_start, + last_chain_nonce, + } => { + if reached_epoch_start { + maybe_mark_epoch_ready(tip_height, state); + } + + let log_len = match context + .ports + .deposit_log + .number_of_deposits_in_epoch(&context.config.nonce_epoch) + .await + { + Ok(v) => v, + Err(err) => { + warn!( + target: "bridge.cursor", + error=%err, + "failed to count deposits in sqlite" + ); + let reason = format!("failed to count deposits in sqlite: {err}"); + context.trigger_local_stop(reason).await; + return outcome; + } + }; + + plan = SigningPlanner::plan_tick(SigningTickPlanInput { + tip_height: input.tip_height, + nonce_epoch_start_height: context.config.nonce_epoch.start_height, + logged_epoch_ready: state.logged_epoch_ready, + last_chain_nonce: Some(last_chain_nonce), + nonce_epoch_base, + first_epoch_nonce, + log_len: Some(log_len), + }); + } + SigningTickPlanAction::WaitForLogCatchup { + tip_height, + reached_epoch_start, + nonce_epoch_base, + log_len, + spent_epoch_nonces, + .. + } => { + if reached_epoch_start { + maybe_mark_epoch_ready(tip_height, state); + } + debug!( + target: "bridge.cursor", + log_len, + spent_epoch_nonces, + nonce_epoch_base, + "deposit log behind chain prefix, waiting for log to catch up" + ); + return outcome; + } + SigningTickPlanAction::Continue { + tip_height, + reached_epoch_start, + next_nonce, + .. + } => { + if reached_epoch_start { + maybe_mark_epoch_ready(tip_height, state); + } + break next_nonce; + } + } + }; + + let candidates = match context + .ports + .deposit_log + .records_from_nonce( + next_nonce, context.config.policy.pipeline_depth, &context.config.nonce_epoch, + ) + .await + { + Ok(v) => v, + Err(err) => { + warn!( + target: "bridge.cursor", + error=%err, + "failed to query candidate deposits from sqlite" + ); + return outcome; + } + }; + + if candidates.is_empty() { + return outcome; + } + + // Sign and gossip signatures for the tip candidate (and optional pipeline). + for (nonce, record) in candidates { + if context.control.stop.is_stopped() { + break; + } + + if matches!( + SigningPlanner::plan_epoch_bounds(SigningEpochBoundsDecisionInput { + is_before_start_key: context + .config + .nonce_epoch + .is_before_start_key(record.block_height, &record.tx_id), + }), + SigningEpochBoundsDecision::StopRecordBeforeStart + ) { + let reason = format!( + "signing candidate is before nonce_epoch start key (record_height={}, start_height={}); candidate should have been filtered before signing", + record.block_height, context.config.nonce_epoch.start_height + ); + context.trigger_local_stop(reason).await; break; } @@ -1368,41 +1914,55 @@ pub async fn run_signing_cursor_loop( let deposit_id = DepositId::from_effect_payload(&req); - // Skip work if we've already signed this proposal. - if let Ok(Some(state)) = proposal_cache.get_state(&deposit_id) { - if state.status == crate::proposal_cache::ProposalStatus::Confirmed { - continue; - } - if state.my_signature.is_some() { + let existing_state = context + .ports + .proposal_cache + .get_state(&deposit_id) + .ok() + .flatten(); + let precheck = SigningPlanner::plan_candidate_precheck(SigningCandidatePrecheckInput { + is_confirmed: existing_state + .as_ref() + .map(|proposal_state| { + proposal_state.status == crate::proposal_cache::ProposalStatus::Confirmed + }) + .unwrap_or(false), + has_my_signature: existing_state + .as_ref() + .and_then(|proposal_state| proposal_state.my_signature.as_ref()) + .is_some(), + }); + match precheck { + SigningCandidatePrecheckDecision::SkipConfirmed + | SigningCandidatePrecheckDecision::SkipAlreadySigned => { continue; } + SigningCandidatePrecheckDecision::CheckProcessedOnChain => {} } // Optimization: skip signing if deposit is already processed on-chain. // Do not block signing on transient Base RPC errors. - match base_bridge.is_deposit_processed(&req.tx_id).await { - Ok(true) => { - if nonce == next_nonce { - let reason = format!( - "epoch mismatch: tip candidate tx_id is already processed on-chain, but lastDepositNonce={last_chain_nonce} implies nonce {next_nonce} is still pending (check nonce_epoch_start_height/base)" - ); - trigger_local_stop( - runtime.clone(), - stop_controller.clone(), - bridge_status.clone(), - reason, - ) - .await; - break; + match context + .ports + .base_bridge + .is_deposit_processed(&req.tx_id) + .await + { + Ok(processed_on_chain) => { + match SigningPlanner::plan_processed(SigningProcessedDecisionInput { + processed_on_chain, + }) { + SigningProcessedDecision::SkipProcessed => { + debug!( + target: "bridge.cursor", + nonce, + "deposit already processed on-chain, skipping signature" + ); + continue; + } + SigningProcessedDecision::ContinueSign => {} } - debug!( - target: "bridge.cursor", - nonce, - "deposit already processed on-chain, skipping signature" - ); - continue; } - Ok(false) => {} Err(e) => { warn!( target: "bridge.cursor", @@ -1413,324 +1973,652 @@ pub async fn run_signing_cursor_loop( } } - let proposal_hash = req.compute_proposal_hash(); - let proposal_id = hex::encode(proposal_hash); + if matches!( + context + .execute_signing_candidate(&req, &deposit_id, now, now_secs) + .await, + SigningCandidateExecutionResult::Broadcasted + ) { + outcome.initial_broadcasts += 1; + } + } - // Ensure the proposal is visible in the TUI even if no kernel `%commit-nock-deposits` - // effect fires on this node (e.g. after restart). - bridge_status.update_proposal(crate::tui::types::Proposal { - id: proposal_id.clone(), - proposal_type: "deposit".to_string(), - description: format!( - "Deposit {} wei to {} (nonce {})", - req.amount, - hex::encode(req.recipient.0), - req.nonce - ), - signatures_collected: 0, - signatures_required: crate::proposal_cache::SIGNATURE_THRESHOLD as u8, - signers: vec![], - created_at: SystemTime::now(), - status: crate::tui::types::ProposalStatus::Pending, - data_hash: proposal_id.clone(), - submitted_at_block: None, - submitted_at: None, - tx_hash: None, - time_to_submit_ms: None, - executed_at_block: None, - source_block: Some(req.block_height), - amount: Some(req.amount as u128), - recipient: Some(format!("0x{}", hex::encode(req.recipient.0))), - nonce: Some(req.nonce), - source_tx_id: Some(format_source_tx_id(&req.tx_id)), - current_proposer: None, - is_my_turn: false, - time_until_takeover: None, - }); + outcome + } +} + +pub async fn signing_tick_once( + context: &SigningTickContext, + state: &mut SigningTickState, + input: SigningTickInput, +) -> SigningTickOutcome { + context.tick_once(state, input).await +} + +/// Background loop that deterministically selects deposits to sign from shared history + chain tip. +/// +/// This decouples signing from `%commit-nock-deposits` effects so nodes can restart at different +/// nock heights and still converge on the same `lastDepositNonce + 1` deposit for signing. +#[allow(clippy::too_many_arguments)] +pub async fn run_signing_cursor_loop( + runtime: Arc, + base_bridge: Arc, + deposit_log: Arc, + nonce_epoch: &NonceEpochConfig, + proposal_cache: Arc, + signer: Arc, + valid_addresses: HashSet
, + peers: Vec, + self_node_id: u64, + bridge_status: BridgeStatus, + address_to_node_id: std::collections::HashMap, + stop_controller: crate::stop::StopController, + stop: crate::stop::StopHandle, +) { + run_signing_cursor_loop_with_policy( + runtime, + base_bridge, + deposit_log, + nonce_epoch, + proposal_cache, + signer, + valid_addresses, + peers, + self_node_id, + bridge_status, + address_to_node_id, + stop_controller, + stop, + SigningLoopPolicy::default(), + ) + .await +} + +#[allow(clippy::too_many_arguments)] +pub async fn run_signing_cursor_loop_with_policy( + runtime: Arc, + base_bridge: Arc, + deposit_log: Arc, + nonce_epoch: &NonceEpochConfig, + proposal_cache: Arc, + signer: Arc, + valid_addresses: HashSet
, + peers: Vec, + self_node_id: u64, + bridge_status: BridgeStatus, + address_to_node_id: std::collections::HashMap, + stop_controller: crate::stop::StopController, + stop: crate::stop::StopHandle, + policy: SigningLoopPolicy, +) { + use tokio::time::{interval, MissedTickBehavior}; + use tracing::{info, warn}; + + info!( + target: "bridge.cursor", + poll_interval_secs=policy.poll_interval.as_secs(), + pipeline_depth=policy.pipeline_depth, + regossip_interval_secs=policy.regossip_interval.as_secs(), + "starting signing cursor loop" + ); - // Step 1: Sign the proposal locally. - let signature = match signer.sign_hash(&proposal_hash).await { - Ok(sig) => sig.as_bytes().to_vec(), + let context = SigningTickContext::new( + SigningTickPorts::new(runtime.clone(), base_bridge, deposit_log, proposal_cache), + SigningTickNodeState::new( + signer, valid_addresses, peers, self_node_id, address_to_node_id, + ), + SigningTickControl::new(bridge_status, stop_controller, stop.clone()), + SigningTickConfig::new(nonce_epoch, policy), + ); + + let mut ticker = interval(policy.poll_interval); + ticker.set_missed_tick_behavior(MissedTickBehavior::Skip); + let mut state = SigningTickState::new(SystemTime::now()); + + loop { + ticker.tick().await; + + if stop.is_stopped() { + continue; + } + + let tip_height = match runtime.nock_hashchain_tip().await { + Ok(height) => height, + Err(err) => { + warn!( + target: "bridge.cursor", + error=%err, + "failed to peek nock hashchain tip height" + ); + continue; + } + }; + let _ = signing_tick_once( + &context, + &mut state, + SigningTickInput { + now: SystemTime::now(), + tip_height, + }, + ) + .await; + } +} + +fn update_proposal_cache_metrics(proposal_cache: &ProposalCache) { + let metrics = metrics::init_metrics(); + let snapshot = match proposal_cache.metrics_snapshot() { + Ok(snapshot) => snapshot, + Err(_) => { + metrics.proposal_cache_metrics_update_error.increment(); + return; + } + }; + + metrics + .proposal_cache_total + .swap(snapshot.proposal_total as f64); + metrics + .proposal_cache_collecting + .swap(snapshot.collecting as f64); + metrics.proposal_cache_ready.swap(snapshot.ready as f64); + metrics.proposal_cache_posting.swap(snapshot.posting as f64); + metrics + .proposal_cache_confirmed + .swap(snapshot.confirmed as f64); + metrics.proposal_cache_failed.swap(snapshot.failed as f64); + metrics + .proposal_cache_total_peer_signatures + .swap(snapshot.total_peer_signatures as f64); + metrics + .proposal_cache_max_peer_signatures_per_proposal + .swap(snapshot.max_peer_signatures_per_proposal as f64); + metrics + .proposal_cache_proposals_with_my_signature + .swap(snapshot.proposals_with_my_signature as f64); + metrics + .proposal_cache_pending_signature_deposit_count + .swap(snapshot.pending_signature_deposit_count as f64); + metrics + .proposal_cache_pending_signature_total + .swap(snapshot.pending_signature_total as f64); + metrics + .proposal_cache_oldest_age_secs + .swap(snapshot.oldest_age_secs as f64); + metrics + .proposal_cache_oldest_confirmed_age_secs + .swap(snapshot.oldest_confirmed_age_secs as f64); + metrics + .proposal_cache_oldest_failed_age_secs + .swap(snapshot.oldest_failed_age_secs as f64); + metrics + .proposal_cache_pending_oldest_age_secs + .swap(snapshot.pending_oldest_age_secs as f64); + metrics + .proposal_cache_approx_state_bytes + .swap(snapshot.approx_state_bytes as f64); + metrics + .proposal_cache_approx_peer_signature_bytes + .swap(snapshot.approx_peer_signature_bytes as f64); + metrics + .proposal_cache_approx_my_signature_bytes + .swap(snapshot.approx_my_signature_bytes as f64); + metrics + .proposal_cache_approx_pending_signature_bytes + .swap(snapshot.approx_pending_signature_bytes as f64); + metrics + .proposal_cache_approx_total_bytes + .swap(snapshot.approx_total_bytes as f64); +} + +/// Background loop that checks ProposalCache for ready proposals and posts to BASE. +/// +/// Runs continuously, checking every second for proposals with threshold signatures. +/// Posts to Base when: +/// 1. Threshold reached (status=Ready) +/// 2. I'm the proposer OR backoff expired (failover logic) +/// +fn spawn_confirmation_broadcast( + peers: &[(u64, String)], + msg: &crate::ingress::proto::ConfirmationBroadcast, + proposal_id: &str, +) { + use tracing::{info, warn}; + + use crate::ingress::proto::bridge_ingress_client::BridgeIngressClient; + + for (peer_node_id, peer_address) in peers { + let msg = msg.clone(); + let addr = peer_address.clone(); + let peer_id = *peer_node_id; + let prop_id = proposal_id.to_string(); + + tokio::spawn(async move { + match BridgeIngressClient::connect(addr.clone()).await { + Ok(mut client) => match client.broadcast_confirmation(msg).await { + Ok(_) => { + info!( + target: "bridge.posting", + peer_node_id=peer_id, + proposal_hash=%prop_id, + "broadcast confirmation to peer" + ); + } + Err(e) => { + warn!( + target: "bridge.posting", + peer_node_id=peer_id, + error=%e, + "failed to broadcast confirmation to peer" + ); + } + }, Err(e) => { - error!( - target: "bridge.cursor", + warn!( + target: "bridge.posting", + peer_node_id=peer_id, + peer_address=%addr, error=%e, - proposal_hash=%proposal_id, - "failed to sign proposal" + "failed to connect to peer for confirmation broadcast" ); - continue; } - }; + } + }); + } +} - // Step 2: Add own signature to cache. - let add_result = proposal_cache.add_signature( - &deposit_id, - crate::proposal_cache::SignatureData { - signer_address: my_eth_address, - signature: signature.clone(), - proposal_hash, - is_mine: true, - }, - Some(req.clone()), - |hash, sig| verify_bridge_signature(hash, sig, &valid_addresses), +#[derive(Clone, Debug, Default)] +pub struct PostingTickState { + pub ticks_executed: u64, +} + +#[derive(Clone, Copy, Debug)] +pub struct PostingTickInput { + pub now: SystemTime, +} + +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] +pub struct PostingTickOutcome { + pub ready_proposals: usize, + pub submitted: usize, +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +enum PostingSubmissionExecutionResult { + Submitted, + ProposalNoLongerReady, + SignatureFetchFailed, + MarkPostingFailed, + SubmitFailed, + SubmitTimedOut, +} + +#[derive(Clone)] +pub struct PostingTickPorts { + proposal_cache: Arc, + base_bridge: Arc, +} + +impl PostingTickPorts { + pub fn new(proposal_cache: Arc, base_bridge: Arc) -> Self { + Self { + proposal_cache, + base_bridge, + } + } +} + +#[derive(Clone)] +pub struct PostingTickNodeState { + node_config: NodeConfig, + peers: Arc>, + my_node_id: usize, +} + +impl PostingTickNodeState { + pub fn new(node_config: NodeConfig) -> Self { + let my_node_id = node_config.node_id as usize; + let peers: Vec<(u64, String)> = node_config + .nodes + .iter() + .enumerate() + .filter(|(idx, _)| *idx != my_node_id) + .map(|(idx, node)| (idx as u64, crate::health::normalize_endpoint(&node.ip))) + .collect(); + Self { + node_config, + peers: Arc::new(peers), + my_node_id, + } + } +} + +#[derive(Clone)] +pub struct PostingTickControl { + bridge_status: BridgeStatus, + status_state: crate::status::BridgeStatusState, +} + +impl PostingTickControl { + pub fn new( + bridge_status: BridgeStatus, + status_state: crate::status::BridgeStatusState, + ) -> Self { + Self { + bridge_status, + status_state, + } + } +} + +#[derive(Clone, Copy, Debug)] +pub struct PostingTickConfig { + failover_backoff_secs: u64, +} + +impl PostingTickConfig { + pub fn new(failover_backoff_secs: u64) -> Self { + Self { + failover_backoff_secs, + } + } +} + +#[derive(Clone)] +pub struct PostingTickContext { + ports: PostingTickPorts, + node: PostingTickNodeState, + control: PostingTickControl, + config: PostingTickConfig, +} + +impl PostingTickContext { + pub fn new( + ports: PostingTickPorts, + node: PostingTickNodeState, + control: PostingTickControl, + config: PostingTickConfig, + ) -> Self { + Self { + ports, + node, + control, + config, + } + } + + async fn execute_submission( + &self, + deposit_id: &crate::types::DepositId, + proposal_state: &crate::proposal_cache::ProposalState, + proposal_hash: [u8; 32], + proposal_id: &str, + now: SystemTime, + now_secs: u64, + ) -> PostingSubmissionExecutionResult { + use serde_bytes::ByteBuf; + use tracing::{error, info, warn}; + + use crate::ingress::proto::ConfirmationBroadcast; + use crate::status::LastSubmittedDeposit; + use crate::tui::types::{AlertSeverity, BatchStatus, ProposalStatus}; + use crate::types::DepositSubmission; + + // Get signatures for posting BEFORE marking as posting + // (get_signatures_for_posting requires status == Ready). + let signatures = match self + .ports + .proposal_cache + .get_signatures_for_posting(deposit_id) + { + Ok(Some(sigs)) => sigs, + Ok(None) => { + warn!( + target: "bridge.posting", + proposal_hash=%proposal_id, + "proposal no longer ready for posting" + ); + return PostingSubmissionExecutionResult::ProposalNoLongerReady; + } + Err(e) => { + error!( + target: "bridge.posting", + error=%e, + proposal_hash=%proposal_id, + "failed to get signatures for posting" + ); + let _ = self.ports.proposal_cache.mark_failed(deposit_id); + return PostingSubmissionExecutionResult::SignatureFetchFailed; + } + }; + + // Mark as posting to prevent duplicate submissions. + if let Err(e) = self.ports.proposal_cache.mark_posting(deposit_id) { + error!( + target: "bridge.posting", + error=%e, + proposal_hash=%proposal_id, + "failed to mark proposal as posting" ); + return PostingSubmissionExecutionResult::MarkPostingFailed; + } - // Apply any pending signatures that arrived before we processed this deposit. - let valid_addrs = valid_addresses.clone(); - if let Ok(report) = proposal_cache.apply_pending_signatures(&deposit_id, |hash, sig| { - verify_bridge_signature(hash, sig, &valid_addrs) - }) { - if report.applied > 0 { - debug!( - target: "bridge.cursor", - proposal_hash=%proposal_id, - applied_count=report.applied, - "applied pending signatures from peers" - ); - } - if let Some(first) = report.mismatched.first() { - let deposit_id_hex = hex::encode(deposit_id.to_bytes()); - let expected_hex = hex::encode(first.expected_hash); - let received_hex = hex::encode(first.received_hash); - warn!( - target: "bridge.cursor", - deposit_id=%deposit_id_hex, - expected_hash=%expected_hex, - received_hash=%received_hex, - signer=%first.signer_address, - mismatch_count=report.mismatched.len(), - "peer signature proposal hash mismatch, possible nonce divergence" - ); - bridge_status.push_alert( - crate::tui::types::AlertSeverity::Error, - "Nonce Divergence Suspected".to_string(), - format!( - "Deposit {} has {} peer signature(s) for a different proposal hash. expected={}, received={}, signer={}", - deposit_id_hex, - report.mismatched.len(), - expected_hex, - received_hex, - first.signer_address - ), - "nonce-divergence".to_string(), - ); - } + // Update batch status to Submitting. + self.control + .bridge_status + .update_batch_status(BatchStatus::Submitting { + batch_id: proposal_state.proposal.nonce, + }); + + info!( + target: "bridge.posting", + proposal_hash=%proposal_id, + "posting proposal to BASE" + ); + + // Prepare deposit submission. + let req = &proposal_state.proposal; + let mut recipient_bytes = [0u8; 20]; + recipient_bytes.copy_from_slice(&req.recipient.0); + + let submission = DepositSubmission { + tx_id: req.tx_id.clone(), + name_first: req.name.first.clone(), + name_last: req.name.last.clone(), + recipient: recipient_bytes, + amount: req.amount as u128, + block_height: req.block_height, + as_of: req.as_of.clone(), + nonce: req.nonce, + signatures: crate::types::SignatureSet { + eth_signatures: signatures.into_iter().map(ByteBuf::from).collect(), + nock_signatures: vec![], // Not used for Base deposits + }, + }; + + // Update TUI to Submitted status with timestamp. + if let Some(mut proposal) = self.control.bridge_status.find_proposal(proposal_id) { + proposal.status = ProposalStatus::Submitted; + proposal.submitted_at = Some(now); + if let Ok(duration) = now.duration_since(proposal.created_at) { + proposal.time_to_submit_ms = Some(duration.as_millis() as u64); } + self.control.bridge_status.update_proposal(proposal); + } - if let Ok(Some(state)) = proposal_cache.get_state(&deposit_id) { - bridge_status.sync_proposal_signatures_from_cache( - &proposal_id, &state, &address_to_node_id, self_node_id, + // Submit to BASE with a timeout so a hung RPC can't stall the queue. + match tokio::time::timeout( + Duration::from_secs(SUBMIT_DEPOSIT_TIMEOUT_SECS), + self.ports.base_bridge.submit_deposit(submission), + ) + .await + { + Ok(Ok(result)) => { + info!( + target: "bridge.posting", + proposal_hash=%proposal_id, + tx_hash=%result.tx_hash, + block_number=%result.block_number, + "successfully posted deposit to BASE" ); - } - match add_result { - Ok(crate::proposal_cache::SignatureAddResult::Added) - | Ok(crate::proposal_cache::SignatureAddResult::ThresholdReached) => {} - Ok(crate::proposal_cache::SignatureAddResult::Duplicate) => { - debug!( - target: "bridge.cursor", - proposal_hash=%proposal_id, - "duplicate signature, skipping broadcast" - ); - continue; - } - Ok(crate::proposal_cache::SignatureAddResult::Stale) => { - info!( - target: "bridge.cursor", - proposal_hash=%proposal_id, - "proposal already confirmed, skipping broadcast" - ); - continue; - } - Ok(crate::proposal_cache::SignatureAddResult::Invalid(msg)) => { - warn!( - target: "bridge.cursor", - proposal_hash=%proposal_id, - error=%msg, - "own signature invalid, skipping broadcast" - ); - continue; - } - Err(e) => { - warn!( - target: "bridge.cursor", - proposal_hash=%proposal_id, - error=%e, - "failed to add own signature to cache, skipping broadcast" - ); - continue; + self.control + .status_state + .update_last_submitted_deposit(LastSubmittedDeposit { + deposit: proposal_state.proposal.clone(), + base_tx_hash: result.tx_hash.clone(), + base_block_number: result.block_number, + }); + + // Mark confirmed in cache. + let _ = self.ports.proposal_cache.mark_confirmed(deposit_id); + + // Broadcast confirmation to all peers so they stop waiting. + let confirmation_msg = ConfirmationBroadcast { + deposit_id: deposit_id.to_bytes(), + proposal_hash: proposal_hash.to_vec(), + tx_hash: result.tx_hash.as_bytes().to_vec(), + block_number: result.block_number, + timestamp: now_secs, + }; + spawn_confirmation_broadcast( + self.node.peers.as_ref(), + &confirmation_msg, + proposal_id, + ); + + // Update TUI to Executed status. + if let Some(mut proposal) = self.control.bridge_status.find_proposal(proposal_id) { + proposal.status = ProposalStatus::Executed; + proposal.tx_hash = Some(result.tx_hash); + proposal.submitted_at_block = Some(result.block_number); + proposal.executed_at_block = Some(result.block_number); + self.control.bridge_status.update_proposal(proposal); } + + // Update batch status back to Idle after successful submission. + self.control + .bridge_status + .update_batch_status(BatchStatus::Idle); + PostingSubmissionExecutionResult::Submitted } + Ok(Err(e)) => { + error!( + target: "bridge.posting", + error=%e, + proposal_hash=%proposal_id, + "failed to post deposit to BASE" + ); - // Step 3: Broadcast signature to all peers (fire-and-forget, concurrent). - let timestamp = SystemTime::now() - .duration_since(UNIX_EPOCH) - .unwrap_or_default() - .as_secs(); + // Mark failed in cache. + let _ = self.ports.proposal_cache.mark_failed(deposit_id); - let broadcast_msg = SignatureBroadcast { - deposit_id: deposit_id.to_bytes(), - proposal_hash: proposal_hash.to_vec(), - signature: signature.clone(), - signer_address: my_eth_address.as_slice().to_vec(), - timestamp, - }; + // Update TUI to Failed status. + if let Some(mut proposal) = self.control.bridge_status.find_proposal(proposal_id) { + proposal.status = ProposalStatus::Failed { + reason: format!("BASE submission failed: {}", e), + }; + self.control.bridge_status.update_proposal(proposal); + } - spawn_signature_broadcast( - &peers, - &broadcast_msg, - &proposal_id, - SignatureBroadcastReason::Initial, - ); - } - } -} + // Push alert for failure. + self.control.bridge_status.push_alert( + AlertSeverity::Error, + "Proposal Failed".to_string(), + format!("Failed to post deposit {}: {}", proposal_id, e), + "posting-loop".to_string(), + ); -fn update_proposal_cache_metrics(proposal_cache: &ProposalCache) { - let metrics = metrics::init_metrics(); - let snapshot = match proposal_cache.metrics_snapshot() { - Ok(snapshot) => snapshot, - Err(_) => { - metrics.proposal_cache_metrics_update_error.increment(); - return; - } - }; + // Update batch status back to Idle after failure. + self.control + .bridge_status + .update_batch_status(BatchStatus::Idle); + PostingSubmissionExecutionResult::SubmitFailed + } + Err(_) => { + error!( + target: "bridge.posting", + proposal_hash=%proposal_id, + timeout_secs=SUBMIT_DEPOSIT_TIMEOUT_SECS, + "posting to BASE timed out" + ); - metrics - .proposal_cache_total - .swap(snapshot.proposal_total as f64); - metrics - .proposal_cache_collecting - .swap(snapshot.collecting as f64); - metrics.proposal_cache_ready.swap(snapshot.ready as f64); - metrics.proposal_cache_posting.swap(snapshot.posting as f64); - metrics - .proposal_cache_confirmed - .swap(snapshot.confirmed as f64); - metrics.proposal_cache_failed.swap(snapshot.failed as f64); - metrics - .proposal_cache_total_peer_signatures - .swap(snapshot.total_peer_signatures as f64); - metrics - .proposal_cache_max_peer_signatures_per_proposal - .swap(snapshot.max_peer_signatures_per_proposal as f64); - metrics - .proposal_cache_proposals_with_my_signature - .swap(snapshot.proposals_with_my_signature as f64); - metrics - .proposal_cache_pending_signature_deposit_count - .swap(snapshot.pending_signature_deposit_count as f64); - metrics - .proposal_cache_pending_signature_total - .swap(snapshot.pending_signature_total as f64); - metrics - .proposal_cache_oldest_age_secs - .swap(snapshot.oldest_age_secs as f64); - metrics - .proposal_cache_oldest_confirmed_age_secs - .swap(snapshot.oldest_confirmed_age_secs as f64); - metrics - .proposal_cache_oldest_failed_age_secs - .swap(snapshot.oldest_failed_age_secs as f64); - metrics - .proposal_cache_pending_oldest_age_secs - .swap(snapshot.pending_oldest_age_secs as f64); - metrics - .proposal_cache_approx_state_bytes - .swap(snapshot.approx_state_bytes as f64); - metrics - .proposal_cache_approx_peer_signature_bytes - .swap(snapshot.approx_peer_signature_bytes as f64); - metrics - .proposal_cache_approx_my_signature_bytes - .swap(snapshot.approx_my_signature_bytes as f64); - metrics - .proposal_cache_approx_pending_signature_bytes - .swap(snapshot.approx_pending_signature_bytes as f64); - metrics - .proposal_cache_approx_total_bytes - .swap(snapshot.approx_total_bytes as f64); -} + // Mark failed in cache. + let _ = self.ports.proposal_cache.mark_failed(deposit_id); -/// Background loop that checks ProposalCache for ready proposals and posts to BASE. -/// -/// Runs continuously, checking every second for proposals with threshold signatures. -/// Posts to Base when: -/// 1. Threshold reached (status=Ready) -/// 2. I'm the proposer OR backoff expired (failover logic) -/// -pub async fn run_posting_loop( - proposal_cache: Arc, - base_bridge: Arc, - node_config: NodeConfig, - bridge_status: BridgeStatus, - stop: crate::stop::StopHandle, - status_state: crate::status::BridgeStatusState, -) { - use std::time::{SystemTime, UNIX_EPOCH}; + // Update TUI to Failed status. + if let Some(mut proposal) = self.control.bridge_status.find_proposal(proposal_id) { + proposal.status = ProposalStatus::Failed { + reason: format!( + "BASE submission timed out after {}s", + SUBMIT_DEPOSIT_TIMEOUT_SECS + ), + }; + self.control.bridge_status.update_proposal(proposal); + } - use serde_bytes::ByteBuf; - use tracing::{debug, error, info, warn}; + // Push alert for failure. + self.control.bridge_status.push_alert( + AlertSeverity::Error, + "Proposal Failed".to_string(), + format!( + "Failed to post deposit {}: timed out after {}s", + proposal_id, SUBMIT_DEPOSIT_TIMEOUT_SECS + ), + "posting-loop".to_string(), + ); - use crate::ingress::proto::bridge_ingress_client::BridgeIngressClient; - use crate::ingress::proto::ConfirmationBroadcast; - use crate::proposer::hoon_proposer; - use crate::status::LastSubmittedDeposit; - use crate::tui::types::{AlertSeverity, BatchStatus, ProposalStatus}; - use crate::types::DepositSubmission; - - const FAILOVER_BACKOFF_SECS: u64 = 120; // 2m per failover slot - - // Extract node PKHs for proposer calculation - let node_pkhs: Vec<_> = node_config - .nodes - .iter() - .map(|n| n.nock_pkh.clone()) - .collect(); - let num_nodes = node_pkhs.len(); - let my_node_id = node_config.node_id as usize; - - // Build peer addresses for confirmation broadcast (exclude self) - let peers: Vec<(u64, String)> = node_config - .nodes - .iter() - .enumerate() - .filter(|(idx, _)| *idx != my_node_id) - .map(|(idx, node)| (idx as u64, crate::health::normalize_endpoint(&node.ip))) - .collect(); + // Update batch status back to Idle after timeout. + self.control + .bridge_status + .update_batch_status(BatchStatus::Idle); + PostingSubmissionExecutionResult::SubmitTimedOut + } + } + } +} - info!("Starting proposal posting loop"); +impl PostingTickContext { + pub async fn tick_once( + &self, + state: &mut PostingTickState, + input: PostingTickInput, + ) -> PostingTickOutcome { + use tracing::{debug, error, info}; - loop { - tokio::time::sleep(Duration::from_secs(1)).await; - update_proposal_cache_metrics(&proposal_cache); - if stop.is_stopped() { - continue; - } + use crate::proposer::hoon_proposer; + + let context = self; + state.ticks_executed = state.ticks_executed.saturating_add(1); + let mut outcome = PostingTickOutcome::default(); // Get all ready proposals - let ready_proposals = match proposal_cache.ready_proposals() { + let ready_proposals = match context.ports.proposal_cache.ready_proposals() { Ok(proposals) => proposals, Err(e) => { error!(target: "bridge.posting", error=%e, "failed to fetch ready proposals"); - continue; + return outcome; } }; if ready_proposals.is_empty() { - continue; + return outcome; } + outcome.ready_proposals = ready_proposals.len(); // Query the chain for the last confirmed deposit nonce. // This is the source of truth - we only submit lastDepositNonce + 1. - let last_chain_nonce = match base_bridge.get_last_deposit_nonce().await { + let last_chain_nonce = match context.ports.base_bridge.get_last_deposit_nonce().await { Ok(n) => n, Err(e) => { error!(target: "bridge.posting", error=%e, "failed to query lastDepositNonce from chain"); - continue; + return outcome; } }; - bridge_status.update_last_deposit_nonce(last_chain_nonce); + context + .control + .bridge_status + .update_last_deposit_nonce(last_chain_nonce); let next_nonce = last_chain_nonce + 1; debug!( @@ -1740,23 +2628,26 @@ pub async fn run_posting_loop( "queried chain for deposit nonce" ); - let now = SystemTime::now() - .duration_since(UNIX_EPOCH) - .unwrap_or_default() - .as_secs(); - // NOTE: do not "skip" a stuck nonce. Under runtime-assigned epoch nonces, skipping // strands deposits permanently because subsequent nonces cannot be posted until the // contract advances. We only ever submit `lastDepositNonce + 1`. + let now_secs = system_time_secs(input.now); - // Get current nockchain height for proposer calculation + // Get current nockchain height for proposer calculation. // Use the block_height from the first proposal as a proxy - // (all proposals in a batch should be from the same height) + // (all proposals in a batch should be from the same height). let current_height = ready_proposals .first() - .map(|(_, state)| state.proposal.block_height) + .map(|(_, proposal_state)| proposal_state.proposal.block_height) .unwrap_or(0); - + let node_pkhs: Vec<_> = context + .node + .node_config + .nodes + .iter() + .map(|node| node.nock_pkh.clone()) + .collect(); + let num_nodes = node_pkhs.len(); let current_proposer = hoon_proposer(current_height, &node_pkhs); debug!( @@ -1764,317 +2655,158 @@ pub async fn run_posting_loop( ready_count=ready_proposals.len(), current_height=current_height, current_proposer=current_proposer, - my_node_id=my_node_id, + my_node_id=context.node.my_node_id, "checking ready proposals" ); - for (deposit_id, state) in ready_proposals { - let proposal_hash = state.proposal_hash; - let proposal_id = hex::encode(proposal_hash); - - // Only submit the next expected nonce (lastDepositNonce + 1). - // If this proposal's nonce doesn't match, skip it. - if state.proposal.nonce < next_nonce { - // Already on chain - mark as confirmed and skip - debug!( - target: "bridge.posting", - proposal_hash=%proposal_id, - nonce=state.proposal.nonce, - last_chain_nonce=last_chain_nonce, - "proposal already confirmed on chain, marking confirmed" - ); - let _ = proposal_cache.mark_confirmed(&deposit_id); - continue; - } else if state.proposal.nonce > next_nonce { - // Not ready yet - waiting for earlier nonce - debug!( - target: "bridge.posting", - proposal_hash=%proposal_id, - nonce=state.proposal.nonce, - next_nonce=next_nonce, - "waiting for nonce {} to be ready before posting {}", - next_nonce, - state.proposal.nonce - ); - continue; - } - // state.proposal.nonce == next_nonce - this is the one to submit - - // Calculate if this node should post - let should_post = if my_node_id == current_proposer { - // I'm the proposer, post immediately - true - } else if let Some(ready_at) = state.ready_at { - // Calculate failover slot (position after proposer in rotation) - let failover_slot = if my_node_id > current_proposer { - my_node_id - current_proposer - } else { - num_nodes - current_proposer + my_node_id - }; - let required_wait = FAILOVER_BACKOFF_SECS * failover_slot as u64; - let elapsed = now.saturating_sub(ready_at); - elapsed >= required_wait - } else { - // No ready_at timestamp, shouldn't happen but skip - false - }; - - if !should_post { - debug!( - target: "bridge.posting", - proposal_hash=%proposal_id, - current_proposer=current_proposer, - my_node_id=my_node_id, - "not my turn to post, waiting for proposer or failover" - ); - continue; - } + let decisions = PostingPlanner::plan_tick( + PostingTickPlanInput { + next_nonce, + my_node_id: context.node.my_node_id, + current_proposer, + num_nodes, + now_secs, + failover_backoff_secs: context.config.failover_backoff_secs, + }, + &ready_proposals + .iter() + .map(|(_, proposal_state)| PostingReadyProposal { + nonce: proposal_state.proposal.nonce, + ready_at: proposal_state.ready_at, + }) + .collect::>(), + ); - info!( - target: "bridge.posting", - proposal_hash=%proposal_id, - current_proposer=current_proposer, - my_node_id=my_node_id, - is_proposer=(my_node_id == current_proposer), - "posting proposal to BASE" - ); + for ((deposit_id, proposal_state), decision) in ready_proposals.into_iter().zip(decisions) { + let proposal_hash = proposal_state.proposal_hash; + let proposal_id = hex::encode(proposal_hash); - // Get signatures for posting BEFORE marking as posting - // (get_signatures_for_posting requires status == Ready) - let signatures = match proposal_cache.get_signatures_for_posting(&deposit_id) { - Ok(Some(sigs)) => sigs, - Ok(None) => { - warn!( + let is_proposer = match decision { + PostingCandidateDecision::MarkConfirmedOnChain => { + debug!( + target: "bridge.posting", + proposal_hash=%proposal_id, + nonce=proposal_state.proposal.nonce, + last_chain_nonce=last_chain_nonce, + "proposal already confirmed on chain, marking confirmed" + ); + let _ = context.ports.proposal_cache.mark_confirmed(&deposit_id); + continue; + } + PostingCandidateDecision::WaitForEarlierNonce => { + debug!( target: "bridge.posting", proposal_hash=%proposal_id, - "proposal no longer ready for posting" + nonce=proposal_state.proposal.nonce, + next_nonce=next_nonce, + "waiting for nonce {} to be ready before posting {}", + next_nonce, + proposal_state.proposal.nonce ); continue; } - Err(e) => { - error!( + PostingCandidateDecision::NotMyTurn => { + debug!( target: "bridge.posting", - error=%e, proposal_hash=%proposal_id, - "failed to get signatures for posting" + current_proposer=current_proposer, + my_node_id=context.node.my_node_id, + "not my turn to post, waiting for proposer or failover" ); - let _ = proposal_cache.mark_failed(&deposit_id); continue; } + PostingCandidateDecision::Submit { is_proposer } => is_proposer, }; - // Mark as posting to prevent duplicate submissions - if let Err(e) = proposal_cache.mark_posting(&deposit_id) { - error!( - target: "bridge.posting", - error=%e, - proposal_hash=%proposal_id, - "failed to mark proposal as posting" - ); - continue; - } - - // Update batch status to Submitting - bridge_status.update_batch_status(BatchStatus::Submitting { - batch_id: state.proposal.nonce, - }); - info!( target: "bridge.posting", proposal_hash=%proposal_id, + current_proposer=current_proposer, + my_node_id=context.node.my_node_id, + is_proposer=is_proposer, "posting proposal to BASE" ); - // Prepare deposit submission - let req = &state.proposal; - let mut recipient_bytes = [0u8; 20]; - recipient_bytes.copy_from_slice(&req.recipient.0); - - let submission = DepositSubmission { - tx_id: req.tx_id.clone(), - name_first: req.name.first.clone(), - name_last: req.name.last.clone(), - recipient: recipient_bytes, - amount: req.amount as u128, - block_height: req.block_height, - as_of: req.as_of.clone(), - nonce: req.nonce, - signatures: crate::types::SignatureSet { - eth_signatures: signatures.into_iter().map(ByteBuf::from).collect(), - nock_signatures: vec![], // Not used for Base deposits - }, - }; - - // Update TUI to Submitted status with timestamp - let submit_time = std::time::SystemTime::now(); - if let Some(mut proposal) = bridge_status.find_proposal(&proposal_id) { - proposal.status = ProposalStatus::Submitted; - proposal.submitted_at = Some(submit_time); - if let Ok(duration) = submit_time.duration_since(proposal.created_at) { - proposal.time_to_submit_ms = Some(duration.as_millis() as u64); - } - bridge_status.update_proposal(proposal); + if matches!( + context + .execute_submission( + &deposit_id, &proposal_state, proposal_hash, &proposal_id, input.now, + now_secs, + ) + .await, + PostingSubmissionExecutionResult::Submitted + ) { + outcome.submitted += 1; } + } - // Submit to BASE with a timeout so a hung RPC can't stall the queue - match tokio::time::timeout( - Duration::from_secs(SUBMIT_DEPOSIT_TIMEOUT_SECS), - base_bridge.submit_deposit(submission), - ) - .await - { - Ok(Ok(result)) => { - info!( - target: "bridge.posting", - proposal_hash=%proposal_id, - tx_hash=%result.tx_hash, - block_number=%result.block_number, - "successfully posted deposit to BASE" - ); - - status_state.update_last_submitted_deposit(LastSubmittedDeposit { - deposit: state.proposal.clone(), - base_tx_hash: result.tx_hash.clone(), - base_block_number: result.block_number, - }); - - // Mark confirmed in cache - let _ = proposal_cache.mark_confirmed(&deposit_id); - - // Broadcast confirmation to all peers so they stop waiting - let deposit_id_bytes = deposit_id.to_bytes(); - let timestamp = SystemTime::now() - .duration_since(UNIX_EPOCH) - .unwrap_or_default() - .as_secs(); - - let confirmation_msg = ConfirmationBroadcast { - deposit_id: deposit_id_bytes, - proposal_hash: proposal_hash.to_vec(), - tx_hash: result.tx_hash.as_bytes().to_vec(), - block_number: result.block_number, - timestamp, - }; - - for (peer_node_id, peer_address) in &peers { - let msg = confirmation_msg.clone(); - let addr = peer_address.clone(); - let peer_id = *peer_node_id; - let prop_id = proposal_id.clone(); - - tokio::spawn(async move { - match BridgeIngressClient::connect(addr.clone()).await { - Ok(mut client) => match client.broadcast_confirmation(msg).await { - Ok(_) => { - info!( - target: "bridge.posting", - peer_node_id=peer_id, - proposal_hash=%prop_id, - "broadcast confirmation to peer" - ); - } - Err(e) => { - warn!( - target: "bridge.posting", - peer_node_id=peer_id, - error=%e, - "failed to broadcast confirmation to peer" - ); - } - }, - Err(e) => { - warn!( - target: "bridge.posting", - peer_node_id=peer_id, - peer_address=%addr, - error=%e, - "failed to connect to peer for confirmation broadcast" - ); - } - } - }); - } - - // Update TUI to Executed status - if let Some(mut proposal) = bridge_status.find_proposal(&proposal_id) { - proposal.status = ProposalStatus::Executed; - proposal.tx_hash = Some(result.tx_hash); - proposal.submitted_at_block = Some(result.block_number); - proposal.executed_at_block = Some(result.block_number); - bridge_status.update_proposal(proposal); - } - - // Update batch status back to Idle after successful submission - bridge_status.update_batch_status(BatchStatus::Idle); - } - Ok(Err(e)) => { - error!( - target: "bridge.posting", - error=%e, - proposal_hash=%proposal_id, - "failed to post deposit to BASE" - ); - - // Mark failed in cache - let _ = proposal_cache.mark_failed(&deposit_id); - - // Update TUI to Failed status - if let Some(mut proposal) = bridge_status.find_proposal(&proposal_id) { - proposal.status = ProposalStatus::Failed { - reason: format!("BASE submission failed: {}", e), - }; - bridge_status.update_proposal(proposal); - } + outcome + } +} - // Push alert for failure - bridge_status.push_alert( - AlertSeverity::Error, - "Proposal Failed".to_string(), - format!("Failed to post deposit {}: {}", proposal_id, e), - "posting-loop".to_string(), - ); +pub async fn posting_tick_once( + context: &PostingTickContext, + state: &mut PostingTickState, + input: PostingTickInput, +) -> PostingTickOutcome { + context.tick_once(state, input).await +} - // Update batch status back to Idle after failure - bridge_status.update_batch_status(BatchStatus::Idle); - } - Err(_) => { - error!( - target: "bridge.posting", - proposal_hash=%proposal_id, - timeout_secs=SUBMIT_DEPOSIT_TIMEOUT_SECS, - "posting to BASE timed out" - ); +pub async fn run_posting_loop( + proposal_cache: Arc, + base_bridge: Arc, + node_config: NodeConfig, + bridge_status: BridgeStatus, + stop: crate::stop::StopHandle, + status_state: crate::status::BridgeStatusState, +) { + run_posting_loop_with_policy( + proposal_cache, + base_bridge, + node_config, + bridge_status, + stop, + status_state, + PostingLoopPolicy::default(), + ) + .await +} - // Mark failed in cache - let _ = proposal_cache.mark_failed(&deposit_id); - - // Update TUI to Failed status - if let Some(mut proposal) = bridge_status.find_proposal(&proposal_id) { - proposal.status = ProposalStatus::Failed { - reason: format!( - "BASE submission timed out after {}s", - SUBMIT_DEPOSIT_TIMEOUT_SECS - ), - }; - bridge_status.update_proposal(proposal); - } +pub async fn run_posting_loop_with_policy( + proposal_cache: Arc, + base_bridge: Arc, + node_config: NodeConfig, + bridge_status: BridgeStatus, + stop: crate::stop::StopHandle, + status_state: crate::status::BridgeStatusState, + policy: PostingLoopPolicy, +) { + use tracing::info; - // Push alert for failure - bridge_status.push_alert( - AlertSeverity::Error, - "Proposal Failed".to_string(), - format!( - "Failed to post deposit {}: timed out after {}s", - proposal_id, SUBMIT_DEPOSIT_TIMEOUT_SECS - ), - "posting-loop".to_string(), - ); + info!("Starting proposal posting loop"); + let context = PostingTickContext::new( + PostingTickPorts::new(proposal_cache, base_bridge), + PostingTickNodeState::new(node_config), + PostingTickControl::new(bridge_status, status_state), + PostingTickConfig::new(policy.failover_backoff_secs), + ); + let mut state = PostingTickState::default(); - // Update batch status back to Idle after timeout - bridge_status.update_batch_status(BatchStatus::Idle); - } - } + loop { + tokio::time::sleep(policy.tick_interval).await; + update_proposal_cache_metrics(&context.ports.proposal_cache); + if stop.is_stopped() { + continue; } + + let _ = posting_tick_once( + &context, + &mut state, + PostingTickInput { + now: SystemTime::now(), + }, + ) + .await; } } @@ -2288,7 +3020,11 @@ mod tests { events: Arc::new(Mutex::new(Vec::new())), }); let (mut runtime, handle) = BridgeRuntime::new(builder); - let mut peek_rx = runtime.peek_rx.take().expect("peek receiver missing"); + let mut peek_rx = runtime + .channels + .peek_rx + .take() + .expect("peek receiver missing"); let responder = tokio::spawn(async move { if let Some(request) = peek_rx.recv().await { @@ -2313,7 +3049,11 @@ mod tests { events: Arc::new(Mutex::new(Vec::new())), }); let (mut runtime, handle) = BridgeRuntime::new(builder); - let mut peek_rx = runtime.peek_rx.take().expect("peek receiver missing"); + let mut peek_rx = runtime + .channels + .peek_rx + .take() + .expect("peek receiver missing"); let responder = tokio::spawn(async move { if let Some(request) = peek_rx.recv().await { @@ -2350,7 +3090,11 @@ mod tests { events: Arc::new(Mutex::new(Vec::new())), }); let (mut runtime, handle) = BridgeRuntime::new(builder); - let mut peek_rx = runtime.peek_rx.take().expect("peek receiver missing"); + let mut peek_rx = runtime + .channels + .peek_rx + .take() + .expect("peek receiver missing"); let responder = tokio::spawn(async move { if let Some(request) = peek_rx.recv().await { @@ -2374,7 +3118,11 @@ mod tests { events: Arc::new(Mutex::new(Vec::new())), }); let (mut runtime, handle) = BridgeRuntime::new(builder); - let mut peek_rx = runtime.peek_rx.take().expect("peek receiver missing"); + let mut peek_rx = runtime + .channels + .peek_rx + .take() + .expect("peek receiver missing"); let responder = tokio::spawn(async move { if let Some(request) = peek_rx.recv().await { @@ -2396,7 +3144,11 @@ mod tests { events: Arc::new(Mutex::new(Vec::new())), }); let (mut runtime, handle) = BridgeRuntime::new(builder); - let mut peek_rx = runtime.peek_rx.take().expect("peek receiver missing"); + let mut peek_rx = runtime + .channels + .peek_rx + .take() + .expect("peek receiver missing"); let responder = tokio::spawn(async move { if let Some(request) = peek_rx.recv().await { @@ -2423,7 +3175,11 @@ mod tests { events: Arc::new(Mutex::new(Vec::new())), }); let (mut runtime, handle) = BridgeRuntime::new(builder); - let mut peek_rx = runtime.peek_rx.take().expect("peek receiver missing"); + let mut peek_rx = runtime + .channels + .peek_rx + .take() + .expect("peek receiver missing"); let responder = tokio::spawn(async move { if let Some(request) = peek_rx.recv().await { diff --git a/crates/bridge/src/signing.rs b/crates/bridge/src/signing.rs index 82e2dcbce..5a0b48b80 100644 --- a/crates/bridge/src/signing.rs +++ b/crates/bridge/src/signing.rs @@ -5,7 +5,7 @@ use alloy::primitives::{Address, Signature}; use alloy::signers::local::PrivateKeySigner; use alloy::signers::Signer; use nockapp::nockapp::wire::{Wire, WireRepr}; -use nockvm::noun::Noun; +use nockvm::noun::{Noun, NounSpace}; use noun_serde::NounDecode; use tracing::info; @@ -30,11 +30,15 @@ impl EthereumSigner { /// /// The proposal_hash is the hash of a bundle proposal that needs to be signed. /// This uses EIP-191 message prefix for Ethereum compatibility. - pub async fn sign_proposal(&self, proposal_hash_noun: Noun) -> Result { - let proposal_hash = match proposal_hash_noun.as_atom() { - Ok(_) => Self::proposal_hash_from_noun(proposal_hash_noun)?, + pub async fn sign_proposal( + &self, + proposal_hash_noun: Noun, + space: &NounSpace, + ) -> Result { + let proposal_hash = match proposal_hash_noun.in_space(space).as_atom() { + Ok(_) => Self::proposal_hash_from_noun(proposal_hash_noun, space)?, Err(_) => { - let digest = NounDigest::from_noun(&proposal_hash_noun).map_err(|_| { + let digest = NounDigest::from_noun(&proposal_hash_noun, space).map_err(|_| { BridgeError::Config("proposal-hash noun was not an atom".into()) })?; Self::proposal_hash_from_limbs(digest) @@ -59,8 +63,12 @@ impl EthereumSigner { } /// Convert a cued noun representing a proposal hash into a 32-byte big-endian array - pub fn proposal_hash_from_noun(hash_noun: Noun) -> Result<[u8; 32], BridgeError> { + fn proposal_hash_from_noun( + hash_noun: Noun, + space: &NounSpace, + ) -> Result<[u8; 32], BridgeError> { let atom = hash_noun + .in_space(space) .as_atom() .map_err(|_| BridgeError::Config("proposal-hash noun was not an atom".to_string()))?; let mut b = atom.to_be_bytes(); @@ -229,6 +237,11 @@ pub fn extract_valid_bridge_addresses(node_config: &NodeConfig) -> HashSet v, @@ -96,9 +106,23 @@ pub(crate) async fn trigger_local_stop( }; if !stop_controller.trigger(info) { + metrics.stop_local_duplicate.increment(); + info!( + target: "bridge.stop", + reason=%reason, + "local stop already active, ignoring duplicate request" + ); return; } + metrics.stop_local_triggered.increment(); + info!( + target: "bridge.stop", + reason=%reason, + has_last = last.is_some(), + "local stop activated" + ); + bridge_status.push_alert( AlertSeverity::Error, "Bridge Stopped".to_string(), @@ -107,6 +131,10 @@ pub(crate) async fn trigger_local_stop( ); if let Some(last) = last { + info!( + target: "bridge.stop", + "forwarding local stop cause to kernel" + ); if let Err(err) = runtime.send_stop(last).await { warn!( target: "bridge.stop", @@ -114,6 +142,11 @@ pub(crate) async fn trigger_local_stop( "failed to poke kernel with stop cause after local stop trigger" ); } + } else { + info!( + target: "bridge.stop", + "no stop-info snapshot available; skipping kernel stop poke" + ); } } @@ -147,9 +180,12 @@ pub fn create_stop_driver( }; let root = unsafe { effect.root() }; - let bridge_effect = match BridgeEffect::from_noun(root) { - Ok(effect) => effect, - Err(_) => continue, + let bridge_effect = { + let space = effect.noun_space(); + match BridgeEffect::from_noun(root, &space) { + Ok(effect) => effect, + Err(_) => continue, + } }; let BridgeEffectVariant::Stop(data) = bridge_effect.variant else { diff --git a/crates/bridge/src/tui/types.rs b/crates/bridge/src/tui/types.rs index 92baebbb0..3a8daf1ca 100644 --- a/crates/bridge/src/tui/types.rs +++ b/crates/bridge/src/tui/types.rs @@ -320,12 +320,37 @@ impl TransactionState { } pub fn push(&mut self, tx: BridgeTx) { + if let Some(existing) = self + .transactions + .iter_mut() + .find(|existing| Self::same_event(existing, &tx)) + { + existing.status = tx.status; + if existing.base_block.is_none() { + existing.base_block = tx.base_block; + } + if existing.nock_height.is_none() { + existing.nock_height = tx.nock_height; + } + return; + } + if self.transactions.len() >= self.max_transactions { self.transactions.pop_back(); } self.transactions.push_front(tx); } + fn same_event(a: &BridgeTx, b: &BridgeTx) -> bool { + a.tx_hash == b.tx_hash + && a.direction == b.direction + && a.from == b.from + && a.to == b.to + && a.amount == b.amount + && a.base_block == b.base_block + && a.nock_height == b.nock_height + } + pub fn deposits(&self) -> impl Iterator { self.transactions .iter() @@ -861,4 +886,57 @@ mod tests { assert_eq!(format_nock_from_nicks(1), "0.0000152587890625"); assert_eq!(format_nock_from_nicks(653_410_000_000), "9970245.361328125"); } + + #[test] + fn transaction_state_deduplicates_replayed_event() { + let mut state = TransactionState::new(10); + let tx = BridgeTx { + tx_hash: "0xabc".to_string(), + direction: TxDirection::Deposit, + from: "Base".to_string(), + to: "0x1111".to_string(), + amount: 123, + status: TxStatus::Completed, + timestamp: std::time::UNIX_EPOCH, + base_block: Some(42), + nock_height: None, + }; + let replay = BridgeTx { + timestamp: std::time::UNIX_EPOCH + Duration::from_secs(10), + ..tx.clone() + }; + + state.push(tx.clone()); + state.push(replay); + + assert_eq!(state.transactions.len(), 1); + assert_eq!(state.transactions[0].tx_hash, tx.tx_hash); + assert_eq!(state.transactions[0].timestamp, tx.timestamp); + } + + #[test] + fn transaction_state_keeps_distinct_events_with_same_tx_hash() { + let mut state = TransactionState::new(10); + let tx1 = BridgeTx { + tx_hash: "0xabc".to_string(), + direction: TxDirection::Deposit, + from: "Base".to_string(), + to: "0x1111".to_string(), + amount: 123, + status: TxStatus::Completed, + timestamp: std::time::UNIX_EPOCH, + base_block: Some(42), + nock_height: None, + }; + let tx2 = BridgeTx { + amount: 456, + timestamp: std::time::UNIX_EPOCH + Duration::from_secs(1), + ..tx1.clone() + }; + + state.push(tx1); + state.push(tx2); + + assert_eq!(state.transactions.len(), 2); + } } diff --git a/crates/bridge/src/types.rs b/crates/bridge/src/types.rs index 8632de300..0ce3275a7 100644 --- a/crates/bridge/src/types.rs +++ b/crates/bridge/src/types.rs @@ -4,7 +4,7 @@ pub use nockchain_types::tx_engine::common::Hash as Tip5Hash; use nockchain_types::tx_engine::common::Hash as NockPkh; use nockchain_types::v1::Name; pub use nockchain_types::EthAddress; -use nockvm::noun::{Noun, NounAllocator}; +use nockvm::noun::{Noun, NounAllocator, NounSpace}; use noun_serde::{NounDecode, NounEncode}; use serde::{Deserialize, Serialize}; use serde_bytes::ByteBuf; @@ -215,11 +215,13 @@ impl NounEncode for EthSignatureParts { fn to_noun(&self, allocator: &mut A) -> nockvm::noun::Noun { let r_atom = unsafe { let mut ia = nockvm::noun::IndirectAtom::new_raw_bytes(allocator, 32, self.r.as_ptr()); - ia.normalize_as_atom().as_noun() + let space = allocator.noun_space(); + ia.normalize_as_atom(&space).as_noun() }; let s_atom = unsafe { let mut ia = nockvm::noun::IndirectAtom::new_raw_bytes(allocator, 32, self.s.as_ptr()); - ia.normalize_as_atom().as_noun() + let space = allocator.noun_space(); + ia.normalize_as_atom(&space).as_noun() }; let v_atom = nockvm::noun::Atom::new(allocator, self.v).as_noun(); let inner = nockvm::noun::T(allocator, &[s_atom, v_atom]); @@ -228,8 +230,12 @@ impl NounEncode for EthSignatureParts { } impl NounDecode for EthSignatureParts { - fn from_noun(noun: &nockvm::noun::Noun) -> Result { + fn from_noun( + noun: &nockvm::noun::Noun, + space: &NounSpace, + ) -> Result { let c0 = noun + .in_space(space) .as_cell() .map_err(|_| noun_serde::NounDecodeError::ExpectedCell)?; let r_bytes = c0 @@ -293,13 +299,17 @@ impl NounEncode for ByteArray { } impl NounDecode for ByteArray { - fn from_noun(noun: &nockvm::noun::Noun) -> Result { + fn from_noun( + noun: &nockvm::noun::Noun, + space: &NounSpace, + ) -> Result { let mut bytes = Vec::new(); - let mut current = *noun; + let mut current = noun.in_space(space); while let Ok(cell) = current.as_cell() { - let head = cell.head(); + let head = cell.head().noun(); let byte = head + .in_space(space) .as_atom() .map_err(|_| noun_serde::NounDecodeError::ExpectedAtom)? .as_u64() @@ -378,14 +388,19 @@ impl NounEncode for AtomBytes { unsafe { let mut ia = nockvm::noun::IndirectAtom::new_raw_bytes(allocator, self.0.len(), self.0.as_ptr()); - ia.normalize_as_atom().as_noun() + let space = allocator.noun_space(); + ia.normalize_as_atom(&space).as_noun() } } } impl NounDecode for AtomBytes { - fn from_noun(noun: &nockvm::noun::Noun) -> Result { + fn from_noun( + noun: &nockvm::noun::Noun, + space: &NounSpace, + ) -> Result { let atom = noun + .in_space(space) .as_atom() .map_err(|_| noun_serde::NounDecodeError::ExpectedAtom)?; let bytes = atom.as_ne_bytes(); @@ -420,8 +435,11 @@ impl NounEncode for SchnorrSecretKey { } impl NounDecode for SchnorrSecretKey { - fn from_noun(noun: &nockvm::noun::Noun) -> Result { - Ok(SchnorrSecretKey(decode_belt_array(noun)?)) + fn from_noun( + noun: &nockvm::noun::Noun, + space: &NounSpace, + ) -> Result { + Ok(SchnorrSecretKey(decode_belt_array(noun, space)?)) } } @@ -527,8 +545,9 @@ impl NounEncode for NullTag { } impl NounDecode for NullTag { - fn from_noun(noun: &Noun) -> Result { + fn from_noun(noun: &Noun, space: &NounSpace) -> Result { let atom = noun + .in_space(space) .as_atom() .map_err(|_| noun_serde::NounDecodeError::ExpectedAtom)?; if atom.as_u64()? == 0 { @@ -584,24 +603,29 @@ impl NounEncode for NockchainBlockCause { fn to_noun(&self, allocator: &mut A) -> nockvm::noun::Noun { use nockapp::noun::NounAllocatorExt; - let page_noun = allocator.copy_into(self.page_noun); + let page_noun = allocator.copy_into(self.page_noun, &self.page_slab.noun_space()); let txs_noun = self.txs.to_noun(allocator); nockvm::noun::T(allocator, &[page_noun, txs_noun]) } } impl NounDecode for NockchainBlockCause { - fn from_noun(noun: &nockvm::noun::Noun) -> Result { + fn from_noun( + noun: &nockvm::noun::Noun, + space: &NounSpace, + ) -> Result { use nockapp::noun::slab::{NockJammer, NounSlab}; let cell = noun + .in_space(space) .as_cell() .map_err(|_| noun_serde::NounDecodeError::ExpectedCell)?; let mut page_slab: NounSlab = NounSlab::new(); - let page_noun = page_slab.copy_into(cell.head()); + let page_noun = page_slab.copy_into(cell.head().noun(), space); - let txs = NockchainTxsMap::from_noun(&cell.tail())?; + let txs_noun = cell.tail().noun(); + let txs = NockchainTxsMap::from_noun(&txs_noun, space)?; Ok(Self { page_slab, @@ -628,37 +652,44 @@ impl NounEncode for NockchainTxsMap { } impl NounDecode for NockchainTxsMap { - fn from_noun(noun: &nockvm::noun::Noun) -> Result { + fn from_noun( + noun: &nockvm::noun::Noun, + space: &NounSpace, + ) -> Result { fn traverse( node: &nockvm::noun::Noun, + space: &NounSpace, acc: &mut Vec<(nockchain_types::tx_engine::common::TxId, Tx)>, ) -> Result<(), noun_serde::NounDecodeError> { - if let Ok(atom) = node.as_atom() { + if let Ok(atom) = node.in_space(space).as_atom() { if atom.as_u64()? == 0 { return Ok(()); } return Err(noun_serde::NounDecodeError::ExpectedCell); } let cell = node + .in_space(space) .as_cell() .map_err(|_| noun_serde::NounDecodeError::ExpectedCell)?; let kv = cell .head() .as_cell() .map_err(|_| noun_serde::NounDecodeError::ExpectedCell)?; - let tx_id = nockchain_types::tx_engine::common::TxId::from_noun(&kv.head())?; - let tx = Tx::from_noun(&kv.tail())?; + let tx_id_noun = kv.head().noun(); + let tx_noun = kv.tail().noun(); + let tx_id = nockchain_types::tx_engine::common::TxId::from_noun(&tx_id_noun, space)?; + let tx = Tx::from_noun(&tx_noun, space)?; acc.push((tx_id, tx)); let branches = cell .tail() .as_cell() .map_err(|_| noun_serde::NounDecodeError::ExpectedCell)?; - traverse(&branches.head(), acc)?; - traverse(&branches.tail(), acc)?; + traverse(&branches.head().noun(), space, acc)?; + traverse(&branches.tail().noun(), space, acc)?; Ok(()) } let mut acc = Vec::new(); - traverse(noun, &mut acc)?; + traverse(noun, space, &mut acc)?; Ok(NockchainTxsMap(acc)) } } @@ -677,8 +708,12 @@ impl NounEncode for Tx { } impl NounDecode for Tx { - fn from_noun(noun: &nockvm::noun::Noun) -> Result { + fn from_noun( + noun: &nockvm::noun::Noun, + space: &NounSpace, + ) -> Result { let cell = noun + .in_space(space) .as_cell() .map_err(|_| noun_serde::NounDecodeError::ExpectedCell)?; let tag = cell @@ -688,7 +723,7 @@ impl NounDecode for Tx { .as_u64() .map_err(|_| noun_serde::NounDecodeError::Custom("tx tag too large".into()))?; match tag { - 1 => Ok(Tx::V1(TxV1::from_noun(noun)?)), + 1 => Ok(Tx::V1(TxV1::from_noun(noun, space)?)), _ => Err(noun_serde::NounDecodeError::Custom(format!( "unsupported tx version: {}", tag @@ -721,31 +756,37 @@ impl NounEncode for OutputsV1 { } impl NounDecode for OutputsV1 { - fn from_noun(noun: &nockvm::noun::Noun) -> Result { + fn from_noun( + noun: &nockvm::noun::Noun, + space: &NounSpace, + ) -> Result { fn traverse( node: &nockvm::noun::Noun, + space: &NounSpace, acc: &mut Vec, ) -> Result<(), noun_serde::NounDecodeError> { - if let Ok(atom) = node.as_atom() { + if let Ok(atom) = node.in_space(space).as_atom() { if atom.as_u64()? == 0 { return Ok(()); } return Err(noun_serde::NounDecodeError::ExpectedCell); } let cell = node + .in_space(space) .as_cell() .map_err(|_| noun_serde::NounDecodeError::ExpectedCell)?; - acc.push(OutputV1::from_noun(&cell.head())?); + let output_noun = cell.head().noun(); + acc.push(OutputV1::from_noun(&output_noun, space)?); let branches = cell .tail() .as_cell() .map_err(|_| noun_serde::NounDecodeError::ExpectedCell)?; - traverse(&branches.head(), acc)?; - traverse(&branches.tail(), acc)?; + traverse(&branches.head().noun(), space, acc)?; + traverse(&branches.tail().noun(), space, acc)?; Ok(()) } let mut acc = Vec::new(); - traverse(noun, &mut acc)?; + traverse(noun, space, &mut acc)?; Ok(OutputsV1(acc)) } } @@ -1132,17 +1173,19 @@ fn encode_belt_array( fn decode_belt_array( noun: &Noun, + space: &NounSpace, ) -> Result<[Belt; N], noun_serde::NounDecodeError> { let mut result = [Belt(0); N]; - let mut current = *noun; + let mut current = noun.in_space(space); for (idx, item) in result.iter_mut().enumerate() { if idx == N - 1 { - *item = Belt::from_noun(¤t)?; + *item = Belt::from_noun(¤t.noun(), space)?; } else { let cell = current .as_cell() .map_err(|_| noun_serde::NounDecodeError::ExpectedCell)?; - *item = Belt::from_noun(&cell.head())?; + let head_noun = cell.head().noun(); + *item = Belt::from_noun(&head_noun, space)?; current = cell.tail(); } } @@ -1165,6 +1208,14 @@ mod tests { .try_init(); } + fn decode_from_slab( + noun: &Noun, + slab: &NounSlab, + ) -> Result { + let space = slab.noun_space(); + T::from_noun(noun, &space) + } + fn sample_base_blocks_cause() -> RawBaseBlocks { vec![RawBaseBlockEntry { height: 12345, @@ -1211,7 +1262,7 @@ mod tests { let encoded_noun = original_cause.to_noun(&mut allocator); info!("Encoded cause to noun"); - let decoded_cause = BridgeCause::from_noun(&encoded_noun) + let decoded_cause = decode_from_slab::(&encoded_noun, &allocator) .expect("Failed to decode cfg-load cause from noun"); debug!("Decoded cause successfully"); @@ -1279,7 +1330,7 @@ mod tests { let encoded_noun = original_cause.to_noun(&mut allocator); info!("Encoded base-blocks cause to noun"); - let decoded_cause = BridgeCause::from_noun(&encoded_noun) + let decoded_cause = decode_from_slab::(&encoded_noun, &allocator) .expect("Failed to decode base-blocks cause from noun"); debug!("Decoded base-blocks cause successfully"); @@ -1328,7 +1379,7 @@ mod tests { let encoded_noun = original_cause.to_noun(&mut allocator); info!("Encoded proposed-base-call cause to noun"); - let decoded_cause = BridgeCause::from_noun(&encoded_noun) + let decoded_cause = decode_from_slab::(&encoded_noun, &allocator) .expect("Failed to decode proposed-base-call cause from noun"); debug!("Decoded proposed-base-call cause successfully"); @@ -1370,7 +1421,7 @@ mod tests { let encoded_noun = original_cause.to_noun(&mut allocator); info!("Encoded base-call-sig cause to noun"); - let decoded_cause = BridgeCause::from_noun(&encoded_noun) + let decoded_cause = decode_from_slab::(&encoded_noun, &allocator) .expect("Failed to decode base-call-sig cause from noun"); debug!("Decoded base-call-sig cause successfully"); @@ -1408,8 +1459,8 @@ mod tests { let original_cause = BridgeCause::stop(last.clone()); let encoded_noun = original_cause.to_noun(&mut allocator); - let decoded_cause = - BridgeCause::from_noun(&encoded_noun).expect("Failed to decode stop cause from noun"); + let decoded_cause = decode_from_slab::(&encoded_noun, &allocator) + .expect("Failed to decode stop cause from noun"); assert_eq!(decoded_cause.0, 0, "Version should be 0"); @@ -1434,8 +1485,8 @@ mod tests { let original_cause = BridgeCause::start(); let encoded_noun = original_cause.to_noun(&mut allocator); - let decoded_cause = - BridgeCause::from_noun(&encoded_noun).expect("Failed to decode start cause from noun"); + let decoded_cause = decode_from_slab::(&encoded_noun, &allocator) + .expect("Failed to decode start cause from noun"); assert_eq!(decoded_cause.0, 0, "Version should be 0"); @@ -1502,7 +1553,7 @@ mod tests { for (name, cause) in test_cases { debug!("Testing version for {} variant", name); let encoded = cause.to_noun(&mut allocator); - let decoded = BridgeCause::from_noun(&encoded) + let decoded = decode_from_slab::(&encoded, &allocator) .unwrap_or_else(|_| panic!("Failed to decode {} variant", name)); assert_eq!(decoded.0, 0, "{} variant should have version 0", name); } @@ -1521,8 +1572,8 @@ mod tests { let empty_blocks = BridgeCause(0, BridgeCauseVariant::BaseBlocks(empty_batch)); debug!("Encoding empty blocks list"); let encoded_empty = empty_blocks.to_noun(&mut allocator); - let decoded_empty = - BridgeCause::from_noun(&encoded_empty).expect("Failed to decode empty blocks"); + let decoded_empty = decode_from_slab::(&encoded_empty, &allocator) + .expect("Failed to decode empty blocks"); match decoded_empty.1 { BridgeCauseVariant::BaseBlocks(batch) => { @@ -1538,8 +1589,8 @@ mod tests { ); debug!("Encoding non-empty blocks list"); let encoded_nonempty = nonempty_blocks.to_noun(&mut allocator); - let decoded_nonempty = - BridgeCause::from_noun(&encoded_nonempty).expect("Failed to decode non-empty blocks"); + let decoded_nonempty = decode_from_slab::(&encoded_nonempty, &allocator) + .expect("Failed to decode non-empty blocks"); match decoded_nonempty.1 { BridgeCauseVariant::BaseBlocks(batch) => { @@ -1581,7 +1632,7 @@ mod tests { let encoded_noun = original_effect.to_noun(&mut allocator); info!("Encoded base-call effect to noun"); - let decoded_effect = BridgeEffect::from_noun(&encoded_noun) + let decoded_effect = decode_from_slab::(&encoded_noun, &allocator) .expect("Failed to decode base-call effect from noun"); debug!("Decoded base-call effect successfully"); @@ -1633,7 +1684,7 @@ mod tests { let encoded_noun = original_effect.to_noun(&mut allocator); info!("Encoded assemble-base-call effect to noun"); - let decoded_effect = BridgeEffect::from_noun(&encoded_noun) + let decoded_effect = decode_from_slab::(&encoded_noun, &allocator) .expect("Failed to decode assemble-base-call effect from noun"); debug!("Decoded assemble-base-call effect successfully"); @@ -1683,7 +1734,7 @@ mod tests { let encoded_noun = original_effect.to_noun(&mut allocator); info!("Encoded nock-deposit-request effect to noun"); - let decoded_effect = BridgeEffect::from_noun(&encoded_noun) + let decoded_effect = decode_from_slab::(&encoded_noun, &allocator) .expect("Failed to decode nock-deposit-request effect from noun"); debug!("Decoded nock-deposit-request effect successfully"); @@ -1745,12 +1796,21 @@ mod tests { let encoded_noun = original_effect.to_noun(&mut allocator); info!("Encoded commit-nock-deposits effect to noun"); + let debug_space = allocator.noun_space(); + let debug_cell = encoded_noun + .in_space(&debug_space) + .as_cell() + .expect("encoded noun cell") + .cell(); eprintln!( "encoded_noun: {:?}", - nockvm::noun::FullDebugCell(&encoded_noun.as_cell().unwrap()) + nockvm::noun::FullDebugCell { + cell: &debug_cell, + space: &debug_space, + } ); - let decoded_effect = BridgeEffect::from_noun(&encoded_noun) + let decoded_effect = decode_from_slab::(&encoded_noun, &allocator) .expect("Failed to decode commit-nock-deposits effect from noun"); debug!("Decoded commit-nock-deposits effect successfully"); @@ -1798,7 +1858,7 @@ mod tests { let encoded_noun = original_effect.to_noun(&mut allocator); info!("Encoded nockchain-tx effect to noun"); - let decoded_effect = BridgeEffect::from_noun(&encoded_noun) + let decoded_effect = decode_from_slab::(&encoded_noun, &allocator) .expect("Failed to decode nockchain-tx effect from noun"); debug!("Decoded nockchain-tx effect successfully"); @@ -1846,7 +1906,7 @@ mod tests { let encoded_noun = original_effect.to_noun(&mut allocator); info!("Encoded propose-nockchain-tx effect to noun"); - let decoded_effect = BridgeEffect::from_noun(&encoded_noun) + let decoded_effect = decode_from_slab::(&encoded_noun, &allocator) .expect("Failed to decode propose-nockchain-tx effect from noun"); debug!("Decoded propose-nockchain-tx effect successfully"); @@ -1889,8 +1949,8 @@ mod tests { }; let encoded_noun = original_effect.to_noun(&mut allocator); - let decoded_effect = - BridgeEffect::from_noun(&encoded_noun).expect("Failed to decode stop effect from noun"); + let decoded_effect = decode_from_slab::(&encoded_noun, &allocator) + .expect("Failed to decode stop effect from noun"); assert_eq!(decoded_effect.version, 0, "Version should be 0"); match decoded_effect.variant { @@ -1998,7 +2058,8 @@ mod tests { let cause = BridgeCause(0, BridgeCauseVariant::BaseBlocks(vec![entry])); let encoded = cause.to_noun(&mut allocator); - let decoded = BridgeCause::from_noun(&encoded).expect("Failed to decode large AtomBytes"); + let decoded = decode_from_slab::(&encoded, &allocator) + .expect("Failed to decode large AtomBytes"); match decoded.1 { BridgeCauseVariant::BaseBlocks(blocks) => { @@ -2040,7 +2101,7 @@ mod tests { }; let encoded = effect.to_noun(&mut allocator); - let decoded = BridgeEffect::from_noun(&encoded) + let decoded = decode_from_slab::(&encoded, &allocator) .expect("Failed to decode effect with many signatures"); match decoded.variant { @@ -2077,8 +2138,8 @@ mod tests { }; let encoded = effect.to_noun(&mut allocator); - let decoded = - BridgeEffect::from_noun(&encoded).expect("Failed to decode effect with empty fees"); + let decoded = decode_from_slab::(&encoded, &allocator) + .expect("Failed to decode effect with empty fees"); match decoded.variant { BridgeEffectVariant::AssembleBaseCall(data) => { @@ -2280,8 +2341,8 @@ mod tests { let cause = BridgeCause(0, BridgeCauseVariant::ConfigLoad(Some(config.clone()))); let encoded = cause.to_noun(&mut allocator); - let decoded = - BridgeCause::from_noun(&encoded).expect("Failed to decode cfg-load with Some"); + let decoded = decode_from_slab::(&encoded, &allocator) + .expect("Failed to decode cfg-load with Some"); match decoded.1 { BridgeCauseVariant::ConfigLoad(Some(decoded_config)) => { @@ -2309,8 +2370,8 @@ mod tests { let original = BridgeConstants::default(); let encoded = original.to_noun(&mut allocator); - let decoded = - BridgeConstants::from_noun(&encoded).expect("Failed to decode BridgeConstants"); + let decoded = decode_from_slab::(&encoded, &allocator) + .expect("Failed to decode BridgeConstants"); assert_eq!(decoded.version, original.version); assert_eq!(decoded.min_signers, original.min_signers); @@ -2337,8 +2398,8 @@ mod tests { let original = BridgeCause::set_constants(constants.clone()); let encoded = original.to_noun(&mut allocator); - let decoded = - BridgeCause::from_noun(&encoded).expect("Failed to decode set-constants cause"); + let decoded = decode_from_slab::(&encoded, &allocator) + .expect("Failed to decode set-constants cause"); assert_eq!(decoded.0, 0); match decoded.1 { @@ -2377,7 +2438,7 @@ mod tests { let encoded = event.to_noun(&mut allocator); info!("Encoded BaseEvent with DepositProcessed to noun"); - let decoded = BaseEvent::from_noun(&encoded) + let decoded = decode_from_slab::(&encoded, &allocator) .expect("Failed to decode BaseEvent with DepositProcessed"); info!("Decoded BaseEvent successfully"); @@ -2439,7 +2500,7 @@ mod tests { let encoded = event.to_noun(&mut allocator); info!("Encoded BaseEvent with BurnForWithdrawal to noun"); - let decoded = BaseEvent::from_noun(&encoded) + let decoded = decode_from_slab::(&encoded, &allocator) .expect("Failed to decode BaseEvent with BurnForWithdrawal"); info!("Decoded BaseEvent successfully"); @@ -2510,8 +2571,8 @@ mod tests { let encoded = cause.to_noun(&mut allocator); info!("Encoded BridgeCause with BaseBlocks containing events"); - let decoded = - BridgeCause::from_noun(&encoded).expect("Failed to decode BridgeCause with BaseBlocks"); + let decoded = decode_from_slab::(&encoded, &allocator) + .expect("Failed to decode BridgeCause with BaseBlocks"); info!("Decoded BridgeCause successfully"); match decoded.1 { @@ -2556,7 +2617,8 @@ mod tests { inner: Some(Some(42)), }; let encoded = original.to_noun(&mut allocator); - let decoded = CountPeek::from_noun(&encoded).expect("Failed to decode CountPeek from noun"); + let decoded = decode_from_slab::(&encoded, &allocator) + .expect("Failed to decode CountPeek from noun"); assert_eq!( decoded.inner, Some(Some(42)), @@ -2567,8 +2629,8 @@ mod tests { let mut allocator2: NounSlab = NounSlab::new(); let original_none = CountPeek { inner: Some(None) }; let encoded_none = original_none.to_noun(&mut allocator2); - let decoded_none = - CountPeek::from_noun(&encoded_none).expect("Failed to decode CountPeek None from noun"); + let decoded_none = decode_from_slab::(&encoded_none, &allocator2) + .expect("Failed to decode CountPeek None from noun"); assert_eq!( decoded_none.inner, Some(None), @@ -2590,8 +2652,8 @@ mod tests { inner: Some(Some(true)), }; let encoded_true = original_true.to_noun(&mut allocator); - let decoded_true = - BoolPeek::from_noun(&encoded_true).expect("Failed to decode BoolPeek true from noun"); + let decoded_true = decode_from_slab::(&encoded_true, &allocator) + .expect("Failed to decode BoolPeek true from noun"); assert_eq!( decoded_true.inner, Some(Some(true)), @@ -2604,8 +2666,8 @@ mod tests { inner: Some(Some(false)), }; let encoded_false = original_false.to_noun(&mut allocator2); - let decoded_false = - BoolPeek::from_noun(&encoded_false).expect("Failed to decode BoolPeek false from noun"); + let decoded_false = decode_from_slab::(&encoded_false, &allocator2) + .expect("Failed to decode BoolPeek false from noun"); assert_eq!( decoded_false.inner, Some(Some(false)), diff --git a/crates/bridge/tests/core_runner_integration.rs b/crates/bridge/tests/core_runner_integration.rs new file mode 100644 index 000000000..2bd9ce873 --- /dev/null +++ b/crates/bridge/tests/core_runner_integration.rs @@ -0,0 +1,649 @@ +#![allow(clippy::unwrap_used)] + +use std::collections::{HashMap, VecDeque}; + +use bridge::core::base_observer::{BaseObserverCore, BaseObserverRunner, BasePlanAction}; +use bridge::core::nock_observer::{NockObserverCore, NockObserverRunner, NockPlanAction}; +use bridge::errors::BridgeError; +use bridge::ports::NockTipInfo; +use bridge::runtime::{BaseBlockBatch, ChainEvent, NockBlockEvent}; +use bridge::types::{zero_tip5_hash, AtomBytes, BaseBlockRef}; +use nockapp::noun::slab::{NockJammer, NounSlab}; +use nockchain_math::belt::Belt; +use nockchain_types::tx_engine::common::{BigNum, CoinbaseSplit, Hash as NockHash, Page}; +use noun_serde::NounEncode; + +#[path = "harness/in_memory_ports.rs"] +mod in_memory_ports; + +use in_memory_ports::{InMemoryBaseSource, InMemoryKernelState}; + +fn sample_batch(start: u64, end: u64) -> BaseBlockBatch { + let mut blocks = Vec::new(); + for height in start..=end { + blocks.push(BaseBlockRef { + height, + block_id: AtomBytes(vec![height as u8]), + parent_block_id: AtomBytes(vec![height.saturating_sub(1) as u8]), + }); + } + + BaseBlockBatch { + version: 0, + first_height: start, + last_height: end, + blocks, + withdrawals: Vec::new(), + deposit_settlements: Vec::new(), + block_events: HashMap::new(), + prev: zero_tip5_hash(), + } +} + +fn sample_nock_block_event(height: u64) -> NockBlockEvent { + let page = Page { + digest: NockHash([Belt(1), Belt(2), Belt(3), Belt(4), Belt(5)]), + pow: None, + parent: NockHash([Belt(6), Belt(7), Belt(8), Belt(9), Belt(10)]), + tx_ids: vec![], + coinbase: CoinbaseSplit::V0(vec![]), + timestamp: 0, + epoch_counter: 0, + target: BigNum::from_u64(0), + accumulated_work: BigNum::from_u64(0), + height, + msg: vec![], + }; + + let mut page_slab: NounSlab = NounSlab::new(); + let page_noun = page.to_noun(&mut page_slab); + + NockBlockEvent { + block: page, + page_slab, + page_noun, + txs: vec![], + } +} + +#[tokio::test] +async fn base_runner_fetches_and_emits_batch_event() { + let source = InMemoryBaseSource::default(); + *source.chain_tip.lock().await = 500; + source + .batches + .lock() + .await + .extend(VecDeque::from([sample_batch(101, 105)])); + + let kernel = InMemoryKernelState::default(); + *kernel.base_next_height.lock().await = Some(101); + + let runner = BaseObserverRunner { + core: BaseObserverCore { + batch_size: 5, + confirmation_depth: 10, + }, + source, + kernel: kernel.clone(), + }; + + let action = runner.tick_once().await.unwrap(); + assert_eq!( + action, + BasePlanAction::FetchWindow { + chain_tip: 500, + confirmed_height: 490, + start: 101, + end: 105, + } + ); + + let events = kernel.events.lock().await; + assert_eq!(events.len(), 1); + match &events[0] { + ChainEvent::Base(batch) => { + assert_eq!(batch.first_height, 101); + assert_eq!(batch.last_height, 105); + assert_eq!(batch.blocks.len(), 5); + } + ChainEvent::Nock(_) => panic!("expected base event"), + } +} + +#[tokio::test] +async fn base_runner_no_pending_height_emits_nothing() { + let source = InMemoryBaseSource::default(); + *source.chain_tip.lock().await = 500; + + let kernel = InMemoryKernelState::default(); + *kernel.base_next_height.lock().await = None; + + let runner = BaseObserverRunner { + core: BaseObserverCore { + batch_size: 5, + confirmation_depth: 10, + }, + source, + kernel: kernel.clone(), + }; + + let action = runner.tick_once().await.unwrap(); + assert_eq!(action, BasePlanAction::NoPendingHeight { chain_tip: 500 }); + assert!(kernel.events.lock().await.is_empty()); +} + +#[tokio::test] +async fn nock_runner_fetches_and_emits_block_event() { + let mut source = in_memory_ports::InMemoryNockSource::default(); + source.tips.push_back(NockTipInfo { + height: 250, + tip_hash: "tip-b58".to_string(), + }); + source.blocks.push_back(sample_nock_block_event(150)); + + let kernel = InMemoryKernelState::default(); + *kernel.nock_next_height.lock().await = Some(150); + + let mut runner = NockObserverRunner { + core: NockObserverCore { + confirmation_depth: 100, + }, + source, + kernel: kernel.clone(), + }; + + let action = runner.tick_once().await.unwrap(); + assert_eq!( + action, + NockPlanAction::FetchHeight { + tip_height: 250, + confirmed_target: 150, + height: 150, + } + ); + + let events = kernel.events.lock().await; + assert_eq!(events.len(), 1); + match &events[0] { + ChainEvent::Nock(block) => { + assert_eq!(block.height(), 150); + assert_eq!(block.txs.len(), 0); + } + ChainEvent::Base(_) => panic!("expected nock event"), + } +} + +#[tokio::test] +async fn nock_runner_not_yet_confirmed_emits_nothing() { + let mut source = in_memory_ports::InMemoryNockSource::default(); + source.tips.push_back(NockTipInfo { + height: 250, + tip_hash: "tip-b58".to_string(), + }); + + let kernel = InMemoryKernelState::default(); + *kernel.nock_next_height.lock().await = Some(175); + + let mut runner = NockObserverRunner { + core: NockObserverCore { + confirmation_depth: 100, + }, + source, + kernel: kernel.clone(), + }; + + let action = runner.tick_once().await.unwrap(); + assert_eq!( + action, + NockPlanAction::NotYetConfirmed { + tip_height: 250, + confirmed_target: 150, + next_needed_height: 175, + } + ); + assert!(kernel.events.lock().await.is_empty()); +} + +#[tokio::test] +async fn base_runner_replay_transcript_handles_mixed_paths_deterministically() { + let source = InMemoryBaseSource::default(); + source.chain_tip_responses.lock().await.extend([ + Ok(500), + Ok(2800), + Ok(2800), + Err(BridgeError::Runtime( + "simulated base tip source error".to_string(), + )), + ]); + source + .batch_responses + .lock() + .await + .push_back(Ok(sample_batch(1001, 2000))); + + let kernel = InMemoryKernelState::default(); + kernel.base_next_height_results.lock().await.extend([ + Ok(None), + Ok(Some(2001)), + Ok(Some(1001)), + Ok(Some(1001)), + ]); + + let runner = BaseObserverRunner { + core: BaseObserverCore { + batch_size: 1000, + confirmation_depth: 300, + }, + source, + kernel: kernel.clone(), + }; + + let action1 = runner.tick_once().await.expect("tick1"); + assert_eq!(action1, BasePlanAction::NoPendingHeight { chain_tip: 500 }); + + let action2 = runner.tick_once().await.expect("tick2"); + assert_eq!( + action2, + BasePlanAction::NotYetConfirmed { + chain_tip: 2800, + confirmed_height: 2500, + next_needed_height: 2001, + needed_confirmed_height: 3000, + blocks_until_ready: 500, + } + ); + + let action3 = runner.tick_once().await.expect("tick3"); + assert_eq!( + action3, + BasePlanAction::FetchWindow { + chain_tip: 2800, + confirmed_height: 2500, + start: 1001, + end: 2000, + } + ); + + let err = runner.tick_once().await.expect_err("tick4 should fail"); + assert!( + err.to_string().contains("simulated base tip source error"), + "unexpected error: {err}" + ); + + let events = kernel.events.lock().await; + assert_eq!(events.len(), 1, "only the fetch tick should emit"); + match &events[0] { + ChainEvent::Base(batch) => { + assert_eq!(batch.first_height, 1001); + assert_eq!(batch.last_height, 2000); + } + ChainEvent::Nock(_) => panic!("expected base event"), + } +} + +#[tokio::test] +async fn nock_runner_replay_transcript_handles_fetch_none_and_error_paths() { + let mut source = in_memory_ports::InMemoryNockSource::default(); + source.tip_results.extend([ + Ok(Some(NockTipInfo { + height: 250, + tip_hash: "tip-a".to_string(), + })), + Ok(Some(NockTipInfo { + height: 250, + tip_hash: "tip-b".to_string(), + })), + Ok(Some(NockTipInfo { + height: 250, + tip_hash: "tip-c".to_string(), + })), + Err(BridgeError::Runtime( + "simulated nock tip source error".to_string(), + )), + ]); + source + .block_results + .extend([Ok(Some(sample_nock_block_event(150))), Ok(None)]); + + let kernel = InMemoryKernelState::default(); + kernel.nock_next_height_results.lock().await.extend([ + Ok(Some(175)), + Ok(Some(150)), + Ok(Some(150)), + Ok(Some(150)), + ]); + + let mut runner = NockObserverRunner { + core: NockObserverCore { + confirmation_depth: 100, + }, + source, + kernel: kernel.clone(), + }; + + let action1 = runner.tick_once().await.expect("tick1"); + assert_eq!( + action1, + NockPlanAction::NotYetConfirmed { + tip_height: 250, + confirmed_target: 150, + next_needed_height: 175, + } + ); + + let action2 = runner.tick_once().await.expect("tick2"); + assert_eq!( + action2, + NockPlanAction::FetchHeight { + tip_height: 250, + confirmed_target: 150, + height: 150, + } + ); + + let action3 = runner.tick_once().await.expect("tick3"); + assert_eq!( + action3, + NockPlanAction::FetchHeight { + tip_height: 250, + confirmed_target: 150, + height: 150, + } + ); + + let err = runner.tick_once().await.expect_err("tick4 should fail"); + assert!( + err.to_string().contains("simulated nock tip source error"), + "unexpected error: {err}" + ); + + let events = kernel.events.lock().await; + assert_eq!(events.len(), 1, "only one fetch should emit an event"); + match &events[0] { + ChainEvent::Nock(block) => assert_eq!(block.height(), 150), + ChainEvent::Base(_) => panic!("expected nock event"), + } +} + +#[tokio::test] +async fn base_runner_replay_peek_and_emit_errors_then_recovers() { + let source = InMemoryBaseSource::default(); + source + .chain_tip_responses + .lock() + .await + .extend([Ok(2800), Ok(2800), Ok(2800)]); + source + .batch_responses + .lock() + .await + .extend([Ok(sample_batch(1001, 2000)), Ok(sample_batch(1001, 2000))]); + + let kernel = InMemoryKernelState::default(); + kernel.base_next_height_results.lock().await.extend([ + Err(BridgeError::Runtime( + "simulated base peek source error".to_string(), + )), + Ok(Some(1001)), + Ok(Some(1001)), + ]); + kernel + .emit_results + .lock() + .await + .push_back(Err(BridgeError::Runtime( + "simulated base emit error".to_string(), + ))); + + let runner = BaseObserverRunner { + core: BaseObserverCore { + batch_size: 1000, + confirmation_depth: 300, + }, + source, + kernel: kernel.clone(), + }; + + let err1 = runner.tick_once().await.expect_err("tick1 should fail"); + assert!( + err1.to_string() + .contains("simulated base peek source error"), + "unexpected error: {err1}" + ); + + let err2 = runner.tick_once().await.expect_err("tick2 should fail"); + assert!( + err2.to_string().contains("simulated base emit error"), + "unexpected error: {err2}" + ); + + let action3 = runner.tick_once().await.expect("tick3 should recover"); + assert_eq!( + action3, + BasePlanAction::FetchWindow { + chain_tip: 2800, + confirmed_height: 2500, + start: 1001, + end: 2000, + } + ); + + let events = kernel.events.lock().await; + assert_eq!(events.len(), 1, "only recovered emit should be recorded"); + match &events[0] { + ChainEvent::Base(batch) => { + assert_eq!(batch.first_height, 1001); + assert_eq!(batch.last_height, 2000); + } + ChainEvent::Nock(_) => panic!("expected base event"), + } +} + +#[tokio::test] +async fn base_runner_replay_fetch_error_then_recovers() { + let source = InMemoryBaseSource::default(); + source + .chain_tip_responses + .lock() + .await + .extend([Ok(2800), Ok(2800)]); + source.batch_responses.lock().await.extend([ + Err(BridgeError::Runtime( + "simulated base batch fetch error".to_string(), + )), + Ok(sample_batch(1001, 2000)), + ]); + + let kernel = InMemoryKernelState::default(); + kernel + .base_next_height_results + .lock() + .await + .extend([Ok(Some(1001)), Ok(Some(1001))]); + + let runner = BaseObserverRunner { + core: BaseObserverCore { + batch_size: 1000, + confirmation_depth: 300, + }, + source, + kernel: kernel.clone(), + }; + + let err1 = runner.tick_once().await.expect_err("tick1 should fail"); + assert!( + err1.to_string() + .contains("simulated base batch fetch error"), + "unexpected error: {err1}" + ); + + let action2 = runner.tick_once().await.expect("tick2 should recover"); + assert_eq!( + action2, + BasePlanAction::FetchWindow { + chain_tip: 2800, + confirmed_height: 2500, + start: 1001, + end: 2000, + } + ); + assert_eq!( + kernel.events.lock().await.len(), + 1, + "only recovered fetch should emit" + ); +} + +#[tokio::test] +async fn nock_runner_replay_peek_and_emit_errors_then_recovers() { + let mut source = in_memory_ports::InMemoryNockSource::default(); + source.tip_results.extend([ + Ok(Some(NockTipInfo { + height: 250, + tip_hash: "tip-a".to_string(), + })), + Ok(Some(NockTipInfo { + height: 250, + tip_hash: "tip-b".to_string(), + })), + Ok(Some(NockTipInfo { + height: 250, + tip_hash: "tip-c".to_string(), + })), + ]); + source + .block_results + .extend([Ok(Some(sample_nock_block_event(150))), Ok(Some(sample_nock_block_event(150)))]); + + let kernel = InMemoryKernelState::default(); + kernel.nock_next_height_results.lock().await.extend([ + Err(BridgeError::Runtime( + "simulated nock peek source error".to_string(), + )), + Ok(Some(150)), + Ok(Some(150)), + ]); + kernel + .emit_results + .lock() + .await + .push_back(Err(BridgeError::Runtime( + "simulated nock emit error".to_string(), + ))); + + let mut runner = NockObserverRunner { + core: NockObserverCore { + confirmation_depth: 100, + }, + source, + kernel: kernel.clone(), + }; + + let err1 = runner.tick_once().await.expect_err("tick1 should fail"); + assert!( + err1.to_string() + .contains("simulated nock peek source error"), + "unexpected error: {err1}" + ); + + let err2 = runner.tick_once().await.expect_err("tick2 should fail"); + assert!( + err2.to_string().contains("simulated nock emit error"), + "unexpected error: {err2}" + ); + + let action3 = runner.tick_once().await.expect("tick3 should recover"); + assert_eq!( + action3, + NockPlanAction::FetchHeight { + tip_height: 250, + confirmed_target: 150, + height: 150, + } + ); + assert_eq!( + kernel.events.lock().await.len(), + 1, + "only recovered nock emit should be recorded" + ); +} + +#[tokio::test] +async fn nock_runner_replay_fetch_error_and_none_then_recovers() { + let mut source = in_memory_ports::InMemoryNockSource::default(); + source.tip_results.extend([ + Ok(Some(NockTipInfo { + height: 250, + tip_hash: "tip-a".to_string(), + })), + Ok(Some(NockTipInfo { + height: 250, + tip_hash: "tip-b".to_string(), + })), + Ok(Some(NockTipInfo { + height: 250, + tip_hash: "tip-c".to_string(), + })), + ]); + source.block_results.extend([ + Err(BridgeError::Runtime( + "simulated nock block fetch error".to_string(), + )), + Ok(None), + Ok(Some(sample_nock_block_event(150))), + ]); + + let kernel = InMemoryKernelState::default(); + kernel.nock_next_height_results.lock().await.extend([ + Ok(Some(150)), + Ok(Some(150)), + Ok(Some(150)), + ]); + + let mut runner = NockObserverRunner { + core: NockObserverCore { + confirmation_depth: 100, + }, + source, + kernel: kernel.clone(), + }; + + let err1 = runner.tick_once().await.expect_err("tick1 should fail"); + assert!( + err1.to_string() + .contains("simulated nock block fetch error"), + "unexpected error: {err1}" + ); + + let action2 = runner.tick_once().await.expect("tick2 should continue"); + assert_eq!( + action2, + NockPlanAction::FetchHeight { + tip_height: 250, + confirmed_target: 150, + height: 150, + } + ); + assert!( + kernel.events.lock().await.is_empty(), + "none fetch result should not emit" + ); + + let action3 = runner.tick_once().await.expect("tick3 should recover"); + assert_eq!( + action3, + NockPlanAction::FetchHeight { + tip_height: 250, + confirmed_target: 150, + height: 150, + } + ); + + let events = kernel.events.lock().await; + assert_eq!(events.len(), 1, "recovered fetch should emit one event"); + match &events[0] { + ChainEvent::Nock(block) => assert_eq!(block.height(), 150), + ChainEvent::Base(_) => panic!("expected nock event"), + } +} diff --git a/crates/bridge/tests/harness/in_memory_ports.rs b/crates/bridge/tests/harness/in_memory_ports.rs new file mode 100644 index 000000000..fa3bba2e2 --- /dev/null +++ b/crates/bridge/tests/harness/in_memory_ports.rs @@ -0,0 +1,163 @@ +#![allow(dead_code)] + +use std::collections::VecDeque; +use std::sync::Arc; + +use async_trait::async_trait; +use bridge::errors::BridgeError; +use bridge::ethereum::DepositSubmissionResult; +use bridge::ports::{ + BaseContractPort, BaseSourcePort, KernelStatePort, NockSourcePort, NockTipInfo, +}; +use bridge::runtime::{BaseBlockBatch, ChainEvent, EventId, NockBlockEvent}; +use bridge::types::{DepositSubmission, Tip5Hash}; +use tokio::sync::Mutex; + +type SharedHeightResultQueue = Arc, BridgeError>>>>; +type SharedHoldResultQueue = Arc>>>; + +#[derive(Clone, Default)] +pub struct InMemoryBaseContract { + pub last_nonce: Arc>, + pub processed: Arc>>, + pub submissions: Arc>>, +} + +#[async_trait] +impl BaseContractPort for InMemoryBaseContract { + async fn submit_deposit( + &self, + submission: DepositSubmission, + ) -> Result { + self.submissions.lock().await.push(submission); + let mut nonce = self.last_nonce.lock().await; + *nonce = nonce.saturating_add(1); + Ok(DepositSubmissionResult { + tx_hash: format!("in-memory-{:x}", *nonce), + block_number: *nonce, + }) + } + + async fn get_last_deposit_nonce(&self) -> Result { + Ok(*self.last_nonce.lock().await) + } + + async fn is_deposit_processed(&self, tx_id: &Tip5Hash) -> Result { + Ok(self.processed.lock().await.iter().any(|seen| seen == tx_id)) + } +} + +#[derive(Clone, Default)] +pub struct InMemoryBaseSource { + pub chain_tip: Arc>, + pub batches: Arc>>, + pub chain_tip_responses: Arc>>>, + pub batch_responses: Arc>>>, +} + +#[async_trait] +impl BaseSourcePort for InMemoryBaseSource { + async fn chain_tip_height(&self) -> Result { + if let Some(response) = self.chain_tip_responses.lock().await.pop_front() { + return response; + } + Ok(*self.chain_tip.lock().await) + } + + async fn fetch_batch(&self, _start: u64, _end: u64) -> Result { + if let Some(response) = self.batch_responses.lock().await.pop_front() { + return response; + } + self.batches + .lock() + .await + .pop_front() + .ok_or_else(|| BridgeError::Runtime("in-memory batch queue empty".into())) + } +} + +#[derive(Default)] +pub struct InMemoryNockSource { + pub tips: VecDeque, + pub blocks: VecDeque, + pub tip_results: VecDeque, BridgeError>>, + pub block_results: VecDeque, BridgeError>>, +} + +#[async_trait] +impl NockSourcePort for InMemoryNockSource { + async fn tip_info(&mut self) -> Result, BridgeError> { + if let Some(response) = self.tip_results.pop_front() { + return response; + } + Ok(self.tips.pop_front()) + } + + async fn fetch_block_at_height( + &mut self, + _height: u64, + ) -> Result, BridgeError> { + if let Some(response) = self.block_results.pop_front() { + return response; + } + Ok(self.blocks.pop_front()) + } +} + +#[derive(Clone, Default)] +pub struct InMemoryKernelState { + pub base_hold_active: Arc>, + pub base_next_height: Arc>>, + pub nock_next_height: Arc>>, + pub events: Arc>>, + pub base_tip_hash: Arc>>, + pub base_hold_results: SharedHoldResultQueue, + pub base_next_height_results: SharedHeightResultQueue, + pub nock_next_height_results: SharedHeightResultQueue, + pub emit_results: Arc>>>, +} + +#[async_trait] +impl KernelStatePort for InMemoryKernelState { + async fn peek_base_hold(&self) -> Result { + if let Some(response) = self.base_hold_results.lock().await.pop_front() { + return response; + } + Ok(*self.base_hold_active.lock().await) + } + + async fn peek_base_next_height(&self) -> Result, BridgeError> { + if let Some(response) = self.base_next_height_results.lock().await.pop_front() { + return response; + } + Ok(*self.base_next_height.lock().await) + } + + async fn peek_nock_next_height(&self) -> Result, BridgeError> { + if let Some(response) = self.nock_next_height_results.lock().await.pop_front() { + return response; + } + Ok(*self.nock_next_height.lock().await) + } + + async fn emit_chain_event(&self, event: ChainEvent) -> Result { + if let Some(response) = self.emit_results.lock().await.pop_front() { + if response.is_ok() { + self.events.lock().await.push(event); + } + return response; + } + self.events.lock().await.push(event); + Ok(EventId { + kind: bridge::runtime::BridgeEventKind::ChainBase, + timestamp_ms: 0, + digest: [0u8; 32], + }) + } + + fn set_base_tip_hash(&self, tip_hash: String) { + if let Ok(mut guard) = self.base_tip_hash.try_lock() { + *guard = Some(tip_hash); + } + } +} diff --git a/crates/bridge/tests/loop_policy_shell_tests.rs b/crates/bridge/tests/loop_policy_shell_tests.rs new file mode 100644 index 000000000..5fee16ea5 --- /dev/null +++ b/crates/bridge/tests/loop_policy_shell_tests.rs @@ -0,0 +1,204 @@ +#![allow(clippy::unwrap_used)] + +use std::sync::atomic::{AtomicUsize, Ordering}; +use std::sync::{Arc, RwLock}; +use std::time::{Duration, SystemTime}; + +use alloy::primitives::Address; +use async_trait::async_trait; +use bridge::bridge_status::BridgeStatus; +use bridge::errors::BridgeError; +use bridge::ethereum::DepositSubmissionResult; +use bridge::health::SharedHealthState; +use bridge::loop_policy::PostingLoopPolicy; +use bridge::ports::BaseContractPort; +use bridge::proposal_cache::{ProposalCache, SignatureData}; +use bridge::runtime::run_posting_loop_with_policy; +use bridge::status::BridgeStatusState; +use bridge::stop::{StopController, StopInfo, StopSource}; +use bridge::types::{ + zero_tip5_hash, AtomBytes, DepositId, DepositSubmission, EthAddress, NockDepositRequestData, + NodeConfig, NodeInfo, SchnorrSecretKey, Tip5Hash, +}; +use nockchain_math::belt::Belt; +use nockchain_types::tx_engine::common::Hash as NockPkh; +use nockchain_types::v1::Name; + +#[derive(Clone, Default)] +struct CountingBaseContract { + get_last_nonce_calls: Arc, + is_processed_calls: Arc, + submit_calls: Arc, +} + +impl CountingBaseContract { + fn get_last_nonce_calls(&self) -> usize { + self.get_last_nonce_calls.load(Ordering::SeqCst) + } + + fn submit_calls(&self) -> usize { + self.submit_calls.load(Ordering::SeqCst) + } +} + +#[async_trait] +impl BaseContractPort for CountingBaseContract { + async fn submit_deposit( + &self, + _submission: DepositSubmission, + ) -> Result { + self.submit_calls.fetch_add(1, Ordering::SeqCst); + Ok(DepositSubmissionResult { + tx_hash: "in-memory".to_string(), + block_number: 0, + }) + } + + async fn get_last_deposit_nonce(&self) -> Result { + self.get_last_nonce_calls.fetch_add(1, Ordering::SeqCst); + Ok(0) + } + + async fn is_deposit_processed(&self, _tx_id: &Tip5Hash) -> Result { + self.is_processed_calls.fetch_add(1, Ordering::SeqCst); + Ok(false) + } +} + +fn test_bridge_status() -> BridgeStatus { + let health: SharedHealthState = Arc::new(RwLock::new(Vec::new())); + BridgeStatus::new(health) +} + +fn trigger_stop_now(controller: &StopController) { + controller.trigger(StopInfo { + reason: "test stop".to_string(), + last: None, + source: StopSource::Local, + at: SystemTime::now(), + }); +} + +fn test_node_config() -> NodeConfig { + NodeConfig { + node_id: 0, + nodes: vec![NodeInfo { + ip: "localhost:8001".to_string(), + eth_pubkey: AtomBytes(vec![0u8; 20]), + nock_pkh: NockPkh::from_base58("2222222222222222222222222222222222222222222222222222") + .expect("valid test nock pkh"), + }], + my_eth_key: AtomBytes(vec![]), + my_nock_key: SchnorrSecretKey([Belt(0); 8]), + } +} + +fn sample_proposal(nonce: u64) -> NockDepositRequestData { + NockDepositRequestData { + tx_id: Tip5Hash([Belt(1 + nonce), Belt(2), Belt(3), Belt(4), Belt(5)]), + name: Name::new(zero_tip5_hash(), zero_tip5_hash()), + recipient: EthAddress::ZERO, + amount: 1_000, + block_height: 100, + as_of: zero_tip5_hash(), + nonce, + } +} + +fn seed_ready_proposal(cache: &ProposalCache) -> DepositId { + let proposal = sample_proposal(1); + let proposal_hash = proposal.compute_proposal_hash(); + let deposit_id = DepositId::from_effect_payload(&proposal); + + let signer1 = Address::from([1u8; 20]); + let signer2 = Address::from([2u8; 20]); + let signer3 = Address::from([3u8; 20]); + + cache + .add_signature( + &deposit_id, + SignatureData { + signer_address: signer1, + signature: vec![1, 1, 1], + proposal_hash, + is_mine: true, + }, + Some(proposal.clone()), + move |_, _| Some(signer1), + ) + .expect("seed sig1"); + + cache + .add_signature( + &deposit_id, + SignatureData { + signer_address: signer2, + signature: vec![2, 2, 2], + proposal_hash, + is_mine: false, + }, + None, + move |_, _| Some(signer2), + ) + .expect("seed sig2"); + + cache + .add_signature( + &deposit_id, + SignatureData { + signer_address: signer3, + signature: vec![3, 3, 3], + proposal_hash, + is_mine: false, + }, + None, + move |_, _| Some(signer3), + ) + .expect("seed sig3"); + + assert!(cache.is_ready(&deposit_id).expect("ready check")); + deposit_id +} + +#[tokio::test] +async fn posting_loop_stopped_state_with_ready_proposal_skips_base_contract_calls() { + let proposal_cache = Arc::new(ProposalCache::new()); + let base_contract = Arc::new(CountingBaseContract::default()); + let bridge_status = test_bridge_status(); + let status_state = BridgeStatusState::new(); + let node_config = test_node_config(); + let (stop_controller, stop_handle) = StopController::new(); + trigger_stop_now(&stop_controller); + + let _deposit_id = seed_ready_proposal(&proposal_cache); + assert_eq!( + proposal_cache + .ready_proposals() + .expect("ready proposals") + .len(), + 1 + ); + + let policy = PostingLoopPolicy { + tick_interval: Duration::from_millis(5), + failover_backoff_secs: 1, + }; + + let task = tokio::spawn(run_posting_loop_with_policy( + proposal_cache, + base_contract.clone(), + node_config, + bridge_status, + stop_handle, + status_state, + policy, + )); + + tokio::time::sleep(Duration::from_millis(35)).await; + + assert_eq!(base_contract.get_last_nonce_calls(), 0); + assert_eq!(base_contract.submit_calls(), 0); + + task.abort(); + let _ = task.await; +} diff --git a/crates/bridge/tests/loop_tick_tests.rs b/crates/bridge/tests/loop_tick_tests.rs new file mode 100644 index 000000000..0307490c1 --- /dev/null +++ b/crates/bridge/tests/loop_tick_tests.rs @@ -0,0 +1,884 @@ +#![allow(clippy::unwrap_used)] + +use std::collections::{HashMap, HashSet, VecDeque}; +use std::sync::atomic::{AtomicUsize, Ordering}; +use std::sync::{Arc, Mutex}; +use std::time::{Duration, SystemTime, UNIX_EPOCH}; + +use alloy::primitives::Address; +use async_trait::async_trait; +use bridge::bridge_status::BridgeStatus; +use bridge::config::NonceEpochConfig; +use bridge::deposit_log::{DepositLog, DepositLogEntry, DepositLogInsertOutcome}; +use bridge::errors::BridgeError; +use bridge::ethereum::DepositSubmissionResult; +use bridge::health::SharedHealthState; +use bridge::ports::BaseContractPort; +use bridge::proposal_cache::{ProposalCache, ProposalStatus, SignatureData}; +use bridge::proposer::hoon_proposer; +use bridge::runtime::{ + posting_tick_once, signing_tick_once, BridgeEvent, BridgeRuntime, CauseBuildOutcome, + CauseBuilder, EventEnvelope, PostingTickConfig, PostingTickContext, PostingTickControl, + PostingTickInput, PostingTickNodeState, PostingTickPorts, PostingTickState, + SigningLocalStopMode, SigningTickConfig, SigningTickContext, SigningTickControl, + SigningTickInput, SigningTickNodeState, SigningTickPorts, SigningTickState, +}; +use bridge::signing::BridgeSigner; +use bridge::status::BridgeStatusState; +use bridge::stop::StopController; +use bridge::types::{ + zero_tip5_hash, AtomBytes, DepositId, DepositSubmission, EthAddress, NockDepositRequestData, + NodeConfig, NodeInfo, SchnorrSecretKey, Tip5Hash, +}; +use nockchain_math::belt::Belt; +use nockchain_types::tx_engine::common::Hash as NockPkh; +use nockchain_types::v1::Name; +use tempfile::TempDir; + +const TEST_PRIVATE_KEY: &str = "4c0883a69102937d6231471b5dbb6204fe5129617082792ae468d01a3f362318"; + +#[derive(Clone)] +struct CountingBaseContract { + get_last_nonce_calls: Arc, + is_processed_calls: Arc, + submit_calls: Arc, + last_nonce: u64, + fail_get_last_nonce: bool, + scripted_last_nonce: Arc>>>, + scripted_processed: Arc>>>, + scripted_submit: Arc>>>, +} + +impl CountingBaseContract { + fn with_last_nonce(last_nonce: u64) -> Self { + Self { + get_last_nonce_calls: Arc::new(AtomicUsize::new(0)), + is_processed_calls: Arc::new(AtomicUsize::new(0)), + submit_calls: Arc::new(AtomicUsize::new(0)), + last_nonce, + fail_get_last_nonce: false, + scripted_last_nonce: Arc::new(Mutex::new(VecDeque::new())), + scripted_processed: Arc::new(Mutex::new(VecDeque::new())), + scripted_submit: Arc::new(Mutex::new(VecDeque::new())), + } + } + + fn with_nonce_query_error() -> Self { + Self { + get_last_nonce_calls: Arc::new(AtomicUsize::new(0)), + is_processed_calls: Arc::new(AtomicUsize::new(0)), + submit_calls: Arc::new(AtomicUsize::new(0)), + last_nonce: 0, + fail_get_last_nonce: true, + scripted_last_nonce: Arc::new(Mutex::new(VecDeque::new())), + scripted_processed: Arc::new(Mutex::new(VecDeque::new())), + scripted_submit: Arc::new(Mutex::new(VecDeque::new())), + } + } + + fn script_last_nonce_responses( + &self, + responses: impl IntoIterator>, + ) { + self.scripted_last_nonce + .lock() + .expect("scripted_last_nonce lock poisoned") + .extend(responses); + } + + fn script_processed_responses( + &self, + responses: impl IntoIterator>, + ) { + self.scripted_processed + .lock() + .expect("scripted_processed lock poisoned") + .extend(responses); + } + + fn script_submit_responses( + &self, + responses: impl IntoIterator>, + ) { + self.scripted_submit + .lock() + .expect("scripted_submit lock poisoned") + .extend(responses); + } + + fn get_last_nonce_calls(&self) -> usize { + self.get_last_nonce_calls.load(Ordering::SeqCst) + } + + fn submit_calls(&self) -> usize { + self.submit_calls.load(Ordering::SeqCst) + } + + fn is_processed_calls(&self) -> usize { + self.is_processed_calls.load(Ordering::SeqCst) + } +} + +#[async_trait] +impl BaseContractPort for CountingBaseContract { + async fn submit_deposit( + &self, + _submission: DepositSubmission, + ) -> Result { + self.submit_calls.fetch_add(1, Ordering::SeqCst); + if let Some(response) = self + .scripted_submit + .lock() + .expect("scripted_submit lock poisoned") + .pop_front() + { + return response; + } + Ok(DepositSubmissionResult { + tx_hash: "in-memory".to_string(), + block_number: 0, + }) + } + + async fn get_last_deposit_nonce(&self) -> Result { + self.get_last_nonce_calls.fetch_add(1, Ordering::SeqCst); + if let Some(response) = self + .scripted_last_nonce + .lock() + .expect("scripted_last_nonce lock poisoned") + .pop_front() + { + return response; + } + if self.fail_get_last_nonce { + Err(BridgeError::BaseBridgeQuery( + "simulated nonce query error".to_string(), + )) + } else { + Ok(self.last_nonce) + } + } + + async fn is_deposit_processed(&self, _tx_id: &Tip5Hash) -> Result { + self.is_processed_calls.fetch_add(1, Ordering::SeqCst); + if let Some(response) = self + .scripted_processed + .lock() + .expect("scripted_processed lock poisoned") + .pop_front() + { + return response; + } + Ok(false) + } +} + +struct NoopCauseBuilder; + +impl CauseBuilder for NoopCauseBuilder { + fn build_poke( + &self, + _event: &EventEnvelope, + ) -> Result { + Ok(CauseBuildOutcome::Deferred("test".to_string())) + } +} + +fn test_bridge_status() -> BridgeStatus { + let health: SharedHealthState = Arc::new(std::sync::RwLock::new(Vec::new())); + BridgeStatus::new(health) +} + +fn test_node_config(node_id: u64) -> NodeConfig { + NodeConfig { + node_id, + nodes: vec![ + NodeInfo { + ip: "localhost:8001".to_string(), + eth_pubkey: AtomBytes(vec![0u8; 20]), + nock_pkh: NockPkh::from_base58( + "2222222222222222222222222222222222222222222222222222", + ) + .expect("valid test pkh 0"), + }, + NodeInfo { + ip: "localhost:8002".to_string(), + eth_pubkey: AtomBytes(vec![1u8; 20]), + nock_pkh: NockPkh::from_base58( + "3333333333333333333333333333333333333333333333333333", + ) + .expect("valid test pkh 1"), + }, + NodeInfo { + ip: "localhost:8003".to_string(), + eth_pubkey: AtomBytes(vec![2u8; 20]), + nock_pkh: NockPkh::from_base58( + "4444444444444444444444444444444444444444444444444444", + ) + .expect("valid test pkh 2"), + }, + ], + my_eth_key: AtomBytes(vec![]), + my_nock_key: SchnorrSecretKey([Belt(0); 8]), + } +} + +fn sample_proposal(nonce: u64) -> NockDepositRequestData { + NockDepositRequestData { + tx_id: Tip5Hash([Belt(1 + nonce), Belt(2), Belt(3), Belt(4), Belt(5)]), + name: Name::new(zero_tip5_hash(), zero_tip5_hash()), + recipient: EthAddress::ZERO, + amount: 1_000, + block_height: 100, + as_of: zero_tip5_hash(), + nonce, + } +} + +fn seed_ready_proposal(cache: &ProposalCache, nonce: u64) -> DepositId { + let mut proposal = sample_proposal(nonce); + proposal.as_of = Tip5Hash([Belt(10_000 + nonce), Belt(0), Belt(0), Belt(0), Belt(0)]); + let proposal_hash = proposal.compute_proposal_hash(); + let deposit_id = DepositId::from_effect_payload(&proposal); + + let signer1 = Address::from([1u8; 20]); + let signer2 = Address::from([2u8; 20]); + let signer3 = Address::from([3u8; 20]); + + cache + .add_signature( + &deposit_id, + SignatureData { + signer_address: signer1, + signature: vec![1, 1, 1], + proposal_hash, + is_mine: true, + }, + Some(proposal.clone()), + move |_, _| Some(signer1), + ) + .expect("seed sig1"); + + cache + .add_signature( + &deposit_id, + SignatureData { + signer_address: signer2, + signature: vec![2, 2, 2], + proposal_hash, + is_mine: false, + }, + None, + move |_, _| Some(signer2), + ) + .expect("seed sig2"); + + cache + .add_signature( + &deposit_id, + SignatureData { + signer_address: signer3, + signature: vec![3, 3, 3], + proposal_hash, + is_mine: false, + }, + None, + move |_, _| Some(signer3), + ) + .expect("seed sig3"); + + assert!(cache.is_ready(&deposit_id).expect("ready check")); + deposit_id +} + +fn seed_collecting_proposal_with_my_signature( + cache: &ProposalCache, + signer_address: Address, +) -> DepositId { + let proposal = sample_proposal(1); + let proposal_hash = proposal.compute_proposal_hash(); + let deposit_id = DepositId::from_effect_payload(&proposal); + + cache + .add_signature( + &deposit_id, + SignatureData { + signer_address, + signature: vec![9, 9, 9], + proposal_hash, + is_mine: true, + }, + Some(proposal), + move |_, _| Some(signer_address), + ) + .expect("seed collecting my sig"); + + assert!(!cache.is_ready(&deposit_id).expect("ready check")); + deposit_id +} + +fn non_proposer_node_id_for_height(height: u64) -> u64 { + let base = test_node_config(0); + let node_pkhs: Vec<_> = base + .nodes + .iter() + .map(|node| node.nock_pkh.clone()) + .collect(); + let proposer = hoon_proposer(height, &node_pkhs); + ((proposer + 1) % node_pkhs.len()) as u64 +} + +fn proposer_node_id_for_height(height: u64) -> u64 { + let base = test_node_config(0); + let node_pkhs: Vec<_> = base + .nodes + .iter() + .map(|node| node.nock_pkh.clone()) + .collect(); + hoon_proposer(height, &node_pkhs) as u64 +} + +async fn seed_deposit_log_from_proposal(log: &DepositLog, proposal: &NockDepositRequestData) { + let entry = DepositLogEntry { + block_height: proposal.block_height, + tx_id: proposal.tx_id.clone(), + as_of: proposal.as_of.clone(), + name: proposal.name.clone(), + recipient: proposal.recipient, + amount_to_mint: proposal.amount, + }; + let outcome = log + .insert_entry(&entry) + .await + .expect("seed deposit log entry"); + assert!(matches!( + outcome, + DepositLogInsertOutcome::Inserted | DepositLogInsertOutcome::ExistingMatch + )); +} + +#[tokio::test] +async fn posting_tick_not_my_turn_skips_submit_without_sleep() { + let proposal_cache = Arc::new(ProposalCache::new()); + let deposit_id = seed_ready_proposal(&proposal_cache, 1); + let base_contract = Arc::new(CountingBaseContract::with_last_nonce(0)); + let status_state = BridgeStatusState::new(); + let bridge_status = test_bridge_status(); + let node_config = test_node_config(non_proposer_node_id_for_height(100)); + + let context = PostingTickContext::new( + PostingTickPorts::new(proposal_cache.clone(), base_contract.clone()), + PostingTickNodeState::new(node_config), + PostingTickControl::new(bridge_status, status_state), + PostingTickConfig::new(300), + ); + let mut state = PostingTickState::default(); + + let outcome = posting_tick_once( + &context, + &mut state, + PostingTickInput { + now: SystemTime::now(), + }, + ) + .await; + + assert_eq!(outcome.ready_proposals, 1); + assert_eq!(outcome.submitted, 0); + assert_eq!(base_contract.submit_calls(), 0); + assert!(!proposal_cache.is_confirmed(&deposit_id)); +} + +#[tokio::test] +async fn posting_tick_failover_zero_submits_without_sleep() { + let proposal_cache = Arc::new(ProposalCache::new()); + let deposit_id = seed_ready_proposal(&proposal_cache, 1); + let base_contract = Arc::new(CountingBaseContract::with_last_nonce(0)); + let status_state = BridgeStatusState::new(); + let bridge_status = test_bridge_status(); + let node_config = test_node_config(non_proposer_node_id_for_height(100)); + + let context = PostingTickContext::new( + PostingTickPorts::new(proposal_cache.clone(), base_contract.clone()), + PostingTickNodeState::new(node_config), + PostingTickControl::new(bridge_status, status_state), + PostingTickConfig::new(0), + ); + let mut state = PostingTickState::default(); + + let outcome = posting_tick_once( + &context, + &mut state, + PostingTickInput { + now: SystemTime::now(), + }, + ) + .await; + + assert_eq!(outcome.ready_proposals, 1); + assert_eq!(outcome.submitted, 1); + assert_eq!(base_contract.submit_calls(), 1); + assert!(proposal_cache.is_confirmed(&deposit_id)); +} + +#[tokio::test] +async fn posting_tick_replay_submits_ready_proposals_in_nonce_order() { + let proposal_cache = Arc::new(ProposalCache::new()); + let deposit_id_1 = seed_ready_proposal(&proposal_cache, 1); + let deposit_id_2 = seed_ready_proposal(&proposal_cache, 2); + let base_contract = Arc::new(CountingBaseContract::with_last_nonce(0)); + base_contract.script_last_nonce_responses([Ok(0), Ok(1)]); + base_contract.script_submit_responses([ + Ok(DepositSubmissionResult { + tx_hash: "tx-1".to_string(), + block_number: 101, + }), + Ok(DepositSubmissionResult { + tx_hash: "tx-2".to_string(), + block_number: 102, + }), + ]); + let status_state = BridgeStatusState::new(); + let bridge_status = test_bridge_status(); + let node_config = test_node_config(proposer_node_id_for_height(100)); + + let context = PostingTickContext::new( + PostingTickPorts::new(proposal_cache.clone(), base_contract.clone()), + PostingTickNodeState::new(node_config), + PostingTickControl::new(bridge_status, status_state), + PostingTickConfig::new(300), + ); + let mut state = PostingTickState::default(); + let now = UNIX_EPOCH + Duration::from_secs(1_700_000_000); + + let tick1 = posting_tick_once(&context, &mut state, PostingTickInput { now }).await; + assert_eq!(tick1.ready_proposals, 2); + assert_eq!(tick1.submitted, 1); + assert_eq!(base_contract.get_last_nonce_calls(), 1); + assert_eq!(base_contract.submit_calls(), 1); + assert!(proposal_cache.is_confirmed(&deposit_id_1)); + assert!(!proposal_cache.is_confirmed(&deposit_id_2)); + + let tick2 = posting_tick_once( + &context, + &mut state, + PostingTickInput { + now: now + Duration::from_secs(1), + }, + ) + .await; + assert_eq!(tick2.ready_proposals, 1); + assert_eq!(tick2.submitted, 1); + assert_eq!(base_contract.get_last_nonce_calls(), 2); + assert_eq!(base_contract.submit_calls(), 2); + assert!(proposal_cache.is_confirmed(&deposit_id_2)); + + let tick3 = posting_tick_once( + &context, + &mut state, + PostingTickInput { + now: now + Duration::from_secs(2), + }, + ) + .await; + assert_eq!(tick3.ready_proposals, 0); + assert_eq!(tick3.submitted, 0); + assert_eq!( + base_contract.get_last_nonce_calls(), + 2, + "no chain nonce query should happen when no proposals are ready" + ); +} + +#[tokio::test] +async fn signing_tick_without_tip_skips_chain_nonce_query() { + let base_contract = Arc::new(CountingBaseContract::with_last_nonce(0)); + let deposit_log_dir = TempDir::new().expect("tempdir"); + let deposit_log_path = deposit_log_dir.path().join("deposit-log.sqlite"); + let deposit_log = Arc::new( + DepositLog::open(deposit_log_path) + .await + .expect("open deposit log"), + ); + let (runtime, runtime_handle) = BridgeRuntime::new(Arc::new(NoopCauseBuilder)); + let _runtime = runtime; + let runtime_handle = Arc::new(runtime_handle); + let nonce_epoch = NonceEpochConfig { + base: 0, + start_height: 1, + start_tx_id: None, + }; + let proposal_cache = Arc::new(ProposalCache::new()); + let signer = Arc::new( + BridgeSigner::new(format!("0x{}", TEST_PRIVATE_KEY)).expect("valid test signer key"), + ); + let bridge_status = test_bridge_status(); + let (stop_controller, stop) = StopController::new(); + + let context = SigningTickContext::new( + SigningTickPorts::new( + runtime_handle, + base_contract.clone(), + deposit_log, + proposal_cache, + ), + SigningTickNodeState::new(signer, HashSet::new(), Vec::new(), 0, HashMap::new()), + SigningTickControl::new(bridge_status, stop_controller, stop), + SigningTickConfig::new( + &nonce_epoch, + bridge::loop_policy::SigningLoopPolicy::default(), + ), + ); + let mut state = SigningTickState::new(SystemTime::now()); + + let outcome = signing_tick_once( + &context, + &mut state, + SigningTickInput { + now: SystemTime::now(), + tip_height: None, + }, + ) + .await; + + assert_eq!(outcome.regossip_broadcasts, 0); + assert_eq!(outcome.initial_broadcasts, 0); + assert_eq!(base_contract.get_last_nonce_calls(), 1); +} + +#[tokio::test] +async fn signing_tick_regossip_collecting_my_signature_with_tip() { + let base_contract = Arc::new(CountingBaseContract::with_last_nonce(0)); + let deposit_log_dir = TempDir::new().expect("tempdir"); + let deposit_log_path = deposit_log_dir.path().join("deposit-log.sqlite"); + let deposit_log = Arc::new( + DepositLog::open(deposit_log_path) + .await + .expect("open deposit log"), + ); + let (runtime, runtime_handle) = BridgeRuntime::new(Arc::new(NoopCauseBuilder)); + let _runtime = runtime; + let runtime_handle = Arc::new(runtime_handle); + let nonce_epoch = NonceEpochConfig { + base: 0, + start_height: 1, + start_tx_id: None, + }; + let proposal_cache = Arc::new(ProposalCache::new()); + let signer = Arc::new( + BridgeSigner::new(format!("0x{}", TEST_PRIVATE_KEY)).expect("valid test signer key"), + ); + let _deposit_id = seed_collecting_proposal_with_my_signature(&proposal_cache, signer.address()); + let bridge_status = test_bridge_status(); + let (stop_controller, stop) = StopController::new(); + + let context = SigningTickContext::new( + SigningTickPorts::new( + runtime_handle, + base_contract.clone(), + deposit_log, + proposal_cache, + ), + SigningTickNodeState::new(signer, HashSet::new(), Vec::new(), 0, HashMap::new()), + SigningTickControl::new(bridge_status, stop_controller, stop), + SigningTickConfig::new( + &nonce_epoch, + bridge::loop_policy::SigningLoopPolicy::default(), + ), + ); + let mut state = SigningTickState::default(); + + let outcome = signing_tick_once( + &context, + &mut state, + SigningTickInput { + now: SystemTime::now(), + tip_height: Some(1), + }, + ) + .await; + + assert_eq!(outcome.regossip_broadcasts, 1); + assert_eq!(outcome.initial_broadcasts, 0); + assert_eq!(base_contract.get_last_nonce_calls(), 1); +} + +#[tokio::test] +async fn signing_tick_replay_waits_then_signs_then_skips_already_signed() { + let base_contract = Arc::new(CountingBaseContract::with_last_nonce(0)); + base_contract.script_last_nonce_responses([Ok(0), Ok(0)]); + base_contract.script_processed_responses([Ok(false)]); + let deposit_log_dir = TempDir::new().expect("tempdir"); + let deposit_log_path = deposit_log_dir.path().join("deposit-log.sqlite"); + let deposit_log = Arc::new( + DepositLog::open(deposit_log_path) + .await + .expect("open deposit log"), + ); + let proposal = sample_proposal(1); + seed_deposit_log_from_proposal(&deposit_log, &proposal).await; + let deposit_id = DepositId::from_effect_payload(&proposal); + let (runtime, runtime_handle) = BridgeRuntime::new(Arc::new(NoopCauseBuilder)); + let _runtime = runtime; + let runtime_handle = Arc::new(runtime_handle); + let nonce_epoch = NonceEpochConfig { + base: 0, + start_height: 1, + start_tx_id: None, + }; + let proposal_cache = Arc::new(ProposalCache::new()); + let signer = Arc::new( + BridgeSigner::new(format!("0x{}", TEST_PRIVATE_KEY)).expect("valid test signer key"), + ); + let mut valid_addresses = HashSet::new(); + valid_addresses.insert(signer.address()); + let bridge_status = test_bridge_status(); + let (stop_controller, stop) = StopController::new(); + + let context = SigningTickContext::new( + SigningTickPorts::new( + runtime_handle, + base_contract.clone(), + deposit_log, + proposal_cache.clone(), + ), + SigningTickNodeState::new(signer, valid_addresses, Vec::new(), 0, HashMap::new()), + SigningTickControl::new(bridge_status, stop_controller, stop), + SigningTickConfig::new( + &nonce_epoch, + bridge::loop_policy::SigningLoopPolicy::default(), + ), + ); + let start = UNIX_EPOCH + Duration::from_secs(1_700_000_000); + let mut state = SigningTickState::new(start); + + let tick1 = signing_tick_once( + &context, + &mut state, + SigningTickInput { + now: start, + tip_height: None, + }, + ) + .await; + assert_eq!(tick1.regossip_broadcasts, 0); + assert_eq!(tick1.initial_broadcasts, 0); + assert_eq!(base_contract.get_last_nonce_calls(), 1); + assert_eq!(base_contract.is_processed_calls(), 0); + + let tick2 = signing_tick_once( + &context, + &mut state, + SigningTickInput { + now: start + Duration::from_secs(1), + tip_height: Some(1), + }, + ) + .await; + assert_eq!(tick2.regossip_broadcasts, 0); + assert_eq!(tick2.initial_broadcasts, 1); + assert_eq!(base_contract.get_last_nonce_calls(), 2); + assert_eq!(base_contract.is_processed_calls(), 1); + + let state_after_tick2 = proposal_cache + .get_state(&deposit_id) + .expect("proposal cache read") + .expect("proposal state should exist after signing"); + assert_eq!(state_after_tick2.status, ProposalStatus::Collecting); + assert!( + state_after_tick2.my_signature.is_some(), + "tick2 should persist our signature for replay determinism" + ); + + let tick3 = signing_tick_once( + &context, + &mut state, + SigningTickInput { + now: start + Duration::from_secs(2), + tip_height: Some(1), + }, + ) + .await; + assert_eq!(tick3.regossip_broadcasts, 0); + assert_eq!(tick3.initial_broadcasts, 0); + assert_eq!(base_contract.get_last_nonce_calls(), 3); + assert_eq!( + base_contract.is_processed_calls(), + 1, + "already-signed candidate should skip processedDeposits query on replay tick" + ); +} + +#[tokio::test] +async fn signing_tick_nonce_query_error_returns_without_actions() { + let base_contract = Arc::new(CountingBaseContract::with_nonce_query_error()); + let deposit_log_dir = TempDir::new().expect("tempdir"); + let deposit_log_path = deposit_log_dir.path().join("deposit-log.sqlite"); + let deposit_log = Arc::new( + DepositLog::open(deposit_log_path) + .await + .expect("open deposit log"), + ); + let (runtime, runtime_handle) = BridgeRuntime::new(Arc::new(NoopCauseBuilder)); + let _runtime = runtime; + let runtime_handle = Arc::new(runtime_handle); + let nonce_epoch = NonceEpochConfig { + base: 0, + start_height: 1, + start_tx_id: None, + }; + let proposal_cache = Arc::new(ProposalCache::new()); + let signer = Arc::new( + BridgeSigner::new(format!("0x{}", TEST_PRIVATE_KEY)).expect("valid test signer key"), + ); + let bridge_status = test_bridge_status(); + let (stop_controller, stop) = StopController::new(); + + let context = SigningTickContext::new( + SigningTickPorts::new( + runtime_handle, + base_contract.clone(), + deposit_log, + proposal_cache, + ), + SigningTickNodeState::new(signer, HashSet::new(), Vec::new(), 0, HashMap::new()), + SigningTickControl::new(bridge_status, stop_controller, stop), + SigningTickConfig::new( + &nonce_epoch, + bridge::loop_policy::SigningLoopPolicy::default(), + ), + ); + let mut state = SigningTickState::new(SystemTime::now()); + + let outcome = signing_tick_once( + &context, + &mut state, + SigningTickInput { + now: SystemTime::now(), + tip_height: Some(1), + }, + ) + .await; + + assert_eq!(outcome.regossip_broadcasts, 0); + assert_eq!(outcome.initial_broadcasts, 0); + assert_eq!(base_contract.get_last_nonce_calls(), 1); +} + +#[tokio::test] +async fn signing_tick_nonce_epoch_mismatch_triggers_local_stop_without_runtime_peek() { + let base_contract = Arc::new(CountingBaseContract::with_last_nonce(0)); + let deposit_log_dir = TempDir::new().expect("tempdir"); + let deposit_log_path = deposit_log_dir.path().join("deposit-log.sqlite"); + let deposit_log = Arc::new( + DepositLog::open(deposit_log_path) + .await + .expect("open deposit log"), + ); + let (runtime, runtime_handle) = BridgeRuntime::new(Arc::new(NoopCauseBuilder)); + let _runtime = runtime; + let runtime_handle = Arc::new(runtime_handle); + let nonce_epoch = NonceEpochConfig { + base: 1, + start_height: 1, + start_tx_id: None, + }; + let proposal_cache = Arc::new(ProposalCache::new()); + let signer = Arc::new( + BridgeSigner::new(format!("0x{}", TEST_PRIVATE_KEY)).expect("valid test signer key"), + ); + let bridge_status = test_bridge_status(); + let (stop_controller, stop) = StopController::new(); + let stop_probe = stop.clone(); + + let context = SigningTickContext::new( + SigningTickPorts::new( + runtime_handle, + base_contract.clone(), + deposit_log, + proposal_cache, + ), + SigningTickNodeState::new(signer, HashSet::new(), Vec::new(), 0, HashMap::new()), + SigningTickControl::new(bridge_status, stop_controller, stop) + .with_local_stop_mode(SigningLocalStopMode::LocalTriggerOnly), + SigningTickConfig::new( + &nonce_epoch, + bridge::loop_policy::SigningLoopPolicy::default(), + ), + ); + let mut state = SigningTickState::new(SystemTime::now()); + + let outcome = signing_tick_once( + &context, + &mut state, + SigningTickInput { + now: SystemTime::now(), + tip_height: Some(1), + }, + ) + .await; + + assert_eq!(outcome.regossip_broadcasts, 0); + assert_eq!(outcome.initial_broadcasts, 0); + assert_eq!(base_contract.get_last_nonce_calls(), 1); + assert!(stop_probe.is_stopped()); +} + +#[tokio::test] +async fn signing_tick_sqlite_count_error_triggers_local_stop_without_runtime_peek() { + let base_contract = Arc::new(CountingBaseContract::with_last_nonce(0)); + let deposit_log_dir = TempDir::new().expect("tempdir"); + let deposit_log_path = deposit_log_dir.path().join("deposit-log.sqlite"); + let deposit_log = Arc::new( + DepositLog::open(deposit_log_path) + .await + .expect("open deposit log"), + ); + let (runtime, runtime_handle) = BridgeRuntime::new(Arc::new(NoopCauseBuilder)); + let _runtime = runtime; + let runtime_handle = Arc::new(runtime_handle); + let nonce_epoch = NonceEpochConfig { + base: 0, + start_height: u64::MAX, + start_tx_id: None, + }; + let proposal_cache = Arc::new(ProposalCache::new()); + let signer = Arc::new( + BridgeSigner::new(format!("0x{}", TEST_PRIVATE_KEY)).expect("valid test signer key"), + ); + let bridge_status = test_bridge_status(); + let (stop_controller, stop) = StopController::new(); + let stop_probe = stop.clone(); + + let context = SigningTickContext::new( + SigningTickPorts::new( + runtime_handle, + base_contract.clone(), + deposit_log, + proposal_cache, + ), + SigningTickNodeState::new(signer, HashSet::new(), Vec::new(), 0, HashMap::new()), + SigningTickControl::new(bridge_status, stop_controller, stop) + .with_local_stop_mode(SigningLocalStopMode::LocalTriggerOnly), + SigningTickConfig::new( + &nonce_epoch, + bridge::loop_policy::SigningLoopPolicy::default(), + ), + ); + let mut state = SigningTickState::new(SystemTime::now()); + + let outcome = signing_tick_once( + &context, + &mut state, + SigningTickInput { + now: SystemTime::now(), + tip_height: Some(u64::MAX), + }, + ) + .await; + + assert_eq!(outcome.regossip_broadcasts, 0); + assert_eq!(outcome.initial_broadcasts, 0); + assert_eq!(base_contract.get_last_nonce_calls(), 1); + assert!(stop_probe.is_stopped()); +} diff --git a/crates/chaff/benches/jam_checkpoint.rs b/crates/chaff/benches/jam_checkpoint.rs index e30612bee..cf73f7a54 100644 --- a/crates/chaff/benches/jam_checkpoint.rs +++ b/crates/chaff/benches/jam_checkpoint.rs @@ -12,6 +12,7 @@ use nockapp::noun::slab::{Jammer, NockJammer, NounSlab}; const FALLBACK_CHECKPOINT: &str = concat!(env!("CARGO_MANIFEST_DIR"), "/test-jams/0.chkjam"); +const SNAPSHOT_VERSION_1: u32 = 1; const SNAPSHOT_VERSION_2: u32 = 2; #[derive(Decode)] @@ -20,7 +21,6 @@ struct CheckpointEnvelope { version: u32, payload: Vec, } - fn checkpoint_bytes() -> &'static [u8] { static SAMPLE: OnceLock> = OnceLock::new(); SAMPLE @@ -56,7 +56,7 @@ fn extract_jammed_state(bytes: &[u8]) -> Bytes { if let Ok((checkpoint, _)) = bincode::decode_from_slice::(bytes, config) { - if checkpoint.magic_bytes == JAM_MAGIC_BYTES { + if checkpoint.magic_bytes == JAM_MAGIC_BYTES && checkpoint.version == SNAPSHOT_VERSION_1 { return checkpoint.jam.0; } } diff --git a/crates/chaff/src/bin/large_checkpoint_bench.rs b/crates/chaff/src/bin/large_checkpoint_bench.rs index 18f801894..e91a20d0e 100644 --- a/crates/chaff/src/bin/large_checkpoint_bench.rs +++ b/crates/chaff/src/bin/large_checkpoint_bench.rs @@ -10,8 +10,9 @@ use bincode::Decode; use bytes::Bytes; use chaff::Chaff; use nockapp::nockapp::save::{JammedCheckpointV1, JammedCheckpointV2, JAM_MAGIC_BYTES}; -use nockapp::noun::slab::{slab_noun_equality, NockJammer, NounSlab}; +use nockapp::noun::slab::{slab_equality, NockJammer, NounSlab}; +const SNAPSHOT_VERSION_1: u32 = 1; const SNAPSHOT_VERSION_2: u32 = 2; #[derive(Decode)] @@ -24,7 +25,6 @@ struct CheckpointEnvelope { fn extract_jammed_state(bytes: &[u8]) -> Bytes { let config = config::standard(); - // Try to decode as envelope format (V2) if let Ok((envelope, _)) = bincode::decode_from_slice::(bytes, config) { @@ -37,11 +37,10 @@ fn extract_jammed_state(bytes: &[u8]) -> Bytes { } } - // Try to decode as V1 (non-envelope format) if let Ok((checkpoint, _)) = bincode::decode_from_slice::(bytes, config) { - if checkpoint.magic_bytes == JAM_MAGIC_BYTES { + if checkpoint.magic_bytes == JAM_MAGIC_BYTES && checkpoint.version == SNAPSHOT_VERSION_1 { return checkpoint.jam.0; } } @@ -50,7 +49,6 @@ fn extract_jammed_state(bytes: &[u8]) -> Bytes { } fn main() { - // Get checkpoint path from args or use default let checkpoint_path = std::env::args() .nth(1) .map(PathBuf::from) @@ -80,7 +78,6 @@ fn main() { extract_time.as_secs_f64() ); - // Drop original bytes to free memory drop(checkpoint_bytes); jammed_state } else { @@ -102,7 +99,6 @@ fn main() { println!("\n=== CUE BENCHMARKS ===\n"); - // Benchmark NockJammer cue println!("Cueing with NockJammer (bitvec)..."); let cue_start = Instant::now(); let mut nock_slab = NounSlab::::new(); @@ -117,7 +113,6 @@ fn main() { (jammed_state.len() as f64 / (1024.0 * 1024.0)) / nock_cue_time.as_secs_f64() ); - // Benchmark Chaff cue println!("\nCueing with Chaff (BitReader)..."); let cue_start = Instant::now(); let mut chaff_slab = NounSlab::::new(); @@ -135,16 +130,11 @@ fn main() { let cue_speedup = nock_cue_time.as_secs_f64() / chaff_cue_time.as_secs_f64(); println!("\nCue speedup: {:.2}x", cue_speedup); - // ======================================== - // COMPREHENSIVE VERIFICATION - Part 1: Noun Equality - // (Must do this BEFORE jam benchmarks consume the slabs) - // ======================================== println!("\n=== COMPREHENSIVE VERIFICATION ===\n"); - println!("--- Part 1: Noun Equality (Cue Verification) ---\n"); println!("Comparing cued nouns using slab_noun_equality..."); let noun_eq_start = Instant::now(); - let nouns_equal = slab_noun_equality(unsafe { nock_slab.root() }, unsafe { chaff_slab.root() }); + let nouns_equal = slab_equality(&nock_slab, &chaff_slab); let noun_eq_time = noun_eq_start.elapsed(); if nouns_equal { println!( @@ -155,18 +145,14 @@ fn main() { println!("✗ Cued nouns are NOT equal!"); } - // Clone slabs for the 4x4 jam matrix (coerce_jammer consumes the slab) let nock_slab_for_nock_jam = nock_slab.clone(); let nock_slab_for_chaff_jam = nock_slab.clone(); let chaff_slab_for_nock_jam = chaff_slab.clone(); let chaff_slab_for_chaff_jam = chaff_slab.clone(); - - // Keep one original for round-trip verification let nock_slab_original = nock_slab; println!("\n=== JAM BENCHMARKS ===\n"); - // Benchmark NockJammer jam (using nock-cued slab) println!("Jamming with NockJammer (bitvec)..."); let jam_start = Instant::now(); let nock_nock_jammed = nock_slab_for_nock_jam.coerce_jammer::().jam(); @@ -177,7 +163,6 @@ fn main() { (nock_nock_jammed.len() as f64 / (1024.0 * 1024.0)) / nock_jam_time.as_secs_f64() ); - // Benchmark Chaff jam (using chaff-cued slab) println!("\nJamming with Chaff (BitWriter)..."); let jam_start = Instant::now(); let chaff_chaff_jammed = chaff_slab_for_chaff_jam.coerce_jammer::().jam(); @@ -191,13 +176,7 @@ fn main() { let jam_speedup = nock_jam_time.as_secs_f64() / chaff_jam_time.as_secs_f64(); println!("\nJam speedup: {:.2}x", jam_speedup); - // --- Part 2: 4x4 Jam Matrix (byte equality verification) --- println!("\n--- Part 2: 4x4 Jam Matrix (Byte Equality) ---\n"); - - // Compute the remaining cross-jams: - // - nock_chaff_jammed: nock-cued slab -> jammed with Chaff - // - chaff_nock_jammed: chaff-cued slab -> jammed with NockJammer - println!("Computing cross-jams for 4x4 matrix..."); println!(" [NockJammer cue -> Chaff jam]..."); @@ -236,7 +215,6 @@ fn main() { chaff_chaff_jammed.len() ); - // Helper to check byte equality fn check_byte_eq(name: &str, a: &[u8], b: &[u8]) -> bool { if a == b { println!(" ✓ {}: exact match ({} bytes)", name, a.len()); @@ -250,7 +228,6 @@ fn main() { ); false } else { - // Find first difference for (i, (x, y)) in a.iter().zip(b.iter()).enumerate() { if x != y { println!( @@ -266,7 +243,6 @@ fn main() { println!("\n Pairwise comparisons (6 pairs from 4 outputs):"); - // All 6 pairwise comparisons (C(4,2) = 6) let mut all_match = true; all_match &= check_byte_eq("NN vs NC", &nock_nock_jammed, &nock_chaff_jammed); all_match &= check_byte_eq("NN vs CN", &nock_nock_jammed, &chaff_nock_jammed); @@ -275,10 +251,8 @@ fn main() { all_match &= check_byte_eq("NC vs CC", &nock_chaff_jammed, &chaff_chaff_jammed); all_match &= check_byte_eq("CN vs CC", &chaff_nock_jammed, &chaff_chaff_jammed); - // --- Part 3: Cross-verification (re-cue jammed outputs) --- println!("\n--- Part 3: Round-trip Verification ---\n"); - // Re-cue the chaff-jammed output with NockJammer and compare println!("Re-cueing Chaff jam output (CC) with NockJammer..."); let recue_start = Instant::now(); let mut recue_slab = NounSlab::::new(); @@ -288,12 +262,9 @@ fn main() { recue_slab.set_root(recue_noun); println!(" Done in {:.2}s", recue_start.elapsed().as_secs_f64()); - // Compare re-cued noun with original nock-cued noun println!("Comparing re-cued noun with original NockJammer-cued noun..."); let roundtrip_eq_start = Instant::now(); - let roundtrip_equal = slab_noun_equality(unsafe { nock_slab_original.root() }, unsafe { - recue_slab.root() - }); + let roundtrip_equal = slab_equality(&nock_slab_original, &recue_slab); let roundtrip_eq_time = roundtrip_eq_start.elapsed(); if roundtrip_equal { println!( @@ -305,7 +276,6 @@ fn main() { all_match = false; } - // Also re-cue with Chaff to verify Chaff cue of Chaff jam println!("\nRe-cueing Chaff jam output (CC) with Chaff..."); let recue_start = Instant::now(); let mut recue_chaff_slab = NounSlab::::new(); @@ -317,9 +287,7 @@ fn main() { println!("Comparing Chaff re-cued noun with original..."); let roundtrip_chaff_eq_start = Instant::now(); - let roundtrip_chaff_equal = slab_noun_equality(unsafe { nock_slab_original.root() }, unsafe { - recue_chaff_slab.root() - }); + let roundtrip_chaff_equal = slab_equality(&nock_slab_original, &recue_chaff_slab); let roundtrip_chaff_eq_time = roundtrip_chaff_eq_start.elapsed(); if roundtrip_chaff_equal { println!( @@ -331,7 +299,6 @@ fn main() { all_match = false; } - // --- Final Summary --- println!("\n--- Verification Summary ---\n"); if nouns_equal && all_match { println!("✓ ALL VERIFICATIONS PASSED"); diff --git a/crates/chaff/src/jammer.rs b/crates/chaff/src/jammer.rs new file mode 100644 index 000000000..4981c2a8c --- /dev/null +++ b/crates/chaff/src/jammer.rs @@ -0,0 +1,570 @@ +use std::fmt; + +use bytes::Bytes; +use either::Either; +use habit::{BitReader, BitWriter}; +use intmap::IntMap; +use nockapp::noun::slab::{CueError as SlabCueError, Jammer as SlabJammer, NounSlab}; +use nockvm::ext::{noun_equality, AtomExt}; +use nockvm::mug::{calc_atom_mug_u32, calc_cell_mug_u32, get_mug, set_mug}; +use nockvm::noun::{Atom, Cell, CellMemory, DirectAtom, Noun, NounAllocator, NounSpace, D}; +use nockvm::serialization::{met0_u64_to_usize, met0_usize}; + +pub struct Chaff; + +const MAX_USIZE_BITS: usize = usize::BITS as usize; + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum CueError { + BadBackref, + BackrefTooBig, + TruncatedBuffer, +} + +impl fmt::Display for CueError { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + CueError::BadBackref => f.write_str("cue: Bad backref"), + CueError::BackrefTooBig => f.write_str("cue: backref too big"), + CueError::TruncatedBuffer => f.write_str("cue: truncated buffer"), + } + } +} + +impl std::error::Error for CueError {} + +struct NounMap(IntMap>); + +impl NounMap { + fn new() -> Self { + NounMap(IntMap::new()) + } + + fn insert(&mut self, key: Noun, value: V, space: &NounSpace) { + let key_mug = noun_mug(key, space) as u64; + if let Some(vec) = self.0.get_mut(key_mug) { + let mut chain_iter = vec[..].iter_mut(); + if let Some(entry) = + chain_iter.find(|entry| noun_equality(key.in_space(space), entry.0.in_space(space))) + { + entry.1 = value; + } else { + vec.push((key, value)); + } + } else { + self.0.insert(key_mug, vec![(key, value)]); + } + } + + fn get(&self, key: Noun, space: &NounSpace) -> Option<&V> { + let key_mug = noun_mug(key, space) as u64; + self.0.get(key_mug).and_then(|vec| { + vec.iter() + .find(|entry| noun_equality(key.in_space(space), entry.0.in_space(space))) + .map(|entry| &entry.1) + }) + } +} + +fn noun_mug(a: Noun, space: &NounSpace) -> u32 { + let mut stack = vec![a]; + while let Some(noun) = stack.pop() { + if let Ok(mut allocated) = noun.as_allocated() { + if get_mug(noun, space).is_none() { + match allocated.as_either() { + Either::Left(indirect) => unsafe { + set_mug( + &mut allocated, + calc_atom_mug_u32(indirect.as_atom(), space), + space, + ); + }, + Either::Right(cell) => match ( + get_mug(cell.in_space(space).head().noun(), space), + get_mug(cell.in_space(space).tail().noun(), space), + ) { + (Some(head_mug), Some(tail_mug)) => unsafe { + set_mug( + &mut allocated, + calc_cell_mug_u32(head_mug, tail_mug, space), + space, + ); + }, + _ => { + stack.push(noun); + stack.push(cell.in_space(space).tail().noun()); + stack.push(cell.in_space(space).head().noun()); + } + }, + } + } + } + } + get_mug(a, space).expect("Noun should have a mug once mugged.") +} + +impl Chaff { + pub fn cue_into(allocator: &mut A, bytes: Bytes) -> Result { + fn rub_backref(reader: &mut BitReader) -> Result { + let zeros = reader.read_unary().ok_or(CueError::TruncatedBuffer)?; + if zeros == 0 { + return Ok(0); + } + if zeros > MAX_USIZE_BITS { + return Err(CueError::BackrefTooBig); + } + let size_low = if zeros > 1 { + reader + .read_bits_to_usize(zeros - 1) + .ok_or(CueError::TruncatedBuffer)? + } else { + 0 + }; + let bit_count = (1usize << (zeros - 1)) | size_low; + if bit_count > MAX_USIZE_BITS { + return Err(CueError::BackrefTooBig); + } + reader + .read_bits_to_usize(bit_count) + .ok_or(CueError::TruncatedBuffer) + } + + fn rub_atom( + allocator: &mut A, + reader: &mut BitReader, + ) -> Result { + let zeros = reader.read_unary().ok_or(CueError::TruncatedBuffer)?; + if zeros == 0 { + return unsafe { Ok(DirectAtom::new_unchecked(0).as_atom()) }; + } + if zeros > MAX_USIZE_BITS { + return Err(CueError::TruncatedBuffer); + } + let size_low = if zeros > 1 { + reader + .read_bits_to_usize(zeros - 1) + .ok_or(CueError::TruncatedBuffer)? + } else { + 0 + }; + let bit_count = (1usize << (zeros - 1)) | size_low; + if bit_count < 64 { + let value = reader + .read_bits_to_usize(bit_count) + .ok_or(CueError::TruncatedBuffer)? as u64; + unsafe { Ok(DirectAtom::new_unchecked(value).as_atom()) } + } else { + let byte_len = (bit_count + 7) >> 3; + let mut buffer = vec![0u8; byte_len]; + reader + .read_bits_to_bytes(&mut buffer, bit_count) + .ok_or(CueError::TruncatedBuffer)?; + Ok(::from_bytes(allocator, &buffer)) + } + } + + enum CueStackEntry { + DestinationPointer(*mut Noun), + BackRef(u64, *const Noun), + } + + let mut reader = BitReader::new(bytes); + let mut result = D(0); + let mut stack = vec![CueStackEntry::DestinationPointer(&mut result)]; + let mut backrefs: IntMap = IntMap::new(); + + while let Some(entry) = stack.pop() { + match entry { + CueStackEntry::DestinationPointer(dest) => { + let backref_pos = reader.position() as u64; + let tag = reader.read_bit().ok_or(CueError::TruncatedBuffer)?; + if !tag { + let atom = rub_atom(allocator, &mut reader)?; + unsafe { + *dest = atom.as_noun(); + } + backrefs.insert(backref_pos, unsafe { *dest }); + } else { + let second = reader.read_bit().ok_or(CueError::TruncatedBuffer)?; + if second { + let backref = rub_backref(&mut reader)? as u64; + let noun = + backrefs.get(backref).copied().ok_or(CueError::BadBackref)?; + unsafe { + *dest = noun; + } + } else { + let (cell, cell_mem): (Cell, *mut CellMemory) = + unsafe { Cell::new_raw_mut(allocator) }; + unsafe { + *dest = cell.as_noun(); + } + stack.push(CueStackEntry::BackRef(backref_pos, dest as *const Noun)); + unsafe { + stack + .push(CueStackEntry::DestinationPointer(&mut (*cell_mem).tail)); + stack + .push(CueStackEntry::DestinationPointer(&mut (*cell_mem).head)); + } + } + } + } + CueStackEntry::BackRef(pos, noun_ptr) => { + backrefs.insert(pos, unsafe { *noun_ptr }); + } + } + } + + Ok(result) + } + + pub fn jam(noun: Noun, space: &NounSpace) -> Bytes { + fn mat_backref_fast(writer: &mut BitWriter, backref: usize) { + if backref == 0 { + writer.write_bits_from_value(0b111, 3); + return; + } + let backref_sz = met0_u64_to_usize(backref as u64); + let backref_sz_sz = met0_u64_to_usize(backref_sz as u64); + writer.write_bit(true); + writer.write_bit(true); + writer.write_zeros(backref_sz_sz); + writer.write_bit(true); + writer.write_bits_from_value(backref_sz, backref_sz_sz - 1); + writer.write_bits_from_value(backref, backref_sz); + } + + fn mat_atom_fast(writer: &mut BitWriter, atom: Atom, space: &NounSpace) { + unsafe { + if atom.as_noun().raw_equals(&D(0)) { + writer.write_bits_from_value(0b10, 2); + return; + } + } + let atom_sz = met0_usize(atom, space); + let atom_sz_sz = met0_u64_to_usize(atom_sz as u64); + writer.write_bit(false); + writer.write_zeros(atom_sz_sz); + writer.write_bit(true); + writer.write_bits_from_value(atom_sz, atom_sz_sz - 1); + writer.write_bits_from_le_bytes(atom.in_space(space).as_ne_bytes(), atom_sz); + } + + let mut writer = BitWriter::new(); + let mut backref_map = NounMap::::new(); + let mut stack = vec![noun]; + while let Some(noun) = stack.pop() { + if let Some(backref) = backref_map.get(noun, space) { + if let Ok(atom) = noun.in_space(space).as_atom() { + let atom = atom.atom(); + if met0_u64_to_usize(*backref as u64) < met0_usize(atom, space) { + mat_backref_fast(&mut writer, *backref); + } else { + mat_atom_fast(&mut writer, atom, space); + } + } else { + mat_backref_fast(&mut writer, *backref); + } + } else { + backref_map.insert(noun, writer.bit_len(), space); + match noun.in_space(space).as_either_atom_cell() { + Either::Left(atom) => { + mat_atom_fast(&mut writer, atom.atom(), space); + } + Either::Right(cell) => { + writer.write_bit(true); + writer.write_bit(false); + stack.push(cell.tail().noun()); + stack.push(cell.head().noun()); + } + } + } + } + + writer.into_bytes() + } +} + +impl SlabJammer for Chaff { + fn jam(noun: Noun, space: &NounSpace) -> Bytes { + Chaff::jam(noun, space) + } + + fn cue(slab: &mut NounSlab, bytes: Bytes) -> Result { + let noun = Chaff::cue_into(slab, bytes).map_err(|err| match err { + CueError::BadBackref => SlabCueError::BadBackref, + CueError::BackrefTooBig => SlabCueError::BackrefTooBig, + CueError::TruncatedBuffer => SlabCueError::TruncatedBuffer, + })?; + slab.set_root(noun); + Ok(noun) + } +} + +#[cfg(test)] +mod tests { + use bytes::Bytes; + use habit::BitWriter; + use nockvm::ext::AtomExt; + use nockvm::mem::{NockStack, NOCK_STACK_SIZE_TINY}; + use nockvm::noun::{Atom, Cell, Noun, D, T}; + use quickcheck::{Arbitrary, Gen, TestResult}; + + use crate::{Chaff, CueError}; + + const TOP_SLOTS: usize = 1 << 10; + + fn fresh_stack() -> NockStack { + NockStack::new(NOCK_STACK_SIZE_TINY, TOP_SLOTS) + } + + fn atom_from_bytes(stack: &mut NockStack, bytes: &[u8]) -> Atom { + ::from_bytes(stack, bytes) + } + + fn build_shared_noun(stack: &mut NockStack) -> Noun { + let shared = D(42); + let cell = Cell::new(stack, shared, shared).as_noun(); + T(stack, &[shared, cell, shared]) + } + + fn build_list(stack: &mut NockStack, leaves: &[u16]) -> Noun { + let mut list = D(0); + for value in leaves.iter().rev() { + list = Cell::new(stack, D(*value as u64), list).as_noun(); + } + list + } + + fn roundtrip(noun: Noun, stack: &mut NockStack) { + let jammed = Chaff::jam(noun, &stack.noun_space()); + let mut cue_stack = fresh_stack(); + let cued = Chaff::cue_into(&mut cue_stack, jammed.clone()).expect("cue should succeed"); + let rejam = Chaff::jam(cued, &cue_stack.noun_space()); + assert_eq!(jammed, rejam); + } + + #[test] + fn chaff_roundtrip_direct_atom() { + let mut stack = fresh_stack(); + roundtrip(D(0), &mut stack); + } + + #[test] + fn chaff_roundtrip_indirect_atom() { + let mut stack = fresh_stack(); + let bytes = vec![0xFFu8; 32]; + let atom = atom_from_bytes(&mut stack, &bytes); + roundtrip(atom.as_noun(), &mut stack); + } + + #[test] + fn chaff_roundtrip_nested_cells() { + let mut stack = fresh_stack(); + let noun = T(&mut stack, &[D(1), D(2), D(3), D(4)]); + roundtrip(noun, &mut stack); + } + + #[test] + fn chaff_handles_shared_backrefs() { + let mut stack = fresh_stack(); + let noun = build_shared_noun(&mut stack); + roundtrip(noun, &mut stack); + } + + #[test] + fn chaff_rejects_truncated_input() { + let mut stack = fresh_stack(); + let jammed = Bytes::from_static(&[0b1]); + let result = Chaff::cue_into(&mut stack, jammed); + assert!(matches!(result, Err(CueError::TruncatedBuffer))); + } + + #[test] + fn chaff_rejects_bad_backref() { + let mut stack = fresh_stack(); + let jammed = Bytes::from_static(&[0b0000_0111]); + let result = Chaff::cue_into(&mut stack, jammed); + assert!(matches!(result, Err(CueError::BadBackref))); + } + + #[test] + fn chaff_rejects_backref_before_definition() { + let mut stack = fresh_stack(); + let mut writer = BitWriter::new(); + writer.write_bit(true); + writer.write_bit(true); + writer.write_zeros(1); + writer.write_bit(true); + writer.write_bit(false); + writer.write_bit(true); + let jammed = writer.into_bytes(); + let result = Chaff::cue_into(&mut stack, jammed); + assert!(matches!(result, Err(CueError::BadBackref))); + } + + #[test] + fn chaff_rejects_backref_far_ahead() { + let mut stack = fresh_stack(); + let mut writer = BitWriter::new(); + writer.write_bit(true); + writer.write_bit(true); + writer.write_zeros(3); + writer.write_bit(true); + writer.write_bits_from_value(0b11, 2); + writer.write_bits_from_value(0b1010, 4); + let jammed = writer.into_bytes(); + let result = Chaff::cue_into(&mut stack, jammed); + assert!(matches!(result, Err(CueError::BadBackref))); + } + + #[test] + fn chaff_rejects_self_referential_backref() { + let mut stack = fresh_stack(); + let mut writer = BitWriter::new(); + writer.write_bit(true); + writer.write_bit(true); + writer.write_zeros(1); + writer.write_bit(true); + writer.write_bit(false); + writer.write_bit(false); + let jammed = writer.into_bytes(); + let result = Chaff::cue_into(&mut stack, jammed); + assert!(matches!(result, Err(CueError::BadBackref))); + } + + #[test] + fn chaff_rejects_backref_too_big() { + let mut stack = fresh_stack(); + let mut writer = BitWriter::new(); + writer.write_bit(true); + writer.write_bit(true); + writer.write_zeros(usize::BITS as usize + 1); + writer.write_bit(true); + let jammed = writer.into_bytes(); + let result = Chaff::cue_into(&mut stack, jammed); + assert!(matches!(result, Err(CueError::BackrefTooBig))); + } + + #[test] + fn chaff_rejects_truncated_backref_size() { + let mut stack = fresh_stack(); + let mut writer = BitWriter::new(); + writer.write_bit(true); + writer.write_bit(true); + writer.write_zeros(2); + writer.write_bit(true); + writer.write_bits_from_value(0b01, 2); + let jammed = writer.into_bytes(); + let result = Chaff::cue_into(&mut stack, jammed); + assert!(matches!(result, Err(CueError::TruncatedBuffer))); + } + + #[test] + fn chaff_rejects_atom_missing_size_bits() { + let mut stack = fresh_stack(); + let mut writer = BitWriter::new(); + writer.write_bit(false); + writer.write_bit(false); + writer.write_bit(false); + let jammed = writer.into_bytes(); + assert!(matches!( + Chaff::cue_into(&mut stack, jammed), + Err(CueError::TruncatedBuffer) + )); + } + + #[test] + fn chaff_rejects_truncated_indirect_atom() { + let mut stack = fresh_stack(); + let mut writer = BitWriter::new(); + writer.write_bit(false); + writer.write_zeros(6); + writer.write_bit(true); + writer.write_bits_from_value(0b100000, 6); + writer.write_bits_from_value(0xFFFF, 16); + let jammed = writer.into_bytes(); + let result = Chaff::cue_into(&mut stack, jammed); + assert!(matches!(result, Err(CueError::TruncatedBuffer))); + } + + #[test] + fn chaff_rejects_zero_atom_bad_encoding() { + let mut stack = fresh_stack(); + let mut writer = BitWriter::new(); + writer.write_bit(false); + writer.write_bit(false); + writer.write_bit(false); + let jammed = writer.into_bytes(); + let result = Chaff::cue_into(&mut stack, jammed); + assert!(matches!(result, Err(CueError::TruncatedBuffer))); + } + + #[derive(Clone, Debug)] + struct SmallNoun { + leaves: Vec, + } + + impl Arbitrary for SmallNoun { + fn arbitrary(g: &mut Gen) -> Self { + let len = 1 + (usize::arbitrary(g) % 8); + let mut leaves = Vec::with_capacity(len); + for _ in 0..len { + leaves.push(u16::arbitrary(g)); + } + Self { leaves } + } + } + + #[test] + fn chaff_roundtrip_list_fixture() { + let mut stack = fresh_stack(); + let noun = build_list(&mut stack, &[1, 2, 3, 4, 5]); + roundtrip(noun, &mut stack); + } + + #[test] + fn chaff_roundtrip_larger_atom_fixture() { + let mut stack = fresh_stack(); + let bytes = (0u8..64).collect::>(); + let atom = atom_from_bytes(&mut stack, &bytes); + let noun = T(&mut stack, &[D(7), atom.as_noun(), D(9)]); + roundtrip(noun, &mut stack); + } + + quickcheck::quickcheck! { + fn prop_chaff_roundtrip(small: SmallNoun) -> TestResult { + let mut stack = fresh_stack(); + let noun = build_list(&mut stack, &small.leaves); + let jammed = Chaff::jam(noun, &stack.noun_space()); + let mut cue_stack = fresh_stack(); + let cued = match Chaff::cue_into(&mut cue_stack, jammed.clone()) { + Ok(noun) => noun, + Err(_) => return TestResult::failed(), + }; + let rejam = Chaff::jam(cued, &cue_stack.noun_space()); + TestResult::from_bool(jammed == rejam) + } + } + + quickcheck::quickcheck! { + fn prop_chaff_handles_small_atoms(values: Vec) -> TestResult { + let mut values = values; + values.truncate(8); + let mut stack = fresh_stack(); + let mut list = D(0); + for value in values.iter().rev() { + let bounded = value & (nockvm::noun::DIRECT_MAX >> 1); + list = Cell::new(&mut stack, D(bounded), list).as_noun(); + } + let jammed = Chaff::jam(list, &stack.noun_space()); + let mut cue_stack = fresh_stack(); + let cued = match Chaff::cue_into(&mut cue_stack, jammed.clone()) { + Ok(noun) => noun, + Err(_) => return TestResult::failed(), + }; + let rejam = Chaff::jam(cued, &cue_stack.noun_space()); + TestResult::from_bool(jammed == rejam) + } + } +} diff --git a/crates/chaff/src/legacy_tests.rs b/crates/chaff/src/legacy_tests.rs new file mode 100644 index 000000000..f991dc10d --- /dev/null +++ b/crates/chaff/src/legacy_tests.rs @@ -0,0 +1,1859 @@ +#[cfg(test)] +mod tests { + use std::sync::OnceLock; + + use bincode::config::{self, Configuration}; + use bincode::Decode; + use blake3::hash; + use bytes::Bytes; + use habit::BitWriter; + use nockapp::nockapp::save::{ + CheckpointBootstrapReader, JammedCheckpointV0, JammedCheckpointV1, JammedCheckpointV2, + SaveableCheckpoint, JAM_MAGIC_BYTES, + }; + use nockapp::noun::slab::{slab_equality, CueError, Jammer, NockJammer, NounSlab}; + use nockapp::noun::NounAllocatorExt; + use nockapp::utils::NOCK_STACK_SIZE_TINY; + use nockapp::JammedNoun; + use nockvm::ext::{AtomExt, NounExt}; + use nockvm::mem::NockStack; + use nockvm::noun::{Atom, Cell, Noun, NounAllocator, NounSpace, D, T}; + use quickcheck::{Arbitrary, Gen, TestResult}; + use tempfile::TempDir; + + use super::Chaff; + + const NOCKVM_STACK_WORDS_MIN: usize = NOCK_STACK_SIZE_TINY; + const NOCKVM_STACK_WORDS_JAM_SCALE: usize = 8; + const NOCKVM_STACK_WORDS_MASS_SCALE: usize = 4; + + fn atom_from_bytes(slab: &mut NounSlab, bytes: &[u8]) -> Atom { + ::from_bytes(slab, bytes) + } + + fn atom_with_exact_bit_size(slab: &mut NounSlab, bits: usize) -> Atom { + assert!(bits > 0, "atom bit size must be positive"); + let byte_len = (bits + 7) >> 3; + let mut bytes = vec![0u8; byte_len]; + for (idx, byte) in bytes + .iter_mut() + .enumerate() + .take(byte_len.saturating_sub(1)) + { + *byte = ((idx as u8).wrapping_mul(37)).wrapping_add(0x5A); + } + let high_bit = (bits - 1) & 7; + bytes[byte_len - 1] |= 1u8 << high_bit; + atom_from_bytes(slab, &bytes) + } + + fn build_backref_threshold_fixture( + slab: &mut NounSlab, + prefix_len: usize, + repeated: Atom, + ) -> Noun { + let mut list = D(0); + list = Cell::new(slab, repeated.as_noun(), list).as_noun(); + list = Cell::new(slab, repeated.as_noun(), list).as_noun(); + for idx in (0..prefix_len).rev() { + list = Cell::new(slab, D(10_000 + idx as u64), list).as_noun(); + } + list + } + + fn build_shared_noun(slab: &mut NounSlab) -> Noun { + let shared = D(42); + let cell = Cell::new(slab, shared, shared).as_noun(); + T(slab, &[shared, cell, shared]) + } + + fn nouns_equal(a: Noun, a_space: &NounSpace, b: Noun, b_space: &NounSpace) -> bool { + let mut left: NounSlab = NounSlab::new(); + let left_root = left.copy_into(a, a_space); + left.set_root(left_root); + + let mut right: NounSlab = NounSlab::new(); + let right_root = right.copy_into(b, b_space); + right.set_root(right_root); + + slab_equality(&left, &right) + } + + fn atom_bytes(atom: Atom, space: &NounSpace) -> Bytes { + Bytes::copy_from_slice(atom.in_space(space).as_ne_bytes()) + } + + #[test] + fn chaff_roundtrip_direct_atom() { + let mut slab: NounSlab = NounSlab::new(); + let noun = D(0); + slab.set_root(noun); + let jammed = slab.jam(); + let mut cued_slab: NounSlab = NounSlab::new(); + let cued = cued_slab.cue_into(jammed).expect("cue should succeed"); + cued_slab.set_root(cued); + assert!(slab_equality(&slab, &cued_slab)); + } + + #[test] + fn chaff_roundtrip_indirect_atom() { + let mut slab: NounSlab = NounSlab::new(); + let bytes = vec![0xFFu8; 32]; + let atom = atom_from_bytes(&mut slab, &bytes); + slab.set_root(atom.as_noun()); + let jammed = slab.jam(); + let mut cued_slab: NounSlab = NounSlab::new(); + let cued = cued_slab.cue_into(jammed).expect("cue should succeed"); + cued_slab.set_root(cued); + assert!(slab_equality(&slab, &cued_slab)); + } + + #[test] + fn chaff_roundtrip_nested_cells() { + let mut slab: NounSlab = NounSlab::new(); + let noun = T(&mut slab, &[D(1), D(2), D(3), D(4)]); + slab.set_root(noun); + let jammed = slab.jam(); + let mut cued_slab: NounSlab = NounSlab::new(); + let cued = cued_slab.cue_into(jammed).expect("cue should succeed"); + cued_slab.set_root(cued); + assert!(slab_equality(&slab, &cued_slab)); + } + + #[test] + fn chaff_handles_shared_backrefs() { + let mut slab: NounSlab = NounSlab::new(); + let noun = build_shared_noun(&mut slab); + slab.set_root(noun); + let jammed = slab.jam(); + let mut cued_slab: NounSlab = NounSlab::new(); + let cued = cued_slab.cue_into(jammed).expect("cue should succeed"); + cued_slab.set_root(cued); + assert!(slab_equality(&slab, &cued_slab)); + } + + #[test] + fn chaff_jam_matches_nock_jam() { + let mut slab: NounSlab = NounSlab::new(); + let noun = T(&mut slab, &[D(5), D(23), D(7)]); + slab.set_root(noun); + let chaff_jam = slab.jam(); + let source_space = slab.noun_space(); + let mut nock_slab: NounSlab = NounSlab::new(); + let copied = nock_slab.copy_into(noun, &source_space); + let nock_space = nock_slab.noun_space(); + let nock_jam = NockJammer::jam(copied, &nock_space); + assert_eq!(chaff_jam, nock_jam); + } + + #[test] + fn chaff_rejects_truncated_input() { + let mut slab: NounSlab = NounSlab::new(); + let jammed = Bytes::from_static(&[0b1]); + let result = slab.cue_into(jammed); + assert!(matches!(result, Err(CueError::TruncatedBuffer))); + } + + #[test] + fn chaff_rejects_bad_backref() { + let mut slab: NounSlab = NounSlab::new(); + let jammed = Bytes::from_static(&[0b0000_0111]); + let result = slab.cue_into(jammed); + assert!(matches!(result, Err(CueError::BadBackref))); + } + + #[test] + fn chaff_rejects_backref_before_definition() { + let mut slab: NounSlab = NounSlab::new(); + let mut writer = BitWriter::new(); + writer.write_bit(true); // tag + writer.write_bit(true); // backref + writer.write_zeros(1); // size-of-size zeros + writer.write_bit(true); // delimiter + writer.write_bit(false); // backref size = 1 + writer.write_bit(true); // backref value = 1 (missing) + let jammed = writer.into_bytes(); + let result = slab.cue_into(jammed); + assert!(matches!(result, Err(CueError::BadBackref))); + } + + #[test] + fn chaff_rejects_backref_far_ahead() { + let mut slab: NounSlab = NounSlab::new(); + let mut writer = BitWriter::new(); + writer.write_bit(true); // tag + writer.write_bit(true); // backref + writer.write_zeros(3); // size-of-size zeros + writer.write_bit(true); // delimiter + writer.write_bits_from_value(0b11, 2); // backref size = 4 + writer.write_bits_from_value(0b1010, 4); // backref = 10 + let jammed = writer.into_bytes(); + let result = slab.cue_into(jammed); + assert!(matches!(result, Err(CueError::BadBackref))); + } + + #[test] + fn chaff_rejects_self_referential_backref() { + let mut slab: NounSlab = NounSlab::new(); + let mut writer = BitWriter::new(); + writer.write_bit(true); // tag + writer.write_bit(true); // backref + writer.write_zeros(1); // size-of-size zeros + writer.write_bit(true); // delimiter + writer.write_bit(false); // backref size = 1 + writer.write_bit(false); // backref value = 0 (self) + let jammed = writer.into_bytes(); + let result = slab.cue_into(jammed); + assert!(matches!(result, Err(CueError::BadBackref))); + } + + #[test] + fn chaff_rejects_backref_too_big() { + let mut slab: NounSlab = NounSlab::new(); + let mut writer = BitWriter::new(); + writer.write_bit(true); // tag + writer.write_bit(true); // backref + writer.write_zeros(usize::BITS as usize + 1); + writer.write_bit(true); // delimiter + let jammed = writer.into_bytes(); + let result = slab.cue_into(jammed); + assert!(matches!(result, Err(CueError::BackrefTooBig))); + } + + #[test] + fn chaff_rejects_truncated_backref_size() { + let mut slab: NounSlab = NounSlab::new(); + let mut writer = BitWriter::new(); + writer.write_bit(true); // tag + writer.write_bit(true); // backref + writer.write_zeros(2); // size-of-size zeros (expect 3 bits for size) + writer.write_bit(true); // delimiter + writer.write_bits_from_value(0b01, 2); // only two size bits + let jammed = writer.into_bytes(); + let result = slab.cue_into(jammed); + assert!(matches!(result, Err(CueError::TruncatedBuffer))); + } + + #[test] + fn chaff_rejects_atom_missing_size_bits() { + let mut slab: NounSlab = NounSlab::new(); + let mut writer = BitWriter::new(); + writer.write_bit(false); // atom tag + writer.write_bit(false); // first zero + writer.write_bit(false); // second zero + // zeros should be 3 (two zeros then delimiter), need 2 bits for size_low + // but we end here - truncated + let jammed = writer.into_bytes(); + assert!(slab.cue_into(jammed).is_err()); + } + + #[test] + fn chaff_rejects_truncated_indirect_atom() { + let mut slab: NounSlab = NounSlab::new(); + let mut writer = BitWriter::new(); + writer.write_bit(false); // atom tag + writer.write_zeros(6); // zeros = 7, expect 6 bits for size_low + writer.write_bit(true); // delimiter + writer.write_bits_from_value(0b100000, 6); // size_low = 33 (bit_count = 65) + writer.write_bits_from_value(0xFFFF, 16); // partial value (needs 65 bits) + let jammed = writer.into_bytes(); + let result = slab.cue_into(jammed); + assert!(matches!(result, Err(CueError::TruncatedBuffer))); + } + + #[test] + fn chaff_rejects_atom_size_prefix_too_big() { + let mut slab: NounSlab = NounSlab::new(); + let mut writer = BitWriter::new(); + writer.write_bit(false); // atom tag + writer.write_zeros(usize::BITS as usize + 1); // zeros > MAX_USIZE_BITS + writer.write_bit(true); // unary delimiter + let jammed = writer.into_bytes(); + let result = slab.cue_into(jammed); + assert!(matches!(result, Err(CueError::TruncatedBuffer))); + } + + #[test] + fn chaff_rejects_huge_atom_size_without_panicking() { + let mut slab: NounSlab = NounSlab::new(); + let mut writer = BitWriter::new(); + writer.write_bit(false); // atom tag + writer.write_zeros(usize::BITS as usize); // zeros == MAX_USIZE_BITS + writer.write_bit(true); // unary delimiter + if usize::BITS > 1 { + writer.write_bits_from_value(usize::MAX >> 1, usize::BITS as usize - 1); + } + let jammed = writer.into_bytes(); + let result = + std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| slab.cue_into(jammed))); + assert!( + result.is_ok(), + "cue should not panic on huge atom size prefix" + ); + assert!(matches!( + result.expect("catch_unwind should succeed"), + Err(CueError::TruncatedBuffer) + )); + } + + #[test] + fn chaff_rejects_zero_atom_bad_encoding() { + let mut slab: NounSlab = NounSlab::new(); + let mut writer = BitWriter::new(); + writer.write_bit(false); // atom tag + writer.write_bit(false); // zeros = 0 (should be "0 1" encoding) + writer.write_bit(false); + let jammed = writer.into_bytes(); + let result = slab.cue_into(jammed); + assert!(matches!(result, Err(CueError::TruncatedBuffer))); + } + + const SHARED_PREFIX_BOUNDARIES: [usize; 15] = + [7, 8, 9, 15, 16, 17, 31, 32, 33, 63, 64, 65, 127, 128, 129]; + + #[derive(Clone, Copy, Debug)] + struct NounTreeProfile { + max_depth: usize, + max_indirect_bytes: usize, + } + + const TIER1_PROFILE: NounTreeProfile = NounTreeProfile { + max_depth: 6, + max_indirect_bytes: 256, + }; + + const TIER2_PROFILE: NounTreeProfile = NounTreeProfile { + max_depth: 10, + max_indirect_bytes: 4 * 1024, + }; + + const TIER3_PROFILE: NounTreeProfile = NounTreeProfile { + max_depth: 14, + max_indirect_bytes: 64 * 1024, + }; + + #[derive(Clone, Debug)] + enum NounTree { + Atom(AtomKind), + Cell(Box, Box), + } + + #[derive(Clone, Debug)] + enum AtomKind { + Zero, + Direct(u64), + Indirect(Vec), + } + + #[derive(Clone, Debug)] + struct Tier1Tree(NounTree); + + #[derive(Clone, Debug)] + struct Tier2Tree(NounTree); + + #[derive(Clone, Debug)] + struct Tier3Tree(NounTree); + + #[derive(Clone, Copy, Debug)] + enum JamMutationKind { + TruncateAtBit, + InflateUnaryPrefix, + DeflateUnaryPrefix, + RewriteBackrefFuture, + RewriteBackrefSelf, + FlipTagBit, + DropPayloadBytes, + AppendNonZeroTrailingByte, + } + + #[derive(Clone, Debug)] + struct JamMutationCase { + base_tree: NounTree, + seed_selector: u8, + mutation: JamMutationKind, + salt: u16, + } + + impl JamMutationKind { + fn is_differential_safe(self) -> bool { + matches!( + self, + JamMutationKind::AppendNonZeroTrailingByte + | JamMutationKind::RewriteBackrefFuture + | JamMutationKind::RewriteBackrefSelf + | JamMutationKind::FlipTagBit + ) + } + } + + impl Arbitrary for JamMutationKind { + fn arbitrary(g: &mut Gen) -> Self { + match usize::arbitrary(g) % 8 { + 0 => Self::TruncateAtBit, + 1 => Self::InflateUnaryPrefix, + 2 => Self::DeflateUnaryPrefix, + 3 => Self::RewriteBackrefFuture, + 4 => Self::RewriteBackrefSelf, + 5 => Self::FlipTagBit, + 6 => Self::DropPayloadBytes, + _ => Self::AppendNonZeroTrailingByte, + } + } + + fn shrink(&self) -> Box> { + Box::new(std::iter::empty()) + } + } + + impl Arbitrary for JamMutationCase { + fn arbitrary(g: &mut Gen) -> Self { + Self { + base_tree: arbitrary_noun_tree(g, TIER1_PROFILE), + seed_selector: u8::arbitrary(g), + mutation: JamMutationKind::arbitrary(g), + salt: u16::arbitrary(g), + } + } + + fn shrink(&self) -> Box> { + let mut out = Vec::new(); + for shrunk in shrink_noun_tree(&self.base_tree) { + out.push(Self { + base_tree: shrunk, + seed_selector: self.seed_selector, + mutation: self.mutation, + salt: self.salt, + }); + } + Box::new(out.into_iter()) + } + } + + impl Arbitrary for Tier1Tree { + fn arbitrary(g: &mut Gen) -> Self { + Self(arbitrary_noun_tree(g, TIER1_PROFILE)) + } + + fn shrink(&self) -> Box> { + Box::new(shrink_noun_tree(&self.0).into_iter().map(Self)) + } + } + + impl Arbitrary for Tier2Tree { + fn arbitrary(g: &mut Gen) -> Self { + Self(arbitrary_noun_tree(g, TIER2_PROFILE)) + } + + fn shrink(&self) -> Box> { + Box::new(shrink_noun_tree(&self.0).into_iter().map(Self)) + } + } + + impl Arbitrary for Tier3Tree { + fn arbitrary(g: &mut Gen) -> Self { + Self(arbitrary_noun_tree(g, TIER3_PROFILE)) + } + + fn shrink(&self) -> Box> { + Box::new(shrink_noun_tree(&self.0).into_iter().map(Self)) + } + } + + fn arbitrary_noun_tree(g: &mut Gen, profile: NounTreeProfile) -> NounTree { + gen_noun_tree(g, profile, profile.max_depth) + } + + fn gen_noun_tree(g: &mut Gen, profile: NounTreeProfile, fuel: usize) -> NounTree { + if fuel == 0 { + return NounTree::Atom(gen_atom_kind(g, profile)); + } + + let choose_cell = (usize::arbitrary(g) % 10) < 6; + if !choose_cell { + return NounTree::Atom(gen_atom_kind(g, profile)); + } + + let remaining = fuel - 1; + let head_fuel = if remaining == 0 { + 0 + } else { + usize::arbitrary(g) % (remaining + 1) + }; + let tail_fuel = remaining.saturating_sub(head_fuel); + NounTree::Cell( + Box::new(gen_noun_tree(g, profile, head_fuel)), + Box::new(gen_noun_tree(g, profile, tail_fuel)), + ) + } + + fn gen_atom_kind(g: &mut Gen, profile: NounTreeProfile) -> AtomKind { + let choice = usize::arbitrary(g) % 100; + match choice { + 0..=2 => AtomKind::Zero, + 3..=34 => { + let val = u64::arbitrary(g) & nockvm::noun::DIRECT_MAX; + AtomKind::Direct(val) + } + 35..=54 => { + let bits = *g.choose(&[63usize, 64, 65]).expect("bits boundary choice"); + let bytes = gen_atom_bytes_exact_bits(g, bits); + if bits < 64 { + AtomKind::Direct(bytes_to_u64(&bytes) & nockvm::noun::DIRECT_MAX) + } else { + AtomKind::Indirect(bytes) + } + } + _ => { + let byte_len = choose_indirect_byte_len(g, profile.max_indirect_bytes); + let mut bytes: Vec = (0..byte_len).map(|_| u8::arbitrary(g)).collect(); + if let Some(last) = bytes.last_mut() { + if *last == 0 { + *last = 1 + (u8::arbitrary(g) % 0x7F); + } + } + AtomKind::Indirect(bytes) + } + } + } + + fn choose_indirect_byte_len(g: &mut Gen, max_indirect_bytes: usize) -> usize { + let min_len = 9usize; + let max_len = max_indirect_bytes.max(min_len); + const BOUNDARY_BYTES: [usize; 12] = [9, 10, 15, 16, 17, 31, 32, 33, 63, 64, 65, 129]; + let use_boundary = (usize::arbitrary(g) % 10) < 6; + if use_boundary { + let candidate = BOUNDARY_BYTES[usize::arbitrary(g) % BOUNDARY_BYTES.len()]; + candidate.min(max_len) + } else { + min_len + (usize::arbitrary(g) % (max_len - min_len + 1)) + } + } + + fn gen_atom_bytes_exact_bits(g: &mut Gen, bits: usize) -> Vec { + let byte_len = bits.div_ceil(8); + let mut bytes: Vec = (0..byte_len).map(|_| u8::arbitrary(g)).collect(); + let high_byte_idx = byte_len - 1; + let high_bit = (bits - 1) & 7; + bytes[high_byte_idx] |= 1u8 << high_bit; + if high_bit < 7 { + bytes[high_byte_idx] &= (1u8 << (high_bit + 1)) - 1; + } + bytes + } + + fn bytes_to_u64(bytes: &[u8]) -> u64 { + let mut value = 0u64; + for (idx, byte) in bytes.iter().copied().enumerate() { + if idx >= 8 { + break; + } + value |= (byte as u64) << (idx * 8); + } + value + } + + fn shrink_noun_tree(tree: &NounTree) -> Vec { + match tree { + NounTree::Atom(kind) => shrink_atom_kind(kind) + .into_iter() + .map(NounTree::Atom) + .collect(), + NounTree::Cell(head, tail) => { + let mut out = vec![(**head).clone(), (**tail).clone()]; + out.extend( + shrink_noun_tree(head) + .into_iter() + .map(|shrunk| NounTree::Cell(Box::new(shrunk), Box::new((**tail).clone()))), + ); + out.extend( + shrink_noun_tree(tail) + .into_iter() + .map(|shrunk| NounTree::Cell(Box::new((**head).clone()), Box::new(shrunk))), + ); + out + } + } + } + + fn shrink_atom_kind(kind: &AtomKind) -> Vec { + match kind { + AtomKind::Zero => Vec::new(), + AtomKind::Direct(value) => { + let mut out = vec![AtomKind::Zero]; + for shrunk in value.shrink() { + out.push(AtomKind::Direct(shrunk & nockvm::noun::DIRECT_MAX)); + } + out + } + AtomKind::Indirect(bytes) => { + let mut out = vec![AtomKind::Zero, AtomKind::Direct(1), AtomKind::Direct(63)]; + for bits in [63usize, 64, 65] { + let mut seed = vec![0u8; bits.div_ceil(8)]; + let high_byte_idx = seed.len() - 1; + let high_bit = (bits - 1) & 7; + seed[high_byte_idx] |= 1u8 << high_bit; + if bits < 64 { + out.push(AtomKind::Direct(bytes_to_u64(&seed))); + } else { + out.push(AtomKind::Indirect(seed)); + } + } + + if bytes.len() > 9 { + let mut shorter = bytes[..bytes.len() / 2].to_vec(); + if shorter.len() < 9 { + shorter.resize(9, 0); + } + if let Some(last) = shorter.last_mut() { + if *last == 0 { + *last = 1; + } + } + out.push(AtomKind::Indirect(shorter)); + } + out + } + } + } + + fn build_noun_from_tree(slab: &mut NounSlab, tree: &NounTree) -> Noun { + match tree { + NounTree::Atom(AtomKind::Zero) => D(0), + NounTree::Atom(AtomKind::Direct(value)) => D(*value), + NounTree::Atom(AtomKind::Indirect(bytes)) => atom_from_bytes(slab, bytes).as_noun(), + NounTree::Cell(head, tail) => { + let h = build_noun_from_tree(slab, head); + let t = build_noun_from_tree(slab, tail); + Cell::new(slab, h, t).as_noun() + } + } + } + + fn pair_with_self(slab: &mut NounSlab, noun: Noun) -> Noun { + Cell::new(slab, noun, noun).as_noun() + } + + fn nested_shared(slab: &mut NounSlab, noun: Noun) -> Noun { + let inner = pair_with_self(slab, noun); + Cell::new(slab, noun, inner).as_noun() + } + + fn alternating_shared(slab: &mut NounSlab, noun: Noun, prefix_len: usize) -> Noun { + let mut list = D(0); + for idx in (0..prefix_len).rev() { + list = Cell::new(slab, D(7_000 + idx as u64), list).as_noun(); + } + let nested = nested_shared(slab, noun); + T(slab, &[list, noun, nested, noun]) + } + + fn seed_noun_for_selector(slab: &mut NounSlab, selector: u8, tree: &NounTree) -> Noun { + match selector % 4 { + 0 => build_noun_from_tree(slab, tree), + 1 => build_shared_noun(slab), + 2 => T(slab, &[D(1), D(2), D(3), D(4), D(5)]), + _ => { + let atom = atom_with_exact_bit_size(slab, 65); + T(slab, &[D(11), atom.as_noun(), D(99)]) + } + } + } + + fn canonical_jam_from_case(case: &JamMutationCase) -> Bytes { + let mut slab: NounSlab = NounSlab::new(); + let noun = seed_noun_for_selector(&mut slab, case.seed_selector, &case.base_tree); + slab.set_root(noun); + slab.jam() + } + + fn truncate_bits(bytes: &[u8], keep_bits: usize) -> Bytes { + if keep_bits == 0 { + return Bytes::new(); + } + let bit_len = bytes.len() * 8; + let keep = keep_bits.min(bit_len); + if keep == bit_len { + return Bytes::copy_from_slice(bytes); + } + let byte_len = keep.div_ceil(8); + let mut out = bytes[..byte_len].to_vec(); + let rem = keep & 7; + if rem != 0 { + let mask = (1u8 << rem) - 1; + let last = out.len() - 1; + out[last] &= mask; + } + Bytes::from(out) + } + + fn set_bit(bytes: &mut Vec, bit_idx: usize, value: bool) { + let byte_idx = bit_idx >> 3; + if byte_idx >= bytes.len() { + bytes.resize(byte_idx + 1, 0); + } + let bit_mask = 1u8 << (bit_idx & 7); + if value { + bytes[byte_idx] |= bit_mask; + } else { + bytes[byte_idx] &= !bit_mask; + } + } + + fn get_bit(bytes: &[u8], bit_idx: usize) -> bool { + let byte_idx = bit_idx >> 3; + if byte_idx >= bytes.len() { + return false; + } + ((bytes[byte_idx] >> (bit_idx & 7)) & 1) == 1 + } + + fn clear_first_set_bit(bytes: &mut Vec, start_bit: usize) -> bool { + let total_bits = bytes.len() * 8; + for bit in start_bit..total_bits { + if get_bit(bytes, bit) { + set_bit(bytes, bit, false); + return true; + } + } + false + } + + fn set_first_clear_bit(bytes: &mut Vec, start_bit: usize) -> bool { + let total_bits = bytes.len() * 8; + for bit in start_bit..total_bits { + if !get_bit(bytes, bit) { + set_bit(bytes, bit, true); + return true; + } + } + false + } + + fn overwrite_prefix_bits(bytes: &mut Vec, prefix_bits: &[bool]) { + for (idx, bit) in prefix_bits.iter().copied().enumerate() { + set_bit(bytes, idx, bit); + } + } + + fn mutate_jam_bytes(canonical: &Bytes, mutation: JamMutationKind, salt: u16) -> Bytes { + let mut bytes = canonical.to_vec(); + match mutation { + JamMutationKind::TruncateAtBit => { + let total_bits = bytes.len() * 8; + if total_bits == 0 { + return Bytes::new(); + } + let keep_bits = (salt as usize) % total_bits; + truncate_bits(&bytes, keep_bits) + } + JamMutationKind::InflateUnaryPrefix => { + if !clear_first_set_bit(&mut bytes, 1) && !bytes.is_empty() { + set_bit(&mut bytes, 0, false); + } + Bytes::from(bytes) + } + JamMutationKind::DeflateUnaryPrefix => { + if !set_first_clear_bit(&mut bytes, 1) { + bytes.push(0x01); + } + Bytes::from(bytes) + } + JamMutationKind::RewriteBackrefFuture => { + overwrite_prefix_bits(&mut bytes, &[true, true, false, true, true, false]); + Bytes::from(bytes) + } + JamMutationKind::RewriteBackrefSelf => { + overwrite_prefix_bits(&mut bytes, &[true, true, false, true, false, false]); + Bytes::from(bytes) + } + JamMutationKind::FlipTagBit => { + if bytes.is_empty() { + bytes.push(0); + } + let flip_idx = (salt as usize) % (bytes.len() * 8); + let current = get_bit(&bytes, flip_idx); + set_bit(&mut bytes, flip_idx, !current); + Bytes::from(bytes) + } + JamMutationKind::DropPayloadBytes => { + if bytes.len() <= 1 { + let keep_bits = (salt as usize) % 4; + return truncate_bits(&bytes, keep_bits); + } + let drop = 1 + ((salt as usize) % bytes.len().min(4)); + bytes.truncate(bytes.len() - drop); + Bytes::from(bytes) + } + JamMutationKind::AppendNonZeroTrailingByte => { + let trailing = (salt as u8) | 1; + bytes.push(trailing); + Bytes::from(bytes) + } + } + } + + fn expected_cue_error(err: CueError) -> bool { + matches!( + err, + CueError::TruncatedBuffer | CueError::BadBackref | CueError::BackrefTooBig + ) + } + + fn qc_test_count(env_key: &str, default: u64) -> u64 { + std::env::var(env_key) + .ok() + .and_then(|raw| raw.parse::().ok()) + .filter(|count| *count > 0) + .unwrap_or(default) + } + + fn prop_chaff_roundtrip_tree_impl(tree: &NounTree) -> TestResult { + let mut slab: NounSlab = NounSlab::new(); + let noun = build_noun_from_tree(&mut slab, tree); + slab.set_root(noun); + let jammed = slab.jam(); + let mut cued: NounSlab = NounSlab::new(); + let cued_noun = match cued.cue_into(jammed) { + Ok(noun) => noun, + Err(_) => return TestResult::failed(), + }; + cued.set_root(cued_noun); + TestResult::from_bool(slab_equality(&slab, &cued)) + } + + fn prop_chaff_matches_nock_tree_impl(tree: &NounTree) -> TestResult { + let mut slab: NounSlab = NounSlab::new(); + let noun = build_noun_from_tree(&mut slab, tree); + slab.set_root(noun); + let chaff_jam = slab.jam(); + let source_space = slab.noun_space(); + let mut nock_slab: NounSlab = NounSlab::new(); + let copied = nock_slab.copy_into(noun, &source_space); + let nock_space = nock_slab.noun_space(); + let nock_jam = NockJammer::jam(copied, &nock_space); + TestResult::from_bool(chaff_jam == nock_jam) + } + + fn prop_chaff_parity_tree_matrix_impl(tree: &NounTree) -> TestResult { + let mut slab: NounSlab = NounSlab::new(); + let noun = build_noun_from_tree(&mut slab, tree); + assert_parity_with_nock(noun, &mut slab); + TestResult::passed() + } + + fn prop_chaff_parity_shared_tree_impl( + tree: &NounTree, + mode: u8, + prefix_seed: usize, + ) -> TestResult { + let mut slab: NounSlab = NounSlab::new(); + let sub = build_noun_from_tree(&mut slab, tree); + let shared = match mode % 3 { + 0 => pair_with_self(&mut slab, sub), + 1 => nested_shared(&mut slab, sub), + _ => { + let prefix = SHARED_PREFIX_BOUNDARIES[prefix_seed % SHARED_PREFIX_BOUNDARIES.len()]; + alternating_shared(&mut slab, sub, prefix) + } + }; + assert_parity_with_nock(shared, &mut slab); + TestResult::passed() + } + + fn prop_chaff_idempotence_chain_tree_impl(tree: &NounTree) -> TestResult { + let mut slab: NounSlab = NounSlab::new(); + let noun = build_noun_from_tree(&mut slab, tree); + slab.set_root(noun); + assert_idempotence_canonical_chain(slab.jam(), None, 4); + TestResult::passed() + } + + fn prop_chaff_adversarial_mutations_no_panic_impl(case: JamMutationCase) -> TestResult { + let canonical = canonical_jam_from_case(&case); + let mutated = mutate_jam_bytes(&canonical, case.mutation, case.salt); + let mut chaff_slab: NounSlab = NounSlab::new(); + let result = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| { + chaff_slab.cue_into(mutated.clone()) + })); + let cue_result = match result { + Ok(outcome) => outcome, + Err(_) => return TestResult::failed(), + }; + + match cue_result { + Ok(noun) => { + chaff_slab.set_root(noun); + let rejam = chaff_slab.jam(); + if matches!(case.mutation, JamMutationKind::AppendNonZeroTrailingByte) { + return TestResult::from_bool(rejam == canonical); + } + TestResult::from_bool(!rejam.is_empty() || canonical.is_empty()) + } + Err(err) => TestResult::from_bool(expected_cue_error(err)), + } + } + + fn prop_chaff_adversarial_mutations_differential_impl(case: JamMutationCase) -> TestResult { + if !case.mutation.is_differential_safe() { + return TestResult::discard(); + } + + let canonical = canonical_jam_from_case(&case); + let mutated = mutate_jam_bytes(&canonical, case.mutation, case.salt); + let mut chaff_slab: NounSlab = NounSlab::new(); + let mut nock_slab: NounSlab = NounSlab::new(); + + let chaff = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| { + chaff_slab.cue_into(mutated.clone()) + })); + let nock = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| { + nock_slab.cue_into(mutated.clone()) + })); + + let chaff = match chaff { + Ok(value) => value, + Err(_) => return TestResult::failed(), + }; + let nock = match nock { + Ok(value) => value, + Err(_) => return TestResult::failed(), + }; + + match (chaff, nock) { + (Err(chaff_err), Err(nock_err)) => { + TestResult::from_bool(expected_cue_error(chaff_err) && expected_cue_error(nock_err)) + } + (Ok(chaff_noun), Ok(nock_noun)) => { + chaff_slab.set_root(chaff_noun); + nock_slab.set_root(nock_noun); + if !slab_equality(&chaff_slab, &nock_slab) { + return TestResult::failed(); + } + let chaff_rejam = chaff_slab.jam(); + let nock_rejam = nock_slab.jam(); + if matches!(case.mutation, JamMutationKind::AppendNonZeroTrailingByte) + && chaff_rejam != canonical + { + return TestResult::failed(); + } + TestResult::from_bool(chaff_rejam == nock_rejam) + } + _ => TestResult::failed(), + } + } + + fn build_list(slab: &mut NounSlab, leaves: &[u16]) -> Noun { + let mut list = D(0); + for value in leaves.iter().rev() { + list = Cell::new(slab, D(*value as u64), list).as_noun(); + } + list + } + + fn parity_fixtures(slab: &mut NounSlab) -> Vec { + let mut fixtures = Vec::new(); + fixtures.push(D(0)); + + let bytes = vec![0xFFu8; 64]; + let atom = atom_from_bytes(slab, &bytes); + fixtures.push(atom.as_noun()); + + fixtures.push(T(slab, &[D(1), D(2), D(3), D(4)])); + fixtures.push(build_shared_noun(slab)); + fixtures.push(build_list(slab, &[1, 2, 3, 4, 5])); + fixtures + } + + fn stack_words_for_jam(bytes_len: usize) -> usize { + let words = bytes_len.div_ceil(8); + let scaled = words.saturating_mul(NOCKVM_STACK_WORDS_JAM_SCALE); + scaled.max(NOCKVM_STACK_WORDS_MIN) + } + + fn stack_words_for_large_checkpoint(bytes_len: usize) -> usize { + stack_words_for_jam(bytes_len).max(NOCK_STACK_SIZE_TINY) + } + + fn stack_words_for_noun(noun: Noun, space: &NounSpace, jam_len: usize) -> usize { + let jam_words = stack_words_for_jam(jam_len); + let mass_words = noun + .in_space(space) + .mass() + .saturating_mul(NOCKVM_STACK_WORDS_MASS_SCALE); + jam_words.max(mass_words) + } + + #[derive(Decode)] + struct CheckpointEnvelope { + magic_bytes: u64, + version: u32, + payload: Vec, + } + + const SNAPSHOT_VERSION_2: u32 = 2; + + fn jammed_state_from_checkpoint(bytes: &[u8]) -> Bytes { + let config = config::standard(); + + if let Ok((envelope, _)) = + bincode::decode_from_slice::(bytes, config) + { + if envelope.magic_bytes == JAM_MAGIC_BYTES && envelope.version == SNAPSHOT_VERSION_2 { + let (checkpoint, _) = + bincode::decode_from_slice::( + &envelope.payload, config, + ) + .expect("V2 checkpoint payload should decode"); + return checkpoint.state_jam.0; + } + } + + if let Ok((checkpoint, _)) = + bincode::decode_from_slice::(bytes, config) + { + if checkpoint.magic_bytes == JAM_MAGIC_BYTES { + return checkpoint.jam.0; + } + } + + panic!("Failed to decode checkpoint as either V1 or V2 format"); + } + + fn legacy_pair_jam(state_value: u64, cold_value: u64) -> JammedNoun { + let mut slab = NounSlab::::new(); + let space = NounSpace::empty(); + let state = slab.copy_into(D(state_value), &space); + let cold = slab.copy_into(D(cold_value), &space); + let root = T(&mut slab, &[state, cold]); + slab.set_root(root); + JammedNoun::new(slab.coerce_jammer::().jam()) + } + + fn jam_atom_with_chaff(value: u64) -> JammedNoun { + let mut slab = NounSlab::::new(); + let space = NounSpace::empty(); + let atom = slab.copy_into(D(value), &space); + slab.set_root(atom); + JammedNoun::new(slab.coerce_jammer::().jam()) + } + + fn atom_value(noun: Noun, space: &NounSpace) -> u64 { + noun.in_space(space) + .as_atom() + .expect("expected atom") + .as_u64() + .expect("expected atom to fit in u64") + } + + async fn load_saveable_with_chaff(temp: &TempDir, bytes: Vec) -> SaveableCheckpoint { + std::fs::write(temp.path().join("0.chkjam"), bytes).expect("write checkpoint"); + CheckpointBootstrapReader::::new(temp.path().to_path_buf()) + .load_latest(None) + .await + .expect("load checkpoint") + .expect("expected a checkpoint") + } + + fn assert_loaded_values( + saveable: &SaveableCheckpoint, + expected_hash: blake3::Hash, + event_num: u64, + state_value: u64, + cold_value: u64, + ) { + assert_eq!(saveable.ker_hash, expected_hash); + assert_eq!(saveable.event_num, event_num); + let state_space = saveable.state.noun_space(); + let cold_space = saveable.cold.noun_space(); + assert_eq!( + atom_value(unsafe { *saveable.state.root() }, &state_space), + state_value + ); + assert_eq!( + atom_value(unsafe { *saveable.cold.root() }, &cold_space), + cold_value + ); + } + + fn large_checkpoint_jam() -> Bytes { + static JAM: OnceLock = OnceLock::new(); + JAM.get_or_init(|| { + let bytes = include_bytes!(concat!(env!("CARGO_MANIFEST_DIR"), "/test-jams/0.chkjam")); + jammed_state_from_checkpoint(bytes) + }) + .clone() + } + + fn jam_nockstack_trimmed(noun: Noun, space: &NounSpace, expected_len: usize) -> Bytes { + let mut jam_stack = NockStack::new(stack_words_for_noun(noun, space, expected_len), 0); + let jam_noun = jam_stack.copy_into(noun, space); + let jammed = nockvm::serialization::jam(&mut jam_stack, jam_noun); + let jam_space = jam_stack.noun_space(); + let jam_handle = jammed.in_space(&jam_space); + let bytes = jam_handle.as_ne_bytes(); + assert!( + bytes.len() >= expected_len, + "nockvm jam shorter than expected" + ); + assert!( + bytes[expected_len..].iter().all(|b| *b == 0), + "nockvm jam has non-zero padding bytes" + ); + Bytes::copy_from_slice(&bytes[..expected_len]) + } + + fn cue_all_with_stack( + jam: &Bytes, + stack_words: usize, + ) -> ( + NounSlab, + Noun, + NounSlab, + Noun, + NockStack, + Noun, + ) { + let mut chaff_slab: NounSlab = NounSlab::new(); + let chaff_noun = chaff_slab + .cue_into(jam.clone()) + .expect("chaff cue should succeed"); + chaff_slab.set_root(chaff_noun); + + let mut nock_slab: NounSlab = NounSlab::new(); + let nock_noun = nock_slab + .cue_into(jam.clone()) + .expect("NockJammer cue should succeed"); + nock_slab.set_root(nock_noun); + + let mut nockvm_stack = NockStack::new(stack_words, 0); + let nockvm_noun = ::cue_bytes(&mut nockvm_stack, jam) + .expect("nockvm cue should succeed"); + + ( + chaff_slab, chaff_noun, nock_slab, nock_noun, nockvm_stack, nockvm_noun, + ) + } + + fn assert_parity_with_nock(noun: Noun, chaff_slab: &mut NounSlab) { + chaff_slab.set_root(noun); + let chaff_jam = chaff_slab.jam(); + let chaff_root = unsafe { *chaff_slab.root() }; + let chaff_space = chaff_slab.noun_space(); + + let nock_jam = NockJammer::jam(chaff_root, &chaff_space); + assert_eq!(chaff_jam, nock_jam, "chaff jam differs from NockJammer"); + + let stack_words = stack_words_for_noun(chaff_root, &chaff_space, chaff_jam.len()); + let mut jam_stack = NockStack::new(stack_words, 0); + let stack_noun = jam_stack.copy_into(chaff_root, &chaff_space); + let nockvm_jam = nockvm::serialization::jam(&mut jam_stack, stack_noun); + let jam_space = jam_stack.noun_space(); + assert!( + nockvm_jam.in_space(&jam_space).eq_bytes(&chaff_jam), + "chaff jam differs from nockvm jam (allowing zero padding)" + ); + let nockvm_bytes = atom_bytes(nockvm_jam, &jam_space); + + let mut chaff_cue_slab: NounSlab = NounSlab::new(); + let chaff_cued = chaff_cue_slab + .cue_into(chaff_jam.clone()) + .expect("chaff cue should succeed"); + let chaff_cue_space = chaff_cue_slab.noun_space(); + assert!( + nouns_equal(chaff_root, &chaff_space, chaff_cued, &chaff_cue_space), + "chaff cue differs from original noun" + ); + + let mut nock_cue_slab: NounSlab = NounSlab::new(); + let nock_cued = nock_cue_slab + .cue_into(chaff_jam.clone()) + .expect("NockJammer cue should succeed"); + let nock_cue_space = nock_cue_slab.noun_space(); + assert!( + nouns_equal(chaff_root, &chaff_space, nock_cued, &nock_cue_space), + "NockJammer cue differs from original noun" + ); + + let mut cue_stack = NockStack::new(stack_words, 0); + let nockvm_cued = ::cue_bytes(&mut cue_stack, &chaff_jam) + .expect("nockvm cue should succeed"); + let cue_space = cue_stack.noun_space(); + assert!( + nouns_equal(chaff_root, &chaff_space, nockvm_cued, &cue_space), + "nockvm cue differs from original noun" + ); + + let mut chaff_cue_slab2: NounSlab = NounSlab::new(); + let chaff_cued_from_nockvm = chaff_cue_slab2 + .cue_into(nockvm_bytes.clone()) + .expect("chaff cue should accept nockvm jam bytes"); + let chaff_cue_space2 = chaff_cue_slab2.noun_space(); + assert!( + nouns_equal(chaff_root, &chaff_space, chaff_cued_from_nockvm, &chaff_cue_space2,), + "chaff cue differs when decoding nockvm jam bytes" + ); + + let mut nock_cue_slab2: NounSlab = NounSlab::new(); + let nock_cued_from_nockvm = nock_cue_slab2 + .cue_into(nockvm_bytes.clone()) + .expect("NockJammer cue should accept nockvm jam bytes"); + let nock_cue_space2 = nock_cue_slab2.noun_space(); + assert!( + nouns_equal(chaff_root, &chaff_space, nock_cued_from_nockvm, &nock_cue_space2,), + "NockJammer cue differs when decoding nockvm jam bytes" + ); + } + + fn assert_roundtrip_matrix(jam: Bytes, full_matrix: bool) { + let mut chaff_slab: NounSlab = NounSlab::new(); + let chaff_noun = chaff_slab + .cue_into(jam.clone()) + .expect("chaff cue should succeed"); + chaff_slab.set_root(chaff_noun); + + let mut nock_slab: NounSlab = NounSlab::new(); + let nock_noun = nock_slab + .cue_into(jam.clone()) + .expect("NockJammer cue should succeed"); + nock_slab.set_root(nock_noun); + let chaff_space = chaff_slab.noun_space(); + let nock_space = nock_slab.noun_space(); + + let stack_words = stack_words_for_noun(chaff_noun, &chaff_space, jam.len()); + let mut nockvm_stack = NockStack::new(stack_words, 0); + let nockvm_noun = ::cue_bytes(&mut nockvm_stack, &jam) + .expect("nockvm cue should succeed"); + let nockvm_space = nockvm_stack.noun_space(); + + assert!( + slab_equality(&chaff_slab, &nock_slab), + "NockJammer cue differs from Chaff cue" + ); + assert!( + nouns_equal(chaff_noun, &chaff_space, nockvm_noun, &nockvm_space), + "nockvm cue differs from Chaff cue" + ); + + let chaff_jam = chaff_slab.jam(); + assert_eq!(chaff_jam, jam, "chaff jam differs from input jam"); + + let nock_jam = NockJammer::jam(nock_noun, &nock_space); + assert_eq!(nock_jam, jam, "NockJammer jam differs from input jam"); + + let mut jam_stack = NockStack::new(stack_words, 0); + let jam_noun = jam_stack.copy_into(chaff_noun, &chaff_space); + let nockvm_jam = nockvm::serialization::jam(&mut jam_stack, jam_noun); + let jam_space = jam_stack.noun_space(); + let nockvm_handle = nockvm_jam.in_space(&jam_space); + let nockvm_bytes = nockvm_handle.as_ne_bytes(); + assert!( + nockvm_bytes.len() >= jam.len(), + "nockvm jam shorter than input jam" + ); + assert_eq!( + &nockvm_bytes[..jam.len()], + jam.as_ref(), + "nockvm jam prefix differs from input jam" + ); + assert!( + nockvm_bytes[jam.len()..].iter().all(|b| *b == 0), + "nockvm jam has non-zero padding bytes" + ); + + let nockvm_trimmed = Bytes::copy_from_slice(&nockvm_bytes[..jam.len()]); + if !full_matrix { + let mut chaff = NounSlab::::new(); + let chaff_rt = chaff + .cue_into(nockvm_trimmed.clone()) + .expect("chaff cue should succeed"); + let chaff_rt_space = chaff.noun_space(); + assert!( + nouns_equal(chaff_noun, &chaff_space, chaff_rt, &chaff_rt_space), + "chaff cue mismatch on nockvm jam" + ); + + let mut nock = NounSlab::::new(); + let nock_rt = nock + .cue_into(nockvm_trimmed) + .expect("NockJammer cue should succeed"); + let nock_rt_space = nock.noun_space(); + assert!( + nouns_equal(chaff_noun, &chaff_space, nock_rt, &nock_rt_space), + "NockJammer cue mismatch on nockvm jam" + ); + return; + } + + for jam_bytes in [chaff_jam, nock_jam, nockvm_trimmed] { + let mut chaff = NounSlab::::new(); + let chaff_rt = chaff + .cue_into(jam_bytes.clone()) + .expect("chaff cue should succeed"); + let chaff_rt_space = chaff.noun_space(); + assert!( + nouns_equal(chaff_noun, &chaff_space, chaff_rt, &chaff_rt_space), + "chaff roundtrip noun mismatch" + ); + chaff.set_root(chaff_rt); + assert_eq!(chaff.jam(), jam_bytes, "chaff roundtrip jam mismatch"); + + let mut nock = NounSlab::::new(); + let nock_rt = nock + .cue_into(jam_bytes.clone()) + .expect("NockJammer cue should succeed"); + let nock_rt_space = nock.noun_space(); + assert!( + nouns_equal(chaff_noun, &chaff_space, nock_rt, &nock_rt_space), + "NockJammer roundtrip noun mismatch" + ); + assert_eq!( + NockJammer::jam(nock_rt, &nock_rt_space), + jam_bytes, + "NockJammer roundtrip jam mismatch" + ); + + let mut stack = NockStack::new(stack_words, 0); + let nockvm_rt = ::cue_bytes(&mut stack, &jam_bytes) + .expect("nockvm cue should succeed"); + let stack_space = stack.noun_space(); + assert!( + nouns_equal(chaff_noun, &chaff_space, nockvm_rt, &stack_space), + "nockvm roundtrip noun mismatch" + ); + let nockvm_rt_jam = nockvm::serialization::jam(&mut stack, nockvm_rt); + let nockvm_rt_handle = nockvm_rt_jam.in_space(&stack_space); + let nockvm_rt_bytes = nockvm_rt_handle.as_ne_bytes(); + assert!( + nockvm_rt_bytes.len() >= jam_bytes.len(), + "nockvm roundtrip jam shorter than input jam" + ); + assert_eq!( + &nockvm_rt_bytes[..jam_bytes.len()], + jam_bytes.as_ref(), + "nockvm roundtrip jam prefix mismatch" + ); + assert!( + nockvm_rt_bytes[jam_bytes.len()..].iter().all(|b| *b == 0), + "nockvm roundtrip jam has non-zero padding bytes" + ); + } + } + + fn assert_idempotence_canonical_chain( + mut input_jam: Bytes, + expected_canonical: Option, + steps: usize, + ) { + assert!(steps > 0, "chain length must be positive"); + for step in 0..steps { + let mut chaff_slab: NounSlab = NounSlab::new(); + let chaff_noun = chaff_slab + .cue_into(input_jam.clone()) + .expect("chaff cue should succeed in chain"); + chaff_slab.set_root(chaff_noun); + let chaff_jam = chaff_slab.jam(); + + let mut nock_slab: NounSlab = NounSlab::new(); + let nock_noun = nock_slab + .cue_into(input_jam.clone()) + .expect("NockJammer cue should succeed in chain"); + nock_slab.set_root(nock_noun); + let chaff_space = chaff_slab.noun_space(); + let nock_space = nock_slab.noun_space(); + assert!( + nouns_equal(chaff_noun, &chaff_space, nock_noun, &nock_space), + "noun mismatch between Chaff and NockJammer at chain step {step}" + ); + let nock_jam = NockJammer::jam(nock_noun, &nock_space); + assert_eq!( + nock_jam, chaff_jam, + "jam mismatch between Chaff and NockJammer at chain step {step}" + ); + + let stack_words = stack_words_for_noun(chaff_noun, &chaff_space, chaff_jam.len()); + let mut stack = NockStack::new(stack_words, 0); + let nockvm_noun = ::cue_bytes(&mut stack, &input_jam) + .expect("nockvm cue should succeed in chain"); + let stack_space = stack.noun_space(); + assert!( + nouns_equal(chaff_noun, &chaff_space, nockvm_noun, &stack_space), + "noun mismatch between Chaff and nockvm at chain step {step}" + ); + let nockvm_jam = nockvm::serialization::jam(&mut stack, nockvm_noun); + let nockvm_handle = nockvm_jam.in_space(&stack_space); + let nockvm_bytes = nockvm_handle.as_ne_bytes(); + assert!( + nockvm_bytes.len() >= chaff_jam.len(), + "nockvm jam shorter than canonical jam at chain step {step}" + ); + assert_eq!( + &nockvm_bytes[..chaff_jam.len()], + chaff_jam.as_ref(), + "nockvm jam prefix mismatch at chain step {step}" + ); + assert!( + nockvm_bytes[chaff_jam.len()..] + .iter() + .all(|byte| *byte == 0), + "nockvm jam has non-zero padding bytes at chain step {step}" + ); + + if step == 0 { + if let Some(ref canonical) = expected_canonical { + assert_eq!( + &chaff_jam, canonical, + "first chain step should canonicalize to expected bytes" + ); + } else { + assert_eq!( + chaff_jam, input_jam, + "canonical jam should be idempotent on first chain step" + ); + } + } else { + assert_eq!( + chaff_jam, input_jam, + "canonical chain changed unexpectedly at step {step}" + ); + } + + input_jam = chaff_jam; + } + } + + #[test] + fn chaff_roundtrip_list_fixture() { + let mut slab: NounSlab = NounSlab::new(); + let noun = build_list(&mut slab, &[1, 2, 3, 4, 5]); + slab.set_root(noun); + let jammed = slab.jam(); + let mut cued: NounSlab = NounSlab::new(); + let cued_noun = cued.cue_into(jammed).expect("cue should succeed"); + cued.set_root(cued_noun); + assert!(slab_equality(&slab, &cued)); + } + + #[test] + fn chaff_matches_nock_for_shared_fixture() { + let mut slab: NounSlab = NounSlab::new(); + let noun = build_shared_noun(&mut slab); + slab.set_root(noun); + let chaff_jam = slab.jam(); + let source_space = slab.noun_space(); + let mut nock_slab: NounSlab = NounSlab::new(); + let copied = nock_slab.copy_into(noun, &source_space); + let nock_space = nock_slab.noun_space(); + let nock_jam = NockJammer::jam(copied, &nock_space); + assert_eq!(chaff_jam, nock_jam); + } + + #[test] + fn chaff_roundtrip_larger_atom_fixture() { + let mut slab: NounSlab = NounSlab::new(); + let bytes = (0u8..64).collect::>(); + let atom = atom_from_bytes(&mut slab, &bytes); + let noun = T(&mut slab, &[D(7), atom.as_noun(), D(9)]); + slab.set_root(noun); + let jammed = slab.jam(); + let mut cued: NounSlab = NounSlab::new(); + let cued_noun = cued.cue_into(jammed).expect("cue should succeed"); + cued.set_root(cued_noun); + assert!(slab_equality(&slab, &cued)); + } + + #[test] + fn chaff_parity_jam_fixtures() { + let mut slab: NounSlab = NounSlab::new(); + for noun in parity_fixtures(&mut slab) { + slab.set_root(noun); + let jammed = slab.jam(); + assert_roundtrip_matrix(jammed, true); + } + } + + #[test] + fn chaff_idempotence_chain_jam_fixtures() { + let mut slab: NounSlab = NounSlab::new(); + for noun in parity_fixtures(&mut slab) { + slab.set_root(noun); + assert_idempotence_canonical_chain(slab.jam(), None, 4); + } + } + + #[test] + fn chaff_parity_atom_bit_size_boundaries() { + let mut slab: NounSlab = NounSlab::new(); + for bits in [63usize, 64, 65] { + let atom = atom_with_exact_bit_size(&mut slab, bits); + let noun = T(&mut slab, &[D(bits as u64), atom.as_noun(), atom.as_noun()]); + assert_parity_with_nock(noun, &mut slab); + } + } + + #[test] + fn chaff_parity_backref_threshold_boundaries() { + let mut slab: NounSlab = NounSlab::new(); + for prefix_len in [7usize, 8, 9, 15, 16, 17, 31, 32, 33, 63, 64, 65, 127, 128, 129] { + let repeated = atom_with_exact_bit_size(&mut slab, 9); + let noun = build_backref_threshold_fixture(&mut slab, prefix_len, repeated); + assert_parity_with_nock(noun, &mut slab); + } + } + + #[test] + fn cue_non_zero_trailing_bytes_policy_is_accepted_and_canonicalized() { + let mut source: NounSlab = NounSlab::new(); + let original = T(&mut source, &[D(1), D(2), D(3), D(4), D(5)]); + source.set_root(original); + let canonical = source.jam(); + let original_root = unsafe { *source.root() }; + let source_space = source.noun_space(); + + let mut trailing = canonical.to_vec(); + trailing.push(0b1010_0101); + let with_trailing = Bytes::from(trailing); + + let mut chaff_slab: NounSlab = NounSlab::new(); + let chaff_noun = chaff_slab + .cue_into(with_trailing.clone()) + .expect("chaff cue should accept trailing bytes"); + let chaff_space = chaff_slab.noun_space(); + assert!( + nouns_equal(original_root, &source_space, chaff_noun, &chaff_space), + "chaff cue with non-zero trailing bytes should preserve noun" + ); + chaff_slab.set_root(chaff_noun); + assert_eq!( + chaff_slab.jam(), + canonical, + "chaff re-jam should canonicalize trailing bytes away" + ); + + let mut nock_slab: NounSlab = NounSlab::new(); + let nock_noun = nock_slab + .cue_into(with_trailing.clone()) + .expect("NockJammer cue should accept trailing bytes"); + let nock_space = nock_slab.noun_space(); + assert!( + nouns_equal(original_root, &source_space, nock_noun, &nock_space), + "NockJammer cue with non-zero trailing bytes should preserve noun" + ); + assert_eq!( + NockJammer::jam(nock_noun, &nock_space), + canonical, + "NockJammer re-jam should canonicalize trailing bytes away" + ); + + let mut nockvm_stack = NockStack::new( + stack_words_for_noun(original_root, &source_space, with_trailing.len()), + 0, + ); + let nockvm_noun = ::cue_bytes(&mut nockvm_stack, &with_trailing) + .expect("nockvm cue should accept trailing bytes"); + let nockvm_space = nockvm_stack.noun_space(); + assert!( + nouns_equal(original_root, &source_space, nockvm_noun, &nockvm_space), + "nockvm cue with non-zero trailing bytes should preserve noun" + ); + } + + #[test] + fn chaff_chain_canonicalizes_non_zero_trailing_bytes() { + let mut source: NounSlab = NounSlab::new(); + let original = T(&mut source, &[D(1), D(2), D(3), D(4), D(5)]); + source.set_root(original); + let canonical = source.jam(); + + let mut trailing = canonical.to_vec(); + trailing.extend_from_slice(&[0xA5, 0x01]); + assert_idempotence_canonical_chain(Bytes::from(trailing), Some(canonical), 4); + } + + #[test] + fn chaff_chain_canonicalizes_nockvm_padded_jam() { + let mut slab: NounSlab = NounSlab::new(); + let noun = T(&mut slab, &[D(7), D(8), D(9), D(10), D(11)]); + slab.set_root(noun); + let canonical = slab.jam(); + let root = unsafe { *slab.root() }; + let root_space = slab.noun_space(); + let mut stack = NockStack::new(stack_words_for_noun(root, &root_space, canonical.len()), 0); + let stack_noun = stack.copy_into(root, &root_space); + let nockvm_jam = nockvm::serialization::jam(&mut stack, stack_noun); + let stack_space = stack.noun_space(); + let padded = atom_bytes(nockvm_jam, &stack_space); + assert_idempotence_canonical_chain(padded, Some(canonical), 4); + } + + #[test] + fn nockjammer_parity_jam_fixtures() { + let mut slab: NounSlab = NounSlab::new(); + for noun in parity_fixtures(&mut slab) { + let space = slab.noun_space(); + let jammed = NockJammer::jam(noun, &space); + assert_roundtrip_matrix(jammed, true); + } + } + + #[test] + fn nockstack_parity_jam_fixtures() { + let mut slab: NounSlab = NounSlab::new(); + for noun in parity_fixtures(&mut slab) { + slab.set_root(noun); + let expected = slab.jam(); + let space = slab.noun_space(); + let jammed = jam_nockstack_trimmed(noun, &space, expected.len()); + assert_roundtrip_matrix(jammed, true); + } + } + + #[test] + fn chaff_parity_cue_large_checkpoint() { + let jammed = large_checkpoint_jam(); + let stack_words = stack_words_for_large_checkpoint(jammed.len()); + let (chaff_slab, chaff_noun, nock_slab, nock_noun, stack, nockvm_noun) = + cue_all_with_stack(&jammed, stack_words); + let chaff_space = chaff_slab.noun_space(); + let nock_space = nock_slab.noun_space(); + let stack_space = stack.noun_space(); + assert!( + nouns_equal(chaff_noun, &chaff_space, nock_noun, &nock_space), + "NockJammer cue differs from Chaff cue" + ); + assert!( + nouns_equal(chaff_noun, &chaff_space, nockvm_noun, &stack_space), + "nockvm cue differs from Chaff cue" + ); + assert_eq!(chaff_slab.jam(), jammed, "chaff jam differs from input jam"); + } + + #[test] + fn nockjammer_parity_cue_large_checkpoint() { + let jammed = large_checkpoint_jam(); + let stack_words = stack_words_for_large_checkpoint(jammed.len()); + let (chaff_slab, chaff_noun, nock_slab, nock_noun, stack, nockvm_noun) = + cue_all_with_stack(&jammed, stack_words); + let chaff_space = chaff_slab.noun_space(); + let nock_space = nock_slab.noun_space(); + let stack_space = stack.noun_space(); + assert!( + nouns_equal(nock_noun, &nock_space, chaff_noun, &chaff_space), + "Chaff cue differs from NockJammer cue" + ); + assert!( + nouns_equal(nock_noun, &nock_space, nockvm_noun, &stack_space), + "nockvm cue differs from NockJammer cue" + ); + assert_eq!( + NockJammer::jam(nock_noun, &nock_space), + jammed, + "NockJammer jam differs from input jam" + ); + } + + #[test] + fn nockstack_parity_cue_large_checkpoint() { + let jammed = large_checkpoint_jam(); + let stack_words = stack_words_for_large_checkpoint(jammed.len()); + let (chaff_slab, chaff_noun, nock_slab, nock_noun, mut stack, nockvm_noun) = + cue_all_with_stack(&jammed, stack_words); + let chaff_space = chaff_slab.noun_space(); + let nock_space = nock_slab.noun_space(); + let stack_space = stack.noun_space(); + assert!( + nouns_equal(nockvm_noun, &stack_space, chaff_noun, &chaff_space), + "Chaff cue differs from nockvm cue" + ); + assert!( + nouns_equal(nockvm_noun, &stack_space, nock_noun, &nock_space), + "NockJammer cue differs from nockvm cue" + ); + let nockvm_jam = nockvm::serialization::jam(&mut stack, nockvm_noun); + let nockvm_handle = nockvm_jam.in_space(&stack_space); + let nockvm_bytes = nockvm_handle.as_ne_bytes(); + assert!( + nockvm_bytes.len() >= jammed.len(), + "nockvm jam shorter than input jam" + ); + assert_eq!( + &nockvm_bytes[..jammed.len()], + jammed.as_ref(), + "nockvm jam prefix differs from input jam" + ); + assert!( + nockvm_bytes[jammed.len()..].iter().all(|b| *b == 0), + "nockvm jam has non-zero padding bytes" + ); + } + + #[tokio::test] + async fn saver_chaff_loads_v1_checkpoint() { + let temp = TempDir::new().expect("create temp dir"); + let state_value = 5; + let cold_value = 9; + let checkpoint = JammedCheckpointV1::new( + hash(b"legacy-v1-chaff"), + 7, + legacy_pair_jam(state_value, cold_value), + ); + let saveable = + load_saveable_with_chaff(&temp, checkpoint.encode().expect("encode v1")).await; + assert_loaded_values( + &saveable, + hash(b"legacy-v1-chaff"), + 7, + state_value, + cold_value, + ); + } + + #[tokio::test] + async fn saver_chaff_loads_v0_checkpoint() { + let temp = TempDir::new().expect("create temp dir"); + let state_value = 11; + let cold_value = 22; + let checkpoint = JammedCheckpointV0::new( + false, + hash(b"legacy-v0-chaff"), + 3, + legacy_pair_jam(state_value, cold_value), + ); + let saveable = + load_saveable_with_chaff(&temp, checkpoint.encode().expect("encode v0")).await; + assert_loaded_values( + &saveable, + hash(b"legacy-v0-chaff"), + 3, + state_value, + cold_value, + ); + } + + #[tokio::test] + async fn saver_chaff_loads_v2_checkpoint() { + let temp = TempDir::new().expect("create temp dir"); + let state_value = 1_000_001; + let cold_value = 2_000_002; + let checkpoint = JammedCheckpointV2::new( + hash(b"v2-chaff"), + 42, + jam_atom_with_chaff(cold_value), + jam_atom_with_chaff(state_value), + ); + let saveable = + load_saveable_with_chaff(&temp, checkpoint.encode().expect("encode v2")).await; + assert_loaded_values(&saveable, hash(b"v2-chaff"), 42, state_value, cold_value); + } + + #[test] + fn prop_chaff_roundtrip_tree_tier1() { + let tests = qc_test_count("CHAFF_QC_TIER1_ROUNDTRIP_TESTS", 96); + quickcheck::QuickCheck::new() + .tests(tests) + .quickcheck(prop_chaff_roundtrip_tree_tier1_case as fn(Tier1Tree) -> TestResult); + } + + fn prop_chaff_roundtrip_tree_tier1_case(tree: Tier1Tree) -> TestResult { + prop_chaff_roundtrip_tree_impl(&tree.0) + } + + #[test] + fn prop_chaff_roundtrip_tree_tier2() { + let tests = qc_test_count("CHAFF_QC_TIER2_ROUNDTRIP_TESTS", 24); + quickcheck::QuickCheck::new() + .tests(tests) + .quickcheck(prop_chaff_roundtrip_tree_tier2_case as fn(Tier2Tree) -> TestResult); + } + + fn prop_chaff_roundtrip_tree_tier2_case(tree: Tier2Tree) -> TestResult { + prop_chaff_roundtrip_tree_impl(&tree.0) + } + + #[test] + fn prop_chaff_roundtrip_tree_tier3() { + let tests = qc_test_count("CHAFF_QC_TIER3_ROUNDTRIP_TESTS", 8); + quickcheck::QuickCheck::new() + .tests(tests) + .quickcheck(prop_chaff_roundtrip_tree_tier3_case as fn(Tier3Tree) -> TestResult); + } + + fn prop_chaff_roundtrip_tree_tier3_case(tree: Tier3Tree) -> TestResult { + prop_chaff_roundtrip_tree_impl(&tree.0) + } + + #[test] + fn prop_chaff_matches_nock_tree() { + let tests = qc_test_count("CHAFF_QC_MATCH_NOCK_TESTS", 64); + quickcheck::QuickCheck::new() + .tests(tests) + .quickcheck(prop_chaff_matches_nock_tree_case as fn(Tier1Tree) -> TestResult); + } + + fn prop_chaff_matches_nock_tree_case(tree: Tier1Tree) -> TestResult { + prop_chaff_matches_nock_tree_impl(&tree.0) + } + + #[test] + fn prop_chaff_parity_tree_matrix() { + let tests = qc_test_count("CHAFF_QC_PARITY_MATRIX_TESTS", 40); + quickcheck::QuickCheck::new() + .tests(tests) + .quickcheck(prop_chaff_parity_tree_matrix_case as fn(Tier1Tree) -> TestResult); + } + + fn prop_chaff_parity_tree_matrix_case(tree: Tier1Tree) -> TestResult { + prop_chaff_parity_tree_matrix_impl(&tree.0) + } + + #[test] + fn prop_chaff_parity_shared_tree() { + let tests = qc_test_count("CHAFF_QC_PARITY_SHARED_TESTS", 40); + quickcheck::QuickCheck::new().tests(tests).quickcheck( + prop_chaff_parity_shared_tree_case as fn(Tier1Tree, u8, usize) -> TestResult, + ); + } + + fn prop_chaff_parity_shared_tree_case( + tree: Tier1Tree, + mode: u8, + prefix_seed: usize, + ) -> TestResult { + prop_chaff_parity_shared_tree_impl(&tree.0, mode, prefix_seed) + } + + #[test] + fn prop_chaff_idempotence_chain_tree() { + let tests = qc_test_count("CHAFF_QC_CHAIN_TESTS", 32); + quickcheck::QuickCheck::new() + .tests(tests) + .quickcheck(prop_chaff_idempotence_chain_tree_case as fn(Tier1Tree) -> TestResult); + } + + fn prop_chaff_idempotence_chain_tree_case(tree: Tier1Tree) -> TestResult { + prop_chaff_idempotence_chain_tree_impl(&tree.0) + } + + #[test] + fn prop_chaff_adversarial_mutations_no_panic() { + let tests = qc_test_count("CHAFF_QC_ADVERSARIAL_TESTS", 80); + quickcheck::QuickCheck::new().tests(tests).quickcheck( + prop_chaff_adversarial_mutations_no_panic_case as fn(JamMutationCase) -> TestResult, + ); + } + + fn prop_chaff_adversarial_mutations_no_panic_case(case: JamMutationCase) -> TestResult { + prop_chaff_adversarial_mutations_no_panic_impl(case) + } + + #[test] + fn prop_chaff_adversarial_mutations_differential() { + let tests = qc_test_count("CHAFF_QC_ADVERSARIAL_DIFF_TESTS", 64); + quickcheck::QuickCheck::new().tests(tests).quickcheck( + prop_chaff_adversarial_mutations_differential_case as fn(JamMutationCase) -> TestResult, + ); + } + + fn prop_chaff_adversarial_mutations_differential_case(case: JamMutationCase) -> TestResult { + prop_chaff_adversarial_mutations_differential_impl(case) + } +} diff --git a/crates/chaff/src/lib.rs b/crates/chaff/src/lib.rs index d1e633652..7c6a9b720 100644 --- a/crates/chaff/src/lib.rs +++ b/crates/chaff/src/lib.rs @@ -1,1973 +1,6 @@ -use bytes::Bytes; -use either::Either; -use habit::{BitReader, BitWriter}; -use intmap::IntMap; -use nockapp::noun::slab::{CueError, Jammer, NounMap, NounSlab}; -use nockvm::noun::{Atom, Cell, CellMemory, DirectAtom, IndirectAtom, Noun, D}; -use nockvm::serialization::{met0_u64_to_usize, met0_usize}; +mod jammer; -pub struct Chaff; - -const MAX_USIZE_BITS: usize = usize::BITS as usize; - -impl Jammer for Chaff { - fn cue(slab: &mut NounSlab, bytes: Bytes) -> Result { - fn rub_backref(reader: &mut BitReader) -> Result { - let zeros = reader.read_unary().ok_or(CueError::TruncatedBuffer)?; - if zeros == 0 { - return Ok(0); - } - if zeros > MAX_USIZE_BITS { - return Err(CueError::BackrefTooBig); - } - let size_low = if zeros > 1 { - reader - .read_bits_to_usize(zeros - 1) - .ok_or(CueError::TruncatedBuffer)? - } else { - 0 - }; - let bit_count = (1usize << (zeros - 1)) | size_low; - if bit_count > MAX_USIZE_BITS { - return Err(CueError::BackrefTooBig); - } - reader - .read_bits_to_usize(bit_count) - .ok_or(CueError::TruncatedBuffer) - } - - fn rub_atom(slab: &mut NounSlab, reader: &mut BitReader) -> Result { - let zeros = reader.read_unary().ok_or(CueError::TruncatedBuffer)?; - if zeros == 0 { - return unsafe { Ok(DirectAtom::new_unchecked(0).as_atom()) }; - } - if zeros > MAX_USIZE_BITS { - return Err(CueError::TruncatedBuffer); - } - let size_low = if zeros > 1 { - reader - .read_bits_to_usize(zeros - 1) - .ok_or(CueError::TruncatedBuffer)? - } else { - 0 - }; - let bit_count = (1usize << (zeros - 1)) | size_low; - if bit_count < 64 { - let value = reader - .read_bits_to_usize(bit_count) - .ok_or(CueError::TruncatedBuffer)? as u64; - unsafe { Ok(DirectAtom::new_unchecked(value).as_atom()) } - } else { - if reader.bits_remaining() < bit_count { - return Err(CueError::TruncatedBuffer); - } - let byte_len = bit_count.checked_add(7).ok_or(CueError::TruncatedBuffer)? >> 3; - let (mut atom, buffer) = unsafe { IndirectAtom::new_raw_mut_bytes(slab, byte_len) }; - reader - .read_bits_to_bytes(buffer, bit_count) - .ok_or(CueError::TruncatedBuffer)?; - unsafe { Ok(atom.normalize_as_atom()) } - } - } - - enum CueStackEntry { - DestinationPointer(*mut Noun), - BackRef(u64, *const Noun), - } - - let mut reader = BitReader::new(bytes); - let mut result = D(0); - let mut stack = vec![CueStackEntry::DestinationPointer(&mut result)]; - let mut backrefs: IntMap = IntMap::new(); - - while let Some(entry) = stack.pop() { - match entry { - CueStackEntry::DestinationPointer(dest) => { - let backref_pos = reader.position() as u64; - let tag = reader.read_bit().ok_or(CueError::TruncatedBuffer)?; - if !tag { - let atom = rub_atom(slab, &mut reader)?; - unsafe { - *dest = atom.as_noun(); - } - backrefs.insert(backref_pos, unsafe { *dest }); - } else { - let second = reader.read_bit().ok_or(CueError::TruncatedBuffer)?; - if second { - let backref = rub_backref(&mut reader)? as u64; - let noun = - backrefs.get(backref).copied().ok_or(CueError::BadBackref)?; - unsafe { - *dest = noun; - } - } else { - let (cell, cell_mem): (Cell, *mut CellMemory) = - unsafe { Cell::new_raw_mut(slab) }; - unsafe { - *dest = cell.as_noun(); - } - stack.push(CueStackEntry::BackRef(backref_pos, dest as *const Noun)); - unsafe { - stack - .push(CueStackEntry::DestinationPointer(&mut (*cell_mem).tail)); - stack - .push(CueStackEntry::DestinationPointer(&mut (*cell_mem).head)); - } - } - } - } - CueStackEntry::BackRef(pos, noun_ptr) => { - backrefs.insert(pos, unsafe { *noun_ptr }); - } - } - } - - slab.set_root(result); - Ok(result) - } - - fn jam(noun: Noun) -> Bytes { - fn mat_backref_fast(writer: &mut BitWriter, backref: usize) { - if backref == 0 { - writer.write_bits_from_value(0b111, 3); // 1 1 1 - return; - } - let backref_sz = met0_u64_to_usize(backref as u64); - let backref_sz_sz = met0_u64_to_usize(backref_sz as u64); - // backref tag 1 1 - writer.write_bit(true); - writer.write_bit(true); - // write backref_sz_sz zeros - writer.write_zeros(backref_sz_sz); - // delimiter 1 - writer.write_bit(true); - // write backref_sz_sz-1 bits of backref_sz (LSB first) - writer.write_bits_from_value(backref_sz, backref_sz_sz - 1); - // write backref bits (backref_sz bits) - writer.write_bits_from_value(backref, backref_sz); - } - - fn mat_atom_fast(writer: &mut BitWriter, atom: Atom) { - unsafe { - if atom.as_noun().raw_equals(&D(0)) { - writer.write_bits_from_value(0b10, 2); // 0 1 - return; - } - } - let atom_sz = met0_usize(atom); - let atom_sz_sz = met0_u64_to_usize(atom_sz as u64); - writer.write_bit(false); // atom tag 0 - writer.write_zeros(atom_sz_sz); // size zeros - writer.write_bit(true); // delimiter - // write size bits (atom_sz_sz - 1) - writer.write_bits_from_value(atom_sz, atom_sz_sz - 1); - // write atom bits (little-endian order) - writer.write_bits_from_le_bytes(atom.as_ne_bytes(), atom_sz); - } - - // Main jam implementation ---------------------------------------- - let mut writer = BitWriter::new(); - let mut backref_map = NounMap::::new(); - let mut stack = vec![noun]; - while let Some(noun) = stack.pop() { - if let Some(backref) = backref_map.get(noun) { - // already seen this noun - if let Ok(atom) = noun.as_atom() { - if met0_u64_to_usize(*backref as u64) < met0_usize(atom) { - mat_backref_fast(&mut writer, *backref); - } else { - mat_atom_fast(&mut writer, atom); - } - } else { - mat_backref_fast(&mut writer, *backref); - } - } else { - backref_map.insert(noun, writer.bit_len()); - match noun.as_either_atom_cell() { - Either::Left(atom) => { - mat_atom_fast(&mut writer, atom); - } - Either::Right(cell) => { - // cell tag 1 0 - writer.write_bit(true); - writer.write_bit(false); - // push tail then head (LIFO stack) - stack.push(cell.tail()); - stack.push(cell.head()); - } - } - } - } - - writer.into_bytes() - } -} +pub use jammer::{Chaff, CueError}; #[cfg(test)] -mod tests { - use std::sync::OnceLock; - - use bincode::config::{self, Configuration}; - use bincode::Decode; - use blake3::hash; - use bytes::Bytes; - use habit::BitWriter; - use nockapp::nockapp::save::{ - JammedCheckpointV0, JammedCheckpointV1, JammedCheckpointV2, SaveableCheckpoint, Saver, - JAM_MAGIC_BYTES, - }; - use nockapp::noun::slab::{slab_noun_equality, CueError, Jammer, NockJammer, NounSlab}; - use nockapp::noun::NounAllocatorExt; - use nockapp::utils::NOCK_STACK_SIZE_TINY; - use nockapp::JammedNoun; - use nockvm::ext::{AtomExt, NounExt}; - use nockvm::mem::NockStack; - use nockvm::noun::{Atom, Cell, Noun, D, T}; - use quickcheck::{Arbitrary, Gen, TestResult}; - use tempfile::TempDir; - - use super::Chaff; - - const NOCKVM_STACK_WORDS_MIN: usize = 1 << 20; - const NOCKVM_STACK_WORDS_JAM_SCALE: usize = 8; - const NOCKVM_STACK_WORDS_MASS_SCALE: usize = 4; - - fn atom_from_bytes(slab: &mut NounSlab, bytes: &[u8]) -> Atom { - ::from_bytes(slab, bytes) - } - - fn atom_with_exact_bit_size(slab: &mut NounSlab, bits: usize) -> Atom { - assert!(bits > 0, "atom bit size must be positive"); - let byte_len = (bits + 7) >> 3; - let mut bytes = vec![0u8; byte_len]; - for (idx, byte) in bytes - .iter_mut() - .enumerate() - .take(byte_len.saturating_sub(1)) - { - *byte = ((idx as u8).wrapping_mul(37)).wrapping_add(0x5A); - } - let high_bit = (bits - 1) & 7; - bytes[byte_len - 1] |= 1u8 << high_bit; - atom_from_bytes(slab, &bytes) - } - - fn build_backref_threshold_fixture( - slab: &mut NounSlab, - prefix_len: usize, - repeated: Atom, - ) -> Noun { - let mut list = D(0); - list = Cell::new(slab, repeated.as_noun(), list).as_noun(); - list = Cell::new(slab, repeated.as_noun(), list).as_noun(); - for idx in (0..prefix_len).rev() { - list = Cell::new(slab, D(10_000 + idx as u64), list).as_noun(); - } - list - } - - fn build_shared_noun(slab: &mut NounSlab) -> Noun { - let shared = D(42); - let cell = Cell::new(slab, shared, shared).as_noun(); - T(slab, &[shared, cell, shared]) - } - - #[test] - fn chaff_roundtrip_direct_atom() { - let mut slab: NounSlab = NounSlab::new(); - let noun = D(0); - slab.set_root(noun); - let jammed = slab.jam(); - let mut cued_slab: NounSlab = NounSlab::new(); - let cued = cued_slab.cue_into(jammed).expect("cue should succeed"); - assert!(slab_noun_equality(unsafe { slab.root() }, &cued)); - } - - #[test] - fn chaff_roundtrip_indirect_atom() { - let mut slab: NounSlab = NounSlab::new(); - let bytes = vec![0xFFu8; 32]; - let atom = atom_from_bytes(&mut slab, &bytes); - slab.set_root(atom.as_noun()); - let jammed = slab.jam(); - let mut cued_slab: NounSlab = NounSlab::new(); - let cued = cued_slab.cue_into(jammed).expect("cue should succeed"); - assert!(slab_noun_equality(unsafe { slab.root() }, &cued)); - } - - #[test] - fn chaff_roundtrip_nested_cells() { - let mut slab: NounSlab = NounSlab::new(); - let noun = T(&mut slab, &[D(1), D(2), D(3), D(4)]); - slab.set_root(noun); - let jammed = slab.jam(); - let mut cued_slab: NounSlab = NounSlab::new(); - let cued = cued_slab.cue_into(jammed).expect("cue should succeed"); - assert!(slab_noun_equality(unsafe { slab.root() }, &cued)); - } - - #[test] - fn chaff_handles_shared_backrefs() { - let mut slab: NounSlab = NounSlab::new(); - let noun = build_shared_noun(&mut slab); - slab.set_root(noun); - let jammed = slab.jam(); - let mut cued_slab: NounSlab = NounSlab::new(); - let cued = cued_slab.cue_into(jammed).expect("cue should succeed"); - assert!(slab_noun_equality(unsafe { slab.root() }, &cued)); - } - - #[test] - fn chaff_jam_matches_nock_jam() { - let mut slab: NounSlab = NounSlab::new(); - let noun = T(&mut slab, &[D(5), D(23), D(7)]); - slab.set_root(noun); - let chaff_jam = slab.jam(); - let mut nock_slab: NounSlab = NounSlab::new(); - let copied = nock_slab.copy_into(noun); - let nock_jam = NockJammer::jam(copied); - assert_eq!(chaff_jam, nock_jam); - } - - #[test] - fn chaff_rejects_truncated_input() { - let mut slab: NounSlab = NounSlab::new(); - let jammed = Bytes::from_static(&[0b1]); - let result = slab.cue_into(jammed); - assert!(matches!(result, Err(CueError::TruncatedBuffer))); - } - - #[test] - fn chaff_rejects_bad_backref() { - let mut slab: NounSlab = NounSlab::new(); - let jammed = Bytes::from_static(&[0b0000_0111]); - let result = slab.cue_into(jammed); - assert!(matches!(result, Err(CueError::BadBackref))); - } - - #[test] - fn chaff_rejects_backref_before_definition() { - let mut slab: NounSlab = NounSlab::new(); - let mut writer = BitWriter::new(); - writer.write_bit(true); // tag - writer.write_bit(true); // backref - writer.write_zeros(1); // size-of-size zeros - writer.write_bit(true); // delimiter - writer.write_bit(false); // backref size = 1 - writer.write_bit(true); // backref value = 1 (missing) - let jammed = writer.into_bytes(); - let result = slab.cue_into(jammed); - assert!(matches!(result, Err(CueError::BadBackref))); - } - - #[test] - fn chaff_rejects_backref_far_ahead() { - let mut slab: NounSlab = NounSlab::new(); - let mut writer = BitWriter::new(); - writer.write_bit(true); // tag - writer.write_bit(true); // backref - writer.write_zeros(3); // size-of-size zeros - writer.write_bit(true); // delimiter - writer.write_bits_from_value(0b11, 2); // backref size = 4 - writer.write_bits_from_value(0b1010, 4); // backref = 10 - let jammed = writer.into_bytes(); - let result = slab.cue_into(jammed); - assert!(matches!(result, Err(CueError::BadBackref))); - } - - #[test] - fn chaff_rejects_self_referential_backref() { - let mut slab: NounSlab = NounSlab::new(); - let mut writer = BitWriter::new(); - writer.write_bit(true); // tag - writer.write_bit(true); // backref - writer.write_zeros(1); // size-of-size zeros - writer.write_bit(true); // delimiter - writer.write_bit(false); // backref size = 1 - writer.write_bit(false); // backref value = 0 (self) - let jammed = writer.into_bytes(); - let result = slab.cue_into(jammed); - assert!(matches!(result, Err(CueError::BadBackref))); - } - - #[test] - fn chaff_rejects_backref_too_big() { - let mut slab: NounSlab = NounSlab::new(); - let mut writer = BitWriter::new(); - writer.write_bit(true); // tag - writer.write_bit(true); // backref - writer.write_zeros(usize::BITS as usize + 1); - writer.write_bit(true); // delimiter - let jammed = writer.into_bytes(); - let result = slab.cue_into(jammed); - assert!(matches!(result, Err(CueError::BackrefTooBig))); - } - - #[test] - fn chaff_rejects_truncated_backref_size() { - let mut slab: NounSlab = NounSlab::new(); - let mut writer = BitWriter::new(); - writer.write_bit(true); // tag - writer.write_bit(true); // backref - writer.write_zeros(2); // size-of-size zeros (expect 3 bits for size) - writer.write_bit(true); // delimiter - writer.write_bits_from_value(0b01, 2); // only two size bits - let jammed = writer.into_bytes(); - let result = slab.cue_into(jammed); - assert!(matches!(result, Err(CueError::TruncatedBuffer))); - } - - #[test] - fn chaff_rejects_atom_missing_size_bits() { - let mut slab: NounSlab = NounSlab::new(); - let mut writer = BitWriter::new(); - writer.write_bit(false); // atom tag - writer.write_bit(false); // first zero - writer.write_bit(false); // second zero - // zeros should be 3 (two zeros then delimiter), need 2 bits for size_low - // but we end here - truncated - let jammed = writer.into_bytes(); - assert!(slab.cue_into(jammed).is_err()); - } - - #[test] - fn chaff_rejects_truncated_indirect_atom() { - let mut slab: NounSlab = NounSlab::new(); - let mut writer = BitWriter::new(); - writer.write_bit(false); // atom tag - writer.write_zeros(6); // zeros = 7, expect 6 bits for size_low - writer.write_bit(true); // delimiter - writer.write_bits_from_value(0b100000, 6); // size_low = 33 (bit_count = 65) - writer.write_bits_from_value(0xFFFF, 16); // partial value (needs 65 bits) - let jammed = writer.into_bytes(); - let result = slab.cue_into(jammed); - assert!(matches!(result, Err(CueError::TruncatedBuffer))); - } - - #[test] - fn chaff_rejects_atom_size_prefix_too_big() { - let mut slab: NounSlab = NounSlab::new(); - let mut writer = BitWriter::new(); - writer.write_bit(false); // atom tag - writer.write_zeros(usize::BITS as usize + 1); // zeros > MAX_USIZE_BITS - writer.write_bit(true); // unary delimiter - let jammed = writer.into_bytes(); - let result = slab.cue_into(jammed); - assert!(matches!(result, Err(CueError::TruncatedBuffer))); - } - - #[test] - fn chaff_rejects_huge_atom_size_without_panicking() { - let mut slab: NounSlab = NounSlab::new(); - let mut writer = BitWriter::new(); - writer.write_bit(false); // atom tag - writer.write_zeros(usize::BITS as usize); // zeros == MAX_USIZE_BITS - writer.write_bit(true); // unary delimiter - if usize::BITS > 1 { - writer.write_bits_from_value(usize::MAX >> 1, usize::BITS as usize - 1); - } - let jammed = writer.into_bytes(); - let result = - std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| slab.cue_into(jammed))); - assert!( - result.is_ok(), - "cue should not panic on huge atom size prefix" - ); - assert!(matches!( - result.expect("catch_unwind should succeed"), - Err(CueError::TruncatedBuffer) - )); - } - - #[test] - fn chaff_rejects_zero_atom_bad_encoding() { - let mut slab: NounSlab = NounSlab::new(); - let mut writer = BitWriter::new(); - writer.write_bit(false); // atom tag - writer.write_bit(false); // zeros = 0 (should be "0 1" encoding) - writer.write_bit(false); - let jammed = writer.into_bytes(); - let result = slab.cue_into(jammed); - assert!(matches!(result, Err(CueError::TruncatedBuffer))); - } - - const SHARED_PREFIX_BOUNDARIES: [usize; 15] = - [7, 8, 9, 15, 16, 17, 31, 32, 33, 63, 64, 65, 127, 128, 129]; - - #[derive(Clone, Copy, Debug)] - struct NounTreeProfile { - max_depth: usize, - max_indirect_bytes: usize, - } - - const TIER1_PROFILE: NounTreeProfile = NounTreeProfile { - max_depth: 6, - max_indirect_bytes: 256, - }; - - const TIER2_PROFILE: NounTreeProfile = NounTreeProfile { - max_depth: 10, - max_indirect_bytes: 4 * 1024, - }; - - const TIER3_PROFILE: NounTreeProfile = NounTreeProfile { - max_depth: 14, - max_indirect_bytes: 64 * 1024, - }; - - #[derive(Clone, Debug)] - enum NounTree { - Atom(AtomKind), - Cell(Box, Box), - } - - #[derive(Clone, Debug)] - enum AtomKind { - Zero, - Direct(u64), - Indirect(Vec), - } - - #[derive(Clone, Debug)] - struct Tier1Tree(NounTree); - - #[derive(Clone, Debug)] - struct Tier2Tree(NounTree); - - #[derive(Clone, Debug)] - struct Tier3Tree(NounTree); - - #[derive(Clone, Copy, Debug)] - enum JamMutationKind { - TruncateAtBit, - InflateUnaryPrefix, - DeflateUnaryPrefix, - RewriteBackrefFuture, - RewriteBackrefSelf, - FlipTagBit, - DropPayloadBytes, - AppendNonZeroTrailingByte, - } - - #[derive(Clone, Debug)] - struct JamMutationCase { - base_tree: NounTree, - seed_selector: u8, - mutation: JamMutationKind, - salt: u16, - } - - impl JamMutationKind { - fn is_differential_safe(self) -> bool { - matches!( - self, - JamMutationKind::AppendNonZeroTrailingByte - | JamMutationKind::RewriteBackrefFuture - | JamMutationKind::RewriteBackrefSelf - | JamMutationKind::FlipTagBit - ) - } - } - - impl Arbitrary for JamMutationKind { - fn arbitrary(g: &mut Gen) -> Self { - match usize::arbitrary(g) % 8 { - 0 => Self::TruncateAtBit, - 1 => Self::InflateUnaryPrefix, - 2 => Self::DeflateUnaryPrefix, - 3 => Self::RewriteBackrefFuture, - 4 => Self::RewriteBackrefSelf, - 5 => Self::FlipTagBit, - 6 => Self::DropPayloadBytes, - _ => Self::AppendNonZeroTrailingByte, - } - } - - fn shrink(&self) -> Box> { - Box::new(std::iter::empty()) - } - } - - impl Arbitrary for JamMutationCase { - fn arbitrary(g: &mut Gen) -> Self { - Self { - base_tree: arbitrary_noun_tree(g, TIER1_PROFILE), - seed_selector: u8::arbitrary(g), - mutation: JamMutationKind::arbitrary(g), - salt: u16::arbitrary(g), - } - } - - fn shrink(&self) -> Box> { - let mut out = Vec::new(); - for shrunk in shrink_noun_tree(&self.base_tree) { - out.push(Self { - base_tree: shrunk, - seed_selector: self.seed_selector, - mutation: self.mutation, - salt: self.salt, - }); - } - Box::new(out.into_iter()) - } - } - - impl Arbitrary for Tier1Tree { - fn arbitrary(g: &mut Gen) -> Self { - Self(arbitrary_noun_tree(g, TIER1_PROFILE)) - } - - fn shrink(&self) -> Box> { - Box::new(shrink_noun_tree(&self.0).into_iter().map(Self)) - } - } - - impl Arbitrary for Tier2Tree { - fn arbitrary(g: &mut Gen) -> Self { - Self(arbitrary_noun_tree(g, TIER2_PROFILE)) - } - - fn shrink(&self) -> Box> { - Box::new(shrink_noun_tree(&self.0).into_iter().map(Self)) - } - } - - impl Arbitrary for Tier3Tree { - fn arbitrary(g: &mut Gen) -> Self { - Self(arbitrary_noun_tree(g, TIER3_PROFILE)) - } - - fn shrink(&self) -> Box> { - Box::new(shrink_noun_tree(&self.0).into_iter().map(Self)) - } - } - - fn arbitrary_noun_tree(g: &mut Gen, profile: NounTreeProfile) -> NounTree { - gen_noun_tree(g, profile, profile.max_depth) - } - - fn gen_noun_tree(g: &mut Gen, profile: NounTreeProfile, fuel: usize) -> NounTree { - if fuel == 0 { - return NounTree::Atom(gen_atom_kind(g, profile)); - } - - let choose_cell = (usize::arbitrary(g) % 10) < 6; - if !choose_cell { - return NounTree::Atom(gen_atom_kind(g, profile)); - } - - let remaining = fuel - 1; - let head_fuel = if remaining == 0 { - 0 - } else { - usize::arbitrary(g) % (remaining + 1) - }; - let tail_fuel = remaining.saturating_sub(head_fuel); - NounTree::Cell( - Box::new(gen_noun_tree(g, profile, head_fuel)), - Box::new(gen_noun_tree(g, profile, tail_fuel)), - ) - } - - fn gen_atom_kind(g: &mut Gen, profile: NounTreeProfile) -> AtomKind { - let choice = usize::arbitrary(g) % 100; - match choice { - 0..=2 => AtomKind::Zero, - 3..=34 => { - let val = u64::arbitrary(g) & nockvm::noun::DIRECT_MAX; - AtomKind::Direct(val) - } - 35..=54 => { - let bits = *g.choose(&[63usize, 64, 65]).expect("bits boundary choice"); - let bytes = gen_atom_bytes_exact_bits(g, bits); - if bits < 64 { - AtomKind::Direct(bytes_to_u64(&bytes) & nockvm::noun::DIRECT_MAX) - } else { - AtomKind::Indirect(bytes) - } - } - _ => { - let byte_len = choose_indirect_byte_len(g, profile.max_indirect_bytes); - let mut bytes: Vec = (0..byte_len).map(|_| u8::arbitrary(g)).collect(); - if let Some(last) = bytes.last_mut() { - if *last == 0 { - *last = 1 + (u8::arbitrary(g) % 0x7F); - } - } - AtomKind::Indirect(bytes) - } - } - } - - fn choose_indirect_byte_len(g: &mut Gen, max_indirect_bytes: usize) -> usize { - let min_len = 9usize; - let max_len = max_indirect_bytes.max(min_len); - const BOUNDARY_BYTES: [usize; 12] = [9, 10, 15, 16, 17, 31, 32, 33, 63, 64, 65, 129]; - let use_boundary = (usize::arbitrary(g) % 10) < 6; - if use_boundary { - let candidate = BOUNDARY_BYTES[usize::arbitrary(g) % BOUNDARY_BYTES.len()]; - candidate.min(max_len) - } else { - min_len + (usize::arbitrary(g) % (max_len - min_len + 1)) - } - } - - fn gen_atom_bytes_exact_bits(g: &mut Gen, bits: usize) -> Vec { - let byte_len = bits.div_ceil(8); - let mut bytes: Vec = (0..byte_len).map(|_| u8::arbitrary(g)).collect(); - let high_byte_idx = byte_len - 1; - let high_bit = (bits - 1) & 7; - bytes[high_byte_idx] |= 1u8 << high_bit; - if high_bit < 7 { - bytes[high_byte_idx] &= (1u8 << (high_bit + 1)) - 1; - } - bytes - } - - fn bytes_to_u64(bytes: &[u8]) -> u64 { - let mut value = 0u64; - for (idx, byte) in bytes.iter().copied().enumerate() { - if idx >= 8 { - break; - } - value |= (byte as u64) << (idx * 8); - } - value - } - - fn shrink_noun_tree(tree: &NounTree) -> Vec { - match tree { - NounTree::Atom(kind) => shrink_atom_kind(kind) - .into_iter() - .map(NounTree::Atom) - .collect(), - NounTree::Cell(head, tail) => { - let mut out = vec![(**head).clone(), (**tail).clone()]; - out.extend( - shrink_noun_tree(head) - .into_iter() - .map(|shrunk| NounTree::Cell(Box::new(shrunk), Box::new((**tail).clone()))), - ); - out.extend( - shrink_noun_tree(tail) - .into_iter() - .map(|shrunk| NounTree::Cell(Box::new((**head).clone()), Box::new(shrunk))), - ); - out - } - } - } - - fn shrink_atom_kind(kind: &AtomKind) -> Vec { - match kind { - AtomKind::Zero => Vec::new(), - AtomKind::Direct(value) => { - let mut out = vec![AtomKind::Zero]; - for shrunk in value.shrink() { - out.push(AtomKind::Direct(shrunk & nockvm::noun::DIRECT_MAX)); - } - out - } - AtomKind::Indirect(bytes) => { - let mut out = vec![AtomKind::Zero, AtomKind::Direct(1), AtomKind::Direct(63)]; - for bits in [63usize, 64, 65] { - let mut seed = vec![0u8; bits.div_ceil(8)]; - let high_byte_idx = seed.len() - 1; - let high_bit = (bits - 1) & 7; - seed[high_byte_idx] |= 1u8 << high_bit; - if bits < 64 { - out.push(AtomKind::Direct(bytes_to_u64(&seed))); - } else { - out.push(AtomKind::Indirect(seed)); - } - } - - if bytes.len() > 9 { - let mut shorter = bytes[..bytes.len() / 2].to_vec(); - if shorter.len() < 9 { - shorter.resize(9, 0); - } - if let Some(last) = shorter.last_mut() { - if *last == 0 { - *last = 1; - } - } - out.push(AtomKind::Indirect(shorter)); - } - out - } - } - } - - fn build_noun_from_tree(slab: &mut NounSlab, tree: &NounTree) -> Noun { - match tree { - NounTree::Atom(AtomKind::Zero) => D(0), - NounTree::Atom(AtomKind::Direct(value)) => D(*value), - NounTree::Atom(AtomKind::Indirect(bytes)) => atom_from_bytes(slab, bytes).as_noun(), - NounTree::Cell(head, tail) => { - let h = build_noun_from_tree(slab, head); - let t = build_noun_from_tree(slab, tail); - Cell::new(slab, h, t).as_noun() - } - } - } - - fn pair_with_self(slab: &mut NounSlab, noun: Noun) -> Noun { - Cell::new(slab, noun, noun).as_noun() - } - - fn nested_shared(slab: &mut NounSlab, noun: Noun) -> Noun { - let inner = pair_with_self(slab, noun); - Cell::new(slab, noun, inner).as_noun() - } - - fn alternating_shared(slab: &mut NounSlab, noun: Noun, prefix_len: usize) -> Noun { - let mut list = D(0); - for idx in (0..prefix_len).rev() { - list = Cell::new(slab, D(7_000 + idx as u64), list).as_noun(); - } - let nested = nested_shared(slab, noun); - T(slab, &[list, noun, nested, noun]) - } - - fn seed_noun_for_selector(slab: &mut NounSlab, selector: u8, tree: &NounTree) -> Noun { - match selector % 4 { - 0 => build_noun_from_tree(slab, tree), - 1 => build_shared_noun(slab), - 2 => T(slab, &[D(1), D(2), D(3), D(4), D(5)]), - _ => { - let atom = atom_with_exact_bit_size(slab, 65); - T(slab, &[D(11), atom.as_noun(), D(99)]) - } - } - } - - fn canonical_jam_from_case(case: &JamMutationCase) -> Bytes { - let mut slab: NounSlab = NounSlab::new(); - let noun = seed_noun_for_selector(&mut slab, case.seed_selector, &case.base_tree); - slab.set_root(noun); - slab.jam() - } - - fn truncate_bits(bytes: &[u8], keep_bits: usize) -> Bytes { - if keep_bits == 0 { - return Bytes::new(); - } - let bit_len = bytes.len() * 8; - let keep = keep_bits.min(bit_len); - if keep == bit_len { - return Bytes::copy_from_slice(bytes); - } - let byte_len = keep.div_ceil(8); - let mut out = bytes[..byte_len].to_vec(); - let rem = keep & 7; - if rem != 0 { - let mask = (1u8 << rem) - 1; - let last = out.len() - 1; - out[last] &= mask; - } - Bytes::from(out) - } - - fn set_bit(bytes: &mut Vec, bit_idx: usize, value: bool) { - let byte_idx = bit_idx >> 3; - if byte_idx >= bytes.len() { - bytes.resize(byte_idx + 1, 0); - } - let bit_mask = 1u8 << (bit_idx & 7); - if value { - bytes[byte_idx] |= bit_mask; - } else { - bytes[byte_idx] &= !bit_mask; - } - } - - fn get_bit(bytes: &[u8], bit_idx: usize) -> bool { - let byte_idx = bit_idx >> 3; - if byte_idx >= bytes.len() { - return false; - } - ((bytes[byte_idx] >> (bit_idx & 7)) & 1) == 1 - } - - fn clear_first_set_bit(bytes: &mut Vec, start_bit: usize) -> bool { - let total_bits = bytes.len() * 8; - for bit in start_bit..total_bits { - if get_bit(bytes, bit) { - set_bit(bytes, bit, false); - return true; - } - } - false - } - - fn set_first_clear_bit(bytes: &mut Vec, start_bit: usize) -> bool { - let total_bits = bytes.len() * 8; - for bit in start_bit..total_bits { - if !get_bit(bytes, bit) { - set_bit(bytes, bit, true); - return true; - } - } - false - } - - fn overwrite_prefix_bits(bytes: &mut Vec, prefix_bits: &[bool]) { - for (idx, bit) in prefix_bits.iter().copied().enumerate() { - set_bit(bytes, idx, bit); - } - } - - fn mutate_jam_bytes(canonical: &Bytes, mutation: JamMutationKind, salt: u16) -> Bytes { - let mut bytes = canonical.to_vec(); - match mutation { - JamMutationKind::TruncateAtBit => { - let total_bits = bytes.len() * 8; - if total_bits == 0 { - return Bytes::new(); - } - let keep_bits = (salt as usize) % total_bits; - truncate_bits(&bytes, keep_bits) - } - JamMutationKind::InflateUnaryPrefix => { - if !clear_first_set_bit(&mut bytes, 1) && !bytes.is_empty() { - set_bit(&mut bytes, 0, false); - } - Bytes::from(bytes) - } - JamMutationKind::DeflateUnaryPrefix => { - if !set_first_clear_bit(&mut bytes, 1) { - bytes.push(0x01); - } - Bytes::from(bytes) - } - JamMutationKind::RewriteBackrefFuture => { - overwrite_prefix_bits(&mut bytes, &[true, true, false, true, true, false]); - Bytes::from(bytes) - } - JamMutationKind::RewriteBackrefSelf => { - overwrite_prefix_bits(&mut bytes, &[true, true, false, true, false, false]); - Bytes::from(bytes) - } - JamMutationKind::FlipTagBit => { - if bytes.is_empty() { - bytes.push(0); - } - let flip_idx = (salt as usize) % (bytes.len() * 8); - let current = get_bit(&bytes, flip_idx); - set_bit(&mut bytes, flip_idx, !current); - Bytes::from(bytes) - } - JamMutationKind::DropPayloadBytes => { - if bytes.len() <= 1 { - let keep_bits = (salt as usize) % 4; - return truncate_bits(&bytes, keep_bits); - } - let drop = 1 + ((salt as usize) % bytes.len().min(4)); - bytes.truncate(bytes.len() - drop); - Bytes::from(bytes) - } - JamMutationKind::AppendNonZeroTrailingByte => { - let trailing = (salt as u8) | 1; - bytes.push(trailing); - Bytes::from(bytes) - } - } - } - - fn expected_cue_error(err: CueError) -> bool { - matches!( - err, - CueError::TruncatedBuffer | CueError::BadBackref | CueError::BackrefTooBig - ) - } - - fn qc_test_count(env_key: &str, default: u64) -> u64 { - std::env::var(env_key) - .ok() - .and_then(|raw| raw.parse::().ok()) - .filter(|count| *count > 0) - .unwrap_or(default) - } - - fn prop_chaff_roundtrip_tree_impl(tree: &NounTree) -> TestResult { - let mut slab: NounSlab = NounSlab::new(); - let noun = build_noun_from_tree(&mut slab, tree); - slab.set_root(noun); - let jammed = slab.jam(); - let mut cued: NounSlab = NounSlab::new(); - let cued_noun = match cued.cue_into(jammed) { - Ok(noun) => noun, - Err(_) => return TestResult::failed(), - }; - TestResult::from_bool(slab_noun_equality(unsafe { slab.root() }, &cued_noun)) - } - - fn prop_chaff_matches_nock_tree_impl(tree: &NounTree) -> TestResult { - let mut slab: NounSlab = NounSlab::new(); - let noun = build_noun_from_tree(&mut slab, tree); - slab.set_root(noun); - let chaff_jam = slab.jam(); - let mut nock_slab: NounSlab = NounSlab::new(); - let copied = nock_slab.copy_into(noun); - let nock_jam = NockJammer::jam(copied); - TestResult::from_bool(chaff_jam == nock_jam) - } - - fn prop_chaff_parity_tree_matrix_impl(tree: &NounTree) -> TestResult { - let mut slab: NounSlab = NounSlab::new(); - let noun = build_noun_from_tree(&mut slab, tree); - assert_parity_with_nock(noun, &mut slab); - TestResult::passed() - } - - fn prop_chaff_parity_shared_tree_impl( - tree: &NounTree, - mode: u8, - prefix_seed: usize, - ) -> TestResult { - let mut slab: NounSlab = NounSlab::new(); - let sub = build_noun_from_tree(&mut slab, tree); - let shared = match mode % 3 { - 0 => pair_with_self(&mut slab, sub), - 1 => nested_shared(&mut slab, sub), - _ => { - let prefix = SHARED_PREFIX_BOUNDARIES[prefix_seed % SHARED_PREFIX_BOUNDARIES.len()]; - alternating_shared(&mut slab, sub, prefix) - } - }; - assert_parity_with_nock(shared, &mut slab); - TestResult::passed() - } - - fn prop_chaff_idempotence_chain_tree_impl(tree: &NounTree) -> TestResult { - let mut slab: NounSlab = NounSlab::new(); - let noun = build_noun_from_tree(&mut slab, tree); - slab.set_root(noun); - assert_idempotence_canonical_chain(slab.jam(), None, 4); - TestResult::passed() - } - - fn prop_chaff_adversarial_mutations_no_panic_impl(case: JamMutationCase) -> TestResult { - let canonical = canonical_jam_from_case(&case); - let mutated = mutate_jam_bytes(&canonical, case.mutation, case.salt); - let mut chaff_slab: NounSlab = NounSlab::new(); - let result = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| { - chaff_slab.cue_into(mutated.clone()) - })); - let cue_result = match result { - Ok(outcome) => outcome, - Err(_) => return TestResult::failed(), - }; - - match cue_result { - Ok(noun) => { - chaff_slab.set_root(noun); - let rejam = chaff_slab.jam(); - if matches!(case.mutation, JamMutationKind::AppendNonZeroTrailingByte) { - return TestResult::from_bool(rejam == canonical); - } - TestResult::from_bool(!rejam.is_empty() || canonical.is_empty()) - } - Err(err) => TestResult::from_bool(expected_cue_error(err)), - } - } - - fn prop_chaff_adversarial_mutations_differential_impl(case: JamMutationCase) -> TestResult { - if !case.mutation.is_differential_safe() { - return TestResult::discard(); - } - - let canonical = canonical_jam_from_case(&case); - let mutated = mutate_jam_bytes(&canonical, case.mutation, case.salt); - let mut chaff_slab: NounSlab = NounSlab::new(); - let mut nock_slab: NounSlab = NounSlab::new(); - - let chaff = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| { - chaff_slab.cue_into(mutated.clone()) - })); - let nock = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| { - nock_slab.cue_into(mutated.clone()) - })); - - let chaff = match chaff { - Ok(value) => value, - Err(_) => return TestResult::failed(), - }; - let nock = match nock { - Ok(value) => value, - Err(_) => return TestResult::failed(), - }; - - match (chaff, nock) { - (Err(chaff_err), Err(nock_err)) => { - TestResult::from_bool(expected_cue_error(chaff_err) && expected_cue_error(nock_err)) - } - (Ok(chaff_noun), Ok(nock_noun)) => { - if !slab_noun_equality(&chaff_noun, &nock_noun) { - return TestResult::failed(); - } - chaff_slab.set_root(chaff_noun); - let chaff_rejam = chaff_slab.jam(); - let nock_rejam = NockJammer::jam(nock_noun); - if matches!(case.mutation, JamMutationKind::AppendNonZeroTrailingByte) - && chaff_rejam != canonical - { - return TestResult::failed(); - } - TestResult::from_bool(chaff_rejam == nock_rejam) - } - _ => TestResult::failed(), - } - } - - fn build_list(slab: &mut NounSlab, leaves: &[u16]) -> Noun { - let mut list = D(0); - for value in leaves.iter().rev() { - list = Cell::new(slab, D(*value as u64), list).as_noun(); - } - list - } - - fn parity_fixtures(slab: &mut NounSlab) -> Vec { - let mut fixtures = Vec::new(); - fixtures.push(D(0)); - - let bytes = vec![0xFFu8; 64]; - let atom = atom_from_bytes(slab, &bytes); - fixtures.push(atom.as_noun()); - - fixtures.push(T(slab, &[D(1), D(2), D(3), D(4)])); - fixtures.push(build_shared_noun(slab)); - fixtures.push(build_list(slab, &[1, 2, 3, 4, 5])); - fixtures - } - - fn stack_words_for_jam(bytes_len: usize) -> usize { - let words = bytes_len.div_ceil(8); - let scaled = words.saturating_mul(NOCKVM_STACK_WORDS_JAM_SCALE); - scaled.max(NOCKVM_STACK_WORDS_MIN) - } - - fn stack_words_for_large_checkpoint(bytes_len: usize) -> usize { - stack_words_for_jam(bytes_len).max(NOCK_STACK_SIZE_TINY) - } - - fn stack_words_for_noun(noun: Noun, jam_len: usize) -> usize { - let jam_words = stack_words_for_jam(jam_len); - let mass_words = noun.mass().saturating_mul(NOCKVM_STACK_WORDS_MASS_SCALE); - jam_words.max(mass_words) - } - - #[derive(Decode)] - struct CheckpointEnvelope { - magic_bytes: u64, - version: u32, - payload: Vec, - } - - const SNAPSHOT_VERSION_2: u32 = 2; - - fn jammed_state_from_checkpoint(bytes: &[u8]) -> Bytes { - let config = config::standard(); - - if let Ok((envelope, _)) = - bincode::decode_from_slice::(bytes, config) - { - if envelope.magic_bytes == JAM_MAGIC_BYTES && envelope.version == SNAPSHOT_VERSION_2 { - let (checkpoint, _) = - bincode::decode_from_slice::( - &envelope.payload, config, - ) - .expect("V2 checkpoint payload should decode"); - return checkpoint.state_jam.0; - } - } - - if let Ok((checkpoint, _)) = - bincode::decode_from_slice::(bytes, config) - { - if checkpoint.magic_bytes == JAM_MAGIC_BYTES { - return checkpoint.jam.0; - } - } - - panic!("Failed to decode checkpoint as either V1 or V2 format"); - } - - fn legacy_pair_jam(state_value: u64, cold_value: u64) -> JammedNoun { - let mut slab = NounSlab::::new(); - let state = slab.copy_into(D(state_value)); - let cold = slab.copy_into(D(cold_value)); - let root = T(&mut slab, &[state, cold]); - slab.set_root(root); - JammedNoun::new(slab.coerce_jammer::().jam()) - } - - fn jam_atom_with_chaff(value: u64) -> JammedNoun { - let mut slab = NounSlab::::new(); - let atom = slab.copy_into(D(value)); - slab.set_root(atom); - JammedNoun::new(slab.coerce_jammer::().jam()) - } - - fn atom_value(noun: Noun) -> u64 { - noun.as_atom() - .expect("expected atom") - .as_u64() - .expect("expected atom to fit in u64") - } - - async fn load_saveable_with_chaff(temp: &TempDir, bytes: Vec) -> SaveableCheckpoint { - std::fs::write(temp.path().join("0.chkjam"), bytes).expect("write checkpoint"); - let (_, maybe_saveable) = - Saver::::try_load::(&temp.path().to_path_buf(), None) - .await - .expect("load checkpoint"); - maybe_saveable.expect("expected a checkpoint") - } - - fn assert_loaded_values( - saveable: &SaveableCheckpoint, - expected_hash: blake3::Hash, - event_num: u64, - state_value: u64, - cold_value: u64, - ) { - assert_eq!(saveable.ker_hash, expected_hash); - assert_eq!(saveable.event_num, event_num); - assert_eq!(atom_value(unsafe { *saveable.state.root() }), state_value); - assert_eq!(atom_value(unsafe { *saveable.cold.root() }), cold_value); - } - - fn large_checkpoint_jam() -> Bytes { - static JAM: OnceLock = OnceLock::new(); - JAM.get_or_init(|| { - let bytes = include_bytes!(concat!(env!("CARGO_MANIFEST_DIR"), "/test-jams/0.chkjam")); - jammed_state_from_checkpoint(bytes) - }) - .clone() - } - - fn jam_nockstack_trimmed(noun: Noun, expected_len: usize) -> Bytes { - let mut jam_stack = NockStack::new(stack_words_for_noun(noun, expected_len), 0); - let jam_noun = jam_stack.copy_into(noun); - let jammed = nockvm::serialization::jam(&mut jam_stack, jam_noun); - let bytes = jammed.as_ne_bytes(); - assert!( - bytes.len() >= expected_len, - "nockvm jam shorter than expected" - ); - assert!( - bytes[expected_len..].iter().all(|b| *b == 0), - "nockvm jam has non-zero padding bytes" - ); - Bytes::copy_from_slice(&bytes[..expected_len]) - } - - fn cue_all_with_stack( - jam: &Bytes, - stack_words: usize, - ) -> ( - NounSlab, - Noun, - NounSlab, - Noun, - NockStack, - Noun, - ) { - let mut chaff_slab: NounSlab = NounSlab::new(); - let chaff_noun = chaff_slab - .cue_into(jam.clone()) - .expect("chaff cue should succeed"); - chaff_slab.set_root(chaff_noun); - - let mut nock_slab: NounSlab = NounSlab::new(); - let nock_noun = nock_slab - .cue_into(jam.clone()) - .expect("NockJammer cue should succeed"); - - let mut nockvm_stack = NockStack::new(stack_words, 0); - let nockvm_noun = ::cue_bytes(&mut nockvm_stack, jam) - .expect("nockvm cue should succeed"); - - ( - chaff_slab, chaff_noun, nock_slab, nock_noun, nockvm_stack, nockvm_noun, - ) - } - - fn assert_parity_with_nock(noun: Noun, chaff_slab: &mut NounSlab) { - chaff_slab.set_root(noun); - let chaff_jam = chaff_slab.jam(); - let chaff_root = unsafe { *chaff_slab.root() }; - - let nock_jam = NockJammer::jam(chaff_root); - assert_eq!(chaff_jam, nock_jam, "chaff jam differs from NockJammer"); - - let stack_words = stack_words_for_noun(chaff_root, chaff_jam.len()); - let mut jam_stack = NockStack::new(stack_words, 0); - let stack_noun = jam_stack.copy_into(chaff_root); - let nockvm_jam = nockvm::serialization::jam(&mut jam_stack, stack_noun); - assert!( - nockvm_jam.eq_bytes(&chaff_jam), - "chaff jam differs from nockvm jam (allowing zero padding)" - ); - let nockvm_bytes = Bytes::copy_from_slice(nockvm_jam.as_ne_bytes()); - - let mut chaff_cue_slab: NounSlab = NounSlab::new(); - let chaff_cued = chaff_cue_slab - .cue_into(chaff_jam.clone()) - .expect("chaff cue should succeed"); - assert!( - slab_noun_equality(&chaff_root, &chaff_cued), - "chaff cue differs from original noun" - ); - - let mut nock_cue_slab: NounSlab = NounSlab::new(); - let nock_cued = nock_cue_slab - .cue_into(chaff_jam.clone()) - .expect("NockJammer cue should succeed"); - assert!( - slab_noun_equality(&chaff_root, &nock_cued), - "NockJammer cue differs from original noun" - ); - - let mut cue_stack = NockStack::new(stack_words, 0); - let nockvm_cued = ::cue_bytes(&mut cue_stack, &chaff_jam) - .expect("nockvm cue should succeed"); - assert!( - slab_noun_equality(&chaff_root, &nockvm_cued), - "nockvm cue differs from original noun" - ); - - let mut chaff_cue_slab2: NounSlab = NounSlab::new(); - let chaff_cued_from_nockvm = chaff_cue_slab2 - .cue_into(nockvm_bytes.clone()) - .expect("chaff cue should accept nockvm jam bytes"); - assert!( - slab_noun_equality(&chaff_root, &chaff_cued_from_nockvm), - "chaff cue differs when decoding nockvm jam bytes" - ); - - let mut nock_cue_slab2: NounSlab = NounSlab::new(); - let nock_cued_from_nockvm = nock_cue_slab2 - .cue_into(nockvm_bytes.clone()) - .expect("NockJammer cue should accept nockvm jam bytes"); - assert!( - slab_noun_equality(&chaff_root, &nock_cued_from_nockvm), - "NockJammer cue differs when decoding nockvm jam bytes" - ); - } - - fn assert_roundtrip_matrix(jam: Bytes, full_matrix: bool) { - let mut chaff_slab: NounSlab = NounSlab::new(); - let chaff_noun = chaff_slab - .cue_into(jam.clone()) - .expect("chaff cue should succeed"); - chaff_slab.set_root(chaff_noun); - - let mut nock_slab: NounSlab = NounSlab::new(); - let nock_noun = nock_slab - .cue_into(jam.clone()) - .expect("NockJammer cue should succeed"); - - let stack_words = stack_words_for_noun(chaff_noun, jam.len()); - let mut nockvm_stack = NockStack::new(stack_words, 0); - let nockvm_noun = ::cue_bytes(&mut nockvm_stack, &jam) - .expect("nockvm cue should succeed"); - - assert!( - slab_noun_equality(&chaff_noun, &nock_noun), - "NockJammer cue differs from Chaff cue" - ); - assert!( - slab_noun_equality(&chaff_noun, &nockvm_noun), - "nockvm cue differs from Chaff cue" - ); - - let chaff_jam = chaff_slab.jam(); - assert_eq!(chaff_jam, jam, "chaff jam differs from input jam"); - - let nock_jam = NockJammer::jam(nock_noun); - assert_eq!(nock_jam, jam, "NockJammer jam differs from input jam"); - - let mut jam_stack = NockStack::new(stack_words, 0); - let jam_noun = jam_stack.copy_into(chaff_noun); - let nockvm_jam = nockvm::serialization::jam(&mut jam_stack, jam_noun); - let nockvm_bytes = nockvm_jam.as_ne_bytes(); - assert!( - nockvm_bytes.len() >= jam.len(), - "nockvm jam shorter than input jam" - ); - assert_eq!( - &nockvm_bytes[..jam.len()], - jam.as_ref(), - "nockvm jam prefix differs from input jam" - ); - assert!( - nockvm_bytes[jam.len()..].iter().all(|b| *b == 0), - "nockvm jam has non-zero padding bytes" - ); - - let nockvm_trimmed = Bytes::copy_from_slice(&nockvm_bytes[..jam.len()]); - if !full_matrix { - let mut chaff = NounSlab::::new(); - let chaff_rt = chaff - .cue_into(nockvm_trimmed.clone()) - .expect("chaff cue should succeed"); - assert!( - slab_noun_equality(&chaff_noun, &chaff_rt), - "chaff cue mismatch on nockvm jam" - ); - - let mut nock = NounSlab::::new(); - let nock_rt = nock - .cue_into(nockvm_trimmed) - .expect("NockJammer cue should succeed"); - assert!( - slab_noun_equality(&chaff_noun, &nock_rt), - "NockJammer cue mismatch on nockvm jam" - ); - return; - } - - for jam_bytes in [chaff_jam, nock_jam, nockvm_trimmed] { - let mut chaff = NounSlab::::new(); - let chaff_rt = chaff - .cue_into(jam_bytes.clone()) - .expect("chaff cue should succeed"); - assert!( - slab_noun_equality(&chaff_noun, &chaff_rt), - "chaff roundtrip noun mismatch" - ); - chaff.set_root(chaff_rt); - assert_eq!(chaff.jam(), jam_bytes, "chaff roundtrip jam mismatch"); - - let mut nock = NounSlab::::new(); - let nock_rt = nock - .cue_into(jam_bytes.clone()) - .expect("NockJammer cue should succeed"); - assert!( - slab_noun_equality(&chaff_noun, &nock_rt), - "NockJammer roundtrip noun mismatch" - ); - assert_eq!( - NockJammer::jam(nock_rt), - jam_bytes, - "NockJammer roundtrip jam mismatch" - ); - - let mut stack = NockStack::new(stack_words, 0); - let nockvm_rt = ::cue_bytes(&mut stack, &jam_bytes) - .expect("nockvm cue should succeed"); - assert!( - slab_noun_equality(&chaff_noun, &nockvm_rt), - "nockvm roundtrip noun mismatch" - ); - let nockvm_rt_jam = nockvm::serialization::jam(&mut stack, nockvm_rt); - let nockvm_rt_bytes = nockvm_rt_jam.as_ne_bytes(); - assert!( - nockvm_rt_bytes.len() >= jam_bytes.len(), - "nockvm roundtrip jam shorter than input jam" - ); - assert_eq!( - &nockvm_rt_bytes[..jam_bytes.len()], - jam_bytes.as_ref(), - "nockvm roundtrip jam prefix mismatch" - ); - assert!( - nockvm_rt_bytes[jam_bytes.len()..].iter().all(|b| *b == 0), - "nockvm roundtrip jam has non-zero padding bytes" - ); - } - } - - fn assert_idempotence_canonical_chain( - mut input_jam: Bytes, - expected_canonical: Option, - steps: usize, - ) { - assert!(steps > 0, "chain length must be positive"); - for step in 0..steps { - let mut chaff_slab: NounSlab = NounSlab::new(); - let chaff_noun = chaff_slab - .cue_into(input_jam.clone()) - .expect("chaff cue should succeed in chain"); - chaff_slab.set_root(chaff_noun); - let chaff_jam = chaff_slab.jam(); - - let mut nock_slab: NounSlab = NounSlab::new(); - let nock_noun = nock_slab - .cue_into(input_jam.clone()) - .expect("NockJammer cue should succeed in chain"); - assert!( - slab_noun_equality(&chaff_noun, &nock_noun), - "noun mismatch between Chaff and NockJammer at chain step {step}" - ); - let nock_jam = NockJammer::jam(nock_noun); - assert_eq!( - nock_jam, chaff_jam, - "jam mismatch between Chaff and NockJammer at chain step {step}" - ); - - let stack_words = stack_words_for_noun(chaff_noun, chaff_jam.len()); - let mut stack = NockStack::new(stack_words, 0); - let nockvm_noun = ::cue_bytes(&mut stack, &input_jam) - .expect("nockvm cue should succeed in chain"); - assert!( - slab_noun_equality(&chaff_noun, &nockvm_noun), - "noun mismatch between Chaff and nockvm at chain step {step}" - ); - let nockvm_jam = nockvm::serialization::jam(&mut stack, nockvm_noun); - let nockvm_bytes = nockvm_jam.as_ne_bytes(); - assert!( - nockvm_bytes.len() >= chaff_jam.len(), - "nockvm jam shorter than canonical jam at chain step {step}" - ); - assert_eq!( - &nockvm_bytes[..chaff_jam.len()], - chaff_jam.as_ref(), - "nockvm jam prefix mismatch at chain step {step}" - ); - assert!( - nockvm_bytes[chaff_jam.len()..] - .iter() - .all(|byte| *byte == 0), - "nockvm jam has non-zero padding bytes at chain step {step}" - ); - - if step == 0 { - if let Some(ref canonical) = expected_canonical { - assert_eq!( - &chaff_jam, canonical, - "first chain step should canonicalize to expected bytes" - ); - } else { - assert_eq!( - chaff_jam, input_jam, - "canonical jam should be idempotent on first chain step" - ); - } - } else { - assert_eq!( - chaff_jam, input_jam, - "canonical chain changed unexpectedly at step {step}" - ); - } - - input_jam = chaff_jam; - } - } - - #[test] - fn chaff_roundtrip_list_fixture() { - let mut slab: NounSlab = NounSlab::new(); - let noun = build_list(&mut slab, &[1, 2, 3, 4, 5]); - slab.set_root(noun); - let jammed = slab.jam(); - let mut cued: NounSlab = NounSlab::new(); - let cued_noun = cued.cue_into(jammed).expect("cue should succeed"); - assert!(slab_noun_equality(unsafe { slab.root() }, &cued_noun)); - } - - #[test] - fn chaff_matches_nock_for_shared_fixture() { - let mut slab: NounSlab = NounSlab::new(); - let noun = build_shared_noun(&mut slab); - slab.set_root(noun); - let chaff_jam = slab.jam(); - let mut nock_slab: NounSlab = NounSlab::new(); - let copied = nock_slab.copy_into(noun); - let nock_jam = NockJammer::jam(copied); - assert_eq!(chaff_jam, nock_jam); - } - - #[test] - fn chaff_roundtrip_larger_atom_fixture() { - let mut slab: NounSlab = NounSlab::new(); - let bytes = (0u8..64).collect::>(); - let atom = atom_from_bytes(&mut slab, &bytes); - let noun = T(&mut slab, &[D(7), atom.as_noun(), D(9)]); - slab.set_root(noun); - let jammed = slab.jam(); - let mut cued: NounSlab = NounSlab::new(); - let cued_noun = cued.cue_into(jammed).expect("cue should succeed"); - assert!(slab_noun_equality(unsafe { slab.root() }, &cued_noun)); - } - - #[test] - fn chaff_parity_jam_fixtures() { - let mut slab: NounSlab = NounSlab::new(); - for noun in parity_fixtures(&mut slab) { - slab.set_root(noun); - let jammed = slab.jam(); - assert_roundtrip_matrix(jammed, true); - } - } - - #[test] - fn chaff_idempotence_chain_jam_fixtures() { - let mut slab: NounSlab = NounSlab::new(); - for noun in parity_fixtures(&mut slab) { - slab.set_root(noun); - assert_idempotence_canonical_chain(slab.jam(), None, 4); - } - } - - #[test] - fn chaff_parity_atom_bit_size_boundaries() { - let mut slab: NounSlab = NounSlab::new(); - for bits in [63usize, 64, 65] { - let atom = atom_with_exact_bit_size(&mut slab, bits); - let noun = T(&mut slab, &[D(bits as u64), atom.as_noun(), atom.as_noun()]); - assert_parity_with_nock(noun, &mut slab); - } - } - - #[test] - fn chaff_parity_backref_threshold_boundaries() { - let mut slab: NounSlab = NounSlab::new(); - for prefix_len in [7usize, 8, 9, 15, 16, 17, 31, 32, 33, 63, 64, 65, 127, 128, 129] { - let repeated = atom_with_exact_bit_size(&mut slab, 9); - let noun = build_backref_threshold_fixture(&mut slab, prefix_len, repeated); - assert_parity_with_nock(noun, &mut slab); - } - } - - #[test] - fn cue_non_zero_trailing_bytes_policy_is_accepted_and_canonicalized() { - let mut source: NounSlab = NounSlab::new(); - let original = T(&mut source, &[D(1), D(2), D(3), D(4), D(5)]); - source.set_root(original); - let canonical = source.jam(); - let original_root = unsafe { *source.root() }; - - let mut trailing = canonical.to_vec(); - trailing.push(0b1010_0101); - let with_trailing = Bytes::from(trailing); - - let mut chaff_slab: NounSlab = NounSlab::new(); - let chaff_noun = chaff_slab - .cue_into(with_trailing.clone()) - .expect("chaff cue should accept trailing bytes"); - assert!( - slab_noun_equality(&original_root, &chaff_noun), - "chaff cue with non-zero trailing bytes should preserve noun" - ); - chaff_slab.set_root(chaff_noun); - assert_eq!( - chaff_slab.jam(), - canonical, - "chaff re-jam should canonicalize trailing bytes away" - ); - - let mut nock_slab: NounSlab = NounSlab::new(); - let nock_noun = nock_slab - .cue_into(with_trailing.clone()) - .expect("NockJammer cue should accept trailing bytes"); - assert!( - slab_noun_equality(&original_root, &nock_noun), - "NockJammer cue with non-zero trailing bytes should preserve noun" - ); - assert_eq!( - NockJammer::jam(nock_noun), - canonical, - "NockJammer re-jam should canonicalize trailing bytes away" - ); - - let mut nockvm_stack = - NockStack::new(stack_words_for_noun(original_root, with_trailing.len()), 0); - let nockvm_noun = ::cue_bytes(&mut nockvm_stack, &with_trailing) - .expect("nockvm cue should accept trailing bytes"); - assert!( - slab_noun_equality(&original_root, &nockvm_noun), - "nockvm cue with non-zero trailing bytes should preserve noun" - ); - } - - #[test] - fn chaff_chain_canonicalizes_non_zero_trailing_bytes() { - let mut source: NounSlab = NounSlab::new(); - let original = T(&mut source, &[D(1), D(2), D(3), D(4), D(5)]); - source.set_root(original); - let canonical = source.jam(); - - let mut trailing = canonical.to_vec(); - trailing.extend_from_slice(&[0xA5, 0x01]); - assert_idempotence_canonical_chain(Bytes::from(trailing), Some(canonical), 4); - } - - #[test] - fn chaff_chain_canonicalizes_nockvm_padded_jam() { - let mut slab: NounSlab = NounSlab::new(); - let noun = T(&mut slab, &[D(7), D(8), D(9), D(10), D(11)]); - slab.set_root(noun); - let canonical = slab.jam(); - let root = unsafe { *slab.root() }; - let mut stack = NockStack::new(stack_words_for_noun(root, canonical.len()), 0); - let stack_noun = stack.copy_into(root); - let nockvm_jam = nockvm::serialization::jam(&mut stack, stack_noun); - let padded = Bytes::copy_from_slice(nockvm_jam.as_ne_bytes()); - assert_idempotence_canonical_chain(padded, Some(canonical), 4); - } - - #[test] - fn nockjammer_parity_jam_fixtures() { - let mut slab: NounSlab = NounSlab::new(); - for noun in parity_fixtures(&mut slab) { - let jammed = NockJammer::jam(noun); - assert_roundtrip_matrix(jammed, true); - } - } - - #[test] - fn nockstack_parity_jam_fixtures() { - let mut slab: NounSlab = NounSlab::new(); - for noun in parity_fixtures(&mut slab) { - slab.set_root(noun); - let expected = slab.jam(); - let jammed = jam_nockstack_trimmed(noun, expected.len()); - assert_roundtrip_matrix(jammed, true); - } - } - - #[test] - fn chaff_parity_cue_large_checkpoint() { - let jammed = large_checkpoint_jam(); - let stack_words = stack_words_for_large_checkpoint(jammed.len()); - let (chaff_slab, chaff_noun, _nock_slab, nock_noun, _stack, nockvm_noun) = - cue_all_with_stack(&jammed, stack_words); - assert!( - slab_noun_equality(&chaff_noun, &nock_noun), - "NockJammer cue differs from Chaff cue" - ); - assert!( - slab_noun_equality(&chaff_noun, &nockvm_noun), - "nockvm cue differs from Chaff cue" - ); - assert_eq!(chaff_slab.jam(), jammed, "chaff jam differs from input jam"); - } - - #[test] - fn nockjammer_parity_cue_large_checkpoint() { - let jammed = large_checkpoint_jam(); - let stack_words = stack_words_for_large_checkpoint(jammed.len()); - let (_chaff_slab, chaff_noun, _nock_slab, nock_noun, _stack, nockvm_noun) = - cue_all_with_stack(&jammed, stack_words); - assert!( - slab_noun_equality(&nock_noun, &chaff_noun), - "Chaff cue differs from NockJammer cue" - ); - assert!( - slab_noun_equality(&nock_noun, &nockvm_noun), - "nockvm cue differs from NockJammer cue" - ); - assert_eq!( - NockJammer::jam(nock_noun), - jammed, - "NockJammer jam differs from input jam" - ); - } - - #[test] - fn nockstack_parity_cue_large_checkpoint() { - let jammed = large_checkpoint_jam(); - let stack_words = stack_words_for_large_checkpoint(jammed.len()); - let (_chaff_slab, chaff_noun, _nock_slab, nock_noun, mut stack, nockvm_noun) = - cue_all_with_stack(&jammed, stack_words); - assert!( - slab_noun_equality(&nockvm_noun, &chaff_noun), - "Chaff cue differs from nockvm cue" - ); - assert!( - slab_noun_equality(&nockvm_noun, &nock_noun), - "NockJammer cue differs from nockvm cue" - ); - let nockvm_jam = nockvm::serialization::jam(&mut stack, nockvm_noun); - let nockvm_bytes = nockvm_jam.as_ne_bytes(); - assert!( - nockvm_bytes.len() >= jammed.len(), - "nockvm jam shorter than input jam" - ); - assert_eq!( - &nockvm_bytes[..jammed.len()], - jammed.as_ref(), - "nockvm jam prefix differs from input jam" - ); - assert!( - nockvm_bytes[jammed.len()..].iter().all(|b| *b == 0), - "nockvm jam has non-zero padding bytes" - ); - } - - #[tokio::test] - async fn saver_chaff_loads_v1_checkpoint() { - let temp = TempDir::new().expect("create temp dir"); - let state_value = 5; - let cold_value = 9; - let checkpoint = JammedCheckpointV1::new( - hash(b"legacy-v1-chaff"), - 7, - legacy_pair_jam(state_value, cold_value), - ); - let saveable = - load_saveable_with_chaff(&temp, checkpoint.encode().expect("encode v1")).await; - assert_loaded_values( - &saveable, - hash(b"legacy-v1-chaff"), - 7, - state_value, - cold_value, - ); - } - - #[tokio::test] - async fn saver_chaff_loads_v0_checkpoint() { - let temp = TempDir::new().expect("create temp dir"); - let state_value = 11; - let cold_value = 22; - let checkpoint = JammedCheckpointV0::new( - false, - hash(b"legacy-v0-chaff"), - 3, - legacy_pair_jam(state_value, cold_value), - ); - let saveable = - load_saveable_with_chaff(&temp, checkpoint.encode().expect("encode v0")).await; - assert_loaded_values( - &saveable, - hash(b"legacy-v0-chaff"), - 3, - state_value, - cold_value, - ); - } - - #[tokio::test] - async fn saver_chaff_loads_v2_checkpoint() { - let temp = TempDir::new().expect("create temp dir"); - let state_value = 1_000_001; - let cold_value = 2_000_002; - let checkpoint = JammedCheckpointV2::new( - hash(b"v2-chaff"), - 42, - jam_atom_with_chaff(cold_value), - jam_atom_with_chaff(state_value), - ); - let saveable = - load_saveable_with_chaff(&temp, checkpoint.encode().expect("encode v2")).await; - assert_loaded_values(&saveable, hash(b"v2-chaff"), 42, state_value, cold_value); - } - - #[test] - fn prop_chaff_roundtrip_tree_tier1() { - let tests = qc_test_count("CHAFF_QC_TIER1_ROUNDTRIP_TESTS", 96); - quickcheck::QuickCheck::new() - .tests(tests) - .quickcheck(prop_chaff_roundtrip_tree_tier1_case as fn(Tier1Tree) -> TestResult); - } - - fn prop_chaff_roundtrip_tree_tier1_case(tree: Tier1Tree) -> TestResult { - prop_chaff_roundtrip_tree_impl(&tree.0) - } - - #[test] - fn prop_chaff_roundtrip_tree_tier2() { - let tests = qc_test_count("CHAFF_QC_TIER2_ROUNDTRIP_TESTS", 24); - quickcheck::QuickCheck::new() - .tests(tests) - .quickcheck(prop_chaff_roundtrip_tree_tier2_case as fn(Tier2Tree) -> TestResult); - } - - fn prop_chaff_roundtrip_tree_tier2_case(tree: Tier2Tree) -> TestResult { - prop_chaff_roundtrip_tree_impl(&tree.0) - } - - #[test] - fn prop_chaff_roundtrip_tree_tier3() { - let tests = qc_test_count("CHAFF_QC_TIER3_ROUNDTRIP_TESTS", 8); - quickcheck::QuickCheck::new() - .tests(tests) - .quickcheck(prop_chaff_roundtrip_tree_tier3_case as fn(Tier3Tree) -> TestResult); - } - - fn prop_chaff_roundtrip_tree_tier3_case(tree: Tier3Tree) -> TestResult { - prop_chaff_roundtrip_tree_impl(&tree.0) - } - - #[test] - fn prop_chaff_matches_nock_tree() { - let tests = qc_test_count("CHAFF_QC_MATCH_NOCK_TESTS", 64); - quickcheck::QuickCheck::new() - .tests(tests) - .quickcheck(prop_chaff_matches_nock_tree_case as fn(Tier1Tree) -> TestResult); - } - - fn prop_chaff_matches_nock_tree_case(tree: Tier1Tree) -> TestResult { - prop_chaff_matches_nock_tree_impl(&tree.0) - } - - #[test] - fn prop_chaff_parity_tree_matrix() { - let tests = qc_test_count("CHAFF_QC_PARITY_MATRIX_TESTS", 40); - quickcheck::QuickCheck::new() - .tests(tests) - .quickcheck(prop_chaff_parity_tree_matrix_case as fn(Tier1Tree) -> TestResult); - } - - fn prop_chaff_parity_tree_matrix_case(tree: Tier1Tree) -> TestResult { - prop_chaff_parity_tree_matrix_impl(&tree.0) - } - - #[test] - fn prop_chaff_parity_shared_tree() { - let tests = qc_test_count("CHAFF_QC_PARITY_SHARED_TESTS", 40); - quickcheck::QuickCheck::new().tests(tests).quickcheck( - prop_chaff_parity_shared_tree_case as fn(Tier1Tree, u8, usize) -> TestResult, - ); - } - - fn prop_chaff_parity_shared_tree_case( - tree: Tier1Tree, - mode: u8, - prefix_seed: usize, - ) -> TestResult { - prop_chaff_parity_shared_tree_impl(&tree.0, mode, prefix_seed) - } - - #[test] - fn prop_chaff_idempotence_chain_tree() { - let tests = qc_test_count("CHAFF_QC_CHAIN_TESTS", 32); - quickcheck::QuickCheck::new() - .tests(tests) - .quickcheck(prop_chaff_idempotence_chain_tree_case as fn(Tier1Tree) -> TestResult); - } - - fn prop_chaff_idempotence_chain_tree_case(tree: Tier1Tree) -> TestResult { - prop_chaff_idempotence_chain_tree_impl(&tree.0) - } - - #[test] - fn prop_chaff_adversarial_mutations_no_panic() { - let tests = qc_test_count("CHAFF_QC_ADVERSARIAL_TESTS", 80); - quickcheck::QuickCheck::new().tests(tests).quickcheck( - prop_chaff_adversarial_mutations_no_panic_case as fn(JamMutationCase) -> TestResult, - ); - } - - fn prop_chaff_adversarial_mutations_no_panic_case(case: JamMutationCase) -> TestResult { - prop_chaff_adversarial_mutations_no_panic_impl(case) - } - - #[test] - fn prop_chaff_adversarial_mutations_differential() { - let tests = qc_test_count("CHAFF_QC_ADVERSARIAL_DIFF_TESTS", 64); - quickcheck::QuickCheck::new().tests(tests).quickcheck( - prop_chaff_adversarial_mutations_differential_case as fn(JamMutationCase) -> TestResult, - ); - } - - fn prop_chaff_adversarial_mutations_differential_case(case: JamMutationCase) -> TestResult { - prop_chaff_adversarial_mutations_differential_impl(case) - } -} +include!("legacy_tests.rs"); diff --git a/crates/chaff/src/stream.rs b/crates/chaff/src/stream.rs new file mode 100644 index 000000000..46b079edc --- /dev/null +++ b/crates/chaff/src/stream.rs @@ -0,0 +1,430 @@ +use std::collections::HashMap; +use std::io::{self, Write}; + +use either::Either; +use nockvm::noun::{Atom, Noun, NounSpace, D}; +use nockvm::pma::{classify_pma_noun, PmaDirectJamError, PmaDirectReader, PmaRawNounKind}; +use nockvm::serialization::{met0_u64_to_usize, met0_usize}; + +const STREAM_BUF_BYTES: usize = 32 * 1024; + +#[derive(Debug, Clone, Copy)] +pub struct JamStreamStats { + pub bit_len: usize, + pub byte_len: usize, +} + +pub struct StreamingBitWriter<'a, W: Write> { + writer: &'a mut W, + buf: Vec, + current_byte: u8, + current_bits: u8, + bit_len: usize, +} + +impl<'a, W: Write> StreamingBitWriter<'a, W> { + pub fn new(writer: &'a mut W) -> Self { + Self { + writer, + buf: Vec::with_capacity(STREAM_BUF_BYTES), + current_byte: 0, + current_bits: 0, + bit_len: 0, + } + } + + pub fn bit_len(&self) -> usize { + self.bit_len + } + + fn flush_buf(&mut self) -> io::Result<()> { + if !self.buf.is_empty() { + self.writer.write_all(&self.buf)?; + self.buf.clear(); + } + Ok(()) + } + + fn push_bytes(&mut self, bytes: &[u8]) -> io::Result<()> { + if bytes.is_empty() { + return Ok(()); + } + if self.buf.is_empty() && bytes.len() >= STREAM_BUF_BYTES { + self.writer.write_all(bytes)?; + return Ok(()); + } + let mut start = 0; + while start < bytes.len() { + let remaining = STREAM_BUF_BYTES - self.buf.len(); + if remaining == 0 { + self.flush_buf()?; + } + let end = (start + remaining).min(bytes.len()); + self.buf.extend_from_slice(&bytes[start..end]); + start = end; + } + Ok(()) + } + + fn push_zero_bytes(&mut self, mut count: usize) -> io::Result<()> { + let zeros = [0u8; 4096]; + while count > 0 { + let chunk = count.min(zeros.len()); + self.push_bytes(&zeros[..chunk])?; + count -= chunk; + } + Ok(()) + } + + pub fn write_bit(&mut self, bit: bool) -> io::Result<()> { + if bit { + self.current_byte |= 1u8 << self.current_bits; + } + self.current_bits += 1; + self.bit_len += 1; + if self.current_bits == 8 { + self.push_bytes(&[self.current_byte])?; + self.current_byte = 0; + self.current_bits = 0; + } + Ok(()) + } + + pub fn write_zeros(&mut self, count: usize) -> io::Result<()> { + let mut remaining = count; + if self.current_bits == 0 { + let full_bytes = remaining / 8; + if full_bytes > 0 { + self.push_zero_bytes(full_bytes)?; + self.bit_len += full_bytes * 8; + remaining -= full_bytes * 8; + } + } + for _ in 0..remaining { + self.write_bit(false)?; + } + Ok(()) + } + + pub fn write_bits_from_value(&mut self, mut value: u64, bits: usize) -> io::Result<()> { + let mut remaining = bits; + if self.current_bits == 0 { + while remaining >= 8 { + self.push_bytes(&[value as u8])?; + self.bit_len += 8; + value >>= 8; + remaining -= 8; + } + } + for _ in 0..remaining { + self.write_bit((value & 1) != 0)?; + value >>= 1; + } + Ok(()) + } + + pub fn write_bits_from_le_bytes(&mut self, bytes: &[u8], bits: usize) -> io::Result<()> { + let mut remaining = bits; + let mut offset = 0usize; + if self.current_bits == 0 { + let full_bytes = remaining / 8; + if full_bytes > 0 { + self.push_bytes(&bytes[..full_bytes])?; + self.bit_len += full_bytes * 8; + remaining -= full_bytes * 8; + offset = full_bytes; + } + } + if remaining > 0 { + if self.current_bits == 0 { + let byte = bytes + .get(offset) + .copied() + .ok_or_else(|| io::Error::new(io::ErrorKind::UnexpectedEof, "atom bytes"))?; + for bit in 0..remaining { + self.write_bit((byte & (1u8 << bit)) != 0)?; + } + } else { + let mut idx = offset; + while remaining > 0 { + let byte = bytes.get(idx).copied().ok_or_else(|| { + io::Error::new(io::ErrorKind::UnexpectedEof, "atom bytes") + })?; + let take = remaining.min(8); + for bit in 0..take { + self.write_bit((byte & (1u8 << bit)) != 0)?; + } + remaining -= take; + idx += 1; + } + } + } + Ok(()) + } + + pub fn finish(&mut self) -> io::Result { + if self.current_bits != 0 { + self.push_bytes(&[self.current_byte])?; + self.current_byte = 0; + self.current_bits = 0; + } + self.flush_buf()?; + self.writer.flush()?; + let byte_len = (self.bit_len + 7) / 8; + Ok(JamStreamStats { + bit_len: self.bit_len, + byte_len, + }) + } +} + +pub fn jam_pma_to_writer( + reader: &mut PmaDirectReader, + root_raw: u64, + writer: &mut W, +) -> Result { + let mut bit_writer = StreamingBitWriter::new(writer); + let mut backrefs: HashMap = HashMap::new(); + let mut stack = Vec::new(); + stack.push(root_raw); + + while let Some(noun_raw) = stack.pop() { + let kind = classify_pma_noun(noun_raw)?; + if let Some(backref) = backrefs.get(&noun_raw).copied() { + match kind { + PmaRawNounKind::Direct(value) => { + let atom_bits = met0_u64_to_usize(value); + if met0_u64_to_usize(backref as u64) < atom_bits { + mat_backref(&mut bit_writer, backref)?; + } else { + mat_direct_atom(&mut bit_writer, value)?; + } + } + PmaRawNounKind::Indirect { offset } => { + let atom_bits = reader.indirect_atom_bits(offset)?; + if met0_u64_to_usize(backref as u64) < atom_bits { + mat_backref(&mut bit_writer, backref)?; + } else { + mat_indirect_atom(reader, &mut bit_writer, offset, atom_bits)?; + } + } + PmaRawNounKind::Cell { .. } => { + mat_backref(&mut bit_writer, backref)?; + } + } + continue; + } + + backrefs.insert(noun_raw, bit_writer.bit_len()); + match kind { + PmaRawNounKind::Direct(value) => { + mat_direct_atom(&mut bit_writer, value)?; + } + PmaRawNounKind::Indirect { offset } => { + let atom_bits = reader.indirect_atom_bits(offset)?; + mat_indirect_atom(reader, &mut bit_writer, offset, atom_bits)?; + } + PmaRawNounKind::Cell { offset } => { + mat_cell(&mut bit_writer)?; + let (head, tail) = reader.read_cell(offset)?; + stack.push(tail); + stack.push(head); + } + } + } + + Ok(bit_writer.finish()?) +} + +pub fn jam_noun_to_writer( + noun: Noun, + space: &NounSpace, + writer: &mut W, +) -> io::Result { + fn mat_backref( + writer: &mut StreamingBitWriter<'_, W>, + backref: usize, + ) -> io::Result<()> { + if backref == 0 { + writer.write_bits_from_value(0b111, 3)?; + return Ok(()); + } + let backref_sz = met0_u64_to_usize(backref as u64); + let backref_sz_sz = met0_u64_to_usize(backref_sz as u64); + writer.write_bit(true)?; + writer.write_bit(true)?; + writer.write_zeros(backref_sz_sz)?; + writer.write_bit(true)?; + writer.write_bits_from_value(backref_sz as u64, backref_sz_sz - 1)?; + writer.write_bits_from_value(backref as u64, backref_sz)?; + Ok(()) + } + + fn mat_atom( + writer: &mut StreamingBitWriter<'_, W>, + atom: Atom, + space: &NounSpace, + ) -> io::Result<()> { + unsafe { + if atom.as_noun().raw_equals(&D(0)) { + writer.write_bits_from_value(0b10, 2)?; + return Ok(()); + } + } + let atom_sz = met0_usize(atom, space); + let atom_sz_sz = met0_u64_to_usize(atom_sz as u64); + writer.write_bit(false)?; + writer.write_zeros(atom_sz_sz)?; + writer.write_bit(true)?; + writer.write_bits_from_value(atom_sz as u64, atom_sz_sz - 1)?; + writer.write_bits_from_le_bytes(atom.in_space(space).as_ne_bytes(), atom_sz)?; + Ok(()) + } + + let mut bit_writer = StreamingBitWriter::new(writer); + let mut backrefs: HashMap = HashMap::new(); + let mut stack = vec![noun]; + + while let Some(noun) = stack.pop() { + let raw = unsafe { noun.as_raw() }; + if let Some(backref) = backrefs.get(&raw).copied() { + if let Ok(atom) = noun.as_atom() { + if met0_u64_to_usize(backref as u64) < met0_usize(atom, space) { + mat_backref(&mut bit_writer, backref)?; + } else { + mat_atom(&mut bit_writer, atom, space)?; + } + } else { + mat_backref(&mut bit_writer, backref)?; + } + continue; + } + + backrefs.insert(raw, bit_writer.bit_len()); + match noun.as_either_atom_cell() { + Either::Left(atom) => { + mat_atom(&mut bit_writer, atom, space)?; + } + Either::Right(cell) => { + bit_writer.write_bit(true)?; + bit_writer.write_bit(false)?; + let cell = cell.in_space(space); + stack.push(cell.tail().noun()); + stack.push(cell.head().noun()); + } + } + } + + bit_writer.finish() +} + +fn mat_backref( + writer: &mut StreamingBitWriter<'_, W>, + backref: usize, +) -> Result<(), PmaDirectJamError> { + if backref == 0 { + writer.write_bits_from_value(0b111, 3)?; + return Ok(()); + } + let backref_sz = met0_u64_to_usize(backref as u64); + let backref_sz_sz = met0_u64_to_usize(backref_sz as u64); + writer.write_bit(true)?; + writer.write_bit(true)?; + writer.write_zeros(backref_sz_sz)?; + writer.write_bit(true)?; + writer.write_bits_from_value(backref_sz as u64, backref_sz_sz - 1)?; + writer.write_bits_from_value(backref as u64, backref_sz)?; + Ok(()) +} + +fn mat_direct_atom( + writer: &mut StreamingBitWriter<'_, W>, + value: u64, +) -> Result<(), PmaDirectJamError> { + if value == 0 { + writer.write_bits_from_value(0b10, 2)?; + return Ok(()); + } + let atom_bits = met0_u64_to_usize(value); + mat_atom_header(writer, atom_bits)?; + writer.write_bits_from_value(value, atom_bits)?; + Ok(()) +} + +fn mat_indirect_atom( + reader: &mut PmaDirectReader, + writer: &mut StreamingBitWriter<'_, W>, + offset: u64, + atom_bits: usize, +) -> Result<(), PmaDirectJamError> { + if atom_bits == 0 { + writer.write_bits_from_value(0b10, 2)?; + return Ok(()); + } + let size_words = reader.indirect_atom_words(offset)?; + mat_atom_header(writer, atom_bits)?; + let last_bits = atom_bits.saturating_sub((size_words - 1).saturating_mul(64)); + for i in 0..size_words { + let word = reader.read_u64(offset + 2 + i as u64)?; + let bits = if i + 1 == size_words { last_bits } else { 64 }; + writer.write_bits_from_value(word, bits)?; + } + Ok(()) +} + +fn mat_atom_header( + writer: &mut StreamingBitWriter<'_, W>, + atom_bits: usize, +) -> Result<(), PmaDirectJamError> { + let atom_sz_sz = met0_u64_to_usize(atom_bits as u64); + writer.write_bit(false)?; + writer.write_zeros(atom_sz_sz)?; + writer.write_bit(true)?; + writer.write_bits_from_value(atom_bits as u64, atom_sz_sz - 1)?; + Ok(()) +} + +fn mat_cell(writer: &mut StreamingBitWriter<'_, W>) -> Result<(), PmaDirectJamError> { + writer.write_bits_from_value(0b01, 2)?; + Ok(()) +} + +#[cfg(test)] +mod tests { + use nockvm::ext::AtomExt; + use nockvm::mem::{NockStack, NOCK_STACK_SIZE_TINY}; + use nockvm::noun::{Atom, D, T}; + + use crate::stream::jam_noun_to_writer; + use crate::Chaff; + + const TOP_SLOTS: usize = 1 << 10; + + fn fresh_stack() -> NockStack { + NockStack::new(NOCK_STACK_SIZE_TINY, TOP_SLOTS) + } + + #[test] + fn stream_jam_matches_chaff_simple_cell() { + let mut stack = fresh_stack(); + let noun = T(&mut stack, &[D(1), D(2), D(3)]); + let jammed = Chaff::jam(noun, &stack.noun_space()); + let space = stack.noun_space(); + let mut out = Vec::new(); + let _stats = jam_noun_to_writer(noun, &space, &mut out).expect("stream jam"); + assert_eq!(out, jammed.as_ref()); + } + + #[test] + fn stream_jam_matches_chaff_indirect_atom() { + let mut stack = fresh_stack(); + let bytes = vec![0xA5u8; 64]; + let atom = ::from_bytes(&mut stack, &bytes); + let noun = T(&mut stack, &[D(7), atom.as_noun()]); + let jammed = Chaff::jam(noun, &stack.noun_space()); + let space = stack.noun_space(); + let mut out = Vec::new(); + let _stats = jam_noun_to_writer(noun, &space, &mut out).expect("stream jam"); + assert_eq!(out, jammed.as_ref()); + } +} diff --git a/crates/habit/src/lib.rs b/crates/habit/src/lib.rs index 79fd670d7..3eaa250d5 100644 --- a/crates/habit/src/lib.rs +++ b/crates/habit/src/lib.rs @@ -486,7 +486,6 @@ mod tests { } } } - #[test] fn reads_unary_across_bytes() { let bytes = Bytes::from(vec![0b0000_0000, 0b0001_0000]); @@ -641,7 +640,6 @@ mod tests { } } } - quickcheck::quickcheck! { fn prop_write_then_read_bits(payload: Vec, bit_count: usize) -> TestResult { let bits = bit_count % 129; diff --git a/crates/hoonc/Cargo.toml b/crates/hoonc/Cargo.toml index 23490d573..2886de28b 100644 --- a/crates/hoonc/Cargo.toml +++ b/crates/hoonc/Cargo.toml @@ -18,6 +18,7 @@ no_check_oom = ["nockvm/no_check_oom"] bincode = { workspace = true } blake3 = { workspace = true } bytes = { workspace = true } +chaff = { workspace = true } clap = { workspace = true, features = ["derive", "cargo", "color", "env"] } dirs = { workspace = true } futures = { workspace = true } diff --git a/crates/hoonc/src/bin/prewarm.rs b/crates/hoonc/src/bin/prewarm.rs index 619fdaa5c..f45a61703 100644 --- a/crates/hoonc/src/bin/prewarm.rs +++ b/crates/hoonc/src/bin/prewarm.rs @@ -1,15 +1,13 @@ use std::fs; -use std::path::{Path, PathBuf}; -use std::time::SystemTime; +use std::path::PathBuf; +use chaff::Chaff; use clap::Parser; use hoonc::Error; use nockapp::export::ExportedState; use nockapp::kernel::boot::{self, default_boot_cli}; -use nockapp::kernel::form::LoadState; use nockapp::noun::slab::{NockJammer, NounSlab}; use nockapp::one_punch::OnePunchWire; -use nockapp::save::JammedCheckpoint; use nockapp::wire::Wire; use nockapp::{exit_driver, file_driver, AtomExt}; use nockvm::noun::{Atom, D, T}; @@ -52,22 +50,14 @@ async fn main() -> Result<(), Box> { nockapp.add_io_driver(exit_driver()).await; run_boot_poke(&mut nockapp).await?; - nockapp.save_blocking().await?; - - let checkpoints_dir = base_data_dir.join("hoonc").join("checkpoints"); - let latest_checkpoint = find_latest_checkpoint(&checkpoints_dir)?; - let exported = checkpoint_to_exported_state(&latest_checkpoint)?; + let exported = ExportedState::from_loadstate::(nockapp.export().await?).encode()?; if let Some(parent) = args.output.parent() { fs::create_dir_all(parent)?; } fs::write(&args.output, &exported)?; - println!( - "Wrote prewarmed kernel state from {} to {}", - latest_checkpoint.display(), - args.output.display() - ); + println!("Wrote prewarmed kernel state to {}", args.output.display()); if args.keep_data_dir { if let Some(dir) = temp_dir { @@ -105,45 +95,3 @@ async fn run_boot_poke(nockapp: &mut nockapp::NockApp) -> Result<(), .map(|_| ()) .map_err(|e| -> Error { Box::new(e) }) } - -fn find_latest_checkpoint(dir: &Path) -> Result> { - let mut latest: Option<(SystemTime, PathBuf)> = None; - - for entry in fs::read_dir(dir)? { - let entry = entry?; - if !entry.file_type()?.is_file() { - continue; - } - let metadata = entry.metadata()?; - let modified = metadata.modified()?; - if latest - .as_ref() - .map(|(time, _)| modified > *time) - .unwrap_or(true) - { - latest = Some((modified, entry.path())); - } - } - - latest - .map(|(_, path)| path) - .ok_or_else(|| format!("No checkpoint found in {}", dir.display()).into()) -} - -fn checkpoint_to_exported_state(path: &Path) -> Result, Box> { - let bytes = fs::read(path)?; - let checkpoint = JammedCheckpoint::decode_from_bytes(&bytes) - .map_err(|e| -> Box { Box::new(e) })?; - - let mut kernel_slab = NounSlab::::new(); - let kernel_root = kernel_slab.cue_into(checkpoint.state_jam.0.clone())?; - kernel_slab.set_root(kernel_root); - - let load_state = LoadState { - ker_hash: checkpoint.ker_hash, - event_num: checkpoint.event_num, - kernel_state: kernel_slab, - }; - let exported = ExportedState::from_loadstate(load_state); - Ok(exported.encode()?) -} diff --git a/crates/hoonc/src/lib.rs b/crates/hoonc/src/lib.rs index e4d403278..f49db0459 100644 --- a/crates/hoonc/src/lib.rs +++ b/crates/hoonc/src/lib.rs @@ -1,7 +1,7 @@ use std::env::current_dir; use std::ffi::OsStr; use std::io::Write; -use std::path::PathBuf; +use std::path::{Path, PathBuf}; use clap::{ColorChoice, Parser}; use nockapp::driver::Operation; @@ -22,9 +22,6 @@ use walkdir::{DirEntry, WalkDir}; pub const OUT_JAM_NAME: &str = "out.jam"; -// save interval in milliseconds -const DEFAULT_SAVE_INTERVAL: u64 = 600000; - pub type Error = Box; pub static KERNEL_JAM: &[u8] = include_bytes!("../bootstrap/hoonc.jam"); @@ -57,10 +54,6 @@ pub struct HoonCli { pub output: Option, } -pub fn default_save_interval() -> u64 { - DEFAULT_SAVE_INTERVAL -} - pub async fn hoonc_data_dir() -> PathBuf { let hoonc_data_dir = system_data_dir().join("hoonc"); if !hoonc_data_dir.exists() { @@ -78,6 +71,28 @@ pub async fn hoonc_data_dir() -> PathBuf { hoonc_data_dir } +fn dir_has_regular_files(path: &Path) -> bool { + path.exists() + && std::fs::read_dir(path) + .map(|entries| { + entries + .filter_map(Result::ok) + .any(|entry| entry.file_type().map(|ft| ft.is_file()).unwrap_or(false)) + }) + .unwrap_or(false) +} + +fn has_existing_hoonc_durability_state(hoonc_data_dir: &Path) -> bool { + let checkpoints_dir = hoonc_data_dir.join("checkpoints"); + let pma_dir = hoonc_data_dir.join("pma"); + + dir_has_regular_files(&checkpoints_dir) + || dir_has_regular_files(&pma_dir) + || hoonc_data_dir.join("event-log.sqlite3").exists() + || hoonc_data_dir.join("event-log.sqlite3-wal").exists() + || hoonc_data_dir.join("event-log.sqlite3-shm").exists() +} + /// Builds and interprets a Hoon generator. /// /// This function: @@ -229,26 +244,18 @@ async fn initialize_hoonc_inner( let mut boot_cli = boot_cli; let disable_prewarm = std::env::var("HOONC_DISABLE_PREWARM").is_ok(); let hoonc_data_dir = data_dir.join("hoonc"); - let checkpoints_dir = hoonc_data_dir.join("checkpoints"); - let has_existing_checkpoint = checkpoints_dir.exists() - && std::fs::read_dir(&checkpoints_dir) - .map(|entries| { - entries.filter_map(Result::ok).any(|entry| { - let is_file = entry.file_type().map(|ft| ft.is_file()).unwrap_or(false); - is_file && entry.file_name().to_string_lossy().ends_with(".chkjam") - }) - }) - .unwrap_or(false); + let has_existing_durability_state = has_existing_hoonc_durability_state(&hoonc_data_dir); let should_use_prewarm = !disable_prewarm && boot_cli.state_jam.is_none() - && (boot_cli.new || !has_existing_checkpoint); + && (boot_cli.new || !has_existing_durability_state); // Keep the prewarm tempfile alive for the duration of this function when used. let mut _prewarm_state_file: Option = None; if should_use_prewarm { let mut tmp = NamedTempFile::new()?; tmp.write_all(PREWARM_STATE_JAM)?; + boot_cli.new = true; boot_cli.state_jam = Some(tmp.path().to_string_lossy().into_owned()); _prewarm_state_file = Some(tmp); } @@ -420,6 +427,41 @@ pub fn is_valid_file_or_dir(entry: &DirEntry) -> bool { is_dir || is_valid_file } +#[cfg(test)] +mod durability_state_tests { + use super::has_existing_hoonc_durability_state; + + #[test] + fn detects_existing_pma_state_without_checkpoints() { + let temp = tempfile::tempdir().expect("tempdir"); + let hoonc_data_dir = temp.path().join("hoonc"); + std::fs::create_dir_all(hoonc_data_dir.join("pma")).expect("create pma dir"); + std::fs::write(hoonc_data_dir.join("pma").join("0.pma"), b"pma").expect("write pma"); + + assert!(has_existing_hoonc_durability_state(&hoonc_data_dir)); + } + + #[test] + fn detects_existing_event_log_state() { + let temp = tempfile::tempdir().expect("tempdir"); + let hoonc_data_dir = temp.path().join("hoonc"); + std::fs::create_dir_all(&hoonc_data_dir).expect("create hoonc dir"); + std::fs::write(hoonc_data_dir.join("event-log.sqlite3"), b"sqlite") + .expect("write event log"); + + assert!(has_existing_hoonc_durability_state(&hoonc_data_dir)); + } + + #[test] + fn empty_hoonc_data_dir_does_not_block_prewarm() { + let temp = tempfile::tempdir().expect("tempdir"); + let hoonc_data_dir = temp.path().join("hoonc"); + std::fs::create_dir_all(&hoonc_data_dir).expect("create hoonc dir"); + + assert!(!has_existing_hoonc_durability_state(&hoonc_data_dir)); + } +} + #[instrument] pub fn canonicalize_and_string(path: &std::path::Path) -> String { let path = path.canonicalize().expect("Failed to canonicalize path"); diff --git a/crates/hoonc/src/main.rs b/crates/hoonc/src/main.rs index e67ecbe75..8369b4e74 100644 --- a/crates/hoonc/src/main.rs +++ b/crates/hoonc/src/main.rs @@ -6,8 +6,7 @@ use nockvm::mem::{AllocationError, NewStackError}; #[tokio::main] async fn main() -> Result<(), Error> { - let mut cli = HoonCli::parse(); - cli.boot.save_interval = None; + let cli = HoonCli::parse(); boot::init_default_tracing(&cli.boot.clone()); let result = std::panic::AssertUnwindSafe(async { let (mut nockapp, _) = initialize_hoonc(cli).await?; diff --git a/crates/kernels/Cargo.toml b/crates/kernels/Cargo.toml new file mode 100644 index 000000000..29c1eb299 --- /dev/null +++ b/crates/kernels/Cargo.toml @@ -0,0 +1,19 @@ +[package] +name = "kernels" +version.workspace = true +edition.workspace = true +license = "MIT OR Apache-2.0" + +[features] +default = [] +bazel_build = [] + +# Specific kernels +approver = [] +dumb = [] +wallet = [] +miner = [] +nockchain-peek = [] +verifier = [] + +[dependencies] diff --git a/crates/kernels/approver/Cargo.toml b/crates/kernels/approver/Cargo.toml new file mode 100644 index 000000000..ade34e3e1 --- /dev/null +++ b/crates/kernels/approver/Cargo.toml @@ -0,0 +1,7 @@ +[package] +name = "kernels-open-approver" +version.workspace = true +edition.workspace = true +license = "MIT OR Apache-2.0" + +[dependencies] diff --git a/crates/kernels/approver/build.rs b/crates/kernels/approver/build.rs new file mode 100644 index 000000000..d8796d3bc --- /dev/null +++ b/crates/kernels/approver/build.rs @@ -0,0 +1,15 @@ +use std::env; +use std::path::PathBuf; + +fn main() { + let manifest_dir = PathBuf::from(env::var("CARGO_MANIFEST_DIR").expect("CARGO_MANIFEST_DIR")); + let repo_root = manifest_dir.ancestors().nth(3).expect("repo root"); + let jam_path = repo_root.join("assets/approver.jam"); + + println!("cargo:rerun-if-env-changed=KERNEL_JAM_PATH"); + println!("cargo:rerun-if-changed={}", jam_path.display()); + + if env::var_os("KERNEL_JAM_PATH").is_none() { + println!("cargo:rustc-env=KERNEL_JAM_PATH={}", jam_path.display()); + } +} diff --git a/crates/kernels/approver/src/lib.rs b/crates/kernels/approver/src/lib.rs new file mode 100644 index 000000000..625a4ae13 --- /dev/null +++ b/crates/kernels/approver/src/lib.rs @@ -0,0 +1 @@ +pub static KERNEL: &[u8] = include_bytes!(env!("KERNEL_JAM_PATH")); diff --git a/crates/kernels/build.rs b/crates/kernels/build.rs new file mode 100644 index 000000000..80cf54202 --- /dev/null +++ b/crates/kernels/build.rs @@ -0,0 +1,35 @@ +use std::env; +use std::error::Error; +use std::path::PathBuf; + +fn main() -> Result<(), Box> { + let manifest_dir = PathBuf::from(env::var("CARGO_MANIFEST_DIR")?); + let assets_dir = manifest_dir.join("../../assets"); + + println!( + "cargo:rustc-env=DUMB_JAM_PATH={}", + assets_dir.join("dumb.jam").display() + ); + println!( + "cargo:rustc-env=WALLET_JAM_PATH={}", + assets_dir.join("wal.jam").display() + ); + println!( + "cargo:rustc-env=MINER_JAM_PATH={}", + assets_dir.join("miner.jam").display() + ); + println!( + "cargo:rerun-if-changed={}", + assets_dir.join("dumb.jam").display() + ); + println!( + "cargo:rerun-if-changed={}", + assets_dir.join("wal.jam").display() + ); + println!( + "cargo:rerun-if-changed={}", + assets_dir.join("miner.jam").display() + ); + + Ok(()) +} diff --git a/crates/kernels/src/approver.rs b/crates/kernels/src/approver.rs new file mode 100644 index 000000000..e78489b5b --- /dev/null +++ b/crates/kernels/src/approver.rs @@ -0,0 +1,8 @@ +#[cfg(feature = "bazel_build")] +pub static KERNEL: &[u8] = include_bytes!(env!("APPROVER_JAM_PATH")); + +#[cfg(not(feature = "bazel_build"))] +pub const KERNEL: &[u8] = include_bytes!(concat!( + env!("CARGO_MANIFEST_DIR"), + "/../../assets/approver.jam" +)); diff --git a/crates/kernels/src/dumb.rs b/crates/kernels/src/dumb.rs new file mode 100644 index 000000000..235cf3e2e --- /dev/null +++ b/crates/kernels/src/dumb.rs @@ -0,0 +1 @@ +pub const KERNEL: &[u8] = include_bytes!(env!("DUMB_JAM_PATH")); diff --git a/crates/kernels/src/lib.rs b/crates/kernels/src/lib.rs new file mode 100644 index 000000000..57f55fb26 --- /dev/null +++ b/crates/kernels/src/lib.rs @@ -0,0 +1,17 @@ +#[cfg(feature = "wallet")] +pub mod wallet; + +#[cfg(feature = "approver")] +pub mod approver; + +#[cfg(feature = "dumb")] +pub mod dumb; + +#[cfg(feature = "miner")] +pub mod miner; + +#[cfg(feature = "nockchain-peek")] +pub mod nockchain_peek; + +#[cfg(feature = "verifier")] +pub mod verifier; diff --git a/crates/kernels/src/miner.rs b/crates/kernels/src/miner.rs new file mode 100644 index 000000000..c47faf73c --- /dev/null +++ b/crates/kernels/src/miner.rs @@ -0,0 +1 @@ +pub const KERNEL: &[u8] = include_bytes!(env!("MINER_JAM_PATH")); diff --git a/crates/kernels/src/wallet.rs b/crates/kernels/src/wallet.rs new file mode 100644 index 000000000..2d32a18e3 --- /dev/null +++ b/crates/kernels/src/wallet.rs @@ -0,0 +1 @@ +pub const KERNEL: &[u8] = include_bytes!(env!("WALLET_JAM_PATH")); diff --git a/crates/nockapp-grpc-proto/proto/nockchain/common/v2/blockchain.proto b/crates/nockapp-grpc-proto/proto/nockchain/common/v2/blockchain.proto index 4433e6ae2..77296a564 100644 --- a/crates/nockapp-grpc-proto/proto/nockchain/common/v2/blockchain.proto +++ b/crates/nockapp-grpc-proto/proto/nockchain/common/v2/blockchain.proto @@ -36,6 +36,8 @@ message NoteV1 { } message NoteData { + // TODO(grpc): Add an optional typed/decoded representation (e.g. DecodedNoteData) + // so clients can consume known note-data payloads without re-decoding `blob`. repeated NoteDataEntry entries = 1; } diff --git a/crates/nockapp-grpc-proto/src/v2/convert.rs b/crates/nockapp-grpc-proto/src/v2/convert.rs index bebefd395..de50d6d38 100644 --- a/crates/nockapp-grpc-proto/src/v2/convert.rs +++ b/crates/nockapp-grpc-proto/src/v2/convert.rs @@ -237,12 +237,7 @@ impl From for PbSpendCondition { impl From for PbPkhLock { fn from(pkh: Pkh) -> Self { - let mut hashes = pkh - .hashes - .into_iter() - .map(PbHash::from) - .collect::>(); - hashes.dedup(); + let hashes = pkh.hashes.into_iter().map(PbHash::from).collect::>(); PbPkhLock { m: pkh.m, hashes } } } @@ -258,8 +253,7 @@ impl From for PbLockTim { impl From for PbHaxLock { fn from(hax: Hax) -> Self { - let mut hashes = hax.0.into_iter().map(PbHash::from).collect::>(); - hashes.dedup(); + let hashes = hax.0.into_iter().map(PbHash::from).collect::>(); PbHaxLock { hashes } } } @@ -431,7 +425,7 @@ impl TryFrom for LockPrimitive { .into_iter() .map(v1::Hash::try_from) .collect::, _>>()?; - Ok(LockPrimitive::Pkh(Pkh { m: pkh.m, hashes })) + Ok(LockPrimitive::Pkh(Pkh::new(pkh.m, hashes))) } lock_primitive::Primitive::Tim(tim) => Ok(LockPrimitive::Tim(LockTim { rel: tim.rel.required("LockTim", "rel")?.into(), @@ -443,7 +437,7 @@ impl TryFrom for LockPrimitive { .into_iter() .map(v1::Hash::try_from) .collect::, _>>()?; - Ok(LockPrimitive::Hax(Hax(hashes))) + Ok(LockPrimitive::Hax(Hax::new(hashes))) } lock_primitive::Primitive::Burn(_) => Ok(LockPrimitive::Burn), } @@ -657,3 +651,255 @@ impl TryFrom for V1RawTx { }) } } + +#[cfg(test)] +mod tests { + use nockchain_types::tx_engine::common::{Hash, Name, Nicks, Version}; + + use super::*; + + fn sample_hash(seed: u64) -> Hash { + Hash::from_limbs(&[seed, seed + 1, seed + 2, seed + 3, seed + 4]) + } + + fn sample_spend_condition() -> SpendCondition { + SpendCondition::new(vec![LockPrimitive::Burn]) + } + + fn sample_merkle_proof() -> MerkleProof { + MerkleProof { + root: Hash::from_limbs(&[1, 2, 3, 4, 5]), + path: vec![Hash::from_limbs(&[6, 7, 8, 9, 10])], + } + } + + #[test] + fn test_lock_merkle_proof_stub_roundtrip() { + let spend_condition = sample_spend_condition(); + let merkle_proof = sample_merkle_proof(); + let stub = LockMerkleProof::new_stub(spend_condition, 42, merkle_proof); + + let pb = PbLockMerkleProof::from(stub.clone()); + assert!(pb.lmp_version.is_none()); + + let decoded = LockMerkleProof::try_from(pb).expect("decode stub"); + assert_eq!(decoded, stub); + } + + #[test] + fn test_lock_merkle_proof_full_roundtrip() { + let spend_condition = sample_spend_condition(); + let merkle_proof = sample_merkle_proof(); + let full = LockMerkleProof::new_full(spend_condition, 84, merkle_proof); + let expected_version = match &full { + LockMerkleProof::Full(proof) => proof.version, + LockMerkleProof::Stub(_) => panic!("expected full proof"), + }; + + let pb = PbLockMerkleProof::from(full.clone()); + assert_eq!(pb.lmp_version, Some(expected_version)); + + let decoded = LockMerkleProof::try_from(pb).expect("decode full"); + assert_eq!(decoded, full); + } + + #[test] + fn test_lock_merkle_proof_invalid_version_rejected() { + let spend_condition = sample_spend_condition(); + let merkle_proof = sample_merkle_proof(); + let stub = LockMerkleProof::new_stub(spend_condition.clone(), 7, merkle_proof.clone()); + let expected_version = match LockMerkleProof::new_full(spend_condition, 7, merkle_proof) { + LockMerkleProof::Full(proof) => proof.version, + LockMerkleProof::Stub(_) => panic!("expected full proof"), + }; + + let mut pb = PbLockMerkleProof::from(stub); + pb.lmp_version = Some(expected_version + 1); + + let err = LockMerkleProof::try_from(pb).expect_err("invalid version should fail"); + match err { + ConversionError::Invalid(_) => {} + other => panic!("unexpected error: {other:?}"), + } + } + + #[test] + fn test_pkh_lock_decode_canonicalizes_hash_set() { + let hash_a = sample_hash(11); + let hash_b = sample_hash(21); + let pb = PbLockPrimitive { + primitive: Some(lock_primitive::Primitive::Pkh(PbPkhLock { + m: 2, + hashes: vec![ + PbHash::from(hash_b.clone()), + PbHash::from(hash_a.clone()), + PbHash::from(hash_b.clone()), + ], + })), + }; + + let decoded = LockPrimitive::try_from(pb).expect("decode pkh lock"); + let expected = LockPrimitive::Pkh(Pkh::new(2, vec![hash_a, hash_b])); + assert_eq!(decoded, expected); + + let roundtrip = PbLockPrimitive::from(decoded.clone()); + let decoded_roundtrip = LockPrimitive::try_from(roundtrip).expect("re-decode pkh lock"); + assert_eq!(decoded_roundtrip, expected); + } + + #[test] + fn test_hax_lock_decode_canonicalizes_hash_set() { + let hash_a = sample_hash(31); + let hash_b = sample_hash(41); + let pb = PbLockPrimitive { + primitive: Some(lock_primitive::Primitive::Hax(PbHaxLock { + hashes: vec![ + PbHash::from(hash_b.clone()), + PbHash::from(hash_a.clone()), + PbHash::from(hash_b.clone()), + ], + })), + }; + + let decoded = LockPrimitive::try_from(pb).expect("decode hax lock"); + let expected = LockPrimitive::Hax(Hax::new(vec![hash_a, hash_b])); + assert_eq!(decoded, expected); + + let roundtrip = PbLockPrimitive::from(decoded.clone()); + let decoded_roundtrip = LockPrimitive::try_from(roundtrip).expect("re-decode hax lock"); + assert_eq!(decoded_roundtrip, expected); + } + + #[test] + fn test_spend_condition_decode_canonicalizes_nested_set_locks() { + let pkh_a = sample_hash(51); + let pkh_b = sample_hash(61); + let hax_a = sample_hash(71); + let hax_b = sample_hash(81); + let pb = PbSpendCondition { + primitives: vec![ + PbLockPrimitive { + primitive: Some(lock_primitive::Primitive::Pkh(PbPkhLock { + m: 2, + hashes: vec![ + PbHash::from(pkh_b.clone()), + PbHash::from(pkh_a.clone()), + PbHash::from(pkh_b.clone()), + ], + })), + }, + PbLockPrimitive { + primitive: Some(lock_primitive::Primitive::Hax(PbHaxLock { + hashes: vec![ + PbHash::from(hax_b.clone()), + PbHash::from(hax_a.clone()), + PbHash::from(hax_b.clone()), + ], + })), + }, + PbLockPrimitive { + primitive: Some(lock_primitive::Primitive::Burn(PbBurnLock {})), + }, + ], + }; + + let decoded = SpendCondition::try_from(pb).expect("decode spend condition"); + let expected = SpendCondition::new(vec![ + LockPrimitive::Pkh(Pkh::new(2, vec![pkh_a, pkh_b])), + LockPrimitive::Hax(Hax::new(vec![hax_a, hax_b])), + LockPrimitive::Burn, + ]); + assert_eq!(decoded, expected); + + let roundtrip = PbSpendCondition::from(decoded.clone()); + let decoded_roundtrip = + SpendCondition::try_from(roundtrip).expect("re-decode spend condition"); + assert_eq!(decoded_roundtrip, expected); + } + + #[test] + fn test_raw_transaction_roundtrip_canonicalizes_nested_set_locks() { + let pkh_a = sample_hash(91); + let pkh_b = sample_hash(101); + let hax_a = sample_hash(111); + let hax_b = sample_hash(121); + let expected = V1RawTx { + version: Version::V1, + id: sample_hash(131), + spends: nockchain_types::tx_engine::v1::Spends(vec![( + Name::new(sample_hash(141), sample_hash(151)), + V1Spend::Witness(Spend1 { + witness: V1Witness::new( + LockMerkleProof::new_stub( + SpendCondition::new(vec![ + LockPrimitive::Pkh(Pkh::new(2, vec![pkh_a.clone(), pkh_b.clone()])), + LockPrimitive::Hax(Hax::new(vec![hax_a.clone(), hax_b.clone()])), + ]), + 42, + sample_merkle_proof(), + ), + PkhSignature(Vec::new()), + Vec::new(), + ), + seeds: nockchain_types::tx_engine::v1::Seeds(Vec::new()), + fee: Nicks(7), + }), + )]), + }; + + let mut pb = PbRawTransaction::from(expected.clone()); + let spend = pb.spends.first_mut().expect("raw tx spend entry"); + let witness = match spend + .spend + .as_mut() + .and_then(|spend| spend.spend_kind.as_mut()) + .expect("witness spend kind") + { + spend::SpendKind::Witness(witness) => witness, + other => panic!("expected witness spend, got {other:?}"), + }; + let lock_merkle_proof = witness + .witness + .as_mut() + .and_then(|witness| witness.lock_merkle_proof.as_mut()) + .expect("lock merkle proof"); + let primitives = &mut lock_merkle_proof + .spend_condition + .as_mut() + .expect("spend condition") + .primitives; + + match primitives + .get_mut(0) + .and_then(|primitive| primitive.primitive.as_mut()) + .expect("pkh primitive") + { + lock_primitive::Primitive::Pkh(pkh) => { + pkh.hashes = vec![ + PbHash::from(pkh_b.clone()), + PbHash::from(pkh_a.clone()), + PbHash::from(pkh_b.clone()), + ]; + } + other => panic!("expected pkh primitive, got {other:?}"), + } + + match primitives + .get_mut(1) + .and_then(|primitive| primitive.primitive.as_mut()) + .expect("hax primitive") + { + lock_primitive::Primitive::Hax(hax) => { + hax.hashes = vec![ + PbHash::from(hax_b.clone()), + PbHash::from(hax_a.clone()), + PbHash::from(hax_b.clone()), + ]; + } + other => panic!("expected hax primitive, got {other:?}"), + } + + let decoded = V1RawTx::try_from(pb).expect("decode raw transaction"); + assert_eq!(decoded, expected); + } +} diff --git a/crates/nockapp-grpc/src/services/private_nockapp/driver.rs b/crates/nockapp-grpc/src/services/private_nockapp/driver.rs index 542f4b3e9..6d6fb9e84 100644 --- a/crates/nockapp-grpc/src/services/private_nockapp/driver.rs +++ b/crates/nockapp-grpc/src/services/private_nockapp/driver.rs @@ -4,7 +4,7 @@ use nockapp::driver::{make_driver, IODriverFn, NockAppHandle}; use nockapp::noun::slab::NounSlab; use nockapp::wire::{WireRepr, WireTag as AppWireTag}; use nockapp::{Bytes, NockAppError, Noun}; -use nockvm::noun::{D, T}; +use nockvm::noun::{NounAllocator, NounSpace, D, T}; use nockvm_macros::tas; use noun_serde::prelude::*; use noun_serde::NounDecodeError; @@ -67,33 +67,37 @@ pub enum PrivateGrpcEffect { } impl NounDecode for PrivateGrpcEffect { - fn from_noun(effect: &Noun) -> Result { - let Ok(effect_cell) = effect.as_cell() else { + fn from_noun(effect: &Noun, space: &NounSpace) -> Result { + let Ok(effect_cell) = effect.in_space(space).as_cell() else { return Err(NounDecodeError::ExpectedCell); }; - if unsafe { effect_cell.head().raw_equals(&D(tas!(b"grpc"))) } { + if unsafe { effect_cell.head().noun().raw_equals(&D(tas!(b"grpc"))) } { let effect_payload = effect_cell.tail().as_cell()?; - match effect_payload.head().as_direct() { + match effect_payload.head().noun().as_direct() { // [%grpc %poke pid payload] Ok(tag) if tag.data() == tas!(b"poke") => { let eff = effect_payload.tail().as_cell()?; - let pid = u64::from_noun(&eff.head())?; + let pid_noun = eff.head().noun(); + let pid = u64::from_noun(&pid_noun, space)?; let mut slab: NounSlab = NounSlab::new(); - slab.copy_into(eff.tail()); + slab.copy_into(eff.tail().noun(), space); let payload = slab.jam().to_vec(); Ok(PrivateGrpcEffect::Poke { pid, payload }) } // [%grpc %peek pid [%type path]] Ok(tag) if tag.data() == tas!(b"peek") => { let peek_tail = effect_payload.tail().as_cell()?; - let pid: u64 = ::from_noun(&peek_tail.head())?; + let pid_noun = peek_tail.head().noun(); + let pid: u64 = ::from_noun(&pid_noun, space)?; let meta = peek_tail.tail().as_cell()?; // [%type path] - let typ = String::from_noun(&meta.head())?; + let typ_noun = meta.head().noun(); + let typ = String::from_noun(&typ_noun, space)?; - let path_vec: Vec = >::from_noun(&meta.tail())?; + let path_vec: Vec = + >::from_noun(&meta.tail().noun(), space)?; Ok(PrivateGrpcEffect::Peek { pid, typ, @@ -119,13 +123,16 @@ pub fn grpc_listener_driver(addr: String) -> IODriverFn { loop { match handle.next_effect().await { Ok(effect) => { - let effect_noun = unsafe { effect.root() }; - let grpc_effect = PrivateGrpcEffect::from_noun(&effect_noun).map_err(|err| { - NockAppError::OtherError(format!( - "Failed to decode gRPC effect noun: {}", - err - )) - }); + let grpc_effect = { + let effect_noun = unsafe { effect.root() }; + let space = effect.noun_space(); + PrivateGrpcEffect::from_noun(&effect_noun, &space).map_err(|err| { + NockAppError::OtherError(format!( + "Failed to decode gRPC effect noun: {}", + err + )) + }) + }; let grpc_effect = match grpc_effect { Ok(effect) => effect, Err(_) => continue, diff --git a/crates/nockapp-grpc/src/services/public_nockchain/v1/driver.rs b/crates/nockapp-grpc/src/services/public_nockchain/v1/driver.rs index bd302c280..dd4d324af 100644 --- a/crates/nockapp-grpc/src/services/public_nockchain/v1/driver.rs +++ b/crates/nockapp-grpc/src/services/public_nockchain/v1/driver.rs @@ -2,7 +2,7 @@ use std::net::SocketAddr; use nockapp::driver::{make_driver, IODriverFn, NockAppHandle}; use nockchain_types::tx_engine::v0; -use nockvm::ext::NounExt; +use nockvm::noun::{NounAllocator, NounSpace}; use nockvm_macros::tas; use noun_serde::{NounDecode, NounDecodeError}; use tracing::{error, info, warn}; @@ -16,8 +16,8 @@ pub enum PublicNockchainEffect { } impl NounDecode for PublicNockchainEffect { - fn from_noun(effect: &nockapp::Noun) -> Result { - let effect_cell = effect.as_cell()?; + fn from_noun(effect: &nockapp::Noun, space: &NounSpace) -> Result { + let effect_cell = effect.in_space(space).as_cell()?; if !effect_cell.head().eq_bytes(b"nockchain-grpc") { return Err(NounDecodeError::InvalidTag); } @@ -25,13 +25,15 @@ impl NounDecode for PublicNockchainEffect { let payload_cell = effect_cell.tail().as_cell()?; let tag_atom = payload_cell.head().as_atom()?; let tag = tag_atom + .atom() .as_direct() .map_err(|_| NounDecodeError::InvalidTag)? .data(); match tag { t if t == tas!(b"send-tx") => { - let raw_tx = v0::RawTx::from_noun(&payload_cell.tail())?; + let raw_tx_noun = payload_cell.tail().noun(); + let raw_tx = v0::RawTx::from_noun(&raw_tx_noun, space)?; Ok(PublicNockchainEffect::SendTx { raw_tx }) } _ => Err(NounDecodeError::InvalidTag), @@ -79,7 +81,12 @@ pub fn grpc_listener_driver(addr: String) -> IODriverFn { Err(_) => continue, }; - let effect = match PublicNockchainEffect::from_noun(unsafe { effect.root() }) { + let effect = { + let effect_noun = unsafe { effect.root() }; + let space = effect.noun_space(); + PublicNockchainEffect::from_noun(&effect_noun, &space) + }; + let effect = match effect { Ok(effect) => effect, Err(NounDecodeError::InvalidTag) => continue, Err(err) => { diff --git a/crates/nockapp-grpc/src/services/public_nockchain/v1/server.rs b/crates/nockapp-grpc/src/services/public_nockchain/v1/server.rs index a33b98391..9e3c1df63 100644 --- a/crates/nockapp-grpc/src/services/public_nockchain/v1/server.rs +++ b/crates/nockapp-grpc/src/services/public_nockchain/v1/server.rs @@ -8,7 +8,7 @@ use nockapp::driver::{NockAppHandle, PokeResult}; use nockapp::noun::slab::NounSlab; use nockapp::wire::WireRepr; use nockchain_types::tx_engine::v0; -use nockvm::noun::SIG; +use nockvm::noun::{NounAllocator, SIG}; use noun_serde::{NounDecode, NounEncode}; use tokio::sync::RwLock; use tokio::time::{self, Duration}; @@ -161,7 +161,9 @@ impl PublicNockchainGrpcServer { let result = match peek_result { Ok(Some(result_slab)) => { let result_noun = unsafe { result_slab.root() }; - match >>::from_noun(&result_noun) { + let space = result_slab.noun_space(); + match >>::from_noun(&result_noun, &space) + { Ok(opt) => Ok(opt.flatten()), // Peek either returned [~ ~] or ~ Err(_) => Err(NockAppGrpcError::PeekReturnedNoData), @@ -413,7 +415,8 @@ impl NockchainService for PublicNockchainGrpcServer { match peek_result { Ok(Some(result_slab)) => { let result_noun = unsafe { result_slab.root() }; - let result = >>::from_noun(&result_noun); + let space = result_slab.noun_space(); + let result = >>::from_noun(&result_noun, &space); match result { Ok(update) => { @@ -723,7 +726,8 @@ impl NockchainService for PublicNockchainGrpcServer { match peek_result { Ok(Some(result_slab)) => { let result_noun = unsafe { result_slab.root() }; - match >>::from_noun(&result_noun) { + let space = result_slab.noun_space(); + match >>::from_noun(&result_noun, &space) { Ok(opt) => { let accepted = opt.flatten().unwrap_or(false); timed_return( @@ -812,8 +816,9 @@ mod tests { &self, path: NounSlab, ) -> std::result::Result, nockapp::nockapp::error::NockAppError> { + let space = path.noun_space(); let root = unsafe { path.root() }; - if let Ok(segments) = >::from_noun(&root) { + if let Ok(segments) = >::from_noun(&root, &space) { if segments.first().map(String::as_str) == Some("heaviest-chain") { let mut slab = NounSlab::new(); let noun = Some(Some(( @@ -842,7 +847,7 @@ mod tests { } } - #[tokio::test] + #[tokio::test(flavor = "current_thread")] async fn wallet_get_balance_uses_cache_for_subsequent_pages() { let (update, expected_names) = fixtures::make_balance_update(4); let handle = Arc::new(MockHandle::new(update)); diff --git a/crates/nockapp-grpc/src/services/public_nockchain/v2/block_explorer.rs b/crates/nockapp-grpc/src/services/public_nockchain/v2/block_explorer.rs index ba92c08d2..98d694194 100644 --- a/crates/nockapp-grpc/src/services/public_nockchain/v2/block_explorer.rs +++ b/crates/nockapp-grpc/src/services/public_nockchain/v2/block_explorer.rs @@ -1,5 +1,4 @@ use std::collections::{BTreeMap, HashMap, VecDeque}; -use std::convert::TryFrom; use std::sync::atomic::{AtomicBool, AtomicU64, Ordering}; use std::sync::{Arc, Mutex}; use std::time::{Duration, Instant}; @@ -11,11 +10,11 @@ use nockapp_grpc_proto::pb::public::v2::{ CoinbaseSplitV1 as ProtoCoinbaseSplitV1, CoinbaseSplitV1Entry, PageMsg as ProtoPageMsg, ProofOfWork, TransactionDetails, TransactionInput, TransactionOutput, }; -use nockchain_math::noun_ext::NounMathExt; +use nockchain_math::noun_ext::NounMathExtHandle; use nockchain_math::structs::HoonMapIter; use nockchain_types::tx_engine::common::{BlockHeight, Hash, Name, Page}; use nockchain_types::tx_engine::v0::{Lock, NoteV0, RawTx}; -use nockvm::noun::{Noun, SIG}; +use nockvm::noun::{Noun, NounAllocator, NounHandle, NounSpace, SIG}; use noun_serde::{NounDecode, NounDecodeError, NounEncode}; use tokio::sync::{RwLock, Semaphore}; use tracing::{debug, error, info, warn}; @@ -212,7 +211,7 @@ pub struct BlockExplorerCache { } impl BlockExplorerCache { - const RANGE_CHUNK: u64 = 1024; + const RANGE_CHUNK: u64 = 256; const INITIAL_SEED_RETRY_DELAY: Duration = Duration::from_secs(2); const INITIAL_SEED_MAX_WAIT: Duration = Duration::from_secs(120); const MIN_TX_PREFIX_LEN: usize = 8; @@ -669,24 +668,29 @@ impl BlockExplorerCache { .ok_or(NockAppGrpcError::PeekFailed)?; let result_noun = unsafe { result.root() }; - + let space = result.noun_space(); + let result_noun_handle = result_noun.in_space(&space); tracing::debug!( noun_is_atom = result_noun.is_atom(), "peek_full_page raw result" ); - let opt: Option>> = NounDecode::from_noun(&result_noun) - .map_err(|e| { - tracing::error!("Failed to decode FullPageEntryNoun list: {:?}", e); - NockAppGrpcError::NounDecode(e) - })?; - let entries = opt.flatten().ok_or(NockAppGrpcError::PeekReturnedNoData)?; + let entries = decode_optional_optional_vec_handles(result_noun_handle).map_err(|e| { + tracing::error!("Failed to decode full page entry list: {:?}", e); + NockAppGrpcError::NounDecode(e) + })?; + let entries = entries.ok_or(NockAppGrpcError::PeekReturnedNoData)?; let parsed: Vec = entries .into_iter() .map(|entry| { - let entry_height = entry.height.0 .0; - FullPageDetails::try_from(entry).map_err(|e| { + let entry_height = entry + .slot(2) + .ok() + .and_then(|noun| noun.as_atom().ok()) + .and_then(|atom| atom.as_u64().ok()) + .unwrap_or(u64::MAX); + FullPageDetails::from_noun_handle(&entry).map_err(|e| { tracing::error!( height = entry_height, error = %e, @@ -858,9 +862,10 @@ impl BlockExplorerCache { .ok_or(NockAppGrpcError::PeekFailed)?; let result_noun = unsafe { result.root() }; + let space = result.noun_space(); // Decode Option> - let opt: Option> = NounDecode::from_noun(&result_noun) + let opt: Option> = NounDecode::from_noun(&result_noun, &space) .map_err(|e| { error!( "Failed to decode heaviest-chain peek result.\n\ @@ -913,17 +918,19 @@ impl BlockExplorerCache { .ok_or(NockAppGrpcError::PeekFailed)?; let result_noun = unsafe { result.root() }; - + let space = result.noun_space(); + let result_noun_handle = result_noun.in_space(&space); // Decode Option> - let opt: Option> = NounDecode::from_noun(&result_noun).map_err(|e| { - error!( - "Failed to decode heaviest-block peek result.\n\ + let opt: Option> = + NounDecode::from_noun_handle(&result_noun_handle).map_err(|e| { + error!( + "Failed to decode heaviest-block peek result.\n\ Decode error: {:?}\n\ Expected: Option>", - e - ); - NockAppGrpcError::NounDecode(e) - })?; + e + ); + NockAppGrpcError::NounDecode(e) + })?; debug!( "peek_heaviest_block decoded: outer={:?}", @@ -1006,13 +1013,17 @@ impl BlockExplorerCache { .ok_or(NockAppGrpcError::PeekFailed)?; let result_noun = unsafe { result.root() }; - let opt: Option>> = - NounDecode::from_noun(&result_noun).map_err(NockAppGrpcError::NounDecode)?; - let entries = opt.flatten().ok_or(NockAppGrpcError::PeekReturnedNoData)?; + let space = result.noun_space(); + let entries = decode_optional_optional_vec_handles(result_noun.in_space(&space)) + .map_err(NockAppGrpcError::NounDecode)? + .ok_or(NockAppGrpcError::PeekReturnedNoData)?; let mut parsed = Vec::new(); for entry in entries { - parsed.push(BlockEntryWithTxs::try_from(entry).map_err(NockAppGrpcError::NounDecode)?); + parsed.push( + BlockEntryWithTxs::from_noun_handle(&entry) + .map_err(NockAppGrpcError::NounDecode)?, + ); } parsed @@ -1254,13 +1265,12 @@ impl BlockExplorerCache { .ok_or(NockAppGrpcError::PeekFailed)?; let result_noun = unsafe { result.root() }; - - // Debug: log raw noun structure + let space = result.noun_space(); // Debug: log raw noun structure let is_atom = result_noun.is_atom(); let is_cell = result_noun.is_cell(); if is_atom { if let Ok(atom) = result_noun.as_atom() { - let val = atom.as_u64().unwrap_or(u64::MAX); + let val = atom.in_space(&space).as_u64().unwrap_or(u64::MAX); info!( is_atom, atom_val = val, @@ -1269,8 +1279,8 @@ impl BlockExplorerCache { } } else if is_cell { if let Ok(cell) = result_noun.as_cell() { - let head_is_atom = cell.head().is_atom(); - let tail_is_atom = cell.tail().is_atom(); + let head_is_atom = cell.in_space(&space).head().is_atom(); + let tail_is_atom = cell.in_space(&space).tail().is_atom(); info!( head_is_atom, tail_is_atom, "peek_blocks_range raw result is cell" @@ -1280,21 +1290,21 @@ impl BlockExplorerCache { // Decode Option>> // We need to extract fields from the page and txs - let opt: Option>> = - NounDecode::from_noun(&result_noun).map_err(|e| { + let decoded_entries = + decode_optional_optional_vec_handles(result_noun.in_space(&space)).map_err(|e| { // Log detailed noun structure to help diagnose format issues let noun_debug = if result_noun.is_atom() { format!( "atom (value: {:?})", - result_noun.as_atom().ok().and_then(|a| a.as_u64().ok()) + result_noun.as_atom().ok().and_then(|a| a.in_space(&space).as_u64().ok()) ) } else if let Ok(cell) = result_noun.as_cell() { - let head_type = if cell.head().is_atom() { + let head_type = if cell.in_space(&space).head().is_atom() { "atom" } else { "cell" }; - let tail_type = if cell.tail().is_atom() { + let tail_type = if cell.in_space(&space).tail().is_atom() { "atom" } else { "cell" @@ -1303,25 +1313,23 @@ impl BlockExplorerCache { } else { "unknown".to_string() }; - error!( - "Failed to decode BlockRangeEntryNoun list.\n\ + "Failed to decode block range entry list.\n\ Decode error: {:?}\n\ Result noun structure: {}\n\ This is likely a Page decoder issue - check tx_ids (z-set vs list), bignum, or coinbase format.", e, noun_debug ); - NockAppGrpcError::NounDecode(e) - })?; + NockAppGrpcError::NounDecode(e)})?; - let outer_some = opt.is_some(); - let inner_some = opt.as_ref().map(|v| v.is_some()).unwrap_or(false); + let outer_some = result_noun.is_cell(); + let inner_some = decoded_entries.is_some(); info!( outer_some, inner_some, "peek_blocks_range decoded outer options" ); - let entries = opt.flatten().ok_or_else(|| { + let entries = decoded_entries.ok_or_else(|| { warn!("peek_blocks_range: opt.flatten() returned None"); NockAppGrpcError::PeekReturnedNoData })?; @@ -1333,8 +1341,8 @@ impl BlockExplorerCache { let entries: Vec = entries .into_iter() .map(|entry| { - BlockRangeEntry::try_from(entry).map_err(|e| { - error!("Failed to convert BlockRangeEntryNoun: {:?}", e); + BlockRangeEntry::from_noun_handle(&entry).map_err(|e| { + error!("Failed to convert block range entry: {:?}", e); e }) }) @@ -1366,31 +1374,77 @@ struct BlockRangeEntry { tx_ids: Vec, } -#[derive(Debug, Clone, NounDecode)] -struct BlockRangeEntryNoun { - height: BlockHeight, - tail: BlockRangeEntryTail, +fn decode_option_handle<'a>( + noun: NounHandle<'a>, +) -> Result>, NounDecodeError> { + if let Ok(atom) = noun.as_atom() { + if atom.as_u64()? == 0 { + return Ok(None); + } + return Err(NounDecodeError::Custom("Invalid Option encoding".into())); + } + + let cell = noun.as_cell().map_err(|_| NounDecodeError::ExpectedCell)?; + let head = cell + .head() + .as_atom() + .map_err(|_| NounDecodeError::ExpectedAtom)?; + if head.as_u64()? != 0 { + return Err(NounDecodeError::Custom( + "Invalid Option encoding - expected ~".into(), + )); + } + Ok(Some(cell.tail())) } -#[derive(Debug, Clone, NounDecode)] -struct BlockRangeEntryTail { - block_id: Hash, - tail: PageAndTxs, +fn decode_vec_handles<'a>(noun: NounHandle<'a>) -> Result>, NounDecodeError> { + let mut result = Vec::new(); + let mut current = noun; + + while let Ok(cell) = current.as_cell() { + result.push(cell.head()); + current = cell.tail(); + } + + let atom = current + .as_atom() + .map_err(|_| NounDecodeError::ExpectedAtom)?; + if atom.as_u64()? != 0 { + return Err(NounDecodeError::Custom("Invalid list termination".into())); + } + + Ok(result) } -#[derive(Debug, Clone, NounDecode)] -struct PageAndTxs { - page: Page, - txs: Noun, +fn decode_optional_optional_vec_handles<'a>( + noun: NounHandle<'a>, +) -> Result>>, NounDecodeError> { + let Some(inner) = decode_option_handle(noun)? else { + return Ok(None); + }; + let Some(list) = decode_option_handle(inner)? else { + return Ok(None); + }; + Ok(Some(decode_vec_handles(list)?)) } -impl TryFrom for BlockRangeEntry { - type Error = NounDecodeError; +impl BlockRangeEntry { + fn from_noun_handle(noun: &NounHandle) -> std::result::Result { + let cell = noun.as_cell().map_err(|_| NounDecodeError::ExpectedCell)?; + let height = BlockHeight::from_noun_handle(&cell.head())?; - fn try_from(raw: BlockRangeEntryNoun) -> std::result::Result { - let BlockRangeEntryNoun { height, tail } = raw; - let BlockRangeEntryTail { block_id, tail } = tail; - let PageAndTxs { page, txs } = tail; + let tail = cell + .tail() + .as_cell() + .map_err(|_| NounDecodeError::ExpectedCell)?; + let block_id = Hash::from_noun_handle(&tail.head())?; + + let page_and_txs = tail + .tail() + .as_cell() + .map_err(|_| NounDecodeError::ExpectedCell)?; + let page = Page::from_noun_handle(&page_and_txs.head())?; + let txs = page_and_txs.tail(); let parent_id = page.parent; let timestamp = page.timestamp; @@ -1408,7 +1462,7 @@ impl TryFrom for BlockRangeEntry { /// Extract transaction IDs from transactions map (z-map tx-id tx) fn extract_tx_ids_from_map( - txs_noun: &Noun, + txs_noun: &NounHandle, ) -> std::result::Result, noun_serde::NounDecodeError> { // Check if it's an empty map (atom 0) if let Ok(atom) = txs_noun.as_atom() { @@ -1419,7 +1473,7 @@ fn extract_tx_ids_from_map( // Iterate over the z-map and collect keys (tx-ids) let mut tx_ids = Vec::new(); - for (idx, entry) in HoonMapIter::from(*txs_noun).enumerate() { + for (idx, entry) in HoonMapIter::new(txs_noun).enumerate() { if !entry.is_cell() { return Err(NounDecodeError::Custom(format!( "extract_tx_ids_from_map: entry {} is not a cell (expected z-map [key value] pair)", @@ -1427,7 +1481,7 @@ fn extract_tx_ids_from_map( ))); } let [key, _value] = entry.uncell().map_err(|_| NounDecodeError::ExpectedCell)?; - let hash = Hash::from_noun(&key).map_err(|e| { + let hash = Hash::from_noun_handle(&key).map_err(|e| { NounDecodeError::Custom(format!( "extract_tx_ids_from_map: failed to decode tx_id at entry {}: {}", idx, e @@ -1444,13 +1498,23 @@ struct BlockEntryWithTxs { txs: Vec<(Hash, DecodedTx)>, } -impl TryFrom for BlockEntryWithTxs { - type Error = NounDecodeError; +impl BlockEntryWithTxs { + fn from_noun_handle(noun: &NounHandle) -> std::result::Result { + let cell = noun.as_cell().map_err(|_| NounDecodeError::ExpectedCell)?; + let height = BlockHeight::from_noun_handle(&cell.head())?; + + let tail = cell + .tail() + .as_cell() + .map_err(|_| NounDecodeError::ExpectedCell)?; + let block_id = Hash::from_noun_handle(&tail.head())?; - fn try_from(raw: BlockRangeEntryNoun) -> std::result::Result { - let BlockRangeEntryNoun { height, tail } = raw; - let BlockRangeEntryTail { block_id, tail } = tail; - let PageAndTxs { page, txs } = tail; + let page_and_txs = tail + .tail() + .as_cell() + .map_err(|_| NounDecodeError::ExpectedCell)?; + let page = Page::from_noun_handle(&page_and_txs.head())?; + let txs = page_and_txs.tail(); let parent_id = page.parent; let timestamp = page.timestamp; @@ -1471,7 +1535,7 @@ impl TryFrom for BlockEntryWithTxs { } fn extract_transactions_from_map( - txs_noun: &Noun, + txs_noun: &NounHandle, ) -> std::result::Result, noun_serde::NounDecodeError> { if let Ok(atom) = txs_noun.as_atom() { if atom.as_u64()? == 0 { @@ -1480,7 +1544,7 @@ fn extract_transactions_from_map( } let mut txs = Vec::new(); - for (idx, entry) in HoonMapIter::from(*txs_noun).enumerate() { + for (idx, entry) in HoonMapIter::new(txs_noun).enumerate() { if !entry.is_cell() { return Err(NounDecodeError::Custom(format!( "extract_transactions_from_map: entry {} is not a cell (expected z-map [key value] pair)", @@ -1488,13 +1552,13 @@ fn extract_transactions_from_map( ))); } let [key, value] = entry.uncell().map_err(|_| NounDecodeError::ExpectedCell)?; - let hash = Hash::from_noun(&key).map_err(|e| { + let hash = Hash::from_noun_handle(&key).map_err(|e| { NounDecodeError::Custom(format!( "extract_transactions_from_map: failed to decode tx hash at entry {}: {}", idx, e )) })?; - let tx = DecodedTx::from_noun(&value).map_err(|e| { + let tx = DecodedTx::from_noun_handle(&value).map_err(|e| { NounDecodeError::Custom(format!( "extract_transactions_from_map: failed to decode tx at entry {} (hash={}): {}", idx, @@ -1552,20 +1616,20 @@ struct TxOutputV0 { } impl NounDecode for DecodedTx { - fn from_noun(noun: &Noun) -> Result { - let cell = noun.as_cell()?; - let version = u64::from_noun(&cell.head())?; + fn from_noun(noun: &Noun, space: &NounSpace) -> Result { + let cell = noun.in_space(space).as_cell()?; + let version = u64::from_noun_handle(&cell.head())?; match version { 0 => { // v0 tx: [%0 raw-tx:v0 total-size outputs:v0] let tail = cell.tail(); let cell = tail.as_cell()?; - let raw_tx = RawTx::from_noun(&cell.head())?; + let raw_tx = RawTx::from_noun_handle(&cell.head())?; let tail = cell.tail(); let cell = tail.as_cell()?; - let total_size = u64::from_noun(&cell.head())?; + let total_size = u64::from_noun_handle(&cell.head())?; let outputs = decode_outputs_v0(&cell.tail())?; Ok(DecodedTx::V0(TxV0Data { @@ -1588,7 +1652,7 @@ impl NounDecode for DecodedTx { let raw_cell = raw_tx_noun .as_cell() .map_err(|e| NounDecodeError::Custom(format!("v1 raw-tx not cell: {e:?}")))?; - let raw_version = u64::from_noun(&raw_cell.head()).map_err(|e| { + let raw_version = u64::from_noun_handle(&raw_cell.head()).map_err(|e| { NounDecodeError::Custom(format!("v1 raw-tx version not atom: {e:?}")) })?; if raw_version != 1 { @@ -1600,7 +1664,7 @@ impl NounDecode for DecodedTx { let raw_tail = raw_cell.tail().as_cell().map_err(|e| { NounDecodeError::Custom(format!("v1 raw-tx tail not cell: {e:?}")) })?; - let _tx_id = Hash::from_noun(&raw_tail.head()).map_err(|e| { + let _tx_id = Hash::from_noun_handle(&raw_tail.head()).map_err(|e| { NounDecodeError::Custom(format!("v1 raw-tx id parse failed: {e:?}")) })?; let spends_noun = raw_tail.tail(); @@ -1614,7 +1678,7 @@ impl NounDecode for DecodedTx { let cell = tail.as_cell().map_err(|e| { NounDecodeError::Custom(format!("v1 tx size/outputs tail not cell: {e:?}")) })?; - let total_size = u64::from_noun(&cell.head()).map_err(|e| { + let total_size = u64::from_noun_handle(&cell.head()).map_err(|e| { NounDecodeError::Custom(format!("v1 tx total_size not atom: {e:?}")) })?; let outputs = decode_outputs_v1(&cell.tail()).map_err(|e| { @@ -1637,7 +1701,7 @@ impl NounDecode for DecodedTx { } /// Returns (inputs, total_fee) -fn decode_v1_spends(noun: &Noun) -> Result<(Vec, u64), NounDecodeError> { +fn decode_v1_spends(noun: &NounHandle) -> Result<(Vec, u64), NounDecodeError> { if let Ok(atom) = noun.as_atom() { if atom.as_u64()? == 0 { return Ok((Vec::new(), 0)); @@ -1646,7 +1710,7 @@ fn decode_v1_spends(noun: &Noun) -> Result<(Vec, u64), NounDecodeErro let mut inputs = Vec::new(); let mut total_fee = 0u64; - for (idx, entry) in HoonMapIter::from(*noun).enumerate() { + for (idx, entry) in HoonMapIter::new(noun).enumerate() { if !entry.is_cell() { return Err(NounDecodeError::Custom(format!( "decode_v1_spends: entry {} is not a cell (expected z-map [nname spend] pair)", @@ -1655,7 +1719,7 @@ fn decode_v1_spends(noun: &Noun) -> Result<(Vec, u64), NounDecodeErro } let [key, value] = entry.uncell().map_err(|_| NounDecodeError::ExpectedCell)?; // key is nname (Name type) - let name = Name::from_noun(&key).map_err(|e| { + let name = Name::from_noun_handle(&key).map_err(|e| { NounDecodeError::Custom(format!( "decode_v1_spends: failed to decode name at entry {}: {}", idx, e @@ -1687,7 +1751,7 @@ fn decode_v1_spends(noun: &Noun) -> Result<(Vec, u64), NounDecodeErro Ok((inputs, total_fee)) } -fn decode_outputs_v1(noun: &Noun) -> Result, NounDecodeError> { +fn decode_outputs_v1(noun: &NounHandle) -> Result, NounDecodeError> { // outputs:v1 can be either: // 1. Tagged form from polymorphic outputs: [%1 (z-set output)] // 2. Untagged form from tx:v1: (z-set output) @@ -1728,7 +1792,7 @@ fn decode_outputs_v1(noun: &Noun) -> Result, NounDecodeError> { let mut outputs = Vec::new(); // z-set structure: [n=value l=z-set r=z-set] // HoonMapIter returns n (the value) directly for each node - for (idx, entry) in HoonMapIter::from(zset_noun).enumerate() { + for (idx, entry) in HoonMapIter::new(&zset_noun).enumerate() { if !entry.is_cell() { return Err(NounDecodeError::Custom(format!( "decode_outputs_v1: entry {} is not a cell (expected z-set output entry)", @@ -1758,7 +1822,7 @@ fn decode_outputs_v1(noun: &Noun) -> Result, NounDecodeError> { continue; } - let note_version = u64::from_noun(¬e_head).map_err(|e| { + let note_version = u64::from_noun_handle(¬e_head).map_err(|e| { NounDecodeError::Custom(format!( "decode_outputs_v1: output {} note version not atom: {e:?}", idx @@ -1774,7 +1838,7 @@ fn decode_outputs_v1(noun: &Noun) -> Result, NounDecodeError> { idx )) })?; - let _origin_page = u64::from_noun(¬e_tail.head()).map_err(|e| { + let _origin_page = u64::from_noun_handle(¬e_tail.head()).map_err(|e| { NounDecodeError::Custom(format!( "decode_outputs_v1: output {} origin-page not atom: {e:?}", idx @@ -1786,7 +1850,7 @@ fn decode_outputs_v1(noun: &Noun) -> Result, NounDecodeError> { idx )) })?; - let name = Name::from_noun(¬e_tail.head()).map_err(|e| { + let name = Name::from_noun_handle(¬e_tail.head()).map_err(|e| { NounDecodeError::Custom(format!( "decode_outputs_v1: output {} name parse failed: {e:?}", idx @@ -1799,7 +1863,7 @@ fn decode_outputs_v1(noun: &Noun) -> Result, NounDecodeError> { )) })?; let _note_data = note_tail.head(); // z-map @tas * - skip - let assets = u64::from_noun(¬e_tail.tail()).map_err(|e| { + let assets = u64::from_noun_handle(¬e_tail.tail()).map_err(|e| { NounDecodeError::Custom(format!( "decode_outputs_v1: output {} assets not atom: {e:?}", idx @@ -1815,7 +1879,7 @@ fn decode_outputs_v1(noun: &Noun) -> Result, NounDecodeError> { Ok(outputs) } -fn decode_outputs_v0(noun: &Noun) -> Result, NounDecodeError> { +fn decode_outputs_v0(noun: &NounHandle) -> Result, NounDecodeError> { if let Ok(atom) = noun.as_atom() { if atom.as_u64()? == 0 { return Ok(Vec::new()); @@ -1823,7 +1887,7 @@ fn decode_outputs_v0(noun: &Noun) -> Result, NounDecodeError> { } let mut outputs = Vec::new(); - for (idx, entry) in HoonMapIter::from(*noun).enumerate() { + for (idx, entry) in HoonMapIter::new(noun).enumerate() { if !entry.is_cell() { return Err(NounDecodeError::Custom(format!( "decode_outputs_v0: entry {} is not a cell (expected z-map [lock note] pair)", @@ -1831,7 +1895,7 @@ fn decode_outputs_v0(noun: &Noun) -> Result, NounDecodeError> { ))); } let [key, value] = entry.uncell().map_err(|_| NounDecodeError::ExpectedCell)?; - let lock = Lock::from_noun(&key).map_err(|e| { + let lock = Lock::from_noun_handle(&key).map_err(|e| { NounDecodeError::Custom(format!( "decode_outputs_v0: output {} lock parse failed: {}", idx, e @@ -1840,7 +1904,7 @@ fn decode_outputs_v0(noun: &Noun) -> Result, NounDecodeError> { let value_cell = value.as_cell().map_err(|_| { NounDecodeError::Custom(format!("decode_outputs_v0: output {} value not cell", idx)) })?; - let note = NoteV0::from_noun(&value_cell.head()).map_err(|e| { + let note = NoteV0::from_noun_handle(&value_cell.head()).map_err(|e| { NounDecodeError::Custom(format!( "decode_outputs_v0: output {} note parse failed: {}", idx, e @@ -2020,66 +2084,40 @@ fn lock_summary(lock: &Lock) -> String { // Full Page Details - Noun Decoding Types // ============================================================================ -/// Noun decoder for full page entry -#[derive(Debug, Clone, NounDecode)] -struct FullPageEntryNoun { - height: BlockHeight, - tail: FullPageEntryTail, -} - -#[derive(Debug, Clone, NounDecode)] -struct FullPageEntryTail { - block_id: Hash, - tail: FullPageData, -} - -#[derive(Debug, Clone, NounDecode)] -struct FullPageData { - page: FullPageNoun, - txs: Noun, -} - -/// Full page structure matching Hoon's page type -#[derive(Debug, Clone)] -struct FullPageNoun { - /// For v1: version tag (%1), for v0: this is actually digest - version_or_digest: Noun, - /// Remaining page fields - rest: Noun, -} - -impl NounDecode for FullPageNoun { - fn from_noun(noun: &Noun) -> Result { - let cell = noun.as_cell()?; - Ok(Self { - version_or_digest: cell.head(), - rest: cell.tail(), - }) - } -} - -impl TryFrom for FullPageDetails { - type Error = NounDecodeError; - - fn try_from(raw: FullPageEntryNoun) -> Result { - let height = raw.height.0 .0; - let block_id = raw.tail.block_id; - let page = raw.tail.tail.page; - let txs_noun = raw.tail.tail.txs; +impl FullPageDetails { + fn from_noun_handle(noun: &NounHandle) -> std::result::Result { + let cell = noun.as_cell().map_err(|_| NounDecodeError::ExpectedCell)?; + let height = BlockHeight::from_noun_handle(&cell.head())?.0 .0; + + let tail = cell + .tail() + .as_cell() + .map_err(|_| NounDecodeError::ExpectedCell)?; + let block_id = Hash::from_noun_handle(&tail.head())?; + + let page_and_txs = tail + .tail() + .as_cell() + .map_err(|_| NounDecodeError::ExpectedCell)?; + let page = page_and_txs.head(); + let txs_noun = page_and_txs.tail(); + + let page_cell = page.as_cell().map_err(|_| NounDecodeError::ExpectedCell)?; + let version_or_digest = page_cell.head(); + let rest = page_cell.tail(); // Determine if v0 or v1 page: // - v0 page: [digest pow parent ...] where digest is a Hash (cell of 5 Belts) // - v1 page: [%1 digest pow parent ...] where %1 is an atom // So if head is an atom, it's v1; if head is a cell (Hash), it's v0 - let head_is_atom = page.version_or_digest.is_atom(); - let head_is_cell = page.version_or_digest.is_cell(); - let head_as_u64 = page - .version_or_digest + let head_is_atom = version_or_digest.is_atom(); + let head_is_cell = version_or_digest.is_cell(); + let head_as_u64 = version_or_digest .as_atom() .ok() .and_then(|a| a.as_u64().ok()); - let head_as_bytes = page.version_or_digest.as_atom().ok().map(|a| { - let bytes = a.as_ne_bytes(); + let head_as_bytes = version_or_digest.as_atom().ok().map(|atom_handle| { + let bytes = atom_handle.as_ne_bytes(); format!("{:?} (len={})", bytes, bytes.len()) }); tracing::info!( @@ -2093,8 +2131,7 @@ impl TryFrom for FullPageDetails { if head_is_atom { // v1 page - head is the version tag - let version = page - .version_or_digest + let version = version_or_digest .as_atom() .map_err(|_| NounDecodeError::Custom("Expected atom for version".into()))? .as_u64() @@ -2106,12 +2143,10 @@ impl TryFrom for FullPageDetails { version ))); } - decode_v1_page(height, block_id, page.rest, txs_noun) + decode_v1_page(height, block_id, rest, txs_noun) } else { // v0 page - head is the digest - decode_v0_page( - height, block_id, page.version_or_digest, page.rest, txs_noun, - ) + decode_v0_page(height, block_id, version_or_digest, rest, txs_noun) } } } @@ -2119,12 +2154,12 @@ impl TryFrom for FullPageDetails { fn decode_v0_page( height: u64, block_id: Hash, - digest_noun: Noun, - rest: Noun, - txs_noun: Noun, + digest_noun: NounHandle, + rest: NounHandle, + txs_noun: NounHandle, ) -> Result { // v0 page: [digest pow parent tx-ids coinbase timestamp epoch-counter target accumulated-work height msg] - let _digest = Hash::from_noun(&digest_noun) + let _digest = Hash::from_noun_handle(&digest_noun) .map_err(|e| NounDecodeError::Custom(format!("v0 page digest: {}", e)))?; let cell = rest @@ -2137,7 +2172,7 @@ fn decode_v0_page( .tail() .as_cell() .map_err(|_| NounDecodeError::Custom("v0 page: expected cell after pow".into()))?; - let parent = Hash::from_noun(&cell.head()) + let parent = Hash::from_noun_handle(&cell.head()) .map_err(|e| NounDecodeError::Custom(format!("v0 page parent: {}", e)))?; let cell = cell @@ -2223,14 +2258,15 @@ fn decode_v0_page( fn decode_v1_page( height: u64, block_id: Hash, - rest: Noun, - txs_noun: Noun, + rest: NounHandle, + txs_noun: NounHandle, ) -> Result { + let _space = rest.space(); // v1 page (after version tag): [digest pow parent tx-ids coinbase timestamp epoch-counter target accumulated-work height msg] let cell = rest .as_cell() .map_err(|_| NounDecodeError::Custom("v1 page: rest should be cell".into()))?; - let _digest = Hash::from_noun(&cell.head()) + let _digest = Hash::from_noun_handle(&cell.head()) .map_err(|e| NounDecodeError::Custom(format!("v1 page digest: {}", e)))?; let cell = cell @@ -2244,7 +2280,7 @@ fn decode_v1_page( .tail() .as_cell() .map_err(|_| NounDecodeError::Custom("v1 page: expected cell after pow".into()))?; - let parent = Hash::from_noun(&cell.head()) + let parent = Hash::from_noun_handle(&cell.head()) .map_err(|e| NounDecodeError::Custom(format!("v1 page parent: {}", e)))?; let cell = cell @@ -2327,7 +2363,7 @@ fn decode_v1_page( }) } -fn decode_pow(noun: &Noun) -> Result<(bool, Option>), NounDecodeError> { +fn decode_pow(noun: &NounHandle) -> Result<(bool, Option>), NounDecodeError> { if noun.is_atom() { let atom = noun.as_atom()?; if atom.as_u64()? == 0 { @@ -2340,7 +2376,7 @@ fn decode_pow(noun: &Noun) -> Result<(bool, Option>), NounDecodeError> { } } -fn decode_bignum(noun: &Noun) -> Result { +fn decode_bignum(noun: &NounHandle) -> Result { // Bignum can be [%bn list-of-u32] or just a raw atom if let Ok(cell) = noun.as_cell() { if let Ok(tag) = cell.head().as_atom() { @@ -2373,7 +2409,7 @@ fn decode_bignum(noun: &Noun) -> Result { Ok(BigNumValue { raw_bytes: bytes }) } -fn decode_coinbase(noun: &Noun) -> Result { +fn decode_coinbase(noun: &NounHandle) -> Result { // Coinbase in pages can be: // - Atom 0: empty map (genesis or no coinbase) // - v0 raw format: (z-map sig coins) where sig is a complex structure @@ -2417,7 +2453,7 @@ fn decode_coinbase(noun: &Noun) -> Result { } } -fn decode_coinbase_v1_map(noun: &Noun) -> Result, NounDecodeError> { +fn decode_coinbase_v1_map(noun: &NounHandle) -> Result, NounDecodeError> { if let Ok(atom) = noun.as_atom() { if atom.as_u64()? == 0 { return Ok(Vec::new()); @@ -2425,21 +2461,21 @@ fn decode_coinbase_v1_map(noun: &Noun) -> Result, NounDecodeErr } let mut entries = Vec::new(); - for entry in HoonMapIter::from(*noun) { + for entry in HoonMapIter::new(noun) { if !entry.is_cell() { continue; } let [key, value] = entry.uncell().map_err(|_| NounDecodeError::ExpectedCell)?; - let hash = Hash::from_noun(&key)?; + let hash = Hash::from_noun_handle(&key)?; let coins = value.as_atom()?.as_u64()?; entries.push((hash, coins)); } Ok(entries) } -fn count_map_entries(noun: &Noun) -> usize { +fn count_map_entries(noun: &NounHandle) -> usize { let mut count = 0; - for entry in HoonMapIter::from(*noun) { + for entry in HoonMapIter::new(noun) { if entry.is_cell() { count += 1; } @@ -2447,7 +2483,7 @@ fn count_map_entries(noun: &Noun) -> usize { count } -fn decode_page_msg(noun: &Noun) -> Result { +fn decode_page_msg(noun: &NounHandle) -> Result { // PageMsg is a list of u32, we'll just get the raw bytes if let Ok(atom) = noun.as_atom() { if atom.as_u64()? == 0 { @@ -2564,6 +2600,7 @@ impl FullPageDetails { mod tests { use nockchain_math::belt::Belt; use nockchain_types::tx_engine::common::{BlockHeight, Hash}; + use nockvm::noun::NounAllocator; use noun_serde::{NounDecode, NounEncode}; use super::*; @@ -2575,12 +2612,13 @@ mod tests { // Test that BlockHeight encodes as a simple atom let height = BlockHeight(Belt(105)); let height_noun = height.to_noun(&mut slab); + let space = slab.noun_space(); // Verify it's an atom assert!(height_noun.is_atom(), "BlockHeight should encode as atom"); // Try to decode as u64 directly - let result_u64 = u64::from_noun(&height_noun); + let result_u64 = u64::from_noun(&height_noun, &space); match result_u64 { Ok(val) => { println!("BlockHeight decoded as u64: {}", val); @@ -2592,7 +2630,7 @@ mod tests { } // Try to decode as BlockHeight - let result_height = BlockHeight::from_noun(&height_noun); + let result_height = BlockHeight::from_noun(&height_noun, &space); match result_height { Ok(h) => { println!("BlockHeight decoded correctly: {:?}", h); @@ -2660,8 +2698,9 @@ mod tests { let entry_noun = nockvm::noun::T(&mut slab, &[height_noun, block_page_cell]); // Try to decode it - let raw = BlockRangeEntryNoun::from_noun(&entry_noun).expect("decode raw entry"); - let entry = BlockRangeEntry::try_from(raw).expect("convert raw entry"); + let space = slab.noun_space(); + let entry = BlockRangeEntry::from_noun_handle(&entry_noun.in_space(&space)) + .expect("decode block range entry"); assert_eq!(entry.height, 42); assert_eq!(entry.block_id, block_id); diff --git a/crates/nockapp-grpc/src/services/public_nockchain/v2/driver.rs b/crates/nockapp-grpc/src/services/public_nockchain/v2/driver.rs index 6dfd411d3..ebb8f8873 100644 --- a/crates/nockapp-grpc/src/services/public_nockchain/v2/driver.rs +++ b/crates/nockapp-grpc/src/services/public_nockchain/v2/driver.rs @@ -2,7 +2,7 @@ use std::net::SocketAddr; use nockapp::driver::{make_driver, IODriverFn, NockAppHandle}; use nockchain_types::tx_engine::v1; -use nockvm::ext::NounExt; +use nockvm::noun::{NounAllocator, NounSpace}; use nockvm_macros::tas; use noun_serde::{NounDecode, NounDecodeError}; use tracing::{error, info, warn}; @@ -16,8 +16,8 @@ pub enum PublicNockchainEffect { } impl NounDecode for PublicNockchainEffect { - fn from_noun(effect: &nockapp::Noun) -> Result { - let effect_cell = effect.as_cell()?; + fn from_noun(effect: &nockapp::Noun, space: &NounSpace) -> Result { + let effect_cell = effect.in_space(space).as_cell()?; if !effect_cell.head().eq_bytes(b"nockchain-grpc") { return Err(NounDecodeError::InvalidTag); } @@ -25,13 +25,15 @@ impl NounDecode for PublicNockchainEffect { let payload_cell = effect_cell.tail().as_cell()?; let tag_atom = payload_cell.head().as_atom()?; let tag = tag_atom + .atom() .as_direct() .map_err(|_| NounDecodeError::InvalidTag)? .data(); match tag { t if t == tas!(b"send-tx") => { - let raw_tx = v1::RawTx::from_noun(&payload_cell.tail())?; + let raw_tx_noun = payload_cell.tail().noun(); + let raw_tx = v1::RawTx::from_noun(&raw_tx_noun, space)?; Ok(PublicNockchainEffect::SendTx { raw_tx }) } _ => Err(NounDecodeError::InvalidTag), @@ -78,7 +80,12 @@ pub fn grpc_listener_driver(addr: String) -> IODriverFn { Err(_) => continue, }; - let effect = match PublicNockchainEffect::from_noun(unsafe { effect.root() }) { + let effect = { + let effect_noun = unsafe { effect.root() }; + let space = effect.noun_space(); + PublicNockchainEffect::from_noun(&effect_noun, &space) + }; + let effect = match effect { Ok(effect) => effect, Err(NounDecodeError::InvalidTag) => continue, Err(err) => { diff --git a/crates/nockapp-grpc/src/services/public_nockchain/v2/ip_blocklist.rs b/crates/nockapp-grpc/src/services/public_nockchain/v2/ip_blocklist.rs new file mode 100644 index 000000000..1bf62449b --- /dev/null +++ b/crates/nockapp-grpc/src/services/public_nockchain/v2/ip_blocklist.rs @@ -0,0 +1,195 @@ +//! IP blocklist enforced at the public gRPC API edge. +//! +//! Public API requests reach us behind a load balancer which sets the +//! `x-forwarded-for` header in the form `RECENT_IP,LOAD_BALANCER_IP`, where +//! `RECENT_IP` is the real client. This module rejects a request when that +//! client IP is on the blocklist (the load balancer's own IP is ignored). +//! +//! The blocklist is the union of: +//! * a compiled-in default set (so the list survives a missing env var), and +//! * any addresses supplied via the `NOCKCHAIN_API_IP_BLOCKLIST` environment +//! variable (comma- or whitespace-separated). +//! +//! To change the blocklist operationally, set/extend `NOCKCHAIN_API_IP_BLOCKLIST` +//! and restart the API. It is read once at server startup. + +use std::collections::HashSet; +use std::net::{IpAddr, SocketAddr}; +use std::sync::Arc; + +use tonic::service::InterceptorLayer; +use tonic::{Request, Status}; +use tracing::{info, warn}; + +use super::metrics::NockchainGrpcApiMetrics; + +/// Environment variable holding additional blocked IPs (comma/whitespace separated). +pub const BLOCKLIST_ENV_VAR: &str = "NOCKCHAIN_API_IP_BLOCKLIST"; + +/// Always-blocked addresses, compiled in so the list survives a missing env var. +const DEFAULT_BLOCKED_IPS: &[&str] = &["90.65.165.234"]; + +/// A set of client IPs that are denied access to the public gRPC API. +/// +/// Cheap to clone (the underlying set is shared via `Arc`). +#[derive(Clone)] +pub struct IpBlocklist { + blocked: Arc>, +} + +impl IpBlocklist { + /// Build the blocklist from the compiled-in defaults plus + /// `NOCKCHAIN_API_IP_BLOCKLIST`. Unparseable entries are logged and skipped. + pub fn from_env_and_defaults() -> Self { + let mut blocked = HashSet::new(); + + for ip in DEFAULT_BLOCKED_IPS { + match ip.parse::() { + Ok(addr) => { + blocked.insert(addr); + } + Err(e) => warn!("Invalid compiled-in blocked IP {ip:?}: {e}"), + } + } + + if let Ok(raw) = std::env::var(BLOCKLIST_ENV_VAR) { + for token in raw.split([',', ' ', '\t', '\n', '\r']) { + let token = token.trim(); + if token.is_empty() { + continue; + } + match parse_ip(token) { + Some(addr) => { + blocked.insert(addr); + } + None => { + warn!("Ignoring unparseable entry {token:?} in {BLOCKLIST_ENV_VAR}") + } + } + } + } + + if blocked.is_empty() { + info!("API IP blocklist is empty"); + } else { + info!("API IP blocklist active with {} address(es)", blocked.len()); + } + + Self { + blocked: Arc::new(blocked), + } + } + + #[cfg(test)] + pub fn from_ips(ips: impl IntoIterator) -> Self { + Self { + blocked: Arc::new(ips.into_iter().collect()), + } + } + + pub fn is_empty(&self) -> bool { + self.blocked.is_empty() + } + + /// Checks the client IP in an `x-forwarded-for` header value. + /// + /// Our load balancer emits exactly `RECENT_IP,LOAD_BALANCER_IP`, where + /// `RECENT_IP` (the leftmost entry) is the real client. We therefore match + /// only the first token and ignore the load balancer's own address. + /// Returns the client IP if it is blocked. + pub fn blocked_in_xff(&self, xff: &str) -> Option { + let recent_ip = parse_ip(xff.split(',').next()?.trim())?; + self.blocked.contains(&recent_ip).then_some(recent_ip) + } +} + +/// Parse an IP from an XFF token, tolerating an optional `:port`, +/// `[v6]:port`, or bracketed `[v6]` form that some proxies emit. +fn parse_ip(token: &str) -> Option { + if let Ok(ip) = token.parse::() { + return Some(ip); + } + if let Ok(sa) = token.parse::() { + return Some(sa.ip()); + } + let unbracketed = token.trim_start_matches('[').trim_end_matches(']'); + unbracketed.parse::().ok() +} + +/// A tonic layer that rejects requests whose `x-forwarded-for` header contains +/// a blocked IP, before they reach any service handler. +/// +/// Applied server-wide via `Server::builder().layer(..)`, so it also guards the +/// health, reflection and block-explorer/metrics services. Requests with no +/// `x-forwarded-for` (e.g. proxy/LB health probes) are passed through. +pub fn blocklist_layer( + blocklist: IpBlocklist, + metrics: Arc, +) -> InterceptorLayer) -> Result, Status> + Clone> { + InterceptorLayer::new(move |req: Request<()>| { + if let Some(xff) = req + .metadata() + .get("x-forwarded-for") + .and_then(|v| v.to_str().ok()) + { + if let Some(blocked) = blocklist.blocked_in_xff(xff) { + metrics.api_request_blocked.increment(); + warn!("Rejecting blocked client IP {blocked} (x-forwarded-for: {xff:?})"); + return Err(Status::permission_denied("client IP is blocked")); + } + } + Ok(req) + }) +} + +#[cfg(test)] +mod tests { + use super::*; + + fn ip(s: &str) -> IpAddr { + s.parse().unwrap() + } + + #[test] + fn default_bad_ip_is_blocked() { + let bl = IpBlocklist::from_env_and_defaults(); + assert_eq!( + bl.blocked_in_xff("90.65.165.234"), + Some(ip("90.65.165.234")) + ); + } + + #[test] + fn matches_recent_ip_only() { + let bl = IpBlocklist::from_ips([ip("90.65.165.234")]); + // Real format: RECENT_IP,LOAD_BALANCER_IP — RECENT_IP is blocked. + assert_eq!( + bl.blocked_in_xff("90.65.165.234,10.0.0.1"), + Some(ip("90.65.165.234")) + ); + } + + #[test] + fn ignores_load_balancer_ip() { + // The blocked IP only appears as the load balancer entry; the real + // client (RECENT_IP) is clean, so the request must pass. + let bl = IpBlocklist::from_ips([ip("10.0.0.1")]); + assert_eq!(bl.blocked_in_xff("90.65.165.234,10.0.0.1"), None); + } + + #[test] + fn clean_recent_ip_passes() { + let bl = IpBlocklist::from_ips([ip("90.65.165.234")]); + assert_eq!(bl.blocked_in_xff("1.2.3.4,10.0.0.1"), None); + } + + #[test] + fn tolerates_port_and_brackets() { + let bl = IpBlocklist::from_ips([ip("90.65.165.234"), ip("2001:db8::1")]); + assert_eq!( + bl.blocked_in_xff("90.65.165.234:54321"), + Some(ip("90.65.165.234")) + ); + assert_eq!(bl.blocked_in_xff("[2001:db8::1]"), Some(ip("2001:db8::1"))); + } +} diff --git a/crates/nockapp-grpc/src/services/public_nockchain/v2/metrics.rs b/crates/nockapp-grpc/src/services/public_nockchain/v2/metrics.rs index 5c816f71d..82ef13daa 100644 --- a/crates/nockapp-grpc/src/services/public_nockchain/v2/metrics.rs +++ b/crates/nockapp-grpc/src/services/public_nockchain/v2/metrics.rs @@ -269,7 +269,8 @@ metrics_struct![ ( block_explorer_get_block_details_p99_ms, "nockchain_public_grpc.block_explorer.get_block_details.latency_p99_ms", Gauge - ) + ), + (api_request_blocked, "nockchain_public_grpc.api_request_blocked", Count) ]; static METRICS: OnceCell> = OnceCell::new(); diff --git a/crates/nockapp-grpc/src/services/public_nockchain/v2/mod.rs b/crates/nockapp-grpc/src/services/public_nockchain/v2/mod.rs index 81bb4a2ce..3b21d28d2 100644 --- a/crates/nockapp-grpc/src/services/public_nockchain/v2/mod.rs +++ b/crates/nockapp-grpc/src/services/public_nockchain/v2/mod.rs @@ -2,6 +2,7 @@ pub mod block_explorer; mod cache; pub mod client; pub mod driver; +pub mod ip_blocklist; pub mod metrics; pub mod server; diff --git a/crates/nockapp-grpc/src/services/public_nockchain/v2/server.rs b/crates/nockapp-grpc/src/services/public_nockchain/v2/server.rs index 2cc297213..ba20e3757 100644 --- a/crates/nockapp-grpc/src/services/public_nockchain/v2/server.rs +++ b/crates/nockapp-grpc/src/services/public_nockchain/v2/server.rs @@ -9,7 +9,7 @@ use nockapp::nockapp::NockAppExit; use nockapp::noun::slab::NounSlab; use nockapp::wire::WireRepr; use nockchain_types::tx_engine::{v0, v1}; -use nockvm::noun::SIG; +use nockvm::noun::{NounAllocator, SIG}; use noun_serde::{NounDecode, NounEncode}; use tokio::sync::RwLock; use tokio::time::{self, Duration}; @@ -22,6 +22,7 @@ use super::block_explorer::BlockExplorerCache; use super::cache::{ AddressBalanceCache, DEFAULT_PAGE_BYTES, DEFAULT_PAGE_SIZE, MAX_PAGE_BYTES, MAX_PAGE_SIZE, }; +use super::ip_blocklist::{blocklist_layer, IpBlocklist}; use super::metrics::{init_metrics, NockchainGrpcApiMetrics}; use crate::error::{NockAppGrpcError, Result}; use crate::pb::common::v1::{Acknowledged, ErrorCode, ErrorStatus}; @@ -44,6 +45,7 @@ use crate::v2::pagination::{ use crate::wire_conversion::{create_grpc_wire, grpc_wire_to_nockapp}; const DEFAULT_HEAVIEST_CHAIN_REFRESH_INTERVAL: Duration = Duration::from_secs(60); +const DEFAULT_BLOCK_EXPLORER_REFRESH_INTERVAL: Duration = Duration::from_secs(120); #[async_trait] pub trait BalanceHandle: Send + Sync { @@ -175,7 +177,13 @@ impl PublicNockchainGrpcServer { self.metrics.clone(), )); + // Reject blocked client IPs (from the front proxy's x-forwarded-for) + // before requests reach any service. Configured via the + // NOCKCHAIN_API_IP_BLOCKLIST env var on top of compiled-in defaults. + let blocklist = IpBlocklist::from_env_and_defaults(); + Server::builder() + .layer(blocklist_layer(blocklist, self.metrics.clone())) .add_service(health_service) .add_service(reflection_service_v1) .add_service(nockchain_api) @@ -221,7 +229,9 @@ impl PublicNockchainGrpcServer { let result = match peek_result { Ok(Some(result_slab)) => { let result_noun = unsafe { result_slab.root() }; - match >>::from_noun(&result_noun) { + let space = result_slab.noun_space(); + match >>::from_noun(&result_noun, &space) + { Ok(opt) => Ok(opt.flatten()), // Peek either returned [~ ~] or ~ Err(_) => Err(NockAppGrpcError::PeekReturnedNoData), @@ -273,7 +283,7 @@ impl PublicNockchainGrpcServer { }; info!("Block explorer refresh worker starting"); - let mut interval = time::interval(Duration::from_secs(15)); + let mut interval = time::interval(DEFAULT_BLOCK_EXPLORER_REFRESH_INTERVAL); let mut initialized = false; let mut backfill_started = false; @@ -758,7 +768,9 @@ impl NockchainService for PublicNockchainGrpcServer { match peek_result { Ok(Some(result_slab)) => { let result_noun = unsafe { result_slab.root() }; - let result = >>::from_noun(&result_noun); + let space = result_slab.noun_space(); + let result = + >>::from_noun(&result_noun, &space); match result { Ok(update) => { @@ -985,7 +997,9 @@ impl NockchainService for PublicNockchainGrpcServer { match peek_result { Ok(Some(result_slab)) => { let result_noun = unsafe { result_slab.root() }; - let result = >>::from_noun(&result_noun); + let space = result_slab.noun_space(); + let result = + >>::from_noun(&result_noun, &space); match result { Ok(update) => { @@ -1320,7 +1334,8 @@ impl NockchainService for PublicNockchainGrpcServer { match peek_result { Ok(Some(result_slab)) => { let result_noun = unsafe { result_slab.root() }; - match >>::from_noun(&result_noun) { + let space = result_slab.noun_space(); + match >>::from_noun(&result_noun, &space) { Ok(opt) => { let accepted = opt.flatten().unwrap_or(false); timed_return( @@ -1914,8 +1929,9 @@ mod tests { &self, path: NounSlab, ) -> std::result::Result, nockapp::nockapp::error::NockAppError> { + let space = path.noun_space(); let root = unsafe { path.root() }; - if let Ok(segments) = >::from_noun(&root) { + if let Ok(segments) = >::from_noun(&root, &space) { if segments.first().map(String::as_str) == Some("heaviest-chain") { let mut slab = NounSlab::new(); let noun = Some(Some(( @@ -1950,8 +1966,9 @@ mod tests { &self, path: NounSlab, ) -> std::result::Result, nockapp::nockapp::error::NockAppError> { + let space = path.noun_space(); let root = unsafe { path.root() }; - if let Ok(segments) = >::from_noun(&root) { + if let Ok(segments) = >::from_noun(&root, &space) { if segments.first().map(String::as_str) == Some("heaviest-chain") { let mut slab = NounSlab::new(); let noun = Some(Some(( @@ -1980,7 +1997,7 @@ mod tests { } } - #[tokio::test] + #[tokio::test(flavor = "current_thread")] async fn wallet_get_balance_uses_cache_for_subsequent_pages() { let (update, expected_names) = fixtures_v1::make_balance_update(4); let handle = Arc::new(MockHandleV0::new(update)); @@ -2056,7 +2073,7 @@ mod tests { assert_eq!(handle.peek_calls(), 1, "cache should prevent second peek"); } - #[tokio::test] + #[tokio::test(flavor = "current_thread")] async fn wallet_get_balance_by_first_name_uses_cache_for_subsequent_pages() { // TODO: finish test let (update, expected_names) = fixtures::make_balance_update(4); diff --git a/crates/nockapp/Cargo.toml b/crates/nockapp/Cargo.toml index b66b47172..29f6c2f2d 100644 --- a/crates/nockapp/Cargo.toml +++ b/crates/nockapp/Cargo.toml @@ -14,6 +14,7 @@ slog-tracing = [] trait-alias = [] bazel_build = [] tracing-tracy = ["dep:tracing-tracy"] +pma-assert = ["nockvm/pma-assert"] [dependencies] @@ -29,14 +30,17 @@ bytes = { workspace = true, features = ["serde"] } chrono = { workspace = true } clap = { workspace = true, features = ["derive", "cargo", "color", "env"] } config = { workspace = true } +diesel = { workspace = true } +diesel_migrations = { workspace = true } dirs = { workspace = true } either = { workspace = true } futures = { workspace = true } -getrandom = { workspace = true } gnort = { workspace = true } ibig = { workspace = true } instant-acme = { workspace = true } intmap = { workspace = true } +libc = { workspace = true } +libsqlite3-sys = { workspace = true } nockvm = { workspace = true } nockvm_macros = { workspace = true } noun-serde = { workspace = true } @@ -68,6 +72,10 @@ tracing-tracy = { workspace = true, optional = true, features = [ "manual-lifetime", "ondemand", ] } +uuid = { workspace = true } webpki-roots = { workspace = true } x509-parser = { workspace = true } yaque = { workspace = true } + +[dev-dependencies] +rusqlite = { workspace = true } diff --git a/crates/nockapp/build.rs b/crates/nockapp/build.rs new file mode 100644 index 000000000..d2632c513 --- /dev/null +++ b/crates/nockapp/build.rs @@ -0,0 +1,17 @@ +use std::env; +use std::error::Error; +use std::path::PathBuf; + +fn main() -> Result<(), Box> { + let manifest_dir = PathBuf::from(env::var("CARGO_MANIFEST_DIR")?); + let dumb_jam = manifest_dir.join("../../assets/dumb.jam"); + + println!("cargo:rustc-env=DUMB_JAM_PATH={}", dumb_jam.display()); + println!("cargo:rerun-if-changed={}", dumb_jam.display()); + println!( + "cargo:rerun-if-changed={}", + manifest_dir.join("migrations").display() + ); + + Ok(()) +} diff --git a/crates/nockapp/migrations/20260505000000_initial_event_log_schema/down.sql b/crates/nockapp/migrations/20260505000000_initial_event_log_schema/down.sql new file mode 100644 index 000000000..4778223d5 --- /dev/null +++ b/crates/nockapp/migrations/20260505000000_initial_event_log_schema/down.sql @@ -0,0 +1,6 @@ +DROP INDEX IF EXISTS snapshots_event_idx; +DROP INDEX IF EXISTS snapshots_kind_ts_idx; +DROP TABLE IF EXISTS snapshots; +DROP INDEX IF EXISTS events_event_num_idx; +DROP TABLE IF EXISTS events; +DROP TABLE IF EXISTS meta; diff --git a/crates/nockapp/migrations/20260505000000_initial_event_log_schema/up.sql b/crates/nockapp/migrations/20260505000000_initial_event_log_schema/up.sql new file mode 100644 index 000000000..6b8fc8ed4 --- /dev/null +++ b/crates/nockapp/migrations/20260505000000_initial_event_log_schema/up.sql @@ -0,0 +1,42 @@ +-- IF NOT EXISTS lets pre-Diesel PMA event logs adopt Diesel migration tracking. +CREATE TABLE IF NOT EXISTS meta ( + key TEXT PRIMARY KEY, + value BLOB NOT NULL +); + +CREATE TABLE IF NOT EXISTS events ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + event_num INTEGER NOT NULL UNIQUE, + job_jam BLOB NOT NULL, + wire_source TEXT NOT NULL, + wire_version INTEGER NOT NULL, + wire_tags_json TEXT NOT NULL, + cause_hash BLOB NOT NULL, + job_hash BLOB NOT NULL, + event_processing_duration_us INTEGER NOT NULL DEFAULT 0, + created_at_ms INTEGER NOT NULL +); + +CREATE INDEX IF NOT EXISTS events_event_num_idx ON events(event_num); + +CREATE TABLE IF NOT EXISTS snapshots ( + snapshot_id INTEGER PRIMARY KEY AUTOINCREMENT, + kind TEXT NOT NULL CHECK(kind IN ('epoch','rotating')), + state TEXT NOT NULL CHECK(state IN ('writing','ready','failed','retired')), + event_num INTEGER NOT NULL, + pma_path TEXT NOT NULL, + manifest_path TEXT NOT NULL, + alloc_words INTEGER NOT NULL, + kernel_root_raw INTEGER NOT NULL, + cold_offset INTEGER NOT NULL, + used_blake3 BLOB NOT NULL, + structure_blake3 BLOB, + created_at_ms INTEGER NOT NULL, + activated_at_ms INTEGER, + base_snapshot_id INTEGER, + timestamp_tag TEXT NOT NULL, + UNIQUE(kind, timestamp_tag) +); + +CREATE INDEX IF NOT EXISTS snapshots_kind_ts_idx ON snapshots(kind, timestamp_tag DESC); +CREATE INDEX IF NOT EXISTS snapshots_event_idx ON snapshots(event_num DESC); diff --git a/crates/nockapp/nockapp_tests.bzl b/crates/nockapp/nockapp_tests.bzl new file mode 100644 index 000000000..a5cd6d4b7 --- /dev/null +++ b/crates/nockapp/nockapp_tests.bzl @@ -0,0 +1,107 @@ +"""Bazel helpers for nockapp Rust test targets.""" + +load("@crates//:defs.bzl", "aliases", "all_crate_deps") +load("@rules_rust//rust:defs.bzl", "rust_test") + +NOCKAPP_BOOT_PMA_TESTS = [ + "kernel::boot::tests::bootstraps_checkpoint_copy_into_empty_event_log", + "kernel::boot::tests::bootstraps_pma_from_checkpoint_once", + "kernel::boot::tests::checkpoint_ahead_of_event_log_recovers_to_sqlite_boundary", + "kernel::boot::tests::checkpoint_behind_event_log_replays_to_sqlite_boundary", + "kernel::boot::tests::export_state_jam_creates_parent_dir", + "kernel::boot::tests::falls_back_from_corrupt_newest_rotating_snapshot", + "kernel::boot::tests::falls_back_from_manifest_only_corruption", + "kernel::boot::tests::fresh_boot_with_committed_event_log_replays_from_zero", + "kernel::boot::tests::honors_active_snapshot_selection_before_ordering", + "kernel::boot::tests::moves_orphan_snapshot_files_to_corrupted_pma", + "kernel::boot::tests::pma_ahead_of_event_log_recovers_to_sqlite_boundary", + "kernel::boot::tests::pma_gc_switches_slabs_and_rebuilds_runtime_state", + "kernel::boot::tests::pma_lagging_event_log_replays_to_sqlite_boundary", + "kernel::boot::tests::refuses_boot_on_event_log_gap_after_snapshot", + "kernel::boot::tests::replay_rejected_logged_event_fails_instead_of_synthesizing_crud", + "kernel::boot::tests::replays_logged_events_after_snapshot_restore", + "kernel::boot::tests::restores_epoch_snapshot_when_pma_is_missing", + "kernel::boot::tests::rotates_snapshots_and_retires_oldest", + "kernel::boot::tests::setup_allows_new_for_empty_data_dir", + "kernel::boot::tests::shutdown_flush_rewrites_missing_active_pma_metadata", + "kernel::boot::tests::snapshot_kernel_hash_mismatch_loads_state_like_checkpoint", + "kernel::boot::tests::valid_pma_skips_corrupt_checkpoint_files", + "kernel::boot::tests::valid_pma_with_unopenable_event_log_fails_closed", +] + +_NOCKAPP_TEST_COMPILE_DATA = [ + "//open/assets:dumb", + "//open/crates/nockapp/test-jams:cue-test.jam", + "//open/crates/nockapp/test-jams:test-ker.jam", +] + +def nockapp_boot_pma_skip_args(): + """Returns libtest arguments that skip heavyweight PMA boot tests.""" + return _skip_args(NOCKAPP_BOOT_PMA_TESTS) + +def nockapp_boot_pma_test_args(): + """Returns libtest arguments for the isolated PMA boot target.""" + return NOCKAPP_BOOT_PMA_TESTS + ["--test-threads=1"] + +def nockapp_unit_rust_test(name, args = None, size = "medium", timeout = "moderate"): + """Defines a nockapp unit-test target with the crate's shared test setup. + + Args: + name: Bazel target name. + args: Optional libtest arguments. + size: Bazel test size. + timeout: Bazel test timeout. + """ + if args == None: + args = [] + # Embedded Diesel migrations are read by a proc macro at compile time. + migration_srcs = native.glob(["migrations/**/*.sql"]) + test_compile_data = migration_srcs + _NOCKAPP_TEST_COMPILE_DATA + rust_test( + name = name, + size = size, + timeout = timeout, + srcs = native.glob([ + "src/**/*.rs", + "src/*.rs", + ]), + aliases = aliases(), + args = args, + compile_data = test_compile_data, + crate_features = [ + "bazel_build", + "default", + "slog-tracing", + ], + crate_root = "src/lib.rs", + data = _NOCKAPP_TEST_COMPILE_DATA, + edition = "2021", + proc_macro_deps = [ + "//open/crates/nockvm/rust/nockvm_macros", + ] + all_crate_deps(proc_macro = True), + rustc_env = { + # rust_test compiles from bazel-out; point Diesel's embed_migrations! + # at the compile_data tree instead of the source-tree manifest dir. + "CARGO_MANIFEST_DIR": "$(BINDIR)/open/crates/nockapp", + "DUMB_JAM_PATH": "$(location //open/assets:dumb)", + }, + deps = all_crate_deps( + normal = True, + normal_dev = True, + ) + [ + "//open/crates/nockvm/rust/ibig", + "//open/crates/nockvm/rust/nockvm", + "//open/crates/noun-serde", + ], + ) + +def _skip_args(tests): + """Builds repeated --skip arguments for libtest. + + Args: + tests: Fully-qualified test names to skip. + """ + args = [] + for test in tests: + args.extend(["--skip", test]) + return args diff --git a/crates/nockapp/src/bin/nockapp-chkjam-to-state-jam.rs b/crates/nockapp/src/bin/nockapp-chkjam-to-state-jam.rs new file mode 100644 index 000000000..08b668ff8 --- /dev/null +++ b/crates/nockapp/src/bin/nockapp-chkjam-to-state-jam.rs @@ -0,0 +1,171 @@ +use std::path::{Path, PathBuf}; + +use anyhow::{bail, Context, Result}; +use clap::Parser; +use nockapp::export::ExportedState; +use nockapp::kernel::form::LoadState; +use nockapp::noun::slab::NockJammer; +use nockapp::save::{CheckpointBootstrapReader, SaveableCheckpoint}; +use tempfile::TempDir; +use tokio::fs; + +#[derive(Debug, Parser)] +#[command( + name = "nockapp-chkjam-to-state-jam", + about = "Convert checkpoint jam file(s) into state jam file(s) compatible with --state-jam" +)] +struct Cli { + /// Input checkpoint jam file(s) or checkpoint directories containing 0.chkjam/1.chkjam + #[arg(long = "input", required = true)] + inputs: Vec, + + /// Output file path (single input only) + #[arg(long, conflicts_with = "output_dir")] + output: Option, + + /// Output directory for generated state jam files + #[arg(long)] + output_dir: Option, + + /// Suffix for generated files when using --output-dir + #[arg(long, default_value = ".state.jam")] + suffix: String, + + /// Overwrite existing output files + #[arg(long, default_value_t = false)] + overwrite: bool, +} + +#[tokio::main(flavor = "current_thread")] +async fn main() -> Result<()> { + let cli = Cli::parse(); + + if cli.output.is_some() && cli.inputs.len() != 1 { + bail!("--output only supports exactly one --input"); + } + + let default_output_dir = + std::env::current_dir().context("failed to resolve current directory")?; + let output_dir = cli.output_dir.clone().unwrap_or(default_output_dir); + + if cli.output.is_none() { + fs::create_dir_all(&output_dir) + .await + .with_context(|| format!("failed to create output dir {}", output_dir.display()))?; + } + + for input in &cli.inputs { + let checkpoint = load_checkpoint(input).await?; + let output_path = match &cli.output { + Some(path) => path.clone(), + None => output_dir.join(output_name_for(input, &cli.suffix)), + }; + + if output_path.exists() && !cli.overwrite { + bail!( + "output already exists (pass --overwrite): {}", + output_path.display() + ); + } + + if let Some(parent) = output_path.parent() { + fs::create_dir_all(parent).await.with_context(|| { + format!( + "failed to create output parent directory {}", + parent.display() + ) + })?; + } + + let load_state = LoadState { + ker_hash: checkpoint.ker_hash, + event_num: checkpoint.event_num, + kernel_state: checkpoint.state, + }; + let exported = ExportedState::from_loadstate::(load_state); + let event_num = exported.event_num; + let ker_hash = exported.ker_hash; + let encoded = exported + .encode() + .context("failed to encode exported state")?; + + // Verify output can be decoded back into a load state before writing. + let decoded = ExportedState::decode(&encoded).context("failed to decode output bytes")?; + let _ = decoded.to_loadstate::().map_err(|err| { + anyhow::anyhow!("decoded output did not produce a valid load state: {err}") + })?; + + fs::write(&output_path, encoded) + .await + .with_context(|| format!("failed to write {}", output_path.display()))?; + + println!( + "wrote {} (input={} event_num={} ker_hash={})", + output_path.display(), + input.display(), + event_num, + ker_hash, + ); + } + + Ok(()) +} + +async fn load_checkpoint(input: &Path) -> Result { + if input.is_dir() { + let path = input.to_path_buf(); + let checkpoint = CheckpointBootstrapReader::::new(path) + .load_latest(None) + .await + .with_context(|| format!("failed to load checkpoint directory {}", input.display()))?; + return checkpoint.with_context(|| format!("no checkpoint found in {}", input.display())); + } + + if !input.exists() { + bail!("input does not exist: {}", input.display()); + } + if !input.is_file() { + bail!("input must be a file or directory: {}", input.display()); + } + + let temp = TempDir::new().context("failed to create temporary directory")?; + let checkpoint_dir = temp.path().join("checkpoints"); + fs::create_dir_all(&checkpoint_dir) + .await + .context("failed to create temporary checkpoints directory")?; + + let copied = checkpoint_dir.join("0.chkjam"); + fs::copy(input, &copied) + .await + .with_context(|| format!("failed to copy {} to {}", input.display(), copied.display()))?; + + let checkpoint = CheckpointBootstrapReader::::new(checkpoint_dir) + .load_latest(None) + .await + .with_context(|| { + format!( + "failed to decode checkpoint from temporary copy of {}", + input.display() + ) + })?; + + checkpoint.with_context(|| format!("no checkpoint found in {}", input.display())) +} + +fn output_name_for(input: &Path, suffix: &str) -> String { + let base = if input.is_dir() { + input + .file_name() + .and_then(|name| name.to_str()) + .unwrap_or("checkpoint") + .to_owned() + } else { + input + .file_stem() + .and_then(|name| name.to_str()) + .unwrap_or("checkpoint") + .to_owned() + }; + + format!("{base}{suffix}") +} diff --git a/crates/nockapp/src/drivers/exit.rs b/crates/nockapp/src/drivers/exit.rs index d43590cd6..1b4d7f18d 100644 --- a/crates/nockapp/src/drivers/exit.rs +++ b/crates/nockapp/src/drivers/exit.rs @@ -1,7 +1,7 @@ +use nockvm::noun::NounAllocator; use tracing::{debug, error}; use crate::nockapp::driver::{make_driver, IODriverFn}; -use crate::NounExt; /// Creates an IO driver function for handling exit signals. /// @@ -20,17 +20,31 @@ pub fn exit() -> IODriverFn { match eff { Ok(eff) => { unsafe { - let noun = eff.root(); - if let Ok(cell) = noun.as_cell() { - if cell.head().eq_bytes(b"exit") && cell.tail().is_atom() { - // Exit with the code provided in the tail - if let Ok(exit_code) = cell.tail().as_atom().and_then(|atom| atom.as_u64()) { - handle.exit.exit(exit_code as usize).await?; + let exit_code = { + let noun = eff.root(); + if let Ok(cell) = noun.as_cell() { + let space = eff.noun_space(); + let cell = cell.in_space(&space); + if cell.head().eq_bytes(b"exit") + && cell.tail().is_atom() + { + Some( + cell.tail() + .as_atom() + .and_then(|atom| atom.as_u64()) + .ok(), + ) } else { - // Default to error code 1 if we can't get a valid exit code - handle.exit.exit(1).await?; + None } + } else { + None } + }; + + if let Some(exit_code) = exit_code { + let exit_code = exit_code.unwrap_or(1); + handle.exit.exit(exit_code as usize).await?; } } } diff --git a/crates/nockapp/src/drivers/file.rs b/crates/nockapp/src/drivers/file.rs index c89f98a19..29125f643 100644 --- a/crates/nockapp/src/drivers/file.rs +++ b/crates/nockapp/src/drivers/file.rs @@ -1,7 +1,7 @@ use std::path::Path; use bytes::Bytes; -use nockvm::noun::{Atom, IndirectAtom, Noun, D, SIG, T}; +use nockvm::noun::{Atom, IndirectAtom, Noun, NounAllocator, NounSpace, D, SIG, T}; use nockvm_macros::tas; use noun_serde::{NounDecode, NounEncode}; use tokio::fs::{self, File, OpenOptions}; @@ -12,6 +12,7 @@ use crate::nockapp::driver::{make_driver, IODriverFn}; use crate::nockapp::error::NockAppError; use crate::nockapp::wire::{Wire, WireRepr}; use crate::noun::slab::NounSlab; +use crate::utils::durability; use crate::{AtomExt, IndirectAtomExt}; #[derive(Clone, Debug, NounDecode)] @@ -27,8 +28,8 @@ struct BatchWriteResultEntry { success: bool, } -fn decode_from_noun(noun: Noun) -> Result { - T::from_noun(&noun).map_err(|err| NockAppError::NounDecodeError(Box::new(err))) +fn decode_from_noun(noun: Noun, space: &NounSpace) -> Result { + T::from_noun(&noun, space).map_err(|err| NockAppError::NounDecodeError(Box::new(err))) } async fn ensure_parent_dirs(path: &str) -> std::io::Result<()> { @@ -53,7 +54,7 @@ async fn prepare_file_write(path: &str, contents: &[u8]) -> std::io::Result std::io::Result<()> { debug!("file driver: writing {} bytes to: {}", contents.len(), path); let file = prepare_file_write(path, contents).await?; - file.sync_all().await + durability::sync_all_async(&file, "file_driver_write_fsync", Some(Path::new(path))).await } pub enum FileWire { @@ -94,6 +95,12 @@ impl Wire for FileWire { /// `[%file %batch-write (list [path=@t contents=@ success=?])]` pub fn file() -> IODriverFn { make_driver(|handle| async move { + enum FileOp { + Read(String), + Write { path: String, contents: Bytes }, + BatchWrite(Vec), + } + loop { let effect_res = handle.next_effect().await; let slab = match effect_res { @@ -104,59 +111,71 @@ pub fn file() -> IODriverFn { } }; - let Ok(effect_cell) = unsafe { slab.root() }.as_cell() else { - continue; - }; + let parsed = (|| { + let space = slab.noun_space(); + let effect_cell = unsafe { slab.root() }.in_space(&space).as_cell().ok()?; - if !unsafe { effect_cell.head().raw_equals(&D(tas!(b"file"))) } { - continue; - } + if !unsafe { effect_cell.head().noun().raw_equals(&D(tas!(b"file"))) } { + return None; + } - let Ok(file_cell) = effect_cell.tail().as_cell() else { - continue; - }; + let file_cell = effect_cell.tail().as_cell().ok()?; + let operation = decode_from_noun::(file_cell.head().noun(), &space).ok()?; - let Ok(operation) = decode_from_noun::(file_cell.head()) else { + match operation.as_str() { + "read" => { + let path = + decode_from_noun::(file_cell.tail().noun(), &space).ok()?; + Some(FileOp::Read(path)) + } + "write" => { + let (path, contents) = + decode_from_noun::<(String, Bytes)>(file_cell.tail().noun(), &space) + .ok()?; + Some(FileOp::Write { path, contents }) + } + "batch-write" => { + let batch_entries = decode_from_noun::>( + file_cell.tail().noun(), + &space, + ) + .ok()?; + Some(FileOp::BatchWrite(batch_entries)) + } + _ => None, + } + })(); + + let Some(operation) = parsed else { continue; }; - match operation.as_str() { - "read" => { - let Ok(path) = decode_from_noun::(file_cell.tail()) else { - continue; - }; - match fs::read(&path).await { - Ok(contents) => { - let mut poke_slab = NounSlab::new(); - let contents_atom = ::from_bytes( - &mut poke_slab, - contents.as_slice(), - ); - let poke_noun: Noun = T( - &mut poke_slab, - &[D(tas!(b"file")), D(tas!(b"read")), SIG, contents_atom.as_noun()], - ); - poke_slab.set_root(poke_noun); - let wire = FileWire::Read.to_wire(); - handle.poke(wire, poke_slab).await?; - } - Err(_) => { - let mut poke_slab = NounSlab::new(); - let poke_items: Vec = - vec![D(tas!(b"file")), D(tas!(b"read")), D(0)]; - let poke_noun = poke_items.to_noun(&mut poke_slab); - poke_slab.set_root(poke_noun); - let wire = FileWire::Read.to_wire(); - handle.poke(wire, poke_slab).await?; - } + match operation { + FileOp::Read(path) => match fs::read(&path).await { + Ok(contents) => { + let mut poke_slab = NounSlab::new(); + let contents_atom = ::from_bytes( + &mut poke_slab, + contents.as_slice(), + ); + let poke_noun: Noun = T( + &mut poke_slab, + &[D(tas!(b"file")), D(tas!(b"read")), SIG, contents_atom.as_noun()], + ); + poke_slab.set_root(poke_noun); + let wire = FileWire::Read.to_wire(); + handle.poke(wire, poke_slab).await?; } - } - "write" => { - let Ok((path, contents)) = - decode_from_noun::<(String, Bytes)>(file_cell.tail()) - else { - continue; - }; + Err(_) => { + let mut poke_slab = NounSlab::new(); + let poke_noun: Noun = + T(&mut poke_slab, &[D(tas!(b"file")), D(tas!(b"read")), D(0)]); + poke_slab.set_root(poke_noun); + let wire = FileWire::Read.to_wire(); + handle.poke(wire, poke_slab).await?; + } + }, + FileOp::Write { path, contents } => { let success = match write_then_flush(&path, contents.as_ref()).await { Ok(_) => true, Err(e) => { @@ -186,12 +205,7 @@ pub fn file() -> IODriverFn { let wire = FileWire::Write.to_wire(); handle.poke(wire, poke_slab).await?; } - "batch-write" => { - let Ok(batch_entries) = - decode_from_noun::>(file_cell.tail()) - else { - continue; - }; + FileOp::BatchWrite(batch_entries) => { let mut results: Vec = Vec::with_capacity(batch_entries.len()); @@ -214,7 +228,13 @@ pub fn file() -> IODriverFn { } for (idx, path, file) in pending_flushes { - match file.sync_all().await { + match durability::sync_all_async( + &file, + "file_driver_batch_write_fsync", + Some(Path::new(&path)), + ) + .await + { Ok(_) => results[idx].success = true, Err(e) => error!("file driver: error flushing path {}: {}", path, e), } @@ -231,7 +251,6 @@ pub fn file() -> IODriverFn { let wire = FileWire::BatchWrite.to_wire(); handle.poke(wire, poke_slab).await?; } - _ => continue, } } }) diff --git a/crates/nockapp/src/drivers/http/http.rs b/crates/nockapp/src/drivers/http/http.rs index da279f032..9cda123d0 100644 --- a/crates/nockapp/src/drivers/http/http.rs +++ b/crates/nockapp/src/drivers/http/http.rs @@ -11,7 +11,7 @@ use axum::response::Response; use axum::routing::get; use axum::{serve, Router}; use axum_server::tls_rustls::RustlsConfig; -use nockvm::noun::{Atom, D, T}; +use nockvm::noun::{Atom, NounAllocator, D, T}; use nockvm_macros::tas; use tokio::select; use tokio::sync::{oneshot, RwLock}; @@ -525,69 +525,105 @@ pub fn http() -> IODriverFn { return Ok(()); } }; - let effect = unsafe { slab.root() }; - let res_list = effect.as_cell()?; - - let head_tag = res_list.head().as_atom()?; - let tag_val = head_tag.as_u64().map_err(|e| HttpError::AtomCreationError(e.to_string()))?; - if tag_val != tas!(b"res") && tag_val != tas!(b"cache") && tag_val != tas!(b"htmx") && tag_val != tas!(b"h-cache") { - debug!("http: not an HTTP response effect, skipping. Got tag: {:?}", head_tag); - return Ok(()); + struct ParsedEffect { + tag_val: u64, + id: u64, + status_code: u64, + headers: Vec<(String, String)>, + body: Option, } - debug!("processing HTTP response effect"); - let mut res = res_list.tail().as_cell()?; - let id = res.head().as_atom()?.as_u64() - .map_err(|e| HttpError::AtomCreationError(e.to_string()))?; - debug!("HTTP response for request id: {}", id); - - res = res.tail().as_cell()?; - let status_code = res - .head() - .as_atom()? - .direct() - .expect("not a valid status code!") - .data(); - debug!("HTTP response status code: {}", status_code); - - let mut header_list = res.tail().as_cell()?.head(); - let mut header_vec: Vec<(String, String)> = Vec::new(); - loop { - if header_list.is_atom() { - break; - } else { - let header = header_list.as_cell()?.head().as_cell()?; - let key_vec = header.head().as_atom()?; - let val_vec = header.tail().as_atom()?; - - if let Ok(key) = key_vec.to_bytes_until_nul() { - if let Ok(val) = val_vec.to_bytes_until_nul() { - let key_str = String::from_utf8(key)?; - let val_str = String::from_utf8(val)?; - debug!("HTTP response header: {}: {}", key_str, val_str); - header_vec.push((key_str, val_str)); - header_list = header_list.as_cell()?.tail(); + let parsed = (|| -> Result, HttpError> { + let effect = unsafe { slab.root() }; + let space = slab.noun_space(); + let res_list = effect.in_space(&space).as_cell()?; + + let head_tag = res_list.head().as_atom()?; + let tag_val = head_tag + .as_u64() + .map_err(|e| HttpError::AtomCreationError(e.to_string()))?; + if tag_val != tas!(b"res") + && tag_val != tas!(b"cache") + && tag_val != tas!(b"htmx") + && tag_val != tas!(b"h-cache") + { + debug!( + "http: not an HTTP response effect, skipping. Got tag: {:?}", + head_tag.atom() + ); + return Ok(None); + } + + debug!("processing HTTP response effect"); + let mut res = res_list.tail().as_cell()?; + let id = res + .head() + .as_atom()? + .as_u64() + .map_err(|e| HttpError::AtomCreationError(e.to_string()))?; + debug!("HTTP response for request id: {}", id); + + res = res.tail().as_cell()?; + let status_code = res + .head() + .as_atom()? + .atom() + .as_direct() + .expect("not a valid status code!") + .data(); + debug!("HTTP response status code: {}", status_code); + + let mut header_list = res.tail().as_cell()?.head().noun(); + let mut headers: Vec<(String, String)> = Vec::new(); + loop { + if header_list.is_atom() { + break; + } else { + let header = header_list + .in_space(&space) + .as_cell()? + .head() + .as_cell()?; + let key_vec = header.head().as_atom()?; + let val_vec = header.tail().as_atom()?; + + if let Ok(key) = key_vec.to_bytes_until_nul() { + if let Ok(val) = val_vec.to_bytes_until_nul() { + let key_str = String::from_utf8(key)?; + let val_str = String::from_utf8(val)?; + debug!( + "HTTP response header: {}: {}", + key_str, val_str + ); + headers.push((key_str, val_str)); + header_list = + header_list.in_space(&space).as_cell()?.tail().noun(); + } else { + break; + } } else { break; } - } else { - break; } } - } - let maybe_body = res.tail().as_cell()?.tail(); + let maybe_body = res.tail().as_cell()?.tail().noun(); - let body: Option = { - if maybe_body.is_cell() { - let body_octs = maybe_body.as_cell()?.tail().as_cell()?; + let body: Option = if maybe_body.is_cell() { + let body_octs = maybe_body + .in_space(&space) + .as_cell()? + .tail() + .as_cell()?; let body_len = body_octs .head() .as_atom()? - .direct() + .atom() + .as_direct() .expect("body len") .data(); - let len: usize = body_len.try_into().map_err(|_| HttpError::BodyLengthConversion)?; + let len: usize = + body_len.try_into().map_err(|_| HttpError::BodyLengthConversion)?; let mut body_vec: Vec = vec![0; len]; let body_atom = body_octs.tail().as_atom()?; @@ -603,7 +639,9 @@ pub fn http() -> IODriverFn { } }; - body_vec.copy_from_slice(&body_bytes[..std::cmp::min(body_bytes.len(), len)]); + body_vec.copy_from_slice( + &body_bytes[..std::cmp::min(body_bytes.len(), len)], + ); let bytes = Bytes::from(body_vec); // Log the response body as string if possible, using lossy conversion @@ -617,9 +655,29 @@ pub fn http() -> IODriverFn { } else { debug!("HTTP response has no body"); None - } + }; + + Ok(Some(ParsedEffect { + tag_val, + id, + status_code, + headers, + body, + })) + })(); + + let Some(parsed) = parsed? else { + return Ok(()); }; + let ParsedEffect { + tag_val, + id, + status_code, + headers: header_vec, + body, + } = parsed; + let resp = if let Ok(status) = StatusCode::from_u16(status_code as u16) { debug!("Building HTTP response with status: {}", status); let res_builder = ResponseBuilder { @@ -657,7 +715,6 @@ pub fn http() -> IODriverFn { Ok(response) } else { error!("http: not a valid status code: {}", status_code); - error!("http: res: {:?}", res); Err(StatusCode::INTERNAL_SERVER_ERROR) }; diff --git a/crates/nockapp/src/drivers/markdown.rs b/crates/nockapp/src/drivers/markdown.rs index 3818bd2eb..8d637a4f6 100644 --- a/crates/nockapp/src/drivers/markdown.rs +++ b/crates/nockapp/src/drivers/markdown.rs @@ -1,11 +1,9 @@ -use nockvm::noun::D; +use nockvm::noun::{NounAllocator, D}; use nockvm_macros::tas; use termimad::MadSkin; use tracing::error; use crate::nockapp::driver::{make_driver, IODriverFn}; -use crate::AtomExt; - pub fn markdown() -> IODriverFn { make_driver(|handle| async move { let skin = MadSkin::default_dark(); @@ -13,13 +11,15 @@ pub fn markdown() -> IODriverFn { loop { match handle.next_effect().await { Ok(effect) => { - let Ok(effect_cell) = unsafe { effect.root() }.as_cell() else { + let space = effect.noun_space(); + let Ok(effect_cell) = unsafe { effect.root() }.in_space(&space).as_cell() + else { continue; }; - if unsafe { effect_cell.head().raw_equals(&D(tas!(b"markdown"))) } { - let markdown_text = effect_cell.tail(); + if unsafe { effect_cell.head().noun().raw_equals(&D(tas!(b"markdown"))) } { + let markdown_text = effect_cell.tail().noun(); - let text = if let Ok(atom) = markdown_text.as_atom() { + let text = if let Ok(atom) = markdown_text.in_space(&space).as_atom() { String::from_utf8_lossy(&atom.to_bytes_until_nul()?).to_string() } else { error!("Failed to convert markdown text to string"); diff --git a/crates/nockapp/src/event_log.rs b/crates/nockapp/src/event_log.rs new file mode 100644 index 000000000..cfe8eb8c5 --- /dev/null +++ b/crates/nockapp/src/event_log.rs @@ -0,0 +1,657 @@ +use std::fs; +use std::path::{Path, PathBuf}; +use std::time::Duration; + +use diesel::connection::SimpleConnection; +use diesel::dsl::{max, sql}; +use diesel::prelude::*; +use diesel::sql_types::{BigInt, Integer, Text}; +use diesel::sqlite::SqliteConnection; +use diesel::{sql_query, OptionalExtension}; +use diesel_migrations::{embed_migrations, EmbeddedMigrations, MigrationHarness}; +use nockvm::offset::PmaOffsetWords; +use thiserror::Error; + +use crate::utils::durability; + +const ACTIVE_SNAPSHOT_ID_KEY: &str = "active_snapshot_id"; +const EVENT_LOG_MIGRATIONS: EmbeddedMigrations = embed_migrations!("migrations"); + +diesel::table! { + events (id) { + id -> BigInt, + event_num -> BigInt, + job_jam -> Binary, + wire_source -> Text, + wire_version -> BigInt, + wire_tags_json -> Text, + cause_hash -> Binary, + job_hash -> Binary, + event_processing_duration_us -> BigInt, + created_at_ms -> BigInt, + } +} + +diesel::table! { + snapshots (snapshot_id) { + snapshot_id -> BigInt, + kind -> Text, + state -> Text, + event_num -> BigInt, + pma_path -> Text, + manifest_path -> Text, + alloc_words -> BigInt, + kernel_root_raw -> BigInt, + cold_offset -> BigInt, + used_blake3 -> Binary, + structure_blake3 -> Nullable, + created_at_ms -> BigInt, + activated_at_ms -> Nullable, + base_snapshot_id -> Nullable, + timestamp_tag -> Text, + } +} + +#[derive(Debug, Clone)] +pub(crate) struct EventLogConfig { + pub path: PathBuf, +} + +#[derive(Debug, Clone)] +pub(crate) struct EventLogEntry { + pub event_num: u64, + pub job_jam: Vec, + pub wire_source: String, + pub wire_version: i64, + pub wire_tags_json: String, + pub cause_hash: Vec, + pub job_hash: Vec, + pub event_processing_duration: Duration, + pub created_at_ms: i64, +} + +#[derive(Debug, Clone)] +pub(crate) struct ReadySnapshotRecord { + pub snapshot_id: i64, + pub kind: String, + pub event_num: u64, + pub pma_path: String, + pub manifest_path: String, + pub alloc_words: u64, + pub kernel_root_raw: u64, + pub cold_offset: PmaOffsetWords, + pub used_blake3: Vec, + pub structure_blake3: Option>, + pub created_at_ms: i64, + pub activated_at_ms: Option, + pub base_snapshot_id: Option, + pub timestamp_tag: String, +} + +#[derive(Debug, Clone)] +pub(crate) struct ReplayLogEntry { + pub event_num: u64, + pub job_jam: Vec, +} + +#[derive(Debug, Error)] +pub(crate) enum EventLogError { + #[error("event log io error: {0}")] + Io(#[from] std::io::Error), + #[error("event log sqlite connection error: {0}")] + Connection(#[from] diesel::ConnectionError), + #[error("event log sqlite query error: {0}")] + Query(#[from] diesel::result::Error), + #[error("event log path is not valid UTF-8: {0}")] + NonUtf8Path(PathBuf), + #[error("event log migration error: {0}")] + Migration(String), + #[error("invalid event number {0}")] + InvalidEventNum(i64), + #[error("sqlite INTEGER field {field} has out-of-range value {value}")] + FieldOutOfRange { field: &'static str, value: i64 }, + #[error("integer field {field} out of range for sqlite INTEGER: {value}")] + IntegerOutOfRange { field: &'static str, value: u64 }, + #[error("duration field {field} out of range for sqlite INTEGER microseconds: {micros}")] + DurationOutOfRange { field: &'static str, micros: u128 }, + #[error("event log quick_check failed: {0}")] + QuickCheck(String), + #[error("event sequence gap detected: expected event_num {expected}, found {found}")] + EventSequenceGap { expected: u64, found: u64 }, +} + +pub(crate) struct EventLog { + path: PathBuf, + conn: SqliteConnection, +} + +#[derive(Insertable)] +#[diesel(table_name = events)] +struct NewEventRow<'a> { + event_num: i64, + job_jam: &'a [u8], + wire_source: &'a str, + wire_version: i64, + wire_tags_json: &'a str, + cause_hash: &'a [u8], + job_hash: &'a [u8], + event_processing_duration_us: i64, + created_at_ms: i64, +} + +impl<'a> NewEventRow<'a> { + fn try_from_entry(event: &'a EventLogEntry) -> Result { + Ok(Self { + event_num: i64::try_from(event.event_num).map_err(|_| { + EventLogError::InvalidEventNum(event.event_num.try_into().unwrap_or(i64::MAX)) + })?, + job_jam: &event.job_jam, + wire_source: &event.wire_source, + wire_version: event.wire_version, + wire_tags_json: &event.wire_tags_json, + cause_hash: &event.cause_hash, + job_hash: &event.job_hash, + event_processing_duration_us: sqlite_duration_micros( + "event_processing_duration_us", event.event_processing_duration, + )?, + created_at_ms: event.created_at_ms, + }) + } +} + +#[derive(Insertable)] +#[diesel(table_name = snapshots)] +struct NewSnapshotRow<'a> { + kind: &'a str, + state: &'a str, + event_num: i64, + pma_path: &'a str, + manifest_path: &'a str, + alloc_words: i64, + kernel_root_raw: i64, + cold_offset: i64, + used_blake3: &'a [u8], + structure_blake3: Option<&'a [u8]>, + created_at_ms: i64, + activated_at_ms: Option, + base_snapshot_id: Option, + timestamp_tag: &'a str, +} + +impl<'a> NewSnapshotRow<'a> { + fn try_from_record(snapshot: &'a ReadySnapshotRecord) -> Result { + Ok(Self { + kind: &snapshot.kind, + state: "ready", + event_num: sqlite_i64("event_num", snapshot.event_num)?, + pma_path: &snapshot.pma_path, + manifest_path: &snapshot.manifest_path, + alloc_words: sqlite_i64("alloc_words", snapshot.alloc_words)?, + kernel_root_raw: sqlite_bitcast_i64(snapshot.kernel_root_raw), + cold_offset: i64::try_from(snapshot.cold_offset.words()).map_err(|_| { + EventLogError::IntegerOutOfRange { + field: "cold_offset", + value: snapshot.cold_offset.words(), + } + })?, + used_blake3: &snapshot.used_blake3, + structure_blake3: snapshot.structure_blake3.as_deref(), + created_at_ms: snapshot.created_at_ms, + activated_at_ms: snapshot.activated_at_ms, + base_snapshot_id: snapshot.base_snapshot_id, + timestamp_tag: &snapshot.timestamp_tag, + }) + } +} + +#[derive(Queryable)] +struct SnapshotRow { + snapshot_id: i64, + kind: String, + _state: String, + event_num: i64, + pma_path: String, + manifest_path: String, + alloc_words: i64, + kernel_root_raw: i64, + cold_offset: i64, + used_blake3: Vec, + structure_blake3: Option>, + created_at_ms: i64, + activated_at_ms: Option, + base_snapshot_id: Option, + timestamp_tag: String, +} + +impl SnapshotRow { + fn try_into_record(self) -> Result { + Ok(ReadySnapshotRecord { + snapshot_id: self.snapshot_id, + kind: self.kind, + event_num: event_num_from_sqlite(self.event_num)?, + pma_path: self.pma_path, + manifest_path: self.manifest_path, + alloc_words: u64_from_sqlite("alloc_words", self.alloc_words)?, + kernel_root_raw: u64::from_ne_bytes(self.kernel_root_raw.to_ne_bytes()), + cold_offset: PmaOffsetWords::from_words(u64_from_sqlite( + "cold_offset", self.cold_offset, + )?), + used_blake3: self.used_blake3, + structure_blake3: self.structure_blake3, + created_at_ms: self.created_at_ms, + activated_at_ms: self.activated_at_ms, + base_snapshot_id: self.base_snapshot_id, + timestamp_tag: self.timestamp_tag, + }) + } +} + +#[derive(QueryableByName)] +struct I64ValueRow { + #[diesel(sql_type = BigInt)] + value: i64, +} + +#[derive(QueryableByName)] +struct QuickCheckRow { + #[diesel(sql_type = Text)] + quick_check: String, +} + +impl EventLog { + pub(crate) fn open(config: EventLogConfig) -> Result { + if let Some(parent) = config.path.parent() { + fs::create_dir_all(parent)?; + } + let mut conn = establish_connection(&config.path)?; + Self::configure(&mut conn)?; + Self::migrate(&mut conn)?; + Ok(Self { + path: config.path, + conn, + }) + } + + pub(crate) fn path(&self) -> &Path { + &self.path + } + + pub(crate) fn append_event(&mut self, event: &EventLogEntry) -> Result<(), EventLogError> { + let row = NewEventRow::try_from_entry(event)?; + self.conn.immediate_transaction(|conn| { + diesel::insert_into(events::table) + .values(&row) + .execute(conn)?; + Ok(()) + }) + } + + pub(crate) fn has_ready_snapshot(&mut self) -> Result { + let count = snapshots::table + .filter(snapshots::state.eq("ready")) + .count() + .get_result::(&mut self.conn)?; + Ok(count > 0) + } + + pub(crate) fn insert_ready_snapshot( + &mut self, + snapshot: &ReadySnapshotRecord, + ) -> Result { + let row = NewSnapshotRow::try_from_record(snapshot)?; + self.conn.immediate_transaction(|conn| { + diesel::insert_into(snapshots::table) + .values(&row) + .execute(conn)?; + let snapshot_id = last_insert_rowid(conn)?; + store_meta_i64(conn, ACTIVE_SNAPSHOT_ID_KEY, snapshot_id)?; + Ok(snapshot_id) + }) + } + + pub(crate) fn quick_check(&mut self) -> Result<(), EventLogError> { + let result = sql_query("SELECT quick_check FROM pragma_quick_check") + .get_result::(&mut self.conn)? + .quick_check; + if result.eq_ignore_ascii_case("ok") { + Ok(()) + } else { + Err(EventLogError::QuickCheck(result)) + } + } + + pub(crate) fn list_ready_snapshots( + &mut self, + ) -> Result, EventLogError> { + snapshots::table + .filter(snapshots::state.eq("ready")) + .order(( + sql::("CASE kind WHEN 'rotating' THEN 0 ELSE 1 END").asc(), + snapshots::timestamp_tag.desc(), + snapshots::snapshot_id.desc(), + )) + .load::(&mut self.conn)? + .into_iter() + .map(SnapshotRow::try_into_record) + .collect() + } + + pub(crate) fn latest_ready_snapshot_event_num(&mut self) -> Result, EventLogError> { + let latest_event_num = snapshots::table + .filter(snapshots::state.eq("ready")) + .select(max(snapshots::event_num)) + .first::>(&mut self.conn)?; + latest_event_num.map(event_num_from_sqlite).transpose() + } + + pub(crate) fn mark_snapshot_failed(&mut self, snapshot_id: i64) -> Result<(), EventLogError> { + diesel::update(snapshots::table.filter(snapshots::snapshot_id.eq(snapshot_id))) + .set(snapshots::state.eq("failed")) + .execute(&mut self.conn)?; + self.refresh_active_snapshot_after_state_change(snapshot_id) + } + + pub(crate) fn active_snapshot_id(&mut self) -> Result, EventLogError> { + load_meta_i64(&mut self.conn, ACTIVE_SNAPSHOT_ID_KEY) + } + + pub(crate) fn set_active_snapshot_id(&mut self, snapshot_id: i64) -> Result<(), EventLogError> { + store_meta_i64(&mut self.conn, ACTIVE_SNAPSHOT_ID_KEY, snapshot_id) + } + + pub(crate) fn ready_rotating_snapshots( + &mut self, + ) -> Result, EventLogError> { + snapshots::table + .filter( + snapshots::state + .eq("ready") + .and(snapshots::kind.eq("rotating")), + ) + .order(( + snapshots::timestamp_tag.desc(), + snapshots::snapshot_id.desc(), + )) + .load::(&mut self.conn)? + .into_iter() + .map(SnapshotRow::try_into_record) + .collect() + } + + pub(crate) fn retire_snapshot(&mut self, snapshot_id: i64) -> Result<(), EventLogError> { + diesel::update(snapshots::table.filter(snapshots::snapshot_id.eq(snapshot_id))) + .set(snapshots::state.eq("retired")) + .execute(&mut self.conn)?; + self.refresh_active_snapshot_after_state_change(snapshot_id) + } + + pub(crate) fn replay_events_after( + &mut self, + event_num: u64, + ) -> Result, EventLogError> { + let rows = events::table + .select((events::event_num, events::job_jam)) + .filter(events::event_num.gt(sqlite_i64("event_num", event_num)?)) + .order(events::event_num.asc()) + .load::<(i64, Vec)>(&mut self.conn)?; + let entries = rows + .into_iter() + .map(|(event_num, job_jam)| { + Ok(ReplayLogEntry { + event_num: event_num_from_sqlite(event_num)?, + job_jam, + }) + }) + .collect::, EventLogError>>()?; + let mut expected = event_num.saturating_add(1); + for entry in &entries { + if entry.event_num != expected { + return Err(EventLogError::EventSequenceGap { + expected, + found: entry.event_num, + }); + } + expected = expected.saturating_add(1); + } + Ok(entries) + } + + pub(crate) fn event_processing_time_after( + &mut self, + event_num: u64, + ) -> Result { + let total_micros = sql_query( + "SELECT COALESCE(SUM(event_processing_duration_us), 0) AS value FROM events WHERE event_num > ?", + ) + .bind::(sqlite_i64("event_num", event_num)?) + .get_result::(&mut self.conn)? + .value; + duration_from_sqlite("event_processing_duration_us", total_micros) + } + + #[allow(dead_code)] + pub(crate) fn max_event_num(&mut self) -> Result, EventLogError> { + let max_event_num = events::table + .select(max(events::event_num)) + .first::>(&mut self.conn)?; + max_event_num.map(event_num_from_sqlite).transpose() + } + + fn configure(conn: &mut SqliteConnection) -> Result<(), EventLogError> { + conn.batch_execute( + r#" +PRAGMA journal_mode = WAL; +PRAGMA temp_store = MEMORY; +PRAGMA foreign_keys = 1; +"#, + )?; + let sync_mode = if durability::fsync_disabled() { + "OFF" + } else { + "FULL" + }; + conn.batch_execute(&format!("PRAGMA synchronous = {sync_mode};"))?; + Ok(()) + } + + fn migrate(conn: &mut SqliteConnection) -> Result<(), EventLogError> { + conn.run_pending_migrations(EVENT_LOG_MIGRATIONS) + .map(|_| ()) + .map_err(|err| EventLogError::Migration(err.to_string())) + } + + fn refresh_active_snapshot_after_state_change( + &mut self, + changed_snapshot_id: i64, + ) -> Result<(), EventLogError> { + let active_snapshot_id = self.active_snapshot_id()?; + if active_snapshot_id != Some(changed_snapshot_id) { + return Ok(()); + } + let replacement = snapshots::table + .select(snapshots::snapshot_id) + .filter(snapshots::state.eq("ready")) + .order(( + sql::("CASE kind WHEN 'rotating' THEN 0 ELSE 1 END").asc(), + snapshots::timestamp_tag.desc(), + snapshots::snapshot_id.desc(), + )) + .first::(&mut self.conn) + .optional()?; + if let Some(replacement) = replacement { + self.set_active_snapshot_id(replacement)?; + } else { + delete_meta_key(&mut self.conn, ACTIVE_SNAPSHOT_ID_KEY)?; + } + Ok(()) + } +} + +fn establish_connection(path: &Path) -> Result { + let path_str = path + .to_str() + .ok_or_else(|| EventLogError::NonUtf8Path(path.to_path_buf()))?; + Ok(SqliteConnection::establish(path_str)?) +} + +fn load_meta_i64(conn: &mut SqliteConnection, key: &str) -> Result, EventLogError> { + sql_query("SELECT CAST(value AS INTEGER) AS value FROM meta WHERE key = ?") + .bind::(key) + .get_result::(conn) + .optional() + .map(|row| row.map(|row| row.value)) + .map_err(Into::into) +} + +fn store_meta_i64(conn: &mut SqliteConnection, key: &str, value: i64) -> Result<(), EventLogError> { + sql_query( + r#" +INSERT INTO meta (key, value) +VALUES (?, ?) +ON CONFLICT(key) DO UPDATE SET value = excluded.value +"#, + ) + .bind::(key) + .bind::(value) + .execute(conn)?; + Ok(()) +} + +fn delete_meta_key(conn: &mut SqliteConnection, key: &str) -> Result<(), EventLogError> { + sql_query("DELETE FROM meta WHERE key = ?") + .bind::(key) + .execute(conn)?; + Ok(()) +} + +fn last_insert_rowid(conn: &mut SqliteConnection) -> Result { + Ok(sql_query("SELECT last_insert_rowid() AS value") + .get_result::(conn)? + .value) +} + +fn sqlite_i64(field: &'static str, value: u64) -> Result { + i64::try_from(value).map_err(|_| EventLogError::IntegerOutOfRange { field, value }) +} + +fn sqlite_bitcast_i64(value: u64) -> i64 { + i64::from_ne_bytes(value.to_ne_bytes()) +} + +fn event_num_from_sqlite(value: i64) -> Result { + u64::try_from(value).map_err(|_| EventLogError::InvalidEventNum(value)) +} + +fn u64_from_sqlite(field: &'static str, value: i64) -> Result { + u64::try_from(value).map_err(|_| EventLogError::FieldOutOfRange { field, value }) +} + +fn sqlite_duration_micros(field: &'static str, value: Duration) -> Result { + let micros = value.as_micros(); + if micros > i64::MAX as u128 { + return Err(EventLogError::DurationOutOfRange { field, micros }); + } + Ok(micros as i64) +} + +fn duration_from_sqlite(field: &'static str, value: i64) -> Result { + let micros = u64_from_sqlite(field, value)?; + Ok(Duration::from_micros(micros)) +} + +#[cfg(test)] +mod tests { + use std::time::Duration; + + use tempfile::TempDir; + + use super::*; + + fn sample_entry(event_num: u64) -> EventLogEntry { + EventLogEntry { + event_num, + job_jam: vec![0, 1, 2, event_num as u8], + wire_source: "sys".to_string(), + wire_version: 1, + wire_tags_json: "[]".to_string(), + cause_hash: vec![3; 32], + job_hash: vec![4; 32], + event_processing_duration: Duration::from_micros(event_num), + created_at_ms: 42, + } + } + + #[test] + fn initializes_schema_and_appends_events() { + let temp = TempDir::new().expect("tempdir"); + let path = temp.path().join("event-log.sqlite3"); + let mut log = EventLog::open(EventLogConfig { path }).expect("open event log"); + assert_eq!(log.max_event_num().expect("max event num"), None); + + log.append_event(&sample_entry(1)).expect("append event 1"); + log.append_event(&sample_entry(2)).expect("append event 2"); + + assert_eq!(log.max_event_num().expect("max event num"), Some(2)); + assert_eq!( + log.event_processing_time_after(0) + .expect("event processing time after 0"), + Duration::from_micros(3) + ); + } + + #[test] + fn inserts_ready_snapshot_rows() { + let temp = TempDir::new().expect("tempdir"); + let path = temp.path().join("event-log.sqlite3"); + let mut log = EventLog::open(EventLogConfig { path }).expect("open event log"); + assert!(!log.has_ready_snapshot().expect("ready snapshot count")); + + log.insert_ready_snapshot(&ReadySnapshotRecord { + snapshot_id: 0, + kind: "epoch".to_string(), + event_num: 7, + pma_path: "epoch.pma".to_string(), + manifest_path: "epoch.manifest".to_string(), + alloc_words: 128, + kernel_root_raw: u64::MAX, + cold_offset: PmaOffsetWords::from_words(3), + used_blake3: vec![5; 32], + structure_blake3: None, + created_at_ms: 99, + activated_at_ms: Some(99), + base_snapshot_id: None, + timestamp_tag: "epoch".to_string(), + }) + .expect("insert ready snapshot"); + + assert!(log.has_ready_snapshot().expect("ready snapshot count")); + let ready = log.list_ready_snapshots().expect("ready snapshots"); + assert_eq!(ready.len(), 1); + assert_eq!(ready[0].event_num, 7); + assert_eq!( + log.latest_ready_snapshot_event_num() + .expect("latest ready snapshot event num"), + Some(7) + ); + } + + #[test] + fn replay_events_detects_gaps() { + let temp = TempDir::new().expect("tempdir"); + let path = temp.path().join("event-log.sqlite3"); + let mut log = EventLog::open(EventLogConfig { path }).expect("open event log"); + log.append_event(&sample_entry(1)).expect("append event 1"); + log.append_event(&sample_entry(3)).expect("append event 3"); + + let err = log + .replay_events_after(0) + .expect_err("gap should be detected"); + assert!(matches!( + err, + EventLogError::EventSequenceGap { + expected: 2, + found: 3 + } + )); + } +} diff --git a/crates/nockapp/src/kernel/boot.rs b/crates/nockapp/src/kernel/boot.rs index 85dc45c14..695086d19 100644 --- a/crates/nockapp/src/kernel/boot.rs +++ b/crates/nockapp/src/kernel/boot.rs @@ -1,13 +1,16 @@ #![allow(clippy::items_after_test_module)] -use std::path::PathBuf; +use std::path::{Path, PathBuf}; +use std::sync::Arc; +use std::time::Duration; use chrono; use clap::{Args, ColorChoice, Parser, ValueEnum}; use nockvm::jets::hot::HotEntry; use nockvm::noun::Atom; +use nockvm::pma::Pma; use nockvm::trace::{IntervalFilter, KeywordFilter, TraceFilter, TraceInfo, TracingBackend}; use tokio::fs; -use tracing::{debug, info, Level, Subscriber}; +use tracing::{debug, info, warn, Level, Subscriber}; use tracing_subscriber::fmt::format::Writer; use tracing_subscriber::fmt::{FmtContext, FormatEvent, FormatFields}; use tracing_subscriber::layer::SubscriberExt; @@ -17,18 +20,136 @@ use tracing_subscriber::util::SubscriberInitExt; use tracing_subscriber::Layer as _; use tracing_subscriber::{fmt, EnvFilter}; +use crate::event_log::{EventLogConfig, ReadySnapshotRecord, ReplayLogEntry}; use crate::export::ExportedState; -use crate::kernel::form::Kernel; +use crate::kernel::form::{inspect_existing_pma, ExistingPmaStatus, Kernel, PmaConfig}; +use crate::metrics::NockAppMetrics; use crate::noun::slab::{Jammer, NounSlab}; -use crate::save::SaveableCheckpoint; +use crate::save::{CheckpointBootstrapReader, SaveableCheckpoint}; +use crate::snapshot::{cleanup_snapshot_artifacts, restore_verified_snapshot, SnapshotManifest}; use crate::utils::error::{CrownError, ExternalError}; +use crate::utils::{ + durability, NOCK_STACK_SIZE, NOCK_STACK_SIZE_HUGE, NOCK_STACK_SIZE_LARGE, + NOCK_STACK_SIZE_MEDIUM, NOCK_STACK_SIZE_SMALL, NOCK_STACK_SIZE_TINY, +}; use crate::{default_data_dir, AtomExt, NockApp}; -pub const DEFAULT_SAVE_INTERVAL: u64 = 120000; -const DEFAULT_SAVE_INTERVAL_STR: &str = "120000"; +pub const DEFAULT_GC_INTERVAL_SECS: u64 = 60 * 60; +const DEFAULT_GC_INTERVAL_SECS_STR: &str = "3600"; +const DEFAULT_ROTATING_SNAPSHOT_INTERVAL_EVENT_TIME_SECS: u64 = 15 * 60; +const DEFAULT_ROTATING_SNAPSHOT_INTERVAL_EVENT_TIME_SECS_STR: &str = "900"; + const DEFAULT_LOG_FILTER: &str = "info"; +const NOCK_PMA_INITIAL_WORDS_FOR_REGRESSION_ENV: &str = "NOCK_PMA_INITIAL_WORDS_FOR_REGRESSION"; +const DEFAULT_PMA_INITIAL_MIN_BYTES: usize = 256 * 1024 * 1024; +const DEFAULT_PMA_INITIAL_KERNEL_MULTIPLIER: usize = 16; +const WORD_BYTES: usize = std::mem::size_of::(); + +#[derive(Debug)] +enum BootSource { + Pma { path: PathBuf, event_num: u64 }, + Snapshot { path: PathBuf, event_num: u64 }, + Checkpoint { path: PathBuf, event_num: u64 }, + Fresh, +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct PmaSize { + words: usize, +} + +impl PmaSize { + pub const fn from_words(words: usize) -> Self { + Self { words } + } + + pub const fn from_bytes_ceil(bytes: usize) -> Self { + Self::from_words(bytes.div_ceil(WORD_BYTES)) + } + + pub const fn words(self) -> usize { + self.words + } +} + +fn default_pma_initial_words(kernel_jam_bytes: usize) -> usize { + let target_bytes = kernel_jam_bytes + .saturating_mul(DEFAULT_PMA_INITIAL_KERNEL_MULTIPLIER) + .max(DEFAULT_PMA_INITIAL_MIN_BYTES); + let words = PmaSize::from_bytes_ceil(target_bytes).words().max(1); + words.checked_next_power_of_two().unwrap_or(words) +} + +fn pma_initial_words_for_boot(configured_size: Option, kernel_jam_bytes: usize) -> usize { + let configured_words = configured_size + .map(PmaSize::words) + .unwrap_or_else(|| default_pma_initial_words(kernel_jam_bytes)); + let Some(value) = std::env::var_os(NOCK_PMA_INITIAL_WORDS_FOR_REGRESSION_ENV) else { + return configured_words; + }; + match value.to_string_lossy().parse::() { + Ok(words) if words > 0 => { + warn!( + configured_words, + override_words = words, + env = NOCK_PMA_INITIAL_WORDS_FOR_REGRESSION_ENV, + "using regression PMA initial-size override" + ); + words + } + _ => { + warn!( + configured_words, + env = NOCK_PMA_INITIAL_WORDS_FOR_REGRESSION_ENV, + value = %value.to_string_lossy(), + "ignoring invalid regression PMA initial-size override" + ); + configured_words + } + } +} + +fn parse_pma_size(input: &str) -> Result { + let trimmed = input.trim(); + if trimmed.is_empty() { + return Err("PMA size cannot be empty".to_string()); + } + let normalized = trimmed.replace('_', ""); + let split_at = normalized + .find(|c: char| !c.is_ascii_digit()) + .unwrap_or(normalized.len()); + if split_at == 0 { + return Err(format!("PMA size '{trimmed}' is missing a numeric value")); + } + let value = normalized[..split_at] + .parse::() + .map_err(|err| format!("invalid PMA size '{trimmed}': {err}"))?; + let suffix = normalized[split_at..].trim().to_ascii_lowercase(); + let multiplier = match suffix.as_str() { + "" | "b" | "byte" | "bytes" => 1u128, + "k" | "kb" | "kib" => 1024u128, + "m" | "mb" | "mib" => 1024u128.pow(2), + "g" | "gb" | "gib" => 1024u128.pow(3), + "t" | "tb" | "tib" => 1024u128.pow(4), + _ => { + return Err(format!( + "invalid PMA size suffix '{suffix}' in '{trimmed}'; use bytes, KiB, MiB, GiB, or TiB" + )); + } + }; + let bytes = value + .checked_mul(multiplier) + .ok_or_else(|| format!("PMA size '{trimmed}' overflowed"))?; + let bytes = usize::try_from(bytes) + .map_err(|_| format!("PMA size '{trimmed}' exceeds this platform's address size"))?; + let size = PmaSize::from_bytes_ceil(bytes); + if size.words() == 0 { + return Err("PMA size must be at least one word".to_string()); + } + Ok(size) +} -#[derive(Debug, Clone, ValueEnum)] +#[derive(Debug, Clone, Copy, ValueEnum)] pub enum NockStackSize { Tiny, Small, @@ -38,6 +159,19 @@ pub enum NockStackSize { Huge, } +impl NockStackSize { + pub const fn stack_words(self) -> usize { + match self { + Self::Tiny => NOCK_STACK_SIZE_TINY, + Self::Small => NOCK_STACK_SIZE_SMALL, + Self::Normal => NOCK_STACK_SIZE, + Self::Medium => NOCK_STACK_SIZE_MEDIUM, + Self::Large => NOCK_STACK_SIZE_LARGE, + Self::Huge => NOCK_STACK_SIZE_HUGE, + } + } +} + #[derive(Clone, Copy, Debug, ValueEnum)] pub enum TraceMode { Tracing, @@ -86,7 +220,7 @@ impl From for Option { pub struct Cli { #[arg( long, - help = "Start with a new data directory, removing any existing data", + help = "Start with a fresh data directory, aborting if the target already contains data", default_value = "false" )] pub new: bool, @@ -96,20 +230,41 @@ pub struct Cli { #[arg( long, - help = "Set the save interval for checkpoints (in ms). Use 'none' or '0' to disable periodic saves.", - default_value = DEFAULT_SAVE_INTERVAL_STR, - value_parser = parse_save_interval + help = "Set the PMA GC interval (in seconds). Use 'none' or '0' to disable PMA GC.", + default_value = DEFAULT_GC_INTERVAL_SECS_STR, + value_parser = parse_optional_u64 )] - pub save_interval: Option, + pub gc_interval: Option, + + #[arg( + long, + help = "Set the rotating snapshot interval in cumulative event-processing seconds. Use 'none' or '0' to disable rotating snapshots.", + default_value = DEFAULT_ROTATING_SNAPSHOT_INTERVAL_EVENT_TIME_SECS_STR, + value_parser = parse_optional_u64 + )] + pub rotating_snapshot_interval_event_time: Option, + + #[arg( + long, + help = "Run with in-memory NockStack state only, disabling PMA durability, event logs, snapshots, and GC." + )] + pub ephemeral: bool, #[arg(long, help = "Control colored output", value_enum, default_value_t = ColorChoice::Auto)] pub color: ColorChoice, #[arg( long, + requires = "new", help = "Path to a jam file containing existing kernel state. Supports both JammedCheckpoint and ExportedState formats." )] pub state_jam: Option, + #[arg( + long, + help = "Path to a jammed checkpoint (.chkjam) to bootstrap from. Copies into the checkpoints dir as 0.chkjam before boot. Conflicts with --state-jam and --export-state-jam.", + conflicts_with_all = &["state_jam", "export_state_jam"] + )] + pub bootstrap_from_chkjam: Option, #[arg( long, @@ -124,16 +279,59 @@ pub struct Cli { default_value_t = NockStackSize::Normal )] pub stack_size: NockStackSize, + + #[arg( + long, + help = "Initial file-backed PMA capacity. Accepts byte units like 256MiB or 4GiB. Defaults to auto: round_up_power_of_two(max(256MiB, kernel_jam_bytes * 16)).", + value_parser = parse_pma_size + )] + pub pma_initial_size: Option, + + #[arg( + long, + help = "Maximum virtual PMA reservation. Accepts byte units like 64GiB or 1TiB. Defaults to 1TiB, or NOCK_PMA_RESERVED_WORDS when set.", + value_parser = parse_pma_size + )] + pub pma_reserved_size: Option, + #[arg( + long, + help = "Override the full data directory for this nockapp instance (expects the directory that contains checkpoints/)" + )] + pub data_dir: Option, + #[arg(long, help = "Override the SQLite event-log path")] + pub event_log_path: Option, + + #[arg( + long, + help = "Disable all fsync/fdatasync calls (including SQLite FULL-sync durability)" + )] + pub disable_fsync: bool, } impl Cli { - fn normalized_save_interval(&self) -> Option { - self.save_interval - .and_then(|value| if value == 0 { None } else { Some(value) }) + fn normalized_gc_interval(&self) -> Option { + self.gc_interval.and_then(|value| { + if value == 0 { + None + } else { + Some(Duration::from_secs(value)) + } + }) + } + + fn normalized_rotating_snapshot_interval_event_time(&self) -> Option { + self.rotating_snapshot_interval_event_time + .and_then(|value| { + if value == 0 { + None + } else { + Some(Duration::from_secs(value)) + } + }) } } -fn parse_save_interval(input: &str) -> Result { +fn parse_optional_u64(input: &str) -> Result { let trimmed = input.trim(); if trimmed.eq_ignore_ascii_case("none") { @@ -141,45 +339,1492 @@ fn parse_save_interval(input: &str) -> Result { } else { let value = trimmed .parse::() - .map_err(|e| format!("Invalid save interval '{trimmed}': {e}"))?; + .map_err(|e| format!("Invalid value '{trimmed}': {e}"))?; Ok(value) } } #[cfg(test)] mod tests { - use super::parse_save_interval; + use std::fs; + use std::path::{Path, PathBuf}; + use std::sync::atomic::Ordering; + use std::sync::Arc; + use std::time::Duration; + + use clap::Parser; + use nockvm::jets::util::slot; + use nockvm::noun::{CellHandle, NounAllocator, NounSpace, D, T}; + use nockvm_macros::tas; + use rusqlite::Connection; + use tempfile::TempDir; + + use super::{ + default_boot_cli, default_pma_initial_words, export_kernel_state, parse_optional_u64, + parse_pma_size, select_boot_state, setup_, BootEventLogPolicy, BootSelection, SetupResult, + DEFAULT_GC_INTERVAL_SECS, + }; + use crate::metrics::NockAppMetrics; + use crate::nockapp::wire::{wire_to_noun, SystemWire, Wire}; + use crate::noun::slab::{slab_equality, NockJammer, NounSlab}; + use crate::save::SaveableCheckpoint; + use crate::snapshot::SnapshotManifest; + use crate::NockApp; + + fn load_test_jam_bytes() -> Vec { + let possible_paths = [ + Path::new(env!("CARGO_MANIFEST_DIR")) + .join("test-jams") + .join("test-ker.jam"), + Path::new("open/crates/nockapp/test-jams").join("test-ker.jam"), + Path::new("test-jams").join("test-ker.jam"), + ]; + + possible_paths + .iter() + .find_map(|path| fs::read(path).ok()) + .expect("read test kernel") + } + + fn inc_poke() -> NounSlab { + let mut slab = NounSlab::new(); + let space = NounSpace::empty(); + slab.copy_into(D(tas!(b"inc")), &space); + slab + } + + fn state_peek() -> NounSlab { + let mut slab = NounSlab::new(); + let peek = T(&mut slab, &[D(tas!(b"state")), D(0)]); + slab.set_root(peek); + slab + } + + fn atom_slab(value: u64) -> NounSlab { + let mut slab = NounSlab::new(); + let space = NounSpace::empty(); + slab.copy_into(D(value), &space); + slab + } + + async fn assert_counter_state(app: &mut NockApp, expected: u64) { + assert_eq!( + app.kernel.serf.event_number.load(Ordering::SeqCst), + expected, + "booted event number must match the committed SQLite boundary" + ); + let mut actual = app.kernel.peek(state_peek()).await.expect("peek state"); + let actual_space = actual.noun_space(); + actual.modify_noun(|root| { + let cell = slot(root, 7, &actual_space) + .expect("peek state result cell") + .as_cell() + .expect("peek state result payload cell"); + CellHandle::new(cell, &actual_space).tail().noun() + }); + let expected_slab = atom_slab(expected); + assert!( + slab_equality(&actual, &expected_slab), + "counter state did not match committed event boundary: actual={actual:?}, expected={expected_slab:?}" + ); + } + + fn logged_poke_job_jam(event_num: u64, cause: u64) -> Vec { + let mut slab = NounSlab::::new(); + let base_wire = wire_to_noun(&mut slab, &SystemWire.to_wire()); + let wire = T(&mut slab, &[D(tas!(b"poke")), base_wire]); + let job = T(&mut slab, &[D(event_num), wire, D(0), D(0), D(0), D(cause)]); + slab.set_root(job); + slab.jam().to_vec() + } + + fn durable_test_boot_cli(new: bool) -> super::Cli { + let mut cli = default_boot_cli(new); + cli.gc_interval = None; + cli.rotating_snapshot_interval_event_time = None; + cli.disable_fsync = true; + cli + } + + async fn setup_test_app(data_dir: &Path) -> NockApp { + try_setup_test_app(data_dir, None) + .await + .expect("setup boot test app") + } + + async fn setup_test_app_from_chkjam( + data_dir: &Path, + chkjam_path: &Path, + ) -> NockApp { + let jam = load_test_jam_bytes(); + let mut cli = durable_test_boot_cli(false); + cli.data_dir = Some(data_dir.to_path_buf()); + cli.bootstrap_from_chkjam = Some(chkjam_path.to_string_lossy().into_owned()); + match setup_::(&jam, cli, &[], "boot-test", None) + .await + .expect("setup boot test app from checkpoint") + { + SetupResult::App(app) => app, + SetupResult::ExportedState => panic!("unexpected export"), + } + } + + async fn try_setup_test_app( + data_dir: &Path, + rotating_snapshot_interval_event_time_secs: Option, + ) -> Result, Box> { + let jam = load_test_jam_bytes(); + let mut cli = durable_test_boot_cli(false); + cli.data_dir = Some(data_dir.to_path_buf()); + cli.rotating_snapshot_interval_event_time = rotating_snapshot_interval_event_time_secs; + Ok( + match setup_::(&jam, cli, &[], "boot-test", None).await? { + SetupResult::App(app) => app, + SetupResult::ExportedState => panic!("unexpected export"), + }, + ) + } + + async fn try_setup_test_app_with_gc_interval( + data_dir: &Path, + gc_interval_secs: Option, + ) -> Result, Box> { + let jam = load_test_jam_bytes(); + let mut cli = durable_test_boot_cli(false); + cli.data_dir = Some(data_dir.to_path_buf()); + cli.gc_interval = gc_interval_secs; + Ok( + match setup_::(&jam, cli, &[], "boot-test", None).await? { + SetupResult::App(app) => app, + SetupResult::ExportedState => panic!("unexpected export"), + }, + ) + } + + async fn write_checkpoint_bootstrap_fixture(app: &NockApp, data_dir: &Path) { + let checkpoint: SaveableCheckpoint = + app.kernel.checkpoint().await.expect("checkpoint saveable"); + let jammed = checkpoint.to_jammed_checkpoint::(); + let bytes = jammed.encode().expect("encode checkpoint"); + fs::write(data_dir.join("checkpoints").join("0.chkjam"), bytes) + .expect("write checkpoint fixture"); + } + + async fn poke_inc(app: &NockApp) { + app.kernel + .poke(SystemWire.to_wire(), inc_poke()) + .await + .expect("poke inc"); + } + + async fn wait_for_serf_idle(app: &NockApp) { + app.export().await.expect("export barrier"); + } + + async fn stop_app(app: &mut NockApp) { + app.kernel.serf.stop().await.expect("stop kernel"); + } + + fn clear_pma_files(data_dir: &Path) { + let pma_dir = data_dir.join("pma"); + for file_name in ["0.pma", "1.pma", "0.meta", "1.meta"] { + let path = pma_dir.join(file_name); + if path.exists() { + fs::remove_file(path).expect("remove PMA artifact"); + } + } + } + + fn copy_runtime_pma_files(from_dir: &Path, to_dir: &Path) { + fs::create_dir_all(to_dir).expect("create PMA backup dir"); + for file_name in ["0.pma", "1.pma", "0.meta", "1.meta"] { + let from = from_dir.join(file_name); + if from.exists() { + fs::copy(&from, to_dir.join(file_name)).expect("copy PMA artifact"); + } + } + } + + fn restore_runtime_pma_files(data_dir: &Path, backup_dir: &Path) { + clear_pma_files(data_dir); + let pma_dir = data_dir.join("pma"); + for file_name in ["0.pma", "1.pma", "0.meta", "1.meta"] { + let backup = backup_dir.join(file_name); + if backup.exists() { + fs::copy(&backup, pma_dir.join(file_name)).expect("restore PMA artifact"); + } + } + } + + fn ready_snapshot_count(data_dir: &Path) -> i64 { + let conn = + Connection::open(data_dir.join("event-log.sqlite3")).expect("open event log sqlite"); + conn.query_row( + "SELECT COUNT(1) FROM snapshots WHERE state = 'ready'", + [], + |row| row.get::<_, i64>(0), + ) + .expect("count ready snapshots") + } + + fn ready_rotating_snapshots(data_dir: &Path) -> Vec<(i64, String, String, i64)> { + let conn = + Connection::open(data_dir.join("event-log.sqlite3")).expect("open event log sqlite"); + let mut stmt = conn + .prepare( + "SELECT snapshot_id, pma_path, manifest_path, event_num FROM snapshots WHERE state = 'ready' AND kind = 'rotating' ORDER BY timestamp_tag DESC", + ) + .expect("prepare rotating snapshots query"); + let rows = stmt + .query_map([], |row| { + Ok(( + row.get::<_, i64>(0)?, + row.get::<_, String>(1)?, + row.get::<_, String>(2)?, + row.get::<_, i64>(3)?, + )) + }) + .expect("query rotating snapshots"); + rows.collect::, _>>() + .expect("collect rotating snapshots") + } + + fn ready_snapshots(data_dir: &Path) -> Vec<(i64, String, String, String, i64)> { + let conn = + Connection::open(data_dir.join("event-log.sqlite3")).expect("open event log sqlite"); + let mut stmt = conn + .prepare( + "SELECT snapshot_id, kind, pma_path, manifest_path, event_num FROM snapshots WHERE state = 'ready' ORDER BY event_num DESC, snapshot_id DESC", + ) + .expect("prepare ready snapshots query"); + let rows = stmt + .query_map([], |row| { + Ok(( + row.get::<_, i64>(0)?, + row.get::<_, String>(1)?, + row.get::<_, String>(2)?, + row.get::<_, String>(3)?, + row.get::<_, i64>(4)?, + )) + }) + .expect("query ready snapshots"); + rows.collect::, _>>() + .expect("collect ready snapshots") + } + + fn retain_only_snapshot_for_test(data_dir: &Path, snapshot_id: i64) { + let conn = + Connection::open(data_dir.join("event-log.sqlite3")).expect("open event log sqlite"); + let stale_paths = { + let mut stmt = conn + .prepare("SELECT pma_path, manifest_path FROM snapshots WHERE snapshot_id != ?1") + .expect("prepare stale snapshot path query"); + let rows = stmt + .query_map([snapshot_id], |row| { + Ok((row.get::<_, String>(0)?, row.get::<_, String>(1)?)) + }) + .expect("query stale snapshot paths"); + rows.collect::, _>>() + .expect("collect stale snapshot paths") + }; + for (pma_path, manifest_path) in stale_paths { + for path in [PathBuf::from(pma_path), PathBuf::from(manifest_path)] { + if path.exists() { + fs::remove_file(path).expect("remove stale snapshot artifact"); + } + } + } + conn.execute( + "DELETE FROM snapshots WHERE snapshot_id != ?1", + [snapshot_id], + ) + .expect("delete stale snapshots"); + set_active_snapshot_id_for_test(data_dir, snapshot_id); + } + + fn retired_rotating_snapshot_count(data_dir: &Path) -> i64 { + let conn = + Connection::open(data_dir.join("event-log.sqlite3")).expect("open event log sqlite"); + conn.query_row( + "SELECT COUNT(1) FROM snapshots WHERE state = 'retired' AND kind = 'rotating'", + [], + |row| row.get::<_, i64>(0), + ) + .expect("count retired rotating snapshots") + } + + fn active_snapshot_id_for_test(data_dir: &Path) -> Option { + let conn = + Connection::open(data_dir.join("event-log.sqlite3")).expect("open event log sqlite"); + conn.query_row( + "SELECT CAST(value AS INTEGER) FROM meta WHERE key = 'active_snapshot_id'", + [], + |row| row.get::<_, i64>(0), + ) + .ok() + } + + fn set_active_snapshot_id_for_test(data_dir: &Path, snapshot_id: i64) { + let conn = + Connection::open(data_dir.join("event-log.sqlite3")).expect("open event log sqlite"); + conn.execute( + r#" +INSERT INTO meta (key, value) +VALUES ('active_snapshot_id', ?1) +ON CONFLICT(key) DO UPDATE SET value = excluded.value +"#, + [snapshot_id], + ) + .expect("set active snapshot id"); + } + + fn snapshot_state_for_test(data_dir: &Path, snapshot_id: i64) -> String { + let conn = + Connection::open(data_dir.join("event-log.sqlite3")).expect("open event log sqlite"); + conn.query_row( + "SELECT state FROM snapshots WHERE snapshot_id = ?1", + [snapshot_id], + |row| row.get::<_, String>(0), + ) + .expect("snapshot state") + } + + fn clear_ready_snapshots_for_test(data_dir: &Path) { + let conn = + Connection::open(data_dir.join("event-log.sqlite3")).expect("open event log sqlite"); + conn.execute("DELETE FROM snapshots", []) + .expect("delete snapshots"); + conn.execute("DELETE FROM meta WHERE key = 'active_snapshot_id'", []) + .expect("delete active snapshot id"); + } + + fn max_event_num_for_test(data_dir: &Path) -> Option { + let conn = + Connection::open(data_dir.join("event-log.sqlite3")).expect("open event log sqlite"); + conn.query_row("SELECT max(event_num) FROM events", [], |row| { + row.get::<_, Option>(0) + }) + .expect("read max event num") + } + + fn insert_event_job_for_test(data_dir: &Path, event_num: u64, job_jam: Vec) { + let conn = + Connection::open(data_dir.join("event-log.sqlite3")).expect("open event log sqlite"); + conn.execute( + r#" +INSERT INTO events ( + event_num, + job_jam, + wire_source, + wire_version, + wire_tags_json, + cause_hash, + job_hash, + event_processing_duration_us, + created_at_ms +) VALUES (?1, ?2, 'sys', 1, '[]', ?3, ?4, 0, 42) +"#, + ( + i64::try_from(event_num).expect("event num fits in sqlite"), + job_jam, + vec![0_u8; 32], + vec![1_u8; 32], + ), + ) + .expect("insert event job"); + } + + fn rewrite_snapshot_manifest_ker_hash(manifest_path: &Path, ker_hash: [u8; 32]) { + let manifest = + SnapshotManifest::read_from_path(manifest_path).expect("read snapshot manifest"); + let rewritten = SnapshotManifest::new( + manifest.kind, + manifest.timestamp_tag.clone(), + blake3::Hash::from_bytes(ker_hash), + manifest.event_num, + manifest.pma_words, + manifest.alloc_words, + manifest.kernel_root_raw, + manifest.cold_offset, + blake3::Hash::from_bytes(manifest.used_blake3), + manifest.structure_blake3.map(blake3::Hash::from_bytes), + manifest.created_at_ms, + ) + .expect("rewrite snapshot manifest"); + rewritten + .write_to_path(manifest_path) + .expect("write rewritten snapshot manifest"); + } + + fn set_event_processing_duration_for_test(data_dir: &Path, event_num: u64, duration: Duration) { + let conn = + Connection::open(data_dir.join("event-log.sqlite3")).expect("open event log sqlite"); + let duration_us = + i64::try_from(duration.as_micros()).expect("event processing duration fits in i64"); + conn.execute( + "UPDATE events SET event_processing_duration_us = ?1 WHERE event_num = ?2", + ( + duration_us, + i64::try_from(event_num).expect("event num fits in i64"), + ), + ) + .expect("update event processing duration"); + } + + #[test] + fn parse_optional_u64_none_variants() { + assert_eq!(parse_optional_u64("none").unwrap(), 0); + assert_eq!(parse_optional_u64("NoNe").unwrap(), 0); + assert_eq!(parse_optional_u64("0").unwrap(), 0); + assert_eq!(parse_optional_u64(" 0 ").unwrap(), 0); + } + + #[test] + fn parse_optional_u64_positive_values() { + assert_eq!(parse_optional_u64("1").unwrap(), 1); + assert_eq!(parse_optional_u64(" 900 ").unwrap(), 900); + } + + #[test] + fn parse_optional_u64_rejects_invalid() { + assert!(parse_optional_u64("abc").is_err()); + } + + #[test] + fn parse_pma_size_accepts_binary_byte_units() { + assert_eq!(parse_pma_size("1").unwrap().words(), 1); + assert_eq!(parse_pma_size("8").unwrap().words(), 1); + assert_eq!(parse_pma_size("256MiB").unwrap().words(), 32 * 1024 * 1024); + assert_eq!(parse_pma_size("1GiB").unwrap().words(), 128 * 1024 * 1024); + assert_eq!( + parse_pma_size("1_TiB").unwrap().words(), + 128 * 1024 * 1024 * 1024 + ); + } + + #[test] + fn default_pma_initial_size_uses_kernel_jam_size_floor_and_power_of_two() { + assert_eq!( + default_pma_initial_words(12 * 1024 * 1024), + 32 * 1024 * 1024 + ); + assert_eq!( + default_pma_initial_words(18 * 1024 * 1024), + 64 * 1024 * 1024 + ); + } #[test] - fn parse_save_interval_none_variants() { - assert_eq!(parse_save_interval("none").expect("should parse"), 0); - assert_eq!(parse_save_interval("NoNe").expect("should parse"), 0); - assert_eq!(parse_save_interval("0").expect("should parse"), 0); - assert_eq!(parse_save_interval(" 0 ").expect("should parse"), 0); + fn pma_size_cli_decouples_from_stack_size() { + let parsed = super::Cli::try_parse_from([ + "boot-test", "--stack-size", "large", "--pma-initial-size", "512MiB", + "--pma-reserved-size", "2TiB", + ]) + .expect("parse PMA sizing cli"); + assert!(matches!(parsed.stack_size, super::NockStackSize::Large)); + assert_eq!(parsed.pma_initial_size.unwrap().words(), 64 * 1024 * 1024); + assert_eq!( + parsed.pma_reserved_size.unwrap().words(), + 256 * 1024 * 1024 * 1024 + ); + } + + #[test] + fn normalized_rotating_snapshot_interval_event_time_filters_zero() { + let mut cli = super::default_boot_cli(false); + cli.rotating_snapshot_interval_event_time = Some(0); + assert_eq!(cli.normalized_rotating_snapshot_interval_event_time(), None); + + cli.rotating_snapshot_interval_event_time = Some(5); + assert_eq!( + cli.normalized_rotating_snapshot_interval_event_time(), + Some(Duration::from_secs(5)) + ); + } + + #[test] + fn normalized_gc_interval_defaults_enabled_and_filters_zero() { + let mut cli = super::default_boot_cli(false); + assert_eq!( + cli.normalized_gc_interval(), + Some(Duration::from_secs(DEFAULT_GC_INTERVAL_SECS)) + ); + + cli.gc_interval = Some(0); + assert_eq!(cli.normalized_gc_interval(), None); + + cli.gc_interval = None; + assert_eq!(cli.normalized_gc_interval(), None); } #[test] - fn parse_save_interval_positive_values() { - assert_eq!(parse_save_interval("1").expect("should parse"), 1); + fn gc_interval_cli_defaults_enabled_and_allows_disable() { + let parsed = super::Cli::try_parse_from(["boot-test"]).expect("parse default cli"); + assert_eq!( + parsed.normalized_gc_interval(), + Some(Duration::from_secs(DEFAULT_GC_INTERVAL_SECS)) + ); + + let parsed = super::Cli::try_parse_from(["boot-test", "--gc-interval", "none"]) + .expect("parse disabled gc interval"); + assert_eq!(parsed.normalized_gc_interval(), None); + + let parsed = super::Cli::try_parse_from(["boot-test", "--gc-interval", "0"]) + .expect("parse zero gc interval"); + assert_eq!(parsed.normalized_gc_interval(), None); + + let parsed = super::Cli::try_parse_from(["boot-test", "--gc-interval", "1"]) + .expect("parse one-second gc interval"); assert_eq!( - parse_save_interval(" 120000 ").expect("should parse"), - 120000 + parsed.normalized_gc_interval(), + Some(Duration::from_secs(1)) ); } #[test] - fn parse_save_interval_rejects_invalid() { - assert!(parse_save_interval("abc").is_err()); + fn ephemeral_test_boot_cli_disables_durable_side_effects() { + let cli = super::ephemeral_test_boot_cli(true); + assert!(cli.new); + assert!(cli.ephemeral); + assert_eq!(cli.gc_interval, None); + assert_eq!(cli.rotating_snapshot_interval_event_time, None); + assert!(cli.disable_fsync); + } + + #[test] + fn durable_test_boot_cli_keeps_pma_with_fast_io_defaults() { + let cli = durable_test_boot_cli(false); + assert!(!cli.new); + assert!(!cli.ephemeral); + assert_eq!(cli.gc_interval, None); + assert_eq!(cli.rotating_snapshot_interval_event_time, None); + assert!(cli.disable_fsync); } #[test] - fn normalized_save_interval_filters_zero() { + fn state_jam_cli_requires_new() { + let err = + super::Cli::try_parse_from(["boot-test", "--state-jam", "/tmp/state.jam"]).unwrap_err(); + assert!( + err.to_string().contains("--new"), + "expected clap error to mention --new, got: {err}" + ); + + let parsed = + super::Cli::try_parse_from(["boot-test", "--new", "--state-jam", "/tmp/state.jam"]) + .expect("parse with --new"); + assert!(parsed.new); + assert_eq!(parsed.state_jam.as_deref(), Some("/tmp/state.jam")); + } + + #[tokio::test(flavor = "current_thread")] + async fn setup_rejects_state_jam_without_new_for_programmatic_callers() { + let temp = TempDir::new().expect("tempdir"); let mut cli = super::default_boot_cli(false); - cli.save_interval = Some(0); - assert_eq!(cli.normalized_save_interval(), None); + cli.state_jam = Some(temp.path().join("state.jam").display().to_string()); + + let err = + match setup_::(&[], cli, &[], "boot-test", Some(temp.path().to_path_buf())) + .await + { + Ok(_) => panic!("setup should reject state_jam without --new"), + Err(err) => err, + }; + + assert!( + err.to_string().contains("--state-jam requires --new"), + "unexpected error: {err}" + ); + } + + #[tokio::test(flavor = "current_thread")] + #[cfg_attr(miri, ignore)] + async fn export_state_jam_creates_parent_dir() { + let temp = TempDir::new().expect("tempdir"); + let data_dir = temp.path().join("export-state-jam-data"); + let export_path = temp + .path() + .join("nested") + .join("backup") + .join("kernel.state.jam"); + + let mut app = setup_test_app(&data_dir).await; + wait_for_serf_idle(&app).await; + + export_kernel_state::( + &app.kernel, + export_path.to_str().expect("export path string"), + ) + .await + .expect("export state jam"); + + assert!(export_path.exists()); + assert!(export_path.parent().expect("export parent").exists()); + + stop_app(&mut app).await; + } + + #[tokio::test(flavor = "current_thread")] + async fn setup_rejects_new_when_data_dir_is_nonempty() { + let temp = TempDir::new().expect("tempdir"); + let data_dir = temp.path().join("existing-data-dir"); + let checkpoints_dir = data_dir.join("checkpoints"); + let checkpoint_path = checkpoints_dir.join("existing.chkjam"); + fs::create_dir_all(&checkpoints_dir).expect("create checkpoints dir"); + fs::write(&checkpoint_path, b"keep").expect("write existing checkpoint"); + + let jam = load_test_jam_bytes(); + let mut cli = default_boot_cli(true); + cli.data_dir = Some(data_dir.clone()); + + let err = match setup_::(&jam, cli, &[], "boot-test", None).await { + Ok(_) => panic!("setup should reject --new for a non-empty data dir"), + Err(err) => err, + }; + + assert!( + err.to_string() + .contains("--new requires an empty data directory"), + "unexpected error: {err}" + ); + assert_eq!( + fs::read(&checkpoint_path).expect("read existing checkpoint"), + b"keep" + ); + } + + #[tokio::test(flavor = "current_thread")] + #[cfg_attr(miri, ignore)] + async fn setup_allows_new_for_empty_data_dir() { + let temp = TempDir::new().expect("tempdir"); + let data_dir = temp.path().join("fresh-empty-dir"); + fs::create_dir_all(&data_dir).expect("create empty data dir"); + + let jam = load_test_jam_bytes(); + let mut cli = durable_test_boot_cli(true); + cli.data_dir = Some(data_dir.clone()); + + let mut app = match setup_::(&jam, cli, &[], "boot-test", None) + .await + .expect("setup should allow --new for an empty data dir") + { + SetupResult::App(app) => app, + SetupResult::ExportedState => panic!("unexpected export"), + }; + + assert!(data_dir.join("checkpoints").exists()); + assert!(data_dir.join("pma").exists()); + stop_app(&mut app).await; + } + + #[tokio::test(flavor = "current_thread")] + #[cfg_attr(miri, ignore)] + async fn bootstraps_pma_from_checkpoint_once() { + let temp = TempDir::new().expect("tempdir"); + let data_dir = temp.path().join("bootstrapped-pma"); + + let mut first = setup_test_app(&data_dir).await; + poke_inc(&first).await; + write_checkpoint_bootstrap_fixture(&first, &data_dir).await; + stop_app(&mut first).await; + drop(first); + assert_eq!(ready_snapshot_count(&data_dir), 1); + fs::remove_file(data_dir.join("pma").join("epoch.pma")).expect("remove epoch pma"); + fs::remove_file(data_dir.join("pma").join("epoch.manifest")) + .expect("remove epoch manifest"); + clear_ready_snapshots_for_test(&data_dir); + clear_pma_files(&data_dir); + + let mut second = setup_test_app(&data_dir).await; + assert_eq!(second.kernel.serf.event_number.load(Ordering::SeqCst), 1); + assert!( + data_dir.join("pma").join("0.meta").exists() + || data_dir.join("pma").join("1.meta").exists() + ); + assert!(data_dir.join("pma").join("epoch.pma").exists()); + assert!(data_dir.join("pma").join("epoch.manifest").exists()); + assert_eq!(ready_snapshot_count(&data_dir), 1); + poke_inc(&second).await; + assert_eq!(second.kernel.serf.event_number.load(Ordering::SeqCst), 2); + stop_app(&mut second).await; + drop(second); + } + + #[tokio::test(flavor = "current_thread")] + #[cfg_attr(miri, ignore)] + async fn bootstraps_explicit_checkpoint_copy_into_empty_event_log() { + let temp = TempDir::new().expect("tempdir"); + let source_data_dir = temp.path().join("checkpoint-source"); + let copied_data_dir = temp.path().join("checkpoint-copy-empty-event-log"); + + let mut first = setup_test_app(&source_data_dir).await; + poke_inc(&first).await; + assert_counter_state(&mut first, 1).await; + write_checkpoint_bootstrap_fixture(&first, &source_data_dir).await; + stop_app(&mut first).await; + drop(first); + + fs::create_dir_all(copied_data_dir.join("checkpoints")).expect("create checkpoint dir"); + fs::copy( + source_data_dir.join("checkpoints").join("0.chkjam"), + copied_data_dir.join("checkpoints").join("0.chkjam"), + ) + .expect("copy checkpoint into fresh data dir"); + + let implicit_err = match try_setup_test_app(&copied_data_dir, None).await { + Ok(mut app) => { + stop_app(&mut app).await; + panic!("implicit checkpoint bootstrap unexpectedly succeeded") + } + Err(err) => err, + }; + assert!( + implicit_err + .to_string() + .contains("SQLite event log is empty or missing"), + "unexpected implicit checkpoint bootstrap error: {implicit_err}" + ); + + let mut second = setup_test_app_from_chkjam( + &copied_data_dir, + &source_data_dir.join("checkpoints").join("0.chkjam"), + ) + .await; + assert_counter_state(&mut second, 1).await; + assert_eq!(max_event_num_for_test(&copied_data_dir), None); + stop_app(&mut second).await; + drop(second); + + let mut third = setup_test_app(&copied_data_dir).await; + assert_counter_state(&mut third, 1).await; + assert_eq!(max_event_num_for_test(&copied_data_dir), None); + poke_inc(&third).await; + assert_counter_state(&mut third, 2).await; + assert_eq!(max_event_num_for_test(&copied_data_dir), Some(2)); + stop_app(&mut third).await; + drop(third); + } + + #[tokio::test(flavor = "current_thread")] + #[cfg_attr(miri, ignore)] + async fn valid_pma_skips_corrupt_checkpoint_files() { + let temp = TempDir::new().expect("tempdir"); + let data_dir = temp.path().join("skip-corrupt-checkpoint"); + + let mut first = setup_test_app(&data_dir).await; + poke_inc(&first).await; + write_checkpoint_bootstrap_fixture(&first, &data_dir).await; + stop_app(&mut first).await; + drop(first); + clear_pma_files(&data_dir); + + let mut second = setup_test_app(&data_dir).await; + poke_inc(&second).await; + assert_eq!(second.kernel.serf.event_number.load(Ordering::SeqCst), 2); + stop_app(&mut second).await; + drop(second); + + let checkpoints_dir = data_dir.join("checkpoints"); + fs::write(checkpoints_dir.join("0.chkjam"), b"corrupt checkpoint 0").expect("corrupt chk0"); + fs::write(checkpoints_dir.join("1.chkjam"), b"corrupt checkpoint 1").expect("corrupt chk1"); + + let boot_selection: BootSelection = select_boot_state::( + &data_dir.join("checkpoints"), + &load_test_jam_bytes(), + &data_dir.join("event-log.sqlite3"), + &data_dir.join("pma").join("0.pma"), + &data_dir.join("pma").join("1.pma"), + Arc::new(NockAppMetrics::default()), + BootEventLogPolicy { + preexisting: true, + allow_empty_bootstrap: false, + }, + ) + .await + .expect("select boot state"); + assert!(boot_selection.checkpoint.is_none()); + assert!(boot_selection.pma_open_existing); + assert!(boot_selection.snapshot_manifest.is_none()); + assert!(boot_selection.replay_jobs.is_empty()); + } + + #[tokio::test(flavor = "current_thread")] + #[cfg_attr(miri, ignore)] + async fn valid_pma_with_unopenable_event_log_fails_closed() { + let temp = TempDir::new().expect("tempdir"); + let data_dir = temp.path().join("valid-pma-unopenable-event-log"); + + let mut first = setup_test_app(&data_dir).await; + poke_inc(&first).await; + assert_counter_state(&mut first, 1).await; + stop_app(&mut first).await; + drop(first); - cli.save_interval = Some(5000); - assert_eq!(cli.normalized_save_interval(), Some(5000)); + let event_log_path = data_dir.join("event-log.sqlite3"); + fs::remove_file(&event_log_path).expect("remove event log sqlite"); + fs::create_dir(&event_log_path).expect("replace event log sqlite with directory"); + + let result = select_boot_state::( + &data_dir.join("checkpoints"), + &load_test_jam_bytes(), + &event_log_path, + &data_dir.join("pma").join("0.pma"), + &data_dir.join("pma").join("1.pma"), + Arc::new(NockAppMetrics::default()), + BootEventLogPolicy { + preexisting: true, + allow_empty_bootstrap: false, + }, + ) + .await; + + match result { + Ok(_) => panic!("boot selection trusted PMA even though SQLite could not be opened"), + Err(err) => assert!( + err.to_string().contains("event log"), + "unexpected boot selection error: {err}" + ), + } + } + + #[tokio::test(flavor = "current_thread")] + #[cfg_attr(miri, ignore)] + async fn pma_ahead_of_event_log_recovers_to_sqlite_boundary() { + let temp = TempDir::new().expect("tempdir"); + let data_dir = temp.path().join("pma-ahead-of-event-log"); + + let mut first = setup_test_app(&data_dir).await; + poke_inc(&first).await; + poke_inc(&first).await; + assert_eq!(first.kernel.serf.event_number.load(Ordering::SeqCst), 2); + stop_app(&mut first).await; + drop(first); + + let conn = + Connection::open(data_dir.join("event-log.sqlite3")).expect("open event log sqlite"); + conn.execute("DELETE FROM events WHERE event_num = 2", []) + .expect("delete event 2"); + drop(conn); + assert_eq!(max_event_num_for_test(&data_dir), Some(1)); + + let mut second = setup_test_app(&data_dir).await; + assert_counter_state(&mut second, 1).await; + stop_app(&mut second).await; + drop(second); + } + + #[tokio::test(flavor = "current_thread")] + #[cfg_attr(miri, ignore)] + async fn pma_lagging_event_log_replays_to_sqlite_boundary() { + let temp = TempDir::new().expect("tempdir"); + let data_dir = temp.path().join("pma-lagging-event-log"); + let pma_dir = data_dir.join("pma"); + let event_1_pma_backup = temp.path().join("event-1-pma-backup"); + + let mut first = setup_test_app(&data_dir).await; + poke_inc(&first).await; + assert_eq!(first.kernel.serf.event_number.load(Ordering::SeqCst), 1); + stop_app(&mut first).await; + drop(first); + copy_runtime_pma_files(&pma_dir, &event_1_pma_backup); + + let mut second = setup_test_app(&data_dir).await; + assert_eq!(second.kernel.serf.event_number.load(Ordering::SeqCst), 1); + poke_inc(&second).await; + assert_eq!(second.kernel.serf.event_number.load(Ordering::SeqCst), 2); + stop_app(&mut second).await; + drop(second); + + assert_eq!(max_event_num_for_test(&data_dir), Some(2)); + restore_runtime_pma_files(&data_dir, &event_1_pma_backup); + + let mut third = setup_test_app(&data_dir).await; + assert_counter_state(&mut third, 2).await; + stop_app(&mut third).await; + drop(third); + } + + #[tokio::test(flavor = "current_thread")] + #[cfg_attr(miri, ignore)] + async fn checkpoint_behind_event_log_replays_to_sqlite_boundary() { + let temp = TempDir::new().expect("tempdir"); + let data_dir = temp.path().join("checkpoint-behind-event-log"); + + let mut first = setup_test_app(&data_dir).await; + poke_inc(&first).await; + assert_counter_state(&mut first, 1).await; + write_checkpoint_bootstrap_fixture(&first, &data_dir).await; + stop_app(&mut first).await; + drop(first); + + let mut second = setup_test_app(&data_dir).await; + assert_counter_state(&mut second, 1).await; + poke_inc(&second).await; + assert_counter_state(&mut second, 2).await; + stop_app(&mut second).await; + drop(second); + + assert_eq!(max_event_num_for_test(&data_dir), Some(2)); + clear_ready_snapshots_for_test(&data_dir); + clear_pma_files(&data_dir); + + let mut third = setup_test_app(&data_dir).await; + assert_counter_state(&mut third, 2).await; + stop_app(&mut third).await; + drop(third); + } + + #[tokio::test(flavor = "current_thread")] + #[cfg_attr(miri, ignore)] + async fn checkpoint_ahead_of_event_log_recovers_to_sqlite_boundary() { + let temp = TempDir::new().expect("tempdir"); + let data_dir = temp.path().join("checkpoint-ahead-event-log"); + + let mut first = setup_test_app(&data_dir).await; + poke_inc(&first).await; + assert_counter_state(&mut first, 1).await; + stop_app(&mut first).await; + drop(first); + + let mut second = setup_test_app(&data_dir).await; + assert_counter_state(&mut second, 1).await; + poke_inc(&second).await; + assert_counter_state(&mut second, 2).await; + write_checkpoint_bootstrap_fixture(&second, &data_dir).await; + stop_app(&mut second).await; + drop(second); + + let conn = + Connection::open(data_dir.join("event-log.sqlite3")).expect("open event log sqlite"); + conn.execute("DELETE FROM events WHERE event_num = 2", []) + .expect("delete uncommitted event"); + drop(conn); + assert_eq!(max_event_num_for_test(&data_dir), Some(1)); + clear_ready_snapshots_for_test(&data_dir); + clear_pma_files(&data_dir); + + let mut third = setup_test_app(&data_dir).await; + assert_counter_state(&mut third, 1).await; + stop_app(&mut third).await; + drop(third); + } + + #[tokio::test(flavor = "current_thread")] + #[cfg_attr(miri, ignore)] + async fn fresh_boot_with_committed_event_log_replays_from_zero() { + let temp = TempDir::new().expect("tempdir"); + let data_dir = temp.path().join("fresh-replay-from-event-log"); + + let mut first = setup_test_app(&data_dir).await; + stop_app(&mut first).await; + drop(first); + + clear_ready_snapshots_for_test(&data_dir); + clear_pma_files(&data_dir); + insert_event_job_for_test(&data_dir, 1, logged_poke_job_jam(1, tas!(b"inc"))); + + let mut second = setup_test_app(&data_dir).await; + assert_counter_state(&mut second, 1).await; + stop_app(&mut second).await; + drop(second); + } + + #[tokio::test(flavor = "current_thread")] + #[cfg_attr(miri, ignore)] + async fn replay_rejected_logged_event_fails_instead_of_synthesizing_crud() { + let temp = TempDir::new().expect("tempdir"); + let data_dir = temp.path().join("replay-rejected-logged-event"); + + let mut first = setup_test_app(&data_dir).await; + stop_app(&mut first).await; + drop(first); + + insert_event_job_for_test(&data_dir, 1, logged_poke_job_jam(1, tas!(b"bad"))); + clear_pma_files(&data_dir); + + let err = match try_setup_test_app(&data_dir, None).await { + Ok(_) => panic!("replay should fail instead of synthesizing a replacement event"), + Err(err) => err, + }; + + let err_text = err.to_string(); + assert!( + err_text.contains("event replay after snapshot restore") + || err_text.to_ascii_lowercase().contains("kernel error"), + "unexpected replay error: {err}" + ); + } + + #[tokio::test(flavor = "current_thread")] + #[cfg_attr(miri, ignore)] + async fn snapshot_kernel_hash_mismatch_loads_state_like_checkpoint() { + let temp = TempDir::new().expect("tempdir"); + let data_dir = temp.path().join("snapshot-kernel-hash-mismatch"); + + let mut first = setup_test_app(&data_dir).await; + poke_inc(&first).await; + assert_counter_state(&mut first, 1).await; + stop_app(&mut first).await; + drop(first); + set_event_processing_duration_for_test(&data_dir, 1, Duration::from_secs(1)); + clear_pma_files(&data_dir); + + let mut second = try_setup_test_app(&data_dir, Some(1)) + .await + .expect("setup app for rotating snapshot"); + assert_counter_state(&mut second, 1).await; + poke_inc(&second).await; + wait_for_serf_idle(&second).await; + assert_counter_state(&mut second, 2).await; + stop_app(&mut second).await; + drop(second); + + let rotating = ready_rotating_snapshots(&data_dir); + assert!(!rotating.is_empty(), "expected a rotating snapshot"); + let (snapshot_id, _pma_path, manifest_path, event_num) = &rotating[0]; + assert_eq!(*event_num, 2); + retain_only_snapshot_for_test(&data_dir, *snapshot_id); + rewrite_snapshot_manifest_ker_hash(Path::new(manifest_path), [0x42; 32]); + clear_pma_files(&data_dir); + + let ready = ready_snapshots(&data_dir); + assert_eq!(ready.len(), 1); + assert_eq!(ready[0].0, *snapshot_id); + + let mut third = setup_test_app(&data_dir).await; + assert_counter_state(&mut third, 2).await; + stop_app(&mut third).await; + drop(third); + } + + #[tokio::test(flavor = "current_thread")] + #[cfg_attr(miri, ignore)] + async fn restores_epoch_snapshot_when_pma_is_missing() { + let temp = TempDir::new().expect("tempdir"); + let data_dir = temp.path().join("restore-epoch-snapshot"); + + let mut first = setup_test_app(&data_dir).await; + poke_inc(&first).await; + stop_app(&mut first).await; + drop(first); + clear_pma_files(&data_dir); + + let mut second = setup_test_app(&data_dir).await; + assert_eq!(second.kernel.serf.event_number.load(Ordering::SeqCst), 1); + stop_app(&mut second).await; + drop(second); + + clear_pma_files(&data_dir); + let checkpoints_dir = data_dir.join("checkpoints"); + fs::write(checkpoints_dir.join("0.chkjam"), b"corrupt checkpoint 0").expect("corrupt chk0"); + fs::write(checkpoints_dir.join("1.chkjam"), b"corrupt checkpoint 1").expect("corrupt chk1"); + + let mut third = setup_test_app(&data_dir).await; + assert_eq!(third.kernel.serf.event_number.load(Ordering::SeqCst), 1); + assert!(data_dir.join("pma").join("0.meta").exists()); + stop_app(&mut third).await; + drop(third); + } + + #[tokio::test(flavor = "current_thread")] + #[cfg_attr(miri, ignore)] + async fn shutdown_flush_rewrites_missing_active_pma_metadata() { + let temp = TempDir::new().expect("tempdir"); + let data_dir = temp.path().join("shutdown-rewrites-pma-meta"); + + let mut first = setup_test_app(&data_dir).await; + poke_inc(&first).await; + for meta_name in ["0.meta", "1.meta"] { + let meta_path = data_dir.join("pma").join(meta_name); + if meta_path.exists() { + fs::remove_file(&meta_path).expect("remove active meta before shutdown"); + } + assert!(!meta_path.exists()); + } + + stop_app(&mut first).await; + drop(first); + + assert!( + data_dir.join("pma").join("0.meta").exists() + || data_dir.join("pma").join("1.meta").exists() + ); + } + + #[tokio::test(flavor = "current_thread")] + #[cfg_attr(miri, ignore)] + async fn pma_gc_switches_slabs_and_rebuilds_runtime_state() { + let temp = TempDir::new().expect("tempdir"); + let data_dir = temp.path().join("pma-gc-slab-switch"); + fs::create_dir_all(&data_dir).expect("create data dir"); + + let mut first = try_setup_test_app_with_gc_interval(&data_dir, Some(1)) + .await + .expect("setup with forced GC interval"); + tokio::time::sleep(Duration::from_secs(1) + Duration::from_millis(50)).await; + poke_inc(&first).await; + wait_for_serf_idle(&first).await; + + assert!( + data_dir.join("pma").join("1.meta").exists(), + "GC should publish metadata for the alternate slab" + ); + assert!( + !data_dir.join("pma").join("0.meta").exists(), + "GC should invalidate the old active slab metadata before switching slabs" + ); + assert_counter_state(&mut first, 1).await; + stop_app(&mut first).await; + + let mut second = setup_test_app(&data_dir).await; + assert_counter_state(&mut second, 1).await; + stop_app(&mut second).await; + } + + #[tokio::test(flavor = "current_thread")] + #[cfg_attr(miri, ignore)] + async fn replays_logged_events_after_snapshot_restore() { + let temp = TempDir::new().expect("tempdir"); + let data_dir = temp.path().join("replay-after-snapshot-restore"); + + let mut first = setup_test_app(&data_dir).await; + poke_inc(&first).await; + stop_app(&mut first).await; + drop(first); + clear_pma_files(&data_dir); + + let mut second = setup_test_app(&data_dir).await; + assert_eq!(second.kernel.serf.event_number.load(Ordering::SeqCst), 1); + poke_inc(&second).await; + assert_eq!(second.kernel.serf.event_number.load(Ordering::SeqCst), 2); + stop_app(&mut second).await; + drop(second); + + clear_pma_files(&data_dir); + let checkpoints_dir = data_dir.join("checkpoints"); + fs::write(checkpoints_dir.join("0.chkjam"), b"corrupt checkpoint 0").expect("corrupt chk0"); + fs::write(checkpoints_dir.join("1.chkjam"), b"corrupt checkpoint 1").expect("corrupt chk1"); + + let mut third = setup_test_app(&data_dir).await; + assert_eq!(third.kernel.serf.event_number.load(Ordering::SeqCst), 2); + stop_app(&mut third).await; + drop(third); + } + + #[tokio::test(flavor = "current_thread")] + #[cfg_attr(miri, ignore)] + async fn refuses_boot_on_event_log_gap_after_snapshot() { + let temp = TempDir::new().expect("tempdir"); + let data_dir = temp.path().join("gap-after-snapshot"); + + let mut first = setup_test_app(&data_dir).await; + poke_inc(&first).await; + stop_app(&mut first).await; + drop(first); + clear_pma_files(&data_dir); + + let mut second = setup_test_app(&data_dir).await; + poke_inc(&second).await; + poke_inc(&second).await; + stop_app(&mut second).await; + drop(second); + + let conn = + Connection::open(data_dir.join("event-log.sqlite3")).expect("open event log sqlite"); + conn.execute("DELETE FROM events WHERE event_num = 2", []) + .expect("delete event"); + + clear_pma_files(&data_dir); + let checkpoints_dir = data_dir.join("checkpoints"); + fs::write(checkpoints_dir.join("0.chkjam"), b"corrupt checkpoint 0").expect("corrupt chk0"); + fs::write(checkpoints_dir.join("1.chkjam"), b"corrupt checkpoint 1").expect("corrupt chk1"); + + let err = match try_setup_test_app(&data_dir, None).await { + Ok(_) => panic!("boot should fail on continuity gap"), + Err(err) => err, + }; + assert!(err + .to_string() + .contains("event log continuity check failed")); + } + + #[tokio::test(flavor = "current_thread")] + #[cfg_attr(miri, ignore)] + async fn rotates_snapshots_and_retires_oldest() { + let temp = TempDir::new().expect("tempdir"); + let data_dir = temp.path().join("rotating-snapshot-retention"); + + let mut first = setup_test_app(&data_dir).await; + poke_inc(&first).await; + wait_for_serf_idle(&first).await; + stop_app(&mut first).await; + drop(first); + set_event_processing_duration_for_test(&data_dir, 1, Duration::from_secs(1)); + clear_pma_files(&data_dir); + + let mut second = try_setup_test_app(&data_dir, Some(1)) + .await + .expect("setup rotating snapshot retention app"); + poke_inc(&second).await; + poke_inc(&second).await; + wait_for_serf_idle(&second).await; + stop_app(&mut second).await; + drop(second); + set_event_processing_duration_for_test(&data_dir, 3, Duration::from_secs(1)); + clear_pma_files(&data_dir); + + let mut third = try_setup_test_app(&data_dir, Some(1)) + .await + .expect("setup rotating snapshot retention app after event 3"); + poke_inc(&third).await; + poke_inc(&third).await; + wait_for_serf_idle(&third).await; + stop_app(&mut third).await; + drop(third); + set_event_processing_duration_for_test(&data_dir, 5, Duration::from_secs(1)); + clear_pma_files(&data_dir); + + let mut fourth = try_setup_test_app(&data_dir, Some(1)) + .await + .expect("setup rotating snapshot retention app after event 5"); + poke_inc(&fourth).await; + wait_for_serf_idle(&fourth).await; + stop_app(&mut fourth).await; + drop(fourth); + + let rotating = ready_rotating_snapshots(&data_dir); + assert_eq!(rotating.len(), 2); + assert_eq!(rotating[0].3, 6); + assert_eq!(rotating[1].3, 4); + assert_eq!(retired_rotating_snapshot_count(&data_dir), 1); + for (_, pma_path, manifest_path, _) in rotating { + assert!(Path::new(&pma_path).exists()); + assert!(Path::new(&manifest_path).exists()); + } + } + + #[tokio::test(flavor = "current_thread")] + #[cfg_attr(miri, ignore)] + async fn falls_back_from_corrupt_newest_rotating_snapshot() { + let temp = TempDir::new().expect("tempdir"); + let data_dir = temp.path().join("rotating-fallback"); + + let mut first = setup_test_app(&data_dir).await; + poke_inc(&first).await; + wait_for_serf_idle(&first).await; + stop_app(&mut first).await; + drop(first); + set_event_processing_duration_for_test(&data_dir, 1, Duration::from_secs(1)); + clear_pma_files(&data_dir); + + let mut second = try_setup_test_app(&data_dir, Some(1)) + .await + .expect("setup rotating fallback app"); + poke_inc(&second).await; + poke_inc(&second).await; + wait_for_serf_idle(&second).await; + stop_app(&mut second).await; + drop(second); + set_event_processing_duration_for_test(&data_dir, 3, Duration::from_secs(1)); + clear_pma_files(&data_dir); + + let mut third = try_setup_test_app(&data_dir, Some(1)) + .await + .expect("setup rotating fallback app after event 3"); + poke_inc(&third).await; + wait_for_serf_idle(&third).await; + assert_eq!(third.kernel.serf.event_number.load(Ordering::SeqCst), 4); + stop_app(&mut third).await; + drop(third); + + let rotating = ready_rotating_snapshots(&data_dir); + assert_eq!(rotating.len(), 2); + fs::write(&rotating[0].1, b"corrupt newest rotating snapshot") + .expect("corrupt newest rotating pma"); + + clear_pma_files(&data_dir); + let checkpoints_dir = data_dir.join("checkpoints"); + fs::write(checkpoints_dir.join("0.chkjam"), b"corrupt checkpoint 0").expect("corrupt chk0"); + fs::write(checkpoints_dir.join("1.chkjam"), b"corrupt checkpoint 1").expect("corrupt chk1"); + + let mut fourth = setup_test_app(&data_dir).await; + assert_eq!(fourth.kernel.serf.event_number.load(Ordering::SeqCst), 4); + stop_app(&mut fourth).await; + drop(fourth); + } + + #[tokio::test(flavor = "current_thread")] + #[cfg_attr(miri, ignore)] + async fn falls_back_from_manifest_only_corruption() { + let temp = TempDir::new().expect("tempdir"); + let data_dir = temp.path().join("manifest-only-fallback"); + + let mut first = setup_test_app(&data_dir).await; + poke_inc(&first).await; + wait_for_serf_idle(&first).await; + stop_app(&mut first).await; + drop(first); + set_event_processing_duration_for_test(&data_dir, 1, Duration::from_secs(1)); + clear_pma_files(&data_dir); + + let mut second = try_setup_test_app(&data_dir, Some(1)) + .await + .expect("setup manifest-only fallback app"); + poke_inc(&second).await; + poke_inc(&second).await; + wait_for_serf_idle(&second).await; + stop_app(&mut second).await; + drop(second); + set_event_processing_duration_for_test(&data_dir, 3, Duration::from_secs(1)); + clear_pma_files(&data_dir); + + let mut third = try_setup_test_app(&data_dir, Some(1)) + .await + .expect("setup manifest-only fallback app after event 3"); + poke_inc(&third).await; + wait_for_serf_idle(&third).await; + assert_eq!(third.kernel.serf.event_number.load(Ordering::SeqCst), 4); + stop_app(&mut third).await; + drop(third); + + let rotating = ready_rotating_snapshots(&data_dir); + assert_eq!(rotating.len(), 2); + fs::write(&rotating[0].2, b"corrupt newest rotating manifest") + .expect("corrupt newest rotating manifest"); + + clear_pma_files(&data_dir); + let checkpoints_dir = data_dir.join("checkpoints"); + fs::write(checkpoints_dir.join("0.chkjam"), b"corrupt checkpoint 0").expect("corrupt chk0"); + fs::write(checkpoints_dir.join("1.chkjam"), b"corrupt checkpoint 1").expect("corrupt chk1"); + + let mut fourth = setup_test_app(&data_dir).await; + assert_eq!(fourth.kernel.serf.event_number.load(Ordering::SeqCst), 4); + stop_app(&mut fourth).await; + drop(fourth); + } + + #[tokio::test(flavor = "current_thread")] + #[cfg_attr(miri, ignore)] + async fn honors_active_snapshot_selection_before_ordering() { + let temp = TempDir::new().expect("tempdir"); + let data_dir = temp.path().join("active-snapshot-selection"); + + let mut first = setup_test_app(&data_dir).await; + poke_inc(&first).await; + wait_for_serf_idle(&first).await; + stop_app(&mut first).await; + drop(first); + set_event_processing_duration_for_test(&data_dir, 1, Duration::from_secs(1)); + clear_pma_files(&data_dir); + + let mut second = try_setup_test_app(&data_dir, Some(1)) + .await + .expect("setup active snapshot selection app"); + poke_inc(&second).await; + poke_inc(&second).await; + wait_for_serf_idle(&second).await; + stop_app(&mut second).await; + drop(second); + set_event_processing_duration_for_test(&data_dir, 3, Duration::from_secs(1)); + clear_pma_files(&data_dir); + + let mut third = try_setup_test_app(&data_dir, Some(1)) + .await + .expect("setup active snapshot selection app after event 3"); + poke_inc(&third).await; + wait_for_serf_idle(&third).await; + assert_eq!(third.kernel.serf.event_number.load(Ordering::SeqCst), 4); + stop_app(&mut third).await; + drop(third); + + let rotating = ready_rotating_snapshots(&data_dir); + assert_eq!(rotating.len(), 2); + let older_snapshot_id = rotating[1].0; + let newer_snapshot_id = rotating[0].0; + set_active_snapshot_id_for_test(&data_dir, older_snapshot_id); + fs::write(&rotating[1].1, b"corrupt active rotating snapshot") + .expect("corrupt active rotating pma"); + + clear_pma_files(&data_dir); + let checkpoints_dir = data_dir.join("checkpoints"); + fs::write(checkpoints_dir.join("0.chkjam"), b"corrupt checkpoint 0").expect("corrupt chk0"); + fs::write(checkpoints_dir.join("1.chkjam"), b"corrupt checkpoint 1").expect("corrupt chk1"); + + let mut fourth = setup_test_app(&data_dir).await; + assert_eq!(fourth.kernel.serf.event_number.load(Ordering::SeqCst), 4); + stop_app(&mut fourth).await; + drop(fourth); + + assert_eq!( + snapshot_state_for_test(&data_dir, older_snapshot_id), + "failed" + ); + assert_eq!( + active_snapshot_id_for_test(&data_dir), + Some(newer_snapshot_id) + ); + } + + #[tokio::test(flavor = "current_thread")] + #[cfg_attr(miri, ignore)] + async fn moves_orphan_snapshot_files_to_corrupted_pma() { + let temp = TempDir::new().expect("tempdir"); + let data_dir = temp.path().join("orphan-snapshot-cleanup"); + + let mut first = setup_test_app(&data_dir).await; + poke_inc(&first).await; + wait_for_serf_idle(&first).await; + stop_app(&mut first).await; + drop(first); + set_event_processing_duration_for_test(&data_dir, 1, Duration::from_secs(1)); + clear_pma_files(&data_dir); + + let mut second = try_setup_test_app(&data_dir, Some(1)) + .await + .expect("setup orphan cleanup app"); + poke_inc(&second).await; + assert_eq!(second.kernel.serf.event_number.load(Ordering::SeqCst), 2); + stop_app(&mut second).await; + drop(second); + + let pma_dir = data_dir.join("pma"); + let orphan_pma = pma_dir.join("snap-orphan.pma"); + let orphan_manifest = pma_dir.join("snap-orphan.manifest"); + let orphan_pma_tmp = pma_dir.join("snap-orphan.pma.tmp"); + let orphan_manifest_tmp = pma_dir.join("snap-orphan.manifest.tmp"); + fs::write(&orphan_pma, b"orphan pma").expect("write orphan pma"); + fs::write(&orphan_manifest, b"orphan manifest").expect("write orphan manifest"); + fs::write(&orphan_pma_tmp, b"orphan pma tmp").expect("write orphan pma tmp"); + fs::write(&orphan_manifest_tmp, b"orphan manifest tmp").expect("write orphan manifest tmp"); + + let boot_selection: BootSelection = select_boot_state::( + &data_dir.join("checkpoints"), + &load_test_jam_bytes(), + &data_dir.join("event-log.sqlite3"), + &data_dir.join("pma").join("0.pma"), + &data_dir.join("pma").join("1.pma"), + Arc::new(NockAppMetrics::default()), + BootEventLogPolicy { + preexisting: true, + allow_empty_bootstrap: false, + }, + ) + .await + .expect("select boot state"); + assert!(boot_selection.checkpoint.is_none()); + assert!(boot_selection.pma_open_existing); + assert!(boot_selection.snapshot_manifest.is_none()); + assert!(boot_selection.replay_jobs.is_empty()); + + let corrupted_dir = pma_dir.join("corrupted_pma"); + assert!(!orphan_pma.exists()); + assert!(!orphan_manifest.exists()); + assert!(!orphan_pma_tmp.exists()); + assert!(!orphan_manifest_tmp.exists()); + assert!(corrupted_dir.join("snap-orphan.pma").exists()); + assert!(corrupted_dir.join("snap-orphan.manifest").exists()); + assert!(corrupted_dir.join("snap-orphan.pma.tmp").exists()); + assert!(corrupted_dir.join("snap-orphan.manifest.tmp").exists()); } } @@ -192,16 +1837,476 @@ pub enum SetupResult { ExportedState, } +struct BootSelection { + checkpoint: Option, + pma_open_existing: bool, + snapshot_manifest: Option, + replay_jobs: Vec, +} + +#[derive(Clone, Copy)] +struct BootEventLogPolicy { + preexisting: bool, + allow_empty_bootstrap: bool, +} + +fn order_snapshot_candidates( + active_snapshot_id: Option, + ready_snapshots: Vec, +) -> Vec { + if let Some(active_snapshot_id) = active_snapshot_id { + if let Some(active_idx) = ready_snapshots + .iter() + .position(|snapshot| snapshot.snapshot_id == active_snapshot_id) + { + let mut ordered = Vec::with_capacity(ready_snapshots.len()); + ordered.push(ready_snapshots[active_idx].clone()); + ordered.extend( + ready_snapshots + .into_iter() + .filter(|snapshot| snapshot.snapshot_id != active_snapshot_id), + ); + return ordered; + } + } + ready_snapshots +} + +async fn select_boot_state( + jams_dir: &Path, + kernel_bytes: &[u8], + event_log_path: &Path, + pma_path_0: &Path, + pma_path_1: &Path, + metrics: Arc, + event_log_policy: BootEventLogPolicy, +) -> Result> { + let expected_ker_hash = { + let mut hasher = blake3::Hasher::new(); + hasher.update(kernel_bytes); + hasher.finalize() + }; + let existing_pma = Some(inspect_existing_pma(pma_path_0, pma_path_1, kernel_bytes)); + let mut recovery_event_log = match crate::event_log::EventLog::open(EventLogConfig { + path: event_log_path.to_path_buf(), + }) { + Ok(mut event_log) => { + let cleanup_start = std::time::Instant::now(); + if let Err(err) = cleanup_snapshot_artifacts( + &mut event_log, + pma_path_0 + .parent() + .unwrap_or_else(|| std::path::Path::new(".")), + ) { + metrics.snapshot_cleanup_failures.increment(); + warn!("snapshot cleanup failed during boot: {err}"); + } else { + metrics + .snapshot_cleanup + .add_timing(&cleanup_start.elapsed()); + } + Some(event_log) + } + Err(err) => { + if !matches!(existing_pma.as_ref(), Some(ExistingPmaStatus::Missing)) { + return Err(CrownError::Unknown(format!( + "event log could not be opened while PMA artifacts exist: {err}" + ))); + } + warn!("recovery skipped because event log could not be opened: {err}"); + None + } + }; + + if let Some(event_log) = recovery_event_log.as_mut() { + if let Err(err) = event_log.quick_check() { + return Err(CrownError::Unknown(format!( + "event log quick_check failed during snapshot recovery: {err}" + ))); + } + } + let event_log_max = recovery_event_log + .as_mut() + .map(|event_log| { + event_log + .max_event_num() + .map_err(|err| { + CrownError::Unknown(format!( + "failed to read max event number from event log: {err}" + )) + }) + .map(|max| max.unwrap_or(0)) + }) + .transpose()? + .unwrap_or(0); + + if let Some(ExistingPmaStatus::Valid { path, event_num }) = existing_pma.as_ref() { + if event_log_max == 0 && *event_num > 0 { + if !event_log_policy.preexisting && !event_log_policy.allow_empty_bootstrap { + return Err(CrownError::Unknown(format!( + "refusing to boot PMA event_num={} because the SQLite event log was missing; restore the event log or use explicit bootstrap/discard intent", + event_num + ))); + } + let boot_source = BootSource::Pma { + path: path.clone(), + event_num: *event_num, + }; + match &boot_source { + BootSource::Pma { path, event_num } => { + info!( + "Boot source: PMA path={} event_num={} event_log_max=0 empty_event_log_bootstrap=true", + path.display(), + event_num + ); + } + _ => unreachable!(), + } + return Ok(BootSelection { + checkpoint: None, + pma_open_existing: true, + snapshot_manifest: None, + replay_jobs: Vec::new(), + }); + } + match event_num.cmp(&event_log_max) { + std::cmp::Ordering::Equal => { + let boot_source = BootSource::Pma { + path: path.clone(), + event_num: *event_num, + }; + match &boot_source { + BootSource::Pma { path, event_num } => { + info!( + "Boot source: PMA path={} event_num={}", + path.display(), + event_num + ); + } + _ => unreachable!(), + } + return Ok(BootSelection { + checkpoint: None, + pma_open_existing: true, + snapshot_manifest: None, + replay_jobs: Vec::new(), + }); + } + std::cmp::Ordering::Greater => { + warn!( + "Ignoring PMA at {} event_num={} because it is ahead of event log max {}", + path.display(), + event_num, + event_log_max + ); + } + std::cmp::Ordering::Less => { + warn!( + "Ignoring PMA at {} event_num={} because it is behind event log max {}", + path.display(), + event_num, + event_log_max + ); + } + } + } + + match existing_pma.as_ref() { + Some(ExistingPmaStatus::Invalid { path, reason }) => { + warn!("Ignoring invalid PMA at {}: {}", path.display(), reason); + } + Some(ExistingPmaStatus::Missing) => { + info!("No valid PMA found; checking snapshot and checkpoint recovery"); + } + None => {} + Some(ExistingPmaStatus::Valid { .. }) => {} + } + + if let Some(event_log) = recovery_event_log.as_mut() { + let active_snapshot_id = event_log.active_snapshot_id().map_err(|err| { + CrownError::Unknown(format!( + "failed to read active_snapshot_id from event log: {err}" + )) + })?; + let ready_snapshots = event_log.list_ready_snapshots().map_err(|err| { + CrownError::Unknown(format!( + "failed to list ready snapshots from event log: {err}" + )) + })?; + for snapshot in order_snapshot_candidates(active_snapshot_id, ready_snapshots) { + if snapshot.event_num > event_log_max { + warn!( + "Snapshot {} event_num={} is ahead of event log max {}; marking failed", + snapshot.pma_path, snapshot.event_num, event_log_max + ); + let _ = event_log.mark_snapshot_failed(snapshot.snapshot_id); + continue; + } + let replay_entries = + event_log + .replay_events_after(snapshot.event_num) + .map_err(|err| { + CrownError::Unknown(format!( + "event log continuity check failed from snapshot {} event_num={}: {err}", + snapshot.pma_path, snapshot.event_num + )) + })?; + let replay_reaches_event_log_max = replay_entries + .last() + .map(|entry| entry.event_num) + .unwrap_or(snapshot.event_num) + == event_log_max; + if !replay_reaches_event_log_max { + warn!( + "Snapshot {} event_num={} does not replay to event log max {}; marking failed", + snapshot.pma_path, snapshot.event_num, event_log_max + ); + let _ = event_log.mark_snapshot_failed(snapshot.snapshot_id); + continue; + } + let verify_start = std::time::Instant::now(); + match restore_verified_snapshot(&snapshot, pma_path_0) { + Ok(manifest) => { + metrics.snapshot_verify.add_timing(&verify_start.elapsed()); + if manifest.ker_hash != *expected_ker_hash.as_bytes() { + warn!( + "Snapshot {} kernel hash mismatch; loading snapshot state into current kernel", + snapshot.pma_path + ); + } + let _ = event_log.set_active_snapshot_id(snapshot.snapshot_id); + for stale in [ + pma_path_1.to_path_buf(), + pma_path_0.with_extension("meta"), + pma_path_1.with_extension("meta"), + ] { + if stale.exists() { + let _ = std::fs::remove_file(&stale); + } + } + let boot_source = BootSource::Snapshot { + path: PathBuf::from(&snapshot.pma_path), + event_num: snapshot.event_num, + }; + match &boot_source { + BootSource::Snapshot { path, event_num } => info!( + "Boot source: snapshot path={} event_num={}", + path.display(), + event_num + ), + _ => unreachable!(), + } + return Ok(BootSelection { + checkpoint: None, + pma_open_existing: true, + snapshot_manifest: Some(manifest), + replay_jobs: replay_entries, + }); + } + Err(err) => { + metrics.snapshot_verify_failures.increment(); + warn!( + "Snapshot restore failed for {}: {}; marking snapshot failed", + snapshot.pma_path, err + ); + let _ = event_log.mark_snapshot_failed(snapshot.snapshot_id); + } + } + } + } + + let checkpoint_reader = CheckpointBootstrapReader::::new(jams_dir.to_path_buf()); + let checkpoint_candidate = checkpoint_reader + .load_latest_state_only_with_summary(None) + .await + .map_err(|err| match existing_pma.as_ref() { + Some(ExistingPmaStatus::Invalid { path, reason }) => CrownError::Unknown(format!( + "checkpoint bootstrap inspection failed after PMA validation failed for {}: {} ({err})", + path.display(), + reason + )), + Some(ExistingPmaStatus::Missing) => { + CrownError::Unknown(format!("checkpoint bootstrap inspection failed: {err}")) + } + None => CrownError::Unknown(format!("checkpoint bootstrap inspection failed: {err}")), + Some(ExistingPmaStatus::Valid { .. }) => { + CrownError::Unknown(format!("checkpoint bootstrap inspection failed after PMA was rejected against event log: {err}")) + } + })?; + + if let Some((checkpoint, summary)) = checkpoint_candidate { + let checkpoint_bootstraps_empty_event_log = recovery_event_log.is_some() + && event_log_max == 0 + && event_log_policy.allow_empty_bootstrap; + if recovery_event_log.is_some() + && event_log_max == 0 + && summary.event_num > 0 + && !event_log_policy.allow_empty_bootstrap + { + return Err(CrownError::Unknown(format!( + "refusing to boot checkpoint {} event_num={} because the SQLite event log is empty or missing; use --new/--bootstrap-from-chkjam for explicit checkpoint bootstrap", + summary.path.display(), + summary.event_num + ))); + } + if summary.event_num <= event_log_max || checkpoint_bootstraps_empty_event_log { + let replay_entries = if checkpoint_bootstraps_empty_event_log { + Vec::new() + } else if let Some(event_log) = recovery_event_log.as_mut() { + event_log + .replay_events_after(summary.event_num) + .map_err(|err| { + CrownError::Unknown(format!( + "event log continuity check failed from checkpoint {} event_num={}: {err}", + summary.path.display(), + summary.event_num + )) + })? + } else { + Vec::new() + }; + let replay_reaches_event_log_max = replay_entries + .last() + .map(|entry| entry.event_num) + .unwrap_or(summary.event_num) + == event_log_max + || checkpoint_bootstraps_empty_event_log; + if replay_reaches_event_log_max { + let boot_source = BootSource::Checkpoint { + path: summary.path.clone(), + event_num: summary.event_num, + }; + match &boot_source { + BootSource::Checkpoint { path, event_num } => { + info!( + "Boot source: checkpoint path={} event_num={} event_log_max={} empty_event_log_bootstrap={} checkpoint_cold_state=empty", + path.display(), + event_num, + event_log_max, + checkpoint_bootstraps_empty_event_log + ); + } + _ => unreachable!(), + } + return Ok(BootSelection { + checkpoint: Some(checkpoint), + pma_open_existing: false, + snapshot_manifest: None, + replay_jobs: replay_entries, + }); + } + warn!( + "Checkpoint {} event_num={} did not replay to event log max {}; ignoring", + summary.path.display(), + summary.event_num, + event_log_max + ); + } else { + warn!( + "Ignoring checkpoint {} event_num={} because it is ahead of event log max {}", + summary.path.display(), + summary.event_num, + event_log_max + ); + } + } + + if event_log_max > 0 { + if let Some(event_log) = recovery_event_log.as_mut() { + let replay_entries = event_log.replay_events_after(0).map_err(|err| { + CrownError::Unknown(format!( + "event log continuity check failed from fresh boot event_num=0: {err}" + )) + })?; + let replay_reaches_event_log_max = replay_entries + .last() + .map(|entry| entry.event_num) + .unwrap_or(0) + == event_log_max; + if replay_reaches_event_log_max { + let boot_source = BootSource::Fresh; + match boot_source { + BootSource::Fresh => { + info!("Boot source: fresh kernel state with event-log replay") + } + _ => unreachable!(), + } + return Ok(BootSelection { + checkpoint: None, + pma_open_existing: false, + snapshot_manifest: None, + replay_jobs: replay_entries, + }); + } + } + return Err(CrownError::Unknown(format!( + "no valid boot base can recover event log max {event_log_max}" + ))); + } + + let boot_source = BootSource::Fresh; + match boot_source { + BootSource::Fresh => info!("Boot source: fresh kernel state"), + _ => unreachable!(), + } + Ok(BootSelection { + checkpoint: None, + pma_open_existing: false, + snapshot_manifest: None, + replay_jobs: Vec::new(), + }) +} + pub fn default_boot_cli(new: bool) -> Cli { Cli { - save_interval: Some(DEFAULT_SAVE_INTERVAL), + gc_interval: Some(DEFAULT_GC_INTERVAL_SECS), + rotating_snapshot_interval_event_time: Some( + DEFAULT_ROTATING_SNAPSHOT_INTERVAL_EVENT_TIME_SECS, + ), + ephemeral: false, new, trace_opts: Default::default(), color: ColorChoice::Auto, state_jam: None, + bootstrap_from_chkjam: None, export_state_jam: None, stack_size: NockStackSize::Normal, + pma_initial_size: None, + pma_reserved_size: None, + data_dir: None, + event_log_path: None, + disable_fsync: false, + } +} + +pub fn ephemeral_test_boot_cli(new: bool) -> Cli { + let mut cli = default_boot_cli(new); + cli.ephemeral = true; + cli.gc_interval = None; + cli.rotating_snapshot_interval_event_time = None; + cli.disable_fsync = true; + cli +} + +fn dir_has_entries(path: &std::path::Path) -> std::io::Result { + if !path.exists() { + return Ok(false); } + + if !path.is_dir() { + return Ok(true); + } + + let mut entries = std::fs::read_dir(path)?; + Ok(entries.next().transpose()?.is_some()) +} + +fn event_log_sidecar_paths(event_log_path: &std::path::Path) -> [PathBuf; 3] { + [ + event_log_path.to_path_buf(), + PathBuf::from(format!("{}-wal", event_log_path.display())), + PathBuf::from(format!("{}-shm", event_log_path.display())), + ] } /// A minimal event formatter for development mode @@ -356,9 +2461,9 @@ pub async fn setup( cli: Cli, hot_state: &[HotEntry], name: &str, - data_dir: Option, + data_root: Option, ) -> Result, Box> { - let result = setup_(jam, cli, hot_state, name, data_dir).await?; + let result = setup_(jam, cli, hot_state, name, data_root).await?; match result { SetupResult::App(app) => Ok(app), SetupResult::ExportedState => { @@ -370,122 +2475,387 @@ pub async fn setup( pub async fn setup_( jam: &[u8], - cli: Cli, + mut cli: Cli, hot_state: &[HotEntry], name: &str, - data_dir: Option, + data_root: Option, ) -> Result, Box> { + if cli.state_jam.is_some() && !cli.new { + return Err(Box::new(std::io::Error::new( + std::io::ErrorKind::InvalidInput, + "--state-jam requires --new", + ))); + } + if cli.ephemeral && cli.bootstrap_from_chkjam.is_some() { + return Err(Box::new(std::io::Error::new( + std::io::ErrorKind::InvalidInput, + "--bootstrap-from-chkjam requires durable PMA boot", + ))); + } + if cli.ephemeral { + cli.gc_interval = None; + cli.rotating_snapshot_interval_event_time = None; + cli.disable_fsync = true; + } + durability::set_fsync_disabled(cli.disable_fsync); let nock_test_jets_env = std::env::var("NOCK_TEST_JETS").unwrap_or_default(); let test_jets = parse_test_jets(nock_test_jets_env.as_str()); - let data_dir = if let Some(data_path) = data_dir.clone() { - data_path.join(name) + let ephemeral = cli.ephemeral; + let data_dir = if let Some(explicit_dir) = cli.data_dir.clone() { + explicit_dir + } else if let Some(root) = data_root { + root.join(name) } else { default_data_dir(name) }; let pma_dir = data_dir.join("pma"); let jams_dir = data_dir.join("checkpoints"); + let event_log_path = cli + .event_log_path + .clone() + .unwrap_or_else(|| data_dir.join("event-log.sqlite3")); + + if cli.new && !ephemeral { + if dir_has_entries(&data_dir)? { + warn!( + path = %data_dir.display(), + "Refusing --new because the target data directory already contains data or setup artifacts; use a fresh path or remove it manually" + ); + return Err(Box::new(std::io::Error::new( + std::io::ErrorKind::AlreadyExists, + format!( + "--new requires an empty data directory, found existing contents at {}", + data_dir.display() + ), + ))); + } + + for path in event_log_sidecar_paths(&event_log_path) { + if path.exists() { + warn!( + path = %path.display(), + "Refusing --new because the target event-log path already exists; use a fresh path or remove it manually" + ); + return Err(Box::new(std::io::Error::new( + std::io::ErrorKind::AlreadyExists, + format!( + "--new requires an unused event-log path, found existing file at {}", + path.display() + ), + ))); + } + } + } - if !jams_dir.exists() { + if !ephemeral && !jams_dir.exists() { std::fs::create_dir_all(&jams_dir)?; debug!("Created jams directory: {:?}", jams_dir); } - if pma_dir.exists() { - std::fs::remove_dir_all(&pma_dir)?; - debug!("Deleted existing pma directory: {:?}", pma_dir); + if !ephemeral && !pma_dir.exists() { + std::fs::create_dir_all(&pma_dir)?; + debug!("Created pma directory: {:?}", pma_dir); } - if cli.new && jams_dir.exists() { - std::fs::remove_dir_all(&jams_dir)?; - debug!("Deleted existing checkpoint directory: {:?}", jams_dir); + if !ephemeral { + if let Some(chkjam_path) = cli.bootstrap_from_chkjam.as_deref() { + let src = PathBuf::from(chkjam_path); + if !src.exists() { + return Err(format!("bootstrap chkjam not found: {}", src.display()).into()); + } + let dst = jams_dir.join("0.chkjam"); + std::fs::copy(&src, &dst)?; + let dst_alt = jams_dir.join("1.chkjam"); + if dst_alt.exists() { + std::fs::remove_file(&dst_alt)?; + } + info!( + "Bootstrapping from checkpoint: {} -> {}", + src.display(), + dst.display() + ); + } } info!("kernel: starting"); debug!("kernel: pma directory: {:?}", pma_dir); debug!("kernel: snapshots directory: {:?}", jams_dir); + debug!("kernel: event-log path: {:?}", event_log_path); info!("NockApp boot cli: {:?}", cli); - let save_interval = cli - .normalized_save_interval() - .map(std::time::Duration::from_millis); - - let kernel_f = async |checkpoint| { - let kernel: Kernel = match cli.stack_size { + if cli.disable_fsync { + warn!("All fsync/fdatasync durability calls are disabled"); + } + let gc_interval = if ephemeral { + None + } else { + cli.normalized_gc_interval() + }; + let rotating_snapshot_interval_event_time = if ephemeral { + None + } else { + cli.normalized_rotating_snapshot_interval_event_time() + }; + if let Some(interval) = gc_interval { + info!("PMA GC interval duration: {:?}", interval); + } else { + info!("PMA GC interval disabled"); + } + if let Some(interval) = rotating_snapshot_interval_event_time { + info!("Rotating snapshot interval event time: {:?}", interval); + } else { + info!("Rotating snapshots disabled"); + } + if ephemeral { + info!("Ephemeral NockStack active; PMA durability, event log, snapshots, and GC disabled"); + } else { + info!("PMA durability active"); + } + let pma_path_0 = pma_dir.join("0.pma"); + let pma_path_1 = pma_dir.join("1.pma"); + let stack_size = cli.stack_size; + let pma_initial_words = pma_initial_words_for_boot(cli.pma_initial_size, jam.len()); + let pma_reserved_words = Some( + cli.pma_reserved_size + .map(PmaSize::words) + .unwrap_or_else(|| Pma::default_reserved_words_for_capacity(pma_initial_words)) + .max(pma_initial_words), + ); + let trace_opts = cli.trace_opts.clone(); + let event_log_path_for_kernel = event_log_path.clone(); + let event_log_preexisting = event_log_sidecar_paths(&event_log_path) + .iter() + .any(|path| path.exists()); + let allow_empty_event_log_bootstrap = cli.new || cli.bootstrap_from_chkjam.is_some(); + if !ephemeral { + info!( + kernel_jam_bytes = jam.len(), + pma_initial_words, + pma_reserved_words = ?pma_reserved_words, + "PMA sizing selected" + ); + } + let kernel_f = move |metrics: Arc| async move { + let boot_selection = if ephemeral { + BootSelection { + checkpoint: None, + pma_open_existing: false, + snapshot_manifest: None, + replay_jobs: Vec::new(), + } + } else { + select_boot_state::( + &jams_dir, + jam, + &event_log_path, + &pma_path_0, + &pma_path_1, + metrics.clone(), + BootEventLogPolicy { + preexisting: event_log_preexisting, + allow_empty_bootstrap: allow_empty_event_log_bootstrap, + }, + ) + .await? + }; + let mut checkpoint = boot_selection.checkpoint; + let pma_open_existing = boot_selection.pma_open_existing; + let snapshot_manifest = boot_selection.snapshot_manifest.clone(); + let replay_jobs = boot_selection.replay_jobs; + let pma_config = |_nock_stack_words| { + if ephemeral { + None + } else { + Some(PmaConfig { + path_0: pma_path_0.clone(), + path_1: pma_path_1.clone(), + words: pma_initial_words, + reserved_words: pma_reserved_words, + open_existing: pma_open_existing, + create_snapshots: true, + rotating_snapshot_interval_event_time, + restore_manifest: snapshot_manifest.clone(), + gc_interval, + }) + } + }; + let event_log_config = if ephemeral { + None + } else { + Some(EventLogConfig { + path: event_log_path_for_kernel.clone(), + }) + }; + let kernel: Kernel = match stack_size { NockStackSize::Tiny => { - Kernel::load_with_hot_state_tiny( - jam, checkpoint, hot_state, test_jets, cli.trace_opts, + Kernel::load_with_hot_state_tiny_with_event_log( + jam, + checkpoint.take(), + hot_state, + test_jets, + trace_opts.clone(), + pma_config(NOCK_STACK_SIZE_TINY), + event_log_config.clone(), ) .await? } NockStackSize::Small => { - Kernel::load_with_hot_state_small( - jam, checkpoint, hot_state, test_jets, cli.trace_opts, + Kernel::load_with_hot_state_small_with_event_log( + jam, + checkpoint.take(), + hot_state, + test_jets, + trace_opts.clone(), + pma_config(NOCK_STACK_SIZE_SMALL), + event_log_config.clone(), ) .await? } NockStackSize::Normal => { - Kernel::load_with_hot_state(jam, checkpoint, hot_state, test_jets, cli.trace_opts) - .await? + Kernel::load_with_hot_state_with_event_log( + jam, + checkpoint.take(), + hot_state, + test_jets, + trace_opts.clone(), + pma_config(NOCK_STACK_SIZE), + event_log_config.clone(), + ) + .await? } NockStackSize::Medium => { - Kernel::load_with_hot_state_medium( - jam, checkpoint, hot_state, test_jets, cli.trace_opts, + Kernel::load_with_hot_state_medium_with_event_log( + jam, + checkpoint.take(), + hot_state, + test_jets, + trace_opts.clone(), + pma_config(NOCK_STACK_SIZE_MEDIUM), + event_log_config.clone(), ) .await? } NockStackSize::Large => { - Kernel::load_with_hot_state_large( - jam, checkpoint, hot_state, test_jets, cli.trace_opts, + Kernel::load_with_hot_state_large_with_event_log( + jam, + checkpoint.take(), + hot_state, + test_jets, + trace_opts.clone(), + pma_config(NOCK_STACK_SIZE_LARGE), + event_log_config.clone(), ) .await? } NockStackSize::Huge => { - Kernel::load_with_hot_state_huge( - jam, checkpoint, hot_state, test_jets, cli.trace_opts, + Kernel::load_with_hot_state_huge_with_event_log( + jam, + checkpoint.take(), + hot_state, + test_jets, + trace_opts, + pma_config(NOCK_STACK_SIZE_HUGE), + event_log_config, ) .await? } }; + if !replay_jobs.is_empty() { + let replay_start = std::time::Instant::now(); + let replay_job_count = replay_jobs.len(); + info!( + jobs = replay_job_count, + "event replay after snapshot restore start" + ); + if let Err(err) = kernel.replay_event_jobs(replay_jobs).await { + metrics.replay_failures.increment(); + warn!( + jobs = replay_job_count, + elapsed_ms = replay_start.elapsed().as_secs_f64() * 1000.0, + error = %err, + "event replay after snapshot restore failed" + ); + return Err(err); + } + for _ in 0..replay_job_count { + metrics.replay_events.increment(); + } + let elapsed = replay_start.elapsed(); + metrics.replay_apply.add_timing(&elapsed); + info!( + jobs = replay_job_count, + elapsed_ms = elapsed.as_secs_f64() * 1000.0, + "event replay after snapshot restore done" + ); + } let res: Result, CrownError> = Ok(kernel); res }; - let app: NockApp = NockApp::new(kernel_f, &jams_dir, save_interval).await?; + let app: NockApp = NockApp::new(kernel_f).await?; if let Some(export_path) = cli.export_state_jam.clone() { - export_kernel_state(&app.kernel, &export_path).await?; + export_kernel_state::(&app.kernel, &export_path).await?; return Ok(SetupResult::ExportedState); } if let Some(import_path) = cli.state_jam.clone() { - import_kernel_state(&app.kernel, &import_path).await?; + import_kernel_state::(&app.kernel, &import_path).await?; } Ok(SetupResult::App(app)) } /// Exports the kernel state to a jam file at the specified path -async fn export_kernel_state( +async fn export_kernel_state( kernel: &Kernel, export_path: &str, ) -> Result<(), Box> { + info!( + path = %export_path, + jammer = std::any::type_name::(), + "state jam export start" + ); + if let Some(parent) = Path::new(export_path).parent() { + if !parent.as_os_str().is_empty() { + fs::create_dir_all(parent).await?; + } + } let kernel_state = kernel.export().await?; - let exported_state = ExportedState::from_loadstate(kernel_state); + let exported_state = ExportedState::from_loadstate::(kernel_state); let state_bytes = exported_state.encode()?; + info!( + path = %export_path, + bytes = state_bytes.len(), + "state jam write start" + ); fs::write(export_path, state_bytes).await?; - info!("Successfully exported kernel state to: {:?}", export_path); + info!( + path = %export_path, + jammer = std::any::type_name::(), + "state jam export done" + ); Ok(()) } /// Imports the kernel state from a jam file at the specified path -async fn import_kernel_state( +async fn import_kernel_state( kernel: &Kernel, import_path: &str, ) -> Result<(), Box> { + info!( + path = %import_path, + jammer = std::any::type_name::(), + "state jam import start" + ); let state_bytes = fs::read(import_path).await?; let exported_state = ExportedState::decode(&state_bytes)?; - let kernel_state = exported_state.to_loadstate()?; + let kernel_state = exported_state.to_loadstate::()?; kernel.import(kernel_state).await?; - info!("Successfully imported kernel state from: {:?}", import_path); + info!( + path = %import_path, + jammer = std::any::type_name::(), + "state jam import done" + ); Ok(()) } diff --git a/crates/nockapp/src/kernel/form.rs b/crates/nockapp/src/kernel/form.rs index 5ff3227ef..94ee2da48 100644 --- a/crates/nockapp/src/kernel/form.rs +++ b/crates/nockapp/src/kernel/form.rs @@ -1,11 +1,15 @@ #![allow(dead_code)] #![allow(clippy::items_after_test_module)] use std::any::Any; +use std::fs; use std::future::Future; +use std::panic::{catch_unwind, AssertUnwindSafe}; +use std::path::{Path, PathBuf}; use std::sync::atomic::{AtomicBool, AtomicU64, Ordering}; -use std::sync::Arc; -use std::time::Instant; +use std::sync::{Arc, Mutex}; +use std::time::{Instant, SystemTime, UNIX_EPOCH}; +use bincode::{config, Decode, Encode}; use blake3::{Hash, Hasher}; use byteorder::{LittleEndian, WriteBytesExt}; use nockvm::hamt::Hamt; @@ -13,24 +17,30 @@ use nockvm::interpreter::{self, interpret, Error, Mote, NockCancelToken}; use nockvm::jets::cold::{Cold, Nounable}; use nockvm::jets::hot::{HotEntry, URBIT_HOT_STATE}; use nockvm::jets::nock::util::mook; -use nockvm::mem::NockStack; +use nockvm::mem::{AllocationError, NewStackError, NockStack}; use nockvm::mug::met3_usize; -use nockvm::noun::{Atom, Cell, DirectAtom, IndirectAtom, Noun, Slots, D, T}; +use nockvm::noun::{Atom, Cell, DirectAtom, IndirectAtom, Noun, D, T}; +use nockvm::offset::PmaOffsetWords; +use nockvm::pma::{Pma, PmaCopy, PmaCopyFrom}; use nockvm::trace::{path_to_cord, write_serf_trace_safe}; use nockvm_macros::tas; use tokio::sync::{mpsc, oneshot}; use tokio::time::Duration; -use tracing::{debug, warn}; +use tracing::{debug, error, info, warn}; +use crate::event_log::{EventLog, EventLogConfig, EventLogEntry, ReplayLogEntry}; use crate::kernel::boot::TraceOpts; use crate::metrics::NockAppMetrics; use crate::nockapp::wire::{wire_to_noun, WireRepr}; -use crate::noun::slab::NounSlab; +use crate::noun::slab::{Jammer, NockJammer, NounSlab}; use crate::noun::slam; use crate::save::SaveableCheckpoint; +use crate::snapshot::{ + maybe_create_epoch_snapshot, maybe_create_rotating_snapshot, SnapshotManifest, +}; use crate::utils::{ - create_context, current_da, NOCK_STACK_SIZE, NOCK_STACK_SIZE_HUGE, NOCK_STACK_SIZE_LARGE, - NOCK_STACK_SIZE_MEDIUM, NOCK_STACK_SIZE_SMALL, NOCK_STACK_SIZE_TINY, + create_context, current_da, durability, NOCK_STACK_SIZE, NOCK_STACK_SIZE_HUGE, + NOCK_STACK_SIZE_LARGE, NOCK_STACK_SIZE_MEDIUM, NOCK_STACK_SIZE_SMALL, NOCK_STACK_SIZE_TINY, }; use crate::{AtomExt, CrownError, IndirectAtomExt, NounExt, Result, ToBytesExt}; @@ -41,6 +51,185 @@ const POKE_AXIS: u64 = 23; const SERF_FINISHED_INTERVAL: Duration = Duration::from_millis(100); const SERF_THREAD_STACK_SIZE: usize = 256 * 1024 * 1024; // 8MB +const REPLAY_PRESERVE_BATCH: usize = 64; +const PMA_GC_DROP_ALLOCATED_PREFIX_NUMERATOR: usize = 1; +const PMA_GC_DROP_ALLOCATED_PREFIX_DENOMINATOR: usize = 1; +const PMA_EVENT_PREFLIGHT_MIN_FREE_WORDS: usize = 4096; +const PMA_EVENT_PREFLIGHT_FREE_WORDS_ENV: &str = "NOCK_PMA_EVENT_PREFLIGHT_FREE_WORDS"; +const NOCK_STACK_FREE_GAP_TRIM_ENV: &str = "NOCK_STACK_FREE_GAP_TRIM"; +const NOCK_STACK_FREE_GAP_TRIM_THRESHOLD_BYTES: usize = 512 * 1024 * 1024; +const NOCK_STACK_FREE_GAP_TRIM_THRESHOLD_WORDS: usize = + NOCK_STACK_FREE_GAP_TRIM_THRESHOLD_BYTES / std::mem::size_of::(); + +fn duration_ms(elapsed: Duration) -> f64 { + elapsed.as_secs_f64() * 1000.0 +} + +fn words_to_mib(words: usize) -> f64 { + (words as f64 * 8.0) / (1024.0 * 1024.0) +} + +fn bytes_to_mib(bytes: usize) -> f64 { + bytes as f64 / (1024.0 * 1024.0) +} + +fn pma_event_preflight_free_words(pma: &Pma) -> usize { + if let Ok(value) = std::env::var(PMA_EVENT_PREFLIGHT_FREE_WORDS_ENV) { + if let Ok(words) = value.parse::() { + return words; + } + } + PMA_EVENT_PREFLIGHT_MIN_FREE_WORDS.max(pma.size_words() / 100) +} + +fn stack_free_gap_trim_enabled() -> bool { + match std::env::var(NOCK_STACK_FREE_GAP_TRIM_ENV) { + Ok(value) => !matches!( + value.trim().to_ascii_lowercase().as_str(), + "0" | "false" | "off" | "no" | "disable" | "disabled" + ), + Err(std::env::VarError::NotPresent) => true, + Err(std::env::VarError::NotUnicode(_)) => true, + } +} + +fn log_preserve_component(event_num: u64, component: &'static str, elapsed: Duration) { + debug!( + event_num, + component, + elapsed_ms = duration_ms(elapsed), + "preserve component done" + ); +} + +fn log_pma_preserve_component(event_num: u64, component: &'static str, segment: PmaCopySegment) { + debug!( + event_num, + component, + elapsed_ms = duration_ms(segment.elapsed), + alloc_words = segment.alloc_words, + alloc_mib = words_to_mib(segment.alloc_words), + "preserve component done" + ); +} + +#[derive(Clone, Debug)] +struct SerfInitDiagnostics { + phase: &'static str, + stack_size_words: usize, + checkpoint_event_num: Option, + checkpoint_ker_hash: Option, + current_ker_hash: Option, +} + +impl SerfInitDiagnostics { + fn new(stack_size_words: usize) -> Self { + Self { + phase: "starting serf initialization", + stack_size_words, + checkpoint_event_num: None, + checkpoint_ker_hash: None, + current_ker_hash: None, + } + } + + fn with_phase(&self, phase: &'static str) -> Self { + let mut diagnostics = self.clone(); + diagnostics.phase = phase; + diagnostics + } + + fn with_current_ker_hash(&self, current_ker_hash: Hash) -> Self { + let mut diagnostics = self.clone(); + diagnostics.current_ker_hash = Some(current_ker_hash); + diagnostics + } + + fn with_checkpoint(&self, saveable: &SaveableCheckpoint) -> Self { + let mut diagnostics = self.clone(); + diagnostics.checkpoint_event_num = Some(saveable.event_num); + diagnostics.checkpoint_ker_hash = Some(saveable.ker_hash); + diagnostics + } +} + +fn run_serf_init_phase( + diagnostics: &SerfInitDiagnostics, + phase: &'static str, + f: F, +) -> Result +where + F: FnOnce() -> T, +{ + catch_unwind(AssertUnwindSafe(f)) + .map_err(|payload| serf_init_panic_to_error(diagnostics.with_phase(phase), payload)) +} + +fn serf_init_panic_to_error( + diagnostics: SerfInitDiagnostics, + payload: Box, +) -> CrownError { + if let Some(err) = payload.downcast_ref::() { + return CrownError::SerfInitAllocationError(serf_init_allocation_message( + &diagnostics, + &err.to_string(), + )); + } + if let Some(err) = payload.downcast_ref::() { + return CrownError::SerfInitAllocationError(serf_init_allocation_message( + &diagnostics, + &err.to_string(), + )); + } + + CrownError::SerfInitPanic(format!( + "Serf initialization panicked during {}: {}. Preserve the checkpoint or state jam, rerun with RUST_BACKTRACE=1, and report this as a bug.", + diagnostics.phase, + panic_payload_message(payload.as_ref()) + )) +} + +fn serf_init_allocation_message(diagnostics: &SerfInitDiagnostics, err: &str) -> String { + let stack_bytes = diagnostics + .stack_size_words + .saturating_mul(std::mem::size_of::()); + let stack_mib = stack_bytes / (1024 * 1024); + let checkpoint = match ( + diagnostics.checkpoint_event_num, diagnostics.checkpoint_ker_hash, + ) { + (Some(event_num), Some(ker_hash)) => { + format!( + "checkpoint event_num={event_num} checkpoint_ker_hash={}", + ker_hash.to_hex() + ) + } + _ => "no checkpoint metadata was available".to_string(), + }; + let current = diagnostics + .current_ker_hash + .map(|ker_hash| ker_hash.to_hex().to_string()) + .unwrap_or_else(|| "unknown".to_string()); + + format!( + "Nock stack exhausted during Serf initialization phase `{}`. Configured stack size: {} words (~{} MiB). {} current_ker_hash={}. Try restarting this peer with a larger `--stack-size` (`large` or `huge`). If it still fails with `huge`, restore a checkpoint or state jam from a synced peer and preserve the failing artifact for debugging. Allocation error: {}", + diagnostics.phase, + diagnostics.stack_size_words, + stack_mib, + checkpoint, + current, + err + ) +} + +fn panic_payload_message(payload: &(dyn Any + Send)) -> String { + if let Some(message) = payload.downcast_ref::<&'static str>() { + (*message).to_string() + } else if let Some(message) = payload.downcast_ref::() { + message.clone() + } else { + "unknown panic payload".to_string() + } +} pub struct LoadState { pub ker_hash: Hash, @@ -48,8 +237,393 @@ pub struct LoadState { pub kernel_state: NounSlab, } +#[derive(Clone, Debug)] +pub struct PmaConfig { + pub path_0: PathBuf, + pub path_1: PathBuf, + pub words: usize, + pub reserved_words: Option, + pub open_existing: bool, + pub create_snapshots: bool, + pub rotating_snapshot_interval_event_time: Option, + pub(crate) restore_manifest: Option, + pub gc_interval: Option, +} + +#[derive(Clone, Copy, Debug)] +enum PmaSlab { + Slab0, + Slab1, +} + +impl PmaSlab { + fn next(self) -> Self { + match self { + PmaSlab::Slab0 => PmaSlab::Slab1, + PmaSlab::Slab1 => PmaSlab::Slab0, + } + } +} + +#[derive(Clone, Debug)] +struct PmaSlabPaths { + path_0: PathBuf, + path_1: PathBuf, +} + +impl PmaSlabPaths { + fn path(&self, slab: PmaSlab) -> &PathBuf { + match slab { + PmaSlab::Slab0 => &self.path_0, + PmaSlab::Slab1 => &self.path_1, + } + } +} + +const PMA_PERSIST_MAGIC: u64 = u64::from_le_bytes(*b"PMAPERS1"); +const PMA_PERSIST_VERSION: u32 = 5; +const PMA_PERSIST_VERSION_V4: u32 = 4; +const SNAPSHOT_UNUSED_COLD_OFFSET: PmaOffsetWords = PmaOffsetWords::from_words(0); + +#[derive(Clone, Encode, Decode, Debug)] +struct PmaPersistMetadata { + magic: u64, + version: u32, + #[bincode(with_serde)] + ker_hash: Hash, + event_num: u64, + kernel_state_raw: u64, + pma_reserved_words: u64, + #[bincode(with_serde)] + checksum: Hash, +} + +#[derive(Clone, Encode, Decode, Debug)] +struct PmaPersistMetadataV4 { + magic: u64, + version: u32, + #[bincode(with_serde)] + ker_hash: Hash, + event_num: u64, + kernel_state_raw: u64, + #[bincode(with_serde)] + checksum: Hash, +} + +impl PmaPersistMetadata { + fn new(ker_hash: Hash, event_num: u64, kernel_state_raw: u64, pma_reserved_words: u64) -> Self { + let checksum = Self::checksum(ker_hash, event_num, kernel_state_raw, pma_reserved_words); + Self { + magic: PMA_PERSIST_MAGIC, + version: PMA_PERSIST_VERSION, + ker_hash, + event_num, + kernel_state_raw, + pma_reserved_words, + checksum, + } + } + + fn checksum( + ker_hash: Hash, + event_num: u64, + kernel_state_raw: u64, + pma_reserved_words: u64, + ) -> Hash { + let mut hasher = Hasher::new(); + hasher.update(ker_hash.as_bytes()); + hasher.update(&event_num.to_le_bytes()); + hasher.update(&kernel_state_raw.to_le_bytes()); + hasher.update(&pma_reserved_words.to_le_bytes()); + hasher.finalize() + } + + fn validate(&self) -> bool { + if self.magic != PMA_PERSIST_MAGIC || self.version != PMA_PERSIST_VERSION { + return false; + } + self.checksum + == Self::checksum( + self.ker_hash, self.event_num, self.kernel_state_raw, self.pma_reserved_words, + ) + } + + fn load_from_path(path: &Path) -> Option { + let bytes = fs::read(path).ok()?; + if let Ok((meta, _)) = + bincode::decode_from_slice::(&bytes, config::standard()) + { + if meta.validate() { + return Some(meta); + } + } + let (legacy, _) = + bincode::decode_from_slice::( + &bytes, + config::standard(), + ) + .ok()?; + legacy.validate().then(|| legacy.into_current()) + } + + fn save_to_path(&self, path: &Path) -> std::io::Result<()> { + let bytes = + bincode::encode_to_vec(self, config::standard()).map_err(std::io::Error::other)?; + durability::write_atomic(path, &bytes, "pma_meta_write")?; + Ok(()) + } +} + +impl PmaPersistMetadataV4 { + fn checksum(ker_hash: Hash, event_num: u64, kernel_state_raw: u64) -> Hash { + let mut hasher = Hasher::new(); + hasher.update(ker_hash.as_bytes()); + hasher.update(&event_num.to_le_bytes()); + hasher.update(&kernel_state_raw.to_le_bytes()); + hasher.finalize() + } + + fn validate(&self) -> bool { + if self.magic != PMA_PERSIST_MAGIC || self.version != PMA_PERSIST_VERSION_V4 { + return false; + } + self.checksum == Self::checksum(self.ker_hash, self.event_num, self.kernel_state_raw) + } + + fn into_current(self) -> PmaPersistMetadata { + PmaPersistMetadata::new(self.ker_hash, self.event_num, self.kernel_state_raw, 0) + } +} + +fn pma_meta_path(path: &Path) -> PathBuf { + path.with_extension("meta") +} + +// Slab selection is intentionally kernel-hash-agnostic: a kernel upgrade +// (new bytecode -> new BLAKE3 hash) must not hide the genuinely-current slab. +// Gating this on `ker_hash` caused both slabs to report `None` after an +// upgrade, so selection fell through to its `Slab0` default and then reported +// the (possibly stale) slab-0 metadata as "missing or invalid" even when the +// live state lived in slab 1. Structural integrity is still guarded by the +// metadata checksum (`PmaPersistMetadata::validate`) and the PMA trailer +// (`Pma::open`); cross-kernel state compatibility is the kernel's own +// Hoon-level `+load` responsibility, consistent with the checkpoint and +// snapshot restore paths. +fn pma_meta_status(path: &Path) -> Option<(u64, SystemTime)> { + let meta_path = pma_meta_path(path); + let meta = PmaPersistMetadata::load_from_path(&meta_path)?; + let modified = std::fs::metadata(&meta_path) + .and_then(|meta| meta.modified()) + .unwrap_or(SystemTime::UNIX_EPOCH); + Some((meta.event_num, modified)) +} + +fn select_active_pma_slab(paths: &PmaSlabPaths) -> PmaSlab { + let status_0 = pma_meta_status(&paths.path_0); + let status_1 = pma_meta_status(&paths.path_1); + match (status_0, status_1) { + (Some((event_0, mod_0)), Some((event_1, mod_1))) => { + if event_0 > event_1 { + PmaSlab::Slab0 + } else if event_1 > event_0 || mod_1 > mod_0 { + PmaSlab::Slab1 + } else { + PmaSlab::Slab0 + } + } + (Some(_), None) => PmaSlab::Slab0, + (None, Some(_)) => PmaSlab::Slab1, + (None, None) => PmaSlab::Slab0, + } +} + +#[derive(Clone, Debug)] +pub(crate) enum ExistingPmaStatus { + Missing, + Valid { path: PathBuf, event_num: u64 }, + Invalid { path: PathBuf, reason: String }, +} + +pub(crate) fn inspect_existing_pma( + path_0: &Path, + path_1: &Path, + kernel_bytes: &[u8], +) -> ExistingPmaStatus { + let mut hasher = Hasher::new(); + hasher.update(kernel_bytes); + let ker_hash = hasher.finalize(); + let paths = PmaSlabPaths { + path_0: path_0.to_path_buf(), + path_1: path_1.to_path_buf(), + }; + let active = select_active_pma_slab(&paths); + let active_path = paths.path(active).clone(); + let inactive_path = paths.path(active.next()).clone(); + let active_meta_path = pma_meta_path(&active_path); + let inactive_meta_path = pma_meta_path(&inactive_path); + let any_pma_artifacts = active_path.exists() + || inactive_path.exists() + || active_meta_path.exists() + || inactive_meta_path.exists(); + + if !active_path.exists() { + return if any_pma_artifacts { + ExistingPmaStatus::Invalid { + path: active_path, + reason: "selected PMA slab is missing".to_string(), + } + } else { + ExistingPmaStatus::Missing + }; + } + + let Some(meta) = PmaPersistMetadata::load_from_path(&active_meta_path) else { + return ExistingPmaStatus::Invalid { + path: active_path, + reason: format!( + "missing or invalid PMA metadata at {}", + active_meta_path.display() + ), + }; + }; + + // A kernel-hash mismatch does NOT invalidate the PMA. The persisted kernel + // state is a kernel-agnostic noun and is loaded into the freshly-cued + // current kernel, which performs its own Hoon-level state migration. This + // mirrors the checkpoint (`form.rs` "loading checkpoint state into current + // kernel") and snapshot (`boot.rs` "loading snapshot state into current + // kernel") restore paths. Forcing a full event-log replay on every kernel + // upgrade is the operational cost we are eliminating here. + if meta.ker_hash != ker_hash { + warn!( + metadata = %meta.ker_hash, + kernel = %ker_hash, + "PMA kernel hash mismatch; loading existing PMA state into current kernel" + ); + } + + match Pma::read_file_metadata(&active_path) { + Ok(_) => ExistingPmaStatus::Valid { + path: active_path, + event_num: meta.event_num, + }, + Err(err) => ExistingPmaStatus::Invalid { + path: active_path, + reason: format!("failed to open PMA: {err}"), + }, + } +} + +struct PmaGcState { + paths: PmaSlabPaths, + active: PmaSlab, + interval: Duration, + last_gc: Instant, + words: usize, + reserved_words: Option, +} + +impl PmaGcState { + fn new( + paths: PmaSlabPaths, + active: PmaSlab, + interval: Duration, + words: usize, + reserved_words: Option, + ) -> Self { + Self { + paths, + active, + interval, + last_gc: Instant::now(), + words, + reserved_words, + } + } + + fn active_path(&self) -> &PathBuf { + self.paths.path(self.active) + } + + fn inactive_path(&self) -> &PathBuf { + self.paths.path(self.active.next()) + } + + fn should_gc(&self) -> bool { + self.last_gc.elapsed() >= self.interval + } + + fn mark_gc_completed(&mut self) { + self.active = self.active.next(); + self.last_gc = Instant::now(); + } +} + +#[derive(Clone, Copy, Debug, Default)] +pub struct PmaCopySegment { + pub elapsed: Duration, + pub alloc_words: usize, +} + +#[derive(Clone, Copy, Debug, Default)] +pub struct PmaCopyDetail { + pub warm: PmaCopySegment, + pub test_jets: PmaCopySegment, + pub hot: PmaCopySegment, + pub cache: PmaCopySegment, + pub cold: PmaCopySegment, + pub arvo: PmaCopySegment, +} + +#[derive(Clone, Copy, Debug)] +pub struct PmaTimingSample { + pub event: Duration, + pub pma_copy: Duration, + pub detail: Option, +} + +#[derive(Default)] +pub(crate) struct PmaTiming { + samples: Mutex>, +} + +impl PmaTiming { + pub(crate) fn record( + &self, + event: Duration, + pma_copy: Duration, + detail: Option, + ) { + let mut samples = self.samples.lock().unwrap_or_else(|err| err.into_inner()); + samples.push(PmaTimingSample { + event, + pma_copy, + detail, + }); + } + + pub(crate) fn take_samples(&self) -> Vec { + let mut samples = self.samples.lock().unwrap_or_else(|err| err.into_inner()); + std::mem::take(&mut *samples) + } +} + +#[derive(Debug, Clone)] +struct AcceptedEventMetadata { + wire_source: String, + wire_version: i64, + wire_tags_json: String, + cause_hash: Vec, + created_at_ms: i64, +} + +struct AcceptedPoke { + effects: Noun, + durable_event: Option, +} + // Actions to request of the serf thread -pub enum SerfAction { +pub(crate) enum SerfAction { // Make a CheckPoint Checkpoint { result: oneshot::Sender, @@ -74,6 +648,10 @@ pub enum SerfAction { ovo: NounSlab, result: oneshot::Sender>, }, + Replay { + jobs: Vec, + result: oneshot::Sender>, + }, // Run a poke // // TODO: send back the event number after each poke @@ -88,8 +666,10 @@ pub enum SerfAction { metrics: Arc, result: oneshot::Sender<()>, }, - // Stop the loop - Stop, + // Flush durable PMA state and stop the loop + Stop { + result: oneshot::Sender>, + }, } pub struct SerfThread { @@ -98,6 +678,7 @@ pub struct SerfThread { pub cancel_token: NockCancelToken, inhibit: Arc, pub event_number: Arc, + pub(crate) pma_timing: Option>, } impl SerfThread { @@ -106,39 +687,190 @@ impl SerfThread { checkpoint: Option, constant_hot_state: Vec, nock_stack_size: usize, + pma: Option, + test_jets: Vec, + trace: TraceOpts, + ) -> Result { + Self::new_with_event_log( + kernel_bytes, checkpoint, constant_hot_state, nock_stack_size, pma, None, test_jets, + trace, + ) + .await + } + + #[allow(clippy::too_many_arguments)] + pub(crate) async fn new_with_event_log( + kernel_bytes: Vec, + checkpoint: Option, + constant_hot_state: Vec, + nock_stack_size: usize, + pma: Option, + event_log: Option, test_jets: Vec, trace: TraceOpts, ) -> Result { let (action_sender, action_receiver) = mpsc::channel(1); - let (event_number_sender, event_number_receiver) = oneshot::channel(); - let (cancel_token_sender, cancel_token_receiver) = oneshot::channel(); + let pma_timing = std::env::var_os("NOCK_PMA_TIMING") + .is_some() + .then(|| Arc::new(PmaTiming::default())); + let (init_sender, init_receiver) = oneshot::channel(); let inhibit = Arc::new(AtomicBool::new(false)); let inhibit_clone = inhibit.clone(); + let pma_timing_thread = pma_timing.clone(); let handle = std::thread::Builder::new() .name("serf".to_string()) .stack_size(SERF_THREAD_STACK_SIZE) .spawn(move || { - let stack = NockStack::new(nock_stack_size, 0); + let diagnostics = SerfInitDiagnostics::new(nock_stack_size); + let ker_hash = { + let mut hasher = Hasher::new(); + hasher.update(&kernel_bytes); + hasher.finalize() + }; + let diagnostics = diagnostics.with_current_ker_hash(ker_hash); + let mut pma_meta_load = false; + let ( + pma, + pma_meta_path, + pma_gc_state, + create_snapshots, + rotating_snapshot_interval_event_time, + restore_manifest, + ) = match pma { + Some(config) => { + let PmaConfig { + path_0, + path_1, + words, + reserved_words, + open_existing, + create_snapshots, + rotating_snapshot_interval_event_time, + restore_manifest, + gc_interval, + } = config; + let paths = PmaSlabPaths { path_0, path_1 }; + let active = if open_existing { + select_active_pma_slab(&paths) + } else { + PmaSlab::Slab0 + }; + let active_path = paths.path(active).clone(); + let pma_meta_path = pma_meta_path(&active_path); + let pma_result = if open_existing && active_path.exists() { + pma_meta_load = true; + match reserved_words { + Some(reserved_words) => Pma::open_with_min_and_reserved( + active_path.clone(), + words, + reserved_words, + ), + None => Pma::open_with_min(active_path.clone(), words), + } + } else { + pma_meta_load = false; + match reserved_words { + Some(reserved_words) => Pma::new_with_reserved( + words, + reserved_words, + active_path.clone(), + ), + None => Pma::new(words, active_path.clone()), + } + }; + match pma_result { + Ok(pma) => ( + Some(pma), + Some(pma_meta_path), + gc_interval.map(|interval| { + PmaGcState::new(paths, active, interval, words, reserved_words) + }), + create_snapshots, + rotating_snapshot_interval_event_time, + restore_manifest, + ), + Err(err) => { + let _ = init_sender.send(Err(CrownError::Unknown(err.to_string()))); + return; + } + } + } + None => (None, None, None, false, None, None), + }; + let event_log = if let Some(config) = event_log { + match EventLog::open(config.clone()) { + Ok(log) => Some(log), + Err(err) => { + let _ = init_sender.send(Err(CrownError::Unknown(format!( + "failed to open event log at {}: {err}", + config.path.display() + )))); + return; + } + } + } else { + None + }; + if let (Some(pma), Some(meta_path), Some(manifest)) = ( + pma.as_ref(), + pma_meta_path.as_ref(), + restore_manifest.as_ref(), + ) { + let pma_reserved_words = match u64::try_from(pma.reserved_words()) { + Ok(words) => words, + Err(_) => { + let _ = init_sender.send(Err(CrownError::Unknown( + "PMA reservation exceeds u64 while synthesizing metadata" + .to_string(), + ))); + return; + } + }; + let synthesized = PmaPersistMetadata::new( + ker_hash, manifest.event_num, manifest.kernel_root_raw, pma_reserved_words, + ); + if let Err(err) = synthesized.save_to_path(meta_path) { + let _ = init_sender.send(Err(CrownError::Unknown(format!( + "failed to synthesize PMA metadata from snapshot manifest at {}: {err}", + meta_path.display() + )))); + return; + } + } + let stack = match run_serf_init_phase(&diagnostics, "allocate Nock stack", || { + NockStack::new(nock_stack_size, 0) + }) { + Ok(stack) => stack, + Err(err) => { + let _ = init_sender.send(Err(err)); + return; + } + }; let serf = Serf::new( - stack, checkpoint, &kernel_bytes, &constant_hot_state, test_jets, trace, + stack, pma, pma_meta_path, pma_meta_load, pma_gc_state, create_snapshots, + rotating_snapshot_interval_event_time, event_log, checkpoint, &kernel_bytes, + &constant_hot_state, test_jets, trace, nock_stack_size, ); - event_number_sender - .send(serf.event_num.clone()) - .expect("Could not send event number out of serf thread"); - cancel_token_sender - .send(serf.context.cancel_token()) - .expect("Could not send cancel token out of serf thread"); - serf_loop(serf, action_receiver, inhibit_clone); + let serf = match serf { + Ok(serf) => serf, + Err(err) => { + let _ = init_sender.send(Err(err)); + return; + } + }; + let _ = init_sender.send(Ok((serf.event_num.clone(), serf.context.cancel_token()))); + serf_loop(serf, action_receiver, inhibit_clone, pma_timing_thread); })?; - - let event_number = event_number_receiver.await?; - let cancel_token = cancel_token_receiver.await?; + let (event_number, cancel_token) = init_receiver + .await + .map_err(|err| CrownError::Unknown(err.to_string()))??; Ok(SerfThread { inhibit, handle: Some(handle), action_sender, event_number, cancel_token, + pma_timing, }) } } @@ -160,21 +892,27 @@ impl SerfThread { pub(crate) fn stop(&mut self) -> impl Future> { let action_sender = self.action_sender.clone(); - let cancel_token = self.cancel_token.clone(); let join_handle = self.handle.take().expect("Serf join handle already taken."); let tokio_join_handle = tokio::task::spawn_blocking(move || join_handle.join()); - self.inhibit.store(true, Ordering::SeqCst); + let (result, result_recv) = oneshot::channel(); async move { - cancel_token.cancel(); - action_sender - .send(SerfAction::Stop) - .await - .expect("Failed to send stop action"); - match tokio_join_handle.await { + if let Err(err) = action_sender.send(SerfAction::Stop { result }).await { + let join_res = tokio_join_handle.await; + return match join_res { + Ok(Ok(())) => Err(err.into()), + Ok(Err(e)) => Err(CrownError::Unknown(format!("Serf thread panicked: {e:?}"))), + Err(e) => Err(CrownError::JoinError(e)), + }; + } + + let stop_res = result_recv.await?; + let join_res = tokio_join_handle.await; + match join_res { Ok(Ok(())) => Ok(()), Ok(Err(e)) => Err(CrownError::Unknown(format!("Serf thread panicked: {e:?}"))), Err(e) => Err(CrownError::JoinError(e)), - } + }?; + stop_res } } @@ -294,6 +1032,20 @@ impl SerfThread { } } + pub(crate) fn replay_event_jobs( + &self, + jobs: Vec, + ) -> impl Future> { + let (result, result_fut) = oneshot::channel(); + let action_sender = self.action_sender.clone(); + async move { + action_sender + .send(SerfAction::Replay { jobs, result }) + .await?; + result_fut.await? + } + } + pub fn import(&self, state: LoadState) -> impl Future> { let (result, result_fut) = oneshot::channel(); let action_sender = self.action_sender.clone(); @@ -319,6 +1071,7 @@ fn serf_loop( mut serf: Serf, mut action_receiver: mpsc::Receiver>, inhibit: Arc, + pma_timing: Option>, ) { loop { let start = std::time::Instant::now(); @@ -333,16 +1086,25 @@ fn serf_loop( }; let action_start = std::time::Instant::now(); match action { - SerfAction::Stop => { + SerfAction::Stop { result } => { + let stop_res = serf.flush_pma_state_for_shutdown(); + let _ = result.send(stop_res).inspect_err(|_err| { + debug!("Failed to send shutdown flush result to dropped channel"); + }); break; } SerfAction::Export { result } => { - let kernel_state_noun = serf.arvo.slot(STATE_AXIS); + let space = serf.context.stack.noun_space(); + let kernel_state_noun = serf + .arvo + .in_space(&space) + .slot(STATE_AXIS) + .map(|handle| handle.noun()); let kernel_state = kernel_state_noun.map_or_else( |err| Err(CrownError::from(err)), |noun| { let mut slab = NounSlab::new(); - slab.copy_into(noun); + slab.copy_into(noun, &space); Ok(slab) }, ); @@ -373,7 +1135,7 @@ fn serf_loop( } unsafe { serf.event_update(state.event_num, arvo); - serf.preserve_event_update_leftovers(); + let _ = serf.preserve_event_update_leftovers(); } let _ = result.send(Ok(())).map_err(|err| { debug!("Tried to send to dropped channel: {:?}", err); @@ -382,12 +1144,17 @@ fn serf_loop( } } SerfAction::GetKernelStateSlab { result } => { - let kernel_state_noun = serf.arvo.slot(STATE_AXIS); + let space = serf.context.stack.noun_space(); + let kernel_state_noun = serf + .arvo + .in_space(&space) + .slot(STATE_AXIS) + .map(|handle| handle.noun()); let kernel_state_slab = kernel_state_noun.map_or_else( |err| Err(CrownError::from(err)), |noun| { let mut slab = NounSlab::new(); - slab.copy_into(noun); + slab.copy_into(noun, &space); Ok(slab) }, ); @@ -405,7 +1172,8 @@ fn serf_loop( let cold_state_noun = serf.context.cold.into_noun(serf.stack()); let cold_state_slab = { let mut slab = NounSlab::new(); - slab.copy_into(cold_state_noun); + let space = serf.context.stack.noun_space(); + slab.copy_into(cold_state_noun, &space); slab }; let _ = result.send(cold_state_slab).inspect_err(|_e| { @@ -444,9 +1212,10 @@ fn serf_loop( } else { let ovo_noun = ovo.copy_to_stack(serf.stack()); let noun_res = serf.peek(ovo_noun); + let space = serf.context.stack.noun_space(); let noun_slab_res = noun_res.map(|noun| { let mut slab = NounSlab::new(); - slab.copy_into(noun); + slab.copy_into(noun, &space); slab }); let _ = result.send(noun_slab_res).inspect_err(|_e| { @@ -458,6 +1227,20 @@ fn serf_loop( nockapp_metrics.serf_loop_peek.add_timing(&action_elapsed); }; } + SerfAction::Replay { jobs, result } => { + if inhibit.load(Ordering::SeqCst) { + let _ = result + .send(Err(CrownError::Unknown("Serf stopping".to_string()))) + .inspect_err(|_e| { + debug!("Failed to send inhibited replay result from serf thread"); + }); + } else { + let replay_res = serf.replay_event_jobs(jobs); + let _ = result.send(replay_res).inspect_err(|_e| { + debug!("Failed to send replay result from serf thread"); + }); + } + } SerfAction::Poke { wire, cause, @@ -471,20 +1254,127 @@ fn serf_loop( debug!("Failed to send inihibited poke result from serf thread"); }); } else { + if let Err(err) = serf.ensure_pma_capacity_before_event() { + let _ = result.send(Err(err)).inspect_err(|_e| { + debug!("Failed to send PMA preflight failure from serf thread"); + }); + let _ = result_ack.blocking_recv().inspect_err(|_e| { + debug!("Failed to receive result ack after PMA preflight failure"); + }); + continue; + } let cause_noun = cause.copy_to_stack(serf.stack()); + let event_num_before = serf.event_num.load(Ordering::SeqCst) + 1; + debug!( + event_num = event_num_before, + source = %wire.source, + "poke action start" + ); + let event_start = Instant::now(); let noun_res = serf.poke(wire, cause_noun); - let noun_slab_res = noun_res.map(|noun| { - let mut slab = NounSlab::new(); - slab.copy_into(noun); - slab - }); + let event_elapsed = event_start.elapsed(); + let space = serf.context.stack.noun_space(); + let (noun_slab_res, did_update, mut durable_event) = match noun_res { + Ok(accepted_poke) => { + let mut slab = NounSlab::new(); + slab.copy_into(accepted_poke.effects, &space); + (Ok(slab), true, accepted_poke.durable_event) + } + Err(err) => (Err(err), false, None), + }; + if let Some(durable_event) = durable_event.as_mut() { + durable_event.event_processing_duration = event_elapsed; + } + let mut pma_elapsed = None; + let mut pma_detail = None; + let mut durable_append_elapsed = None; + let cleanup_start = did_update.then(Instant::now); + if did_update { + debug!(event_num = event_num_before, "poke cleanup start"); + let pma_start = Instant::now(); + unsafe { + pma_detail = + serf.preserve_event_update_leftovers_with_pre_persist(|serf| { + if let Some(durable_event) = durable_event.as_ref() { + durable_append_elapsed = + Some(serf.append_durable_event(durable_event)); + serf.cumulative_event_processing_time_since_snapshot = serf + .cumulative_event_processing_time_since_snapshot + .saturating_add( + durable_event.event_processing_duration, + ); + } + }); + } + pma_elapsed = Some(pma_start.elapsed()); + debug!( + event_num = event_num_before, + elapsed_ms = duration_ms(pma_start.elapsed()), + "poke cleanup stage done: preserve_event_update_leftovers" + ); + } let _ = result.send(noun_slab_res).inspect_err(|_e| { debug!("Failed to send poke result from serf thread"); }); + if let Some(timing) = &pma_timing { + let pma_elapsed = pma_elapsed.unwrap_or_else(|| Duration::from_millis(0)); + timing.record(event_elapsed, pma_elapsed, pma_detail); + } + if std::env::var_os("NOCK_PMA_TIMING").is_some() { + let event_ms = event_elapsed.as_secs_f64() * 1000.0; + let pma_ms = pma_elapsed + .map(|elapsed| elapsed.as_secs_f64() * 1000.0) + .unwrap_or(0.0); + let total_ms = event_ms + pma_ms; + let total_alloc_words = pma_detail.map_or(0usize, |detail| { + detail.warm.alloc_words + + detail.test_jets.alloc_words + + detail.hot.alloc_words + + detail.cache.alloc_words + + detail.cold.alloc_words + + detail.arvo.alloc_words + }); + let total_alloc_mib = words_to_mib(total_alloc_words); + let event_num = serf.event_num.load(Ordering::SeqCst); + info!( + "pma-timing: event_ms={:.3} pma_copy_ms={:.3} total_ms={:.3} alloc_words={} alloc_mib={:.3} event_num={}", + event_ms, + pma_ms, + total_ms, + total_alloc_words, + total_alloc_mib, + event_num + ); + } + let _ = result_ack.blocking_recv().inspect_err(|_e| { + debug!("Failed to receive result ack in serf thread"); + }); + let mut snapshot_stage_elapsed = None; + if did_update { + let snapshot_start = Instant::now(); + serf.maybe_create_rotating_snapshot(); + snapshot_stage_elapsed = Some(snapshot_start.elapsed()); + debug!( + event_num = event_num_before, + elapsed_ms = duration_ms(snapshot_start.elapsed()), + "poke cleanup stage done: rotating_snapshot" + ); + } + let cleanup_elapsed = cleanup_start.map(|start| start.elapsed()); + let poke_total_ms = duration_ms(event_elapsed) + + cleanup_elapsed.map(duration_ms).unwrap_or(0.0); + debug!( + event_num = event_num_before, + did_update, + poke_total_ms, + event_eval_ms = duration_ms(event_elapsed), + preserve_ms = pma_elapsed.map(duration_ms), + durable_append_ms = durable_append_elapsed.map(duration_ms), + snapshot_stage_ms = snapshot_stage_elapsed.map(duration_ms), + cleanup_total_ms = cleanup_elapsed.map(duration_ms), + "poke action done" + ); }; - let _ = result_ack.blocking_recv().inspect_err(|_e| { - debug!("Failed to receive result ack in serf thread"); - }); let action_elapsed = action_start.elapsed(); if let Some(nockapp_metrics) = &serf.metrics { nockapp_metrics.serf_loop_poke.add_timing(&action_elapsed); @@ -516,24 +1406,44 @@ fn create_checkpoint( ) -> C { let ker_hash = serf.ker_hash; let event_num = serf.event_num.load(Ordering::SeqCst); - let ker_state = serf.arvo.slot(STATE_AXIS).unwrap_or_else(|err| { - panic!( - "Panicked with {err:?} at {}:{} (git sha: {:?})", - file!(), - line!(), - option_env!("GIT_SHA") - ) - }); + let space = serf.context.stack.noun_space(); + let ker_state = serf + .arvo + .in_space(&space) + .slot(STATE_AXIS) + .map(|handle| handle.noun()) + .unwrap_or_else(|err| { + panic!( + "Panicked with {err:?} at {}:{} (git sha: {:?})", + file!(), + line!(), + option_env!("GIT_SHA") + ) + }); let cold_state = serf.context.cold; - C::new( + let checkpoint = C::new( serf.stack(), ker_hash, event_num, ker_state, cold_state, metrics, - ) + ); + + if let Some(pma) = serf.pma.as_ref() { + if let Err(err) = pma.sync_all() { + warn!("Failed to msync PMA after checkpoint save: {err}"); + } + if let Err(err) = durability::sync_path_data(pma.path(), "checkpoint_pma_fdatasync") { + warn!("Failed to fdatasync PMA after checkpoint save: {err}"); + } + if let Err(err) = pma.advise_drop_allocated_prefix(4, 5) { + warn!("Failed to madvise PMA prefix after checkpoint save: {err}"); + } + } + + checkpoint } /// Represents a Sword kernel, containing a Serf and snapshot location. @@ -561,45 +1471,96 @@ impl Kernel { hot_state: &[HotEntry], test_jets: Vec, trace: TraceOpts, + pma: Option, ) -> Result { - let kernel_vec = Vec::from(kernel); - let hot_state_vec = Vec::from(hot_state); - let serf = SerfThread::new( - kernel_vec, checkpoint, hot_state_vec, NOCK_STACK_SIZE, test_jets, trace, + Self::load_with_hot_state_with_event_log( + kernel, checkpoint, hot_state, test_jets, trace, pma, None, ) - .await?; - Ok(Self { serf }) + .await } - pub async fn load_with_hot_state_tiny( + pub(crate) async fn load_with_hot_state_with_event_log( kernel: &[u8], checkpoint: Option, hot_state: &[HotEntry], test_jets: Vec, trace: TraceOpts, + pma: Option, + event_log: Option, ) -> Result { let kernel_vec = Vec::from(kernel); let hot_state_vec = Vec::from(hot_state); - let serf = SerfThread::new( - kernel_vec, checkpoint, hot_state_vec, NOCK_STACK_SIZE_TINY, test_jets, trace, + let serf = SerfThread::new_with_event_log( + kernel_vec, checkpoint, hot_state_vec, NOCK_STACK_SIZE, pma, event_log, test_jets, + trace, ) .await?; Ok(Self { serf }) } - pub async fn load_with_hot_state_small( + pub async fn load_with_hot_state_tiny( kernel: &[u8], checkpoint: Option, hot_state: &[HotEntry], test_jets: Vec, trace: TraceOpts, + pma: Option, ) -> Result { - let kernel_vec = Vec::from(kernel); - let hot_state_vec = Vec::from(hot_state); - let serf = SerfThread::new( - kernel_vec, checkpoint, hot_state_vec, NOCK_STACK_SIZE_SMALL, test_jets, trace, + Self::load_with_hot_state_tiny_with_event_log( + kernel, checkpoint, hot_state, test_jets, trace, pma, None, ) - .await?; + .await + } + + pub(crate) async fn load_with_hot_state_tiny_with_event_log( + kernel: &[u8], + checkpoint: Option, + hot_state: &[HotEntry], + test_jets: Vec, + trace: TraceOpts, + pma: Option, + event_log: Option, + ) -> Result { + let kernel_vec = Vec::from(kernel); + let hot_state_vec = Vec::from(hot_state); + let serf = SerfThread::new_with_event_log( + kernel_vec, checkpoint, hot_state_vec, NOCK_STACK_SIZE_TINY, pma, event_log, test_jets, + trace, + ) + .await?; + Ok(Self { serf }) + } + + pub async fn load_with_hot_state_small( + kernel: &[u8], + checkpoint: Option, + hot_state: &[HotEntry], + test_jets: Vec, + trace: TraceOpts, + pma: Option, + ) -> Result { + Self::load_with_hot_state_small_with_event_log( + kernel, checkpoint, hot_state, test_jets, trace, pma, None, + ) + .await + } + + pub(crate) async fn load_with_hot_state_small_with_event_log( + kernel: &[u8], + checkpoint: Option, + hot_state: &[HotEntry], + test_jets: Vec, + trace: TraceOpts, + pma: Option, + event_log: Option, + ) -> Result { + let kernel_vec = Vec::from(kernel); + let hot_state_vec = Vec::from(hot_state); + let serf = SerfThread::new_with_event_log( + kernel_vec, checkpoint, hot_state_vec, NOCK_STACK_SIZE_SMALL, pma, event_log, + test_jets, trace, + ) + .await?; Ok(Self { serf }) } @@ -609,11 +1570,28 @@ impl Kernel { hot_state: &[HotEntry], test_jets: Vec, trace: TraceOpts, + pma: Option, + ) -> Result { + Self::load_with_hot_state_medium_with_event_log( + kernel, checkpoint, hot_state, test_jets, trace, pma, None, + ) + .await + } + + pub(crate) async fn load_with_hot_state_medium_with_event_log( + kernel: &[u8], + checkpoint: Option, + hot_state: &[HotEntry], + test_jets: Vec, + trace: TraceOpts, + pma: Option, + event_log: Option, ) -> Result { let kernel_vec = Vec::from(kernel); let hot_state_vec = Vec::from(hot_state); - let serf = SerfThread::new( - kernel_vec, checkpoint, hot_state_vec, NOCK_STACK_SIZE_MEDIUM, test_jets, trace, + let serf = SerfThread::new_with_event_log( + kernel_vec, checkpoint, hot_state_vec, NOCK_STACK_SIZE_MEDIUM, pma, event_log, + test_jets, trace, ) .await?; Ok(Self { serf }) @@ -625,27 +1603,78 @@ impl Kernel { hot_state: &[HotEntry], test_jets: Vec, trace: TraceOpts, + pma: Option, + ) -> Result { + Self::load_with_hot_state_large_with_event_log( + kernel, checkpoint, hot_state, test_jets, trace, pma, None, + ) + .await + } + + pub(crate) async fn load_with_hot_state_large_with_event_log( + kernel: &[u8], + checkpoint: Option, + hot_state: &[HotEntry], + test_jets: Vec, + trace: TraceOpts, + pma: Option, + event_log: Option, ) -> Result { let kernel_vec = Vec::from(kernel); let hot_state_vec = Vec::from(hot_state); - let serf = SerfThread::new( - kernel_vec, checkpoint, hot_state_vec, NOCK_STACK_SIZE_LARGE, test_jets, trace, + let serf = SerfThread::new_with_event_log( + kernel_vec, checkpoint, hot_state_vec, NOCK_STACK_SIZE_LARGE, pma, event_log, + test_jets, trace, ) .await?; Ok(Self { serf }) } + pub fn take_pma_timing_samples(&self) -> Option> { + self.serf.pma_timing.as_ref().map(|timing| { + timing + .take_samples() + .into_iter() + .map(|sample| (sample.event, sample.pma_copy)) + .collect() + }) + } + + pub fn take_pma_timing_samples_detailed(&self) -> Option> { + self.serf + .pma_timing + .as_ref() + .map(|timing| timing.take_samples()) + } + pub async fn load_with_hot_state_huge( kernel: &[u8], checkpoint: Option, hot_state: &[HotEntry], test_jets: Vec, trace: TraceOpts, + pma: Option, + ) -> Result { + Self::load_with_hot_state_huge_with_event_log( + kernel, checkpoint, hot_state, test_jets, trace, pma, None, + ) + .await + } + + pub(crate) async fn load_with_hot_state_huge_with_event_log( + kernel: &[u8], + checkpoint: Option, + hot_state: &[HotEntry], + test_jets: Vec, + trace: TraceOpts, + pma: Option, + event_log: Option, ) -> Result { let kernel_vec = Vec::from(kernel); let hot_state_vec = Vec::from(hot_state); - let serf = SerfThread::new( - kernel_vec, checkpoint, hot_state_vec, NOCK_STACK_SIZE_HUGE, test_jets, trace, + let serf = SerfThread::new_with_event_log( + kernel_vec, checkpoint, hot_state_vec, NOCK_STACK_SIZE_HUGE, pma, event_log, test_jets, + trace, ) .await?; Ok(Self { serf }) @@ -667,14 +1696,42 @@ impl Kernel { checkpoint: Option, test_jets: Vec, trace: TraceOpts, + pma: Option, + ) -> Result { + Self::load_with_event_log(kernel, checkpoint, test_jets, trace, pma, None).await + } + + pub(crate) async fn load_with_event_log( + kernel: &[u8], + checkpoint: Option, + test_jets: Vec, + trace: TraceOpts, + pma: Option, + event_log: Option, ) -> Result { - Self::load_with_hot_state(kernel, checkpoint, &Vec::new(), test_jets, trace).await + Self::load_with_hot_state_with_event_log( + kernel, + checkpoint, + &Vec::new(), + test_jets, + trace, + pma, + event_log, + ) + .await } /// Produces a checkpoint of the kernel state. pub fn checkpoint(&self) -> impl Future> { self.serf.checkpoint() } + + pub(crate) fn replay_event_jobs( + &self, + jobs: Vec, + ) -> impl Future> { + self.serf.replay_event_jobs(jobs) + } } impl Kernel { @@ -731,6 +1788,17 @@ pub struct Serf { pub arvo: Noun, /// The interpreter context. pub context: interpreter::Context, + /// Persistent memory arena for long-lived state. + pub pma: Option, + /// Optional metadata path for PMA persistence. + pub pma_meta_path: Option, + /// Optional GC configuration for PMA slab compaction. + pma_gc_state: Option, + snapshot_creation_enabled: bool, + rotating_snapshot_interval_event_time: Option, + cumulative_event_processing_time_since_snapshot: Duration, + /// Optional append-only event log used as the durability boundary. + event_log: Option, /// Cancellation pub cancel_token: NockCancelToken, /// The current event number. @@ -753,45 +1821,141 @@ impl Serf { /// # Returns /// /// A new `Serf` instance. + #[allow(clippy::too_many_arguments)] fn new( mut stack: NockStack, + mut pma: Option, + pma_meta_path: Option, + pma_meta_load: bool, + pma_gc_state: Option, + snapshot_creation_enabled: bool, + rotating_snapshot_interval_event_time: Option, + mut event_log: Option, checkpoint: Option, kernel_bytes: &[u8], constant_hot_state: &[HotEntry], test_jets: Vec, trace: TraceOpts, - ) -> Self { + nock_stack_size: usize, + ) -> Result { let hot_state = [URBIT_HOT_STATE, constant_hot_state].concat(); + if let Some(ref pma) = pma { + stack.install_pma_arena(Arc::clone(pma.arena())); + } + let mut hasher = Hasher::new(); hasher.update(kernel_bytes); let ker_hash = hasher.finalize(); + let mut diagnostics = + SerfInitDiagnostics::new(nock_stack_size).with_current_ker_hash(ker_hash); + + let mut reset_pma = false; + let pma_state = if pma_meta_load && checkpoint.is_none() { + if let (Some(_pma), Some(meta_path)) = (pma.as_ref(), pma_meta_path.as_ref()) { + if let Some(meta) = PmaPersistMetadata::load_from_path(meta_path) { + // A kernel-hash mismatch does not discard the PMA: the + // persisted state noun is loaded into the freshly-cued + // current kernel, which runs its own Hoon-level `+load` + // state migration. This matches the checkpoint restore + // path below ("loading checkpoint state into current + // kernel"). The metadata checksum (already validated by + // `load_from_path`) and the PMA trailer guard structural + // integrity; kernel-hash equality is not required for + // memory safety. + if meta.ker_hash != ker_hash { + warn!( + "PMA metadata kernel hash mismatch (metadata: {}, kernel: {}); loading existing PMA state into current kernel", + meta.ker_hash, ker_hash + ); + } + let kernel_state = unsafe { Noun::from_raw(meta.kernel_state_raw) }; + Some((kernel_state, meta.event_num)) + } else { + if meta_path.exists() { + warn!( + "Failed to load PMA metadata at {}; starting fresh", + meta_path.display() + ); + } + reset_pma = true; + None + } + } else { + None + } + } else { + None + }; + + if !pma_meta_load { + if let Some(meta_path) = pma_meta_path.as_ref() { + let _ = fs::remove_file(meta_path); + } + } + + if reset_pma { + if let Some(pma) = pma.as_mut() { + pma.reset(); + } + if let Some(meta_path) = pma_meta_path.as_ref() { + let _ = fs::remove_file(meta_path); + } + } let (maybe_state, cold, event_num_raw) = if let Some(c) = checkpoint { let saveable = c.load(); - - let ker_state = saveable.state.copy_to_stack(&mut stack); - let cold_noun = saveable.cold.copy_to_stack(&mut stack); - let cold_vecs = Cold::from_noun(&mut stack, &cold_noun) - .expect("Could not load cold state from snapshot"); - let cold = Cold::from_vecs(&mut stack, cold_vecs.0, cold_vecs.1, cold_vecs.2); + diagnostics = diagnostics.with_checkpoint(&saveable); + + let ker_state = run_serf_init_phase( + &diagnostics, + "copy checkpoint state into Nock stack", + || saveable.state.copy_to_stack(&mut stack), + )?; + let cold_noun = run_serf_init_phase( + &diagnostics, + "copy checkpoint cold state into Nock stack", + || saveable.cold.copy_to_stack(&mut stack), + )?; + let cold_vecs = + run_serf_init_phase(&diagnostics, "decode checkpoint cold state", || { + let space = stack.noun_space(); + Cold::from_noun(&mut stack, &cold_noun, &space).map_err(|err| { + CrownError::Unknown(format!( + "could not load cold state from checkpoint: {err}" + )) + }) + })??; + let cold = run_serf_init_phase(&diagnostics, "rebuild checkpoint cold state", || { + Cold::from_vecs(&mut stack, cold_vecs.0, cold_vecs.1, cold_vecs.2) + })?; if saveable.ker_hash != ker_hash { - debug!( - "Loading snapshot from kernel {} into kernel {}", - saveable.ker_hash, ker_hash + warn!( + checkpoint = %saveable.ker_hash.to_hex(), + current = %ker_hash.to_hex(), + "checkpoint kernel hash mismatch; loading checkpoint state into current kernel" ); } (Some(ker_state), cold, saveable.event_num) + } else if let Some((ker_state, event_num)) = pma_state { + info!("Loaded PMA state at event_num {}", event_num); + (Some(ker_state), Cold::new(&mut stack), event_num) } else { - (None, Cold::new(&mut stack), 0) + let cold = run_serf_init_phase(&diagnostics, "initialize empty cold state", || { + Cold::new(&mut stack) + })?; + (None, cold, 0) }; let event_num = Arc::new(AtomicU64::new(event_num_raw)); - let mut context = create_context(stack, &hot_state, cold, trace.into(), test_jets); + let mut context = + run_serf_init_phase(&diagnostics, "create Nock interpreter context", || { + create_context(stack, &hot_state, cold, trace.into(), test_jets) + })?; let cancel_token = context.cancel_token(); - let mut arvo = { + let mut arvo = run_serf_init_phase(&diagnostics, "boot kernel", || { let kernel_trap = Noun::cue_bytes_slice(&mut context.stack, kernel_bytes) .expect("invalid kernel jam"); let fol = T(&mut context.stack, &[D(9), D(2), D(0), D(1)]); @@ -818,26 +1982,62 @@ impl Serf { ) }) } + })?; + + let cumulative_event_processing_time_since_snapshot = if snapshot_creation_enabled + && rotating_snapshot_interval_event_time.is_some() + { + match event_log.as_mut() { + Some(event_log) => { + let latest_ready_snapshot_event_num = event_log + .latest_ready_snapshot_event_num() + .unwrap_or_else(|err| { + panic!("serf: failed to load latest ready snapshot event_num: {err}") + }) + .unwrap_or(0); + event_log + .event_processing_time_after(latest_ready_snapshot_event_num) + .unwrap_or_else(|err| { + panic!( + "serf: failed to load cumulative event processing time since latest snapshot: {err}" + ) + }) + } + None => Duration::ZERO, + } + } else { + Duration::ZERO }; let mut serf = Self { ker_hash, arvo, context, + pma, + pma_meta_path, + pma_gc_state, + snapshot_creation_enabled, + rotating_snapshot_interval_event_time, + cumulative_event_processing_time_since_snapshot, + event_log, event_num, cancel_token, metrics: None, }; if let Some(kernel_state) = maybe_state { - arvo = serf.load(kernel_state).expect("serf: load failed"); + arvo = + run_serf_init_phase(&diagnostics, "load checkpoint state through kernel", || { + serf.load(kernel_state) + })??; } - unsafe { + run_serf_init_phase(&diagnostics, "preserve loaded kernel state", || unsafe { serf.event_update(event_num_raw, arvo); - serf.preserve_event_update_leftovers(); - } - serf + let _ = serf.preserve_event_update_leftovers(); + })?; + serf.ensure_epoch_snapshot(); + Ok(serf) } /// Performs a peek operation on the Arvo state. @@ -875,9 +2075,12 @@ impl Serf { /// A noun representing the error. pub fn goof(&mut self, mote: Mote, traces: Noun) -> Noun { let tone = Cell::new(&mut self.context.stack, D(2), traces); + let space = self.context.stack.noun_space(); let tang = mook(&mut self.context, tone, false) .expect("serf: goof: +mook crashed on bail") - .tail(); + .in_space(&space) + .tail() + .noun(); T(&mut self.context.stack, &[D(mote as u64), tang]) } @@ -901,13 +2104,18 @@ impl Serf { } pub fn print_goof(&mut self, goof: Noun) { + let space = self.context.stack.noun_space(); let tang = goof + .in_space(&space) .as_cell() .expect("print goof: expected goof to be a cell") - .tail(); - tang.list_iter().for_each(|tank: Noun| { + .tail() + .noun(); + tang.in_space(&space).list_iter().for_each(|tank| { // TODO: Slogger should be emitting Results in case of failure - self.context.slogger.slog(&mut self.context.stack, 1, tank); + self.context + .slogger + .slog(&mut self.context.stack, 1, tank.noun()); }); } @@ -921,21 +2129,33 @@ impl Serf { /// /// Result containing the poke response or an error. #[tracing::instrument(level = "info", skip_all)] - pub fn do_poke(&mut self, job: Noun) -> Result { + fn do_poke( + &mut self, + job: Noun, + metadata: Option<&AcceptedEventMetadata>, + ) -> Result { + let space = self.context.stack.noun_space(); match self.soft(job, POKE_AXIS, Some("poke".to_string())) { Ok(res) => { - let cell = res.as_cell().expect("serf: poke: +slam returned atom"); - let mut fec = cell.head(); + let cell = res + .in_space(&space) + .as_cell() + .expect("serf: poke: +slam returned atom"); + let fec = cell.head().noun(); let eve = self.event_num.load(Ordering::SeqCst); unsafe { - self.event_update(eve + 1, cell.tail()); - self.stack().preserve(&mut fec); - self.preserve_event_update_leftovers(); + self.event_update(eve + 1, cell.tail().noun()); } - Ok(fec) + let durable_event = metadata + .map(|metadata| self.capture_accepted_event(job, eve + 1, metadata)) + .transpose()?; + Ok(AcceptedPoke { + effects: fec, + durable_event, + }) } - Err(goof) => self.poke_swap(job, goof), + Err(goof) => self.poke_swap(job, goof, metadata), } } @@ -1016,10 +2236,11 @@ impl Serf { /// /// Result containing the final Arvo state or an error. fn play_list(&mut self, mut lit: Noun) -> Result { + let space = self.context.stack.noun_space(); let mut eve = self.event_num.load(Ordering::SeqCst); - while let Ok(cell) = lit.as_cell() { - let ovo = cell.head(); - lit = cell.tail(); + while let Ok(cell) = lit.in_space(&space).as_cell() { + let ovo = cell.head().noun(); + lit = cell.tail().noun(); let trace_name = if self.context.trace_info.is_some() { Some(format!("play [{}]", eve)) } else { @@ -1028,17 +2249,17 @@ impl Serf { match self.soft(ovo, POKE_AXIS, trace_name) { Ok(res) => { - let arvo = res.as_cell()?.tail(); + let arvo = res.in_space(&space).as_cell()?.tail().noun(); eve += 1; unsafe { self.event_update(eve, arvo); - self.context.stack.preserve(&mut lit); - self.preserve_event_update_leftovers(); } } Err(goof) => { - return Err(CrownError::KernelError(Some(goof))); + let mut goof_slab = NounSlab::new(); + goof_slab.copy_into(goof, &space); + return Err(CrownError::KernelError(Some(goof_slab))); } } } @@ -1055,22 +2276,31 @@ impl Serf { /// # Returns /// /// Result containing the new event or an error. - fn poke_swap(&mut self, job: Noun, goof: Noun) -> Result { + fn poke_swap( + &mut self, + job: Noun, + goof: Noun, + metadata: Option<&AcceptedEventMetadata>, + ) -> Result { let stack = &mut self.context.stack; + let space = stack.noun_space(); self.context.cache = Hamt::::new(stack); - let job_cell = job.as_cell().expect("serf: poke: job not a cell"); + let job_cell = job + .in_space(&space) + .as_cell() + .expect("serf: poke: job not a cell"); // job data is job without event_num let job_data = job_cell .tail() .as_cell() .expect("serf: poke: data not a cell"); // job input is job without event_num or wire - let job_input = job_data.tail(); + let job_input = job_data.tail().noun(); let wire = T(stack, &[D(0), D(tas!(b"arvo")), D(0)]); let crud = DirectAtom::new_panic(tas!(b"crud")); let event_num = D(self.event_num.load(Ordering::SeqCst) + 1); - let mut ovo = T(stack, &[event_num, wire, goof, job_input]); + let ovo = T(stack, &[event_num, wire, goof, job_input]); let trace_name = if self.context.trace_info.is_some() { Some(Self::poke_trace_name( &mut self.context.stack, @@ -1083,19 +2313,29 @@ impl Serf { match self.soft(ovo, POKE_AXIS, trace_name) { Ok(res) => { - let cell = res.as_cell().expect("serf: poke: crud +slam returned atom"); - let mut fec = cell.head(); + let cell = res + .in_space(&space) + .as_cell() + .expect("serf: poke: crud +slam returned atom"); + let fec = cell.head().noun(); let eve = self.event_num.load(Ordering::SeqCst); unsafe { - self.event_update(eve + 1, cell.tail()); - self.context.stack.preserve(&mut ovo); - self.context.stack.preserve(&mut fec); - self.preserve_event_update_leftovers(); + self.event_update(eve + 1, cell.tail().noun()); } - Ok(fec) + let durable_event = metadata + .map(|metadata| self.capture_accepted_event(ovo, eve + 1, metadata)) + .transpose()?; + Ok(AcceptedPoke { + effects: fec, + durable_event, + }) + } + Err(goof_crud) => { + let mut goof_slab = NounSlab::new(); + goof_slab.copy_into(goof_crud, &space); + Err(CrownError::KernelError(Some(goof_slab))) } - Err(goof_crud) => Err(CrownError::KernelError(Some(goof_crud))), } } @@ -1112,8 +2352,10 @@ impl Serf { /// A string representing the trace name. fn poke_trace_name(stack: &mut NockStack, wire: Noun, vent: Atom) -> String { let wpc = path_to_cord(stack, wire); - let wpc_len = met3_usize(wpc); - let wpc_bytes = &wpc.as_ne_bytes()[0..wpc_len]; + let space = stack.noun_space(); + let wpc_len = met3_usize(wpc, &space); + let wpc_handle = wpc.in_space(&space); + let wpc_bytes = &wpc_handle.as_ne_bytes()[0..wpc_len]; let wpc_str = match std::str::from_utf8(wpc_bytes) { Ok(valid) => valid, Err(error) => { @@ -1122,8 +2364,9 @@ impl Serf { } }; - let vc_len = met3_usize(vent); - let vc_bytes = &vent.as_ne_bytes()[0..vc_len]; + let vc_len = met3_usize(vent, &space); + let vent_handle = vent.in_space(&space); + let vc_bytes = &vent_handle.as_ne_bytes()[0..vc_len]; let vc_str = match std::str::from_utf8(vc_bytes) { Ok(valid) => valid, Err(error) => { @@ -1148,7 +2391,8 @@ impl Serf { #[tracing::instrument(level = "info", skip_all, fields( src = wire.source ))] - pub fn poke(&mut self, wire: WireRepr, cause: Noun) -> Result { + fn poke(&mut self, wire: WireRepr, cause: Noun) -> Result { + let metadata = self.prepare_accepted_event_metadata(&wire, cause)?; let random_bytes = rand::random::(); let bytes = random_bytes.as_bytes()?; let eny: Atom = Atom::from_bytes(&mut self.context.stack, &bytes); @@ -1168,7 +2412,376 @@ impl Serf { &[event_num, wire, eny.as_noun(), our.as_noun(), now.as_noun(), cause], ); - self.do_poke(poke) + self.do_poke(poke, metadata.as_ref()) + } + + fn prepare_accepted_event_metadata( + &self, + wire: &WireRepr, + cause: Noun, + ) -> Result> { + if self.event_log.is_none() { + return Ok(None); + } + let space = self.context.stack.noun_space(); + let cause_jam = NockJammer::jam(cause, &space).to_vec(); + let cause_hash = blake3::hash(&cause_jam); + let wire_tags = wire + .tags + .iter() + .map(std::string::ToString::to_string) + .collect::>(); + let wire_tags_json = serde_json::to_string(&wire_tags) + .map_err(|err| CrownError::SaveError(format!("wire tags json encode failed: {err}")))?; + Ok(Some(AcceptedEventMetadata { + wire_source: wire.source.to_string(), + wire_version: i64::try_from(wire.version)?, + wire_tags_json, + cause_hash: cause_hash.as_bytes().to_vec(), + created_at_ms: Self::current_time_ms()?, + })) + } + + fn capture_accepted_event( + &self, + accepted_job: Noun, + event_num: u64, + metadata: &AcceptedEventMetadata, + ) -> Result { + let space = self.context.stack.noun_space(); + let job_jam = NockJammer::jam(accepted_job, &space).to_vec(); + let job_hash = blake3::hash(&job_jam); + Ok(EventLogEntry { + event_num, + job_jam, + wire_source: metadata.wire_source.clone(), + wire_version: metadata.wire_version, + wire_tags_json: metadata.wire_tags_json.clone(), + cause_hash: metadata.cause_hash.clone(), + job_hash: job_hash.as_bytes().to_vec(), + event_processing_duration: Duration::ZERO, + created_at_ms: metadata.created_at_ms, + }) + } + + fn current_time_ms() -> Result { + Ok(i64::try_from( + SystemTime::now() + .duration_since(UNIX_EPOCH) + .map_err(|err| CrownError::SaveError(format!("system clock error: {err}")))? + .as_millis(), + )?) + } + + fn append_durable_event(&mut self, event: &EventLogEntry) -> Duration { + let Some(event_log) = self.event_log.as_mut() else { + debug!( + event_num = event.event_num, + "event-log append skipped: event log disabled" + ); + return Duration::from_millis(0); + }; + debug!( + event_num = event.event_num, + path = %event_log.path().display(), + sqlite_sync_mode = if durability::fsync_disabled() { "OFF" } else { "FULL" }, + "event-log append start" + ); + let start = Instant::now(); + if let Err(err) = event_log.append_event(event) { + if let Some(metrics) = &self.metrics { + metrics.event_log_commit_failures.increment(); + } + let elapsed = start.elapsed(); + error!( + event_num = event.event_num, + path = %event_log.path().display(), + elapsed_ms = duration_ms(elapsed), + error = %err, + "event-log append failed" + ); + std::process::abort(); + } + let elapsed = start.elapsed(); + debug!( + event_num = event.event_num, + path = %event_log.path().display(), + elapsed_ms = duration_ms(elapsed), + "event-log append done" + ); + if let Some(metrics) = &self.metrics { + metrics.event_log_append.add_timing(&elapsed); + } + elapsed + } + + fn ensure_pma_capacity_before_event(&mut self) -> Result<()> { + let Some(pma) = self.pma.as_mut() else { + return Ok(()); + }; + let reserve_words = pma_event_preflight_free_words(pma); + if pma.free_words() >= reserve_words { + return Ok(()); + } + let event_num = self.event_num.load(Ordering::SeqCst).saturating_add(1); + info!( + event_num, + path = %pma.path().display(), + capacity_words = pma.size_words(), + alloc_words = pma.alloc_offset(), + free_words = pma.free_words(), + reserve_words, + "event PMA preflight growth start" + ); + pma.ensure_free_words(reserve_words).map_err(|err| { + CrownError::SaveError(format!( + "PMA preflight growth failed before event {event_num}: {err}" + )) + })?; + info!( + event_num, + path = %pma.path().display(), + capacity_words = pma.size_words(), + alloc_words = pma.alloc_offset(), + free_words = pma.free_words(), + reserve_words, + "event PMA preflight growth done" + ); + Ok(()) + } + + fn ensure_epoch_snapshot(&mut self) { + if !self.snapshot_creation_enabled { + debug!("epoch snapshot skipped: creation disabled"); + return; + } + let kernel_root_raw = { + let space = self.context.stack.noun_space(); + let kernel_state = self + .arvo + .in_space(&space) + .slot(STATE_AXIS) + .map(|handle| handle.noun()); + match kernel_state { + Ok(noun) => unsafe { noun.as_raw() }, + Err(err) => { + warn!("epoch snapshot skipped: failed to resolve kernel root: {err:?}"); + return; + } + } + }; + + let event_num = self.event_num.load(Ordering::SeqCst); + let (Some(event_log), Some(pma)) = (&mut self.event_log, &self.pma) else { + return; + }; + let build_start = Instant::now(); + info!(event_num, "epoch snapshot build start"); + match maybe_create_epoch_snapshot( + event_log, pma, self.ker_hash, event_num, kernel_root_raw, SNAPSHOT_UNUSED_COLD_OFFSET, + ) { + Ok(created) => { + let elapsed = build_start.elapsed(); + if created { + self.cumulative_event_processing_time_since_snapshot = Duration::ZERO; + } + info!( + event_num, + created, + elapsed_ms = duration_ms(elapsed), + "epoch snapshot build done" + ); + if let Some(metrics) = &self.metrics { + metrics.snapshot_build.add_timing(&elapsed); + } + } + Err(err) => { + let elapsed = build_start.elapsed(); + if let Some(metrics) = &self.metrics { + metrics.snapshot_build_failures.increment(); + } + warn!( + event_num, + elapsed_ms = duration_ms(elapsed), + error = %err, + "epoch snapshot build failed" + ); + } + } + } + + fn replay_event_jobs(&mut self, jobs: Vec) -> Result<()> { + let replay_start = Instant::now(); + info!(jobs = jobs.len(), "event replay start"); + let mut dirty_since_preserve = false; + for (idx, entry) in jobs.into_iter().enumerate() { + let expected_current = entry.event_num.checked_sub(1).ok_or_else(|| { + CrownError::Unknown("event replay encountered event_num 0".to_string()) + })?; + let current = self.event_num.load(Ordering::SeqCst); + if current != expected_current { + return Err(CrownError::Unknown(format!( + "event replay expected current event_num {expected_current} before applying {}, found {current}", + entry.event_num + ))); + } + let job = Noun::cue_bytes_slice(&mut self.context.stack, &entry.job_jam)?; + self.replay_one_event(entry.event_num, job)?; + dirty_since_preserve = true; + if (idx + 1) % REPLAY_PRESERVE_BATCH == 0 { + let preserve_start = Instant::now(); + unsafe { + let _ = self.preserve_event_update_leftovers(); + } + debug!( + replay_batch_end = idx + 1, + elapsed_ms = duration_ms(preserve_start.elapsed()), + "event replay preserve batch done" + ); + dirty_since_preserve = false; + } + } + if dirty_since_preserve { + let preserve_start = Instant::now(); + unsafe { + let _ = self.preserve_event_update_leftovers(); + } + debug!( + elapsed_ms = duration_ms(preserve_start.elapsed()), + "event replay final preserve done" + ); + } + info!( + elapsed_ms = duration_ms(replay_start.elapsed()), + "event replay done" + ); + Ok(()) + } + + fn replay_one_event(&mut self, event_num: u64, job: Noun) -> Result<()> { + let space = self.context.stack.noun_space(); + let job_cell = job.in_space(&space).as_cell().map_err(CrownError::from)?; + let job_event_num = job_cell.head().as_atom()?.as_u64()?; + if job_event_num != event_num { + return Err(CrownError::Unknown(format!( + "event replay job event_num {job_event_num} does not match SQLite event_num {event_num}" + ))); + } + let trace_name = if self.context.trace_info.is_some() { + Some(format!("replay [{event_num}]")) + } else { + None + }; + match self.soft(job, POKE_AXIS, trace_name) { + Ok(res) => { + let cell = res + .in_space(&space) + .as_cell() + .expect("serf: replay: +slam returned atom"); + unsafe { + self.event_update(event_num, cell.tail().noun()); + } + let current = self.event_num.load(Ordering::SeqCst); + if current != event_num { + return Err(CrownError::Unknown(format!( + "event replay applied event {event_num}, but current event_num is {current}" + ))); + } + Ok(()) + } + Err(goof) => { + let mut goof_slab = NounSlab::new(); + goof_slab.copy_into(goof, &space); + Err(CrownError::KernelError(Some(goof_slab))) + } + } + } + + fn maybe_create_rotating_snapshot(&mut self) { + if !self.snapshot_creation_enabled { + debug!("rotating snapshot skipped: creation disabled"); + return; + } + let Some(interval) = self.rotating_snapshot_interval_event_time else { + return; + }; + if self.cumulative_event_processing_time_since_snapshot < interval { + return; + } + let kernel_root_raw = { + let space = self.context.stack.noun_space(); + let kernel_state = self + .arvo + .in_space(&space) + .slot(STATE_AXIS) + .map(|handle| handle.noun()); + match kernel_state { + Ok(noun) => unsafe { noun.as_raw() }, + Err(err) => { + warn!("rotating snapshot skipped: failed to resolve kernel root: {err:?}"); + return; + } + } + }; + + let event_num = self.event_num.load(Ordering::SeqCst); + let (Some(event_log), Some(pma)) = (&mut self.event_log, &self.pma) else { + return; + }; + let build_start = Instant::now(); + info!( + event_num, + rotating_interval_event_time = ?self.rotating_snapshot_interval_event_time, + cumulative_event_processing_time_since_snapshot = ?self.cumulative_event_processing_time_since_snapshot, + "rotating snapshot build start" + ); + match maybe_create_rotating_snapshot( + event_log, pma, self.ker_hash, event_num, kernel_root_raw, SNAPSHOT_UNUSED_COLD_OFFSET, + self.cumulative_event_processing_time_since_snapshot, + self.rotating_snapshot_interval_event_time, + ) { + Ok(status) => { + let elapsed = build_start.elapsed(); + let created = status.created(); + if created { + self.cumulative_event_processing_time_since_snapshot = Duration::ZERO; + } + if let Some(metrics) = &self.metrics { + metrics.snapshot_build.add_timing(&elapsed); + } + if let Some(err) = status.cleanup_error() { + if let Some(metrics) = &self.metrics { + metrics.snapshot_build_failures.increment(); + } + warn!( + event_num, + created, + elapsed_ms = duration_ms(elapsed), + error = %err, + "rotating snapshot build completed with cleanup error" + ); + return; + } + info!( + event_num, + created, + elapsed_ms = duration_ms(elapsed), + "rotating snapshot build done" + ); + } + Err(err) => { + let elapsed = build_start.elapsed(); + if let Some(metrics) = &self.metrics { + metrics.snapshot_build_failures.increment(); + } + warn!( + event_num, + elapsed_ms = duration_ms(elapsed), + error = %err, + "rotating snapshot build failed" + ); + } + } } /// Updates the Serf's state after an event. @@ -1190,21 +2803,562 @@ impl Serf { self.context.scry_stack = D(0); } - /// Preserves leftovers after an event update. - /// - /// # Safety - /// - /// This function is unsafe because it modifies the Serf's state directly. - #[tracing::instrument(level = "info", skip_all)] - pub unsafe fn preserve_event_update_leftovers(&mut self) { + unsafe fn copy_segment( + label: &str, + value: &mut T, + stack: &NockStack, + pma: &mut Pma, + trace_pma: bool, + segment: Option<&mut PmaCopySegment>, + ) -> PmaCopySegment { + if trace_pma { + debug!("pma-copy: {label}"); + } + let before = pma.alloc_offset(); + let start = Instant::now(); + value.copy_to_pma(stack, pma); + let copy_segment = PmaCopySegment { + elapsed: start.elapsed(), + alloc_words: pma.alloc_offset().saturating_sub(before), + }; + if let Some(segment) = segment { + *segment = copy_segment; + } + copy_segment + } + + unsafe fn copy_durable_state_to_pma( + &mut self, + pma: &mut Pma, + trace_pma: bool, + detail: Option<&mut PmaCopyDetail>, + ) -> PmaCopySegment { + let stack = &self.context.stack; + Self::copy_segment( + "arvo", + &mut self.arvo, + stack, + pma, + trace_pma, + detail.map(|detail| &mut detail.arvo), + ) + } + + unsafe fn copy_runtime_state_to_pma( + &mut self, + pma: &mut Pma, + trace_pma: bool, + mut detail: Option<&mut PmaCopyDetail>, + ) { + let event_num = self.event_num.load(Ordering::SeqCst); + let context = &mut self.context; + let stack = &context.stack; + + let segment = Self::copy_segment("warm", &mut context.warm, stack, pma, trace_pma, None); + if let Some(detail) = detail.as_mut() { + detail.warm = segment; + } + log_pma_preserve_component(event_num, "pma_warm_copy", segment); + + let segment = Self::copy_segment( + "test_jets", &mut context.test_jets, stack, pma, trace_pma, None, + ); + if let Some(detail) = detail.as_mut() { + detail.test_jets = segment; + } + log_pma_preserve_component(event_num, "pma_test_jets_copy", segment); + + let segment = Self::copy_segment("hot", &mut context.hot, stack, pma, trace_pma, None); + if let Some(detail) = detail.as_mut() { + detail.hot = segment; + } + log_pma_preserve_component(event_num, "pma_hot_copy", segment); + + let segment = Self::copy_segment("cache", &mut context.cache, stack, pma, trace_pma, None); + if let Some(detail) = detail.as_mut() { + detail.cache = segment; + } + log_pma_preserve_component(event_num, "pma_cache_copy", segment); + + let segment = Self::copy_segment("cold", &mut context.cold, stack, pma, trace_pma, None); + if let Some(detail) = detail.as_mut() { + detail.cold = segment; + } + log_pma_preserve_component(event_num, "pma_cold_copy", segment); + } + + unsafe fn copy_from_pma_segment( + value: &mut T, + from_pma: &Pma, + to_pma: &mut Pma, + ) -> PmaCopySegment { + let before = to_pma.alloc_offset(); + let start = Instant::now(); + value.copy_from_pma(from_pma, to_pma); + PmaCopySegment { + elapsed: start.elapsed(), + alloc_words: to_pma.alloc_offset().saturating_sub(before), + } + } + + unsafe fn copy_runtime_state_from_pma( + &mut self, + from_pma: &Pma, + to_pma: &mut Pma, + ) -> PmaCopyDetail { + let context = &mut self.context; + PmaCopyDetail { + warm: Self::copy_from_pma_segment(&mut context.warm, from_pma, to_pma), + test_jets: Self::copy_from_pma_segment(&mut context.test_jets, from_pma, to_pma), + hot: Self::copy_from_pma_segment(&mut context.hot, from_pma, to_pma), + cache: Self::copy_from_pma_segment(&mut context.cache, from_pma, to_pma), + cold: Self::copy_from_pma_segment(&mut context.cold, from_pma, to_pma), + arvo: PmaCopySegment::default(), + } + } + + #[cfg(feature = "pma-assert")] + fn assert_durable_state_in_pma(&self, pma: &Pma) { + self.arvo.assert_in_pma(pma); + } + + #[cfg(feature = "pma-assert")] + fn assert_runtime_state_in_pma(&self, pma: &Pma) { + self.context.warm.assert_in_pma(pma); + self.context.test_jets.assert_in_pma(pma); + self.context.hot.assert_in_pma(pma); + self.context.cache.assert_in_pma(pma); + self.context.cold.assert_in_pma(pma); + } + + unsafe fn preserve_runtime_state_in_stack(&mut self, event_num: u64) { let stack = &mut self.context.stack; + let start = Instant::now(); stack.preserve(&mut self.context.warm); + log_preserve_component(event_num, "warm", start.elapsed()); + + let start = Instant::now(); stack.preserve(&mut self.context.test_jets); + log_preserve_component(event_num, "test_jets", start.elapsed()); + + let start = Instant::now(); stack.preserve(&mut self.context.hot); + log_preserve_component(event_num, "hot", start.elapsed()); + + let start = Instant::now(); stack.preserve(&mut self.context.cache); + log_preserve_component(event_num, "cache", start.elapsed()); + + let start = Instant::now(); stack.preserve(&mut self.context.cold); + log_preserve_component(event_num, "cold", start.elapsed()); + } + + unsafe fn preserve_persistent_state_in_stack(&mut self, event_num: u64) { + self.preserve_runtime_state_in_stack(event_num); + let stack = &mut self.context.stack; + let start = Instant::now(); stack.preserve(&mut self.arvo); - stack.flip_top_frame(0); + log_preserve_component(event_num, "arvo", start.elapsed()); + } + + fn persist_pma_metadata(&self, pma: &Pma) { + if let Err(err) = self.persist_pma_metadata_strict(pma) { + let Some(meta_path) = self.pma_meta_path.as_ref() else { + return; + }; + error!( + path = %meta_path.display(), + pma_path = %pma.path().display(), + error = %err, + "fatal PMA metadata persistence failure" + ); + std::process::abort(); + } + } + + fn persist_pma_state(&self, pma: &Pma) { + pma.persist_metadata(); + if let Err(err) = self.sync_pma_data_strict(pma, "poke_cleanup_pma_fdatasync") { + error!( + path = %pma.path().display(), + error = %err, + "fatal PMA slab fdatasync failure after SQLite commit" + ); + std::process::abort(); + } + self.persist_pma_metadata(pma); + } + + fn persist_pma_metadata_strict(&self, pma: &Pma) -> Result<()> { + let Some(meta_path) = self.pma_meta_path.as_ref() else { + return Ok(()); + }; + let space = self.context.stack.noun_space(); + let kernel_state = self + .arvo + .in_space(&space) + .slot(STATE_AXIS) + .map(|handle| handle.noun()) + .map_err(|err| { + CrownError::SaveError(format!( + "failed to resolve kernel root for PMA metadata at {}: {err:?}", + meta_path.display() + )) + })?; + let kernel_state_raw = unsafe { kernel_state.as_raw() }; + let event_num = self.event_num.load(Ordering::SeqCst); + let pma_reserved_words = u64::try_from(pma.reserved_words()).map_err(|_| { + CrownError::SaveError( + "PMA reservation exceeds u64 while persisting metadata".to_string(), + ) + })?; + let meta = PmaPersistMetadata::new( + self.ker_hash, event_num, kernel_state_raw, pma_reserved_words, + ); + meta.save_to_path(meta_path)?; + Ok(()) + } + + fn sync_pma_data_strict(&self, pma: &Pma, context: &str) -> Result<()> { + durability::sync_path_data(pma.path(), context)?; + Ok(()) + } + + fn flush_pma_state_for_shutdown(&self) -> Result<()> { + let Some(pma) = self.pma.as_ref() else { + return Ok(()); + }; + if pma.is_growth_poisoned() { + return Err(CrownError::SaveError( + "PMA growth failed after partial file growth; restart the node so PMA growth journal recovery can complete before publishing metadata".to_string(), + )); + } + let event_num = self.event_num.load(Ordering::SeqCst); + info!( + event_num, + path = %pma.path().display(), + "shutdown PMA flush start" + ); + pma.persist_metadata(); + pma.sync_all()?; + self.sync_pma_data_strict(pma, "shutdown_pma_fdatasync")?; + self.persist_pma_metadata_strict(pma)?; + info!( + event_num, + path = %pma.path().display(), + "shutdown PMA flush done" + ); + Ok(()) + } + + fn maybe_pma_gc(&mut self, pma: Pma) -> (Pma, bool) { + let Some(gc_state) = self.pma_gc_state.as_ref() else { + return (pma, false); + }; + if !gc_state.should_gc() { + return (pma, false); + } + + let from_path = gc_state.active_path().clone(); + let to_path = gc_state.inactive_path().clone(); + let from_meta_path = pma_meta_path(&from_path); + let to_meta_path = pma_meta_path(&to_path); + let gc_words = gc_state.words; + let gc_reserved_words = gc_state.reserved_words; + let event_num = self.event_num.load(Ordering::SeqCst); + if !self.has_pma_gc_recovery_anchor() { + warn!( + event_num, + from = %from_path.display(), + to = %to_path.display(), + "pma-gc: skipped because no ready snapshot recovery anchor is available" + ); + return (pma, false); + } + + let gc_start = Instant::now(); + let from_alloc = pma.alloc_offset(); + info!( + "pma-gc: start: event_num={} from={} to={} from_alloc_words={}", + event_num, + from_path.display(), + to_path.display(), + from_alloc + ); + + if let Err(err) = Self::remove_pma_meta_for_gc(&to_meta_path, "pma_gc_inactive_meta_remove") + { + warn!( + event_num, + path = %to_meta_path.display(), + error = %err, + "pma-gc: skipped because inactive PMA metadata could not be removed" + ); + return (pma, false); + } + + let create_start = Instant::now(); + let to_pma_result = match gc_reserved_words { + Some(reserved_words) => { + Pma::new_with_reserved(gc_words, reserved_words, to_path.clone()) + } + None => Pma::new(gc_words, to_path.clone()), + }; + let mut to_pma = match to_pma_result { + Ok(pma) => pma, + Err(err) => { + warn!( + "pma-gc: failed to create new PMA slab at {}: {err}", + to_path.display() + ); + return (pma, false); + } + }; + let create_elapsed = create_start.elapsed(); + + if let Err(err) = + Self::remove_pma_meta_for_gc(&from_meta_path, "pma_gc_active_meta_invalidate") + { + warn!( + event_num, + path = %from_meta_path.display(), + error = %err, + "pma-gc: skipped because active PMA metadata could not be invalidated" + ); + return (pma, false); + } + + let arvo_start = Instant::now(); + let arvo_alloc_before = to_pma.alloc_offset(); + unsafe { + self.arvo.copy_from_pma(&pma, &mut to_pma); + } + let arvo_segment = PmaCopySegment { + elapsed: arvo_start.elapsed(), + alloc_words: to_pma.alloc_offset().saturating_sub(arvo_alloc_before), + }; + let mut runtime_segments = unsafe { self.copy_runtime_state_from_pma(&pma, &mut to_pma) }; + runtime_segments.arvo = arvo_segment; + + debug!( + "pma-gc: copy timings: arvo_ms={} warm_ms={} test_jets_ms={} hot_ms={} cache_ms={} cold_ms={} arvo_alloc_words={} warm_alloc_words={} test_jets_alloc_words={} hot_alloc_words={} cache_alloc_words={} cold_alloc_words={}", + runtime_segments.arvo.elapsed.as_millis(), + runtime_segments.warm.elapsed.as_millis(), + runtime_segments.test_jets.elapsed.as_millis(), + runtime_segments.hot.elapsed.as_millis(), + runtime_segments.cache.elapsed.as_millis(), + runtime_segments.cold.elapsed.as_millis(), + runtime_segments.arvo.alloc_words, + runtime_segments.warm.alloc_words, + runtime_segments.test_jets.alloc_words, + runtime_segments.hot.alloc_words, + runtime_segments.cache.alloc_words, + runtime_segments.cold.alloc_words + ); + + self.context + .stack + .install_pma_arena(Arc::clone(to_pma.arena())); + self.pma_meta_path = Some(to_meta_path); + self.persist_pma_state(&to_pma); + + let to_alloc = to_pma.alloc_offset(); + let pageout_start = Instant::now(); + match to_pma.advise_drop_allocated_prefix( + PMA_GC_DROP_ALLOCATED_PREFIX_NUMERATOR, PMA_GC_DROP_ALLOCATED_PREFIX_DENOMINATOR, + ) { + Ok(advised_bytes) => { + debug!( + event_num, + path = %to_pma.path().display(), + advised_bytes, + elapsed_ms = duration_ms(pageout_start.elapsed()), + "pma-gc: advised compacted PMA pages out" + ); + } + Err(err) => { + warn!( + event_num, + path = %to_pma.path().display(), + error = %err, + elapsed_ms = duration_ms(pageout_start.elapsed()), + "pma-gc: failed to advise compacted PMA pages out" + ); + } + } + if let Some(gc_state) = self.pma_gc_state.as_mut() { + gc_state.mark_gc_completed(); + } + info!( + "pma-gc: done: total_ms={} create_ms={} to_alloc_words={}", + gc_start.elapsed().as_millis(), + create_elapsed.as_millis(), + to_alloc + ); + + (to_pma, true) + } + + fn has_pma_gc_recovery_anchor(&mut self) -> bool { + let Some(event_log) = self.event_log.as_mut() else { + return false; + }; + match event_log.has_ready_snapshot() { + Ok(has_ready_snapshot) => has_ready_snapshot, + Err(err) => { + warn!("pma-gc: failed to check ready snapshot recovery anchor: {err}"); + false + } + } + } + + fn remove_pma_meta_for_gc(meta_path: &Path, context: &str) -> Result<()> { + match fs::remove_file(meta_path) { + Ok(()) => durability::sync_parent_dir(meta_path, context).map_err(|err| { + CrownError::SaveError(format!( + "failed to sync PMA metadata parent directory for {}: {err}", + meta_path.display() + )) + }), + Err(err) if err.kind() == std::io::ErrorKind::NotFound => Ok(()), + Err(err) => Err(CrownError::SaveError(format!( + "failed to remove PMA metadata at {}: {err}", + meta_path.display() + ))), + } + } + + unsafe fn flip_top_frame_and_trim_free_gap(&mut self, event_num: u64) { + let flip_start = Instant::now(); + self.context.stack.flip_top_frame(0); + log_preserve_component(event_num, "stack_flip_top_frame", flip_start.elapsed()); + self.trim_stack_free_gap(event_num); + } + + fn trim_stack_free_gap(&self, event_num: u64) { + if !stack_free_gap_trim_enabled() { + return; + } + let trim_start = Instant::now(); + match self + .context + .stack + .advise_free_gap(NOCK_STACK_FREE_GAP_TRIM_THRESHOLD_WORDS) + { + Ok(advice) if advice.advised_bytes > 0 => { + debug!( + event_num, + free_gap_words = advice.free_gap_words, + free_gap_mib = words_to_mib(advice.free_gap_words), + advised_bytes = advice.advised_bytes, + advised_mib = bytes_to_mib(advice.advised_bytes), + threshold_mib = bytes_to_mib(NOCK_STACK_FREE_GAP_TRIM_THRESHOLD_BYTES), + elapsed_ms = duration_ms(trim_start.elapsed()), + "nockstack free-gap madvise done" + ); + } + Ok(_) => {} + Err(err) => { + warn!( + event_num, + error = %err, + elapsed_ms = duration_ms(trim_start.elapsed()), + "nockstack free-gap madvise failed" + ); + } + } + } + + /// Preserves leftovers after an event update. + /// + /// # Safety + /// + /// This function is unsafe because it modifies the Serf's state directly. + #[tracing::instrument(level = "info", skip_all)] + pub unsafe fn preserve_event_update_leftovers(&mut self) -> Option { + self.preserve_event_update_leftovers_with_pre_persist(|_| {}) + } + + /// Preserves leftovers after an event update, with a hook after durable PMA copy succeeds + /// but before durable PMA metadata is published. + /// + /// # Safety + /// + /// This function is unsafe because it modifies the Serf's state directly. + #[tracing::instrument(level = "info", skip_all)] + pub unsafe fn preserve_event_update_leftovers_with_pre_persist( + &mut self, + pre_persist: F, + ) -> Option + where + F: FnOnce(&mut Self), + { + let event_num = self.event_num.load(Ordering::SeqCst); + assert!( + self.context.scry_stack.is_direct(), + "scry_stack must be cleared before resetting the NockStack" + ); + if std::env::var_os("NOCK_STACK_TIMING_DETAIL").is_some() { + let total_words = self.context.stack.arena().words() as u64; + let least_space = self.context.stack.least_space() as u64; + let used_words = total_words.saturating_sub(least_space); + let used_mib = (used_words as f64 * 8.0) / (1024.0 * 1024.0); + info!( + "stack-usage: used_words={} used_mib={:.3} least_space_words={} total_words={} event_num={}", + used_words, used_mib, least_space, total_words, event_num + ); + } + if self.pma.is_some() { + let trace_pma = std::env::var_os("NOCK_PMA_TRACE").is_some(); + let detail_enabled = std::env::var_os("NOCK_PMA_TIMING_DETAIL").is_some(); + let mut pma = self.pma.take().expect("checked is_some"); + let mut detail = if detail_enabled { + Some(PmaCopyDetail::default()) + } else { + None + }; + let arvo_segment = self.copy_durable_state_to_pma(&mut pma, trace_pma, detail.as_mut()); + log_pma_preserve_component(event_num, "pma_arvo_copy", arvo_segment); + #[cfg(feature = "pma-assert")] + { + // Enforce: durable PMA state must not reference the NockStack. + self.assert_durable_state_in_pma(&pma); + } + + pre_persist(self); + + let persist_start = Instant::now(); + self.persist_pma_state(&pma); + log_preserve_component(event_num, "pma_persist_state", persist_start.elapsed()); + + unsafe { + self.copy_runtime_state_to_pma(&mut pma, trace_pma, detail.as_mut()); + } + #[cfg(feature = "pma-assert")] + { + self.assert_runtime_state_in_pma(&pma); + } + + let gc_start = Instant::now(); + let (pma_after_gc, did_pma_gc) = self.maybe_pma_gc(pma); + log_preserve_component(event_num, "pma_gc", gc_start.elapsed()); + pma = pma_after_gc; + if did_pma_gc { + #[cfg(feature = "pma-assert")] + { + self.assert_runtime_state_in_pma(&pma); + } + } + self.flip_top_frame_and_trim_free_gap(event_num); + self.pma = Some(pma); + detail + } else { + pre_persist(self); + self.preserve_persistent_state_in_stack(event_num); + self.flip_top_frame_and_trim_free_gap(event_num); + None + } } /// Returns a mutable reference to the Nock stack. @@ -1249,17 +3403,45 @@ impl Serf { } } -fn slot(noun: Noun, axis: u64) -> Result { - Ok(noun.slot(axis)?) -} - #[cfg(test)] mod tests { use std::fs; use std::path::Path; + use blake3::hash; + use bytes::Bytes; + use nockvm::jets::cold::Cold; + use nockvm::jets::hot::HotEntry; + use nockvm::jets::warm::Warm; + use tempfile::TempDir; + use super::*; + const DUMB_KERNEL_JAM: &[u8] = include_bytes!(env!("DUMB_JAM_PATH")); + + fn dummy_serf() -> Serf { + let mut stack = NockStack::new(NOCK_STACK_SIZE_TINY, 0); + let cold = Cold::new(&mut stack); + let hot_state: [HotEntry; 0] = []; + let context = create_context(stack, &hot_state, cold, None, vec![]); + let cancel_token = context.cancel_token(); + Serf { + ker_hash: Hash::from([0; 32]), + arvo: D(0), + context, + pma: None, + pma_meta_path: None, + pma_gc_state: None, + snapshot_creation_enabled: false, + rotating_snapshot_interval_event_time: None, + cumulative_event_processing_time_since_snapshot: Duration::ZERO, + event_log: None, + cancel_token, + event_num: Arc::new(AtomicU64::new(0)), + metrics: None, + } + } + async fn setup_kernel(jam: &str) -> Kernel { let jam_path = Path::new(env!("CARGO_MANIFEST_DIR")) .join("..") @@ -1267,11 +3449,176 @@ mod tests { .join(jam); let jam_bytes = fs::read(jam_path).unwrap_or_else(|_| panic!("Failed to read {} file", jam)); - Kernel::load(&jam_bytes, None, vec![], TraceOpts::default()) + Kernel::load(&jam_bytes, None, vec![], TraceOpts::default(), None) .await .expect("Could not load kernel") } + fn bench_serf(kernel_bytes: &[u8]) -> Serf { + let stack = NockStack::new(NOCK_STACK_SIZE_TINY, 0); + Serf::new( + stack, + None, + None, + false, + None, + false, + None, + None, + None::, + kernel_bytes, + &[], + vec![], + TraceOpts::default(), + NOCK_STACK_SIZE_TINY, + ) + .expect("bench serf should initialize") + } + + fn kernel_state_slab_from_serf(serf: &mut Serf) -> NounSlab { + let space = serf.context.stack.noun_space(); + let kernel_state_noun = serf + .arvo + .in_space(&space) + .slot(STATE_AXIS) + .map(|handle| handle.noun()) + .expect("resolve kernel state noun"); + let mut kernel_state_slab = NounSlab::new(); + kernel_state_slab.copy_into(kernel_state_noun, &space); + kernel_state_slab + } + + #[derive(Debug)] + struct ColdPersistenceBenchReport { + first_boot_with_cold_ready: Duration, + load_kernel_state_into_fresh_vm: Duration, + cold_to_noun: Duration, + noun_to_jam: Duration, + save_jam_to_disk: Duration, + cue_noun_from_disk: Duration, + reinject_cued_cold_into_vm: Duration, + jam_bytes: usize, + } + + impl ColdPersistenceBenchReport { + fn print(&self) { + println!("cold_state_persistence_bench"); + println!(" kernel_asset=assets/dumb.jam"); + println!( + " first_boot_with_cold_ready_ms={:.3}", + duration_ms(self.first_boot_with_cold_ready) + ); + println!( + " load_kernel_state_into_fresh_vm_ms={:.3}", + duration_ms(self.load_kernel_state_into_fresh_vm) + ); + println!(" cold_to_noun_ms={:.3}", duration_ms(self.cold_to_noun)); + println!(" noun_to_jam_ms={:.3}", duration_ms(self.noun_to_jam)); + println!( + " save_jam_to_disk_ms={:.3}", + duration_ms(self.save_jam_to_disk) + ); + println!( + " cue_noun_from_disk_ms={:.3}", + duration_ms(self.cue_noun_from_disk) + ); + println!( + " reinject_cued_cold_into_vm_ms={:.3}", + duration_ms(self.reinject_cued_cold_into_vm) + ); + println!(" jam_bytes={}", self.jam_bytes); + } + } + + #[test] + #[ignore = "benchmark harness"] + #[cfg_attr(miri, ignore)] + fn cold_state_noun_jam_persistence_bench() { + let boot_start = Instant::now(); + let mut serf = bench_serf(DUMB_KERNEL_JAM); + let first_boot_with_cold_ready = boot_start.elapsed(); + let kernel_state_slab = kernel_state_slab_from_serf(&mut serf); + + let mut load_serf = bench_serf(DUMB_KERNEL_JAM); + let load_state_noun = kernel_state_slab.copy_to_stack(load_serf.stack()); + let load_kernel_state_into_fresh_vm_start = Instant::now(); + let _loaded_arvo = load_serf + .load(load_state_noun) + .expect("load saved kernel state into fresh vm"); + let load_kernel_state_into_fresh_vm = load_kernel_state_into_fresh_vm_start.elapsed(); + + let cold_to_noun_start = Instant::now(); + let cold_noun = serf.context.cold.into_noun(serf.stack()); + let cold_to_noun = cold_to_noun_start.elapsed(); + + let noun_to_jam_start = Instant::now(); + let cold_jam = { + let space = serf.context.stack.noun_space(); + NockJammer::jam(cold_noun, &space) + }; + let noun_to_jam = noun_to_jam_start.elapsed(); + + let temp_dir = TempDir::new().expect("create temp dir"); + let cold_jam_path = temp_dir.path().join("cold-state.jam"); + + let save_jam_to_disk_start = Instant::now(); + durability::write_atomic(&cold_jam_path, &cold_jam, "cold_state_bench_write") + .expect("persist cold jam"); + let save_jam_to_disk = save_jam_to_disk_start.elapsed(); + + let cue_noun_from_disk_start = Instant::now(); + let jam_from_disk = Bytes::from(fs::read(&cold_jam_path).expect("read cold jam")); + let mut cued_cold_slab: NounSlab = NounSlab::new(); + let cued_cold_noun = cued_cold_slab + .cue_into(jam_from_disk) + .expect("cue cold noun"); + cued_cold_slab.set_root(cued_cold_noun); + let cue_noun_from_disk = cue_noun_from_disk_start.elapsed(); + + let mut inject_stack = NockStack::new(NOCK_STACK_SIZE_TINY, 0); + let inject_cold = Cold::new(&mut inject_stack); + let mut inject_context = + create_context(inject_stack, URBIT_HOT_STATE, inject_cold, None, vec![]); + + let reinject_cued_cold_into_vm_start = Instant::now(); + let cued_cold_in_vm = cued_cold_slab.copy_to_stack(&mut inject_context.stack); + let space = inject_context.stack.noun_space(); + let cold_vecs = Cold::from_noun(&mut inject_context.stack, &cued_cold_in_vm, &space) + .expect("decode cold noun"); + let rebuilt_cold = Cold::from_vecs( + &mut inject_context.stack, cold_vecs.0, cold_vecs.1, cold_vecs.2, + ); + inject_context.cold = rebuilt_cold; + let hot = inject_context.hot; + let test_jets = inject_context.test_jets; + inject_context.warm = Warm::init( + &mut inject_context.stack, &mut inject_context.cold, &hot, &test_jets, + ); + let reinject_cued_cold_into_vm = reinject_cued_cold_into_vm_start.elapsed(); + + let reinjected_cold_noun = inject_context.cold.into_noun(&mut inject_context.stack); + let reinjected_cold_jam = { + let space = inject_context.stack.noun_space(); + NockJammer::jam(reinjected_cold_noun, &space) + }; + assert!( + !reinjected_cold_jam.is_empty(), + "re-injected cold state should remain serializable" + ); + + let report = ColdPersistenceBenchReport { + first_boot_with_cold_ready, + load_kernel_state_into_fresh_vm, + cold_to_noun, + noun_to_jam, + save_jam_to_disk, + cue_noun_from_disk, + reinject_cued_cold_into_vm, + jam_bytes: cold_jam.len(), + }; + report.print(); + } + // Convert this to an integration test and feed it the kernel.jam from Choo in CI/CD // https://www.youtube.com/watch?v=4m1EFMoRFvY // #[test] @@ -1288,6 +3635,121 @@ mod tests { // let (kernel, _temp_dir) = setup_kernel("kernel.jam"); // // Add your custom assertions here to test the kernel's behavior // } + + fn checkpoint_with_state_too_large_for_stack(state_bytes_len: usize) -> SaveableCheckpoint { + let mut state = NounSlab::new(); + let state_bytes = Bytes::from(vec![0x41; state_bytes_len]); + let state_root = Atom::from_bytes(&mut state, &state_bytes).as_noun(); + state.set_root(state_root); + + let mut cold_stack = NockStack::new(1024, 0); + let cold_noun = Cold::new(&mut cold_stack).into_noun(&mut cold_stack); + let cold_space = cold_stack.noun_space(); + let mut cold = NounSlab::new(); + let cold_root = cold.copy_into(cold_noun, &cold_space); + cold.set_root(cold_root); + + SaveableCheckpoint { + ker_hash: hash(b"stack-oom-repro"), + event_num: 1, + state, + cold, + } + } + + #[tokio::test] + async fn serf_thread_init_stack_oom_is_not_collapsed_into_oneshot_error() { + let stack_words = 9 * 1024; + let checkpoint = checkpoint_with_state_too_large_for_stack((stack_words - 2) * 8); + let result = SerfThread::::new( + Vec::new(), + Some(checkpoint), + Vec::new(), + stack_words, + None, + Vec::new(), + TraceOpts::default(), + ) + .await; + + let Err(err) = result else { + panic!("expected serf initialization to fail from stack exhaustion"); + }; + let err = err.to_string(); + + assert!( + !err.contains("oneshot channel error"), + "serf stack exhaustion must be reported directly, not as {err:?}" + ); + assert!( + err.contains("Out of memory") || err.contains("allocation"), + "expected a stack allocation error, got {err:?}" + ); + assert!( + err.contains("--stack-size") && err.contains("checkpoint event_num="), + "expected operator guidance and checkpoint context, got {err:?}" + ); + } + + /// Create a structurally-valid PMA slab plus its `.meta` sidecar recording + /// `ker_hash`/`event_num`, as if written by some prior kernel. + fn write_valid_pma(path: &Path, ker_hash: Hash, event_num: u64) { + let pma_reserved_words = { + let pma = Pma::new(1 << 10, path.to_path_buf()).expect("create test PMA"); + let pma_reserved_words = + u64::try_from(pma.reserved_words()).expect("test PMA reservation fits in u64"); + pma.sync_all().expect("sync test PMA"); + pma_reserved_words + }; + PmaPersistMetadata::new(ker_hash, event_num, 0, pma_reserved_words) + .save_to_path(&pma_meta_path(path)) + .expect("write test PMA metadata"); + } + + // Regression for the reported symptom: after a kernel upgrade the hash gate + // made *both* slabs report `None`, so selection fell through to its + // `Slab0` default and then reported slab 0's metadata as "missing or + // invalid" even though the live state was in slab 1. + #[test] + fn select_active_pma_slab_is_kernel_hash_agnostic_and_picks_latest_event() { + let dir = TempDir::new().expect("temp dir"); + let path_0 = dir.path().join("0.pma"); + let path_1 = dir.path().join("1.pma"); + // The two slabs were written by *different* kernels. + write_valid_pma(&path_0, hash(b"kernel-a"), 1); + write_valid_pma(&path_1, hash(b"kernel-b"), 5); + let paths = PmaSlabPaths { + path_0: path_0.clone(), + path_1: path_1.clone(), + }; + assert!( + matches!(select_active_pma_slab(&paths), PmaSlab::Slab1), + "slab selection must pick the most-recent slab regardless of kernel hash" + ); + } + + // A kernel-hash mismatch must not invalidate the PMA; the state is reused + // and migrated by the current kernel, mirroring checkpoint/snapshot restore. + #[test] + fn inspect_existing_pma_is_valid_on_kernel_hash_mismatch() { + let dir = TempDir::new().expect("temp dir"); + let path_0 = dir.path().join("0.pma"); + let path_1 = dir.path().join("1.pma"); + write_valid_pma(&path_0, hash(b"old-kernel-bytes"), 7); + // Current kernel bytes hash to something different from the stored meta. + let status = inspect_existing_pma(path_0.as_path(), path_1.as_path(), b"new-kernel-bytes"); + match status { + ExistingPmaStatus::Valid { event_num, .. } => { + assert_eq!( + event_num, 7, + "must reuse PMA state at its recorded event_num" + ); + } + other => { + panic!("kernel-hash mismatch must NOT invalidate the PMA, got {other:?}") + } + } + } } pub trait SerfCheckpoint: Send { @@ -1316,14 +3778,15 @@ impl SerfCheckpoint for SaveableCheckpoint { // Cold state has nouns in it which are *not* copied in into_noun // TODO: FIX THIS FOOTGUN let cold_stack_noun = cold_state.into_noun(stack); + let space = stack.noun_space(); let mut cold_slab: NounSlab = NounSlab::new(); - let cold_copy = cold_slab.copy_into(cold_stack_noun); + let cold_copy = cold_slab.copy_into(cold_stack_noun, &space); cold_slab.set_root(cold_copy); let cold_noun_elapsed = cold_noun_start.elapsed(); let state_copy_start = Instant::now(); let mut state_slab: NounSlab = NounSlab::new(); - let state_copy = state_slab.copy_into(kernel_state); + let state_copy = state_slab.copy_into(kernel_state, &space); state_slab.set_root(state_copy); let state_copy_elapsed = state_copy_start.elapsed(); diff --git a/crates/nockapp/src/lib.rs b/crates/nockapp/src/lib.rs index 925a40b9a..5a3790dc5 100644 --- a/crates/nockapp/src/lib.rs +++ b/crates/nockapp/src/lib.rs @@ -1,3 +1,4 @@ +#![feature(negative_impls)] #![feature(slice_pattern)] // Allow unwrap in test code - standard practice for test assertions #![cfg_attr(test, allow(clippy::unwrap_used))] @@ -15,10 +16,12 @@ //! - `utils`: Errors, misc functions and extensions. //! pub mod drivers; +pub(crate) mod event_log; pub mod kernel; pub mod nockapp; pub mod noun; pub mod observability; +pub(crate) mod snapshot; pub mod utils; use std::path::PathBuf; @@ -68,5 +71,131 @@ pub fn system_data_dir() -> PathBuf { home_dir.join(".nockapp") } -/// Default size for the Nock stack (1 GB) -pub const DEFAULT_NOCK_STACK_SIZE: usize = 1 << 27; +/// Default size for the Nock stack. +pub const DEFAULT_NOCK_STACK_SIZE: usize = nockvm::mem::NOCK_STACK_SIZE_TINY; + +#[cfg(test)] +pub mod test_support { + use std::fs; + use std::path::{Path, PathBuf}; + use std::sync::Arc; + + use nockvm::mem::{NockStack, NOCK_STACK_SIZE_TINY}; + use nockvm::pma::Pma; + use uuid::Uuid; + + /// Per-test PMA filesystem sandbox. + /// + /// PMA tests should allocate their persistence files under a unique directory + /// instead of serializing the whole test crate behind a global lock. This keeps + /// tests parallel-safe while making the persistence boundary explicit. + pub struct TestPmaSandbox { + path: PathBuf, + } + + impl TestPmaSandbox { + pub fn new() -> Self { + let root = PathBuf::from("/tmp/.test-pma"); + fs::create_dir_all(&root).expect("create PMA test root"); + + for _ in 0..16 { + let path = root.join(Uuid::new_v4().to_string()); + match fs::create_dir(&path) { + Ok(()) => return Self { path }, + Err(err) if err.kind() == std::io::ErrorKind::AlreadyExists => continue, + Err(err) => panic!("create PMA test sandbox {}: {err}", path.display()), + } + } + + panic!( + "failed to allocate unique PMA test sandbox under {}", + root.display() + ); + } + + pub fn path(&self) -> &Path { + &self.path + } + } + + impl Default for TestPmaSandbox { + fn default() -> Self { + Self::new() + } + } + + impl Drop for TestPmaSandbox { + fn drop(&mut self) { + let _ = fs::remove_dir_all(&self.path); + } + } + + /// Reified PMA-equipped stack for tests that need stack + PMA noun resolution. + pub struct TestPmaStack { + stack: NockStack, + pma: Pma, + // Keep the sandbox last so file-backed mappings are dropped before the + // directory cleanup runs. + sandbox: TestPmaSandbox, + } + + impl TestPmaStack { + pub fn with_words(stack_words: usize, pma_words: usize) -> Self { + let sandbox = TestPmaSandbox::new(); + let pma_path = sandbox.path().join("test.pma"); + let pma = Pma::new(pma_words, pma_path).expect("create test PMA"); + let mut stack = NockStack::new(stack_words, 0); + stack.install_pma_arena(Arc::clone(pma.arena())); + Self { + stack, + pma, + sandbox, + } + } + + pub fn stack(&self) -> &NockStack { + &self.stack + } + + pub fn stack_mut(&mut self) -> &mut NockStack { + &mut self.stack + } + + pub fn pma(&self) -> &Pma { + &self.pma + } + + pub fn pma_mut(&mut self) -> &mut Pma { + &mut self.pma + } + + pub fn stack_pma_mut(&mut self) -> (&mut NockStack, &mut Pma) { + (&mut self.stack, &mut self.pma) + } + + pub fn pma_path(&self) -> &Path { + self.pma.path().as_path() + } + + pub fn sandbox_path(&self) -> &Path { + self.sandbox.path() + } + + pub fn into_sandbox(self) -> TestPmaSandbox { + let Self { + stack, + pma, + sandbox, + } = self; + drop(stack); + drop(pma); + sandbox + } + } + + impl Default for TestPmaStack { + fn default() -> Self { + Self::with_words(NOCK_STACK_SIZE_TINY, 4096) + } + } +} diff --git a/crates/nockapp/src/nockapp/actors/mod.rs b/crates/nockapp/src/nockapp/actors/mod.rs index 882de23e7..8214f07ec 100644 --- a/crates/nockapp/src/nockapp/actors/mod.rs +++ b/crates/nockapp/src/nockapp/actors/mod.rs @@ -1,2 +1 @@ pub(crate) mod kernel; -pub(crate) mod save; diff --git a/crates/nockapp/src/nockapp/actors/save.rs b/crates/nockapp/src/nockapp/actors/save.rs deleted file mode 100644 index d21b9f447..000000000 --- a/crates/nockapp/src/nockapp/actors/save.rs +++ /dev/null @@ -1,107 +0,0 @@ -use std::path::PathBuf; -use std::sync::atomic::{AtomicBool, Ordering}; -use std::sync::Arc; -use std::time::Duration; - -use tokio::fs; -use tokio::io::AsyncWriteExt as _; -use tokio::sync::mpsc; -use tracing::{error, trace}; - -use crate::kernel::checkpoint::JammedCheckpoint; -use crate::nockapp::NockAppError; - -// Save actor messages -pub(crate) enum SaveMessage { - SaveCheckpoint(JammedCheckpoint), - // SaveCheckpoint(JammedCheckpoint, Option>), - // SaveCheckpointWithResponse(JammedCheckpoint, tokio::sync::oneshot::Sender), - // IntervalSaveCheckpoint(JammedCheckpoint), - Shutdown, -} - -// enum ShutdownMessage { -// Shutdown, -// ShutdownNow, -// } - -pub(crate) struct SaveActor { - receiver: mpsc::Receiver, - jam_paths: (PathBuf, PathBuf), - save_interval: Duration, - last_save: std::time::Instant, - buffer_toggle: Arc, - event_sender: tokio::sync::watch::Sender, -} - -impl SaveActor { - pub(crate) fn new( - receiver: mpsc::Receiver, - jam_paths: (PathBuf, PathBuf), - save_interval: Duration, - buffer_toggle: Arc, - event_sender: tokio::sync::watch::Sender, - ) -> Self { - Self { - receiver, - jam_paths, - save_interval, - last_save: std::time::Instant::now(), - buffer_toggle, - event_sender, - } - } - - pub(crate) async fn run(mut self) { - while let Some(msg) = self.receiver.recv().await { - match msg { - SaveMessage::SaveCheckpoint(checkpoint) => { - if let Err(e) = self.handle_save(checkpoint).await { - error!("Save error: {:?}", e); - } - } - SaveMessage::Shutdown => break, - } - } - } - - async fn handle_save(&mut self, checkpoint: JammedCheckpoint) -> Result<(), NockAppError> { - // Enforce minimum interval between saves - let elapsed = self.last_save.elapsed(); - if elapsed < self.save_interval { - tokio::time::sleep(self.save_interval - elapsed).await; - } - - let bytes = checkpoint.encode()?; - let path = if self.buffer_toggle.load(Ordering::SeqCst) { - &self.jam_paths.1 - } else { - &self.jam_paths.0 - }; - - let mut file = fs::File::create(path) - .await - .map_err(NockAppError::SaveError)?; - - file.write_all(&bytes) - .await - .map_err(NockAppError::SaveError)?; - - file.sync_all().await.map_err(NockAppError::SaveError)?; - - trace!( - "Write to {:?} successful, ker_hash: {}, event: {}", - path.display(), - checkpoint.ker_hash, - checkpoint.event_num - ); - - // Flip toggle after successful write - self.buffer_toggle - .store(!self.buffer_toggle.load(Ordering::SeqCst), Ordering::SeqCst); - self.event_sender.send(checkpoint.event_num)?; - self.last_save = std::time::Instant::now(); - - Ok(()) - } -} diff --git a/crates/nockapp/src/nockapp/artifact.rs b/crates/nockapp/src/nockapp/artifact.rs new file mode 100644 index 000000000..0433a9250 --- /dev/null +++ b/crates/nockapp/src/nockapp/artifact.rs @@ -0,0 +1,184 @@ +use blake3::Hash; +use thiserror::Error; + +const CHECKPOINT_ADVICE: &str = + "Restore the file from a known-good peer or move it to a backup directory so Nockchain can try another checkpoint."; + +#[derive(Debug, Error)] +pub enum ArtifactError { + #[error("{format} is malformed: {details}. {advice}")] + Malformed { + format: &'static str, + details: String, + advice: &'static str, + }, + #[error( + "{format} has unsupported version {version}; supported versions: {supported}. Use a compatible Nockchain binary or regenerate the artifact." + )] + UnsupportedVersion { + format: &'static str, + version: u32, + supported: &'static str, + }, + #[error( + "{format} has invalid magic bytes; expected {expected}. Use the correct artifact type, or regenerate the artifact from a known-good node." + )] + InvalidMagic { + format: &'static str, + expected: &'static str, + }, +} + +impl ArtifactError { + pub fn malformed(format: &'static str, details: impl Into) -> Self { + Self::Malformed { + format, + details: details.into(), + advice: CHECKPOINT_ADVICE, + } + } + + pub fn malformed_with_advice( + format: &'static str, + details: impl Into, + advice: &'static str, + ) -> Self { + Self::Malformed { + format, + details: details.into(), + advice, + } + } +} + +pub(crate) struct CheckedReader<'a> { + input: &'a [u8], + offset: usize, + format: &'static str, + advice: &'static str, +} + +impl<'a> CheckedReader<'a> { + pub(crate) fn new(input: &'a [u8], format: &'static str) -> Self { + Self { + input, + offset: 0, + format, + advice: CHECKPOINT_ADVICE, + } + } + + pub(crate) fn new_with_advice( + input: &'a [u8], + format: &'static str, + advice: &'static str, + ) -> Self { + Self { + input, + offset: 0, + format, + advice, + } + } + + pub(crate) fn read_u64(&mut self, field: &'static str) -> Result { + match self.read_varint_discriminant(field)? { + byte @ 0..=250 => Ok(u64::from(byte)), + 251 => { + let bytes = self.read_array::<2>(field)?; + Ok(u64::from(u16::from_le_bytes(bytes))) + } + 252 => { + let bytes = self.read_array::<4>(field)?; + Ok(u64::from(u32::from_le_bytes(bytes))) + } + 253 => { + let bytes = self.read_array::<8>(field)?; + Ok(u64::from_le_bytes(bytes)) + } + 254 => Err(self.malformed(format!( + "{field} uses a u128 varint marker where u64 was expected" + ))), + _ => Err(self.malformed(format!("{field} uses a reserved varint marker"))), + } + } + + pub(crate) fn read_u32(&mut self, field: &'static str) -> Result { + let value = self.read_u64(field)?; + u32::try_from(value) + .map_err(|_| self.malformed(format!("{field} value {value} does not fit in u32"))) + } + + pub(crate) fn read_bool(&mut self, field: &'static str) -> Result { + match self.read_byte(field)? { + 0 => Ok(false), + 1 => Ok(true), + value => Err(self.malformed(format!("{field} has invalid boolean value {value}"))), + } + } + + pub(crate) fn read_hash(&mut self, field: &'static str) -> Result { + let bytes = self.read_array::<32>(field)?; + Ok(Hash::from(bytes)) + } + + pub(crate) fn read_bytes(&mut self, field: &'static str) -> Result<&'a [u8], ArtifactError> { + let len = self.read_u64(field)?; + let len = usize::try_from(len) + .map_err(|_| self.malformed(format!("{field} length does not fit in usize")))?; + self.read_exact(field, len) + } + + pub(crate) fn finish(&self) -> Result<(), ArtifactError> { + let trailing = self.input.len().saturating_sub(self.offset); + if trailing == 0 { + Ok(()) + } else { + Err(self.malformed(format!("{trailing} trailing bytes after artifact payload"))) + } + } + + fn read_varint_discriminant(&mut self, field: &'static str) -> Result { + self.read_byte(field) + } + + fn read_byte(&mut self, field: &'static str) -> Result { + let byte = self + .input + .get(self.offset) + .ok_or_else(|| self.malformed(format!("{field} is missing")))?; + self.offset += 1; + Ok(*byte) + } + + fn read_array( + &mut self, + field: &'static str, + ) -> Result<[u8; N], ArtifactError> { + let bytes = self.read_exact(field, N)?; + bytes + .try_into() + .map_err(|_| self.malformed(format!("{field} could not be read"))) + } + + fn read_exact(&mut self, field: &'static str, len: usize) -> Result<&'a [u8], ArtifactError> { + let end = self + .offset + .checked_add(len) + .ok_or_else(|| self.malformed(format!("{field} length overflows usize")))?; + if end > self.input.len() { + return Err(self.malformed(format!( + "{field} declares {len} bytes, but only {} bytes remain", + self.input.len().saturating_sub(self.offset) + ))); + } + + let bytes = &self.input[self.offset..end]; + self.offset = end; + Ok(bytes) + } + + fn malformed(&self, details: impl Into) -> ArtifactError { + ArtifactError::malformed_with_advice(self.format, details, self.advice) + } +} diff --git a/crates/nockapp/src/nockapp/export.rs b/crates/nockapp/src/nockapp/export.rs index 65f9f0f4f..ff9342732 100644 --- a/crates/nockapp/src/nockapp/export.rs +++ b/crates/nockapp/src/nockapp/export.rs @@ -1,13 +1,20 @@ -use bincode::{config, decode_from_slice, encode_to_vec, Decode, Encode}; +use std::time::Instant; + +use bincode::{config, encode_to_vec, Decode, Encode}; use blake3::Hash; +use bytes::Bytes; use nockvm_macros::tas; +use tracing::info; +use super::artifact::{ArtifactError, CheckedReader}; use crate::kernel::form::LoadState; -use crate::noun::slab::NounSlab; +use crate::noun::slab::{Jammer, NounSlab}; use crate::{JammedNoun, NockAppError}; const EXPORTED_STATE_MAGIC_BYTES: u64 = tas!(b"EXPJAM"); const EXPORTED_STATE_VERSION: u32 = 0; +const EXPORTED_STATE_ADVICE: &str = + "Re-export the state jam from a known-good node, or start from a checkpoint instead."; /// A structure for exporting just the kernel state, without the cold state #[derive(Encode, Decode, PartialEq, Debug)] @@ -30,27 +37,92 @@ impl ExportedState { encode_to_vec(self, config::standard()) } - pub fn decode(data: &[u8]) -> Result { - let (state, _) = decode_from_slice(data, config::standard())?; - Ok(state) + pub fn decode(data: &[u8]) -> Result { + let mut reader = + CheckedReader::new_with_advice(data, "exported state jam", EXPORTED_STATE_ADVICE); + let magic_bytes = reader.read_u64("magic bytes")?; + if magic_bytes != EXPORTED_STATE_MAGIC_BYTES { + return Err(ArtifactError::InvalidMagic { + format: "exported state jam", + expected: "EXPJAM", + }); + } + + let version = reader.read_u32("version")?; + if version != EXPORTED_STATE_VERSION { + return Err(ArtifactError::UnsupportedVersion { + format: "exported state jam", + version, + supported: "0", + }); + } + + let ker_hash = reader.read_hash("kernel hash")?; + let event_num = reader.read_u64("event number")?; + let jam = JammedNoun::new(Bytes::copy_from_slice(reader.read_bytes("state jam")?)); + reader.finish()?; + + Ok(Self { + magic_bytes, + version, + ker_hash, + event_num, + jam, + }) } - pub fn from_loadstate(state: LoadState) -> Self { - let jam = JammedNoun::new(state.kernel_state.jam()); + pub fn from_loadstate(state: LoadState) -> Self { + let LoadState { + kernel_state, + ker_hash, + event_num, + } = state; + + info!( + event_num = event_num, + ker_hash = %ker_hash, + jammer = std::any::type_name::(), + "state jam jamming kernel state start" + ); + let jam_start = Instant::now(); + let jam = JammedNoun::new(kernel_state.coerce_jammer::().jam()); + info!( + event_num = event_num, + ker_hash = %ker_hash, + jammer = std::any::type_name::(), + jam_bytes = jam.0.len(), + elapsed_ms = jam_start.elapsed().as_secs_f64() * 1000.0, + "state jam jamming kernel state done" + ); Self { magic_bytes: EXPORTED_STATE_MAGIC_BYTES, version: EXPORTED_STATE_VERSION, - ker_hash: state.ker_hash, - event_num: state.event_num, + ker_hash, + event_num, jam, } } - pub fn to_loadstate(self) -> Result { - let mut kernel_state = NounSlab::new(); + pub fn to_loadstate(self) -> Result { + info!( + event_num = self.event_num, + ker_hash = %self.ker_hash, + jammer = std::any::type_name::(), + "state jam cueing kernel state start" + ); + let cue_start = Instant::now(); + let mut kernel_state = NounSlab::::new(); let kernel_state_noun = kernel_state.cue_into(self.jam.0)?; kernel_state.set_root(kernel_state_noun); + let kernel_state: NounSlab = kernel_state.coerce_jammer(); + info!( + event_num = self.event_num, + ker_hash = %self.ker_hash, + jammer = std::any::type_name::(), + elapsed_ms = cue_start.elapsed().as_secs_f64() * 1000.0, + "state jam cueing kernel state done" + ); Ok(LoadState { kernel_state, @@ -59,3 +131,62 @@ impl ExportedState { }) } } + +#[cfg(test)] +mod tests { + use std::panic::{catch_unwind, AssertUnwindSafe}; + + use blake3::hash; + + use super::*; + + fn exported_state_with_absurd_jam_length() -> Vec { + let state = ExportedState { + magic_bytes: EXPORTED_STATE_MAGIC_BYTES, + version: EXPORTED_STATE_VERSION, + ker_hash: hash(b"state-jam"), + event_num: 7, + jam: JammedNoun::new(Bytes::new()), + }; + let mut bytes = state.encode().expect("encode exported state"); + + // The final byte is the bincode varint length for the empty Bytes field. + assert_eq!(bytes.pop(), Some(0)); + bytes.push(253); + bytes.extend_from_slice(&u64::MAX.to_le_bytes()); + bytes + } + + #[test] + fn decodes_exported_state_roundtrip() { + let state = ExportedState { + magic_bytes: EXPORTED_STATE_MAGIC_BYTES, + version: EXPORTED_STATE_VERSION, + ker_hash: hash(b"state-jam"), + event_num: 7, + jam: JammedNoun::new(Bytes::from_static(b"state")), + }; + let encoded = state.encode().expect("encode exported state"); + let decoded = ExportedState::decode(&encoded).expect("decode exported state"); + + assert_eq!(decoded, state); + } + + #[test] + fn corrupt_exported_state_length_is_reported_without_panic() { + let bytes = exported_state_with_absurd_jam_length(); + let decode_result = catch_unwind(AssertUnwindSafe(|| ExportedState::decode(&bytes))); + + assert!( + decode_result.is_ok(), + "corrupt exported state length must be reported as a decode error, not a panic" + ); + let err = decode_result + .expect("checked above") + .expect_err("corrupt exported state should fail to decode"); + assert!( + err.to_string().contains("Re-export the state jam"), + "error should give an operator a recovery path: {err}" + ); + } +} diff --git a/crates/nockapp/src/nockapp/metrics.rs b/crates/nockapp/src/nockapp/metrics.rs index 54827f0e9..2fdae4ad8 100644 --- a/crates/nockapp/src/nockapp/metrics.rs +++ b/crates/nockapp/src/nockapp/metrics.rs @@ -1,3 +1,4 @@ +use gnort::instrument::{Count, Gauge, TimingCount}; use gnort::*; metrics_struct![ @@ -11,6 +12,17 @@ metrics_struct![ (least_free_space_seen_in_slam, "nockapp.least_free_space_seen_in_slam", Gauge), (save_jam_time, "nockapp.save_jam_time", TimingCount), (load_cue_time, "nockapp.load_cue_time", TimingCount), + (event_log_append, "nockapp.event_log.append", TimingCount), + (event_log_commit_failures, "nockapp.event_log.commit_failures", Count), + (snapshot_build, "nockapp.snapshot.build", TimingCount), + (snapshot_build_failures, "nockapp.snapshot.build_failures", Count), + (snapshot_verify, "nockapp.snapshot.verify", TimingCount), + (snapshot_verify_failures, "nockapp.snapshot.verify_failures", Count), + (snapshot_cleanup, "nockapp.snapshot.cleanup", TimingCount), + (snapshot_cleanup_failures, "nockapp.snapshot.cleanup_failures", Count), + (replay_apply, "nockapp.replay.apply", TimingCount), + (replay_failures, "nockapp.replay.failures", Count), + (replay_events, "nockapp.replay.events", Count), (serf_loop_blocking_recv, "nockapp.serf_loop.blocking_recv", TimingCount), (serf_loop_all, "nockapp.serf_loop.all", TimingCount), (serf_loop_load_state, "nockapp.serf_loop.load_state", TimingCount), @@ -25,3 +37,44 @@ metrics_struct![ (serf_loop_provide_metrics, "nockapp.serf_loop.provide_metrics", TimingCount), (next_effect_lagged_error, "nockapp.next_effect.lag", Count) ]; + +#[allow(clippy::derivable_impls)] +impl Default for NockAppMetrics { + fn default() -> Self { + Self { + handle_shutdown: Count::default(), + handle_save_permit_res: Count::default(), + handle_action: Count::default(), + handle_exit: Count::default(), + poke_during_exit: Count::default(), + peek_during_exit: Count::default(), + least_free_space_seen_in_slam: Gauge::default(), + save_jam_time: TimingCount::default(), + load_cue_time: TimingCount::default(), + event_log_append: TimingCount::default(), + event_log_commit_failures: Count::default(), + snapshot_build: TimingCount::default(), + snapshot_build_failures: Count::default(), + snapshot_verify: TimingCount::default(), + snapshot_verify_failures: Count::default(), + snapshot_cleanup: TimingCount::default(), + snapshot_cleanup_failures: Count::default(), + replay_apply: TimingCount::default(), + replay_failures: Count::default(), + replay_events: Count::default(), + serf_loop_blocking_recv: TimingCount::default(), + serf_loop_all: TimingCount::default(), + serf_loop_load_state: TimingCount::default(), + serf_loop_get_state_bytes: TimingCount::default(), + serf_loop_get_kernel_state_slab: TimingCount::default(), + serf_loop_get_cold_state_slab: TimingCount::default(), + serf_loop_checkpoint: TimingCount::default(), + serf_loop_noun_encode_cold_state: TimingCount::default(), + serf_loop_copy_state_noun: TimingCount::default(), + serf_loop_peek: TimingCount::default(), + serf_loop_poke: TimingCount::default(), + serf_loop_provide_metrics: TimingCount::default(), + next_effect_lagged_error: Count::default(), + } + } +} diff --git a/crates/nockapp/src/nockapp/mod.rs b/crates/nockapp/src/nockapp/mod.rs index 46514d5e1..32dfc6346 100644 --- a/crates/nockapp/src/nockapp/mod.rs +++ b/crates/nockapp/src/nockapp/mod.rs @@ -1,4 +1,5 @@ // pub(crate) mod actors; +pub mod artifact; pub mod driver; pub mod error; pub mod export; @@ -8,29 +9,27 @@ pub mod test; pub mod wire; use std::future::Future; -use std::path::PathBuf; use std::sync::atomic::{AtomicBool, Ordering}; use std::sync::Arc; +use std::time::Duration; use driver::{IOAction, IODriverFn, NockAppHandle, PokeResult}; pub use error::NockAppError; -use futures::future::{pending, Either}; use futures::stream::StreamExt; use metrics::*; -use nockvm::noun::SIG; +use nockvm::noun::{NounAllocator, SIG}; use signal_hook::consts::signal::*; use signal_hook::consts::TERM_SIGNALS; use signal_hook_tokio::Signals; use tokio::select; -use tokio::sync::{broadcast, mpsc, Mutex, OwnedMutexGuard}; -use tokio::time::{interval_at, Duration, Instant, Interval}; +use tokio::sync::{broadcast, mpsc, Mutex}; use tokio_util::task::TaskTracker; -use tracing::{debug, error, info, instrument, trace, warn}; +use tracing::{debug, error, instrument, trace, warn}; use wire::WireRepr; use crate::kernel::form::Kernel; use crate::noun::slab::{Jammer, NockJammer, NounSlab}; -use crate::save::{SaveableCheckpoint, Saver}; +use crate::save::SaveableCheckpoint; type NockAppResult = Result<(), NockAppError>; @@ -49,7 +48,6 @@ pub const EXIT_SIGINT: usize = 130; pub const EXIT_SIGQUIT: usize = 131; /// SIGTERM: Termination signal from OS or process manager pub const EXIT_SIGTERM: usize = 143; - pub struct NockApp { /// Nock kernel pub(crate) kernel: Kernel, @@ -69,13 +67,10 @@ pub struct NockApp { action_channel_sender: mpsc::Sender, /// Effect broadcast channel effect_broadcast: Arc>, - /// Save interval - save_interval: Option, - /// Mutex to ensure only one save at a time - pub(crate) save_mutex: Arc>>, metrics: Arc, /// Signals handled by the work loop signals: Signals, + _phantom: std::marker::PhantomData, } pub enum NockAppRun { @@ -138,27 +133,53 @@ impl NockAppExit { } impl NockApp { + fn metrics_enabled() -> bool { + if std::env::var_os("NOCKAPP_DISABLE_METRICS").is_some() { + return false; + } + if std::env::var_os("GNORT_DISABLE").is_some() { + return false; + } + std::env::var_os("RUST_TEST_THREADS").is_none() + } + + pub fn take_pma_timing_samples(&self) -> Option> { + self.kernel.take_pma_timing_samples() + } + + pub fn take_pma_timing_samples_detailed( + &self, + ) -> Option> { + self.kernel.take_pma_timing_samples_detailed() + } + /// This constructs a Tokio interval, even though it doesn't look like it, a Tokio runtime is _required_. - pub async fn new( - kernel_from_checkpoint: F, - snapshot_path: &PathBuf, - save_interval_duration: Option, - ) -> Result + pub async fn new(kernel_from_boot: F) -> Result where - F: FnOnce(Option) -> U, + F: FnOnce(Arc) -> U, U: Future, E>>, NockAppError: From, { // let cancel_token = tokio_util::sync::CancellationToken::new(); - let metrics = Arc::new( - NockAppMetrics::register(gnort::global_metrics_registry()) - .expect("Failed to register metrics!"), - ); - let (saver, checkpoint) = Saver::::try_load(snapshot_path, Some(metrics.clone())) - .await - .expect("Failed to set up snapshotting"); - let save_mutex = Arc::new(Mutex::new(saver)); - let mut kernel = kernel_from_checkpoint(checkpoint).await?; + let metrics = if Self::metrics_enabled() { + match gnort::GnortClient::default() { + Ok(client) => { + let registry = gnort::MetricsRegistry::new( + gnort::RegistryConfig::default().with_client(client), + ); + Arc::new( + NockAppMetrics::register(®istry).expect("Failed to register metrics!"), + ) + } + Err(err) => { + warn!("Failed to initialize metrics client, disabling metrics: {err}"); + Arc::new(NockAppMetrics::default()) + } + } + } else { + Arc::new(NockAppMetrics::default()) + }; + let mut kernel = kernel_from_boot(metrics.clone()).await?; // important: we are tracking this separately here because // what matters is the last poke *we* received an ack for. Using // the Arc in the serf would result in a race condition! @@ -166,20 +187,7 @@ impl NockApp { let (action_channel_sender, action_channel) = mpsc::channel(100); let (effect_broadcast_sender, _) = broadcast::channel(100); let effect_broadcast = Arc::new(effect_broadcast_sender); - // let tasks = Arc::new(Mutex::new(TaskJoinSet::new())); - // let tasks = TaskJoinSet::new(); - // let tasks = Arc::new(TaskJoinSet::new()); let tasks = TaskTracker::new(); - let save_interval = save_interval_duration.map(|duration| { - info!("Nockapp save interval duration: {:?}", duration); - let first_tick_at = Instant::now() + duration; - let mut interval = interval_at(first_tick_at, duration); - interval.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Skip); // important so we don't stack ticks when lagging - interval - }); - if save_interval.is_none() { - info!("Nockapp save interval disabled; periodic saves off"); - } let exit_status = AtomicBool::new(false); let abort_immediately = AtomicBool::new(false); @@ -202,11 +210,9 @@ impl NockApp { action_channel, action_channel_sender, effect_broadcast, - save_interval, - save_mutex, - // cancel_token, metrics, signals, + _phantom: std::marker::PhantomData, }) } @@ -267,60 +273,6 @@ impl NockApp { io_sender } - /// Purely for testing purposes (injecting delays) for now. - #[instrument(skip(self, f, save_permit))] - pub(crate) async fn save_f( - &mut self, - f: impl std::future::Future + Send + 'static, - mut save_permit: OwnedMutexGuard>, - ) -> Result, NockAppError> { - let checkpoint_fut = self.kernel.checkpoint(); - let metrics = self.metrics.clone(); - - trace!("Spawning save task from save_f"); - let join_handle = self.tasks.spawn(async move { - f.await; - trace!("Save task from save_f: f.await done"); - let checkpoint = checkpoint_fut.await?; - trace!("Save task from save_f: checkpoint_fut.await done"); - save_permit.save(checkpoint, metrics).await?; - trace!("Save task from save_f: save_permit.save done"); - - drop(save_permit); - Ok::<(), NockAppError>(()) - }); - // We don't want to close and re-open the tasktracker from multiple places - // so we're just returning the join_handle to let the caller decide. - Ok(join_handle) - } - - /// Except in tests, save should only be called by the permit handler. - pub(crate) async fn save(&mut self, save_permit: OwnedMutexGuard>) -> NockAppResult { - let _join_handle = self.save_f(async {}, save_permit).await?; - Ok(()) - } - - pub async fn save_locked(&mut self) -> NockAppResult { - trace!("save_locked: locking save_mutex"); - let guard = self.save_mutex.clone().lock_owned().await; - trace!("save_locked: save_mutex locked"); - self.save(guard).await.map_err(|e| { - error!("Failed to save: {:?}", e); - e - })?; - Ok(()) - } - - /// Save the kernel to disk, blocking operation - pub async fn save_blocking(&mut self) -> NockAppResult { - trace!("save_blocking: locking save_mutex"); - let guard = self.save_mutex.clone().lock_owned().await; - trace!("save_blocking: save_mutex locked"); - let join_handle = self.save_f(async {}, guard).await?; - join_handle.await.map_err(NockAppError::JoinError)??; - Ok(()) - } - /// Peek at a noun in the kernel, blocking operation #[tracing::instrument(skip(self, path))] pub fn peek_sync(&mut self, path: NounSlab) -> Result { @@ -333,9 +285,13 @@ impl NockApp { #[tracing::instrument(skip(self, path))] pub async fn peek(&mut self, path: NounSlab) -> Result { trace!("Peeking at noun: {:?}", path); - let res = self.kernel.peek(path).await?; + let res = self + .kernel + .peek(path.clone()) + .await + .map_err(NockAppError::CrownError); trace!("Peeked noun: {:?}", res); - Ok(res) + res } // Peek at a noun in the kernel with result munging. A `~`, which denotes an invalid @@ -349,13 +305,15 @@ impl NockApp { return Err(NockAppError::PeekFailed); } - let tail = unsafe { res.root().as_cell()?.tail() }; + let space = res.noun_space(); + let root_cell = unsafe { res.root().in_space(&space).as_cell()? }; + let tail = root_cell.tail().noun(); if unsafe { tail.raw_equals(&SIG) } { Ok(None) } else { - let res_noun = tail.as_cell()?.tail(); + let res_noun = tail.in_space(&space).as_cell()?.tail().noun(); let mut slab = NounSlab::new(); - slab.copy_into(res_noun); + slab.copy_into(res_noun, &space); Ok(Some(slab)) } } @@ -382,6 +340,10 @@ impl NockApp { Ok(effects_slab.to_vec()) } + pub async fn export(&self) -> Result { + Ok(self.kernel.export().await?) + } + pub async fn poke_timeout( &mut self, wire: WireRepr, @@ -427,20 +389,6 @@ impl NockApp { } async fn work(&mut self) -> Result { - // Track SIGINT (C-c) presses for immediate termination - // Fires when there is a save interval tick *and* an available permit in the save semaphore - let save_ready = if let Some(interval) = self.save_interval.as_mut() { - let save_mutex = self.save_mutex.clone(); - Either::Left(async move { - interval.tick().await; - trace!("save_interval tick: locking save_mutex"); - let guard = save_mutex.lock_owned().await; - trace!("save_interval tick: save_mutex locked"); - guard - }) - } else { - Either::Right(pending::>>()) - }; select!( exit_status_res = self.exit_recv.recv() => { let Some(exit_status) = exit_status_res else { @@ -481,10 +429,6 @@ impl NockApp { }, } }, - save_guard = save_ready => { - self.metrics.handle_save_permit_res.increment(); - self.handle_save_permit_res(save_guard).await - }, maybe_signal = self.signals.next() => { debug!("Signal received"); if let Some(signal) = maybe_signal { @@ -533,22 +477,6 @@ impl NockApp { ) } - #[instrument(skip_all, level = "trace")] - async fn handle_save_permit_res( - &mut self, - save_guard: OwnedMutexGuard>, - ) -> Result { - // Check if we should write in the first place - let curr_event_num = self.kernel.serf.event_number.load(Ordering::SeqCst); - if !save_guard.save_needed(curr_event_num) { - return Ok(NockAppRun::Pending); - } - - let res = self.save(save_guard).await; - - res.map(|_| NockAppRun::Pending) - } - #[instrument(skip_all)] async fn handle_action(&self, action: IOAction) { // Stop processing events if we are exiting @@ -643,8 +571,6 @@ impl NockApp { })); } - // TODO: We should explicitly kick off a save somehow - // TOOD: :>) spawn a task which awaits the signal stream and if there is a SIGINT, then call std::process::exit(1) #[instrument(skip_all)] async fn handle_exit(&mut self, code: usize) -> Result { // We should only run handle_exit once, break out if we are already exiting. @@ -660,59 +586,13 @@ impl NockApp { } } - let exit_event_num = self.kernel.serf.event_number.load(Ordering::SeqCst); - debug!( - "Exit request received, waiting for save checkpoint with event_num {} (code {})", - exit_event_num, code - ); - - let waiter_mutex_arc = self.save_mutex.clone(); - let waiter = { - trace!("Waiting for save event_num {}", exit_event_num); - let mut guard = waiter_mutex_arc.lock().await; - trace!("Locked save mutex for event_num {}", exit_event_num); - let oneshot = guard.wait_for_snapshot(exit_event_num).await; - trace!("Acquired the oneshot for snapshot on save event_num {}", exit_event_num); - drop(guard); - oneshot - }; - - // Force an immediate save to ensure we have the latest state - debug!( - "Exit signal received with code {}, forcing immediate save", - code - ); - if let Err(e) = self.save_locked().await { - error!( - "Failed to save during exit: {:?} - continuing with shutdown anyway", - e - ); - } - - // let cancel_token = self.cancel_token.clone(); let exit = self.exit.clone(); - // self.tasks.close(); - // self.tasks.wait().await; - // recv from the watch channel until we reach the exit event_num, wrapped up in a future - // that will send the shutdown result when we're done. - // TODO: Break this out as a separate select! handler with no spawn self.tasks.spawn(async move { - debug!("Waiting for save event_num {}", exit_event_num); - let result = waiter.await; - if let Err(e) = result { - error!("Error waiting for snapshot: {e}"); - panic!("Error waiting for snapshot: {e}"); - }; - debug!("Save event_num reached, finishing with code {}", code); let shutdown_result = if code == EXIT_OK { Ok(()) } else { Err(NockAppError::Exit(code)) }; - // Ensure we send the shutdown result before canceling so that - // we don't get a race condition where the yielded result is - // "canceled" instead of the actual result. - debug!("Sending shutdown result"); if let Err(e) = exit.shutdown(shutdown_result).await { error!("Error sending shutdown: {e:}") } diff --git a/crates/nockapp/src/nockapp/save.rs b/crates/nockapp/src/nockapp/save.rs index 2f601bb14..1d86583ca 100644 --- a/crates/nockapp/src/nockapp/save.rs +++ b/crates/nockapp/src/nockapp/save.rs @@ -1,18 +1,17 @@ -use std::future::Future; use std::path::{Path, PathBuf}; use std::sync::Arc; use std::time::Instant; -use bincode::config::Configuration; use bincode::{config, encode_to_vec, Decode, Encode}; use blake3::{Hash, Hasher}; use bytes::Bytes; +use nockvm::noun::{NounAllocator, D, T}; use nockvm_macros::tas; use thiserror::Error; use tokio::fs::create_dir_all; -use tokio::sync::oneshot; -use tracing::{debug, error, trace, warn}; +use tracing::{debug, error, warn}; +use super::artifact::{ArtifactError, CheckedReader}; use crate::metrics::NockAppMetrics; use crate::noun::slab::{Jammer, NockJammer, NounSlab}; use crate::JammedNoun; @@ -23,188 +22,49 @@ const SNAPSHOT_VERSION_1: u32 = 1; const SNAPSHOT_VERSION_2: u32 = 2; pub const LATEST_SNAPSHOT_VERSION: u32 = SNAPSHOT_VERSION_2; -pub enum WhichSnapshot { - Snapshot0, - Snapshot1, -} - -impl WhichSnapshot { - pub fn next(&self) -> Self { - match self { - WhichSnapshot::Snapshot0 => WhichSnapshot::Snapshot1, - WhichSnapshot::Snapshot1 => WhichSnapshot::Snapshot0, - } - } +#[derive(Clone, Debug)] +pub(crate) struct CheckpointSummary { + pub path: PathBuf, + pub event_num: u64, } -/// State object which handles all NockApp saves and loads -pub struct Saver { - path_0: PathBuf, - path_1: PathBuf, - save_to_next: WhichSnapshot, - waiters: Vec<(u64, oneshot::Sender<()>)>, - last_event_num: u64, +pub struct CheckpointBootstrapReader { + path: PathBuf, _phantom: std::marker::PhantomData, } -impl Saver { - pub fn last_path(&self) -> PathBuf { - match self.save_to_next { - WhichSnapshot::Snapshot1 => self.path_0.clone(), - WhichSnapshot::Snapshot0 => self.path_1.clone(), - } - } - - pub fn next_path(&self) -> PathBuf { - match self.save_to_next { - WhichSnapshot::Snapshot1 => self.path_1.clone(), - WhichSnapshot::Snapshot0 => self.path_0.clone(), - } - } - - /// The future from this function should not be awaited before any mutex - /// around the 'Saver' is released, or a deadlock will result. - #[tracing::instrument(skip(self))] - #[allow(clippy::async_yields_async)] - pub async fn wait_for_snapshot( - &mut self, - wait_for_event_num: u64, - ) -> impl Future> { - if self.last_event_num >= wait_for_event_num { - return futures::future::Either::Left(std::future::ready(Ok(()))); +impl CheckpointBootstrapReader { + pub fn new(path: PathBuf) -> Self { + Self { + path, + _phantom: std::marker::PhantomData, } - - let (tx, rx) = tokio::sync::oneshot::channel(); - self.waiters.push((wait_for_event_num, tx)); - futures::future::Either::Right(rx) - } - - /// Check if we need to save - pub fn save_needed(&self, event_num: u64) -> bool { - self.last_event_num < event_num } } -impl Saver { - pub async fn try_load( - path: &PathBuf, +impl CheckpointBootstrapReader { + pub async fn load_latest( + &self, metrics: Option>, - ) -> Result<(Self, Option), CheckpointError> { - let path_0 = path.join("0.chkjam"); - let path_1 = path.join("1.chkjam"); - let waiters = Vec::new(); - - // No snapshot to load - if !path_0.exists() && !path_1.exists() { - create_dir_all(path).await?; - return Ok(( - Self { - path_0, - path_1, - save_to_next: WhichSnapshot::Snapshot0, - waiters, - last_event_num: 0, - _phantom: std::marker::PhantomData, - }, - None, - )); - } - - let checkpoint_0 = load_checkpoint_file(&path_0).await; - let checkpoint_1 = load_checkpoint_file(&path_1).await; - - let (loaded_checkpoint, save_to_next) = match (checkpoint_0, checkpoint_1) { - (Ok(c0), Ok(c1)) => { - if c0.event_num() > c1.event_num() { - debug!( - "Loading checkpoint at: {}, checksum: {}", - path_0.display(), - c0.checksum() - ); - (c0, WhichSnapshot::Snapshot1) - } else { - debug!( - "Loading checkpoint at: {}, checksum: {}", - path_1.display(), - c1.checksum() - ); - (c1, WhichSnapshot::Snapshot0) - } - } - (Ok(c0), Err(e1)) => { - warn!("checkpoint at {} failed to load: {}", path_1.display(), e1); - debug!( - "Loading checkpoint at: {}, checksum: {}", - path_0.display(), - c0.checksum() - ); - (c0, WhichSnapshot::Snapshot1) - } - (Err(e0), Ok(c1)) => { - warn!("checkpoint at {} failed to load: {}", path_0.display(), e0); - debug!( - "Loading checkpoint at: {}, checksum: {}", - path_1.display(), - c1.checksum() - ); - (c1, WhichSnapshot::Snapshot0) - } - (Err(e0), Err(e1)) => { - error!("checkpoint at {} failed to load: {}", path_0.display(), e0); - error!("checkpoint at {} failed to load: {}", path_1.display(), e1); - return Err(CheckpointError::BothCheckpointsFailed( - Box::new(e0), - Box::new(e1), - )); - } - }; - let last_event_num = loaded_checkpoint.event_num(); - let saveable = loaded_checkpoint.into_saveable::(metrics.clone())?; - trace!("After from_jammed_checkpoint"); - let c = C::from_saveable(saveable)?; - Ok(( - Self { - path_0, - path_1, - save_to_next, - waiters, - last_event_num, - _phantom: std::marker::PhantomData, - }, - Some(c), - )) + ) -> Result, CheckpointError> { + inspect_latest(&self.path) + .await? + .map(|(checkpoint, _)| checkpoint.into_saveable::(metrics)) + .transpose() } - #[tracing::instrument(skip_all)] - pub async fn save( - &mut self, - checkpoint: C, - metrics: Arc, - ) -> Result<(), CheckpointError> { - let event_num = checkpoint.event_num(); - trace!("Saving checkpoint at event_num {}", event_num); - let saveable = checkpoint.to_saveable(); - trace!("Converted checkpoint to saveable"); - let jammed = saveable.to_jammed_checkpoint::(metrics); - trace!("Converted saveable to jammed"); - let path = self.next_path(); - jammed.save_to_file(&path).await?; - self.save_to_next = self.save_to_next.next(); - std::mem::drop(jammed); - debug!("Saved checkpoint to file: {}", path.display()); - let mut still_waiting = Vec::new(); - for (waiting_event_num, waiter) in self.waiters.drain(..) { - if waiting_event_num <= event_num { - let _ = waiter.send(()); // An error means the receiver was dropped - } else { - still_waiting.push((waiting_event_num, waiter)); - } - } - - self.last_event_num = event_num; - self.waiters = still_waiting; - - Ok(()) + pub(crate) async fn load_latest_state_only_with_summary( + &self, + metrics: Option>, + ) -> Result, CheckpointError> { + inspect_latest(&self.path) + .await? + .map(|(checkpoint, summary)| { + checkpoint + .into_saveable_state_only::(metrics) + .map(|checkpoint| (checkpoint, summary)) + }) + .transpose() } } @@ -225,9 +85,16 @@ pub struct SaveableCheckpoint { } impl SaveableCheckpoint { + fn empty_cold_slab() -> NounSlab { + let mut slab = NounSlab::new(); + let root = T(&mut slab, &[D(0), D(0), D(0)]); + slab.set_root(root); + slab + } + #[allow(clippy::wrong_self_convention)] - #[tracing::instrument(skip(self, metrics))] - fn to_jammed_checkpoint(self, metrics: Arc) -> JammedCheckpointV2 { + #[tracing::instrument(skip(self))] + pub fn to_jammed_checkpoint(self) -> JammedCheckpointV2 { let SaveableCheckpoint { ker_hash, event_num, @@ -235,11 +102,8 @@ impl SaveableCheckpoint { cold, } = self; - let jam_start = Instant::now(); let state_jam = JammedNoun::new(state.coerce_jammer::().jam()); let cold_jam = JammedNoun::new(cold.coerce_jammer::().jam()); - metrics.save_jam_time.add_timing(&jam_start.elapsed()); - JammedCheckpointV2::new(ker_hash, event_num, cold_jam, state_jam) } @@ -252,16 +116,15 @@ impl SaveableCheckpoint { let root = slab.cue_into(jammed.jam.0)?; metrics.map(|m| m.load_cue_time.add_timing(&cue_start.elapsed())); slab.set_root(root); - let cell = root - .as_cell() - .expect("legacy checkpoint root should be a cell"); + let space = slab.noun_space(); + let cell = root.in_space(&space).as_cell()?; let mut state_slab: NounSlab = NounSlab::new(); - let state_copy = state_slab.copy_into(cell.head()); + let state_copy = state_slab.copy_into(cell.head().noun(), &space); state_slab.set_root(state_copy); let mut cold_slab: NounSlab = NounSlab::new(); - let cold_copy = cold_slab.copy_into(cell.tail()); + let cold_copy = cold_slab.copy_into(cell.tail().noun(), &space); cold_slab.set_root(cold_copy); Ok(Self { @@ -272,6 +135,41 @@ impl SaveableCheckpoint { }) } + fn from_legacy_jammed_checkpoint_state_only( + ker_hash: Hash, + event_num: u64, + jam: JammedNoun, + metrics: Option>, + ) -> Result { + let mut slab: NounSlab = NounSlab::new(); + let cue_start = Instant::now(); + let root = slab.cue_into(jam.0)?; + metrics.map(|m| m.load_cue_time.add_timing(&cue_start.elapsed())); + slab.set_root(root); + let space = slab.noun_space(); + let cell = root.in_space(&space).as_cell()?; + + let mut state_slab: NounSlab = NounSlab::new(); + let state_copy = state_slab.copy_into(cell.head().noun(), &space); + state_slab.set_root(state_copy); + + Ok(Self { + ker_hash, + event_num, + state: state_slab, + cold: Self::empty_cold_slab(), + }) + } + + fn from_jammed_checkpoint_v1_state_only( + jammed: JammedCheckpointV1, + metrics: Option>, + ) -> Result { + Self::from_legacy_jammed_checkpoint_state_only::( + jammed.ker_hash, jammed.event_num, jammed.jam, metrics, + ) + } + fn from_jammed_checkpoint_v2( jammed: JammedCheckpointV2, metrics: Option>, @@ -303,19 +201,25 @@ impl SaveableCheckpoint { cold: cold_slab, }) } -} - -impl Checkpoint for SaveableCheckpoint { - fn to_saveable(self) -> SaveableCheckpoint { - self - } - fn from_saveable(saveable: SaveableCheckpoint) -> Result { - Ok(saveable) - } + fn from_jammed_checkpoint_v2_state_only( + jammed: JammedCheckpointV2, + metrics: Option>, + ) -> Result { + let mut state_slab: NounSlab = NounSlab::new(); + let state_start = Instant::now(); + let state_root = state_slab.cue_into(jammed.state_jam.0.clone())?; + if let Some(metrics) = metrics { + metrics.load_cue_time.add_timing(&state_start.elapsed()); + } + state_slab.set_root(state_root); - fn event_num(&self) -> u64 { - self.event_num + Ok(Self { + ker_hash: jammed.ker_hash, + event_num: jammed.event_num, + state: state_slab.coerce_jammer::(), + cold: Self::empty_cold_slab(), + }) } } @@ -325,11 +229,17 @@ pub enum CheckpointError { IOError(#[from] std::io::Error), #[error("Bincode decoding error: {0}")] DecodeError(#[from] bincode::error::DecodeError), + #[error("Artifact decoding error: {0}")] + ArtifactError(#[from] ArtifactError), #[error("Bincode encoding error: {0}")] EncodeError(#[from] bincode::error::EncodeError), - #[error("Invalid checksum at {0}")] + #[error( + "Invalid checksum at {0}. The checkpoint is corrupt or incomplete; restore it from a known-good peer or remove it so Nockchain can try another checkpoint." + )] InvalidChecksum(PathBuf), - #[error("Invalid version at {0}")] + #[error( + "Invalid checkpoint version at {0}. Use a compatible Nockchain binary for this checkpoint, or restore/remove the checkpoint so the peer can boot from a valid one." + )] InvalidVersion(PathBuf), #[error("Sword noun error: {0}")] SwordNounError(#[from] nockvm::noun::Error), @@ -420,19 +330,32 @@ impl JammedCheckpointV1 { async fn load_from_file(path: &Path) -> Result { debug!("Loading jammed checkpoint from file: {}", path.display()); let bytes = tokio::fs::read(path).await?; - let config = bincode::config::standard(); - let (checkpoint, _) = bincode::decode_from_slice::(&bytes, config)?; + let checkpoint = Self::decode_from_bytes(&bytes, path)?; checkpoint.validate(path)?; Ok(checkpoint) } - #[allow(dead_code)] - #[tracing::instrument(skip(self))] - async fn save_to_file(&self, path: &Path) -> Result<(), CheckpointError> { - let bytes = self.encode()?; - trace!("Saving jammed checkpoint to file: {}", path.display()); - tokio::fs::write(path, bytes).await?; - Ok(()) + fn decode_from_bytes(bytes: &[u8], path: &Path) -> Result { + let mut reader = CheckedReader::new(bytes, "checkpoint v1"); + let magic_bytes = reader.read_u64("magic bytes")?; + let version = reader.read_u32("version")?; + if magic_bytes != JAM_MAGIC_BYTES || version != SNAPSHOT_VERSION_1 { + return Err(CheckpointError::InvalidVersion(path.to_path_buf())); + } + let ker_hash = reader.read_hash("kernel hash")?; + let checksum = reader.read_hash("checksum")?; + let event_num = reader.read_u64("event number")?; + let jam = JammedNoun::new(Bytes::copy_from_slice(reader.read_bytes("jam")?)); + reader.finish()?; + + Ok(Self { + magic_bytes, + version, + ker_hash, + checksum, + event_num, + jam, + }) } } @@ -511,46 +434,50 @@ impl JammedCheckpointV2 { async fn load_from_file(path: &Path) -> Result { debug!("Loading jammed checkpoint from file: {}", path.display()); let bytes = tokio::fs::read(path).await?; - let config = bincode::config::standard(); - let (envelope, _) = bincode::decode_from_slice::( - &bytes, config, - )?; - let checkpoint = Self::from_envelope(envelope, Some(path))?; + let checkpoint = Self::decode_from_bytes_with_path(&bytes, Some(path))?; checkpoint.validate(path)?; Ok(checkpoint) } - #[tracing::instrument(skip(self))] - async fn save_to_file(&self, path: &Path) -> Result<(), CheckpointError> { - let bytes = self.encode()?; - trace!("Saving jammed checkpoint to file: {}", path.display()); - tokio::fs::write(path, bytes).await?; - Ok(()) + fn from_payload(payload: &[u8]) -> Result { + let mut reader = CheckedReader::new(payload, "checkpoint v2 payload"); + let ker_hash = reader.read_hash("kernel hash")?; + let checksum = reader.read_hash("checksum")?; + let event_num = reader.read_u64("event number")?; + let cold_jam = JammedNoun::new(Bytes::copy_from_slice(reader.read_bytes("cold jam")?)); + let state_jam = JammedNoun::new(Bytes::copy_from_slice(reader.read_bytes("state jam")?)); + reader.finish()?; + + Ok(Self { + ker_hash, + checksum, + event_num, + cold_jam, + state_jam, + }) } - fn from_envelope( - envelope: JammedCheckpointV2Envelope, + fn decode_from_bytes_with_path( + bytes: &[u8], path: Option<&Path>, ) -> Result { - if envelope.magic_bytes != JAM_MAGIC_BYTES { + let mut reader = CheckedReader::new(bytes, "checkpoint v2 envelope"); + let magic_bytes = reader.read_u64("magic bytes")?; + let version = reader.read_u32("version")?; + if magic_bytes != JAM_MAGIC_BYTES { return Err(CheckpointError::InvalidVersion(path_or_memory(path))); } - if envelope.version != LATEST_SNAPSHOT_VERSION { + if version != LATEST_SNAPSHOT_VERSION { return Err(CheckpointError::InvalidVersion(path_or_memory(path))); } - - let config = bincode::config::standard(); - let (checkpoint, _) = - bincode::decode_from_slice::(&envelope.payload, config)?; - + let payload = reader.read_bytes("payload")?; + reader.finish()?; + let checkpoint = Self::from_payload(payload)?; Ok(checkpoint) } pub fn decode_from_bytes(bytes: &[u8]) -> Result { - let config = bincode::config::standard(); - let (envelope, _) = - bincode::decode_from_slice::(bytes, config)?; - let checkpoint = Self::from_envelope(envelope, None)?; + let checkpoint = Self::decode_from_bytes_with_path(bytes, None)?; let fake_path = path_or_memory(None); checkpoint.validate(&fake_path)?; Ok(checkpoint) @@ -566,6 +493,7 @@ fn path_or_memory(path: Option<&Path>) -> PathBuf { enum LoadedCheckpoint { V2(JammedCheckpointV2), V1(JammedCheckpointV1), + V0(JammedCheckpointV0), } impl LoadedCheckpoint { @@ -573,6 +501,7 @@ impl LoadedCheckpoint { match self { LoadedCheckpoint::V2(cp) => cp.event_num, LoadedCheckpoint::V1(cp) => cp.event_num, + LoadedCheckpoint::V0(cp) => cp.event_num, } } @@ -580,6 +509,7 @@ impl LoadedCheckpoint { match self { LoadedCheckpoint::V2(cp) => cp.checksum, LoadedCheckpoint::V1(cp) => cp.checksum, + LoadedCheckpoint::V0(cp) => cp.checksum, } } @@ -594,17 +524,106 @@ impl LoadedCheckpoint { LoadedCheckpoint::V1(cp) => { SaveableCheckpoint::from_jammed_checkpoint_v1::(cp, metrics) } + LoadedCheckpoint::V0(cp) => SaveableCheckpoint::from_jammed_checkpoint_v2::( + JammedCheckpoint::from(cp), + metrics, + ), + } + } + + fn into_saveable_state_only( + self, + metrics: Option>, + ) -> Result { + match self { + LoadedCheckpoint::V2(cp) => { + SaveableCheckpoint::from_jammed_checkpoint_v2_state_only::(cp, metrics) + } + LoadedCheckpoint::V1(cp) => { + SaveableCheckpoint::from_jammed_checkpoint_v1_state_only::(cp, metrics) + } + LoadedCheckpoint::V0(cp) => { + SaveableCheckpoint::from_legacy_jammed_checkpoint_state_only::( + cp.ker_hash, cp.event_num, cp.jam, metrics, + ) + } } } } +async fn inspect_latest( + path: &Path, +) -> Result, CheckpointError> { + let path_0 = path.join("0.chkjam"); + let path_1 = path.join("1.chkjam"); + + if !path_0.exists() && !path_1.exists() { + create_dir_all(path).await?; + return Ok(None); + } + + let checkpoint_0 = load_checkpoint_file(&path_0).await; + let checkpoint_1 = load_checkpoint_file(&path_1).await; + + let (loaded_checkpoint, selected_path) = match (checkpoint_0, checkpoint_1) { + (Ok(c0), Ok(c1)) => { + if c0.event_num() > c1.event_num() { + debug!( + "Loading checkpoint at: {}, checksum: {}", + path_0.display(), + c0.checksum() + ); + (c0, path_0) + } else { + debug!( + "Loading checkpoint at: {}, checksum: {}", + path_1.display(), + c1.checksum() + ); + (c1, path_1) + } + } + (Ok(c0), Err(e1)) => { + warn!("checkpoint at {} failed to load: {}", path_1.display(), e1); + debug!( + "Loading checkpoint at: {}, checksum: {}", + path_0.display(), + c0.checksum() + ); + (c0, path_0) + } + (Err(e0), Ok(c1)) => { + warn!("checkpoint at {} failed to load: {}", path_0.display(), e0); + debug!( + "Loading checkpoint at: {}, checksum: {}", + path_1.display(), + c1.checksum() + ); + (c1, path_1) + } + (Err(e0), Err(e1)) => { + error!("checkpoint at {} failed to load: {}", path_0.display(), e0); + error!("checkpoint at {} failed to load: {}", path_1.display(), e1); + return Err(CheckpointError::BothCheckpointsFailed( + Box::new(e0), + Box::new(e1), + )); + } + }; + let summary = CheckpointSummary { + path: selected_path, + event_num: loaded_checkpoint.event_num(), + }; + Ok(Some((loaded_checkpoint, summary))) +} + async fn load_checkpoint_file(path: &Path) -> Result { match JammedCheckpointV2::load_from_file(path).await { Ok(cp) => Ok(LoadedCheckpoint::V2(cp)), Err(e_v2) => match JammedCheckpointV1::load_from_file(path).await { Ok(cp) => Ok(LoadedCheckpoint::V1(cp)), Err(e_v1) => match JammedCheckpointV0::load_from_file(path).await { - Ok(cp0) => Ok(LoadedCheckpoint::V2(JammedCheckpoint::from(cp0))), + Ok(cp0) => Ok(LoadedCheckpoint::V0(cp0)), Err(e_v0) => Err(CheckpointError::VersionsFailedV2 { v2: Box::new(e_v2), v1: Box::new(e_v1), @@ -630,17 +649,20 @@ impl From for JammedCheckpoint { let root = slab .cue_into(v1.jam.0.clone()) .expect("legacy checkpoint jam should cue"); + slab.set_root(root); + let space = slab.noun_space(); let cell = root + .in_space(&space) .as_cell() .expect("legacy checkpoint root should be a cell"); let mut state_slab: NounSlab = NounSlab::new(); - let state_copy = state_slab.copy_into(cell.head()); + let state_copy = state_slab.copy_into(cell.head().noun(), &space); state_slab.set_root(state_copy); let state_jam = JammedNoun::new(state_slab.jam()); let mut cold_slab: NounSlab = NounSlab::new(); - let cold_copy = cold_slab.copy_into(cell.tail()); + let cold_copy = cold_slab.copy_into(cell.tail().noun(), &space); cold_slab.set_root(cold_copy); let cold_jam = JammedNoun::new(cold_slab.jam()); @@ -711,48 +733,110 @@ impl JammedCheckpointV0 { async fn load_from_file(path: &Path) -> Result { debug!("Loading jammed checkpoint from file: {}", path.display()); let bytes = tokio::fs::read(path).await?; - let config = bincode::config::standard(); - let (checkpoint, _) = bincode::decode_from_slice::(&bytes, config)?; + let checkpoint = Self::decode_from_bytes(&bytes, path)?; checkpoint.validate(path)?; Ok(checkpoint) } - #[tracing::instrument(skip(self))] - #[allow(dead_code)] // Preserving this for posterity - async fn save_to_file(&self, path: &Path) -> Result<(), CheckpointError> { - let bytes = self.encode()?; - trace!("Saving jammed checkpoint to file: {}", path.display()); - tokio::fs::write(path, bytes).await?; - Ok(()) + fn decode_from_bytes(bytes: &[u8], path: &Path) -> Result { + let mut reader = CheckedReader::new(bytes, "checkpoint v0"); + let magic_bytes = reader.read_u64("magic bytes")?; + let version = reader.read_u32("version")?; + if magic_bytes != JAM_MAGIC_BYTES || version != SNAPSHOT_VERSION_0 { + return Err(CheckpointError::InvalidVersion(path.to_path_buf())); + } + let buff_index = reader.read_bool("buffer index")?; + let ker_hash = reader.read_hash("kernel hash")?; + let checksum = reader.read_hash("checksum")?; + let event_num = reader.read_u64("event number")?; + let jam = JammedNoun::new(Bytes::copy_from_slice(reader.read_bytes("jam")?)); + reader.finish()?; + + Ok(Self { + magic_bytes, + version, + buff_index, + ker_hash, + checksum, + event_num, + jam, + }) } } #[cfg(test)] mod version_tests { + use std::panic::AssertUnwindSafe; + use blake3::hash; - use nockvm::noun::{Noun, D, T}; + use futures::FutureExt; + use nockvm::noun::{Noun, NounSpace, D, T}; use tempfile::TempDir; use super::*; fn legacy_pair_jam(state_value: u64, cold_value: u64) -> JammedNoun { let mut slab = NounSlab::::new(); - let state = slab.copy_into(D(state_value)); - let cold = slab.copy_into(D(cold_value)); + let space = NounSpace::empty(); + let state = slab.copy_into(D(state_value), &space); + let cold = slab.copy_into(D(cold_value), &space); let root = T(&mut slab, &[state, cold]); slab.set_root(root); JammedNoun::new(slab.coerce_jammer::().jam()) } - fn atom_value(noun: Noun) -> u64 { - noun.as_atom() + fn atom_value(noun: Noun, space: &NounSpace) -> u64 { + noun.in_space(space) + .as_atom() .expect("expected atom") .as_u64() .expect("expected atom to fit in u64") } - #[tokio::test] - async fn loads_v1_checkpoint_via_saver() { + fn v1_checkpoint_with_absurd_jam_length(event_num: u64) -> Vec { + let checkpoint = JammedCheckpointV1::new( + hash(b"corrupt-v1"), + event_num, + JammedNoun::new(Bytes::new()), + ); + let mut bytes = checkpoint.encode().expect("encode v1 checkpoint"); + + // The final byte is the bincode varint length for the empty Bytes field. + assert_eq!(bytes.pop(), Some(0)); + + // bincode standard varint marker 253 means the next eight bytes are a u64. + // On 64-bit targets this decodes to usize::MAX and currently panics in Vec allocation + // before checkpoint fallback can try the older checkpoint. + bytes.push(253); + bytes.extend_from_slice(&u64::MAX.to_le_bytes()); + bytes + } + + fn v2_checkpoint_with_absurd_payload_length() -> Vec { + let checkpoint = JammedCheckpointV2::new( + hash(b"corrupt-v2"), + 9, + JammedNoun::new(Bytes::from_static(b"cold")), + JammedNoun::new(Bytes::from_static(b"state")), + ); + let mut bytes = checkpoint.encode().expect("encode v2 checkpoint"); + + // The V2 envelope is magic (u64 varint: 9 bytes for CHKJAM), version (one byte for 2), + // then a bincode Vec length for the payload. + let payload_len_offset = 10; + assert_eq!( + bytes[payload_len_offset] as usize, + bytes.len() - payload_len_offset - 1 + ); + bytes.truncate(payload_len_offset); + bytes.push(253); + bytes.extend_from_slice(&u64::MAX.to_le_bytes()); + bytes + } + + #[tokio::test(flavor = "current_thread")] + #[cfg_attr(miri, ignore = "memfd_create unsupported in Miri")] + async fn loads_v1_checkpoint_via_reader() { let temp = TempDir::new().expect("create temp dir"); let state_value = 5; let cold_value = 9; @@ -762,8 +846,9 @@ mod version_tests { let bytes = checkpoint.encode().expect("encode v1 checkpoint"); std::fs::write(temp.path().join("0.chkjam"), bytes).expect("write checkpoint"); - let (_, maybe_saveable) = - Saver::::try_load::(&temp.path().to_path_buf(), None) + let maybe_saveable = + CheckpointBootstrapReader::::new(temp.path().to_path_buf()) + .load_latest(None) .await .expect("load checkpoint"); @@ -771,14 +856,17 @@ mod version_tests { assert_eq!(saveable.ker_hash, ker_hash); assert_eq!(saveable.event_num, 7); - let loaded_state = atom_value(unsafe { *saveable.state.root() }); - let loaded_cold = atom_value(unsafe { *saveable.cold.root() }); + let state_space = saveable.state.noun_space(); + let cold_space = saveable.cold.noun_space(); + let loaded_state = atom_value(unsafe { *saveable.state.root() }, &state_space); + let loaded_cold = atom_value(unsafe { *saveable.cold.root() }, &cold_space); assert_eq!(loaded_state, state_value); assert_eq!(loaded_cold, cold_value); } - #[tokio::test] - async fn loads_v0_checkpoint_via_saver() { + #[tokio::test(flavor = "current_thread")] + #[cfg_attr(miri, ignore = "memfd_create unsupported in Miri")] + async fn loads_v0_checkpoint_via_reader() { let temp = TempDir::new().expect("create temp dir"); let state_value = 11; let cold_value = 22; @@ -788,8 +876,9 @@ mod version_tests { let bytes = checkpoint.encode().expect("encode v0 checkpoint"); std::fs::write(temp.path().join("0.chkjam"), bytes).expect("write checkpoint"); - let (_, maybe_saveable) = - Saver::::try_load::(&temp.path().to_path_buf(), None) + let maybe_saveable = + CheckpointBootstrapReader::::new(temp.path().to_path_buf()) + .load_latest(None) .await .expect("load checkpoint"); @@ -797,11 +886,94 @@ mod version_tests { assert_eq!(saveable.ker_hash, ker_hash); assert_eq!(saveable.event_num, 3); - let loaded_state = atom_value(unsafe { *saveable.state.root() }); - let loaded_cold = atom_value(unsafe { *saveable.cold.root() }); + let state_space = saveable.state.noun_space(); + let cold_space = saveable.cold.noun_space(); + let loaded_state = atom_value(unsafe { *saveable.state.root() }, &state_space); + let loaded_cold = atom_value(unsafe { *saveable.cold.root() }, &cold_space); assert_eq!(loaded_state, state_value); assert_eq!(loaded_cold, cold_value); } + + #[test] + fn decodes_v2_checkpoint_roundtrip() { + let checkpoint = JammedCheckpointV2::new( + hash(b"valid-v2"), + 11, + JammedNoun::new(Bytes::from_static(b"cold")), + JammedNoun::new(Bytes::from_static(b"state")), + ); + let encoded = checkpoint.encode().expect("encode v2 checkpoint"); + let decoded = + JammedCheckpointV2::decode_from_bytes(&encoded).expect("decode v2 checkpoint"); + + assert_eq!(decoded, checkpoint); + } + + #[test] + fn corrupt_v2_payload_length_is_reported_without_panic() { + let bytes = v2_checkpoint_with_absurd_payload_length(); + let load_result = + std::panic::catch_unwind(|| JammedCheckpointV2::decode_from_bytes(&bytes)); + + assert!( + load_result.is_ok(), + "corrupt v2 payload length must be reported as a decode error, not a panic" + ); + assert!(load_result.expect("checked above").is_err()); + } + + #[tokio::test] + async fn corrupt_checkpoint_length_does_not_panic_and_falls_back_to_previous_checkpoint() { + let temp = TempDir::new().expect("create temp dir"); + let valid_state_value = 5; + let valid_cold_value = 9; + let valid_ker_hash = hash(b"valid-v1"); + let valid_checkpoint = JammedCheckpointV1::new( + valid_ker_hash, + 7, + legacy_pair_jam(valid_state_value, valid_cold_value), + ); + + std::fs::write( + temp.path().join("0.chkjam"), + valid_checkpoint.encode().expect("encode valid checkpoint"), + ) + .expect("write valid checkpoint"); + std::fs::write( + temp.path().join("1.chkjam"), + v1_checkpoint_with_absurd_jam_length(8), + ) + .expect("write corrupt checkpoint"); + + let load_result = AssertUnwindSafe( + CheckpointBootstrapReader::::new(temp.path().to_path_buf()) + .load_latest(None), + ) + .catch_unwind() + .await; + + assert!( + load_result.is_ok(), + "corrupt checkpoint length must be reported as a load error, not a panic" + ); + + let maybe_saveable = load_result + .expect("checked above") + .expect("valid older checkpoint should be used"); + let saveable = maybe_saveable.expect("expected fallback checkpoint"); + assert_eq!(saveable.ker_hash, valid_ker_hash); + assert_eq!(saveable.event_num, 7); + let state_space = saveable.state.noun_space(); + let cold_space = saveable.cold.noun_space(); + assert_eq!( + atom_value(unsafe { *saveable.state.root() }, &state_space), + valid_state_value + ); + assert_eq!( + atom_value(unsafe { *saveable.cold.root() }, &cold_space), + valid_cold_value + ); + } } /* diff --git a/crates/nockapp/src/nockapp/test.rs b/crates/nockapp/src/nockapp/test.rs index 0d9ec8038..51a23f186 100644 --- a/crates/nockapp/src/nockapp/test.rs +++ b/crates/nockapp/src/nockapp/test.rs @@ -5,10 +5,9 @@ use tempfile::TempDir; use super::NockApp; use crate::kernel::form::Kernel; +use crate::save::SaveableCheckpoint; -pub async fn setup_nockapp(jam: &str) -> (TempDir, NockApp) { - let temp_dir = TempDir::new().expect("Failed to create temp directory"); - let temp_dir_path = temp_dir.path().to_path_buf(); +fn load_jam_bytes(jam: &str) -> Vec { // Try multiple possible locations for the jam file let possible_paths = [ Path::new(env!("CARGO_MANIFEST_DIR")) @@ -18,370 +17,43 @@ pub async fn setup_nockapp(jam: &str) -> (TempDir, NockApp) { // Add other potential paths ]; - let jam_bytes = possible_paths + possible_paths .iter() .find_map(|path| fs::read(path).ok()) - .unwrap_or_else(|| panic!("Failed to read {} file from any known location", jam)); + .unwrap_or_else(|| panic!("Failed to read {} file from any known location", jam)) +} + +pub async fn setup_nockapp(jam: &str) -> (TempDir, NockApp) { + let temp_dir = TempDir::new().expect("Failed to create temp directory"); + let jam_bytes = load_jam_bytes(jam); - let kernel_f = - async |checkpoint| Kernel::load(&jam_bytes, checkpoint, vec![], Default::default()).await; + let kernel_f = move |_| async move { + let kernel = Kernel::load(&jam_bytes, None, vec![], Default::default(), None).await?; + Ok::, crate::CrownError>(kernel) + }; ( temp_dir, - NockApp::new( - kernel_f, - &temp_dir_path, - Some(std::time::Duration::from_secs(1)), - ) - .await - .expect("Could not create NockApp"), + NockApp::new(kernel_f) + .await + .expect("Could not create NockApp"), ) } #[cfg(test)] pub mod tests { - use std::sync::atomic::Ordering; - use std::time::Duration; - use bytes::Bytes; - use nockvm::jets::util::slot; + use nockvm::ext::noun_equality; use nockvm::mem::NockStack; - use nockvm::noun::{Noun, D, T}; + use nockvm::noun::{Noun, NounAllocator}; use nockvm::serialization::{cue, jam}; use nockvm::unifying_equality::unifying_equality; - use nockvm_macros::tas; - use tracing::info; - use tracing_test::traced_test; use super::setup_nockapp; - use crate::nockapp::wire::{SystemWire, Wire}; - use crate::noun::slab::{slab_equality, slab_noun_equality, NockJammer, NounSlab}; - use crate::save::{SaveableCheckpoint, Saver}; + use crate::noun::slab::{slab_equality, NounSlab}; use crate::utils::NOCK_STACK_SIZE; - use crate::{NockApp, NounExt}; - - async fn save_nockapp(nockapp: &mut NockApp) { - nockapp.tasks.close(); - let permit = nockapp.save_mutex.clone().lock_owned().await; - let _ = nockapp.save(permit).await; - let _ = nockapp.tasks.wait().await; - nockapp.tasks.reopen(); - } - - // Panics if checkpoint failed to load, only permissible because this is expressly for testing - async fn spawn_save_t(nockapp: &mut NockApp, sleep_t: std::time::Duration) { - let sleepy_time = tokio::time::sleep(sleep_t); - let permit = nockapp.save_mutex.clone().lock_owned().await; - let _join_handle = nockapp - .save_f(sleepy_time, permit) - .await - .expect("Failed to spawn nockapp save task"); - // join_handle.await.expect("Failed to save nockapp").expect("Failed to save nockapp 2"); - } - - // Test nockapp save - // TODO: bump the actual serf event number (can we do a poke to the test kernel?) - #[test] - #[traced_test] - #[cfg_attr(miri, ignore)] - fn test_nockapp_save_race_condition() { - let runtime = tokio::runtime::Builder::new_multi_thread() - .worker_threads(2) - .enable_all() - .build() - .unwrap_or_else(|err| { - panic!( - "Panicked with {err:?} at {}:{} (git sha: {:?})", - file!(), - line!(), - option_env!("GIT_SHA") - ) - }); - let (temp, mut nockapp) = runtime.block_on(setup_nockapp("test-ker.jam")); - assert_eq!(nockapp.kernel.serf.event_number.load(Ordering::SeqCst), 0); - // first run - runtime.block_on(spawn_save_t(&mut nockapp, Duration::from_millis(1000))); - // second run - nockapp.kernel.serf.event_number.store(1, Ordering::SeqCst); // we need to set the actual serf event number - runtime.block_on(spawn_save_t(&mut nockapp, Duration::from_millis(5000))); - // Simulate what the event handlers would be doing and wait for the task tracker to be done - nockapp.tasks.close(); - runtime.block_on(nockapp.tasks.wait()); - nockapp.tasks.reopen(); - // Shutdown the runtime immediately - runtime.shutdown_timeout(std::time::Duration::from_secs(0)); - - let runtime = tokio::runtime::Builder::new_multi_thread() - .worker_threads(2) - .enable_all() - .build() - .expect("Failed to build runtime"); - - let (_, checkpoint_opt) = runtime - .block_on(Saver::::try_load( - &temp.path().to_path_buf(), - None, - )) - .expect("Failed trying to load checkpoint"); - let checkpoint: SaveableCheckpoint = checkpoint_opt.expect("No checkpoint found"); - info!("checkpoint: {:?}", checkpoint); - assert_eq!(checkpoint.event_num, 1); - } - - // Test nockapp save - // TODO: need a way to grab arvo state from the serf. Probably a serf action - // TODO: use slab equality, not unifying equality - #[tokio::test] - #[traced_test] - #[cfg_attr(miri, ignore)] - async fn test_nockapp_save() { - // console_subscriber::init(); - let (temp, mut nockapp) = setup_nockapp("test-ker.jam").await; - let first_checkpoint = nockapp - .kernel - .checkpoint() - .await - .expect("Couldn't get kernel checkpoint"); - - assert_eq!(nockapp.kernel.serf.event_number.load(Ordering::SeqCst), 0); - // Save - info!("Saving nockapp"); - save_nockapp(&mut nockapp).await; - // Permit should be dropped - - // A valid checkpoint should exist in one of the jam files - let (_, checkpoint_opt) = Saver::::try_load(&temp.path().to_path_buf(), None) - .await - .expect("Could not load checkpoint"); - let checkpoint: SaveableCheckpoint = checkpoint_opt.expect("No checkpoint loaded"); - - // Checkpoint event number should be 0 - assert_eq!(checkpoint.event_num, 0); - - info!("Asserting checkpoint and arvo equality"); - // Checkpoint kernel should be equal to the saved kernel - assert!(slab_equality(&checkpoint.state, &first_checkpoint.state)); - assert!(slab_equality(&checkpoint.cold, &first_checkpoint.cold)); - } - - // Test nockapp poke - #[tokio::test] - #[traced_test] - #[cfg_attr(miri, ignore)] - async fn test_nockapp_poke_save() { - let (temp, mut nockapp) = setup_nockapp("test-ker.jam").await; - assert_eq!(nockapp.kernel.serf.event_number.load(Ordering::SeqCst), 0); - let state_before_poke = nockapp - .kernel - .checkpoint() - .await - .expect("Can't get kernel state before poke"); - - let poke_noun = D(tas!(b"inc")); - let poke = { - let mut slab = NounSlab::new(); - slab.copy_into(poke_noun); - slab - }; - - let wire = SystemWire.to_wire(); - let _ = nockapp.kernel.poke(wire, poke).await.unwrap_or_else(|err| { - panic!( - "Panicked with {err:?} at {}:{} (git sha: {:?})", - file!(), - line!(), - option_env!("GIT_SHA") - ) - }); - - // Save - save_nockapp(&mut nockapp).await; - - // A valid checkpoint should exist in one of the jam files - let (_, checkpoint_opt) = Saver::::try_load(&temp.path().to_path_buf(), None) - .await - .expect("Failed to load checkpoint"); - let checkpoint: SaveableCheckpoint = checkpoint_opt.expect("No checkpoint"); - - // Checkpoint event number should be 1 - assert!(checkpoint.event_num == 1); - let state_after_poke = nockapp - .kernel - .checkpoint() - .await - .expect("Failed to get checkpoint after poke"); - - assert!(slab_equality(&checkpoint.state, &state_after_poke.state)); - assert!(slab_equality(&checkpoint.cold, &state_after_poke.cold)); - assert!(!slab_equality(&checkpoint.state, &state_before_poke.state)); - } - - #[tokio::test] - #[traced_test] - #[cfg_attr(miri, ignore)] - async fn test_nockapp_save_multiple() { - let (temp, mut nockapp) = setup_nockapp("test-ker.jam").await; - assert_eq!(nockapp.kernel.serf.event_number.load(Ordering::SeqCst), 0); - let mut stack = NockStack::new(NOCK_STACK_SIZE, 0); - - for i in 1..4 { - // Poke to increment the state - let poke_noun = D(tas!(b"inc")); - let poke = { - let mut slab = NounSlab::new(); - slab.copy_into(poke_noun); - slab - }; - let wire = SystemWire.to_wire(); - let _ = nockapp.kernel.poke(wire, poke).await.unwrap_or_else(|err| { - panic!( - "Panicked with {err:?} at {}:{} (git sha: {:?})", - file!(), - line!(), - option_env!("GIT_SHA") - ) - }); - - // Save - save_nockapp(&mut nockapp).await; - - // A valid checkpoint should exist in one of the jam files - let (_, checkpoint_opt) = - Saver::::try_load(&temp.path().to_path_buf(), None) - .await - .expect("Failed to load checkpoint"); - let checkpoint: SaveableCheckpoint = checkpoint_opt.expect("No checkpoint found"); - - // Checkpoint event number should be i - assert!(checkpoint.event_num == i); - - // Checkpointed state should have been incremented - let peek_noun = T(&mut stack, &[D(tas!(b"state")), D(0)]); - let peek = { - let mut slab = NounSlab::new(); - slab.copy_into(peek_noun); - slab - }; - - // res should be [~ ~ [%0 val]] - let mut res = nockapp.kernel.peek(peek).await.unwrap_or_else(|err| { - panic!( - "Panicked with {err:?} at {}:{} (git sha: {:?})", - file!(), - line!(), - option_env!("GIT_SHA") - ) - }); - res.modify_noun(|r| { - slot(r, 7) - .unwrap_or_else(|err| { - panic!( - "Panicked with {err:?} at {}:{} (git sha: {:?})", - file!(), - line!(), - option_env!("GIT_SHA") - ) - }) - .as_cell() - .unwrap_or_else(|err| { - panic!( - "Panicked with {err:?} at {}:{} (git sha: {:?})", - file!(), - line!(), - option_env!("GIT_SHA") - ) - }) - .tail() - }); - - let comp = { - let mut slab = NounSlab::new(); - slab.copy_into(D(i)); - slab - }; - - assert!( - slab_equality(&res, &comp), - "res: {:?} != comp: {:?}", - res, - comp - ); - } - } - - // Tests for fallback to previous checkpoint if checkpoint is corrupt - // TODO: ask about this test and reframe it for 'Saver' - /* - #[tokio::test] - #[traced_test] - #[cfg_attr(miri, ignore)] - async fn test_nockapp_corrupt_check() { - let (temp, mut nockapp) = setup_nockapp("test-ker.jam").await; - assert_eq!(nockapp.kernel.serf.event_number.load(Ordering::SeqCst), 0); - - // Save a valid checkpoint - save_nockapp(&mut nockapp).await; - - // Generate an invalid checkpoint by incrementing the event number - let mut invalid = nockapp - .kernel - .checkpoint() - .await - .expect("Could not get kernel checkpoint"); - invalid.event_num += 1; - assert!(!invalid.validate()); - - // The invalid checkpoint has a higher event number than the valid checkpoint - let mut checkpoint_stack = NockStack::new(NOCK_STACK_SIZE, 0); - let valid = jam_paths - .load_checkpoint(&mut checkpoint_stack) - .unwrap_or_else(|err| { - panic!( - "Panicked with {err:?} at {}:{} (git sha: {:?})", - file!(), - line!(), - option_env!("GIT_SHA") - ) - }); - assert!(valid.event_num < invalid.event_num); - - // Save the corrupted checkpoint, because of the toggle buffer, we will write to jam file 1 - assert!(!jam_paths.1.exists()); - let jam_path = &jam_paths.1; - let jam_bytes = invalid.encode().unwrap_or_else(|err| { - panic!( - "Panicked with {err:?} at {}:{} (git sha: {:?})", - file!(), - line!(), - option_env!("GIT_SHA") - ) - }); - tokio::fs::write(jam_path, jam_bytes) - .await - .unwrap_or_else(|err| { - panic!( - "Panicked with {err:?} at {}:{} (git sha: {:?})", - file!(), - line!(), - option_env!("GIT_SHA") - ) - }); - - // The loaded checkpoint will be the valid one - let chk = jam_paths - .load_checkpoint(&mut checkpoint_stack) - .unwrap_or_else(|err| { - panic!( - "Panicked with {err:?} at {}:{} (git sha: {:?})", - file!(), - line!(), - option_env!("GIT_SHA") - ) - }); - assert!(chk.event_num == valid.event_num); - } - */ + use crate::NounExt; - #[tokio::test] + #[tokio::test(flavor = "current_thread")] #[cfg_attr(miri, ignore)] async fn test_jam_equality_stack() { let (_temp, nockapp) = setup_nockapp("test-ker.jam").await; @@ -425,7 +97,7 @@ pub mod tests { }); let jammed_bytes = slab1.jam(); let mut slab2: NounSlab = NounSlab::new(); - let c = slab2.cue_into(jammed_bytes).unwrap_or_else(|err| { + let _c = slab2.cue_into(jammed_bytes).unwrap_or_else(|err| { panic!( "Panicked with {err:?} at {}:{} (git sha: {:?})", file!(), @@ -433,10 +105,10 @@ pub mod tests { option_env!("GIT_SHA") ) }); - unsafe { assert!(slab_noun_equality(slab1.root(), &c)) } + assert!(slab_equality(&slab1, &slab2)); } - #[tokio::test] + #[tokio::test(flavor = "current_thread")] #[cfg_attr(miri, ignore)] async fn test_jam_equality_slab() { let (_temp, nockapp) = setup_nockapp("test-ker.jam").await; @@ -455,10 +127,12 @@ pub mod tests { option_env!("GIT_SHA") ) }); - unsafe { assert!(slab_noun_equality(state_slab.root(), &c)) } + let space = state_slab.noun_space(); + let root = unsafe { state_slab.root() }; + assert!(noun_equality(root.in_space(&space), c.in_space(&space))); } - #[tokio::test] + #[tokio::test(flavor = "current_thread")] #[cfg_attr(miri, ignore)] async fn test_jam_equality_slab_stack() { let (_temp, nockapp) = setup_nockapp("test-ker.jam").await; diff --git a/crates/nockapp/src/noun/extensions.rs b/crates/nockapp/src/noun/extensions.rs index be32f3dde..054ddbddd 100644 --- a/crates/nockapp/src/noun/extensions.rs +++ b/crates/nockapp/src/noun/extensions.rs @@ -4,7 +4,7 @@ use bytes::Bytes; use either::Either; use nockvm::ext::AtomExt as CoreAtomExt; pub use nockvm::ext::{IndirectAtomExt, JammedNoun, NounExt}; -use nockvm::noun::{Atom, Cell, IndirectAtom, NounAllocator, D}; +use nockvm::noun::{Atom, Cell, IndirectAtom, NounAllocator, NounSpace, D}; use crate::noun::slab::NounSlab; use crate::{Noun, Result, ToBytes, ToBytesExt}; @@ -16,9 +16,6 @@ use crate::{Noun, Result, ToBytes, ToBytesExt}; pub trait AtomExt: CoreAtomExt { fn from_bytes(allocator: &mut A, bytes: &Bytes) -> Atom; fn from_value(allocator: &mut A, value: T) -> Result; - fn eq_bytes(self, bytes: impl AsRef<[u8]>) -> bool; - fn to_bytes_until_nul(self) -> Result>; - fn into_string(self) -> Result; } impl AtomExt for Atom { @@ -33,22 +30,13 @@ impl AtomExt for Atom { Ok(::from_bytes(allocator, data.as_ref())) } - /** Test for byte equality, ignoring trailing 0s in the Atom representation - beyond the length of the bytes compared to - */ - fn eq_bytes(self, bytes: impl AsRef<[u8]>) -> bool { - CoreAtomExt::eq_bytes(&self, bytes) - } - - fn to_bytes_until_nul(self) -> Result> { - CoreAtomExt::to_bytes_until_nul(&self).map_err(Into::into) - } - - fn into_string(self) -> Result { - CoreAtomExt::into_string(self).map_err(Into::into) - } + // NounSpace-dependent helpers moved to NounHandle/AtomHandle. } +#[diagnostic::on_unimplemented( + message = "`{Self}` cannot implement `IntoNoun` safely", + label = "use `IntoSlab` or allocate into a caller-owned allocator" +)] pub trait IntoNoun { fn into_noun(self) -> Noun; } @@ -65,8 +53,8 @@ impl IntoNoun for u64 { } impl FromAtom for u64 { - fn from_atom(atom: Atom) -> Self { - atom.as_u64().unwrap_or_else(|err| { + fn from_atom(atom: Atom, space: &NounSpace) -> Self { + atom.in_space(space).as_u64().unwrap_or_else(|err| { panic!( "Panicked with {err:?} at {}:{} (git sha: {:?})", file!(), @@ -82,34 +70,19 @@ impl IntoNoun for Noun { self } } -impl IntoNoun for &str { - fn into_noun(self) -> Noun { - let mut slab: NounSlab = NounSlab::new(); - let bytes = self.to_bytes().unwrap_or_else(|err| { - panic!( - "Panicked with {err:?} at {}:{} (git sha: {:?})", - file!(), - line!(), - option_env!("GIT_SHA") - ) - }); - let contents_atom = - ::from_bytes(&mut slab, bytes.as_slice()); - Noun::from_atom(contents_atom) - } -} +impl !IntoNoun for &str {} pub trait AsSlabVec { - fn as_slab_vec(&self) -> Vec; + fn as_slab_vec(&self, space: &NounSpace) -> Vec; } impl AsSlabVec for Noun { - fn as_slab_vec(&self) -> Vec { + fn as_slab_vec(&self, space: &NounSpace) -> Vec { let noun_list = *self; let mut slab_vec = Vec::new(); - for noun in noun_list.list_iter() { + for noun in noun_list.in_space(space).list_iter() { let mut new_slab = NounSlab::new(); - new_slab.copy_into(noun); + new_slab.copy_into(noun.noun(), space); slab_vec.push(new_slab); } slab_vec @@ -117,17 +90,18 @@ impl AsSlabVec for Noun { } impl AsSlabVec for NounSlab { - fn as_slab_vec(&self) -> Vec { + fn as_slab_vec(&self, _space: &NounSpace) -> Vec { let noun_list = unsafe { self.root() }; - noun_list.as_slab_vec() + let space = self.noun_space(); + noun_list.as_slab_vec(&space) } } pub trait FromAtom { - fn from_atom(atom: Atom) -> Self; + fn from_atom(atom: Atom, space: &NounSpace) -> Self; } impl FromAtom for Noun { - fn from_atom(atom: Atom) -> Self { + fn from_atom(atom: Atom, _space: &NounSpace) -> Self { atom.as_noun() } } @@ -139,18 +113,27 @@ pub trait IntoSlab { impl IntoSlab for &str { fn into_slab(self) -> NounSlab { let mut slab = NounSlab::new(); - let noun = self.into_noun(); + let bytes = self.to_bytes().unwrap_or_else(|err| { + panic!( + "Panicked with {err:?} at {}:{} (git sha: {:?})", + file!(), + line!(), + option_env!("GIT_SHA") + ) + }); + let noun = + ::from_bytes(&mut slab, bytes.as_slice()).as_noun(); slab.set_root(noun); slab } } pub trait NounAllocatorExt { - fn copy_into(&mut self, noun: Noun) -> Noun; + fn copy_into(&mut self, noun: Noun, space: &NounSpace) -> Noun; } impl NounAllocatorExt for A { - fn copy_into(&mut self, noun: Noun) -> Noun { + fn copy_into(&mut self, noun: Noun, space: &NounSpace) -> Noun { let mut stack = Vec::with_capacity(32); let mut res = D(0); stack.push((noun, &mut res as *mut Noun)); @@ -161,16 +144,18 @@ impl NounAllocatorExt for A { }, Either::Right(a) => match a.as_either() { Either::Left(i) => unsafe { - let word_size = i.size(); + let i_handle = i.as_atom().in_space(space); + let word_size = i_handle.size(); let ia = self.alloc_indirect(word_size); - copy_nonoverlapping(i.to_raw_pointer(), ia, word_size + 2); + copy_nonoverlapping(i_handle.raw_pointer(), ia, word_size + 2); *dest = IndirectAtom::from_raw_pointer(ia).as_noun(); }, Either::Right(c) => unsafe { let cm = self.alloc_cell(); *dest = Cell::from_raw_pointer(cm).as_noun(); - stack.push((c.tail(), &mut (*cm).tail)); - stack.push((c.head(), &mut (*cm).head)); + let c_handle = c.in_space(space); + stack.push((c_handle.tail().noun(), &mut (*cm).tail)); + stack.push((c_handle.head().noun(), &mut (*cm).head)); }, }, } @@ -178,3 +163,25 @@ impl NounAllocatorExt for A { res } } + +#[cfg(test)] +mod tests { + use nockvm::noun::NounAllocator; + + use super::IntoSlab; + + #[test] + fn str_into_slab_allocates_in_destination_slab() { + let slab = "hello".into_slab(); + let root = unsafe { *slab.root() }; + let space = slab.noun_space(); + let atom = root + .in_space(&space) + .as_atom() + .expect("root should be an atom"); + let text = atom + .into_string() + .expect("root atom should decode to utf-8"); + assert_eq!(text, "hello"); + } +} diff --git a/crates/nockapp/src/noun/slab.rs b/crates/nockapp/src/noun/slab.rs index c0bf12357..c7e9ae877 100644 --- a/crates/nockapp/src/noun/slab.rs +++ b/crates/nockapp/src/noun/slab.rs @@ -10,19 +10,18 @@ use bytes::Bytes; use either::Either; use ibig::Stack; use intmap::IntMap; +use nockvm::ext::noun_equality; use nockvm::mem::NockStack; use nockvm::mug::{calc_atom_mug_u32, calc_cell_mug_u32, get_mug, set_mug}; -use nockvm::noun::{Atom, Cell, CellMemory, DirectAtom, IndirectAtom, Noun, NounAllocator, D}; +use nockvm::noun::{ + Atom, Cell, CellMemory, DirectAtom, IndirectAtom, Noun, NounAllocator, NounSpace, D, DIRECT_MAX, +}; use nockvm::serialization::{met0_u64_to_usize, met0_usize}; use thiserror::Error; -use crate::noun::NounExt; - const CELL_MEM_WORD_SIZE: usize = (size_of::() + 7) >> 3; -/// A (mostly*) self-contained arena for allocating nouns. -/// -/// *Nouns may contain references to the PMA, but not other allocation arenas. +/// A self-contained arena for allocating nouns. pub struct NounSlab { root: Noun, slabs: Vec<(*mut u8, Layout)>, @@ -43,6 +42,79 @@ impl Debug for NounSlab { } impl NounSlab { + fn contains_ptr(&self, ptr: *const u8) -> bool { + self.slabs.iter().any(|(base, layout)| unsafe { + !base.is_null() && ptr >= *base as *const u8 && ptr < base.add(layout.size()) + }) + } + + fn rehome_noun(&mut self, noun: Noun) -> Noun { + let mut copied = IntMap::new(); + self.rehome_noun_inner(noun, &mut copied) + } + + fn rehome_noun_inner(&mut self, noun: Noun, copied: &mut IntMap) -> Noun { + match noun.as_either_direct_allocated() { + Either::Left(direct) => direct.as_noun(), + Either::Right(allocated) => match allocated.as_either() { + Either::Left(indirect) => { + let Some(data_ptr) = indirect.data_pointer_stack() else { + panic!( + "Cannot splice offset-form noun into NounSlab without a source NounSpace" + ); + }; + if self.contains_ptr(data_ptr as *const u8) { + return noun; + } + + let src_ptr = unsafe { indirect.to_raw_pointer_stack() }; + let src_key = src_ptr as u64; + if let Some(copied_noun) = copied.get(src_key) { + return *copied_noun; + } + + let size = unsafe { *src_ptr.add(1) as usize }; + let new_mem = unsafe { self.alloc_indirect(size) }; + unsafe { + copy_nonoverlapping(src_ptr, new_mem, size + 2); + } + let copied_noun = + unsafe { IndirectAtom::from_raw_pointer(new_mem).as_atom().as_noun() }; + copied.insert(src_key, copied_noun); + copied_noun + } + Either::Right(cell) => { + let Some(cell_ptr) = cell.stack_memory_pointer() else { + panic!( + "Cannot splice offset-form noun into NounSlab without a source NounSpace" + ); + }; + let src_key = cell_ptr as u64; + if let Some(copied_noun) = copied.get(src_key) { + return *copied_noun; + } + + let source_head = unsafe { (*cell_ptr).head }; + let source_tail = unsafe { (*cell_ptr).tail }; + let rehomed_head = self.rehome_noun_inner(source_head, copied); + let rehomed_tail = self.rehome_noun_inner(source_tail, copied); + + if self.contains_ptr(cell_ptr as *const u8) + && unsafe { rehomed_head.raw_equals(&source_head) } + && unsafe { rehomed_tail.raw_equals(&source_tail) } + { + copied.insert(src_key, noun); + return noun; + } + + let copied_noun = Cell::new(self, rehomed_head, rehomed_tail).as_noun(); + copied.insert(src_key, copied_noun); + copied_noun + } + }, + } + } + pub fn coerce_jammer(mut self) -> NounSlab { let slabs = std::mem::take(&mut self.slabs); NounSlab { @@ -68,11 +140,13 @@ impl NounSlab { } pub fn to_vec(&self) -> Vec { + let space = self.noun_space(); self.root + .in_space(&space) .list_iter() .map(|n| { let mut slab = Self::new(); - slab.copy_into(n); + slab.copy_into(n.noun(), &space); slab }) .collect() @@ -80,12 +154,16 @@ impl NounSlab { pub fn modify Vec>(&mut self, f: F) { let new_root_base = f(self.root); - let new_root = nockvm::noun::T(self, &new_root_base); + let rehomed_root_base: Vec = new_root_base + .into_iter() + .map(|noun| self.rehome_noun(noun)) + .collect(); + let new_root = nockvm::noun::T(self, &rehomed_root_base); self.set_root(new_root); } pub fn modify_noun Noun>(&mut self, f: F) { - let new_root = f(self.root); + let new_root = self.rehome_noun(f(self.root)); self.set_root(new_root); } @@ -93,12 +171,20 @@ impl NounSlab { &mut self, f: F, imports: (Noun, Noun, Noun), + space: &NounSpace, ) { - self.copy_into(imports.0); - self.copy_into(imports.1); - self.copy_into(imports.2); - let new_root_base = f(imports, self.root); - let new_root = nockvm::noun::T(self, &new_root_base); + let old_root = self.root; + let imported = ( + self.copy_into(imports.0, space), + self.copy_into(imports.1, space), + self.copy_into(imports.2, space), + ); + let new_root_base = f(imported, old_root); + let rehomed_root_base: Vec = new_root_base + .into_iter() + .map(|noun| self.rehome_noun(noun)) + .collect(); + let new_root = nockvm::noun::T(self, &rehomed_root_base); self.set_root(new_root); } } @@ -106,7 +192,8 @@ impl NounSlab { impl Clone for NounSlab { fn clone(&self) -> Self { let mut slab = Self::new(); - slab.copy_into(self.root); + let space = self.noun_space(); + slab.copy_into(self.root, &space); slab } } @@ -209,11 +296,17 @@ impl NounAllocator for NounSlab { unsafe fn equals(&mut self, a: *mut Noun, b: *mut Noun) -> bool { let a = unsafe { &mut *a }; let b = unsafe { &mut *b }; - slab_noun_equality(a, b) + let space = self.noun_space(); + noun_equality((*a).in_space(&space), (*b).in_space(&space)) + } + + fn noun_space(&self) -> NounSpace { + NounSpace::empty().with_extra_ptr_ranges(self.ptr_ranges()) } } -/// # Safety: no noun in this slab references a noun outside the slab, except in the PMA +/// # Safety: sending a slab across threads relies on the invariant that every noun reachable from +/// its root is allocated inside the slab. unsafe impl Send for NounSlab {} impl Default for NounSlab { @@ -222,10 +315,10 @@ impl Default for NounSlab { } } -impl From for NounSlab { - fn from(noun: Noun) -> Self { +impl NounSlab { + pub fn from_noun(noun: Noun, space: &NounSpace) -> Self { let mut slab = Self::new(); - slab.copy_into(noun); + slab.copy_into(noun, space); slab } } @@ -233,6 +326,7 @@ impl From for NounSlab { impl From<[Noun; N]> for NounSlab { fn from(nouns: [Noun; N]) -> Self { let mut slab = Self::new(); + let nouns = nouns.map(|noun| slab.rehome_noun(noun)); let new_root = nockvm::noun::T(&mut slab, &nouns); slab.set_root(new_root); slab @@ -258,12 +352,12 @@ impl NounSlab { /// Copy the root from another slab into this slab, set this slab's root to the copied root pub fn copy_from_slab(&mut self, other: &NounSlab) { - self.copy_into(other.root); + let space = other.noun_space(); + self.copy_into(other.root, &space); } - /// Copy a noun into this slab, only leaving references into the PMA. Set that noun as the root - /// noun. - pub fn copy_into(&mut self, copy_root: Noun) -> Noun { + /// Copy a noun into this slab and set that noun as the root noun. + pub fn copy_into(&mut self, copy_root: Noun, space: &NounSpace) -> Noun { let mut copied: IntMap = IntMap::new(); // let mut copy_stack = vec![(copy_root, &mut self.root as *mut Noun)]; let mut copy_stack = vec![(copy_root, std::ptr::addr_of_mut!(self.root))]; @@ -274,13 +368,16 @@ impl NounSlab { } Either::Right(allocated) => match allocated.as_either() { Either::Left(indirect) => { - let indirect_ptr = unsafe { indirect.to_raw_pointer() }; - let indirect_mem_size = indirect.raw_size(); + let indirect_ptr = + unsafe { indirect.as_atom().in_space(space).raw_pointer() }; + let indirect_mem_size = indirect.as_atom().in_space(space).raw_size(); if let Some(copied_noun) = copied.get(indirect_ptr as u64) { unsafe { *dest = *copied_noun }; continue; } - let indirect_new_mem = unsafe { self.alloc_indirect(indirect.size()) }; + let indirect_new_mem = unsafe { + self.alloc_indirect(indirect.as_atom().in_space(space).size()) + }; unsafe { copy_nonoverlapping(indirect_ptr, indirect_new_mem, indirect_mem_size) }; @@ -293,7 +390,7 @@ impl NounSlab { unsafe { *dest = copied_noun }; } Either::Right(cell) => { - let cell_ptr = unsafe { cell.to_raw_pointer() }; + let cell_ptr = unsafe { cell.in_space(space).raw_pointer() }; if let Some(copied_noun) = copied.get(cell_ptr as u64) { unsafe { *dest = *copied_noun }; continue; @@ -308,10 +405,14 @@ impl NounSlab { // .push((cell.tail(), &mut (*cell_new_mem).tail as *mut Noun)); // copy_stack // .push((cell.head(), &mut (*cell_new_mem).head as *mut Noun)); - copy_stack - .push((cell.tail(), std::ptr::addr_of_mut!((*cell_new_mem).tail))); - copy_stack - .push((cell.head(), std::ptr::addr_of_mut!((*cell_new_mem).head))); + copy_stack.push(( + cell.in_space(space).tail().noun(), + std::ptr::addr_of_mut!((*cell_new_mem).tail), + )); + copy_stack.push(( + cell.in_space(space).head().noun(), + std::ptr::addr_of_mut!((*cell_new_mem).head), + )); } } }, @@ -320,40 +421,53 @@ impl NounSlab { self.root } - /// Copy the root noun from this slab into the given NockStack, only leaving references into the PMA + /// Copy the root noun from this slab into the given NockStack. /// /// Note that this consumes the slab, the slab will be freed after and the root noun returned /// referencing the stack. Nouns referencing the slab should not be used past this point. #[tracing::instrument(skip(self, stack), level = "trace")] pub fn copy_to_stack(self, stack: &mut NockStack) -> Noun { + let space = stack.noun_space().with_extra_ptr_ranges(self.ptr_ranges()); let mut res = D(0); let mut copy_stack = vec![(self.root, &mut res as *mut Noun)]; while let Some((noun, dest)) = copy_stack.pop() { if let Ok(allocated) = noun.as_allocated() { - if let Some(forward) = unsafe { allocated.forwarding_pointer() } { - unsafe { *dest = forward.as_noun() }; + let noun_handle = noun.in_space(&space); + if let Some(forward) = unsafe { noun_handle.forwarding_pointer() } { + unsafe { *dest = forward.noun() }; } else { match allocated.as_either() { - Either::Left(mut indirect) => { - let raw_pointer = unsafe { indirect.to_raw_pointer() }; - let raw_size = indirect.raw_size(); + Either::Left(indirect) => { + let raw_pointer = + unsafe { indirect.as_atom().in_space(&space).raw_pointer() }; + let raw_size = indirect.as_atom().in_space(&space).raw_size(); unsafe { - let indirect_mem = stack.alloc_indirect(indirect.size()); + let indirect_mem = stack + .alloc_indirect(indirect.as_atom().in_space(&space).size()); std::ptr::copy_nonoverlapping(raw_pointer, indirect_mem, raw_size); - indirect.set_forwarding_pointer(indirect_mem); + indirect + .as_atom() + .in_space(&space) + .set_forwarding_pointer(indirect_mem); *dest = IndirectAtom::from_raw_pointer(indirect_mem) .as_atom() .as_noun(); } } - Either::Right(mut cell) => { - let raw_pointer = unsafe { cell.to_raw_pointer() }; + Either::Right(cell) => { + let raw_pointer = unsafe { cell.in_space(&space).raw_pointer() }; unsafe { let cell_mem = stack.alloc_cell(); copy_nonoverlapping(raw_pointer, cell_mem, 1); - copy_stack.push((cell.tail(), &mut (*cell_mem).tail as *mut Noun)); - copy_stack.push((cell.head(), &mut (*cell_mem).head as *mut Noun)); - cell.set_forwarding_pointer(cell_mem); + copy_stack.push(( + cell.in_space(&space).tail().noun(), + &mut (*cell_mem).tail as *mut Noun, + )); + copy_stack.push(( + cell.in_space(&space).head().noun(), + &mut (*cell_mem).head as *mut Noun, + )); + cell.in_space(&space).set_forwarding_pointer(cell_mem); *dest = Cell::from_raw_pointer(cell_mem).as_noun() } } @@ -368,31 +482,44 @@ impl NounSlab { res } + fn ptr_ranges(&self) -> Vec<(usize, usize)> { + let mut ranges = Vec::with_capacity(self.slabs.len()); + for (base, layout) in &self.slabs { + if base.is_null() || layout.size() == 0 { + continue; + } + let start = *base as usize; + let end = start + layout.size(); + ranges.push((start, end)); + } + ranges + } + /// Set the root of the noun slab. /// - /// Panics if the given root is not in the noun slab or PMA. + /// Panics if the given root is not in the noun slab. pub fn set_root(&mut self, root: Noun) { if let Ok(allocated) = root.as_allocated() { match allocated.as_either() { Either::Left(indirect) => { - let ptr = unsafe { indirect.to_raw_pointer() }; + let Some(ptr) = indirect.data_pointer_stack() else { + panic!("Set root of NounSlab to noun from outside slab"); + }; let u8_ptr = ptr as *const u8; - for slab in &self.slabs { - if unsafe { u8_ptr >= slab.0 && u8_ptr < slab.0.add(slab.1.size()) } { - self.root = root; - return; - } + if self.contains_ptr(u8_ptr) { + self.root = root; + return; } panic!("Set root of NounSlab to noun from outside slab"); } Either::Right(cell) => { - let ptr = unsafe { cell.to_raw_pointer() }; + let Some(ptr) = cell.stack_memory_pointer() else { + panic!("Set root of NounSlab to noun from outside slab"); + }; let u8_ptr = ptr as *const u8; - for slab in &self.slabs { - if unsafe { u8_ptr >= slab.0 && u8_ptr < slab.0.add(slab.1.size()) } { - self.root = root; - return; - } + if self.contains_ptr(u8_ptr) { + self.root = root; + return; } panic!("Set root of NounSlab to noun from outside slab"); } @@ -403,14 +530,18 @@ impl NounSlab { /// Get the root noun /// - /// # Safety: The noun must not be used past the lifetime of the slab. + /// # Safety + /// + /// The noun must not be used past the lifetime of the slab. pub unsafe fn root(&self) -> &Noun { &self.root } /// Get the root noun /// - /// # Safety: The noun must not be used past the lifetime of the slab. + /// # Safety + /// + /// The noun must not be used past the lifetime of the slab. pub unsafe fn root_mut(&mut self) -> &mut Noun { &mut self.root } @@ -418,7 +549,8 @@ impl NounSlab { impl NounSlab { pub fn jam(&self) -> Bytes { - J::jam(unsafe { *self.root() }) + let space = self.noun_space(); + J::jam(unsafe { *self.root() }, &space) } pub fn cue_into(&mut self, jammed: Bytes) -> Result { @@ -510,11 +642,13 @@ impl NounMap { pub fn new() -> Self { NounMap(IntMap::new()) } - pub fn insert(&mut self, key: Noun, value: V) { - let key_mug = slab_mug(key) as u64; + pub fn insert(&mut self, key: Noun, value: V, space: &NounSpace) { + let key_mug = slab_mug(key, space) as u64; if let Some(vec) = self.0.get_mut(key_mug) { let mut chain_iter = vec[..].iter_mut(); - if let Some(entry) = chain_iter.find(|entry| slab_noun_equality(&key, &entry.0)) { + if let Some(entry) = + chain_iter.find(|entry| noun_equality(key.in_space(space), entry.0.in_space(space))) + { entry.1 = value; } else { vec.push((key, value)) @@ -524,12 +658,12 @@ impl NounMap { } } - pub fn get(&self, key: Noun) -> Option<&V> { - let key_mug = slab_mug(key) as u64; + pub fn get(&self, key: Noun, space: &NounSpace) -> Option<&V> { + let key_mug = slab_mug(key, space) as u64; if let Some(vec) = self.0.get(key_mug) { let mut chain_iter = vec[..].iter(); - if let Some(entry) = - chain_iter.find(|entry| slab_noun_equality(&(key as Noun), &entry.0)) + if let Some(entry) = chain_iter + .find(|entry| noun_equality((key as Noun).in_space(space), entry.0.in_space(space))) { Some(&entry.1) } else { @@ -541,119 +675,48 @@ impl NounMap { } } -pub fn slab_equality(a: &NounSlab, b: &NounSlab) -> bool { - slab_noun_equality(&a.root, &b.root) -} - -// Does not unify: slabs are collected all-at-once so there's no point. -pub fn slab_noun_equality(a: &Noun, b: &Noun) -> bool { - let mut already_equal: IntMap = IntMap::new(); - - fn ae_keys(a: Noun, b: Noun) -> (u128, u128) { - let a_raw = unsafe { a.as_raw() } as u128; - let b_raw = unsafe { b.as_raw() } as u128; - (a_raw << 64 | b_raw, b_raw << 64 | a_raw) - } - - fn check_ae(ae: &IntMap, a: Noun, b: Noun) -> bool { - let (key1, key2) = ae_keys(a, b); - ae.contains_key(key1) | ae.contains_key(key2) - } - - fn set_ae(ae: &mut IntMap, a: Noun, b: Noun) { - let (key1, _key2) = ae_keys(a, b); - ae.insert(key1, ()); - } - - enum StackEntry { - Nouns(Noun, Noun), - Cells(Noun, Noun), - } - - let mut stack = vec![StackEntry::Nouns(*a, *b)]; - loop { - if let Some(entry) = stack.pop() { - match entry { - StackEntry::Cells(a, b) => { - set_ae(&mut already_equal, a, b); - } - StackEntry::Nouns(a, b) => { - if unsafe { a.raw_equals(&b) } { - continue; - } - - if check_ae(&already_equal, a, b) { - continue; - } - - match ( - a.as_ref_either_direct_allocated(), - b.as_ref_either_direct_allocated(), - ) { - (Either::Right(a_allocated), Either::Right(b_allocated)) => { - if let Some(a_mug) = a_allocated.get_cached_mug() { - if let Some(b_mug) = b_allocated.get_cached_mug() { - if a_mug != b_mug { - break false; - } - } - }; - - match (a_allocated.as_ref_either(), b_allocated.as_ref_either()) { - (Either::Left(a_indirect), Either::Left(b_indirect)) => { - if a_indirect.as_slice() != b_indirect.as_slice() { - break false; - } - set_ae(&mut already_equal, a, b); - continue; - } - (Either::Right(a_cell), Either::Right(b_cell)) => { - stack.push(StackEntry::Cells(a, b)); - stack.push(StackEntry::Nouns(a_cell.tail(), b_cell.tail())); - stack.push(StackEntry::Nouns(a_cell.head(), b_cell.head())); - continue; - } - _ => { - break false; - } - } - } - _ => { - break false; - } - } - } - } - } else { - break true; - } - } +pub fn slab_equality(a: &NounSlab, b: &NounSlab) -> bool { + let mut ranges = a.ptr_ranges(); + ranges.extend(b.ptr_ranges()); + let space = NounSpace::empty().with_extra_ptr_ranges(ranges); + noun_equality(a.root.in_space(&space), b.root.in_space(&space)) } -fn slab_mug(a: Noun) -> u32 { +fn slab_mug(a: Noun, space: &NounSpace) -> u32 { let mut stack = vec![a]; while let Some(noun) = stack.pop() { if let Ok(mut allocated) = noun.as_allocated() { - if allocated.get_cached_mug().is_none() { + if get_mug(noun, space).is_none() { match allocated.as_either() { Either::Left(indirect) => unsafe { - set_mug(&mut allocated, calc_atom_mug_u32(indirect.as_atom())); + set_mug( + &mut allocated, + calc_atom_mug_u32(indirect.as_atom(), space), + space, + ); }, - Either::Right(cell) => match (get_mug(cell.head()), get_mug(cell.tail())) { + Either::Right(cell) => match ( + get_mug(cell.in_space(space).head().noun(), space), + get_mug(cell.in_space(space).tail().noun(), space), + ) { (Some(head_mug), Some(tail_mug)) => unsafe { - set_mug(&mut allocated, calc_cell_mug_u32(head_mug, tail_mug)); + set_mug( + &mut allocated, + calc_cell_mug_u32(head_mug, tail_mug, space), + space, + ); }, _ => { stack.push(noun); - stack.push(cell.tail()); - stack.push(cell.head()); + stack.push(cell.in_space(space).tail().noun()); + stack.push(cell.in_space(space).head().noun()); } }, } } } } - get_mug(a).expect("Noun should have a mug once mugged.") + get_mug(a, space).expect("Noun should have a mug once mugged.") } enum CueStackEntry { @@ -664,14 +727,14 @@ enum CueStackEntry { // gonna use this like an ML module /// This makes us modular over different implementations of jam and cue pub trait Jammer: Sized { - fn jam(noun: Noun) -> Bytes; + fn jam(noun: Noun, space: &NounSpace) -> Bytes; fn cue(slab: &mut NounSlab, bytes: Bytes) -> Result; } pub struct NockJammer; impl Jammer for NockJammer { - fn jam(noun: Noun) -> Bytes { + fn jam(noun: Noun, space: &NounSpace) -> Bytes { fn mat_backref(buffer: &mut BitVec, backref: usize) { if backref == 0 { buffer.extend_from_bitslice(bits![u8, Lsb0; 1, 1, 1]); @@ -690,12 +753,12 @@ impl Jammer for NockJammer { .extend_from_bitslice(&BitSlice::<_, Lsb0>::from_element(&backref)[0..backref_sz]); } - fn mat_atom(buffer: &mut BitVec, atom: Atom) { + fn mat_atom(buffer: &mut BitVec, atom: Atom, space: &NounSpace) { if unsafe { atom.as_noun().raw_equals(&D(0)) } { buffer.extend_from_bitslice(bits![u8, Lsb0; 0, 1]); return; } - let atom_sz = met0_usize(atom); + let atom_sz = met0_usize(atom, space); let atom_sz_sz = met0_u64_to_usize(atom_sz as u64); buffer.push(false); // atom tag let buffer_len = buffer.len(); @@ -704,32 +767,33 @@ impl Jammer for NockJammer { buffer.extend_from_bitslice( &BitSlice::<_, Lsb0>::from_element(&atom_sz)[0..atom_sz_sz - 1], ); - buffer.extend_from_bitslice(&atom.as_bitslice()[0..atom_sz]); + buffer.extend_from_bitslice(&atom.in_space(space).as_bitslice()[0..atom_sz]); } let mut backref_map = NounMap::::new(); let mut stack = vec![noun]; let mut buffer = bitvec![u8, Lsb0; 0; 0]; while let Some(noun) = stack.pop() { - if let Some(backref) = backref_map.get(noun) { + if let Some(backref) = backref_map.get(noun, space) { if let Ok(atom) = noun.as_atom() { - if met0_u64_to_usize(*backref as u64) < met0_usize(atom) { + if met0_u64_to_usize(*backref as u64) < met0_usize(atom, space) { mat_backref(&mut buffer, *backref); } else { - mat_atom(&mut buffer, atom) + mat_atom(&mut buffer, atom, space) } } else { mat_backref(&mut buffer, *backref); } } else { - backref_map.insert(noun, buffer.len()); + backref_map.insert(noun, buffer.len(), space); match noun.as_either_atom_cell() { Either::Left(atom) => { - mat_atom(&mut buffer, atom); + mat_atom(&mut buffer, atom, space); } Either::Right(cell) => { buffer.extend_from_bitslice(bits![u8, Lsb0; 1, 0]); // cell tag - stack.push(cell.tail()); - stack.push(cell.head()); + let cell = cell.in_space(space); + stack.push(cell.tail().noun()); + stack.push(cell.head().noun()); } } } @@ -803,11 +867,35 @@ impl Jammer for NockJammer { } else { // Indirect atom let indirect_words = (sz + 63) >> 6; // fast round to 64-bit words - let (mut indirect, slice) = - unsafe { IndirectAtom::new_raw_mut_bitslice(slab, indirect_words) }; + let (indirect, data_ptr) = + unsafe { IndirectAtom::new_raw_mut_zeroed(slab, indirect_words) }; + let slice = unsafe { + BitSlice::::from_slice_mut(std::slice::from_raw_parts_mut( + data_ptr, indirect_words, + )) + }; slice[0..sz].clone_from_bitslice(&buffer[*cursor..*cursor + sz]); *cursor += sz; - Ok(unsafe { indirect.normalize_as_atom() }) + let words = + unsafe { std::slice::from_raw_parts_mut(data_ptr, indirect_words) }; + let mut used_words = words.len(); + while used_words > 1 && words[used_words - 1] == 0 { + used_words -= 1; + } + if used_words == 0 { + return Ok(unsafe { DirectAtom::new_unchecked(0).as_atom() }); + } + if used_words == 1 { + let value = words[0]; + if value <= DIRECT_MAX { + return Ok(unsafe { DirectAtom::new_unchecked(value).as_atom() }); + } + } + unsafe { + let meta_ptr = words.as_mut_ptr().sub(2); + *meta_ptr.add(1) = used_words as u64; + } + Ok(indirect.as_atom()) } } } else { @@ -878,36 +966,55 @@ impl Jammer for NockJammer { #[cfg(test)] mod tests { + use std::panic::{catch_unwind, AssertUnwindSafe}; + use bitvec::prelude::*; use ibig::ubig; - use nockvm::noun::{D, T}; + use nockvm::mem::{NockStack, NOCK_STACK_SIZE_TINY}; + use nockvm::noun::{AllocLocation, NounRepr, D, T}; + use nockvm::pma::{Pma, PmaCopy}; use nockvm_macros::tas; + use tempfile::TempDir; use super::*; use crate::AtomExt; + + fn assert_set_root_rejects(root: Noun) { + let mut slab: NounSlab = NounSlab::new(); + let res = catch_unwind(AssertUnwindSafe(|| slab.set_root(root))); + assert!( + res.is_err(), + "set_root should reject roots outside the slab" + ); + } #[test] + #[cfg_attr(miri, ignore = "memfd_create unsupported in Miri")] fn test_ubig_alloc() { let mut slab: NounSlab = NounSlab::new(); let big_exp = ubig!(0xFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF); let atom = Atom::from_ubig(&mut slab, &big_exp); - let big = atom.as_ubig(&mut slab); + let space = slab.noun_space(); + let big = atom.in_space(&space).as_ubig(&mut slab); assert_eq!(big, big_exp); } #[test] - #[cfg_attr(miri, ignore)] + #[cfg_attr(miri, ignore = "memfd_create unsupported in Miri")] fn test_jam() { let mut slab: NounSlab = NounSlab::new(); - let test_noun = T( + let slab_noun = T( &mut slab, &[D(tas!(b"request")), D(tas!(b"block")), D(tas!(b"by-id")), D(0)], ); - slab.set_root(test_noun); + slab.set_root(slab_noun); let jammed: Vec = slab.jam().to_vec(); println!("jammed: {:?}", jammed); - let mut stack = NockStack::new(1000, 0); - let mut nockvm_jammed: Vec = nockvm::serialization::jam(&mut stack, test_noun) + let mut stack = NockStack::new(NOCK_STACK_SIZE_TINY, 0); + let stack_noun = slab.copy_to_stack(&mut stack); + let space = stack.noun_space(); + let mut nockvm_jammed: Vec = nockvm::serialization::jam(&mut stack, stack_noun) + .in_space(&space) .as_ne_bytes() .to_vec(); let nockvm_suffix: Vec = nockvm_jammed.split_off(jammed.len()); @@ -921,6 +1028,7 @@ mod tests { } #[test] + #[cfg_attr(miri, ignore = "memfd_create unsupported in Miri")] fn test_jam_cue_roundtrip() { let mut original_slab: NounSlab = NounSlab::new(); let original_noun = T(&mut original_slab, &[D(5), D(23)]); @@ -939,13 +1047,20 @@ mod tests { println!("cued_noun: {:?}", cued_noun); // Compare the original and cued nouns + let mut ranges = original_slab.ptr_ranges(); + ranges.extend(cued_slab.ptr_ranges()); + let space = NounSpace::empty().with_extra_ptr_ranges(ranges); assert!( - slab_noun_equality(unsafe { original_slab.root() }, &cued_noun), + noun_equality( + unsafe { original_slab.root() }.in_space(&space), + cued_noun.in_space(&space), + ), "Original and cued nouns should be equal" ); } #[test] + #[cfg_attr(miri, ignore = "memfd_create unsupported in Miri")] fn test_complex_noun() { let mut slab: NounSlab = NounSlab::new(); let complex_noun = T( @@ -958,13 +1073,20 @@ mod tests { let mut cued_slab: NounSlab = NounSlab::new(); let cued_noun = cued_slab.cue_into(jammed).expect("Cue should succeed"); + let mut ranges = slab.ptr_ranges(); + ranges.extend(cued_slab.ptr_ranges()); + let space = NounSpace::empty().with_extra_ptr_ranges(ranges); assert!( - slab_noun_equality(unsafe { slab.root() }, &cued_noun), + noun_equality( + unsafe { slab.root() }.in_space(&space), + cued_noun.in_space(&space), + ), "Complex nouns should be equal after jam/cue roundtrip" ); } #[test] + #[cfg_attr(miri, ignore = "memfd_create unsupported in Miri")] fn test_indirect_atoms() { let mut slab: NounSlab = NounSlab::new(); let large_number = u64::MAX as u128 + 1; @@ -979,13 +1101,20 @@ mod tests { let cued_noun = cued_slab.cue_into(jammed).expect("Cue should succeed"); println!("cued_noun: {:?}", cued_noun); + let mut ranges = slab.ptr_ranges(); + ranges.extend(cued_slab.ptr_ranges()); + let space = NounSpace::empty().with_extra_ptr_ranges(ranges); assert!( - slab_noun_equality(&noun_with_indirect, &cued_noun), + noun_equality( + noun_with_indirect.in_space(&space), + cued_noun.in_space(&space), + ), "Nouns with indirect atoms should be equal after jam/cue roundtrip" ); } #[test] + #[cfg_attr(miri, ignore = "memfd_create unsupported in Miri")] fn test_tas_macro() { let mut slab: NounSlab = NounSlab::new(); let tas_noun = T( @@ -998,14 +1127,20 @@ mod tests { let mut cued_slab: NounSlab = NounSlab::new(); let cued_noun = cued_slab.cue_into(jammed).expect("Cue should succeed"); + let mut ranges = slab.ptr_ranges(); + ranges.extend(cued_slab.ptr_ranges()); + let space = NounSpace::empty().with_extra_ptr_ranges(ranges); assert!( - slab_noun_equality(unsafe { slab.root() }, &cued_noun), + noun_equality( + unsafe { slab.root() }.in_space(&space), + cued_noun.in_space(&space), + ), "Nouns with tas! macros should be equal after jam/cue roundtrip" ); } #[test] - #[cfg_attr(miri, ignore)] + #[cfg_attr(miri, ignore = "memfd_create unsupported in Miri")] fn test_cue_from_file() { use std::fs::File; use std::io::Read; @@ -1051,6 +1186,7 @@ mod tests { } #[test] + #[cfg_attr(miri, ignore = "memfd_create unsupported in Miri")] fn test_cyclic_structure() { let mut slab: NounSlab = NounSlab::new(); @@ -1078,6 +1214,7 @@ mod tests { } #[test] + #[cfg_attr(miri, ignore = "memfd_create unsupported in Miri")] fn test_cue_simple_cell() { let mut slab: NounSlab = NounSlab::new(); @@ -1091,32 +1228,255 @@ mod tests { assert!(result.is_ok(), "cue_into should succeed"); if let Ok(cued_noun) = result { let expected_noun = T(&mut slab, &[D(1), D(0)]); + let space = slab.noun_space(); assert!( - slab_noun_equality(&cued_noun, &expected_noun), + noun_equality(cued_noun.in_space(&space), expected_noun.in_space(&space),), "Cued noun should equal [1 0]" ); } } #[test] + #[cfg_attr(miri, ignore = "memfd_create unsupported in Miri")] fn test_cell_construction_for_noun_slab() { let mut slab: NounSlab = NounSlab::new(); let (cell, cell_mem_ptr) = unsafe { Cell::new_raw_mut(&mut slab) }; - unsafe { - assert!(std::ptr::eq( - cell_mem_ptr as *const CellMemory, - cell.to_raw_pointer() - )) - }; + let space = slab.noun_space(); + let cell_mem_ptr = cell_mem_ptr as *const CellMemory; + let cell_ptr = cell + .in_space(&space) + .cell() + .stack_memory_pointer() + .expect("cell not in stack"); + assert!(cell_mem_ptr == cell_ptr); } #[test] + #[cfg_attr(miri, ignore = "memfd_create unsupported in Miri")] fn test_noun_slab_copy_into() { let mut slab: NounSlab = NounSlab::new(); let test_noun = T(&mut slab, &[D(5), D(23)]); slab.set_root(test_noun); let mut copy_slab: NounSlab = NounSlab::new(); - copy_slab.copy_into(test_noun); + let space = slab.noun_space(); + copy_slab.copy_into(test_noun, &space); + } + + #[test] + #[cfg_attr(miri, ignore = "memfd_create unsupported in Miri")] + fn test_modify_with_imports3_uses_imported_nouns_and_preserves_root() { + let mut local_slab: NounSlab = NounSlab::new(); + let local_root = T(&mut local_slab, &[D(42), D(43)]); + local_slab.set_root(local_root); + + let mut foreign_slab: NounSlab = NounSlab::new(); + let foreign_a = T(&mut foreign_slab, &[D(1), D(2)]); + let foreign_b = T(&mut foreign_slab, &[D(3), D(4)]); + let foreign_c = T(&mut foreign_slab, &[D(5), D(6)]); + foreign_slab.set_root(foreign_c); + let foreign_space = foreign_slab.noun_space(); + + local_slab.modify_with_imports3( + |(import_a, import_b, import_c), root| vec![root, import_a, import_b, import_c, D(0)], + (foreign_a, foreign_b, foreign_c), + &foreign_space, + ); + + let local_space = local_slab.noun_space(); + let elems: Vec = unsafe { *local_slab.root() } + .in_space(&local_space) + .list_iter() + .map(|handle| handle.noun()) + .collect(); + + assert_eq!( + elems.len(), + 4, + "modified slab should contain root plus 3 imports" + ); + assert!( + unsafe { elems[0].raw_equals(&local_root) }, + "closure should receive the original slab root" + ); + let imported_a_slab: NounSlab = NounSlab::from_noun(elems[1], &local_space); + let imported_b_slab: NounSlab = NounSlab::from_noun(elems[2], &local_space); + let imported_c_slab: NounSlab = NounSlab::from_noun(elems[3], &local_space); + let expected_a_slab: NounSlab = NounSlab::from_noun(foreign_a, &foreign_space); + let expected_b_slab: NounSlab = NounSlab::from_noun(foreign_b, &foreign_space); + let expected_c_slab: NounSlab = NounSlab::from_noun(foreign_c, &foreign_space); + assert!( + slab_equality(&imported_a_slab, &expected_a_slab), + "first imported noun should match structurally" + ); + assert!( + slab_equality(&imported_b_slab, &expected_b_slab), + "second imported noun should match structurally" + ); + assert!( + slab_equality(&imported_c_slab, &expected_c_slab), + "third imported noun should match structurally" + ); + assert!( + !unsafe { elems[1].raw_equals(&foreign_a) }, + "first imported noun should be copied into the destination slab" + ); + assert!( + !unsafe { elems[2].raw_equals(&foreign_b) }, + "second imported noun should be copied into the destination slab" + ); + assert!( + !unsafe { elems[3].raw_equals(&foreign_c) }, + "third imported noun should be copied into the destination slab" + ); + } + + #[test] + #[cfg_attr(miri, ignore = "memfd_create unsupported in Miri")] + fn test_modify_rehomes_foreign_children_into_destination_slab() { + let mut local_slab: NounSlab = NounSlab::new(); + let local_root = T(&mut local_slab, &[D(42), D(43)]); + local_slab.set_root(local_root); + + let mut foreign_slab: NounSlab = NounSlab::new(); + let foreign_root = T(&mut foreign_slab, &[D(1), D(2)]); + foreign_slab.set_root(foreign_root); + let foreign_space = foreign_slab.noun_space(); + + local_slab.modify(|root| vec![root, foreign_root, D(0)]); + + let local_space = local_slab.noun_space(); + let elems: Vec = unsafe { *local_slab.root() } + .in_space(&local_space) + .list_iter() + .map(|handle| handle.noun()) + .collect(); + + assert_eq!( + elems.len(), + 2, + "modified slab should contain 2 list elements" + ); + assert!( + unsafe { elems[0].raw_equals(&local_root) }, + "closure should receive the original slab root" + ); + let imported_slab: NounSlab = NounSlab::from_noun(elems[1], &local_space); + let expected_slab: NounSlab = NounSlab::from_noun(foreign_root, &foreign_space); + assert!( + slab_equality(&imported_slab, &expected_slab), + "foreign child should be copied structurally into the destination slab" + ); + assert!( + !unsafe { elems[1].raw_equals(&foreign_root) }, + "foreign child should not retain the foreign slab pointer" + ); + } + + #[test] + #[cfg_attr(miri, ignore = "memfd_create unsupported in Miri")] + fn test_modify_noun_rehomes_mixed_local_root() { + let mut local_slab: NounSlab = NounSlab::new(); + let local_root = T(&mut local_slab, &[D(42), D(43)]); + local_slab.set_root(local_root); + + let mut foreign_slab: NounSlab = NounSlab::new(); + let foreign_root = T(&mut foreign_slab, &[D(1), D(2)]); + foreign_slab.set_root(foreign_root); + let foreign_space = foreign_slab.noun_space(); + + let mixed_root = T(&mut local_slab, &[local_root, foreign_root, D(0)]); + local_slab.modify_noun(|_| mixed_root); + + let local_space = local_slab.noun_space(); + let elems: Vec = unsafe { *local_slab.root() } + .in_space(&local_space) + .list_iter() + .map(|handle| handle.noun()) + .collect(); + + assert_eq!( + elems.len(), + 2, + "modified slab should contain 2 list elements" + ); + assert!( + unsafe { elems[0].raw_equals(&local_root) }, + "local root should be preserved" + ); + let imported_slab: NounSlab = NounSlab::from_noun(elems[1], &local_space); + let expected_slab: NounSlab = NounSlab::from_noun(foreign_root, &foreign_space); + assert!( + slab_equality(&imported_slab, &expected_slab), + "modify_noun should re-home foreign descendants before setting the root" + ); + assert!( + !unsafe { elems[1].raw_equals(&foreign_root) }, + "foreign child should not retain the foreign slab pointer" + ); + } + + #[test] + #[cfg_attr(miri, ignore = "memfd_create unsupported in Miri")] + fn test_from_array_rehomes_foreign_children_into_destination_slab() { + let mut foreign_slab: NounSlab = NounSlab::new(); + let foreign_root = T(&mut foreign_slab, &[D(1), D(2)]); + foreign_slab.set_root(foreign_root); + let foreign_space = foreign_slab.noun_space(); + + let slab: NounSlab = [D(7), foreign_root, D(0)].into(); + let local_space = slab.noun_space(); + let elems: Vec = unsafe { *slab.root() } + .in_space(&local_space) + .list_iter() + .map(|handle| handle.noun()) + .collect(); + + assert_eq!(elems.len(), 2, "slab root should be a 2-element list"); + assert!( + unsafe { elems[0].raw_equals(&D(7)) }, + "first list element should remain direct" + ); + let imported_slab: NounSlab = NounSlab::from_noun(elems[1], &local_space); + let expected_slab: NounSlab = NounSlab::from_noun(foreign_root, &foreign_space); + assert!( + slab_equality(&imported_slab, &expected_slab), + "foreign child should be copied structurally into the destination slab" + ); + assert!( + !unsafe { elems[1].raw_equals(&foreign_root) }, + "foreign child should not retain the foreign slab pointer" + ); + } + + #[test] + #[cfg_attr(miri, ignore = "memfd_create unsupported in Miri")] + fn test_set_root_rejects_roots_outside_slab() { + let mut local_slab: NounSlab = NounSlab::new(); + let local_root = T(&mut local_slab, &[D(1), D(2)]); + local_slab.set_root(local_root); + assert!(unsafe { local_slab.root().raw_equals(&local_root) }); + + let mut foreign_slab: NounSlab = NounSlab::new(); + let foreign_root = T(&mut foreign_slab, &[D(3), D(4)]); + assert_set_root_rejects(foreign_root); + + let mut stack = NockStack::new(NOCK_STACK_SIZE_TINY, 0); + let stack_root = Cell::new(&mut stack, D(5), D(6)).as_noun(); + assert_set_root_rejects(stack_root); + + let tempdir = TempDir::new().expect("create temp dir for PMA"); + let pma_path = tempdir.path().join("set-root-test.pma"); + let mut pma = Pma::new(1 << 10, pma_path).expect("create test PMA"); + let mut pma_root = Cell::new(&mut stack, D(7), D(8)).as_noun(); + unsafe { + pma_root.copy_to_pma(&stack, &mut pma); + } + let pma_space = NounSpace::pma_only(&pma); + assert!(matches!( + pma_root.in_space(&pma_space).repr(), + NounRepr::Cell(AllocLocation::PmaOffset) + )); + assert_set_root_rejects(pma_root); } // Fails in Miri @@ -1152,14 +1512,16 @@ mod tests { } #[test] + #[cfg_attr(miri, ignore = "memfd_create unsupported in Miri")] fn test_nounslab_modify() { let mut slab: NounSlab = NounSlab::new(); slab.modify(|root| vec![D(0), D(tas!(b"bind")), root]); let mut test_slab: NounSlab = NounSlab::new(); - slab_noun_equality( - &slab.root, - &T(&mut test_slab, &[D(0), D(tas!(b"bind")), D(0)]), - ); + let test_noun = T(&mut test_slab, &[D(0), D(tas!(b"bind")), D(0)]); + let mut ranges = slab.ptr_ranges(); + ranges.extend(test_slab.ptr_ranges()); + let space = NounSpace::empty().with_extra_ptr_ranges(ranges); + noun_equality(slab.root.in_space(&space), test_noun.in_space(&space)); // let peek_res = unsafe { bind_slab.root_owned() }; // let bind_noun = T(&mut bind_slab, &[D(pid), D(tas!(b"bind")), peek_res]); } diff --git a/crates/nockapp/src/observability.rs b/crates/nockapp/src/observability.rs index 05ff9382a..56dc29c3f 100644 --- a/crates/nockapp/src/observability.rs +++ b/crates/nockapp/src/observability.rs @@ -69,8 +69,15 @@ pub fn init_tracing() -> Result tracing_subscriber::EnvFilter::new(format!("{},{}", base_directives, user)), + Err(_) => tracing_subscriber::EnvFilter::new(base_directives), + }; let subscriber = tracing_subscriber::Registry::default() - .with(tracing_subscriber::EnvFilter::from_default_env()) + .with(env_filter) .with(fmt_layer) .with(telemetry); Ok(subscriber) diff --git a/crates/nockapp/src/snapshot.rs b/crates/nockapp/src/snapshot.rs new file mode 100644 index 000000000..2bcba9f27 --- /dev/null +++ b/crates/nockapp/src/snapshot.rs @@ -0,0 +1,1282 @@ +#![allow(dead_code)] + +use std::collections::{HashMap, HashSet}; +use std::fs::{self, File}; +use std::io::{self, Read}; +use std::path::{Component, Path, PathBuf}; +use std::time::{Duration, Instant, SystemTime, UNIX_EPOCH}; + +use bincode::{config, Decode, Encode}; +use blake3::{Hash, Hasher, OUT_LEN}; +use nockvm::offset::PmaOffsetWords; +use nockvm::pma::{ + classify_pma_noun, Pma, PmaDirectJamConfig, PmaDirectJamError, PmaDirectReader, + PmaFileMetadata, PmaRawNounKind, +}; +use nockvm::serialization::met0_u64_to_usize; +use thiserror::Error; +use tracing::debug; + +use crate::event_log::{EventLog, EventLogError, ReadySnapshotRecord}; +use crate::utils::durability; + +const SNAPSHOT_MANIFEST_MAGIC: u64 = u64::from_le_bytes(*b"SNAPMAN1"); +const SNAPSHOT_MANIFEST_VERSION: u32 = 2; +type HashBytes = [u8; OUT_LEN]; + +fn duration_ms(elapsed: Duration) -> f64 { + elapsed.as_secs_f64() * 1000.0 +} + +#[derive(Clone, Copy, Debug, Encode, Decode, PartialEq, Eq)] +pub(crate) enum SnapshotKind { + Epoch, + Rotating, +} + +#[derive(Clone, Debug, Encode, Decode, PartialEq, Eq)] +struct SnapshotManifestPayload { + magic: u64, + version: u32, + kind: SnapshotKind, + timestamp_tag: String, + ker_hash: HashBytes, + event_num: u64, + pma_words: u64, + alloc_words: u64, + kernel_root_raw: u64, + cold_offset: PmaOffsetWords, + used_blake3: HashBytes, + structure_blake3: Option, + created_at_ms: i64, +} + +#[derive(Clone, Debug, Encode, Decode, PartialEq, Eq)] +pub(crate) struct SnapshotManifest { + pub magic: u64, + pub version: u32, + pub kind: SnapshotKind, + pub timestamp_tag: String, + pub ker_hash: HashBytes, + pub event_num: u64, + pub pma_words: u64, + pub alloc_words: u64, + pub kernel_root_raw: u64, + pub cold_offset: PmaOffsetWords, + pub used_blake3: HashBytes, + pub structure_blake3: Option, + pub created_at_ms: i64, + pub checksum: HashBytes, +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(crate) enum SnapshotVerifyMode { + Fast, + Full, +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(crate) struct JamStreamStats { + pub bit_len: usize, + pub byte_len: usize, +} + +#[derive(Debug)] +pub(crate) struct SnapshotVerification { + pub manifest: SnapshotManifest, + pub file_metadata: PmaFileMetadata, + pub used_blake3: HashBytes, + pub structure_stats: Option, +} + +#[derive(Debug, Error)] +pub(crate) enum SnapshotManifestError { + #[error("snapshot manifest io error: {0}")] + Io(#[from] io::Error), + #[error("snapshot manifest encode error: {0}")] + Encode(#[from] bincode::error::EncodeError), + #[error("snapshot manifest decode error: {0}")] + Decode(#[from] bincode::error::DecodeError), + #[error("snapshot manifest magic mismatch: expected {expected:#x}, found {found:#x}")] + BadMagic { expected: u64, found: u64 }, + #[error("snapshot manifest version mismatch: expected {expected}, found {found}")] + BadVersion { expected: u32, found: u32 }, + #[error("snapshot manifest checksum mismatch")] + ChecksumMismatch, +} + +#[derive(Debug, Error)] +pub(crate) enum SnapshotVerifyError { + #[error(transparent)] + Manifest(#[from] SnapshotManifestError), + #[error("snapshot PMA metadata error: {0}")] + Pma(#[from] nockvm::pma::PmaError), + #[error("snapshot PMA direct-reader error: {0}")] + Direct(#[from] PmaDirectJamError), + #[error("snapshot io error: {0}")] + Io(#[from] io::Error), + #[error("snapshot PMA words mismatch: manifest={manifest}, file={file}")] + PmaWordsMismatch { manifest: u64, file: u64 }, + #[error("snapshot alloc words mismatch: manifest={manifest}, file={file}")] + AllocWordsMismatch { manifest: u64, file: u64 }, + #[error("snapshot used-range hash mismatch")] + UsedHashMismatch { + expected: HashBytes, + actual: HashBytes, + }, + #[error("snapshot structure hash mismatch")] + StructureHashMismatch { + expected: HashBytes, + actual: HashBytes, + }, + #[error("snapshot cold_offset {cold_offset} is out of bounds for alloc_words {alloc_words}")] + ColdOffsetOutOfBounds { + cold_offset: PmaOffsetWords, + alloc_words: u64, + }, +} + +#[derive(Debug, Error)] +pub(crate) enum SnapshotBuildError { + #[error(transparent)] + Verify(#[from] SnapshotVerifyError), + #[error(transparent)] + Manifest(#[from] SnapshotManifestError), + #[error(transparent)] + EventLog(#[from] EventLogError), + #[error("snapshot build io error: {0}")] + Io(#[from] io::Error), +} + +#[derive(Debug, Error)] +pub(crate) enum SnapshotRestoreError { + #[error(transparent)] + Verify(#[from] SnapshotVerifyError), + #[error("snapshot restore io error: {0}")] + Io(#[from] io::Error), +} + +#[derive(Debug, Error)] +pub(crate) enum SnapshotCleanupError { + #[error(transparent)] + EventLog(#[from] EventLogError), + #[error("snapshot cleanup io error: {0}")] + Io(#[from] io::Error), +} + +#[derive(Debug)] +pub(crate) enum RotatingSnapshotBuildStatus { + NotCreated, + Created, + CreatedWithCleanupError(SnapshotBuildError), +} + +impl RotatingSnapshotBuildStatus { + pub(crate) fn created(&self) -> bool { + matches!( + self, + RotatingSnapshotBuildStatus::Created + | RotatingSnapshotBuildStatus::CreatedWithCleanupError(_) + ) + } + + pub(crate) fn cleanup_error(&self) -> Option<&SnapshotBuildError> { + match self { + RotatingSnapshotBuildStatus::CreatedWithCleanupError(err) => Some(err), + _ => None, + } + } +} + +impl SnapshotManifest { + #[allow(clippy::too_many_arguments)] + pub(crate) fn new( + kind: SnapshotKind, + timestamp_tag: String, + ker_hash: Hash, + event_num: u64, + pma_words: u64, + alloc_words: u64, + kernel_root_raw: u64, + cold_offset: PmaOffsetWords, + used_blake3: Hash, + structure_blake3: Option, + created_at_ms: i64, + ) -> Result { + let mut manifest = Self { + magic: SNAPSHOT_MANIFEST_MAGIC, + version: SNAPSHOT_MANIFEST_VERSION, + kind, + timestamp_tag, + ker_hash: to_hash_bytes(ker_hash), + event_num, + pma_words, + alloc_words, + kernel_root_raw, + cold_offset, + used_blake3: to_hash_bytes(used_blake3), + structure_blake3: structure_blake3.map(to_hash_bytes), + created_at_ms, + checksum: [0; OUT_LEN], + }; + manifest.checksum = manifest.compute_checksum()?; + Ok(manifest) + } + + pub(crate) fn validate(&self) -> Result<(), SnapshotManifestError> { + if self.magic != SNAPSHOT_MANIFEST_MAGIC { + return Err(SnapshotManifestError::BadMagic { + expected: SNAPSHOT_MANIFEST_MAGIC, + found: self.magic, + }); + } + if self.version != SNAPSHOT_MANIFEST_VERSION { + return Err(SnapshotManifestError::BadVersion { + expected: SNAPSHOT_MANIFEST_VERSION, + found: self.version, + }); + } + if self.compute_checksum()? != self.checksum { + return Err(SnapshotManifestError::ChecksumMismatch); + } + Ok(()) + } + + pub(crate) fn encode(&self) -> Result, SnapshotManifestError> { + self.validate()?; + Ok(bincode::encode_to_vec(self, config::standard())?) + } + + pub(crate) fn decode(bytes: &[u8]) -> Result { + let (manifest, _) = bincode::decode_from_slice::(bytes, config::standard())?; + manifest.validate()?; + Ok(manifest) + } + + pub(crate) fn read_from_path(path: &Path) -> Result { + let bytes = fs::read(path)?; + Self::decode(&bytes) + } + + pub(crate) fn write_to_path(&self, path: &Path) -> Result<(), SnapshotManifestError> { + let bytes = self.encode()?; + durability::write_atomic(path, &bytes, "snapshot_manifest_write")?; + Ok(()) + } + + fn payload(&self) -> SnapshotManifestPayload { + SnapshotManifestPayload { + magic: self.magic, + version: self.version, + kind: self.kind, + timestamp_tag: self.timestamp_tag.clone(), + ker_hash: self.ker_hash, + event_num: self.event_num, + pma_words: self.pma_words, + alloc_words: self.alloc_words, + kernel_root_raw: self.kernel_root_raw, + cold_offset: self.cold_offset, + used_blake3: self.used_blake3, + structure_blake3: self.structure_blake3, + created_at_ms: self.created_at_ms, + } + } + + fn compute_checksum(&self) -> Result { + let payload = self.payload(); + compute_checksum(&payload) + } +} + +pub(crate) fn verify_snapshot( + manifest_path: &Path, + pma_path: &Path, + mode: SnapshotVerifyMode, +) -> Result { + let verify_start = Instant::now(); + debug!( + mode = ?mode, + manifest_path = ?manifest_path, + pma_path = ?pma_path, + "snapshot verify start" + ); + let stage_start = Instant::now(); + let manifest = SnapshotManifest::read_from_path(manifest_path)?; + debug!( + mode = ?mode, + event_num = manifest.event_num, + elapsed_ms = duration_ms(stage_start.elapsed()), + "snapshot verify stage done: read_manifest" + ); + let stage_start = Instant::now(); + let file_metadata = Pma::read_file_metadata(pma_path)?; + debug!( + mode = ?mode, + event_num = manifest.event_num, + elapsed_ms = duration_ms(stage_start.elapsed()), + "snapshot verify stage done: read_pma_metadata" + ); + let stage_start = Instant::now(); + if manifest.pma_words != file_metadata.data_words { + return Err(SnapshotVerifyError::PmaWordsMismatch { + manifest: manifest.pma_words, + file: file_metadata.data_words, + }); + } + if manifest.alloc_words != file_metadata.alloc_words { + return Err(SnapshotVerifyError::AllocWordsMismatch { + manifest: manifest.alloc_words, + file: file_metadata.alloc_words, + }); + } + debug!( + mode = ?mode, + event_num = manifest.event_num, + elapsed_ms = duration_ms(stage_start.elapsed()), + "snapshot verify stage done: validate_file_metadata" + ); + + let stage_start = Instant::now(); + let used_blake3 = hash_file_prefix(pma_path, file_metadata.alloc_words.saturating_mul(8))?; + debug!( + mode = ?mode, + event_num = manifest.event_num, + elapsed_ms = duration_ms(stage_start.elapsed()), + "snapshot verify stage done: hash_used_prefix" + ); + if used_blake3 != manifest.used_blake3 { + return Err(SnapshotVerifyError::UsedHashMismatch { + expected: manifest.used_blake3, + actual: used_blake3, + }); + } + + let stage_start = Instant::now(); + let mut reader = PmaDirectReader::from_path( + pma_path, + file_metadata.data_words, + file_metadata.alloc_words, + PmaDirectJamConfig { + require_direct_io: false, + ..PmaDirectJamConfig::default() + }, + )?; + debug!( + mode = ?mode, + event_num = manifest.event_num, + elapsed_ms = duration_ms(stage_start.elapsed()), + "snapshot verify stage done: open_direct_reader" + ); + let stage_start = Instant::now(); + validate_root_raw(&mut reader, manifest.kernel_root_raw)?; + debug!( + mode = ?mode, + event_num = manifest.event_num, + elapsed_ms = duration_ms(stage_start.elapsed()), + "snapshot verify stage done: validate_root_raw" + ); + let stage_start = Instant::now(); + validate_cold_offset(&mut reader, manifest.cold_offset)?; + debug!( + mode = ?mode, + event_num = manifest.event_num, + elapsed_ms = duration_ms(stage_start.elapsed()), + "snapshot verify stage done: validate_cold_offset" + ); + + let structure_stats = match mode { + SnapshotVerifyMode::Fast => None, + SnapshotVerifyMode::Full => { + let stage_start = Instant::now(); + let stats = verify_structure(&mut reader, &manifest)?; + debug!( + mode = ?mode, + event_num = manifest.event_num, + bit_len = stats.bit_len, + byte_len = stats.byte_len, + elapsed_ms = duration_ms(stage_start.elapsed()), + "snapshot verify stage done: verify_structure" + ); + Some(stats) + } + }; + + let verification = SnapshotVerification { + manifest, + file_metadata, + used_blake3, + structure_stats, + }; + debug!( + mode = ?mode, + event_num = verification.manifest.event_num, + elapsed_ms = duration_ms(verify_start.elapsed()), + "snapshot verify done" + ); + Ok(verification) +} + +pub(crate) fn maybe_create_epoch_snapshot( + event_log: &mut EventLog, + pma: &Pma, + ker_hash: Hash, + event_num: u64, + kernel_root_raw: u64, + cold_offset: PmaOffsetWords, +) -> Result { + if event_log.has_ready_snapshot()? { + return Ok(false); + } + create_ready_snapshot( + event_log, + pma, + SnapshotKind::Epoch, + "epoch".to_string(), + "epoch".to_string(), + ker_hash, + event_num, + kernel_root_raw, + cold_offset, + None, + )?; + Ok(true) +} + +#[allow(clippy::too_many_arguments)] +pub(crate) fn maybe_create_rotating_snapshot( + event_log: &mut EventLog, + pma: &Pma, + ker_hash: Hash, + event_num: u64, + kernel_root_raw: u64, + cold_offset: PmaOffsetWords, + cumulative_processing_time: Duration, + rotating_snapshot_interval_time: Option, +) -> Result { + let Some(interval) = rotating_snapshot_interval_time else { + return Ok(RotatingSnapshotBuildStatus::NotCreated); + }; + if cumulative_processing_time < interval { + return Ok(RotatingSnapshotBuildStatus::NotCreated); + } + let created_at_ms = current_time_ms()?; + let timestamp_tag = format!("{created_at_ms:020}-{event_num:020}"); + let file_stem = format!("snap-{timestamp_tag}"); + let base_snapshot_id = event_log.active_snapshot_id()?; + create_ready_snapshot( + event_log, + pma, + SnapshotKind::Rotating, + timestamp_tag, + file_stem, + ker_hash, + event_num, + kernel_root_raw, + cold_offset, + base_snapshot_id, + )?; + if let Err(err) = retire_old_rotating_snapshots(event_log).map_err(snapshot_cleanup_into_build) + { + return Ok(RotatingSnapshotBuildStatus::CreatedWithCleanupError(err)); + } + Ok(RotatingSnapshotBuildStatus::Created) +} + +pub(crate) fn restore_verified_snapshot( + record: &ReadySnapshotRecord, + operative_pma_path: &Path, +) -> Result { + let verification = verify_snapshot( + Path::new(&record.manifest_path), + Path::new(&record.pma_path), + SnapshotVerifyMode::Full, + )?; + let tmp_path = operative_pma_path.with_extension("restore.tmp"); + copy_snapshot_file(Path::new(&record.pma_path), &tmp_path)?; + replace_file(&tmp_path, operative_pma_path)?; + Ok(verification.manifest) +} + +pub(crate) fn cleanup_snapshot_artifacts( + event_log: &mut EventLog, + pma_dir: &Path, +) -> Result<(), SnapshotCleanupError> { + retire_old_rotating_snapshots(event_log)?; + if !pma_dir.exists() { + return Ok(()); + } + + let tracked_paths: HashSet<_> = event_log + .list_ready_snapshots()? + .into_iter() + .flat_map(|snapshot| { + [ + tracked_snapshot_path(pma_dir, &snapshot.pma_path), + tracked_snapshot_path(pma_dir, &snapshot.manifest_path), + ] + }) + .collect(); + + let corrupted_dir = pma_dir.join("corrupted_pma"); + for entry in fs::read_dir(pma_dir)? { + let entry = entry?; + let path = entry.path(); + if !path.is_file() { + continue; + } + let Some(name) = path.file_name().and_then(|name| name.to_str()) else { + continue; + }; + if !is_snapshot_artifact(name) || tracked_paths.contains(&normalize_snapshot_path(&path)) { + continue; + } + fs::create_dir_all(&corrupted_dir)?; + move_to_corrupted_dir(&path, &corrupted_dir)?; + } + Ok(()) +} + +#[allow(clippy::too_many_arguments)] +fn create_ready_snapshot( + event_log: &mut EventLog, + pma: &Pma, + kind: SnapshotKind, + timestamp_tag: String, + file_stem: String, + ker_hash: Hash, + event_num: u64, + kernel_root_raw: u64, + cold_offset: PmaOffsetWords, + base_snapshot_id: Option, +) -> Result { + let pma_dir = pma.path().parent().ok_or_else(|| { + io::Error::new( + io::ErrorKind::InvalidInput, + "PMA path missing parent directory for snapshot", + ) + })?; + fs::create_dir_all(pma_dir)?; + let snapshot_pma_path = pma_dir.join(format!("{file_stem}.pma")); + let snapshot_manifest_path = pma_dir.join(format!("{file_stem}.manifest")); + let tmp_snapshot_pma_path = pma_dir.join(format!("{file_stem}.pma.tmp")); + let created_at_ms = current_time_ms()?; + + pma.sync_used_data()?; + pma.sync_trailer()?; + durability::sync_path_data(pma.path(), "snapshot_source_pma_fdatasync")?; + + let copy_start = Instant::now(); + copy_snapshot_file(pma.path(), &tmp_snapshot_pma_path)?; + debug!( + event_num, + src = ?pma.path(), + dst = ?tmp_snapshot_pma_path, + elapsed_ms = duration_ms(copy_start.elapsed()), + "snapshot file copy done" + ); + replace_file(&tmp_snapshot_pma_path, &snapshot_pma_path)?; + let hash_start = Instant::now(); + let used_blake3 = Hash::from_bytes(hash_file_prefix( + &snapshot_pma_path, + pma.alloc_offset_words() + .checked_bytes() + .expect("PMA alloc offset exceeds u64 byte range"), + )?); + debug!( + event_num, + path = ?snapshot_pma_path, + len_bytes = pma.alloc_offset_words() + .checked_bytes() + .expect("PMA alloc offset exceeds u64 byte range"), + elapsed_ms = duration_ms(hash_start.elapsed()), + "snapshot used-range hash done" + ); + let manifest = SnapshotManifest::new( + kind, + timestamp_tag.clone(), + ker_hash, + event_num, + u64::try_from(pma.size_words()).expect("PMA size exceeds u64 addressable range"), + pma.alloc_offset_words().into(), + kernel_root_raw, + cold_offset, + used_blake3, + None, + created_at_ms, + )?; + manifest.write_to_path(&snapshot_manifest_path)?; + + let verify_start = Instant::now(); + if let Err(err) = verify_snapshot( + &snapshot_manifest_path, + &snapshot_pma_path, + SnapshotVerifyMode::Fast, + ) { + let _ = fs::remove_file(&snapshot_manifest_path); + let _ = fs::remove_file(&snapshot_pma_path); + return Err(err.into()); + } + debug!( + event_num, + manifest_path = ?snapshot_manifest_path, + pma_path = ?snapshot_pma_path, + elapsed_ms = duration_ms(verify_start.elapsed()), + "snapshot verify build stage done" + ); + + event_log.insert_ready_snapshot(&ReadySnapshotRecord { + snapshot_id: 0, + kind: snapshot_kind_name(kind).to_string(), + event_num, + pma_path: snapshot_pma_path.to_string_lossy().into_owned(), + manifest_path: snapshot_manifest_path.to_string_lossy().into_owned(), + alloc_words: pma.alloc_offset_words().into(), + kernel_root_raw, + cold_offset, + used_blake3: manifest.used_blake3.to_vec(), + structure_blake3: manifest.structure_blake3.map(|hash| hash.to_vec()), + created_at_ms, + activated_at_ms: Some(created_at_ms), + base_snapshot_id, + timestamp_tag, + })?; + Ok(manifest) +} + +fn retire_old_rotating_snapshots(event_log: &mut EventLog) -> Result<(), SnapshotCleanupError> { + let retire_start = Instant::now(); + let rotating = event_log.ready_rotating_snapshots()?; + let mut retired_count = 0usize; + for snapshot in rotating.into_iter().skip(2) { + let pma_path = Path::new(&snapshot.pma_path); + if pma_path.exists() { + fs::remove_file(pma_path)?; + } + let manifest_path = Path::new(&snapshot.manifest_path); + if manifest_path.exists() { + fs::remove_file(manifest_path)?; + } + event_log.retire_snapshot(snapshot.snapshot_id)?; + retired_count += 1; + } + debug!( + retired_count, + elapsed_ms = duration_ms(retire_start.elapsed()), + "retire old rotating snapshots done" + ); + Ok(()) +} + +fn snapshot_cleanup_into_build(err: SnapshotCleanupError) -> SnapshotBuildError { + match err { + SnapshotCleanupError::EventLog(err) => SnapshotBuildError::EventLog(err), + SnapshotCleanupError::Io(err) => SnapshotBuildError::Io(err), + } +} + +fn tracked_snapshot_path(pma_dir: &Path, raw: &str) -> PathBuf { + let pma_dir = normalize_snapshot_path(pma_dir); + let path = Path::new(raw); + if path.is_absolute() { + return normalize_snapshot_path(path); + } + + let path = normalize_snapshot_path(path); + let Some(parent) = path + .parent() + .filter(|parent| !parent.as_os_str().is_empty()) + else { + return pma_dir.join(path); + }; + + // Older event-log rows may store paths like `./.data.nockchain/pma/epoch.pma`. + // If `pma_dir` already names that directory, keep the tracked file in `pma_dir` + // instead of treating the whole relative path as a child of `pma_dir`. + if path_has_suffix(&pma_dir, parent) { + if let Some(file_name) = path.file_name() { + return pma_dir.join(file_name); + } + } + + pma_dir.join(path) +} + +fn normalize_snapshot_path(path: &Path) -> PathBuf { + let mut normalized = PathBuf::new(); + for component in path.components() { + match component { + Component::CurDir => {} + Component::ParentDir => { + if !normalized.pop() { + normalized.push(component.as_os_str()); + } + } + Component::Prefix(_) | Component::RootDir | Component::Normal(_) => { + normalized.push(component.as_os_str()); + } + } + } + normalized +} + +fn path_has_suffix(path: &Path, suffix: &Path) -> bool { + let path_components: Vec<_> = path.components().collect(); + let suffix_components: Vec<_> = suffix.components().collect(); + path_components.ends_with(&suffix_components) +} + +fn is_snapshot_artifact(name: &str) -> bool { + matches!( + name, + "epoch.pma" | "epoch.manifest" | "epoch.tmp" | "epoch.pma.tmp" | "epoch.manifest.tmp" + ) || name.starts_with("snap-") +} + +fn move_to_corrupted_dir(path: &Path, corrupted_dir: &Path) -> Result<(), io::Error> { + let file_name = path + .file_name() + .and_then(|name| name.to_str()) + .ok_or_else(|| { + io::Error::new( + io::ErrorKind::InvalidInput, + "snapshot artifact missing filename", + ) + })?; + let mut target = corrupted_dir.join(file_name); + let mut suffix = 1usize; + while target.exists() { + target = corrupted_dir.join(format!("{file_name}.{suffix}")); + suffix = suffix.saturating_add(1); + } + fs::rename(path, &target)?; + sync_parent_dir(&target)?; + Ok(()) +} + +fn snapshot_kind_name(kind: SnapshotKind) -> &'static str { + match kind { + SnapshotKind::Epoch => "epoch", + SnapshotKind::Rotating => "rotating", + } +} + +fn validate_root_raw( + reader: &mut PmaDirectReader, + kernel_root_raw: u64, +) -> Result<(), SnapshotVerifyError> { + match classify_pma_noun(kernel_root_raw)? { + PmaRawNounKind::Direct(_) => Ok(()), + PmaRawNounKind::Indirect { offset } => { + let _ = reader.indirect_atom_words(offset.words())?; + Ok(()) + } + PmaRawNounKind::Cell { offset } => { + let _ = reader.read_cell(offset.words())?; + Ok(()) + } + } +} + +fn validate_cold_offset( + reader: &mut PmaDirectReader, + cold_offset: PmaOffsetWords, +) -> Result<(), SnapshotVerifyError> { + let alloc_words = reader.alloc_words(); + if cold_offset.words() >= alloc_words { + return Err(SnapshotVerifyError::ColdOffsetOutOfBounds { + cold_offset, + alloc_words, + }); + } + let _ = reader.read_u64(cold_offset.words())?; + Ok(()) +} + +fn verify_structure( + reader: &mut PmaDirectReader, + manifest: &SnapshotManifest, +) -> Result { + let mut writer = StructureBitWriter::new(manifest.structure_blake3.is_some()); + let mut backrefs: HashMap = HashMap::new(); + let mut stack = vec![manifest.kernel_root_raw]; + + while let Some(noun_raw) = stack.pop() { + let kind = classify_pma_noun(noun_raw)?; + if let Some(backref) = backrefs.get(&noun_raw).copied() { + match kind { + PmaRawNounKind::Direct(value) => { + let atom_bits = met0_u64_to_usize(value); + if met0_u64_to_usize(backref as u64) < atom_bits { + mat_backref(&mut writer, backref)?; + } else { + mat_direct_atom(&mut writer, value)?; + } + } + PmaRawNounKind::Indirect { offset } => { + let atom_bits = reader.indirect_atom_bits(offset.words())?; + if met0_u64_to_usize(backref as u64) < atom_bits { + mat_backref(&mut writer, backref)?; + } else { + mat_indirect_atom(reader, &mut writer, offset.words(), atom_bits)?; + } + } + PmaRawNounKind::Cell { .. } => { + mat_backref(&mut writer, backref)?; + } + } + continue; + } + + backrefs.insert(noun_raw, writer.bit_len()); + match kind { + PmaRawNounKind::Direct(value) => { + mat_direct_atom(&mut writer, value)?; + } + PmaRawNounKind::Indirect { offset } => { + let atom_bits = reader.indirect_atom_bits(offset.words())?; + mat_indirect_atom(reader, &mut writer, offset.words(), atom_bits)?; + } + PmaRawNounKind::Cell { offset } => { + let (head, tail) = reader.read_cell(offset.words())?; + mat_cell(&mut writer)?; + stack.push(tail); + stack.push(head); + } + } + } + + let (stats, actual_hash) = writer.finish(); + if let Some(expected) = manifest.structure_blake3 { + let actual = actual_hash.expect("structure hash requested"); + if actual != expected { + return Err(SnapshotVerifyError::StructureHashMismatch { expected, actual }); + } + } + Ok(stats) +} + +fn compute_checksum(value: &T) -> Result { + let encoded = bincode::encode_to_vec(value, config::standard())?; + let mut hasher = Hasher::new(); + hasher.update(&encoded); + Ok(*hasher.finalize().as_bytes()) +} + +fn to_hash_bytes(hash: Hash) -> HashBytes { + *hash.as_bytes() +} + +fn hash_file_prefix(path: &Path, len_bytes: u64) -> Result { + let mut file = File::open(path)?; + let mut hasher = Hasher::new(); + let mut remaining = len_bytes; + let mut buf = [0u8; 8192]; + while remaining > 0 { + let read_len = remaining.min(buf.len() as u64) as usize; + file.read_exact(&mut buf[..read_len])?; + hasher.update(&buf[..read_len]); + remaining -= read_len as u64; + } + Ok(*hasher.finalize().as_bytes()) +} + +fn sync_parent_dir(path: &Path) -> Result<(), io::Error> { + durability::sync_parent_dir(path, "snapshot_parent_dir_fsync") +} + +fn copy_snapshot_file(src: &Path, dst: &Path) -> Result<(), io::Error> { + if let Some(parent) = dst.parent() { + fs::create_dir_all(parent)?; + } + fs::copy(src, dst)?; + let file = File::open(dst)?; + durability::sync_all(&file, "snapshot_file_fsync", Some(dst))?; + Ok(()) +} + +fn replace_file(src: &Path, dst: &Path) -> Result<(), io::Error> { + if dst.exists() { + fs::remove_file(dst)?; + } + fs::rename(src, dst)?; + sync_parent_dir(dst)?; + Ok(()) +} + +fn current_time_ms() -> Result { + i64::try_from( + SystemTime::now() + .duration_since(UNIX_EPOCH) + .map_err(io::Error::other)? + .as_millis(), + ) + .map_err(io::Error::other) +} + +struct StructureBitWriter { + hasher: Option, + current_byte: u8, + current_bits: u8, + bit_len: usize, +} + +impl StructureBitWriter { + fn new(hash_output: bool) -> Self { + Self { + hasher: hash_output.then(Hasher::new), + current_byte: 0, + current_bits: 0, + bit_len: 0, + } + } + + fn bit_len(&self) -> usize { + self.bit_len + } + + fn push_bytes(&mut self, bytes: &[u8]) { + if let Some(hasher) = self.hasher.as_mut() { + hasher.update(bytes); + } + } + + fn push_zero_bytes(&mut self, mut count: usize) { + const ZERO_BUF: [u8; 4096] = [0; 4096]; + while count > 0 { + let chunk = count.min(ZERO_BUF.len()); + self.push_bytes(&ZERO_BUF[..chunk]); + count -= chunk; + } + } + + fn flush_current_byte(&mut self) { + if self.current_bits == 0 { + return; + } + self.push_bytes(&[self.current_byte]); + self.current_byte = 0; + self.current_bits = 0; + } + + fn write_bit(&mut self, bit: bool) { + if bit { + self.current_byte |= 1u8 << self.current_bits; + } + self.current_bits += 1; + self.bit_len += 1; + if self.current_bits == 8 { + self.flush_current_byte(); + } + } + + fn write_zeros(&mut self, count: usize) { + let mut remaining = count; + if self.current_bits == 0 { + let full_bytes = remaining / 8; + if full_bytes > 0 { + self.push_zero_bytes(full_bytes); + self.bit_len += full_bytes * 8; + remaining -= full_bytes * 8; + } + } + for _ in 0..remaining { + self.write_bit(false); + } + } + + fn write_bits_from_value(&mut self, mut value: u64, bits: usize) { + let mut remaining = bits; + if self.current_bits == 0 { + while remaining >= 8 { + self.push_bytes(&[value as u8]); + self.bit_len += 8; + value >>= 8; + remaining -= 8; + } + } + for _ in 0..remaining { + self.write_bit((value & 1) != 0); + value >>= 1; + } + } + + fn finish(mut self) -> (JamStreamStats, Option) { + self.flush_current_byte(); + let byte_len = self.bit_len.div_ceil(8); + let hash = self + .hasher + .take() + .map(|hasher| *hasher.finalize().as_bytes()); + ( + JamStreamStats { + bit_len: self.bit_len, + byte_len, + }, + hash, + ) + } +} + +fn mat_backref(writer: &mut StructureBitWriter, backref: usize) -> Result<(), PmaDirectJamError> { + if backref == 0 { + writer.write_bits_from_value(0b111, 3); + return Ok(()); + } + let backref_sz = met0_u64_to_usize(backref as u64); + let backref_sz_sz = met0_u64_to_usize(backref_sz as u64); + writer.write_bit(true); + writer.write_bit(true); + writer.write_zeros(backref_sz_sz); + writer.write_bit(true); + writer.write_bits_from_value(backref_sz as u64, backref_sz_sz - 1); + writer.write_bits_from_value(backref as u64, backref_sz); + Ok(()) +} + +fn mat_direct_atom(writer: &mut StructureBitWriter, value: u64) -> Result<(), PmaDirectJamError> { + if value == 0 { + writer.write_bits_from_value(0b10, 2); + return Ok(()); + } + let atom_bits = met0_u64_to_usize(value); + mat_atom_header(writer, atom_bits)?; + writer.write_bits_from_value(value, atom_bits); + Ok(()) +} + +fn mat_indirect_atom( + reader: &mut PmaDirectReader, + writer: &mut StructureBitWriter, + offset: u64, + atom_bits: usize, +) -> Result<(), PmaDirectJamError> { + if atom_bits == 0 { + writer.write_bits_from_value(0b10, 2); + return Ok(()); + } + let size_words = reader.indirect_atom_words(offset)?; + mat_atom_header(writer, atom_bits)?; + let last_bits = atom_bits.saturating_sub((size_words - 1).saturating_mul(64)); + for i in 0..size_words { + let word = reader.read_u64(offset + 2 + i as u64)?; + let bits = if i + 1 == size_words { last_bits } else { 64 }; + writer.write_bits_from_value(word, bits); + } + Ok(()) +} + +fn mat_atom_header( + writer: &mut StructureBitWriter, + atom_bits: usize, +) -> Result<(), PmaDirectJamError> { + let atom_sz_sz = met0_u64_to_usize(atom_bits as u64); + writer.write_bit(false); + writer.write_zeros(atom_sz_sz); + writer.write_bit(true); + writer.write_bits_from_value(atom_bits as u64, atom_sz_sz - 1); + Ok(()) +} + +fn mat_cell(writer: &mut StructureBitWriter) -> Result<(), PmaDirectJamError> { + writer.write_bits_from_value(0b01, 2); + Ok(()) +} + +#[cfg(test)] +mod tests { + use std::fs::OpenOptions; + use std::io::{Seek, SeekFrom, Write}; + use std::path::PathBuf; + + use nockvm::jets::cold::Cold; + use nockvm::noun::{Noun, D, T}; + use nockvm::pma::{classify_pma_noun, PmaCopy, PmaRawNounKind}; + use tempfile::TempDir; + + use super::*; + use crate::event_log::EventLogConfig; + use crate::test_support::{TestPmaSandbox, TestPmaStack}; + + fn build_test_snapshot() -> (TestPmaSandbox, PathBuf, PathBuf, SnapshotManifest, u64) { + let mut test_pma = TestPmaStack::default(); + let pma_path = test_pma.pma_path().to_path_buf(); + let manifest_path = test_pma.sandbox_path().join("snapshot.manifest"); + let pma_words = u64::try_from(test_pma.pma().size_words()).expect("pma size fits in u64"); + + let (cold_offset, root_raw, alloc_words) = { + let (stack, pma) = test_pma.stack_pma_mut(); + let mut cold = Cold::new(stack); + let mut root: Noun = T(stack, &[D(42), D(0)]); + unsafe { + cold.copy_to_pma(stack, pma); + root.copy_to_pma(stack, pma); + } + let cold_offset = cold.pma_offset(pma).expect("cold offset"); + let root_raw = unsafe { root.as_raw() }; + pma.sync_used_data().expect("sync used"); + pma.sync_trailer().expect("sync trailer"); + durability::sync_path_data(pma.path(), "snapshot_test_source_fdatasync") + .expect("sync file"); + (cold_offset, root_raw, pma.alloc_offset_words()) + }; + + let manifest = SnapshotManifest::new( + SnapshotKind::Epoch, + "epoch".to_string(), + blake3::hash(b"kernel"), + 7, + pma_words, + alloc_words.into(), + root_raw, + cold_offset, + Hash::from_bytes( + hash_file_prefix( + &pma_path, + alloc_words + .checked_bytes() + .expect("pma alloc fits in bytes"), + ) + .expect("used hash"), + ), + None, + 1234, + ) + .expect("manifest"); + manifest + .write_to_path(&manifest_path) + .expect("write manifest"); + let sandbox = test_pma.into_sandbox(); + (sandbox, pma_path, manifest_path, manifest, root_raw) + } + + fn overwrite_u64(path: &Path, offset_words: u64, value: u64) { + let mut file = OpenOptions::new() + .read(true) + .write(true) + .open(path) + .expect("open pma"); + file.seek(SeekFrom::Start(offset_words * 8)) + .expect("seek pma"); + file.write_all(&value.to_ne_bytes()).expect("write word"); + durability::sync_all(&file, "snapshot_test_file_fsync", None).expect("sync file"); + } + + #[test] + fn cleanup_preserves_ready_snapshot_stored_with_relative_pma_dir_path() { + let temp = TempDir::new().expect("tempdir"); + let pma_dir = temp.path().join(".data.nockchain").join("pma"); + fs::create_dir_all(&pma_dir).expect("create pma dir"); + let pma_path = pma_dir.join("epoch.pma"); + let manifest_path = pma_dir.join("epoch.manifest"); + fs::write(&pma_path, b"tracked epoch pma").expect("write pma"); + fs::write(&manifest_path, b"tracked epoch manifest").expect("write manifest"); + + let mut event_log = EventLog::open(EventLogConfig { + path: temp.path().join("event-log.sqlite3"), + }) + .expect("open event log"); + event_log + .insert_ready_snapshot(&ReadySnapshotRecord { + snapshot_id: 0, + kind: "epoch".to_string(), + event_num: 7, + pma_path: "./.data.nockchain/pma/epoch.pma".to_string(), + manifest_path: "./.data.nockchain/pma/epoch.manifest".to_string(), + alloc_words: 128, + kernel_root_raw: u64::MAX, + cold_offset: PmaOffsetWords::from_words(3), + used_blake3: vec![5; 32], + structure_blake3: None, + created_at_ms: 99, + activated_at_ms: Some(99), + base_snapshot_id: None, + timestamp_tag: "epoch".to_string(), + }) + .expect("insert ready snapshot"); + + cleanup_snapshot_artifacts(&mut event_log, &pma_dir).expect("cleanup snapshots"); + + assert!(pma_path.exists(), "tracked PMA should not be moved"); + assert!( + manifest_path.exists(), + "tracked manifest should not be moved" + ); + assert!(!pma_dir.join("corrupted_pma").join("epoch.pma").exists()); + assert!(!pma_dir + .join("corrupted_pma") + .join("epoch.manifest") + .exists()); + } + + #[test] + fn manifest_roundtrips_with_checksum() { + let manifest = SnapshotManifest::new( + SnapshotKind::Rotating, + "snap-123".to_string(), + blake3::hash(b"ker"), + 11, + 1024, + 64, + 0, + PmaOffsetWords::from_words(3), + blake3::hash(b"used"), + Some(blake3::hash(b"struct")), + 99, + ) + .expect("manifest"); + let encoded = manifest.encode().expect("encode"); + let decoded = SnapshotManifest::decode(&encoded).expect("decode"); + assert_eq!(decoded, manifest); + } + + #[test] + fn verify_snapshot_accepts_valid_snapshot() { + let (_sandbox, pma_path, manifest_path, manifest, _root_raw) = build_test_snapshot(); + let verification = + verify_snapshot(&manifest_path, &pma_path, SnapshotVerifyMode::Full).expect("verify"); + assert_eq!(verification.manifest, manifest); + assert_eq!(verification.file_metadata.alloc_words, manifest.alloc_words); + assert!(verification.structure_stats.is_some()); + } + + #[test] + fn verify_snapshot_rejects_used_hash_mismatch() { + let (_sandbox, pma_path, manifest_path, mut manifest, _root_raw) = build_test_snapshot(); + manifest.used_blake3 = [7; OUT_LEN]; + manifest.checksum = manifest.compute_checksum().expect("checksum"); + manifest + .write_to_path(&manifest_path) + .expect("write manifest"); + + let err = verify_snapshot(&manifest_path, &pma_path, SnapshotVerifyMode::Fast) + .expect_err("mismatch"); + assert!(matches!(err, SnapshotVerifyError::UsedHashMismatch { .. })); + } + + #[test] + fn verify_snapshot_rejects_structural_corruption() { + let (_sandbox, pma_path, manifest_path, mut manifest, root_raw) = build_test_snapshot(); + let root_offset = match classify_pma_noun(root_raw).expect("classify root") { + PmaRawNounKind::Cell { offset } => offset, + other => panic!("expected cell root, got {other:?}"), + }; + overwrite_u64( + &pma_path, + root_offset + .checked_add_words(1) + .expect("root offset + 1 fits") + .into(), + u64::MAX, + ); + manifest.used_blake3 = + hash_file_prefix(&pma_path, manifest.alloc_words * 8).expect("rehash used range"); + manifest.checksum = manifest.compute_checksum().expect("checksum"); + manifest + .write_to_path(&manifest_path) + .expect("write manifest"); + + let err = verify_snapshot(&manifest_path, &pma_path, SnapshotVerifyMode::Full) + .expect_err("corrupt"); + assert!(matches!(err, SnapshotVerifyError::Direct(_))); + } +} diff --git a/crates/nockapp/src/utils/bytes.rs b/crates/nockapp/src/utils/bytes.rs index 679c5449c..264f4481f 100644 --- a/crates/nockapp/src/utils/bytes.rs +++ b/crates/nockapp/src/utils/bytes.rs @@ -3,7 +3,7 @@ use std::any; use bytes::Bytes; use ibig::UBig; use nockvm::jets::cold::{Nounable, NounableResult}; -use nockvm::noun::{Atom, NounAllocator, Slots, D, T}; +use nockvm::noun::{Atom, NounAllocator, NounSpace, D, T}; use crate::utils::error::ConversionError; use crate::{Noun, Result}; @@ -142,9 +142,13 @@ impl Nounable for Byts { let dat = Atom::from_ubig(stack, &big).as_noun(); T(stack, &[wid, dat]) } - fn from_noun(_stack: &mut A, noun: &Noun) -> NounableResult { - let size = noun.slot(2)?; - let dat = noun.slot(3)?.as_atom()?; + fn from_noun( + _stack: &mut A, + noun: &Noun, + space: &NounSpace, + ) -> NounableResult { + let size = noun.in_space(space).slot(2)?; + let dat = noun.in_space(space).slot(3)?.as_atom()?; let wid = size.as_atom()?.as_u64()? as usize; let mut res = vec![0; wid]; @@ -182,8 +186,9 @@ mod test { fn test_byt_direct_atom(context: &mut Context, n: u64) -> Result<(), FromNounError> { // Start with a byt_noun which is a direct atom and consists of zeroes let byt_noun = T(&mut context.stack, &[D(n), D(0x0)]); + let space = context.stack.noun_space(); // Convert it to byt - let byt = super::Byts::from_noun(&mut context.stack, &byt_noun)?; + let byt = super::Byts::from_noun(&mut context.stack, &byt_noun, &space)?; // Check that it is equal assert_eq!(byt.0, vec![0x00; n as usize]); // Convert it back to noun @@ -201,8 +206,9 @@ mod test { // Start with a byt_noun which is a direct atom and a trailing zero let byt_noun = T(&mut context.stack, &[D(3), D(0x8765)]); + let space = context.stack.noun_space(); // Convert it to byt - let byt = super::Byts::from_noun(&mut context.stack, &byt_noun)?; + let byt = super::Byts::from_noun(&mut context.stack, &byt_noun, &space)?; // Check that it is equal assert_eq!(byt.0, vec![0x87, 0x65, 0x00]); // Convert it back to noun @@ -243,7 +249,8 @@ mod test { let expected_byt_noun = T(&mut context.stack, &[D(8), expected_byt_dat]); assert_noun_eq(&mut context.stack, byt_noun, expected_byt_noun); // Convert it back to a byt and check if it matches - let byt_roundtrip = Byts::from_noun(&mut context.stack, &byt_noun)?; + let space = context.stack.noun_space(); + let byt_roundtrip = Byts::from_noun(&mut context.stack, &byt_noun, &space)?; assert_eq!(byt.0, byt_roundtrip.0); // Start with a byt_noun is an indirect atom which does not fit in a u64 @@ -255,7 +262,8 @@ mod test { // Check that the noun is as expected assert_noun_eq(&mut context.stack, byt_noun, expected_byt_noun); // Convert it back to a byt - let byt_roundtrip = Byts::from_noun(&mut context.stack, &byt_noun)?; + let space = context.stack.noun_space(); + let byt_roundtrip = Byts::from_noun(&mut context.stack, &byt_noun, &space)?; // Check that it is the same assert_eq!(byt.0, byt_roundtrip.0); Ok(()) diff --git a/crates/nockapp/src/utils/durability.rs b/crates/nockapp/src/utils/durability.rs new file mode 100644 index 000000000..161eba9d6 --- /dev/null +++ b/crates/nockapp/src/utils/durability.rs @@ -0,0 +1,355 @@ +use std::fs::File; +use std::io; +#[cfg(target_os = "macos")] +use std::os::fd::AsRawFd; +use std::path::{Path, PathBuf}; +use std::sync::atomic::{AtomicBool, Ordering}; +use std::time::Instant; + +use tokio::fs::File as TokioFile; +use tracing::{debug, warn}; + +static FSYNC_DISABLED: AtomicBool = AtomicBool::new(false); + +pub(crate) fn set_fsync_disabled(disabled: bool) { + FSYNC_DISABLED.store(disabled, Ordering::Relaxed); +} + +pub(crate) fn fsync_disabled() -> bool { + FSYNC_DISABLED.load(Ordering::Relaxed) +} + +pub(crate) fn sync_all(file: &File, context: &str, path: Option<&Path>) -> io::Result<()> { + sync_std_file(file, SyncKind::All, context, path) +} + +pub(crate) fn sync_data(file: &File, context: &str, path: Option<&Path>) -> io::Result<()> { + sync_std_file(file, SyncKind::Data, context, path) +} + +pub(crate) async fn sync_all_async( + file: &TokioFile, + context: &str, + path: Option<&Path>, +) -> io::Result<()> { + if fsync_disabled() { + log_sync_skipped("fsync", context, path); + return Ok(()); + } + let strategy = async_sync_strategy(file).await?; + let op = sync_op_name(SyncKind::All, strategy); + log_sync_start(op, context, path); + let start = Instant::now(); + let result = sync_tokio_file(file, SyncKind::All, strategy).await; + log_sync_result(op, context, path, start.elapsed(), &result); + result +} + +pub(crate) fn sync_parent_dir(path: &Path, context: &str) -> io::Result<()> { + #[cfg(unix)] + { + if let Some(parent) = path.parent() { + let parent_file = File::open(parent)?; + return sync_all(&parent_file, context, Some(parent)); + } + } + #[cfg(not(unix))] + { + let _ = path; + } + Ok(()) +} + +pub(crate) fn sync_path_data(path: &Path, context: &str) -> io::Result<()> { + let file = File::options().read(true).write(true).open(path)?; + sync_data(&file, context, Some(path)) +} + +pub(crate) fn write_atomic(path: &Path, bytes: &[u8], context: &str) -> io::Result<()> { + let tmp_path = tmp_path(path); + std::fs::write(&tmp_path, bytes)?; + let tmp_file = File::open(&tmp_path)?; + sync_all(&tmp_file, context, Some(&tmp_path))?; + std::fs::rename(&tmp_path, path)?; + sync_parent_dir(path, context)?; + Ok(()) +} + +fn tmp_path(path: &Path) -> PathBuf { + let mut os = path.as_os_str().to_os_string(); + os.push(".tmp"); + PathBuf::from(os) +} + +#[derive(Copy, Clone)] +enum SyncKind { + All, + Data, +} + +#[derive(Copy, Clone)] +enum SyncStrategy { + Portable, + #[cfg(target_os = "macos")] + FullFlush, +} + +fn sync_std_file( + file: &File, + kind: SyncKind, + context: &str, + path: Option<&Path>, +) -> io::Result<()> { + if fsync_disabled() { + log_sync_skipped(default_sync_op_name(kind), context, path); + return Ok(()); + } + + let strategy = sync_strategy(file)?; + let op = sync_op_name(kind, strategy); + run_sync(op, context, path, || { + sync_std_file_with_strategy(file, kind, strategy) + }) +} + +fn sync_std_file_with_strategy( + file: &File, + kind: SyncKind, + strategy: SyncStrategy, +) -> io::Result<()> { + match kind { + SyncKind::All => file.sync_all()?, + SyncKind::Data => file.sync_data()?, + } + + #[cfg(target_os = "macos")] + if matches!(strategy, SyncStrategy::FullFlush) { + full_fsync_file(file)?; + } + + #[cfg(not(target_os = "macos"))] + let _ = strategy; + + Ok(()) +} + +async fn sync_tokio_file( + file: &TokioFile, + kind: SyncKind, + strategy: SyncStrategy, +) -> io::Result<()> { + match kind { + SyncKind::All => file.sync_all().await?, + SyncKind::Data => file.sync_data().await?, + } + + #[cfg(target_os = "macos")] + if matches!(strategy, SyncStrategy::FullFlush) { + full_fsync_fd(file.as_raw_fd())?; + } + + #[cfg(not(target_os = "macos"))] + let _ = strategy; + + Ok(()) +} + +fn default_sync_op_name(kind: SyncKind) -> &'static str { + match kind { + SyncKind::All => "fsync", + SyncKind::Data => "fdatasync", + } +} + +#[cfg(target_os = "macos")] +fn sync_op_name(kind: SyncKind, strategy: SyncStrategy) -> &'static str { + match (kind, strategy) { + (SyncKind::All, SyncStrategy::Portable) => "fsync", + (SyncKind::Data, SyncStrategy::Portable) => "fdatasync", + (SyncKind::All, SyncStrategy::FullFlush) => "fsync+fullfsync", + (SyncKind::Data, SyncStrategy::FullFlush) => "fdatasync+fullfsync", + } +} + +#[cfg(not(target_os = "macos"))] +fn sync_op_name(kind: SyncKind, _strategy: SyncStrategy) -> &'static str { + default_sync_op_name(kind) +} + +#[cfg(target_os = "macos")] +fn sync_strategy(file: &File) -> io::Result { + let metadata = file.metadata()?; + Ok(if metadata.is_file() { + SyncStrategy::FullFlush + } else { + SyncStrategy::Portable + }) +} + +#[cfg(not(target_os = "macos"))] +fn sync_strategy(_file: &File) -> io::Result { + Ok(SyncStrategy::Portable) +} + +#[cfg(target_os = "macos")] +async fn async_sync_strategy(file: &TokioFile) -> io::Result { + let metadata = file.metadata().await?; + Ok(if metadata.is_file() { + SyncStrategy::FullFlush + } else { + SyncStrategy::Portable + }) +} + +#[cfg(not(target_os = "macos"))] +async fn async_sync_strategy(_file: &TokioFile) -> io::Result { + Ok(SyncStrategy::Portable) +} + +fn run_sync(op: &'static str, context: &str, path: Option<&Path>, f: F) -> io::Result<()> +where + F: FnOnce() -> io::Result<()>, +{ + log_sync_start(op, context, path); + let start = Instant::now(); + let result = f(); + log_sync_result(op, context, path, start.elapsed(), &result); + result +} + +#[cfg(target_os = "macos")] +fn full_fsync_file(file: &File) -> io::Result<()> { + full_fsync_fd(file.as_raw_fd()) +} + +#[cfg(target_os = "macos")] +fn full_fsync_fd(fd: std::os::fd::RawFd) -> io::Result<()> { + let rc = unsafe { libc::fcntl(fd, libc::F_FULLFSYNC) }; + if rc == -1 { + return Err(io::Error::last_os_error()); + } + Ok(()) +} + +fn log_sync_start(op: &str, context: &str, path: Option<&Path>) { + match path { + Some(path) => debug!( + sync_op = op, + sync_context = context, + path = %path.display(), + "durability sync start" + ), + None => debug!( + sync_op = op, + sync_context = context, + "durability sync start" + ), + } +} + +fn log_sync_skipped(op: &str, context: &str, path: Option<&Path>) { + match path { + Some(path) => debug!( + sync_op = op, + sync_context = context, + path = %path.display(), + "durability sync skipped (fsync disabled)" + ), + None => debug!( + sync_op = op, + sync_context = context, + "durability sync skipped (fsync disabled)" + ), + } +} + +fn log_sync_result( + op: &str, + context: &str, + path: Option<&Path>, + elapsed: std::time::Duration, + result: &io::Result<()>, +) { + let elapsed_ms = duration_ms(elapsed); + match (path, result) { + (Some(path), Ok(())) => debug!( + sync_op = op, + sync_context = context, + path = %path.display(), + elapsed_ms, + "durability sync done" + ), + (None, Ok(())) => debug!( + sync_op = op, + sync_context = context, + elapsed_ms, + "durability sync done" + ), + (Some(path), Err(err)) => warn!( + sync_op = op, + sync_context = context, + path = %path.display(), + elapsed_ms, + error = %err, + "durability sync failed" + ), + (None, Err(err)) => warn!( + sync_op = op, + sync_context = context, + elapsed_ms, + error = %err, + "durability sync failed" + ), + } +} + +fn duration_ms(d: std::time::Duration) -> f64 { + d.as_secs_f64() * 1000.0 +} + +#[cfg(test)] +mod tests { + use std::fs; + + use tempfile::tempdir; + + use super::{sync_all, sync_parent_dir, sync_path_data, write_atomic}; + + #[test] + fn sync_all_accepts_read_only_regular_files() { + let temp = tempdir().expect("tempdir"); + let path = temp.path().join("sync-all.bin"); + fs::write(&path, b"abc").expect("write test file"); + + let file = std::fs::File::open(&path).expect("open test file"); + sync_all(&file, "durability_test_sync_all", Some(&path)).expect("sync file"); + } + + #[test] + fn sync_path_data_accepts_regular_files() { + let temp = tempdir().expect("tempdir"); + let path = temp.path().join("sync-data.bin"); + fs::write(&path, b"abc").expect("write test file"); + + sync_path_data(&path, "durability_test_sync_data").expect("sync path data"); + } + + #[test] + fn sync_parent_dir_accepts_directory_handles() { + let temp = tempdir().expect("tempdir"); + let path = temp.path().join("child.bin"); + fs::write(&path, b"abc").expect("write child file"); + + sync_parent_dir(&path, "durability_test_sync_parent_dir").expect("sync parent dir"); + } + + #[test] + fn write_atomic_syncs_file_and_parent_dir() { + let temp = tempdir().expect("tempdir"); + let path = temp.path().join("atomic.bin"); + + write_atomic(&path, b"payload", "durability_test_write_atomic").expect("write atomic"); + + assert_eq!(fs::read(&path).expect("read atomic file"), b"payload"); + } +} diff --git a/crates/nockapp/src/utils/error.rs b/crates/nockapp/src/utils/error.rs index 87129c4eb..b4a54dc67 100644 --- a/crates/nockapp/src/utils/error.rs +++ b/crates/nockapp/src/utils/error.rs @@ -1,6 +1,8 @@ use noun_serde::NounDecodeError; use thiserror::Error; +use crate::noun::slab::NounSlab; + #[derive(Debug, Error)] pub enum ExternalError { #[error("unknown error: {0}")] @@ -27,7 +29,7 @@ pub enum CrownError { #[error("{0}")] InterpreterError(#[from] SwordError), #[error("kernel error")] - KernelError(Option), + KernelError(Option), #[error("{0}")] Utf8FromError(#[from] std::string::FromUtf8Error), #[error("{0}")] @@ -40,6 +42,14 @@ pub enum CrownError { BootError, #[error("Serf load error")] SerfLoadError, + #[error("{0}")] + SerfInitAllocationError(String), + #[error("{0}")] + SerfInitPanic(String), + #[error( + "checkpoint ker_hash != current ker_hash: checkpoint={checkpoint} current={current}. Rebuild with matching kernel assets or resync checkpoints." + )] + CheckpointKernelHashMismatch { checkpoint: String, current: String }, #[error("work bail")] WorkBail, #[error("peek bail")] diff --git a/crates/nockapp/src/utils/mod.rs b/crates/nockapp/src/utils/mod.rs index 2206173a1..6311148d1 100644 --- a/crates/nockapp/src/utils/mod.rs +++ b/crates/nockapp/src/utils/mod.rs @@ -1,4 +1,5 @@ pub mod bytes; +pub(crate) mod durability; pub mod error; pub mod scry; pub mod slogger; @@ -11,7 +12,6 @@ use std::time::{SystemTime, UNIX_EPOCH}; use byteorder::{LittleEndian, ReadBytesExt}; pub use bytes::ToBytes; -use either::Either; pub use error::{CrownError, Result}; use nockvm::hamt::Hamt; use nockvm::interpreter::{self, Context, NockCancelToken}; @@ -19,7 +19,11 @@ use nockvm::jets::cold::Cold; use nockvm::jets::hot::{Hot, HotEntry}; use nockvm::jets::warm::Warm; use nockvm::mem::NockStack; -use nockvm::noun::{Noun, D}; +pub use nockvm::mem::{ + NOCK_STACK_1KB, NOCK_STACK_SIZE, NOCK_STACK_SIZE_HUGE, NOCK_STACK_SIZE_LARGE, + NOCK_STACK_SIZE_MEDIUM, NOCK_STACK_SIZE_SMALL, NOCK_STACK_SIZE_TINY, +}; +use nockvm::noun::{Noun, NounSpace, D}; use nockvm::serialization::jam; use nockvm::trace::TraceInfo; use slogger::CrownSlogger; @@ -34,24 +38,6 @@ const EPOCH_DA: u128 = 170141184475152167957503069145530368000; // ~s1 const S1: u128 = 18446744073709551616; -pub const NOCK_STACK_1KB: usize = 1 << 7; - -pub const NOCK_STACK_SIZE_TINY: usize = (NOCK_STACK_1KB << 10 << 10) * 2; // 2GB - -pub const NOCK_STACK_SIZE_SMALL: usize = (NOCK_STACK_1KB << 10 << 10) * 4; // 4GB - -// nock stack size -pub const NOCK_STACK_SIZE: usize = (NOCK_STACK_1KB << 10 << 10) * 8; // 8GB - -// nock stack size -pub const NOCK_STACK_SIZE_MEDIUM: usize = (NOCK_STACK_1KB << 10 << 10) * 16; // 16GB - -// LARGE nock stack size -pub const NOCK_STACK_SIZE_LARGE: usize = (NOCK_STACK_1KB << 10 << 10) * 32; // 32GB - -// HUGE nock stack size -pub const NOCK_STACK_SIZE_HUGE: usize = (NOCK_STACK_1KB << 10 << 10) * 64; // 64GB - /** * :: +from-unix: unix seconds to @da * :: @@ -98,7 +84,9 @@ pub use nockvm::ext::make_tas; // serialize a noun for writing over a socket or a file descriptor pub fn serialize_noun(stack: &mut NockStack, noun: Noun) -> Result> { let atom = jam(stack, noun); - let size = atom.size() << 3; + let space = stack.noun_space(); + let atom_handle = atom.in_space(&space); + let size = atom_handle.size() << 3; let buf = unsafe { from_raw_parts_mut(stack.struct_alloc::(size + 5), size + 5) }; buf[0] = 0u8; @@ -107,27 +95,32 @@ pub fn serialize_noun(stack: &mut NockStack, noun: Noun) -> Result> { buf[3] = (size >> 16) as u8; buf[4] = (size >> 24) as u8; - match atom.as_either() { - Either::Left(direct) => unsafe { + if atom_handle.is_direct() { + let direct = atom_handle + .atom() + .as_direct() + .expect("direct atom expected"); + unsafe { copy_nonoverlapping( &direct.data() as *const u64 as *const u8, buf.as_mut_ptr().add(5), size, ); - }, - Either::Right(indirect) => unsafe { + } + } else { + unsafe { copy_nonoverlapping( - indirect.data_pointer() as *const u8, + atom_handle.data_pointer() as *const u8, buf.as_mut_ptr().add(5), size, ); - }, - }; + } + } Ok(buf.to_vec()) } -pub fn compute_timer_time(time: Noun) -> Result { - let time_atom = time.as_atom()?; +pub fn compute_timer_time(time: Noun, space: &NounSpace) -> Result { + let time_atom = time.in_space(space).as_atom()?; let mut time_bytes: &[u8] = time_atom.as_ne_bytes(); let timer_time: u128 = da_to_unix_ms(DA(ReadBytesExt::read_u128::( &mut time_bytes, @@ -156,6 +149,7 @@ pub fn create_context( trace_info: Option, test_jets: Vec, ) -> Context { + let arena = stack.arena().clone(); let cache = Hamt::::new(&mut stack); let test_jets = { let mut hamt = Hamt::<()>::new(&mut stack); @@ -182,5 +176,6 @@ pub fn create_context( trace_info, test_jets, running_status: cancel, + arena, } } diff --git a/crates/nockapp/src/utils/scry.rs b/crates/nockapp/src/utils/scry.rs index 970ef2480..53f801477 100644 --- a/crates/nockapp/src/utils/scry.rs +++ b/crates/nockapp/src/utils/scry.rs @@ -1,18 +1,18 @@ use either::{Left, Right}; -use nockvm::noun::Noun; +use nockvm::noun::{Noun, NounHandle, NounSpace}; -pub enum ScryResult { - BadPath, // ~ - Nothing, // [~ ~] - Some(Noun), // [~ ~ foo] - Invalid, // anything that isn't one of the above +pub enum ScryResult<'a> { + BadPath, // ~ + Nothing, // [~ ~] + Some(NounHandle<'a>), // [~ ~ foo] + Invalid, // anything that isn't one of the above } -impl From<&Noun> for ScryResult { - fn from(noun: &Noun) -> ScryResult { - match noun.as_either_atom_cell() { +impl<'a> ScryResult<'a> { + pub fn from_noun(noun: &Noun, space: &'a NounSpace) -> ScryResult<'a> { + match noun.in_space(space).as_either_atom_cell() { Left(atom) => { - let Ok(direct) = atom.as_direct() else { + let Ok(direct) = atom.atom().as_direct() else { return ScryResult::Invalid; }; if direct.data() == 0 { @@ -20,13 +20,13 @@ impl From<&Noun> for ScryResult { } } Right(cell) => { - let Ok(head) = cell.head().as_direct() else { + let Ok(head) = cell.head().noun().as_direct() else { return ScryResult::Invalid; }; if head.data() == 0 { match cell.tail().as_either_atom_cell() { Left(atom) => { - let Ok(direct) = atom.as_direct() else { + let Ok(direct) = atom.atom().as_direct() else { return ScryResult::Invalid; }; if direct.data() == 0 { @@ -34,7 +34,7 @@ impl From<&Noun> for ScryResult { } } Right(tail) => { - let Ok(tail_head) = tail.head().as_direct() else { + let Ok(tail_head) = tail.head().noun().as_direct() else { return ScryResult::Invalid; }; if tail_head.data() == 0 { diff --git a/crates/nockapp/src/utils/slogger.rs b/crates/nockapp/src/utils/slogger.rs index 8fb02fc80..acd1ec064 100644 --- a/crates/nockapp/src/utils/slogger.rs +++ b/crates/nockapp/src/utils/slogger.rs @@ -4,7 +4,7 @@ use either::Either::*; use nockvm::interpreter::Slogger; use nockvm::jets::list::util::lent; use nockvm::mem::NockStack; -use nockvm::noun::{Atom, DirectAtom, IndirectAtom, Noun, Slots}; +use nockvm::noun::{Atom, DirectAtom, IndirectAtom, Noun, NounSpace}; use nockvm_macros::tas; use tracing::{debug, error, info, trace, warn}; @@ -55,8 +55,9 @@ impl Slogger for CrownSlogger { option_env!("GIT_SHA") ) }); + let space = _stack.noun_space(); let mut buffer = Vec::new(); - match slog_cord(cord_atom, &mut buffer) { + match slog_cord(cord_atom, &mut buffer, &space) { Ok(_) => { let message = String::from_utf8_lossy(&buffer) .trim_matches('\0') @@ -81,30 +82,32 @@ impl Slogger for CrownSlogger { } } -fn slog_cord(cord: Atom, out: &mut W) -> Result<()> { - out.write_all(cord.as_ne_bytes())?; +fn slog_cord(cord: Atom, out: &mut W, space: &NounSpace) -> Result<()> { + out.write_all(cord.in_space(space).as_ne_bytes())?; Ok(()) } fn slog_tape(stack: &mut NockStack, tape: Noun, out: &mut W) -> Result<()> { let cord = crip(stack, tape)?; - slog_cord(cord, out) + let space = stack.noun_space(); + slog_cord(cord, out, &space) } // XX TODO: pre-crip all tapes fn slog_palm(stack: &mut NockStack, palm: Noun, out: &mut W) -> Result<()> { - let ds = palm.slot(6)?; - let fore1 = ds.slot(6)?; - let fore2 = ds.slot(14)?; + let space = stack.noun_space(); + let ds = palm.in_space(&space).slot(6)?.noun(); + let fore1 = ds.in_space(&space).slot(6)?.noun(); + let fore2 = ds.in_space(&space).slot(14)?.noun(); slog_tape(stack, fore1, out)?; slog_tape(stack, fore2, out)?; - let mid = ds.slot(2)?; - let end = ds.slot(15)?; - let mut tanks = palm.slot(7)?; + let mid = ds.in_space(&space).slot(2)?.noun(); + let end = ds.in_space(&space).slot(15)?.noun(); + let mut tanks = palm.in_space(&space).slot(7)?.noun(); loop { - if let Ok(tanks_it) = tanks.as_cell() { - slog_tank(stack, tanks_it.head(), out)?; - tanks = tanks_it.tail(); + if let Ok(tanks_it) = tanks.in_space(&space).as_cell() { + slog_tank(stack, tanks_it.head().noun(), out)?; + tanks = tanks_it.tail().noun(); if tanks.is_cell() { slog_tape(stack, mid, out)?; } @@ -116,18 +119,19 @@ fn slog_palm(stack: &mut NockStack, palm: Noun, out: &mut W) -> Result // XX todo: pre-crip all tapes fn slog_rose(stack: &mut NockStack, rose: Noun, out: &mut W) -> Result<()> { - let ds = rose.slot(6)?; - let fore = ds.slot(6)?; + let space = stack.noun_space(); + let ds = rose.in_space(&space).slot(6)?.noun(); + let fore = ds.in_space(&space).slot(6)?.noun(); slog_tape(stack, fore, out)?; - let mid = ds.slot(2)?; - let end = ds.slot(7)?; + let mid = ds.in_space(&space).slot(2)?.noun(); + let end = ds.in_space(&space).slot(7)?.noun(); - let mut tanks = rose.slot(7)?; + let mut tanks = rose.in_space(&space).slot(7)?.noun(); loop { - if let Ok(tanks_it) = tanks.as_cell() { - slog_tank(stack, tanks_it.head(), out)?; - tanks = tanks_it.tail(); + if let Ok(tanks_it) = tanks.in_space(&space).as_cell() { + slog_tank(stack, tanks_it.head().noun(), out)?; + tanks = tanks_it.tail().noun(); if tanks.is_cell() { slog_tape(stack, mid, out)?; } @@ -138,12 +142,13 @@ fn slog_rose(stack: &mut NockStack, rose: Noun, out: &mut W) -> Result } fn slog_tank(stack: &mut NockStack, tank: Noun, out: &mut W) -> Result<()> { - match tank.as_either_atom_cell() { - Left(cord) => slog_cord(cord, out), + let space = stack.noun_space(); + match tank.in_space(&space).as_either_atom_cell() { + Left(cord) => slog_cord(cord.atom(), out, &space), Right(cell) => { - let tag = cell.head().as_direct()?; + let tag = cell.head().noun().as_direct()?; match tag.data() { - tas!(b"leaf") => slog_tape(stack, cell.tail(), out), + tas!(b"leaf") => slog_tape(stack, cell.tail().noun(), out), tas!(b"palm") => slog_palm(stack, tank, out), tas!(b"rose") => slog_rose(stack, tank, out), _ => Err(CrownError::Unknown("Bad tank".to_string())), @@ -153,17 +158,18 @@ fn slog_tank(stack: &mut NockStack, tank: Noun, out: &mut W) -> Result } fn crip(stack: &mut NockStack, mut tape: Noun) -> Result { - let l = lent(tape)?; + let space = stack.noun_space(); + let l = lent(tape, &space)?; if l == 0 { return Ok(unsafe { DirectAtom::new_unchecked(0).as_atom() }); } - let (mut indirect, buf) = unsafe { IndirectAtom::new_raw_mut_bytes(stack, l) }; + let (indirect, buf) = unsafe { IndirectAtom::new_raw_mut_bytes(stack, l) }; let mut idx = 0; loop { - if let Ok(tape_it) = tape.as_cell() { - let tape_byte = tape_it.head().as_direct()?; - tape = tape_it.tail(); + if let Ok(tape_it) = tape.in_space(&space).as_cell() { + let tape_byte = tape_it.head().noun().as_direct()?; + tape = tape_it.tail().noun(); if tape_byte.data() >= 256 { break Err(CrownError::Unknown("Bad tape".to_string())); } else { @@ -171,7 +177,8 @@ fn crip(stack: &mut NockStack, mut tape: Noun) -> Result { idx += 1; } } else { - break Ok(unsafe { indirect.normalize_as_atom() }); + let normalized = unsafe { indirect.as_atom().in_space(&space).normalize().atom() }; + break Ok(normalized); } } } diff --git a/crates/nockapp/tests/integration.rs b/crates/nockapp/tests/integration.rs index b49bb2073..c09052a42 100644 --- a/crates/nockapp/tests/integration.rs +++ b/crates/nockapp/tests/integration.rs @@ -2,14 +2,15 @@ use nockapp::noun::slab::NounSlab; use nockapp::test::setup_nockapp; use nockapp::wire::{SystemWire, Wire}; use nockapp::NockApp; -use nockvm::noun::{Noun, Slots, D}; +use nockvm::mem::{NockStack, NOCK_STACK_SIZE_TINY}; +use nockvm::noun::{Noun, NounAllocator, D}; use nockvm_macros::tas; use tracing::info; #[tracing::instrument(skip(nockapp))] fn run_once(nockapp: &mut NockApp, i: u64) { info!("before poke construction"); - let poke = D(tas!(b"inc")).into(); + let poke = make_inc_poke(); info!("Poke constructed"); let wire = SystemWire.to_wire(); info!("Wire constructed"); @@ -34,15 +35,20 @@ fn run_once(nockapp: &mut NockApp, i: u64) { option_env!("GIT_SHA") ) }); - let root = unsafe { res.root() }; - let val: Noun = root.slot(15).unwrap_or_else(|err| { - panic!( - "Panicked with {err:?} at {}:{} (git sha: {:?})", - file!(), - line!(), - option_env!("GIT_SHA") - ) - }); + let space = res.noun_space(); + let root = unsafe { *res.root() }; + let val: Noun = root + .in_space(&space) + .slot(15) + .map(|handle| handle.noun()) + .unwrap_or_else(|err| { + panic!( + "Panicked with {err:?} at {}:{} (git sha: {:?})", + file!(), + line!(), + option_env!("GIT_SHA") + ) + }); unsafe { assert!(val.raw_equals(&D(i)), "Expected {} but got {:?}", i, val); } @@ -83,8 +89,9 @@ async fn test_looped_sync_peek_and_poke() { async fn test_sync_peek_and_poke() { let (_temp, mut nockapp) = setup_nockapp("test-ker.jam").await; tokio::task::spawn_blocking(move || { + let _test_arena = install_test_arena(); for i in 1..4 { - let poke = D(tas!(b"inc")).into(); + let poke = make_inc_poke(); let wire = SystemWire.to_wire(); let _ = nockapp.poke_sync(wire, poke).unwrap_or_else(|err| { panic!( @@ -105,15 +112,20 @@ async fn test_sync_peek_and_poke() { option_env!("GIT_SHA") ) }); - let root = unsafe { res.root() }; - let val: Noun = root.slot(15).unwrap_or_else(|err| { - panic!( - "Panicked with {err:?} at {}:{} (git sha: {:?})", - file!(), - line!(), - option_env!("GIT_SHA") - ) - }); + let space = res.noun_space(); + let root = unsafe { *res.root() }; + let val: Noun = root + .in_space(&space) + .slot(15) + .map(|handle| handle.noun()) + .unwrap_or_else(|err| { + panic!( + "Panicked with {err:?} at {}:{} (git sha: {:?})", + file!(), + line!(), + option_env!("GIT_SHA") + ) + }); unsafe { assert!(val.raw_equals(&D(i))); } @@ -122,3 +134,13 @@ async fn test_sync_peek_and_poke() { .await .expect("Synchronous test thread failed"); } + +fn install_test_arena() -> NockStack { + NockStack::new(NOCK_STACK_SIZE_TINY, 0) +} + +fn make_inc_poke() -> NounSlab { + let mut poke = NounSlab::new(); + poke.set_root(D(tas!(b"inc"))); + poke +} diff --git a/crates/nockapp/tests/pma_regressions.rs b/crates/nockapp/tests/pma_regressions.rs new file mode 100644 index 000000000..c343b530f --- /dev/null +++ b/crates/nockapp/tests/pma_regressions.rs @@ -0,0 +1,77 @@ +use std::error::Error; +use std::sync::Mutex; + +mod pma_regressions { + pub(crate) mod boot_active_resize; + pub(crate) mod checkpoint_bootstrap_size; + pub(crate) mod event_preflight_growth; + pub(crate) mod event_resize_failure_boundary; + pub(crate) mod failed_preserve_recovery; + pub(crate) mod fresh_replay_boundary; + pub(crate) mod pma_meta; + pub(crate) mod snapshot_restore_expand; + pub(crate) mod sqlite_boundary_recovery; + pub(crate) mod stale_checkpoint_refusal; + pub(crate) mod upgrade_from_65a; +} + +type TestResult = Result<(), Box>; + +static PMA_REGRESSION_LOCK: Mutex<()> = Mutex::new(()); + +fn run_serialized(test: fn() -> TestResult) -> TestResult { + let _guard = PMA_REGRESSION_LOCK + .lock() + .expect("PMA regression test lock poisoned"); + test() +} + +#[test] +fn pma_boot_active_resize_regression() -> TestResult { + run_serialized(pma_regressions::boot_active_resize::run_regression) +} + +#[test] +fn pma_checkpoint_bootstrap_size_regression() -> TestResult { + run_serialized(pma_regressions::checkpoint_bootstrap_size::run_regression) +} + +#[test] +fn pma_event_preflight_growth_regression() -> TestResult { + run_serialized(pma_regressions::event_preflight_growth::run_regression) +} + +#[test] +fn pma_event_resize_failure_boundary_regression() -> TestResult { + run_serialized(pma_regressions::event_resize_failure_boundary::run_regression) +} + +#[test] +fn pma_failed_preserve_recovery_regression() -> TestResult { + run_serialized(pma_regressions::failed_preserve_recovery::run_regression) +} + +#[test] +fn pma_fresh_replay_boundary_regression() -> TestResult { + run_serialized(pma_regressions::fresh_replay_boundary::run_regression) +} + +#[test] +fn pma_snapshot_restore_expand_regression() -> TestResult { + run_serialized(pma_regressions::snapshot_restore_expand::run_regression) +} + +#[test] +fn pma_sqlite_boundary_recovery_regression() -> TestResult { + run_serialized(pma_regressions::sqlite_boundary_recovery::run_regression) +} + +#[test] +fn pma_stale_checkpoint_refusal_regression() -> TestResult { + run_serialized(pma_regressions::stale_checkpoint_refusal::run_regression) +} + +#[test] +fn pma_upgrade_from_65a_regression() -> TestResult { + run_serialized(pma_regressions::upgrade_from_65a::run_regression) +} diff --git a/crates/nockapp/tests/pma_regressions/boot_active_resize.rs b/crates/nockapp/tests/pma_regressions/boot_active_resize.rs new file mode 100644 index 000000000..05388f23f --- /dev/null +++ b/crates/nockapp/tests/pma_regressions/boot_active_resize.rs @@ -0,0 +1,209 @@ +use std::error::Error; +use std::fs; +use std::path::{Path, PathBuf}; + +use nockapp::kernel::boot::{default_boot_cli, setup_, NockStackSize, PmaSize, SetupResult}; +use nockapp::nockapp::wire::{SystemWire, Wire}; +use nockapp::noun::slab::{NockJammer, NounSlab}; +use nockapp::NockApp; +use nockvm::noun::{NounSpace, D}; +use nockvm::offset::PmaOffsetWords; +use nockvm::pma::Pma; +use nockvm_macros::tas; +use tempfile::TempDir; + +const FORCED_FREE_WORDS: u64 = 2; + +pub(crate) fn run_regression() -> Result<(), Box> { + nockvm::check_endian(); + let runtime = tokio::runtime::Builder::new_current_thread() + .enable_all() + .build()?; + runtime.block_on(run()) +} + +async fn run() -> Result<(), Box> { + let temp = TempDir::new()?; + let data_dir = temp.path().join("pma-boot-active-resize-regression"); + let jam = load_test_jam()?; + + println!("stage 1: create valid active tiny PMA at event 1"); + let mut first = boot_app(&jam, &data_dir, NockStackSize::Tiny).await?; + poke_inc(&mut first).await?; + let event_num = first.export().await?.event_num; + assert_eq!(event_num, 1, "fixture must be at event 1 before PMA fill"); + stop_app(first).await?; + + let pma_path = data_dir.join("pma").join("0.pma"); + let meta_path = data_dir.join("pma").join("0.meta"); + if !pma_path.exists() || !meta_path.exists() { + return Err(std::io::Error::other(format!( + "expected active 0.pma/0.meta after first boot, got pma_exists={} meta_exists={}", + pma_path.exists(), + meta_path.exists() + )) + .into()); + } + + force_pma_free_words(&pma_path, FORCED_FREE_WORDS)?; + let filled = Pma::read_file_metadata(&pma_path)?; + println!( + "fixture active PMA forced near full: path={} data_words={} alloc_words={} free_words={}", + pma_path.display(), + filled.data_words, + filled.alloc_words, + filled.data_words.saturating_sub(filled.alloc_words) + ); + assert_eq!( + filled.data_words.saturating_sub(filled.alloc_words), + FORCED_FREE_WORDS, + "fixture must reproduce production free-space condition" + ); + + println!("stage 2: reboot with larger configured stack/PMA minimum"); + match boot_app(&jam, &data_dir, NockStackSize::Small).await { + Ok(second) => { + let event_num = second.export().await?.event_num; + assert_eq!( + event_num, 1, + "boot must preserve the committed event boundary after active PMA resize" + ); + let active = largest_meta_paired_pma(&data_dir)?; + let active_metadata = Pma::read_file_metadata(&active)?; + println!( + "post-boot active PMA candidate: path={} data_words={} alloc_words={} free_words={}", + active.display(), + active_metadata.data_words, + active_metadata.alloc_words, + active_metadata + .data_words + .saturating_sub(active_metadata.alloc_words) + ); + if active_metadata.data_words <= filled.data_words { + return Err(std::io::Error::other(format!( + "boot succeeded but active PMA was not grown: before_data_words={} after_data_words={}", + filled.data_words, active_metadata.data_words + )) + .into()); + } + stop_app(second).await?; + println!("active full PMA was resized by production boot before Serf initialization"); + Ok(()) + } + Err(err) => { + let message = err.to_string(); + if message.contains("PMA is full") { + eprintln!( + "reproduced pre-fix production failure: active valid PMA selected with only {FORCED_FREE_WORDS} free words, then Serf initialization hit PMA OOM" + ); + } + Err(std::io::Error::other(format!( + "production boot did not resize active full PMA before Serf initialization: {message}" + )) + .into()) + } + } +} + +fn load_test_jam() -> Result, Box> { + let mut possible_paths = Vec::new(); + if let Some(manifest_dir) = option_env!("CARGO_MANIFEST_DIR") { + possible_paths.push( + Path::new(manifest_dir) + .join("test-jams") + .join("test-ker.jam"), + ); + } + possible_paths.push(Path::new("open/crates/nockapp/test-jams").join("test-ker.jam")); + possible_paths.push(Path::new("test-jams").join("test-ker.jam")); + + for path in &possible_paths { + if let Ok(bytes) = fs::read(path) { + return Ok(bytes); + } + } + + Err(std::io::Error::other(format!( + "failed to read test-ker.jam from any candidate path: {:?}", + possible_paths + )) + .into()) +} + +async fn boot_app( + jam: &[u8], + data_dir: &Path, + stack_size: NockStackSize, +) -> Result, Box> { + let mut cli = default_boot_cli(false); + cli.data_dir = Some(data_dir.to_path_buf()); + cli.pma_initial_size = Some(PmaSize::from_words(stack_size.stack_words())); + cli.stack_size = stack_size; + cli.gc_interval = None; + cli.rotating_snapshot_interval_event_time = None; + cli.disable_fsync = true; + match setup_::(jam, cli, &[], "pma-boot-active-resize-regression", None).await? { + SetupResult::App(app) => Ok(app), + SetupResult::ExportedState => Err(std::io::Error::other("unexpected state export").into()), + } +} + +fn inc_poke() -> NounSlab { + let mut slab = NounSlab::new(); + let space = NounSpace::empty(); + slab.copy_into(D(tas!(b"inc")), &space); + slab +} + +async fn poke_inc(app: &mut NockApp) -> Result<(), Box> { + app.poke(SystemWire.to_wire(), inc_poke()).await?; + Ok(()) +} + +async fn stop_app(mut app: NockApp) -> Result<(), Box> { + let handle = app.get_handle(); + handle.exit.exit(0).await?; + app.run().await?; + Ok(()) +} + +fn force_pma_free_words(path: &Path, free_words: u64) -> Result<(), Box> { + let metadata = Pma::read_file_metadata(path)?; + if metadata.data_words <= free_words { + return Err(std::io::Error::other(format!( + "PMA too small to force {free_words} free words: data_words={}", + metadata.data_words + )) + .into()); + } + let new_alloc_words = metadata.data_words - free_words; + let mut pma = Pma::open(path.to_path_buf())?; + pma.reset_to(PmaOffsetWords::from_words(new_alloc_words)); + pma.sync_trailer()?; + pma.sync_file()?; + Ok(()) +} + +fn largest_meta_paired_pma(data_dir: &Path) -> Result> { + let pma_dir = data_dir.join("pma"); + let mut best: Option<(PathBuf, u64)> = None; + for idx in [0, 1] { + let pma_path = pma_dir.join(format!("{idx}.pma")); + let meta_path = pma_dir.join(format!("{idx}.meta")); + if !pma_path.exists() || !meta_path.exists() { + continue; + } + let metadata = Pma::read_file_metadata(&pma_path)?; + match &best { + Some((_, best_words)) if *best_words >= metadata.data_words => {} + _ => best = Some((pma_path, metadata.data_words)), + } + } + best.map(|(path, _)| path).ok_or_else(|| { + std::io::Error::other(format!( + "no meta-paired runtime PMA found in {}", + pma_dir.display() + )) + .into() + }) +} diff --git a/crates/nockapp/tests/pma_regressions/checkpoint_bootstrap_size.rs b/crates/nockapp/tests/pma_regressions/checkpoint_bootstrap_size.rs new file mode 100644 index 000000000..c485bcb86 --- /dev/null +++ b/crates/nockapp/tests/pma_regressions/checkpoint_bootstrap_size.rs @@ -0,0 +1,464 @@ +use std::error::Error; +use std::ffi::OsString; +use std::fs; +use std::path::{Path, PathBuf}; + +use diesel::prelude::*; +use diesel::sql_query; +use diesel::sql_types::BigInt; +use diesel::sqlite::SqliteConnection; +use nockapp::kernel::boot::{default_boot_cli, setup_, NockStackSize, PmaSize, SetupResult}; +use nockapp::nockapp::wire::{SystemWire, Wire}; +use nockapp::noun::slab::{NockJammer, NounSlab}; +use nockapp::save::SaveableCheckpoint; +use nockapp::NockApp; +use nockvm::mem::NOCK_STACK_SIZE_SMALL; +use nockvm::noun::{NounAllocator, NounSpace, D, T}; +use nockvm::pma::Pma; +use nockvm_macros::tas; +use tempfile::TempDir; + +use crate::pma_regressions::pma_meta::PmaPersistMetadataForTest; + +const PMA_INITIAL_OVERRIDE_ENV: &str = "NOCK_PMA_INITIAL_WORDS_FOR_REGRESSION"; +const PMA_RESERVED_WORDS_ENV: &str = "NOCK_PMA_RESERVED_WORDS"; +const PMA_GROWTH_EVENTS_ENV: &str = "NOCK_PMA_GROWTH_EVENTS_PATH"; +const LARGE_BOOTSTRAP_INITIAL_WORDS: usize = 1024; +const LARGE_BOOTSTRAP_RESERVED_WORDS: usize = 1 << 22; +const LARGE_BOOTSTRAP_MIN_GROWTH_EVENTS: usize = 2; + +#[derive(QueryableByName)] +struct I64ValueRow { + #[diesel(sql_type = BigInt)] + value: i64, +} + +pub(crate) fn run_regression() -> Result<(), Box> { + nockvm::check_endian(); + let runtime = tokio::runtime::Builder::new_current_thread() + .enable_all() + .build()?; + runtime.block_on(run()) +} + +async fn run() -> Result<(), Box> { + let temp = TempDir::new()?; + let data_dir = temp.path().join("pma-checkpoint-bootstrap-size-regression"); + let jam = load_test_jam()?; + + println!("stage 1: boot tiny PMA, checkpoint event 1, then commit event 2"); + let mut app = boot_app(&jam, &data_dir, NockStackSize::Tiny).await?; + poke_inc(&mut app).await?; + assert_counter_state(&mut app, 1).await?; + let checkpoint_path = write_checkpoint_from_export(&mut app, &data_dir).await?; + println!( + "wrote valid event-1 checkpoint at {}", + checkpoint_path.display() + ); + poke_inc(&mut app).await?; + assert_counter_state(&mut app, 2).await?; + stop_app(app).await?; + + assert_eq!(sqlite_max_event_num(&data_dir)?, 2); + assert_eq!(max_runtime_pma_meta_event(&data_dir)?, 2); + delete_snapshot_rows_and_artifacts(&data_dir)?; + clear_runtime_pma_files(&data_dir)?; + println!("stage 2: boot from checkpoint base with small PMA config and replay SQLite tail"); + + let mut recovered = boot_app(&jam, &data_dir, NockStackSize::Small).await?; + assert_counter_state(&mut recovered, 2).await?; + assert_eq!(sqlite_max_event_num(&data_dir)?, 2); + assert_eq!(max_runtime_pma_meta_event(&data_dir)?, 2); + + let active_pma = active_runtime_pma(&data_dir)?; + let active_metadata = Pma::read_file_metadata(&active_pma)?; + println!( + "checkpoint-restored active PMA: path={} data_words={} alloc_words={} free_words={}", + active_pma.display(), + active_metadata.data_words, + active_metadata.alloc_words, + active_metadata + .data_words + .saturating_sub(active_metadata.alloc_words) + ); + if active_metadata.data_words < NOCK_STACK_SIZE_SMALL as u64 { + return Err(std::io::Error::other(format!( + "checkpoint recovery replayed to SQLite max but did not create the configured larger PMA: required_min_words={} actual_words={}", + NOCK_STACK_SIZE_SMALL, active_metadata.data_words + )) + .into()); + } + + stop_app(recovered).await?; + println!("checkpoint recovery created configured larger PMA and replayed to SQLite max"); + + run_large_checkpoint_bootstrap_growth_subcase(&jam).await?; + Ok(()) +} + +async fn run_large_checkpoint_bootstrap_growth_subcase(jam: &[u8]) -> Result<(), Box> { + let temp = TempDir::new()?; + let data_dir = temp + .path() + .join("pma-checkpoint-large-bootstrap-growth-regression"); + println!("stage 3: checkpoint bootstrap from tiny initial PMA with large virtual reservation"); + + let mut app = boot_app(jam, &data_dir, NockStackSize::Tiny).await?; + poke_inc(&mut app).await?; + assert_counter_state(&mut app, 1).await?; + let checkpoint_path = write_checkpoint_from_export(&mut app, &data_dir).await?; + poke_inc(&mut app).await?; + assert_counter_state(&mut app, 2).await?; + stop_app(app).await?; + + assert_eq!(sqlite_max_event_num(&data_dir)?, 2); + delete_snapshot_rows_and_artifacts(&data_dir)?; + clear_runtime_pma_files(&data_dir)?; + + let growth_events_path = data_dir.join("large-bootstrap-growth-events.log"); + let _initial_guard = EnvVarGuard::set( + PMA_INITIAL_OVERRIDE_ENV, + LARGE_BOOTSTRAP_INITIAL_WORDS.to_string(), + ); + let _reserved_guard = EnvVarGuard::set( + PMA_RESERVED_WORDS_ENV, + LARGE_BOOTSTRAP_RESERVED_WORDS.to_string(), + ); + let _events_guard = EnvVarGuard::set( + PMA_GROWTH_EVENTS_ENV, + growth_events_path.as_os_str().to_os_string(), + ); + + let mut recovered = boot_app(jam, &data_dir, NockStackSize::Tiny).await?; + assert_counter_state(&mut recovered, 2).await?; + assert_eq!(sqlite_max_event_num(&data_dir)?, 2); + assert_eq!(max_runtime_pma_meta_event(&data_dir)?, 2); + + let active_pma = active_runtime_pma(&data_dir)?; + let active_meta = PmaPersistMetadataForTest::load(&active_pma.with_extension("meta"))?; + let metadata = Pma::read_file_metadata(&active_pma)?; + if active_meta.pma_reserved_words != Some(metadata.reserved_words) { + return Err(std::io::Error::other(format!( + "PMA sidecar reservation did not match trailer: sidecar={:?} trailer={}", + active_meta.pma_reserved_words, metadata.reserved_words + )) + .into()); + } + let growth_events = read_growth_events(&growth_events_path)?; + println!( + "large checkpoint bootstrap active PMA: checkpoint={} path={} capacity_words={} alloc_words={} free_words={} reserved_words={} growth_events={}", + checkpoint_path.display(), + active_pma.display(), + metadata.capacity_words, + metadata.alloc_words, + metadata.free_words, + metadata.reserved_words, + growth_events.len() + ); + for event in &growth_events { + println!("large checkpoint bootstrap growth event: {event}"); + } + + let ten_x_initial = u64::try_from(LARGE_BOOTSTRAP_INITIAL_WORDS * 10)?; + if metadata.alloc_words <= ten_x_initial { + return Err(std::io::Error::other(format!( + "checkpoint bootstrap PMA allocation was not more than 10x the initial capacity: initial_words={} alloc_words={}", + LARGE_BOOTSTRAP_INITIAL_WORDS, metadata.alloc_words + )) + .into()); + } + if growth_events.len() < LARGE_BOOTSTRAP_MIN_GROWTH_EVENTS { + return Err(std::io::Error::other(format!( + "expected at least {LARGE_BOOTSTRAP_MIN_GROWTH_EVENTS} PMA growth events during large checkpoint bootstrap, got {}", + growth_events.len() + )) + .into()); + } + let reserved_bytes = u64::try_from(LARGE_BOOTSTRAP_RESERVED_WORDS)? + .checked_mul(8) + .ok_or_else(|| std::io::Error::other("reserved byte calculation overflowed"))?; + if metadata.apparent_file_bytes >= reserved_bytes { + return Err(std::io::Error::other(format!( + "large checkpoint bootstrap materialized the reserved maximum: apparent_file_bytes={} reserved_bytes={reserved_bytes}", + metadata.apparent_file_bytes + )) + .into()); + } + + stop_app(recovered).await?; + println!( + "checkpoint bootstrap grew a PMA payload more than 10x its initial capacity without retrying copy_to_pma" + ); + Ok(()) +} + +fn load_test_jam() -> Result, Box> { + let mut possible_paths = Vec::new(); + if let Some(manifest_dir) = option_env!("CARGO_MANIFEST_DIR") { + possible_paths.push( + Path::new(manifest_dir) + .join("test-jams") + .join("test-ker.jam"), + ); + } + possible_paths.push(Path::new("open/crates/nockapp/test-jams").join("test-ker.jam")); + possible_paths.push(Path::new("test-jams").join("test-ker.jam")); + + for path in &possible_paths { + if let Ok(bytes) = fs::read(path) { + return Ok(bytes); + } + } + + Err(std::io::Error::other(format!( + "failed to read test-ker.jam from any candidate path: {:?}", + possible_paths + )) + .into()) +} + +async fn boot_app( + jam: &[u8], + data_dir: &Path, + stack_size: NockStackSize, +) -> Result, Box> { + let mut cli = default_boot_cli(false); + cli.data_dir = Some(data_dir.to_path_buf()); + cli.pma_initial_size = Some(PmaSize::from_words(stack_size.stack_words())); + cli.stack_size = stack_size; + cli.gc_interval = None; + cli.rotating_snapshot_interval_event_time = None; + cli.disable_fsync = true; + match setup_::( + jam, + cli, + &[], + "pma-checkpoint-bootstrap-size-regression", + None, + ) + .await? + { + SetupResult::App(app) => Ok(app), + SetupResult::ExportedState => Err(std::io::Error::other("unexpected state export").into()), + } +} + +fn inc_poke() -> NounSlab { + let mut slab = NounSlab::new(); + let space = NounSpace::empty(); + slab.copy_into(D(tas!(b"inc")), &space); + slab +} + +fn state_peek() -> NounSlab { + let mut slab = NounSlab::new(); + let peek = T(&mut slab, &[D(tas!(b"state")), D(0)]); + slab.set_root(peek); + slab +} + +fn empty_cold_slab() -> NounSlab { + let mut slab = NounSlab::new(); + let root = T(&mut slab, &[D(0), D(0), D(0)]); + slab.set_root(root); + slab +} + +async fn write_checkpoint_from_export( + app: &mut NockApp, + data_dir: &Path, +) -> Result> { + let exported = app.export().await?; + let checkpoint = SaveableCheckpoint { + ker_hash: exported.ker_hash, + event_num: exported.event_num, + state: exported.kernel_state, + cold: empty_cold_slab(), + }; + let jammed = checkpoint.to_jammed_checkpoint::(); + let bytes = jammed.encode()?; + let checkpoint_dir = data_dir.join("checkpoints"); + fs::create_dir_all(&checkpoint_dir)?; + let checkpoint_path = checkpoint_dir.join("0.chkjam"); + fs::write(&checkpoint_path, bytes)?; + Ok(checkpoint_path) +} + +async fn poke_inc(app: &mut NockApp) -> Result<(), Box> { + app.poke(SystemWire.to_wire(), inc_poke()).await?; + Ok(()) +} + +async fn assert_counter_state( + app: &mut NockApp, + expected: u64, +) -> Result<(), Box> { + let exported = app.export().await?; + if exported.event_num != expected { + return Err(std::io::Error::other(format!( + "event number mismatch: expected={expected} actual={}", + exported.event_num + )) + .into()); + } + let actual = app.peek(state_peek()).await?; + let space = actual.noun_space(); + let root = unsafe { *actual.root() }; + let value = root + .in_space(&space) + .slot(7)? + .as_cell()? + .tail() + .noun() + .in_space(&space) + .as_atom()? + .as_u64()?; + if value != expected { + return Err(std::io::Error::other(format!( + "counter state mismatch: expected={expected} actual={value}" + )) + .into()); + } + Ok(()) +} + +async fn stop_app(mut app: NockApp) -> Result<(), Box> { + let handle = app.get_handle(); + handle.exit.exit(0).await?; + app.run().await?; + Ok(()) +} + +fn sqlite_connection(data_dir: &Path) -> Result> { + let path = data_dir.join("event-log.sqlite3"); + Ok(SqliteConnection::establish(path.to_str().ok_or_else( + || std::io::Error::other(format!("non-utf8 sqlite path: {path:?}")), + )?)?) +} + +fn sqlite_max_event_num(data_dir: &Path) -> Result> { + let mut conn = sqlite_connection(data_dir)?; + let row = sql_query("SELECT COALESCE(MAX(event_num), 0) AS value FROM events") + .get_result::(&mut conn)?; + Ok(u64::try_from(row.value)?) +} + +fn max_runtime_pma_meta_event(data_dir: &Path) -> Result> { + let pma_dir = data_dir.join("pma"); + let mut max_event = None; + for idx in [0, 1] { + let meta_path = pma_dir.join(format!("{idx}.meta")); + if !meta_path.exists() { + continue; + } + let meta = PmaPersistMetadataForTest::load(&meta_path)?; + max_event = Some(max_event.map_or(meta.event_num, |event: u64| event.max(meta.event_num))); + } + max_event.ok_or_else(|| { + std::io::Error::other(format!( + "no runtime PMA metadata found in {}", + pma_dir.display() + )) + .into() + }) +} + +fn active_runtime_pma(data_dir: &Path) -> Result> { + let pma_dir = data_dir.join("pma"); + let mut candidates = Vec::new(); + for idx in [0, 1] { + let pma_path = pma_dir.join(format!("{idx}.pma")); + let meta_path = pma_dir.join(format!("{idx}.meta")); + if pma_path.exists() && meta_path.exists() { + let meta = PmaPersistMetadataForTest::load(&meta_path)?; + let modified = fs::metadata(&meta_path) + .and_then(|metadata| metadata.modified()) + .unwrap_or(std::time::SystemTime::UNIX_EPOCH); + candidates.push((pma_path, meta.event_num, modified)); + } + } + candidates.sort_by_key(|(_, event_num, modified)| (*event_num, *modified)); + candidates.pop().map(|(path, _, _)| path).ok_or_else(|| { + std::io::Error::other(format!( + "no meta-paired runtime PMA found in {}", + pma_dir.display() + )) + .into() + }) +} + +fn read_growth_events(path: &Path) -> Result, Box> { + if !path.exists() { + return Ok(Vec::new()); + } + Ok(fs::read_to_string(path)? + .lines() + .map(str::trim) + .filter(|line| !line.is_empty()) + .map(str::to_owned) + .collect()) +} + +struct EnvVarGuard { + key: &'static str, + previous: Option, +} + +impl EnvVarGuard { + fn set(key: &'static str, value: impl Into) -> Self { + let previous = std::env::var_os(key); + std::env::set_var(key, value.into()); + Self { key, previous } + } +} + +impl Drop for EnvVarGuard { + fn drop(&mut self) { + if let Some(previous) = &self.previous { + std::env::set_var(self.key, previous); + } else { + std::env::remove_var(self.key); + } + } +} + +fn clear_runtime_pma_files(data_dir: &Path) -> Result<(), Box> { + let pma_dir = data_dir.join("pma"); + for file_name in ["0.pma", "1.pma", "0.meta", "1.meta"] { + let path = pma_dir.join(file_name); + if path.exists() { + fs::remove_file(path)?; + } + } + Ok(()) +} + +fn delete_snapshot_rows_and_artifacts(data_dir: &Path) -> Result<(), Box> { + let pma_dir = data_dir.join("pma"); + let mut conn = sqlite_connection(data_dir)?; + sql_query("DELETE FROM snapshots").execute(&mut conn)?; + sql_query("DELETE FROM meta WHERE key = 'active_snapshot_id'").execute(&mut conn)?; + drop(conn); + + if !pma_dir.exists() { + return Ok(()); + } + for entry in fs::read_dir(&pma_dir)? { + let path = entry?.path(); + if !path.is_file() { + continue; + } + let Some(name) = path.file_name().and_then(|name| name.to_str()) else { + continue; + }; + if name == "epoch.pma" + || name == "epoch.manifest" + || name == "epoch.pma.tmp" + || name == "epoch.manifest.tmp" + || name.starts_with("snap-") + { + fs::remove_file(path)?; + } + } + Ok(()) +} diff --git a/crates/nockapp/tests/pma_regressions/event_preflight_growth.rs b/crates/nockapp/tests/pma_regressions/event_preflight_growth.rs new file mode 100644 index 000000000..a1ba147f1 --- /dev/null +++ b/crates/nockapp/tests/pma_regressions/event_preflight_growth.rs @@ -0,0 +1,290 @@ +use std::error::Error; +use std::fs; +use std::path::{Path, PathBuf}; + +use diesel::prelude::*; +use diesel::sql_query; +use diesel::sql_types::BigInt; +use diesel::sqlite::SqliteConnection; +use nockapp::kernel::boot::{default_boot_cli, setup_, NockStackSize, PmaSize, SetupResult}; +use nockapp::nockapp::wire::{SystemWire, Wire}; +use nockapp::noun::slab::{NockJammer, NounSlab}; +use nockapp::NockApp; +use nockvm::mem::NOCK_STACK_SIZE_TINY; +use nockvm::noun::{NounAllocator, NounSpace, D, T}; +use nockvm::offset::PmaOffsetWords; +use nockvm::pma::Pma; +use nockvm_macros::tas; +use tempfile::TempDir; + +// Leave roughly one existing PMA allocation worth of free space plus this small headroom. Boot can re-preserve the loaded state, then the next event has too little free space unless online growth runs before the durable append path. +const LOW_FREE_FIXTURE_EXTRA_WORDS: u64 = 700; + +#[derive(QueryableByName)] +struct I64ValueRow { + #[diesel(sql_type = BigInt)] + value: i64, +} + +pub(crate) fn run_regression() -> Result<(), Box> { + nockvm::check_endian(); + let runtime = tokio::runtime::Builder::new_current_thread() + .enable_all() + .build()?; + runtime.block_on(run()) +} + +async fn run() -> Result<(), Box> { + let temp = TempDir::new()?; + let data_dir = temp.path().join("pma-event-preflight-growth-regression"); + let jam = load_test_jam()?; + + println!("stage 1: create durable event-1 fixture"); + let mut first = boot_app(&jam, &data_dir).await?; + poke_inc(&mut first).await?; + assert_counter_state(&mut first, 1).await?; + stop_app(first).await?; + assert_eq!(sqlite_max_event_num(&data_dir)?, 1); + + let active_pma = active_runtime_pma(&data_dir)?; + let before_force = Pma::read_file_metadata(&active_pma)?; + println!( + "active PMA before filler: path={} data_words={} alloc_words={} free_words={}", + active_pma.display(), + before_force.data_words, + before_force.alloc_words, + before_force + .data_words + .saturating_sub(before_force.alloc_words) + ); + let forced_free_words = before_force + .alloc_words + .checked_add(LOW_FREE_FIXTURE_EXTRA_WORDS) + .ok_or_else(|| std::io::Error::other("forced free word calculation overflowed"))?; + force_pma_free_words(&active_pma, forced_free_words)?; + let forced = Pma::read_file_metadata(&active_pma)?; + println!( + "active PMA forced low-free before event: path={} data_words={} alloc_words={} free_words={} fixture_extra_words={}", + active_pma.display(), + forced.data_words, + forced.alloc_words, + forced.data_words.saturating_sub(forced.alloc_words), + LOW_FREE_FIXTURE_EXTRA_WORDS + ); + + println!("stage 2: reboot low-free fixture and issue one accepted inc event"); + let mut second = boot_app(&jam, &data_dir).await?; + assert_counter_state(&mut second, 1).await?; + let sqlite_before = sqlite_max_event_num(&data_dir)?; + let pma_before_event = active_runtime_pma(&data_dir)?; + let pma_before_event_metadata = Pma::read_file_metadata(&pma_before_event)?; + println!( + "before event: sqlite_max={} active_pma={} data_words={} alloc_words={} free_words={}", + sqlite_before, + pma_before_event.display(), + pma_before_event_metadata.data_words, + pma_before_event_metadata.alloc_words, + pma_before_event_metadata + .data_words + .saturating_sub(pma_before_event_metadata.alloc_words) + ); + + let poke_result = second.poke(SystemWire.to_wire(), inc_poke()).await; + let sqlite_after = sqlite_max_event_num(&data_dir)?; + match poke_result { + Ok(_) => { + assert_counter_state(&mut second, 2).await?; + let pma_after_event = active_runtime_pma(&data_dir)?; + let pma_after_event_metadata = Pma::read_file_metadata(&pma_after_event)?; + println!( + "after event: sqlite_max={} active_pma={} data_words={} alloc_words={} free_words={}", + sqlite_after, + pma_after_event.display(), + pma_after_event_metadata.data_words, + pma_after_event_metadata.alloc_words, + pma_after_event_metadata + .data_words + .saturating_sub(pma_after_event_metadata.alloc_words) + ); + if sqlite_after != sqlite_before + 1 { + return Err(std::io::Error::other(format!( + "accepted event did not advance SQLite exactly once: before={sqlite_before} after={sqlite_after}" + )) + .into()); + } + if pma_after_event_metadata.data_words <= pma_before_event_metadata.data_words { + return Err(std::io::Error::other(format!( + "event succeeded but active PMA was not grown from low-free fixture: before_data_words={} after_data_words={}", + pma_before_event_metadata.data_words, pma_after_event_metadata.data_words + )) + .into()); + } + stop_app(second).await?; + println!("event-time PMA preflight grew PMA before accepting/logging the event"); + Ok(()) + } + Err(err) => { + let message = err.to_string(); + if sqlite_after > sqlite_before { + return Err(std::io::Error::other(format!( + "event log advanced despite failed low-free PMA event: sqlite_before={sqlite_before} sqlite_after={sqlite_after} poke_error={message}" + )) + .into()); + } + Err(std::io::Error::other(format!( + "low-free PMA event was rejected before SQLite append; this is fail-safe but does not satisfy growth regression: sqlite_before={sqlite_before} sqlite_after={sqlite_after} poke_error={message}" + )) + .into()) + } + } +} + +fn load_test_jam() -> Result, Box> { + let mut possible_paths = Vec::new(); + if let Some(manifest_dir) = option_env!("CARGO_MANIFEST_DIR") { + possible_paths.push( + Path::new(manifest_dir) + .join("test-jams") + .join("test-ker.jam"), + ); + } + possible_paths.push(Path::new("open/crates/nockapp/test-jams").join("test-ker.jam")); + possible_paths.push(Path::new("test-jams").join("test-ker.jam")); + + for path in &possible_paths { + if let Ok(bytes) = fs::read(path) { + return Ok(bytes); + } + } + + Err(std::io::Error::other(format!( + "failed to read test-ker.jam from any candidate path: {:?}", + possible_paths + )) + .into()) +} + +async fn boot_app(jam: &[u8], data_dir: &Path) -> Result, Box> { + let mut cli = default_boot_cli(false); + cli.data_dir = Some(data_dir.to_path_buf()); + cli.pma_initial_size = Some(PmaSize::from_words(NOCK_STACK_SIZE_TINY)); + cli.stack_size = NockStackSize::Tiny; + cli.gc_interval = None; + cli.rotating_snapshot_interval_event_time = None; + cli.disable_fsync = true; + match setup_::(jam, cli, &[], "pma-event-preflight-growth-regression", None).await? + { + SetupResult::App(app) => Ok(app), + SetupResult::ExportedState => Err(std::io::Error::other("unexpected state export").into()), + } +} + +fn inc_poke() -> NounSlab { + let mut slab = NounSlab::new(); + let space = NounSpace::empty(); + slab.copy_into(D(tas!(b"inc")), &space); + slab +} + +fn state_peek() -> NounSlab { + let mut slab = NounSlab::new(); + let peek = T(&mut slab, &[D(tas!(b"state")), D(0)]); + slab.set_root(peek); + slab +} + +async fn poke_inc(app: &mut NockApp) -> Result<(), Box> { + app.poke(SystemWire.to_wire(), inc_poke()).await?; + Ok(()) +} + +async fn assert_counter_state( + app: &mut NockApp, + expected: u64, +) -> Result<(), Box> { + let exported = app.export().await?; + if exported.event_num != expected { + return Err(std::io::Error::other(format!( + "event number mismatch: expected={expected} actual={}", + exported.event_num + )) + .into()); + } + let actual = app.peek(state_peek()).await?; + let space = actual.noun_space(); + let root = unsafe { *actual.root() }; + let value = root + .in_space(&space) + .slot(7)? + .as_cell()? + .tail() + .noun() + .in_space(&space) + .as_atom()? + .as_u64()?; + if value != expected { + return Err(std::io::Error::other(format!( + "counter state mismatch: expected={expected} actual={value}" + )) + .into()); + } + Ok(()) +} + +async fn stop_app(mut app: NockApp) -> Result<(), Box> { + let handle = app.get_handle(); + handle.exit.exit(0).await?; + app.run().await?; + Ok(()) +} + +fn sqlite_max_event_num(data_dir: &Path) -> Result> { + let path = data_dir.join("event-log.sqlite3"); + let mut conn = SqliteConnection::establish( + path.to_str() + .ok_or_else(|| std::io::Error::other(format!("non-utf8 sqlite path: {path:?}")))?, + )?; + let row = sql_query("SELECT COALESCE(MAX(event_num), 0) AS value FROM events") + .get_result::(&mut conn)?; + Ok(u64::try_from(row.value)?) +} + +fn force_pma_free_words(path: &Path, free_words: u64) -> Result<(), Box> { + let metadata = Pma::read_file_metadata(path)?; + if metadata.data_words <= free_words { + return Err(std::io::Error::other(format!( + "PMA too small to force {free_words} free words: data_words={}", + metadata.data_words + )) + .into()); + } + let new_alloc_words = metadata.data_words - free_words; + let mut pma = Pma::open(path.to_path_buf())?; + pma.reset_to(PmaOffsetWords::from_words(new_alloc_words)); + pma.sync_trailer()?; + pma.sync_file()?; + Ok(()) +} + +fn active_runtime_pma(data_dir: &Path) -> Result> { + let pma_dir = data_dir.join("pma"); + let mut candidates = Vec::new(); + for idx in [0, 1] { + let pma_path = pma_dir.join(format!("{idx}.pma")); + let meta_path = pma_dir.join(format!("{idx}.meta")); + if pma_path.exists() && meta_path.exists() { + let modified = fs::metadata(&meta_path) + .and_then(|metadata| metadata.modified()) + .unwrap_or(std::time::SystemTime::UNIX_EPOCH); + candidates.push((pma_path, modified)); + } + } + candidates.sort_by_key(|(_, modified)| *modified); + candidates.pop().map(|(path, _)| path).ok_or_else(|| { + std::io::Error::other(format!( + "no meta-paired runtime PMA found in {}", + pma_dir.display() + )) + .into() + }) +} diff --git a/crates/nockapp/tests/pma_regressions/event_resize_failure_boundary.rs b/crates/nockapp/tests/pma_regressions/event_resize_failure_boundary.rs new file mode 100644 index 000000000..a687aa268 --- /dev/null +++ b/crates/nockapp/tests/pma_regressions/event_resize_failure_boundary.rs @@ -0,0 +1,345 @@ +use std::error::Error; +use std::ffi::OsString; +use std::fs; +use std::path::{Path, PathBuf}; + +use diesel::prelude::*; +use diesel::sql_query; +use diesel::sql_types::BigInt; +use diesel::sqlite::SqliteConnection; +use nockapp::kernel::boot::{default_boot_cli, setup_, NockStackSize, PmaSize, SetupResult}; +use nockapp::nockapp::wire::{SystemWire, Wire}; +use nockapp::noun::slab::{NockJammer, NounSlab}; +use nockapp::NockApp; +use nockvm::mem::NOCK_STACK_SIZE_TINY; +use nockvm::noun::{NounAllocator, NounSpace, D, T}; +use nockvm::offset::PmaOffsetWords; +use nockvm::pma::Pma; +use nockvm_macros::tas; +use tempfile::TempDir; + +use crate::pma_regressions::pma_meta::PmaPersistMetadataForTest; + +// Leave roughly one existing PMA allocation worth of free space plus this small headroom. Boot can re-preserve the loaded state, then the next event has too little free space unless online growth runs before the durable append path. +const LOW_FREE_FIXTURE_EXTRA_WORDS: u64 = 700; +const RESIZE_FAIL_ENV: &str = "NOCK_PMA_RESIZE_FAIL_AT"; +const RESIZE_FAIL_POINT: &str = "create_destination"; +#[derive(QueryableByName)] +struct I64ValueRow { + #[diesel(sql_type = BigInt)] + value: i64, +} + +pub(crate) fn run_regression() -> Result<(), Box> { + nockvm::check_endian(); + let runtime = tokio::runtime::Builder::new_current_thread() + .enable_all() + .build()?; + runtime.block_on(run()) +} + +async fn run() -> Result<(), Box> { + let temp = TempDir::new()?; + let data_dir = temp + .path() + .join("pma-event-resize-failure-boundary-regression"); + let jam = load_test_jam()?; + + println!("stage 1: create durable event-1 fixture"); + let mut first = boot_app(&jam, &data_dir).await?; + poke_inc(&mut first).await?; + assert_counter_state(&mut first, 1).await?; + stop_app(first).await?; + assert_eq!(sqlite_max_event_num(&data_dir)?, 1); + assert_eq!(max_runtime_pma_meta_event(&data_dir)?, 1); + + let active_pma = active_runtime_pma(&data_dir)?; + let before_force = Pma::read_file_metadata(&active_pma)?; + println!( + "active PMA before filler: path={} data_words={} alloc_words={} free_words={}", + active_pma.display(), + before_force.data_words, + before_force.alloc_words, + before_force + .data_words + .saturating_sub(before_force.alloc_words) + ); + let forced_free_words = before_force + .alloc_words + .checked_add(LOW_FREE_FIXTURE_EXTRA_WORDS) + .ok_or_else(|| std::io::Error::other("forced free word calculation overflowed"))?; + force_pma_free_words(&active_pma, forced_free_words)?; + let forced = Pma::read_file_metadata(&active_pma)?; + println!( + "active PMA forced low-free before resize-failure event: path={} data_words={} alloc_words={} free_words={} fixture_extra_words={}", + active_pma.display(), + forced.data_words, + forced.alloc_words, + forced.data_words.saturating_sub(forced.alloc_words), + LOW_FREE_FIXTURE_EXTRA_WORDS + ); + + println!("stage 2: reboot low-free fixture with deterministic resize failure enabled"); + let mut second = boot_app(&jam, &data_dir).await?; + assert_counter_state(&mut second, 1).await?; + let sqlite_before = sqlite_max_event_num(&data_dir)?; + let pma_meta_before = max_runtime_pma_meta_event(&data_dir)?; + let pma_before_event = active_runtime_pma(&data_dir)?; + let pma_before_event_metadata = Pma::read_file_metadata(&pma_before_event)?; + println!( + "before event: sqlite_max={} pma_meta_event={} active_pma={} data_words={} alloc_words={} free_words={}", + sqlite_before, + pma_meta_before, + pma_before_event.display(), + pma_before_event_metadata.data_words, + pma_before_event_metadata.alloc_words, + pma_before_event_metadata + .data_words + .saturating_sub(pma_before_event_metadata.alloc_words) + ); + + let _resize_failure = EnvVarGuard::set(RESIZE_FAIL_ENV, RESIZE_FAIL_POINT); + let poke_result = second.poke(SystemWire.to_wire(), inc_poke()).await; + let sqlite_after = sqlite_max_event_num(&data_dir)?; + let pma_meta_after = max_runtime_pma_meta_event(&data_dir)?; + + match poke_result { + Ok(_) => { + return Err(std::io::Error::other(format!( + "poke succeeded despite injected PMA resize failure at {RESIZE_FAIL_POINT}: sqlite_before={sqlite_before} sqlite_after={sqlite_after} pma_meta_before={pma_meta_before} pma_meta_after={pma_meta_after}" + )) + .into()); + } + Err(err) => { + println!("poke failed under injected resize failure: {err}"); + if sqlite_after != sqlite_before { + return Err(std::io::Error::other(format!( + "event log advanced despite injected PMA resize failure: sqlite_before={sqlite_before} sqlite_after={sqlite_after} pma_meta_before={pma_meta_before} pma_meta_after={pma_meta_after} poke_error={err}" + )) + .into()); + } + if pma_meta_after != pma_meta_before { + return Err(std::io::Error::other(format!( + "PMA metadata advanced despite injected resize failure: sqlite_before={sqlite_before} sqlite_after={sqlite_after} pma_meta_before={pma_meta_before} pma_meta_after={pma_meta_after} poke_error={err}" + )) + .into()); + } + } + } + drop(_resize_failure); + + stop_app(second).await?; + + println!("stage 3: reboot previous durable boundary after failed pre-append resize"); + let mut third = boot_app(&jam, &data_dir).await?; + assert_eq!(sqlite_max_event_num(&data_dir)?, sqlite_before); + assert_eq!(max_runtime_pma_meta_event(&data_dir)?, pma_meta_before); + assert_counter_state(&mut third, sqlite_before).await?; + stop_app(third).await?; + + println!( + "pre-append resize failure left previous SQLite/PMA boundary bootable at event {sqlite_before}" + ); + Ok(()) +} + +fn load_test_jam() -> Result, Box> { + let mut possible_paths = Vec::new(); + if let Some(manifest_dir) = option_env!("CARGO_MANIFEST_DIR") { + possible_paths.push( + Path::new(manifest_dir) + .join("test-jams") + .join("test-ker.jam"), + ); + } + possible_paths.push(Path::new("open/crates/nockapp/test-jams").join("test-ker.jam")); + possible_paths.push(Path::new("test-jams").join("test-ker.jam")); + + for path in &possible_paths { + if let Ok(bytes) = fs::read(path) { + return Ok(bytes); + } + } + + Err(std::io::Error::other(format!( + "failed to read test-ker.jam from any candidate path: {:?}", + possible_paths + )) + .into()) +} + +async fn boot_app(jam: &[u8], data_dir: &Path) -> Result, Box> { + let mut cli = default_boot_cli(false); + cli.data_dir = Some(data_dir.to_path_buf()); + cli.pma_initial_size = Some(PmaSize::from_words(NOCK_STACK_SIZE_TINY)); + cli.stack_size = NockStackSize::Tiny; + cli.gc_interval = None; + cli.rotating_snapshot_interval_event_time = None; + cli.disable_fsync = true; + match setup_::( + jam, + cli, + &[], + "pma-event-resize-failure-boundary-regression", + None, + ) + .await? + { + SetupResult::App(app) => Ok(app), + SetupResult::ExportedState => Err(std::io::Error::other("unexpected state export").into()), + } +} + +fn inc_poke() -> NounSlab { + let mut slab = NounSlab::new(); + let space = NounSpace::empty(); + slab.copy_into(D(tas!(b"inc")), &space); + slab +} + +fn state_peek() -> NounSlab { + let mut slab = NounSlab::new(); + let peek = T(&mut slab, &[D(tas!(b"state")), D(0)]); + slab.set_root(peek); + slab +} + +async fn poke_inc(app: &mut NockApp) -> Result<(), Box> { + app.poke(SystemWire.to_wire(), inc_poke()).await?; + Ok(()) +} + +async fn assert_counter_state( + app: &mut NockApp, + expected: u64, +) -> Result<(), Box> { + let exported = app.export().await?; + if exported.event_num != expected { + return Err(std::io::Error::other(format!( + "event number mismatch: expected={expected} actual={}", + exported.event_num + )) + .into()); + } + let actual = app.peek(state_peek()).await?; + let space = actual.noun_space(); + let root = unsafe { *actual.root() }; + let value = root + .in_space(&space) + .slot(7)? + .as_cell()? + .tail() + .noun() + .in_space(&space) + .as_atom()? + .as_u64()?; + if value != expected { + return Err(std::io::Error::other(format!( + "counter state mismatch: expected={expected} actual={value}" + )) + .into()); + } + Ok(()) +} + +async fn stop_app(mut app: NockApp) -> Result<(), Box> { + let handle = app.get_handle(); + handle.exit.exit(0).await?; + app.run().await?; + Ok(()) +} + +fn sqlite_max_event_num(data_dir: &Path) -> Result> { + let path = data_dir.join("event-log.sqlite3"); + let mut conn = SqliteConnection::establish( + path.to_str() + .ok_or_else(|| std::io::Error::other(format!("non-utf8 sqlite path: {path:?}")))?, + )?; + let row = sql_query("SELECT COALESCE(MAX(event_num), 0) AS value FROM events") + .get_result::(&mut conn)?; + Ok(u64::try_from(row.value)?) +} + +fn max_runtime_pma_meta_event(data_dir: &Path) -> Result> { + let pma_dir = data_dir.join("pma"); + let mut max_event = None; + for idx in [0, 1] { + let meta_path = pma_dir.join(format!("{idx}.meta")); + if !meta_path.exists() { + continue; + } + let meta = PmaPersistMetadataForTest::load(&meta_path)?; + max_event = Some(max_event.map_or(meta.event_num, |event: u64| event.max(meta.event_num))); + } + max_event.ok_or_else(|| { + std::io::Error::other(format!( + "no runtime PMA metadata found in {}", + pma_dir.display() + )) + .into() + }) +} + +fn force_pma_free_words(path: &Path, free_words: u64) -> Result<(), Box> { + let metadata = Pma::read_file_metadata(path)?; + if metadata.data_words <= free_words { + return Err(std::io::Error::other(format!( + "PMA too small to force {free_words} free words: data_words={}", + metadata.data_words + )) + .into()); + } + let new_alloc_words = metadata.data_words - free_words; + let mut pma = Pma::open(path.to_path_buf())?; + pma.reset_to(PmaOffsetWords::from_words(new_alloc_words)); + pma.sync_trailer()?; + pma.sync_file()?; + Ok(()) +} + +fn active_runtime_pma(data_dir: &Path) -> Result> { + let pma_dir = data_dir.join("pma"); + let mut candidates = Vec::new(); + for idx in [0, 1] { + let pma_path = pma_dir.join(format!("{idx}.pma")); + let meta_path = pma_dir.join(format!("{idx}.meta")); + if pma_path.exists() && meta_path.exists() { + let meta = PmaPersistMetadataForTest::load(&meta_path)?; + let modified = fs::metadata(&meta_path) + .and_then(|metadata| metadata.modified()) + .unwrap_or(std::time::SystemTime::UNIX_EPOCH); + candidates.push((pma_path, meta.event_num, modified)); + } + } + candidates.sort_by_key(|(_, event_num, modified)| (*event_num, *modified)); + candidates.pop().map(|(path, _, _)| path).ok_or_else(|| { + std::io::Error::other(format!( + "no meta-paired runtime PMA found in {}", + pma_dir.display() + )) + .into() + }) +} + +struct EnvVarGuard { + key: &'static str, + previous: Option, +} + +impl EnvVarGuard { + fn set(key: &'static str, value: &str) -> Self { + let previous = std::env::var_os(key); + std::env::set_var(key, value); + Self { key, previous } + } +} + +impl Drop for EnvVarGuard { + fn drop(&mut self) { + if let Some(previous) = self.previous.take() { + std::env::set_var(self.key, previous); + } else { + std::env::remove_var(self.key); + } + } +} diff --git a/crates/nockapp/tests/pma_regressions/failed_preserve_recovery.rs b/crates/nockapp/tests/pma_regressions/failed_preserve_recovery.rs new file mode 100644 index 000000000..9fb573338 --- /dev/null +++ b/crates/nockapp/tests/pma_regressions/failed_preserve_recovery.rs @@ -0,0 +1,360 @@ +use std::error::Error; +use std::path::{Path, PathBuf}; +use std::process::Command; +use std::{env, fs}; + +use diesel::prelude::*; +use diesel::sql_query; +use diesel::sql_types::BigInt; +use diesel::sqlite::SqliteConnection; +use nockapp::kernel::boot::{default_boot_cli, setup_, NockStackSize, PmaSize, SetupResult}; +use nockapp::nockapp::wire::{SystemWire, Wire}; +use nockapp::noun::slab::{NockJammer, NounSlab}; +use nockapp::NockApp; +use nockvm::mem::NOCK_STACK_SIZE_SMALL; +use nockvm::noun::{NounAllocator, NounSpace, D, T}; +use nockvm::offset::PmaOffsetWords; +use nockvm::pma::Pma; +use nockvm_macros::tas; +use tempfile::TempDir; + +use crate::pma_regressions::pma_meta::PmaPersistMetadataForTest; + +const CHILD_ENV: &str = "NOCK_PMA_FAILED_PRESERVE_CHILD"; +const DATA_DIR_ENV: &str = "NOCK_PMA_FAILED_PRESERVE_DATA_DIR"; +const DISABLE_RESIZE_ENV: &str = "NOCK_PMA_DISABLE_RESIZE_FOR_REGRESSION"; +const CHILD_TEST_NAME: &str = "pma_failed_preserve_recovery_regression"; +#[derive(QueryableByName)] +struct I64ValueRow { + #[diesel(sql_type = BigInt)] + value: i64, +} + +pub(crate) fn run_regression() -> Result<(), Box> { + nockvm::check_endian(); + let runtime = tokio::runtime::Builder::new_current_thread() + .enable_all() + .build()?; + if env::var_os(CHILD_ENV).is_some() { + let data_dir = env::var_os(DATA_DIR_ENV) + .map(PathBuf::from) + .ok_or_else(|| std::io::Error::other(format!("{DATA_DIR_ENV} not set")))?; + return runtime.block_on(child_no_resize_boot_attempt(&data_dir)); + } + runtime.block_on(run_parent()) +} + +async fn run_parent() -> Result<(), Box> { + let temp = TempDir::new()?; + let data_dir = temp.path().join("pma-failed-preserve-recovery-regression"); + let jam = load_test_jam()?; + + println!("stage 1: create durable event-1 active PMA fixture"); + let mut first = boot_app(&jam, &data_dir, NockStackSize::Tiny).await?; + poke_inc(&mut first).await?; + assert_counter_state(&mut first, 1).await?; + stop_app(first).await?; + assert_eq!(sqlite_max_event_num(&data_dir)?, 1); + assert_eq!(max_runtime_pma_meta_event(&data_dir)?, 1); + + let active_pma = active_runtime_pma(&data_dir)?; + let before_force = Pma::read_file_metadata(&active_pma)?; + println!( + "active PMA before low-free fixture: path={} data_words={} alloc_words={} free_words={}", + active_pma.display(), + before_force.data_words, + before_force.alloc_words, + before_force + .data_words + .saturating_sub(before_force.alloc_words) + ); + + let forced_free_words = before_force.alloc_words; + force_pma_free_words(&active_pma, forced_free_words)?; + let forced = Pma::read_file_metadata(&active_pma)?; + println!( + "active PMA forced for partial preserve failure: path={} data_words={} alloc_words={} free_words={}", + active_pma.display(), + forced.data_words, + forced.alloc_words, + forced.data_words.saturating_sub(forced.alloc_words) + ); + + println!("stage 2: run failing no-resize boot attempt in child process"); + let child_output = Command::new(env::current_exe()?) + .arg("--exact") + .arg(CHILD_TEST_NAME) + .arg("--nocapture") + .env(CHILD_ENV, "1") + .env(DATA_DIR_ENV, &data_dir) + .env(DISABLE_RESIZE_ENV, "1") + .output()?; + let child_stdout = String::from_utf8_lossy(&child_output.stdout); + let child_stderr = String::from_utf8_lossy(&child_output.stderr); + if !child_stdout.trim().is_empty() { + println!("child stdout:\n{child_stdout}"); + } + if !child_stderr.trim().is_empty() { + println!("child stderr:\n{child_stderr}"); + } + let combined = format!("{child_stdout}\n{child_stderr}"); + if child_output.status.success() { + return Err(std::io::Error::other( + "no-resize child boot unexpectedly succeeded; this regression requires a failing first boot attempt", + ) + .into()); + } + if !combined.contains("PMA is full") { + return Err(std::io::Error::other(format!( + "no-resize child boot failed for the wrong reason: status={} output={combined}", + child_output.status + )) + .into()); + } + println!("child reproduced initialization preserve PMA OOM"); + + let after_child = Pma::read_file_metadata(&active_pma)?; + println!( + "active PMA after failed child boot: path={} data_words={} alloc_words={} free_words={} alloc_delta={}", + active_pma.display(), + after_child.data_words, + after_child.alloc_words, + after_child.data_words.saturating_sub(after_child.alloc_words), + after_child.alloc_words.saturating_sub(forced.alloc_words) + ); + assert_eq!(sqlite_max_event_num(&data_dir)?, 1); + assert_eq!(max_runtime_pma_meta_event(&data_dir)?, 1); + + println!("stage 3: fixed production boot must grow or recover after failed preserve"); + match boot_app(&jam, &data_dir, NockStackSize::Small).await { + Ok(mut recovered) => { + assert_counter_state(&mut recovered, 1).await?; + assert_eq!(sqlite_max_event_num(&data_dir)?, 1); + assert_eq!(max_runtime_pma_meta_event(&data_dir)?, 1); + let recovered_pma = active_runtime_pma(&data_dir)?; + let recovered_metadata = Pma::read_file_metadata(&recovered_pma)?; + println!( + "recovered active PMA: path={} data_words={} alloc_words={} free_words={}", + recovered_pma.display(), + recovered_metadata.data_words, + recovered_metadata.alloc_words, + recovered_metadata + .data_words + .saturating_sub(recovered_metadata.alloc_words) + ); + if recovered_metadata.data_words < NOCK_STACK_SIZE_SMALL as u64 { + return Err(std::io::Error::other(format!( + "recovery boot succeeded but active PMA was not grown to configured small size: required_min_words={} actual_words={}", + NOCK_STACK_SIZE_SMALL, recovered_metadata.data_words + )) + .into()); + } + stop_app(recovered).await?; + println!("failed initialization preserve did not poison later recovery"); + Ok(()) + } + Err(err) => Err(std::io::Error::other(format!( + "production boot did not recover after failed initialization preserve: {err}" + )) + .into()), + } +} + +async fn child_no_resize_boot_attempt(data_dir: &Path) -> Result<(), Box> { + let jam = load_test_jam()?; + match boot_app(&jam, data_dir, NockStackSize::Tiny).await { + Ok(app) => { + let _ = stop_app(app).await; + Err( + std::io::Error::other("deliberate no-resize child boot unexpectedly succeeded") + .into(), + ) + } + Err(err) => Err(err), + } +} + +fn load_test_jam() -> Result, Box> { + let mut possible_paths = Vec::new(); + if let Some(manifest_dir) = option_env!("CARGO_MANIFEST_DIR") { + possible_paths.push( + Path::new(manifest_dir) + .join("test-jams") + .join("test-ker.jam"), + ); + } + possible_paths.push(Path::new("open/crates/nockapp/test-jams").join("test-ker.jam")); + possible_paths.push(Path::new("test-jams").join("test-ker.jam")); + + for path in &possible_paths { + if let Ok(bytes) = fs::read(path) { + return Ok(bytes); + } + } + + Err(std::io::Error::other(format!( + "failed to read test-ker.jam from any candidate path: {:?}", + possible_paths + )) + .into()) +} + +async fn boot_app( + jam: &[u8], + data_dir: &Path, + stack_size: NockStackSize, +) -> Result, Box> { + let mut cli = default_boot_cli(false); + cli.data_dir = Some(data_dir.to_path_buf()); + cli.pma_initial_size = Some(PmaSize::from_words(stack_size.stack_words())); + cli.stack_size = stack_size; + cli.gc_interval = None; + cli.rotating_snapshot_interval_event_time = None; + cli.disable_fsync = true; + match setup_::( + jam, + cli, + &[], + "pma-failed-preserve-recovery-regression", + None, + ) + .await? + { + SetupResult::App(app) => Ok(app), + SetupResult::ExportedState => Err(std::io::Error::other("unexpected state export").into()), + } +} + +fn inc_poke() -> NounSlab { + let mut slab = NounSlab::new(); + let space = NounSpace::empty(); + slab.copy_into(D(tas!(b"inc")), &space); + slab +} + +fn state_peek() -> NounSlab { + let mut slab = NounSlab::new(); + let peek = T(&mut slab, &[D(tas!(b"state")), D(0)]); + slab.set_root(peek); + slab +} + +async fn poke_inc(app: &mut NockApp) -> Result<(), Box> { + app.poke(SystemWire.to_wire(), inc_poke()).await?; + Ok(()) +} + +async fn assert_counter_state( + app: &mut NockApp, + expected: u64, +) -> Result<(), Box> { + let exported = app.export().await?; + if exported.event_num != expected { + return Err(std::io::Error::other(format!( + "event number mismatch: expected={expected} actual={}", + exported.event_num + )) + .into()); + } + let actual = app.peek(state_peek()).await?; + let space = actual.noun_space(); + let root = unsafe { *actual.root() }; + let value = root + .in_space(&space) + .slot(7)? + .as_cell()? + .tail() + .noun() + .in_space(&space) + .as_atom()? + .as_u64()?; + if value != expected { + return Err(std::io::Error::other(format!( + "counter state mismatch: expected={expected} actual={value}" + )) + .into()); + } + Ok(()) +} + +async fn stop_app(mut app: NockApp) -> Result<(), Box> { + let handle = app.get_handle(); + handle.exit.exit(0).await?; + app.run().await?; + Ok(()) +} + +fn sqlite_connection(data_dir: &Path) -> Result> { + let path = data_dir.join("event-log.sqlite3"); + Ok(SqliteConnection::establish(path.to_str().ok_or_else( + || std::io::Error::other(format!("non-utf8 sqlite path: {path:?}")), + )?)?) +} + +fn sqlite_max_event_num(data_dir: &Path) -> Result> { + let mut conn = sqlite_connection(data_dir)?; + let row = sql_query("SELECT COALESCE(MAX(event_num), 0) AS value FROM events") + .get_result::(&mut conn)?; + Ok(u64::try_from(row.value)?) +} + +fn max_runtime_pma_meta_event(data_dir: &Path) -> Result> { + let pma_dir = data_dir.join("pma"); + let mut max_event = None; + for idx in [0, 1] { + let meta_path = pma_dir.join(format!("{idx}.meta")); + if !meta_path.exists() { + continue; + } + let meta = PmaPersistMetadataForTest::load(&meta_path)?; + max_event = Some(max_event.map_or(meta.event_num, |event: u64| event.max(meta.event_num))); + } + max_event.ok_or_else(|| { + std::io::Error::other(format!( + "no runtime PMA metadata found in {}", + pma_dir.display() + )) + .into() + }) +} + +fn active_runtime_pma(data_dir: &Path) -> Result> { + let pma_dir = data_dir.join("pma"); + let mut candidates = Vec::new(); + for idx in [0, 1] { + let pma_path = pma_dir.join(format!("{idx}.pma")); + let meta_path = pma_dir.join(format!("{idx}.meta")); + if pma_path.exists() && meta_path.exists() { + let meta = PmaPersistMetadataForTest::load(&meta_path)?; + let modified = fs::metadata(&meta_path) + .and_then(|metadata| metadata.modified()) + .unwrap_or(std::time::SystemTime::UNIX_EPOCH); + candidates.push((pma_path, meta.event_num, modified)); + } + } + candidates.sort_by_key(|(_, event_num, modified)| (*event_num, *modified)); + candidates.pop().map(|(path, _, _)| path).ok_or_else(|| { + std::io::Error::other(format!( + "no meta-paired runtime PMA found in {}", + pma_dir.display() + )) + .into() + }) +} + +fn force_pma_free_words(path: &Path, free_words: u64) -> Result<(), Box> { + let metadata = Pma::read_file_metadata(path)?; + if metadata.data_words <= free_words { + return Err(std::io::Error::other(format!( + "PMA too small to force {free_words} free words: data_words={}", + metadata.data_words + )) + .into()); + } + let new_alloc_words = metadata.data_words - free_words; + let mut pma = Pma::open(path.to_path_buf())?; + pma.reset_to(PmaOffsetWords::from_words(new_alloc_words)); + pma.sync_trailer()?; + pma.sync_file()?; + Ok(()) +} diff --git a/crates/nockapp/tests/pma_regressions/fresh_replay_boundary.rs b/crates/nockapp/tests/pma_regressions/fresh_replay_boundary.rs new file mode 100644 index 000000000..057f67527 --- /dev/null +++ b/crates/nockapp/tests/pma_regressions/fresh_replay_boundary.rs @@ -0,0 +1,279 @@ +use std::error::Error; +use std::fs; +use std::path::Path; + +use diesel::prelude::*; +use diesel::sql_query; +use diesel::sql_types::BigInt; +use diesel::sqlite::SqliteConnection; +use nockapp::kernel::boot::{default_boot_cli, setup_, NockStackSize, SetupResult}; +use nockapp::nockapp::wire::{SystemWire, Wire}; +use nockapp::noun::slab::{NockJammer, NounSlab}; +use nockapp::NockApp; +use nockvm::noun::{NounAllocator, NounSpace, D, T}; +use nockvm_macros::tas; +use tempfile::TempDir; + +#[derive(QueryableByName)] +struct I64ValueRow { + #[diesel(sql_type = BigInt)] + value: i64, +} + +pub(crate) fn run_regression() -> Result<(), Box> { + nockvm::check_endian(); + let runtime = tokio::runtime::Builder::new_current_thread() + .enable_all() + .build()?; + runtime.block_on(run()) +} + +async fn run() -> Result<(), Box> { + let temp = TempDir::new()?; + let jam = load_test_jam()?; + + continuous_event_log_replays_from_fresh_kernel(&jam, &temp.path().join("continuous-log")) + .await?; + gapped_event_log_fails_closed(&jam, &temp.path().join("gapped-log")).await?; + + println!("fresh event-log replay boundary regression passed"); + Ok(()) +} + +async fn continuous_event_log_replays_from_fresh_kernel( + jam: &[u8], + data_dir: &Path, +) -> Result<(), Box> { + println!("subcase continuous-log: remove PMA/snapshot/checkpoint bases and replay events 1..2 from fresh kernel"); + let mut first = boot_app(jam, data_dir).await?; + poke_inc(&mut first).await?; + poke_inc(&mut first).await?; + assert_counter_state(&mut first, 2).await?; + stop_app(first).await?; + assert_eq!(sqlite_max_event_num(data_dir)?, 2); + + remove_all_non_event_log_boot_bases(data_dir)?; + let mut recovered = boot_app(jam, data_dir).await?; + assert_counter_state(&mut recovered, 2).await?; + assert_eq!(sqlite_max_event_num(data_dir)?, 2); + stop_app(recovered).await?; + println!("subcase continuous-log passed: fresh replay reached SQLite max event 2"); + Ok(()) +} + +async fn gapped_event_log_fails_closed(jam: &[u8], data_dir: &Path) -> Result<(), Box> { + println!("subcase gapped-log: remove event 1 and require continuity failure instead of skipped replay"); + let mut first = boot_app(jam, data_dir).await?; + poke_inc(&mut first).await?; + poke_inc(&mut first).await?; + assert_counter_state(&mut first, 2).await?; + stop_app(first).await?; + assert_eq!(sqlite_max_event_num(data_dir)?, 2); + + remove_all_non_event_log_boot_bases(data_dir)?; + delete_sqlite_event(data_dir, 1)?; + assert_eq!(sqlite_max_event_num(data_dir)?, 2); + + match boot_app(jam, data_dir).await { + Ok(app) => { + let event_num = app.export().await?.event_num; + let _ = stop_app(app).await; + Err(std::io::Error::other(format!( + "fresh replay unexpectedly booted from a gapped SQLite log: booted_event={event_num}" + )) + .into()) + } + Err(err) => { + let text = err.to_string(); + if !text.contains("event log continuity") + && !text.contains("EventSequenceGap") + && !text.contains("event sequence gap") + { + return Err(std::io::Error::other(format!( + "gapped event log failed for the wrong reason: {text}" + )) + .into()); + } + println!("subcase gapped-log passed: boot failed closed with continuity error: {text}"); + Ok(()) + } + } +} + +fn load_test_jam() -> Result, Box> { + let mut possible_paths = Vec::new(); + if let Some(manifest_dir) = option_env!("CARGO_MANIFEST_DIR") { + possible_paths.push( + Path::new(manifest_dir) + .join("test-jams") + .join("test-ker.jam"), + ); + } + possible_paths.push(Path::new("open/crates/nockapp/test-jams").join("test-ker.jam")); + possible_paths.push(Path::new("test-jams").join("test-ker.jam")); + + for path in &possible_paths { + if let Ok(bytes) = fs::read(path) { + return Ok(bytes); + } + } + + Err(std::io::Error::other(format!( + "failed to read test-ker.jam from any candidate path: {:?}", + possible_paths + )) + .into()) +} + +async fn boot_app(jam: &[u8], data_dir: &Path) -> Result, Box> { + let mut cli = default_boot_cli(false); + cli.data_dir = Some(data_dir.to_path_buf()); + cli.stack_size = NockStackSize::Tiny; + cli.gc_interval = None; + cli.rotating_snapshot_interval_event_time = None; + cli.disable_fsync = true; + match setup_::(jam, cli, &[], "pma-fresh-replay-boundary-regression", None).await? { + SetupResult::App(app) => Ok(app), + SetupResult::ExportedState => Err(std::io::Error::other("unexpected state export").into()), + } +} + +fn inc_poke() -> NounSlab { + let mut slab = NounSlab::new(); + let space = NounSpace::empty(); + slab.copy_into(D(tas!(b"inc")), &space); + slab +} + +fn state_peek() -> NounSlab { + let mut slab = NounSlab::new(); + let peek = T(&mut slab, &[D(tas!(b"state")), D(0)]); + slab.set_root(peek); + slab +} + +async fn poke_inc(app: &mut NockApp) -> Result<(), Box> { + app.poke(SystemWire.to_wire(), inc_poke()).await?; + Ok(()) +} + +async fn assert_counter_state( + app: &mut NockApp, + expected: u64, +) -> Result<(), Box> { + let exported = app.export().await?; + if exported.event_num != expected { + return Err(std::io::Error::other(format!( + "event number mismatch: expected={expected} actual={}", + exported.event_num + )) + .into()); + } + let actual = app.peek(state_peek()).await?; + let space = actual.noun_space(); + let root = unsafe { *actual.root() }; + let value = root + .in_space(&space) + .slot(7)? + .as_cell()? + .tail() + .noun() + .in_space(&space) + .as_atom()? + .as_u64()?; + if value != expected { + return Err(std::io::Error::other(format!( + "counter state mismatch: expected={expected} actual={value}" + )) + .into()); + } + Ok(()) +} + +async fn stop_app(mut app: NockApp) -> Result<(), Box> { + let handle = app.get_handle(); + handle.exit.exit(0).await?; + app.run().await?; + Ok(()) +} + +fn sqlite_connection(data_dir: &Path) -> Result> { + let path = data_dir.join("event-log.sqlite3"); + Ok(SqliteConnection::establish(path.to_str().ok_or_else( + || std::io::Error::other(format!("non-utf8 sqlite path: {path:?}")), + )?)?) +} + +fn sqlite_max_event_num(data_dir: &Path) -> Result> { + let mut conn = sqlite_connection(data_dir)?; + let row = sql_query("SELECT COALESCE(MAX(event_num), 0) AS value FROM events") + .get_result::(&mut conn)?; + Ok(u64::try_from(row.value)?) +} + +fn delete_sqlite_event(data_dir: &Path, event_num: u64) -> Result<(), Box> { + let mut conn = sqlite_connection(data_dir)?; + sql_query("DELETE FROM events WHERE event_num = ?") + .bind::(i64::try_from(event_num)?) + .execute(&mut conn)?; + Ok(()) +} + +fn remove_all_non_event_log_boot_bases(data_dir: &Path) -> Result<(), Box> { + clear_runtime_pma_files(data_dir)?; + delete_snapshot_rows_and_artifacts(data_dir)?; + remove_checkpoint_files(data_dir)?; + Ok(()) +} + +fn clear_runtime_pma_files(data_dir: &Path) -> Result<(), Box> { + let pma_dir = data_dir.join("pma"); + for file_name in ["0.pma", "1.pma", "0.meta", "1.meta"] { + let path = pma_dir.join(file_name); + if path.exists() { + fs::remove_file(path)?; + } + } + Ok(()) +} + +fn delete_snapshot_rows_and_artifacts(data_dir: &Path) -> Result<(), Box> { + let pma_dir = data_dir.join("pma"); + let mut conn = sqlite_connection(data_dir)?; + sql_query("DELETE FROM snapshots").execute(&mut conn)?; + sql_query("DELETE FROM meta WHERE key = 'active_snapshot_id'").execute(&mut conn)?; + drop(conn); + + if !pma_dir.exists() { + return Ok(()); + } + for entry in fs::read_dir(&pma_dir)? { + let path = entry?.path(); + if !path.is_file() { + continue; + } + let Some(name) = path.file_name().and_then(|name| name.to_str()) else { + continue; + }; + if name == "epoch.pma" + || name == "epoch.manifest" + || name == "epoch.pma.tmp" + || name == "epoch.manifest.tmp" + || name.starts_with("snap-") + { + fs::remove_file(path)?; + } + } + Ok(()) +} + +fn remove_checkpoint_files(data_dir: &Path) -> Result<(), Box> { + let checkpoint_dir = data_dir.join("checkpoints"); + for file_name in ["0.chkjam", "1.chkjam"] { + let path = checkpoint_dir.join(file_name); + if path.exists() { + fs::remove_file(path)?; + } + } + Ok(()) +} diff --git a/crates/nockapp/tests/pma_regressions/pma_meta.rs b/crates/nockapp/tests/pma_regressions/pma_meta.rs new file mode 100644 index 000000000..edbd20e61 --- /dev/null +++ b/crates/nockapp/tests/pma_regressions/pma_meta.rs @@ -0,0 +1,151 @@ +use std::error::Error; +use std::fs; +use std::path::Path; + +use bincode::{config, Decode, Encode}; +use blake3::{Hash, Hasher}; + +const PMA_PERSIST_MAGIC: u64 = u64::from_le_bytes(*b"PMAPERS1"); +const PMA_PERSIST_VERSION: u32 = 5; +const PMA_PERSIST_VERSION_V4: u32 = 4; + +#[derive(Clone, Debug)] +pub(crate) struct PmaPersistMetadataForTest { + pub(crate) ker_hash: Hash, + pub(crate) event_num: u64, + pub(crate) kernel_state_raw: u64, + pub(crate) pma_reserved_words: Option, +} + +#[derive(Clone, Encode, Decode, Debug)] +struct PmaPersistMetadataV5ForTest { + magic: u64, + version: u32, + #[bincode(with_serde)] + ker_hash: Hash, + event_num: u64, + kernel_state_raw: u64, + pma_reserved_words: u64, + #[bincode(with_serde)] + checksum: Hash, +} + +#[derive(Clone, Encode, Decode, Debug)] +struct PmaPersistMetadataV4ForTest { + magic: u64, + version: u32, + #[bincode(with_serde)] + ker_hash: Hash, + event_num: u64, + kernel_state_raw: u64, + #[bincode(with_serde)] + checksum: Hash, +} + +impl PmaPersistMetadataForTest { + pub(crate) fn load(path: &Path) -> Result> { + let bytes = fs::read(path)?; + if let Ok((meta, _)) = bincode::decode_from_slice::< + PmaPersistMetadataV5ForTest, + config::Configuration, + >(&bytes, config::standard()) + { + if meta.validate() { + return Ok(meta.into_current()); + } + } + if let Ok((meta, _)) = bincode::decode_from_slice::< + PmaPersistMetadataV4ForTest, + config::Configuration, + >(&bytes, config::standard()) + { + if meta.validate() { + return Ok(meta.into_current()); + } + } + Err(std::io::Error::other(format!( + "invalid PMA persist metadata at {}", + path.display() + )) + .into()) + } + + pub(crate) fn save_v4_to_path(&self, path: &Path) -> Result<(), Box> { + let meta = + PmaPersistMetadataV4ForTest::new(self.ker_hash, self.event_num, self.kernel_state_raw); + let bytes = bincode::encode_to_vec(meta, config::standard())?; + fs::write(path, bytes)?; + Ok(()) + } +} + +impl PmaPersistMetadataV5ForTest { + fn validate(&self) -> bool { + self.magic == PMA_PERSIST_MAGIC + && self.version == PMA_PERSIST_VERSION + && self.checksum + == Self::checksum( + self.ker_hash, self.event_num, self.kernel_state_raw, self.pma_reserved_words, + ) + } + + fn checksum( + ker_hash: Hash, + event_num: u64, + kernel_state_raw: u64, + pma_reserved_words: u64, + ) -> Hash { + let mut hasher = Hasher::new(); + hasher.update(ker_hash.as_bytes()); + hasher.update(&event_num.to_le_bytes()); + hasher.update(&kernel_state_raw.to_le_bytes()); + hasher.update(&pma_reserved_words.to_le_bytes()); + hasher.finalize() + } + + fn into_current(self) -> PmaPersistMetadataForTest { + PmaPersistMetadataForTest { + ker_hash: self.ker_hash, + event_num: self.event_num, + kernel_state_raw: self.kernel_state_raw, + pma_reserved_words: Some(self.pma_reserved_words), + } + } +} + +impl PmaPersistMetadataV4ForTest { + fn new(ker_hash: Hash, event_num: u64, kernel_state_raw: u64) -> Self { + let checksum = Self::checksum(ker_hash, event_num, kernel_state_raw); + Self { + magic: PMA_PERSIST_MAGIC, + version: PMA_PERSIST_VERSION_V4, + ker_hash, + event_num, + kernel_state_raw, + checksum, + } + } + + fn validate(&self) -> bool { + self.magic == PMA_PERSIST_MAGIC + && self.version == PMA_PERSIST_VERSION_V4 + && self.checksum == Self::checksum(self.ker_hash, self.event_num, self.kernel_state_raw) + } + + fn checksum(ker_hash: Hash, event_num: u64, kernel_state_raw: u64) -> Hash { + let mut hasher = Hasher::new(); + hasher.update(ker_hash.as_bytes()); + hasher.update(&event_num.to_le_bytes()); + hasher.update(&kernel_state_raw.to_le_bytes()); + hasher.finalize() + } + + fn into_current(self) -> PmaPersistMetadataForTest { + PmaPersistMetadataForTest { + ker_hash: self.ker_hash, + event_num: self.event_num, + kernel_state_raw: self.kernel_state_raw, + pma_reserved_words: None, + } + } +} diff --git a/crates/nockapp/tests/pma_regressions/snapshot_restore_expand.rs b/crates/nockapp/tests/pma_regressions/snapshot_restore_expand.rs new file mode 100644 index 000000000..21a13a35b --- /dev/null +++ b/crates/nockapp/tests/pma_regressions/snapshot_restore_expand.rs @@ -0,0 +1,363 @@ +use std::error::Error; +use std::fs; +use std::path::{Path, PathBuf}; + +use diesel::prelude::*; +use diesel::sql_query; +use diesel::sql_types::{BigInt, Text}; +use diesel::sqlite::SqliteConnection; +use nockapp::kernel::boot::{default_boot_cli, setup_, NockStackSize, PmaSize, SetupResult}; +use nockapp::nockapp::wire::{SystemWire, Wire}; +use nockapp::noun::slab::{NockJammer, NounSlab}; +use nockapp::NockApp; +use nockvm::mem::{NOCK_STACK_SIZE_SMALL, NOCK_STACK_SIZE_TINY}; +use nockvm::noun::{NounAllocator, NounSpace, D, T}; +use nockvm::pma::Pma; +use nockvm_macros::tas; +use tempfile::TempDir; + +use crate::pma_regressions::pma_meta::PmaPersistMetadataForTest; + +#[derive(QueryableByName)] +struct I64ValueRow { + #[diesel(sql_type = BigInt)] + value: i64, +} + +#[derive(QueryableByName)] +struct SnapshotRow { + #[diesel(sql_type = Text)] + pma_path: String, + #[diesel(sql_type = Text)] + manifest_path: String, + #[diesel(sql_type = BigInt)] + event_num: i64, +} + +pub(crate) fn run_regression() -> Result<(), Box> { + nockvm::check_endian(); + let runtime = tokio::runtime::Builder::new_current_thread() + .enable_all() + .build()?; + runtime.block_on(run()) +} + +async fn run() -> Result<(), Box> { + let temp = TempDir::new()?; + let data_dir = temp.path().join("pma-snapshot-restore-expand-regression"); + let jam = load_test_jam()?; + + println!("stage 1: create tiny-PMA snapshot plus two committed events"); + let mut first = boot_app(&jam, &data_dir, NockStackSize::Tiny).await?; + poke_inc(&mut first).await?; + poke_inc(&mut first).await?; + assert_counter_state(&mut first, 2).await?; + stop_app(first).await?; + + assert_eq!(sqlite_max_event_num(&data_dir)?, 2); + assert_eq!(max_runtime_pma_meta_event(&data_dir)?, 2); + + let snapshot = newest_ready_snapshot(&data_dir)?; + if snapshot.event_num >= sqlite_max_event_num(&data_dir)? { + return Err(std::io::Error::other(format!( + "snapshot must be a replay base behind SQLite max: snapshot_event={} sqlite_max={}", + snapshot.event_num, + sqlite_max_event_num(&data_dir)? + )) + .into()); + } + let snapshot_pma_metadata = Pma::read_file_metadata(&snapshot.pma_path)?; + let snapshot_pma = Pma::open(snapshot.pma_path.clone())?; + let snapshot_runtime_metadata = snapshot_pma.file_metadata()?; + println!( + "ready snapshot selected: pma={} manifest={} event_num={} data_words={} alloc_words={} free_words={}", + snapshot.pma_path.display(), + snapshot.manifest_path.display(), + snapshot.event_num, + snapshot_pma_metadata.data_words, + snapshot_pma_metadata.alloc_words, + snapshot_pma_metadata + .data_words + .saturating_sub(snapshot_pma_metadata.alloc_words) + ); + if snapshot_pma_metadata.data_words != NOCK_STACK_SIZE_TINY as u64 { + return Err(std::io::Error::other(format!( + "snapshot fixture should have tiny PMA size: expected={} actual={}", + NOCK_STACK_SIZE_TINY, snapshot_pma_metadata.data_words + )) + .into()); + } + if snapshot_runtime_metadata.reserved_words <= snapshot_runtime_metadata.capacity_words { + return Err(std::io::Error::other(format!( + "snapshot PMA fixture should have a reservation larger than current capacity: capacity_words={} reserved_words={}", + snapshot_runtime_metadata.capacity_words, snapshot_runtime_metadata.reserved_words + )) + .into()); + } + let reserved_data_bytes = snapshot_runtime_metadata.reserved_words.saturating_mul(8); + if snapshot_runtime_metadata.apparent_file_bytes >= reserved_data_bytes { + return Err(std::io::Error::other(format!( + "snapshot copied reserved maximum instead of current capacity: apparent_file_bytes={} reserved_data_bytes={}", + snapshot_runtime_metadata.apparent_file_bytes, reserved_data_bytes + )) + .into()); + } + + clear_runtime_pma_files(&data_dir)?; + write_corrupt_checkpoint_pair(&data_dir)?; + println!( + "stage 2: restore from snapshot into configured small PMA and replay to SQLite max={}", + sqlite_max_event_num(&data_dir)? + ); + + let mut second = boot_app(&jam, &data_dir, NockStackSize::Small).await?; + assert_counter_state(&mut second, 2).await?; + assert_eq!(sqlite_max_event_num(&data_dir)?, 2); + assert_eq!(max_runtime_pma_meta_event(&data_dir)?, 2); + + let active_pma = active_runtime_pma(&data_dir)?; + let active_metadata = Pma::read_file_metadata(&active_pma)?; + println!( + "restored active PMA: path={} data_words={} alloc_words={} free_words={}", + active_pma.display(), + active_metadata.data_words, + active_metadata.alloc_words, + active_metadata + .data_words + .saturating_sub(active_metadata.alloc_words) + ); + if active_metadata.data_words < NOCK_STACK_SIZE_SMALL as u64 { + return Err(std::io::Error::other(format!( + "snapshot recovery replayed to SQLite max but did not restore into configured larger PMA: required_min_words={} actual_words={}", + NOCK_STACK_SIZE_SMALL, active_metadata.data_words + )) + .into()); + } + + stop_app(second).await?; + println!("snapshot restore expanded PMA and replayed to SQLite max"); + Ok(()) +} + +#[derive(Debug)] +struct ReadySnapshotForTest { + pma_path: PathBuf, + manifest_path: PathBuf, + event_num: u64, +} + +fn load_test_jam() -> Result, Box> { + let mut possible_paths = Vec::new(); + if let Some(manifest_dir) = option_env!("CARGO_MANIFEST_DIR") { + possible_paths.push( + Path::new(manifest_dir) + .join("test-jams") + .join("test-ker.jam"), + ); + } + possible_paths.push(Path::new("open/crates/nockapp/test-jams").join("test-ker.jam")); + possible_paths.push(Path::new("test-jams").join("test-ker.jam")); + + for path in &possible_paths { + if let Ok(bytes) = fs::read(path) { + return Ok(bytes); + } + } + + Err(std::io::Error::other(format!( + "failed to read test-ker.jam from any candidate path: {:?}", + possible_paths + )) + .into()) +} + +async fn boot_app( + jam: &[u8], + data_dir: &Path, + stack_size: NockStackSize, +) -> Result, Box> { + let mut cli = default_boot_cli(false); + cli.data_dir = Some(data_dir.to_path_buf()); + cli.pma_initial_size = Some(PmaSize::from_words(stack_size.stack_words())); + cli.stack_size = stack_size; + cli.gc_interval = None; + cli.rotating_snapshot_interval_event_time = None; + cli.disable_fsync = true; + match setup_::( + jam, + cli, + &[], + "pma-snapshot-restore-expand-regression", + None, + ) + .await? + { + SetupResult::App(app) => Ok(app), + SetupResult::ExportedState => Err(std::io::Error::other("unexpected state export").into()), + } +} + +fn inc_poke() -> NounSlab { + let mut slab = NounSlab::new(); + let space = NounSpace::empty(); + slab.copy_into(D(tas!(b"inc")), &space); + slab +} + +fn state_peek() -> NounSlab { + let mut slab = NounSlab::new(); + let peek = T(&mut slab, &[D(tas!(b"state")), D(0)]); + slab.set_root(peek); + slab +} + +async fn poke_inc(app: &mut NockApp) -> Result<(), Box> { + app.poke(SystemWire.to_wire(), inc_poke()).await?; + Ok(()) +} + +async fn assert_counter_state( + app: &mut NockApp, + expected: u64, +) -> Result<(), Box> { + let exported = app.export().await?; + if exported.event_num != expected { + return Err(std::io::Error::other(format!( + "event number mismatch: expected={expected} actual={}", + exported.event_num + )) + .into()); + } + let actual = app.peek(state_peek()).await?; + let space = actual.noun_space(); + let root = unsafe { *actual.root() }; + let value = root + .in_space(&space) + .slot(7)? + .as_cell()? + .tail() + .noun() + .in_space(&space) + .as_atom()? + .as_u64()?; + if value != expected { + return Err(std::io::Error::other(format!( + "counter state mismatch: expected={expected} actual={value}" + )) + .into()); + } + Ok(()) +} + +async fn stop_app(mut app: NockApp) -> Result<(), Box> { + let handle = app.get_handle(); + handle.exit.exit(0).await?; + app.run().await?; + Ok(()) +} + +fn sqlite_connection(data_dir: &Path) -> Result> { + let path = data_dir.join("event-log.sqlite3"); + Ok(SqliteConnection::establish(path.to_str().ok_or_else( + || std::io::Error::other(format!("non-utf8 sqlite path: {path:?}")), + )?)?) +} + +fn sqlite_max_event_num(data_dir: &Path) -> Result> { + let mut conn = sqlite_connection(data_dir)?; + let row = sql_query("SELECT COALESCE(MAX(event_num), 0) AS value FROM events") + .get_result::(&mut conn)?; + Ok(u64::try_from(row.value)?) +} + +fn newest_ready_snapshot(data_dir: &Path) -> Result> { + let mut conn = sqlite_connection(data_dir)?; + let row = sql_query( + "SELECT pma_path, manifest_path, event_num FROM snapshots WHERE state = 'ready' ORDER BY event_num DESC, snapshot_id DESC LIMIT 1", + ) + .get_result::(&mut conn)?; + let pma_path = PathBuf::from(row.pma_path); + let manifest_path = PathBuf::from(row.manifest_path); + if !pma_path.exists() || !manifest_path.exists() { + return Err(std::io::Error::other(format!( + "ready snapshot artifacts missing: pma_exists={} manifest_exists={} pma={} manifest={}", + pma_path.exists(), + manifest_path.exists(), + pma_path.display(), + manifest_path.display() + )) + .into()); + } + Ok(ReadySnapshotForTest { + pma_path, + manifest_path, + event_num: u64::try_from(row.event_num)?, + }) +} + +fn max_runtime_pma_meta_event(data_dir: &Path) -> Result> { + let pma_dir = data_dir.join("pma"); + let mut max_event = None; + for idx in [0, 1] { + let meta_path = pma_dir.join(format!("{idx}.meta")); + if !meta_path.exists() { + continue; + } + let meta = PmaPersistMetadataForTest::load(&meta_path)?; + max_event = Some(max_event.map_or(meta.event_num, |event: u64| event.max(meta.event_num))); + } + max_event.ok_or_else(|| { + std::io::Error::other(format!( + "no runtime PMA metadata found in {}", + pma_dir.display() + )) + .into() + }) +} + +fn active_runtime_pma(data_dir: &Path) -> Result> { + let pma_dir = data_dir.join("pma"); + let mut candidates = Vec::new(); + for idx in [0, 1] { + let pma_path = pma_dir.join(format!("{idx}.pma")); + let meta_path = pma_dir.join(format!("{idx}.meta")); + if pma_path.exists() && meta_path.exists() { + let meta = PmaPersistMetadataForTest::load(&meta_path)?; + let modified = fs::metadata(&meta_path) + .and_then(|metadata| metadata.modified()) + .unwrap_or(std::time::SystemTime::UNIX_EPOCH); + candidates.push((pma_path, meta.event_num, modified)); + } + } + candidates.sort_by_key(|(_, event_num, modified)| (*event_num, *modified)); + candidates.pop().map(|(path, _, _)| path).ok_or_else(|| { + std::io::Error::other(format!( + "no meta-paired runtime PMA found in {}", + pma_dir.display() + )) + .into() + }) +} + +fn clear_runtime_pma_files(data_dir: &Path) -> Result<(), Box> { + let pma_dir = data_dir.join("pma"); + for file_name in ["0.pma", "1.pma", "0.meta", "1.meta"] { + let path = pma_dir.join(file_name); + if path.exists() { + fs::remove_file(path)?; + } + } + Ok(()) +} + +fn write_corrupt_checkpoint_pair(data_dir: &Path) -> Result<(), Box> { + let checkpoints_dir = data_dir.join("checkpoints"); + fs::create_dir_all(&checkpoints_dir)?; + for file_name in ["0.chkjam", "1.chkjam"] { + fs::write( + checkpoints_dir.join(file_name), + b"corrupt checkpoint fixture", + )?; + } + Ok(()) +} diff --git a/crates/nockapp/tests/pma_regressions/sqlite_boundary_recovery.rs b/crates/nockapp/tests/pma_regressions/sqlite_boundary_recovery.rs new file mode 100644 index 000000000..6ee47fe97 --- /dev/null +++ b/crates/nockapp/tests/pma_regressions/sqlite_boundary_recovery.rs @@ -0,0 +1,372 @@ +use std::error::Error; +use std::fs; +use std::path::{Path, PathBuf}; + +use diesel::prelude::*; +use diesel::sql_query; +use diesel::sql_types::BigInt; +use diesel::sqlite::SqliteConnection; +use nockapp::kernel::boot::{default_boot_cli, setup_, NockStackSize, SetupResult}; +use nockapp::nockapp::wire::{SystemWire, Wire}; +use nockapp::noun::slab::{NockJammer, NounSlab}; +use nockapp::NockApp; +use nockvm::noun::{NounAllocator, NounSpace, D, T}; +use nockvm::pma::Pma; +use nockvm_macros::tas; +use tempfile::TempDir; + +use crate::pma_regressions::pma_meta::PmaPersistMetadataForTest; + +#[derive(QueryableByName)] +struct I64ValueRow { + #[diesel(sql_type = BigInt)] + value: i64, +} + +pub(crate) fn run_regression() -> Result<(), Box> { + nockvm::check_endian(); + let runtime = tokio::runtime::Builder::new_current_thread() + .enable_all() + .build()?; + runtime.block_on(run()) +} + +async fn run() -> Result<(), Box> { + let temp = TempDir::new()?; + let jam = load_test_jam()?; + + pma_ahead_subcase(&jam, &temp.path().join("pma-ahead")).await?; + pma_behind_subcase(&jam, &temp.path().join("pma-behind")).await?; + meta_stale_subcase(&jam, &temp.path().join("meta-stale")).await?; + + println!("PMA/SQLite boundary recovery matrix passed"); + Ok(()) +} + +async fn pma_ahead_subcase(jam: &[u8], data_dir: &Path) -> Result<(), Box> { + println!("subcase pma-ahead: create event-2 PMA, then delete SQLite event 2"); + let mut first = boot_app(jam, data_dir).await?; + poke_inc(&mut first).await?; + poke_inc(&mut first).await?; + assert_counter_state(&mut first, 2).await?; + stop_app(first).await?; + assert_eq!(sqlite_max_event_num(data_dir)?, 2); + assert_eq!(max_runtime_pma_meta_event(data_dir)?, 2); + + delete_sqlite_event(data_dir, 2)?; + assert_eq!(sqlite_max_event_num(data_dir)?, 1); + assert_eq!(max_runtime_pma_meta_event(data_dir)?, 2); + print_active_pma_state("pma-ahead before recovery", data_dir)?; + + let mut recovered = boot_app(jam, data_dir).await?; + assert_counter_state(&mut recovered, 1).await?; + assert_eq!(sqlite_max_event_num(data_dir)?, 1); + assert_eq!(max_runtime_pma_meta_event(data_dir)?, 1); + stop_app(recovered).await?; + println!("subcase pma-ahead passed: boot recovered to SQLite boundary event 1"); + Ok(()) +} + +async fn pma_behind_subcase(jam: &[u8], data_dir: &Path) -> Result<(), Box> { + println!("subcase pma-behind: restore event-1 runtime PMA while SQLite remains at event 2"); + let backup_dir = data_dir.with_extension("event-1-pma-backup"); + + let mut first = boot_app(jam, data_dir).await?; + poke_inc(&mut first).await?; + assert_counter_state(&mut first, 1).await?; + stop_app(first).await?; + assert_eq!(sqlite_max_event_num(data_dir)?, 1); + assert_eq!(max_runtime_pma_meta_event(data_dir)?, 1); + copy_runtime_pma_files(&data_dir.join("pma"), &backup_dir)?; + + let mut second = boot_app(jam, data_dir).await?; + assert_counter_state(&mut second, 1).await?; + poke_inc(&mut second).await?; + assert_counter_state(&mut second, 2).await?; + stop_app(second).await?; + assert_eq!(sqlite_max_event_num(data_dir)?, 2); + assert_eq!(max_runtime_pma_meta_event(data_dir)?, 2); + + restore_runtime_pma_files(data_dir, &backup_dir)?; + assert_eq!(sqlite_max_event_num(data_dir)?, 2); + assert_eq!(max_runtime_pma_meta_event(data_dir)?, 1); + print_active_pma_state("pma-behind before recovery", data_dir)?; + + let mut recovered = boot_app(jam, data_dir).await?; + assert_counter_state(&mut recovered, 2).await?; + assert_eq!(sqlite_max_event_num(data_dir)?, 2); + assert_eq!(max_runtime_pma_meta_event(data_dir)?, 2); + stop_app(recovered).await?; + println!("subcase pma-behind passed: boot replayed to SQLite boundary event 2"); + Ok(()) +} + +async fn meta_stale_subcase(jam: &[u8], data_dir: &Path) -> Result<(), Box> { + println!( + "subcase meta-stale: keep event-2 PMA file but replace active .meta with event-1 metadata" + ); + let stale_meta_path = data_dir.with_extension("event-1-active.meta"); + + let mut first = boot_app(jam, data_dir).await?; + poke_inc(&mut first).await?; + assert_counter_state(&mut first, 1).await?; + stop_app(first).await?; + let event_1_pma = active_runtime_pma(data_dir)?; + let event_1_meta = event_1_pma.with_extension("meta"); + let event_1_metadata = Pma::read_file_metadata(&event_1_pma)?; + fs::copy(&event_1_meta, &stale_meta_path)?; + assert_eq!(sqlite_max_event_num(data_dir)?, 1); + assert_eq!(max_runtime_pma_meta_event(data_dir)?, 1); + + let mut second = boot_app(jam, data_dir).await?; + assert_counter_state(&mut second, 1).await?; + poke_inc(&mut second).await?; + assert_counter_state(&mut second, 2).await?; + stop_app(second).await?; + assert_eq!(sqlite_max_event_num(data_dir)?, 2); + assert_eq!(max_runtime_pma_meta_event(data_dir)?, 2); + + let event_2_pma = active_runtime_pma(data_dir)?; + let event_2_metadata = Pma::read_file_metadata(&event_2_pma)?; + if event_2_metadata.alloc_words <= event_1_metadata.alloc_words { + return Err(std::io::Error::other(format!( + "meta-stale fixture did not advance the PMA trailer: event1_alloc={} event2_alloc={}", + event_1_metadata.alloc_words, event_2_metadata.alloc_words + )) + .into()); + } + fs::copy(&stale_meta_path, event_2_pma.with_extension("meta"))?; + assert_eq!(sqlite_max_event_num(data_dir)?, 2); + assert_eq!(max_runtime_pma_meta_event(data_dir)?, 1); + print_active_pma_state("meta-stale before recovery", data_dir)?; + + let mut recovered = boot_app(jam, data_dir).await?; + assert_counter_state(&mut recovered, 2).await?; + assert_eq!(sqlite_max_event_num(data_dir)?, 2); + assert_eq!(max_runtime_pma_meta_event(data_dir)?, 2); + stop_app(recovered).await?; + println!("subcase meta-stale passed: boot ignored stale .meta and replayed to SQLite boundary event 2"); + Ok(()) +} + +fn load_test_jam() -> Result, Box> { + let mut possible_paths = Vec::new(); + if let Some(manifest_dir) = option_env!("CARGO_MANIFEST_DIR") { + possible_paths.push( + Path::new(manifest_dir) + .join("test-jams") + .join("test-ker.jam"), + ); + } + possible_paths.push(Path::new("open/crates/nockapp/test-jams").join("test-ker.jam")); + possible_paths.push(Path::new("test-jams").join("test-ker.jam")); + + for path in &possible_paths { + if let Ok(bytes) = fs::read(path) { + return Ok(bytes); + } + } + + Err(std::io::Error::other(format!( + "failed to read test-ker.jam from any candidate path: {:?}", + possible_paths + )) + .into()) +} + +async fn boot_app(jam: &[u8], data_dir: &Path) -> Result, Box> { + let mut cli = default_boot_cli(false); + cli.data_dir = Some(data_dir.to_path_buf()); + cli.stack_size = NockStackSize::Tiny; + cli.gc_interval = None; + cli.rotating_snapshot_interval_event_time = None; + cli.disable_fsync = true; + match setup_::( + jam, + cli, + &[], + "pma-sqlite-boundary-recovery-regression", + None, + ) + .await? + { + SetupResult::App(app) => Ok(app), + SetupResult::ExportedState => Err(std::io::Error::other("unexpected state export").into()), + } +} + +fn inc_poke() -> NounSlab { + let mut slab = NounSlab::new(); + let space = NounSpace::empty(); + slab.copy_into(D(tas!(b"inc")), &space); + slab +} + +fn state_peek() -> NounSlab { + let mut slab = NounSlab::new(); + let peek = T(&mut slab, &[D(tas!(b"state")), D(0)]); + slab.set_root(peek); + slab +} + +async fn poke_inc(app: &mut NockApp) -> Result<(), Box> { + app.poke(SystemWire.to_wire(), inc_poke()).await?; + Ok(()) +} + +async fn assert_counter_state( + app: &mut NockApp, + expected: u64, +) -> Result<(), Box> { + let exported = app.export().await?; + if exported.event_num != expected { + return Err(std::io::Error::other(format!( + "event number mismatch: expected={expected} actual={}", + exported.event_num + )) + .into()); + } + let actual = app.peek(state_peek()).await?; + let space = actual.noun_space(); + let root = unsafe { *actual.root() }; + let value = root + .in_space(&space) + .slot(7)? + .as_cell()? + .tail() + .noun() + .in_space(&space) + .as_atom()? + .as_u64()?; + if value != expected { + return Err(std::io::Error::other(format!( + "counter state mismatch: expected={expected} actual={value}" + )) + .into()); + } + Ok(()) +} + +async fn stop_app(mut app: NockApp) -> Result<(), Box> { + let handle = app.get_handle(); + handle.exit.exit(0).await?; + app.run().await?; + Ok(()) +} + +fn sqlite_connection(data_dir: &Path) -> Result> { + let path = data_dir.join("event-log.sqlite3"); + Ok(SqliteConnection::establish(path.to_str().ok_or_else( + || std::io::Error::other(format!("non-utf8 sqlite path: {path:?}")), + )?)?) +} + +fn sqlite_max_event_num(data_dir: &Path) -> Result> { + let mut conn = sqlite_connection(data_dir)?; + let row = sql_query("SELECT COALESCE(MAX(event_num), 0) AS value FROM events") + .get_result::(&mut conn)?; + Ok(u64::try_from(row.value)?) +} + +fn delete_sqlite_event(data_dir: &Path, event_num: u64) -> Result<(), Box> { + let mut conn = sqlite_connection(data_dir)?; + sql_query("DELETE FROM events WHERE event_num = ?") + .bind::(i64::try_from(event_num)?) + .execute(&mut conn)?; + Ok(()) +} + +fn max_runtime_pma_meta_event(data_dir: &Path) -> Result> { + let pma_dir = data_dir.join("pma"); + let mut max_event = None; + for idx in [0, 1] { + let meta_path = pma_dir.join(format!("{idx}.meta")); + if !meta_path.exists() { + continue; + } + let meta = PmaPersistMetadataForTest::load(&meta_path)?; + max_event = Some(max_event.map_or(meta.event_num, |event: u64| event.max(meta.event_num))); + } + max_event.ok_or_else(|| { + std::io::Error::other(format!( + "no runtime PMA metadata found in {}", + pma_dir.display() + )) + .into() + }) +} + +fn active_runtime_pma(data_dir: &Path) -> Result> { + let pma_dir = data_dir.join("pma"); + let mut candidates = Vec::new(); + for idx in [0, 1] { + let pma_path = pma_dir.join(format!("{idx}.pma")); + let meta_path = pma_dir.join(format!("{idx}.meta")); + if pma_path.exists() && meta_path.exists() { + let meta = PmaPersistMetadataForTest::load(&meta_path)?; + let modified = fs::metadata(&meta_path) + .and_then(|metadata| metadata.modified()) + .unwrap_or(std::time::SystemTime::UNIX_EPOCH); + candidates.push((pma_path, meta.event_num, modified)); + } + } + candidates.sort_by_key(|(_, event_num, modified)| (*event_num, *modified)); + candidates.pop().map(|(path, _, _)| path).ok_or_else(|| { + std::io::Error::other(format!( + "no meta-paired runtime PMA found in {}", + pma_dir.display() + )) + .into() + }) +} + +fn copy_runtime_pma_files(from_dir: &Path, to_dir: &Path) -> Result<(), Box> { + fs::create_dir_all(to_dir)?; + for file_name in ["0.pma", "1.pma", "0.meta", "1.meta"] { + let from = from_dir.join(file_name); + if from.exists() { + fs::copy(&from, to_dir.join(file_name))?; + } + } + Ok(()) +} + +fn clear_runtime_pma_files(data_dir: &Path) -> Result<(), Box> { + let pma_dir = data_dir.join("pma"); + for file_name in ["0.pma", "1.pma", "0.meta", "1.meta"] { + let path = pma_dir.join(file_name); + if path.exists() { + fs::remove_file(path)?; + } + } + Ok(()) +} + +fn restore_runtime_pma_files(data_dir: &Path, backup_dir: &Path) -> Result<(), Box> { + clear_runtime_pma_files(data_dir)?; + let pma_dir = data_dir.join("pma"); + fs::create_dir_all(&pma_dir)?; + for file_name in ["0.pma", "1.pma", "0.meta", "1.meta"] { + let backup = backup_dir.join(file_name); + if backup.exists() { + fs::copy(&backup, pma_dir.join(file_name))?; + } + } + Ok(()) +} + +fn print_active_pma_state(label: &str, data_dir: &Path) -> Result<(), Box> { + let active = active_runtime_pma(data_dir)?; + let meta = PmaPersistMetadataForTest::load(&active.with_extension("meta"))?; + let metadata = Pma::read_file_metadata(&active)?; + println!( + "{label}: sqlite_max={} active_pma={} meta_event={} data_words={} alloc_words={} free_words={}", + sqlite_max_event_num(data_dir)?, + active.display(), + meta.event_num, + metadata.data_words, + metadata.alloc_words, + metadata.data_words.saturating_sub(metadata.alloc_words) + ); + Ok(()) +} diff --git a/crates/nockapp/tests/pma_regressions/stale_checkpoint_refusal.rs b/crates/nockapp/tests/pma_regressions/stale_checkpoint_refusal.rs new file mode 100644 index 000000000..f67125ce3 --- /dev/null +++ b/crates/nockapp/tests/pma_regressions/stale_checkpoint_refusal.rs @@ -0,0 +1,361 @@ +use std::error::Error; +use std::fs; +use std::path::{Path, PathBuf}; + +use diesel::prelude::*; +use diesel::sql_query; +use diesel::sql_types::BigInt; +use diesel::sqlite::SqliteConnection; +use nockapp::kernel::boot::{default_boot_cli, setup_, NockStackSize, SetupResult}; +use nockapp::nockapp::wire::{SystemWire, Wire}; +use nockapp::noun::slab::{NockJammer, NounSlab}; +use nockapp::save::SaveableCheckpoint; +use nockapp::NockApp; +use nockvm::noun::{NounAllocator, NounSpace, D, T}; +use nockvm_macros::tas; +use tempfile::TempDir; + +use crate::pma_regressions::pma_meta::PmaPersistMetadataForTest; + +#[derive(QueryableByName)] +struct I64ValueRow { + #[diesel(sql_type = BigInt)] + value: i64, +} + +pub(crate) fn run_regression() -> Result<(), Box> { + nockvm::check_endian(); + let runtime = tokio::runtime::Builder::new_current_thread() + .enable_all() + .build()?; + runtime.block_on(run()) +} + +async fn run() -> Result<(), Box> { + let temp = TempDir::new()?; + let jam = load_test_jam()?; + + sqlite_present_checkpoint_replays_to_newer_history( + &jam, + &temp.path().join("sqlite-present-replay"), + ) + .await?; + missing_sqlite_requires_explicit_checkpoint_bootstrap( + &jam, + &temp.path().join("missing-sqlite-normal"), + &temp.path().join("explicit-bootstrap"), + ) + .await?; + + println!("stale checkpoint refusal regression passed"); + Ok(()) +} + +async fn sqlite_present_checkpoint_replays_to_newer_history( + jam: &[u8], + data_dir: &Path, +) -> Result<(), Box> { + println!("subcase sqlite-present: checkpoint event 1 must replay to SQLite max event 2"); + let mut app = boot_app(jam, data_dir, false, None).await?; + poke_inc(&mut app).await?; + assert_counter_state(&mut app, 1).await?; + let checkpoint_path = write_checkpoint_from_export(&mut app, data_dir).await?; + println!( + "wrote valid event-1 checkpoint at {}", + checkpoint_path.display() + ); + poke_inc(&mut app).await?; + assert_counter_state(&mut app, 2).await?; + stop_app(app).await?; + + assert_eq!(sqlite_max_event_num(data_dir)?, 2); + assert_eq!(max_runtime_pma_meta_event(data_dir)?, 2); + delete_snapshot_rows_and_artifacts(data_dir)?; + clear_runtime_pma_files(data_dir)?; + + let mut recovered = boot_app(jam, data_dir, false, None).await?; + assert_counter_state(&mut recovered, 2).await?; + assert_eq!(sqlite_max_event_num(data_dir)?, 2); + assert_eq!(max_runtime_pma_meta_event(data_dir)?, 2); + stop_app(recovered).await?; + println!("subcase sqlite-present passed: stale checkpoint was only used as a replay base"); + Ok(()) +} + +async fn missing_sqlite_requires_explicit_checkpoint_bootstrap( + jam: &[u8], + data_dir: &Path, + explicit_bootstrap_dir: &Path, +) -> Result<(), Box> { + println!("subcase missing-sqlite: normal recovery must not silently use stale checkpoint"); + let mut app = boot_app(jam, data_dir, false, None).await?; + poke_inc(&mut app).await?; + assert_counter_state(&mut app, 1).await?; + let checkpoint_path = write_checkpoint_from_export(&mut app, data_dir).await?; + poke_inc(&mut app).await?; + assert_counter_state(&mut app, 2).await?; + stop_app(app).await?; + + assert_eq!(sqlite_max_event_num(data_dir)?, 2); + assert_eq!(max_runtime_pma_meta_event(data_dir)?, 2); + delete_snapshot_rows_and_artifacts(data_dir)?; + clear_runtime_pma_files(data_dir)?; + remove_event_log_files(data_dir)?; + + match boot_app(jam, data_dir, false, None).await { + Ok(silently_recovered) => { + let event_num = silently_recovered.export().await?.event_num; + let _ = stop_app(silently_recovered).await; + return Err(std::io::Error::other(format!( + "normal recovery silently used a stale checkpoint after the SQLite event log was removed: booted_event={event_num}; expected fail-closed unless explicit bootstrap/discard intent is provided" + )) + .into()); + } + Err(err) => { + println!("normal recovery failed closed with missing SQLite event log: {err}"); + } + } + + println!( + "subcase explicit-bootstrap: the same checkpoint is allowed only in a fresh --new boot" + ); + let mut explicit = boot_app( + jam, + explicit_bootstrap_dir, + true, + Some(checkpoint_path.clone()), + ) + .await?; + assert_counter_state(&mut explicit, 1).await?; + stop_app(explicit).await?; + println!("subcase missing-sqlite passed: stale checkpoint required explicit bootstrap intent"); + Ok(()) +} + +fn load_test_jam() -> Result, Box> { + let mut possible_paths = Vec::new(); + if let Some(manifest_dir) = option_env!("CARGO_MANIFEST_DIR") { + possible_paths.push( + Path::new(manifest_dir) + .join("test-jams") + .join("test-ker.jam"), + ); + } + possible_paths.push(Path::new("open/crates/nockapp/test-jams").join("test-ker.jam")); + possible_paths.push(Path::new("test-jams").join("test-ker.jam")); + + for path in &possible_paths { + if let Ok(bytes) = fs::read(path) { + return Ok(bytes); + } + } + + Err(std::io::Error::other(format!( + "failed to read test-ker.jam from any candidate path: {:?}", + possible_paths + )) + .into()) +} + +async fn boot_app( + jam: &[u8], + data_dir: &Path, + new: bool, + bootstrap_from_chkjam: Option, +) -> Result, Box> { + let mut cli = default_boot_cli(new); + cli.data_dir = Some(data_dir.to_path_buf()); + cli.stack_size = NockStackSize::Tiny; + cli.gc_interval = None; + cli.rotating_snapshot_interval_event_time = None; + cli.disable_fsync = true; + cli.bootstrap_from_chkjam = + bootstrap_from_chkjam.map(|path| path.to_string_lossy().into_owned()); + match setup_::( + jam, + cli, + &[], + "pma-stale-checkpoint-refusal-regression", + None, + ) + .await? + { + SetupResult::App(app) => Ok(app), + SetupResult::ExportedState => Err(std::io::Error::other("unexpected state export").into()), + } +} + +fn inc_poke() -> NounSlab { + let mut slab = NounSlab::new(); + let space = NounSpace::empty(); + slab.copy_into(D(tas!(b"inc")), &space); + slab +} + +fn state_peek() -> NounSlab { + let mut slab = NounSlab::new(); + let peek = T(&mut slab, &[D(tas!(b"state")), D(0)]); + slab.set_root(peek); + slab +} + +fn empty_cold_slab() -> NounSlab { + let mut slab = NounSlab::new(); + let root = T(&mut slab, &[D(0), D(0), D(0)]); + slab.set_root(root); + slab +} + +async fn write_checkpoint_from_export( + app: &mut NockApp, + data_dir: &Path, +) -> Result> { + let exported = app.export().await?; + let checkpoint = SaveableCheckpoint { + ker_hash: exported.ker_hash, + event_num: exported.event_num, + state: exported.kernel_state, + cold: empty_cold_slab(), + }; + let jammed = checkpoint.to_jammed_checkpoint::(); + let bytes = jammed.encode()?; + let checkpoint_dir = data_dir.join("checkpoints"); + fs::create_dir_all(&checkpoint_dir)?; + let checkpoint_path = checkpoint_dir.join("0.chkjam"); + fs::write(&checkpoint_path, bytes)?; + Ok(checkpoint_path) +} + +async fn poke_inc(app: &mut NockApp) -> Result<(), Box> { + app.poke(SystemWire.to_wire(), inc_poke()).await?; + Ok(()) +} + +async fn assert_counter_state( + app: &mut NockApp, + expected: u64, +) -> Result<(), Box> { + let exported = app.export().await?; + if exported.event_num != expected { + return Err(std::io::Error::other(format!( + "event number mismatch: expected={expected} actual={}", + exported.event_num + )) + .into()); + } + let actual = app.peek(state_peek()).await?; + let space = actual.noun_space(); + let root = unsafe { *actual.root() }; + let value = root + .in_space(&space) + .slot(7)? + .as_cell()? + .tail() + .noun() + .in_space(&space) + .as_atom()? + .as_u64()?; + if value != expected { + return Err(std::io::Error::other(format!( + "counter state mismatch: expected={expected} actual={value}" + )) + .into()); + } + Ok(()) +} + +async fn stop_app(mut app: NockApp) -> Result<(), Box> { + let handle = app.get_handle(); + handle.exit.exit(0).await?; + app.run().await?; + Ok(()) +} + +fn sqlite_connection(data_dir: &Path) -> Result> { + let path = data_dir.join("event-log.sqlite3"); + Ok(SqliteConnection::establish(path.to_str().ok_or_else( + || std::io::Error::other(format!("non-utf8 sqlite path: {path:?}")), + )?)?) +} + +fn sqlite_max_event_num(data_dir: &Path) -> Result> { + let mut conn = sqlite_connection(data_dir)?; + let row = sql_query("SELECT COALESCE(MAX(event_num), 0) AS value FROM events") + .get_result::(&mut conn)?; + Ok(u64::try_from(row.value)?) +} + +fn max_runtime_pma_meta_event(data_dir: &Path) -> Result> { + let pma_dir = data_dir.join("pma"); + let mut max_event = None; + for idx in [0, 1] { + let meta_path = pma_dir.join(format!("{idx}.meta")); + if !meta_path.exists() { + continue; + } + let meta = PmaPersistMetadataForTest::load(&meta_path)?; + max_event = Some(max_event.map_or(meta.event_num, |event: u64| event.max(meta.event_num))); + } + max_event.ok_or_else(|| { + std::io::Error::other(format!( + "no runtime PMA metadata found in {}", + pma_dir.display() + )) + .into() + }) +} + +fn clear_runtime_pma_files(data_dir: &Path) -> Result<(), Box> { + let pma_dir = data_dir.join("pma"); + for file_name in ["0.pma", "1.pma", "0.meta", "1.meta"] { + let path = pma_dir.join(file_name); + if path.exists() { + fs::remove_file(path)?; + } + } + Ok(()) +} + +fn delete_snapshot_rows_and_artifacts(data_dir: &Path) -> Result<(), Box> { + let pma_dir = data_dir.join("pma"); + let mut conn = sqlite_connection(data_dir)?; + sql_query("DELETE FROM snapshots").execute(&mut conn)?; + sql_query("DELETE FROM meta WHERE key = 'active_snapshot_id'").execute(&mut conn)?; + drop(conn); + + if !pma_dir.exists() { + return Ok(()); + } + for entry in fs::read_dir(&pma_dir)? { + let path = entry?.path(); + if !path.is_file() { + continue; + } + let Some(name) = path.file_name().and_then(|name| name.to_str()) else { + continue; + }; + if name == "epoch.pma" + || name == "epoch.manifest" + || name == "epoch.pma.tmp" + || name == "epoch.manifest.tmp" + || name.starts_with("snap-") + { + fs::remove_file(path)?; + } + } + Ok(()) +} + +fn remove_event_log_files(data_dir: &Path) -> Result<(), Box> { + let event_log = data_dir.join("event-log.sqlite3"); + for path in [ + event_log.clone(), + PathBuf::from(format!("{}-wal", event_log.display())), + PathBuf::from(format!("{}-shm", event_log.display())), + ] { + if path.exists() { + fs::remove_file(path)?; + } + } + Ok(()) +} diff --git a/crates/nockapp/tests/pma_regressions/upgrade_from_65a.rs b/crates/nockapp/tests/pma_regressions/upgrade_from_65a.rs new file mode 100644 index 000000000..cc95c80d2 --- /dev/null +++ b/crates/nockapp/tests/pma_regressions/upgrade_from_65a.rs @@ -0,0 +1,295 @@ +use std::error::Error; +use std::fs; +use std::fs::OpenOptions; +use std::io::{Seek, SeekFrom, Write}; +use std::path::{Path, PathBuf}; + +use diesel::prelude::*; +use diesel::sql_query; +use diesel::sql_types::BigInt; +use diesel::sqlite::SqliteConnection; +use nockapp::kernel::boot::{default_boot_cli, setup_, NockStackSize, PmaSize, SetupResult}; +use nockapp::nockapp::wire::{SystemWire, Wire}; +use nockapp::noun::slab::{NockJammer, NounSlab}; +use nockapp::NockApp; +use nockvm::noun::{NounAllocator, NounSpace, D, T}; +use nockvm::pma::Pma; +use nockvm_macros::tas; +use tempfile::TempDir; + +use crate::pma_regressions::pma_meta::PmaPersistMetadataForTest; + +const PMA_MAGIC: u64 = u64::from_le_bytes(*b"NOCKPMA1"); +const PMA_VERSION_V1: u64 = 1; +const PMA_LEGACY_TRAILER_BYTES: u64 = 32; + +#[derive(QueryableByName)] +struct I64ValueRow { + #[diesel(sql_type = BigInt)] + value: i64, +} + +pub(crate) fn run_regression() -> Result<(), Box> { + nockvm::check_endian(); + let runtime = tokio::runtime::Builder::new_current_thread() + .enable_all() + .build()?; + runtime.block_on(run()) +} + +async fn run() -> Result<(), Box> { + let temp = TempDir::new()?; + let data_dir = temp.path().join("pma-upgrade-from-65a-regression"); + let jam = load_test_jam()?; + let configured_reserved_words = NockStackSize::Small.stack_words(); + + println!("stage 1: create current PMA, then rewrite it as a 65a-style PMA fixture"); + let mut first = boot_app(&jam, &data_dir, configured_reserved_words).await?; + poke_inc(&mut first).await?; + poke_inc(&mut first).await?; + assert_counter_state(&mut first, 2).await?; + stop_app(first).await?; + assert_eq!(sqlite_max_event_num(&data_dir)?, 2); + + let active_pma = active_runtime_pma(&data_dir)?; + let active_meta_path = active_pma.with_extension("meta"); + let current_metadata = Pma::read_file_metadata(&active_pma)?; + let current_meta = PmaPersistMetadataForTest::load(&active_meta_path)?; + if current_meta.event_num != 2 { + return Err(std::io::Error::other(format!( + "expected event-2 sidecar before downgrade, got {}", + current_meta.event_num + )) + .into()); + } + + downgrade_pma_file_to_65a( + &active_pma, current_metadata.data_words, current_metadata.alloc_words, + )?; + current_meta.save_v4_to_path(&active_meta_path)?; + + let legacy_metadata = Pma::read_file_metadata(&active_pma)?; + if legacy_metadata.version != PMA_VERSION_V1 { + return Err(std::io::Error::other(format!( + "downgraded fixture should read as PMA v1, got version {}", + legacy_metadata.version + )) + .into()); + } + if legacy_metadata.data_words != current_metadata.data_words + || legacy_metadata.alloc_words != current_metadata.alloc_words + || legacy_metadata.reserved_words != legacy_metadata.data_words + { + return Err(std::io::Error::other(format!( + "65a fixture metadata mismatch: legacy={legacy_metadata:?} current={current_metadata:?}" + )) + .into()); + } + let legacy_meta = PmaPersistMetadataForTest::load(&active_meta_path)?; + if legacy_meta.event_num != 2 || legacy_meta.pma_reserved_words.is_some() { + return Err(std::io::Error::other(format!( + "65a fixture sidecar mismatch: event_num={} reserved={:?}", + legacy_meta.event_num, legacy_meta.pma_reserved_words + )) + .into()); + } + + println!("stage 2: boot current code from the 65a-style PMA and continue event processing"); + let mut upgraded = boot_app(&jam, &data_dir, configured_reserved_words).await?; + assert_counter_state(&mut upgraded, 2).await?; + poke_inc(&mut upgraded).await?; + assert_counter_state(&mut upgraded, 3).await?; + stop_app(upgraded).await?; + assert_eq!(sqlite_max_event_num(&data_dir)?, 3); + + let upgraded_pma = active_runtime_pma(&data_dir)?; + let upgraded_metadata = Pma::read_file_metadata(&upgraded_pma)?; + if upgraded_metadata.version != 2 { + return Err(std::io::Error::other(format!( + "upgraded PMA should be v2 after boot, got version {}", + upgraded_metadata.version + )) + .into()); + } + if upgraded_metadata.data_words < current_metadata.data_words { + return Err(std::io::Error::other(format!( + "upgrade shrank PMA capacity: before={} after={}", + current_metadata.data_words, upgraded_metadata.data_words + )) + .into()); + } + if upgraded_metadata.reserved_words != u64::try_from(configured_reserved_words)? { + return Err(std::io::Error::other(format!( + "upgrade did not apply configured reservation: expected={} actual={}", + configured_reserved_words, upgraded_metadata.reserved_words + )) + .into()); + } + let upgraded_meta = PmaPersistMetadataForTest::load(&upgraded_pma.with_extension("meta"))?; + if upgraded_meta.event_num != 3 + || upgraded_meta.pma_reserved_words != Some(upgraded_metadata.reserved_words) + { + return Err(std::io::Error::other(format!( + "upgraded sidecar mismatch: event_num={} reserved={:?} trailer_reserved={}", + upgraded_meta.event_num, + upgraded_meta.pma_reserved_words, + upgraded_metadata.reserved_words + )) + .into()); + } + + println!("65a PMA fixture upgraded, preserved state, and accepted a new event"); + Ok(()) +} + +fn downgrade_pma_file_to_65a( + path: &Path, + data_words: u64, + alloc_words: u64, +) -> Result<(), Box> { + let mut file = OpenOptions::new().read(true).write(true).open(path)?; + let data_bytes = data_words + .checked_mul(8) + .ok_or_else(|| std::io::Error::other("PMA data byte length overflowed"))?; + file.set_len(data_bytes + PMA_LEGACY_TRAILER_BYTES)?; + file.seek(SeekFrom::Start(data_bytes))?; + file.write_all(&PMA_MAGIC.to_le_bytes())?; + file.write_all(&PMA_VERSION_V1.to_le_bytes())?; + file.write_all(&data_words.to_le_bytes())?; + file.write_all(&alloc_words.to_le_bytes())?; + file.sync_all()?; + Ok(()) +} + +fn load_test_jam() -> Result, Box> { + let mut possible_paths = Vec::new(); + if let Some(manifest_dir) = option_env!("CARGO_MANIFEST_DIR") { + possible_paths.push( + Path::new(manifest_dir) + .join("test-jams") + .join("test-ker.jam"), + ); + } + possible_paths.push(Path::new("open/crates/nockapp/test-jams").join("test-ker.jam")); + possible_paths.push(Path::new("test-jams").join("test-ker.jam")); + + for path in &possible_paths { + if let Ok(bytes) = fs::read(path) { + return Ok(bytes); + } + } + + Err(std::io::Error::other(format!( + "failed to read test-ker.jam from any candidate path: {:?}", + possible_paths + )) + .into()) +} + +async fn boot_app( + jam: &[u8], + data_dir: &Path, + pma_reserved_words: usize, +) -> Result, Box> { + let mut cli = default_boot_cli(false); + cli.data_dir = Some(data_dir.to_path_buf()); + cli.pma_initial_size = Some(PmaSize::from_words(NockStackSize::Tiny.stack_words())); + cli.pma_reserved_size = Some(PmaSize::from_words(pma_reserved_words)); + cli.stack_size = NockStackSize::Tiny; + cli.gc_interval = None; + cli.rotating_snapshot_interval_event_time = None; + cli.disable_fsync = true; + match setup_::(jam, cli, &[], "pma-upgrade-from-65a-regression", None).await? { + SetupResult::App(app) => Ok(app), + SetupResult::ExportedState => Err(std::io::Error::other("unexpected state export").into()), + } +} + +fn inc_poke() -> NounSlab { + let mut slab = NounSlab::new(); + let space = NounSpace::empty(); + slab.copy_into(D(tas!(b"inc")), &space); + slab +} + +fn state_peek() -> NounSlab { + let mut slab = NounSlab::new(); + let peek = T(&mut slab, &[D(tas!(b"state")), D(0)]); + slab.set_root(peek); + slab +} + +async fn poke_inc(app: &mut NockApp) -> Result<(), Box> { + app.poke(SystemWire.to_wire(), inc_poke()).await?; + Ok(()) +} + +async fn assert_counter_state( + app: &mut NockApp, + expected: u64, +) -> Result<(), Box> { + let result = app.peek(state_peek()).await?; + let space = result.noun_space(); + let root = unsafe { *result.root() }; + let value = root + .in_space(&space) + .slot(7)? + .as_cell()? + .tail() + .noun() + .in_space(&space) + .as_atom()? + .as_u64()?; + if value != expected { + return Err(std::io::Error::other(format!( + "counter state mismatch: expected={expected} actual={value}" + )) + .into()); + } + Ok(()) +} + +async fn stop_app(mut app: NockApp) -> Result<(), Box> { + let handle = app.get_handle(); + handle.exit.exit(0).await?; + app.run().await?; + Ok(()) +} + +fn sqlite_connection(data_dir: &Path) -> Result> { + let path = data_dir.join("event-log.sqlite3"); + Ok(SqliteConnection::establish(path.to_str().ok_or_else( + || std::io::Error::other(format!("invalid sqlite path: {}", path.display())), + )?)?) +} + +fn sqlite_max_event_num(data_dir: &Path) -> Result> { + let mut conn = sqlite_connection(data_dir)?; + let row = sql_query("SELECT COALESCE(MAX(event_num), 0) AS value FROM events") + .get_result::(&mut conn)?; + Ok(u64::try_from(row.value)?) +} + +fn active_runtime_pma(data_dir: &Path) -> Result> { + let pma_dir = data_dir.join("pma"); + let mut candidates = Vec::new(); + for idx in [0, 1] { + let pma_path = pma_dir.join(format!("{idx}.pma")); + let meta_path = pma_dir.join(format!("{idx}.meta")); + if pma_path.exists() && meta_path.exists() { + let meta = PmaPersistMetadataForTest::load(&meta_path)?; + let modified = fs::metadata(&meta_path) + .and_then(|metadata| metadata.modified()) + .unwrap_or(std::time::SystemTime::UNIX_EPOCH); + candidates.push((pma_path, meta.event_num, modified)); + } + } + candidates.sort_by_key(|(_, event_num, modified)| (*event_num, *modified)); + candidates.pop().map(|(path, _, _)| path).ok_or_else(|| { + std::io::Error::other(format!( + "no meta-paired runtime PMA found in {}", + pma_dir.display() + )) + .into() + }) +} diff --git a/crates/nockchain-api/README.md b/crates/nockchain-api/README.md index 45129b46a..06402503c 100644 --- a/crates/nockchain-api/README.md +++ b/crates/nockchain-api/README.md @@ -70,3 +70,24 @@ That’s it—the API surface piggybacks on the running node; there is no separa - This binary shares the same hot prover state (`zkvm-jetpack::produce_prover_hot_state`) as every other Nockchain node; make sure the host has enough RAM for the prover plus the gRPC caches. Deployments today are integration testbeds, not hardened services. Control access, scrape the metrics, and expect breaking changes until we tag an official release. + +## PMA quickstart (local snapshots) + +The repository ships with a 2.5 GiB checkpoint in `test-api/.data.nockchain/checkpoints`. You can boot the public API directly on top of that snapshot and exercise the memfd-backed PMA without running a full sync: + +```bash +cargo run --release -p nockchain-api -- \ + --data-dir test-api/.data.nockchain \ + --identity-path test-api/.nockchain_identity \ + --no-default-peers \ + --bind /ip4/127.0.0.1/udp/0/quic-v1 \ + --bind-public-grpc-addr 127.0.0.1:5555 +``` + +Key points: + +- `--data-dir` now takes the exact directory containing your `checkpoints/` folder; point it at `test-api/.data.nockchain` (or any other PMA workspace). +- `--identity-path` overrides the default `.nockchain_identity`; pass the matching key from `test-api` so the node can re-use the provided peer ID. +- All stack allocations now go through the memfd-backed PMA implementation, so once the node is up you can watch the `nockvm.pma.*` gnort metrics (paging ratios, touched pages, etc.) while issuing peeks. + +This is the minimal way to reproduce RSS measurements for the PMA work: start the API against the canned data dir, point your `nockchain-peek` clients at `127.0.0.1:5555`, and observe the metrics as the OS pages the slab in and out. diff --git a/crates/nockchain-api/src/main.rs b/crates/nockchain-api/src/main.rs index 0032c4484..daa904f63 100644 --- a/crates/nockchain-api/src/main.rs +++ b/crates/nockchain-api/src/main.rs @@ -1,7 +1,6 @@ use std::error::Error; use chaff::Chaff; -use clap::Parser; use kernels_open_dumb::KERNEL; use nockapp::kernel::boot; use nockchain::NockchainAPIConfig; @@ -25,7 +24,8 @@ static ALLOC: tracy_client::ProfiledAllocator = #[tokio::main] async fn main() -> Result<(), Box> { nockvm::check_endian(); - let mut cli = nockchain::NockchainCli::parse(); + let mut cli = + nockchain::NockchainCli::parse_with_default_stack_size(boot::NockStackSize::Large); cli.nockapp_cli.color = clap::ColorChoice::Never; boot::init_default_tracing(&cli.nockapp_cli); diff --git a/crates/nockchain-libp2p-io/Cargo.toml b/crates/nockchain-libp2p-io/Cargo.toml index 03469e894..a7a1dada7 100644 --- a/crates/nockchain-libp2p-io/Cargo.toml +++ b/crates/nockchain-libp2p-io/Cargo.toml @@ -44,5 +44,7 @@ tracing = { workspace = true } [dev-dependencies] cbor4ii = { workspace = true, features = ["serde1", "use_std"] } +hex = { workspace = true } quickcheck = { workspace = true } serde_cbor = { workspace = true } +serde_json = { workspace = true } diff --git a/crates/nockchain-libp2p-io/src/behaviour.rs b/crates/nockchain-libp2p-io/src/behaviour.rs index 7bf3f2b04..d1f5b93da 100644 --- a/crates/nockchain-libp2p-io/src/behaviour.rs +++ b/crates/nockchain-libp2p-io/src/behaviour.rs @@ -9,6 +9,7 @@ use libp2p::{ }; use crate::config::LibP2PConfig; +use crate::ip_block::{self, PeerExclusions}; use crate::messages::{NockchainRequest, NockchainResponse}; #[derive(NetworkBehaviour)] @@ -20,9 +21,11 @@ pub(crate) struct NockchainBehaviour { /// Connectivity testing ping: ping::Behaviour, /// Peer discovery via a DHT - pub kad: kad::Behaviour, - /// Peer banning + pub kad: ip_block::IpFilteredKad, + /// Peer banning (by peer id) pub allow_block_list: allow_block_list::Behaviour, + /// Connection gating by IP (deny banned IPs regardless of peer id) + pub ip_block: ip_block::Behaviour, /// Peer whitelisting pub allow_peers: Toggle>, /// Connection limiting @@ -41,6 +44,7 @@ impl NockchainBehaviour { allowed: Option>, limits: connection_limits::ConnectionLimits, memory_limits: Option, + peer_exclusions: PeerExclusions, ) -> impl FnOnce(&libp2p::identity::Keypair) -> Self { move |keypair: &libp2p::identity::Keypair| { let peer_id = libp2p::identity::PeerId::from_public_key(&keypair.public()); @@ -92,8 +96,9 @@ impl NockchainBehaviour { NockchainBehaviour { ping: ping::Behaviour::default(), identify: identify_behaviour, - kad: kad_behaviour, + kad: ip_block::IpFilteredKad::new(kad_behaviour, peer_exclusions.clone()), allow_block_list: allow_block_list::Behaviour::default(), + ip_block: ip_block::Behaviour::new(peer_exclusions), allow_peers, request_response: request_response_behaviour, connection_limits: connection_limits_behaviour, diff --git a/crates/nockchain-libp2p-io/src/cbor_tests.rs b/crates/nockchain-libp2p-io/src/cbor_tests.rs index 1292bcc5a..10082a658 100644 --- a/crates/nockchain-libp2p-io/src/cbor_tests.rs +++ b/crates/nockchain-libp2p-io/src/cbor_tests.rs @@ -1133,4 +1133,203 @@ mod tests { println!("SUCCESS: Fix confirmed - adding boolean field resolves the EOF enum serialization issue"); } + + #[derive(Debug, serde::Deserialize)] + struct Gen1CborConformanceVectors { + schema_version: String, + request_vectors: Vec, + response_vectors: Vec, + invalid_vectors: Vec, + } + + #[derive(Debug, serde::Deserialize)] + struct RequestVector { + id: String, + variant: RequestVariant, + cbor_hex: String, + message_hex: String, + pow_hex: Option, + nonce: Option, + } + + #[derive(Debug, serde::Deserialize)] + #[serde(rename_all = "snake_case")] + enum RequestVariant { + Gossip, + Request, + } + + #[derive(Debug, serde::Deserialize)] + struct ResponseVector { + id: String, + variant: ResponseVariant, + cbor_hex: String, + acked: Option, + message_hex: Option, + } + + #[derive(Debug, serde::Deserialize)] + #[serde(rename_all = "snake_case")] + enum ResponseVariant { + Ack, + Result, + } + + #[derive(Debug, serde::Deserialize)] + struct InvalidVector { + id: String, + target: InvalidTarget, + cbor_hex: String, + error_substring: Option, + } + + #[derive(Debug, serde::Deserialize)] + #[serde(rename_all = "snake_case")] + enum InvalidTarget { + Request, + Response, + } + + fn load_gen1_cbor_conformance_vectors() -> Gen1CborConformanceVectors { + serde_json::from_str(include_str!("../testdata/req_res_gen1_cbor_vectors.json")) + .expect("gen1 cbor vector fixture must be valid JSON") + } + + fn decode_hex(hex_value: &str) -> Vec { + hex::decode(hex_value).expect("vector hex must be valid") + } + + fn request_from_vector(vector: &RequestVector) -> NockchainRequest { + match vector.variant { + RequestVariant::Gossip => NockchainRequest::Gossip { + message: ByteBuf::from(decode_hex(&vector.message_hex)), + }, + RequestVariant::Request => { + let pow_hex = vector + .pow_hex + .as_ref() + .expect("request vectors require pow_hex for Request variant"); + let pow_bytes = decode_hex(pow_hex); + let pow: [u8; 16] = pow_bytes + .try_into() + .expect("pow_hex must decode to 16 bytes"); + let nonce = vector + .nonce + .expect("request vectors require nonce for Request variant"); + NockchainRequest::Request { + pow, + nonce, + message: ByteBuf::from(decode_hex(&vector.message_hex)), + } + } + } + } + + fn response_from_vector(vector: &ResponseVector) -> NockchainResponse { + match vector.variant { + ResponseVariant::Ack => { + let acked = vector + .acked + .expect("response vectors require acked for Ack variant"); + NockchainResponse::Ack { acked } + } + ResponseVariant::Result => { + let message_hex = vector + .message_hex + .as_ref() + .expect("response vectors require message_hex for Result variant"); + NockchainResponse::Result { + message: ByteBuf::from(decode_hex(message_hex)), + } + } + } + } + + #[test] + fn test_gen1_cbor_vector_schema_version() { + let vectors = load_gen1_cbor_conformance_vectors(); + assert_eq!(vectors.schema_version, "req_res_gen1_cbor_v1"); + } + + #[test] + fn test_gen1_request_cbor_vectors_roundtrip() { + let vectors = load_gen1_cbor_conformance_vectors(); + assert!( + !vectors.request_vectors.is_empty(), + "request vector fixture must not be empty" + ); + for vector in vectors.request_vectors { + let expected = request_from_vector(&vector); + let encoded = serde_cbor::to_vec(&expected).expect("request should serialize"); + assert_eq!( + hex::encode(&encoded), + vector.cbor_hex, + "request vector '{}' cbor mismatch", + vector.id + ); + let decoded: NockchainRequest = serde_cbor::from_slice(&decode_hex(&vector.cbor_hex)) + .expect("request vector cbor should deserialize"); + assert_eq!( + decoded, expected, + "request vector '{}' roundtrip mismatch", + vector.id + ); + } + } + + #[test] + fn test_gen1_response_cbor_vectors_roundtrip() { + let vectors = load_gen1_cbor_conformance_vectors(); + assert!( + !vectors.response_vectors.is_empty(), + "response vector fixture must not be empty" + ); + for vector in vectors.response_vectors { + let expected = response_from_vector(&vector); + let encoded = serde_cbor::to_vec(&expected).expect("response should serialize"); + assert_eq!( + hex::encode(&encoded), + vector.cbor_hex, + "response vector '{}' cbor mismatch", + vector.id + ); + let decoded: NockchainResponse = serde_cbor::from_slice(&decode_hex(&vector.cbor_hex)) + .expect("response vector cbor should deserialize"); + assert_eq!( + decoded, expected, + "response vector '{}' roundtrip mismatch", + vector.id + ); + } + } + + #[test] + fn test_gen1_invalid_cbor_vectors_fail_decode() { + let vectors = load_gen1_cbor_conformance_vectors(); + assert!( + !vectors.invalid_vectors.is_empty(), + "invalid vector fixture must not be empty" + ); + for vector in vectors.invalid_vectors { + let bytes = decode_hex(&vector.cbor_hex); + let err = match vector.target { + InvalidTarget::Request => serde_cbor::from_slice::(&bytes) + .expect_err("invalid request vector should fail decode"), + InvalidTarget::Response => serde_cbor::from_slice::(&bytes) + .expect_err("invalid response vector should fail decode"), + }; + if let Some(substring) = vector.error_substring { + if !substring.is_empty() { + let err_text = format!("{err:?}"); + assert!( + err_text.contains(&substring), + "invalid vector '{}' error mismatch. expected substring '{}', got '{}'", + vector.id, + substring, + err_text + ); + } + } + } + } } diff --git a/crates/nockchain-libp2p-io/src/config.rs b/crates/nockchain-libp2p-io/src/config.rs index 9b27223ec..b31389878 100644 --- a/crates/nockchain-libp2p-io/src/config.rs +++ b/crates/nockchain-libp2p-io/src/config.rs @@ -1,4 +1,7 @@ +use std::collections::HashSet; +use std::net::IpAddr; use std::num::NonZero; +use std::str::FromStr; use std::time::Duration; use config::{Config, ConfigError, Environment}; @@ -70,6 +73,21 @@ const POKE_TIMEOUT_SECS: u64 = 180; // Default max failed pings before closing connection const FAILED_PINGS_BEFORE_CLOSE: u64 = 4; +const IP_HYGIENE_ENABLED: bool = true; +const ADDRESS_COOLDOWN: Duration = Duration::from_secs(3600); +const IP_EXCLUSION: Duration = Duration::from_secs(3600); +const IP_EXTENDED_EXCLUSION: Duration = Duration::from_secs(21600); +const EVIDENCE_WINDOW: Duration = Duration::from_secs(600); +const IP_EXCLUSION_HISTORY: Duration = Duration::from_secs(86400); +const PERMISSION_DENIED_COOLDOWN: Duration = Duration::from_secs(300); +const WRONG_PEER_ID_IP_THRESHOLD: usize = 3; +const DIAL_FAILURE_IP_THRESHOLD: usize = 5; +const SAME_IP_KAD_ENTRY_THRESHOLD: usize = 8; +const MAX_AUTO_EXCLUSION: Duration = Duration::from_secs(21600); +const MAX_EXCLUSION_ENTRIES: usize = 4096; +const REQUEST_PEER_COOLDOWN: Duration = Duration::from_secs(300); +const FAIL2BAN_ON_TEMP_EXCLUSION: bool = false; + /// Configuration struct that allows overriding default constants from environment variables #[derive(Debug, Deserialize, Clone)] pub struct LibP2PConfig { @@ -179,6 +197,66 @@ pub struct LibP2PConfig { /// Number of failed pings before closing connection #[serde(default = "default_failed_pings_before_close")] pub failed_pings_before_close: u64, + + /// Whether temporary IP and endpoint hygiene is active + #[serde(default = "default_ip_hygiene_enabled")] + pub ip_hygiene_enabled: bool, + + /// Address cooldown duration after endpoint-local evidence + #[serde(default = "default_address_cooldown_secs")] + pub address_cooldown_secs: u64, + + /// First IP exclusion duration after same-IP evidence crosses a threshold + #[serde(default = "default_ip_exclusion_secs")] + pub ip_exclusion_secs: u64, + + /// Recurrent IP exclusion duration + #[serde(default = "default_ip_extended_exclusion_secs")] + pub ip_extended_exclusion_secs: u64, + + /// Evidence window for escalation + #[serde(default = "default_evidence_window_secs")] + pub evidence_window_secs: u64, + + /// Time horizon for repeated exclusion escalation + #[serde(default = "default_ip_exclusion_history_secs")] + pub ip_exclusion_history_secs: u64, + + /// Cooldown for local PermissionDenied transport failures + #[serde(default = "default_permission_denied_cooldown_secs")] + pub permission_denied_cooldown_secs: u64, + + /// Distinct wrong peer IDs or ports needed before IP exclusion + #[serde(default = "default_wrong_peer_id_ip_threshold")] + pub wrong_peer_id_ip_threshold: usize, + + /// Failed endpoints on one IP needed before IP exclusion + #[serde(default = "default_dial_failure_ip_threshold")] + pub dial_failure_ip_threshold: usize, + + /// Local Kademlia cardinality that marks same-IP pollution + #[serde(default = "default_same_ip_kad_entry_threshold")] + pub same_ip_kad_entry_threshold: usize, + + /// Upper bound for automatic exclusion duration + #[serde(default = "default_max_auto_exclusion_secs")] + pub max_auto_exclusion_secs: u64, + + /// Max evidence events retained in memory + #[serde(default = "default_max_exclusion_entries")] + pub max_exclusion_entries: usize, + + /// Peer request cooldown after request-response failure + #[serde(default = "default_request_peer_cooldown_secs")] + pub request_peer_cooldown_secs: u64, + + /// Comma-separated IPs exempt from automatic exclusion + #[serde(default)] + pub exclusion_allow_ips: String, + + /// Emit fail2ban-compatible lines for local peer blocks and temporary exclusions + #[serde(default = "default_fail2ban_on_temp_exclusion")] + pub fail2ban_on_temp_exclusion: bool, } // Default value functions @@ -259,6 +337,48 @@ fn default_poke_timeout_secs() -> u64 { fn default_failed_pings_before_close() -> u64 { FAILED_PINGS_BEFORE_CLOSE // Number of failed pings before closing connection } +fn default_ip_hygiene_enabled() -> bool { + IP_HYGIENE_ENABLED +} +fn default_address_cooldown_secs() -> u64 { + ADDRESS_COOLDOWN.as_secs() +} +fn default_ip_exclusion_secs() -> u64 { + IP_EXCLUSION.as_secs() +} +fn default_ip_extended_exclusion_secs() -> u64 { + IP_EXTENDED_EXCLUSION.as_secs() +} +fn default_evidence_window_secs() -> u64 { + EVIDENCE_WINDOW.as_secs() +} +fn default_ip_exclusion_history_secs() -> u64 { + IP_EXCLUSION_HISTORY.as_secs() +} +fn default_permission_denied_cooldown_secs() -> u64 { + PERMISSION_DENIED_COOLDOWN.as_secs() +} +fn default_wrong_peer_id_ip_threshold() -> usize { + WRONG_PEER_ID_IP_THRESHOLD +} +fn default_dial_failure_ip_threshold() -> usize { + DIAL_FAILURE_IP_THRESHOLD +} +fn default_same_ip_kad_entry_threshold() -> usize { + SAME_IP_KAD_ENTRY_THRESHOLD +} +fn default_max_auto_exclusion_secs() -> u64 { + MAX_AUTO_EXCLUSION.as_secs() +} +fn default_max_exclusion_entries() -> usize { + MAX_EXCLUSION_ENTRIES +} +fn default_request_peer_cooldown_secs() -> u64 { + REQUEST_PEER_COOLDOWN.as_secs() +} +fn default_fail2ban_on_temp_exclusion() -> bool { + FAIL2BAN_ON_TEMP_EXCLUSION +} // Do _not_ use this default implementation in production code. It's just a fallback. // Use from_env() to load from environment variables with sensible defaults. @@ -288,10 +408,138 @@ impl Default for LibP2PConfig { seen_tx_clear_interval: default_seen_tx_clear_interval(), poke_timeout_secs: default_poke_timeout_secs(), failed_pings_before_close: default_failed_pings_before_close(), + ip_hygiene_enabled: default_ip_hygiene_enabled(), + address_cooldown_secs: default_address_cooldown_secs(), + ip_exclusion_secs: default_ip_exclusion_secs(), + ip_extended_exclusion_secs: default_ip_extended_exclusion_secs(), + evidence_window_secs: default_evidence_window_secs(), + ip_exclusion_history_secs: default_ip_exclusion_history_secs(), + permission_denied_cooldown_secs: default_permission_denied_cooldown_secs(), + wrong_peer_id_ip_threshold: default_wrong_peer_id_ip_threshold(), + dial_failure_ip_threshold: default_dial_failure_ip_threshold(), + same_ip_kad_entry_threshold: default_same_ip_kad_entry_threshold(), + max_auto_exclusion_secs: default_max_auto_exclusion_secs(), + max_exclusion_entries: default_max_exclusion_entries(), + request_peer_cooldown_secs: default_request_peer_cooldown_secs(), + exclusion_allow_ips: String::new(), + fail2ban_on_temp_exclusion: default_fail2ban_on_temp_exclusion(), } } } +#[derive(Debug, Clone)] +pub(crate) struct PeerExclusionConfig { + pub(crate) enabled: bool, + pub(crate) address_cooldown_secs: u64, + pub(crate) ip_exclusion_secs: u64, + pub(crate) ip_extended_exclusion_secs: u64, + pub(crate) evidence_window_secs: u64, + pub(crate) ip_exclusion_history_secs: u64, + pub(crate) permission_denied_cooldown_secs: u64, + pub(crate) wrong_peer_id_ip_threshold: usize, + pub(crate) dial_failure_ip_threshold: usize, + pub(crate) same_ip_kad_entry_threshold: usize, + pub(crate) max_auto_exclusion_secs: u64, + pub(crate) max_exclusion_entries: usize, + pub(crate) request_peer_cooldown_secs: u64, + pub(crate) allow_ips: HashSet, + pub(crate) fail2ban_on_temp_exclusion: bool, +} + +impl Default for PeerExclusionConfig { + fn default() -> Self { + Self { + enabled: default_ip_hygiene_enabled(), + address_cooldown_secs: default_address_cooldown_secs(), + ip_exclusion_secs: default_ip_exclusion_secs(), + ip_extended_exclusion_secs: default_ip_extended_exclusion_secs(), + evidence_window_secs: default_evidence_window_secs(), + ip_exclusion_history_secs: default_ip_exclusion_history_secs(), + permission_denied_cooldown_secs: default_permission_denied_cooldown_secs(), + wrong_peer_id_ip_threshold: default_wrong_peer_id_ip_threshold(), + dial_failure_ip_threshold: default_dial_failure_ip_threshold(), + same_ip_kad_entry_threshold: default_same_ip_kad_entry_threshold(), + max_auto_exclusion_secs: default_max_auto_exclusion_secs(), + max_exclusion_entries: default_max_exclusion_entries(), + request_peer_cooldown_secs: default_request_peer_cooldown_secs(), + allow_ips: HashSet::new(), + fail2ban_on_temp_exclusion: default_fail2ban_on_temp_exclusion(), + } + } +} + +impl PeerExclusionConfig { + pub(crate) fn from_libp2p_config(config: &LibP2PConfig) -> Result { + let mut allow_ips = HashSet::new(); + for raw in config.exclusion_allow_ips.split(',') { + let raw = raw.trim(); + if raw.is_empty() { + continue; + } + let ip = IpAddr::from_str(raw).map_err(|err| { + ConfigError::Message(format!( + "invalid NOCKCHAIN_LIBP2P_EXCLUSION_ALLOW_IPS entry {raw}: {err}" + )) + })?; + allow_ips.insert(ip); + } + + Ok(Self { + enabled: config.ip_hygiene_enabled, + address_cooldown_secs: config.address_cooldown_secs, + ip_exclusion_secs: config.ip_exclusion_secs, + ip_extended_exclusion_secs: config.ip_extended_exclusion_secs, + evidence_window_secs: config.evidence_window_secs, + ip_exclusion_history_secs: config.ip_exclusion_history_secs, + permission_denied_cooldown_secs: config.permission_denied_cooldown_secs, + wrong_peer_id_ip_threshold: config.wrong_peer_id_ip_threshold, + dial_failure_ip_threshold: config.dial_failure_ip_threshold, + same_ip_kad_entry_threshold: config.same_ip_kad_entry_threshold, + max_auto_exclusion_secs: config.max_auto_exclusion_secs, + max_exclusion_entries: config.max_exclusion_entries, + request_peer_cooldown_secs: config.request_peer_cooldown_secs, + allow_ips, + fail2ban_on_temp_exclusion: config.fail2ban_on_temp_exclusion, + }) + } + + pub(crate) fn address_cooldown(&self) -> Duration { + Duration::from_secs(self.address_cooldown_secs) + } + + pub(crate) fn ip_exclusion(&self) -> Duration { + Duration::from_secs(self.ip_exclusion_secs) + } + + pub(crate) fn ip_extended_exclusion(&self) -> Duration { + Duration::from_secs(self.ip_extended_exclusion_secs) + } + + pub(crate) fn evidence_window(&self) -> Duration { + Duration::from_secs(self.evidence_window_secs) + } + + pub(crate) fn ip_exclusion_history(&self) -> Duration { + Duration::from_secs(self.ip_exclusion_history_secs) + } + + pub(crate) fn permission_denied_cooldown(&self) -> Duration { + Duration::from_secs(self.permission_denied_cooldown_secs) + } + + pub(crate) fn max_auto_exclusion(&self) -> Duration { + Duration::from_secs(self.max_auto_exclusion_secs) + } + + pub(crate) fn event_history(&self) -> Duration { + self.evidence_window().max(self.ip_exclusion_history()) + } + + pub(crate) fn request_peer_cooldown(&self) -> Duration { + Duration::from_secs(self.request_peer_cooldown_secs) + } +} + impl LibP2PConfig { /// Load configuration from environment variables with NOCKCHAIN_LIBP2P_ prefix pub fn from_env() -> Result { @@ -401,4 +649,61 @@ impl LibP2PConfig { pub fn failed_pings_before_close(&self) -> u64 { self.failed_pings_before_close } + + pub(crate) fn peer_exclusion_config(&self) -> Result { + PeerExclusionConfig::from_libp2p_config(self) + } +} + +#[cfg(test)] +mod tests { + use std::net::{IpAddr, Ipv4Addr, Ipv6Addr}; + + use crate::config::{LibP2PConfig, PeerExclusionConfig}; + + #[test] + fn peer_exclusion_config_parses_allow_ips_and_overrides() { + let config = LibP2PConfig { + ip_hygiene_enabled: false, + address_cooldown_secs: 7, + ip_exclusion_secs: 11, + ip_extended_exclusion_secs: 13, + evidence_window_secs: 17, + ip_exclusion_history_secs: 19, + permission_denied_cooldown_secs: 23, + wrong_peer_id_ip_threshold: 3, + dial_failure_ip_threshold: 5, + same_ip_kad_entry_threshold: 8, + max_auto_exclusion_secs: 29, + max_exclusion_entries: 31, + request_peer_cooldown_secs: 37, + exclusion_allow_ips: String::from("203.0.113.1,2001:db8::1"), + fail2ban_on_temp_exclusion: true, + ..LibP2PConfig::default() + }; + + let exclusion = + PeerExclusionConfig::from_libp2p_config(&config).expect("valid exclusion config"); + + assert!(!exclusion.enabled); + assert_eq!(exclusion.address_cooldown_secs, 7); + assert_eq!(exclusion.max_exclusion_entries, 31); + assert!(exclusion.fail2ban_on_temp_exclusion); + assert!(exclusion + .allow_ips + .contains(&IpAddr::V4(Ipv4Addr::new(203, 0, 113, 1)))); + assert!(exclusion.allow_ips.contains(&IpAddr::V6( + "2001:db8::1".parse::().expect("valid ipv6") + ))); + } + + #[test] + fn peer_exclusion_config_rejects_invalid_allow_ip() { + let config = LibP2PConfig { + exclusion_allow_ips: String::from("203.0.113.1,nope"), + ..LibP2PConfig::default() + }; + + assert!(PeerExclusionConfig::from_libp2p_config(&config).is_err()); + } } diff --git a/crates/nockchain-libp2p-io/src/driver.rs b/crates/nockchain-libp2p-io/src/driver.rs index fd97977cf..1d923225a 100644 --- a/crates/nockchain-libp2p-io/src/driver.rs +++ b/crates/nockchain-libp2p-io/src/driver.rs @@ -1,4 +1,6 @@ +use std::collections::{BTreeMap, BTreeSet}; use std::error::Error; +use std::net::IpAddr; use std::pin::Pin; use std::str::FromStr; use std::sync::Arc; @@ -23,16 +25,13 @@ use libp2p::{ }; use nockapp::driver::{IODriverFn, PokeResult}; use nockapp::noun::slab::NounSlab; -use nockapp::noun::FromAtom; use nockapp::utils::error::{CrownError, ExternalError}; use nockapp::utils::make_tas; use nockapp::utils::scry::*; use nockapp::wire::{Wire, WireRepr}; use nockapp::{AtomExt, NockAppError}; -use nockvm::ext::NounExt; -use nockvm::noun::{Atom, Noun, D, T}; +use nockvm::noun::{Atom, Noun, NounAllocator, NounSpace, D, T}; use nockvm_macros::tas; -use rand::rng; use rand::seq::SliceRandom; use tokio::sync::{mpsc, Mutex, MutexGuard}; use tokio::time::{Duration, MissedTickBehavior}; @@ -40,6 +39,9 @@ use tracing::{debug, error, info, instrument, trace, warn}; use crate::behaviour::{NockchainBehaviour, NockchainEvent}; use crate::config::LibP2PConfig; +use crate::ip_block::{ + AddressCooldownOutcome, ExclusionOutcome, IpExclusionOutcome, PeerExclusions, +}; use crate::messages::{NockchainDataRequest, NockchainFact, NockchainRequest, NockchainResponse}; use crate::metrics::NockchainP2PMetrics; use crate::p2p_state::{CacheResponse, P2PState}; @@ -109,12 +111,14 @@ impl EffectType { return EffectType::Unknown; }; - let head = effect_cell.head(); - let Ok(atom) = head.as_atom() else { + let space = noun_slab.noun_space(); + let effect_cell = effect_cell.in_space(&space); + let head = effect_cell.head().noun(); + let Ok(atom) = head.in_space(&space).as_atom() else { return EffectType::Unknown; }; let Ok(bytes) = atom.to_bytes_until_nul() else { - warn!("atom was not properly formatted: {:?}", atom); + warn!("atom was not properly formatted: {:?}", atom.atom()); return EffectType::Unknown; }; @@ -171,6 +175,8 @@ pub fn make_libp2p_driver( Box::pin(async move { let libp2p_config = LibP2PConfig::from_env()?; debug!("Libp2p config: {:?}", libp2p_config); + let peer_exclusion_config = libp2p_config.peer_exclusion_config()?; + let peer_exclusions = PeerExclusions::new(peer_exclusion_config); let kademlia_bootstrap_interval = libp2p_config.kademlia_bootstrap_interval(); let force_peer_dial_interval = libp2p_config.force_peer_dial_interval(); let request_high_reset = libp2p_config.request_high_reset(); @@ -182,22 +188,29 @@ pub fn make_libp2p_driver( let min_peers = libp2p_config.min_peers(); let poke_timeout = libp2p_config.poke_timeout(); let failed_pings_before_close = libp2p_config.failed_pings_before_close(); - let mut swarm = - match start_swarm(libp2p_config, keypair, bind, allowed, limits, memory_limits) { - Ok(swarm) => swarm, - Err(e) => { - error!("Could not create swarm: {}", e); - let (_, handle_clone) = handle.dup(); - tokio::spawn(async move { - if let Err(e) = handle_clone.exit.exit(1).await { - error!("Failed to send exit signal: {}", e); - } - }); - return Err(NockAppError::OtherError(String::from( - "Could not start swarm", - ))); - } - }; + let mut swarm = match start_swarm( + libp2p_config, + keypair, + bind, + allowed, + limits, + memory_limits, + peer_exclusions.clone(), + ) { + Ok(swarm) => swarm, + Err(e) => { + error!("Could not create swarm: {}", e); + let (_, handle_clone) = handle.dup(); + tokio::spawn(async move { + if let Err(e) = handle_clone.exit.exit(1).await { + error!("Failed to send exit signal: {}", e); + } + }); + return Err(NockAppError::OtherError(String::from( + "Could not start swarm", + ))); + } + }; let (swarm_tx, mut swarm_rx) = mpsc::channel::(1000); // number needs to be high enough to send gossips to peers let mut join_set = TrackedJoinSet::>::new(); let driver_state = Arc::new(Mutex::new(P2PState::new( @@ -236,10 +249,15 @@ pub fn make_libp2p_driver( join_set.spawn("timer".to_string(), send_timer_poke(guard, traffic_cop.clone(), metrics.clone())) } _ = connectivity_interval.tick() => { - let peer_count = log_peer_status(&mut swarm, &metrics).await; + let peer_count = log_peer_status( + &mut swarm, + &metrics, + &peer_exclusions, + &driver_state + ).await; if peer_count < min_peers { let state_guard = driver_state.lock().await; - dial_more_peers(&mut swarm, state_guard); + dial_more_peers(&mut swarm, state_guard, &peer_exclusions); } }, Ok(noun_slab) = effect_handle.next_effect() => { @@ -250,8 +268,9 @@ pub fn make_libp2p_driver( let connected_peers: Vec = swarm.connected_peers().cloned().collect(); let state_guard = Arc::clone(&driver_state); // Clone the Arc, not the P2P state let metrics_clone = metrics.clone(); + let peer_exclusions_clone = peer_exclusions.clone(); join_set.spawn("handle_effect".to_string(), async move { - handle_effect(noun_slab, swarm_tx_clone, equix_builder_clone, local_peer_id, connected_peers, fast_sync, state_guard, metrics_clone).await + handle_effect(noun_slab, swarm_tx_clone, equix_builder_clone, local_peer_id, connected_peers, fast_sync, state_guard, metrics_clone, peer_exclusions_clone).await }); }, Some(event) = swarm.next() => { @@ -271,20 +290,58 @@ pub fn make_libp2p_driver( }, SwarmEvent::Behaviour(NockchainEvent::Identify(Received { connection_id: _, peer_id, info })) => { trace!("SEvent: identify_received"); - identify_received(&mut swarm, peer_id, info)?; + identify_received(&mut swarm, peer_id, info, &peer_exclusions, &metrics)?; + }, + SwarmEvent::Behaviour(NockchainEvent::Kad(event)) => { + trace!("SEvent: kad event {event:?}"); + observe_kad_cardinality_and_exclude( + &mut swarm, + &driver_state, + &peer_exclusions, + &metrics, + ).await; + prune_excluded_swarm_state( + &mut swarm, + &driver_state, + &peer_exclusions, + &metrics, + ).await; }, SwarmEvent::ConnectionEstablished { connection_id, peer_id, endpoint, .. } => { driver_state.lock().await.track_connection(connection_id, peer_id, endpoint.get_remote_address(), endpoint.clone()); debug!("SEvent: {peer_id} is new friend via: {endpoint:?}"); }, SwarmEvent::ConnectionClosed { connection_id, peer_id, endpoint, cause, .. } => { - let mut state_guard = driver_state.lock().await; - let _ = state_guard.lost_connection(connection_id); - if let Some(cause) = cause { + { + let mut state_guard = driver_state.lock().await; + let _ = state_guard.lost_connection(connection_id); + } + let eperm = cause + .as_ref() + .is_some_and(|c| chain_has_permission_denied(c)); + if let Some(cause) = &cause { debug!("SEvent: friendship ended with {peer_id} via: {endpoint:?}. cause: {cause:?}"); } else { debug!("SEvent: friendship ended by us with {peer_id} via: {endpoint:?}."); } + if eperm { + if let Some(ip) = endpoint.get_remote_address().ip_addr() { + record_exclusion_outcome( + &mut swarm, + &driver_state, + &peer_exclusions, + &metrics, + peer_exclusions.record_permission_denied( + endpoint.get_remote_address() + ), + &[peer_id], + ) + .await; + if !peer_exclusions.is_ip_excluded(&ip) { + trace!("PermissionDenied on {ip} stayed below IP exclusion threshold"); + } + } + } }, SwarmEvent::IncomingConnectionError { local_addr, send_back_addr, error, .. } => { trace!("SEvent: Failed incoming connection from {} to {}: {}", @@ -310,12 +367,19 @@ pub fn make_libp2p_driver( let traffic_clone = traffic_cop.clone(); let metrics = metrics.clone(); let state_arc = Arc::clone(&driver_state); // Clone the Arc, not the MessageTracker + let peer_exclusions_clone = peer_exclusions.clone(); join_set.spawn("handle_request_response".to_string(), async move { - handle_request_response(peer, connection_id, message, swarm_tx_clone, &mut equix_builder_clone, local_peer_id, traffic_clone, metrics.clone(), state_arc, request_high_threshold).await + handle_request_response(peer, connection_id, message, swarm_tx_clone, &mut equix_builder_clone, local_peer_id, traffic_clone, metrics.clone(), state_arc, request_high_threshold, peer_exclusions_clone).await }); }, SwarmEvent::Behaviour(NockchainEvent::RequestResponse(OutboundFailure { peer, error, ..})) => { - log_outbound_failure(peer, error, metrics.clone()); + handle_outbound_failure( + peer, + error, + metrics.clone(), + driver_state.clone(), + peer_exclusions.clone(), + ).await; } SwarmEvent::Behaviour(NockchainEvent::RequestResponse(InboundFailure { peer, error, .. })) => { log_inbound_failure(peer, error, metrics.clone()); @@ -326,11 +390,26 @@ pub fn make_libp2p_driver( match result { Ok(duration) => { state_guard.ping_succeeded(connection); + if let Some(ip) = connection_address.as_ref().and_then(|addr| addr.ip_addr()) { + peer_exclusions.record_positive_ip(ip); + } log_ping_success(peer, connection_address, duration); } Err(error) => { let failures = state_guard.ping_failed(connection); log_ping_failure(peer, connection_address.clone(), error); + drop(state_guard); + if let Some(addr) = connection_address.as_ref() { + let outcome = peer_exclusions.record_ping_failure(addr); + record_exclusion_outcome( + &mut swarm, + &driver_state, + &peer_exclusions, + &metrics, + outcome, + &[peer], + ).await; + } if failures >= failed_pings_before_close { if let Some(ip) = connection_address.and_then(|c| c.ip_addr()) { info!("Closing connection to {peer} on {ip} after {failures} failed pings."); @@ -343,7 +422,13 @@ pub fn make_libp2p_driver( } } SwarmEvent::OutgoingConnectionError { error, .. } => { - log_dial_error(error); + handle_outgoing_connection_error( + &mut swarm, + &driver_state, + &peer_exclusions, + &metrics, + error + ).await; }, SwarmEvent::IncomingConnection { local_addr, @@ -378,8 +463,7 @@ pub fn make_libp2p_driver( warn!("SAction: Blocking peer {peer_id}"); // Block the peer in the allow_block_list swarm.behaviour_mut().allow_block_list.block_peer(peer_id); - { - // get peer IP address from the swarm + if peer_exclusions.fail2ban_enabled() { let peer_addresses = swarm.behaviour_mut().peer_store.store().addresses_of_peer(&peer_id); if let Some(peer_multi_addrs) = peer_addresses { for multi_addr in peer_multi_addrs { @@ -489,28 +573,45 @@ async fn handle_effect( fast_sync: bool, driver_state: Arc>, metrics: Arc, + peer_exclusions: PeerExclusions, ) -> Result<(), NockAppError> { match EffectType::from_noun_slab(&noun_slab) { EffectType::Gossip => { - // Get the tail of the gossip effect (after %gossip head) - let mut tail_slab = NounSlab::new(); - let gossip_cell = unsafe { noun_slab.root().as_cell()?.tail() }; - - // Skip version number - // TODO: add version negotiation, reject unknown/incompatible versions - let data_cell = gossip_cell.as_cell()?.tail(); - tail_slab.copy_into(data_cell); - - // Check if this is a heard-block gossip - let gossip_noun = unsafe { tail_slab.root() }; - if let Ok(data_cell) = gossip_noun.as_cell() { - if data_cell.head().eq_bytes(b"heard-block") { - trace!("Gossip effect for heard-block, clearing block and elders cache"); - let mut state_guard = driver_state.lock().await; - state_guard.block_cache.clear(); - state_guard.elders_cache.clear(); - state_guard.elders_negative_cache.clear(); - } + let (tail_slab, is_heard_block) = { + // Get the tail of the gossip effect (after %gossip head) + let mut tail_slab = NounSlab::new(); + let space = noun_slab.noun_space(); + let gossip_cell = unsafe { *noun_slab.root() } + .in_space(&space) + .as_cell()? + .tail() + .noun(); + + // Skip version number + // TODO: add version negotiation, reject unknown/incompatible versions + let data_cell = gossip_cell.in_space(&space).as_cell()?.tail().noun(); + tail_slab.copy_into(data_cell, &space); + + // Check if this is a heard-block gossip + let is_heard_block = { + let tail_space = tail_slab.noun_space(); + let gossip_noun = unsafe { *tail_slab.root() }; + if let Ok(data_cell) = gossip_noun.in_space(&tail_space).as_cell() { + data_cell.head().eq_bytes(b"heard-block") + } else { + false + } + }; + + (tail_slab, is_heard_block) + }; + + if is_heard_block { + trace!("Gossip effect for heard-block, clearing block and elders cache"); + let mut state_guard = driver_state.lock().await; + state_guard.block_cache.clear(); + state_guard.elders_cache.clear(); + state_guard.elders_negative_cache.clear(); } let gossip_request = NockchainRequest::new_gossip(&tail_slab); @@ -529,61 +630,94 @@ async fn handle_effect( } } EffectType::Request => { - // Extract request details to check if it's a peer-specific request - let request_cell = unsafe { noun_slab.root().as_cell()? }; - let request_body = request_cell.tail().as_cell()?; - let request_type = request_body.head().as_direct()?; - - let mut is_limited_request = false; - - let target_peers = if request_type.data() == tas!(b"block") { - let block_cell = request_body.tail().as_cell()?; - if block_cell.head().eq_bytes(b"elders") { - // Extract peer ID from elders request - let elders_cell = block_cell.tail().as_cell()?; - let peer_id_atom = elders_cell.tail().as_atom()?; - if let Ok(bytes) = peer_id_atom.to_bytes_until_nul() { - if let Ok(peer_id) = PeerId::from_bytes(&bytes) { - vec![peer_id] + let (target_peers, is_limited_request, raw_tx_id, request_desc) = { + let space = noun_slab.noun_space(); + // Extract request details to check if it's a peer-specific request + let request_cell = unsafe { *noun_slab.root() }.in_space(&space).as_cell()?; + let request_body = request_cell.tail().as_cell()?; + let request_type = request_body.head().noun().as_direct()?; + + let mut is_limited_request = false; + let request_tag = request_type.data(); + let mut request_desc: String = if request_tag == tas!(b"block") { + "block".to_string() + } else if request_tag == tas!(b"raw-tx") { + "raw-tx".to_string() + } else { + format!("tag-{request_tag}") + }; + + let target_peers = if request_tag == tas!(b"block") { + let block_cell = request_body.tail().as_cell()?; + if block_cell.head().eq_bytes(b"elders") { + request_desc = "block/elders".to_string(); + // Extract peer ID from elders request + let elders_cell = block_cell.tail().as_cell()?; + let peer_id_atom = elders_cell.tail().as_atom()?; + if let Ok(bytes) = peer_id_atom.to_bytes_until_nul() { + if let Ok(peer_id) = PeerId::from_bytes(&bytes) { + vec![peer_id] + } else { + is_limited_request = fast_sync; + connected_peers.clone() + } } else { is_limited_request = fast_sync; connected_peers.clone() } } else { - is_limited_request = fast_sync; connected_peers.clone() } } else { connected_peers.clone() - } - } else { - connected_peers.clone() - }; + }; - if request_type.data() == tas!(b"raw-tx") { - if let Ok(raw_tx_cell) = request_body.tail().as_cell() { - if raw_tx_cell.head().eq_bytes(b"by-id") { - is_limited_request = fast_sync; - trace!("Requesting raw transaction by ID, removing ID from seen set"); - let tx_id = tip5_hash_to_base58_stack(&mut noun_slab, raw_tx_cell.tail())?; - let mut state_guard = driver_state.clone().lock_owned().await; - state_guard.seen_txs.remove(&tx_id); + let raw_tx_id = if request_tag == tas!(b"raw-tx") { + if let Ok(raw_tx_cell) = request_body.tail().as_cell() { + if raw_tx_cell.head().eq_bytes(b"by-id") { + is_limited_request = fast_sync; + request_desc = "raw-tx/by-id".to_string(); + trace!("Requesting raw transaction by ID, removing ID from seen set"); + Some(tip5_hash_to_base58_stack( + &mut noun_slab, + raw_tx_cell.tail().noun(), + &space, + )?) + } else { + None + } + } else { + None } - } + } else { + None + }; + + Ok::<_, NockAppError>((target_peers, is_limited_request, raw_tx_id, request_desc)) + }?; + + if let Some(tx_id) = raw_tx_id { + let mut state_guard = driver_state.clone().lock_owned().await; + state_guard.seen_txs.remove(&tx_id); } - let request_peers: Vec<_> = if is_limited_request { - let mut rng = rng(); - let mut request_peers = target_peers.clone(); - request_peers.shuffle(&mut rng); - request_peers.into_iter().take(2).collect() - } else { - let mut rng = rng(); - let mut request_peers = target_peers.clone(); - request_peers.shuffle(&mut rng); - request_peers.into_iter().take(8).collect() + let request_limit = if is_limited_request { 2 } else { 8 }; + let request_peers = { + let state_guard = driver_state.lock().await; + state_guard.select_request_peers( + target_peers.clone(), + request_limit, + &peer_exclusions, + ) }; - debug!("Sending request to {} peers", request_peers.len()); + info!( + "Sending {request_desc} request to {} peer(s): {:?}", + request_peers.len(), + request_peers + .iter() + .map(|p| p.to_base58()) + .collect::>() + ); for peer_id in request_peers { let local_peer_id_clone = local_peer_id; let mut equix_builder_clone = equix_builder.clone(); @@ -599,29 +733,32 @@ async fn handle_effect( } } EffectType::LiarPeer => { - let effect_cell = unsafe { noun_slab.root().as_cell()? }; - let liar_peer_cell = effect_cell.tail().as_cell().map_err(|_| { - NockAppError::IoError(std::io::Error::other( - "Expected peer ID cell in liar-peer effect", - )) - })?; - let peer_id_atom = liar_peer_cell.head().as_atom().map_err(|_| { - NockAppError::IoError(std::io::Error::other( - "Expected peer ID atom in liar-peer effect", - )) - })?; - - let bytes = peer_id_atom - .to_bytes_until_nul() - .expect("failed to strip null bytes"); - - let peer_id_str = String::from_utf8(bytes).map_err(|_| { - NockAppError::IoError(std::io::Error::other("Invalid UTF-8 in peer ID")) - })?; - - let peer_id = PeerId::from_str(&peer_id_str).map_err(|_| { - NockAppError::IoError(std::io::Error::other("Invalid peer ID format")) - })?; + let peer_id = { + let space = noun_slab.noun_space(); + let effect_cell = unsafe { *noun_slab.root() }.in_space(&space).as_cell()?; + let liar_peer_cell = effect_cell.tail().as_cell().map_err(|_| { + NockAppError::IoError(std::io::Error::other( + "Expected peer ID cell in liar-peer effect", + )) + })?; + let peer_id_atom = liar_peer_cell.head().as_atom().map_err(|_| { + NockAppError::IoError(std::io::Error::other( + "Expected peer ID atom in liar-peer effect", + )) + })?; + + let bytes = peer_id_atom + .to_bytes_until_nul() + .expect("failed to strip null bytes"); + + let peer_id_str = String::from_utf8(bytes).map_err(|_| { + NockAppError::IoError(std::io::Error::other("Invalid UTF-8 in peer ID")) + })?; + + PeerId::from_str(&peer_id_str).map_err(|_| { + NockAppError::IoError(std::io::Error::other("Invalid peer ID format")) + })? + }; swarm_tx .send(SwarmAction::BlockPeer { peer_id }) @@ -631,18 +768,24 @@ async fn handle_effect( })?; } EffectType::LiarBlockId => { - let effect_cell = unsafe { noun_slab.root().as_cell()? }; - let liar_block_cell = effect_cell.tail().as_cell().map_err(|_| { - NockAppError::IoError(std::io::Error::other( - "Expected block ID cell in liar-block-id effect", - )) - })?; + let block_id = { + let space = noun_slab.noun_space(); + let effect_cell = unsafe { *noun_slab.root() }.in_space(&space).as_cell()?; + let liar_block_cell = effect_cell.tail().as_cell().map_err(|_| { + NockAppError::IoError(std::io::Error::other( + "Expected block ID cell in liar-block-id effect", + )) + })?; - let block_id = liar_block_cell.head(); + liar_block_cell.head().noun() + }; // Add the bad block ID let mut state_guard = driver_state.lock().await; - let peers_to_ban = state_guard.process_bad_block_id(block_id)?; + let peers_to_ban = { + let space = noun_slab.noun_space(); + state_guard.process_bad_block_id(block_id, &space)? + }; // Ban each peer that sent this block for peer_id in peers_to_ban { @@ -655,82 +798,131 @@ async fn handle_effect( } } EffectType::Track => { - let effect_cell = unsafe { noun_slab.root().as_cell()? }; - let track_cell = effect_cell.tail().as_cell()?; - let action = track_cell.head(); - - if action.eq_bytes(b"add") { - // Handle [%track %add block-id peer-id] - let data_cell = track_cell.tail().as_cell()?; - let block_id = data_cell.head(); - let peer_id_atom = data_cell.tail().as_atom()?; - - // Convert peer_id from base58 string to PeerId - let Ok(peer_id) = PeerId::from_noun(peer_id_atom.as_noun()) else { - return Err(NockAppError::OtherError(String::from( - "Invalid peer ID format", - ))); - }; + enum TrackAction { + Add { block_id: Noun, peer_id: PeerId }, + Remove { block_id: Noun }, + } - // Add to message tracker - let mut state_guard = driver_state.lock().await; - state_guard.track_block_id_and_peer(block_id, peer_id)?; - } else if action.eq_bytes(b"remove") { - // Handle [%track %remove block-id] - let block_id = track_cell.tail(); + let track_action = { + let space = noun_slab.noun_space(); + let effect_cell = unsafe { *noun_slab.root() }.in_space(&space).as_cell()?; + let track_cell = effect_cell.tail().as_cell()?; + let action = track_cell.head(); + + if action.eq_bytes(b"add") { + // Handle [%track %add block-id peer-id] + let data_cell = track_cell.tail().as_cell()?; + let block_id = data_cell.head().noun(); + let peer_id_atom = data_cell.tail().as_atom()?; + + // Convert peer_id from base58 string to PeerId + let Ok(peer_id) = PeerId::from_noun(peer_id_atom.atom().as_noun(), &space) + else { + return Err(NockAppError::OtherError(String::from( + "Invalid peer ID format", + ))); + }; - // Remove from message tracker - let mut state_guard = driver_state.lock().await; - state_guard.remove_block_id(block_id)?; - } else { - return Err(NockAppError::IoError(std::io::Error::other( - "Invalid track action", - ))); + Ok(TrackAction::Add { block_id, peer_id }) + } else if action.eq_bytes(b"remove") { + // Handle [%track %remove block-id] + let block_id = track_cell.tail().noun(); + Ok(TrackAction::Remove { block_id }) + } else { + Err(NockAppError::IoError(std::io::Error::other( + "Invalid track action", + ))) + } + }?; + + match track_action { + TrackAction::Add { block_id, peer_id } => { + // Add to message tracker + let mut state_guard = driver_state.lock().await; + let space = noun_slab.noun_space(); + state_guard.track_block_id_and_peer(block_id, peer_id, &space)?; + } + TrackAction::Remove { block_id } => { + // Remove from message tracker + let mut state_guard = driver_state.lock().await; + let space = noun_slab.noun_space(); + state_guard.remove_block_id(block_id, &space)?; + } } } EffectType::Seen => { - let effect_cell = unsafe { noun_slab.root().as_cell()? }; - let seen_cell = effect_cell.tail().as_cell()?; - let seen_type = seen_cell.head(); + enum SeenAction { + Block { id: String, height: Option }, + Tx { id: String }, + Unknown, + } - if seen_type.eq_bytes(b"block") { - let seen_pq = seen_cell.tail().as_cell()?; - let block_id = seen_pq.head().as_cell()?; - let mut state_guard = driver_state.lock().await; - let block_id_str = tip5_hash_to_base58_stack(&mut noun_slab, block_id.as_noun()) - .expect("failed to convert block ID to base58"); - trace!("seen block id: {:?}", &block_id_str); - state_guard.seen_blocks.insert(block_id_str); - - if let Ok(block_height_unit_cell) = seen_pq.tail().as_cell() { - let block_height = block_height_unit_cell.tail().as_atom()?.as_u64()?; - if state_guard.first_negative <= block_height { - metrics.highest_block_height_seen.swap(block_height as f64); - state_guard.first_negative = block_height + 1; - trace!( - "Setting state_guard.first_negative to {:?}", - state_guard.first_negative - ); - - // Check if we should clear the tx cache - if block_height - >= state_guard.last_tx_cache_clear_height - + state_guard.seen_tx_clear_interval - { - debug!("Clearing seen_txs cache at block height {}", block_height); - debug!("Cache before clearing: {:?}", state_guard.seen_txs); - state_guard.seen_txs.clear(); - state_guard.last_tx_cache_clear_height = block_height; + let seen_action = { + let space = noun_slab.noun_space(); + let effect_cell = unsafe { *noun_slab.root() }.in_space(&space).as_cell()?; + let seen_cell = effect_cell.tail().as_cell()?; + let seen_type = seen_cell.head(); + + if seen_type.eq_bytes(b"block") { + let seen_pq = seen_cell.tail().as_cell()?; + let block_id = seen_pq.head().noun(); + let block_id_str = tip5_hash_to_base58_stack(&mut noun_slab, block_id, &space) + .expect("failed to convert block ID to base58"); + let block_height = if let Ok(block_height_unit_cell) = seen_pq.tail().as_cell() + { + Some(block_height_unit_cell.tail().as_atom()?.as_u64()?) + } else { + None + }; + SeenAction::Block { + id: block_id_str, + height: block_height, + } + } else if seen_type.eq_bytes(b"tx") { + let tx_id = seen_cell.tail().as_cell()?; + let tx_id_str = + tip5_hash_to_base58_stack(&mut noun_slab, tx_id.cell().as_noun(), &space) + .expect("failed to convert tx ID to base58"); + SeenAction::Tx { id: tx_id_str } + } else { + SeenAction::Unknown + } + }; + + match seen_action { + SeenAction::Block { id, height } => { + let mut state_guard = driver_state.lock().await; + trace!("seen block id: {:?}", &id); + state_guard.seen_blocks.insert(id); + + if let Some(block_height) = height { + if state_guard.first_negative <= block_height { + metrics.highest_block_height_seen.swap(block_height as f64); + state_guard.first_negative = block_height + 1; + trace!( + "Setting state_guard.first_negative to {:?}", + state_guard.first_negative + ); + + // Check if we should clear the tx cache + if block_height + >= state_guard.last_tx_cache_clear_height + + state_guard.seen_tx_clear_interval + { + debug!("Clearing seen_txs cache at block height {}", block_height); + debug!("Cache before clearing: {:?}", state_guard.seen_txs); + state_guard.seen_txs.clear(); + state_guard.last_tx_cache_clear_height = block_height; + } } } } - } else if seen_type.eq_bytes(b"tx") { - let tx_id = seen_cell.tail().as_cell()?; - let mut state_guard = driver_state.lock().await; - let tx_id_str = tip5_hash_to_base58_stack(&mut noun_slab, tx_id.as_noun()) - .expect("failed to convert tx ID to base58"); - trace!("seen tx id: {:?}", &tx_id_str); - state_guard.seen_txs.insert(tx_id_str); + SeenAction::Tx { id } => { + let mut state_guard = driver_state.lock().await; + trace!("seen tx id: {:?}", &id); + state_guard.seen_txs.insert(id); + } + SeenAction::Unknown => {} } } EffectType::Unknown => { @@ -754,6 +946,7 @@ async fn handle_request_response( metrics: Arc, driver_state: Arc>, request_high_threshold: u64, + peer_exclusions: PeerExclusions, ) -> Result<(), NockAppError> { trace!("handle_request_response peer: {peer}"); match message { @@ -776,6 +969,7 @@ async fn handle_request_response( let addr_str = addr.to_string(); debug!("Request received from peer at address {addr_str} with id {peer}"); if let Some(ip) = addr.ip_addr() { + peer_exclusions.record_positive_ip(ip); let threshold_exceeded = driver_state .lock() .await @@ -800,7 +994,10 @@ async fn handle_request_response( let message_bytes = Bytes::from(message.to_vec()); let request_noun = request_slab.cue_into(message_bytes)?; - let data_request = NockchainDataRequest::from_noun(request_noun)?; + let data_request = { + let space = request_slab.noun_space(); + NockchainDataRequest::from_noun(request_noun, &space)? + }; let cached = { let cache_result = { @@ -850,11 +1047,20 @@ async fn handle_request_response( } Ok(None) => { metrics.requests_peeked_none.increment(); + let request_type = { + let space = request_slab.noun_space(); + request_noun + .in_space(&space) + .as_cell() + .ok() + .and_then(|cell| cell.tail().as_cell().ok()) + .map(|cell| cell.head().noun()) + }; trace!( - "No data found for incoming request from: {}, request type: {:?}", - peer, - request_noun.as_cell()?.tail().as_cell().map(|c| c.head()) - ); + "No data found for incoming request from: {}, request type: {:?}", + peer, + request_type + ); None } Err(NockAppError::MPSCFullError(act)) => { @@ -906,7 +1112,13 @@ async fn handle_request_response( let response = match data_request { NockchainDataRequest::BlockByHeight(height) => { let scry_res = unsafe { scry_res_slab.root() }; - match create_scry_response(scry_res, "heard-block", &mut res_slab) { + let res = { + let scry_space = scry_res_slab.noun_space(); + create_scry_response( + scry_res, &scry_space, "heard-block", &mut res_slab, + ) + }; + match res { Left(()) => { trace!("No data found for incoming block by-height request"); NockchainResponse::Ack { acked: true } @@ -924,7 +1136,13 @@ async fn handle_request_response( } NockchainDataRequest::EldersById(id, _, _) => { let scry_res = unsafe { scry_res_slab.root() }; - match create_scry_response(scry_res, "heard-elders", &mut res_slab) { + let res = { + let scry_space = scry_res_slab.noun_space(); + create_scry_response( + scry_res, &scry_space, "heard-elders", &mut res_slab, + ) + }; + match res { Left(()) => { trace!("No data found for incoming elders request"); let mut state_guard = driver_state.lock().await; @@ -942,7 +1160,13 @@ async fn handle_request_response( } NockchainDataRequest::RawTransactionById(ref id, _) => { let scry_res = unsafe { scry_res_slab.root() }; - match create_scry_response(scry_res, "heard-tx", &mut res_slab) { + let res = { + let scry_space = scry_res_slab.noun_space(); + create_scry_response( + scry_res, &scry_space, "heard-tx", &mut res_slab, + ) + }; + match res { Left(()) => { trace!("No data found for incoming raw-tx request"); NockchainResponse::Ack { acked: true } @@ -1036,11 +1260,17 @@ async fn handle_request_response( let wire = Libp2pWire::Gossip(peer); - trace!( - "Poking kernel with wire: {:?} noun: {:?}", - wire, - nockvm::noun::FullDebugCell(unsafe { &request_slab.root().as_cell()? }) - ); + { + let space = request_slab.noun_space(); + trace!( + "Poking kernel with wire: {:?} noun: {:?}", + wire, + nockvm::noun::FullDebugCell { + cell: unsafe { &request_slab.root().as_cell()? }, + space: &space, + } + ); + } let poke = gossip.fact_poke(); let (timing, timing_rx) = tokio::sync::oneshot::channel(); @@ -1147,10 +1377,16 @@ async fn handle_request_response( let response_noun = response_slab.cue_into(message_bytes)?; response_slab.set_root(response_noun); - trace!( - "Response noun: {:?}", - nockvm::noun::FullDebugCell(&response_noun.as_cell()?) - ); + { + let trace_space = response_slab.noun_space(); + trace!( + "Response noun: {:?}", + nockvm::noun::FullDebugCell { + cell: &response_noun.as_cell()?, + space: &trace_space, + } + ); + } let response = NockchainFact::from_noun_slab(&mut response_slab)?; let response_cell = unsafe { response_slab.root().as_cell() }?; @@ -1219,12 +1455,23 @@ async fn handle_request_response( .await; let elapsed = timing_rx.await?; - if response_cell.head().eq_bytes(b"heard-block") { - metrics.heard_block_poke_time.add_timing(&elapsed); - } else if response_cell.head().eq_bytes(b"heard-tx") { - metrics.heard_tx_poke_time.add_timing(&elapsed); - } else if response_cell.head().eq_bytes(b"heard-elders") { - metrics.heard_elders_poke_time.add_timing(&elapsed); + { + let space = response_slab.noun_space(); + if response_cell + .in_space(&space) + .head() + .eq_bytes(b"heard-block") + { + metrics.heard_block_poke_time.add_timing(&elapsed); + } else if response_cell.in_space(&space).head().eq_bytes(b"heard-tx") { + metrics.heard_tx_poke_time.add_timing(&elapsed); + } else if response_cell + .in_space(&space) + .head() + .eq_bytes(b"heard-elders") + { + metrics.heard_elders_poke_time.add_timing(&elapsed); + } } match poke_result { @@ -1289,6 +1536,8 @@ async fn handle_request_response( trace!("Error sending poke") } } + peer_exclusions.record_peer_request_success(&peer); + driver_state.lock().await.record_request_success(peer); trace!("handle_request_response: Poke successful"); } NockchainResponse::Ack { acked } => { @@ -1305,7 +1554,20 @@ async fn handle_request_response( async fn log_peer_status( swarm: &mut Swarm, metrics: &NockchainP2PMetrics, + peer_exclusions: &PeerExclusions, + driver_state: &Arc>, ) -> usize { + let expired = peer_exclusions.expire(); + for _ in 0..expired.ips { + metrics.ip_exclusions_expired.increment(); + } + let _ = metrics + .ip_exclusions_active + .swap(peer_exclusions.active_ip_exclusion_count() as f64); + let _ = metrics + .address_cooldowns_active + .swap(peer_exclusions.active_address_cooldown_count() as f64); + let connected_peer_count = { info!("Logging current peer status..."); let connected_peers: Vec<_> = swarm.connected_peers().cloned().collect(); @@ -1346,6 +1608,8 @@ async fn log_peer_status( "Routing table has {} entries", routing_table_size ); }; + observe_kad_cardinality_and_exclude(swarm, driver_state, peer_exclusions, metrics).await; + prune_excluded_swarm_state(swarm, driver_state, peer_exclusions, metrics).await; connected_peer_count } @@ -1378,7 +1642,7 @@ fn request_to_scry_slab(request: NockchainDataRequest) -> Result { debug!("Requesting block by height: {}", height); let mut slab = NounSlab::new(); - let height_atom = Noun::from_atom(Atom::new(&mut slab, height)); + let height_atom = Atom::new(&mut slab, height).as_noun(); let noun = T(&mut slab, &[D(tas!(b"heavy-n")), height_atom, D(0)]); slab.set_root(noun); Ok(slab) @@ -1417,10 +1681,11 @@ fn request_to_scry_slab(request: NockchainDataRequest) -> Result Either<(), Result> { - match ScryResult::from(scry_res) { + match ScryResult::from_noun(scry_res, space) { ScryResult::BadPath => { warn!("Bad scry path"); Left(()) @@ -1430,7 +1695,8 @@ fn create_scry_response( Left(()) } ScryResult::Some(x) => { - let nouns = vec![x]; + let imported = res_slab.copy_into(x.noun(), x.space()); + let nouns = vec![imported]; if let Ok(response_noun) = prepend_tas(res_slab, heard_type, nouns) { res_slab.set_root(response_noun); Right(Ok(NockchainResponse::new_response_result(res_slab.jam()))) @@ -1509,6 +1775,17 @@ fn record_crown_error_metric(error: &CrownError, metrics: &Nockch CrownError::SerfLoadError => { metrics.requests_crown_error_serf_load_error.increment(); } + CrownError::SerfInitAllocationError(_) => { + metrics + .requests_crown_error_serf_init_allocation_error + .increment(); + } + CrownError::SerfInitPanic(_) => { + metrics.requests_crown_error_serf_init_panic.increment(); + } + CrownError::CheckpointKernelHashMismatch { .. } => { + metrics.requests_crown_error_serf_load_error.increment(); + } CrownError::WorkBail => { metrics.requests_crown_error_work_bail.increment(); } @@ -1576,14 +1853,19 @@ mod tests { use std::sync::LazyLock; use nockapp::noun::slab::NounSlab; - use nockvm::noun::{D, T}; + use nockvm::mem::{NockStack, NOCK_STACK_SIZE_TINY}; + use nockvm::noun::{NounAllocator, D, T}; use nockvm_macros::tas; use serde_bytes::ByteBuf; - use super::*; + use crate::driver::*; pub static LIBP2P_CONFIG: LazyLock = LazyLock::new(LibP2PConfig::default); + fn install_test_arena() -> NockStack { + NockStack::new(NOCK_STACK_SIZE_TINY, 0) + } + #[test] #[cfg_attr(miri, ignore)] // ibig has a memory leak so miri fails this test fn test_request_to_scry_slab() { @@ -1597,7 +1879,8 @@ mod tests { let request = T(&mut slab, &[D(tas!(b"request")), block_cell]); slab.set_root(request); - let data_request = NockchainDataRequest::from_noun(request) + let space = slab.noun_space(); + let data_request = NockchainDataRequest::from_noun(request, &space) .expect("Failed to create request from noun"); let result_slab = request_to_scry_slab(data_request).unwrap_or_else(|_| { @@ -1609,6 +1892,7 @@ mod tests { ) }); let result = unsafe { result_slab.root() }; + let result_space = result_slab.noun_space(); assert!(result.is_cell()); let result_cell = result.as_cell().unwrap_or_else(|_| { @@ -1619,6 +1903,7 @@ mod tests { option_env!("GIT_SHA").unwrap_or("unknown") ) }); + let result_cell = result_cell.in_space(&result_space); assert!(result_cell.head().eq_bytes(b"heavy-n")); // Get the tail cell and check its components @@ -1658,14 +1943,18 @@ mod tests { ) }); assert_eq!( - tail_atom.as_u64().unwrap_or_else(|err| { - panic!( - "Panicked with {err:?} at {}:{} (git sha: {:?})", - file!(), - line!(), - option_env!("GIT_SHA") - ) - }), + tail_atom + .atom() + .as_direct() + .unwrap_or_else(|err| { + panic!( + "Panicked with {err:?} at {}:{} (git sha: {:?})", + file!(), + line!(), + option_env!("GIT_SHA") + ) + }) + .data(), 0 ); } @@ -1674,7 +1963,8 @@ mod tests { { let mut slab: NounSlab = NounSlab::new(); slab.set_root(D(123)); - let result = NockchainDataRequest::from_noun(*unsafe { slab.root() }) + let space = slab.noun_space(); + let result = NockchainDataRequest::from_noun(*unsafe { slab.root() }, &space) .and_then(request_to_scry_slab); assert!(result.is_err()); } @@ -1696,7 +1986,8 @@ mod tests { let request = T(&mut slab, &[D(tas!(b"request")), block_cell]); slab.set_root(request); - let data_request = NockchainDataRequest::from_noun(request) + let space = slab.noun_space(); + let data_request = NockchainDataRequest::from_noun(request, &space) .expect("Could not create request from noun"); let result_slab = request_to_scry_slab(data_request).unwrap_or_else(|_| { @@ -1708,6 +1999,7 @@ mod tests { ) }); let result = unsafe { result_slab.root() }; + let result_space = result_slab.noun_space(); // Verify the structure: [%elders block_id_b58 0] assert!(result.is_cell()); @@ -1721,6 +2013,7 @@ mod tests { }); // Check %elders tag + let result_cell = result_cell.in_space(&result_space); assert!(result_cell.head().eq_bytes(b"elders")); // Get the tail cell @@ -1760,7 +2053,7 @@ mod tests { }); // Get the expected base58 string - let expected_b58 = tip5_hash_to_base58(five_tuple).unwrap_or_else(|_| { + let expected_b58 = tip5_hash_to_base58(five_tuple, &space).unwrap_or_else(|_| { panic!( "Called `expect()` at {}:{} (git sha: {})", file!(), @@ -1771,9 +2064,17 @@ mod tests { assert_eq!(block_id_str, expected_b58); // Check final 0 + let tail_atom = tail_cell.tail().as_atom().unwrap_or_else(|_| { + panic!( + "Called `expect()` at {}:{} (git sha: {})", + file!(), + line!(), + option_env!("GIT_SHA").unwrap_or("unknown") + ) + }); assert_eq!( - tail_cell - .tail() + tail_atom + .atom() .as_direct() .unwrap_or_else(|err| { panic!( @@ -1797,16 +2098,80 @@ mod tests { ); slab.set_root(invalid_request); - let result = - NockchainDataRequest::from_noun(invalid_request).and_then(request_to_scry_slab); + let space = slab.noun_space(); + let result = NockchainDataRequest::from_noun(invalid_request, &space) + .and_then(request_to_scry_slab); assert!(result.is_err()); drop(slab); } } + #[test] + #[cfg_attr(miri, ignore = "memfd_create unsupported in Miri")] + fn test_create_scry_response_wraps_peek_payload_in_response() { + let mut scry_res_slab: NounSlab = NounSlab::new(); + let payload = T(&mut scry_res_slab, &[D(1), D(2)]); + let scry_result = T(&mut scry_res_slab, &[D(0), D(0), payload]); + scry_res_slab.set_root(scry_result); + + let scry_space = scry_res_slab.noun_space(); + match ScryResult::from_noun(unsafe { scry_res_slab.root() }, &scry_space) { + ScryResult::Some(found) => { + assert!(unsafe { found.noun().raw_equals(&payload) }); + } + _ => panic!("expected Some(payload) scry result"), + } + + let mut res_slab = NounSlab::new(); + let response = create_scry_response( + unsafe { scry_res_slab.root() }, + &scry_space, + "heard-block", + &mut res_slab, + ); + let Right(Ok(NockchainResponse::Result { message })) = response else { + panic!("expected a successful response result"); + }; + + let mut decoded_slab: NounSlab = NounSlab::new(); + decoded_slab + .cue_into(message.into_vec().into()) + .expect("response jam should decode"); + let decoded_space = decoded_slab.noun_space(); + let response_cell = unsafe { decoded_slab.root() } + .in_space(&decoded_space) + .as_cell() + .expect("response root should be a cell"); + assert!(response_cell.head().eq_bytes(b"heard-block")); + + let payload_cell = response_cell + .tail() + .as_cell() + .expect("heard-block payload should be a cell"); + assert_eq!( + payload_cell + .head() + .as_atom() + .expect("payload head should be an atom") + .as_u64() + .expect("payload head should fit in u64"), + 1 + ); + assert_eq!( + payload_cell + .tail() + .as_atom() + .expect("payload tail should be an atom") + .as_u64() + .expect("payload tail should fit in u64"), + 2 + ); + } + #[test] #[cfg_attr(miri, ignore)] // equix uses a foreign function so miri fails this tes fn test_equix_pow_verification() { + let _arena = install_test_arena(); use equix::EquiXBuilder; // Create EquiX builder - new() doesn't return Result let mut builder = EquiXBuilder::new(); @@ -1874,9 +2239,10 @@ mod tests { ); } - #[tokio::test] + #[tokio::test(flavor = "current_thread")] #[cfg_attr(miri, ignore)] async fn test_liar_peer_effect() { + let _arena = install_test_arena(); use equix::EquiXBuilder; use tokio::sync::mpsc; @@ -1917,6 +2283,7 @@ mod tests { LIBP2P_CONFIG.seen_tx_clear_interval, ))), metrics, + PeerExclusions::default(), ) .await; @@ -1937,9 +2304,10 @@ mod tests { assert!(swarm_rx.try_recv().is_err(), "Should only send one action"); } - #[tokio::test] + #[tokio::test(flavor = "current_thread")] #[cfg_attr(miri, ignore)] // ibig has a memory leak so miri fails this test async fn test_track_add_effect() { + let _arena = install_test_arena(); use equix::EquiXBuilder; use tokio::sync::mpsc; @@ -1962,6 +2330,7 @@ mod tests { let add_cell = T(&mut effect_slab, &[add_atom.as_noun(), data_cell]); let track_cell = T(&mut effect_slab, &[track_atom.as_noun(), add_cell]); effect_slab.set_root(track_cell); + let space = effect_slab.noun_space(); // Create message tracker and other required components let (swarm_tx, _swarm_rx) = mpsc::channel(1); @@ -1985,6 +2354,7 @@ mod tests { false, state_arc.clone(), metrics, + PeerExclusions::default(), ) .await; @@ -1992,7 +2362,7 @@ mod tests { assert!(result.is_ok(), "handle_effect should succeed"); // Get the expected block ID string (base58 of [1 2 3 4 5]) - let block_id_str = tip5_hash_to_base58(block_id_tuple).unwrap_or_else(|_| { + let block_id_str = tip5_hash_to_base58(block_id_tuple, &space).unwrap_or_else(|_| { panic!( "Called `expect()` at {}:{} (git sha: {})", file!(), @@ -2005,7 +2375,7 @@ mod tests { let state_guard = state_arc.lock().await; // Check block_id_to_peers mapping - let peers = state_guard.get_peers_for_block_id(block_id_tuple); + let peers = state_guard.get_peers_for_block_id(block_id_tuple, &space); assert!( peers.contains(&peer_id), "Peer ID should be associated with block ID" @@ -2019,9 +2389,10 @@ mod tests { ); } - #[tokio::test] + #[tokio::test(flavor = "current_thread")] #[cfg_attr(miri, ignore)] // ibig has a memory leak so miri fails this test async fn test_track_remove_effect() { + let _arena = install_test_arena(); use equix::EquiXBuilder; use tokio::sync::mpsc; @@ -2041,11 +2412,12 @@ mod tests { // Create block ID as [1 2 3 4 5] let mut setup_slab: NounSlab = NounSlab::new(); let block_id_tuple = T(&mut setup_slab, &[D(1), D(2), D(3), D(4), D(5)]); + let setup_space = setup_slab.noun_space(); { let mut state_guard = state_arc.lock().await; state_guard - .track_block_id_and_peer(block_id_tuple, peer_id) + .track_block_id_and_peer(block_id_tuple, peer_id, &setup_space) .unwrap_or_else(|_| { panic!( "Called `expect()` at {}:{} (git sha: {})", @@ -2056,7 +2428,7 @@ mod tests { }); // Verify it was added correctly - assert!(state_guard.is_tracking_block_id(block_id_tuple)); + assert!(state_guard.is_tracking_block_id(block_id_tuple, &setup_space)); assert!(state_guard.is_tracking_peer(peer_id)); } @@ -2094,6 +2466,7 @@ mod tests { false, state_arc.clone(), metrics, + PeerExclusions::default(), ) .await; @@ -2105,7 +2478,7 @@ mod tests { // Check that the block ID was removed from block_id_to_peers assert!( - !state_guard.is_tracking_block_id(block_id_tuple), + !state_guard.is_tracking_block_id(block_id_tuple, &setup_space), "Block ID should be removed" ); @@ -2117,9 +2490,10 @@ mod tests { ); } - #[tokio::test] + #[tokio::test(flavor = "current_thread")] #[cfg_attr(miri, ignore)] // ibig has a memory leak so miri fails this test async fn test_liar_block_id_effect() { + let _arena = install_test_arena(); use equix::EquiXBuilder; use tokio::sync::mpsc; @@ -2150,6 +2524,7 @@ mod tests { let bad_block_id = T(&mut setup_slab, &[D(1), D(2), D(3), D(4), D(5)]); // Good block ID as [6 7 8 9 10] let good_block_id = T(&mut setup_slab, &[D(6), D(7), D(8), D(9), D(10)]); + let setup_space = setup_slab.noun_space(); println!("Created block_ids"); { @@ -2158,7 +2533,7 @@ mod tests { // Associate bad_peer1 with the bad block state_guard - .track_block_id_and_peer(bad_block_id, bad_peer1) + .track_block_id_and_peer(bad_block_id, bad_peer1, &setup_space) .unwrap_or_else(|_| { panic!( "Called `expect()` at {}:{} (git sha: {})", @@ -2170,7 +2545,7 @@ mod tests { // Associate bad_peer2 with the bad block state_guard - .add_peer_if_tracking_block_id(bad_block_id, bad_peer2) + .add_peer_if_tracking_block_id(bad_block_id, bad_peer2, &setup_space) .unwrap_or_else(|_| { panic!( "Called `expect()` at {}:{} (git sha: {})", @@ -2182,7 +2557,7 @@ mod tests { // Associate good_peer with a different block state_guard - .track_block_id_and_peer(good_block_id, good_peer) + .track_block_id_and_peer(good_block_id, good_peer, &setup_space) .unwrap_or_else(|_| { panic!( "Called `expect()` at {}:{} (git sha: {})", @@ -2193,8 +2568,8 @@ mod tests { }); // Verify tracking is working - assert!(state_guard.is_tracking_block_id(bad_block_id)); - assert!(state_guard.is_tracking_block_id(good_block_id)); + assert!(state_guard.is_tracking_block_id(bad_block_id, &setup_space)); + assert!(state_guard.is_tracking_block_id(good_block_id, &setup_space)); assert!(state_guard.is_tracking_peer(bad_peer1)); assert!(state_guard.is_tracking_peer(bad_peer2)); assert!(state_guard.is_tracking_peer(good_peer)); @@ -2238,6 +2613,7 @@ mod tests { false, state_arc.clone(), metrics, + PeerExclusions::default(), ) .await; @@ -2288,13 +2664,13 @@ mod tests { // Bad block should be removed assert!( - !state_guard.is_tracking_block_id(bad_block_id), + !state_guard.is_tracking_block_id(bad_block_id, &setup_space), "Bad block ID should be removed" ); // Good block should still be tracked assert!( - state_guard.is_tracking_block_id(good_block_id), + state_guard.is_tracking_block_id(good_block_id, &setup_space), "Good block ID should still be tracked" ); @@ -2318,16 +2694,18 @@ mod tests { } } - #[tokio::test] + #[tokio::test(flavor = "current_thread")] #[cfg_attr(miri, ignore)] // ibig has a memory leak so miri fails this test async fn test_seen_block_effect() { + let _arena = install_test_arena(); use equix::EquiXBuilder; use tokio::sync::mpsc; let mut effect_slab = NounSlab::new(); let block_id = T(&mut effect_slab, &[D(1), D(2), D(3), D(4), D(5)]); - let block_id_str = tip5_hash_to_base58(block_id).unwrap_or_else(|_| { + let space = effect_slab.noun_space(); + let block_id_str = tip5_hash_to_base58(block_id, &space).unwrap_or_else(|_| { panic!( "Called `expect()` at {}:{} (git sha: {})", file!(), @@ -2362,6 +2740,7 @@ mod tests { false, state_arc_clone, metrics, + PeerExclusions::default(), ) .await; @@ -2373,15 +2752,17 @@ mod tests { assert!(contains, "Block ID should be marked as seen"); } - #[tokio::test] + #[tokio::test(flavor = "current_thread")] #[cfg_attr(miri, ignore)] // ibig has a memory leak so miri fails this test async fn test_seen_tx_effect() { + let _arena = install_test_arena(); use equix::EquiXBuilder; use tokio::sync::mpsc; let mut effect_slab = NounSlab::new(); let tx_id = T(&mut effect_slab, &[D(1), D(2), D(3), D(4), D(5)]); - let tx_id_str = tip5_hash_to_base58(tx_id).unwrap_or_else(|_| { + let space = effect_slab.noun_space(); + let tx_id_str = tip5_hash_to_base58(tx_id, &space).unwrap_or_else(|_| { panic!( "Called `expect()` at {}:{} (git sha: {})", file!(), @@ -2413,6 +2794,7 @@ mod tests { false, state_arc_clone, metrics, + PeerExclusions::default(), ) .await; @@ -2442,6 +2824,303 @@ fn dial_peers( Ok(()) } +/// Walks an error's `source()` chain looking for an [`std::io::Error`] of +/// kind [`PermissionDenied`](std::io::ErrorKind::PermissionDenied). This is +/// how a firewall-blocked egress surfaces: the quinn UDP socket's `sendmsg` +/// returns `EPERM`, which the QUIC transport reports as a `PermissionDenied` +/// io error inside `DialError::Transport`. +fn chain_has_permission_denied(err: &(dyn std::error::Error + 'static)) -> bool { + let mut cur: Option<&(dyn std::error::Error + 'static)> = Some(err); + while let Some(e) = cur { + if let Some(io) = e.downcast_ref::() { + if io.kind() == std::io::ErrorKind::PermissionDenied { + return true; + } + } + cur = e.source(); + } + false +} + +/// The peer id encoded as the trailing `/p2p/` of a multiaddr, if +/// any (Kademlia keys its routing table by it). +fn p2p_peer_id(addr: &Multiaddr) -> Option { + addr.iter().find_map(|p| match p { + Protocol::P2p(peer_id) => Some(peer_id), + _ => None, + }) +} + +fn without_p2p(addr: &Multiaddr) -> Multiaddr { + addr.iter() + .filter(|protocol| !matches!(protocol, Protocol::P2p(_))) + .collect() +} + +async fn handle_outgoing_connection_error( + swarm: &mut Swarm, + driver_state: &Arc>, + peer_exclusions: &PeerExclusions, + metrics: &NockchainP2PMetrics, + error: DialError, +) { + match &error { + // The host answered with a different identity than Kademlia + // advertised. Poisoned peers exploit this with many ports / fresh + // ids on one IP, so we never connect but keep retrying forever. + DialError::WrongPeerId { obtained, address } => { + let obtained = *obtained; + let expected = p2p_peer_id(address); + if address.ip_addr().is_none() { + warn!("WrongPeerId for {address} had no IP component; cannot exclude by IP"); + return; + } + metrics.wrong_peer_id_observed.increment(); + let mut ids: Vec = expected.into_iter().collect(); + ids.push(obtained); + let outcome = peer_exclusions.record_wrong_peer_id(address, expected, obtained); + record_exclusion_outcome(swarm, driver_state, peer_exclusions, metrics, outcome, &ids) + .await; + } + // A firewall is dropping our egress to this address: quinn's + // `sendmsg` returned EPERM (PermissionDenied). Treat this as local + // reachability evidence first, with IP-wide action only after repeats. + DialError::Transport(addr_errs) => { + for (addr, transport_err) in addr_errs { + let dyn_err: &(dyn std::error::Error + 'static) = transport_err; + let ids: Vec = p2p_peer_id(addr).into_iter().collect(); + if chain_has_permission_denied(dyn_err) { + let outcome = peer_exclusions.record_permission_denied(addr); + record_exclusion_outcome( + swarm, driver_state, peer_exclusions, metrics, outcome, &ids, + ) + .await; + } else { + let outcome = peer_exclusions.record_dial_failure(addr, p2p_peer_id(addr)); + record_exclusion_outcome( + swarm, driver_state, peer_exclusions, metrics, outcome, &ids, + ) + .await; + trace!("Failed to dial address {}: {}", addr, transport_err); + } + } + } + _ => log_dial_error(error), + } +} + +async fn record_exclusion_outcome( + swarm: &mut Swarm, + driver_state: &Arc>, + peer_exclusions: &PeerExclusions, + metrics: &NockchainP2PMetrics, + outcome: ExclusionOutcome, + related_peers: &[PeerId], +) { + if let Some(address) = outcome.address_cooldown { + log_address_cooldown(&address); + metrics.address_cooldown_dial_denied.increment(); + let mut peers_to_prune = related_peers.iter().copied().collect::>(); + if let Some(peer_id) = address.key.expected_peer { + peers_to_prune.insert(peer_id); + } + if peers_to_prune.is_empty() { + prune_one_address(swarm, metrics, None, &address.address).await; + } else { + for peer_id in peers_to_prune { + prune_one_address(swarm, metrics, Some(peer_id), &address.address).await; + } + } + } + + if let Some(ip) = outcome.ip_exclusion { + log_ip_exclusion(&ip, related_peers); + metrics.ip_exclusions_created.increment(); + if ip.fail2ban { + let log_peer = related_peers + .first() + .copied() + .unwrap_or_else(PeerId::random); + match ip.ip { + IpAddr::V4(v4) => log_fail2ban_ipv4(&log_peer, &v4), + IpAddr::V6(v6) => log_fail2ban_ipv6(&log_peer, &v6), + } + } + prune_excluded_swarm_state(swarm, driver_state, peer_exclusions, metrics).await; + } +} + +fn log_address_cooldown(outcome: &AddressCooldownOutcome) { + info!( + address = %outcome.address, + ip = %outcome.key.ip, + ttl_secs = outcome.ttl.as_secs(), + reason = %outcome.reason, + "temporarily cooling down peer endpoint" + ); +} + +fn log_ip_exclusion(outcome: &IpExclusionOutcome, related_peers: &[PeerId]) { + warn!( + ip = %outcome.ip, + ttl_secs = outcome.ttl.as_secs(), + reason = %outcome.reason, + peers = ?related_peers.iter().map(|peer| peer.to_base58()).collect::>(), + "temporarily excluding peer IP" + ); +} + +async fn prune_one_address( + swarm: &mut Swarm, + metrics: &NockchainP2PMetrics, + peer_id: Option, + address: &Multiaddr, +) { + let Some(peer_id) = peer_id.or_else(|| p2p_peer_id(address)) else { + return; + }; + + let stripped = without_p2p(address); + let mut address_candidates = vec![address.clone()]; + if stripped != *address { + address_candidates.push(stripped); + } + + let mut removed_address = false; + let mut removed_peer = false; + for candidate in address_candidates { + let had_kad_address = swarm.behaviour_mut().kad.kbuckets().any(|bucket| { + bucket.iter().any(|peer| { + peer.node.key.into_preimage() == peer_id + && peer.node.value.iter().any(|addr| addr == &candidate) + }) + }); + if swarm + .behaviour_mut() + .kad + .remove_address(&peer_id, &candidate) + .is_some() + { + removed_peer = true; + } + let removed_from_peer_store = swarm + .behaviour_mut() + .peer_store + .store_mut() + .remove_address(&peer_id, &candidate); + removed_address = removed_address || had_kad_address || removed_from_peer_store; + } + if removed_address { + metrics.kad_addresses_pruned_for_exclusion.increment(); + } + if removed_peer { + metrics.kad_peers_pruned_for_exclusion.increment(); + } +} + +async fn observe_kad_cardinality_and_exclude( + swarm: &mut Swarm, + driver_state: &Arc>, + peer_exclusions: &PeerExclusions, + metrics: &NockchainP2PMetrics, +) { + let mut by_ip: BTreeMap, BTreeSet)> = BTreeMap::new(); + for bucket in swarm.behaviour_mut().kad.kbuckets() { + for peer in bucket.iter() { + let peer_id = peer.node.key.into_preimage(); + for address in peer.node.value.iter() { + let Some(key) = peer_exclusions.address_key(address, Some(peer_id)) else { + continue; + }; + let (peers, ports) = by_ip.entry(key.ip).or_default(); + peers.insert(peer_id); + if let Some(port) = key.port { + ports.insert(port); + } + } + } + } + + let mut max_cardinality = 0usize; + for (ip, (peers, ports)) in by_ip { + max_cardinality = max_cardinality.max(peers.len()).max(ports.len()); + if let Some(outcome) = peer_exclusions.record_kad_cardinality(ip, peers.len(), ports.len()) + { + let related_peers = peers.iter().copied().collect::>(); + record_exclusion_outcome( + swarm, + driver_state, + peer_exclusions, + metrics, + ExclusionOutcome { + address_cooldown: None, + ip_exclusion: Some(outcome), + }, + &related_peers, + ) + .await; + } + } + let _ = metrics.same_ip_kad_cardinality.swap(max_cardinality as f64); +} + +async fn prune_excluded_swarm_state( + swarm: &mut Swarm, + driver_state: &Arc>, + peer_exclusions: &PeerExclusions, + metrics: &NockchainP2PMetrics, +) { + let mut addresses_to_remove = Vec::new(); + for bucket in swarm.behaviour_mut().kad.kbuckets() { + for peer in bucket.iter() { + let peer_id = peer.node.key.into_preimage(); + for address in peer.node.value.iter() { + if peer_exclusions.is_address_excluded(address, Some(peer_id)) { + addresses_to_remove.push((peer_id, address.clone())); + } + } + } + } + + for (peer_id, address) in addresses_to_remove { + metrics.kad_addresses_pruned_for_exclusion.increment(); + if swarm + .behaviour_mut() + .kad + .remove_address(&peer_id, &address) + .is_some() + { + metrics.kad_peers_pruned_for_exclusion.increment(); + } + let _ = swarm + .behaviour_mut() + .peer_store + .store_mut() + .remove_address(&peer_id, &address); + } + + let connections_to_close = { + let state_guard = driver_state.lock().await; + state_guard + .peer_connections + .iter() + .flat_map(|(peer_id, connections)| { + connections.iter().filter_map(|(connection_id, address)| { + let ip = address.ip_addr()?; + peer_exclusions + .is_ip_excluded(&ip) + .then_some((*peer_id, *connection_id)) + }) + }) + .collect::>() + }; + + for (peer_id, connection_id) in connections_to_close { + debug!("Closing connection {connection_id} to excluded peer {peer_id}"); + swarm.close_connection(connection_id); + } +} + fn log_dial_error(error: DialError) { match error { DialError::NoAddresses => debug!("No addresses to dial"), @@ -2467,12 +3146,18 @@ fn log_dial_error(error: DialError) { } } -fn log_outbound_failure( +async fn handle_outbound_failure( peer: PeerId, error: request_response::OutboundFailure, metrics: Arc, + driver_state: Arc>, + peer_exclusions: PeerExclusions, ) { metrics.request_failed.increment(); + driver_state.lock().await.record_request_failure(peer); + if peer_exclusions.record_peer_request_failure(peer) { + metrics.request_peer_cooldowns_created.increment(); + } match error { request_response::OutboundFailure::DialFailure => { debug!("Failed to dial peer {} for request", peer) @@ -2517,7 +3202,11 @@ fn log_inbound_failure( }; } -fn dial_more_peers(swarm: &mut Swarm, state_guard: MutexGuard) { +fn dial_more_peers( + swarm: &mut Swarm, + state_guard: MutexGuard, + peer_exclusions: &PeerExclusions, +) { let mut addresses_to_dial = Vec::new(); for bucket in swarm.behaviour_mut().kad.kbuckets() { for peer in bucket.iter() { @@ -2530,6 +3219,12 @@ fn dial_more_peers(swarm: &mut Swarm, state_guard: MutexGuar for address in peer.node.value.iter() { let mut address = address.clone(); + if peer_exclusions + .is_address_excluded(&address, Some(peer.node.key.into_preimage())) + { + continue; + } + if let Ok(address_with_peer_id) = address.clone().with_p2p(peer.node.key.into_preimage()) { @@ -2566,6 +3261,7 @@ pub(crate) fn start_swarm( allowed: Option>, limits: connection_limits::ConnectionLimits, memory_limits: Option, + peer_exclusions: PeerExclusions, ) -> Result, Box> { let (resolver_config, resolver_opts) = if let Ok(sys) = hickory_resolver::system_conf::read_system_conf() { @@ -2590,7 +3286,7 @@ pub(crate) fn start_swarm( }) .with_dns_config(resolver_config, resolver_opts) .with_behaviour(NockchainBehaviour::pre_new( - libp2p_config, allowed, limits, memory_limits, + libp2p_config, allowed, limits, memory_limits, peer_exclusions, ))? .with_swarm_config(|cfg| cfg.with_idle_connection_timeout(swarm_idle_timeout)) .with_connection_timeout(connection_timeout) @@ -2611,8 +3307,13 @@ pub(crate) fn identify_received( swarm: &mut Swarm, peer_id: PeerId, info: libp2p::identify::Info, + peer_exclusions: &PeerExclusions, + metrics: &NockchainP2PMetrics, ) -> Result<(), NockAppError> { swarm.add_external_address(info.observed_addr.clone()); + if let Some(ip) = info.observed_addr.ip_addr() { + peer_exclusions.record_positive_ip(ip); + } let us = *swarm.local_peer_id(); let kad = &mut swarm.behaviour_mut().kad; trace!("identify received for peer {}", peer_id); @@ -2622,6 +3323,11 @@ pub(crate) fn identify_received( if let Some(Protocol::Dnsaddr(_)) = addr.iter().next() { continue; } + if peer_exclusions.is_address_excluded(&addr, Some(peer_id)) { + trace!("Skipping excluded address {addr} for peer {peer_id}"); + metrics.identify_addresses_skipped_for_exclusion.increment(); + continue; + } trace!("Adding address {} for peer {}", addr, peer_id); kad.add_address(&peer_id, addr); } diff --git a/crates/nockchain-libp2p-io/src/ip_block.rs b/crates/nockchain-libp2p-io/src/ip_block.rs new file mode 100644 index 000000000..7a3ec8c4f --- /dev/null +++ b/crates/nockchain-libp2p-io/src/ip_block.rs @@ -0,0 +1,1402 @@ +//! IP and endpoint-level connection hygiene. +//! +//! [`crate::behaviour::NockchainBehaviour`]'s peer-id block list cannot stop a +//! host that repeatedly advertises fresh peer IDs on the same IP. Kademlia can +//! then keep relearning `new-peer-id -> same-address` mappings from DHT query +//! responses and dial them without an application-level choke point. +//! +//! This module keeps a local, expiry-aware exclusion store. The swarm-facing +//! [`NetworkBehaviour`] hooks deny active IP exclusions before transport work, +//! while [`IpFilteredKad`] also filters Kademlia-owned dial candidates before +//! they leave the routing behaviour. + +use std::collections::{HashMap, HashSet, VecDeque}; +use std::convert::Infallible; +use std::fmt; +use std::net::IpAddr; +use std::ops::{Deref, DerefMut}; +use std::sync::{Arc, RwLock, RwLockReadGuard, RwLockWriteGuard}; +use std::task::{Context, Poll}; +use std::time::{Duration, Instant}; + +use libp2p::core::transport::PortUse; +use libp2p::core::{Endpoint, Multiaddr}; +use libp2p::identity::PeerId; +use libp2p::kad; +use libp2p::multiaddr::Protocol; +use libp2p::swarm::{ + dummy, ConnectionDenied, ConnectionId, FromSwarm, NetworkBehaviour, THandler, THandlerInEvent, + THandlerOutEvent, ToSwarm, +}; + +use crate::config::PeerExclusionConfig; +use crate::p2p_util::MultiaddrExt; + +#[derive(Debug, Clone, Copy, Eq, PartialEq, Hash)] +pub(crate) enum TransportKind { + Tcp, + Udp, + Other, +} + +#[derive(Debug, Clone, Copy, Eq, PartialEq, Hash)] +pub(crate) struct AddressKey { + pub(crate) ip: IpAddr, + pub(crate) transport: TransportKind, + pub(crate) port: Option, + pub(crate) expected_peer: Option, +} + +#[derive(Debug, Clone, Copy, Eq, PartialEq)] +pub(crate) enum ExclusionReason { + WrongPeerId, + RepeatedWrongPeerId, + PermissionDenied, + RepeatedDialFailure, + KadSameIpCardinality, +} + +impl fmt::Display for ExclusionReason { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + ExclusionReason::WrongPeerId => write!(f, "wrong-peer-id"), + ExclusionReason::RepeatedWrongPeerId => write!(f, "repeated-wrong-peer-id"), + ExclusionReason::PermissionDenied => write!(f, "permission-denied"), + ExclusionReason::RepeatedDialFailure => write!(f, "repeated-dial-failure"), + ExclusionReason::KadSameIpCardinality => write!(f, "kad-same-ip-cardinality"), + } + } +} + +#[derive(Debug, Clone)] +pub(crate) struct AddressCooldownOutcome { + pub(crate) key: AddressKey, + pub(crate) address: Multiaddr, + pub(crate) ttl: Duration, + pub(crate) reason: ExclusionReason, +} + +#[derive(Debug, Clone)] +pub(crate) struct IpExclusionOutcome { + pub(crate) ip: IpAddr, + pub(crate) ttl: Duration, + pub(crate) reason: ExclusionReason, + pub(crate) fail2ban: bool, +} + +#[derive(Debug, Clone, Default)] +pub(crate) struct ExclusionOutcome { + pub(crate) address_cooldown: Option, + pub(crate) ip_exclusion: Option, +} + +#[derive(Debug, Clone, Copy, Eq, PartialEq)] +enum EvidenceKind { + WrongPeerId, + DialFailure, + PermissionDenied, + PingFailure, + KadCardinality, +} + +#[derive(Debug, Clone)] +struct PeerHealthEvent { + ip: IpAddr, + port: Option, + expected_peer: Option, + obtained_peer: Option, + at: Instant, + kind: EvidenceKind, +} + +#[derive(Debug, Clone)] +struct IpExclusion { + expires_at: Instant, + reason: ExclusionReason, + strike_count: u32, + last_seen: Instant, +} + +#[derive(Debug, Clone)] +struct AddressExclusion { + expires_at: Instant, + reason: ExclusionReason, + last_seen: Instant, +} + +#[derive(Debug, Clone)] +struct PeerPenalty { + expires_at: Instant, + failure_count: u32, + last_failure: Instant, +} + +#[derive(Debug, Clone)] +struct IpHistory { + last_excluded_at: Instant, + exclusion_count: u32, +} + +#[derive(Debug, Default)] +struct ExclusionState { + ips: HashMap, + addresses: HashMap, + peers: HashMap, + events: VecDeque, + ip_history: HashMap, +} + +fn read_state(lock: &RwLock) -> RwLockReadGuard<'_, ExclusionState> { + match lock.read() { + Ok(guard) => guard, + Err(poisoned) => poisoned.into_inner(), + } +} + +fn write_state(lock: &RwLock) -> RwLockWriteGuard<'_, ExclusionState> { + match lock.write() { + Ok(guard) => guard, + Err(poisoned) => poisoned.into_inner(), + } +} + +#[derive(Debug, Default, Clone, Copy, Eq, PartialEq)] +pub(crate) struct ExpireOutcome { + pub(crate) ips: usize, + pub(crate) addresses: usize, + pub(crate) peers: usize, +} + +/// Cheaply cloneable, shared exclusion state. +#[derive(Clone)] +pub(crate) struct PeerExclusions { + inner: Arc>, + config: Arc, +} + +impl Default for PeerExclusions { + fn default() -> Self { + Self::new(PeerExclusionConfig::default()) + } +} + +impl PeerExclusions { + pub(crate) fn new(config: PeerExclusionConfig) -> Self { + Self { + inner: Arc::new(RwLock::new(ExclusionState::default())), + config: Arc::new(config), + } + } + + pub(crate) fn address_key( + &self, + addr: &Multiaddr, + expected_peer: Option, + ) -> Option { + address_key(addr, expected_peer) + } + + pub(crate) fn is_ip_excluded(&self, ip: &IpAddr) -> bool { + self.is_ip_excluded_at(ip, Instant::now()) + } + + pub(crate) fn is_address_excluded( + &self, + addr: &Multiaddr, + expected_peer: Option, + ) -> bool { + self.is_address_excluded_at(addr, expected_peer, Instant::now()) + } + + pub(crate) fn is_ip_excluded_at(&self, ip: &IpAddr, now: Instant) -> bool { + if !self.config.enabled || self.config.allow_ips.contains(ip) { + return false; + } + read_state(&self.inner) + .ips + .get(ip) + .is_some_and(|entry| entry.expires_at > now) + } + + pub(crate) fn is_address_excluded_at( + &self, + addr: &Multiaddr, + expected_peer: Option, + now: Instant, + ) -> bool { + let Some(ip) = addr.ip_addr() else { + return false; + }; + if self.is_ip_excluded_at(&ip, now) { + return true; + } + if !self.config.enabled || self.config.allow_ips.contains(&ip) { + return false; + } + let Some(key) = address_key(addr, expected_peer) else { + return false; + }; + address_is_excluded(&read_state(&self.inner), key, now) + } + + pub(crate) fn record_wrong_peer_id( + &self, + address: &Multiaddr, + expected_peer: Option, + obtained_peer: PeerId, + ) -> ExclusionOutcome { + self.record_wrong_peer_id_at(address, expected_peer, obtained_peer, Instant::now()) + } + + fn record_wrong_peer_id_at( + &self, + address: &Multiaddr, + expected_peer: Option, + obtained_peer: PeerId, + now: Instant, + ) -> ExclusionOutcome { + let Some(key) = address_key(address, expected_peer) else { + return ExclusionOutcome::default(); + }; + if !self.config.enabled || self.config.allow_ips.contains(&key.ip) { + return ExclusionOutcome::default(); + } + + let mut state = write_state(&self.inner); + prune_state(&mut state, now, &self.config); + push_event( + &mut state, + PeerHealthEvent { + ip: key.ip, + port: key.port, + expected_peer, + obtained_peer: Some(obtained_peer), + at: now, + kind: EvidenceKind::WrongPeerId, + }, + now, + &self.config, + ); + let address_cooldown = insert_address_cooldown( + &mut state, + key, + address.clone(), + self.config.address_cooldown(), + ExclusionReason::WrongPeerId, + now, + ); + + let ip_exclusion = if wrong_peer_threshold_met(&state, key.ip, now, &self.config) { + insert_ip_exclusion( + &mut state, + key.ip, + ExclusionReason::RepeatedWrongPeerId, + now, + &self.config, + ) + } else { + None + }; + + ExclusionOutcome { + address_cooldown, + ip_exclusion, + } + } + + pub(crate) fn record_permission_denied(&self, address: &Multiaddr) -> ExclusionOutcome { + self.record_address_failure_at( + address, + None, + EvidenceKind::PermissionDenied, + ExclusionReason::PermissionDenied, + self.config.permission_denied_cooldown(), + Instant::now(), + ) + } + + pub(crate) fn record_dial_failure( + &self, + address: &Multiaddr, + expected_peer: Option, + ) -> ExclusionOutcome { + self.record_address_failure_at( + address, + expected_peer, + EvidenceKind::DialFailure, + ExclusionReason::RepeatedDialFailure, + self.config.address_cooldown(), + Instant::now(), + ) + } + + fn record_address_failure_at( + &self, + address: &Multiaddr, + expected_peer: Option, + kind: EvidenceKind, + reason: ExclusionReason, + address_ttl: Duration, + now: Instant, + ) -> ExclusionOutcome { + let Some(key) = address_key(address, expected_peer) else { + return ExclusionOutcome::default(); + }; + if !self.config.enabled || self.config.allow_ips.contains(&key.ip) { + return ExclusionOutcome::default(); + } + + let mut state = write_state(&self.inner); + prune_state(&mut state, now, &self.config); + push_event( + &mut state, + PeerHealthEvent { + ip: key.ip, + port: key.port, + expected_peer, + obtained_peer: None, + at: now, + kind, + }, + now, + &self.config, + ); + + let failure_count = state + .events + .iter() + .filter(|event| { + event.ip == key.ip + && event.at + self.config.evidence_window() >= now + && matches!( + event.kind, + EvidenceKind::DialFailure + | EvidenceKind::PermissionDenied + | EvidenceKind::PingFailure + ) + }) + .count(); + + let address_cooldown = + insert_address_cooldown(&mut state, key, address.clone(), address_ttl, reason, now); + let ip_exclusion = if failure_count >= self.config.dial_failure_ip_threshold { + insert_ip_exclusion( + &mut state, + key.ip, + ExclusionReason::RepeatedDialFailure, + now, + &self.config, + ) + } else { + None + }; + + ExclusionOutcome { + address_cooldown, + ip_exclusion, + } + } + + pub(crate) fn record_ping_failure(&self, address: &Multiaddr) -> ExclusionOutcome { + self.record_address_failure_at( + address, + None, + EvidenceKind::PingFailure, + ExclusionReason::RepeatedDialFailure, + self.config.address_cooldown(), + Instant::now(), + ) + } + + pub(crate) fn record_positive_ip(&self, ip: IpAddr) { + if !self.config.enabled || self.config.allow_ips.contains(&ip) { + return; + } + let now = Instant::now(); + let mut state = write_state(&self.inner); + prune_state(&mut state, now, &self.config); + + let mut remaining_events = VecDeque::with_capacity(state.events.len()); + let mut removed = 0usize; + while let Some(event) = state.events.pop_front() { + if event.ip == ip && removed < 2 { + removed += 1; + continue; + } + remaining_events.push_back(event); + } + state.events = remaining_events; + } + + pub(crate) fn record_peer_request_failure(&self, peer_id: PeerId) -> bool { + if !self.config.enabled { + return false; + } + let now = Instant::now(); + let expires_at = now + self.config.request_peer_cooldown(); + let mut state = write_state(&self.inner); + prune_state(&mut state, now, &self.config); + let was_active = state + .peers + .get(&peer_id) + .is_some_and(|entry| entry.expires_at > now); + state + .peers + .entry(peer_id) + .and_modify(|entry| { + entry.expires_at = expires_at; + entry.failure_count = entry.failure_count.saturating_add(1); + entry.last_failure = now; + }) + .or_insert(PeerPenalty { + expires_at, + failure_count: 1, + last_failure: now, + }); + !was_active + } + + pub(crate) fn record_peer_request_success(&self, peer_id: &PeerId) { + let now = Instant::now(); + let mut state = write_state(&self.inner); + prune_state(&mut state, now, &self.config); + state.peers.remove(peer_id); + } + + pub(crate) fn is_peer_request_cooled_down(&self, peer_id: &PeerId) -> bool { + if !self.config.enabled { + return false; + } + let now = Instant::now(); + read_state(&self.inner) + .peers + .get(peer_id) + .is_some_and(|entry| entry.expires_at > now) + } + + pub(crate) fn record_kad_cardinality( + &self, + ip: IpAddr, + peer_count: usize, + port_count: usize, + ) -> Option { + let now = Instant::now(); + self.record_kad_cardinality_at(ip, peer_count, port_count, now) + } + + fn record_kad_cardinality_at( + &self, + ip: IpAddr, + peer_count: usize, + port_count: usize, + now: Instant, + ) -> Option { + if !self.config.enabled || self.config.allow_ips.contains(&ip) { + return None; + } + if peer_count < self.config.same_ip_kad_entry_threshold + && port_count < self.config.same_ip_kad_entry_threshold + { + return None; + } + + let mut state = write_state(&self.inner); + prune_state(&mut state, now, &self.config); + push_event( + &mut state, + PeerHealthEvent { + ip, + port: None, + expected_peer: None, + obtained_peer: None, + at: now, + kind: EvidenceKind::KadCardinality, + }, + now, + &self.config, + ); + + if has_recent_failure(&state, ip, now, &self.config) { + insert_ip_exclusion( + &mut state, + ip, + ExclusionReason::KadSameIpCardinality, + now, + &self.config, + ) + } else { + None + } + } + + pub(crate) fn expire(&self) -> ExpireOutcome { + let now = Instant::now(); + let mut state = write_state(&self.inner); + prune_state(&mut state, now, &self.config) + } + + pub(crate) fn active_ip_exclusion_count(&self) -> usize { + let now = Instant::now(); + read_state(&self.inner) + .ips + .values() + .filter(|entry| entry.expires_at > now) + .count() + } + + pub(crate) fn active_address_cooldown_count(&self) -> usize { + let now = Instant::now(); + read_state(&self.inner) + .addresses + .values() + .filter(|entry| entry.expires_at > now) + .count() + } + + pub(crate) fn fail2ban_enabled(&self) -> bool { + self.config.fail2ban_on_temp_exclusion + } +} + +fn prune_state( + state: &mut ExclusionState, + now: Instant, + config: &PeerExclusionConfig, +) -> ExpireOutcome { + let ip_before = state.ips.len(); + state.ips.retain(|_, entry| entry.expires_at > now); + let address_before = state.addresses.len(); + state.addresses.retain(|_, entry| entry.expires_at > now); + let peer_before = state.peers.len(); + state.peers.retain(|_, entry| entry.expires_at > now); + + while state + .events + .front() + .is_some_and(|event| event.at + config.event_history() < now) + { + state.events.pop_front(); + } + while state.events.len() > config.max_exclusion_entries { + state.events.pop_front(); + } + + state + .ip_history + .retain(|_, history| history.last_excluded_at + config.ip_exclusion_history() >= now); + + ExpireOutcome { + ips: ip_before.saturating_sub(state.ips.len()), + addresses: address_before.saturating_sub(state.addresses.len()), + peers: peer_before.saturating_sub(state.peers.len()), + } +} + +fn push_event( + state: &mut ExclusionState, + event: PeerHealthEvent, + now: Instant, + config: &PeerExclusionConfig, +) { + state.events.push_back(event); + prune_state(state, now, config); +} + +fn insert_address_cooldown( + state: &mut ExclusionState, + key: AddressKey, + address: Multiaddr, + ttl: Duration, + reason: ExclusionReason, + now: Instant, +) -> Option { + let expires_at = now + ttl; + match state.addresses.get_mut(&key) { + Some(existing) if existing.expires_at >= expires_at => { + existing.last_seen = now; + None + } + Some(existing) => { + existing.expires_at = expires_at; + existing.reason = reason; + existing.last_seen = now; + Some(AddressCooldownOutcome { + key, + address, + ttl, + reason, + }) + } + None => { + state.addresses.insert( + key, + AddressExclusion { + expires_at, + reason, + last_seen: now, + }, + ); + Some(AddressCooldownOutcome { + key, + address, + ttl, + reason, + }) + } + } +} + +fn insert_ip_exclusion( + state: &mut ExclusionState, + ip: IpAddr, + reason: ExclusionReason, + now: Instant, + config: &PeerExclusionConfig, +) -> Option { + let history = state.ip_history.entry(ip).or_insert(IpHistory { + last_excluded_at: now, + exclusion_count: 0, + }); + let recent_recurrence = history.exclusion_count > 0 + && history.last_excluded_at + config.ip_exclusion_history() >= now; + history.exclusion_count = if recent_recurrence { + history.exclusion_count.saturating_add(1) + } else { + 1 + }; + history.last_excluded_at = now; + + let ttl = if recent_recurrence { + config.ip_extended_exclusion() + } else { + config.ip_exclusion() + } + .min(config.max_auto_exclusion()); + let expires_at = now + ttl; + + match state.ips.get_mut(&ip) { + Some(existing) if existing.expires_at >= expires_at => { + existing.last_seen = now; + existing.strike_count = existing.strike_count.saturating_add(1); + None + } + Some(existing) => { + existing.expires_at = expires_at; + existing.reason = reason; + existing.last_seen = now; + existing.strike_count = existing.strike_count.saturating_add(1); + Some(IpExclusionOutcome { + ip, + ttl, + reason, + fail2ban: config.fail2ban_on_temp_exclusion, + }) + } + None => { + state.ips.insert( + ip, + IpExclusion { + expires_at, + reason, + strike_count: 1, + last_seen: now, + }, + ); + Some(IpExclusionOutcome { + ip, + ttl, + reason, + fail2ban: config.fail2ban_on_temp_exclusion, + }) + } + } +} + +fn wrong_peer_threshold_met( + state: &ExclusionState, + ip: IpAddr, + now: Instant, + config: &PeerExclusionConfig, +) -> bool { + let mut peers = HashSet::new(); + let mut obtained_peers = HashSet::new(); + let mut ports = HashSet::new(); + for event in state.events.iter().filter(|event| { + event.ip == ip + && event.kind == EvidenceKind::WrongPeerId + && event.at + config.evidence_window() >= now + }) { + if let Some(peer) = event.expected_peer { + peers.insert(peer); + } + if let Some(peer) = event.obtained_peer { + obtained_peers.insert(peer); + } + if let Some(port) = event.port { + ports.insert(port); + } + } + peers.len() >= config.wrong_peer_id_ip_threshold + || obtained_peers.len() >= config.wrong_peer_id_ip_threshold + || ports.len() >= config.wrong_peer_id_ip_threshold +} + +fn has_recent_failure( + state: &ExclusionState, + ip: IpAddr, + now: Instant, + config: &PeerExclusionConfig, +) -> bool { + state.events.iter().any(|event| { + event.ip == ip + && event.at + config.evidence_window() >= now + && matches!( + event.kind, + EvidenceKind::WrongPeerId + | EvidenceKind::DialFailure + | EvidenceKind::PermissionDenied + | EvidenceKind::PingFailure + ) + }) +} + +fn address_key(addr: &Multiaddr, expected_peer: Option) -> Option { + let mut ip = None; + let mut transport = TransportKind::Other; + let mut port = None; + let mut peer = expected_peer; + + for protocol in addr.iter() { + match protocol { + Protocol::Ip4(addr) => ip = Some(IpAddr::V4(addr)), + Protocol::Ip6(addr) => ip = Some(IpAddr::V6(addr)), + Protocol::Tcp(p) => { + transport = TransportKind::Tcp; + port = Some(p); + } + Protocol::Udp(p) => { + transport = TransportKind::Udp; + port = Some(p); + } + Protocol::P2p(peer_id) if peer.is_none() => peer = Some(peer_id), + _ => {} + } + } + + ip.map(|ip| AddressKey { + ip, + transport, + port, + expected_peer: peer, + }) +} + +fn address_is_excluded(state: &ExclusionState, key: AddressKey, now: Instant) -> bool { + if state + .addresses + .get(&key) + .is_some_and(|entry| entry.expires_at > now) + { + return true; + } + + if key.expected_peer.is_none() { + return false; + } + + let wildcard_key = AddressKey { + expected_peer: None, + ..key + }; + state + .addresses + .get(&wildcard_key) + .is_some_and(|entry| entry.expires_at > now) +} + +/// A connection was refused because its remote endpoint is under exclusion. +#[derive(Debug)] +pub(crate) struct BlockedEndpoint { + addr: Multiaddr, +} + +impl fmt::Display for BlockedEndpoint { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "endpoint {} is temporarily excluded", self.addr) + } +} + +impl std::error::Error for BlockedEndpoint {} + +fn enforce_not_excluded( + exclusions: &PeerExclusions, + addr: &Multiaddr, + expected_peer: Option, +) -> Result<(), ConnectionDenied> { + if exclusions.is_address_excluded(addr, expected_peer) { + return Err(ConnectionDenied::new(BlockedEndpoint { + addr: addr.clone(), + })); + } + Ok(()) +} + +/// Kademlia plus IP/endpoint filtering for the addresses Kademlia contributes +/// to dials. +pub(crate) struct IpFilteredKad { + inner: kad::Behaviour, + exclusions: PeerExclusions, +} + +impl IpFilteredKad { + pub(crate) fn new( + inner: kad::Behaviour, + exclusions: PeerExclusions, + ) -> Self { + Self { inner, exclusions } + } + + fn is_allowed_addr(&self, addr: &Multiaddr, expected_peer: Option) -> bool { + !self.exclusions.is_address_excluded(addr, expected_peer) + } +} + +impl Deref for IpFilteredKad { + type Target = kad::Behaviour; + + fn deref(&self) -> &Self::Target { + &self.inner + } +} + +impl DerefMut for IpFilteredKad { + fn deref_mut(&mut self) -> &mut Self::Target { + &mut self.inner + } +} + +impl NetworkBehaviour for IpFilteredKad { + type ConnectionHandler = + as NetworkBehaviour>::ConnectionHandler; + type ToSwarm = as NetworkBehaviour>::ToSwarm; + + fn handle_pending_inbound_connection( + &mut self, + connection_id: ConnectionId, + local_addr: &Multiaddr, + remote_addr: &Multiaddr, + ) -> Result<(), ConnectionDenied> { + self.inner + .handle_pending_inbound_connection(connection_id, local_addr, remote_addr) + } + + fn handle_established_inbound_connection( + &mut self, + connection_id: ConnectionId, + peer: PeerId, + local_addr: &Multiaddr, + remote_addr: &Multiaddr, + ) -> Result, ConnectionDenied> { + self.inner + .handle_established_inbound_connection(connection_id, peer, local_addr, remote_addr) + } + + fn handle_pending_outbound_connection( + &mut self, + connection_id: ConnectionId, + maybe_peer: Option, + addresses: &[Multiaddr], + effective_role: Endpoint, + ) -> Result, ConnectionDenied> { + let addresses = self.inner.handle_pending_outbound_connection( + connection_id, maybe_peer, addresses, effective_role, + )?; + Ok(addresses + .into_iter() + .filter(|addr| self.is_allowed_addr(addr, maybe_peer)) + .collect()) + } + + fn handle_established_outbound_connection( + &mut self, + connection_id: ConnectionId, + peer: PeerId, + addr: &Multiaddr, + role_override: Endpoint, + port_use: PortUse, + ) -> Result, ConnectionDenied> { + enforce_not_excluded(&self.exclusions, addr, Some(peer))?; + self.inner.handle_established_outbound_connection( + connection_id, peer, addr, role_override, port_use, + ) + } + + fn on_swarm_event(&mut self, event: FromSwarm) { + self.inner.on_swarm_event(event); + } + + fn on_connection_handler_event( + &mut self, + peer_id: PeerId, + connection_id: ConnectionId, + event: THandlerOutEvent, + ) { + self.inner + .on_connection_handler_event(peer_id, connection_id, event); + } + + fn poll( + &mut self, + cx: &mut Context<'_>, + ) -> Poll>> { + self.inner.poll(cx) + } +} + +/// A [`NetworkBehaviour`] that denies any connection whose remote endpoint is +/// under active local exclusion. +pub(crate) struct Behaviour { + exclusions: PeerExclusions, +} + +impl Behaviour { + pub(crate) fn new(exclusions: PeerExclusions) -> Self { + Self { exclusions } + } + + fn enforce( + &self, + addr: &Multiaddr, + expected_peer: Option, + ) -> Result<(), ConnectionDenied> { + enforce_not_excluded(&self.exclusions, addr, expected_peer) + } +} + +impl NetworkBehaviour for Behaviour { + type ConnectionHandler = dummy::ConnectionHandler; + type ToSwarm = Infallible; + + fn handle_pending_inbound_connection( + &mut self, + _: ConnectionId, + _local_addr: &Multiaddr, + remote_addr: &Multiaddr, + ) -> Result<(), ConnectionDenied> { + self.enforce(remote_addr, None) + } + + fn handle_established_inbound_connection( + &mut self, + _: ConnectionId, + peer: PeerId, + _local_addr: &Multiaddr, + remote_addr: &Multiaddr, + ) -> Result, ConnectionDenied> { + self.enforce(remote_addr, Some(peer))?; + Ok(dummy::ConnectionHandler) + } + + fn handle_pending_outbound_connection( + &mut self, + _: ConnectionId, + maybe_peer: Option, + addresses: &[Multiaddr], + _: Endpoint, + ) -> Result, ConnectionDenied> { + if addresses.is_empty() { + return Ok(vec![]); + } + + if addresses + .iter() + .any(|addr| !self.exclusions.is_address_excluded(addr, maybe_peer)) + { + return Ok(vec![]); + } + + self.enforce(&addresses[0], maybe_peer)?; + Ok(vec![]) + } + + fn handle_established_outbound_connection( + &mut self, + _: ConnectionId, + peer: PeerId, + addr: &Multiaddr, + _: Endpoint, + _: PortUse, + ) -> Result, ConnectionDenied> { + self.enforce(addr, Some(peer))?; + Ok(dummy::ConnectionHandler) + } + + fn on_swarm_event(&mut self, _event: FromSwarm) {} + + fn on_connection_handler_event( + &mut self, + _: PeerId, + _: ConnectionId, + event: THandlerOutEvent, + ) { + match event {} + } + + fn poll( + &mut self, + _cx: &mut Context<'_>, + ) -> Poll>> { + Poll::Pending + } +} + +#[cfg(test)] +mod tests { + use std::collections::HashSet; + use std::net::{IpAddr, Ipv4Addr}; + + use libp2p::swarm::NetworkBehaviour; + + use crate::ip_block::*; + + fn quic_addr(ip: Ipv4Addr, port: u16) -> Multiaddr { + format!("/ip4/{ip}/udp/{port}/quic-v1") + .parse() + .expect("valid multiaddr") + } + + fn config() -> PeerExclusionConfig { + PeerExclusionConfig { + address_cooldown_secs: 60, + ip_exclusion_secs: 120, + ip_extended_exclusion_secs: 360, + evidence_window_secs: 60, + ip_exclusion_history_secs: 300, + permission_denied_cooldown_secs: 30, + wrong_peer_id_ip_threshold: 3, + dial_failure_ip_threshold: 3, + same_ip_kad_entry_threshold: 3, + max_auto_exclusion_secs: 360, + max_exclusion_entries: 128, + request_peer_cooldown_secs: 60, + ..PeerExclusionConfig::default() + } + } + + #[test] + fn exclusion_is_shared_across_clones_and_expires() { + let exclusions = PeerExclusions::new(config()); + let clone = exclusions.clone(); + let ip: IpAddr = Ipv4Addr::new(15, 235, 216, 78).into(); + let now = Instant::now(); + + assert!(!clone.is_ip_excluded_at(&ip, now)); + let mut state = exclusions.inner.write().expect("lock"); + let outcome = insert_ip_exclusion( + &mut state, + ip, + ExclusionReason::RepeatedWrongPeerId, + now, + exclusions.config.as_ref(), + ); + drop(state); + + assert!(outcome.is_some()); + assert!(clone.is_ip_excluded_at(&ip, now + Duration::from_secs(1))); + assert!(!clone.is_ip_excluded_at(&ip, now + Duration::from_secs(121))); + } + + #[test] + fn address_cooldown_does_not_block_new_identity_on_same_endpoint() { + let exclusions = PeerExclusions::new(config()); + let now = Instant::now(); + let addr = quic_addr(Ipv4Addr::new(15, 235, 216, 78), 3602); + let old_peer = PeerId::random(); + let new_peer = PeerId::random(); + let obtained = PeerId::random(); + + let outcome = exclusions.record_wrong_peer_id_at(&addr, Some(old_peer), obtained, now); + + assert!(outcome.address_cooldown.is_some()); + assert!(exclusions.is_address_excluded_at(&addr, Some(old_peer), now)); + assert!(!exclusions.is_address_excluded_at(&addr, Some(new_peer), now)); + assert!(!exclusions.is_address_excluded_at(&addr, None, now)); + } + + #[test] + fn wildcard_address_cooldown_blocks_peer_specific_lookup() { + let exclusions = PeerExclusions::new(config()); + let addr = quic_addr(Ipv4Addr::new(15, 235, 216, 78), 3602); + let peer = PeerId::random(); + + let outcome = exclusions.record_permission_denied(&addr); + + assert!(outcome.address_cooldown.is_some()); + assert!(exclusions.is_address_excluded(&addr, None)); + assert!(exclusions.is_address_excluded(&addr, Some(peer))); + } + + #[test] + fn repeated_wrong_peer_ids_trigger_ip_exclusion() { + let exclusions = PeerExclusions::new(config()); + let now = Instant::now(); + let ip = Ipv4Addr::new(15, 235, 216, 78); + + let first = exclusions.record_wrong_peer_id_at( + &quic_addr(ip, 3602), + Some(PeerId::random()), + PeerId::random(), + now, + ); + let second = exclusions.record_wrong_peer_id_at( + &quic_addr(ip, 3603), + Some(PeerId::random()), + PeerId::random(), + now + Duration::from_secs(1), + ); + let third = exclusions.record_wrong_peer_id_at( + &quic_addr(ip, 3604), + Some(PeerId::random()), + PeerId::random(), + now + Duration::from_secs(2), + ); + + assert!(first.ip_exclusion.is_none()); + assert!(second.ip_exclusion.is_none()); + assert!(third.ip_exclusion.is_some()); + assert!(exclusions.is_ip_excluded_at(&IpAddr::V4(ip), now + Duration::from_secs(3))); + } + + #[test] + fn repeated_obtained_wrong_peer_ids_trigger_ip_exclusion() { + let exclusions = PeerExclusions::new(config()); + let now = Instant::now(); + let ip = Ipv4Addr::new(15, 235, 216, 78); + let addr = quic_addr(ip, 3602); + let expected = PeerId::random(); + + let first = + exclusions.record_wrong_peer_id_at(&addr, Some(expected), PeerId::random(), now); + let second = exclusions.record_wrong_peer_id_at( + &addr, + Some(expected), + PeerId::random(), + now + Duration::from_secs(1), + ); + let third = exclusions.record_wrong_peer_id_at( + &addr, + Some(expected), + PeerId::random(), + now + Duration::from_secs(2), + ); + + assert!(first.ip_exclusion.is_none()); + assert!(second.ip_exclusion.is_none()); + assert!(third.ip_exclusion.is_some()); + assert!(exclusions.is_ip_excluded_at(&IpAddr::V4(ip), now + Duration::from_secs(3))); + } + + #[test] + fn recurrent_ip_exclusion_uses_extended_ttl_with_cap() { + let exclusions = PeerExclusions::new(PeerExclusionConfig { + wrong_peer_id_ip_threshold: 1, + ip_exclusion_secs: 10, + ip_extended_exclusion_secs: 60, + max_auto_exclusion_secs: 30, + ip_exclusion_history_secs: 300, + ..config() + }); + let now = Instant::now(); + let ip = Ipv4Addr::new(15, 235, 216, 78); + let addr = quic_addr(ip, 3602); + + let first = exclusions.record_wrong_peer_id_at( + &addr, + Some(PeerId::random()), + PeerId::random(), + now, + ); + assert_eq!( + first.ip_exclusion.as_ref().map(|outcome| outcome.ttl), + Some(Duration::from_secs(10)) + ); + + let second = exclusions.record_wrong_peer_id_at( + &addr, + Some(PeerId::random()), + PeerId::random(), + now + Duration::from_secs(11), + ); + + assert_eq!( + second.ip_exclusion.as_ref().map(|outcome| outcome.ttl), + Some(Duration::from_secs(30)) + ); + } + + #[test] + fn allow_list_bypasses_evidence_and_dials() { + let ip = IpAddr::V4(Ipv4Addr::new(15, 235, 216, 78)); + let exclusions = PeerExclusions::new(PeerExclusionConfig { + wrong_peer_id_ip_threshold: 1, + allow_ips: HashSet::from([ip]), + ..config() + }); + let addr = quic_addr(Ipv4Addr::new(15, 235, 216, 78), 3602); + let outcome = + exclusions.record_wrong_peer_id(&addr, Some(PeerId::random()), PeerId::random()); + let mut behaviour = Behaviour::new(exclusions.clone()); + + assert!(outcome.address_cooldown.is_none()); + assert!(outcome.ip_exclusion.is_none()); + assert!(!exclusions.is_ip_excluded(&ip)); + assert!(behaviour + .handle_pending_outbound_connection( + ConnectionId::new_unchecked(7), + None, + std::slice::from_ref(&addr), + Endpoint::Dialer, + ) + .is_ok()); + } + + #[test] + fn disabled_hygiene_records_no_penalties() { + let exclusions = PeerExclusions::new(PeerExclusionConfig { + enabled: false, + wrong_peer_id_ip_threshold: 1, + ..config() + }); + let addr = quic_addr(Ipv4Addr::new(15, 235, 216, 78), 3602); + let outcome = + exclusions.record_wrong_peer_id(&addr, Some(PeerId::random()), PeerId::random()); + + assert!(outcome.address_cooldown.is_none()); + assert!(outcome.ip_exclusion.is_none()); + assert!(!exclusions.is_address_excluded(&addr, Some(PeerId::random()))); + } + + #[test] + fn permission_denied_is_address_cooldown_first() { + let exclusions = PeerExclusions::new(config()); + let addr = quic_addr(Ipv4Addr::new(15, 235, 216, 78), 3602); + + let outcome = exclusions.record_permission_denied(&addr); + + assert!(outcome.address_cooldown.is_some()); + assert!(outcome.ip_exclusion.is_none()); + } + + #[test] + fn kad_cardinality_needs_recent_failure_before_ip_exclusion() { + let exclusions = PeerExclusions::new(config()); + let now = Instant::now(); + let ip: IpAddr = Ipv4Addr::new(15, 235, 216, 78).into(); + let addr = quic_addr(Ipv4Addr::new(15, 235, 216, 78), 3602); + + assert!(exclusions + .record_kad_cardinality_at(ip, 8, 8, now) + .is_none()); + exclusions.record_wrong_peer_id_at(&addr, Some(PeerId::random()), PeerId::random(), now); + assert!(exclusions + .record_kad_cardinality_at(ip, 8, 8, now + Duration::from_secs(1)) + .is_some()); + } + + #[test] + fn behaviour_denies_excluded_ip_and_allows_others() { + let exclusions = PeerExclusions::new(config()); + let bad: IpAddr = Ipv4Addr::new(15, 235, 216, 78).into(); + let now = Instant::now(); + { + let mut state = exclusions.inner.write().expect("lock"); + insert_ip_exclusion( + &mut state, + bad, + ExclusionReason::RepeatedWrongPeerId, + now, + exclusions.config.as_ref(), + ); + } + let mut behaviour = Behaviour::new(exclusions); + + let bad_addr = quic_addr(Ipv4Addr::new(15, 235, 216, 78), 3602); + let good_addr = quic_addr(Ipv4Addr::new(203, 0, 113, 9), 3006); + let cid = ConnectionId::new_unchecked(0); + + assert!(behaviour + .handle_pending_inbound_connection(cid, &good_addr, &bad_addr) + .is_err()); + assert!(behaviour + .handle_pending_inbound_connection(cid, &good_addr, &good_addr) + .is_ok()); + assert!(behaviour + .handle_pending_outbound_connection( + cid, + None, + std::slice::from_ref(&bad_addr), + Endpoint::Dialer, + ) + .is_err()); + assert!(behaviour + .handle_pending_outbound_connection( + cid, + None, + std::slice::from_ref(&good_addr), + Endpoint::Dialer, + ) + .is_ok()); + assert!(behaviour + .handle_pending_outbound_connection( + cid, + None, + &[bad_addr.clone(), good_addr], + Endpoint::Dialer, + ) + .is_ok()); + } + + #[test] + fn filters_excluded_kad_addresses_returned_for_peer_dial() { + let exclusions = PeerExclusions::new(config()); + let bad: IpAddr = Ipv4Addr::new(15, 235, 216, 78).into(); + let now = Instant::now(); + { + let mut state = exclusions.inner.write().expect("lock"); + insert_ip_exclusion( + &mut state, + bad, + ExclusionReason::RepeatedWrongPeerId, + now, + exclusions.config.as_ref(), + ); + } + + let local_peer = PeerId::random(); + let target_peer = PeerId::random(); + let store = kad::store::MemoryStore::new(local_peer); + let kad = kad::Behaviour::new(local_peer, store); + let mut behaviour = IpFilteredKad::new(kad, exclusions); + + let bad_addr = quic_addr(Ipv4Addr::new(15, 235, 216, 78), 3602); + let good_addr = quic_addr(Ipv4Addr::new(203, 0, 113, 9), 3006); + behaviour.add_address(&target_peer, bad_addr.clone()); + behaviour.add_address(&target_peer, good_addr.clone()); + + let returned = behaviour + .handle_pending_outbound_connection( + ConnectionId::new_unchecked(1), + Some(target_peer), + &[], + Endpoint::Dialer, + ) + .expect("kad address filtering should not deny the dial"); + + assert!(returned + .iter() + .any(|addr| addr.ip_addr() == good_addr.ip_addr())); + assert!(!returned + .iter() + .any(|addr| addr.ip_addr() == bad_addr.ip_addr())); + } +} diff --git a/crates/nockchain-libp2p-io/src/lib.rs b/crates/nockchain-libp2p-io/src/lib.rs index 480678ca5..4a15942d1 100644 --- a/crates/nockchain-libp2p-io/src/lib.rs +++ b/crates/nockchain-libp2p-io/src/lib.rs @@ -7,6 +7,7 @@ mod behaviour; // Nockchain libp2p behavior type pub mod config; // Configurable values for the Nockchain libp2p driver pub mod driver; // Nockchain libp2p driver for NockApp +mod ip_block; // IP-level connection gating (deny banned IPs) mod key_fair_queue; // Fair queue for key-value pairs, allowing replacement mod messages; // Messages exchanged between Nockchain nodes pub mod metrics; // Nockchain libp2p metrics (gnort) diff --git a/crates/nockchain-libp2p-io/src/messages.rs b/crates/nockchain-libp2p-io/src/messages.rs index 7a056733c..bf3939bfd 100644 --- a/crates/nockchain-libp2p-io/src/messages.rs +++ b/crates/nockchain-libp2p-io/src/messages.rs @@ -1,8 +1,7 @@ use libp2p::PeerId; use nockapp::noun::slab::NounSlab; use nockapp::NockAppError; -use nockvm::ext::NounExt; -use nockvm::noun::{Noun, D}; +use nockvm::noun::{Noun, NounAllocator, NounHandle, NounSpace, D}; use nockvm_macros::tas; use serde_bytes::ByteBuf; @@ -29,26 +28,29 @@ impl NockchainFact { poke_slab.modify(|response_noun| vec![D(tas!(b"fact")), D(POKE_VERSION), response_noun]); let noun = unsafe { slab.root() }; - let head = noun.as_cell()?.head(); + let space = slab.noun_space(); + let head = noun.in_space(&space).as_cell()?.head(); if head.eq_bytes(b"heard-block") { - let page = noun.as_cell()?.tail(); + let page = noun.in_space(&space).as_cell()?.tail(); let block_id = block_id_from_page(page)?; - let block_id_str = tip5_hash_to_base58_stack(slab, block_id)?; + let block_id_str = tip5_hash_to_base58_stack(slab, block_id.noun(), block_id.space())?; Ok(NockchainFact::HeardBlock(block_id_str, poke_slab)) } else if head.eq_bytes(b"heard-tx") { - let raw_tx = noun.as_cell()?.tail(); + let raw_tx = noun.in_space(&space).as_cell()?.tail(); let tx_id = tx_id_from_raw_tx(raw_tx)?; - let tx_id_str = tip5_hash_to_base58_stack(slab, tx_id)?; + let tx_id_str = tip5_hash_to_base58_stack(slab, tx_id.noun(), tx_id.space())?; Ok(NockchainFact::HeardTx(tx_id_str, poke_slab)) } else if head.eq_bytes(b"heard-elders") { - let elders_dat = noun.as_cell()?.tail(); - let oldest = elders_dat.as_cell()?.head().as_atom()?.as_u64()?; - let elder_ids = elders_dat.as_cell()?.tail(); + let elders_noun = noun.in_space(&space).as_cell()?.tail().noun(); + let elders_dat = elders_noun.in_space(&space); + let elders_cell = elders_dat.as_cell()?; + let oldest = elders_cell.head().as_atom()?.as_u64()?; + let elder_ids = elders_cell.tail(); // Need to handle the closure capturing mutable reference let mut elder_id_strings = Vec::new(); for id_noun in elder_ids.list_iter() { - elder_id_strings.push(tip5_hash_to_base58_stack(slab, id_noun)?); + elder_id_strings.push(tip5_hash_to_base58_stack(slab, id_noun.noun(), &space)?); } Ok(NockchainFact::HeardElders( oldest, elder_id_strings, poke_slab, @@ -68,7 +70,7 @@ impl NockchainFact { } } -fn block_id_from_page(page: Noun) -> Result { +fn block_id_from_page<'a>(page: NounHandle<'a>) -> Result, NockAppError> { let page_cell = page.as_cell()?; // page v0: [block-id ...] // page v1: [%1 block-id ...] @@ -76,19 +78,18 @@ fn block_id_from_page(page: Noun) -> Result { Ok(version_atom) => { let version = version_atom.as_u64()?; if version == 1 { - Ok(page_cell.tail().as_cell()?.head()) - } else { - Err(NockAppError::OtherError(format!( - "Unsupported page version {}", - version - ))) + return Ok(page_cell.tail().as_cell()?.head()); } + Err(NockAppError::OtherError(format!( + "Unsupported page version {}", + version + ))) } Err(_) => Ok(page_cell.head()), } } -fn tx_id_from_raw_tx(raw_tx: Noun) -> Result { +fn tx_id_from_raw_tx<'a>(raw_tx: NounHandle<'a>) -> Result, NockAppError> { let raw_tx_cell = raw_tx.as_cell()?; // raw-tx v0: [tx-id ...] // raw-tx v1: [%1 tx-id ...] @@ -96,13 +97,12 @@ fn tx_id_from_raw_tx(raw_tx: Noun) -> Result { Ok(version_atom) => { let version = version_atom.as_u64()?; if version == 1 { - Ok(raw_tx_cell.tail().as_cell()?.head()) - } else { - Err(NockAppError::OtherError(format!( - "Unsupported raw-tx version {}", - version - ))) + return Ok(raw_tx_cell.tail().as_cell()?.head()); } + Err(NockAppError::OtherError(format!( + "Unsupported raw-tx version {}", + version + ))) } Err(_) => Ok(raw_tx_cell.head()), } @@ -119,9 +119,9 @@ pub enum NockchainDataRequest { impl NockchainDataRequest { /// Takes noun of type [%request p=request] - pub fn from_noun(noun: Noun) -> Result { + pub fn from_noun(noun: Noun, space: &NounSpace) -> Result { let res = (|| { - let request_cell = noun.as_cell()?; + let request_cell = noun.in_space(space).as_cell()?; if !request_cell.head().eq_bytes(b"request") { return Err(NockAppError::OtherError(String::from( "Missing %request tag", @@ -140,11 +140,11 @@ impl NockchainDataRequest { Ok(Self::BlockByHeight(height)) } else if block_cell.head().eq_bytes(b"elders") { let elders_cell = block_cell.tail().as_cell()?; - let block_id = tip5_hash_to_base58(elders_cell.head())?; - let peer_id = PeerId::from_noun(elders_cell.tail())?; + let block_id = tip5_hash_to_base58(elders_cell.head().noun(), space)?; + let peer_id = PeerId::from_noun(elders_cell.tail().noun(), space)?; let slab = { let mut slab = NounSlab::new(); - slab.copy_into(elders_cell.head()); + slab.copy_into(elders_cell.head().noun(), space); slab }; Ok(Self::EldersById(block_id, peer_id, slab)) @@ -156,10 +156,10 @@ impl NockchainDataRequest { } else if kind_cell.head().eq_bytes(b"raw-tx") { // has type [%by-id p=tx-id:dt] let raw_tx_cell = kind_cell.tail().as_cell()?; - let raw_tx_id = tip5_hash_to_base58(raw_tx_cell.tail())?; + let raw_tx_id = tip5_hash_to_base58(raw_tx_cell.tail().noun(), space)?; let slab = { let mut slab = NounSlab::new(); - slab.copy_into(raw_tx_cell.tail()); + slab.copy_into(raw_tx_cell.tail().noun(), space); slab }; Ok(Self::RawTransactionById(raw_tx_id, slab)) @@ -290,3 +290,84 @@ impl NockchainResponse { } } } + +#[cfg(test)] +mod tests { + use nockapp::utils::make_tas; + use nockvm::noun::T; + + use super::*; + + #[test] + fn test_block_id_from_page_supports_v0_and_v1() { + let mut slab: NounSlab = NounSlab::new(); + let block_id = T(&mut slab, &[D(1), D(2), D(3), D(4), D(5)]); + let v0_page = T(&mut slab, &[block_id, D(99)]); + let v1_page = T(&mut slab, &[D(1), block_id, D(99)]); + let space = slab.noun_space(); + + let v0 = block_id_from_page(v0_page.in_space(&space)).expect("v0 page should decode"); + let v1 = block_id_from_page(v1_page.in_space(&space)).expect("v1 page should decode"); + + assert!(unsafe { v0.noun().raw_equals(&block_id) }); + assert!(unsafe { v1.noun().raw_equals(&block_id) }); + } + + #[test] + fn test_tx_id_from_raw_tx_supports_v0_and_v1() { + let mut slab: NounSlab = NounSlab::new(); + let tx_id = T(&mut slab, &[D(6), D(7), D(8), D(9), D(10)]); + let v0_raw_tx = T(&mut slab, &[tx_id, D(11)]); + let v1_raw_tx = T(&mut slab, &[D(1), tx_id, D(11)]); + let space = slab.noun_space(); + + let v0 = tx_id_from_raw_tx(v0_raw_tx.in_space(&space)).expect("v0 raw-tx should decode"); + let v1 = tx_id_from_raw_tx(v1_raw_tx.in_space(&space)).expect("v1 raw-tx should decode"); + + assert!(unsafe { v0.noun().raw_equals(&tx_id) }); + assert!(unsafe { v1.noun().raw_equals(&tx_id) }); + } + + #[test] + fn test_from_noun_slab_extracts_v1_block_and_tx_ids() { + let mut block_slab: NounSlab = NounSlab::new(); + let block_id = T(&mut block_slab, &[D(1), D(2), D(3), D(4), D(5)]); + let block_page = T(&mut block_slab, &[D(1), block_id, D(0)]); + let heard_block_tag = make_tas(&mut block_slab, "heard-block").as_noun(); + let heard_block = T(&mut block_slab, &[heard_block_tag, block_page]); + block_slab.set_root(heard_block); + let block_space = block_slab.noun_space(); + let expected_block_id = + tip5_hash_to_base58(block_id, &block_space).expect("block id should encode"); + + let parsed_block = + NockchainFact::from_noun_slab(&mut block_slab).expect("heard-block should parse"); + match parsed_block { + NockchainFact::HeardBlock(id, poke) => { + assert_eq!(id, expected_block_id); + let poke_space = poke.noun_space(); + assert!(unsafe { poke.root().in_space(&poke_space).as_cell().is_ok() }); + } + other => panic!("expected HeardBlock, got {other:?}"), + } + + let mut tx_slab: NounSlab = NounSlab::new(); + let tx_id = T(&mut tx_slab, &[D(6), D(7), D(8), D(9), D(10)]); + let raw_tx = T(&mut tx_slab, &[D(1), tx_id, D(0)]); + let heard_tx_tag = make_tas(&mut tx_slab, "heard-tx").as_noun(); + let heard_tx = T(&mut tx_slab, &[heard_tx_tag, raw_tx]); + tx_slab.set_root(heard_tx); + let tx_space = tx_slab.noun_space(); + let expected_tx_id = tip5_hash_to_base58(tx_id, &tx_space).expect("tx id should encode"); + + let parsed_tx = NockchainFact::from_noun_slab(&mut tx_slab).expect("heard-tx should parse"); + match parsed_tx { + NockchainFact::HeardTx(id, poke) => { + assert_eq!(id, expected_tx_id); + let poke_space = poke.noun_space(); + assert!(unsafe { poke.root().in_space(&poke_space).as_cell().is_ok() }); + } + other => panic!("expected HeardTx, got {other:?}"), + } + } +} diff --git a/crates/nockchain-libp2p-io/src/metrics.rs b/crates/nockchain-libp2p-io/src/metrics.rs index d0013edda..333a8833d 100644 --- a/crates/nockchain-libp2p-io/src/metrics.rs +++ b/crates/nockchain-libp2p-io/src/metrics.rs @@ -61,6 +61,14 @@ metrics_struct![ requests_crown_error_serf_load_error, "nockchain-libp2p-io.requests_crown_error_serf_load_error", Count ), + ( + requests_crown_error_serf_init_allocation_error, + "nockchain-libp2p-io.requests_crown_error_serf_init_allocation_error", Count + ), + ( + requests_crown_error_serf_init_panic, + "nockchain-libp2p-io.requests_crown_error_serf_init_panic", Count + ), (requests_crown_error_work_bail, "nockchain-libp2p-io.requests_crown_error_work_bail", Count), (requests_crown_error_peek_bail, "nockchain-libp2p-io.requests_crown_error_peek_bail", Count), (requests_crown_error_work_swap, "nockchain-libp2p-io.requests_crown_error_work_swap", Count), @@ -162,6 +170,28 @@ metrics_struct![ (request_failed, "nockchain-libp2p-io.request_failed", Count), (response_failed_not_dropped, "nockchain-libp2p-io.response_failed_not_dropped", Count), (response_dropped, "nockchain-libp2p-io.response_dropped", Count), + (ip_exclusions_active, "nockchain-libp2p-io.ip_exclusions_active", Gauge), + (address_cooldowns_active, "nockchain-libp2p-io.address_cooldowns_active", Gauge), + (ip_exclusions_created, "nockchain-libp2p-io.ip_exclusions_created", Count), + (ip_exclusions_expired, "nockchain-libp2p-io.ip_exclusions_expired", Count), + (ip_exclusion_dial_denied, "nockchain-libp2p-io.ip_exclusion_dial_denied", Count), + (address_cooldown_dial_denied, "nockchain-libp2p-io.address_cooldown_dial_denied", Count), + ( + kad_addresses_pruned_for_exclusion, + "nockchain-libp2p-io.kad_addresses_pruned_for_exclusion", Count + ), + (kad_peers_pruned_for_exclusion, "nockchain-libp2p-io.kad_peers_pruned_for_exclusion", Count), + ( + identify_addresses_skipped_for_exclusion, + "nockchain-libp2p-io.identify_addresses_skipped_for_exclusion", Count + ), + ( + fast_sync_peers_skipped_for_health, + "nockchain-libp2p-io.fast_sync_peers_skipped_for_health", Count + ), + (request_peer_cooldowns_created, "nockchain-libp2p-io.request_peer_cooldowns_created", Count), + (wrong_peer_id_observed, "nockchain-libp2p-io.wrong_peer_id_observed", Count), + (same_ip_kad_cardinality, "nockchain-libp2p-io.same_ip_kad_cardinality", Gauge), // Per-cause poke timings (timer_poke_time, "nockchain-libp2p-io.timer_poke_time", TimingCount), (heard_tx_poke_time, "nockchain-libp2p-io.heard_tx_poke_time", TimingCount), diff --git a/crates/nockchain-libp2p-io/src/p2p_state.rs b/crates/nockchain-libp2p-io/src/p2p_state.rs index b0570ffcb..e7a2513c0 100644 --- a/crates/nockchain-libp2p-io/src/p2p_state.rs +++ b/crates/nockchain-libp2p-io/src/p2p_state.rs @@ -1,16 +1,19 @@ +use std::cmp::Reverse; use std::collections::{BTreeMap, BTreeSet}; use std::net::IpAddr; use std::sync::Arc; +use std::time::Instant; use libp2p::core::ConnectedPoint; use libp2p::swarm::ConnectionId; use libp2p::{Multiaddr, PeerId, Swarm}; use nockapp::noun::slab::NounSlab; use nockapp::NockAppError; -use nockvm::noun::Noun; +use nockvm::noun::{Noun, NounSpace}; use rand::prelude::SliceRandom; use tracing::{debug, info, trace}; +use crate::ip_block::PeerExclusions; use crate::messages::NockchainDataRequest; use crate::metrics::NockchainP2PMetrics; use crate::p2p_util::MultiaddrExt; @@ -23,11 +26,20 @@ struct IpInfo { connections: BTreeSet, } +#[derive(Default)] +struct PeerRequestHealth { + successes: u64, + failures: u64, + last_success: Option, + last_failure: Option, +} + pub struct P2PState { metrics: Arc, block_id_to_peers: BTreeMap>, peer_to_block_ids: BTreeMap>, - // It's stupid that we must track this state instead of just getting it from libp2p. + // It's stupid that we must track this state locally. libp2p does not expose + // the lookup shape this driver needs. connections: BTreeMap, // subset of connections: all inbound connections inbound_connections: BTreeMap, @@ -40,6 +52,7 @@ pub struct P2PState { pub elders_cache: BTreeMap, pub elders_negative_cache: BTreeSet, pub seen_elders: BTreeSet, + peer_request_health: BTreeMap, // Highest block height seen pub first_negative: u64, pub seen_tx_clear_interval: u64, @@ -63,6 +76,7 @@ impl P2PState { elders_cache: BTreeMap::new(), elders_negative_cache: BTreeSet::new(), seen_elders: BTreeSet::new(), + peer_request_health: BTreeMap::new(), first_negative: 0, seen_tx_clear_interval, last_tx_cache_clear_height: 0, @@ -96,7 +110,7 @@ impl P2PState { self.ip_info.insert( ip, IpInfo { - connections: BTreeSet::new(), + connections, request_count: 0, ping_failure_count: 0, }, @@ -172,6 +186,78 @@ impl P2PState { } } + pub(crate) fn record_request_success(&mut self, peer_id: PeerId) { + let now = Instant::now(); + let health = self.peer_request_health.entry(peer_id).or_default(); + health.successes = health.successes.saturating_add(1); + health.last_success = Some(now); + } + + pub(crate) fn record_request_failure(&mut self, peer_id: PeerId) { + let now = Instant::now(); + let health = self.peer_request_health.entry(peer_id).or_default(); + health.failures = health.failures.saturating_add(1); + health.last_failure = Some(now); + } + + pub(crate) fn select_request_peers( + &self, + target_peers: Vec, + limit: usize, + exclusions: &PeerExclusions, + ) -> Vec { + let mut healthy = Vec::new(); + let mut fallback = Vec::new(); + + for peer_id in target_peers { + if self.peer_has_only_excluded_ips(&peer_id, exclusions) { + self.metrics.fast_sync_peers_skipped_for_health.increment(); + continue; + } + + fallback.push(peer_id); + if exclusions.is_peer_request_cooled_down(&peer_id) { + self.metrics.fast_sync_peers_skipped_for_health.increment(); + continue; + } + healthy.push(peer_id); + } + + let mut selected_from = if healthy.is_empty() { + fallback + } else { + healthy + }; + selected_from.shuffle(&mut rand::rng()); + selected_from.sort_by_key(|peer_id| Reverse(self.request_score(peer_id))); + selected_from.truncate(limit); + selected_from + } + + fn request_score(&self, peer_id: &PeerId) -> i64 { + self.peer_request_health + .get(peer_id) + .map(|health| health.successes as i64 * 2 - health.failures as i64) + .unwrap_or_default() + } + + fn peer_has_only_excluded_ips(&self, peer_id: &PeerId, exclusions: &PeerExclusions) -> bool { + let Some(connections) = self.peer_connections.get(peer_id) else { + return false; + }; + let mut saw_ip = false; + for addr in connections.values() { + let Some(ip) = addr.ip_addr() else { + continue; + }; + saw_ip = true; + if !exclusions.is_ip_excluded(&ip) { + return false; + } + } + saw_ip + } + pub(crate) fn ping_succeeded(&mut self, connection: ConnectionId) { let addr = self.connection_address(connection); let Some(addr) = addr else { @@ -269,8 +355,9 @@ impl P2PState { &mut self, block_id: Noun, peer_id: PeerId, + space: &NounSpace, ) -> Result<(), NockAppError> { - let block_id_str = tip5_hash_to_base58(block_id)?; + let block_id_str = tip5_hash_to_base58(block_id, space)?; self.track_block_id_str_and_peer(block_id_str, peer_id); Ok(()) } @@ -282,8 +369,9 @@ impl P2PState { &mut self, block_id: Noun, peer_id: PeerId, + space: &NounSpace, ) -> Result { - let block_id_str = tip5_hash_to_base58(block_id)?; + let block_id_str = tip5_hash_to_base58(block_id, space)?; if self.block_id_to_peers.contains_key(&block_id_str) { self.track_block_id_str_and_peer(block_id_str, peer_id); @@ -295,16 +383,20 @@ impl P2PState { /// Removes a block ID from the tracker. /// implements [%track %remove block-id] effect - pub fn remove_block_id(&mut self, block_id: Noun) -> Result<(), NockAppError> { - let block_id_str = tip5_hash_to_base58(block_id)?; + pub fn remove_block_id( + &mut self, + block_id: Noun, + space: &NounSpace, + ) -> Result<(), NockAppError> { + let block_id_str = tip5_hash_to_base58(block_id, space)?; self.remove_block_id_str(&block_id_str); Ok(()) } /// Returns a list of peers that have sent us a given block ID. #[allow(dead_code)] - pub fn get_peers_for_block_id(&self, block_id: Noun) -> Vec { - let Ok(block_id_str) = tip5_hash_to_base58(block_id) else { + pub fn get_peers_for_block_id(&self, block_id: Noun, space: &NounSpace) -> Vec { + let Ok(block_id_str) = tip5_hash_to_base58(block_id, space) else { panic!("Invalid block ID"); }; self.block_id_to_peers @@ -324,8 +416,8 @@ impl P2PState { /// Returns true if we are tracking a given block ID. #[allow(dead_code)] - pub fn is_tracking_block_id(&self, block_id: Noun) -> bool { - let Ok(block_id_str) = tip5_hash_to_base58(block_id) else { + pub fn is_tracking_block_id(&self, block_id: Noun, space: &NounSpace) -> bool { + let Ok(block_id_str) = tip5_hash_to_base58(block_id, space) else { return false; }; self.block_id_to_peers.contains_key(&block_id_str) @@ -338,8 +430,12 @@ impl P2PState { // Removes the block id from the MessageTracker maps and returns all the // peers who had sent us that block. - pub fn process_bad_block_id(&mut self, block_id: Noun) -> Result, NockAppError> { - let block_id_str = tip5_hash_to_base58(block_id)?; + pub fn process_bad_block_id( + &mut self, + block_id: Noun, + space: &NounSpace, + ) -> Result, NockAppError> { + let block_id_str = tip5_hash_to_base58(block_id, space)?; let peers_to_ban = self .block_id_to_peers .get(&block_id_str) @@ -351,7 +447,7 @@ impl P2PState { self.remove_peer(peer); } - self.remove_block_id(block_id)?; + self.remove_block_id(block_id, space)?; Ok(peers_to_ban) } @@ -416,14 +512,19 @@ mod tests { use nockapp::noun::slab::NounSlab; use nockapp::AtomExt; - use nockvm::noun::{D, T}; + use nockvm::mem::{NockStack, NOCK_STACK_SIZE_TINY}; + use nockvm::noun::{NounAllocator, D, T}; - use super::*; - use crate::config::LibP2PConfig; + use crate::config::{LibP2PConfig, PeerExclusionConfig}; + use crate::p2p_state::*; use crate::p2p_util::PeerIdExt; pub static LIBP2P_CONFIG: LazyLock = LazyLock::new(LibP2PConfig::default); + fn install_test_arena() -> NockStack { + NockStack::new(NOCK_STACK_SIZE_TINY, 0) + } + #[test] #[cfg_attr(miri, ignore)] // ibig has a memory leak so miri fails this test fn test_message_tracker_basic() { @@ -437,10 +538,11 @@ mod tests { // Create a block ID as [1 2 3 4 5] let mut slab: NounSlab = NounSlab::new(); let block_id_tuple = T(&mut slab, &[D(1), D(2), D(3), D(4), D(5)]); + let space = slab.noun_space(); // Add the block ID tracker - .track_block_id_and_peer(block_id_tuple, peer_id) + .track_block_id_and_peer(block_id_tuple, peer_id, &space) .unwrap_or_else(|_| { panic!( "Called `expect()` at {}:{} (git sha: {})", @@ -451,7 +553,7 @@ mod tests { }); // Get the block ID string - let block_id_str = tip5_hash_to_base58(block_id_tuple).unwrap_or_else(|_| { + let block_id_str = tip5_hash_to_base58(block_id_tuple, &space).unwrap_or_else(|_| { panic!( "Called `expect()` at {}:{} (git sha: {})", file!(), @@ -465,14 +567,16 @@ mod tests { assert!(tracker.peer_to_block_ids.contains_key(&peer_id)); // Remove the block ID - tracker.remove_block_id(block_id_tuple).unwrap_or_else(|_| { - panic!( - "Called `expect()` at {}:{} (git sha: {})", - file!(), - line!(), - option_env!("GIT_SHA").unwrap_or("unknown") - ) - }); + tracker + .remove_block_id(block_id_tuple, &space) + .unwrap_or_else(|_| { + panic!( + "Called `expect()` at {}:{} (git sha: {})", + file!(), + line!(), + option_env!("GIT_SHA").unwrap_or("unknown") + ) + }); // Verify it was removed assert!(!tracker.block_id_to_peers.contains_key(&block_id_str)); @@ -482,6 +586,7 @@ mod tests { #[test] #[cfg_attr(miri, ignore)] // ibig has a memory leak so miri fails this test fn test_bad_block_id() { + let _arena = install_test_arena(); let metrics = Arc::new( NockchainP2PMetrics::register(gnort::global_metrics_registry()) .expect("Could not register metrics"), @@ -492,10 +597,11 @@ mod tests { // Create a block ID let mut slab: NounSlab = NounSlab::new(); let block_id_tuple = T(&mut slab, &[D(1), D(2), D(3), D(4), D(5)]); + let space = slab.noun_space(); // Track the block ID tracker - .track_block_id_and_peer(block_id_tuple, peer_id) + .track_block_id_and_peer(block_id_tuple, peer_id, &space) .unwrap_or_else(|_| { panic!( "Called `expect()` at {}:{} (git sha: {})", @@ -507,7 +613,7 @@ mod tests { // Mark it as bad let peers_to_ban = tracker - .process_bad_block_id(block_id_tuple) + .process_bad_block_id(block_id_tuple, &space) .unwrap_or_else(|_| { panic!( "Called `expect()` at {}:{} (git sha: {})", @@ -524,6 +630,7 @@ mod tests { #[test] fn test_peer_id_base58_roundtrip() { + let _arena = install_test_arena(); use nockvm::noun::Atom; // Generate a random PeerId let original_peer_id = PeerId::random(); @@ -536,14 +643,16 @@ mod tests { .expect("Failed to create peer ID atom"); // Use the from_noun method to convert back to PeerId - let recovered_peer_id = PeerId::from_noun(peer_id_atom.as_noun()).unwrap_or_else(|_| { - panic!( - "Called `expect()` at {}:{} (git sha: {})", - file!(), - line!(), - option_env!("GIT_SHA").unwrap_or("unknown") - ) - }); + let space = slab.noun_space(); + let recovered_peer_id = + PeerId::from_noun(peer_id_atom.as_noun(), &space).unwrap_or_else(|_| { + panic!( + "Called `expect()` at {}:{} (git sha: {})", + file!(), + line!(), + option_env!("GIT_SHA").unwrap_or("unknown") + ) + }); // Verify round trip assert_eq!(original_peer_id, recovered_peer_id); @@ -552,6 +661,7 @@ mod tests { #[test] #[cfg_attr(miri, ignore)] // ibig has a memory leak so miri fails this test fn test_add_peer_if_tracking_block_id() { + let _arena = install_test_arena(); let metrics = Arc::new( NockchainP2PMetrics::register(gnort::global_metrics_registry()) .expect("Could not register metrics"), @@ -563,10 +673,11 @@ mod tests { // Create a block ID let mut slab: NounSlab = NounSlab::new(); let block_id_tuple = T(&mut slab, &[D(1), D(2), D(3), D(4), D(5)]); + let space = slab.noun_space(); // First, try to add a peer to a non-existent block ID let result = tracker - .add_peer_if_tracking_block_id(block_id_tuple, peer_id1) + .add_peer_if_tracking_block_id(block_id_tuple, peer_id1, &space) .unwrap_or_else(|_| { panic!( "Called `expect()` at {}:{} (git sha: {})", @@ -579,7 +690,7 @@ mod tests { // Now track the block ID with peer1 tracker - .track_block_id_and_peer(block_id_tuple, peer_id1) + .track_block_id_and_peer(block_id_tuple, peer_id1, &space) .unwrap_or_else(|_| { panic!( "Called `expect()` at {}:{} (git sha: {})", @@ -591,7 +702,7 @@ mod tests { // Add peer2 to the existing block ID let result = tracker - .add_peer_if_tracking_block_id(block_id_tuple, peer_id2) + .add_peer_if_tracking_block_id(block_id_tuple, peer_id2, &space) .unwrap_or_else(|_| { panic!( "Called `expect()` at {}:{} (git sha: {})", @@ -603,7 +714,7 @@ mod tests { assert!(result); // Should return true since block ID exists // Verify both peers are associated with the block ID - let peers = tracker.get_peers_for_block_id(block_id_tuple); + let peers = tracker.get_peers_for_block_id(block_id_tuple, &space); assert_eq!(peers.len(), 2); assert!(peers.contains(&peer_id1)); assert!(peers.contains(&peer_id2)); @@ -612,6 +723,7 @@ mod tests { #[test] #[cfg_attr(miri, ignore)] // ibig has a memory leak so miri fails this test fn test_add_peer_if_tracking_block_id_then_remove() { + let _arena = install_test_arena(); let metrics = Arc::new( NockchainP2PMetrics::register(gnort::global_metrics_registry()) .expect("Could not register metrics"), @@ -623,7 +735,8 @@ mod tests { // Create a block ID let mut slab: NounSlab = NounSlab::new(); let block_id_tuple = T(&mut slab, &[D(1), D(2), D(3), D(4), D(5)]); - let block_id_str = tip5_hash_to_base58(block_id_tuple).unwrap_or_else(|_| { + let space = slab.noun_space(); + let block_id_str = tip5_hash_to_base58(block_id_tuple, &space).unwrap_or_else(|_| { panic!( "Called `expect()` at {}:{} (git sha: {})", file!(), @@ -634,7 +747,7 @@ mod tests { // Track the block ID with peer1 tracker - .track_block_id_and_peer(block_id_tuple, peer_id1) + .track_block_id_and_peer(block_id_tuple, peer_id1, &space) .unwrap_or_else(|_| { panic!( "Called `expect()` at {}:{} (git sha: {})", @@ -646,7 +759,7 @@ mod tests { // Add peer2 to the existing block ID let result = tracker - .add_peer_if_tracking_block_id(block_id_tuple, peer_id2) + .add_peer_if_tracking_block_id(block_id_tuple, peer_id2, &space) .unwrap_or_else(|_| { panic!( "Called `expect()` at {}:{} (git sha: {})", @@ -658,23 +771,25 @@ mod tests { assert!(result); // Should return true since block ID exists // Verify both peers are associated with the block ID - let peers = tracker.get_peers_for_block_id(block_id_tuple); + let peers = tracker.get_peers_for_block_id(block_id_tuple, &space); assert_eq!(peers.len(), 2); assert!(peers.contains(&peer_id1)); assert!(peers.contains(&peer_id2)); // Now remove the block ID - tracker.remove_block_id(block_id_tuple).unwrap_or_else(|_| { - panic!( - "Called `expect()` at {}:{} (git sha: {})", - file!(), - line!(), - option_env!("GIT_SHA").unwrap_or("unknown") - ) - }); + tracker + .remove_block_id(block_id_tuple, &space) + .unwrap_or_else(|_| { + panic!( + "Called `expect()` at {}:{} (git sha: {})", + file!(), + line!(), + option_env!("GIT_SHA").unwrap_or("unknown") + ) + }); // Verify the block ID is no longer tracked - let peers_after_removal = tracker.get_peers_for_block_id(block_id_tuple); + let peers_after_removal = tracker.get_peers_for_block_id(block_id_tuple, &space); assert_eq!(peers_after_removal.len(), 0); // Verify the block ID is removed from block_id_to_peers @@ -694,6 +809,7 @@ mod tests { #[test] #[cfg_attr(miri, ignore)] // ibig has a memory leak so miri fails this test fn test_process_bad_block_id_removes_peers() { + let _arena = install_test_arena(); let metrics = Arc::new( NockchainP2PMetrics::register(gnort::global_metrics_registry()) .expect("Could not register metrics"), @@ -705,13 +821,14 @@ mod tests { // Create a block ID let mut slab: NounSlab = NounSlab::new(); let block_id_tuple = T(&mut slab, &[D(1), D(2), D(3), D(4), D(5)]); + let space = slab.noun_space(); // Create another block ID that both peers will share let other_block_id = T(&mut slab, &[D(6), D(7), D(8), D(9), D(10)]); // Track both block IDs with both peers tracker - .track_block_id_and_peer(block_id_tuple, peer_id1) + .track_block_id_and_peer(block_id_tuple, peer_id1, &space) .unwrap_or_else(|_| { panic!( "Called `expect()` at {}:{} (git sha: {})", @@ -721,7 +838,7 @@ mod tests { ) }); tracker - .add_peer_if_tracking_block_id(block_id_tuple, peer_id2) + .add_peer_if_tracking_block_id(block_id_tuple, peer_id2, &space) .unwrap_or_else(|_| { panic!( "Called `expect()` at {}:{} (git sha: {})", @@ -731,7 +848,7 @@ mod tests { ) }); tracker - .track_block_id_and_peer(other_block_id, peer_id1) + .track_block_id_and_peer(other_block_id, peer_id1, &space) .unwrap_or_else(|_| { panic!( "Called `expect()` at {}:{} (git sha: {})", @@ -741,7 +858,7 @@ mod tests { ) }); tracker - .add_peer_if_tracking_block_id(other_block_id, peer_id2) + .add_peer_if_tracking_block_id(other_block_id, peer_id2, &space) .unwrap_or_else(|_| { panic!( "Called `expect()` at {}:{} (git sha: {})", @@ -757,7 +874,7 @@ mod tests { // Process the bad block ID let banned_peers = tracker - .process_bad_block_id(block_id_tuple) + .process_bad_block_id(block_id_tuple, &space) .unwrap_or_else(|_| { panic!( "Called `expect()` at {}:{} (git sha: {})", @@ -778,7 +895,148 @@ mod tests { // Verify the other block ID is also no longer tracked // (since we removed the peers entirely) - assert!(!tracker.is_tracking_block_id(other_block_id)); + assert!(!tracker.is_tracking_block_id(other_block_id, &space)); + } + + #[test] + fn select_request_peers_skips_cooled_peer_when_healthy_peer_exists() { + let metrics = Arc::new( + NockchainP2PMetrics::register(gnort::global_metrics_registry()) + .expect("Could not register metrics"), + ); + let tracker = P2PState::new(metrics, LIBP2P_CONFIG.seen_tx_clear_interval); + let exclusions = PeerExclusions::new(PeerExclusionConfig { + request_peer_cooldown_secs: 60, + ..PeerExclusionConfig::default() + }); + let cooled_peer = PeerId::random(); + let healthy_peer = PeerId::random(); + + assert!(exclusions.record_peer_request_failure(cooled_peer)); + + let selected = + tracker.select_request_peers(vec![cooled_peer, healthy_peer], 10, &exclusions); + + assert_eq!(selected, vec![healthy_peer]); + } + + #[test] + fn select_request_peers_falls_back_when_every_peer_is_cooled() { + let metrics = Arc::new( + NockchainP2PMetrics::register(gnort::global_metrics_registry()) + .expect("Could not register metrics"), + ); + let tracker = P2PState::new(metrics, LIBP2P_CONFIG.seen_tx_clear_interval); + let exclusions = PeerExclusions::new(PeerExclusionConfig { + request_peer_cooldown_secs: 60, + ..PeerExclusionConfig::default() + }); + let left = PeerId::random(); + let right = PeerId::random(); + + assert!(exclusions.record_peer_request_failure(left)); + assert!(exclusions.record_peer_request_failure(right)); + + let selected = tracker.select_request_peers(vec![left, right], 10, &exclusions); + + assert_eq!(selected.len(), 2); + assert!(selected.contains(&left)); + assert!(selected.contains(&right)); + } + + #[test] + fn select_request_peers_drops_peer_with_only_excluded_ips() { + let metrics = Arc::new( + NockchainP2PMetrics::register(gnort::global_metrics_registry()) + .expect("Could not register metrics"), + ); + let mut tracker = P2PState::new(metrics, LIBP2P_CONFIG.seen_tx_clear_interval); + let exclusions = PeerExclusions::new(PeerExclusionConfig { + wrong_peer_id_ip_threshold: 1, + ..PeerExclusionConfig::default() + }); + let excluded_peer = PeerId::random(); + let healthy_peer = PeerId::random(); + let excluded_addr = "/ip4/15.235.216.78/udp/3602/quic-v1" + .parse::() + .expect("valid multiaddr"); + let healthy_addr = "/ip4/203.0.113.9/udp/3006/quic-v1" + .parse::() + .expect("valid multiaddr"); + + tracker + .peer_connections + .entry(excluded_peer) + .or_default() + .insert(ConnectionId::new_unchecked(1), excluded_addr.clone()); + tracker + .peer_connections + .entry(healthy_peer) + .or_default() + .insert(ConnectionId::new_unchecked(2), healthy_addr); + + let outcome = + exclusions.record_wrong_peer_id(&excluded_addr, Some(excluded_peer), PeerId::random()); + assert!(outcome.ip_exclusion.is_some()); + + let selected = + tracker.select_request_peers(vec![excluded_peer, healthy_peer], 10, &exclusions); + + assert_eq!(selected, vec![healthy_peer]); + } + + #[test] + fn select_request_peers_keeps_peer_with_a_clean_connection() { + let metrics = Arc::new( + NockchainP2PMetrics::register(gnort::global_metrics_registry()) + .expect("Could not register metrics"), + ); + let mut tracker = P2PState::new(metrics, LIBP2P_CONFIG.seen_tx_clear_interval); + let exclusions = PeerExclusions::new(PeerExclusionConfig { + wrong_peer_id_ip_threshold: 1, + ..PeerExclusionConfig::default() + }); + let peer = PeerId::random(); + let excluded_addr = "/ip4/15.235.216.78/udp/3602/quic-v1" + .parse::() + .expect("valid multiaddr"); + let clean_addr = "/ip4/203.0.113.9/udp/3006/quic-v1" + .parse::() + .expect("valid multiaddr"); + + tracker + .peer_connections + .entry(peer) + .or_default() + .insert(ConnectionId::new_unchecked(1), excluded_addr.clone()); + tracker + .peer_connections + .entry(peer) + .or_default() + .insert(ConnectionId::new_unchecked(2), clean_addr); + + let outcome = exclusions.record_wrong_peer_id(&excluded_addr, Some(peer), PeerId::random()); + assert!(outcome.ip_exclusion.is_some()); + + let selected = tracker.select_request_peers(vec![peer], 10, &exclusions); + + assert_eq!(selected, vec![peer]); + } + + #[test] + fn request_success_clears_request_cooldown() { + let exclusions = PeerExclusions::new(PeerExclusionConfig { + request_peer_cooldown_secs: 60, + ..PeerExclusionConfig::default() + }); + let peer = PeerId::random(); + + assert!(exclusions.record_peer_request_failure(peer)); + assert!(exclusions.is_peer_request_cooled_down(&peer)); + + exclusions.record_peer_request_success(&peer); + + assert!(!exclusions.is_peer_request_cooled_down(&peer)); } #[test] diff --git a/crates/nockchain-libp2p-io/src/p2p_util.rs b/crates/nockchain-libp2p-io/src/p2p_util.rs index 46065a49d..e85f4f5e7 100644 --- a/crates/nockchain-libp2p-io/src/p2p_util.rs +++ b/crates/nockchain-libp2p-io/src/p2p_util.rs @@ -2,8 +2,8 @@ use std::net::{IpAddr, Ipv4Addr, Ipv6Addr}; use std::str::FromStr; use libp2p::{Multiaddr, PeerId}; -use nockapp::{AtomExt, NockAppError}; -use nockvm::noun::Noun; +use nockapp::NockAppError; +use nockvm::noun::{Noun, NounSpace}; use tracing::warn; // The warn logs are specifically constructed for fail2ban @@ -16,12 +16,12 @@ pub fn log_fail2ban_ipv6(peer_id: &PeerId, ip: &Ipv6Addr) { } pub trait PeerIdExt { - fn from_noun(noun: Noun) -> Result; + fn from_noun(noun: Noun, space: &NounSpace) -> Result; } impl PeerIdExt for PeerId { - fn from_noun(noun: Noun) -> Result { - let peer_id_bytes = noun.as_atom()?.to_bytes_until_nul()?; + fn from_noun(noun: Noun, space: &NounSpace) -> Result { + let peer_id_bytes = noun.in_space(space).as_atom()?.to_bytes_until_nul()?; let peer_id_str = String::from_utf8(peer_id_bytes)?; PeerId::from_str(&peer_id_str) .map_err(|_| NockAppError::OtherError(String::from("Failed to parse PeerId from noun"))) diff --git a/crates/nockchain-libp2p-io/src/tip5_util.rs b/crates/nockchain-libp2p-io/src/tip5_util.rs index aeca4da38..5d3cd8dbe 100644 --- a/crates/nockchain-libp2p-io/src/tip5_util.rs +++ b/crates/nockchain-libp2p-io/src/tip5_util.rs @@ -1,7 +1,7 @@ use bs58; use ibig::{ubig, Stack, UBig}; use nockapp::NockAppError; -use nockvm::noun::Noun; +use nockvm::noun::{Noun, NounSpace}; use noun_serde::prelude::*; use num_bigint::BigUint; @@ -21,8 +21,8 @@ const P: u64 = 0xffffffff00000001; /// /// # Returns /// The Noun as a Base58 string -pub fn tip5_hash_to_base58(noun: Noun) -> Result { - let tuple: [u64; 5] = noun.decode()?; +pub fn tip5_hash_to_base58(noun: Noun, space: &NounSpace) -> Result { + let tuple: [u64; 5] = noun.decode(space)?; let decimal_value = base_p_to_decimal(tuple)?; let base58_string = ubig_to_base58(decimal_value); @@ -33,8 +33,9 @@ pub fn tip5_hash_to_base58(noun: Noun) -> Result { pub fn tip5_hash_to_base58_stack( stack: &mut S, noun: Noun, + space: &NounSpace, ) -> Result { - let tuple: [u64; 5] = noun.decode()?; + let tuple: [u64; 5] = noun.decode(space)?; let decimal_value = base_p_to_decimal_stack(stack, tuple)?; let base58_string = ubig_to_base58(decimal_value); @@ -128,7 +129,7 @@ pub fn decimal_to_base_p(value: UBig) -> Result<[u64; 5], NockAppError> { #[allow(clippy::unwrap_used)] mod tests { use nockapp::noun::slab::NounSlab; - use nockvm::noun::{D, T}; + use nockvm::noun::{NounAllocator, D, T}; use quickcheck::{Arbitrary, Gen, QuickCheck, TestResult}; use super::*; @@ -158,7 +159,8 @@ mod tests { // Test case 1: Simple tuple [1 2 3 4 5] let tuple1 = T(&mut slab, &[D(1), D(2), D(3), D(4), D(5)]); let expected1 = "2V9arU36gvtaofWmNowewoj9u7gbNA2qsJZEQ3WPky5mQ"; - let result1 = tip5_hash_to_base58_stack(&mut slab, tuple1).unwrap(); + let space = slab.noun_space(); + let result1 = tip5_hash_to_base58_stack(&mut slab, tuple1, &space).unwrap(); assert_eq!(result1, expected1); // Test case 2: Complex values @@ -173,7 +175,8 @@ mod tests { &[a1.as_noun(), a2.as_noun(), a3.as_noun(), a4.as_noun(), a5.as_noun()], ); let expected2 = "6UkUko9WTwwR6VVRXwPQpUy5pswdvNtoyHspY5n9nLVnBxzAgEyMwPR"; - let result2 = tip5_hash_to_base58_stack(&mut slab, tuple2).unwrap(); + let space = slab.noun_space(); + let result2 = tip5_hash_to_base58_stack(&mut slab, tuple2, &space).unwrap(); assert_eq!(result2, expected2); } @@ -187,7 +190,8 @@ mod tests { // Test case 1: Simple tuple [1 2 3 4 5] let tuple1 = T(&mut slab, &[D(1), D(2), D(3), D(4), D(5)]); let expected1 = "2V9arU36gvtaofWmNowewoj9u7gbNA2qsJZEQ3WPky5mQ"; - let result1 = tip5_hash_to_base58(tuple1).unwrap_or_else(|_| { + let space = slab.noun_space(); + let result1 = tip5_hash_to_base58(tuple1, &space).unwrap_or_else(|_| { panic!( "Called `expect()` at {}:{} (git sha: {})", file!(), @@ -209,7 +213,8 @@ mod tests { &[a1.as_noun(), a2.as_noun(), a3.as_noun(), a4.as_noun(), a5.as_noun()], ); let expected2 = "6UkUko9WTwwR6VVRXwPQpUy5pswdvNtoyHspY5n9nLVnBxzAgEyMwPR"; - let result2 = tip5_hash_to_base58(tuple2).unwrap_or_else(|err| { + let space = slab.noun_space(); + let result2 = tip5_hash_to_base58(tuple2, &space).unwrap_or_else(|err| { panic!( "Panicked with {err:?} at {}:{} (git sha: {:?})", file!(), @@ -431,9 +436,10 @@ mod tests { &mut slab, &[D(tip5[0]), D(tip5[1]), D(tip5[2]), D(tip5[3]), D(tip5[4])], ); + let space = slab.noun_space(); // Convert to base58 and back - let base58 = match tip5_hash_to_base58(noun) { + let base58 = match tip5_hash_to_base58(noun, &space) { Ok(s) => s, Err(_) => return TestResult::discard(), }; diff --git a/crates/nockchain-libp2p-io/testdata/README.md b/crates/nockchain-libp2p-io/testdata/README.md new file mode 100644 index 000000000..6caa455d3 --- /dev/null +++ b/crates/nockchain-libp2p-io/testdata/README.md @@ -0,0 +1,42 @@ +# Req-Res Conformance Vectors + +This directory holds machine-readable CBOR conformance vectors for the libp2p request-response transport. + +Current fixture: +- `req_res_gen1_cbor_vectors.json` + +## Fixture goals + +- Keep canonical bytes explicit and reviewable. +- Make serialization behavior executable in tests. +- Give third-party implementations a deterministic target. + +## Schema summary + +Top-level fields: +- `schema_version` +- `request_vectors` +- `response_vectors` +- `invalid_vectors` + +`request_vectors` and `response_vectors` include: +- a stable `id` +- structured fields (`variant`, payload fields) +- canonical `cbor_hex` + +`invalid_vectors` include: +- a stable `id` +- decode target (`request` or `response`) +- malformed `cbor_hex` +- optional `error_substring` assertion + +## Validation + +Vectors are executed by tests in: +- `open/crates/nockchain-libp2p-io/src/cbor_tests.rs` + +Current entry points: +- `test_gen1_cbor_vector_schema_version` +- `test_gen1_request_cbor_vectors_roundtrip` +- `test_gen1_response_cbor_vectors_roundtrip` +- `test_gen1_invalid_cbor_vectors_fail_decode` diff --git a/crates/nockchain-libp2p-io/testdata/req_res_gen1_cbor_vectors.json b/crates/nockchain-libp2p-io/testdata/req_res_gen1_cbor_vectors.json new file mode 100644 index 000000000..d1e82d2c3 --- /dev/null +++ b/crates/nockchain-libp2p-io/testdata/req_res_gen1_cbor_vectors.json @@ -0,0 +1,96 @@ +{ + "schema_version": "req_res_gen1_cbor_v1", + "request_vectors": [ + { + "id": "request_gossip_small", + "variant": "gossip", + "message_hex": "01020304", + "cbor_hex": "a166476f73736970a1676d6573736167654401020304" + }, + { + "id": "request_gossip_empty", + "variant": "gossip", + "message_hex": "", + "cbor_hex": "a166476f73736970a1676d65737361676540" + }, + { + "id": "request_pow_zero_nonce_zero", + "variant": "request", + "pow_hex": "00000000000000000000000000000000", + "nonce": 0, + "message_hex": "aabbcc", + "cbor_hex": "a16752657175657374a363706f779000000000000000000000000000000000656e6f6e636500676d65737361676543aabbcc" + }, + { + "id": "request_pow_ff_nonce_42", + "variant": "request", + "pow_hex": "ffffffffffffffffffffffffffffffff", + "nonce": 42, + "message_hex": "10203040", + "cbor_hex": "a16752657175657374a363706f779018ff18ff18ff18ff18ff18ff18ff18ff18ff18ff18ff18ff18ff18ff18ff18ff656e6f6e6365182a676d6573736167654410203040" + } + ], + "response_vectors": [ + { + "id": "response_ack_true", + "variant": "ack", + "acked": true, + "cbor_hex": "a16341636ba16561636b6564f5" + }, + { + "id": "response_ack_false", + "variant": "ack", + "acked": false, + "cbor_hex": "a16341636ba16561636b6564f4" + }, + { + "id": "response_result_small", + "variant": "result", + "message_hex": "deadbeef", + "cbor_hex": "a166526573756c74a1676d65737361676544deadbeef" + }, + { + "id": "response_result_empty", + "variant": "result", + "message_hex": "", + "cbor_hex": "a166526573756c74a1676d65737361676540" + } + ], + "invalid_vectors": [ + { + "id": "request_empty_payload", + "target": "request", + "cbor_hex": "", + "error_substring": "Eof" + }, + { + "id": "request_truncated_gossip_small", + "target": "request", + "cbor_hex": "a166476f73736970a1676d65737361676544010203", + "error_substring": "Eof" + }, + { + "id": "response_empty_payload", + "target": "response", + "cbor_hex": "", + "error_substring": "Eof" + }, + { + "id": "response_truncated_ack_true", + "target": "response", + "cbor_hex": "a16341636ba16561636b6564", + "error_substring": "Eof" + }, + { + "id": "response_truncated_result_small", + "target": "response", + "cbor_hex": "a166526573756c74a1676d65737361676544deadbe", + "error_substring": "Eof" + }, + { + "id": "request_invalid_ff", + "target": "request", + "cbor_hex": "ff" + } + ] +} diff --git a/crates/nockchain-math/Cargo.toml b/crates/nockchain-math/Cargo.toml index 018c2061b..78d8710e1 100644 --- a/crates/nockchain-math/Cargo.toml +++ b/crates/nockchain-math/Cargo.toml @@ -18,7 +18,7 @@ num-traits.workspace = true once_cell.workspace = true quickcheck.workspace = true rkyv.workspace = true -serde = { version = "1.0.217", features = ["derive"], default-features = false } +serde = { workspace = true, features = ["derive"], default-features = false } thiserror.workspace = true tracing.workspace = true diff --git a/crates/nockchain-math/src/belt.rs b/crates/nockchain-math/src/belt.rs index 453f3a895..56417e018 100644 --- a/crates/nockchain-math/src/belt.rs +++ b/crates/nockchain-math/src/belt.rs @@ -1,10 +1,11 @@ #![allow(clippy::len_without_is_empty)] +use std::fmt::LowerHex; use std::ops::{Add, Div, Mul, Neg, Sub}; use nockvm::jets::util::BAIL_EXIT; use nockvm::jets::JetErr; -use nockvm::noun::Noun; +use nockvm::noun::NounSpace; use noun_serde::{NounDecode, NounEncode}; use num_traits::Pow; use rkyv::{Archive, Deserialize, Serialize}; @@ -37,8 +38,20 @@ pub const ORDER: u64 = 2_u64.pow(32); Default, )] #[repr(transparent)] +/// Prime-field element modulo [`PRIME`]. +/// +/// Do not blanket-convert with `D(belt.0)` when encoding nouns. A `Belt` value +/// may be valid in the field but still exceed the atom direct-immediate limit +/// (`DIRECT_MAX`). Always allocate through `Atom::new` (or equivalent) so large +/// field elements are encoded correctly. pub struct Belt(pub u64); +impl LowerHex for Belt { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "{:x}", self.0) + } +} + impl SerdeSerialize for Belt { fn serialize(&self, serializer: S) -> Result where @@ -79,13 +92,18 @@ macro_rules! based { // Manual implementation for Belt to encode/decode as atom impl NounEncode for Belt { fn to_noun(&self, allocator: &mut A) -> nockvm::noun::Noun { + // Intentionally avoid `D(self.0)`: field elements can be above DIRECT_MAX. nockvm::noun::Atom::new(allocator, self.0).as_noun() } } impl NounDecode for Belt { - fn from_noun(noun: &nockvm::noun::Noun) -> Result { + fn from_noun( + noun: &nockvm::noun::Noun, + space: &NounSpace, + ) -> Result { let atom = noun + .in_space(space) .as_atom() .map_err(|_| noun_serde::NounDecodeError::ExpectedAtom)?; let value = atom @@ -261,19 +279,6 @@ impl TryFrom<&u64> for Belt { } } -impl TryFrom for Belt { - type Error = (); - - #[inline(always)] - fn try_from(n: Noun) -> std::result::Result { - if !n.is_atom() { - Err(()) - } else { - Belt::try_from(&n.as_atom()?.as_u64()?) - } - } -} - impl From for Belt { #[inline(always)] fn from(f: u64) -> Self { diff --git a/crates/nockchain-math/src/convert.rs b/crates/nockchain-math/src/convert.rs index 481cb897c..49076ce6e 100644 --- a/crates/nockchain-math/src/convert.rs +++ b/crates/nockchain-math/src/convert.rs @@ -1,14 +1,15 @@ -use either::{Left, Right}; use nockvm::jets::util::BAIL_FAIL; use nockvm::jets::JetErr; -use nockvm::noun::{Atom, Cell, Error, IndirectAtom, Noun, Result, D}; +use nockvm::noun::{ + Atom, AtomHandle, CellHandle, Error, IndirectAtom, Noun, NounHandle, NounSpace, Result, D, +}; use noun_serde::{NounDecode, NounEncode}; use crate::belt::*; use crate::felt::*; use crate::handle::{finalize_poly, new_handle_mut_slice}; use crate::mary::*; -use crate::noun_ext::{AtomMathExt, NounMathExt}; +use crate::noun_ext::{AtomMathExt, AtomMathExtHandle, NounMathExt, NounMathExtHandle}; use crate::poly::*; impl AtomMathExt for Atom { @@ -24,50 +25,150 @@ impl AtomMathExt for Atom { } } - fn as_belt(&self) -> Result { - if let Ok(x) = self.as_u64() { + fn as_belt(&self, space: &NounSpace) -> Result { + if let Ok(x) = self.in_space(space).as_u64() { Ok(Belt(x)) } else { Err(Error::NotRepresentable) } } - fn as_felt<'a>(&self) -> Result<&'a Felt> { - if let Ok(atom) = self.as_indirect() { - if atom.size() == 4 { - let buf_ptr = atom.data_pointer(); - unsafe { - assert!(*(buf_ptr.add(3)) == 0x1); - } - let felt_ref: &Felt = unsafe { &*(buf_ptr as *const Felt) }; - Ok(felt_ref) - } else { + fn as_felt<'a>(&self, space: &NounSpace) -> Result<&'a Felt> { + let handle = self.in_space(space); + if handle.is_direct() { + return Err(Error::NotRepresentable); + } + if handle.size() != 4 { + return Err(Error::NotRepresentable); + } + let buf_ptr = handle.data_pointer(); + unsafe { + if *(buf_ptr.add(3)) != 0x1 { + return Err(Error::NotRepresentable); + } + Ok(&*(buf_ptr as *const Felt)) + } + } + + fn as_mut_felt<'a>(&self, space: &NounSpace) -> Result<&'a mut Felt> { + let handle = self.in_space(space); + if handle.is_direct() { + return Err(Error::NotRepresentable); + } + if handle.size() != 4 { + return Err(Error::NotRepresentable); + } + unsafe { + let buf_ptr = handle.raw_pointer_mut().add(2); + if *(buf_ptr.add(3)) != 0x1 { + return Err(Error::NotRepresentable); + } + Ok(&mut *(buf_ptr as *mut Felt)) + } + } +} + +impl<'a> AtomMathExtHandle for AtomHandle<'a> { + fn as_u32(&self) -> Result { + if let Ok(a) = self.as_direct() { + if a.bit_size() > 32 { Err(Error::NotRepresentable) + } else { + Ok(a.data() as u32) } } else { Err(Error::NotRepresentable) } } - fn as_mut_felt<'a>(&self) -> Result<&'a mut Felt> { - if let Ok(mut atom) = self.as_indirect() { - if atom.size() == 4 { - let buf_ptr = atom.data_pointer_mut(); - unsafe { - assert!(*(buf_ptr.add(3)) == 0x1); - } - let felt_ref: &mut Felt = unsafe { &mut *(buf_ptr as *mut Felt) }; - Ok(felt_ref) - } else { - Err(Error::NotRepresentable) - } + fn as_belt(&self) -> Result { + if let Ok(x) = self.as_u64() { + Ok(Belt(x)) } else { Err(Error::NotRepresentable) } } + + fn as_felt<'b>(&self) -> Result<&'b Felt> { + if self.is_direct() { + return Err(Error::NotRepresentable); + } + if self.size() != 4 { + return Err(Error::NotRepresentable); + } + let buf_ptr = self.data_pointer(); + unsafe { + if *(buf_ptr.add(3)) != 0x1 { + return Err(Error::NotRepresentable); + } + Ok(&*(buf_ptr as *const Felt)) + } + } + + fn as_mut_felt<'b>(&self) -> Result<&'b mut Felt> { + if self.is_direct() { + return Err(Error::NotRepresentable); + } + if self.size() != 4 { + return Err(Error::NotRepresentable); + } + unsafe { + let buf_ptr = self.raw_pointer_mut().add(2); + if *(buf_ptr.add(3)) != 0x1 { + return Err(Error::NotRepresentable); + } + Ok(&mut *(buf_ptr as *mut Felt)) + } + } } impl NounMathExt for Noun { + fn as_belt(&self, space: &NounSpace) -> Result { + if let Ok(atom) = self.in_space(space).as_atom() { + atom.atom().as_belt(space) + } else { + Err(Error::NotRepresentable) + } + } + + fn as_felt<'a>(&self, space: &NounSpace) -> Result<&'a Felt> { + if let Ok(atom) = self.in_space(space).as_atom() { + atom.atom().as_felt(space) + } else { + Err(Error::NotRepresentable) + } + } + + fn as_mut_felt<'a>(&self, space: &NounSpace) -> Result<&'a mut Felt> { + if let Ok(atom) = self.in_space(space).as_atom() { + atom.atom().as_mut_felt(space) + } else { + Err(Error::NotRepresentable) + } + } + + fn uncell(&self, space: &NounSpace) -> Result<[Self; N]> { + let mut inp = *self; + let mut cnt = 0; + let mut ret: [Result; N] = [(); N].map(|_| { + cnt += 1; + if cnt == N { + Ok(inp) + } else { + let c = inp.in_space(space).as_cell()?; + inp = c.tail().noun(); + Ok(c.head().noun()) + } + }); + if let Some(e) = ret.iter_mut().find(|v| v.is_err()) { + let n = core::mem::replace(e, Ok(D(0))); + return Err(n.expect_err("checked is_err above")); + } + Ok(ret.map(|v| v.expect("all results are Ok after filtering errors"))) + } +} + +impl<'a> NounMathExtHandle for NounHandle<'a> { fn as_belt(&self) -> Result { if let Ok(atom) = self.as_atom() { atom.as_belt() @@ -76,7 +177,7 @@ impl NounMathExt for Noun { } } - fn as_felt<'a>(&self) -> Result<&'a Felt> { + fn as_felt<'b>(&self) -> Result<&'b Felt> { if let Ok(atom) = self.as_atom() { atom.as_felt() } else { @@ -84,7 +185,7 @@ impl NounMathExt for Noun { } } - fn as_mut_felt<'a>(&self) -> Result<&'a mut Felt> { + fn as_mut_felt<'b>(&self) -> Result<&'b mut Felt> { if let Ok(atom) = self.as_atom() { atom.as_mut_felt() } else { @@ -95,7 +196,7 @@ impl NounMathExt for Noun { fn uncell(&self) -> Result<[Self; N]> { let mut inp = *self; let mut cnt = 0; - let mut ret = [(); N].map(|_| { + let mut ret: [Result; N] = [(); N].map(|_| { cnt += 1; if cnt == N { Ok(inp) @@ -106,62 +207,41 @@ impl NounMathExt for Noun { } }); if let Some(e) = ret.iter_mut().find(|v| v.is_err()) { - let n = core::mem::replace(e, Ok(D(0))); - return Err(n.expect_err("checked is_err above")); + let n = core::mem::replace(e, Ok(D(0).in_space(self.space()))); + return match n { + Ok(_) => unreachable!("checked is_err above"), + Err(err) => Err(err), + }; } Ok(ret.map(|v| v.expect("all results are Ok after filtering errors"))) } } -impl TryFrom for MarySlice<'_> { - type Error = (); - - fn try_from(n: Noun) -> std::result::Result { - if n.is_atom() { - Err(()) - } else { - MarySlice::try_from(n.as_cell()?) - } - } -} - -impl TryFrom for Mary { - type Error = (); - - fn try_from(n: Noun) -> std::result::Result { - if n.is_atom() { +impl MarySlice<'_> { + #[allow(clippy::result_unit_err)] + pub fn try_from(noun: Noun, space: &NounSpace) -> std::result::Result { + if noun.is_atom() { Err(()) } else { - let slice = MarySlice::try_from(n.as_cell()?)?; - Ok(Mary { - step: slice.step, - len: slice.len, - dat: slice.dat.to_vec(), - }) + MarySlice::try_from_cell(noun.in_space(space).as_cell()?, space) } } -} - -impl TryFrom for MarySlice<'_> { - type Error = (); #[inline(always)] - fn try_from(c: Cell) -> std::result::Result { - let step = c.head().as_atom()?.as_u32()?; - let len = c.tail().as_cell()?.head().as_atom()?.as_u32()?; - let cell: Cell = c.tail().as_cell()?; - let dat_noun: Atom = c.tail().as_cell()?.tail().as_atom()?; - let dat_slice: &[u64] = match dat_noun.as_either() { - Left(_direct) => unsafe { - let tail_ptr2 = &(*(cell.to_raw_pointer())).tail as *const Noun; - std::slice::from_raw_parts(tail_ptr2 as *const u64, (len * step) as usize) - }, - Right(indirect) => unsafe { - std::slice::from_raw_parts( - indirect.data_pointer() as *mut u64, - (len * step) as usize, - ) - }, + #[allow(clippy::result_unit_err)] + pub fn try_from_cell(c: CellHandle<'_>, _space: &NounSpace) -> std::result::Result { + let step = c.head().as_atom()?.atom().as_u32()?; + let len = c.tail().as_cell()?.head().as_atom()?.atom().as_u32()?; + let cell = c.tail().as_cell()?; + let dat_atom = cell.tail().as_atom()?; + let dat_slice: &[u64] = if dat_atom.is_direct() { + unsafe { + let cell_ptr = cell.raw_pointer(); + let tail_ptr = &(*cell_ptr).tail as *const Noun; + std::slice::from_raw_parts(tail_ptr as *const u64, (len * step) as usize) + } + } else { + unsafe { std::slice::from_raw_parts(dat_atom.data_pointer(), (len * step) as usize) } }; Ok(MarySlice { step, @@ -171,26 +251,38 @@ impl TryFrom for MarySlice<'_> { } } -impl TryFrom for Table<'_> { - type Error = (); - - fn try_from(n: Noun) -> std::result::Result { - if n.is_atom() { +impl Mary { + #[allow(clippy::result_unit_err)] + pub fn try_from(noun: Noun, space: &NounSpace) -> std::result::Result { + if noun.is_atom() { Err(()) } else { - Table::try_from(n.as_cell()?) + let slice = MarySlice::try_from_cell(noun.in_space(space).as_cell()?, space)?; + Ok(Mary { + step: slice.step, + len: slice.len, + dat: slice.dat.to_vec(), + }) } } } -impl TryFrom for Table<'_> { - type Error = (); +impl Table<'_> { + #[allow(clippy::result_unit_err)] + pub fn try_from(noun: Noun, space: &NounSpace) -> std::result::Result { + if noun.is_atom() { + Err(()) + } else { + Table::try_from_cell(noun.in_space(space).as_cell()?, space) + } + } #[inline(always)] - fn try_from(c: Cell) -> std::result::Result { - let full_width = c.head().as_atom()?.as_u32()?; + #[allow(clippy::result_unit_err)] + pub fn try_from_cell(c: CellHandle<'_>, space: &NounSpace) -> std::result::Result { + let full_width = c.head().as_atom()?.atom().as_u32()?; let mary_cell = c.tail().as_cell()?; - let mary = MarySlice::try_from(mary_cell)?; + let mary = MarySlice::try_from_cell(mary_cell, space)?; Ok(Table { num_cols: full_width, @@ -201,54 +293,28 @@ impl TryFrom for Table<'_> { // TODO: use Ares::noun::Result or Error somehow for the methods that // convert our structs from nouns -impl TryFrom for BPolySlice<'_> { - type Error = JetErr; - +impl BPolySlice<'_> { #[inline(always)] - fn try_from(n: Noun) -> std::result::Result { - if n.is_atom() { + pub fn try_from(noun: Noun, space: &NounSpace) -> std::result::Result { + if noun.is_atom() { Err(BAIL_FAIL) } else { - BPolySlice::try_from(n.as_cell()?) + BPolySlice::try_from_cell(noun.in_space(space).as_cell()?, space) } } -} - -impl TryFrom for FPolySlice<'_> { - type Error = JetErr; #[inline(always)] - fn try_from(n: Noun) -> std::result::Result { - if n.is_atom() { - Err(BAIL_FAIL) - } else { - FPolySlice::try_from(n.as_cell()?) - } - } -} - -impl TryFrom<&Noun> for FPolySlice<'_> { - type Error = JetErr; - - #[inline(always)] - fn try_from(n: &Noun) -> std::result::Result { - if n.is_atom() { - Err(BAIL_FAIL) - } else { - FPolySlice::try_from(n.as_cell()?) - } - } -} - -impl TryFrom for BPolySlice<'_> { - type Error = JetErr; - - #[inline(always)] - fn try_from(c: Cell) -> std::result::Result { + pub fn try_from_cell( + c: CellHandle<'_>, + _space: &NounSpace, + ) -> std::result::Result { let head = c.head().as_atom(); let tail = c.tail().as_atom(); if let (Ok(head), Ok(tail)) = (head, tail) { - let len32 = head.as_u32()?; + let len32 = head.atom().as_u32()?; + if tail.is_direct() { + return Err(BAIL_FAIL); + } let dat_slice: BPolySlice = unsafe { PolySlice(std::slice::from_raw_parts( tail.data_pointer() as *const Belt, @@ -262,15 +328,28 @@ impl TryFrom for BPolySlice<'_> { } } -impl TryFrom for FPolySlice<'_> { - type Error = JetErr; +impl FPolySlice<'_> { + #[inline(always)] + pub fn try_from(noun: Noun, space: &NounSpace) -> std::result::Result { + if noun.is_atom() { + Err(BAIL_FAIL) + } else { + FPolySlice::try_from_cell(noun.in_space(space).as_cell()?, space) + } + } #[inline(always)] - fn try_from(c: Cell) -> std::result::Result { + pub fn try_from_cell( + c: CellHandle<'_>, + _space: &NounSpace, + ) -> std::result::Result { let head = c.head().as_atom(); let tail = c.tail().as_atom(); if let (Ok(head), Ok(tail)) = (head, tail) { - let len32 = head.as_u32()?; + let len32 = head.atom().as_u32()?; + if tail.is_direct() { + return Err(BAIL_FAIL); + } let dat_slice: FPolySlice = unsafe { PolySlice(std::slice::from_raw_parts( tail.data_pointer() as *const Felt, @@ -284,15 +363,28 @@ impl TryFrom for FPolySlice<'_> { } } -impl TryFrom for FPolyVec { - type Error = JetErr; +impl FPolyVec { + #[inline(always)] + pub fn try_from(noun: Noun, space: &NounSpace) -> std::result::Result { + if noun.is_atom() { + Err(BAIL_FAIL) + } else { + FPolyVec::try_from_cell(noun.in_space(space).as_cell()?, space) + } + } #[inline(always)] - fn try_from(c: Cell) -> std::result::Result { + pub fn try_from_cell( + c: CellHandle<'_>, + _space: &NounSpace, + ) -> std::result::Result { let head = c.head().as_atom(); let tail = c.tail().as_atom(); if let (Ok(head), Ok(tail)) = (head, tail) { - let len32 = head.as_u32()?; + let len32 = head.atom().as_u32()?; + if tail.is_direct() { + return Err(BAIL_FAIL); + } let dat_vec: FPolyVec = unsafe { PolyVec( std::slice::from_raw_parts(tail.data_pointer() as *const Felt, len32 as usize) @@ -306,15 +398,28 @@ impl TryFrom for FPolyVec { } } -impl TryFrom for BPolyVec { - type Error = JetErr; +impl BPolyVec { + #[inline(always)] + pub fn try_from(noun: Noun, space: &NounSpace) -> std::result::Result { + if noun.is_atom() { + Err(BAIL_FAIL) + } else { + BPolyVec::try_from_cell(noun.in_space(space).as_cell()?, space) + } + } #[inline(always)] - fn try_from(c: Cell) -> std::result::Result { + pub fn try_from_cell( + c: CellHandle<'_>, + _space: &NounSpace, + ) -> std::result::Result { let head = c.head().as_atom(); let tail = c.tail().as_atom(); if let (Ok(head), Ok(tail)) = (head, tail) { - let len32 = head.as_u32()?; + let len32 = head.atom().as_u32()?; + if tail.is_direct() { + return Err(BAIL_FAIL); + } let dat_vec: BPolyVec = unsafe { PolyVec( std::slice::from_raw_parts(tail.data_pointer() as *const Belt, len32 as usize) @@ -331,9 +436,9 @@ impl TryFrom for BPolyVec { impl NounDecode for FPolyVec { fn from_noun( noun: &nockvm::noun::Noun, + space: &NounSpace, ) -> std::result::Result { - FPolyVec::try_from(noun.as_cell().expect("not a cell")) - .map_err(|_| noun_serde::NounDecodeError::FPolyDecodeError) + FPolyVec::try_from(*noun, space).map_err(|_| noun_serde::NounDecodeError::FPolyDecodeError) } } @@ -349,9 +454,9 @@ impl NounEncode for FPolyVec { impl NounDecode for BPolyVec { fn from_noun( noun: &nockvm::noun::Noun, + space: &NounSpace, ) -> std::result::Result { - BPolyVec::try_from(noun.as_cell().expect("not a cell")) - .map_err(|_| noun_serde::NounDecodeError::FPolyDecodeError) + BPolyVec::try_from(*noun, space).map_err(|_| noun_serde::NounDecodeError::FPolyDecodeError) } } diff --git a/crates/nockchain-math/src/crypto/argon2.rs b/crates/nockchain-math/src/crypto/argon2.rs index de2ed91f5..d6125f2ca 100644 --- a/crates/nockchain-math/src/crypto/argon2.rs +++ b/crates/nockchain-math/src/crypto/argon2.rs @@ -1,8 +1,8 @@ use argon2::{Algorithm, Argon2, AssociatedData, Params, Version}; use ibig::UBig; -use nockvm::ext::{make_tas, AtomExt}; +use nockvm::ext::make_tas; use nockvm::jets::cold::{Nounable, NounableResult}; -use nockvm::noun::{Atom, Noun, NounAllocator, Slots, D, T}; +use nockvm::noun::{Atom, Noun, NounAllocator, NounSpace, D, T}; /// Wrapper for the `$byts` Hoon cell `[wid=@ dat=@]`. /// `wid` records the bit-width, but the Argon2 jet only needs the big-endian payload, @@ -27,9 +27,13 @@ impl Nounable for Byts { let dat = Atom::from_ubig(stack, &big).as_noun(); T(stack, &[wid, dat]) } - fn from_noun(_stack: &mut A, noun: &Noun) -> NounableResult { - let size = noun.slot(2)?; - let dat = noun.slot(3)?.as_atom()?; + fn from_noun( + _stack: &mut A, + noun: &Noun, + space: &NounSpace, + ) -> NounableResult { + let size = noun.in_space(space).slot(2)?; + let dat = noun.in_space(space).slot(3)?.as_atom()?; let wid = size.as_atom()?.as_u64()? as usize; let mut res = vec![0; wid]; @@ -105,9 +109,14 @@ impl Nounable for Argon2Args { &[out, typ_noun, vers, threads, mem_cost, time_cost, secret, extra], ) } - fn from_noun(stack: &mut A, params: &Noun) -> NounableResult { - let out = params.slot(2)?.as_atom()?.as_u64()? as usize; + fn from_noun( + stack: &mut A, + params: &Noun, + space: &NounSpace, + ) -> NounableResult { + let out = params.in_space(space).slot(2)?.as_atom()?.as_u64()? as usize; let typ = params + .in_space(space) .slot(6)? .as_atom()? .into_string() @@ -119,12 +128,14 @@ impl Nounable for Argon2Args { option_env!("GIT_SHA") ) }); - let version = params.slot(14)?.as_atom()?.as_u64()? as u8; - let threads = params.slot(30)?.as_atom()?.as_u64()? as u32; - let mem_cost = params.slot(62)?.as_atom()?.as_u64()? as u32; - let time_cost = params.slot(126)?.as_atom()?.as_u64()? as u32; - let secret = Byts::from_noun(stack, ¶ms.slot(254)?)?; - let extra = Byts::from_noun(stack, ¶ms.slot(255)?)?; + let version = params.in_space(space).slot(14)?.as_atom()?.as_u64()? as u8; + let threads = params.in_space(space).slot(30)?.as_atom()?.as_u64()? as u32; + let mem_cost = params.in_space(space).slot(62)?.as_atom()?.as_u64()? as u32; + let time_cost = params.in_space(space).slot(126)?.as_atom()?.as_u64()? as u32; + let secret_noun = params.in_space(space).slot(254)?.noun(); + let extra_noun = params.in_space(space).slot(255)?.noun(); + let secret = Byts::from_noun(stack, &secret_noun, space)?; + let extra = Byts::from_noun(stack, &extra_noun, space)?; // prepare parameters let data = AssociatedData::new(&extra.0).unwrap_or_else(|err| { diff --git a/crates/nockchain-math/src/felt.rs b/crates/nockchain-math/src/felt.rs index bb08fa718..4138cee46 100644 --- a/crates/nockchain-math/src/felt.rs +++ b/crates/nockchain-math/src/felt.rs @@ -1,6 +1,6 @@ use core::ops::{Add, Div, Mul, Neg, Sub}; -use nockvm::noun::IndirectAtom; +use nockvm::noun::{IndirectAtom, NounSpace}; use noun_serde::{NounDecode, NounEncode}; use num_traits::{MulAdd, Pow}; @@ -25,8 +25,11 @@ impl NounEncode for Felt { // Custom NounDecode implementation for Felt impl NounDecode for Felt { - fn from_noun(noun: &nockvm::noun::Noun) -> Result { - let felt_slice = noun.as_felt()?; + fn from_noun( + noun: &nockvm::noun::Noun, + space: &NounSpace, + ) -> Result { + let felt_slice = noun.as_felt(space)?; let res: [Belt; 3] = [felt_slice[0], felt_slice[1], felt_slice[2]]; Ok(Felt(res)) } diff --git a/crates/nockchain-math/src/fpoly.rs b/crates/nockchain-math/src/fpoly.rs index 0c5502818..dd44f7e15 100644 --- a/crates/nockchain-math/src/fpoly.rs +++ b/crates/nockchain-math/src/fpoly.rs @@ -1,6 +1,7 @@ use std::cmp::max; use std::vec; +use nockvm::noun::NounSpace; use noun_serde::NounDecode; use num_traits::MulAdd; @@ -302,9 +303,9 @@ pub fn fpeval(a: &[Felt], x: Felt) -> Felt { } #[inline(always)] -pub fn lift_to_fpoly(belts: HoonList, res: &mut [Felt]) { +pub fn lift_to_fpoly(belts: HoonList<'_>, res: &mut [Felt], space: &NounSpace) { for (i, b) in belts.into_iter().enumerate() { - let belt = Belt::from_noun(&b).unwrap_or_else(|err| { + let belt = Belt::from_noun(&b, space).unwrap_or_else(|err| { panic!( "Panicked with {err:?} at {}:{} (git sha: {:?})", file!(), diff --git a/crates/nockchain-math/src/handle.rs b/crates/nockchain-math/src/handle.rs index c9ba68fbb..5e3d3a35e 100644 --- a/crates/nockchain-math/src/handle.rs +++ b/crates/nockchain-math/src/handle.rs @@ -64,16 +64,20 @@ where pub fn new_handle_mut_felt<'a, T: NounAllocator>(alloc: &mut T) -> (IndirectAtom, &'a mut Felt) { let (felt_atom, dat_ptr) = unsafe { IndirectAtom::new_raw_mut_words(alloc, 4) }; dat_ptr[3] = 0x1; + let space = alloc.noun_space(); ( felt_atom, - felt_atom.as_atom().as_mut_felt().unwrap_or_else(|err| { - panic!( - "Panicked with {err:?} at {}:{} (git sha: {:?})", - file!(), - line!(), - option_env!("GIT_SHA") - ) - }), + felt_atom + .as_atom() + .as_mut_felt(&space) + .unwrap_or_else(|err| { + panic!( + "Panicked with {err:?} at {}:{} (git sha: {:?})", + file!(), + line!(), + option_env!("GIT_SHA") + ) + }), ) } @@ -107,12 +111,11 @@ pub fn finalize_mary( allocator: &mut A, step: usize, len: usize, - mut res: IndirectAtom, + res: IndirectAtom, ) -> Noun { - unsafe { - res.normalize(); - } - let array = T(allocator, &[D(len as u64), res.as_noun()]); + let space = allocator.noun_space(); + let res_atom = unsafe { res.as_atom().in_space(&space).normalize().atom() }; + let array = T(allocator, &[D(len as u64), res_atom.as_noun()]); T(allocator, &[D(step as u64), array]) } @@ -120,11 +123,10 @@ pub fn finalize_mary( pub fn finalize_poly( allocator: &mut A, len: Option, - mut res: IndirectAtom, + res: IndirectAtom, ) -> Noun { - unsafe { - res.normalize(); - } + let space = allocator.noun_space(); + let res_atom = unsafe { res.as_atom().in_space(&space).normalize().atom() }; let head = Atom::new( allocator, len.unwrap_or_else(|| { @@ -137,6 +139,6 @@ pub fn finalize_poly( }) as u64, ) .as_noun(); - let res_cell = Cell::new(allocator, head, res.as_noun()); + let res_cell = Cell::new(allocator, head, res_atom.as_noun()); res_cell.as_noun() } diff --git a/crates/nockchain-math/src/lib.rs b/crates/nockchain-math/src/lib.rs index 54bce6525..d1bffe5e1 100644 --- a/crates/nockchain-math/src/lib.rs +++ b/crates/nockchain-math/src/lib.rs @@ -8,6 +8,7 @@ pub mod fpoly; pub mod handle; pub mod mary; pub mod noun_ext; +pub mod owned_based_noun; pub mod poly; pub mod shape; pub mod structs; diff --git a/crates/nockchain-math/src/mary.rs b/crates/nockchain-math/src/mary.rs index 3302c573d..cd8b4bb79 100644 --- a/crates/nockchain-math/src/mary.rs +++ b/crates/nockchain-math/src/mary.rs @@ -1,4 +1,4 @@ -use nockvm::noun::{IndirectAtom, Noun, NounAllocator}; +use nockvm::noun::{IndirectAtom, Noun, NounAllocator, NounSpace}; use noun_serde::{NounDecode, NounDecodeError, NounEncode}; use crate::belt::Belt; @@ -92,8 +92,8 @@ impl TryFrom> for &mut [Felt] { } impl NounDecode for Mary { - fn from_noun(noun: &Noun) -> Result { - Mary::try_from(*noun).map_err(|_| NounDecodeError::MaryDecodeError) + fn from_noun(noun: &Noun, space: &NounSpace) -> Result { + Mary::try_from(*noun, space).map_err(|_| NounDecodeError::MaryDecodeError) } } diff --git a/crates/nockchain-math/src/noun_ext.rs b/crates/nockchain-math/src/noun_ext.rs index a39305ed9..753927adb 100644 --- a/crates/nockchain-math/src/noun_ext.rs +++ b/crates/nockchain-math/src/noun_ext.rs @@ -1,4 +1,4 @@ -use nockvm::noun::{IndirectAtom, NounAllocator, Result}; +use nockvm::noun::{IndirectAtom, NounAllocator, NounSpace, Result}; use crate::belt::Belt; use crate::felt::Felt; @@ -7,14 +7,28 @@ use crate::felt::Felt; // their implementations are found in hand/convert pub trait AtomMathExt { fn as_u32(&self) -> Result; - fn as_belt(&self) -> Result; - fn as_felt<'a>(&self) -> Result<&'a Felt>; - fn as_mut_felt<'a>(&self) -> Result<&'a mut Felt>; + fn as_belt(&self, space: &NounSpace) -> Result; + fn as_felt<'a>(&self, space: &NounSpace) -> Result<&'a Felt>; + fn as_mut_felt<'a>(&self, space: &NounSpace) -> Result<&'a mut Felt>; } // Note: since these are methods for converting to other types, // their implementations are found in hand/convert pub trait NounMathExt: Sized { + fn as_belt(&self, space: &NounSpace) -> Result; + fn as_felt<'a>(&self, space: &NounSpace) -> Result<&'a Felt>; + fn as_mut_felt<'a>(&self, space: &NounSpace) -> Result<&'a mut Felt>; + fn uncell(&self, space: &NounSpace) -> Result<[Self; N]>; +} + +pub trait AtomMathExtHandle { + fn as_u32(&self) -> Result; + fn as_belt(&self) -> Result; + fn as_felt<'a>(&self) -> Result<&'a Felt>; + fn as_mut_felt<'a>(&self) -> Result<&'a mut Felt>; +} + +pub trait NounMathExtHandle: Sized { fn as_belt(&self) -> Result; fn as_felt<'a>(&self) -> Result<&'a Felt>; fn as_mut_felt<'a>(&self) -> Result<&'a mut Felt>; diff --git a/crates/nockchain-math/src/owned_based_noun.rs b/crates/nockchain-math/src/owned_based_noun.rs new file mode 100644 index 000000000..293a89fc9 --- /dev/null +++ b/crates/nockchain-math/src/owned_based_noun.rs @@ -0,0 +1,237 @@ +use nockvm::noun::{Noun, NounSpace}; +use noun_serde::NounDecodeError; + +use crate::belt::{based_check, Belt}; +use crate::tip5; + +/// Errors raised while converting allocator-backed nouns into owned based-noun trees. +#[derive(Debug, thiserror::Error, PartialEq, Eq)] +pub enum OwnedBasedNounError { + /// A source atom did not fit into the owned `u64` representation used here. + #[error("owned based noun atom exceeded u64 range")] + AtomTooLarge, + /// A source atom fit into `u64` but was outside the base field. + #[error("owned based noun atom is not based: {0}")] + AtomNotBased(u64), + /// The source noun did not match the expected atom/cell structure. + #[error("{0}")] + Malformed(&'static str), +} + +/// Maps owned based-noun conversion failures into the generic noun-serde decode error. +pub fn owned_based_noun_decode_error(err: OwnedBasedNounError) -> NounDecodeError { + NounDecodeError::Custom(err.to_string()) +} + +/// Allocator-free owned representation of a noun tree whose atom leaves are all +/// base-field elements. +/// +/// We need this type when we want noun structure semantics without holding on +/// to an allocator-backed `nockvm::Noun`, while also making the current +/// protocol invariant explicit: every atom leaf participating in these paths +/// must already be a valid [`Belt`]. +/// +/// That comes up in two places: +/// 1. canonical z-set/z-map ordering, which caches noun-derived ordering keys +/// 2. direct hashable helpers, which need leaf-sequence and dyck-shape hashing +/// +/// By owning the tree in plain Rust boxes, callers can compute noun-structural +/// properties without threading a `NounAllocator` through every operation. +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum OwnedBasedNoun { + /// Atom leaf storing a validated base-field element. + Atom(Belt), + /// Cell node storing its owned head and tail children. + Cell(Box, Box), +} + +impl OwnedBasedNoun { + /// Copies an allocator-backed noun into an owned tree that can outlive the + /// source allocator. + /// + /// This rejects any atom that is either larger than `u64` or not a valid + /// base-field element. + pub fn from_noun(noun: Noun, space: &NounSpace) -> Result { + if noun.is_atom() { + let atom = noun + .in_space(space) + .as_atom() + .map_err(|_| OwnedBasedNounError::Malformed("expected atom"))?; + let atom = atom + .as_u64() + .map_err(|_| OwnedBasedNounError::AtomTooLarge)?; + if !based_check(atom) { + return Err(OwnedBasedNounError::AtomNotBased(atom)); + } + let atom = Belt(atom); + Ok(Self::Atom(atom)) + } else { + let cell = noun + .in_space(space) + .as_cell() + .map_err(|_| OwnedBasedNounError::Malformed("expected cell"))?; + Ok(Self::cell( + Self::from_noun(cell.head().noun(), space)?, + Self::from_noun(cell.tail().noun(), space)?, + )) + } + } + + /// Builds an owned atom noun directly from a validated base-field element. + pub fn atom(atom: Belt) -> Self { + Self::Atom(atom) + } + + /// Builds an owned atom noun from a raw `u64`, rejecting values outside the + /// base field. + pub fn try_atom(atom: u64) -> Result { + if !based_check(atom) { + return Err(OwnedBasedNounError::AtomNotBased(atom)); + } + Ok(Self::atom(Belt(atom))) + } + + /// Builds an owned cell noun from owned head and tail children. + pub fn cell(head: Self, tail: Self) -> Self { + Self::Cell(Box::new(head), Box::new(tail)) + } + + /// Builds the right-associated tuple noun for a slice of raw atoms. + /// + /// Every atom must already be based. For example, `[1, 2, 3]` becomes + /// `[1 [2 3]]`. The empty tuple is represented as the null atom `0`, + /// matching the existing hashable helper behavior. + pub fn tuple_atoms(atoms: &[u64]) -> Result { + let mut iter = atoms.iter().rev(); + let Some(&last) = iter.next() else { + return Self::try_atom(0); + }; + let mut noun = Self::try_atom(last)?; + for &atom in iter { + noun = Self::cell(Self::try_atom(atom)?, noun); + } + Ok(noun) + } + + /// Builds a proper Hoon list terminated by the based null atom `0`. + pub fn list(items: Vec) -> Self { + items + .into_iter() + .rev() + .fold(Self::atom(Belt(0)), |tail, head| Self::cell(head, tail)) + } + + /// Counts the number of atom leaves in the noun tree. + /// + /// Tip5 `hash-noun-varlen` prefixes the flattened leaf stream with this + /// count, so we cache it by traversal when producing the input belt list. + pub fn leaf_count(&self) -> usize { + match self { + Self::Atom(_) => 1, + Self::Cell(left, right) => left.leaf_count() + right.leaf_count(), + } + } + + /// Appends the noun's leaf-sequence encoding to `out`. + /// + /// This is the flattened left-to-right list of atom leaves used by + /// `hash-noun-varlen`. + pub fn push_leaf_sequence(&self, out: &mut Vec) { + match self { + Self::Atom(atom) => out.push(*atom), + Self::Cell(left, right) => { + left.push_leaf_sequence(out); + right.push_leaf_sequence(out); + } + } + } + + /// Appends the noun's dyck-shape encoding to `out`. + /// + /// Cells contribute `0` before the left subtree and `1` before the right + /// subtree, while atoms contribute nothing. + pub fn push_dyck(&self, out: &mut Vec) { + match self { + Self::Atom(_) => {} + Self::Cell(left, right) => { + out.push(Belt(0)); + left.push_dyck(out); + out.push(Belt(1)); + right.push_dyck(out); + } + } + } +} + +/// Computes the tip5 `hash-noun-varlen` digest for an owned noun tree. +/// +/// This mirrors the jet path's noun hashing, but operates on an allocator-free +/// owned tree so higher-level code can hash noun structure directly. +pub fn hash_owned_based_noun_varlen(noun: &OwnedBasedNoun) -> [u64; 5] { + let mut input = Vec::with_capacity(1 + noun.leaf_count() * 2); + input.push(Belt(noun.leaf_count() as u64)); + noun.push_leaf_sequence(&mut input); + noun.push_dyck(&mut input); + tip5::hash::hash_varlen(&mut input) +} + +#[cfg(test)] +mod tests { + use ibig::ubig; + use nockvm::mem::NockStack; + use nockvm::noun::Atom; + + use super::{OwnedBasedNoun, OwnedBasedNounError}; + use crate::belt::{Belt, PRIME}; + + #[test] + fn from_noun_accepts_direct_based_atoms() { + let mut stack = NockStack::new(nockvm::mem::NOCK_STACK_SIZE_TINY, 0); + let noun = Atom::new(&mut stack, 7).as_noun(); + let space = stack.noun_space(); + + assert_eq!( + OwnedBasedNoun::from_noun(noun, &space), + Ok(OwnedBasedNoun::Atom(Belt(7))) + ); + } + + #[test] + fn from_noun_accepts_indirect_based_atoms_that_fit_u64() { + let mut stack = NockStack::new(nockvm::mem::NOCK_STACK_SIZE_TINY, 0); + let noun = Atom::new(&mut stack, PRIME - 1).as_noun(); + let space = stack.noun_space(); + let atom = noun.as_atom().expect("noun should be atom"); + assert!(atom.is_indirect()); + + assert_eq!( + OwnedBasedNoun::from_noun(noun, &space), + Ok(OwnedBasedNoun::Atom(Belt(PRIME - 1))) + ); + } + + #[test] + fn from_noun_rejects_non_based_atoms() { + let mut stack = NockStack::new(nockvm::mem::NOCK_STACK_SIZE_TINY, 0); + let noun = Atom::new(&mut stack, PRIME).as_noun(); + let space = stack.noun_space(); + + assert_eq!( + OwnedBasedNoun::from_noun(noun, &space), + Err(OwnedBasedNounError::AtomNotBased(PRIME)) + ); + } + + #[test] + fn from_noun_rejects_atoms_larger_than_u64() { + let mut stack = NockStack::new(nockvm::mem::NOCK_STACK_SIZE_TINY, 0); + let big = ubig!(1) << 80; + let noun = Atom::from_ubig(&mut stack, &big).as_noun(); + let space = stack.noun_space(); + + assert_eq!( + OwnedBasedNoun::from_noun(noun, &space), + Err(OwnedBasedNounError::AtomTooLarge) + ); + } +} diff --git a/crates/nockchain-math/src/poly.rs b/crates/nockchain-math/src/poly.rs index 2f0c53c2a..32bd631df 100644 --- a/crates/nockchain-math/src/poly.rs +++ b/crates/nockchain-math/src/poly.rs @@ -144,6 +144,10 @@ where pub struct PolyVec(pub Vec); impl PolyVec { + #[inline(always)] + pub fn new() -> Self { + Self(Vec::new()) + } #[inline(always)] pub fn as_slice(&self) -> &[T] { self.0.as_slice() @@ -152,6 +156,25 @@ impl PolyVec { pub fn as_mut_slice(&mut self) -> &mut [T] { self.0.as_mut_slice() } + + #[inline] + pub fn as_poly_slice(&self) -> PolySlice<'_, T> { + PolySlice(self.0.as_slice()) + } + #[inline(always)] + pub fn as_poly_slice_mut(&mut self) -> PolySliceMut<'_, T> { + PolySliceMut(self.0.as_mut_slice()) + } + #[inline(always)] + pub fn iter(&self) -> Iter<'_, T> { + self.0.iter() + } +} + +impl Default for PolyVec { + fn default() -> Self { + Self::new() + } } #[derive(Clone, Copy, Debug)] diff --git a/crates/nockchain-math/src/shape.rs b/crates/nockchain-math/src/shape.rs index d70964d57..77ca41c11 100644 --- a/crates/nockchain-math/src/shape.rs +++ b/crates/nockchain-math/src/shape.rs @@ -1,40 +1,54 @@ use nockvm::jets::list::util::flop; use nockvm::jets::JetErr; -use nockvm::noun::{Noun, NounAllocator, D, T}; +use nockvm::noun::{Noun, NounAllocator, NounSpace, D, T}; use noun_serde::NounEncode; -pub fn dyck(stack: &mut A, t: Noun) -> Result { - let vec = dyck_recursive(stack, t, D(0))?; - flop(stack, vec) +pub fn dyck(stack: &mut A, t: Noun, space: &NounSpace) -> Result { + let vec = dyck_recursive(stack, t, D(0), space)?; + let stack_space = stack.noun_space(); + flop(stack, vec, &stack_space) } -fn dyck_recursive(stack: &mut A, t: Noun, vec: Noun) -> Result { +fn dyck_recursive( + stack: &mut A, + t: Noun, + vec: Noun, + space: &NounSpace, +) -> Result { if t.is_atom() { Ok(vec) } else { - let t_cell = t.as_cell()?; + let t_cell = t.in_space(space).as_cell()?; let vec_inner = T(stack, &[D(0), vec]); - let dyck_inner = dyck_recursive(stack, t_cell.head(), vec_inner)?; + let head = t_cell.head().noun(); + let dyck_inner = dyck_recursive(stack, head, vec_inner, space)?; let vec_outer = T(stack, &[D(1), dyck_inner]); - dyck_recursive(stack, t_cell.tail(), vec_outer) + let tail = t_cell.tail().noun(); + dyck_recursive(stack, tail, vec_outer, space) } } -pub fn leaf_sequence(stack: &mut A, t: Noun) -> Result { +pub fn leaf_sequence( + stack: &mut A, + t: Noun, + space: &NounSpace, +) -> Result { let mut leaf: Vec = Vec::::new(); - do_leaf_sequence(t, &mut leaf)?; + do_leaf_sequence(t, &mut leaf, space)?; let res = leaf.to_noun(stack); Ok(res) } -pub fn do_leaf_sequence(noun: Noun, vec: &mut Vec) -> Result<(), JetErr> { +pub fn do_leaf_sequence(noun: Noun, vec: &mut Vec, space: &NounSpace) -> Result<(), JetErr> { if noun.is_atom() { - vec.push(noun.as_atom()?.as_u64()?); + vec.push(noun.in_space(space).as_atom()?.as_u64()?); Ok(()) } else { - let cell = noun.as_cell()?; - do_leaf_sequence(cell.head(), vec)?; - do_leaf_sequence(cell.tail(), vec)?; + let cell = noun.in_space(space).as_cell()?; + let head = cell.head().noun(); + let tail = cell.tail().noun(); + do_leaf_sequence(head, vec, space)?; + do_leaf_sequence(tail, vec, space)?; Ok(()) } } diff --git a/crates/nockchain-math/src/structs.rs b/crates/nockchain-math/src/structs.rs index d9909b459..bb95babd2 100644 --- a/crates/nockchain-math/src/structs.rs +++ b/crates/nockchain-math/src/structs.rs @@ -1,74 +1,85 @@ use nockvm::jets::sort::util::gor; use nockvm::mem::NockStack; -use nockvm::noun::{Cell, Noun}; +use nockvm::noun::{Cell, CellHandle, Noun, NounHandle, NounSpace}; use nockvm::unifying_equality::unifying_equality; use crate::noun_ext::NounMathExt; #[derive(Copy, Clone)] -pub struct HoonList { - pub(super) next: Option, +pub struct HoonList<'a> { + pub(super) next: Option, + pub(super) space: &'a NounSpace, } -impl Iterator for HoonList { +impl<'a> Iterator for HoonList<'a> { type Item = Noun; #[inline(always)] fn next(&mut self) -> Option { - self.next.take().map(|cell| { - let tail = cell.tail(); - self.next = if tail.is_cell() { - Some(tail.as_cell().unwrap_or_else(|err| { - panic!( - "Panicked with {err:?} at {}:{} (git sha: {:?})", - file!(), - line!(), - option_env!("GIT_SHA") - ) - })) - } else { - None - }; - cell.head() + self.next.take().map(|noun| { + let cell = noun.in_space(self.space).as_cell().unwrap_or_else(|err| { + panic!( + "Panicked with {err:?} at {}:{} (git sha: {:?})", + file!(), + line!(), + option_env!("GIT_SHA") + ) + }); + let tail = cell.tail().noun(); + self.next = if tail.is_cell() { Some(tail) } else { None }; + cell.head().noun() }) } } -impl TryFrom for HoonList { - type Error = nockvm::noun::Error; - fn try_from(n: Noun) -> core::result::Result { +impl<'a> HoonList<'a> { + pub fn try_from( + n: Noun, + space: &'a NounSpace, + ) -> core::result::Result { if n.is_cell() { - Ok(HoonList::from(n.as_cell().unwrap_or_else(|err| { + let _ = n.in_space(space).as_cell().unwrap_or_else(|err| { panic!( "Panicked with {err:?} at {}:{} (git sha: {:?})", file!(), line!(), option_env!("GIT_SHA") ) - }))) + }); + Ok(HoonList { + next: Some(n), + space, + }) } else { - Ok(HoonList { next: None }) + Ok(HoonList { next: None, space }) } } -} -impl From for HoonList { - fn from(c: Cell) -> Self { - Self { next: Some(c) } + pub fn from_cell(c: Cell, space: &'a NounSpace) -> Self { + Self { + next: Some(c.as_noun()), + space, + } + } + + pub fn space(&self) -> &'a NounSpace { + self.space } } -pub fn next_cell(cell: Cell) -> Option { - let tail = cell.tail(); +pub fn next_cell(cell: Cell, space: &NounSpace) -> Option { + let cell_handle = CellHandle::new(cell, space); + let tail = cell_handle.tail().noun(); if tail.is_cell() { - Some(tail.as_cell().unwrap_or_else(|err| { + let handle = tail.in_space(space).as_cell().unwrap_or_else(|err| { panic!( "Panicked with {err:?} at {}:{} (git sha: {:?})", file!(), line!(), option_env!("GIT_SHA") ) - })) + }); + Some(handle.cell()) } else { None } @@ -76,49 +87,86 @@ pub fn next_cell(cell: Cell) -> Option { #[allow(dead_code)] #[derive(Copy, Clone)] -pub struct HoonMap { +pub struct HoonMap<'a> { pub(super) node: Noun, - pub(super) left: Option, - pub(super) right: Option, + pub(super) left: Option, + pub(super) right: Option, + pub(super) space: &'a NounSpace, } -impl HoonMap { +impl<'a> HoonMap<'a> { pub fn get(&self, stack: &mut NockStack, mut k: Noun) -> Option { - let [mut ck, cv] = self.node.uncell().ok()?; + let [mut ck, cv] = self.node.uncell(self.space).ok()?; if unsafe { unifying_equality(stack, &mut ck, &mut k) } { // ?: =(b p.n.a) // (some q.n.a) Some(cv) - } else if gor(stack, k, ck).as_direct().map(|v| v.data()) == Ok(0) { + } else if gor(stack, k, ck, self.space).as_direct().map(|v| v.data()) == Ok(0) { // ?: (gor b p.n.a) // $(a l.a) - let map: Self = self.left?.try_into().ok()?; + let map = Self::try_from(self.left?.in_space(self.space)).ok()?; map.get(stack, k) } else { // $(a r.a) - let map: Self = self.right?.try_into().ok()?; + let map = Self::try_from(self.right?.in_space(self.space)).ok()?; map.get(stack, k) } } + + pub fn try_from(n: NounHandle<'a>) -> std::result::Result { + if n.is_cell() { + let cell = n.as_cell().unwrap_or_else(|err| { + panic!( + "Panicked with {err:?} at {}:{} (git sha: {:?})", + file!(), + line!(), + option_env!("GIT_SHA") + ) + }); + HoonMap::try_from_cell(cell) + } else { + not_cell() + } + } + + pub fn try_from_cell(c: CellHandle<'a>) -> std::result::Result { + let tail = c.tail(); + if let Ok(cell_tail) = tail.as_cell() { + let left = cell_tail.head().noun(); + let right = cell_tail.tail().noun(); + + Ok(Self { + node: c.head().noun(), + left: left.is_cell().then_some(left), + right: right.is_cell().then_some(right), + space: c.space(), + }) + } else { + not_cell() + } + } } #[allow(dead_code)] #[derive(Clone)] -pub struct HoonMapIter { - pub(super) stack: Vec>, +pub struct HoonMapIter<'a> { + pub(super) stack: Vec>>, + pub(super) space: &'a NounSpace, } -impl Iterator for HoonMapIter { - type Item = Noun; +impl<'a> Iterator for HoonMapIter<'a> { + type Item = NounHandle<'a>; #[inline(always)] fn next(&mut self) -> Option { if let Some(maybe_cell) = self.stack.pop() { - if let Some(cell) = maybe_cell { - if let Ok(cell_trie) = HoonMap::try_from(cell) { - self.stack.push(cell_trie.right); - self.stack.push(cell_trie.left); - return Some(cell_trie.node); + if let Some(noun) = maybe_cell { + if let Ok(cell_trie) = HoonMap::try_from(noun) { + self.stack + .push(cell_trie.right.map(|n| n.in_space(self.space))); + self.stack + .push(cell_trie.left.map(|n| n.in_space(self.space))); + return Some(cell_trie.node.in_space(self.space)); } else { return self.next(); } @@ -133,53 +181,185 @@ fn not_cell() -> core::result::Result { Err(nockvm::noun::Error::NotCell) } -impl TryFrom for HoonMap { - type Error = nockvm::noun::Error; - - fn try_from(n: Noun) -> std::result::Result { +impl<'a> HoonMapIter<'a> { + pub fn new(n: &'a NounHandle) -> Self { if n.is_cell() { - HoonMap::try_from(n.as_cell().unwrap_or_else(|err| { - panic!( - "Panicked with {err:?} at {}:{} (git sha: {:?})", - file!(), - line!(), - option_env!("GIT_SHA") - ) - })) + Self { + stack: vec![Some(*n)], + space: n.space(), + } } else { - not_cell() + Self { + stack: vec![None], + space: n.space(), + } } } } -impl TryFrom for HoonMap { - type Error = nockvm::noun::Error; +#[cfg(test)] +mod tests { + use std::path::PathBuf; + use std::sync::atomic::{AtomicUsize, Ordering}; - fn try_from(c: Cell) -> std::result::Result { - let tail: Noun = c.tail(); - if let Ok(cell_tail) = tail.as_cell() { - let left = cell_tail.head(); - let right = cell_tail.tail(); + use nockvm::noun::{AllocLocation, NounRepr, D, T}; + use nockvm::pma::{Pma, PmaCopy}; + use noun_serde::NounEncode; - Ok(Self { - node: c.head(), - left: left.as_cell().ok(), - right: right.as_cell().ok(), - }) - } else { - not_cell() + use super::*; + use crate::zoon::zmap::ZMap; + + fn test_pma_path(label: &str) -> PathBuf { + static COUNTER: AtomicUsize = AtomicUsize::new(0); + + let id = COUNTER.fetch_add(1, Ordering::Relaxed); + let pid = std::process::id(); + let mut path = std::env::temp_dir(); + path.push(format!("nockchain_math_{label}_{pid}_{id}.mmap")); + path + } + + fn test_pma(label: &str) -> Pma { + Pma::new(100000, test_pma_path(label)).expect("failed to create test PMA") + } + + #[test] + #[cfg_attr(miri, ignore = "file-backed PMA unsupported in Miri")] + fn hoon_list_reads_pma_backed_cells() { + let mut stack = NockStack::new(nockvm::mem::NOCK_STACK_SIZE_TINY, 0); + let mut pma = test_pma("hoon_list"); + + let first = T(&mut stack, &[D(1), D(2)]); + let second = T(&mut stack, &[D(3), D(4)]); + let tail = T(&mut stack, &[second, D(0)]); + let mut list = T(&mut stack, &[first, tail]); + + unsafe { + list.copy_to_pma(&stack, &mut pma); } + + let space = NounSpace::pma_only(&pma); + let items: Vec = HoonList::try_from(list, &space) + .expect("PMA-backed list should decode") + .collect(); + + assert_eq!(items.len(), 2, "list length should be preserved"); + assert_eq!( + items[0].in_space(&space).repr(), + NounRepr::Cell(AllocLocation::PmaOffset), + "first element should remain a PMA offset-form cell" + ); + assert_eq!( + items[1].in_space(&space).repr(), + NounRepr::Cell(AllocLocation::PmaOffset), + "second element should remain a PMA offset-form cell" + ); + + let first = items[0] + .in_space(&space) + .as_cell() + .expect("first item should be a cell"); + assert_eq!( + first + .head() + .as_atom() + .expect("first head should be an atom") + .as_u64() + .expect("first head should fit in u64"), + 1 + ); + assert_eq!( + first + .tail() + .as_atom() + .expect("first tail should be an atom") + .as_u64() + .expect("first tail should fit in u64"), + 2 + ); + + let second = items[1] + .in_space(&space) + .as_cell() + .expect("second item should be a cell"); + assert_eq!( + second + .head() + .as_atom() + .expect("second head should be an atom") + .as_u64() + .expect("second head should fit in u64"), + 3 + ); + assert_eq!( + second + .tail() + .as_atom() + .expect("second tail should be an atom") + .as_u64() + .expect("second tail should fit in u64"), + 4 + ); } -} -impl From for HoonMapIter { - fn from(n: Noun) -> Self { - if let Ok(c) = n.as_cell() { - Self { - stack: vec![Some(c)], - } - } else { - Self { stack: vec![None] } + #[test] + #[cfg_attr(miri, ignore = "file-backed PMA unsupported in Miri")] + fn hoon_map_get_and_iter_work_for_pma_backed_maps() { + let mut stack = NockStack::new(nockvm::mem::NOCK_STACK_SIZE_TINY, 0); + let map = ZMap::try_from_entries(vec![(7u64, 70u64), (3u64, 30u64), (11u64, 110u64)]) + .expect("owned z-map should build"); + let mut map_noun = map.to_noun(&mut stack); + let mut pma = test_pma("hoon_map"); + + unsafe { + map_noun.copy_to_pma(&stack, &mut pma); } + + let space = NounSpace::pma_only(&pma); + let hoon_map = + HoonMap::try_from(map_noun.in_space(&space)).expect("PMA-backed map should decode"); + + let mut lookup_stack = NockStack::new(nockvm::mem::NOCK_STACK_SIZE_TINY, 0); + let value = hoon_map + .get(&mut lookup_stack, D(11)) + .expect("existing key should be found"); + assert_eq!( + value + .in_space(&space) + .as_atom() + .expect("map lookup value should be an atom") + .as_u64() + .expect("map lookup value should fit in u64"), + 110 + ); + assert!( + hoon_map.get(&mut lookup_stack, D(99)).is_none(), + "missing key should not resolve" + ); + + let map_handle = map_noun.in_space(&space); + let mut entries: Vec<(u64, u64)> = HoonMapIter::new(&map_handle) + .map(|entry| { + let pair = entry + .as_cell() + .expect("map iterator should yield entry pairs"); + let key = pair + .head() + .as_atom() + .expect("map iterator key should be an atom") + .as_u64() + .expect("map iterator key should fit in u64"); + let value = pair + .tail() + .as_atom() + .expect("map iterator value should be an atom") + .as_u64() + .expect("map iterator value should fit in u64"); + (key, value) + }) + .collect(); + entries.sort_unstable(); + + assert_eq!(entries, vec![(3, 30), (7, 70), (11, 110)]); } } diff --git a/crates/nockchain-math/src/tip5/hash.rs b/crates/nockchain-math/src/tip5/hash.rs index 0608881d8..0ca580837 100644 --- a/crates/nockchain-math/src/tip5/hash.rs +++ b/crates/nockchain-math/src/tip5/hash.rs @@ -1,6 +1,6 @@ use nockvm::jets::list::util::{lent, weld}; use nockvm::jets::JetErr; -use nockvm::noun::{Noun, NounAllocator, D, T}; +use nockvm::noun::{Noun, NounAllocator, NounSpace, D, T}; use noun_serde::{NounDecode, NounEncode}; use super::*; @@ -124,29 +124,41 @@ pub fn hash_10(input_vec: &mut Vec) -> [u64; 5] { tip5_calc_digest(&sponge) } -pub fn hash_noun_varlen(stack: &mut A, n: Noun) -> Result { - let leaf = leaf_sequence(stack, n)?; - let dyck = dyck(stack, n)?; - let size = lent(leaf).map(|x| D(x as u64))?; +pub fn hash_noun_varlen( + stack: &mut A, + n: Noun, + space: &NounSpace, +) -> Result { + let leaf = leaf_sequence(stack, n, space)?; + let dyck = dyck(stack, n, space)?; + let stack_space = stack.noun_space(); + let size = lent(leaf, &stack_space).map(|x| D(x as u64))?; // [size (weld leaf dyck)] - let weld = weld(stack, leaf, dyck)?; + let weld = weld(stack, leaf, dyck, &stack_space)?; let arg = T(stack, &[size, weld]); - hash_belts_list(stack, arg) + let stack_space = stack.noun_space(); + hash_belts_list(stack, arg, &stack_space) } pub fn hash_noun_varlen_digest( stack: &mut A, n: Noun, + space: &NounSpace, ) -> Result<[u64; 5], JetErr> { - let noun_res = hash_noun_varlen(stack, n)?; - let digest = <[u64; 5]>::from_noun(&noun_res)?; + let noun_res = hash_noun_varlen(stack, n, space)?; + let stack_space = stack.noun_space(); + let digest = <[u64; 5]>::from_noun(&noun_res, &stack_space)?; Ok(digest) } -pub fn hash_belts_list(alloc: &mut A, input: Noun) -> Result { - let mut input_vec = >::from_noun(&input)?; +pub fn hash_belts_list( + alloc: &mut A, + input: Noun, + space: &NounSpace, +) -> Result { + let mut input_vec = >::from_noun(&input, space)?; let digest = hash_varlen(&mut input_vec); let res = digest.to_noun(alloc); Ok(res) diff --git a/crates/nockchain-math/src/zoon/common.rs b/crates/nockchain-math/src/zoon/common.rs index d3455aa78..5261f45bb 100644 --- a/crates/nockchain-math/src/zoon/common.rs +++ b/crates/nockchain-math/src/zoon/common.rs @@ -1,9 +1,15 @@ use nockvm::jets::util::BAIL_FAIL; use nockvm::jets::JetErr; -use nockvm::noun::{Noun, NounAllocator}; -use noun_serde::NounDecode; +use nockvm::mem::{NockStack, NOCK_STACK_SIZE_TINY}; +use nockvm::noun::{Noun, NounAllocator, NounSpace, D, T}; +use noun_serde::{NounDecode, NounDecodeError, NounEncode}; use crate::belt::Belt; +use crate::owned_based_noun::{ + hash_owned_based_noun_varlen, owned_based_noun_decode_error, OwnedBasedNoun, + OwnedBasedNounError, +}; +use crate::tip5; pub trait TipHasher { fn hash_noun_varlen( @@ -21,8 +27,10 @@ impl TipHasher for DefaultTipHasher { stack: &mut A, noun: Noun, ) -> Result<[u64; 5], JetErr> { - let noun_res = crate::tip5::hash::hash_noun_varlen(stack, noun)?; - let digest = <[u64; 5]>::from_noun(&noun_res)?; + let input_space = stack.noun_space(); + let noun_res = crate::tip5::hash::hash_noun_varlen(stack, noun, &input_space)?; + let output_space = stack.noun_space(); + let digest = <[u64; 5]>::from_noun(&noun_res, &output_space)?; Ok(digest) } fn hash_ten_cell(&self, ten: [u64; 10]) -> Result<[u64; 5], JetErr> { @@ -102,30 +110,443 @@ pub fn dor_tip( a: &mut Noun, b: &mut Noun, ) -> Result { - use nockvm::jets::math::util::lth_b; + use nockvm::jets::math::util::lth; + let space = stack.noun_space(); if unsafe { stack.equals(a, b) } { Ok(true) } else if !a.is_atom() { if b.is_atom() { Ok(false) } else { - let a_cell = a.as_cell()?; - let b_cell = b.as_cell()?; - - let mut a_head = a_cell.head(); - let mut b_head = b_cell.head(); + let a_cell = a.in_space(&space).as_cell()?; + let b_cell = b.in_space(&space).as_cell()?; + let mut a_head = a_cell.head().noun(); + let mut b_head = b_cell.head().noun(); if unsafe { stack.equals(&mut a_head, &mut b_head) } { - let mut a_tail = a_cell.tail(); - let mut b_tail = b_cell.tail(); + let mut a_tail = a_cell.tail().noun(); + let mut b_tail = b_cell.tail().noun(); dor_tip(stack, &mut a_tail, &mut b_tail) } else { + let mut a_head = a_cell.head().noun(); + let mut b_head = b_cell.head().noun(); dor_tip(stack, &mut a_head, &mut b_head) } } } else if !b.is_atom() { Ok(true) } else { - Ok(lth_b(stack, a.as_atom()?, b.as_atom()?)) + let cmp = lth(stack, a.as_atom()?, b.as_atom()?, &space); + Ok(unsafe { cmp.raw_equals(&D(0)) }) + } +} + +pub type OwnedZoonError = OwnedBasedNounError; + +pub(crate) fn owned_zoon_decode_error(err: OwnedZoonError) -> NounDecodeError { + owned_based_noun_decode_error(err) +} + +fn hash_ten_limbs(ten: [u64; 10]) -> [u64; 5] { + let mut input: Vec = ten.into_iter().map(Belt).collect(); + tip5::hash::hash_10(&mut input) +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub(crate) struct OrderedNoun { + pub(crate) noun: OwnedBasedNoun, + pub(crate) tip: [u64; 5], + pub(crate) double_tip: [u64; 5], +} + +impl OrderedNoun { + pub(crate) fn from_noun(noun: Noun, space: &NounSpace) -> Result { + let noun = OwnedBasedNoun::from_noun(noun, space)?; + Ok(Self::from_owned(noun)) + } + + pub(crate) fn encode(value: &T) -> Result { + let mut stack = NockStack::new(NOCK_STACK_SIZE_TINY, 0); + let noun = value.to_noun(&mut stack); + let space = stack.noun_space(); + Self::from_noun(noun, &space) + } + + fn from_owned(noun: OwnedBasedNoun) -> Self { + let tip = hash_owned_based_noun_varlen(&noun); + let mut ten = [0u64; 10]; + ten[0..5].copy_from_slice(&tip); + ten[5..10].copy_from_slice(&tip); + let double_tip = hash_ten_limbs(ten); + Self { + noun, + tip, + double_tip, + } + } +} + +pub(crate) fn dor_owned(left: &OwnedBasedNoun, right: &OwnedBasedNoun) -> bool { + if left == right { + return true; + } + + match (left, right) { + (OwnedBasedNoun::Atom(left), OwnedBasedNoun::Atom(right)) => left < right, + (OwnedBasedNoun::Atom(_), OwnedBasedNoun::Cell(_, _)) => true, + (OwnedBasedNoun::Cell(_, _), OwnedBasedNoun::Atom(_)) => false, + ( + OwnedBasedNoun::Cell(left_head, left_tail), + OwnedBasedNoun::Cell(right_head, right_tail), + ) => { + if left_head == right_head { + dor_owned(left_tail, right_tail) + } else { + dor_owned(left_head, right_head) + } + } + } +} + +pub(crate) fn gor_owned(left: &OrderedNoun, right: &OrderedNoun) -> bool { + if left.tip == right.tip { + dor_owned(&left.noun, &right.noun) + } else { + lth_tip(&left.tip, &right.tip) + } +} + +pub(crate) fn mor_owned(left: &OrderedNoun, right: &OrderedNoun) -> bool { + if left.double_tip == right.double_tip { + dor_owned(&left.noun, &right.noun) + } else { + lth_tip(&left.double_tip, &right.double_tip) + } +} + +pub(crate) trait ZTreeValue: Sized { + type Output; + + fn ordered(&self) -> &OrderedNoun; + fn same_slot(&self, other: &Self) -> bool; + fn merge_duplicate(existing: Self, incoming: Self) -> Self; + fn into_output(self) -> Self::Output; +} + +pub(crate) trait ZTreeEncode { + fn encode_payload(&self, allocator: &mut A) -> Noun; +} + +pub(crate) trait ZTreeDecode: Sized { + const KIND: &'static str; + + fn decode_payload(noun: &Noun, space: &NounSpace) -> Result; +} + +#[derive(Debug, Clone, PartialEq, Eq, Default)] +pub(crate) enum ZTree

{ + #[default] + Empty, + Node { + payload: P, + left: Box>, + right: Box>, + }, +} + +impl

ZTree

{ + fn split_node(self) -> (P, Box, Box) { + match self { + Self::Node { + payload, + left, + right, + } => (payload, left, right), + Self::Empty => unreachable!("expected z-tree node"), + } + } + + pub(crate) fn len(&self) -> usize { + match self { + Self::Empty => 0, + Self::Node { left, right, .. } => 1 + left.len() + right.len(), + } + } + + pub(crate) fn iter(&self) -> ZTreeIter<'_, P> { + ZTreeIter { stack: vec![self] } + } + + pub(crate) fn fold(&self, empty: &R, node: &F) -> R + where + R: Clone, + F: Fn(&P, R, R) -> R, + { + match self { + Self::Empty => empty.clone(), + Self::Node { + payload, + left, + right, + } => { + let left = left.fold(empty, node); + let right = right.fold(empty, node); + node(payload, left, right) + } + } + } +} + +pub(crate) struct ZTreeIter<'a, P> { + stack: Vec<&'a ZTree

>, +} + +impl<'a, P> Iterator for ZTreeIter<'a, P> { + type Item = &'a P; + + fn next(&mut self) -> Option { + while let Some(tree) = self.stack.pop() { + match tree { + ZTree::Empty => continue, + ZTree::Node { + payload, + left, + right, + } => { + self.stack.push(right); + self.stack.push(left); + return Some(payload); + } + } + } + None + } +} + +impl ZTree

{ + pub(crate) fn insert(self, candidate: P) -> (Self, bool) { + match self { + Self::Empty => ( + Self::Node { + payload: candidate, + left: Box::new(Self::Empty), + right: Box::new(Self::Empty), + }, + true, + ), + Self::Node { + payload, + left, + right, + } => { + if candidate.same_slot(&payload) { + return ( + Self::Node { + payload: P::merge_duplicate(payload, candidate), + left, + right, + }, + false, + ); + } + + if gor_owned(candidate.ordered(), payload.ordered()) { + let (inserted, added) = (*left).insert(candidate); + let (inserted_payload, inserted_left, inserted_right) = inserted.split_node(); + + if mor_owned(payload.ordered(), inserted_payload.ordered()) { + ( + Self::Node { + payload, + left: Box::new(Self::Node { + payload: inserted_payload, + left: inserted_left, + right: inserted_right, + }), + right, + }, + added, + ) + } else { + let new_root = Self::Node { + payload, + left: inserted_right, + right, + }; + ( + Self::Node { + payload: inserted_payload, + left: inserted_left, + right: Box::new(new_root), + }, + added, + ) + } + } else { + let (inserted, added) = (*right).insert(candidate); + let (inserted_payload, inserted_left, inserted_right) = inserted.split_node(); + + if mor_owned(payload.ordered(), inserted_payload.ordered()) { + ( + Self::Node { + payload, + left, + right: Box::new(Self::Node { + payload: inserted_payload, + left: inserted_left, + right: inserted_right, + }), + }, + added, + ) + } else { + let new_root = Self::Node { + payload, + left, + right: inserted_left, + }; + ( + Self::Node { + payload: inserted_payload, + left: Box::new(new_root), + right: inserted_right, + }, + added, + ) + } + } + } + } + } + + pub(crate) fn into_outputs(self, out: &mut Vec) { + match self { + Self::Empty => {} + Self::Node { + payload, + left, + right, + } => { + out.push(payload.into_output()); + left.into_outputs(out); + right.into_outputs(out); + } + } + } +} + +impl ZTree

{ + pub(crate) fn to_noun(&self, allocator: &mut A) -> Noun { + match self { + Self::Empty => D(0), + Self::Node { + payload, + left, + right, + } => { + let payload = payload.encode_payload(allocator); + let left = left.to_noun(allocator); + let right = right.to_noun(allocator); + T(allocator, &[payload, left, right]) + } + } + } +} + +impl ZTree

{ + pub(crate) fn from_noun(noun: &Noun, space: &NounSpace) -> Result { + if let Ok(atom) = noun.in_space(space).as_atom() { + if atom.as_u64()? == 0 { + return Ok(Self::Empty); + } + return Err(NounDecodeError::Custom(format!( + "{} encountered unexpected non-zero atom", + P::KIND + ))); + } + + let cell = noun.in_space(space).as_cell()?; + let payload = P::decode_payload(&cell.head().noun(), space)?; + let branches = cell + .tail() + .as_cell() + .map_err(|_| NounDecodeError::Custom(format!("{} branches must be a cell", P::KIND)))?; + + Ok(Self::Node { + payload, + left: Box::new(Self::from_noun(&branches.head().noun(), space)?), + right: Box::new(Self::from_noun(&branches.tail().noun(), space)?), + }) + } +} + +#[cfg(test)] +pub(crate) mod test_support { + use noun_serde::{NounDecode, NounDecodeError, NounEncode}; + use quickcheck::{empty_shrinker, Arbitrary, Gen}; + + use super::*; + + #[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord)] + pub(crate) enum BoundedTreeValue { + Atom(u16), + Cell(Box, Box), + } + + impl BoundedTreeValue { + fn arbitrary_at_depth(g: &mut Gen, depth: usize) -> Self { + let max_depth = 3; + if depth >= max_depth || bool::arbitrary(g) { + Self::Atom(u16::arbitrary(g)) + } else { + Self::Cell( + Box::new(Self::arbitrary_at_depth(g, depth + 1)), + Box::new(Self::arbitrary_at_depth(g, depth + 1)), + ) + } + } + } + + impl Arbitrary for BoundedTreeValue { + fn arbitrary(g: &mut Gen) -> Self { + Self::arbitrary_at_depth(g, 0) + } + + fn shrink(&self) -> Box> { + empty_shrinker() + } + } + + impl NounEncode for BoundedTreeValue { + fn to_noun(&self, allocator: &mut A) -> Noun { + match self { + Self::Atom(atom) => u64::from(*atom).to_noun(allocator), + Self::Cell(left, right) => { + let left = left.to_noun(allocator); + let right = right.to_noun(allocator); + T(allocator, &[left, right]) + } + } + } + } + + impl NounDecode for BoundedTreeValue { + fn from_noun(noun: &Noun, space: &NounSpace) -> Result { + if let Ok(atom) = noun.in_space(space).as_atom() { + let atom = atom.as_u64().map_err(|_| { + NounDecodeError::Custom("atom too large for bounded value".into()) + })?; + let atom = u16::try_from(atom).map_err(|_| { + NounDecodeError::Custom("atom too large for bounded value".into()) + })?; + Ok(Self::Atom(atom)) + } else { + let cell = noun.in_space(space).as_cell()?; + Ok(Self::Cell( + Box::new(Self::from_noun(&cell.head().noun(), space)?), + Box::new(Self::from_noun(&cell.tail().noun(), space)?), + )) + } + } } } @@ -138,7 +559,7 @@ mod tests { #[test] fn dor_tip_matches_hoon_for_mixed_atom_cell_inputs() { - let mut stack = NockStack::new(8 << 10 << 10, 0); + let mut stack = NockStack::new(nockvm::mem::NOCK_STACK_SIZE_TINY, 0); let cell = T(&mut stack, &[D(1), D(2)]); let mut atom_vs_cell_left = D(7); diff --git a/crates/nockchain-math/src/zoon/zmap.rs b/crates/nockchain-math/src/zoon/zmap.rs index 636849727..d84dc7e5b 100644 --- a/crates/nockchain-math/src/zoon/zmap.rs +++ b/crates/nockchain-math/src/zoon/zmap.rs @@ -1,12 +1,20 @@ use nockvm::interpreter::Context; use nockvm::jets::util::slot; use nockvm::jets::JetErr; -use nockvm::noun::{Noun, NounAllocator, D, T}; +use nockvm::noun::{Noun, NounAllocator, NounSpace, D, T}; use nockvm::site::{site_slam, Site}; +use noun_serde::{NounDecode, NounDecodeError, NounEncode}; use super::common::*; use crate::noun_ext::NounMathExt; +pub trait ZMapHasher { + type Output: Clone; + + fn empty(&self) -> Self::Output; + fn node(&self, key: &K, value: &V, left: Self::Output, right: Self::Output) -> Self::Output; +} + pub fn z_map_put( stack: &mut A, a: &Noun, @@ -14,12 +22,13 @@ pub fn z_map_put( c: &mut Noun, hasher: &H, ) -> Result { + let space = stack.noun_space(); if unsafe { a.raw_equals(&D(0)) } { let kv = T(stack, &[*b, *c]); Ok(T(stack, &[kv, D(0), D(0)])) } else { - let [mut an, al, ar] = a.uncell()?; - let [mut anp, mut anq] = an.uncell()?; + let [mut an, al, ar] = a.uncell(&space)?; + let [mut anp, mut anq] = an.uncell(&space)?; if unsafe { stack.equals(b, &mut anp) } { if unsafe { stack.equals(c, &mut anq) } { return Ok(*a); @@ -29,8 +38,9 @@ pub fn z_map_put( Ok(anbc) } else if gor_tip(stack, b, &mut anp, hasher)? { let d = z_map_put(stack, &al, b, c, hasher)?; - let [dn, dl, dr] = d.uncell()?; - let [mut dnp, _dnq] = dn.uncell()?; + let updated_space = stack.noun_space(); + let [dn, dl, dr] = d.uncell(&updated_space)?; + let [mut dnp, _dnq] = dn.uncell(&updated_space)?; if mor_tip(stack, &mut anp, &mut dnp, hasher)? { Ok(T(stack, &[an, d, ar])) } else { @@ -39,8 +49,9 @@ pub fn z_map_put( } } else { let d = z_map_put(stack, &ar, b, c, hasher)?; - let [dn, dl, dr] = d.uncell()?; - let [mut dnp, _dnq] = dn.uncell()?; + let updated_space = stack.noun_space(); + let [dn, dl, dr] = d.uncell(&updated_space)?; + let [mut dnp, _dnq] = dn.uncell(&updated_space)?; if mor_tip(stack, &mut anp, &mut dnp, hasher)? { Ok(T(stack, &[an, al, d])) } else { @@ -53,16 +64,17 @@ pub fn z_map_put( /// Reduce a z-map using the gate's cached `Site`, mirroring Hoon `++rep`. pub fn z_map_rep(context: &mut Context, map: &Noun, gate: &mut Noun) -> Result { - let prod = slot(*gate, 13)?; + let space = context.stack.noun_space(); + let prod = slot(*gate, 13, &space)?; let site = Site::new(context, gate); let mut reducer = |node: Noun, acc: Noun| -> Result { let sam = T(&mut context.stack, &[node, acc]); site_slam(context, &site, sam) }; - rep_fold(*map, prod, &mut reducer) + rep_fold(*map, prod, &space, &mut reducer) } -fn rep_fold(tree: Noun, acc: Noun, reducer: &mut F) -> Result +fn rep_fold(tree: Noun, acc: Noun, space: &NounSpace, reducer: &mut F) -> Result where F: FnMut(Noun, Noun) -> Result, { @@ -70,8 +82,415 @@ where return Ok(acc); } - let [entry, left, right] = tree.uncell()?; + let [entry, left, right] = tree.uncell(space)?; let acc = reducer(entry, acc)?; - let acc = rep_fold(left, acc, reducer)?; - rep_fold(right, acc, reducer) + let acc = rep_fold(left, acc, space, reducer)?; + rep_fold(right, acc, space, reducer) +} + +#[derive(Debug, Clone, PartialEq, Eq)] +struct ZMapEntry { + key: K, + value: V, + ordered_key: OrderedNoun, +} + +impl ZMapEntry { + fn encode(key: K, value: V) -> Result { + let ordered_key = OrderedNoun::encode(&key)?; + Ok(Self { + key, + value, + ordered_key, + }) + } +} + +impl ZTreeValue for ZMapEntry { + type Output = (K, V); + + fn ordered(&self) -> &OrderedNoun { + &self.ordered_key + } + + fn same_slot(&self, other: &Self) -> bool { + self.ordered_key.noun == other.ordered_key.noun + } + + fn merge_duplicate(existing: Self, incoming: Self) -> Self { + Self { + key: incoming.key, + value: incoming.value, + ordered_key: existing.ordered_key, + } + } + + fn into_output(self) -> Self::Output { + (self.key, self.value) + } +} + +impl ZTreeEncode for ZMapEntry { + fn encode_payload(&self, allocator: &mut A) -> Noun { + let key = self.key.to_noun(allocator); + let value = self.value.to_noun(allocator); + T(allocator, &[key, value]) + } +} + +impl ZTreeDecode for ZMapEntry { + const KIND: &'static str = "z-map"; + + fn decode_payload(noun: &Noun, space: &NounSpace) -> Result { + let entry_cell = noun + .in_space(space) + .as_cell() + .map_err(|_| NounDecodeError::Custom("z-map entry must be a pair".into()))?; + let raw_key = entry_cell.head().noun(); + let raw_value = entry_cell.tail().noun(); + let key = K::from_noun(&raw_key, space)?; + let value = V::from_noun(&raw_value, space)?; + let ordered_key = + OrderedNoun::from_noun(raw_key, space).map_err(owned_zoon_decode_error)?; + Ok(Self { + key, + value, + ordered_key, + }) + } +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct ZMap { + tree: ZTree>, +} + +impl Default for ZMap { + fn default() -> Self { + Self { tree: ZTree::Empty } + } +} + +impl ZMap { + pub fn new() -> Self { + Self::default() + } + + pub fn is_empty(&self) -> bool { + matches!(self.tree, ZTree::Empty) + } + + pub fn into_entries(self) -> Vec<(K, V)> { + let mut out = Vec::new(); + self.tree.into_outputs(&mut out); + out + } + + pub fn fold_tree(&self, empty: R, node: F) -> R + where + R: Clone, + F: Fn(&K, &V, R, R) -> R, + { + self.tree.fold(&empty, &|payload, left, right| { + node(&payload.key, &payload.value, left, right) + }) + } + + pub fn hash_with(&self, hasher: &H) -> H::Output + where + H: ZMapHasher, + { + self.fold_tree(hasher.empty(), |key, value, left, right| { + hasher.node(key, value, left, right) + }) + } +} + +impl ZMap { + pub fn try_insert(&mut self, key: K, value: V) -> Result { + let candidate = ZMapEntry::encode(key, value)?; + let (tree, added) = std::mem::take(&mut self.tree).insert(candidate); + self.tree = tree; + Ok(added) + } + + pub fn try_from_entries(entries: I) -> Result + where + I: IntoIterator, + { + let mut map = Self::new(); + for (key, value) in entries { + map.try_insert(key, value)?; + } + Ok(map) + } +} + +impl NounEncode for ZMap { + fn to_noun(&self, allocator: &mut A) -> Noun { + self.tree.to_noun(allocator) + } +} + +impl NounDecode for ZMap { + fn from_noun(noun: &Noun, space: &NounSpace) -> Result { + Ok(Self { + tree: ZTree::from_noun(noun, space)?, + }) + } +} + +#[cfg(test)] +mod tests { + use std::collections::BTreeMap; + + use nockvm::mem::NockStack; + use nockvm::noun::{Atom, Noun, NounAllocator, NounSpace, D, T}; + use noun_serde::{NounDecode, NounEncode}; + use quickcheck::QuickCheck; + + use super::{z_map_put, DefaultTipHasher, ZMap, ZMapHasher}; + use crate::belt::PRIME; + use crate::zoon::common::test_support::BoundedTreeValue; + + struct DebugMapHasher; + + impl ZMapHasher for DebugMapHasher { + type Output = String; + + fn empty(&self) -> Self::Output { + "~".into() + } + + fn node( + &self, + key: &BoundedTreeValue, + value: &BoundedTreeValue, + left: Self::Output, + right: Self::Output, + ) -> Self::Output { + format!("{key:?}=>{value:?}<{left}|{right}>") + } + } + + fn bounded_entries( + entries: Vec<(BoundedTreeValue, BoundedTreeValue)>, + ) -> Vec<(BoundedTreeValue, BoundedTreeValue)> { + entries.into_iter().take(12).collect() + } + + fn last_write_wins( + entries: Vec<(BoundedTreeValue, BoundedTreeValue)>, + ) -> Vec<(BoundedTreeValue, BoundedTreeValue)> { + let mut deduped = BTreeMap::new(); + for (key, value) in entries { + deduped.insert(key, value); + } + deduped.into_iter().collect() + } + + fn nouns_equal(stack: &mut NockStack, left: Noun, right: Noun) -> bool { + let mut left = left; + let mut right = right; + unsafe { stack.equals(&mut left, &mut right) } + } + + #[test] + fn owned_zmap_matches_raw_zmap_encoding() { + let entries = vec![(7u64, 70u64), (3u64, 30u64), (11u64, 110u64)]; + let mut stack = NockStack::new(nockvm::mem::NOCK_STACK_SIZE_TINY, 0); + let raw = entries.iter().fold(D(0), |acc, (key, value): &(u64, u64)| { + let mut key = key.to_noun(&mut stack); + let mut value = value.to_noun(&mut stack); + z_map_put(&mut stack, &acc, &mut key, &mut value, &DefaultTipHasher) + .expect("raw z-map put") + }); + + let owned = ZMap::try_from_entries(entries).expect("owned z-map should build"); + let mut owned_noun = owned.to_noun(&mut stack); + let mut raw = raw; + assert!(unsafe { stack.equals(&mut raw, &mut owned_noun) }); + } + + #[test] + fn owned_zmap_updates_duplicate_keys() { + let map = ZMap::try_from_entries(vec![(1u64, 10u64), (1u64, 11u64)]) + .expect("owned z-map should build"); + assert_eq!(map.into_entries(), vec![(1u64, 11u64)]); + } + + #[test] + fn owned_zmap_roundtrips_from_noun() { + let mut stack = NockStack::new(nockvm::mem::NOCK_STACK_SIZE_TINY, 0); + let original = + ZMap::try_from_entries(vec![(9u64, 90u64), (2u64, 20u64)]).expect("build z-map"); + let noun = original.to_noun(&mut stack); + let space = stack.noun_space(); + let decoded = ZMap::::from_noun(&noun, &space).expect("decode z-map"); + let mut roundtrip = decoded.to_noun(&mut stack); + let mut noun = noun; + assert!(unsafe { stack.equals(&mut noun, &mut roundtrip) }); + } + + #[test] + fn quickcheck_owned_zmap_matches_raw_zmap_for_bounded_nouns() { + fn prop(entries: Vec<(BoundedTreeValue, BoundedTreeValue)>) -> bool { + let entries = bounded_entries(entries); + let mut stack = NockStack::new(nockvm::mem::NOCK_STACK_SIZE_TINY, 0); + let raw = entries.iter().fold(D(0), |acc, (key, value)| { + let mut key = key.to_noun(&mut stack); + let mut value = value.to_noun(&mut stack); + z_map_put(&mut stack, &acc, &mut key, &mut value, &DefaultTipHasher) + .expect("raw z-map put") + }); + + let owned = ZMap::try_from_entries(entries).expect("owned z-map should build"); + let owned_noun = owned.to_noun(&mut stack); + nouns_equal(&mut stack, raw, owned_noun) + } + + QuickCheck::new() + .tests(256) + .quickcheck(prop as fn(Vec<(BoundedTreeValue, BoundedTreeValue)>) -> bool); + } + + #[test] + fn quickcheck_owned_zmap_is_insertion_order_independent_after_dedup() { + fn prop(entries: Vec<(BoundedTreeValue, BoundedTreeValue)>) -> bool { + let unique = last_write_wins(bounded_entries(entries)); + let mut reversed = unique.clone(); + reversed.reverse(); + + let left = ZMap::try_from_entries(unique).expect("left z-map should build"); + let right = ZMap::try_from_entries(reversed).expect("right z-map should build"); + let mut stack = NockStack::new(nockvm::mem::NOCK_STACK_SIZE_TINY, 0); + let left_noun = left.to_noun(&mut stack); + let right_noun = right.to_noun(&mut stack); + nouns_equal(&mut stack, left_noun, right_noun) + } + + QuickCheck::new() + .tests(256) + .quickcheck(prop as fn(Vec<(BoundedTreeValue, BoundedTreeValue)>) -> bool); + } + + #[test] + fn quickcheck_owned_zmap_uses_last_write_wins_for_duplicate_keys() { + fn prop(entries: Vec<(BoundedTreeValue, BoundedTreeValue)>) -> bool { + let entries = bounded_entries(entries); + let canonical_entries = last_write_wins(entries.clone()); + + let with_duplicates = + ZMap::try_from_entries(entries).expect("z-map with duplicates should build"); + let canonical = + ZMap::try_from_entries(canonical_entries).expect("canonical z-map should build"); + let mut stack = NockStack::new(nockvm::mem::NOCK_STACK_SIZE_TINY, 0); + let with_duplicates_noun = with_duplicates.to_noun(&mut stack); + let canonical_noun = canonical.to_noun(&mut stack); + nouns_equal(&mut stack, with_duplicates_noun, canonical_noun) + } + + QuickCheck::new() + .tests(256) + .quickcheck(prop as fn(Vec<(BoundedTreeValue, BoundedTreeValue)>) -> bool); + } + + #[test] + fn zmap_hash_with_is_insertion_order_independent_after_dedup() { + let left = ZMap::try_from_entries(vec![ + (BoundedTreeValue::Atom(1), BoundedTreeValue::Atom(10)), + (BoundedTreeValue::Atom(7), BoundedTreeValue::Atom(70)), + (BoundedTreeValue::Atom(3), BoundedTreeValue::Atom(30)), + ]) + .expect("left z-map should build"); + let right = ZMap::try_from_entries(vec![ + (BoundedTreeValue::Atom(3), BoundedTreeValue::Atom(30)), + (BoundedTreeValue::Atom(1), BoundedTreeValue::Atom(10)), + (BoundedTreeValue::Atom(7), BoundedTreeValue::Atom(70)), + ]) + .expect("right z-map should build"); + + assert_eq!( + left.hash_with(&DebugMapHasher), + right.hash_with(&DebugMapHasher) + ); + } + + #[test] + fn zmap_hash_with_uses_last_write_wins() { + let with_duplicates = ZMap::try_from_entries(vec![ + (BoundedTreeValue::Atom(1), BoundedTreeValue::Atom(10)), + (BoundedTreeValue::Atom(1), BoundedTreeValue::Atom(11)), + (BoundedTreeValue::Atom(7), BoundedTreeValue::Atom(70)), + ]) + .expect("z-map with duplicates should build"); + let canonical = ZMap::try_from_entries(vec![ + (BoundedTreeValue::Atom(1), BoundedTreeValue::Atom(11)), + (BoundedTreeValue::Atom(7), BoundedTreeValue::Atom(70)), + ]) + .expect("canonical z-map should build"); + + assert_eq!( + with_duplicates.hash_with(&DebugMapHasher), + canonical.hash_with(&DebugMapHasher) + ); + } + + #[test] + fn zmap_decode_rejects_nonzero_atom() { + let space = NounSpace::empty(); + assert!(ZMap::::from_noun(&D(7), &space).is_err()); + } + + #[test] + fn zmap_decode_rejects_non_pair_entry() { + let mut stack = NockStack::new(nockvm::mem::NOCK_STACK_SIZE_TINY, 0); + let malformed = T(&mut stack, &[D(1), D(0), D(0)]); + let space = stack.noun_space(); + assert!(ZMap::::from_noun(&malformed, &space).is_err()); + } + + #[test] + fn zmap_decode_rejects_non_cell_branches() { + let mut stack = NockStack::new(nockvm::mem::NOCK_STACK_SIZE_TINY, 0); + let entry = T(&mut stack, &[D(1), D(2)]); + let malformed = T(&mut stack, &[entry, D(0)]); + let space = stack.noun_space(); + assert!(ZMap::::from_noun(&malformed, &space).is_err()); + } + + #[test] + fn zmap_decode_rejects_invalid_key_payload() { + let mut stack = NockStack::new(nockvm::mem::NOCK_STACK_SIZE_TINY, 0); + let key = T(&mut stack, &[D(1), D(2)]); + let entry = T(&mut stack, &[key, D(3)]); + let malformed = T(&mut stack, &[entry, D(0), D(0)]); + let space = stack.noun_space(); + assert!(ZMap::::from_noun(&malformed, &space).is_err()); + } + + #[test] + fn zmap_decode_rejects_invalid_value_payload() { + let mut stack = NockStack::new(nockvm::mem::NOCK_STACK_SIZE_TINY, 0); + let value = T(&mut stack, &[D(1), D(2)]); + let entry = T(&mut stack, &[D(3), value]); + let malformed = T(&mut stack, &[entry, D(0), D(0)]); + let space = stack.noun_space(); + assert!(ZMap::::from_noun(&malformed, &space).is_err()); + } + + #[test] + fn zmap_decode_rejects_non_based_atoms() { + let mut stack = NockStack::new(nockvm::mem::NOCK_STACK_SIZE_TINY, 0); + let key = Atom::new(&mut stack, PRIME).as_noun(); + let entry = T(&mut stack, &[key, D(3)]); + let malformed = T(&mut stack, &[entry, D(0), D(0)]); + let space = stack.noun_space(); + assert!(ZMap::::from_noun(&malformed, &space).is_err()); + } + + #[test] + fn zmap_try_from_entries_rejects_non_based_atoms() { + assert!(ZMap::try_from_entries(vec![(PRIME, 1u64)]).is_err()); + } } diff --git a/crates/nockchain-math/src/zoon/zset.rs b/crates/nockchain-math/src/zoon/zset.rs index 725a38f32..ec050b8b1 100644 --- a/crates/nockchain-math/src/zoon/zset.rs +++ b/crates/nockchain-math/src/zoon/zset.rs @@ -1,24 +1,34 @@ use nockvm::jets::JetErr; -use nockvm::noun::{Noun, NounAllocator, D, T}; +use nockvm::noun::{Noun, NounAllocator, NounSpace, D, T}; +use noun_serde::{NounDecode, NounDecodeError, NounEncode}; use super::common::*; use crate::noun_ext::NounMathExt; +pub trait ZSetHasher { + type Output: Clone; + + fn empty(&self) -> Self::Output; + fn node(&self, value: &T, left: Self::Output, right: Self::Output) -> Self::Output; +} + pub fn z_set_put( stack: &mut A, a: &Noun, b: &mut Noun, hasher: &H, ) -> Result { + let space = stack.noun_space(); if unsafe { a.raw_equals(&D(0)) } { Ok(T(stack, &[*b, D(0), D(0)])) } else { - let [mut an, al, ar] = a.uncell()?; + let [mut an, al, ar] = a.uncell(&space)?; if unsafe { stack.equals(b, &mut an) } { Ok(*a) } else if gor_tip(stack, b, &mut an, hasher)? { let c = z_set_put(stack, &al, b, hasher)?; - let [mut cn, cl, cr] = c.uncell()?; + let space = stack.noun_space(); + let [mut cn, cl, cr] = c.uncell(&space)?; if mor_tip(stack, &mut an, &mut cn, hasher)? { Ok(T(stack, &[an, c, ar])) } else { @@ -27,7 +37,8 @@ pub fn z_set_put( } } else { let c = z_set_put(stack, &ar, b, hasher)?; - let [mut cn, cl, cr] = c.uncell()?; + let space = stack.noun_space(); + let [mut cn, cl, cr] = c.uncell(&space)?; if mor_tip(stack, &mut an, &mut cn, hasher)? { Ok(T(stack, &[an, al, c])) } else { @@ -50,28 +61,32 @@ pub fn z_set_bif( b: &mut Noun, hasher: &H, ) -> Result { + let space = stack.noun_space(); if unsafe { a.raw_equals(&D(0)) } { Ok(T(stack, &[*b, D(0), D(0)])) } else { - let [mut n, mut l, mut r] = a.uncell()?; + let [mut n, mut l, mut r] = a.uncell(&space)?; if unsafe { stack.equals(b, &mut n) } { Ok(*a) } else if gor_tip(stack, b, &mut n, hasher)? { // could also parameterize Hasher if needed let c = do_bif(stack, &mut l, b, hasher)?; - let [cn, cl, cr] = c.uncell()?; + let space = stack.noun_space(); + let [cn, cl, cr] = c.uncell(&space)?; let new_a = T(stack, &[n, cr, r]); Ok(T(stack, &[cn, cl, new_a])) } else { let c = do_bif(stack, &mut r, b, hasher)?; - let [cn, cl, cr] = c.uncell()?; + let space = stack.noun_space(); + let [cn, cl, cr] = c.uncell(&space)?; let new_a = T(stack, &[n, l, cl]); Ok(T(stack, &[cn, new_a, cr])) } } } let res = do_bif(stack, a, b, hasher)?; - Ok(res.as_cell()?.tail()) + let space = stack.noun_space(); + Ok(res.in_space(&space).as_cell()?.tail().noun()) } pub fn z_set_dif( @@ -86,13 +101,14 @@ pub fn z_set_dif( e: &mut Noun, hasher: &H, ) -> Result { + let space = stack.noun_space(); if unsafe { d.raw_equals(&D(0)) } { Ok(*e) } else if unsafe { e.raw_equals(&D(0)) } { Ok(*d) } else { - let [mut dn, dl, mut dr] = d.uncell()?; - let [mut en, mut el, er] = e.uncell()?; + let [mut dn, dl, mut dr] = d.uncell(&space)?; + let [mut en, mut el, er] = e.uncell(&space)?; if mor_tip(stack, &mut dn, &mut en, hasher)? { let df = dif_helper(stack, &mut dr, e, hasher)?; Ok(T(stack, &[dn, dl, df])) @@ -106,11 +122,377 @@ pub fn z_set_dif( if unsafe { b.raw_equals(&D(0)) } { Ok(*a) } else { - let [mut bn, mut bl, mut br] = b.uncell()?; + let space = stack.noun_space(); + let [mut bn, mut bl, mut br] = b.uncell(&space)?; let c = z_set_bif(stack, a, &mut bn, hasher)?; // could also be generic if needed - let [mut cl, mut cr] = c.uncell()?; + let space = stack.noun_space(); + let [mut cl, mut cr] = c.uncell(&space)?; let mut d = z_set_dif(stack, &mut cl, &mut bl, hasher)?; let mut e = z_set_dif(stack, &mut cr, &mut br, hasher)?; dif_helper(stack, &mut d, &mut e, hasher) } } + +#[derive(Debug, Clone, PartialEq, Eq)] +struct ZSetValue { + value: T, + ordered: OrderedNoun, +} + +impl ZSetValue { + fn encode(value: T) -> Result { + let ordered = OrderedNoun::encode(&value)?; + Ok(Self { value, ordered }) + } +} + +impl ZTreeValue for ZSetValue { + type Output = T; + + fn ordered(&self) -> &OrderedNoun { + &self.ordered + } + + fn same_slot(&self, other: &Self) -> bool { + self.ordered.noun == other.ordered.noun + } + + fn merge_duplicate(existing: Self, _incoming: Self) -> Self { + existing + } + + fn into_output(self) -> Self::Output { + self.value + } +} + +impl ZTreeEncode for ZSetValue { + fn encode_payload(&self, allocator: &mut A) -> Noun { + self.value.to_noun(allocator) + } +} + +impl ZTreeDecode for ZSetValue { + const KIND: &'static str = "z-set"; + + fn decode_payload(noun: &Noun, space: &NounSpace) -> Result { + let value = T::from_noun(noun, space)?; + let ordered = OrderedNoun::from_noun(*noun, space).map_err(owned_zoon_decode_error)?; + Ok(Self { value, ordered }) + } +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct ZSet { + tree: ZTree>, +} + +impl Default for ZSet { + fn default() -> Self { + Self { tree: ZTree::Empty } + } +} + +impl ZSet { + pub fn new() -> Self { + Self::default() + } + + pub fn is_empty(&self) -> bool { + matches!(self.tree, ZTree::Empty) + } + + pub fn into_items(self) -> Vec { + let mut out = Vec::new(); + self.tree.into_outputs(&mut out); + out + } + + pub fn len(&self) -> usize { + self.tree.len() + } + + pub fn iter(&self) -> impl Iterator { + self.tree.iter().map(|payload| &payload.value) + } + + pub fn first(&self) -> Option<&T> { + self.iter().next() + } + + pub fn contains(&self, value: &T) -> bool + where + T: PartialEq, + { + self.iter().any(|item| item == value) + } + + pub fn fold_tree(&self, empty: R, node: F) -> R + where + R: Clone, + F: Fn(&T, R, R) -> R, + { + self.tree.fold(&empty, &|payload, left, right| { + node(&payload.value, left, right) + }) + } + + pub fn hash_with(&self, hasher: &H) -> H::Output + where + H: ZSetHasher, + { + self.fold_tree(hasher.empty(), |value, left, right| { + hasher.node(value, left, right) + }) + } +} + +impl ZSet { + pub fn try_insert(&mut self, value: T) -> Result { + let candidate = ZSetValue::encode(value)?; + let (tree, added) = std::mem::take(&mut self.tree).insert(candidate); + self.tree = tree; + Ok(added) + } + + pub fn try_from_items(items: I) -> Result + where + I: IntoIterator, + { + let mut set = Self::new(); + for item in items { + set.try_insert(item)?; + } + Ok(set) + } +} + +impl NounEncode for ZSet { + fn to_noun(&self, allocator: &mut A) -> Noun { + self.tree.to_noun(allocator) + } +} + +impl NounDecode for ZSet { + fn from_noun(noun: &Noun, space: &NounSpace) -> Result { + Ok(Self { + tree: ZTree::from_noun(noun, space)?, + }) + } +} + +impl IntoIterator for ZSet { + type Item = T; + type IntoIter = std::vec::IntoIter; + + fn into_iter(self) -> Self::IntoIter { + self.into_items().into_iter() + } +} + +#[cfg(test)] +mod tests { + use std::collections::BTreeSet; + + use nockvm::mem::NockStack; + use nockvm::noun::{Atom, Noun, NounAllocator, NounSpace, D, T}; + use noun_serde::{NounDecode, NounEncode}; + use quickcheck::QuickCheck; + + use super::{z_set_put, DefaultTipHasher, ZSet, ZSetHasher}; + use crate::belt::PRIME; + use crate::zoon::common::test_support::BoundedTreeValue; + + struct DebugSetHasher; + + impl ZSetHasher for DebugSetHasher { + type Output = String; + + fn empty(&self) -> Self::Output { + "~".into() + } + + fn node( + &self, + value: &BoundedTreeValue, + left: Self::Output, + right: Self::Output, + ) -> Self::Output { + format!("{value:?}<{left}|{right}>") + } + } + + fn bounded_items(items: Vec) -> Vec { + items.into_iter().take(12).collect() + } + + fn nouns_equal(stack: &mut NockStack, left: Noun, right: Noun) -> bool { + let mut left = left; + let mut right = right; + unsafe { stack.equals(&mut left, &mut right) } + } + + #[test] + fn owned_zset_matches_raw_zset_encoding() { + let items = vec![7u64, 3u64, 11u64, 5u64]; + let mut stack = NockStack::new(nockvm::mem::NOCK_STACK_SIZE_TINY, 0); + let raw = items.iter().fold(D(0), |acc, item: &u64| { + let mut noun = item.to_noun(&mut stack); + z_set_put(&mut stack, &acc, &mut noun, &DefaultTipHasher).expect("raw z-set put") + }); + + let owned = ZSet::try_from_items(items).expect("owned z-set should build"); + let mut owned_noun = owned.to_noun(&mut stack); + let mut raw = raw; + assert!(unsafe { stack.equals(&mut raw, &mut owned_noun) }); + } + + #[test] + fn owned_zset_is_insertion_order_independent() { + let mut stack = NockStack::new(nockvm::mem::NOCK_STACK_SIZE_TINY, 0); + let left = ZSet::try_from_items(vec![1u64, 7u64, 3u64]).expect("left z-set"); + let right = ZSet::try_from_items(vec![7u64, 3u64, 1u64]).expect("right z-set"); + let mut left_noun = left.to_noun(&mut stack); + let mut right_noun = right.to_noun(&mut stack); + assert!(unsafe { stack.equals(&mut left_noun, &mut right_noun) }); + } + + #[test] + fn owned_zset_roundtrips_from_noun() { + let mut stack = NockStack::new(nockvm::mem::NOCK_STACK_SIZE_TINY, 0); + let original = ZSet::try_from_items(vec![9u64, 2u64, 6u64]).expect("build z-set"); + let noun = original.to_noun(&mut stack); + let space = stack.noun_space(); + let decoded = ZSet::::from_noun(&noun, &space).expect("decode z-set"); + let mut roundtrip = decoded.to_noun(&mut stack); + let mut noun = noun; + assert!(unsafe { stack.equals(&mut noun, &mut roundtrip) }); + } + + #[test] + fn quickcheck_owned_zset_matches_raw_zset_for_bounded_nouns() { + fn prop(items: Vec) -> bool { + let items = bounded_items(items); + let mut stack = NockStack::new(nockvm::mem::NOCK_STACK_SIZE_TINY, 0); + let raw = items.iter().fold(D(0), |acc, item| { + let mut noun = item.to_noun(&mut stack); + z_set_put(&mut stack, &acc, &mut noun, &DefaultTipHasher).expect("raw z-set put") + }); + + let owned = ZSet::try_from_items(items).expect("owned z-set should build"); + let owned_noun = owned.to_noun(&mut stack); + nouns_equal(&mut stack, raw, owned_noun) + } + + QuickCheck::new() + .tests(256) + .quickcheck(prop as fn(Vec) -> bool); + } + + #[test] + fn quickcheck_owned_zset_is_insertion_order_independent_for_unique_items() { + fn prop(items: Vec) -> bool { + let unique: Vec<_> = bounded_items(items) + .into_iter() + .collect::>() + .into_iter() + .collect(); + let mut reversed = unique.clone(); + reversed.reverse(); + + let left = ZSet::try_from_items(unique).expect("left z-set should build"); + let right = ZSet::try_from_items(reversed).expect("right z-set should build"); + let mut stack = NockStack::new(nockvm::mem::NOCK_STACK_SIZE_TINY, 0); + let left_noun = left.to_noun(&mut stack); + let right_noun = right.to_noun(&mut stack); + nouns_equal(&mut stack, left_noun, right_noun) + } + + QuickCheck::new() + .tests(256) + .quickcheck(prop as fn(Vec) -> bool); + } + + #[test] + fn quickcheck_owned_zset_ignores_duplicate_items() { + fn prop(items: Vec) -> bool { + let items = bounded_items(items); + let deduped: Vec<_> = items + .iter() + .cloned() + .collect::>() + .into_iter() + .collect(); + + let with_duplicates = + ZSet::try_from_items(items).expect("z-set with duplicates should build"); + let deduped = ZSet::try_from_items(deduped).expect("deduped z-set should build"); + let mut stack = NockStack::new(nockvm::mem::NOCK_STACK_SIZE_TINY, 0); + let with_duplicates_noun = with_duplicates.to_noun(&mut stack); + let deduped_noun = deduped.to_noun(&mut stack); + nouns_equal(&mut stack, with_duplicates_noun, deduped_noun) + } + + QuickCheck::new() + .tests(256) + .quickcheck(prop as fn(Vec) -> bool); + } + + #[test] + fn zset_hash_with_is_insertion_order_independent() { + let left = ZSet::try_from_items(vec![ + BoundedTreeValue::Atom(1), + BoundedTreeValue::Atom(7), + BoundedTreeValue::Atom(3), + ]) + .expect("left z-set should build"); + let right = ZSet::try_from_items(vec![ + BoundedTreeValue::Atom(7), + BoundedTreeValue::Atom(3), + BoundedTreeValue::Atom(1), + ]) + .expect("right z-set should build"); + + assert_eq!( + left.hash_with(&DebugSetHasher), + right.hash_with(&DebugSetHasher) + ); + } + + #[test] + fn zset_decode_rejects_nonzero_atom() { + let space = NounSpace::empty(); + assert!(ZSet::::from_noun(&D(7), &space).is_err()); + } + + #[test] + fn zset_decode_rejects_non_cell_branches() { + let mut stack = NockStack::new(nockvm::mem::NOCK_STACK_SIZE_TINY, 0); + let malformed = T(&mut stack, &[D(1), D(2)]); + let space = stack.noun_space(); + assert!(ZSet::::from_noun(&malformed, &space).is_err()); + } + + #[test] + fn zset_decode_rejects_invalid_item_payload() { + let mut stack = NockStack::new(nockvm::mem::NOCK_STACK_SIZE_TINY, 0); + let payload = T(&mut stack, &[D(1), D(2)]); + let malformed = T(&mut stack, &[payload, D(0), D(0)]); + let space = stack.noun_space(); + assert!(ZSet::::from_noun(&malformed, &space).is_err()); + } + + #[test] + fn zset_decode_rejects_non_based_atoms() { + let mut stack = NockStack::new(nockvm::mem::NOCK_STACK_SIZE_TINY, 0); + let payload = Atom::new(&mut stack, PRIME).as_noun(); + let malformed = T(&mut stack, &[payload, D(0), D(0)]); + let space = stack.noun_space(); + assert!(ZSet::::from_noun(&malformed, &space).is_err()); + } + + #[test] + fn zset_try_from_items_rejects_non_based_atoms() { + assert!(ZSet::try_from_items(vec![PRIME]).is_err()); + } +} diff --git a/crates/nockchain-types/Cargo.toml b/crates/nockchain-types/Cargo.toml index 3b7c9e4cb..032785979 100644 --- a/crates/nockchain-types/Cargo.toml +++ b/crates/nockchain-types/Cargo.toml @@ -17,7 +17,7 @@ nockvm = { workspace = true } nockvm_macros = { workspace = true } noun-serde = { workspace = true } num-bigint.workspace = true -serde = { version = "1.0.217", features = ["derive"], default-features = false } +serde = { workspace = true, features = ["derive"], default-features = false } thiserror = { workspace = true } tracing = { workspace = true } tracing-subscriber = { workspace = true } diff --git a/crates/nockchain-types/jams/README.md b/crates/nockchain-types/jams/README.md index b9bb1600f..d37af28c9 100644 --- a/crates/nockchain-types/jams/README.md +++ b/crates/nockchain-types/jams/README.md @@ -26,3 +26,88 @@ To refresh coverage: 3. Update assertions in the matching tests when fixture structure changes. These tests use fixed `include_bytes!` paths, so fixture files must exist at compile time. + +## Compatibility note +- If you change `tx_engine` encoding/decoding or noun shape without a version bump, regenerate these fixtures. +- Otherwise tests may fail for the wrong reason, or pass against stale data that no longer matches current consensus encoding. + +## Regeneration Flow +`hoon-closed` regeneration flow +- Run all commands from the repo root. +- Each `hoon-closed` run writes `out.jam`; move it to the fixture path shown below. + +## Fixture map +- `open/crates/nockchain-types/jams/v0/raw-tx.jam` + - generator: `closed/hoon/scripts/fixtures/v0/generate-raw-tx.hoon` + - command: +```bash +cargo run --profile release --bin hoon-closed -- \ + closed/hoon/scripts/fixtures/v0/generate-raw-tx.hoon \ + closed/hoon +cp out.jam open/crates/nockchain-types/jams/v0/raw-tx.jam +``` +- `open/crates/nockchain-types/jams/v0/note.jam` + - generator: `closed/hoon/scripts/fixtures/v0/generate-note.hoon` + - command: +```bash +cargo run --profile release --bin hoon-closed -- \ + closed/hoon/scripts/fixtures/v0/generate-note.hoon \ + closed/hoon +cp out.jam open/crates/nockchain-types/jams/v0/note.jam +``` +- `open/crates/nockchain-types/jams/v0/balance.jam` + - generator: `closed/hoon/scripts/fixtures/v0/generate-balance.hoon` + - command: +```bash +cargo run --profile release --bin hoon-closed -- \ + closed/hoon/scripts/fixtures/v0/generate-balance.hoon \ + closed/hoon +cp out.jam open/crates/nockchain-types/jams/v0/balance.jam +``` +- `open/crates/nockchain-types/jams/v0/timelock.jam` + - generator: `closed/hoon/scripts/fixtures/v0/generate-timelock.hoon` + - command: +```bash +cargo run --profile release --bin hoon-closed -- \ + closed/hoon/scripts/fixtures/v0/generate-timelock.hoon \ + closed/hoon +cp out.jam open/crates/nockchain-types/jams/v0/timelock.jam +``` +- `open/crates/nockchain-types/jams/v1/raw-tx.jam` + - generator: `closed/hoon/scripts/fixtures/v1/generate-v1-raw-tx.hoon` + - command: +```bash +cargo run --profile release --bin hoon-closed -- \ + closed/hoon/scripts/fixtures/v1/generate-v1-raw-tx.hoon \ + closed/hoon +cp out.jam open/crates/nockchain-types/jams/v1/raw-tx.jam +``` +- `open/crates/nockchain-types/jams/v1/note.jam` + - generator: `closed/hoon/scripts/fixtures/v1/generate-v1-note.hoon` + - command: +```bash +cargo run --profile release --bin hoon-closed -- \ + closed/hoon/scripts/fixtures/v1/generate-v1-note.hoon \ + closed/hoon +cp out.jam open/crates/nockchain-types/jams/v1/note.jam +``` +- `open/crates/nockchain-types/jams/v1/raw-tx-word-count-oracle.jam` + - generator: `closed/hoon/scripts/fixtures/v1/generate-v1-raw-tx-word-count-golden.hoon` + - command: +```bash +cargo run --profile release --bin hoon-closed -- \ + closed/hoon/scripts/fixtures/v1/generate-v1-raw-tx-word-count-golden.hoon \ + closed/hoon +cp out.jam open/crates/nockchain-types/jams/v1/raw-tx-word-count-oracle.jam +``` + +## Captured fixture (no direct Hoon script) +- `open/crates/nockchain-types/jams/v0/early-balance.jam` + - source: captured `%balance-by-pubkey` peek payload. + - capture command: +```bash +cargo run -p nockapp-grpc --example dump_balance_peek -- \ + http://127.0.0.1:50051 \ + \ + /tmp +``` diff --git a/crates/nockchain-types/jams/v1/raw-tx-word-count-oracle.jam b/crates/nockchain-types/jams/v1/raw-tx-word-count-oracle.jam new file mode 100644 index 000000000..3d4b3947c Binary files /dev/null and b/crates/nockchain-types/jams/v1/raw-tx-word-count-oracle.jam differ diff --git a/crates/nockchain-types/src/blockchain_constants.rs b/crates/nockchain-types/src/blockchain_constants.rs new file mode 100644 index 000000000..ae6858c02 --- /dev/null +++ b/crates/nockchain-types/src/blockchain_constants.rs @@ -0,0 +1,633 @@ +use std::time::Duration; + +use ibig::UBig; +use nockapp::noun::slab::NounSlab; +use nockapp::noun::IntoSlab; +use nockvm::noun::{Atom, Noun, NounAllocator, T}; +use noun_serde::NounEncode; +use tracing::info; + +pub const DEFAULT_FAKENET_POW_LEN: u64 = 2; +pub const DEFAULT_FAKENET_LOG_DIFFICULTY: u64 = 1; +pub const FAKENET_V1_PHASE: u64 = 1; +pub const FAKENET_BYTHOS_PHASE: u64 = 1; +pub const FAKENET_BASE_FEE: u64 = 128; + +#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, NounEncode)] +pub struct Seconds(pub u64); + +impl Seconds { + pub fn new(seconds: u64) -> Self { + Self(seconds) + } + + pub fn as_u64(&self) -> u64 { + self.0 + } + + pub fn to_duration(&self) -> Duration { + Duration::from_secs(self.0) + } +} + +impl From for Seconds { + fn from(seconds: u64) -> Self { + Self(seconds) + } +} + +impl TryFrom for Seconds { + type Error = &'static str; + + fn try_from(duration: Duration) -> Result { + if duration.subsec_nanos() != 0 { + return Err("Duration must be whole seconds only"); + } + Ok(Self(duration.as_secs())) + } +} + +impl From for Duration { + fn from(seconds: Seconds) -> Self { + Duration::from_secs(seconds.0) + } +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, NounEncode)] +pub struct NoteDataConstraints { + pub max_size: u64, + pub min_fee: u64, +} + +/// Shared blockchain constants model used for explicit kernel constants pokes. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct BlockchainConstants { + pub max_block_size: u64, + pub blocks_per_epoch: u64, + pub target_epoch_duration: Seconds, + pub update_candidate_timestamp_interval: Seconds, + pub max_future_timestamp: Seconds, + pub min_past_blocks: u64, + pub genesis_target_atom: UBig, + pub max_target_atom: UBig, + pub check_pow_flag: bool, + pub coinbase_timelock_min: u64, + pub pow_len: u64, + pub max_coinbase_split: u64, + pub first_month_coinbase_min: u64, + pub v1_phase: u64, + pub bythos_phase: u64, + pub note_data: NoteDataConstraints, + pub base_fee: u64, + pub input_fee_divisor: u64, + pub asert_phase: u64, + pub asert_anchor_height: u64, + pub asert_anchor_target_atom: UBig, + pub asert_ideal_block_time: u64, + pub asert_half_life: u64, + pub asert_anchor_min_timestamp: u64, +} + +impl BlockchainConstants { + pub const DEFAULT_MAX_BLOCK_SIZE: u64 = 8_000_000; + pub const DEFAULT_BLOCKS_PER_EPOCH: u64 = 2_016; + pub const DEFAULT_TARGET_EPOCH_DURATION: u64 = 1_209_600; + pub const DEFAULT_UPDATE_CANDIDATE_TIMESTAMP_INTERVAL_SECS: u64 = 300; + pub const DEFAULT_MAX_FUTURE_TIMESTAMP: u64 = 7_200; + pub const DEFAULT_GENESIS_TARGET_ATOM: &str = + "0x3ffffffec0000003bffffff88000000b3ffffff34000000b3ffffff880000003bffffffec0000"; + pub const DEFAULT_MAX_TIP5_ATOM: &str = + "0xfffffffb0000000effffffe20000002cffffffcd0000002cffffffe20000000efffffffb00000000"; + pub const DEFAULT_MIN_PAST_BLOCKS: u64 = 11; + pub const DEFAULT_CHECK_POW_FLAG: bool = true; + pub const DEFAULT_COINBASE_TIMELOCK_MIN: u64 = 100; + pub const DEFAULT_POW_LEN: u64 = 64; + pub const DEFAULT_MAX_COINBASE_SPLIT: u64 = 2; + pub const DEFAULT_FIRST_MONTH_COINBASE_MIN: u64 = 4_383; + pub const DEFAULT_V1_PHASE: u64 = 39_000; + pub const DEFAULT_BYTHOS_PHASE: u64 = 54_000; + pub const DEFAULT_NOTE_DATA_MAX_SIZE: u64 = 2_048; + pub const DEFAULT_NOTE_DATA_MIN_FEE: u64 = 256; + pub const DEFAULT_BASE_FEE: u64 = 16_384; + pub const DEFAULT_INPUT_FEE_DIVISOR: u64 = 4; + pub const DEFAULT_ASERT_PHASE: u64 = 65_500; + pub const DEFAULT_ASERT_ANCHOR_HEIGHT: u64 = 65_499; + pub const DEFAULT_ASERT_ANCHOR_TARGET_BEX: u64 = 291; + pub const DEFAULT_ASERT_IDEAL_BLOCK_TIME: u64 = 150; + pub const DEFAULT_ASERT_HALF_LIFE: u64 = 43_200; + /// Median-of-11 timestamp at the canonical ASERT anchor block (mainnet + /// height 65,499), pinned at phase-2 cutover of 014-aletheia. See + /// `open/hoon/common/tx-engine-1.hoon`'s `blockchain-constants:v1` + /// bunt — must stay in sync. + pub const DEFAULT_ASERT_ANCHOR_MIN_TIMESTAMP: u64 = 9_223_372_093_639_027_842; + + pub fn new() -> Self { + let max_target_atom = UBig::from_str_with_radix_prefix(Self::DEFAULT_MAX_TIP5_ATOM) + .expect("Failed to parse max tip5 atom"); + let genesis_target_atom = + UBig::from_str_with_radix_prefix(Self::DEFAULT_GENESIS_TARGET_ATOM) + .expect("Failed to parse genesis target atom"); + let asert_anchor_target_atom = + UBig::from(1u64) << (Self::DEFAULT_ASERT_ANCHOR_TARGET_BEX as usize); + + Self { + max_block_size: Self::DEFAULT_MAX_BLOCK_SIZE, + blocks_per_epoch: Self::DEFAULT_BLOCKS_PER_EPOCH, + target_epoch_duration: Self::DEFAULT_TARGET_EPOCH_DURATION.into(), + update_candidate_timestamp_interval: + Self::DEFAULT_UPDATE_CANDIDATE_TIMESTAMP_INTERVAL_SECS.into(), + max_future_timestamp: Self::DEFAULT_MAX_FUTURE_TIMESTAMP.into(), + min_past_blocks: Self::DEFAULT_MIN_PAST_BLOCKS, + genesis_target_atom, + max_target_atom, + check_pow_flag: Self::DEFAULT_CHECK_POW_FLAG, + coinbase_timelock_min: Self::DEFAULT_COINBASE_TIMELOCK_MIN, + pow_len: Self::DEFAULT_POW_LEN, + max_coinbase_split: Self::DEFAULT_MAX_COINBASE_SPLIT, + first_month_coinbase_min: Self::DEFAULT_FIRST_MONTH_COINBASE_MIN, + v1_phase: Self::DEFAULT_V1_PHASE, + bythos_phase: Self::DEFAULT_BYTHOS_PHASE, + note_data: NoteDataConstraints { + max_size: Self::DEFAULT_NOTE_DATA_MAX_SIZE, + min_fee: Self::DEFAULT_NOTE_DATA_MIN_FEE, + }, + base_fee: Self::DEFAULT_BASE_FEE, + input_fee_divisor: Self::DEFAULT_INPUT_FEE_DIVISOR, + asert_phase: Self::DEFAULT_ASERT_PHASE, + asert_anchor_height: Self::DEFAULT_ASERT_ANCHOR_HEIGHT, + asert_anchor_target_atom, + asert_ideal_block_time: Self::DEFAULT_ASERT_IDEAL_BLOCK_TIME, + asert_half_life: Self::DEFAULT_ASERT_HALF_LIFE, + asert_anchor_min_timestamp: Self::DEFAULT_ASERT_ANCHOR_MIN_TIMESTAMP, + } + } + + pub fn with_genesis_target_atom_bex(mut self, bex: u128) -> Self { + let difficulty = UBig::from((1 << bex) as u128); + self.genesis_target_atom = self.max_target_atom.clone() / difficulty; + info!("Genesis target atom set to {}", self.genesis_target_atom); + self + } + + pub fn with_update_candidate_timestamp_interval(mut self, interval_secs: Seconds) -> Self { + self.update_candidate_timestamp_interval = interval_secs; + self + } + + pub fn with_pow_len(mut self, pow_len: u64) -> Self { + self.pow_len = pow_len; + self + } + + pub fn with_v1_phase(mut self, v1_phase: u64) -> Self { + self.v1_phase = v1_phase; + self + } + + pub fn with_bythos_phase(mut self, bythos_phase: u64) -> Self { + self.bythos_phase = bythos_phase; + self + } + + pub fn with_first_month_coinbase_min(mut self, coinbase_min: u64) -> Self { + self.first_month_coinbase_min = coinbase_min; + self + } + + pub fn with_coinbase_timelock_min(mut self, coinbase_min: u64) -> Self { + self.coinbase_timelock_min = coinbase_min; + self + } + + pub fn with_base_fee(mut self, base_fee: u64) -> Self { + self.base_fee = base_fee; + self + } + + pub fn with_asert_phase(mut self, asert_phase: u64) -> Self { + self.asert_phase = asert_phase; + self + } + + pub fn with_asert_anchor_height(mut self, asert_anchor_height: u64) -> Self { + self.asert_anchor_height = asert_anchor_height; + self + } + + pub fn with_asert_anchor_target_atom(mut self, asert_anchor_target_atom: UBig) -> Self { + self.asert_anchor_target_atom = asert_anchor_target_atom; + self + } + + pub fn with_asert_anchor_target_bex(mut self, bex: u64) -> Self { + self.asert_anchor_target_atom = UBig::from(1u64) << (bex as usize); + self + } + + fn to_blockchain_constants_v0_fields(&self, allocator: &mut A) -> Vec { + let max_block_size = Atom::new(allocator, self.max_block_size).as_noun(); + let blocks_per_epoch = Atom::new(allocator, self.blocks_per_epoch).as_noun(); + let target_epoch_duration = self.target_epoch_duration.to_noun(allocator); + let update_candidate_timestamp_interval_atoms = + UBig::from(self.update_candidate_timestamp_interval.0) << 64; + let update_candidate_timestamp_interval = + Atom::from_ubig(allocator, &update_candidate_timestamp_interval_atoms).as_noun(); + let max_future_timestamp = self.max_future_timestamp.to_noun(allocator); + let min_past_blocks = Atom::new(allocator, self.min_past_blocks).as_noun(); + let genesis_target_atom = Atom::from_ubig(allocator, &self.genesis_target_atom).as_noun(); + let max_target_atom = Atom::from_ubig(allocator, &self.max_target_atom).as_noun(); + let check_pow_flag = self.check_pow_flag.to_noun(allocator); + let coinbase_timelock_min = Atom::new(allocator, self.coinbase_timelock_min).as_noun(); + let pow_len = Atom::new(allocator, self.pow_len).as_noun(); + let max_coinbase_split = Atom::new(allocator, self.max_coinbase_split).as_noun(); + let first_month_coinbase_min = + Atom::new(allocator, self.first_month_coinbase_min).as_noun(); + + vec![ + max_block_size, blocks_per_epoch, target_epoch_duration, + update_candidate_timestamp_interval, max_future_timestamp, min_past_blocks, + genesis_target_atom, max_target_atom, check_pow_flag, coinbase_timelock_min, pow_len, + max_coinbase_split, first_month_coinbase_min, + ] + } +} + +impl Default for BlockchainConstants { + fn default() -> Self { + Self::new() + } +} + +impl NounEncode for BlockchainConstants { + fn to_noun(&self, allocator: &mut A) -> Noun { + let v1_phase = Atom::new(allocator, self.v1_phase).as_noun(); + let bythos_phase = Atom::new(allocator, self.bythos_phase).as_noun(); + let note_data = self.note_data.to_noun(allocator); + let base_fee = Atom::new(allocator, self.base_fee).as_noun(); + let input_fee_divisor = Atom::new(allocator, self.input_fee_divisor).as_noun(); + let v0_fields = self.to_blockchain_constants_v0_fields(allocator); + let v0_constants = T(allocator, &v0_fields); + let asert_phase = Atom::new(allocator, self.asert_phase).as_noun(); + let asert_anchor_height = Atom::new(allocator, self.asert_anchor_height).as_noun(); + let asert_anchor_target_atom = + Atom::from_ubig(allocator, &self.asert_anchor_target_atom).as_noun(); + let asert_ideal_block_time = Atom::new(allocator, self.asert_ideal_block_time).as_noun(); + let asert_half_life = Atom::new(allocator, self.asert_half_life).as_noun(); + let asert_anchor_min_timestamp = + Atom::new(allocator, self.asert_anchor_min_timestamp).as_noun(); + + T( + allocator, + &[ + v1_phase, bythos_phase, note_data, base_fee, input_fee_divisor, v0_constants, + asert_phase, asert_anchor_height, asert_anchor_target_atom, asert_ideal_block_time, + asert_half_life, asert_anchor_min_timestamp, + ], + ) + } +} + +impl IntoSlab for BlockchainConstants { + fn into_slab(self) -> NounSlab { + let mut slab = NounSlab::new(); + let noun = self.to_noun(&mut slab); + slab.set_root(noun); + slab + } +} + +pub fn fakenet_blockchain_constants(pow_len: u64, target_bex: u64) -> BlockchainConstants { + // Fakenet starts from the mainnet defaults and overrides only the fields + // that are intentionally relaxed for local testing. + BlockchainConstants::new() + .with_update_candidate_timestamp_interval(Seconds(5 * 60)) + .with_pow_len(pow_len) + .with_genesis_target_atom_bex(target_bex as u128) + .with_first_month_coinbase_min(0) + .with_coinbase_timelock_min(1) + .with_base_fee(FAKENET_BASE_FEE) + .with_v1_phase(FAKENET_V1_PHASE) + .with_bythos_phase(FAKENET_BYTHOS_PHASE) +} + +pub fn default_fakenet_blockchain_constants() -> BlockchainConstants { + fakenet_blockchain_constants(DEFAULT_FAKENET_POW_LEN, DEFAULT_FAKENET_LOG_DIFFICULTY) +} + +#[cfg(test)] +mod tests { + use ibig::UBig; + + use super::*; + + fn tuple_len(noun: Noun, space: &nockvm::noun::NounSpace) -> usize { + let mut len = 0; + let mut cur = noun; + loop { + if let Ok(cell) = cur.in_space(space).as_cell() { + len += 1; + cur = cell.tail().noun(); + } else { + len += 1; + break; + } + } + len + } + + #[test] + fn blockchain_constants_new_defaults_are_valid() { + let constants = BlockchainConstants::new(); + + assert_eq!( + constants.max_block_size, 8_000_000, + "max-block-size mismatch" + ); + assert_eq!( + constants.blocks_per_epoch, 2_016, + "blocks-per-epoch mismatch" + ); + assert_eq!( + constants.target_epoch_duration, + Seconds::new(14 * 24 * 60 * 60), + "target-epoch-duration mismatch", + ); + assert_eq!( + constants.update_candidate_timestamp_interval, + Seconds::new(5 * 60), + "update-candidate-interval mismatch", + ); + assert_eq!( + constants.max_future_timestamp, + Seconds::new(60 * 120), + "max-future-timestamp mismatch", + ); + assert_eq!(constants.min_past_blocks, 11, "min-past-blocks mismatch"); + + let max_tip5_atom = + UBig::from_str_with_radix_prefix(BlockchainConstants::DEFAULT_MAX_TIP5_ATOM) + .expect("parse max tip5 atom"); + assert_eq!( + constants.max_target_atom, max_tip5_atom, + "max-target-atom mismatch", + ); + + let expected_genesis_target = &max_tip5_atom / (UBig::from(1u64) << 14); + assert_eq!( + constants.genesis_target_atom, expected_genesis_target, + "genesis-target-atom mismatch", + ); + + assert!(constants.check_pow_flag, "check-pow-flag mismatch"); + assert_eq!( + constants.coinbase_timelock_min, 100, + "coinbase-timelock-min mismatch" + ); + assert_eq!(constants.pow_len, 64, "pow-len mismatch"); + assert_eq!( + constants.max_coinbase_split, 2, + "max-coinbase-split mismatch" + ); + assert_eq!( + constants.first_month_coinbase_min, 4_383, + "first-month-coinbase-min mismatch", + ); + assert_eq!(constants.v1_phase, 39_000, "v1-phase mismatch"); + assert_eq!(constants.bythos_phase, 54_000, "bythos-phase mismatch"); + assert_eq!( + constants.note_data, + NoteDataConstraints { + max_size: 2_048, + min_fee: 256, + }, + "note-data mismatch", + ); + assert_eq!(constants.base_fee, 16_384, "base-fee mismatch"); + assert_eq!(constants.input_fee_divisor, 4, "input-fee-divisor mismatch"); + assert_eq!(constants.asert_phase, 65_500, "asert-phase mismatch"); + assert_eq!( + constants.asert_anchor_height, 65_499, + "asert-anchor-height mismatch" + ); + assert_eq!( + constants.asert_anchor_target_atom, + UBig::from(1u64) << 291, + "asert-anchor-target-atom mismatch" + ); + assert_eq!( + constants.asert_ideal_block_time, 150, + "asert-ideal-block-time mismatch" + ); + assert_eq!( + constants.asert_half_life, 43_200, + "asert-half-life mismatch" + ); + } + + #[test] + fn fakenet_blockchain_constants_activate_early_phases() { + let constants = default_fakenet_blockchain_constants(); + + assert_eq!( + constants.pow_len, DEFAULT_FAKENET_POW_LEN, + "pow-len mismatch" + ); + assert_eq!( + constants.coinbase_timelock_min, 1, + "coinbase-timelock-min mismatch" + ); + assert_eq!( + constants.first_month_coinbase_min, 0, + "first-month-coinbase-min mismatch", + ); + assert_eq!(constants.base_fee, FAKENET_BASE_FEE, "base-fee mismatch"); + assert_eq!(constants.v1_phase, FAKENET_V1_PHASE, "v1-phase mismatch"); + assert_eq!( + constants.bythos_phase, FAKENET_BYTHOS_PHASE, + "bythos-phase mismatch" + ); + } + + #[test] + fn with_v1_phase_overrides_default() { + let constants = BlockchainConstants::new().with_v1_phase(54_321); + + assert_eq!(constants.v1_phase, 54_321); + } + + #[test] + fn with_asert_overrides_default() { + let constants = BlockchainConstants::new() + .with_asert_phase(10) + .with_asert_anchor_height(9) + .with_asert_anchor_target_bex(2); + + assert_eq!(constants.asert_phase, 10); + assert_eq!(constants.asert_anchor_height, 9); + assert_eq!(constants.asert_anchor_target_atom, UBig::from(1u64) << 2); + } + + #[test] + fn blockchain_constants_encode_in_new_v1_wrapper() { + let slab = BlockchainConstants::new().into_slab(); + let root = unsafe { *slab.root() }; + let space = slab.noun_space(); + + let outer = root.in_space(&space).as_cell().expect("outer tuple"); + let v1_phase_atom = outer.head().as_atom().expect("v1-phase should be atom"); + assert_eq!( + v1_phase_atom.as_u64().expect("v1-phase as u64"), + BlockchainConstants::DEFAULT_V1_PHASE + ); + + let rest = outer.tail().as_cell().expect("rest tuple"); + let bythos_phase_atom = rest.head().as_atom().expect("bythos-phase should be atom"); + assert_eq!( + bythos_phase_atom.as_u64().expect("bythos-phase as u64"), + BlockchainConstants::DEFAULT_BYTHOS_PHASE + ); + + let rest = rest.tail().as_cell().expect("note-data and rest tuple"); + let note_data = rest.head().as_cell().expect("note-data tuple"); + let note_data_max_size = note_data + .head() + .as_atom() + .expect("note-data max-size atom") + .as_u64() + .expect("note-data max-size as u64"); + let note_data_min_fee = note_data + .tail() + .as_atom() + .expect("note-data min-fee atom") + .as_u64() + .expect("note-data min-fee as u64"); + assert_eq!( + note_data_max_size, + BlockchainConstants::DEFAULT_NOTE_DATA_MAX_SIZE + ); + assert_eq!( + note_data_min_fee, + BlockchainConstants::DEFAULT_NOTE_DATA_MIN_FEE + ); + + let base_fee_and_rest = rest.tail().as_cell().expect("base-fee and rest tuple"); + let base_fee_atom = base_fee_and_rest.head().as_atom().expect("base-fee atom"); + assert_eq!( + base_fee_atom.as_u64().expect("base-fee as u64"), + BlockchainConstants::DEFAULT_BASE_FEE + ); + + let input_fee_divisor_and_rest = base_fee_and_rest + .tail() + .as_cell() + .expect("input-fee-divisor and rest tuple"); + let input_fee_divisor_atom = input_fee_divisor_and_rest + .head() + .as_atom() + .expect("input-fee-divisor atom"); + assert_eq!( + input_fee_divisor_atom + .as_u64() + .expect("input-fee-divisor as u64"), + BlockchainConstants::DEFAULT_INPUT_FEE_DIVISOR + ); + + let v0_and_rest = input_fee_divisor_and_rest + .tail() + .as_cell() + .expect("v0 constants and asert tail tuple"); + let v0_constants = v0_and_rest.head().noun(); + assert_eq!( + tuple_len(v0_constants, &space), + 13, + "v0 constants should be a 13-tuple" + ); + let v0_cell = v0_constants + .in_space(&space) + .as_cell() + .expect("v0 constants tuple"); + let max_block_size_atom = v0_cell.head().as_atom().expect("max-block-size atom"); + assert_eq!( + max_block_size_atom.as_u64().expect("max-block-size as u64"), + BlockchainConstants::DEFAULT_MAX_BLOCK_SIZE + ); + + let asert_phase_and_rest = v0_and_rest + .tail() + .as_cell() + .expect("asert-phase and rest tuple"); + let asert_phase_atom = asert_phase_and_rest + .head() + .as_atom() + .expect("asert-phase atom"); + assert_eq!( + asert_phase_atom.as_u64().expect("asert-phase as u64"), + BlockchainConstants::DEFAULT_ASERT_PHASE + ); + + let asert_anchor_height_and_rest = asert_phase_and_rest + .tail() + .as_cell() + .expect("asert-anchor-height and rest tuple"); + let asert_anchor_height_atom = asert_anchor_height_and_rest + .head() + .as_atom() + .expect("asert-anchor-height atom"); + assert_eq!( + asert_anchor_height_atom + .as_u64() + .expect("asert-anchor-height as u64"), + BlockchainConstants::DEFAULT_ASERT_ANCHOR_HEIGHT + ); + + let asert_anchor_target_and_rest = asert_anchor_height_and_rest + .tail() + .as_cell() + .expect("asert-anchor-target and rest tuple"); + let _asert_anchor_target_atom = asert_anchor_target_and_rest + .head() + .as_atom() + .expect("asert-anchor-target atom"); + + let asert_ideal_and_rest = asert_anchor_target_and_rest + .tail() + .as_cell() + .expect("asert-ideal-block-time and rest tuple"); + let asert_ideal_atom = asert_ideal_and_rest + .head() + .as_atom() + .expect("asert-ideal-block-time atom"); + assert_eq!( + asert_ideal_atom + .as_u64() + .expect("asert-ideal-block-time as u64"), + BlockchainConstants::DEFAULT_ASERT_IDEAL_BLOCK_TIME + ); + + let asert_half_life_and_rest = asert_ideal_and_rest + .tail() + .as_cell() + .expect("asert-half-life and rest tuple"); + let asert_half_life_atom = asert_half_life_and_rest + .head() + .as_atom() + .expect("asert-half-life atom"); + assert_eq!( + asert_half_life_atom + .as_u64() + .expect("asert-half-life as u64"), + BlockchainConstants::DEFAULT_ASERT_HALF_LIFE + ); + + let asert_anchor_min_timestamp_atom = asert_half_life_and_rest + .tail() + .as_atom() + .expect("asert-anchor-min-timestamp atom"); + assert_eq!( + asert_anchor_min_timestamp_atom + .as_u64() + .expect("asert-anchor-min-timestamp as u64"), + BlockchainConstants::DEFAULT_ASERT_ANCHOR_MIN_TIMESTAMP + ); + } +} diff --git a/crates/nockchain-types/src/eth.rs b/crates/nockchain-types/src/eth.rs index 75f8305ac..5ad158560 100644 --- a/crates/nockchain-types/src/eth.rs +++ b/crates/nockchain-types/src/eth.rs @@ -1,6 +1,6 @@ use alloy::primitives::Address as AlloyAddress; use hex::FromHex; -use nockvm::noun::{Atom, IndirectAtom, Noun, NounAllocator}; +use nockvm::noun::{Atom, IndirectAtom, Noun, NounAllocator, NounSpace}; use noun_serde::{NounDecode, NounDecodeError, NounEncode}; use thiserror::Error; @@ -98,15 +98,17 @@ impl NounEncode for EthAddress { unsafe { let mut atom = IndirectAtom::new_raw_bytes(allocator, trimmed_len, le_bytes.as_ptr()); - atom.normalize_as_atom().as_noun() + let space = allocator.noun_space(); + atom.normalize_as_atom(&space).as_noun() } } } impl NounDecode for EthAddress { - fn from_noun(noun: &Noun) -> Result { + fn from_noun(noun: &Noun, space: &NounSpace) -> Result { let atom = noun.as_atom().map_err(|_| NounDecodeError::ExpectedAtom)?; - let bytes = atom.as_ne_bytes(); + let atom_handle = atom.in_space(space); + let bytes = atom_handle.as_ne_bytes(); let mut buf = [0u8; EthAddress::LEN]; for (i, byte) in bytes.iter().enumerate() { if i < EthAddress::LEN { @@ -157,7 +159,8 @@ mod tests { let mut slab = NounSlab::::new(); let addr = EthAddress::from([0x11; EthAddress::LEN]); let noun = addr.to_noun(&mut slab); - let decoded = EthAddress::from_noun(&noun).expect("decode"); + let space = slab.noun_space(); + let decoded = EthAddress::from_noun(&noun, &space).expect("decode"); assert_eq!(decoded, addr); } @@ -173,8 +176,9 @@ mod tests { let atom = noun .as_atom() .expect("expected EthAddress noun to be an atom"); + let space = slab.noun_space(); let be_bytes = { - let mut bytes = atom.to_be_bytes(); + let mut bytes = atom.in_space(&space).to_be_bytes(); if bytes.len() < EthAddress::LEN { let mut padded = vec![0u8; EthAddress::LEN]; padded[EthAddress::LEN - bytes.len()..].copy_from_slice(&bytes); @@ -198,7 +202,8 @@ mod tests { let value = UBig::from_be_bytes(addr.as_slice()); let mut slab = NounSlab::::new(); let noun = Atom::from_ubig(&mut slab, &value).as_noun(); - let decoded = EthAddress::from_noun(&noun).expect("decode"); + let space = slab.noun_space(); + let decoded = EthAddress::from_noun(&noun, &space).expect("decode"); assert_eq!(decoded, addr); } diff --git a/crates/nockchain-types/src/lib.rs b/crates/nockchain-types/src/lib.rs index b6bcdd3a6..b1d8ad442 100644 --- a/crates/nockchain-types/src/lib.rs +++ b/crates/nockchain-types/src/lib.rs @@ -1,5 +1,7 @@ +pub mod blockchain_constants; pub mod eth; pub mod tx_engine; +pub use blockchain_constants::*; pub use eth::*; pub use tx_engine::*; diff --git a/crates/nockchain-types/src/tx_engine/common/mod.rs b/crates/nockchain-types/src/tx_engine/common/mod.rs index c1723b776..3cd72f4d8 100644 --- a/crates/nockchain-types/src/tx_engine/common/mod.rs +++ b/crates/nockchain-types/src/tx_engine/common/mod.rs @@ -3,10 +3,9 @@ pub mod page; use anyhow::Result; use nockchain_math::belt::{Belt, PRIME}; use nockchain_math::crypto::cheetah::{CheetahError, CheetahPoint}; -use nockchain_math::noun_ext::NounMathExt; -use nockchain_math::zoon::common::DefaultTipHasher; -use nockchain_math::zoon::zmap; -use nockvm::noun::{Noun, NounAllocator, D}; +use nockchain_math::noun_ext::NounMathExtHandle; +use nockchain_math::zoon::zmap::ZMap; +use nockvm::noun::{Noun, NounAllocator, NounSpace, D}; use noun_serde::{NounDecode, NounDecodeError, NounEncode}; use num_bigint::BigUint; pub use page::{BigNum, BlockId, CoinbaseSplit, Page, PageMsg}; @@ -38,32 +37,29 @@ pub struct Signature(pub Vec<(SchnorrPubkey, SchnorrSignature)>); impl NounEncode for Signature { fn to_noun(&self, stack: &mut A) -> Noun { - self.0.iter().fold(D(0), |map, (pubkey, sig)| { - let mut key = pubkey.to_noun(stack); - let mut value = sig.to_noun(stack); - zmap::z_map_put(stack, &map, &mut key, &mut value, &DefaultTipHasher) - .expect("z-map put for signature should not fail") - }) + ZMap::try_from_entries(self.0.clone()) + .expect("signature z-map should encode") + .to_noun(stack) } } impl NounDecode for Signature { - fn from_noun(noun: &Noun) -> Result { - if let Ok(atom) = noun.as_atom() { + fn from_noun(noun: &Noun, space: &NounSpace) -> Result { + if let Ok(atom) = noun.in_space(space).as_atom() { if atom.as_u64()? == 0 { return Ok(Signature(Vec::new())); } return Err(NounDecodeError::Custom("signature node not a cell".into())); } - let entries = nockchain_math::structs::HoonMapIter::from(*noun) + let entries = nockchain_math::structs::HoonMapIter::new(&noun.in_space(space)) .filter(|entry| entry.is_cell()) .map(|entry| { let [key, value] = entry .uncell() .map_err(|_| NounDecodeError::Custom("signature entry not a pair".into()))?; - let pubkey = SchnorrPubkey::from_noun(&key)?; - let signature = SchnorrSignature::from_noun(&value)?; + let pubkey = SchnorrPubkey::from_noun_handle(&key)?; + let signature = SchnorrSignature::from_noun_handle(&value)?; Ok((pubkey, signature)) }) .collect::, NounDecodeError>>()?; @@ -120,7 +116,7 @@ impl NounEncode for Version { } impl NounDecode for Version { - fn from_noun(noun: &Noun) -> Result { + fn from_noun(noun: &Noun, _space: &NounSpace) -> Result { match noun.as_atom()?.as_direct() { Ok(ver) if ver.data() == 0 => Ok(Version::V0), Ok(ver) if ver.data() == 1 => Ok(Version::V1), @@ -247,6 +243,44 @@ impl Hash { } } +/// Canonical wrapper for a v1 note first-name digest. +#[derive(Debug, Clone, PartialEq, Eq, Hash, NounDecode, NounEncode, Serialize, Deserialize)] +pub struct FirstName(pub Hash); + +impl FirstName { + pub fn as_hash(&self) -> &Hash { + &self.0 + } + + pub fn into_hash(self) -> Hash { + self.0 + } + + pub fn to_base58(&self) -> String { + self.0.to_base58() + } + + pub fn from_base58(s: &str) -> Result { + Ok(Self(Hash::from_base58(s)?)) + } + + pub fn to_array(&self) -> [u64; 5] { + self.0.to_array() + } +} + +impl From for FirstName { + fn from(value: Hash) -> Self { + Self(value) + } +} + +impl From for Hash { + fn from(value: FirstName) -> Self { + value.0 + } +} + /// Peek response for the heaviest block ID. /// Wraps `(unit (unit Hash))` - the Hoon peek response encoding. #[derive(Debug, Clone, PartialEq, Eq, NounDecode, NounEncode)] diff --git a/crates/nockchain-types/src/tx_engine/common/page.rs b/crates/nockchain-types/src/tx_engine/common/page.rs index 68f475f62..60a132fd3 100644 --- a/crates/nockchain-types/src/tx_engine/common/page.rs +++ b/crates/nockchain-types/src/tx_engine/common/page.rs @@ -1,5 +1,5 @@ use nockvm::ext::AtomExt; -use nockvm::noun::{Noun, NounAllocator}; +use nockvm::noun::{Noun, NounAllocator, NounHandle, NounSpace}; use noun_serde::{NounDecode, NounDecodeError, NounEncode}; use num_bigint::BigUint; @@ -9,14 +9,14 @@ pub type BlockId = Hash; /// Decode a z-set into a Vec /// z-set structure: either `~` (atom 0) for empty, or `[n=item l=tree r=tree]` -fn decode_zset(noun: &Noun) -> Result, NounDecodeError> { +fn decode_zset(noun: &NounHandle) -> Result, NounDecodeError> { let mut result = Vec::new(); collect_zset_items(noun, &mut result)?; Ok(result) } fn collect_zset_items( - noun: &Noun, + noun: &NounHandle, result: &mut Vec, ) -> Result<(), NounDecodeError> { // Empty set is atom 0 @@ -41,7 +41,7 @@ fn collect_zset_items( collect_zset_items(&l, result)?; // Add the node item - let item = T::from_noun(&n)?; + let item = T::from_noun_handle(&n)?; result.push(item); // Recursively collect from right subtree @@ -70,17 +70,17 @@ impl NounEncode for BigNum { } impl NounDecode for BigNum { - fn from_noun(noun: &Noun) -> Result { + fn from_noun(noun: &Noun, space: &NounSpace) -> Result { // Bignum in Hoon is [%bn p=(list u32)] - a tagged cell with u32 chunks if let Ok(cell) = noun.as_cell() { // Check for %bn tag - if let Ok(tag) = cell.head().as_atom() { + if let Ok(tag) = cell.in_space(space).head().as_atom() { // %bn = 0x6e62 = 28258 in little-endian ('bn' as cord) let tag_val = tag.as_u64().unwrap_or(u64::MAX); if tag_val == 28258 { // Decode tail as list of u32 chunks (LSB first) let mut chunks: Vec = Vec::new(); - let mut current = cell.tail(); + let mut current = cell.in_space(space).tail(); while let Ok(list_cell) = current.as_cell() { let chunk = list_cell .head() @@ -126,7 +126,8 @@ impl NounDecode for BigNum { let atom = noun.as_atom().map_err(|_| { NounDecodeError::Custom("BigNum: expected atom or [%bn list] cell".into()) })?; - let bytes = atom.as_ne_bytes(); + let atom_handle = atom.in_space(space); + let bytes = atom_handle.as_ne_bytes(); let biguint = BigUint::from_bytes_le(bytes); Ok(BigNum(biguint)) } @@ -159,15 +160,15 @@ impl NounEncode for CoinbaseSplit { } impl NounDecode for CoinbaseSplit { - fn from_noun(noun: &Noun) -> Result { + fn from_noun(noun: &Noun, space: &NounSpace) -> Result { // Check if it's a tagged cell [%0 data] or [%1 data] if let Ok(cell) = noun.as_cell() { - if let Ok(tag_atom) = cell.head().as_atom() { + if let Ok(tag_atom) = cell.in_space(space).head().as_atom() { if let Ok(tag) = tag_atom.as_u64() { match tag { 0 => { // V0: [%0 byte-list] - let bytes = Vec::::from_noun(&cell.tail())?; + let bytes = Vec::::from_noun_handle(&cell.in_space(space).tail())?; return Ok(CoinbaseSplit::V0(bytes)); } 1 => { @@ -180,7 +181,7 @@ impl NounDecode for CoinbaseSplit { } } // Fallback: try to decode as raw byte list (untagged v0) - if let Ok(bytes) = Vec::::from_noun(noun) { + if let Ok(bytes) = Vec::::from_noun(noun, space) { return Ok(CoinbaseSplit::V0(bytes)); } // If all else fails, treat as v1 (unknown structure) @@ -239,16 +240,17 @@ impl NounEncode for Page { impl NounDecode for Page { // TODO: Purge these custom Page NounDecode/NounEncode implementations in favor of // the standard noun-serde path once it can represent this shape directly. - fn from_noun(noun: &Noun) -> Result { + fn from_noun(noun: &Noun, space: &NounSpace) -> Result { let cell = noun .as_cell() .map_err(|_| NounDecodeError::Custom("Page: expected cell at root".into()))?; // Check if this is a v1 page (head is atom %1) or v0 page (head is cell/digest) let (digest, rest_after_digest) = - if cell.head().is_atom() { + if cell.in_space(space).head().is_atom() { // v1 page: [%1 digest pow parent ...] let version = cell + .in_space(space) .head() .as_atom() .map_err(|_| NounDecodeError::Custom("Page: version tag not atom".into()))? @@ -261,10 +263,10 @@ impl NounDecode for Page { ))); } // Skip version tag, get digest from tail - let rest = cell.tail().as_cell().map_err(|_| { + let rest = cell.in_space(space).tail().as_cell().map_err(|_| { NounDecodeError::Custom("Page: expected cell after version".into()) })?; - let digest = BlockId::from_noun(&rest.head()) + let digest = BlockId::from_noun_handle(&rest.head()) .map_err(|e| NounDecodeError::Custom(format!("Page.digest: {}", e)))?; let rest_after = rest.tail().as_cell().map_err(|_| { NounDecodeError::Custom("Page: expected cell after digest".into()) @@ -272,9 +274,9 @@ impl NounDecode for Page { (digest, rest_after) } else { // v0 page: [digest pow parent ...] - let digest = BlockId::from_noun(&cell.head()) + let digest = BlockId::from_noun_handle(&cell.in_space(space).head()) .map_err(|e| NounDecodeError::Custom(format!("Page.digest(v0): {}", e)))?; - let rest_after = cell.tail().as_cell().map_err(|_| { + let rest_after = cell.in_space(space).tail().as_cell().map_err(|_| { NounDecodeError::Custom("Page: expected cell after digest(v0)".into()) })?; (digest, rest_after) @@ -302,7 +304,7 @@ impl NounDecode for Page { .as_cell() .map_err(|_| NounDecodeError::Custom("Page: expected cell after pow".into()))?; - let parent = BlockId::from_noun(&rest.head()) + let parent = BlockId::from_noun_handle(&rest.head()) .map_err(|e| NounDecodeError::Custom(format!("Page.parent: {}", e)))?; let rest = rest @@ -318,7 +320,7 @@ impl NounDecode for Page { .as_cell() .map_err(|_| NounDecodeError::Custom("Page: expected cell after tx_ids".into()))?; - let coinbase = CoinbaseSplit::from_noun(&rest.head()) + let coinbase = CoinbaseSplit::from_noun_handle(&rest.head()) .map_err(|e| NounDecodeError::Custom(format!("Page.coinbase: {}", e)))?; let rest = rest @@ -349,7 +351,7 @@ impl NounDecode for Page { NounDecodeError::Custom("Page: expected cell after epoch_counter".into()) })?; - let target = BigNum::from_noun(&rest.head()) + let target = BigNum::from_noun_handle(&rest.head()) .map_err(|e| NounDecodeError::Custom(format!("Page.target: {}", e)))?; let rest = rest @@ -357,7 +359,7 @@ impl NounDecode for Page { .as_cell() .map_err(|_| NounDecodeError::Custom("Page: expected cell after target".into()))?; - let accumulated_work = BigNum::from_noun(&rest.head()) + let accumulated_work = BigNum::from_noun(&rest.head().noun(), space) .map_err(|e| NounDecodeError::Custom(format!("Page.accumulated_work: {}", e)))?; let rest = rest.tail().as_cell().map_err(|_| { @@ -371,7 +373,7 @@ impl NounDecode for Page { .as_u64() .map_err(|_| NounDecodeError::Custom("Page.height: too large".into()))?; - let msg = PageMsg::from_noun(&rest.tail()) + let msg = PageMsg::from_noun_handle(&rest.tail()) .map_err(|e| NounDecodeError::Custom(format!("Page.msg: {}", e)))?; Ok(Page { diff --git a/crates/nockchain-types/src/tx_engine/v0/note.rs b/crates/nockchain-types/src/tx_engine/v0/note.rs index 6bc1e34ce..26a97d184 100644 --- a/crates/nockchain-types/src/tx_engine/v0/note.rs +++ b/crates/nockchain-types/src/tx_engine/v0/note.rs @@ -2,11 +2,11 @@ use std::collections::HashSet; use anyhow::Result; use nockapp::Noun; -use nockchain_math::noun_ext::NounMathExt; +use nockchain_math::noun_ext::NounMathExtHandle; use nockchain_math::structs::HoonMapIter; -use nockchain_math::zoon::common::DefaultTipHasher; -use nockchain_math::zoon::{zmap, zset}; -use nockvm::noun::{NounAllocator, D, SIG}; +use nockchain_math::zoon::zmap::ZMap; +use nockchain_math::zoon::zset::ZSet; +use nockvm::noun::{NounAllocator, NounSpace, SIG}; use noun_serde::{NounDecode, NounDecodeError, NounEncode}; use crate::tx_engine::common::{ @@ -46,29 +46,29 @@ pub struct Lock { impl NounEncode for Lock { fn to_noun(&self, stack: &mut A) -> Noun { let m = u64::to_noun(&self.keys_required, stack); - let keys_noun_map = self - .pubkeys - .iter() - .fold(SIG, |map, pubkey: &SchnorrPubkey| { - let mut val = pubkey.to_noun(stack); - zset::z_set_put(stack, &map, &mut val, &DefaultTipHasher) - .expect("Failed to put public key into set") - }); + let keys_noun_map = if self.pubkeys.is_empty() { + SIG + } else { + ZSet::try_from_items(self.pubkeys.clone()) + .expect("lock pubkey set should encode") + .to_noun(stack) + }; nockvm::noun::T(stack, &[m, keys_noun_map]) } } impl NounDecode for Lock { - fn from_noun(noun: &Noun) -> Result { - let cell = noun.as_cell()?; + fn from_noun(noun: &Noun, space: &NounSpace) -> Result { + let cell = noun.in_space(space).as_cell()?; let keys_required = cell.head().as_atom()?.as_u64()?; // It is called HoonMapIter, but it can be used for sets as well - let pubkeys_iter = HoonMapIter::from(cell.tail()); + let tail = cell.tail(); + let pubkeys_iter = HoonMapIter::new(&tail); let mut pubkeys = Vec::new(); for pubkey in pubkeys_iter { - let schnorr = SchnorrPubkey::from_noun(&pubkey)?; + let schnorr = SchnorrPubkey::from_noun_handle(&pubkey)?; pubkeys.push(schnorr); } @@ -121,23 +121,21 @@ pub struct Balance(pub Vec<(Name, NoteV0)>); impl NounEncode for Balance { fn to_noun(&self, stack: &mut A) -> Noun { - self.0.iter().fold(D(0), |map, (name, note)| { - let mut key = name.to_noun(stack); - let mut value = note.to_noun(stack); - zmap::z_map_put(stack, &map, &mut key, &mut value, &DefaultTipHasher) - .expect("Failed to put into z_map") - }) + ZMap::try_from_entries(self.0.clone()) + .expect("balance z-map should encode") + .to_noun(stack) } } impl NounDecode for Balance { - fn from_noun(noun: &Noun) -> Result { - let notes = HoonMapIter::from(*noun) + fn from_noun(noun: &Noun, space: &NounSpace) -> Result { + let noun_handle = noun.in_space(space); + let notes = HoonMapIter::new(&noun_handle) .filter(|kv| kv.is_cell()) .map(|kv| { let [k, v] = kv.uncell()?; - let name = Name::from_noun(&k)?; - let note = NoteV0::from_noun(&v)?; + let name = Name::from_noun_handle(&k)?; + let note = NoteV0::from_noun_handle(&v)?; Ok((name, note)) }) .collect::, NounDecodeError>>()?; @@ -189,8 +187,10 @@ mod test { fn try_path(jam: &str) -> Result> { let possible_paths = [ std::path::Path::new(env!("CARGO_MANIFEST_DIR")) - .join("jams/v0") + .join("jams") + .join("v0") .join(jam), + std::path::Path::new("open/crates/nockchain-types/jams").join(jam), std::path::Path::new("open/crates/nockchain-types/jams/v0").join(jam), ]; @@ -206,7 +206,8 @@ mod test { let balance_jam = try_path("balance.jam")?; let mut slab: NounSlab = NounSlab::new(); let mut balance_noun = slab.cue_into(balance_jam)?; - let balance = Balance::from_noun(&balance_noun)?; + let space = slab.noun_space(); + let balance = Balance::from_noun(&balance_noun, &space)?; let mut balance_noun_from_struct = Balance::to_noun(&balance, &mut slab); unsafe { slab.equals(&mut balance_noun, &mut balance_noun_from_struct) }; Ok(()) @@ -217,7 +218,8 @@ mod test { let note_jam = try_path("note.jam")?; let mut slab: NounSlab = NounSlab::new(); let mut note_noun = slab.cue_into(note_jam)?; - let note = NoteV0::from_noun(¬e_noun)?; + let space = slab.noun_space(); + let note = NoteV0::from_noun(¬e_noun, &space)?; let mut note_noun_from_struct = NoteV0::to_noun(¬e, &mut slab); assert!(unsafe { slab.equals(&mut note_noun, &mut note_noun_from_struct) }); //eprintln!("{:?}", utxo); @@ -229,7 +231,8 @@ mod test { let timelock_jam = try_path("timelock.jam")?; let mut slab: NounSlab = NounSlab::new(); let timelock_noun = slab.cue_into(timelock_jam)?; - let _ = >::from_noun(&timelock_noun); + let space = slab.noun_space(); + let _ = >::from_noun(&timelock_noun, &space); Ok(()) } @@ -245,7 +248,8 @@ mod test { let mut slab: NounSlab = NounSlab::new(); let tl = Timelock(None); let mut n1 = Timelock::to_noun(&tl, &mut slab); - let tl2 = Timelock::from_noun(&n1).expect("decode"); + let space = slab.noun_space(); + let tl2 = Timelock::from_noun(&n1, &space).expect("decode"); let mut n2 = Timelock::to_noun(&tl2, &mut slab); assert!(unsafe { slab.equals(&mut n1, &mut n2) }); } @@ -258,7 +262,8 @@ mod test { relative: TimelockRangeRelative::none(), })); let mut n1 = Timelock::to_noun(&tl, &mut slab); - let tl2 = Timelock::from_noun(&n1).expect("decode"); + let space = slab.noun_space(); + let tl2 = Timelock::from_noun(&n1, &space).expect("decode"); let mut n2 = Timelock::to_noun(&tl2, &mut slab); assert!(unsafe { slab.equals(&mut n1, &mut n2) }); } @@ -291,7 +296,8 @@ mod test { relative: TimelockRangeRelative::new(Some(dh(5)), Some(dh(50))), })); let mut n1 = Timelock::to_noun(&tl, &mut slab); - let tl2 = Timelock::from_noun(&n1).expect("decode"); + let space = slab.noun_space(); + let tl2 = Timelock::from_noun(&n1, &space).expect("decode"); let mut n2 = Timelock::to_noun(&tl2, &mut slab); assert!(unsafe { slab.equals(&mut n1, &mut n2) }); } @@ -304,7 +310,8 @@ mod test { relative: TimelockRangeRelative::new(Some(dh(1)), Some(dh(2))), })); let mut n1 = Timelock::to_noun(&tl, &mut slab); - let tl2 = Timelock::from_noun(&n1).expect("decode"); + let space = slab.noun_space(); + let tl2 = Timelock::from_noun(&n1, &space).expect("decode"); let mut n2 = Timelock::to_noun(&tl2, &mut slab); assert!(unsafe { slab.equals(&mut n1, &mut n2) }); } @@ -489,7 +496,8 @@ mod test { fn prop(update: BalanceUpdate) -> bool { let mut slab: NounSlab = NounSlab::new(); let mut n1 = BalanceUpdate::to_noun(&update, &mut slab); - let decoded = match BalanceUpdate::from_noun(&n1) { + let space = slab.noun_space(); + let decoded = match BalanceUpdate::from_noun(&n1, &space) { Ok(v) => v, Err(_) => return false, }; diff --git a/crates/nockchain-types/src/tx_engine/v0/tx.rs b/crates/nockchain-types/src/tx_engine/v0/tx.rs index ed283e5c5..44afc74fb 100644 --- a/crates/nockchain-types/src/tx_engine/v0/tx.rs +++ b/crates/nockchain-types/src/tx_engine/v0/tx.rs @@ -1,8 +1,8 @@ -use nockchain_math::noun_ext::NounMathExt; +use nockchain_math::noun_ext::NounMathExtHandle; use nockchain_math::structs::HoonMapIter; -use nockchain_math::zoon::common::DefaultTipHasher; -use nockchain_math::zoon::{zmap, zset}; -use nockvm::noun::{Noun, NounAllocator, D}; +use nockchain_math::zoon::zmap::ZMap; +use nockchain_math::zoon::zset::ZSet; +use nockvm::noun::{Noun, NounAllocator, NounSpace}; use noun_serde::{NounDecode, NounDecodeError, NounEncode}; use super::note::{Lock, NoteV0, TimelockIntent}; @@ -65,25 +65,22 @@ pub struct Inputs(pub Vec<(Name, Input)>); impl NounEncode for Inputs { fn to_noun(&self, stack: &mut A) -> Noun { - self.0.iter().fold(D(0), |map, (name, input)| { - let mut key = name.to_noun(stack); - let mut value = input.to_noun(stack); - zmap::z_map_put(stack, &map, &mut key, &mut value, &DefaultTipHasher) - .expect("Failed to put into z_map") - }) + ZMap::try_from_entries(self.0.clone()) + .expect("inputs z-map should encode") + .to_noun(stack) } } impl NounDecode for Inputs { - fn from_noun(noun: &Noun) -> Result { - let entries = HoonMapIter::from(*noun) + fn from_noun(noun: &Noun, space: &NounSpace) -> Result { + let entries = HoonMapIter::new(&noun.in_space(space)) .filter(|entry| entry.is_cell()) .map(|entry| { let [key, value] = entry .uncell() .map_err(|_| NounDecodeError::Custom("input entry not a pair".into()))?; - let name = Name::from_noun(&key)?; - let input = Input::from_noun(&value)?; + let name = Name::from_noun_handle(&key)?; + let input = Input::from_noun_handle(&value)?; Ok((name, input)) }) .collect::, NounDecodeError>>()?; @@ -110,19 +107,23 @@ impl NounEncode for RawTx { } impl NounDecode for RawTx { - fn from_noun(noun: &Noun) -> Result { - let cell = noun.as_cell()?; - let id = TxId::from_noun(&cell.head())?; + fn from_noun(noun: &Noun, space: &NounSpace) -> Result { + let cell = noun.in_space(space).as_cell()?; + let id_noun = cell.head().noun(); + let id = TxId::from_noun(&id_noun, space)?; let tail = cell.tail(); let cell = tail.as_cell()?; - let inputs = Inputs::from_noun(&cell.head())?; + let inputs_noun = cell.head().noun(); + let inputs = Inputs::from_noun(&inputs_noun, space)?; let tail = cell.tail(); let cell = tail.as_cell()?; - let timelock_range = TimelockRangeAbsolute::from_noun(&cell.head())?; + let range_noun = cell.head().noun(); + let timelock_range = TimelockRangeAbsolute::from_noun(&range_noun, space)?; - let total_fees = Nicks::from_noun(&cell.tail())?; + let fees_noun = cell.tail().noun(); + let total_fees = Nicks::from_noun(&fees_noun, space)?; Ok(Self { id, @@ -149,18 +150,20 @@ pub struct Seed { impl NounEncode for Seeds { fn to_noun(&self, stack: &mut A) -> Noun { - self.seeds.iter().fold(D(0), |set, seed| { - let mut value = seed.to_noun(stack); - zset::z_set_put(stack, &set, &mut value, &DefaultTipHasher) - .expect("z-set put for seeds should not fail") - }) + ZSet::try_from_items(self.seeds.clone()) + .expect("seed z-set should encode") + .to_noun(stack) } } impl NounDecode for Seeds { - fn from_noun(noun: &Noun) -> Result { - fn traverse(node: &Noun, acc: &mut Vec) -> Result<(), NounDecodeError> { - if let Ok(atom) = node.as_atom() { + fn from_noun(noun: &Noun, space: &NounSpace) -> Result { + fn traverse( + node: &Noun, + space: &NounSpace, + acc: &mut Vec, + ) -> Result<(), NounDecodeError> { + if let Ok(atom) = node.in_space(space).as_atom() { if atom.as_u64()? == 0 { return Ok(()); } @@ -168,22 +171,26 @@ impl NounDecode for Seeds { } let cell = node + .in_space(space) .as_cell() .map_err(|_| NounDecodeError::Custom("seed node not a cell".into()))?; - let seed = Seed::from_noun(&cell.head())?; + let seed_noun = cell.head().noun(); + let seed = Seed::from_noun(&seed_noun, space)?; acc.push(seed); let branches = cell .tail() .as_cell() .map_err(|_| NounDecodeError::Custom("seed branches not a cell".into()))?; - traverse(&branches.head(), acc)?; - traverse(&branches.tail(), acc)?; + let left = branches.head().noun(); + let right = branches.tail().noun(); + traverse(&left, space, acc)?; + traverse(&right, space, acc)?; Ok(()) } let mut seeds = Vec::new(); - traverse(noun, &mut seeds)?; + traverse(noun, space, &mut seeds)?; Ok(Seeds { seeds }) } } @@ -199,12 +206,15 @@ impl NounEncode for Spend { } impl NounDecode for Spend { - fn from_noun(noun: &Noun) -> Result { - let cell = noun.as_cell()?; - let signature = Option::::from_noun(&cell.head())?; + fn from_noun(noun: &Noun, space: &NounSpace) -> Result { + let cell = noun.in_space(space).as_cell()?; + let sig_noun = cell.head().noun(); + let signature = Option::::from_noun(&sig_noun, space)?; let inner = cell.tail().as_cell()?; - let seeds = Seeds::from_noun(&inner.head())?; - let fee = Nicks::from_noun(&inner.tail())?; + let seeds_noun = inner.head().noun(); + let fee_noun = inner.tail().noun(); + let seeds = Seeds::from_noun(&seeds_noun, space)?; + let fee = Nicks::from_noun(&fee_noun, space)?; Ok(Spend { signature, diff --git a/crates/nockchain-types/src/tx_engine/v1/hashable.rs b/crates/nockchain-types/src/tx_engine/v1/hashable.rs new file mode 100644 index 000000000..12dd8959e --- /dev/null +++ b/crates/nockchain-types/src/tx_engine/v1/hashable.rs @@ -0,0 +1,419 @@ +use nockchain_math::belt::Belt; +use nockchain_math::owned_based_noun::{hash_owned_based_noun_varlen, OwnedBasedNoun}; +use nockchain_math::tip5; +use nockchain_math::zoon::zset::ZSetHasher; + +use crate::tx_engine::common::Hash; + +mod noun; +pub use noun::{bpoly_to_list, hash_hashable}; + +/// Errors raised by the tip5 hash-hashable jet wrapper. +#[derive(Debug, thiserror::Error)] +pub enum HashableJetError { + #[error("hash-hashable jet failed: {0}")] + Jet(String), + #[error("hash-hashable digest decode failed: {0}")] + DigestDecode(String), +} + +/// Errors raised while encoding direct hashable digests. +#[derive(Debug, thiserror::Error)] +pub enum HashableEncodingError { + #[error("leaf atom is not based: {0}")] + LeafAtomNotBased(u64), +} + +/// Computes the consensus hashable digest directly, without materializing an +/// allocator-backed noun tree first. +pub trait Hashable { + type Error; + + fn hash_digest(&self) -> Result; +} + +/// Generic digest entrypoint for all handwritten `Hashable` values. +pub fn hash_hashable_value(value: &T) -> Result<[u64; 5], T::Error> +where + T: Hashable, +{ + Ok(value.hash_digest()?.to_array()) +} + +/// Hashes a `%leaf` whose payload is a single based atom. +pub fn hash_leaf_atom(atom: u64) -> Result { + let belt = Belt::try_from(&atom).map_err(|_| HashableEncodingError::LeafAtomNotBased(atom))?; + Ok(hash_leaf_belt(belt)) +} + +/// Hashes a `%leaf` whose payload is a single based field element. +/// This is the allocator-free closed form of `hash_noun_varlen` for an atom: +/// `leaf-sequence(atom) = [atom]`, `dyck(atom) = ~`, so the hashed belt list is +/// exactly `[1 atom]`. +pub fn hash_leaf_belt(belt: Belt) -> Hash { + let mut input = vec![Belt(1), belt]; + Hash::from_limbs(&tip5::hash::hash_varlen(&mut input)) +} + +/// Hashes the canonical null leaf `%leaf ~`. +pub fn hash_leaf_null() -> Hash { + hash_leaf_belt(Belt(0)) +} + +/// Hashes a generic hashable pair by hashing the 10 child digest limbs. +pub fn hash_pair(left: &Hash, right: &Hash) -> Hash { + let mut input = left + .to_array() + .into_iter() + .chain(right.to_array()) + .map(Belt) + .collect::>(); + Hash::from_limbs(&tip5::hash::hash_10(&mut input)) +} + +/// Hashes a hashable `(unit noun)` where the payload is a based atom. +pub fn hash_unit_belt(value: Option) -> Hash { + match value { + None => hash_leaf_null(), + Some(value) => { + let none_leaf = hash_leaf_null(); + let value_leaf = hash_leaf_belt(value); + hash_pair(&none_leaf, &value_leaf) + } + } +} + +/// Hashes a `%list` payload from its already-hashed items. +pub fn hash_list(items: &[Hash]) -> Hash { + let noun = OwnedBasedNoun::list( + items + .iter() + .map(|item| { + OwnedBasedNoun::tuple_atoms(&item.to_array()) + .expect("hash digest limbs are always valid based atoms") + }) + .collect(), + ); + Hash::from_limbs(&hash_owned_based_noun_varlen(&noun)) +} + +/// Hashes a `%mary` payload from its step, original array length, and +/// step-normalized belt payload. +pub fn hash_mary( + step: u64, + array_len: u64, + normalized_belts: &[Belt], +) -> Result { + let step_hash = hash_leaf_atom(step)?; + let len_hash = hash_leaf_atom(array_len)?; + let mut belt_list = normalized_belts.to_vec(); + let belts_hash = Hash::from_limbs(&tip5::hash::hash_varlen(&mut belt_list)); + Ok(hash_pair(&step_hash, &hash_pair(&len_hash, &belts_hash))) +} + +#[derive(Debug, Clone, Copy, Default)] +pub struct HashableTreeHasher; + +impl ZSetHasher for HashableTreeHasher { + type Output = Hash; + + fn empty(&self) -> Self::Output { + hash_leaf_null() + } + + fn node(&self, digest: &Hash, left: Self::Output, right: Self::Output) -> Self::Output { + let branches = hash_pair(&left, &right); + hash_pair(digest, &branches) + } +} + +#[cfg(test)] +mod tests { + use nockapp::noun::slab::{NockJammer, NounSlab}; + use nockchain_math::belt::{Belt, PRIME}; + use nockchain_math::noun_ext::NounMathExtHandle; + use nockchain_math::poly::BPolyVec; + use nockchain_math::shape::do_leaf_sequence; + use nockchain_math::structs::HoonList; + use nockchain_math::tip5; + use nockchain_math::zoon::zset::ZSet; + use nockvm::ext::make_tas; + use nockvm::mem::NockStack; + use nockvm::noun::{Atom, Noun, NounAllocator, D, T}; + use nockvm_macros::tas; + use noun_serde::{NounDecode, NounEncode}; + + use super::{ + bpoly_to_list, hash_hashable, hash_hashable_value, hash_leaf_atom, hash_list, hash_mary, + hash_pair, Hashable, HashableEncodingError, HashableTreeHasher, + }; + use crate::tx_engine::common::Hash; + + fn manual_hash_ten_cell(allocator: &mut A, ten_cell: Noun) -> Noun { + let mut leaf = Vec::::new(); + let space = allocator.noun_space(); + do_leaf_sequence(ten_cell, &mut leaf, &space) + .expect("manual ten-cell leaf sequence should work"); + let mut leaf_belt = leaf.into_iter().map(Belt).collect(); + tip5::hash::hash_10(&mut leaf_belt).to_noun(allocator) + } + + fn manual_hash_hashable(allocator: &mut A, noun: Noun) -> Noun { + let space = allocator.noun_space(); + let cell = noun + .in_space(&space) + .as_cell() + .expect("hashable noun must be a cell"); + let head = cell.head(); + let tail = cell.tail(); + + if let Ok(tag) = head.as_atom().and_then(|atom| atom.as_direct()) { + match tag.data() { + tas!(b"hash") => return tail.noun(), + tas!(b"leaf") => { + return tip5::hash::hash_noun_varlen(allocator, tail.noun(), &space) + .expect("manual leaf hashing should succeed"); + } + tas!(b"list") => { + let mut hashed_list = D(0); + let items = HoonList::try_from(tail.noun(), &space) + .expect("manual list payload must decode"); + let hashed_items = items + .into_iter() + .map(|item| manual_hash_hashable(allocator, item)) + .collect::>(); + for item in hashed_items.into_iter().rev() { + hashed_list = T(allocator, &[item, hashed_list]); + } + return tip5::hash::hash_noun_varlen(allocator, hashed_list, &space) + .expect("manual list hashing should succeed"); + } + _ => {} + } + } + + let left = manual_hash_hashable(allocator, head.noun()); + let right = manual_hash_hashable(allocator, tail.noun()); + let pair = T(allocator, &[left, right]); + manual_hash_ten_cell(allocator, pair) + } + + fn digest_hash(noun: Noun, space: &nockvm::noun::NounSpace) -> Hash { + Hash::from_noun(&noun, space).expect("digest noun should decode") + } + + fn tagged_hashable(allocator: &mut A, tag: &str, payload: Noun) -> Noun { + let tag = make_tas(allocator, tag).as_noun(); + T(allocator, &[tag, payload]) + } + + fn zset_noun_hashable(allocator: &mut A, noun: Noun) -> Noun { + if unsafe { noun.raw_equals(&D(0)) } { + return tagged_hashable(allocator, "leaf", D(0)); + } + + let space = allocator.noun_space(); + let [digest, left, right] = noun + .in_space(&space) + .uncell() + .expect("z-set tree must be a node"); + let hash_node = tagged_hashable(allocator, "hash", digest.noun()); + let left = zset_noun_hashable(allocator, left.noun()); + let right = zset_noun_hashable(allocator, right.noun()); + T(allocator, &[hash_node, left, right]) + } + + struct TestLeafHashable(u64); + + impl Hashable for TestLeafHashable { + type Error = HashableEncodingError; + + fn hash_digest(&self) -> Result { + hash_leaf_atom(self.0) + } + } + + #[test] + fn bpoly_to_list_encodes_large_belt_as_atom() { + let mut stack = NockStack::new(nockvm::mem::NOCK_STACK_SIZE_TINY, 0); + let len_noun = Atom::new(&mut stack, 1).as_noun(); + let belt_noun = Atom::new(&mut stack, PRIME - 1).as_noun(); + let sam = T(&mut stack, &[len_noun, belt_noun]); + + let list = bpoly_to_list(&mut stack, sam).expect("bpoly list conversion should succeed"); + let space = stack.noun_space(); + let cell = list + .in_space(&space) + .as_cell() + .expect("expected non-empty list"); + let head = cell.head().as_atom().expect("expected atom head"); + let value = head.as_u64().expect("expected u64 atom"); + + assert_eq!(value, PRIME - 1); + assert!(unsafe { cell.tail().noun().raw_equals(&D(0)) }); + } + + #[test] + fn hash_hashable_leaf_matches_manual_spec() { + let mut slab: NounSlab = NounSlab::new(); + let hashable = tagged_hashable(&mut slab, "leaf", D(42)); + let actual = hash_hashable(&mut slab, hashable).expect("leaf hash should succeed"); + + let mut expected_slab: NounSlab = NounSlab::new(); + let expected_hashable = tagged_hashable(&mut expected_slab, "leaf", D(42)); + let expected = manual_hash_hashable(&mut expected_slab, expected_hashable); + + let actual_space = slab.noun_space(); + let expected_space = expected_slab.noun_space(); + assert_eq!( + digest_hash(actual, &actual_space), + digest_hash(expected, &expected_space) + ); + } + + #[test] + fn hash_hashable_hash_passthrough_matches_manual_spec() { + let expected_hash = Hash::from_limbs(&[7, 11, 13, 17, 19]); + + let mut slab: NounSlab = NounSlab::new(); + let payload = expected_hash.to_noun(&mut slab); + let hashable = tagged_hashable(&mut slab, "hash", payload); + let actual = hash_hashable(&mut slab, hashable).expect("hash passthrough should succeed"); + + let space = slab.noun_space(); + assert_eq!(digest_hash(actual, &space), expected_hash); + } + + #[test] + fn hash_hashable_pair_recursion_matches_manual_spec() { + let mut slab: NounSlab = NounSlab::new(); + let left = tagged_hashable(&mut slab, "leaf", D(3)); + let right = tagged_hashable(&mut slab, "leaf", D(9)); + let hashable = T(&mut slab, &[left, right]); + let actual = hash_hashable(&mut slab, hashable).expect("pair hash should succeed"); + + let mut expected_slab: NounSlab = NounSlab::new(); + let left = tagged_hashable(&mut expected_slab, "leaf", D(3)); + let right = tagged_hashable(&mut expected_slab, "leaf", D(9)); + let expected_hashable = T(&mut expected_slab, &[left, right]); + let expected = manual_hash_hashable(&mut expected_slab, expected_hashable); + + let actual_space = slab.noun_space(); + let expected_space = expected_slab.noun_space(); + assert_eq!( + digest_hash(actual, &actual_space), + digest_hash(expected, &expected_space) + ); + } + + #[test] + fn hash_hashable_list_matches_manual_spec() { + let mut slab: NounSlab = NounSlab::new(); + let item_one = tagged_hashable(&mut slab, "leaf", D(1)); + let item_two = tagged_hashable(&mut slab, "leaf", D(2)); + let list_tail = T(&mut slab, &[item_two, D(0)]); + let list_payload = T(&mut slab, &[item_one, list_tail]); + let hashable = tagged_hashable(&mut slab, "list", list_payload); + let actual = hash_hashable(&mut slab, hashable).expect("list hash should succeed"); + + let mut expected_slab: NounSlab = NounSlab::new(); + let item_one = tagged_hashable(&mut expected_slab, "leaf", D(1)); + let item_two = tagged_hashable(&mut expected_slab, "leaf", D(2)); + let list_tail = T(&mut expected_slab, &[item_two, D(0)]); + let list_payload = T(&mut expected_slab, &[item_one, list_tail]); + let expected_hashable = tagged_hashable(&mut expected_slab, "list", list_payload); + let expected = manual_hash_hashable(&mut expected_slab, expected_hashable); + + let actual_space = slab.noun_space(); + let expected_space = expected_slab.noun_space(); + assert_eq!( + digest_hash(actual, &actual_space), + digest_hash(expected, &expected_space) + ); + } + + #[test] + fn direct_hashable_helper_matches_manual_leaf_hash() { + let actual = + hash_hashable_value(&TestLeafHashable(42)).expect("direct helper should hash test"); + + let mut expected_slab: NounSlab = NounSlab::new(); + let expected_hashable = tagged_hashable(&mut expected_slab, "leaf", D(42)); + let expected = manual_hash_hashable(&mut expected_slab, expected_hashable); + + let expected_space = expected_slab.noun_space(); + assert_eq!( + Hash::from_limbs(&actual), + digest_hash(expected, &expected_space) + ); + } + + #[test] + fn direct_hash_set_matches_hashable_tree_semantics() { + let hashes = vec![ + Hash::from_limbs(&[1, 2, 3, 4, 5]), + Hash::from_limbs(&[7, 11, 13, 17, 19]), + Hash::from_limbs(&[23, 29, 31, 37, 41]), + ]; + let zset = ZSet::try_from_items(hashes).expect("hash z-set should build"); + + let actual = zset.hash_with(&HashableTreeHasher); + + let mut slab: NounSlab = NounSlab::new(); + let zset_noun = zset.to_noun(&mut slab); + let hashable = zset_noun_hashable(&mut slab, zset_noun); + let expected = hash_hashable(&mut slab, hashable).expect("noun helper should hash"); + + let space = slab.noun_space(); + assert_eq!(actual, digest_hash(expected, &space)); + } + + #[test] + fn direct_hash_list_matches_noun_hashable() { + let items = + vec![hash_leaf_atom(1).expect("based leaf"), hash_leaf_atom(2).expect("based leaf")]; + let actual = hash_list(&items); + + let mut slab: NounSlab = NounSlab::new(); + let item_one = tagged_hashable(&mut slab, "leaf", D(1)); + let item_two = tagged_hashable(&mut slab, "leaf", D(2)); + let list_tail = T(&mut slab, &[item_two, D(0)]); + let list_payload = T(&mut slab, &[item_one, list_tail]); + let hashable = tagged_hashable(&mut slab, "list", list_payload); + let expected = hash_hashable(&mut slab, hashable).expect("noun helper should hash"); + + let space = slab.noun_space(); + assert_eq!(actual, digest_hash(expected, &space)); + } + + #[test] + fn direct_hash_mary_matches_noun_hashable() { + let bpoly = BPolyVec::from(vec![2, 4, 8]); + let actual = hash_mary(1, 3, &bpoly.0).expect("direct mary helper should hash"); + + let mut slab: NounSlab = NounSlab::new(); + let array = bpoly.to_noun(&mut slab); + let payload = T(&mut slab, &[D(1), array]); + let hashable = tagged_hashable(&mut slab, "mary", payload); + let expected = hash_hashable(&mut slab, hashable).expect("noun helper should hash"); + + let space = slab.noun_space(); + assert_eq!(actual, digest_hash(expected, &space)); + } + + #[test] + fn hash_pair_matches_manual_ten_cell() { + let left = Hash::from_limbs(&[2, 3, 5, 7, 11]); + let right = Hash::from_limbs(&[13, 17, 19, 23, 29]); + let actual = hash_pair(&left, &right); + + let mut slab: NounSlab = NounSlab::new(); + let left_noun = left.to_noun(&mut slab); + let right_noun = right.to_noun(&mut slab); + let pair = T(&mut slab, &[left_noun, right_noun]); + let expected = manual_hash_ten_cell(&mut slab, pair); + + let space = slab.noun_space(); + assert_eq!(actual, digest_hash(expected, &space)); + } +} diff --git a/crates/nockchain-types/src/tx_engine/v1/hashable/noun.rs b/crates/nockchain-types/src/tx_engine/v1/hashable/noun.rs new file mode 100644 index 000000000..7088b6d9b --- /dev/null +++ b/crates/nockchain-types/src/tx_engine/v1/hashable/noun.rs @@ -0,0 +1,126 @@ +use nockchain_math::noun_ext::NounMathExtHandle; +use nockchain_math::poly::BPolySlice; +use nockchain_math::structs::HoonList; +use nockchain_math::tip5; +use nockvm::jets::util::BAIL_FAIL; +use nockvm::jets::JetErr; +use nockvm::noun::{Atom, Noun, NounAllocator, NounSpace, D}; +use nockvm_macros::tas; +use noun_serde::{NounDecode, NounEncode}; + +use super::{hash_list, hash_mary, hash_pair}; +use crate::tx_engine::common::Hash; + +/// Compatibility wrapper for noun-facing `hash-hashable` callers. The noun +/// frontend lowers into the same direct digest helpers used by handwritten Rust +/// `Hashable` implementations, then re-encodes the final digest as a noun. +pub fn hash_hashable(stack: &mut A, h: Noun) -> Result { + Ok(hash_hashable_digest(stack, h)?.to_noun(stack)) +} + +fn hash_hashable_digest(stack: &mut A, h: Noun) -> Result { + let space = stack.noun_space(); + let h_cell = h.in_space(&space).as_cell().map_err(|_| BAIL_FAIL)?; + let h_head = h_cell.head(); + let h_tail = h_cell.tail(); + + if h_head.is_direct() { + let tag = h_head.as_atom()?.as_direct()?; + + match tag.data() { + tas!(b"hash") => decode_hash_digest_noun(h_tail.noun(), &space), + tas!(b"leaf") => hash_leaf_noun(stack, h_tail.noun()), + tas!(b"list") => hash_hashable_list_digest(stack, h_tail.noun()), + tas!(b"mary") => hash_hashable_mary_digest(stack, h_tail.noun()), + _ => hash_hashable_other_digest(stack, h_head.noun(), h_tail.noun()), + } + } else { + hash_hashable_other_digest(stack, h_head.noun(), h_tail.noun()) + } +} + +fn decode_hash_digest_noun(noun: Noun, space: &NounSpace) -> Result { + Hash::from_noun(&noun, space).map_err(|_| BAIL_FAIL) +} + +fn hash_leaf_noun(stack: &mut A, noun: Noun) -> Result { + let space = stack.noun_space(); + let digest = tip5::hash::hash_noun_varlen_digest(stack, noun, &space)?; + Ok(Hash::from_limbs(&digest)) +} + +fn hash_hashable_list_digest(stack: &mut A, p: Noun) -> Result { + let space = stack.noun_space(); + let mut hashed_items = Vec::new(); + for item in HoonList::try_from(p, &space)? { + hashed_items.push(hash_hashable_digest(stack, item)?); + } + + Ok(hash_list(&hashed_items)) +} + +fn hash_hashable_mary_digest(stack: &mut A, p: Noun) -> Result { + let space = stack.noun_space(); + let [ma_step_noun, ma_array] = p.in_space(&space).uncell()?; + let [ma_array_len_noun, _ma_array_dat] = ma_array.uncell()?; + let ma_step = ma_step_noun.as_atom()?.as_u64()?; + let ma_array_len = ma_array_len_noun.as_atom()?.as_u64()?; + + let ma_changed = change_mary_step(stack, p, 1)?; + let [_ma_changed_step, ma_changed_array] = ma_changed.in_space(&space).uncell()?; + let normalized_bpoly = BPolySlice::try_from(ma_changed_array.noun(), &space)?; + + hash_mary(ma_step, ma_array_len, normalized_bpoly.0).map_err(|_| BAIL_FAIL) +} + +fn hash_hashable_other_digest( + stack: &mut A, + p: Noun, + q: Noun, +) -> Result { + let ph = hash_hashable_digest(stack, p)?; + let qh = hash_hashable_digest(stack, q)?; + Ok(hash_pair(&ph, &qh)) +} + +pub fn bpoly_to_list(stack: &mut A, sam: Noun) -> Result { + let space = stack.noun_space(); + let sam_bpoly = BPolySlice::try_from(sam, &space)?; + let mut res_list = D(0); + for belt in sam_bpoly.0.iter().rev() { + // Belt values are field elements and may exceed DIRECT_MAX, so they must + // be encoded with Atom::new to allow indirect atoms. + let belt_noun = belt.to_noun(stack); + res_list = nockvm::noun::T(stack, &[belt_noun, res_list]); + } + Ok(res_list) +} + +fn change_mary_step( + stack: &mut A, + ma_noun: Noun, + new_step: u64, +) -> Result { + let space = stack.noun_space(); + let [ma_step_noun, ma_array] = ma_noun.in_space(&space).uncell()?; + let [array_len_noun, array_dat] = ma_array.uncell()?; + + let ma_step = ma_step_noun.as_atom()?.as_u64()?; + let array_len = array_len_noun.as_atom()?.as_u64()?; + + if ma_step == new_step { + return Ok(ma_noun); + } + if (ma_step * array_len) % new_step != 0 { + return Err(BAIL_FAIL); + } + + let new_array_len = ma_step.checked_mul(array_len).ok_or(BAIL_FAIL)? / new_step; + let new_step_noun = Atom::new(stack, new_step).as_noun(); + let new_array_len_noun = Atom::new(stack, new_array_len).as_noun(); + + Ok(nockvm::noun::T( + stack, + &[new_step_noun, new_array_len_noun, array_dat.noun()], + )) +} diff --git a/crates/nockchain-types/src/tx_engine/v1/mod.rs b/crates/nockchain-types/src/tx_engine/v1/mod.rs index e18ccb7a2..d4154b9c4 100644 --- a/crates/nockchain-types/src/tx_engine/v1/mod.rs +++ b/crates/nockchain-types/src/tx_engine/v1/mod.rs @@ -1,6 +1,8 @@ +pub mod hashable; pub mod note; pub mod tx; +pub use hashable::*; pub use note::*; pub use tx::*; diff --git a/crates/nockchain-types/src/tx_engine/v1/note.rs b/crates/nockchain-types/src/tx_engine/v1/note.rs index d78e7982f..4cc85e48a 100644 --- a/crates/nockchain-types/src/tx_engine/v1/note.rs +++ b/crates/nockchain-types/src/tx_engine/v1/note.rs @@ -1,12 +1,12 @@ use nockapp::noun::slab::{NockJammer, NounSlab}; use nockapp::noun::NounAllocatorExt; use nockapp::utils::make_tas; -use nockapp::{AtomExt, Noun}; -use nockchain_math::noun_ext::NounMathExt; +use nockapp::Noun; +use nockchain_math::noun_ext::NounMathExtHandle; use nockchain_math::structs::HoonMapIter; use nockchain_math::zoon::common::DefaultTipHasher; -use nockchain_math::zoon::zmap; -use nockvm::noun::{NounAllocator, D}; +use nockchain_math::zoon::zmap::{self, ZMap}; +use nockvm::noun::{NounAllocator, NounSpace, D}; use noun_serde::{NounDecode, NounDecodeError, NounEncode}; use crate::tx_engine::common::{BlockHeight, Hash, Name, Nicks, Version}; @@ -24,23 +24,20 @@ pub struct Balance(pub Vec<(Name, Note)>); impl NounEncode for Balance { fn to_noun(&self, stack: &mut A) -> Noun { - self.0.iter().fold(D(0), |map, (name, note)| { - let mut key = name.to_noun(stack); - let mut value = note.to_noun(stack); - zmap::z_map_put(stack, &map, &mut key, &mut value, &DefaultTipHasher) - .expect("Failed to put into z_map") - }) + ZMap::try_from_entries(self.0.clone()) + .expect("balance z-map should encode") + .to_noun(stack) } } impl NounDecode for Balance { - fn from_noun(noun: &Noun) -> Result { - let notes = HoonMapIter::from(*noun) + fn from_noun(noun: &Noun, space: &NounSpace) -> Result { + let notes = HoonMapIter::new(&noun.in_space(space)) .filter(|kv| kv.is_cell()) .map(|kv| { let [k, v] = kv.uncell()?; - let name = Name::from_noun(&k)?; - let note = ::from_noun(&v)?; + let name = Name::from_noun_handle(&k)?; + let note = ::from_noun_handle(&v)?; Ok((name, note)) }) @@ -74,11 +71,12 @@ impl NounEncode for Note { } impl NounDecode for Note { - fn from_noun(noun: &Noun) -> Result { - let hed = noun.as_cell()?.head(); + fn from_noun(noun: &Noun, space: &NounSpace) -> Result { + let cell = noun.in_space(space).as_cell()?; + let hed = cell.head().noun(); match hed.is_cell() { - true => Ok(Note::V0(NoteV0::from_noun(noun)?)), - false => Ok(Note::V1(NoteV1::from_noun(noun)?)), + true => Ok(Note::V0(NoteV0::from_noun(noun, space)?)), + false => Ok(Note::V1(NoteV1::from_noun(noun, space)?)), } } } @@ -137,7 +135,8 @@ impl NounEncode for NoteData { .expect("failed to cue blob"); let mut value = unsafe { let &root = slab.root(); - allocator.copy_into(root) + let space = slab.noun_space(); + allocator.copy_into(root, &space) }; zmap::z_map_put(allocator, &map, &mut key, &mut value, &DefaultTipHasher) .expect("failed to encode note-data entry") @@ -146,8 +145,8 @@ impl NounEncode for NoteData { } impl NounDecode for NoteData { - fn from_noun(noun: &Noun) -> Result { - let entries = HoonMapIter::from(*noun) + fn from_noun(noun: &Noun, space: &NounSpace) -> Result { + let entries = HoonMapIter::new(&noun.in_space(space)) .filter(|entry| entry.is_cell()) .map(|entry| { let [raw_key, raw_value] = entry.uncell().map_err(|_| { @@ -165,7 +164,7 @@ impl NounDecode for NoteData { })?; let mut slab: NounSlab = NounSlab::new(); - slab.copy_into(raw_value); + slab.copy_into(raw_value.noun(), space); let jam = slab.jam(); Ok(NoteDataEntry { key, blob: jam }) }) diff --git a/crates/nockchain-types/src/tx_engine/v1/tx.rs b/crates/nockchain-types/src/tx_engine/v1/tx.rs index 65c984964..56bc873d8 100644 --- a/crates/nockchain-types/src/tx_engine/v1/tx.rs +++ b/crates/nockchain-types/src/tx_engine/v1/tx.rs @@ -1,17 +1,23 @@ use nockapp::noun::slab::{NockJammer, NounSlab}; use nockapp::noun::NounAllocatorExt; -use nockchain_math::noun_ext::NounMathExt; +use nockchain_math::belt::Belt; +use nockchain_math::noun_ext::NounMathExtHandle; use nockchain_math::structs::{HoonList, HoonMapIter}; use nockchain_math::zoon::common::DefaultTipHasher; -use nockchain_math::zoon::{zmap, zset}; -use nockvm::ext::{make_tas, AtomExt}; -use nockvm::noun::{Noun, NounAllocator, D}; +use nockchain_math::zoon::zmap::{self, ZMap}; +use nockchain_math::zoon::zset::ZSet; +use nockvm::ext::make_tas; +use nockvm::noun::{Noun, NounAllocator, NounSpace, D}; use noun_serde::{NounDecode, NounDecodeError, NounEncode}; +use super::hashable::{ + hash_leaf_atom, hash_leaf_null, hash_pair, hash_unit_belt, Hashable, HashableEncodingError, + HashableTreeHasher, +}; use super::note::NoteData; use crate::tx_engine::common::{ - BlockHeight, Hash, Name, Nicks, SchnorrPubkey, SchnorrSignature, Signature, Source, TxId, - Version, + BlockHeight, BlockHeightDelta, FirstName, Hash, Name, Nicks, SchnorrPubkey, SchnorrSignature, + Signature, Source, TxId, Version, }; use crate::v0::{TimelockRangeAbsolute, TimelockRangeRelative}; @@ -32,17 +38,20 @@ impl NounEncode for RawTx { } impl NounDecode for RawTx { - fn from_noun(noun: &Noun) -> Result { - let cell = noun.as_cell()?; - let version = Version::from_noun(&cell.head())?; + fn from_noun(noun: &Noun, space: &NounSpace) -> Result { + let cell = noun.in_space(space).as_cell()?; + let version_noun = cell.head().noun(); + let version = Version::from_noun(&version_noun, space)?; let tail = cell.tail(); let cell = tail .as_cell() .map_err(|_| NounDecodeError::Custom("raw-tx tail not a cell".into()))?; - let id = TxId::from_noun(&cell.head())?; + let id_noun = cell.head().noun(); + let id = TxId::from_noun(&id_noun, space)?; - let spends = Spends::from_noun(&cell.tail())?; + let spends_noun = cell.tail().noun(); + let spends = Spends::from_noun(&spends_noun, space)?; if version != Version::V1 { return Err(NounDecodeError::Custom("expected raw-tx version 1".into())); @@ -61,25 +70,22 @@ pub struct Spends(pub Vec<(Name, Spend)>); impl NounEncode for Spends { fn to_noun(&self, allocator: &mut A) -> Noun { - self.0.iter().fold(D(0), |acc, (name, spend)| { - let mut key = name.to_noun(allocator); - let mut value = spend.to_noun(allocator); - zmap::z_map_put(allocator, &acc, &mut key, &mut value, &DefaultTipHasher) - .expect("failed to encode spends map") - }) + ZMap::try_from_entries(self.0.clone()) + .expect("spends z-map should encode") + .to_noun(allocator) } } impl NounDecode for Spends { - fn from_noun(noun: &Noun) -> Result { - let entries = HoonMapIter::from(*noun) + fn from_noun(noun: &Noun, space: &NounSpace) -> Result { + let entries = HoonMapIter::new(&noun.in_space(space)) .filter(|entry| entry.is_cell()) .map(|entry| { let [name_raw, spend_raw] = entry .uncell() .map_err(|_| NounDecodeError::Custom("spend entry must be a pair".into()))?; - let name = Name::from_noun(&name_raw)?; - let spend = Spend::from_noun(&spend_raw)?; + let name = Name::from_noun_handle(&name_raw)?; + let spend = Spend::from_noun_handle(&spend_raw)?; Ok((name, spend)) }) .collect::, NounDecodeError>>()?; @@ -111,12 +117,18 @@ impl NounEncode for Spend { } impl NounDecode for Spend { - fn from_noun(noun: &Noun) -> Result { - let cell = noun.as_cell()?; + fn from_noun(noun: &Noun, space: &NounSpace) -> Result { + let cell = noun.in_space(space).as_cell()?; let tag = cell.head().as_atom()?.as_u64()?; match tag { - 0 => Ok(Spend::Legacy(Spend0::from_noun(&cell.tail())?)), - 1 => Ok(Spend::Witness(Spend1::from_noun(&cell.tail())?)), + 0 => { + let tail_noun = cell.tail().noun(); + Ok(Spend::Legacy(Spend0::from_noun(&tail_noun, space)?)) + } + 1 => { + let tail_noun = cell.tail().noun(); + Ok(Spend::Witness(Spend1::from_noun(&tail_noun, space)?)) + } _ => Err(NounDecodeError::InvalidEnumVariant), } } @@ -141,17 +153,15 @@ pub struct Seeds(pub Vec); impl NounEncode for Seeds { fn to_noun(&self, allocator: &mut A) -> Noun { - self.0.iter().fold(D(0), |acc, seed| { - let mut value = seed.to_noun(allocator); - zset::z_set_put(allocator, &acc, &mut value, &DefaultTipHasher) - .expect("failed to encode seeds set") - }) + ZSet::try_from_items(self.0.clone()) + .expect("seed z-set should encode") + .to_noun(allocator) } } impl NounDecode for Seeds { - fn from_noun(noun: &Noun) -> Result { - decode_zset(noun, Seed::from_noun).map(Self) + fn from_noun(noun: &Noun, space: &NounSpace) -> Result { + decode_zset(noun, space, Seed::from_noun).map(Self) } } @@ -199,7 +209,8 @@ impl NounEncode for Witness { slab.cue_into(entry.value.clone()) .expect("failed to cue value"); let &root = slab.root(); - allocator.copy_into(root) + let space = slab.noun_space(); + allocator.copy_into(root, &space) }; zmap::z_map_put( allocator, &acc, &mut key, &mut value_noun, &DefaultTipHasher, @@ -212,30 +223,32 @@ impl NounEncode for Witness { } impl NounDecode for Witness { - fn from_noun(noun: &Noun) -> Result { - let cell = noun.as_cell()?; - let lock_merkle_proof = LockMerkleProof::from_noun(&cell.head())?; + fn from_noun(noun: &Noun, space: &NounSpace) -> Result { + let cell = noun.in_space(space).as_cell()?; + let lmp_noun = cell.head().noun(); + let lock_merkle_proof = LockMerkleProof::from_noun(&lmp_noun, space)?; let tail = cell.tail(); let cell = tail .as_cell() .map_err(|_| NounDecodeError::Custom("witness tail not a cell".into()))?; - let pkh_signature = PkhSignature::from_noun(&cell.head())?; + let pkh_noun = cell.head().noun(); + let pkh_signature = PkhSignature::from_noun(&pkh_noun, space)?; let tail = cell.tail(); let cell = tail .as_cell() .map_err(|_| NounDecodeError::Custom("witness hax tail not a cell".into()))?; - let hax_entries = HoonMapIter::from(cell.head()) + let hax_entries = HoonMapIter::new(&cell.head()) .filter(|entry| entry.is_cell()) .map(|entry| { let [hash_raw, value_noun] = entry.uncell().map_err(|_| { NounDecodeError::Custom("witness hax entry must be a pair".into()) })?; - let hash = Hash::from_noun(&hash_raw)?; + let hash = Hash::from_noun_handle(&hash_raw)?; let mut slab: NounSlab = NounSlab::new(); - slab.copy_into(value_noun); + slab.copy_into(value_noun.noun(), space); let value = slab.jam(); Ok(HaxPreimage { hash, value }) }) @@ -268,31 +281,48 @@ impl PkhSignature { impl NounEncode for PkhSignature { fn to_noun(&self, allocator: &mut A) -> Noun { - self.0.iter().fold(D(0), |acc, entry| { - let mut key = entry.hash.to_noun(allocator); - let mut value = entry.to_noun(allocator); - zmap::z_map_put(allocator, &acc, &mut key, &mut value, &DefaultTipHasher) - .expect("failed to encode pkh-signature map") - }) + let entries = self + .0 + .iter() + .cloned() + .map(|entry| { + ( + entry.hash.clone(), + PkhSignatureValue { + pubkey: entry.pubkey, + signature: entry.signature, + }, + ) + }) + .collect::>(); + ZMap::try_from_entries(entries) + .expect("pkh-signature z-map should encode") + .to_noun(allocator) } } impl NounDecode for PkhSignature { - fn from_noun(noun: &Noun) -> Result { - let entries = HoonMapIter::from(*noun) + fn from_noun(noun: &Noun, space: &NounSpace) -> Result { + let entries = HoonMapIter::new(&noun.in_space(space)) .filter(|entry| entry.is_cell()) .map(|entry| { let [hash_raw, value_raw] = entry.uncell().map_err(|_| { NounDecodeError::Custom("pkh-signature entry must be a pair".into()) })?; - let hash = Hash::from_noun(&hash_raw)?; - PkhSignatureEntry::decode(hash, &value_raw) + let hash = Hash::from_noun_handle(&hash_raw)?; + PkhSignatureEntry::decode(hash, &value_raw.noun(), space) }) .collect::, NounDecodeError>>()?; Ok(Self(entries)) } } +#[derive(Debug, Clone, PartialEq, Eq, NounEncode, NounDecode)] +struct PkhSignatureValue { + pubkey: SchnorrPubkey, + signature: SchnorrSignature, +} + #[derive(Debug, Clone, PartialEq, Eq)] pub struct PkhSignatureEntry { pub hash: Hash, @@ -309,10 +339,12 @@ impl NounEncode for PkhSignatureEntry { } impl PkhSignatureEntry { - fn decode(hash: Hash, noun: &Noun) -> Result { - let cell = noun.as_cell()?; - let pubkey = SchnorrPubkey::from_noun(&cell.head())?; - let signature = SchnorrSignature::from_noun(&cell.tail())?; + fn decode(hash: Hash, noun: &Noun, space: &NounSpace) -> Result { + let cell = noun.in_space(space).as_cell()?; + let pubkey_noun = cell.head().noun(); + let signature_noun = cell.tail().noun(); + let pubkey = SchnorrPubkey::from_noun(&pubkey_noun, space)?; + let signature = SchnorrSignature::from_noun(&signature_noun, space)?; Ok(Self { hash, pubkey, @@ -320,7 +352,6 @@ impl PkhSignatureEntry { }) } } - #[derive(Debug, Clone, PartialEq, Eq, NounEncode, NounDecode)] pub struct LockMerkleProofStub { pub spend_condition: SpendCondition, @@ -392,8 +423,8 @@ impl LockMerkleProof { } impl NounDecode for LockMerkleProof { - fn from_noun(noun: &Noun) -> Result { - if let Ok(full) = LockMerkleProofFull::from_noun(noun) { + fn from_noun(noun: &Noun, space: &NounSpace) -> Result { + if let Ok(full) = LockMerkleProofFull::from_noun(noun, space) { if full.version != nockvm_macros::tas!(b"full") { return Err(NounDecodeError::Custom( "lock-merkle-proof version must be %full".into(), @@ -401,7 +432,7 @@ impl NounDecode for LockMerkleProof { } return Ok(Self::Full(full)); } - Ok(Self::Stub(LockMerkleProofStub::from_noun(noun)?)) + Ok(Self::Stub(LockMerkleProofStub::from_noun(noun, space)?)) } } @@ -424,21 +455,190 @@ impl NounEncode for MerkleProof { } impl NounDecode for MerkleProof { - fn from_noun(noun: &Noun) -> Result { - let cell = noun.as_cell()?; - let root = Hash::from_noun(&cell.head())?; - let path_iter = HoonList::try_from(cell.tail()) + fn from_noun(noun: &Noun, space: &NounSpace) -> Result { + let cell = noun.in_space(space).as_cell()?; + let root_noun = cell.head().noun(); + let root = Hash::from_noun(&root_noun, space)?; + let path_noun = cell.tail().noun(); + let path_iter = HoonList::try_from(path_noun, space) .map_err(|_| NounDecodeError::Custom("merkle proof path must be a list".into()))?; let mut path = Vec::new(); for entry in path_iter { - path.push(Hash::from_noun(&entry)?); + path.push(Hash::from_noun(&entry, space)?); } Ok(Self { root, path }) } } +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +struct NumericTag(u64); + +impl NumericTag { + fn into_inner(self) -> u64 { + self.0 + } +} + +impl NounDecode for NumericTag { + fn from_noun(noun: &Noun, space: &NounSpace) -> Result { + if let Ok(value) = u64::from_noun(noun, space) { + return Ok(Self(value)); + } + + let tag = String::from_noun(noun, space)?; + let value = tag.parse::().map_err(|_| { + NounDecodeError::Custom("lock tree tag must contain only digits".into()) + })?; + Ok(Self(value)) + } +} + +/// Binary lock tree with power-of-two fanout over spend conditions. +/// +/// This mirrors `$lock` in `tx-engine-1.hoon`: a lock is either a single +/// spend-condition leaf, or a tagged `%2/%4/%8/%16` tree. +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum Lock { + SpendCondition(SpendCondition), + V2(LockV2), + V4(LockV4), + V8(LockV8), + V16(LockV16), +} + +#[derive(Debug, Clone, PartialEq, Eq, NounEncode, NounDecode)] +pub struct LockV2 { + pub p: SpendCondition, + pub q: SpendCondition, +} + +#[derive(Debug, Clone, PartialEq, Eq, NounEncode, NounDecode)] +pub struct LockV4 { + pub p: LockV2, + pub q: LockV2, +} + +#[derive(Debug, Clone, PartialEq, Eq, NounEncode, NounDecode)] +pub struct LockV8 { + pub p: LockV4, + pub q: LockV4, +} + +#[derive(Debug, Clone, PartialEq, Eq, NounEncode, NounDecode)] +pub struct LockV16 { + pub p: LockV8, + pub q: LockV8, +} + +impl LockV2 { + fn flatten_spend_conditions(&self) -> Vec { + vec![self.p.clone(), self.q.clone()] + } +} + +impl LockV4 { + fn flatten_spend_conditions(&self) -> Vec { + let mut out = self.p.flatten_spend_conditions(); + out.extend(self.q.flatten_spend_conditions()); + out + } +} + +impl LockV8 { + fn flatten_spend_conditions(&self) -> Vec { + let mut out = self.p.flatten_spend_conditions(); + out.extend(self.q.flatten_spend_conditions()); + out + } +} + +impl LockV16 { + fn flatten_spend_conditions(&self) -> Vec { + let mut out = self.p.flatten_spend_conditions(); + out.extend(self.q.flatten_spend_conditions()); + out + } +} + +impl Lock { + /// Returns how many spend-condition leaves are present in this lock. + pub fn spend_condition_count(&self) -> u64 { + match self { + Self::SpendCondition(_) => 1, + Self::V2(_) => 2, + Self::V4(_) => 4, + Self::V8(_) => 8, + Self::V16(_) => 16, + } + } + + /// Flattens the lock tree in left-to-right order. + pub fn flatten_spend_conditions(&self) -> Vec { + match self { + Self::SpendCondition(spend_condition) => vec![spend_condition.clone()], + Self::V2(v2) => v2.flatten_spend_conditions(), + Self::V4(v4) => v4.flatten_spend_conditions(), + Self::V8(v8) => v8.flatten_spend_conditions(), + Self::V16(v16) => v16.flatten_spend_conditions(), + } + } + + /// Computes the consensus lock root by hashing this lock's handwritten hashable form. + pub fn hash(&self) -> Result { + self.hash_digest() + } +} + +impl NounEncode for Lock { + fn to_noun(&self, allocator: &mut A) -> Noun { + match self { + Self::SpendCondition(spend_condition) => spend_condition.to_noun(allocator), + Self::V2(v2) => { + let value = v2.to_noun(allocator); + nockvm::noun::T(allocator, &[D(2), value]) + } + Self::V4(v4) => { + let value = v4.to_noun(allocator); + nockvm::noun::T(allocator, &[D(4), value]) + } + Self::V8(v8) => { + let value = v8.to_noun(allocator); + nockvm::noun::T(allocator, &[D(8), value]) + } + Self::V16(v16) => { + let value = v16.to_noun(allocator); + nockvm::noun::T(allocator, &[D(16), value]) + } + } + } +} + +impl NounDecode for Lock { + fn from_noun(noun: &Noun, space: &NounSpace) -> Result { + if let Ok(spend_condition) = SpendCondition::from_noun(noun, space) { + return Ok(Self::SpendCondition(spend_condition)); + } + + let cell = noun.in_space(space).as_cell().map_err(|_| { + NounDecodeError::Custom("lock must be spend-condition or lock tree".into()) + })?; + let tag_noun = cell.head().noun(); + let tail_noun = cell.tail().noun(); + let tag = NumericTag::from_noun(&tag_noun, space)?.into_inner(); + match tag { + 2 => Ok(Self::V2(LockV2::from_noun(&tail_noun, space)?)), + 4 => Ok(Self::V4(LockV4::from_noun(&tail_noun, space)?)), + 8 => Ok(Self::V8(LockV8::from_noun(&tail_noun, space)?)), + 16 => Ok(Self::V16(LockV16::from_noun(&tail_noun, space)?)), + _ => Err(NounDecodeError::Custom(format!( + "unsupported lock tree tag: {tag}" + ))), + } + } +} + #[derive(Debug, Clone, PartialEq, Eq)] pub struct SpendCondition(pub Vec); @@ -462,13 +662,13 @@ impl NounEncode for SpendCondition { } impl NounDecode for SpendCondition { - fn from_noun(noun: &Noun) -> Result { - let iter = HoonList::try_from(*noun) + fn from_noun(noun: &Noun, space: &NounSpace) -> Result { + let iter = HoonList::try_from(*noun, space) .map_err(|_| NounDecodeError::Custom("spend-condition must be a list".into()))?; let mut primitives = Vec::new(); for entry in iter { - primitives.push(LockPrimitive::from_noun(&entry)?); + primitives.push(LockPrimitive::from_noun(&entry, space)?); } Ok(Self(primitives)) @@ -511,8 +711,8 @@ impl NounEncode for LockPrimitive { } impl NounDecode for LockPrimitive { - fn from_noun(noun: &Noun) -> Result { - let cell = noun.as_cell()?; + fn from_noun(noun: &Noun, space: &NounSpace) -> Result { + let cell = noun.in_space(space).as_cell()?; let tag_atom = cell .head() .as_atom() @@ -522,9 +722,18 @@ impl NounDecode for LockPrimitive { .map_err(|err| NounDecodeError::Custom(format!("invalid lock-primitive tag: {err}")))?; match tag.as_str() { - "pkh" => Ok(LockPrimitive::Pkh(Pkh::from_noun(&cell.tail())?)), - "tim" => Ok(LockPrimitive::Tim(LockTim::from_noun(&cell.tail())?)), - "hax" => Ok(LockPrimitive::Hax(Hax::from_noun(&cell.tail())?)), + "pkh" => { + let tail_noun = cell.tail().noun(); + Ok(LockPrimitive::Pkh(Pkh::from_noun(&tail_noun, space)?)) + } + "tim" => { + let tail_noun = cell.tail().noun(); + Ok(LockPrimitive::Tim(LockTim::from_noun(&tail_noun, space)?)) + } + "hax" => { + let tail_noun = cell.tail().noun(); + Ok(LockPrimitive::Hax(Hax::from_noun(&tail_noun, space)?)) + } "brn" => Ok(LockPrimitive::Burn), _ => Err(NounDecodeError::InvalidEnumVariant), } @@ -535,26 +744,37 @@ impl NounDecode for LockPrimitive { pub struct Pkh { pub m: u64, // z-set of hashes - pub hashes: Vec, + pub hashes: ZSet, +} + +impl Pkh { + pub fn new(m: u64, hashes: I) -> Self + where + I: IntoIterator, + { + Self { + m, + hashes: ZSet::try_from_items(hashes).expect("pkh hash z-set should build"), + } + } } impl NounEncode for Pkh { fn to_noun(&self, allocator: &mut A) -> Noun { let m = self.m.to_noun(allocator); - let hashes = self.hashes.iter().fold(D(0), |acc, hash| { - let mut value = hash.to_noun(allocator); - zset::z_set_put(allocator, &acc, &mut value, &DefaultTipHasher) - .expect("failed to encode pkh hash set") - }); + let hashes = self.hashes.to_noun(allocator); nockvm::noun::T(allocator, &[m, hashes]) } } impl NounDecode for Pkh { - fn from_noun(noun: &Noun) -> Result { - let cell = noun.as_cell()?; - let m = u64::from_noun(&cell.head())?; - let hashes = decode_zset(&cell.tail(), Hash::from_noun)?; + fn from_noun(noun: &Noun, space: &NounSpace) -> Result { + let cell = noun.in_space(space).as_cell()?; + let m_noun = cell.head().noun(); + let m = u64::from_noun(&m_noun, space)?; + let hashes_noun = cell.tail().noun(); + let hashes = ZSet::try_from_items(decode_zset(&hashes_noun, space, Hash::from_noun)?) + .map_err(|err| NounDecodeError::Custom(format!("pkh hash set invalid: {err}")))?; Ok(Self { m, hashes }) } } @@ -573,33 +793,263 @@ pub struct LockTimeBounds { // Encode into a set of hashes #[derive(Debug, Clone, PartialEq, Eq)] -pub struct Hax(pub Vec); +pub struct Hax(pub ZSet); + +impl Hax { + pub fn new(hashes: I) -> Self + where + I: IntoIterator, + { + Self(ZSet::try_from_items(hashes).expect("hax z-set should build")) + } +} impl NounEncode for Hax { fn to_noun(&self, allocator: &mut A) -> Noun { - self.0.iter().fold(D(0), |acc, hash| { - let mut value = hash.to_noun(allocator); - zset::z_set_put(allocator, &acc, &mut value, &DefaultTipHasher) - .expect("failed to encode hax set") - }) + self.0.to_noun(allocator) } } impl NounDecode for Hax { - fn from_noun(noun: &Noun) -> Result { - decode_zset(noun, Hash::from_noun).map(Self) + fn from_noun(noun: &Noun, space: &NounSpace) -> Result { + let hashes = ZSet::try_from_items(decode_zset(noun, space, Hash::from_noun)?) + .map_err(|err| NounDecodeError::Custom(format!("hax set invalid: {err}")))?; + Ok(Self(hashes)) } } -fn decode_zset(noun: &Noun, mut f: F) -> Result, NounDecodeError> +/// Errors raised while converting consensus values to hashable nouns. +#[derive(Debug, thiserror::Error)] +pub enum LockHashError { + #[error(transparent)] + HashableEncoding(#[from] HashableEncodingError), +} + +#[derive(Debug, thiserror::Error)] +pub enum FirstNameFromLockRootError { + #[error(transparent)] + HashableEncoding(#[from] HashableEncodingError), +} + +struct FirstNameDigestInput<'a> { + lock_root: &'a Hash, +} + +impl Hashable for FirstNameDigestInput<'_> { + type Error = FirstNameFromLockRootError; + + fn hash_digest(&self) -> Result { + let first_tag = hash_leaf_atom(0)?; + Ok(hash_pair(&first_tag, self.lock_root)) + } +} + +impl FirstName { + /// Derives the v1 first-name digest from a lock-root hash. + pub fn from_lock_root(lock_root: &Hash) -> Result { + Ok(Self(FirstNameDigestInput { lock_root }.hash_digest()?)) + } +} + +#[derive(Debug, thiserror::Error)] +pub enum SpendConditionFirstNameError { + #[error(transparent)] + LockHash(#[from] LockHashError), + #[error(transparent)] + FirstNameFromLockRoot(#[from] FirstNameFromLockRootError), +} + +impl SpendCondition { + /// Builds a simple single-signer PKH spend-condition. + pub fn simple_pkh(pkh: Hash) -> Self { + Self::new(vec![LockPrimitive::Pkh(Pkh::new(1, vec![pkh]))]) + } + + /// Builds a coinbase-style single-signer PKH spend-condition with a relative timelock. + pub fn coinbase_pkh(pkh: Hash, coinbase_relative_min: u64) -> Self { + let lock_tim = LockTim { + rel: TimelockRangeRelative::new( + Some(BlockHeightDelta(Belt(coinbase_relative_min))), + None, + ), + abs: TimelockRangeAbsolute::none(), + }; + Self::new(vec![ + LockPrimitive::Pkh(Pkh::new(1, vec![pkh])), + LockPrimitive::Tim(lock_tim), + ]) + } + + /// Computes the consensus spend-condition hash. + pub fn hash(&self) -> Result { + self.hash_digest() + } + + /// Computes the v1 note first-name from this spend-condition. + pub fn first_name(&self) -> Result { + let lock_root = Lock::SpendCondition(self.clone()).hash()?; + Ok(FirstName::from_lock_root(&lock_root)?) + } +} + +impl Hashable for SpendCondition { + type Error = LockHashError; + + fn hash_digest(&self) -> Result { + let mut tail = hash_leaf_null(); + for primitive in self.0.iter().rev() { + let head = primitive.hash_digest()?; + tail = hash_pair(&head, &tail); + } + Ok(tail) + } +} + +impl Hashable for LockPrimitive { + type Error = LockHashError; + + fn hash_digest(&self) -> Result { + match self { + Self::Pkh(pkh) => { + let tag = hash_leaf_atom(nockvm_macros::tas!(b"pkh"))?; + let payload = pkh.hash_digest()?; + Ok(hash_pair(&tag, &payload)) + } + Self::Tim(tim) => { + let tag = hash_leaf_atom(nockvm_macros::tas!(b"tim"))?; + let payload = tim.hash_digest()?; + Ok(hash_pair(&tag, &payload)) + } + Self::Hax(hax) => { + let tag = hash_leaf_atom(nockvm_macros::tas!(b"hax"))?; + let payload = hax.hash_digest()?; + Ok(hash_pair(&tag, &payload)) + } + Self::Burn => { + let burn_tag = hash_leaf_atom(nockvm_macros::tas!(b"brn"))?; + let null_leaf = hash_leaf_null(); + Ok(hash_pair(&burn_tag, &null_leaf)) + } + } + } +} + +impl Hashable for Pkh { + type Error = LockHashError; + + fn hash_digest(&self) -> Result { + let m_hashable = hash_leaf_atom(self.m)?; + let hashes_hashable = self.hashes.hash_with(&HashableTreeHasher); + Ok(hash_pair(&m_hashable, &hashes_hashable)) + } +} + +impl Hashable for Hax { + type Error = LockHashError; + + fn hash_digest(&self) -> Result { + Ok(self.0.hash_with(&HashableTreeHasher)) + } +} + +impl Hashable for LockTim { + type Error = LockHashError; + + fn hash_digest(&self) -> Result { + let rel_min = hash_unit_belt(self.rel.min.as_ref().map(|height| height.0)); + let rel_max = hash_unit_belt(self.rel.max.as_ref().map(|height| height.0)); + let abs_min = hash_unit_belt(self.abs.min.as_ref().map(|height| height.0)); + let abs_max = hash_unit_belt(self.abs.max.as_ref().map(|height| height.0)); + let rel = hash_pair(&rel_min, &rel_max); + let abs = hash_pair(&abs_min, &abs_max); + Ok(hash_pair(&rel, &abs)) + } +} + +impl Hashable for Lock { + type Error = LockHashError; + + fn hash_digest(&self) -> Result { + match self { + Self::SpendCondition(spend_condition) => spend_condition.hash_digest(), + Self::V2(v2) => { + let tag = hash_leaf_atom(2)?; + let payload = v2.hash_digest()?; + Ok(hash_pair(&tag, &payload)) + } + Self::V4(v4) => { + let tag = hash_leaf_atom(4)?; + let payload = v4.hash_digest()?; + Ok(hash_pair(&tag, &payload)) + } + Self::V8(v8) => { + let tag = hash_leaf_atom(8)?; + let payload = v8.hash_digest()?; + Ok(hash_pair(&tag, &payload)) + } + Self::V16(v16) => { + let tag = hash_leaf_atom(16)?; + let payload = v16.hash_digest()?; + Ok(hash_pair(&tag, &payload)) + } + } + } +} + +impl Hashable for LockV2 { + type Error = LockHashError; + + fn hash_digest(&self) -> Result { + let p_hash = self.p.hash()?; + let q_hash = self.q.hash()?; + Ok(hash_pair(&p_hash, &q_hash)) + } +} + +impl Hashable for LockV4 { + type Error = LockHashError; + + fn hash_digest(&self) -> Result { + let left = self.p.hash_digest()?; + let right = self.q.hash_digest()?; + Ok(hash_pair(&left, &right)) + } +} + +impl Hashable for LockV8 { + type Error = LockHashError; + + fn hash_digest(&self) -> Result { + let left = self.p.hash_digest()?; + let right = self.q.hash_digest()?; + Ok(hash_pair(&left, &right)) + } +} + +impl Hashable for LockV16 { + type Error = LockHashError; + + fn hash_digest(&self) -> Result { + let left = self.p.hash_digest()?; + let right = self.q.hash_digest()?; + Ok(hash_pair(&left, &right)) + } +} + +fn decode_zset(noun: &Noun, space: &NounSpace, mut f: F) -> Result, NounDecodeError> where - F: FnMut(&Noun) -> Result, + F: FnMut(&Noun, &NounSpace) -> Result, { - fn traverse(node: &Noun, acc: &mut Vec, f: &mut F) -> Result<(), NounDecodeError> + fn traverse( + node: &Noun, + space: &NounSpace, + acc: &mut Vec, + f: &mut F, + ) -> Result<(), NounDecodeError> where - F: FnMut(&Noun) -> Result, + F: FnMut(&Noun, &NounSpace) -> Result, { - if let Ok(atom) = node.as_atom() { + if let Ok(atom) = node.in_space(space).as_atom() { if atom.as_u64()? == 0 { return Ok(()); } @@ -607,20 +1057,217 @@ where } let cell = node + .in_space(space) .as_cell() .map_err(|_| NounDecodeError::Custom("z-set node must be a cell".into()))?; - acc.push(f(&cell.head())?); + let head_noun = cell.head().noun(); + acc.push(f(&head_noun, space)?); let branches = cell .tail() .as_cell() .map_err(|_| NounDecodeError::Custom("z-set branches must be a cell".into()))?; - traverse(&branches.head(), acc, f)?; - traverse(&branches.tail(), acc, f)?; + let left = branches.head().noun(); + let right = branches.tail().noun(); + traverse(&left, space, acc, f)?; + traverse(&right, space, acc, f)?; Ok(()) } let mut acc = Vec::new(); - traverse(noun, &mut acc, &mut f)?; + traverse(noun, space, &mut acc, &mut f)?; Ok(acc) } + +#[cfg(test)] +mod tests { + use nockapp::noun::slab::{NockJammer, NounSlab}; + use nockchain_math::belt::Belt; + use nockvm::noun::NounAllocator; + use noun_serde::{NounDecode, NounEncode}; + + use super::{Hax, Lock, LockPrimitive, LockTim, LockV2, LockV4, Pkh, SpendCondition}; + use crate::tx_engine::common::{ + BlockHeight, BlockHeightDelta, Hash, TimelockRangeAbsolute, TimelockRangeRelative, + }; + + const ADDRESS_A_B58: &str = "9yPePjfWAdUnzaQKyxcRXKRa5PpUzKKEwtpECBZsUYt9Jd7egSDEWoV"; + const ADDRESS_B_B58: &str = "9phXGACnW4238oqgvn2gpwaUjG3RAqcxq2Ash2vaKp8KjzSd3MQ56Jt"; + const EXPECTED_PKH_ROOT_B58: &str = "DKrgXqE8bXR1uBZ3t4vU13m2KquGCDbnn1PeoPL7dxSHTucGPFDPt53"; + const EXPECTED_MULTISIG_2_OF_2_ROOT_B58: &str = + "4eMAT3BuhLPjYFronoYJ9RSLVSgveCL3nQB7RHSLZzjBTiYCxEzkzEH"; + const EXPECTED_TIM_ROOT_B58: &str = "66FLtgznHvE7v4Fi4wZ6aA9EzsPD6pfaL3qL85apJuiBF8unRKXVsor"; + const EXPECTED_HAX_ROOT_B58: &str = "4kwz3RMCacfRXY3ydNoQ1tsUKuzaBEzGSpX9GpSWf8T3Rj24Ucuj6v4"; + const EXPECTED_LOCK_V2_ROOT_B58: &str = + "e3qeUqDf6ZTkayiiQDpKpax6RqXMBAMRLtrppvL41EdyJYFj743ZKB"; + const EXPECTED_LOCK_V4_ROOT_B58: &str = + "6ezbUN1ozEvZi9TUGVN1pY2TcCJc5KWoCzjj519ihE6LGupvJpnysjo"; + const EXPECTED_MEGA_LOCK_V4_ROOT_B58: &str = + "DaNZuUK5iHhkCiDt3UShiNbz79TLdoLyNs3dKjTX1yzEZ7tjwjzbe8U"; + const BRIDGE_ROOT_B58: &str = "AcsPkuhXQoGeEsF91yynpm1kcW17PQ2Z1MEozgx7YnDPkZwrtzLuuqd"; + + fn pkh_condition(m: u64, hashes: Vec) -> SpendCondition { + SpendCondition::new(vec![pkh_primitive(m, hashes)]) + } + + fn pkh_primitive(m: u64, hashes: Vec) -> LockPrimitive { + LockPrimitive::Pkh(Pkh::new(m, hashes)) + } + + fn tim_condition() -> SpendCondition { + SpendCondition::new(vec![tim_primitive(Some(3), Some(10), Some(20), None)]) + } + + fn tim_primitive( + rel_min: Option, + rel_max: Option, + abs_min: Option, + abs_max: Option, + ) -> LockPrimitive { + LockPrimitive::Tim(LockTim { + rel: TimelockRangeRelative { + min: rel_min.map(|value| BlockHeightDelta(Belt(value))), + max: rel_max.map(|value| BlockHeightDelta(Belt(value))), + }, + abs: TimelockRangeAbsolute { + min: abs_min.map(|value| BlockHeight(Belt(value))), + max: abs_max.map(|value| BlockHeight(Belt(value))), + }, + }) + } + + fn hax_condition(hashes: Vec) -> SpendCondition { + SpendCondition::new(vec![hax_primitive(hashes)]) + } + + fn hax_primitive(hashes: Vec) -> LockPrimitive { + LockPrimitive::Hax(Hax::new(hashes)) + } + + #[test] + fn lock_hash_matches_known_hoon_vectors() { + let address_a = Hash::from_base58(ADDRESS_A_B58).expect("address a should parse"); + let address_b = Hash::from_base58(ADDRESS_B_B58).expect("address b should parse"); + let bridge_root = Hash::from_base58(BRIDGE_ROOT_B58).expect("bridge root should parse"); + + let single_pkh_lock = Lock::SpendCondition(pkh_condition(1, vec![address_a.clone()])); + let multisig_lock = + Lock::SpendCondition(pkh_condition(2, vec![address_a.clone(), address_b.clone()])); + let tim_lock = Lock::SpendCondition(tim_condition()); + let hax_lock = + Lock::SpendCondition(hax_condition(vec![address_a.clone(), address_b.clone()])); + let lock_v2 = Lock::V2(LockV2 { + p: pkh_condition(1, vec![address_a.clone()]), + q: tim_condition(), + }); + let lock_v4 = Lock::V4(LockV4 { + p: LockV2 { + p: pkh_condition(1, vec![address_a.clone()]), + q: tim_condition(), + }, + q: LockV2 { + p: hax_condition(vec![address_a.clone(), address_b.clone()]), + q: SpendCondition::new(vec![LockPrimitive::Burn]), + }, + }); + let mega_lock_v4 = Lock::V4(LockV4 { + p: LockV2 { + p: SpendCondition::new(vec![ + pkh_primitive(2, vec![address_a.clone(), address_b.clone()]), + tim_primitive(Some(5), Some(15), Some(25), None), + hax_primitive(vec![address_a.clone(), bridge_root.clone()]), + ]), + q: SpendCondition::new(vec![ + hax_primitive(vec![address_b.clone()]), + tim_primitive(None, Some(8), None, Some(40)), + pkh_primitive(1, vec![address_b.clone()]), + ]), + }, + q: LockV2 { + p: SpendCondition::new(vec![ + pkh_primitive(1, vec![address_a.clone()]), + tim_primitive(Some(2), None, Some(30), Some(60)), + hax_primitive(vec![address_a.clone(), address_b.clone()]), + ]), + q: SpendCondition::new(vec![ + tim_primitive(Some(1), Some(4), Some(50), Some(90)), + hax_primitive(vec![address_a.clone()]), + pkh_primitive(1, vec![address_a.clone(), address_b.clone()]), + ]), + }, + }); + + assert_eq!( + single_pkh_lock + .hash() + .expect("single pkh lock hash should compute") + .to_base58(), + EXPECTED_PKH_ROOT_B58 + ); + assert_eq!( + multisig_lock + .hash() + .expect("multisig lock hash should compute") + .to_base58(), + EXPECTED_MULTISIG_2_OF_2_ROOT_B58 + ); + assert_eq!( + tim_lock + .hash() + .expect("tim lock hash should compute") + .to_base58(), + EXPECTED_TIM_ROOT_B58 + ); + assert_eq!( + hax_lock + .hash() + .expect("hax lock hash should compute") + .to_base58(), + EXPECTED_HAX_ROOT_B58 + ); + assert_eq!( + lock_v2 + .hash() + .expect("v2 lock hash should compute") + .to_base58(), + EXPECTED_LOCK_V2_ROOT_B58 + ); + assert_eq!( + lock_v4 + .hash() + .expect("v4 lock hash should compute") + .to_base58(), + EXPECTED_LOCK_V4_ROOT_B58 + ); + assert_eq!( + mega_lock_v4 + .hash() + .expect("mega v4 lock hash should compute") + .to_base58(), + EXPECTED_MEGA_LOCK_V4_ROOT_B58 + ); + } + + #[test] + fn lock_tree_roundtrip_preserves_leaf_count() { + fn pkh_with_value(value: u64) -> SpendCondition { + pkh_condition(1, vec![Hash::from_limbs(&[value, 0, 0, 0, 0])]) + } + let lock = Lock::V4(LockV4 { + p: LockV2 { + p: pkh_with_value(11), + q: pkh_with_value(12), + }, + q: LockV2 { + p: pkh_with_value(13), + q: pkh_with_value(14), + }, + }); + let mut slab: NounSlab = NounSlab::new(); + let noun = lock.to_noun(&mut slab); + let space = slab.noun_space(); + let decoded = Lock::from_noun(&noun, &space).expect("lock should decode"); + assert_eq!(decoded.spend_condition_count(), 4); + assert_eq!(decoded.flatten_spend_conditions().len(), 4); + } +} diff --git a/crates/nockchain-types/tests/balance_from_peek_v0.rs b/crates/nockchain-types/tests/balance_from_peek_v0.rs index b6e050302..e1d871997 100644 --- a/crates/nockchain-types/tests/balance_from_peek_v0.rs +++ b/crates/nockchain-types/tests/balance_from_peek_v0.rs @@ -4,6 +4,7 @@ use bytes::Bytes; use nockapp::noun::slab::NounSlab; use nockchain_math::belt::Belt; use nockchain_types::tx_engine::v0; +use nockvm::noun::NounAllocator; use noun_serde::{NounDecode, NounEncode}; #[test] @@ -12,9 +13,10 @@ fn decode_balance_from_peeks_and_snapshots_v1() -> Result<(), Box> = - Option::>::from_noun(&noun)?; + Option::>::from_noun(&noun, &space)?; assert!( matches!(double_option, Some(Some(_))), "jam should decode as Option>" @@ -58,14 +60,17 @@ fn decode_balance_from_peeks_and_snapshots_v1() -> Result<(), Box>::to_noun(&encoded_option, &mut option_slab); - let option_roundtrip = Option::>::from_noun(&option_noun)?; + let option_space = option_slab.noun_space(); + let option_roundtrip = + Option::>::from_noun(&option_noun, &option_space)?; assert_eq!(option_roundtrip, encoded_option); Ok(()) diff --git a/crates/nockchain-types/tests/balance_from_peek_v1.rs b/crates/nockchain-types/tests/balance_from_peek_v1.rs index b6e050302..e1d871997 100644 --- a/crates/nockchain-types/tests/balance_from_peek_v1.rs +++ b/crates/nockchain-types/tests/balance_from_peek_v1.rs @@ -4,6 +4,7 @@ use bytes::Bytes; use nockapp::noun::slab::NounSlab; use nockchain_math::belt::Belt; use nockchain_types::tx_engine::v0; +use nockvm::noun::NounAllocator; use noun_serde::{NounDecode, NounEncode}; #[test] @@ -12,9 +13,10 @@ fn decode_balance_from_peeks_and_snapshots_v1() -> Result<(), Box> = - Option::>::from_noun(&noun)?; + Option::>::from_noun(&noun, &space)?; assert!( matches!(double_option, Some(Some(_))), "jam should decode as Option>" @@ -58,14 +60,17 @@ fn decode_balance_from_peeks_and_snapshots_v1() -> Result<(), Box>::to_noun(&encoded_option, &mut option_slab); - let option_roundtrip = Option::>::from_noun(&option_noun)?; + let option_space = option_slab.noun_space(); + let option_roundtrip = + Option::>::from_noun(&option_noun, &option_space)?; assert_eq!(option_roundtrip, encoded_option); Ok(()) diff --git a/crates/nockchain-types/tests/raw_tx_from_jam_v0.rs b/crates/nockchain-types/tests/raw_tx_from_jam_v0.rs index 01f1afa9d..1627c69f5 100644 --- a/crates/nockchain-types/tests/raw_tx_from_jam_v0.rs +++ b/crates/nockchain-types/tests/raw_tx_from_jam_v0.rs @@ -2,6 +2,7 @@ use bytes::Bytes; use nockapp::noun::slab::NounSlab; use nockchain_math::belt::Belt; use nockchain_types::tx_engine::v0; +use nockvm::noun::NounAllocator; use noun_serde::{NounDecode, NounEncode}; #[test] @@ -10,8 +11,9 @@ fn decode_raw_tx_from_jam_v0() -> Result<(), Box> { let mut slab: NounSlab = NounSlab::new(); let noun = slab.cue_into(Bytes::from_static(RAW_TX_JAM))?; + let space = slab.noun_space(); - let raw_tx = v0::RawTx::from_noun(&noun)?; + let raw_tx = v0::RawTx::from_noun(&noun, &space)?; // basic structural checks assert_eq!(raw_tx.inputs.0.len(), 10, "expected ten named inputs"); @@ -40,7 +42,8 @@ fn decode_raw_tx_from_jam_v0() -> Result<(), Box> { // noun roundtrip let mut encode_slab: NounSlab = NounSlab::new(); let encoded = v0::RawTx::to_noun(&raw_tx, &mut encode_slab); - let round_trip = v0::RawTx::from_noun(&encoded)?; + let encode_space = encode_slab.noun_space(); + let round_trip = v0::RawTx::from_noun(&encoded, &encode_space)?; assert_eq!(round_trip, raw_tx); Ok(()) @@ -52,15 +55,17 @@ fn decode_note_from_jam_v0() -> Result<(), Box> { let mut slab: NounSlab = NounSlab::new(); let noun = slab.cue_into(Bytes::from_static(NOTE_JAM))?; + let space = slab.noun_space(); - let note = v0::NoteV0::from_noun(&noun)?; + let note = v0::NoteV0::from_noun(&noun, &space)?; // basic structural checks // noun roundtrip let mut encode_slab: NounSlab = NounSlab::new(); let encoded = v0::NoteV0::to_noun(¬e, &mut encode_slab); - let round_trip = v0::NoteV0::from_noun(&encoded)?; + let encode_space = encode_slab.noun_space(); + let round_trip = v0::NoteV0::from_noun(&encoded, &encode_space)?; assert_eq!(round_trip, note); Ok(()) diff --git a/crates/nockchain-types/tests/raw_tx_from_jam_v1.rs b/crates/nockchain-types/tests/raw_tx_from_jam_v1.rs index 7dc9829c4..b0f85dfe0 100644 --- a/crates/nockchain-types/tests/raw_tx_from_jam_v1.rs +++ b/crates/nockchain-types/tests/raw_tx_from_jam_v1.rs @@ -1,18 +1,104 @@ +use std::collections::BTreeMap; + use bytes::Bytes; use nockapp::noun::slab::NounSlab; use nockchain_math::belt::Belt; use nockchain_types::common::{BlockHeight, Version}; use nockchain_types::tx_engine::v1; +use nockvm::noun::NounAllocator; use noun_serde::{NounDecode, NounEncode}; +// These constants are pinned to the checked-in raw tx fixture at: +// open/crates/nockchain-types/jams/v1/raw-tx.jam +const EXPECTED_SEED_COUNT: u64 = 7; +const EXPECTED_WITNESS_COUNT: u64 = 133; +const EXPECTED_TX_WORD_COUNT: u64 = 140; +const EXPECTED_MINIMUM_FEE: u64 = 659_456; +const EXPECTED_TOTAL_PAID_FEE: u64 = 1_024; + +fn noun_leaf_count(noun: nockapp::Noun, space: &nockvm::noun::NounSpace) -> u64 { + if noun.is_atom() { + return 1; + } + let cell = noun + .in_space(space) + .as_cell() + .expect("noun should decode as cell"); + noun_leaf_count(cell.head().noun(), space) + .saturating_add(noun_leaf_count(cell.tail().noun(), space)) +} + +fn word_count_from_noun_encode(value: &T) -> u64 { + let mut slab: NounSlab = NounSlab::new(); + let noun = value.to_noun(&mut slab); + let space = slab.noun_space(); + noun_leaf_count(noun, &space) +} + +fn merged_seed_word_count(raw_tx: &v1::RawTx) -> u64 { + let mut merged_by_lock_root = BTreeMap::<[u64; 5], BTreeMap>::new(); + + for (_, spend) in &raw_tx.spends.0 { + let seeds = match spend { + v1::Spend::Legacy(spend0) => &spend0.seeds.0, + v1::Spend::Witness(spend1) => &spend1.seeds.0, + }; + + for seed in seeds { + let merged = merged_by_lock_root + .entry(seed.lock_root.to_array()) + .or_default(); + for note_data_entry in seed.note_data.iter() { + merged.insert(note_data_entry.key.clone(), note_data_entry.blob.clone()); + } + } + } + + merged_by_lock_root + .into_values() + .map(|merged| { + let entries = merged + .into_iter() + .map(|(key, blob)| v1::note::NoteDataEntry::new(key, blob)) + .collect::>(); + word_count_from_noun_encode(&v1::note::NoteData::new(entries)) + }) + .sum() +} + +fn witness_word_count(raw_tx: &v1::RawTx) -> u64 { + raw_tx + .spends + .0 + .iter() + .map(|(_, spend)| match spend { + v1::Spend::Legacy(spend0) => word_count_from_noun_encode(&spend0.signature), + v1::Spend::Witness(spend1) => word_count_from_noun_encode(&spend1.witness), + }) + .sum() +} + +fn total_paid_fee(raw_tx: &v1::RawTx) -> u64 { + raw_tx + .spends + .0 + .iter() + .map(|(_, spend)| match spend { + v1::Spend::Legacy(spend0) => spend0.fee.0 as u64, + v1::Spend::Witness(spend1) => spend1.fee.0 as u64, + }) + .sum() +} + #[test] fn decode_raw_tx_from_jam_v1() -> Result<(), Box> { const RAW_TX_JAM: &[u8] = include_bytes!("../jams/v1/raw-tx.jam"); let mut slab: NounSlab = NounSlab::new(); let noun = slab.cue_into(Bytes::from_static(RAW_TX_JAM))?; + let space = slab.noun_space(); - let raw_tx = v1::RawTx::from_noun(&noun)?; + let raw_tx = v1::RawTx::from_noun(&noun, &space)?; // basic structural checks assert_eq!(raw_tx.version, Version::V1); @@ -20,24 +106,53 @@ fn decode_raw_tx_from_jam_v1() -> Result<(), Box> { // noun roundtrip let mut encode_slab: NounSlab = NounSlab::new(); let encoded = v1::RawTx::to_noun(&raw_tx, &mut encode_slab); - let round_trip = v1::RawTx::from_noun(&encoded)?; + let encode_space = encode_slab.noun_space(); + let round_trip = v1::RawTx::from_noun(&encoded, &encode_space)?; assert_eq!(round_trip, raw_tx); Ok(()) } +#[test] +fn decode_raw_tx_word_count_oracle_v1() -> Result<(), Box> { + const RAW_TX_JAM: &[u8] = include_bytes!("../jams/v1/raw-tx.jam"); + + let mut slab: NounSlab = NounSlab::new(); + let noun = slab.cue_into(Bytes::from_static(RAW_TX_JAM))?; + let space = slab.noun_space(); + let raw_tx = v1::RawTx::from_noun(&noun, &space)?; + + let seed_count = merged_seed_word_count(&raw_tx); + let witness_count = witness_word_count(&raw_tx); + let tx_word_count = seed_count.saturating_add(witness_count); + + // pending-integration defaults in tx-engine: + // base_fee = 2^14, input_fee_divisor = 4, min_fee_floor = 256 + let base_fee: u64 = 1 << 14; + let input_fee_divisor: u64 = 4; + let min_fee_floor: u64 = 256; + let word_fee = seed_count + .saturating_mul(base_fee) + .saturating_add(witness_count.saturating_mul(base_fee) / input_fee_divisor); + let minimum_fee = word_fee.max(min_fee_floor); + + assert_eq!(seed_count, EXPECTED_SEED_COUNT); + assert_eq!(witness_count, EXPECTED_WITNESS_COUNT); + assert_eq!(tx_word_count, EXPECTED_TX_WORD_COUNT); + assert_eq!(minimum_fee, EXPECTED_MINIMUM_FEE); + assert_eq!(total_paid_fee(&raw_tx), EXPECTED_TOTAL_PAID_FEE); + Ok(()) +} + #[test] fn decode_note_from_jam_v1() -> Result<(), Box> { const NOTE_JAM: &[u8] = include_bytes!("../jams/v1/note.jam"); let mut slab: NounSlab = NounSlab::new(); let noun = slab.cue_into(Bytes::from_static(NOTE_JAM))?; + let space = slab.noun_space(); - eprintln!("decoding note"); - let ver = noun.as_cell().expect("not a cell").head(); - eprintln!("version: {:?}", ver); - let note = v1::Note::from_noun(&noun)?; - eprintln!("decoded note"); + let note = v1::Note::from_noun(&noun, &space)?; // basic structural checks match note { @@ -50,7 +165,8 @@ fn decode_note_from_jam_v1() -> Result<(), Box> { // noun roundtrip let mut encode_slab: NounSlab = NounSlab::new(); let encoded = v1::Note::to_noun(¬e, &mut encode_slab); - let round_trip = v1::Note::from_noun(&encoded)?; + let encode_space = encode_slab.noun_space(); + let round_trip = v1::Note::from_noun(&encoded, &encode_space)?; assert_eq!(round_trip, note); Ok(()) @@ -62,12 +178,9 @@ fn decode_name_from_jam_v1() -> Result<(), Box> { let mut slab: NounSlab = NounSlab::new(); let noun = slab.cue_into(Bytes::from_static(NOTE_JAM))?; + let space = slab.noun_space(); - eprintln!("decoding note"); - let ver = noun.as_cell().expect("not a cell").head(); - eprintln!("version: {:?}", ver); - let note = v1::Note::from_noun(&noun)?; - eprintln!("decoded note"); + let note = v1::Note::from_noun(&noun, &space)?; // basic structural checks match note { @@ -80,7 +193,8 @@ fn decode_name_from_jam_v1() -> Result<(), Box> { // noun roundtrip let mut encode_slab: NounSlab = NounSlab::new(); let encoded = v1::Note::to_noun(¬e, &mut encode_slab); - let round_trip = v1::Note::from_noun(&encoded)?; + let encode_space = encode_slab.noun_space(); + let round_trip = v1::Note::from_noun(&encoded, &encode_space)?; assert_eq!(round_trip, note); Ok(()) diff --git a/crates/nockchain-wallet/Cargo.toml b/crates/nockchain-wallet/Cargo.toml index dfa49d303..2d02e5e34 100644 --- a/crates/nockchain-wallet/Cargo.toml +++ b/crates/nockchain-wallet/Cargo.toml @@ -30,4 +30,5 @@ tokio = { workspace = true, features = ["full"] } tonic = { workspace = true } tracing = { workspace = true } tracing-subscriber = { workspace = true } +wallet-tx-builder = { path = "../wallet-tx-builder" } zkvm-jetpack = { workspace = true } diff --git a/crates/nockchain-wallet/README.md b/crates/nockchain-wallet/README.md index bd005a18f..e868a9209 100644 --- a/crates/nockchain-wallet/README.md +++ b/crates/nockchain-wallet/README.md @@ -2,7 +2,7 @@ Status: Active Owner: Nockchain Maintainers -Last Reviewed: 2026-02-19 +Last Reviewed: 2026-04-01 Canonical/Legacy: Canonical (Tier 1 scoped authority for wallet CLI behavior and operational usage; protocol authority remains in [`PROTOCOL.md`](../../PROTOCOL.md)) ## Canonical Scope @@ -218,6 +218,10 @@ Displays the aggregate wallet balance, including the total number of notes and t We support transactions with any amount of input notes going to any number of recipients. ```bash +# Auto-select spendable notes and compute fee +nockchain-wallet create-tx \ + --recipient '{"kind":"p2pkh","address":"","amount":10000}' + # Send to a single P2PKH recipient nockchain-wallet create-tx \ --names "[first1 last1],[first2 last2]" \ @@ -235,12 +239,88 @@ Gifts and fees are denominated in nicks (65536 nicks = 1 nock). #### Common Parameters -- The `names` argument is a list of `[first-name last-name]` pairs specifying funding notes -- The `fee` argument is the transaction fee to pay (in nicks, 65536 nicks to 1 nock) +- The optional `names` argument is a list of `[first-name last-name]` pairs for manual note selection; omit it to auto-select spendable notes +- Auto-selection remains v1-only +- Manual `--names` selection may spend either an all-v1 set or an all-v0 set; mixed-version manual sets are rejected +- The optional `fee` argument overrides the planner-computed fee (in nicks, 65536 nicks to 1 nock) - Provide multiple `--recipient` flags to fan out to several outputs - Each `--recipient` is either a JSON object (preferred) or a legacy `:` string - `address`/`addresses` fields expect base58-encoded pay-to-pubkey-hash values - Provide `--sign-key ` multiple times to explicitly choose signing keys. If omitted, the wallet uses the master key or the `--index/--hardened` pair. +- `--refund-pkh` is required when manually spending legacy v0 notes. For v1 notes, refund defaults to the note owner. + +### Migrating Legacy V0 Notes + +Use `migrate-v0-notes` when you want to sweep spendable legacy v0 notes into a v1 pay-to-pubkey-hash address. + +```bash +nockchain-wallet migrate-v0-notes --destination +``` + +What the command does: + +- Syncs the wallet and finds spendable v0 notes for the active v0 master and any active v0 child signers under that master +- Ignores v1 notes +- Computes the required fee for each signer bucket +- Builds one migration transaction per spend-capable signer bucket +- Writes each saved transaction to `./txs` in the current working directory +- Uses the destination as the refund target, so any leftover value also comes back as v1 +- Prints a signer-by-signer summary with the saved tx path, selected inputs, fee, expected migrated amount, and the exact `send-tx` command for each created transaction + +Typical migration flow: + +```bash +# 1. Pull the latest wallet code +git pull origin master + +# 2. Rebuild the wallet jams and binary +make install-nockchain-wallet + +# 3. Import the legacy seed if you have not already done so +nockchain-wallet import-keys \ + --seedphrase "your legacy seed phrase here" \ + --version 0 + +# 4. Confirm the legacy master address is present +nockchain-wallet list-master-addresses + +# 5. Switch the active master to the legacy v0 master key that owns the signer tree +nockchain-wallet set-active-master-address + +# 6. Run the migration sweep into your v1 P2PKH destination +nockchain-wallet migrate-v0-notes --destination + +# 7. Submit each saved transaction shown in the migration summary +nockchain-wallet send-tx +``` + +Notes: + +- The destination must be a v1 pay-to-pubkey-hash address +- The command is full-sweep only in the current release +- The command may create multiple transactions, not just one: it creates up to one migration tx per active local v0 signer under the active master +- Inspect the migration summary before submitting anything. It tells you which signer each tx belongs to, how many notes were selected, the fee, the expected migrated amount, where the tx was saved, and how to submit it +- Watch-only imports are not enough; the wallet must hold the matching v0 signing key +- If you are using the bridge helper scripts, `open/crates/bridge/scripts/wallet.sh --new` imports both the default v1 fakenet key and the legacy v0 fakenet key + +### Manual V0 Fan-In With `create-tx` + +If you need to pin the exact legacy inputs instead of sweeping every spendable v0 note, you can still use `create-tx` with a manual `--names` set, as long as every selected note is v0 and you provide `--refund-pkh`. + +```bash +nockchain-wallet create-tx \ + --names "[first1 last1],[first2 last2]" \ + --recipient '{"kind":"p2pkh","address":"","amount":10000}' \ + --refund-pkh +``` + +Rules for manual legacy spends: + +- Every selected note must be v0 +- Mixed v0/v1 manual sets are rejected +- `--refund-pkh` is required +- Fee may be planner-computed or overridden with `--fee` +- Omit `--names` if you want normal auto-selection; auto-selection does not pick v0 notes #### Recipient JSON Format diff --git a/crates/nockchain-wallet/src/command.rs b/crates/nockchain-wallet/src/command.rs index 30d7d1086..f32a62df2 100644 --- a/crates/nockchain-wallet/src/command.rs +++ b/crates/nockchain-wallet/src/command.rs @@ -171,6 +171,7 @@ impl FromStr for TimelockRangeCli { } } +/// CLI-facing note selection strategy for create-tx ordering. #[derive(Copy, Clone, Debug, ValueEnum)] pub enum NoteSelectionStrategyCli { Ascending, @@ -186,6 +187,7 @@ impl NoteSelectionStrategyCli { } } +/// Top-level wallet CLI definition. #[derive(Parser, Debug, Clone)] #[command(author, version, about, long_about = None)] pub struct WalletCli { @@ -202,6 +204,7 @@ pub struct WalletCli { pub command: Commands, } +/// Supported watch subcommands for addresses and lock forms. #[derive(Subcommand, Debug, Clone)] pub enum WatchSubcommand { /// Add a watch-only address (base58 pkh or schnorr pubkey) @@ -233,6 +236,7 @@ pub enum WatchSubcommand { }, } +/// gRPC client mode used for wallet network operations. #[derive(clap::ValueEnum, Debug, Clone, PartialEq, Eq)] pub enum ClientType { Public, @@ -241,6 +245,7 @@ pub enum ClientType { #[derive(Debug)] #[allow(dead_code)] +/// Internal wallet event wires used for nockapp routing. pub enum WalletWire { ListNotes, UpdateBalance, @@ -270,6 +275,7 @@ impl Wire for WalletWire { /// Represents a Noun that the wallet kernel can handle pub type CommandNoun = Result<(T, Operation), NockAppError>; +/// Validates label strings accepted by key-derivation CLI paths. fn validate_label(s: &str) -> Result { if s.chars() .all(|c| c.is_ascii_lowercase() || c.is_ascii_digit() || c == '-') @@ -281,6 +287,7 @@ fn validate_label(s: &str) -> Result { } #[derive(Subcommand, Debug, Clone)] +/// Wallet command surface for key, note, and transaction operations. pub enum Commands { /// Generates a new version 1 key pair Keygen, @@ -371,12 +378,12 @@ pub enum Commands { /// Create a transaction (use --refund-pkh when spending legacy v0 notes) #[command( name = "create-tx", - override_usage = "nockchain-wallet create-tx --names --recipient ... --fee [--refund-pkh ] [--include-data ]\n\n# NOTE: --refund-pkh is required when spending from v0 notes. For v1 notes, the refund defaults to the note owner. --include-data defaults to true (pass 'false' to exclude note data).\n# RECIPIENT accepts either legacy ':' strings or JSON objects like '{\"kind\":\"multisig\",\"threshold\":2,\"addresses\":[\"pkh-a\",\"pkh-b\"],\"amount\":9000}'.\n\nExamples:\n # Pay a simple recipient\n nockchain-wallet create-tx \\\n --names \"[first1 last1],[first2 last2]\" \\\n --recipient '{\"kind\":\"p2pkh\",\"address\":\"\",\"amount\":10000}' \\\n --fee 10 \\\n --refund-pkh \n\n # Create a multisig recipient\n nockchain-wallet create-tx \\\n --names \"[first1 last1],[first2 last2]\" \\\n --recipient '{\"kind\":\"multisig\",\"threshold\":2,\"addresses\":[\"\",\"\",\"\"],\"amount\":9000}' \\\n --fee 10" + override_usage = "nockchain-wallet create-tx [--names ] --recipient ... [--fee ] [--refund-pkh ] [--include-data ]\n\n# NOTE: --refund-pkh is required when spending from legacy v0 notes. For v1 notes, the refund defaults to the note owner. --include-data defaults to true (pass 'false' to exclude note data).\n# NOTE: if --names is omitted, the planner auto-selects spendable v1 notes. If provided, names are treated as a manual selection set.\n# NOTE: manual selection may spend either an all-v1 set or an all-v0 set; mixed-version manual sets are rejected.\n# NOTE: --fee is optional. If omitted, the planner computes a fee. If provided, it overrides the planner fee (subject to --allow-low-fee).\n# RECIPIENT accepts either legacy ':' strings or JSON objects like '{\"kind\":\"multisig\",\"threshold\":2,\"addresses\":[\"pkh-a\",\"pkh-b\"],\"amount\":9000}'.\n\nExamples:\n # Auto-select spendable v1 notes and compute fee\n nockchain-wallet create-tx \\\n --recipient '{\"kind\":\"p2pkh\",\"address\":\"\",\"amount\":10000}'\n\n # Manually pin notes and optionally override fee\n nockchain-wallet create-tx \\\n --names \"[first1 last1],[first2 last2]\" \\\n --recipient '{\"kind\":\"p2pkh\",\"address\":\"\",\"amount\":10000}' \\\n --fee 10" )] CreateTx { - /// Names of notes to spend (comma-separated) + /// Optional names of notes to spend (comma-separated) for manual selection. #[arg(long)] - names: String, + names: Option, /// Recipient specifications (repeat --recipient for each output) #[arg( long = "recipient", @@ -385,9 +392,9 @@ pub enum Commands { action = ArgAction::Append )] recipients: Vec, - /// Transaction fee + /// Optional transaction fee override. #[arg(long)] - fee: u64, + fee: Option, /// Allow fees below the estimated minimum (unsafe, testing only) #[arg(long, default_value = "false")] allow_low_fee: bool, @@ -420,6 +427,14 @@ pub enum Commands { note_selection_strategy: NoteSelectionStrategyCli, }, + /// Sweep all spendable legacy v0 notes into one v1 destination address. + #[command(name = "migrate-v0-notes")] + MigrateV0Notes { + /// Base58-encoded v1 pay-to-pubkey-hash address that receives the migrated funds. + #[arg(long = "destination", value_name = "DESTINATION")] + destination: String, + }, + /// Sign a multisig transaction SignMultisigTx { /// Path to transaction file @@ -579,6 +594,7 @@ impl Commands { Commands::ListNotesByAddressCsv { .. } => "list-notes-by-address-csv", Commands::SetActiveMasterAddress { .. } => "set-active-master-address", Commands::CreateTx { .. } => "create-tx", + Commands::MigrateV0Notes { .. } => "migrate-v0-notes", Commands::SignMultisigTx { .. } => "sign-multisig-tx", Commands::SendTx { .. } => "send-tx", Commands::ShowTx { .. } => "show-tx", @@ -605,3 +621,74 @@ impl Commands { } } } + +#[cfg(test)] +mod tests { + use super::*; + + const SAMPLE_P2PKH: &str = "9yPePjfWAdUnzaQKyxcRXKRa5PpUzKKEwtpECBZsUYt9Jd7egSDEWoV"; + + #[test] + fn create_tx_defaults_to_ascending_note_selection() { + let cli = WalletCli::try_parse_from([ + "nockchain-wallet", + "create-tx", + "--recipient", + &format!("{SAMPLE_P2PKH}:100"), + ]) + .expect("create-tx CLI should parse"); + + let Commands::CreateTx { + note_selection_strategy, + .. + } = cli.command + else { + panic!("expected create-tx command"); + }; + + assert!(matches!( + note_selection_strategy, + NoteSelectionStrategyCli::Ascending + )); + } + + #[test] + fn create_tx_accepts_descending_note_selection_override() { + let cli = WalletCli::try_parse_from([ + "nockchain-wallet", + "create-tx", + "--recipient", + &format!("{SAMPLE_P2PKH}:100"), + "--note-selection", + "descending", + ]) + .expect("create-tx CLI should parse"); + + let Commands::CreateTx { + note_selection_strategy, + .. + } = cli.command + else { + panic!("expected create-tx command"); + }; + + assert!(matches!( + note_selection_strategy, + NoteSelectionStrategyCli::Descending + )); + } + + #[test] + fn migrate_v0_notes_requires_destination() { + let cli = WalletCli::try_parse_from([ + "nockchain-wallet", "migrate-v0-notes", "--destination", SAMPLE_P2PKH, + ]) + .expect("migrate-v0-notes CLI should parse"); + + let Commands::MigrateV0Notes { destination } = cli.command else { + panic!("expected migrate-v0-notes command"); + }; + + assert_eq!(destination, SAMPLE_P2PKH); + } +} diff --git a/crates/nockchain-wallet/src/connection.rs b/crates/nockchain-wallet/src/connection.rs index dcbe9acf4..2e5bbf0e5 100644 --- a/crates/nockchain-wallet/src/connection.rs +++ b/crates/nockchain-wallet/src/connection.rs @@ -4,6 +4,7 @@ use nockapp::noun::slab::NounSlab; use nockapp::NockAppError; use nockapp_grpc::{private_nockapp, public_nockchain}; use tracing::info; +use wallet_tx_builder::adapter::NormalizedSnapshot; use crate::command::ClientType; use crate::Wallet; @@ -143,12 +144,20 @@ impl GrpcTarget { } } +/// Output of one wallet balance sync round. +pub(crate) struct BalanceSyncResult { + /// Pokes that must be applied to the wallet kernel to update synced state. + pub pokes: Vec, + /// Optional deduplicated snapshot from the sync source for planner-driven create-tx. + pub normalized_snapshot: Option, +} + pub(crate) async fn sync_wallet_balance( wallet: &mut Wallet, target: &GrpcTarget, pubkeys: Vec, tracked_names: Vec, -) -> Result, NockAppError> { +) -> Result { match target { GrpcTarget::Private { endpoint } => { let mut client = private_nockapp::PrivateNockAppGrpcClient::connect(endpoint.clone()) diff --git a/crates/nockchain-wallet/src/create_tx.rs b/crates/nockchain-wallet/src/create_tx.rs new file mode 100644 index 000000000..e721b5bf6 --- /dev/null +++ b/crates/nockchain-wallet/src/create_tx.rs @@ -0,0 +1,1874 @@ +use std::path::{Path, PathBuf}; + +use nockapp::Bytes; +use nockchain_math::noun_ext::NounMathExtHandle; +use nockchain_math::zoon::zmap::ZMap; +use nockchain_types::tx_engine::common::Signature; +use nockvm::noun::NounSpace; +use wallet_tx_builder::types::CandidateNote; + +use super::*; + +pub(crate) fn ensure_manual_planner_parity( + requested_names: &[Name], + planned_names: &[Name], +) -> Result<(), String> { + let mut normalized_requested = requested_names + .iter() + .map(|name| (name.first.to_array(), name.last.to_array())) + .collect::>(); + let mut normalized_planned = planned_names + .iter() + .map(|name| (name.first.to_array(), name.last.to_array())) + .collect::>(); + normalized_requested.sort_unstable(); + normalized_planned.sort_unstable(); + + if normalized_planned != normalized_requested { + let planned_names_arg = Wallet::format_note_names_for_create_tx(planned_names); + let requested_names_arg = Wallet::format_note_names_for_create_tx(requested_names); + return Err(format!( + "planner parity mismatch: selected names differ from user-provided manual names (planned='{}', requested='{}')", + planned_names_arg, requested_names_arg + )); + } + Ok(()) +} + +#[derive(Debug, Clone, NounEncode, NounDecode)] +/// Subset of chain note-data constants consumed by planner fee logic. +pub(crate) struct PlannerNoteDataConstantsNoun { + pub(crate) _max_size: u64, + pub(crate) min_fee: u64, +} + +#[derive(Debug, Clone, NounEncode)] +/// Blockchain constants payload extracted from wallet state for planning. +pub(crate) struct PlannerBlockchainConstantsNoun { + pub(crate) _v1_phase: u64, + pub(crate) bythos_phase: u64, + pub(crate) data: PlannerNoteDataConstantsNoun, + pub(crate) base_fee: u64, + pub(crate) input_fee_divisor: u64, + pub(crate) coinbase_timelock_min: u64, +} + +impl NounDecode for PlannerBlockchainConstantsNoun { + fn from_noun(noun: &Noun, space: &NounSpace) -> Result { + let fields = noun.in_space(space).uncell::<6>()?; + let legacy_or_timelock = fields[5]; + let coinbase_timelock_min = if let Ok(value) = u64::from_noun_handle(&legacy_or_timelock) { + value + } else { + let legacy_fields = match legacy_or_timelock.uncell::<13>() { + Ok(fields) => fields, + Err(_) => { + let wrapped = legacy_or_timelock.as_cell()?; + wrapped.head().uncell::<13>()? + } + }; + u64::from_noun_handle(&legacy_fields[9])? + }; + + Ok(Self { + _v1_phase: u64::from_noun_handle(&fields[0])?, + bythos_phase: u64::from_noun_handle(&fields[1])?, + data: PlannerNoteDataConstantsNoun::from_noun_handle(&fields[2])?, + base_fee: u64::from_noun_handle(&fields[3])?, + input_fee_divisor: u64::from_noun_handle(&fields[4])?, + coinbase_timelock_min, + }) + } +} + +#[derive(Debug, Clone, NounEncode, NounDecode, PartialEq, Eq)] +pub(crate) struct ActiveSignerEntryNoun { + pub(crate) child_index: Option, + pub(crate) hardened: bool, + pub(crate) absolute_index: Option, + pub(crate) version: u64, + pub(crate) pubkey: SchnorrPubkey, + pub(crate) address_b58: String, +} + +impl ActiveSignerEntryNoun { + fn is_master(&self) -> bool { + self.child_index.is_none() + } + + fn sign_keys(&self) -> Vec<(u64, bool)> { + self.child_index + .map(|index| vec![(index, self.hardened)]) + .unwrap_or_default() + } + + fn sort_key(&self) -> (u8, u64, String) { + ( + if self.is_master() { 0 } else { 1 }, + self.absolute_index.unwrap_or(0), + self.address_b58.clone(), + ) + } + + fn label(&self) -> String { + match self.child_index { + Some(index) => { + let hardened = if self.hardened { + "hardened" + } else { + "unhardened" + }; + format!("child({index}:{hardened})") + } + None => "master".to_string(), + } + } +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub(crate) struct MigrateV0SignerSummary { + pub(crate) signer: ActiveSignerEntryNoun, + pub(crate) note_count: usize, + pub(crate) selected_total: u64, + pub(crate) fee: Option, + pub(crate) migrated_amount: Option, + pub(crate) tx_path: Option, + pub(crate) skip_reason: Option, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub(crate) struct MigrateV0NotesSummary { + pub(crate) destination: String, + pub(crate) block_id: String, + pub(crate) height: u64, + pub(crate) examined_signers: usize, + pub(crate) created_count: usize, + pub(crate) skipped_count: usize, + pub(crate) signers: Vec, +} + +#[cfg(test)] +#[derive(Debug, Clone, NounDecode)] +struct BatchWriteRequestEntry { + path: String, + contents: Bytes, +} + +#[cfg(test)] +#[derive(Debug, Clone, Default, PartialEq, Eq)] +struct AppliedWalletEffects { + tx_paths: Vec, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +struct TxFileSnapshot { + modified: Option, + len: u64, +} + +#[derive(Debug, Clone, Default)] +pub(crate) struct WrittenTxSnapshot(BTreeMap); + +#[derive(Debug, Clone)] +struct CreateTxRequest { + names: String, + recipients: Vec, + fee: u64, + allow_low_fee: bool, + refund_pkh: Option, + sign_keys: Vec<(u64, bool)>, + include_data: bool, + save_raw_tx: bool, + note_selection: NoteSelectionStrategyCli, +} + +#[derive(Debug, Clone)] +struct PendingMigrationTx { + summary_index: usize, + planned_names: Vec, + request: CreateTxRequest, +} + +pub(crate) struct PreparedMigrateV0Notes { + pub(crate) summary: MigrateV0NotesSummary, + poke: Option<(NounSlab, Operation)>, + pending_txs: Vec, +} + +impl PreparedMigrateV0Notes { + pub(crate) fn take_poke(&mut self) -> Option<(NounSlab, Operation)> { + self.poke.take() + } + + fn normalized_name_key(names: &[Name]) -> Vec<([u64; 5], [u64; 5])> { + let mut key = names + .iter() + .map(|name| (name.first.to_array(), name.last.to_array())) + .collect::>(); + key.sort_unstable(); + key + } + + fn assign_tx_paths(&mut self, tx_paths: Vec) -> Result<(), NockAppError> { + if tx_paths.len() != self.pending_txs.len() { + return Err(NockAppError::OtherError(format!( + "migrate-v0-notes expected {} saved transaction files, but found {}", + self.pending_txs.len(), + tx_paths.len() + ))); + } + + let mut expected_by_name_set = BTreeMap::, usize>::new(); + for pending in &self.pending_txs { + let key = Self::normalized_name_key(&pending.planned_names); + if expected_by_name_set + .insert(key, pending.summary_index) + .is_some() + { + return Err(NockAppError::OtherError( + "migrate-v0-notes found duplicate planned note sets while matching saved transactions".to_string(), + )); + } + } + + let mut assigned = BTreeMap::::new(); + for tx_path in tx_paths { + let spends = Wallet::decode_transaction_spends_from_path(&tx_path)?; + let tx_name_key = Self::normalized_name_key( + &spends + .0 + .iter() + .map(|(name, _)| name.clone()) + .collect::>(), + ); + let Some(summary_index) = expected_by_name_set.get(&tx_name_key).copied() else { + return Err(NockAppError::OtherError(format!( + "migrate-v0-notes could not match saved transaction '{}' to any planned signer batch", + tx_path + ))); + }; + if assigned.insert(summary_index, tx_path.clone()).is_some() { + return Err(NockAppError::OtherError(format!( + "migrate-v0-notes matched more than one saved transaction to signer summary index {}", + summary_index + ))); + } + } + + for pending in &self.pending_txs { + let Some(tx_path) = assigned.remove(&pending.summary_index) else { + return Err(NockAppError::OtherError(format!( + "migrate-v0-notes did not find a saved transaction for signer summary index {}", + pending.summary_index + ))); + }; + self.summary.signers[pending.summary_index].tx_path = Some(tx_path); + } + + Ok(()) + } + + pub(crate) fn finalize( + mut self, + tx_paths: Vec, + ) -> Result { + if !self.pending_txs.is_empty() { + self.assign_tx_paths(tx_paths)?; + } + Ok(self.summary) + } +} + +impl PlannerBlockchainConstantsNoun { + /// Returns the consensus coinbase relative timelock minimum. + pub(crate) fn coinbase_timelock_min(&self) -> Result { + Ok(self.coinbase_timelock_min) + } +} + +#[derive(Debug, Clone, Default)] +/// Lock matcher for simple single-signer PKH lock resolution. +/// +/// This matcher is intentionally scoped to single-signer PKH spend conditions +/// that can be satisfied by locally held signer keys. +/// Multisig or otherwise complex lock forms are intentionally not matched here. +pub(crate) struct SigningKeyLockMatcher { + signer_pkhs: std::collections::BTreeSet<[u64; 5]>, +} + +impl SigningKeyLockMatcher { + /// Builds a matcher from signer pubkey-hashes. + pub(crate) fn from_signer_keys(signer_keys: &[Hash]) -> Self { + let signer_pkhs = signer_keys + .iter() + .map(Hash::to_array) + .collect::>(); + Self { signer_pkhs } + } +} + +impl LockMatcher for SigningKeyLockMatcher { + fn matches(&self, note_first_name: &Hash, spend_condition: &SpendCondition) -> bool { + let mut primitive_count = 0usize; + let mut tim_primitive_count = 0usize; + let mut signer_pkh_primitive = None; + for primitive in spend_condition.iter() { + primitive_count = primitive_count.saturating_add(1); + match primitive { + LockPrimitive::Pkh(pkh) => { + if signer_pkh_primitive.is_some() { + return false; + } + signer_pkh_primitive = Some(pkh); + } + LockPrimitive::Tim(_) => { + tim_primitive_count = tim_primitive_count.saturating_add(1); + } + _ => return false, + } + } + let Some(pkh) = signer_pkh_primitive else { + return false; + }; + if pkh.m != 1 || pkh.hashes.len() != 1 { + return false; + } + let Some(hash) = pkh.hashes.first() else { + return false; + }; + if !self.signer_pkhs.contains(&hash.to_array()) { + return false; + } + let is_simple_shape = tim_primitive_count == 0 && primitive_count == 1; + let is_coinbase_shape = tim_primitive_count == 1 && primitive_count == 2; + if !is_simple_shape && !is_coinbase_shape { + return false; + } + let Ok(reconstructed_first_name) = spend_condition.first_name() else { + return false; + }; + note_first_name.to_array() == reconstructed_first_name.as_hash().to_array() + } +} + +impl Wallet { + fn parse_note_names_as_hashes(raw: &str) -> Result, NockAppError> { + Self::parse_note_names(raw)? + .into_iter() + .map(|(first, last)| { + let first_hash = Hash::from_base58(&first).map_err(|err| { + NockAppError::from(CrownError::Unknown(format!( + "Invalid note first-name hash '{}': {}", + first, err + ))) + })?; + let last_hash = Hash::from_base58(&last).map_err(|err| { + NockAppError::from(CrownError::Unknown(format!( + "Invalid note last-name hash '{}': {}", + last, err + ))) + })?; + Ok(Name::new(first_hash, last_hash)) + }) + .collect() + } + + /// Formats selected names into the canonical create-tx `--names` argument. + fn format_note_names_for_create_tx(names: &[Name]) -> String { + names + .iter() + .map(|name| format!("[{} {}]", name.first.to_base58(), name.last.to_base58())) + .collect::>() + .join(",") + } + + /// Determines whether a manual note set is all-v1 or all-v0. + /// Missing notes are ignored here so planner manual-mode errors can report them. + fn manual_candidate_version_policy( + note_names: &[Name], + candidates: &[CandidateNote], + ) -> Result { + if note_names.is_empty() { + return Err("manual mode requires at least one note name".to_string()); + } + + let mut found_v0 = false; + let mut found_v1 = false; + + for name in note_names { + let Some(candidate) = candidates + .iter() + .find(|candidate| candidate.identity().name == *name) + else { + return Err(format!( + "manual mode references unknown note {}/{}", + name.first.to_base58(), + name.last.to_base58() + )); + }; + + match candidate.version() { + nockchain_types::tx_engine::common::Version::V0 => found_v0 = true, + _ => found_v1 = true, + } + } + + match (found_v0, found_v1) { + (true, false) => Ok(CandidateVersionPolicy::V0Only), + (false, true) => Ok(CandidateVersionPolicy::V1Only), + (false, false) => Err("manual mode requires at least one note name".to_string()), + (true, true) => Err( + "manual create-tx cannot mix v0 and v1 notes; select notes from only one version" + .to_string(), + ), + } + } + + /// Maps CLI ordering strategy onto planner selection order semantics. + fn planner_order_direction(strategy: NoteSelectionStrategyCli) -> SelectionOrder { + match strategy { + NoteSelectionStrategyCli::Ascending => SelectionOrder::Ascending, + NoteSelectionStrategyCli::Descending => SelectionOrder::Descending, + } + } + + /// Reads the latest synced balance snapshot from wallet state. + async fn peek_balance_state(&mut self) -> Result { + let mut slab = NounSlab::new(); + let balance_tag = make_tas(&mut slab, "balance").as_noun(); + let path = T(&mut slab, &[balance_tag, SIG]); + slab.set_root(path); + + let result = self.app.peek(slab).await?; + let space = result.noun_space(); + let maybe_balance: Option> = + unsafe { >>::from_noun(result.root(), &space)? }; + match maybe_balance { + Some(Some(balance)) => Ok(balance), + _ => Err(NockAppError::OtherError( + "wallet balance peek returned no balance payload".to_string(), + )), + } + } + + /// Reads blockchain constants from wallet state so the planner uses live fee policy. + async fn peek_planner_blockchain_constants( + &mut self, + ) -> Result { + let mut slab = NounSlab::new(); + let constants_tag = make_tas(&mut slab, "blockchain-constants").as_noun(); + let path = T(&mut slab, &[constants_tag, SIG]); + slab.set_root(path); + + let result = self.app.peek(slab).await?; + let space = result.noun_space(); + let maybe_constants: Option> = unsafe { + >>::from_noun(result.root(), &space)? + }; + let Some(constants) = maybe_constants.flatten() else { + return Err(NockAppError::OtherError( + "wallet blockchain-constants peek returned no payload".to_string(), + )); + }; + Ok(constants) + } + + /// Reads the master signer pubkey-hash from wallet tracked state for lock matching. + async fn peek_master_signing_key(&mut self) -> Result { + let mut slab = NounSlab::new(); + let tracked_tag = make_tas(&mut slab, "master-signing-key").as_noun(); + let path = T(&mut slab, &[tracked_tag, SIG]); + slab.set_root(path); + + let result = self.app.peek(slab).await?; + let space = result.noun_space(); + let maybe_signing_key: Option> = + unsafe { >>::from_noun(result.root(), &space)? }; + maybe_signing_key.flatten().ok_or_else(|| { + NockAppError::OtherError( + "wallet master-signing-key peek returned no payload".to_string(), + ) + }) + } + + async fn peek_master_signing_pubkey(&mut self) -> Result { + let mut slab = NounSlab::new(); + let tracked_tag = make_tas(&mut slab, "master-signing-pubkey").as_noun(); + let path = T(&mut slab, &[tracked_tag, SIG]); + slab.set_root(path); + + let result = self.app.peek(slab).await?; + let space = result.noun_space(); + let maybe_signing_pubkey: Option> = + unsafe { >>::from_noun(result.root(), &space)? }; + maybe_signing_pubkey.flatten().ok_or_else(|| { + NockAppError::OtherError( + "wallet master-signing-pubkey peek returned no payload".to_string(), + ) + }) + } + + async fn peek_active_signers(&mut self) -> Result, NockAppError> { + let mut slab = NounSlab::new(); + let tracked_tag = make_tas(&mut slab, "active-signers").as_noun(); + let path = T(&mut slab, &[tracked_tag, SIG]); + slab.set_root(path); + + let result = self.app.peek(slab).await?; + let space = result.noun_space(); + let maybe_signers: Option>> = unsafe { + >>>::from_noun(result.root(), &space)? + }; + let mut signers = maybe_signers.flatten().unwrap_or_default(); + signers.sort_by_key(ActiveSignerEntryNoun::sort_key); + signers.dedup_by(|left, right| { + left.child_index == right.child_index + && left.hardened == right.hardened + && left.absolute_index == right.absolute_index + && left.address_b58 == right.address_b58 + }); + Ok(signers) + } + + #[cfg(test)] + fn resolve_effect_write_path(path: &str, output_path: Option<&Path>) -> PathBuf { + let raw_path = Path::new(path); + match output_path { + Some(base_path) if !raw_path.is_absolute() => base_path.join(raw_path), + _ => raw_path.to_path_buf(), + } + } + + #[cfg(test)] + async fn apply_wallet_effects_locally( + effects: Vec, + output_path: Option<&Path>, + ) -> Result { + let mut applied = AppliedWalletEffects::default(); + + for effect in effects { + let space = effect.noun_space(); + let noun = unsafe { effect.root() }; + let Ok(cell) = noun.in_space(&space).as_cell() else { + continue; + }; + let Ok(tag) = ::from_noun(&cell.head().noun(), &space) else { + continue; + }; + + match tag.as_str() { + "file" => { + let file_cell = cell.tail().as_cell().map_err(|err| { + NockAppError::OtherError(format!( + "wallet file effect payload did not decode as a cell: {err}" + )) + })?; + let operation = ::from_noun(&file_cell.head().noun(), &space)?; + match operation.as_str() { + "write" => { + let (path, contents): (String, Bytes) = + <(String, Bytes)>::from_noun(&file_cell.tail().noun(), &space)?; + let resolved_path = Self::resolve_effect_write_path(&path, output_path); + if let Some(parent) = resolved_path.parent() { + tokio_fs::create_dir_all(parent) + .await + .map_err(NockAppError::IoError)?; + } + tokio_fs::write(&resolved_path, contents.as_ref()) + .await + .map_err(NockAppError::IoError)?; + if resolved_path + .extension() + .and_then(|ext| ext.to_str()) + .is_some_and(|ext| ext == "tx") + { + applied.tx_paths.push(resolved_path.display().to_string()); + } + } + "batch-write" => { + let entries: Vec = + Vec::from_noun(&file_cell.tail().noun(), &space)?; + for entry in entries { + let resolved_path = + Self::resolve_effect_write_path(&entry.path, output_path); + if let Some(parent) = resolved_path.parent() { + tokio_fs::create_dir_all(parent) + .await + .map_err(NockAppError::IoError)?; + } + tokio_fs::write(&resolved_path, entry.contents.as_ref()) + .await + .map_err(NockAppError::IoError)?; + if resolved_path + .extension() + .and_then(|ext| ext.to_str()) + .is_some_and(|ext| ext == "tx") + { + applied.tx_paths.push(resolved_path.display().to_string()); + } + } + } + _ => {} + } + } + "exit" => { + let code = ::from_noun(&cell.tail().noun(), &space)?; + if code != 0 { + return Err(NockAppError::OtherError(format!( + "wallet command exited with code {code} while running migrate-v0-notes" + ))); + } + } + _ => {} + } + } + + Ok(applied) + } + + pub(crate) async fn snapshot_written_txs( + tx_dir: &Path, + ) -> Result { + let mut snapshots = BTreeMap::new(); + if !tx_dir.exists() { + return Ok(WrittenTxSnapshot(snapshots)); + } + + let mut entries = tokio_fs::read_dir(tx_dir) + .await + .map_err(NockAppError::IoError)?; + while let Some(entry) = entries.next_entry().await.map_err(NockAppError::IoError)? { + let path = entry.path(); + if path.extension().and_then(|ext| ext.to_str()) != Some("tx") { + continue; + } + let metadata = entry.metadata().await.map_err(NockAppError::IoError)?; + let modified = metadata.modified().ok(); + snapshots.insert( + path, + TxFileSnapshot { + modified, + len: metadata.len(), + }, + ); + } + + Ok(WrittenTxSnapshot(snapshots)) + } + + pub(crate) fn detect_written_tx_paths( + before: &WrittenTxSnapshot, + after: &WrittenTxSnapshot, + ) -> Result, NockAppError> { + let changed = after + .0 + .iter() + .filter_map(|(path, metadata)| match before.0.get(path) { + Some(previous) if previous == metadata => None, + _ => Some(path.display().to_string()), + }) + .collect::>(); + + if changed.is_empty() { + return Err(NockAppError::OtherError( + "migrate-v0-notes expected create-tx-batch to write at least one transaction file, but no tx files changed".to_string(), + )); + } + + Ok(changed) + } + + fn decode_transaction_spends_from_bytes(tx_bytes: &[u8]) -> Result { + let mut slab: NounSlab = NounSlab::new(); + let transaction_noun = slab.cue_into(Bytes::copy_from_slice(tx_bytes))?; + let space = slab.noun_space(); + let transaction_cell = transaction_noun.in_space(&space).as_cell().map_err(|err| { + NockAppError::OtherError(format!("transaction jam root not a cell: {err}")) + })?; + let version = ::from_noun(&transaction_cell.head().noun(), &space) + .map_err(|err| { + NockAppError::OtherError(format!("transaction version did not decode: {err}")) + })?; + if version != 1 { + return Err(NockAppError::OtherError(format!( + "expected saved transaction version 1, got {version}" + ))); + } + let name_and_rest = transaction_cell.tail().as_cell().map_err(|err| { + NockAppError::OtherError(format!("transaction jam missing name/rest cell: {err}")) + })?; + let spends_and_rest = name_and_rest.tail().as_cell().map_err(|err| { + NockAppError::OtherError(format!("transaction jam missing spends/rest cell: {err}")) + })?; + let mut spends = + v1::Spends::from_noun(&spends_and_rest.head().noun(), &space).map_err(|err| { + NockAppError::OtherError(format!("saved transaction spends did not decode: {err}")) + })?; + let display_and_witness = spends_and_rest.tail().as_cell().map_err(|err| { + NockAppError::OtherError(format!( + "transaction jam missing display/witness-data cell: {err}" + )) + })?; + let witness_data = display_and_witness.tail(); + let witness_cell = witness_data.as_cell().map_err(|err| { + NockAppError::OtherError(format!("transaction jam witness-data not a cell: {err}")) + })?; + let witness_tag = ::from_noun(&witness_cell.head().noun(), &space) + .map_err(|err| { + NockAppError::OtherError(format!("witness-data tag did not decode: {err}")) + })?; + match witness_tag { + 0 => { + let signatures = + ZMap::::from_noun(&witness_cell.tail().noun(), &space) + .map_err(|err| { + NockAppError::OtherError(format!( + "legacy witness-data signature map did not decode: {err}" + )) + })?; + for (name, signature) in signatures.into_entries() { + let Some((_, v1::Spend::Legacy(spend0))) = spends + .0 + .iter_mut() + .find(|(candidate, _)| *candidate == name) + else { + return Err(NockAppError::OtherError(format!( + "legacy witness-data referenced unknown spend {} / {}", + name.first.to_base58(), + name.last.to_base58() + ))); + }; + spend0.signature = signature; + } + } + 1 => { + let witnesses = + ZMap::::from_noun(&witness_cell.tail().noun(), &space) + .map_err(|err| { + NockAppError::OtherError(format!( + "v1 witness-data map did not decode: {err}" + )) + })?; + for (name, witness) in witnesses.into_entries() { + let Some((_, v1::Spend::Witness(spend1))) = spends + .0 + .iter_mut() + .find(|(candidate, _)| *candidate == name) + else { + return Err(NockAppError::OtherError(format!( + "witness-data referenced unknown spend {} / {}", + name.first.to_base58(), + name.last.to_base58() + ))); + }; + spend1.witness = witness; + } + } + other => { + return Err(NockAppError::OtherError(format!( + "unsupported witness-data tag {other}" + ))); + } + } + Ok(spends) + } + + fn decode_transaction_spends_from_path( + transaction_path: &str, + ) -> Result { + let tx_bytes = std::fs::read(transaction_path).map_err(|err| { + NockAppError::OtherError(format!("failed to read transaction file: {err}")) + })?; + Self::decode_transaction_spends_from_bytes(&tx_bytes) + } + + #[cfg(test)] + /// Builds deterministic signer candidate list used by tests. + pub(crate) fn planner_signer_candidates(mut tracked_signers: Vec) -> Vec> { + tracked_signers.sort_by_key(|signer| signer.to_array()); + tracked_signers.dedup_by(|a, b| a.to_array() == b.to_array()); + let mut candidates = Vec::with_capacity(tracked_signers.len() + 1); + candidates.push(None); + candidates.extend(tracked_signers.into_iter().map(Some)); + candidates + } + + /// Plans create-tx inputs/fee and dispatches final hoon create-tx poke. + pub(crate) async fn create_tx_with_planner( + &mut self, + synced_snapshot: Option, + names: Option, + fee: Option, + recipients: Vec, + allow_low_fee: bool, + refund_pkh: Option, + sign_keys: Vec<(u64, bool)>, + include_data: bool, + save_raw_tx: bool, + note_selection: NoteSelectionStrategyCli, + ) -> CommandNoun { + let planner_error = |reason: String| -> CommandNoun { + Err(CrownError::Unknown(format!("create-tx planner failed: {}", reason)).into()) + }; + + let snapshot = if let Some(snapshot) = synced_snapshot { + snapshot + } else { + let balance = match self.peek_balance_state().await { + Ok(balance) => balance, + Err(err) => { + return planner_error(format!( + "unable to read synced balance from wallet state: {err}" + )); + } + }; + match normalize_balance_pages(&[balance]) { + Ok(snapshot) => snapshot, + Err(err) => { + return planner_error(format!( + "candidate normalization failed for wallet balance snapshot: {err}" + )); + } + } + }; + let v1_candidate_count = snapshot + .candidates + .iter() + .filter(|candidate| { + candidate.version() == nockchain_types::tx_engine::common::Version::V1 + }) + .count(); + let candidate_preview = snapshot + .candidates + .iter() + .take(5) + .map(|candidate| { + let identity = candidate.identity(); + format!( + "{}/{}", + identity.name.first.to_base58(), + identity.name.last.to_base58() + ) + }) + .collect::>(); + info!( + "create-tx planner snapshot block={} height={:?} candidates_total={} candidates_v1={} preview={:?}", + snapshot.metadata.block_id.to_base58(), + snapshot.metadata.height, + snapshot.candidates.len(), + v1_candidate_count, + candidate_preview + ); + + let manual_note_names = match names.as_deref() { + Some(raw_names) => match Self::parse_note_names_as_hashes(raw_names) { + Ok(note_names) => Some(note_names), + Err(err) => { + return planner_error(format!("unable to parse manual note names: {err}")); + } + }, + None => None, + }; + let selection_mode = match &manual_note_names { + Some(note_names) => SelectionMode::Manual { + note_names: note_names.clone(), + }, + None => SelectionMode::Auto, + }; + let parsed_refund_pkh = if let Some(refund) = refund_pkh.as_ref() { + match Hash::from_base58(refund) { + Ok(hash) => Some(hash), + Err(err) => { + return planner_error(format!( + "invalid refund pubkey hash '{}': {}", + refund, err + )); + } + } + } else { + None + }; + let candidate_version_policy = match &manual_note_names { + Some(note_names) => { + match Self::manual_candidate_version_policy(note_names, &snapshot.candidates) { + Ok(policy) => policy, + Err(err) => { + return planner_error(err); + } + } + } + None => CandidateVersionPolicy::V1Only, + }; + if candidate_version_policy == CandidateVersionPolicy::V0Only && parsed_refund_pkh.is_none() + { + return planner_error( + "manual create-tx spending legacy v0 notes requires --refund-pkh".to_string(), + ); + } + if !sign_keys.is_empty() { + info!( + "create-tx planner spendability matching currently uses only the wallet master key" + ); + } + let master_signer_pkh = match self.peek_master_signing_key().await { + Ok(key) => key, + Err(err) => { + warn!( + "create-tx planner could not read master signing key from wallet state: {}", + err + ); + return planner_error( + "wallet has no signer keys for create-tx planner".to_string(), + ); + } + }; + info!( + "create-tx planner master-signer-pkh={}", + master_signer_pkh.to_base58() + ); + let legacy_signer_pubkeys = if candidate_version_policy == CandidateVersionPolicy::V0Only { + let master_signer_pubkey = match self.peek_master_signing_pubkey().await { + Ok(key) => key, + Err(err) => { + return planner_error(format!( + "unable to read master signing pubkey from wallet state: {err}" + )); + } + }; + vec![master_signer_pubkey] + } else { + Vec::new() + }; + // Today lock matching is constrained to the master signer key only. + // We can expand this matcher input to include additional signing keys later. + let matcher_signer_keys = vec![master_signer_pkh.clone()]; + let recipient_outputs = match planner_recipient_outputs(&recipients, include_data) { + Ok(outputs) => outputs, + Err(err) => { + return planner_error(format!( + "unable to derive planner recipient lock roots from recipients: {err}" + )); + } + }; + let refund_output_template = match planner_refund_output_template( + parsed_refund_pkh.as_ref(), + &master_signer_pkh, + include_data, + ) { + Ok(output) => output, + Err(err) => { + return planner_error(format!( + "unable to derive planner refund output template from signer/refund context: {err}" + )); + } + }; + let planner_constants = match self.peek_planner_blockchain_constants().await { + Ok(constants) => constants, + Err(err) => { + return planner_error(format!( + "unable to read blockchain constants from wallet state: {err}" + )); + } + }; + let coinbase_relative_min = match planner_constants.coinbase_timelock_min() { + Ok(min) => min, + Err(err) => { + return planner_error(format!( + "unable to resolve coinbase timelock min from blockchain constants: {err}" + )); + } + }; + info!( + "create-tx planner constants bythos_phase={} base_fee={} input_fee_divisor={} min_fee={} coinbase_relative_min={}", + planner_constants.bythos_phase, + planner_constants.base_fee, + planner_constants.input_fee_divisor, + planner_constants.data.min_fee, + coinbase_relative_min + ); + let order_direction = Self::planner_order_direction(note_selection); + + let request = PlanRequest { + planning_mode: PlanningMode::Standard, + selection_mode: selection_mode.clone(), + order_direction, + include_data, + chain_context: ChainContext { + height: snapshot.metadata.height.clone(), + bythos_phase: nockchain_types::tx_engine::common::BlockHeight( + nockchain_math::belt::Belt(planner_constants.bythos_phase), + ), + base_fee: planner_constants.base_fee, + input_fee_divisor: planner_constants.input_fee_divisor, + min_fee: planner_constants.data.min_fee, + }, + signer_pkh: Some(master_signer_pkh.clone()), + candidate_version_policy, + candidates: snapshot.candidates, + recipient_outputs, + refund_output: refund_output_template, + coinbase_relative_min: Some(coinbase_relative_min), + v0_migration_signer_pubkeys: legacy_signer_pubkeys, + }; + + let matcher = SigningKeyLockMatcher::from_signer_keys(&matcher_signer_keys); + let plan = match plan_create_tx(&request, &matcher) { + Ok(found_plan) => { + info!( + "create-tx planner using master signer {} for lock spendability checks", + master_signer_pkh.to_base58() + ); + found_plan + } + Err(err @ PlanError::CandidateVersionDisabled { .. }) => { + return Err(CrownError::Unknown(format!( + "create-tx planner rejected the manual note set because it does not match the selected note version policy ({})", + err + )) + .into()); + } + Err(err) => { + return planner_error(format!("planner returned an error: {err}")); + } + }; + + for trace in &plan.debug_trace { + info!("create-tx planner trace: {}", trace); + } + + let planned_names = plan + .selected + .iter() + .map(|selected| selected.name.clone()) + .collect::>(); + if let SelectionMode::Manual { note_names } = &selection_mode { + if let Err(reason) = ensure_manual_planner_parity(note_names, &planned_names) { + return planner_error(reason); + } + } + let planned_names_arg = Self::format_note_names_for_create_tx(&planned_names); + let planned_fee = plan.final_fee; + let final_fee = if let Some(requested_fee) = fee { + if requested_fee < planned_fee && !allow_low_fee { + return Err(CrownError::Unknown(format!( + "requested --fee {} is below planner minimum {} (pass --allow-low-fee to override)", + requested_fee, planned_fee + )) + .into()); + } + if requested_fee != planned_fee { + info!( + "create-tx planner fee override requested_fee={} planned_fee={}", + requested_fee, planned_fee + ); + } + requested_fee + } else { + planned_fee + }; + + Self::create_tx(CreateTxRequest { + names: planned_names_arg, + recipients, + fee: final_fee, + allow_low_fee, + refund_pkh, + sign_keys, + include_data, + save_raw_tx, + note_selection, + }) + } + + pub(crate) fn format_migrate_v0_notes_summary(summary: &MigrateV0NotesSummary) -> String { + let mut lines = vec![ + "## V0 Migration Sweep".to_string(), + format!("- destination: `{}`", summary.destination), + format!("- block id: `{}`", summary.block_id), + format!("- height: `{}`", summary.height), + format!( + "- active signing keys examined: `{}`", + summary.examined_signers + ), + format!("- migration txs created: `{}`", summary.created_count), + format!("- signing keys skipped: `{}`", summary.skipped_count), + ]; + + if summary.created_count == 0 { + lines.push( + "- batch create poke: not emitted because every signer bucket was skipped" + .to_string(), + ); + } + + for signer_summary in &summary.signers { + lines.push(String::new()); + lines.push(format!("### {}", signer_summary.signer.label())); + lines.push(format!( + "- signer address: `{}`", + signer_summary.signer.address_b58 + )); + lines.push(format!( + "- signer version: `{}`", + signer_summary.signer.version + )); + lines.push(format!("- selected notes: `{}`", signer_summary.note_count)); + lines.push(format!( + "- selected total: `{}`", + signer_summary.selected_total + )); + match (&signer_summary.migrated_amount, &signer_summary.tx_path) { + (Some(migrated_amount), Some(tx_path)) => { + lines.push("- result: `created`".to_string()); + lines.push(format!( + "- fee: `{}`", + signer_summary.fee.unwrap_or_default() + )); + lines.push(format!("- migrated amount: `{}`", migrated_amount)); + lines.push(format!("- tx path: `{}`", tx_path)); + lines.push(format!( + "- submit with: `nockchain-wallet send-tx \"{}\"`", + tx_path + )); + } + _ => { + lines.push("- result: `skipped`".to_string()); + if let Some(fee) = signer_summary.fee { + lines.push(format!("- fee estimate: `{}`", fee)); + } + if let Some(reason) = &signer_summary.skip_reason { + lines.push(format!("- skip reason: `{}`", reason)); + } + } + } + } + + lines.join("\n") + } + + /// Plans one v0 migration transaction per active local v0 signer. + /// + /// Arguments: + /// - `synced_snapshot`: optional pre-normalized balance snapshot from the caller. When + /// `None`, the helper reads the current synced balance from wallet state and normalizes it. + /// - `destination`: base58-encoded v1 destination address that receives each migrated output. + pub(crate) async fn prepare_migrate_v0_notes_per_signer( + &mut self, + synced_snapshot: Option, + destination: String, + ) -> Result { + let destination_hash = Hash::from_base58(&destination).map_err(|err| { + CrownError::Unknown(format!( + "migrate-v0-notes planner failed: invalid migration destination '{}' : {}", + destination, err + )) + })?; + let snapshot = if let Some(snapshot) = synced_snapshot { + snapshot + } else { + let balance = self.peek_balance_state().await.map_err(|err| { + CrownError::Unknown(format!( + "migrate-v0-notes planner failed: unable to read synced balance from wallet state: {err}" + )) + })?; + normalize_balance_pages(&[balance]).map_err(|err| { + CrownError::Unknown(format!( + "migrate-v0-notes planner failed: candidate normalization failed for wallet balance snapshot: {err}" + )) + })? + }; + let active_signers = self.peek_active_signers().await.map_err(|err| { + CrownError::Unknown(format!( + "migrate-v0-notes planner failed: unable to read active signer entries from wallet state: {err}" + )) + })?; + let active_signers = active_signers + .into_iter() + .filter(|signer| signer.version == 0) + .collect::>(); + if active_signers.is_empty() { + return Err(CrownError::Unknown( + "migrate-v0-notes planner failed: wallet has no active local v0 signing keys under the active master".to_string(), + ) + .into()); + } + + let planner_constants = self.peek_planner_blockchain_constants().await.map_err(|err| { + CrownError::Unknown(format!( + "migrate-v0-notes planner failed: unable to read blockchain constants from wallet state: {err}" + )) + })?; + let coinbase_relative_min = planner_constants.coinbase_timelock_min().map_err(|err| { + CrownError::Unknown(format!( + "migrate-v0-notes planner failed: unable to resolve coinbase timelock min from blockchain constants: {err}" + )) + })?; + let mut destination_outputs = planner_recipient_outputs( + &[RecipientSpec::P2pkh { + address: destination_hash.clone(), + amount: 0, + }], + true, + ) + .map_err(|err| { + CrownError::Unknown(format!( + "migrate-v0-notes planner failed: unable to derive migration destination output from recipient: {err}" + )) + })?; + let destination_output = destination_outputs + .pop() + .expect("single migration recipient should yield one planner output"); + let refund_output = + planner_refund_output_template(Some(&destination_hash), &destination_hash, true) + .expect("p2pkh migration refund template should build"); + let chain_context = ChainContext { + height: snapshot.metadata.height.clone(), + bythos_phase: nockchain_types::tx_engine::common::BlockHeight( + nockchain_math::belt::Belt(planner_constants.bythos_phase), + ), + base_fee: planner_constants.base_fee, + input_fee_divisor: planner_constants.input_fee_divisor, + min_fee: planner_constants.data.min_fee, + }; + + let mut signer_summaries = Vec::with_capacity(active_signers.len()); + let mut pending_txs = Vec::::new(); + let mut skipped_count = 0usize; + + for signer in active_signers { + let request = PlanRequest { + planning_mode: PlanningMode::V0MigrationSweep { + destination_output: destination_output.clone(), + }, + selection_mode: SelectionMode::Auto, + order_direction: SelectionOrder::Ascending, + include_data: true, + chain_context: chain_context.clone(), + signer_pkh: None, + candidate_version_policy: CandidateVersionPolicy::V0Only, + candidates: snapshot.candidates.clone(), + recipient_outputs: Vec::new(), + refund_output: refund_output.clone(), + coinbase_relative_min: Some(coinbase_relative_min), + v0_migration_signer_pubkeys: vec![signer.pubkey.clone()], + }; + + match plan_create_tx(&request, &SigningKeyLockMatcher::default()) { + Ok(plan) => { + for trace in &plan.debug_trace { + info!( + "migrate-v0-notes planner trace signer={} {}", + signer.label(), + trace + ); + } + + let note_count = plan.selected.len(); + let selected_total = plan.selected_total; + let fee = Some(plan.final_fee); + let migrated_amount = plan.outputs.first().map(|output| output.amount); + let planned_names = plan + .selected + .iter() + .map(|selected| selected.name.clone()) + .collect::>(); + let Some(migrated_amount) = migrated_amount else { + skipped_count = skipped_count.saturating_add(1); + signer_summaries.push(MigrateV0SignerSummary { + signer, + note_count, + selected_total, + fee, + migrated_amount: None, + tx_path: None, + skip_reason: Some("planner_returned_no_destination_output".to_string()), + }); + continue; + }; + + let summary_index = signer_summaries.len(); + signer_summaries.push(MigrateV0SignerSummary { + signer: signer.clone(), + note_count, + selected_total, + fee, + migrated_amount: Some(migrated_amount), + tx_path: None, + skip_reason: None, + }); + pending_txs.push(PendingMigrationTx { + summary_index, + planned_names: planned_names.clone(), + request: CreateTxRequest { + names: Self::format_note_names_for_create_tx(&planned_names), + recipients: vec![RecipientSpec::P2pkh { + address: destination_hash.clone(), + amount: migrated_amount, + }], + fee: plan.final_fee, + allow_low_fee: false, + refund_pkh: Some(destination_hash.to_base58()), + sign_keys: signer.sign_keys(), + include_data: true, + save_raw_tx: false, + note_selection: NoteSelectionStrategyCli::Ascending, + }, + }); + } + Err(PlanError::V0MigrationProducesZeroValue { + selected_total, + fee, + }) => { + skipped_count = skipped_count.saturating_add(1); + let skip_reason = if selected_total == 0 { + "no_eligible_v0_notes" + } else { + "zero_value_after_fees" + }; + signer_summaries.push(MigrateV0SignerSummary { + signer, + note_count: 0, + selected_total, + fee: Some(fee), + migrated_amount: None, + tx_path: None, + skip_reason: Some(skip_reason.to_string()), + }); + } + Err(err) => { + skipped_count = skipped_count.saturating_add(1); + signer_summaries.push(MigrateV0SignerSummary { + signer, + note_count: 0, + selected_total: 0, + fee: None, + migrated_amount: None, + tx_path: None, + skip_reason: Some(format!("planner_error:{err}")), + }); + } + } + } + + let poke = if pending_txs.is_empty() { + None + } else { + Some(Self::create_tx_batch( + &pending_txs + .iter() + .map(|pending| pending.request.clone()) + .collect::>(), + )?) + }; + + let created_count = pending_txs.len(); + + Ok(PreparedMigrateV0Notes { + summary: MigrateV0NotesSummary { + destination, + block_id: snapshot.metadata.block_id.to_base58(), + height: (snapshot.metadata.height.0).0, + examined_signers: signer_summaries.len(), + created_count, + skipped_count, + signers: signer_summaries, + }, + poke, + pending_txs, + }) + } + + #[cfg(test)] + pub(crate) async fn migrate_v0_notes_per_signer_for_tests( + &mut self, + synced_snapshot: Option, + destination: String, + output_path: &Path, + ) -> Result { + let mut prepared = self + .prepare_migrate_v0_notes_per_signer(synced_snapshot, destination) + .await?; + let tx_paths = if let Some((poke, _operation)) = prepared.take_poke() { + let effects = self.app.poke(OnePunchWire::Poke.to_wire(), poke).await?; + Self::apply_wallet_effects_locally(effects, Some(output_path)) + .await? + .tx_paths + } else { + Vec::new() + }; + prepared.finalize(tx_paths) + } + + /// Creates a transaction. Use `--refund-pkh` when spending legacy v0 notes so the kernel + /// knows where to return change. When spending v1 notes the refund automatically + /// defaults back to the note owner, so `--refund-pkh` can be omitted. + fn encode_create_tx_request( + slab: &mut NounSlab, + request: &CreateTxRequest, + ) -> Result { + let names_vec = Self::parse_note_names(&request.names)?; + let names_noun = names_vec + .into_iter() + .rev() + .fold(D(0), |acc, (first, last)| { + let first_noun = make_tas(slab, &first).as_noun(); + let last_noun = make_tas(slab, &last).as_noun(); + let name_pair = T(slab, &[first_noun, last_noun]); + Cell::new(slab, name_pair, acc).as_noun() + }); + + let fee_noun = D(request.fee); + let order_noun = request.recipients.to_noun(slab); + let sign_key_noun = Wallet::encode_sign_keys(slab, request.sign_keys.clone()); + + let refund_noun = if let Some(refund) = request.refund_pkh.as_ref() { + let refund_hash = Hash::from_base58(refund).map_err(|err| { + NockAppError::from(CrownError::Unknown(format!( + "Invalid refund pubkey hash '{}': {}", + refund, err + ))) + })?; + let refund_atom = refund_hash.to_noun(slab); + T(slab, &[SIG, refund_atom]) + } else { + SIG + }; + let include_data_noun = request.include_data.to_noun(slab); + let allow_low_fee_noun = request.allow_low_fee.to_noun(slab); + let save_raw_tx_noun = request.save_raw_tx.to_noun(slab); + let note_selection_noun = make_tas(slab, request.note_selection.tas_label()).as_noun(); + + Ok(T( + slab, + &[ + names_noun, order_noun, fee_noun, allow_low_fee_noun, sign_key_noun, refund_noun, + include_data_noun, save_raw_tx_noun, note_selection_noun, + ], + )) + } + + fn create_tx(request: CreateTxRequest) -> CommandNoun { + let mut slab = NounSlab::new(); + let request_noun = Self::encode_create_tx_request(&mut slab, &request)?; + + Self::wallet("create-tx", &[request_noun], Operation::Poke, &mut slab) + } + + fn create_tx_batch(requests: &[CreateTxRequest]) -> CommandNoun { + let mut slab = NounSlab::new(); + let mut request_nouns = Vec::with_capacity(requests.len()); + for request in requests { + request_nouns.push(Self::encode_create_tx_request(&mut slab, request)?); + } + let requests_noun = request_nouns + .into_iter() + .rev() + .fold(D(0), |acc, request_noun| { + Cell::new(&mut slab, request_noun, acc).as_noun() + }); + + Self::wallet( + "create-tx-batch", + &[requests_noun], + Operation::Poke, + &mut slab, + ) + } + + #[cfg(test)] + pub(crate) fn create_tx_command_for_tests( + names: String, + recipients: Vec, + fee: u64, + allow_low_fee: bool, + refund_pkh: Option, + sign_keys: Vec<(u64, bool)>, + include_data: bool, + save_raw_tx: bool, + note_selection: NoteSelectionStrategyCli, + ) -> CommandNoun { + Self::create_tx(CreateTxRequest { + names, + recipients, + fee, + allow_low_fee, + refund_pkh, + sign_keys, + include_data, + save_raw_tx, + note_selection, + }) + } + + /// Encodes optional sign-key tuples for wallet kernel create-tx commands. + fn encode_sign_keys(slab: &mut NounSlab, keys: Vec<(u64, bool)>) -> Noun { + if keys.is_empty() { + SIG + } else { + Some(keys).to_noun(slab) + } + } + + /// Builds one `update-balance-grpc` poke from a fully assembled balance snapshot. + fn update_balance_grpc_poke(balance_update: v1::BalanceUpdate) -> NounSlab { + let mut slab = NounSlab::new(); + let wrapped_balance = Some(Some(balance_update)); + let balance_noun = wrapped_balance.to_noun(&mut slab); + let head = make_tas(&mut slab, "update-balance-grpc").as_noun(); + let full = T(&mut slab, &[head, balance_noun]); + slab.set_root(full); + slab + } + + #[cfg(test)] + pub(crate) fn update_balance_grpc_poke_for_tests( + balance_update: v1::BalanceUpdate, + ) -> NounSlab { + Self::update_balance_grpc_poke(balance_update) + } + + /// Merges fetched balance pages into one consistent deduplicated snapshot. + pub(crate) fn union_balance_pages( + pages: Vec, + ) -> Result, NormalizeSnapshotError> { + if pages.is_empty() { + return Ok(None); + } + + let normalized = normalize_balance_pages(&pages)?; + + let mut deduped_notes = BTreeMap::<([u64; 5], [u64; 5]), (Name, v1::Note)>::new(); + for page in pages { + for (name, note) in page.notes.0 { + let key = (name.first.to_array(), name.last.to_array()); + deduped_notes.entry(key).or_insert((name, note)); + } + } + + let merged = v1::BalanceUpdate { + height: normalized.metadata.height.clone(), + block_id: normalized.metadata.block_id.clone(), + notes: v1::Balance(deduped_notes.into_values().collect()), + }; + Ok(Some((merged, normalized))) + } + + #[cfg(test)] + /// Removes v1 notes that do not match tracked first-name filters. + /// + /// Some balance endpoints can return broader result sets than requested. + /// This keeps wallet state aligned with tracked keys/watch lists by + /// admitting only v1 notes whose first-name matches a tracked query. + fn filter_untracked_v1_notes_from_balance_update( + mut balance_update: v1::BalanceUpdate, + tracked_first_names: &std::collections::BTreeSet<[u64; 5]>, + ) -> v1::BalanceUpdate { + if tracked_first_names.is_empty() { + return balance_update; + } + + let before = balance_update.notes.0.len(); + balance_update.notes.0.retain(|(name, note)| match note { + v1::Note::V1(_) => tracked_first_names.contains(&name.first.to_array()), + v1::Note::V0(_) => true, + }); + let removed = before.saturating_sub(balance_update.notes.0.len()); + if removed > 0 { + info!( + "wallet balance sync dropped {} untracked v1 notes from one page", + removed + ); + } + balance_update + } + + /// Builds one `update-balance-grpc` poke from a private-api peek payload. + fn update_balance_grpc_poke_from_payload( + payload: Option>, + ) -> NounSlab { + let mut slab = NounSlab::new(); + let payload_noun = payload.to_noun(&mut slab); + let head = make_tas(&mut slab, "update-balance-grpc").as_noun(); + let full = T(&mut slab, &[head, payload_noun]); + slab.set_root(full); + slab + } + + #[cfg(test)] + /// Test helper for filtering one balance update against tracked first names. + pub(crate) fn filter_untracked_v1_notes_for_tests( + balance_update: v1::BalanceUpdate, + tracked_first_names: Vec, + ) -> v1::BalanceUpdate { + let tracked = tracked_first_names + .into_iter() + .map(|hash| hash.to_array()) + .collect::>(); + Self::filter_untracked_v1_notes_from_balance_update(balance_update, &tracked) + } + + /// Collects one page set from the public API balance endpoints. + async fn fetch_balance_pages_grpc_public( + client: &mut public_nockchain::PublicNockchainGrpcClient, + pubkeys: &[String], + first_names: &[String], + ) -> Result, NockAppError> { + let mut pages = Vec::::new(); + + for first_name in first_names { + let response = client + .wallet_get_balance(&BalanceRequest::FirstName(first_name.clone())) + .await + .map_err(|e| { + NockAppError::OtherError(format!( + "Failed to request current balance for first name {}: {}", + first_name, e + )) + })?; + let balance_update = v1::BalanceUpdate::try_from(response).map_err(|e| { + NockAppError::OtherError(format!( + "Failed to parse balance update for first name {}: {}", + first_name, e + )) + })?; + pages.push(balance_update); + } + + for key in pubkeys { + let response = client + .wallet_get_balance(&BalanceRequest::Address(key.clone())) + .await + .map_err(|e| { + NockAppError::OtherError(format!( + "Failed to request current balance for pubkey {}: {}", + key, e + )) + })?; + let balance_update = v1::BalanceUpdate::try_from(response).map_err(|e| { + NockAppError::OtherError(format!( + "Failed to parse balance update for pubkey {}: {}", + key, e + )) + })?; + pages.push(balance_update); + } + + Ok(pages) + } + + /// Fetches balances via public gRPC and emits one merged wallet update snapshot. + pub(crate) async fn update_balance_grpc_public( + client: &mut public_nockchain::PublicNockchainGrpcClient, + mut pubkeys: Vec, + mut first_names: Vec, + ) -> Result { + first_names.sort(); + first_names.dedup(); + pubkeys.sort(); + pubkeys.dedup(); + + const SNAPSHOT_DRIFT_MAX_RETRIES: usize = 2; + let mut attempt = 0usize; + let (merged_balance, normalized_snapshot) = loop { + attempt = attempt.saturating_add(1); + let pages = + Self::fetch_balance_pages_grpc_public(client, &pubkeys, &first_names).await?; + + match Self::union_balance_pages(pages) { + Ok(Some((merged_balance, normalized_snapshot))) => { + break (merged_balance, normalized_snapshot); + } + Ok(None) => { + return Ok(connection::BalanceSyncResult { + pokes: Vec::new(), + normalized_snapshot: None, + }); + } + Err( + NormalizeSnapshotError::Snapshot(SnapshotConsistencyError::HeightDrift) + | NormalizeSnapshotError::Snapshot(SnapshotConsistencyError::BlockIdDrift), + ) if attempt <= SNAPSHOT_DRIFT_MAX_RETRIES => { + continue; + } + Err(err) => { + return Err(NockAppError::OtherError(format!( + "Failed to normalize fetched wallet balance pages into one snapshot: {}", + err + ))); + } + } + }; + + Ok(connection::BalanceSyncResult { + pokes: vec![Self::update_balance_grpc_poke(merged_balance)], + normalized_snapshot: Some(normalized_snapshot), + }) + } + + /// Fetches balances via private gRPC peek paths and wraps updates as wallet pokes. + pub(crate) async fn update_balance_grpc_private( + client: &mut private_nockapp::PrivateNockAppGrpcClient, + mut pubkeys: Vec, + mut first_names: Vec, + ) -> Result { + first_names.sort(); + first_names.dedup(); + pubkeys.sort(); + pubkeys.dedup(); + + let mut request_index: i32 = 0; + let mut results = Vec::new(); + + for first_name in first_names { + let mut slab: NounSlab = NounSlab::new(); + + let mut path_slab = NounSlab::::new(); + let path_noun = vec!["balance-by-first-name".to_string(), first_name.clone()] + .to_noun(&mut path_slab); + path_slab.set_root(path_noun); + let path_bytes = path_slab.jam().to_vec(); + + let response = client.peek(request_index, path_bytes).await.map_err(|e| { + NockAppError::OtherError(format!( + "Failed to peek balance for first name {first_name}: {e}" + )) + })?; + request_index = request_index.wrapping_add(1); + + let balance = slab.cue_into(response.as_bytes()?)?; + let space = slab.noun_space(); + let payload: Option> = + >>::from_noun(&balance, &space)?; + results.push(Self::update_balance_grpc_poke_from_payload(payload)); + } + + for key in pubkeys { + let mut slab: NounSlab = NounSlab::new(); + let mut path_slab = NounSlab::::new(); + let path_noun = + vec!["balance-by-pubkey".to_string(), key.clone()].to_noun(&mut path_slab); + path_slab.set_root(path_noun); + let path_bytes = path_slab.jam().to_vec(); + + let response = client.peek(request_index, path_bytes).await.map_err(|e| { + NockAppError::OtherError(format!("Failed to peek balance for pubkey {key}: {e}")) + })?; + request_index = request_index.wrapping_add(1); + + let balance = slab.cue_into(response.as_bytes()?)?; + let space = slab.noun_space(); + let payload: Option> = + >>::from_noun(&balance, &space)?; + results.push(Self::update_balance_grpc_poke_from_payload(payload)); + } + + Ok(connection::BalanceSyncResult { + pokes: results, + normalized_snapshot: None, + }) + } +} + +#[cfg(test)] +mod tests { + use nockchain_math::belt::Belt; + use nockchain_math::crypto::cheetah::A_GEN; + use nockchain_types::tx_engine::common::{BlockHeight, Nicks, SchnorrPubkey}; + use nockchain_types::tx_engine::v0::Lock as V0Lock; + use wallet_tx_builder::note_data::DecodedNoteData; + use wallet_tx_builder::types::{ + CandidateIdentity, CandidateV0Note, CandidateV1Note, CandidateVersionPolicy, + }; + + use super::*; + + fn hash(v: u64) -> Hash { + Hash::from_limbs(&[v, 0, 0, 0, 0]) + } + + fn name(first: u64, last: u64) -> Name { + Name::new(hash(first), hash(last)) + } + + fn candidate_v0(first: u64, last: u64) -> CandidateNote { + CandidateNote::V0(CandidateV0Note { + identity: CandidateIdentity { + name: name(first, last), + origin_page: BlockHeight(Belt(1)), + }, + assets: Nicks(1), + lock: V0Lock { + keys_required: 1, + pubkeys: vec![SchnorrPubkey(A_GEN)], + }, + timelock: None, + }) + } + + fn candidate_v1(first: u64, last: u64) -> CandidateNote { + CandidateNote::V1(CandidateV1Note { + identity: CandidateIdentity { + name: name(first, last), + origin_page: BlockHeight(Belt(1)), + }, + assets: Nicks(1), + raw_note_data: Vec::new(), + decoded_note_data: DecodedNoteData(Vec::new()), + }) + } + + #[test] + fn manual_candidate_version_policy_returns_v0_only_for_all_v0_manual_sets() { + let note_names = vec![name(1, 10), name(2, 20)]; + let candidates = vec![candidate_v0(1, 10), candidate_v0(2, 20), candidate_v1(3, 30)]; + + let policy = + Wallet::manual_candidate_version_policy(¬e_names, &candidates).expect("policy"); + + assert_eq!(policy, CandidateVersionPolicy::V0Only); + } + + #[test] + fn manual_candidate_version_policy_returns_v1_only_for_all_v1_manual_sets() { + let note_names = vec![name(3, 30)]; + let candidates = vec![candidate_v0(1, 10), candidate_v1(3, 30)]; + + let policy = + Wallet::manual_candidate_version_policy(¬e_names, &candidates).expect("policy"); + + assert_eq!(policy, CandidateVersionPolicy::V1Only); + } + + #[test] + fn manual_candidate_version_policy_rejects_mixed_manual_sets() { + let note_names = vec![name(1, 10), name(3, 30)]; + let candidates = vec![candidate_v0(1, 10), candidate_v1(3, 30)]; + + let err = Wallet::manual_candidate_version_policy(¬e_names, &candidates) + .expect_err("mixed version note set should error"); + + assert_eq!( + err, + "manual create-tx cannot mix v0 and v1 notes; select notes from only one version" + ); + } + + #[test] + fn manual_candidate_version_policy_rejects_missing_manual_notes() { + let missing = name(9, 90); + let note_names = vec![missing.clone()]; + let candidates = vec![candidate_v0(1, 10), candidate_v1(3, 30)]; + + let err = Wallet::manual_candidate_version_policy(¬e_names, &candidates) + .expect_err("missing note should error"); + + assert_eq!( + err, + format!( + "manual mode references unknown note {}/{}", + missing.first.to_base58(), + missing.last.to_base58() + ) + ); + } +} diff --git a/crates/nockchain-wallet/src/main.rs b/crates/nockchain-wallet/src/main.rs index 849cfd02a..74554d3d1 100644 --- a/crates/nockchain-wallet/src/main.rs +++ b/crates/nockchain-wallet/src/main.rs @@ -13,12 +13,16 @@ mod command; mod connection; +mod create_tx; mod error; mod recipient; +#[cfg(test)] +mod tests; +use std::collections::BTreeMap; use std::fs; use std::io::{self, Write}; -use std::path::PathBuf; +use std::path::{Path, PathBuf}; use clap::Parser; #[cfg(test)] @@ -30,6 +34,7 @@ use command::{ }; use kernels_open_wallet::KERNEL; use nockapp::driver::*; +use nockapp::drivers::one_punch::OnePunchWire; use nockapp::kernel::boot::{self, NockStackSize}; use nockapp::noun::slab::{NockJammer, NounSlab}; use nockapp::utils::bytes::Byts; @@ -43,15 +48,30 @@ use nockapp_grpc::pb::common::v1::Base58Hash as PbBase58Hash; use nockapp_grpc::pb::public::v2::transaction_accepted_response; use nockapp_grpc::{private_nockapp, public_nockchain}; use nockchain_types::common::{Hash, SchnorrPubkey, TimelockRangeAbsolute, TimelockRangeRelative}; -use nockchain_types::{v0, v1}; +use nockchain_types::tx_engine::common::Name; +use nockchain_types::tx_engine::v1::tx::{LockPrimitive, SpendCondition}; +use nockchain_types::{default_fakenet_blockchain_constants, v0, v1}; use nockvm::jets::cold::Nounable; -use nockvm::noun::{Atom, Cell, IndirectAtom, Noun, D, NO, SIG, T, YES}; +use nockvm::noun::{Atom, Cell, IndirectAtom, Noun, NounAllocator, D, NO, SIG, T, YES}; use noun_serde::prelude::*; use noun_serde::NounDecodeError; -use recipient::{recipient_tokens_to_specs, RecipientSpec}; +#[cfg(test)] +use recipient::BRIDGE_LOCK_ROOT_DEFAULT_B58; +use recipient::{ + planner_recipient_outputs, planner_refund_output_template, recipient_tokens_to_specs, + RecipientSpec, +}; use termimad::MadSkin; use tokio::fs as tokio_fs; use tracing::{error, info, warn}; +use wallet_tx_builder::adapter::{ + normalize_balance_pages, NormalizeSnapshotError, NormalizedSnapshot, SnapshotConsistencyError, +}; +use wallet_tx_builder::lock_resolver::LockMatcher; +use wallet_tx_builder::planner::{plan_create_tx, PlanError}; +use wallet_tx_builder::types::{ + CandidateVersionPolicy, ChainContext, PlanRequest, PlanningMode, SelectionMode, SelectionOrder, +}; use zkvm_jetpack::hot::produce_prover_hot_state; use crate::public_nockchain::v2::client::BalanceRequest; @@ -85,6 +105,7 @@ async fn main() -> Result<(), NockAppError> { .map_err(|e| CrownError::Unknown(format!("Kernel setup failed: {}", e)))?; let mut wallet = Wallet::new(kernel); + let mut synced_snapshot_for_planner: Option = None; if cli.fakenet { wallet.set_fakenet().await?; @@ -123,7 +144,7 @@ async fn main() -> Result<(), NockAppError> { _ => true, }; - let poke = match &cli.command { + let mut poke = match &cli.command { Commands::Keygen => { let mut entropy = [0u8; 32]; let mut salt = [0u8; 16]; @@ -292,32 +313,13 @@ async fn main() -> Result<(), NockAppError> { } } Commands::ListNotesByAddressCsv { address } => Wallet::list_notes_by_address_csv(address), - Commands::CreateTx { - names, - recipients, - fee, - allow_low_fee, - refund_pkh, - index, - hardened, - include_data, - sign_keys, - save_raw_tx, - note_selection_strategy, - } => { - let recipient_specs = recipient_tokens_to_specs(recipients.clone())?; - let signing_keys = Wallet::collect_signing_keys(*index, *hardened, sign_keys)?; - Wallet::create_tx( - names.clone(), - recipient_specs, - *fee, - *allow_low_fee, - refund_pkh.clone(), - signing_keys, - *include_data, - *save_raw_tx, - *note_selection_strategy, - ) + Commands::CreateTx { .. } => { + // Planner-backed create-tx runs after sync once we have a fresh snapshot. + Wallet::show_balance() + } + Commands::MigrateV0Notes { .. } => { + // Planner-backed v0 migration runs after sync once we have a fresh snapshot. + Wallet::show_balance() } Commands::SignMultisigTx { transaction, @@ -363,7 +365,10 @@ async fn main() -> Result<(), NockAppError> { pubkey_slab .to_vec() .iter() - .map(|key| String::from_noun(unsafe { key.root() })) + .map(|key| { + let space = key.noun_space(); + String::from_noun(unsafe { key.root() }, &space) + }) .collect::, NounDecodeError>>()? .into_iter() .filter_map(|value| match normalize_watch_address(value) { @@ -378,17 +383,20 @@ async fn main() -> Result<(), NockAppError> { let first_names: Vec = if let Some(name_slab) = first_name_slab { let names_noun = unsafe { name_slab.root() }; - >::from_noun(names_noun)? + let name_space = name_slab.noun_space(); + >::from_noun(names_noun, &name_space)? } else { Vec::new() }; let connection_target = cli.connection.target(); - let pokes = + let sync_result = connection::sync_wallet_balance(&mut wallet, &connection_target, pubkeys, first_names) .await?; - for poke in pokes { + synced_snapshot_for_planner = sync_result.normalized_snapshot; + + for poke in sync_result.pokes { let _ = wallet .app .poke(SystemWire.to_wire(), poke) @@ -397,6 +405,88 @@ async fn main() -> Result<(), NockAppError> { } } + if let Commands::MigrateV0Notes { destination } = &cli.command { + let mut prepared = wallet + .prepare_migrate_v0_notes_per_signer( + synced_snapshot_for_planner.take(), + destination.clone(), + ) + .await?; + if prepared.summary.created_count == 0 { + let markdown = Wallet::format_migrate_v0_notes_summary(&prepared.summary); + let skin = MadSkin::default_dark(); + println!("{}", skin.term_text(&markdown)); + return Err(NockAppError::OtherError( + "No v0 migration transactions were created".to_string(), + )); + } + + let tx_dir = Path::new("txs"); + let before = Wallet::snapshot_written_txs(tx_dir).await?; + let (noun, operation) = prepared.take_poke().ok_or_else(|| { + NockAppError::from(CrownError::Unknown( + "migrate-v0-notes prepared migration transactions but did not produce a batch create poke" + .to_string(), + )) + })?; + wallet + .app + .add_io_driver(one_punch_driver(noun, operation)) + .await; + wallet.app.add_io_driver(file_driver()).await; + wallet.app.add_io_driver(markdown_driver()).await; + wallet.app.add_io_driver(exit_driver()).await; + + match wallet.app.run().await { + Ok(_) => { + let after = Wallet::snapshot_written_txs(tx_dir).await?; + let tx_paths = Wallet::detect_written_tx_paths(&before, &after)?; + let summary = prepared.finalize(tx_paths)?; + let markdown = Wallet::format_migrate_v0_notes_summary(&summary); + let skin = MadSkin::default_dark(); + println!("{}", skin.term_text(&markdown)); + info!("Command executed successfully"); + } + Err(e) => { + error!("Command failed: {}", e); + return Err(e); + } + } + return Ok(()); + } + + if let Commands::CreateTx { + names, + recipients, + fee, + allow_low_fee, + refund_pkh, + index, + hardened, + include_data, + sign_keys, + save_raw_tx, + note_selection_strategy, + } = &cli.command + { + let recipient_specs = recipient_tokens_to_specs(recipients.clone())?; + let signing_keys = Wallet::collect_signing_keys(*index, *hardened, sign_keys)?; + poke = wallet + .create_tx_with_planner( + synced_snapshot_for_planner.take(), + names.clone(), + *fee, + recipient_specs, + *allow_low_fee, + refund_pkh.clone(), + signing_keys, + *include_data, + *save_raw_tx, + *note_selection_strategy, + ) + .await?; + } + wallet .app .add_io_driver(one_punch_driver(poke.0, poke.1)) @@ -417,17 +507,7 @@ async fn main() -> Result<(), NockAppError> { } } -#[allow(dead_code)] -fn validate_label(s: &str) -> Result { - if s.chars() - .all(|c| c.is_ascii_lowercase() || c.is_ascii_digit() || c == '-') - { - Ok(s.to_string()) - } else { - Err("Label must contain only lowercase letters, numbers, and hyphens".to_string()) - } -} - +/// Wallet runtime wrapper around the underlying nockapp kernel. pub struct Wallet { app: NockApp, } @@ -450,22 +530,25 @@ impl Wallet { Wallet { app: nockapp } } + /// Applies the shared Rust fakenet constants so wallet state matches node fakenet defaults. async fn set_fakenet(&mut self) -> Result<(), NockAppError> { - let mut slab: NounSlab = NounSlab::new(); - let tag = String::from("fakenet").to_noun(&mut slab); - slab.modify(|_| vec![tag, SIG]); - let wire = SystemWire.to_wire(); - let _ = self.app.poke(wire, slab).await?; + let mut slab = NounSlab::new(); + let constants = default_fakenet_blockchain_constants(); + let constants_noun = constants.to_noun(&mut slab); + let (poke, _) = Self::wallet("fakenet", &[constants_noun], Operation::Poke, &mut slab)?; + let wire = OnePunchWire::Poke.to_wire(); + let _ = self.app.poke(wire, poke).await?; Ok(()) } + /// Reads whether current wallet state was initialized in fakenet mode. async fn is_fakenet(&mut self) -> Result { - let mut slab: NounSlab = NounSlab::new(); + let mut slab = NounSlab::new(); let tag = String::from("fakenet").to_noun(&mut slab); slab.modify(|_| vec![tag, SIG]); let result = self.app.peek(slab).await?; let is_fakenet: Option> = - unsafe { >>::from_noun(result.root())? }; + unsafe { >>::from_noun(result.root(), &result.noun_space())? }; match is_fakenet { Some(Some(res)) => Ok(res), _ => Err(NockAppError::OtherError( @@ -513,7 +596,7 @@ impl Wallet { /// * `entropy` - The entropy to use for key generation. /// * `sal` - The salt to use for key generation. fn keygen(entropy: &[u8; 32], sal: &[u8; 16]) -> CommandNoun { - let mut slab = NounSlab::new(); + let mut slab: NounSlab = NounSlab::new(); let ent: Byts = Byts::new(entropy.to_vec()); let ent_noun = ent.into_noun(&mut slab); let sal: Byts = Byts::new(sal.to_vec()); @@ -541,15 +624,15 @@ impl Wallet { // ) //} - // Derives a child key from current master key. - // - // # Arguments - // - // * `index` - The index of the child key to derive - // * `hardened` - Whether the child key should be hardened - // * `label` - Optional label for the child key + /// Derives a child key from the current master key path. + /// + /// # Arguments + /// + /// * `index` - The index of the child key to derive. + /// * `hardened` - Whether the child key should be hardened. + /// * `label` - Optional label persisted alongside the derived key. fn derive_child(index: u64, hardened: bool, label: &Option) -> CommandNoun { - let mut slab = NounSlab::new(); + let mut slab: NounSlab = NounSlab::new(); let index_noun = D(index); let hardened_noun = if hardened { YES } else { NO }; let label_noun = label.as_ref().map_or(SIG, |l| { @@ -620,6 +703,7 @@ impl Wallet { ) } + /// Signs an arbitrary message payload with the requested signing key. fn sign_message( message_bytes: &[u8], index: Option, @@ -654,6 +738,7 @@ impl Wallet { ) } + /// Verifies a signature over an arbitrary message payload. fn verify_message( message_bytes: &[u8], signature_jam: &[u8], @@ -672,6 +757,7 @@ impl Wallet { ) } + /// Signs a base58 tip5 hash directly without message prehashing. fn sign_hash(hash_b58: &str, index: Option, hardened: bool) -> CommandNoun { let mut slab = NounSlab::new(); @@ -701,6 +787,7 @@ impl Wallet { ) } + /// Verifies a signature over a base58 tip5 hash. fn verify_hash( hash_b58: &str, signature_jam: &[u8], @@ -842,6 +929,7 @@ impl Wallet { } #[allow(dead_code)] + /// Builds a kernel timelock intent from optional absolute/relative ranges. fn timelock_intent_from_ranges( absolute: Option, relative: Option, @@ -856,6 +944,7 @@ impl Wallet { } } + /// Parses `"[first last],[first last]"` note-name syntax used by create-tx. fn parse_note_names(raw: &str) -> Result, NockAppError> { let mut names = Vec::new(); @@ -897,6 +986,7 @@ impl Wallet { Ok(names) } + /// Resolves effective sign-key list from explicit `--sign-key` or index/hardened fallback. fn collect_signing_keys( index: Option, hardened: bool, @@ -914,6 +1004,7 @@ impl Wallet { } } + /// Parses one `index[:hardened]` sign-key token from CLI input. fn parse_sign_key_entry(entry: &str) -> Result<(u64, bool), NockAppError> { let trimmed = entry.trim(); if trimmed.is_empty() { @@ -927,183 +1018,7 @@ impl Wallet { Self::parse_sign_key_components(index_part, hardened_part) } - /// Creates a transaction. Use `--refund-pkh` when spending legacy v0 notes so the kernel - /// knows where to return change. When spending v1 notes the refund automatically - /// defaults back to the note owner, so `--refund-pkh` can be omitted. - fn create_tx( - names: String, - recipients: Vec, - fee: u64, - allow_low_fee: bool, - refund_pkh: Option, - sign_keys: Vec<(u64, bool)>, - include_data: bool, - save_raw_tx: bool, - note_selection: NoteSelectionStrategyCli, - ) -> CommandNoun { - let mut slab = NounSlab::new(); - - let names_vec = Self::parse_note_names(&names)?; - let names_noun = names_vec - .into_iter() - .rev() - .fold(D(0), |acc, (first, last)| { - let first_noun = make_tas(&mut slab, &first).as_noun(); - let last_noun = make_tas(&mut slab, &last).as_noun(); - let name_pair = T(&mut slab, &[first_noun, last_noun]); - Cell::new(&mut slab, name_pair, acc).as_noun() - }); - - let fee_noun = D(fee); - let order_noun = recipients.to_noun(&mut slab); - let sign_key_noun = Wallet::encode_sign_keys(&mut slab, sign_keys); - - let refund_noun = if let Some(refund) = refund_pkh { - let refund_hash = Hash::from_base58(&refund).map_err(|err| { - NockAppError::from(CrownError::Unknown(format!( - "Invalid refund pubkey hash '{}': {}", - refund, err - ))) - })?; - let refund_atom = refund_hash.to_noun(&mut slab); - T(&mut slab, &[SIG, refund_atom]) - } else { - SIG - }; - let include_data_noun = include_data.to_noun(&mut slab); - let allow_low_fee_noun = allow_low_fee.to_noun(&mut slab); - let save_raw_tx_noun = save_raw_tx.to_noun(&mut slab); - let note_selection_noun = make_tas(&mut slab, note_selection.tas_label()).as_noun(); - - Self::wallet( - "create-tx", - &[ - names_noun, order_noun, fee_noun, allow_low_fee_noun, sign_key_noun, refund_noun, - include_data_noun, save_raw_tx_noun, note_selection_noun, - ], - Operation::Poke, - &mut slab, - ) - } - - fn encode_sign_keys(slab: &mut NounSlab, keys: Vec<(u64, bool)>) -> Noun { - if keys.is_empty() { - SIG - } else { - Some(keys).to_noun(slab) - } - } - - async fn update_balance_grpc_public( - client: &mut public_nockchain::PublicNockchainGrpcClient, - pubkeys: Vec, - first_names: Vec, - ) -> Result, NockAppError> { - let mut results = Vec::new(); - - for first_name in first_names { - let mut slab = NounSlab::new(); // Define slab - adjust as needed - let response = client - .wallet_get_balance(&BalanceRequest::FirstName(first_name)) - .await - .map_err(|e| { - NockAppError::OtherError(format!("Failed to request current balance: {}", e)) - })?; - let balance_update = v1::BalanceUpdate::try_from(response).map_err(|e| { - NockAppError::OtherError(format!("Failed to parse balance update: {}", e)) - })?; - let wrapped_balance = Some(Some(balance_update)); - let balance_noun = wrapped_balance.to_noun(&mut slab); - let head = make_tas(&mut slab, "update-balance-grpc").as_noun(); - let full = T(&mut slab, &[head, balance_noun]); - slab.set_root(full); - results.push(slab); - } - - for (_index, key) in pubkeys.iter().enumerate() { - let mut slab = NounSlab::new(); // Define slab - adjust as needed - let response = client - .wallet_get_balance(&BalanceRequest::Address(key.to_owned())) - .await - .map_err(|e| { - NockAppError::OtherError(format!("Failed to request current balance: {}", e)) - })?; - let balance_update = v1::BalanceUpdate::try_from(response).map_err(|e| { - NockAppError::OtherError(format!("Failed to parse balance update: {}", e)) - })?; - let wrapped_balance = Some(Some(balance_update)); - let balance_noun = wrapped_balance.to_noun(&mut slab); - let head = make_tas(&mut slab, "update-balance-grpc").as_noun(); - let full = T(&mut slab, &[head, balance_noun]); - slab.set_root(full); - results.push(slab); - } - - Ok(results) - } - - async fn update_balance_grpc_private( - client: &mut private_nockapp::PrivateNockAppGrpcClient, - mut pubkeys: Vec, - mut first_names: Vec, - ) -> Result, NockAppError> { - first_names.sort(); - first_names.dedup(); - pubkeys.sort(); - pubkeys.dedup(); - - let mut request_index: i32 = 0; - let mut results = Vec::new(); - - for first_name in first_names { - let mut slab = NounSlab::new(); - - let mut path_slab = NounSlab::::new(); - let path_noun = vec!["balance-by-first-name".to_string(), first_name.clone()] - .to_noun(&mut path_slab); - path_slab.set_root(path_noun); - let path_bytes = path_slab.jam().to_vec(); - - let response = client.peek(request_index, path_bytes).await.map_err(|e| { - NockAppError::OtherError(format!( - "Failed to peek balance for first name {first_name}: {e}" - )) - })?; - request_index = request_index.wrapping_add(1); - - let balance = slab.cue_into(response.as_bytes()?)?; - let head = make_tas(&mut slab, "update-balance-grpc").as_noun(); - let full = T(&mut slab, &[head, balance]); - slab.set_root(full); - results.push(slab); - } - - for key in pubkeys { - let mut slab = NounSlab::new(); - let mut path_slab = NounSlab::::new(); - let path_noun = - vec!["balance-by-pubkey".to_string(), key.clone()].to_noun(&mut path_slab); - path_slab.set_root(path_noun); - let path_bytes = path_slab.jam().to_vec(); - - let response = client.peek(request_index, path_bytes).await.map_err(|e| { - NockAppError::OtherError(format!("Failed to peek balance for pubkey {key}: {e}")) - })?; - request_index = request_index.wrapping_add(1); - - let balance = slab.cue_into(response.as_bytes()?)?; - let head = make_tas(&mut slab, "update-balance-grpc").as_noun(); - let full = T(&mut slab, &[head, balance]); - slab.set_root(full); - results.push(slab); - } - - Ok(results) - } - /// Lists all notes in the wallet. - /// - /// Retrieves and displays all notes from the wallet's balance, sorted by assets. fn list_notes() -> CommandNoun { let mut slab = NounSlab::new(); Self::wallet("list-notes", &[], Operation::Poke, &mut slab) @@ -1285,6 +1200,7 @@ impl Wallet { Ok((index, hardened)) } + /// Parses permissive bool-like hardened flags used by CLI sign-key input. fn parse_boolish(flag: &str) -> Result { match flag { "true" | "t" | "1" | "yes" | "y" => Ok(true), @@ -1297,6 +1213,7 @@ impl Wallet { } } + /// Parses comma-separated `index:hardened` sign-key tuples from CLI input. fn parse_sign_keys(sign_keys_str: &str) -> Result, NockAppError> { let mut sign_keys = Vec::new(); for piece in sign_keys_str.split(',') { @@ -1322,6 +1239,7 @@ impl Wallet { Ok(sign_keys) } + /// Parses comma-separated base58 pubkey hashes for multisig watch import. fn parse_pubkey_hashes(pubkeys_str: &str) -> Result, NockAppError> { let pubkeys: Vec = pubkeys_str .split(',') @@ -1350,6 +1268,7 @@ impl Wallet { Ok(pubkeys) } + /// Signs a multisig transaction with provided key index/hardened tuples. fn sign_multisig_tx( transaction_path: &str, sign_keys_str: Option<&str>, @@ -1387,6 +1306,7 @@ impl Wallet { } #[allow(dead_code)] + /// Displays a multisig transaction payload without signing. fn show_multisig_tx(transaction_path: &str) -> CommandNoun { let mut slab = NounSlab::new(); @@ -1406,6 +1326,7 @@ impl Wallet { } } +/// Returns wallet data directory path, creating it if missing. pub async fn wallet_data_dir() -> Result { let wallet_data_dir = system_data_dir().join("wallet"); if !wallet_data_dir.exists() { @@ -1419,6 +1340,7 @@ pub async fn wallet_data_dir() -> Result { } #[allow(dead_code)] +/// Confirms dangerous upper-bound timelock usage with explicit user acknowledgement. fn confirm_upper_bound_warning() -> Result<(), NockAppError> { println!( "Warning: specifying an upper timelock bound will make the output unspendable after that height. Only use this feature if you know what you're doing." @@ -1442,6 +1364,7 @@ fn confirm_upper_bound_warning() -> Result<(), NockAppError> { } } +/// Normalizes watch input as either schnorr pubkey or hash base58 value. fn normalize_watch_address(value: String) -> Result, NockAppError> { if value.len() >= SchnorrPubkey::BYTES_BASE58 { match SchnorrPubkey::from_base58(&value) { @@ -1469,6 +1392,7 @@ fn normalize_watch_address(value: String) -> Result, NockAppError } #[allow(dead_code)] +/// Normalizes a first-name hash and filters invalid values. fn normalize_first_name(value: String) -> Result, NockAppError> { match Hash::from_base58(&value) { Ok(hash) => Ok(Some(hash.to_base58())), @@ -1479,6 +1403,7 @@ fn normalize_first_name(value: String) -> Result, NockAppError> { } } +/// Queries the public node for acceptance status of one transaction id. async fn run_transaction_accepted( connection: &connection::ConnectionCli, tx_id: &str, @@ -1539,6 +1464,7 @@ async fn run_transaction_accepted( Ok(()) } +/// Renders a compact markdown summary for transaction acceptance status. fn format_transaction_accepted_markdown(tx_id: &str, accepted: bool) -> String { let status_line = if accepted { "- status: **accepted by node**" @@ -1554,436 +1480,10 @@ fn format_transaction_accepted_markdown(tx_id: &str, accepted: bool) -> String { .join("\n") } +/// Builds an atom from raw bytes using indirect atom allocation. pub fn from_bytes(stack: &mut NounSlab, bytes: &[u8]) -> Atom { unsafe { let mut tas_atom = IndirectAtom::new_raw_bytes(stack, bytes.len(), bytes.as_ptr()); - tas_atom.normalize_as_atom() - } -} - -// TODO: all these tests need to also validate the results and not -// just ensure that the wallet can be poked with the expected noun. -#[allow(warnings)] -#[cfg(test)] -mod tests { - use std::sync::Once; - - use nockapp::kernel::boot::{self, Cli as BootCli}; - use nockapp::wire::SystemWire; - use nockapp::{exit_driver, AtomExt, Bytes}; - use nockchain_math::belt::Belt; - use nockchain_types::tx_engine::common::{BlockHeight, BlockHeightDelta}; - use nockchain_types::tx_engine::v0; - use tokio::sync::mpsc; - - use super::*; - - static INIT: Once = Once::new(); - - fn init_tracing() { - INIT.call_once(|| { - let cli = boot::default_boot_cli(true); - boot::init_default_tracing(&cli); - }); - } - - #[test] - fn timelock_cli_accepts_ascending_bound() { - let range: TimelockRangeCli = "1..5".parse().unwrap(); - let absolute = range.absolute(); - assert_eq!(absolute.min, Some(BlockHeight(Belt(1)))); - assert_eq!(absolute.max, Some(BlockHeight(Belt(5)))); - } - - #[test] - fn timelock_cli_accepts_open_upper_bound() { - let range: TimelockRangeCli = "..5".parse().unwrap(); - let absolute = range.absolute(); - assert_eq!(absolute.min, None); - assert_eq!(absolute.max, Some(BlockHeight(Belt(5)))); - } - - #[test] - fn timelock_cli_accepts_open_lower_bound() { - let range: TimelockRangeCli = "7..".parse().unwrap(); - let relative = range.relative(); - assert_eq!(relative.min, Some(BlockHeightDelta(Belt(7)))); - assert_eq!(relative.max, None); - } - - #[test] - fn timelock_cli_rejects_descending_bounds() { - let err = TimelockRangeCli::from_bounds(Some(10), Some(5)).unwrap_err(); - assert!(err.contains("min <= max")); - } - - #[test] - fn timelock_cli_allows_fully_open_interval() { - let range: TimelockRangeCli = "..".parse().unwrap(); - assert!(range.absolute().min.is_none() && range.absolute().max.is_none()); - assert!(range.relative().min.is_none() && range.relative().max.is_none()); - assert!(!range.has_upper_bound()); - } - - #[test] - fn timelock_intent_from_ranges_handles_none() { - assert!(Wallet::timelock_intent_from_ranges(None, None).is_none()); - let open_range: TimelockRangeCli = "..".parse().unwrap(); - - let explicit_none = Wallet::timelock_intent_from_ranges( - Some(open_range.absolute()), - Some(open_range.relative()), - ) - .expect("expected explicit timelock intent"); - - assert_eq!( - explicit_none, - v0::TimelockIntent { - absolute: TimelockRangeAbsolute::none(), - relative: TimelockRangeRelative::none(), - } - ); - } - - #[test] - fn timelock_intent_from_ranges_accepts_partial_specs() { - let absolute = TimelockRangeAbsolute::none(); - let intent = Wallet::timelock_intent_from_ranges(Some(absolute.clone()), None) - .expect("absolute range should produce intent"); - assert_eq!(intent.absolute, absolute); - assert_eq!(intent.relative, TimelockRangeRelative::none()); - } - - #[test] - fn parse_note_names_accepts_valid_pairs() { - let parsed = Wallet::parse_note_names("[foo bar],[baz qux]").expect("valid names"); - assert_eq!( - parsed, - vec![("foo".to_string(), "bar".to_string()), ("baz".to_string(), "qux".to_string())] - ); - } - - #[test] - fn parse_note_names_rejects_invalid_format() { - let err = Wallet::parse_note_names("foo bar").expect_err("expected failure"); - assert!( - err.to_string().contains("Invalid note name"), - "unexpected error message: {err}" - ); - } - - #[test] - fn collect_signing_keys_prefers_explicit_entries() { - let entries = vec!["0:true".to_string(), "1:false".to_string()]; - let keys = - Wallet::collect_signing_keys(Some(5), false, &entries).expect("valid explicit keys"); - assert_eq!(keys, vec![(0, true), (1, false)]); - } - - #[test] - fn collect_signing_keys_falls_back_to_index() { - let keys = Wallet::collect_signing_keys(Some(3), true, &[]).expect("valid"); - assert_eq!(keys, vec![(3, true)]); - } - - #[test] - fn collect_signing_keys_defaults_to_master() { - let keys = Wallet::collect_signing_keys(None, false, &[]).expect("valid"); - assert!(keys.is_empty()); - } - - #[tokio::test] - #[cfg_attr(miri, ignore)] - async fn test_keygen() -> Result<(), NockAppError> { - init_tracing(); - let cli = BootCli::parse_from(&["--new"]); - - let prover_hot_state = produce_prover_hot_state(); - let nockapp = boot::setup( - KERNEL, - cli.clone(), - prover_hot_state.as_slice(), - "wallet", - None, - ) - .await - .map_err(|e| CrownError::Unknown(e.to_string()))?; - let mut wallet = Wallet::new(nockapp); - let mut entropy = [0u8; 32]; - let mut salt = [0u8; 16]; - getrandom::fill(&mut entropy).map_err(|e| CrownError::Unknown(e.to_string()))?; - getrandom::fill(&mut salt).map_err(|e| CrownError::Unknown(e.to_string()))?; - let (noun, op) = Wallet::keygen(&entropy, &salt)?; - - let wire = WalletWire::Command(Commands::Keygen).to_wire(); - - let keygen_result = wallet.app.poke(wire, noun.clone()).await?; - - println!("keygen result: {:?}", keygen_result); - assert!( - keygen_result.len() == 2, - "Expected keygen result to be a list of 2 noun slabs - markdown and exit" - ); - let exit_cause = unsafe { keygen_result[1].root() }; - let code = exit_cause.as_cell()?.tail(); - assert!(unsafe { code.raw_equals(&D(0)) }, "Expected exit code 0"); - - Ok(()) - } - - #[tokio::test] - #[cfg_attr(miri, ignore)] - async fn test_derive_child() -> Result<(), NockAppError> { - init_tracing(); - let cli = BootCli::parse_from(&["--new"]); - - let prover_hot_state = produce_prover_hot_state(); - let nockapp = boot::setup( - KERNEL, - cli.clone(), - prover_hot_state.as_slice(), - "wallet", - None, - ) - .await - .map_err(|e| CrownError::Unknown(e.to_string()))?; - let mut wallet = Wallet::new(nockapp); - - // Generate a new key pair - let mut entropy = [0u8; 32]; - let mut salt = [0u8; 16]; - let (noun, op) = Wallet::keygen(&entropy, &salt)?; - let wire = WalletWire::Command(Commands::Keygen).to_wire(); - let _ = wallet.app.poke(wire, noun.clone()).await?; - - // Derive a child key - let index = 0; - let hardened = true; - let label = None; - let (noun, op) = Wallet::derive_child(index, hardened, &label)?; - - let wire = WalletWire::Command(Commands::DeriveChild { - index, - hardened, - label, - }) - .to_wire(); - - let derive_result = wallet.app.poke(wire, noun.clone()).await?; - - assert!( - derive_result.len() == 2, - "Expected derive result to be a list of 2 noun slabs - markdown and exit" - ); - - let exit_cause = unsafe { derive_result[1].root() }; - let code = exit_cause.as_cell()?.tail(); - assert!(unsafe { code.raw_equals(&D(0)) }, "Expected exit code 0"); - - Ok(()) - } - - // Tests for Cold Side Commands - #[tokio::test] - #[cfg_attr(miri, ignore)] - async fn test_gen_master_privkey() -> Result<(), NockAppError> { - init_tracing(); - let cli = BootCli::parse_from(&[""]); - let nockapp = boot::setup(KERNEL, cli.clone(), &[], "wallet", None) - .await - .map_err(|e| CrownError::Unknown(e.to_string()))?; - let mut wallet = Wallet::new(nockapp); - let seedphrase = "correct horse battery staple"; - let version = 1; - let (noun, op) = Wallet::import_seed_phrase(seedphrase, version)?; - println!("privkey_slab: {:?}", noun); - let wire = WalletWire::Command(Commands::ImportKeys { - file: None, - key: None, - seedphrase: Some(seedphrase.to_string()), - version: Some(version), - }) - .to_wire(); - let privkey_result = wallet.app.poke(wire, noun.clone()).await?; - println!("privkey_result: {:?}", privkey_result); - Ok(()) - } - - // Tests for Hot Side Commands - // TODO: fix this test by adding a real key file - #[tokio::test] - #[ignore] - async fn test_import_keys() -> Result<(), NockAppError> { - init_tracing(); - let cli = BootCli::parse_from(&["--new"]); - let nockapp = boot::setup(KERNEL, cli.clone(), &[], "wallet", None) - .await - .map_err(|e| CrownError::Unknown(e.to_string()))?; - let mut wallet = Wallet::new(nockapp); - - // Create test key file - let test_path = "test_keys.jam"; - let test_data = vec![0u8; 32]; // TODO: Use real jammed key data - fs::write(test_path, &test_data).expect(&format!( - "Called `expect()` at {}:{} (git sha: {})", - file!(), - line!(), - option_env!("GIT_SHA").unwrap_or("unknown") - )); - - let (noun, op) = Wallet::import_keys(test_path)?; - let wire = WalletWire::Command(Commands::ImportKeys { - file: Some(test_path.to_string()), - key: None, - seedphrase: None, - version: None, - }) - .to_wire(); - let import_result = wallet.app.poke(wire, noun.clone()).await?; - - fs::remove_file(test_path).expect(&format!( - "Called `expect()` at {}:{} (git sha: {})", - file!(), - line!(), - option_env!("GIT_SHA").unwrap_or("unknown") - )); - - println!("import result: {:?}", import_result); - assert!( - !import_result.is_empty(), - "Expected non-empty import result" - ); - - Ok(()) - } - - // TODO: fix this test - #[tokio::test] - #[ignore] - async fn test_spend_multisig_format() -> Result<(), NockAppError> { - // TODO: replace with an end-to-end test that exercises multisig recipient specs. - Ok(()) - } - - #[tokio::test] - #[ignore] - async fn test_spend_single_sig_format() -> Result<(), NockAppError> { - // TODO: replace with an end-to-end test for PKH recipients once fixtures exist. - Ok(()) - } - - #[tokio::test] - #[cfg_attr(miri, ignore)] - async fn test_list_notes() -> Result<(), NockAppError> { - init_tracing(); - let cli = BootCli::parse_from(&[""]); - let nockapp = boot::setup(KERNEL, cli.clone(), &[], "wallet", None) - .await - .map_err(|e| CrownError::Unknown(e.to_string()))?; - let mut wallet = Wallet::new(nockapp); - - // Test listing notes - let (noun, op) = Wallet::list_notes()?; - let wire = WalletWire::Command(Commands::ListNotes {}).to_wire(); - let list_result = wallet.app.poke(wire, noun.clone()).await?; - println!("list_result: {:?}", list_result); - - Ok(()) - } - - // TODO: fix this test by adding a real draft - #[tokio::test] - #[ignore] - async fn test_make_tx_from_draft() -> Result<(), NockAppError> { - init_tracing(); - let cli = BootCli::parse_from(&[""]); - let nockapp = boot::setup(KERNEL, cli.clone(), &[], "wallet", None) - .await - .map_err(|e| CrownError::Unknown(e.to_string()))?; - let mut wallet = Wallet::new(nockapp); - - // use the transaction in txs/ - let transaction_path = "txs/test_transaction.tx"; - let test_data = vec![0u8; 32]; // TODO: Use real transaction data - fs::write(transaction_path, &test_data).expect(&format!( - "Called `expect()` at {}:{} (git sha: {})", - file!(), - line!(), - option_env!("GIT_SHA").unwrap_or("unknown") - )); - - let (noun, op) = Wallet::send_tx(transaction_path)?; - let wire = WalletWire::Command(Commands::SendTx { - transaction: transaction_path.to_string(), - }) - .to_wire(); - let tx_result = wallet.app.poke(wire, noun.clone()).await?; - - fs::remove_file(transaction_path).expect(&format!( - "Called `expect()` at {}:{} (git sha: {})", - file!(), - line!(), - option_env!("GIT_SHA").unwrap_or("unknown") - )); - - println!("transaction result: {:?}", tx_result); - assert!( - !tx_result.is_empty(), - "Expected non-empty transaction result" - ); - - Ok(()) - } - - #[tokio::test] - #[ignore] - async fn test_show_tx() -> Result<(), NockAppError> { - init_tracing(); - let cli = BootCli::parse_from(&[""]); - let nockapp = boot::setup(KERNEL, cli.clone(), &[], "wallet", None) - .await - .map_err(|e| CrownError::Unknown(e.to_string()))?; - let mut wallet = Wallet::new(nockapp); - - // Create a temporary transaction file - let transaction_path = "test_show_transaction.tx"; - let test_data = vec![0u8; 32]; // TODO: Use real transaction data - fs::write(transaction_path, &test_data).expect(&format!( - "Called `expect()` at {}:{} (git sha: {})", - file!(), - line!(), - option_env!("GIT_SHA").unwrap_or("unknown") - )); - - let (noun, op) = Wallet::show_tx(transaction_path)?; - let wire = WalletWire::Command(Commands::ShowTx { - transaction: transaction_path.to_string(), - }) - .to_wire(); - let show_result = wallet.app.poke(wire, noun.clone()).await?; - - fs::remove_file(transaction_path).expect(&format!( - "Called `expect()` at {}:{} (git sha: {})", - file!(), - line!(), - option_env!("GIT_SHA").unwrap_or("unknown") - )); - - println!("show-tx result: {:?}", show_result); - assert!(!show_result.is_empty(), "Expected non-empty show-tx result"); - - Ok(()) - } - - #[test] - fn domain_hash_from_base58_accepts_valid_id() { - let tx_id = "3giXkwW4zbFhoyJu27RbP6VNiYgR6yaTfk2AYnEHvxtVaGbmcVD6jb9"; - Hash::from_base58(tx_id).expect("expected valid base58 hash"); - } - - #[test] - fn domain_hash_from_base58_rejects_invalid_id() { - let invalid_tx_id = "not-a-valid-hash"; - assert!(Hash::from_base58(invalid_tx_id).is_err()); + tas_atom.normalize_as_atom_stack() } } diff --git a/crates/nockchain-wallet/src/recipient.rs b/crates/nockchain-wallet/src/recipient.rs index 11ea5f310..4de11aa85 100644 --- a/crates/nockchain-wallet/src/recipient.rs +++ b/crates/nockchain-wallet/src/recipient.rs @@ -1,12 +1,17 @@ use std::collections::BTreeSet; use nockchain_types::common::Hash; +use nockchain_types::tx_engine::v1::tx::{Lock, LockPrimitive, Pkh, SpendCondition}; use nockchain_types::{EthAddress, EthAddressParseError}; use noun_serde::{NounDecode, NounEncode}; use serde::Deserialize; +use wallet_tx_builder::types::{PlannedOutput, RawNoteDataEntry}; use crate::{CrownError, NockAppError}; +pub const BRIDGE_LOCK_ROOT_DEFAULT_B58: &str = + "AcsPkuhXQoGeEsF91yynpm1kcW17PQ2Z1MEozgx7YnDPkZwrtzLuuqd"; + #[derive(Debug, Clone, Deserialize)] #[serde(tag = "kind", rename_all = "lowercase")] pub enum RecipientSpecToken { @@ -218,10 +223,108 @@ pub fn recipient_tokens_to_specs( .collect() } +fn pkh_lock(threshold: u64, addresses: &[Hash]) -> Lock { + Lock::SpendCondition(SpendCondition::new(vec![LockPrimitive::Pkh(Pkh::new( + threshold, + addresses.to_vec(), + ))])) +} + +fn lock_root(lock: &Lock) -> Result { + lock.hash() + .map_err(|err| CrownError::Unknown(format!("unable to derive lock root: {err}")).into()) +} + +fn evm_address_to_based(evm_address: EthAddress) -> [u64; 3] { + let mut be = [0_u8; 32]; + be[12..].copy_from_slice(evm_address.as_slice()); + let limbs = Hash::from_be_bytes(&be).to_array(); + [limbs[0], limbs[1], limbs[2]] +} + +/// Converts CLI recipient specs into planner outputs with tx-builder-compatible note-data. +pub fn planner_recipient_outputs( + recipients: &[RecipientSpec], + include_data: bool, +) -> Result, NockAppError> { + recipients + .iter() + .map(|recipient| planner_recipient_output(recipient, include_data)) + .collect() +} + +/// Builds one planner output from a recipient, including deterministic lock root + note-data. +pub fn planner_recipient_output( + recipient: &RecipientSpec, + include_data: bool, +) -> Result { + match recipient { + RecipientSpec::P2pkh { address, amount } => { + let lock = pkh_lock(1, std::slice::from_ref(address)); + let note_data = if include_data { + vec![RawNoteDataEntry::from_lock(lock.clone())] + } else { + Vec::new() + }; + Ok(PlannedOutput { + lock_root: lock_root(&lock)?, + amount: *amount, + note_data, + }) + } + RecipientSpec::Multisig { + threshold, + addresses, + amount, + } => { + let lock = pkh_lock(*threshold, addresses); + Ok(PlannedOutput { + lock_root: lock_root(&lock)?, + amount: *amount, + // Hoon always includes lock note-data for multisig outputs. + note_data: vec![RawNoteDataEntry::from_lock(lock.clone())], + }) + } + RecipientSpec::BridgeDeposit { + evm_address, + amount, + } => Ok(PlannedOutput { + lock_root: Hash::from_base58(BRIDGE_LOCK_ROOT_DEFAULT_B58).map_err(|err| { + NockAppError::from(CrownError::Unknown(format!( + "Invalid bridge lock root constant '{}': {}", + BRIDGE_LOCK_ROOT_DEFAULT_B58, err + ))) + })?, + amount: *amount, + note_data: vec![RawNoteDataEntry::from_bridge_deposit(evm_address_to_based( + *evm_address, + ))], + }), + } +} + +pub fn planner_refund_output_template( + refund_pkh: Option<&Hash>, + signer_pkh: &Hash, + include_data: bool, +) -> Result { + let refund_owner = refund_pkh.unwrap_or(signer_pkh).clone(); + let refund_lock = pkh_lock(1, std::slice::from_ref(&refund_owner)); + Ok(PlannedOutput { + lock_root: lock_root(&refund_lock)?, + amount: 0, + note_data: if include_data { + vec![RawNoteDataEntry::from_lock(refund_lock.clone())] + } else { + Vec::new() + }, + }) +} + #[cfg(test)] mod tests { use nockapp::noun::slab::{NockJammer, NounSlab}; - use nockvm::noun::FullDebugCell; + use nockvm::noun::NounAllocator; use noun_serde::NounDecode; use super::*; @@ -394,9 +497,9 @@ mod tests { let mut slab = NounSlab::::new(); for spec in specs { let noun = spec.to_noun(&mut slab); - eprintln!("spec noun: {:?}", FullDebugCell(&noun.as_cell().unwrap())); - let decoded = - RecipientSpec::from_noun(&noun).expect("recipient spec should decode from noun"); + let space = slab.noun_space(); + let decoded = RecipientSpec::from_noun(&noun, &space) + .expect("recipient spec should decode from noun"); assert_eq!(decoded, spec); } } diff --git a/crates/nockchain-wallet/src/tests.rs b/crates/nockchain-wallet/src/tests.rs new file mode 100644 index 000000000..f0c98b51e --- /dev/null +++ b/crates/nockchain-wallet/src/tests.rs @@ -0,0 +1,2058 @@ +#![allow(warnings)] +// TODO: all these tests need to also validate the results and not +// just ensure that the wallet can be poked with the expected noun. + +use std::collections::BTreeMap; +use std::sync::Once; + +use nockapp::kernel::boot::{self, ephemeral_test_boot_cli, Cli as BootCli}; +use nockapp::wire::SystemWire; +use nockapp::{exit_driver, AtomExt, Bytes}; +use nockchain_math::belt::Belt; +use nockchain_math::zoon::zmap::ZMap; +use nockchain_types::default_fakenet_blockchain_constants; +use nockchain_types::tx_engine::common::{BlockHeight, BlockHeightDelta, Nicks, Signature}; +use nockchain_types::tx_engine::v1::note::{NoteData, NoteDataEntry}; +use nockchain_types::tx_engine::v1::tx::{Lock, LockPrimitive, Pkh, SpendCondition}; +use nockchain_types::tx_engine::{v0, v1}; +use nockvm::noun::{NounAllocator, T}; +use noun_serde::{NounDecode, NounEncode}; +use tempfile::TempDir; +use tokio::sync::mpsc; +use wallet_tx_builder::adapter::normalize_balance_pages; +use wallet_tx_builder::fee::{compute_minimum_fee, FeeInputs}; +use wallet_tx_builder::planner::plan_create_tx; +use wallet_tx_builder::types::{ + CandidateVersionPolicy, ChainContext, PlanRequest, PlanningMode, RawNoteDataEntry, + SelectionMode, SelectionOrder, +}; + +use super::*; +use crate::create_tx::{ + ensure_manual_planner_parity, ActiveSignerEntryNoun, MigrateV0NotesSummary, + MigrateV0SignerSummary, PlannerBlockchainConstantsNoun, PlannerNoteDataConstantsNoun, + SigningKeyLockMatcher, +}; +use crate::recipient::{planner_recipient_outputs, RecipientSpec}; + +static INIT: Once = Once::new(); + +fn init_tracing() { + INIT.call_once(|| { + let cli = ephemeral_test_boot_cli(true); + boot::init_default_tracing(&cli); + }); +} + +fn hash(v: u64) -> Hash { + Hash::from_limbs(&[v, 0, 0, 0, 0]) +} + +fn name(first: u64, last: u64) -> Name { + Name::new(hash(first), hash(last)) +} + +fn signer_key(pkh: u64) -> Hash { + hash(pkh) +} + +fn note_v1(first: u64, last: u64, origin_page: u64, assets: u64) -> v1::Note { + v1::Note::V1(v1::NoteV1::new( + BlockHeight(Belt(origin_page)), + name(first, last), + v1::NoteData::new(Vec::new()), + Nicks(assets as usize), + )) +} + +fn balance_page(height: u64, block_id: u64, notes: Vec<(Name, v1::Note)>) -> v1::BalanceUpdate { + v1::BalanceUpdate { + height: BlockHeight(Belt(height)), + block_id: hash(block_id), + notes: v1::Balance(notes), + } +} + +fn simple_pkh_lock(pkh: Hash) -> Lock { + Lock::SpendCondition(SpendCondition::new(vec![LockPrimitive::Pkh(Pkh::new( + 1, + vec![pkh], + ))])) +} + +fn simple_v0_lock(pubkey: SchnorrPubkey) -> v0::Lock { + v0::Lock { + keys_required: 1, + pubkeys: vec![pubkey], + } +} + +fn note_data_from_raw_entries(entries: Vec) -> NoteData { + NoteData::new( + entries + .into_iter() + .map(|entry| NoteDataEntry::new(entry.key, entry.blob)) + .collect(), + ) +} + +fn note_v1_with_lock(name: Name, origin_page: u64, assets: u64, lock: Lock) -> v1::Note { + let note_data = note_data_from_raw_entries(vec![RawNoteDataEntry::from_lock(lock)]); + v1::Note::V1(v1::NoteV1::new( + BlockHeight(Belt(origin_page)), + name, + note_data, + Nicks(assets as usize), + )) +} + +fn note_v0_with_lock(name: Name, origin_page: u64, assets: u64, lock: v0::Lock) -> v1::Note { + v1::Note::V0(v0::NoteV0 { + head: v0::NoteHead { + version: nockchain_types::tx_engine::common::Version::V0, + origin_page: BlockHeight(Belt(origin_page)), + timelock: v0::Timelock(None), + }, + tail: v0::NoteTail { + name, + lock, + source: nockchain_types::tx_engine::common::Source { + hash: hash(origin_page + assets), + is_coinbase: false, + }, + assets: Nicks(assets as usize), + }, + }) +} + +fn noun_leaf_count(noun: nockapp::Noun, space: &nockvm::noun::NounSpace) -> u64 { + if noun.is_atom() { + return 1; + } + let cell = noun + .in_space(space) + .as_cell() + .expect("noun should decode as cell"); + noun_leaf_count(cell.head().noun(), space) + .saturating_add(noun_leaf_count(cell.tail().noun(), space)) +} + +fn word_count_from_noun_encode(value: &T) -> u64 { + let mut slab: NounSlab = NounSlab::new(); + let noun = value.to_noun(&mut slab); + let space = slab.noun_space(); + noun_leaf_count(noun, &space) +} + +fn merged_seed_word_count(spends: &v1::Spends) -> u64 { + let mut merged_by_lock_root = BTreeMap::<[u64; 5], BTreeMap>::new(); + + for (_, spend) in &spends.0 { + let seeds = match spend { + v1::Spend::Legacy(spend0) => &spend0.seeds.0, + v1::Spend::Witness(spend1) => &spend1.seeds.0, + }; + + for seed in seeds { + let merged = merged_by_lock_root + .entry(seed.lock_root.to_array()) + .or_default(); + for note_data_entry in seed.note_data.iter() { + merged.insert(note_data_entry.key.clone(), note_data_entry.blob.clone()); + } + } + } + + merged_by_lock_root + .into_values() + .map(|merged| { + let entries = merged + .into_iter() + .map(|(key, blob)| NoteDataEntry::new(key, blob)) + .collect::>(); + word_count_from_noun_encode(&NoteData::new(entries)) + }) + .sum() +} + +fn witness_word_count(spends: &v1::Spends) -> u64 { + spends + .0 + .iter() + .map(|(_, spend)| match spend { + v1::Spend::Legacy(spend0) => word_count_from_noun_encode(&spend0.signature), + v1::Spend::Witness(spend1) => word_count_from_noun_encode(&spend1.witness), + }) + .sum() +} + +fn total_paid_fee(spends: &v1::Spends) -> u64 { + spends + .0 + .iter() + .map(|(_, spend)| match spend { + v1::Spend::Legacy(spend0) => spend0.fee.0 as u64, + v1::Spend::Witness(spend1) => spend1.fee.0 as u64, + }) + .sum() +} + +fn total_seed_gift(spends: &v1::Spends) -> u64 { + spends + .0 + .iter() + .flat_map(|(_, spend)| match spend { + v1::Spend::Legacy(spend0) => spend0.seeds.0.iter(), + v1::Spend::Witness(spend1) => spend1.seeds.0.iter(), + }) + .map(|seed| seed.gift.0 as u64) + .sum() +} + +fn decode_saved_transaction_spends(effects: &[NounSlab]) -> Result { + let tx_bytes = effects + .iter() + .find_map(|effect| { + let space = effect.noun_space(); + let noun = unsafe { effect.root() }; + let cell = noun.in_space(&space).as_cell().ok()?; + let tag = cell.head().as_atom().ok()?.into_string().ok()?; + if tag != "file" { + return None; + } + let op_cell = cell.tail().as_cell().ok()?; + let op_tag = op_cell.head().as_atom().ok()?.into_string().ok()?; + if op_tag != "write" { + return None; + } + let write_cell = op_cell.tail().as_cell().ok()?; + let path = write_cell.head().as_atom().ok()?.into_string().ok()?; + if !path.ends_with(".tx") { + return None; + } + Some(Bytes::copy_from_slice( + write_cell.tail().as_atom().ok()?.as_ne_bytes(), + )) + }) + .ok_or_else(|| NockAppError::OtherError("missing saved transaction file effect".into()))?; + + decode_transaction_spends_from_bytes(&tx_bytes) +} + +fn decode_saved_transaction_spends_from_path( + transaction_path: &str, +) -> Result { + let tx_bytes = fs::read(transaction_path).map_err(|err| { + NockAppError::OtherError(format!("failed to read transaction file: {err}")) + })?; + decode_transaction_spends_from_bytes(&tx_bytes) +} + +fn decode_transaction_spends_from_bytes(tx_bytes: &[u8]) -> Result { + let mut slab: NounSlab = NounSlab::new(); + let transaction_noun = slab.cue_into(Bytes::copy_from_slice(tx_bytes))?; + let space = slab.noun_space(); + let transaction_cell = transaction_noun.in_space(&space).as_cell().map_err(|err| { + NockAppError::OtherError(format!("transaction jam root not a cell: {err}")) + })?; + let version = + ::from_noun(&transaction_cell.head().noun(), &space).map_err(|err| { + NockAppError::OtherError(format!("transaction version did not decode: {err}")) + })?; + if version != 1 { + return Err(NockAppError::OtherError(format!( + "expected saved transaction version 1, got {version}" + ))); + } + let name_and_rest = transaction_cell.tail().as_cell().map_err(|err| { + NockAppError::OtherError(format!("transaction jam missing name/rest cell: {err}")) + })?; + let spends_and_rest = name_and_rest.tail().as_cell().map_err(|err| { + NockAppError::OtherError(format!("transaction jam missing spends/rest cell: {err}")) + })?; + let mut spends = + v1::Spends::from_noun(&spends_and_rest.head().noun(), &space).map_err(|err| { + NockAppError::OtherError(format!("saved transaction spends did not decode: {err}")) + })?; + let display_and_witness = spends_and_rest.tail().as_cell().map_err(|err| { + NockAppError::OtherError(format!( + "transaction jam missing display/witness-data cell: {err}" + )) + })?; + let witness_data = display_and_witness.tail(); + let witness_cell = witness_data.as_cell().map_err(|err| { + NockAppError::OtherError(format!("transaction jam witness-data not a cell: {err}")) + })?; + let witness_tag = + ::from_noun(&witness_cell.head().noun(), &space).map_err(|err| { + NockAppError::OtherError(format!("witness-data tag did not decode: {err}")) + })?; + match witness_tag { + 0 => { + let signatures = + ZMap::::from_noun(&witness_cell.tail().noun(), &space).map_err( + |err| { + NockAppError::OtherError(format!( + "legacy witness-data signature map did not decode: {err}" + )) + }, + )?; + for (name, signature) in signatures.into_entries() { + let Some((_, v1::Spend::Legacy(spend0))) = spends + .0 + .iter_mut() + .find(|(candidate, _)| *candidate == name) + else { + return Err(NockAppError::OtherError(format!( + "legacy witness-data referenced unknown spend {} / {}", + name.first.to_base58(), + name.last.to_base58() + ))); + }; + spend0.signature = signature; + } + } + 1 => { + let witnesses = ZMap::::from_noun( + &witness_cell.tail().noun(), + &space, + ) + .map_err(|err| { + NockAppError::OtherError(format!("v1 witness-data map did not decode: {err}")) + })?; + for (name, witness) in witnesses.into_entries() { + let Some((_, v1::Spend::Witness(spend1))) = spends + .0 + .iter_mut() + .find(|(candidate, _)| *candidate == name) + else { + return Err(NockAppError::OtherError(format!( + "witness-data referenced unknown spend {} / {}", + name.first.to_base58(), + name.last.to_base58() + ))); + }; + spend1.witness = witness; + } + } + other => { + return Err(NockAppError::OtherError(format!( + "unsupported witness-data tag {other}" + ))); + } + } + Ok(spends) +} + +fn effect_tag(effect: &NounSlab) -> Option { + let space = effect.noun_space(); + let noun = unsafe { effect.root() }; + let cell = noun.in_space(&space).as_cell().ok()?; + cell.head().as_atom().ok()?.into_string().ok() +} + +fn effect_exit_code(effects: &[NounSlab]) -> Option { + effects.iter().find_map(|effect| { + if effect_tag(effect).as_deref() != Some("exit") { + return None; + } + let space = effect.noun_space(); + let noun = unsafe { effect.root() }; + let cell = noun.in_space(&space).as_cell().ok()?; + ::from_noun(&cell.tail().noun(), &space).ok() + }) +} + +fn format_note_names(names: &[Name]) -> String { + names + .iter() + .map(|name| format!("[{} {}]", name.first.to_base58(), name.last.to_base58())) + .collect::>() + .join(",") +} + +fn decode_option_handle<'a>( + noun: nockvm::noun::NounHandle<'a>, +) -> Result>, NockAppError> { + if let Ok(atom) = noun.as_atom() { + let tag = atom + .as_u64() + .map_err(|err| NockAppError::OtherError(format!("option tag did not decode: {err}")))?; + return match tag { + 1 => Ok(None), + other => Err(NockAppError::OtherError(format!( + "unexpected option atom tag {other}" + ))), + }; + } + + let cell = noun + .as_cell() + .map_err(|err| NockAppError::OtherError(format!("option payload not a cell: {err}")))?; + let tag = cell + .head() + .as_atom() + .map_err(|err| NockAppError::OtherError(format!("option head not an atom: {err}")))? + .as_u64() + .map_err(|err| NockAppError::OtherError(format!("option head did not decode: {err}")))?; + if tag != 0 { + return Err(NockAppError::OtherError(format!( + "unexpected option cell tag {tag}" + ))); + } + Ok(Some(cell.tail())) +} + +async fn peek_master_signing_key(wallet: &mut Wallet) -> Result { + let mut slab = NounSlab::new(); + let tag = make_tas(&mut slab, "master-signing-key").as_noun(); + slab.modify(|_| vec![tag, SIG]); + + let result = wallet.app.peek(slab).await?; + let space = result.noun_space(); + let decoded: Option> = unsafe { Option::from_noun(result.root(), &space)? }; + decoded.flatten().ok_or_else(|| { + NockAppError::OtherError("wallet master-signing-key peek returned no payload".to_string()) + }) +} + +async fn peek_master_signing_pubkey(wallet: &mut Wallet) -> Result { + let mut slab = NounSlab::new(); + let tag = make_tas(&mut slab, "master-signing-pubkey").as_noun(); + slab.modify(|_| vec![tag, SIG]); + + let result = wallet.app.peek(slab).await?; + let space = result.noun_space(); + let decoded: Option> = + unsafe { Option::from_noun(result.root(), &space)? }; + decoded.flatten().ok_or_else(|| { + NockAppError::OtherError( + "wallet master-signing-pubkey peek returned no payload".to_string(), + ) + }) +} + +async fn peek_active_signers( + wallet: &mut Wallet, +) -> Result, NockAppError> { + let mut slab = NounSlab::new(); + let tag = make_tas(&mut slab, "active-signers").as_noun(); + slab.modify(|_| vec![tag, SIG]); + + let result = wallet.app.peek(slab).await?; + let space = result.noun_space(); + let decoded: Option>> = + unsafe { Option::from_noun(result.root(), &space)? }; + let mut signers = decoded.flatten().unwrap_or_default(); + signers.sort_by_key(|signer| { + ( + if signer.child_index.is_none() { 0 } else { 1 }, + signer.absolute_index.unwrap_or(0), + signer.address_b58.clone(), + ) + }); + signers.dedup_by(|left, right| { + left.child_index == right.child_index + && left.hardened == right.hardened + && left.absolute_index == right.absolute_index + && left.address_b58 == right.address_b58 + }); + Ok(signers) +} + +async fn import_seed_phrase( + wallet: &mut Wallet, + seedphrase: &str, + version: u64, +) -> Result<(), NockAppError> { + let (noun, _) = Wallet::import_seed_phrase(seedphrase, version)?; + let wire = WalletWire::Command(Commands::ImportKeys { + file: None, + key: None, + seedphrase: Some(seedphrase.to_string()), + version: Some(version), + }) + .to_wire(); + let _ = wallet.app.poke(wire, noun).await?; + Ok(()) +} + +async fn derive_child_key( + wallet: &mut Wallet, + index: u64, + hardened: bool, +) -> Result<(), NockAppError> { + let label = None; + let (noun, _) = Wallet::derive_child(index, hardened, &label)?; + let wire = WalletWire::Command(Commands::DeriveChild { + index, + hardened, + label, + }) + .to_wire(); + let _ = wallet.app.poke(wire, noun).await?; + Ok(()) +} + +async fn apply_balance_update( + wallet: &mut Wallet, + balance_update: v1::BalanceUpdate, +) -> Result<(), NockAppError> { + let poke = Wallet::update_balance_grpc_poke_for_tests(balance_update); + let _ = wallet.app.poke(SystemWire.to_wire(), poke).await?; + Ok(()) +} + +async fn boot_wallet_with( + cli: BootCli, + hot_state: &[nockvm::jets::hot::HotEntry], +) -> Result<(Wallet, TempDir), NockAppError> { + let data_dir = tempfile::tempdir().map_err(NockAppError::IoError)?; + let nockapp = boot::setup( + KERNEL, + cli.clone(), + hot_state, + "wallet", + Some(data_dir.path().to_path_buf()), + ) + .await + .map_err(|e| CrownError::Unknown(e.to_string()))?; + Ok((Wallet::new(nockapp), data_dir)) +} + +async fn boot_test_wallet() -> Result<(Wallet, TempDir), NockAppError> { + let cli = ephemeral_test_boot_cli(true); + let prover_hot_state = produce_prover_hot_state(); + boot_wallet_with(cli, prover_hot_state.as_slice()).await +} + +async fn peek_wallet_blockchain_constants( + wallet: &mut Wallet, +) -> Result { + let mut slab = NounSlab::new(); + let state_tag = make_tas(&mut slab, "state").as_noun(); + slab.modify(|_| vec![state_tag, SIG]); + + let result = wallet.app.peek(slab).await?; + let space = result.noun_space(); + let root = unsafe { *result.root() }.in_space(&space); + let state = decode_option_handle(root)? + .and_then(|inner| decode_option_handle(inner).transpose()) + .transpose()? + .ok_or_else(|| NockAppError::OtherError("missing wallet state payload".to_string()))?; + let constants = state.slot(31).map_err(|err| { + NockAppError::OtherError(format!("wallet state missing blockchain constants: {err}")) + })?; + PlannerBlockchainConstantsNoun::from_noun(&constants.noun(), &space).map_err(|err| { + NockAppError::OtherError(format!("decode blockchain constants failed: {err}")) + }) +} + +async fn peek_balance_state(wallet: &mut Wallet) -> Result { + let mut slab = NounSlab::new(); + let balance_tag = make_tas(&mut slab, "balance").as_noun(); + let path = T(&mut slab, &[balance_tag, SIG]); + slab.set_root(path); + + let result = wallet.app.peek(slab).await?; + let space = result.noun_space(); + let maybe_balance: Option> = + unsafe { >>::from_noun(result.root(), &space)? }; + match maybe_balance { + Some(Some(balance)) => Ok(balance), + _ => Err(NockAppError::OtherError( + "wallet balance peek returned no balance payload".to_string(), + )), + } +} + +#[test] +fn timelock_cli_accepts_ascending_bound() { + let range: TimelockRangeCli = "1..5".parse().unwrap(); + let absolute = range.absolute(); + assert_eq!(absolute.min, Some(BlockHeight(Belt(1)))); + assert_eq!(absolute.max, Some(BlockHeight(Belt(5)))); +} + +#[test] +fn timelock_cli_accepts_open_upper_bound() { + let range: TimelockRangeCli = "..5".parse().unwrap(); + let absolute = range.absolute(); + assert_eq!(absolute.min, None); + assert_eq!(absolute.max, Some(BlockHeight(Belt(5)))); +} + +#[test] +fn timelock_cli_accepts_open_lower_bound() { + let range: TimelockRangeCli = "7..".parse().unwrap(); + let relative = range.relative(); + assert_eq!(relative.min, Some(BlockHeightDelta(Belt(7)))); + assert_eq!(relative.max, None); +} + +#[test] +fn timelock_cli_rejects_descending_bounds() { + let err = TimelockRangeCli::from_bounds(Some(10), Some(5)).unwrap_err(); + assert!(err.contains("min <= max")); +} + +#[test] +fn timelock_cli_allows_fully_open_interval() { + let range: TimelockRangeCli = "..".parse().unwrap(); + assert!(range.absolute().min.is_none() && range.absolute().max.is_none()); + assert!(range.relative().min.is_none() && range.relative().max.is_none()); + assert!(!range.has_upper_bound()); +} + +#[test] +fn timelock_intent_from_ranges_handles_none() { + assert!(Wallet::timelock_intent_from_ranges(None, None).is_none()); + let open_range: TimelockRangeCli = "..".parse().unwrap(); + + let explicit_none = Wallet::timelock_intent_from_ranges( + Some(open_range.absolute()), + Some(open_range.relative()), + ) + .expect("expected explicit timelock intent"); + + assert_eq!( + explicit_none, + v0::TimelockIntent { + absolute: TimelockRangeAbsolute::none(), + relative: TimelockRangeRelative::none(), + } + ); +} + +#[test] +fn timelock_intent_from_ranges_accepts_partial_specs() { + let absolute = TimelockRangeAbsolute::none(); + let intent = Wallet::timelock_intent_from_ranges(Some(absolute.clone()), None) + .expect("absolute range should produce intent"); + assert_eq!(intent.absolute, absolute); + assert_eq!(intent.relative, TimelockRangeRelative::none()); +} + +#[test] +fn parse_note_names_accepts_valid_pairs() { + let parsed = Wallet::parse_note_names("[foo bar],[baz qux]").expect("valid names"); + assert_eq!( + parsed, + vec![("foo".to_string(), "bar".to_string()), ("baz".to_string(), "qux".to_string())] + ); +} + +#[test] +fn parse_note_names_rejects_invalid_format() { + let err = Wallet::parse_note_names("foo bar").expect_err("expected failure"); + assert!( + err.to_string().contains("Invalid note name"), + "unexpected error message: {err}" + ); +} + +#[test] +fn manual_planner_parity_accepts_matching_names() { + let requested = vec![name(1, 101), name(2, 102)]; + let planned = vec![name(1, 101), name(2, 102)]; + ensure_manual_planner_parity(&requested, &planned) + .expect("matching planner output should pass parity check"); +} + +#[test] +fn manual_planner_parity_accepts_reordered_names() { + let requested = vec![name(1, 101), name(2, 102)]; + let planned = vec![name(2, 102), name(1, 101)]; + ensure_manual_planner_parity(&requested, &planned) + .expect("reordered planner output should pass parity check"); +} + +#[test] +fn manual_planner_parity_rejects_name_mismatch() { + let requested = vec![name(1, 101), name(2, 102)]; + let planned = vec![name(1, 101)]; + let err = ensure_manual_planner_parity(&requested, &planned) + .expect_err("name mismatch should fail parity check"); + assert!( + err.contains("selected names differ"), + "unexpected parity error: {err}" + ); +} + +#[test] +fn union_balance_pages_returns_none_for_empty_input() { + let merged = Wallet::union_balance_pages(Vec::new()).expect("empty input should succeed"); + assert!(merged.is_none()); +} + +#[test] +fn union_balance_pages_merges_and_deduplicates_notes() { + let note_a = note_v1(1, 101, 10, 5); + let note_b = note_v1(2, 102, 10, 9); + let page_one = balance_page( + 88, + 777, + vec![(name(1, 101), note_a.clone()), (name(2, 102), note_b.clone())], + ); + let page_two = balance_page( + 88, + 777, + vec![ + (name(1, 101), note_a), + // Duplicate note name with identical payload should dedupe. + (name(2, 102), note_b), + ], + ); + + let (merged_page, normalized) = Wallet::union_balance_pages(vec![page_one, page_two]) + .expect("union should succeed") + .expect("expected merged snapshot"); + + assert_eq!(merged_page.height, BlockHeight(Belt(88))); + assert_eq!(merged_page.block_id, hash(777)); + assert_eq!(merged_page.notes.0.len(), 2); + assert_eq!(normalized.candidates.len(), 2); +} + +#[test] +fn sync_filters_untracked_v1_notes_before_wallet_state_update() { + let tracked = note_v1(1, 101, 10, 5); + let untracked = note_v1(2, 102, 10, 9); + let page = balance_page( + 88, + 777, + vec![(name(1, 101), tracked), (name(2, 102), untracked)], + ); + + let filtered = Wallet::filter_untracked_v1_notes_for_tests(page, vec![hash(1)]); + assert_eq!(filtered.notes.0.len(), 1); + assert_eq!(filtered.notes.0[0].0, name(1, 101)); +} + +#[test] +fn planner_signer_candidates_include_no_signer_and_sorted_unique_signers() { + let candidates = Wallet::planner_signer_candidates(vec![hash(2), hash(1), hash(2)]); + assert_eq!(candidates, vec![None, Some(hash(1)), Some(hash(2))]); +} + +#[test] +fn planner_signer_candidates_still_try_no_signer_when_tracked_set_is_empty() { + let candidates = Wallet::planner_signer_candidates(Vec::new()); + assert_eq!(candidates, vec![None]); +} + +#[test] +fn signing_key_lock_matcher_accepts_simple_lock_for_matching_signer() { + let signer = hash(7); + let matcher = SigningKeyLockMatcher::from_signer_keys(&[signer_key(7)]); + let spend_condition = SpendCondition::new(vec![LockPrimitive::Pkh( + nockchain_types::tx_engine::v1::tx::Pkh::new(1, vec![signer]), + )]); + let first_name = spend_condition + .first_name() + .expect("simple first-name should compute") + .into_hash(); + + assert!(matcher.matches(&first_name, &spend_condition)); + assert!(!matcher.matches(&hash(999), &spend_condition)); +} + +#[test] +fn signing_key_lock_matcher_rejects_when_signer_hash_not_present() { + let matcher = SigningKeyLockMatcher::from_signer_keys(&[signer_key(1)]); + let spend_condition = SpendCondition::new(vec![LockPrimitive::Pkh( + nockchain_types::tx_engine::v1::tx::Pkh::new(1, vec![hash(2)]), + )]); + let first_name = spend_condition + .first_name() + .expect("simple first-name should compute") + .into_hash(); + + assert!(!matcher.matches(&first_name, &spend_condition)); +} + +#[test] +fn signing_key_lock_matcher_rejects_threshold_lock_when_single_signer_cannot_meet_m() { + let signer = hash(5); + let matcher = SigningKeyLockMatcher::from_signer_keys(&[signer_key(5)]); + let spend_condition = SpendCondition::new(vec![LockPrimitive::Pkh( + nockchain_types::tx_engine::v1::tx::Pkh::new(2, vec![hash(9), signer]), + )]); + + assert!(!matcher.matches(&hash(1234), &spend_condition)); +} + +#[test] +fn signing_key_lock_matcher_rejects_multisig_lock_even_when_single_sig_threshold_is_one() { + let signer = hash(5); + let matcher = SigningKeyLockMatcher::from_signer_keys(&[signer_key(5)]); + let spend_condition = SpendCondition::new(vec![LockPrimitive::Pkh( + nockchain_types::tx_engine::v1::tx::Pkh::new(1, vec![hash(9), signer]), + )]); + + assert!(!matcher.matches(&hash(1234), &spend_condition)); +} + +#[test] +fn signing_key_lock_matcher_rejects_multisig_lock_when_signers_meet_threshold() { + let matcher = SigningKeyLockMatcher::from_signer_keys(&[signer_key(5), signer_key(7)]); + let spend_condition = SpendCondition::new(vec![LockPrimitive::Pkh( + nockchain_types::tx_engine::v1::tx::Pkh::new(2, vec![hash(5), hash(6), hash(7)]), + )]); + + assert!(!matcher.matches(&hash(1234), &spend_condition)); +} + +#[test] +fn signing_key_lock_matcher_accepts_coinbase_shape_for_matching_signer() { + let signer = hash(8); + let matcher = SigningKeyLockMatcher::from_signer_keys(&[signer_key(8)]); + let spend_condition = SpendCondition::new(vec![ + LockPrimitive::Pkh(nockchain_types::tx_engine::v1::tx::Pkh::new( + 1, + vec![signer], + )), + LockPrimitive::Tim(nockchain_types::tx_engine::v1::tx::LockTim { + rel: TimelockRangeRelative::new(Some(BlockHeightDelta(Belt(1))), None), + abs: TimelockRangeAbsolute::none(), + }), + ]); + let first_name = spend_condition + .first_name() + .expect("coinbase first-name should compute") + .into_hash(); + + assert!(matcher.matches(&first_name, &spend_condition)); + assert!(!matcher.matches(&hash(82), &spend_condition)); +} + +#[test] +fn signing_key_lock_matcher_rejects_non_pkh_locks() { + let matcher = SigningKeyLockMatcher::from_signer_keys(&[signer_key(1)]); + let spend_condition = SpendCondition::new(vec![LockPrimitive::Burn]); + assert!(!matcher.matches(&hash(10), &spend_condition)); +} + +#[test] +fn signing_key_lock_matcher_rejects_unsupported_primitive_even_with_matching_signer() { + let signer = hash(1); + let matcher = SigningKeyLockMatcher::from_signer_keys(&[signer_key(1)]); + let spend_condition = SpendCondition::new(vec![ + LockPrimitive::Pkh(nockchain_types::tx_engine::v1::tx::Pkh::new( + 1, + vec![signer], + )), + LockPrimitive::Burn, + ]); + assert!(!matcher.matches(&hash(10), &spend_condition)); +} + +#[test] +fn planner_recipient_outputs_match_hoon_lock_root_vectors() { + const EXPECTED_PKH_ROOT_B58: &str = "DKrgXqE8bXR1uBZ3t4vU13m2KquGCDbnn1PeoPL7dxSHTucGPFDPt53"; + const EXPECTED_MULTISIG_2_OF_2_ROOT_B58: &str = + "4eMAT3BuhLPjYFronoYJ9RSLVSgveCL3nQB7RHSLZzjBTiYCxEzkzEH"; + const ADDRESS_A_B58: &str = "9yPePjfWAdUnzaQKyxcRXKRa5PpUzKKEwtpECBZsUYt9Jd7egSDEWoV"; + const ADDRESS_B_B58: &str = "9phXGACnW4238oqgvn2gpwaUjG3RAqcxq2Ash2vaKp8KjzSd3MQ56Jt"; + + let address_a = Hash::from_base58(ADDRESS_A_B58).expect("address a should parse"); + let address_b = Hash::from_base58(ADDRESS_B_B58).expect("address b should parse"); + let bridge_address = + nockchain_types::EthAddress::from_hex_str("0xaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa") + .expect("bridge address should parse"); + let recipients = vec![ + RecipientSpec::P2pkh { + address: address_a.clone(), + amount: 5, + }, + RecipientSpec::Multisig { + threshold: 2, + addresses: vec![address_a.clone(), address_b], + amount: 7, + }, + RecipientSpec::BridgeDeposit { + evm_address: bridge_address, + amount: 9, + }, + ]; + + let outputs = planner_recipient_outputs(&recipients, true).expect("recipient outputs"); + assert_eq!(outputs.len(), 3); + assert_eq!(outputs[0].lock_root.to_base58(), EXPECTED_PKH_ROOT_B58); + assert_eq!( + outputs[1].lock_root.to_base58(), + EXPECTED_MULTISIG_2_OF_2_ROOT_B58 + ); + assert_eq!( + outputs[2].lock_root.to_base58(), + BRIDGE_LOCK_ROOT_DEFAULT_B58 + ); + assert_eq!(outputs[0].note_data.len(), 1); + assert_eq!(outputs[0].note_data[0].key, "lock"); + assert_eq!(outputs[1].note_data.len(), 1); + assert_eq!(outputs[1].note_data[0].key, "lock"); + assert_eq!(outputs[2].note_data.len(), 1); + assert_eq!(outputs[2].note_data[0].key, "bridge"); +} + +#[test] +fn planner_recipient_outputs_respect_include_data_for_p2pkh_but_not_multisig_or_bridge() { + const ADDRESS_A_B58: &str = "9yPePjfWAdUnzaQKyxcRXKRa5PpUzKKEwtpECBZsUYt9Jd7egSDEWoV"; + const ADDRESS_B_B58: &str = "9phXGACnW4238oqgvn2gpwaUjG3RAqcxq2Ash2vaKp8KjzSd3MQ56Jt"; + + let address_a = Hash::from_base58(ADDRESS_A_B58).expect("address a should parse"); + let address_b = Hash::from_base58(ADDRESS_B_B58).expect("address b should parse"); + let bridge_address = + nockchain_types::EthAddress::from_hex_str("0xaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa") + .expect("bridge address should parse"); + let recipients = vec![ + RecipientSpec::P2pkh { + address: address_a.clone(), + amount: 5, + }, + RecipientSpec::Multisig { + threshold: 2, + addresses: vec![address_a.clone(), address_b], + amount: 7, + }, + RecipientSpec::BridgeDeposit { + evm_address: bridge_address, + amount: 9, + }, + ]; + + let outputs = planner_recipient_outputs(&recipients, false).expect("recipient outputs"); + assert_eq!(outputs[0].note_data.len(), 0); + assert_eq!(outputs[1].note_data.len(), 1); + assert_eq!(outputs[1].note_data[0].key, "lock"); + assert_eq!(outputs[2].note_data.len(), 1); + assert_eq!(outputs[2].note_data[0].key, "bridge"); +} + +#[test] +fn planner_refund_output_template_includes_lock_note_data_when_enabled() { + let signer = hash(1234); + let with_data = + planner_refund_output_template(None, &signer, true).expect("refund output with data"); + assert_eq!(with_data.amount, 0); + assert_eq!(with_data.note_data.len(), 1); + assert_eq!(with_data.note_data[0].key, "lock"); + + let without_data = + planner_refund_output_template(None, &signer, false).expect("refund output without data"); + assert_eq!(without_data.amount, 0); + assert_eq!(without_data.note_data.len(), 0); +} + +#[test] +fn planner_constants_decode_from_dedicated_peek_shape() { + let constants = PlannerBlockchainConstantsNoun { + _v1_phase: 40_000, + bythos_phase: 54_000, + data: PlannerNoteDataConstantsNoun { + _max_size: 2_048, + min_fee: 256, + }, + base_fee: 128, + input_fee_divisor: 4, + coinbase_timelock_min: 99, + }; + let wrapped = Some(Some(constants)); + + let mut slab: NounSlab = NounSlab::new(); + let noun = wrapped.to_noun(&mut slab); + let space = slab.noun_space(); + let decoded: Option> = + Option::from_noun(&noun, &space).expect("peek payload should decode"); + let parsed = decoded.flatten().expect("payload should be present"); + assert_eq!(parsed.bythos_phase, 54_000); + assert_eq!(parsed.base_fee, 128); + assert_eq!(parsed.input_fee_divisor, 4); + assert_eq!(parsed.data.min_fee, 256); + assert_eq!(parsed.coinbase_timelock_min().unwrap(), 99); +} + +#[test] +fn planner_constants_extract_coinbase_timelock_min_from_payload() { + let constants = default_fakenet_blockchain_constants(); + let wrapped = Some(Some(constants.clone())); + + let mut slab: NounSlab = NounSlab::new(); + let noun = wrapped.to_noun(&mut slab); + let space = slab.noun_space(); + let decoded: Option> = + Option::from_noun(&noun, &space).expect("peek payload should decode"); + let parsed = decoded.flatten().expect("payload should be present"); + + assert_eq!( + parsed + .coinbase_timelock_min() + .expect("coinbase timelock min should decode"), + constants.coinbase_timelock_min + ); +} + +#[tokio::test] +async fn fakenet_mode_sets_low_bythos_phase_in_wallet_constants() -> Result<(), NockAppError> { + init_tracing(); + let (mut wallet, _data_dir) = boot_test_wallet().await?; + let expected_after = default_fakenet_blockchain_constants(); + + assert!(!wallet.is_fakenet().await?); + + let before = peek_wallet_blockchain_constants(&mut wallet).await?; + assert_eq!(before._v1_phase, 39_000); + assert_eq!(before.bythos_phase, 54_000); + assert_eq!(before.base_fee, 16_384); + assert_eq!(before.input_fee_divisor, 4); + assert_eq!(before.data.min_fee, 256); + assert_eq!(before.coinbase_timelock_min()?, 100); + + wallet.set_fakenet().await?; + assert!(wallet.is_fakenet().await?); + + let after = peek_wallet_blockchain_constants(&mut wallet).await?; + assert_eq!(after._v1_phase, expected_after.v1_phase); + assert_eq!(after.bythos_phase, expected_after.bythos_phase); + assert_eq!(after.base_fee, expected_after.base_fee); + assert_eq!(after.input_fee_divisor, expected_after.input_fee_divisor); + assert_eq!(after.data.min_fee, expected_after.note_data.min_fee); + assert_eq!( + after.coinbase_timelock_min()?, + expected_after.coinbase_timelock_min + ); + + Ok(()) +} + +#[tokio::test] +async fn fakenet_create_tx_accepts_discounted_fee_schedule() -> Result<(), NockAppError> { + init_tracing(); + let (mut wallet, _data_dir) = boot_test_wallet().await?; + let seedphrase = "route run sing warrior light swamp clog flower agent ugly wasp fresh tube snow motion salt salon village raccoon chair demise neutral school confirm"; + + import_seed_phrase(&mut wallet, seedphrase, 1).await?; + wallet.set_fakenet().await?; + + let signer_pkh = peek_master_signing_key(&mut wallet).await?; + let simple = SpendCondition::new(vec![LockPrimitive::Pkh(Pkh::new( + 1, + vec![signer_pkh.clone()], + ))]); + let note_name = Name::new( + simple + .first_name() + .expect("simple first-name should compute") + .into_hash(), + hash(9_999), + ); + let note = note_v1_with_lock( + note_name.clone(), + 1, + 10_000, + simple_pkh_lock(signer_pkh.clone()), + ); + apply_balance_update( + &mut wallet, + balance_page(1, 777, vec![(note_name.clone(), note)]), + ) + .await?; + + let recipient = RecipientSpec::P2pkh { + address: signer_pkh, + amount: 4_000, + }; + let (noun, _) = Wallet::create_tx_command_for_tests( + format_note_names(std::slice::from_ref(¬e_name)), + vec![recipient], + 3_584, + false, + None, + Vec::new(), + true, + false, + NoteSelectionStrategyCli::Ascending, + )?; + let wire = WalletWire::Command(Commands::CreateTx { + names: Some(String::new()), + recipients: Vec::new(), + fee: Some(3_584), + allow_low_fee: false, + refund_pkh: None, + index: None, + hardened: false, + include_data: true, + sign_keys: Vec::new(), + save_raw_tx: false, + note_selection_strategy: NoteSelectionStrategyCli::Ascending, + }) + .to_wire(); + + let result = wallet.app.poke(wire, noun).await?; + assert!( + result.len() > 1, + "fakenet create-tx should emit transaction effects, got {result:?}" + ); + + Ok(()) +} + +#[tokio::test] +async fn signing_keys_support_rust_first_name_reconstruction_in_fakenet() -> Result<(), NockAppError> +{ + init_tracing(); + let (mut wallet, _data_dir) = boot_test_wallet().await?; + let seedphrase = "route run sing warrior light swamp clog flower agent ugly wasp fresh tube snow motion salt salon village raccoon chair demise neutral school confirm"; + + import_seed_phrase(&mut wallet, seedphrase, 1).await?; + wallet.set_fakenet().await?; + + let constants = peek_wallet_blockchain_constants(&mut wallet).await?; + let relative_min = constants.coinbase_timelock_min()?; + let signer_pkh = peek_master_signing_key(&mut wallet).await?; + let simple = SpendCondition::new(vec![LockPrimitive::Pkh(Pkh::new( + 1, + vec![signer_pkh.clone()], + ))]); + let coinbase = SpendCondition::new(vec![ + LockPrimitive::Pkh(Pkh::new(1, vec![signer_pkh.clone()])), + LockPrimitive::Tim(nockchain_types::tx_engine::v1::tx::LockTim { + rel: TimelockRangeRelative::new(Some(BlockHeightDelta(Belt(relative_min))), None), + abs: TimelockRangeAbsolute::none(), + }), + ]); + let rust_simple = simple + .first_name() + .expect("simple first-name should compute"); + let rust_coinbase = coinbase + .first_name() + .expect("coinbase first-name should compute"); + + assert_ne!( + rust_simple, + rust_coinbase, + "simple and coinbase first-name should differ for signer {}", + signer_pkh.to_base58() + ); + + Ok(()) +} + +#[tokio::test] +async fn migrate_v0_notes_per_signer_writes_tx_for_v0_master_bucket() -> Result<(), NockAppError> { + init_tracing(); + let (mut wallet, data_dir) = boot_test_wallet().await?; + let seedphrase = "route run sing warrior light swamp clog flower agent ugly wasp fresh tube snow motion salt salon village raccoon chair demise neutral school confirm"; + + import_seed_phrase(&mut wallet, seedphrase, 0).await?; + wallet.set_fakenet().await?; + + let destination = peek_master_signing_key(&mut wallet).await?.to_base58(); + let signer_pubkey = peek_master_signing_pubkey(&mut wallet).await?; + + let v0_note_name = name(51, 5_151); + let v0_note = note_v0_with_lock( + v0_note_name.clone(), + 1, + 25_000, + simple_v0_lock(signer_pubkey.clone()), + ); + apply_balance_update( + &mut wallet, + balance_page(1, 888, vec![(v0_note_name, v0_note)]), + ) + .await?; + + let summary = wallet + .migrate_v0_notes_per_signer_for_tests(None, destination, data_dir.path()) + .await?; + assert_eq!(summary.examined_signers, 1); + assert_eq!(summary.created_count, 1); + assert_eq!(summary.skipped_count, 0); + + let signer_summary = summary + .signers + .iter() + .find(|signer| signer.tx_path.is_some()) + .expect("master signer migration should create a tx"); + assert!(signer_summary.signer.child_index.is_none()); + assert!(std::path::Path::new( + signer_summary + .tx_path + .as_ref() + .expect("created migration should emit a tx path") + ) + .exists()); + + Ok(()) +} + +#[tokio::test] +async fn active_signers_peek_lists_master_and_children() -> Result<(), NockAppError> { + init_tracing(); + let (mut wallet, _data_dir) = boot_test_wallet().await?; + let seedphrase = "route run sing warrior light swamp clog flower agent ugly wasp fresh tube snow motion salt salon village raccoon chair demise neutral school confirm"; + + import_seed_phrase(&mut wallet, seedphrase, 1).await?; + derive_child_key(&mut wallet, 0, false).await?; + derive_child_key(&mut wallet, 1, true).await?; + + let signers = peek_active_signers(&mut wallet).await?; + + assert_eq!(signers.len(), 3, "expected master plus two child signers"); + assert!(signers[0].child_index.is_none(), "master should sort first"); + assert_eq!(signers[1].child_index, Some(0)); + assert!(!signers[1].hardened); + assert_eq!(signers[1].absolute_index, Some(0)); + assert_eq!(signers[2].child_index, Some(1)); + assert!(signers[2].hardened); + assert_eq!(signers[2].absolute_index, Some((1u64 << 31) + 1)); + + Ok(()) +} + +#[tokio::test] +async fn active_signers_peek_lists_v0_hardened_and_unhardened_children_for_v0_master( +) -> Result<(), NockAppError> { + init_tracing(); + let (mut wallet, _data_dir) = boot_test_wallet().await?; + let seedphrase = "route run sing warrior light swamp clog flower agent ugly wasp fresh tube snow motion salt salon village raccoon chair demise neutral school confirm"; + + import_seed_phrase(&mut wallet, seedphrase, 0).await?; + derive_child_key(&mut wallet, 0, false).await?; + derive_child_key(&mut wallet, 1, true).await?; + + let signers = peek_active_signers(&mut wallet).await?; + + assert_eq!(signers.len(), 3, "expected master plus two child signers"); + assert!(signers.iter().all(|signer| signer.version == 0)); + + let unhardened_child = signers + .iter() + .find(|signer| signer.child_index == Some(0)) + .expect("unhardened v0 child signer should exist"); + assert!(!unhardened_child.hardened); + assert_eq!(unhardened_child.absolute_index, Some(0)); + + let hardened_child = signers + .iter() + .find(|signer| signer.child_index == Some(1)) + .expect("hardened v0 child signer should exist"); + assert!(hardened_child.hardened); + assert_eq!(hardened_child.absolute_index, Some((1u64 << 31) + 1)); + + Ok(()) +} + +#[tokio::test] +async fn migrate_v0_notes_per_signer_creates_txs_for_v0_child_buckets() -> Result<(), NockAppError> +{ + init_tracing(); + let (mut wallet, data_dir) = boot_test_wallet().await?; + let seedphrase = "route run sing warrior light swamp clog flower agent ugly wasp fresh tube snow motion salt salon village raccoon chair demise neutral school confirm"; + + import_seed_phrase(&mut wallet, seedphrase, 1).await?; + wallet.set_fakenet().await?; + derive_child_key(&mut wallet, 1, true).await?; + derive_child_key(&mut wallet, 2, true).await?; + + let destination = peek_master_signing_key(&mut wallet).await?.to_base58(); + let signers = peek_active_signers(&mut wallet).await?; + let child_signer_1 = signers + .iter() + .find(|signer| signer.child_index == Some(1) && signer.hardened && signer.version == 0) + .expect("first v0 child signer should exist") + .clone(); + let child_signer_2 = signers + .iter() + .find(|signer| signer.child_index == Some(2) && signer.hardened && signer.version == 0) + .expect("second v0 child signer should exist") + .clone(); + + let child_note_name_1 = name(71, 7_171); + let child_note_name_2 = name(72, 7_272); + let child_note_1 = note_v0_with_lock( + child_note_name_1.clone(), + 1, + 25_000, + simple_v0_lock(child_signer_1.pubkey.clone()), + ); + let child_note_2 = note_v0_with_lock( + child_note_name_2.clone(), + 1, + 25_000, + simple_v0_lock(child_signer_2.pubkey.clone()), + ); + apply_balance_update( + &mut wallet, + balance_page( + 1, + 1_717, + vec![(child_note_name_1, child_note_1), (child_note_name_2, child_note_2)], + ), + ) + .await?; + + let summary = wallet + .migrate_v0_notes_per_signer_for_tests(None, destination, data_dir.path()) + .await?; + + assert_eq!(summary.examined_signers, 2); + assert_eq!(summary.created_count, 2); + assert_eq!(summary.skipped_count, 0); + assert_eq!(summary.signers.len(), 2); + for signer in &summary.signers { + assert_eq!(signer.signer.version, 0); + assert_eq!(signer.note_count, 1); + let tx_path = signer + .tx_path + .as_ref() + .expect("created migration should emit a tx path"); + assert!( + std::path::Path::new(tx_path).exists(), + "expected tx file to be written at {tx_path}" + ); + } + + Ok(()) +} + +#[tokio::test] +async fn migrate_v0_notes_per_signer_creates_txs_for_mixed_hardened_v0_child_buckets( +) -> Result<(), NockAppError> { + init_tracing(); + let (mut wallet, data_dir) = boot_test_wallet().await?; + let seedphrase = "route run sing warrior light swamp clog flower agent ugly wasp fresh tube snow motion salt salon village raccoon chair demise neutral school confirm"; + + import_seed_phrase(&mut wallet, seedphrase, 0).await?; + wallet.set_fakenet().await?; + derive_child_key(&mut wallet, 0, false).await?; + derive_child_key(&mut wallet, 1, true).await?; + + let destination = hash(8_808).to_base58(); + let signers = peek_active_signers(&mut wallet).await?; + let unhardened_child = signers + .iter() + .find(|signer| signer.child_index == Some(0) && !signer.hardened && signer.version == 0) + .expect("unhardened v0 child signer should exist") + .clone(); + let hardened_child = signers + .iter() + .find(|signer| signer.child_index == Some(1) && signer.hardened && signer.version == 0) + .expect("hardened v0 child signer should exist") + .clone(); + + let unhardened_note_name = name(73, 7_373); + let hardened_note_name = name(74, 7_474); + let unhardened_note = note_v0_with_lock( + unhardened_note_name.clone(), + 1, + 25_000, + simple_v0_lock(unhardened_child.pubkey.clone()), + ); + let hardened_note = note_v0_with_lock( + hardened_note_name.clone(), + 1, + 25_000, + simple_v0_lock(hardened_child.pubkey.clone()), + ); + apply_balance_update( + &mut wallet, + balance_page( + 1, + 1_818, + vec![(unhardened_note_name, unhardened_note), (hardened_note_name, hardened_note)], + ), + ) + .await?; + + let summary = wallet + .migrate_v0_notes_per_signer_for_tests(None, destination, data_dir.path()) + .await?; + + assert_eq!(summary.examined_signers, 3); + assert_eq!(summary.created_count, 2); + assert_eq!(summary.skipped_count, 1); + + let created = summary + .signers + .iter() + .filter(|signer| signer.tx_path.is_some()) + .collect::>(); + assert_eq!(created.len(), 2, "expected one tx per child bucket"); + + let created_unhardened = created + .iter() + .find(|signer| signer.signer.child_index == Some(0)) + .expect("unhardened child summary should exist"); + assert!(!created_unhardened.signer.hardened); + assert_eq!(created_unhardened.signer.version, 0); + assert_eq!(created_unhardened.note_count, 1); + + let created_hardened = created + .iter() + .find(|signer| signer.signer.child_index == Some(1)) + .expect("hardened child summary should exist"); + assert!(created_hardened.signer.hardened); + assert_eq!(created_hardened.signer.version, 0); + assert_eq!(created_hardened.note_count, 1); + + for signer in created { + let tx_path = signer + .tx_path + .as_ref() + .expect("created migration should emit a tx path"); + assert!( + std::path::Path::new(tx_path).exists(), + "expected tx file to be written at {tx_path}" + ); + } + + let skipped_master = summary + .signers + .iter() + .find(|signer| signer.signer.child_index.is_none()) + .expect("master signer summary should exist"); + assert_eq!( + skipped_master.skip_reason.as_deref(), + Some("no_eligible_v0_notes") + ); + + Ok(()) +} + +#[tokio::test] +async fn migrate_v0_notes_per_signer_generated_txs_validate_via_send_tx_for_mixed_v0_children( +) -> Result<(), NockAppError> { + init_tracing(); + let (mut wallet, data_dir) = boot_test_wallet().await?; + let seedphrase = "route run sing warrior light swamp clog flower agent ugly wasp fresh tube snow motion salt salon village raccoon chair demise neutral school confirm"; + + import_seed_phrase(&mut wallet, seedphrase, 0).await?; + wallet.set_fakenet().await?; + derive_child_key(&mut wallet, 0, false).await?; + derive_child_key(&mut wallet, 1, true).await?; + + let destination = hash(9_909).to_base58(); + let signers = peek_active_signers(&mut wallet).await?; + let unhardened_child = signers + .iter() + .find(|signer| signer.child_index == Some(0) && !signer.hardened && signer.version == 0) + .expect("unhardened v0 child signer should exist") + .clone(); + let hardened_child = signers + .iter() + .find(|signer| signer.child_index == Some(1) && signer.hardened && signer.version == 0) + .expect("hardened v0 child signer should exist") + .clone(); + + apply_balance_update( + &mut wallet, + balance_page( + 1, + 1_919, + vec![ + ( + name(75, 7_575), + note_v0_with_lock( + name(75, 7_575), + 1, + 25_000, + simple_v0_lock(unhardened_child.pubkey.clone()), + ), + ), + ( + name(76, 7_676), + note_v0_with_lock( + name(76, 7_676), + 1, + 25_000, + simple_v0_lock(hardened_child.pubkey.clone()), + ), + ), + ], + ), + ) + .await?; + + let summary = wallet + .migrate_v0_notes_per_signer_for_tests(None, destination, data_dir.path()) + .await?; + + let tx_paths = summary + .signers + .iter() + .filter_map(|signer| signer.tx_path.as_deref()) + .collect::>(); + assert_eq!(tx_paths.len(), 2, "expected one tx per child bucket"); + + for tx_path in tx_paths { + let (noun, _) = Wallet::send_tx(tx_path)?; + let wire = WalletWire::Command(Commands::SendTx { + transaction: tx_path.to_string(), + }) + .to_wire(); + let effects = wallet.app.poke(wire, noun).await?; + + assert_eq!( + effect_exit_code(&effects), + Some(0), + "send-tx should validate and exit successfully for {tx_path}" + ); + assert!( + effects + .iter() + .any(|effect| effect_tag(effect).as_deref() == Some("nockchain-grpc")), + "send-tx should emit a nockchain-grpc send effect for {tx_path}" + ); + } + + Ok(()) +} + +#[tokio::test] +async fn create_tx_with_planner_accepts_manual_all_v0_notes() -> Result<(), NockAppError> { + init_tracing(); + let (mut wallet, _data_dir) = boot_test_wallet().await?; + let seedphrase = "route run sing warrior light swamp clog flower agent ugly wasp fresh tube snow motion salt salon village raccoon chair demise neutral school confirm"; + + import_seed_phrase(&mut wallet, seedphrase, 1).await?; + wallet.set_fakenet().await?; + + let destination = peek_master_signing_key(&mut wallet).await?; + let signer_pubkey = peek_master_signing_pubkey(&mut wallet).await?; + + let v0_note_name = name(52, 5_252); + let v0_note = note_v0_with_lock( + v0_note_name.clone(), + 1, + 25_000, + simple_v0_lock(signer_pubkey.clone()), + ); + apply_balance_update( + &mut wallet, + balance_page(1, 889, vec![(v0_note_name.clone(), v0_note)]), + ) + .await?; + + let (noun, _) = wallet + .create_tx_with_planner( + None, + Some(format_note_names(std::slice::from_ref(&v0_note_name))), + None, + vec![RecipientSpec::P2pkh { + address: destination.clone(), + amount: 20_000, + }], + false, + Some(destination.to_base58()), + Vec::new(), + true, + false, + NoteSelectionStrategyCli::Ascending, + ) + .await?; + let result = wallet.app.poke(OnePunchWire::Poke.to_wire(), noun).await?; + assert!( + result.len() > 1, + "manual all-v0 create-tx should emit transaction effects, got {result:?}" + ); + + Ok(()) +} + +#[tokio::test] +async fn migrate_v0_notes_wallet_tx_matches_planner_word_and_fee_counts() -> Result<(), NockAppError> +{ + init_tracing(); + let (mut wallet, data_dir) = boot_test_wallet().await?; + let seedphrase = "route run sing warrior light swamp clog flower agent ugly wasp fresh tube snow motion salt salon village raccoon chair demise neutral school confirm"; + + import_seed_phrase(&mut wallet, seedphrase, 0).await?; + wallet.set_fakenet().await?; + + let destination = peek_master_signing_key(&mut wallet).await?.to_base58(); + let destination_hash = Hash::from_base58(&destination).expect("destination should parse"); + let signer_key = peek_master_signing_key(&mut wallet).await?; + let signer_pubkey = peek_master_signing_pubkey(&mut wallet).await?; + + let v0_note_name = name(61, 6_161); + let v0_note = note_v0_with_lock( + v0_note_name.clone(), + 1, + 25_000, + simple_v0_lock(signer_pubkey.clone()), + ); + apply_balance_update( + &mut wallet, + balance_page(1, 999, vec![(v0_note_name, v0_note)]), + ) + .await?; + + let balance = peek_balance_state(&mut wallet).await?; + let snapshot = normalize_balance_pages(&[balance]) + .map_err(|err| NockAppError::OtherError(format!("snapshot normalization failed: {err}")))?; + let mut destination_outputs = planner_recipient_outputs( + &[RecipientSpec::P2pkh { + address: destination_hash, + amount: 0, + }], + true, + )?; + let destination_output = destination_outputs + .pop() + .expect("single migration destination should yield one output"); + let planner_constants = peek_wallet_blockchain_constants(&mut wallet).await?; + let coinbase_relative_min = planner_constants.coinbase_timelock_min()?; + let request = PlanRequest { + planning_mode: PlanningMode::V0MigrationSweep { + destination_output: destination_output.clone(), + }, + selection_mode: SelectionMode::Auto, + order_direction: SelectionOrder::Ascending, + include_data: true, + chain_context: ChainContext { + height: snapshot.metadata.height.clone(), + bythos_phase: BlockHeight(Belt(planner_constants.bythos_phase)), + base_fee: planner_constants.base_fee, + input_fee_divisor: planner_constants.input_fee_divisor, + min_fee: planner_constants.data.min_fee, + }, + signer_pkh: None, + candidate_version_policy: CandidateVersionPolicy::V0Only, + candidates: snapshot.candidates.clone(), + recipient_outputs: Vec::new(), + refund_output: destination_output, + coinbase_relative_min: Some(coinbase_relative_min), + v0_migration_signer_pubkeys: vec![signer_pubkey.clone()], + }; + let matcher = SigningKeyLockMatcher::from_signer_keys(&[signer_key]); + let plan = plan_create_tx(&request, &matcher) + .map_err(|err| NockAppError::OtherError(format!("planner failed: {err}")))?; + + let summary = wallet + .migrate_v0_notes_per_signer_for_tests(None, destination.clone(), data_dir.path()) + .await?; + assert_eq!(summary.created_count, 1); + assert_eq!(summary.skipped_count, 0); + let tx_path = summary + .signers + .iter() + .find_map(|signer| signer.tx_path.as_deref()) + .expect("migration should create one tx"); + let spends = decode_saved_transaction_spends_from_path(tx_path)?; + + assert_eq!(spends.0.len(), 1, "migration should build one spend"); + assert!( + matches!(spends.0.first(), Some((_, v1::Spend::Legacy(_)))), + "migration should build a legacy v0 spend" + ); + + let hoon_seed_words = merged_seed_word_count(&spends); + let hoon_witness_words = witness_word_count(&spends); + let hoon_fee = total_paid_fee(&spends); + let hoon_gift = total_seed_gift(&spends); + let computed_fee = compute_minimum_fee(FeeInputs { + seed_words: hoon_seed_words, + witness_words: hoon_witness_words, + base_fee: request.chain_context.base_fee, + input_fee_divisor: request.chain_context.input_fee_divisor, + min_fee: request.chain_context.min_fee, + height: request.chain_context.height, + bythos_phase: request.chain_context.bythos_phase, + }); + + // A one-input v0 migration on fakenet should emit one merged destination seed + // note-data payload and one legacy signature map witness. + assert_eq!(hoon_seed_words, 14); + assert_eq!(hoon_witness_words, 31); + assert_eq!(computed_fee.minimum_fee, 2_784); + assert_eq!(hoon_fee, computed_fee.minimum_fee); + assert_eq!(hoon_gift, 22_216); + + assert_eq!(plan.word_counts.seed_words, hoon_seed_words); + assert_eq!(plan.word_counts.witness_words, hoon_witness_words); + assert_eq!(plan.final_fee, computed_fee.minimum_fee); + assert_eq!(plan.outputs.len(), 1); + assert_eq!(plan.outputs[0].amount, hoon_gift); + assert_eq!(plan.selected_total, hoon_fee + hoon_gift); + + Ok(()) +} + +#[test] +fn master_signing_key_decodes_from_hash_payload_shape() { + let wrapped = Some(Some(hash(3))); + + let mut slab: NounSlab = NounSlab::new(); + let noun = wrapped.to_noun(&mut slab); + let space = slab.noun_space(); + let decoded: Option> = + Option::from_noun(&noun, &space).expect("master signing key payload should decode"); + let parsed = decoded.flatten().expect("payload should be present"); + + assert_eq!(parsed, hash(3)); +} + +#[test] +fn migrate_v0_notes_summary_includes_send_tx_command_for_saved_transactions() { + let v0_address = + "2cPnE4Z9RevhTv9is9Hmc1amFubEFbUxzCV2Fxb9GxevJstV5VG92oYt6Sai3d3NjLFcsuVXSLx9hikMbD1agv9M267TVw3hV9MCpMfEnGo5LYtjJ7jPyHg8SERPjJRCWTgZ"; + let summary = MigrateV0NotesSummary { + destination: "9phXGACnW4238oqgvn2gpwaUjG3RAqcxq2Ash2vaKp8KjzSd3MQ56Jt".to_string(), + block_id: "block-id".to_string(), + height: 33, + examined_signers: 1, + created_count: 1, + skipped_count: 0, + signers: vec![MigrateV0SignerSummary { + signer: ActiveSignerEntryNoun { + child_index: Some(0), + hardened: false, + absolute_index: Some(0), + version: 0, + pubkey: SchnorrPubkey::from_base58(v0_address) + .expect("sample v0 signer pubkey should parse"), + address_b58: v0_address.to_string(), + }, + note_count: 1, + selected_total: 25_000, + fee: Some(2_784), + migrated_amount: Some(22_216), + tx_path: Some("./txs/example.tx".to_string()), + skip_reason: None, + }], + }; + + let rendered = Wallet::format_migrate_v0_notes_summary(&summary); + + assert!(rendered.contains("- tx path: `./txs/example.tx`")); + assert!(rendered.contains("- submit with: `nockchain-wallet send-tx \"./txs/example.tx\"`")); +} + +#[test] +fn migrate_v0_notes_summary_mentions_when_no_batch_poke_was_emitted() { + let summary = MigrateV0NotesSummary { + destination: "9phXGACnW4238oqgvn2gpwaUjG3RAqcxq2Ash2vaKp8KjzSd3MQ56Jt".to_string(), + block_id: "block-id".to_string(), + height: 33, + examined_signers: 1, + created_count: 0, + skipped_count: 1, + signers: vec![MigrateV0SignerSummary { + signer: ActiveSignerEntryNoun { + child_index: None, + hardened: false, + absolute_index: None, + version: 0, + pubkey: SchnorrPubkey::from_base58( + "2cPnE4Z9RevhTv9is9Hmc1amFubEFbUxzCV2Fxb9GxevJstV5VG92oYt6Sai3d3NjLFcsuVXSLx9hikMbD1agv9M267TVw3hV9MCpMfEnGo5LYtjJ7jPyHg8SERPjJRCWTgZ", + ) + .expect("sample v0 signer pubkey should parse"), + address_b58: "2cPnE4Z9RevhTv9is9Hmc1amFubEFbUxzCV2Fxb9GxevJstV5VG92oYt6Sai3d3NjLFcsuVXSLx9hikMbD1agv9M267TVw3hV9MCpMfEnGo5LYtjJ7jPyHg8SERPjJRCWTgZ" + .to_string(), + }, + note_count: 0, + selected_total: 0, + fee: Some(2_784), + migrated_amount: None, + tx_path: None, + skip_reason: Some("no_eligible_v0_notes".to_string()), + }], + }; + + let rendered = Wallet::format_migrate_v0_notes_summary(&summary); + + assert!(rendered + .contains("- batch create poke: not emitted because every signer bucket was skipped")); +} + +#[test] +fn collect_signing_keys_prefers_explicit_entries() { + let entries = vec!["0:true".to_string(), "1:false".to_string()]; + let keys = Wallet::collect_signing_keys(Some(5), false, &entries).expect("valid explicit keys"); + assert_eq!(keys, vec![(0, true), (1, false)]); +} + +#[test] +fn collect_signing_keys_falls_back_to_index() { + let keys = Wallet::collect_signing_keys(Some(3), true, &[]).expect("valid"); + assert_eq!(keys, vec![(3, true)]); +} + +#[test] +fn collect_signing_keys_defaults_to_master() { + let keys = Wallet::collect_signing_keys(None, false, &[]).expect("valid"); + assert!(keys.is_empty()); +} + +#[tokio::test] +#[cfg_attr(miri, ignore)] +async fn test_keygen() -> Result<(), NockAppError> { + init_tracing(); + let cli = ephemeral_test_boot_cli(true); + let prover_hot_state = produce_prover_hot_state(); + let (mut wallet, _data_dir) = boot_wallet_with(cli, prover_hot_state.as_slice()).await?; + let mut entropy = [0u8; 32]; + let mut salt = [0u8; 16]; + getrandom::fill(&mut entropy).map_err(|e| CrownError::Unknown(e.to_string()))?; + getrandom::fill(&mut salt).map_err(|e| CrownError::Unknown(e.to_string()))?; + let (noun, op) = Wallet::keygen(&entropy, &salt)?; + + let wire = WalletWire::Command(Commands::Keygen).to_wire(); + + let keygen_result = wallet.app.poke(wire, noun.clone()).await?; + + println!("keygen result: {:?}", keygen_result); + assert!( + keygen_result.len() == 2, + "Expected keygen result to be a list of 2 noun slabs - markdown and exit" + ); + let exit_space = keygen_result[1].noun_space(); + let exit_cause = unsafe { *keygen_result[1].root() }.in_space(&exit_space); + let code = exit_cause.as_cell()?.tail(); + assert_eq!(code.as_atom()?.as_u64()?, 0, "Expected exit code 0"); + + Ok(()) +} + +#[tokio::test] +#[cfg_attr(miri, ignore)] +async fn test_derive_child() -> Result<(), NockAppError> { + init_tracing(); + let cli = ephemeral_test_boot_cli(true); + let prover_hot_state = produce_prover_hot_state(); + let (mut wallet, _data_dir) = boot_wallet_with(cli, prover_hot_state.as_slice()).await?; + + // Generate a new key pair + let mut entropy = [0u8; 32]; + let mut salt = [0u8; 16]; + let (noun, op) = Wallet::keygen(&entropy, &salt)?; + let wire = WalletWire::Command(Commands::Keygen).to_wire(); + let _ = wallet.app.poke(wire, noun.clone()).await?; + + // Derive a child key + let index = 0; + let hardened = true; + let label = None; + let (noun, op) = Wallet::derive_child(index, hardened, &label)?; + + let wire = WalletWire::Command(Commands::DeriveChild { + index, + hardened, + label, + }) + .to_wire(); + + let derive_result = wallet.app.poke(wire, noun.clone()).await?; + + assert!( + derive_result.len() == 2, + "Expected derive result to be a list of 2 noun slabs - markdown and exit" + ); + + let exit_space = derive_result[1].noun_space(); + let exit_cause = unsafe { *derive_result[1].root() }.in_space(&exit_space); + let code = exit_cause.as_cell()?.tail(); + assert_eq!(code.as_atom()?.as_u64()?, 0, "Expected exit code 0"); + + Ok(()) +} + +// Tests for Cold Side Commands +#[tokio::test] +#[cfg_attr(miri, ignore)] +async fn test_gen_master_privkey() -> Result<(), NockAppError> { + init_tracing(); + let cli = ephemeral_test_boot_cli(true); + let (mut wallet, _data_dir) = boot_wallet_with(cli, &[]).await?; + let seedphrase = "correct horse battery staple"; + let version = 1; + let (noun, op) = Wallet::import_seed_phrase(seedphrase, version)?; + println!("privkey_slab: {:?}", noun); + let wire = WalletWire::Command(Commands::ImportKeys { + file: None, + key: None, + seedphrase: Some(seedphrase.to_string()), + version: Some(version), + }) + .to_wire(); + let privkey_result = wallet.app.poke(wire, noun.clone()).await?; + println!("privkey_result: {:?}", privkey_result); + Ok(()) +} + +// Tests for Hot Side Commands +// TODO: fix this test by adding a real key file +#[tokio::test] +#[ignore] +async fn test_import_keys() -> Result<(), NockAppError> { + init_tracing(); + let cli = ephemeral_test_boot_cli(true); + let nockapp = boot::setup(KERNEL, cli.clone(), &[], "wallet", None) + .await + .map_err(|e| CrownError::Unknown(e.to_string()))?; + let mut wallet = Wallet::new(nockapp); + + // Create test key file + let test_path = "test_keys.jam"; + let test_data = vec![0u8; 32]; // TODO: Use real jammed key data + fs::write(test_path, &test_data).expect(&format!( + "Called `expect()` at {}:{} (git sha: {})", + file!(), + line!(), + option_env!("GIT_SHA").unwrap_or("unknown") + )); + + let (noun, op) = Wallet::import_keys(test_path)?; + let wire = WalletWire::Command(Commands::ImportKeys { + file: Some(test_path.to_string()), + key: None, + seedphrase: None, + version: None, + }) + .to_wire(); + let import_result = wallet.app.poke(wire, noun.clone()).await?; + + fs::remove_file(test_path).expect(&format!( + "Called `expect()` at {}:{} (git sha: {})", + file!(), + line!(), + option_env!("GIT_SHA").unwrap_or("unknown") + )); + + println!("import result: {:?}", import_result); + assert!( + !import_result.is_empty(), + "Expected non-empty import result" + ); + + Ok(()) +} + +// TODO: fix this test +#[tokio::test] +#[ignore] +async fn test_spend_multisig_format() -> Result<(), NockAppError> { + // TODO: replace with an end-to-end test that exercises multisig recipient specs. + Ok(()) +} + +#[tokio::test] +#[ignore] +async fn test_spend_single_sig_format() -> Result<(), NockAppError> { + // TODO: replace with an end-to-end test for PKH recipients once fixtures exist. + Ok(()) +} + +#[tokio::test] +#[cfg_attr(miri, ignore)] +async fn test_list_notes() -> Result<(), NockAppError> { + init_tracing(); + let cli = ephemeral_test_boot_cli(true); + let (mut wallet, _data_dir) = boot_wallet_with(cli, &[]).await?; + + // Test listing notes + let (noun, op) = Wallet::list_notes()?; + let wire = WalletWire::Command(Commands::ListNotes {}).to_wire(); + let list_result = wallet.app.poke(wire, noun.clone()).await?; + println!("list_result: {:?}", list_result); + + Ok(()) +} + +// TODO: fix this test by adding a real draft +#[tokio::test] +#[ignore] +async fn test_make_tx_from_draft() -> Result<(), NockAppError> { + init_tracing(); + let cli = ephemeral_test_boot_cli(true); + let nockapp = boot::setup(KERNEL, cli.clone(), &[], "wallet", None) + .await + .map_err(|e| CrownError::Unknown(e.to_string()))?; + let mut wallet = Wallet::new(nockapp); + + // use the transaction in txs/ + let transaction_path = "txs/test_transaction.tx"; + let test_data = vec![0u8; 32]; // TODO: Use real transaction data + fs::write(transaction_path, &test_data).expect(&format!( + "Called `expect()` at {}:{} (git sha: {})", + file!(), + line!(), + option_env!("GIT_SHA").unwrap_or("unknown") + )); + + let (noun, op) = Wallet::send_tx(transaction_path)?; + let wire = WalletWire::Command(Commands::SendTx { + transaction: transaction_path.to_string(), + }) + .to_wire(); + let tx_result = wallet.app.poke(wire, noun.clone()).await?; + + fs::remove_file(transaction_path).expect(&format!( + "Called `expect()` at {}:{} (git sha: {})", + file!(), + line!(), + option_env!("GIT_SHA").unwrap_or("unknown") + )); + + println!("transaction result: {:?}", tx_result); + assert!( + !tx_result.is_empty(), + "Expected non-empty transaction result" + ); + + Ok(()) +} + +#[tokio::test] +#[ignore] +async fn test_show_tx() -> Result<(), NockAppError> { + init_tracing(); + let cli = ephemeral_test_boot_cli(true); + let nockapp = boot::setup(KERNEL, cli.clone(), &[], "wallet", None) + .await + .map_err(|e| CrownError::Unknown(e.to_string()))?; + let mut wallet = Wallet::new(nockapp); + + // Create a temporary transaction file + let transaction_path = "test_show_transaction.tx"; + let test_data = vec![0u8; 32]; // TODO: Use real transaction data + fs::write(transaction_path, &test_data).expect(&format!( + "Called `expect()` at {}:{} (git sha: {})", + file!(), + line!(), + option_env!("GIT_SHA").unwrap_or("unknown") + )); + + let (noun, op) = Wallet::show_tx(transaction_path)?; + let wire = WalletWire::Command(Commands::ShowTx { + transaction: transaction_path.to_string(), + }) + .to_wire(); + let show_result = wallet.app.poke(wire, noun.clone()).await?; + + fs::remove_file(transaction_path).expect(&format!( + "Called `expect()` at {}:{} (git sha: {})", + file!(), + line!(), + option_env!("GIT_SHA").unwrap_or("unknown") + )); + + println!("show-tx result: {:?}", show_result); + assert!(!show_result.is_empty(), "Expected non-empty show-tx result"); + + Ok(()) +} + +#[test] +fn domain_hash_from_base58_accepts_valid_id() { + let tx_id = "3giXkwW4zbFhoyJu27RbP6VNiYgR6yaTfk2AYnEHvxtVaGbmcVD6jb9"; + Hash::from_base58(tx_id).expect("expected valid base58 hash"); +} + +#[test] +fn domain_hash_from_base58_rejects_invalid_id() { + let invalid_tx_id = "not-a-valid-hash"; + assert!(Hash::from_base58(invalid_tx_id).is_err()); +} diff --git a/crates/nockchain/Cargo.toml b/crates/nockchain/Cargo.toml index ab0e9fbdb..f507fddcb 100644 --- a/crates/nockchain/Cargo.toml +++ b/crates/nockchain/Cargo.toml @@ -50,3 +50,19 @@ tracing.workspace = true tracy-client = { workspace = true, optional = true } zkvm-jetpack.workspace = true + +[build-dependencies] +vergen = { workspace = true, features = [ + "build", + "cargo", + "git", + "gitcl", + "rustc", + "si", +] } + +[dev-dependencies] +blake3.workspace = true +libc.workspace = true +memmap2.workspace = true +nockchain-math.workspace = true diff --git a/crates/nockchain/src/bin/bench_nockchain_checkpoint_block.rs b/crates/nockchain/src/bin/bench_nockchain_checkpoint_block.rs new file mode 100644 index 000000000..ab99eb0e9 --- /dev/null +++ b/crates/nockchain/src/bin/bench_nockchain_checkpoint_block.rs @@ -0,0 +1,625 @@ +#![allow(clippy::result_large_err)] + +use std::error::Error; +use std::path::{Path, PathBuf}; +use std::time::{Duration, Instant, SystemTime, UNIX_EPOCH}; + +use chaff::Chaff; +use clap::{ColorChoice, Parser}; +use kernels_open_dumb::KERNEL as NOCKCHAIN_KERNEL; +use libp2p::PeerId; +use nockapp::kernel::boot::{self, NockStackSize, PmaSize, TraceOpts}; +use nockapp::kernel::form::{Kernel, PmaCopyDetail, PmaTimingSample}; +use nockapp::noun::slab::NounSlab; +use nockapp::save::{CheckpointBootstrapReader, SaveableCheckpoint}; +use nockapp::utils::make_tas; +use nockapp::wire::Wire; +use nockapp::{AtomExt, NockApp, NockAppError}; +use nockchain_libp2p_io::driver::Libp2pWire; +use nockchain_libp2p_io::tip5_util::tip5_hash_to_base58_stack; +use nockchain_math::noun_ext::NounMathExtHandle; +use nockchain_math::structs::HoonMapIter; +use nockvm::noun::{Atom, Noun, NounAllocator, NounSpace, D, SIG}; +use nockvm_macros::tas; +use tempfile::{Builder, TempDir}; +use tracing::{info, warn}; +use zkvm_jetpack::hot::produce_prover_hot_state; + +#[derive(Parser, Debug)] +#[command( + name = "bench-nockchain-checkpoint-block", + about = "Bench booting from checkpoint state and poking a single block." +)] +struct BenchArgs { + #[arg(long, default_value = "../replay-checkpoints-51094")] + target_checkpoint_dir: PathBuf, + #[arg(long, default_value = "../nockchain-api-checkpoints-backup")] + source_checkpoint_dir: PathBuf, + #[arg(long, default_value_t = 51095)] + block_height: u64, + #[arg(long)] + previous_height: Option, + #[arg(long, default_value_t = true)] + preload_raw_txs: bool, + #[arg(long, value_enum, default_value_t = NockStackSize::Medium)] + stack_size: NockStackSize, + #[arg(long)] + scratch_root: Option, +} + +enum ScratchDir { + Temp(TempDir), + Persistent(PathBuf), +} + +impl ScratchDir { + fn path(&self) -> &Path { + match self { + Self::Temp(temp) => temp.path(), + Self::Persistent(path) => path.as_path(), + } + } +} + +#[tokio::main] +async fn main() -> Result<(), Box> { + nockvm::check_endian(); + std::env::set_var("NOCKAPP_DISABLE_METRICS", "1"); + std::env::set_var("GNORT_DISABLE", "1"); + std::env::set_var("NOCK_PMA_TIMING", "1"); + std::env::set_var("NOCK_PMA_TIMING_DETAIL", "1"); + + let args = BenchArgs::parse(); + let previous_height = args + .previous_height + .or_else(|| args.block_height.checked_sub(1)) + .ok_or("block height must be greater than zero")?; + + let base_cli = boot::Cli { + new: false, + trace_opts: TraceOpts::default(), + gc_interval: None, + rotating_snapshot_interval_event_time: None, + ephemeral: false, + color: ColorChoice::Auto, + state_jam: None, + bootstrap_from_chkjam: None, + export_state_jam: None, + stack_size: args.stack_size, + pma_initial_size: Some(PmaSize::from_words(args.stack_size.stack_words())), + pma_reserved_size: None, + data_dir: None, + event_log_path: None, + disable_fsync: false, + }; + boot::init_default_tracing(&base_cli); + + let source_scratch = prepare_scratch_dir( + &args.source_checkpoint_dir, + "bench-source", + args.scratch_root.as_deref(), + )?; + let target_scratch = prepare_scratch_dir( + &args.target_checkpoint_dir, + "bench-target", + args.scratch_root.as_deref(), + )?; + + info!( + "bench: source_checkpoint_dir={} source_scratch={}", + args.source_checkpoint_dir.display(), + source_scratch.path().display() + ); + info!( + "bench: target_checkpoint_dir={} target_scratch={}", + args.target_checkpoint_dir.display(), + target_scratch.path().display() + ); + + let source_boot_start = Instant::now(); + info!("bench: loading source checkpoint into in-memory helper kernel"); + let mut source = load_source_peer_from_checkpoint( + source_scratch.path().join("checkpoints"), + args.stack_size, + ) + .await?; + let source_boot = source_boot_start.elapsed(); + let _ = source.take_pma_timing_samples_detailed(); + + info!( + "bench: extracting block at height={} from source peer", + args.block_height + ); + let (block_id, fact_poke) = extract_block_fact(&mut source, args.block_height).await?; + info!( + "bench: extracted block height={} block_id={}", + args.block_height, block_id + ); + let raw_tx_facts = if args.preload_raw_txs { + info!( + "bench: collecting raw transactions for block_id={} from source peer", + block_id + ); + let raw_txs = extract_block_raw_tx_facts(&mut source, &block_id).await?; + info!( + "bench: collected {} raw transaction fact poke(s) for preload", + raw_txs.len() + ); + raw_txs + } else { + Vec::new() + }; + + let target_boot_start = Instant::now(); + info!("bench: booting target peer for measured poke"); + let mut target = boot_peer( + "bench-target", + target_scratch.path().to_path_buf(), + args.stack_size, + ) + .await?; + let target_boot = target_boot_start.elapsed(); + let boot_samples = target + .take_pma_timing_samples_detailed() + .unwrap_or_default(); + if !boot_samples.is_empty() { + info!( + "bench: discarded {} PMA timing sample(s) recorded during target boot", + boot_samples.len() + ); + } + + if let Some(mut page) = peek_page(&mut target, previous_height).await? { + let prior_block_id = page_block_id(&mut page)?; + info!( + "bench: target before poke has height={} block_id={}", + previous_height, prior_block_id + ); + } else { + warn!( + "bench: target before poke did not return a block at height={}", + previous_height + ); + } + + if let Some(mut page) = peek_page(&mut target, args.block_height).await? { + let existing_block_id = page_block_id(&mut page)?; + return Err(format!( + "target already has height {} with block_id {}; checkpoint input is not the expected pre-{} state", + args.block_height, existing_block_id, args.block_height + ) + .into()); + } + + if !raw_tx_facts.is_empty() { + info!( + "bench: preloading {} raw transaction(s) into target before measured block poke", + raw_tx_facts.len() + ); + let preload_start = Instant::now(); + for fact in raw_tx_facts { + let _ = target + .poke(Libp2pWire::Gossip(PeerId::random()).to_wire(), fact) + .await?; + } + info!( + "bench: preload_raw_txs_ms={:.3}", + ms(preload_start.elapsed()) + ); + let preload_samples = target + .take_pma_timing_samples_detailed() + .unwrap_or_default(); + if !preload_samples.is_empty() { + info!( + "bench: discarded {} PMA timing sample(s) recorded during raw-tx preload", + preload_samples.len() + ); + } + } + + let poke_start = Instant::now(); + let effects = target + .poke(Libp2pWire::Gossip(PeerId::random()).to_wire(), fact_poke) + .await?; + let poke_wall = poke_start.elapsed(); + + let poke_sample = target + .take_pma_timing_samples_detailed() + .and_then(|samples| samples.into_iter().last()); + let mut after_page = peek_page(&mut target, args.block_height) + .await? + .ok_or_else(|| { + format!( + "target still has no block at height {} after poke", + args.block_height + ) + })?; + let after_block_id = page_block_id(&mut after_page)?; + if after_block_id != block_id { + return Err(format!( + "target height {} resolved to block_id {} after poke, expected {}", + args.block_height, after_block_id, block_id + ) + .into()); + } + + info!( + "bench: source_boot_ms={:.3} target_boot_ms={:.3} poke_wall_ms={:.3} effects={}", + ms(source_boot), + ms(target_boot), + ms(poke_wall), + effects.len() + ); + info!( + "bench: target now has height={} block_id={}", + args.block_height, after_block_id + ); + if let Some(sample) = poke_sample { + report_pma_sample(sample); + } else { + warn!("bench: no PMA timing sample recorded for the measured poke"); + } + + Ok(()) +} + +fn prepare_scratch_dir( + checkpoint_input: &Path, + label: &str, + scratch_root: Option<&Path>, +) -> Result> { + let checkpoints_dir = resolve_checkpoints_dir(checkpoint_input)?; + let scratch = match scratch_root { + Some(root) => { + std::fs::create_dir_all(root)?; + let path = root.join(format!("{label}-{}", unique_suffix())); + std::fs::create_dir(&path)?; + ScratchDir::Persistent(path) + } + None => ScratchDir::Temp(Builder::new().prefix(label).tempdir()?), + }; + + symlink_checkpoints_dir(&checkpoints_dir, &scratch.path().join("checkpoints"))?; + Ok(scratch) +} + +fn resolve_checkpoints_dir(path: &Path) -> Result> { + let dir = if path.join("checkpoints").is_dir() { + path.join("checkpoints") + } else { + path.to_path_buf() + }; + + if !dir.is_dir() { + return Err(format!( + "checkpoint path {} is not a directory and does not contain checkpoints/", + path.display() + ) + .into()); + } + for file in ["0.chkjam", "1.chkjam"] { + if !dir.join(file).exists() { + warn!( + "bench: checkpoint directory {} is missing {}", + dir.display(), + file + ); + } + } + Ok(dir.canonicalize()?) +} + +#[cfg(unix)] +fn symlink_checkpoints_dir(source: &Path, link: &Path) -> Result<(), Box> { + std::os::unix::fs::symlink(source, link)?; + Ok(()) +} + +#[cfg(not(unix))] +fn symlink_checkpoints_dir(_source: &Path, _link: &Path) -> Result<(), Box> { + Err("checkpoint benchmark scratch dirs require unix symlinks".into()) +} + +async fn boot_peer( + name: &str, + data_dir: PathBuf, + stack_size: NockStackSize, +) -> Result, Box> { + let cli = boot::Cli { + new: false, + trace_opts: TraceOpts::default(), + gc_interval: None, + rotating_snapshot_interval_event_time: None, + ephemeral: false, + color: ColorChoice::Auto, + state_jam: None, + bootstrap_from_chkjam: None, + export_state_jam: None, + stack_size, + pma_initial_size: Some(PmaSize::from_words(stack_size.stack_words())), + pma_reserved_size: None, + data_dir: Some(data_dir), + event_log_path: None, + disable_fsync: false, + }; + let hot_state = produce_prover_hot_state(); + boot::setup::(NOCKCHAIN_KERNEL, cli, hot_state.as_slice(), name, None).await +} + +async fn load_source_peer_from_checkpoint( + checkpoints_dir: PathBuf, + stack_size: NockStackSize, +) -> Result, Box> { + let kernel = load_source_kernel_from_checkpoint(checkpoints_dir, stack_size).await?; + let kernel_f = + move |_| async move { Ok::, nockapp::CrownError>(kernel) }; + Ok(NockApp::new(kernel_f).await?) +} + +async fn load_source_kernel_from_checkpoint( + checkpoints_dir: PathBuf, + stack_size: NockStackSize, +) -> Result, Box> { + let checkpoint = CheckpointBootstrapReader::::new(checkpoints_dir.clone()) + .load_latest(None) + .await? + .ok_or_else(|| { + format!( + "no checkpoint found in source checkpoint dir {}", + checkpoints_dir.display() + ) + })?; + let hot_state = produce_prover_hot_state(); + let test_jets = + boot::parse_test_jets(std::env::var("NOCK_TEST_JETS").unwrap_or_default().as_str()); + let kernel_bytes = Vec::from(NOCKCHAIN_KERNEL); + let mut checkpoint = Some(checkpoint); + let kernel: Kernel = match stack_size { + NockStackSize::Tiny => { + Kernel::load_with_hot_state_tiny( + &kernel_bytes, + checkpoint.take(), + hot_state.as_slice(), + test_jets.clone(), + TraceOpts::default(), + None, + ) + .await? + } + NockStackSize::Small => { + Kernel::load_with_hot_state_small( + &kernel_bytes, + checkpoint.take(), + hot_state.as_slice(), + test_jets.clone(), + TraceOpts::default(), + None, + ) + .await? + } + NockStackSize::Normal => { + Kernel::load_with_hot_state( + &kernel_bytes, + checkpoint.take(), + hot_state.as_slice(), + test_jets.clone(), + TraceOpts::default(), + None, + ) + .await? + } + NockStackSize::Medium => { + Kernel::load_with_hot_state_medium( + &kernel_bytes, + checkpoint.take(), + hot_state.as_slice(), + test_jets.clone(), + TraceOpts::default(), + None, + ) + .await? + } + NockStackSize::Large => { + Kernel::load_with_hot_state_large( + &kernel_bytes, + checkpoint.take(), + hot_state.as_slice(), + test_jets.clone(), + TraceOpts::default(), + None, + ) + .await? + } + NockStackSize::Huge => { + Kernel::load_with_hot_state_huge( + &kernel_bytes, + checkpoint.take(), + hot_state.as_slice(), + test_jets, + TraceOpts::default(), + None, + ) + .await? + } + }; + Ok(kernel) +} + +async fn extract_block_fact( + app: &mut NockApp, + block_height: u64, +) -> Result<(String, NounSlab), Box> { + let mut page = peek_page(app, block_height).await?.ok_or_else(|| { + format!( + "source peer did not return a block at height {}", + block_height + ) + })?; + let block_id = page_block_id(&mut page)?; + Ok((block_id, make_fact_from_payload("heard-block", &page))) +} + +async fn extract_block_raw_tx_facts( + app: &mut NockApp, + block_id: &str, +) -> Result, Box> { + let mut txs_map = peek_block_transactions(app, block_id) + .await? + .ok_or_else(|| { + format!( + "source peer returned no transactions for block {}", + block_id + ) + })?; + let tx_ids = tx_ids_from_map(&mut txs_map)?; + let mut facts = Vec::with_capacity(tx_ids.len()); + for tx_id in tx_ids { + let raw_tx = peek_raw_transaction(app, &tx_id) + .await? + .ok_or_else(|| format!("source peer returned no raw transaction for tx {}", tx_id))?; + facts.push(make_fact_from_payload("heard-tx", &raw_tx)); + } + Ok(facts) +} + +async fn peek_page( + app: &mut NockApp, + block_height: u64, +) -> Result, Box> { + Ok(app.peek_handle(make_heavy_n_path(block_height)).await?) +} + +async fn peek_block_transactions( + app: &mut NockApp, + block_id: &str, +) -> Result, Box> { + let path = make_string_path("block-transactions", block_id)?; + Ok(app.peek_handle(path).await?) +} + +async fn peek_raw_transaction( + app: &mut NockApp, + tx_id: &str, +) -> Result, Box> { + let path = make_string_path("raw-transaction", tx_id)?; + Ok(app.peek_handle(path).await?) +} + +fn make_heavy_n_path(block_height: u64) -> NounSlab { + let mut slab = NounSlab::new(); + let path = nockvm::noun::T(&mut slab, &[D(tas!(b"heavy-n")), D(block_height), SIG]); + slab.set_root(path); + slab +} + +fn make_string_path(tag: &str, value: &str) -> Result> { + let mut slab = NounSlab::new(); + let tag_noun = make_tas(&mut slab, tag).as_noun(); + let value_noun = Atom::from_value(&mut slab, value.as_bytes())?.as_noun(); + let path = nockvm::noun::T(&mut slab, &[tag_noun, value_noun, SIG]); + slab.set_root(path); + Ok(slab) +} + +fn make_fact_from_payload(tag: &str, payload: &NounSlab) -> NounSlab { + let mut heard = NounSlab::new(); + heard.copy_from_slab(payload); + let tag_noun = make_tas(&mut heard, tag).as_noun(); + heard.modify(|payload_noun| vec![tag_noun, payload_noun]); + + let mut fact = NounSlab::new(); + fact.copy_from_slab(&heard); + fact.modify(|heard_noun| vec![D(tas!(b"fact")), D(0), heard_noun]); + fact +} + +fn tx_ids_from_map(txs_map: &mut NounSlab) -> Result, NockAppError> { + let noun = unsafe { txs_map.root() }; + let space = txs_map.noun_space(); + if let Ok(atom) = noun.in_space(&space).as_atom() { + if atom.as_u64()? == 0 { + return Ok(Vec::new()); + } + } + + let mut tx_ids = Vec::new(); + for entry in HoonMapIter::new(&noun.in_space(&space)) { + if !entry.is_cell() { + continue; + } + let [tx_id, _] = entry.uncell()?; + tx_ids.push(tip5_hash_to_base58_stack(txs_map, tx_id.noun(), &space)?); + } + Ok(tx_ids) +} + +fn page_block_id(page: &mut NounSlab) -> Result { + let noun = unsafe { page.root() }; + let space = page.noun_space(); + let block_id = block_id_from_page(*noun, &space)?; + tip5_hash_to_base58_stack(page, block_id, &space) +} + +fn block_id_from_page(page: Noun, space: &NounSpace) -> Result { + let page_cell = page.in_space(space).as_cell()?; + match page_cell.head().as_atom() { + Ok(version_atom) => { + let version = version_atom.as_u64()?; + if version == 1 { + Ok(page_cell.tail().as_cell()?.head().noun()) + } else { + Err(NockAppError::OtherError(format!( + "unsupported page version {}", + version + ))) + } + } + Err(_) => Ok(page_cell.head().noun()), + } +} + +fn report_pma_sample(sample: PmaTimingSample) { + info!( + "bench: poke_event_ms={:.3} poke_pma_copy_ms={:.3} poke_total_ms={:.3}", + ms(sample.event), + ms(sample.pma_copy), + ms(sample.event + sample.pma_copy) + ); + if let Some(detail) = sample.detail { + report_pma_detail(detail); + } +} + +fn report_pma_detail(detail: PmaCopyDetail) { + report_segment("warm", detail.warm); + report_segment("test_jets", detail.test_jets); + report_segment("hot", detail.hot); + report_segment("cache", detail.cache); + report_segment("cold", detail.cold); + report_segment("arvo", detail.arvo); +} + +fn report_segment(label: &str, segment: nockapp::kernel::form::PmaCopySegment) { + info!( + "bench: poke_pma_{}_ms={:.3} poke_pma_{}_alloc_mib={:.3}", + label, + ms(segment.elapsed), + label, + mib(segment.alloc_words) + ); +} + +fn ms(duration: Duration) -> f64 { + duration.as_secs_f64() * 1000.0 +} + +fn mib(words: usize) -> f64 { + (words as f64 * std::mem::size_of::() as f64) / (1024.0 * 1024.0) +} + +fn unique_suffix() -> String { + let now = SystemTime::now() + .duration_since(UNIX_EPOCH) + .unwrap_or_default(); + format!("{}-{}", now.as_secs(), now.subsec_nanos()) +} diff --git a/crates/nockchain/src/bin/bench_nockchain_kernel.rs b/crates/nockchain/src/bin/bench_nockchain_kernel.rs new file mode 100644 index 000000000..1ca564871 --- /dev/null +++ b/crates/nockchain/src/bin/bench_nockchain_kernel.rs @@ -0,0 +1,1122 @@ +#![allow(clippy::result_large_err)] + +use std::collections::{HashSet, VecDeque}; +use std::error::Error; +use std::path::PathBuf; +use std::time::{Duration, Instant}; + +use clap::{ColorChoice, Parser}; +use kernels_open_dumb::KERNEL as NOCKCHAIN_KERNEL; +use kernels_open_miner::KERNEL as MINER_KERNEL; +use libp2p::PeerId; +use nockapp::kernel::boot::{self, NockStackSize, PmaSize, TraceOpts}; +use nockapp::kernel::form::{PmaCopyDetail, PmaTimingSample, SerfThread}; +use nockapp::noun::slab::NounSlab; +use nockapp::save::SaveableCheckpoint; +use nockapp::utils::{make_tas, NOCK_STACK_SIZE_TINY}; +use nockapp::wire::{SystemWire, Wire}; +use nockapp::{AtomExt, Bytes, NockApp, NockAppError}; +use nockchain::mining::MiningWire; +use nockchain::setup::{self, fakenet_blockchain_constants, DEFAULT_GENESIS_BLOCK_HEIGHT}; +use nockchain_libp2p_io::driver::Libp2pWire; +use nockchain_libp2p_io::tip5_util::tip5_hash_to_base58_stack; +use nockvm::noun::{Atom, Noun, NounAllocator, NounSpace, D, NO, T, YES}; +use nockvm_macros::tas; +use noun_serde::NounEncode; +use rand::Rng; +use tempfile::TempDir; +use tracing::{info, warn}; +use zkvm_jetpack::form::belt::PRIME; +use zkvm_jetpack::form::noun_ext::NounMathExt; +use zkvm_jetpack::form::structs::HoonList; +use zkvm_jetpack::hot::produce_prover_hot_state; + +const DEFAULT_BLOCKS: usize = 100; +const DEFAULT_POW_LEN: u64 = 64; +const DEFAULT_LOG_DIFFICULTY: u64 = 2; +const DEFAULT_MINING_PKH: &str = "9yPePjfWAdUnzaQKyxcRXKRa5PpUzKKEwtpECBZsUYt9Jd7egSDEWoV"; +const DEFAULT_V0_PUBKEY: &str = "2cPnE4Z9RevhTv9is9Hmc1amFubEFbUxzCV2Fxb9GxevJstV5VG92oYt6Sai3d3NjLFcsuVXSLx9hikMbD1agv9M267TVw3hV9MCpMfEnGo5LYtjJ7jPyHg8SERPjJRCWTgZ"; + +const GENESIS_POW_64_BEX_2: &[u8] = include_bytes!(concat!( + env!("CARGO_MANIFEST_DIR"), + "/jams/fakenet-genesis-pow-64-bex-2.jam" +)); +const GENESIS_POW_64_BEX_5: &[u8] = include_bytes!(concat!( + env!("CARGO_MANIFEST_DIR"), + "/jams/fakenet-genesis-pow-64-bex-5.jam" +)); + +#[derive(Parser, Debug)] +#[command( + name = "bench-nockchain-kernel", + about = "Kernel-only nockchain peer benchmark (mining + catch-up)." +)] +struct BenchArgs { + #[arg(long, default_value_t = DEFAULT_BLOCKS)] + blocks: usize, + #[arg(long, default_value_t = DEFAULT_POW_LEN)] + pow_len: u64, + #[arg(long, default_value_t = DEFAULT_LOG_DIFFICULTY)] + log_difficulty: u64, + #[arg(long, default_value_t = false)] + skip_mining: bool, + #[arg(long)] + genesis_jam: Option, + #[arg(long, value_enum, default_value_t = NockStackSize::Medium)] + stack_size: NockStackSize, +} + +#[derive(Clone)] +struct MiningCandidate { + version: NounSlab, + header: NounSlab, + target: NounSlab, + pow_len: u64, +} + +struct Miner { + serf: SerfThread, + next_nonce: Option, + total_attempts: u64, +} + +impl Miner { + async fn new() -> Result> { + let hot_state = produce_prover_hot_state(); + let test_jets_str = std::env::var("NOCK_TEST_JETS").unwrap_or_default(); + let test_jets = boot::parse_test_jets(test_jets_str.as_str()); + let serf = SerfThread::::new( + Vec::from(MINER_KERNEL), + None, + hot_state, + NOCK_STACK_SIZE_TINY, + None, + test_jets, + TraceOpts::default(), + ) + .await?; + Ok(Self { + serf, + next_nonce: None, + total_attempts: 0, + }) + } + + async fn mine_candidate( + &mut self, + candidate: &MiningCandidate, + ) -> Result> { + let mut attempts = 0u64; + let mut nonce = self.next_nonce.take(); + loop { + attempts += 1; + let nonce_slab = nonce.take().unwrap_or_else(random_nonce); + let poke_slab = create_candidate_poke(candidate, &nonce_slab); + let result = self + .serf + .poke(MiningWire::Candidate.to_wire(), poke_slab) + .await?; + match parse_mine_result(result)? { + MineResult::Success { poke, next_nonce } => { + self.next_nonce = Some(next_nonce); + self.total_attempts += attempts; + return Ok(poke); + } + MineResult::Retry { next_nonce } => { + nonce = Some(next_nonce); + } + } + } + } +} + +enum MineResult { + Retry { + next_nonce: NounSlab, + }, + Success { + poke: NounSlab, + next_nonce: NounSlab, + }, +} + +struct Poke { + wire: nockapp::wire::WireRepr, + noun: NounSlab, +} + +struct MiningOutput { + gossips: Vec, + duration: Duration, + total_attempts: u64, + poke_timestamps: Vec, +} + +struct CatchupOutput { + duration: Duration, + poke_timestamps: Vec, +} + +#[derive(Clone, Copy)] +struct TimedSample { + idx: usize, + value: Duration, + ts: Duration, +} + +#[derive(Clone, Copy)] +struct ValueSample { + idx: usize, + value_bytes: u64, + ts: Duration, +} + +#[tokio::main] +async fn main() -> Result<(), Box> { + nockvm::check_endian(); + std::env::set_var("NOCKAPP_DISABLE_METRICS", "1"); + std::env::set_var("GNORT_DISABLE", "1"); + std::env::set_var("NOCK_PMA_TIMING", "1"); + std::env::set_var("NOCK_PMA_TIMING_DETAIL", "1"); + + let args = BenchArgs::parse(); + + let boot_cli = boot::Cli { + new: true, + trace_opts: TraceOpts::default(), + gc_interval: None, + rotating_snapshot_interval_event_time: None, + ephemeral: false, + color: ColorChoice::Auto, + state_jam: None, + bootstrap_from_chkjam: None, + export_state_jam: None, + stack_size: args.stack_size, + pma_initial_size: Some(PmaSize::from_words(args.stack_size.stack_words())), + pma_reserved_size: None, + data_dir: None, + event_log_path: None, + disable_fsync: false, + }; + boot::init_default_tracing(&boot_cli); + + let genesis_bytes = load_genesis_bytes(&args)?; + let genesis_id = genesis_block_id(&genesis_bytes)?; + + let (peer1_dir, mut peer1) = build_nockapp("bench-peer-1", boot_cli.clone()).await?; + let (_peer2_dir, mut peer2) = build_nockapp("bench-peer-2", boot_cli).await?; + + let mut constants = fakenet_blockchain_constants(args.pow_len, args.log_difficulty); + if args.skip_mining { + constants.check_pow_flag = false; + } + + let mut peer1_init = build_init_pokes(&constants, &genesis_bytes, true)?; + let mut peer2_init = build_init_pokes(&constants, &genesis_bytes, false)?; + let peer1_init_count = peer1_init.len(); + + apply_init_pokes(&mut peer2, &mut peer2_init).await?; + let _ = peer2.take_pma_timing_samples_detailed(); + + let mut miner = if args.skip_mining { + None + } else { + Some(Miner::new().await?) + }; + let mining_output = run_mining_peer( + &mut peer1, &mut miner, &mut peer1_init, peer1_init_count, args.blocks, &genesis_id, + ) + .await?; + + let peer1_id = PeerId::random(); + let catchup_output = run_catchup_peer(&mut peer2, &mining_output.gossips, peer1_id).await?; + + print_summary(&args, &mining_output, catchup_output.duration); + + let mining_samples = peer1 + .take_pma_timing_samples_detailed() + .map(|mut samples| { + if samples.len() < peer1_init_count { + warn!( + "bench: mining timing samples ({}) less than init pokes ({}); timing output may be incomplete", + samples.len(), + peer1_init_count + ); + samples.clear(); + } else { + samples.drain(..peer1_init_count); + } + samples + }); + report_phase_timings( + "mining_pokes", mining_samples, &mining_output.poke_timestamps, + ); + + let catchup_samples = peer2.take_pma_timing_samples_detailed(); + report_phase_timings( + "catchup_pokes", catchup_samples, &catchup_output.poke_timestamps, + ); + + drop(peer1_dir); + Ok(()) +} + +async fn build_nockapp(name: &str, cli: boot::Cli) -> Result<(TempDir, NockApp), Box> { + let temp_dir = TempDir::new()?; + let hot_state = produce_prover_hot_state(); + let app = boot::setup::( + NOCKCHAIN_KERNEL, + cli, + hot_state.as_slice(), + name, + Some(temp_dir.path().to_path_buf()), + ) + .await?; + Ok((temp_dir, app)) +} + +fn load_genesis_bytes(args: &BenchArgs) -> Result, Box> { + if let Some(path) = &args.genesis_jam { + return Ok(std::fs::read(path)?); + } + match (args.pow_len, args.log_difficulty) { + (2, 1) => Ok(setup::FAKENET_GENESIS_BLOCK.to_vec()), + (64, 2) => Ok(GENESIS_POW_64_BEX_2.to_vec()), + (64, 5) => Ok(GENESIS_POW_64_BEX_5.to_vec()), + _ => Err(format!( + "No built-in genesis jam for pow_len={} log_difficulty={}; supply --genesis-jam", + args.pow_len, args.log_difficulty + ) + .into()), + } +} + +fn genesis_block_id(genesis_bytes: &[u8]) -> Result> { + let mut slab: NounSlab = NounSlab::new(); + let noun = slab.cue_into(Bytes::from(genesis_bytes.to_vec()))?; + let space = slab.noun_space(); + let block_id = block_id_from_page(noun, &space)?; + Ok(tip5_hash_to_base58_stack(&mut slab, block_id, &space)?) +} + +fn build_init_pokes( + constants: &setup::BlockchainConstants, + genesis_bytes: &[u8], + enable_mining: bool, +) -> Result, Box> { + let mut pokes = VecDeque::new(); + + pokes.push_back(Poke { + wire: SystemWire.to_wire(), + noun: make_set_constants_poke(constants), + }); + pokes.push_back(Poke { + wire: SystemWire.to_wire(), + noun: make_set_genesis_seal_poke(setup::FAKENET_GENESIS_MESSAGE), + }); + pokes.push_back(Poke { + wire: SystemWire.to_wire(), + noun: make_set_btc_data_poke(), + }); + if enable_mining { + pokes.push_back(Poke { + wire: MiningWire::SetPubKey.to_wire(), + noun: make_set_mining_key_poke(DEFAULT_V0_PUBKEY, DEFAULT_MINING_PKH), + }); + pokes.push_back(Poke { + wire: MiningWire::Enable.to_wire(), + noun: make_enable_mining_poke(true), + }); + } + pokes.push_back(Poke { + wire: SystemWire.to_wire(), + noun: make_born_poke(), + }); + pokes.push_back(Poke { + wire: SystemWire.to_wire(), + noun: setup::heard_fake_genesis_block(Some(genesis_bytes.to_vec()))?, + }); + + Ok(pokes) +} + +async fn apply_init_pokes( + nockapp: &mut NockApp, + pokes: &mut VecDeque, +) -> Result<(), Box> { + while let Some(poke) = pokes.pop_front() { + let _ = nockapp.poke(poke.wire, poke.noun).await?; + } + Ok(()) +} + +async fn run_mining_peer( + nockapp: &mut NockApp, + miner: &mut Option, + pending: &mut VecDeque, + skip_pokes: usize, + target_blocks: usize, + genesis_id: &str, +) -> Result> { + let mut gossips = Vec::new(); + let mut seen_blocks: HashSet = HashSet::new(); + let mut mining_started = false; + let mut start = None; + let mut total_pokes = 0usize; + let mut phase_start = if skip_pokes == 0 { + Some(Instant::now()) + } else { + None + }; + let mut poke_timestamps = Vec::new(); + + while let Some(poke) = pending.pop_front() { + if phase_start.is_none() && total_pokes == skip_pokes { + phase_start = Some(Instant::now()); + } + let effects = nockapp.poke(poke.wire, poke.noun).await?; + total_pokes += 1; + if let Some(start) = phase_start { + if total_pokes > skip_pokes { + poke_timestamps.push(start.elapsed()); + } + } + for effect in effects { + if let Some(candidate) = parse_mine_effect(&effect)? { + if gossips.len() >= target_blocks { + continue; + } + if !mining_started { + mining_started = true; + start = Some(Instant::now()); + } + let mined_poke = match miner.as_mut() { + Some(miner) => miner.mine_candidate(&candidate).await?, + None => create_pow_poke(&candidate, &random_nonce()), + }; + pending.push_back(Poke { + wire: MiningWire::Mined.to_wire(), + noun: mined_poke, + }); + continue; + } + + if let Some(mut gossip) = extract_gossip_data(&effect)? { + if !mining_started { + continue; + } + if let Some((block_id, fact_poke)) = heard_block_fact(&mut gossip)? { + if block_id == genesis_id { + continue; + } + if seen_blocks.insert(block_id) { + gossips.push(fact_poke); + if gossips.len() >= target_blocks { + break; + } + } + } + } + } + + if gossips.len() >= target_blocks { + break; + } + + if pending.is_empty() && gossips.len() < target_blocks { + return Err("No pending pokes while target not reached; mining stalled" + .to_string() + .into()); + } + } + + if !mining_started { + return Err("Mining never started (no %mine effect observed)" + .to_string() + .into()); + } + if gossips.len() < target_blocks { + return Err(format!( + "Mined {} blocks but target is {}", + gossips.len(), + target_blocks + ) + .into()); + } + + let duration = start.unwrap_or_else(Instant::now).elapsed(); + let total_attempts = miner.as_ref().map(|m| m.total_attempts).unwrap_or(0); + Ok(MiningOutput { + gossips, + duration, + total_attempts, + poke_timestamps, + }) +} + +async fn run_catchup_peer( + nockapp: &mut NockApp, + gossips: &[NounSlab], + peer_id: PeerId, +) -> Result> { + let start = Instant::now(); + let mut poke_timestamps = Vec::with_capacity(gossips.len()); + for gossip in gossips { + let _ = nockapp + .poke(Libp2pWire::Gossip(peer_id).to_wire(), gossip.clone()) + .await?; + poke_timestamps.push(start.elapsed()); + } + Ok(CatchupOutput { + duration: start.elapsed(), + poke_timestamps, + }) +} + +fn parse_mine_effect(effect: &NounSlab) -> Result, NockAppError> { + let Ok(effect_cell) = (unsafe { effect.root().as_cell() }) else { + return Ok(None); + }; + let space = effect.noun_space(); + let effect_cell = effect_cell.in_space(&space); + if !effect_cell.head().eq_bytes("mine") { + return Ok(None); + } + let Ok([version, commit, target, pow_len_noun]) = effect_cell.tail().noun().uncell(&space) + else { + return Err(NockAppError::OtherError( + "Expected four elements in %mine effect".to_string(), + )); + }; + let pow_len = pow_len_noun + .in_space(&space) + .as_atom()? + .as_u64() + .map_err(|_| NockAppError::OtherError("pow-len was not a u64".to_string()))?; + + let mut version_slab = NounSlab::new(); + version_slab.copy_into(version, &space); + let mut header_slab = NounSlab::new(); + header_slab.copy_into(commit, &space); + let mut target_slab = NounSlab::new(); + target_slab.copy_into(target, &space); + + Ok(Some(MiningCandidate { + version: version_slab, + header: header_slab, + target: target_slab, + pow_len, + })) +} + +fn extract_gossip_data(effect: &NounSlab) -> Result, NockAppError> { + let Ok(effect_cell) = (unsafe { effect.root().as_cell() }) else { + return Ok(None); + }; + let space = effect.noun_space(); + let effect_cell = effect_cell.in_space(&space); + if !effect_cell.head().eq_bytes("gossip") { + return Ok(None); + } + let gossip_cell = effect_cell.tail().noun(); + let data = gossip_cell.in_space(&space).as_cell()?.tail().noun(); + let mut data_slab = NounSlab::new(); + data_slab.copy_into(data, &space); + Ok(Some(data_slab)) +} + +fn heard_block_fact(gossip: &mut NounSlab) -> Result, NockAppError> { + let noun = unsafe { gossip.root() }; + let space = gossip.noun_space(); + let head = noun.in_space(&space).as_cell()?.head(); + if !head.eq_bytes(b"heard-block") { + return Ok(None); + } + + let page = noun.in_space(&space).as_cell()?.tail().noun(); + let block_id = block_id_from_page(page, &space)?; + let block_id_str = tip5_hash_to_base58_stack(gossip, block_id, &space)?; + let mut fact_poke = NounSlab::new(); + fact_poke.copy_from_slab(gossip); + fact_poke.modify(|response_noun| vec![D(tas!(b"fact")), D(0), response_noun]); + Ok(Some((block_id_str, fact_poke))) +} + +fn block_id_from_page(page: Noun, space: &NounSpace) -> Result { + let page_cell = page.in_space(space).as_cell()?; + match page_cell.head().as_atom() { + Ok(version_atom) => { + let version = version_atom.as_u64()?; + if version == 1 { + Ok(page_cell.tail().as_cell()?.head().noun()) + } else { + Err(NockAppError::OtherError(format!( + "Unsupported page version {}", + version + ))) + } + } + Err(_) => Ok(page_cell.head().noun()), + } +} + +fn parse_mine_result(result: NounSlab) -> Result { + let result_noun = unsafe { result.root() }; + let space = result.noun_space(); + let Ok(effects) = HoonList::try_from(*result_noun, &space) else { + return Err(NockAppError::OtherError(String::from( + "Mining kernel result was not a list", + ))); + }; + + let mining_result = effects.filter_map(|effect| { + if effect.is_atom() { + return None; + } + let Ok(effect_cell) = effect.in_space(&space).as_cell() else { + return None; + }; + if effect_cell.head().eq_bytes("mine-result") { + Some(effect_cell.tail().noun()) + } else { + None + } + }); + + let Some(mine_result) = mining_result.into_iter().next() else { + return Err(NockAppError::OtherError(String::from( + "Mining kernel result missing %mine-result", + ))); + }; + + let Ok([res, tail]) = mine_result.uncell(&space) else { + return Err(NockAppError::OtherError(String::from( + "Malformed %mine-result payload", + ))); + }; + + if unsafe { res.raw_equals(&D(0)) } { + let Ok([hash, poke]) = tail.uncell(&space) else { + return Err(NockAppError::OtherError(String::from( + "Expected hash and poke in successful %mine-result", + ))); + }; + let mut poke_slab = NounSlab::new(); + poke_slab.copy_into(poke, &space); + let mut nonce_slab = NounSlab::new(); + nonce_slab.copy_into(hash, &space); + Ok(MineResult::Success { + poke: poke_slab, + next_nonce: nonce_slab, + }) + } else { + let mut nonce_slab = NounSlab::new(); + nonce_slab.copy_into(tail, &space); + Ok(MineResult::Retry { + next_nonce: nonce_slab, + }) + } +} + +fn create_candidate_poke(candidate: &MiningCandidate, nonce: &NounSlab) -> NounSlab { + let mut slab = NounSlab::new(); + let header_space = candidate.header.noun_space(); + let version_space = candidate.version.noun_space(); + let target_space = candidate.target.noun_space(); + let nonce_space = nonce.noun_space(); + let header = slab.copy_into(unsafe { *candidate.header.root() }, &header_space); + let version = slab.copy_into(unsafe { *candidate.version.root() }, &version_space); + let target = slab.copy_into(unsafe { *candidate.target.root() }, &target_space); + let nonce = slab.copy_into(unsafe { *nonce.root() }, &nonce_space); + let poke_noun = T( + &mut slab, + &[version, header, nonce, target, D(candidate.pow_len)], + ); + slab.set_root(poke_noun); + slab +} + +fn create_pow_poke(candidate: &MiningCandidate, nonce: &NounSlab) -> NounSlab { + let mut slab = NounSlab::new(); + let version_space = candidate.version.noun_space(); + let header_space = candidate.header.noun_space(); + let nonce_space = nonce.noun_space(); + let version = slab.copy_into(unsafe { *candidate.version.root() }, &version_space); + let header = slab.copy_into(unsafe { *candidate.header.root() }, &header_space); + let nonce = slab.copy_into(unsafe { *nonce.root() }, &nonce_space); + // Dummy proof/digest: skip-mining disables pow checks; 0 passes check-target. + let proof = T(&mut slab, &[version, D(0), D(0), D(0)]); + let poke_noun = T( + &mut slab, + &[D(tas!(b"command")), D(tas!(b"pow")), proof, D(0), header, nonce], + ); + slab.set_root(poke_noun); + slab +} + +fn random_nonce() -> NounSlab { + let mut rng = rand::rng(); + let mut nonce_slab = NounSlab::new(); + let mut nonce_cell = Atom::from_value(&mut nonce_slab, rng.random::() % PRIME) + .expect("Failed to create nonce atom") + .as_noun(); + for _ in 1..5 { + let nonce_atom = Atom::from_value(&mut nonce_slab, rng.random::() % PRIME) + .expect("Failed to create nonce atom") + .as_noun(); + nonce_cell = T(&mut nonce_slab, &[nonce_atom, nonce_cell]); + } + nonce_slab.set_root(nonce_cell); + nonce_slab +} + +fn make_set_constants_poke(constants: &setup::BlockchainConstants) -> NounSlab { + let mut poke_slab = NounSlab::new(); + let tag = make_tas(&mut poke_slab, "set-constants").as_noun(); + let constants_noun = constants.to_noun(&mut poke_slab); + let poke_noun = T(&mut poke_slab, &[D(tas!(b"command")), tag, constants_noun]); + poke_slab.set_root(poke_noun); + poke_slab +} + +fn make_set_genesis_seal_poke(seal: &str) -> NounSlab { + let mut poke_slab = NounSlab::new(); + let block_height_noun = Atom::new(&mut poke_slab, DEFAULT_GENESIS_BLOCK_HEIGHT).as_noun(); + let seal_byts = Bytes::from(seal.to_string().into_bytes()); + let seal_noun = Atom::from_bytes(&mut poke_slab, &seal_byts).as_noun(); + let tag = Bytes::from(b"set-genesis-seal".to_vec()); + let set_genesis_seal = Atom::from_bytes(&mut poke_slab, &tag).as_noun(); + let poke_noun = T( + &mut poke_slab, + &[D(tas!(b"command")), set_genesis_seal, block_height_noun, seal_noun], + ); + poke_slab.set_root(poke_noun); + poke_slab +} + +fn make_set_btc_data_poke() -> NounSlab { + let mut poke_slab = NounSlab::new(); + let poke_noun = T( + &mut poke_slab, + &[D(tas!(b"command")), D(tas!(b"btc-data")), D(0)], + ); + poke_slab.set_root(poke_noun); + poke_slab +} + +fn make_born_poke() -> NounSlab { + let mut poke_slab = NounSlab::new(); + let born = T( + &mut poke_slab, + &[D(tas!(b"command")), D(tas!(b"born")), D(0)], + ); + poke_slab.set_root(born); + poke_slab +} + +fn make_enable_mining_poke(enable: bool) -> NounSlab { + let mut slab = NounSlab::new(); + let enable_mining = + Atom::from_value(&mut slab, "enable-mining").expect("Failed to create enable-mining atom"); + let enable_mining_poke = T( + &mut slab, + &[D(tas!(b"command")), enable_mining.as_noun(), if enable { YES } else { NO }], + ); + slab.set_root(enable_mining_poke); + slab +} + +fn make_set_mining_key_poke(v0_pubkey: &str, pkh: &str) -> NounSlab { + let mut slab = NounSlab::new(); + let set_mining_key_adv = Atom::from_value(&mut slab, "set-mining-key-advanced") + .expect("Failed to create set-mining-key-advanced atom"); + + let mut configs_list = D(0); + let mut keys_noun = D(0); + let key_atom = Atom::from_value(&mut slab, v0_pubkey) + .expect("Failed to create key atom") + .as_noun(); + keys_noun = T(&mut slab, &[key_atom, keys_noun]); + let config_tuple = T(&mut slab, &[D(1), D(1), keys_noun]); + configs_list = T(&mut slab, &[config_tuple, configs_list]); + + let mut pkh_configs_list = D(0); + let pkh_noun = Atom::from_value(&mut slab, pkh) + .expect("Failed to create pkh atom") + .as_noun(); + let pkh_tuple = T(&mut slab, &[D(1), pkh_noun]); + pkh_configs_list = T(&mut slab, &[pkh_tuple, pkh_configs_list]); + + let set_mining_key_poke = T( + &mut slab, + &[ + D(tas!(b"command")), + set_mining_key_adv.as_noun(), + configs_list, + pkh_configs_list, + ], + ); + slab.set_root(set_mining_key_poke); + slab +} + +fn print_summary(args: &BenchArgs, mining: &MiningOutput, catchup: Duration) { + let mined_blocks = mining.gossips.len() as f64; + let mining_ms = duration_ms(mining.duration); + let catchup_ms = duration_ms(catchup); + let avg_mining_ms = if mined_blocks > 0.0 { + mining_ms / mined_blocks + } else { + 0.0 + }; + let avg_catchup_ms = if mined_blocks > 0.0 { + catchup_ms / mined_blocks + } else { + 0.0 + }; + let avg_attempts = if mined_blocks > 0.0 { + (mining.total_attempts as f64) / mined_blocks + } else { + 0.0 + }; + + info!( + "bench: blocks_target={} pow_len={} log_difficulty={} skip_mining={} stack_size={:?}", + args.blocks, args.pow_len, args.log_difficulty, args.skip_mining, args.stack_size + ); + info!( + "bench: mined_blocks={} mining_ms={:.3} avg_ms_per_block={:.3} avg_attempts_per_block={:.2}", + mining.gossips.len(), + mining_ms, + avg_mining_ms, + avg_attempts + ); + info!( + "bench: catchup_blocks={} catchup_ms={:.3} avg_ms_per_block={:.3}", + mining.gossips.len(), + catchup_ms, + avg_catchup_ms + ); +} + +fn report_phase_timings( + label: &str, + samples: Option>, + timestamps: &[Duration], +) { + let Some(samples) = samples else { + info!( + "bench: {} timings unavailable (NOCK_PMA_TIMING not enabled at boot)", + label + ); + return; + }; + if samples.is_empty() || timestamps.is_empty() { + info!("bench: {} timings unavailable (no samples)", label); + return; + } + + let (total_samples, pma_samples, min_len) = build_timed_samples(&samples, timestamps, label); + info!( + "bench: {} timing timestamps are ms since phase start (post-init)", + label + ); + summarize_timed_samples(&format!("{label}_total_ms"), &total_samples); + summarize_timed_samples(&format!("{label}_pma_ms"), &pma_samples); + + if let Some(detail_samples) = build_detail_samples(&samples, timestamps, label, min_len) { + report_detail_samples(label, &detail_samples); + } +} + +struct PmaDetailSamples { + warm_ms: Vec, + warm_alloc: Vec, + test_jets_ms: Vec, + test_jets_alloc: Vec, + hot_ms: Vec, + hot_alloc: Vec, + cache_ms: Vec, + cache_alloc: Vec, + cold_ms: Vec, + cold_alloc: Vec, + arvo_ms: Vec, + arvo_alloc: Vec, +} + +fn build_timed_samples( + samples: &[PmaTimingSample], + timestamps: &[Duration], + label: &str, +) -> (Vec, Vec, usize) { + let min_len = samples.len().min(timestamps.len()); + if samples.len() != timestamps.len() { + warn!( + "bench: {} timing count mismatch: samples={}, timestamps={}, truncating to {}", + label, + samples.len(), + timestamps.len(), + min_len + ); + } + let mut total = Vec::with_capacity(min_len); + let mut pma = Vec::with_capacity(min_len); + for (idx, (sample, ts)) in samples.iter().zip(timestamps).take(min_len).enumerate() { + let total_value = sample.event + sample.pma_copy; + total.push(TimedSample { + idx, + value: total_value, + ts: *ts, + }); + pma.push(TimedSample { + idx, + value: sample.pma_copy, + ts: *ts, + }); + } + (total, pma, min_len) +} + +fn build_detail_samples( + samples: &[PmaTimingSample], + timestamps: &[Duration], + label: &str, + min_len: usize, +) -> Option { + let mut detail = PmaDetailSamples { + warm_ms: Vec::with_capacity(min_len), + warm_alloc: Vec::with_capacity(min_len), + test_jets_ms: Vec::with_capacity(min_len), + test_jets_alloc: Vec::with_capacity(min_len), + hot_ms: Vec::with_capacity(min_len), + hot_alloc: Vec::with_capacity(min_len), + cache_ms: Vec::with_capacity(min_len), + cache_alloc: Vec::with_capacity(min_len), + cold_ms: Vec::with_capacity(min_len), + cold_alloc: Vec::with_capacity(min_len), + arvo_ms: Vec::with_capacity(min_len), + arvo_alloc: Vec::with_capacity(min_len), + }; + + for (idx, (sample, ts)) in samples.iter().zip(timestamps).take(min_len).enumerate() { + let Some(copy) = sample.detail else { + warn!( + "bench: {} detail timings unavailable (NOCK_PMA_TIMING_DETAIL not enabled at boot)", + label + ); + return None; + }; + push_detail_samples(&mut detail, idx, *ts, copy); + } + + Some(detail) +} + +fn push_detail_samples( + detail: &mut PmaDetailSamples, + idx: usize, + ts: Duration, + copy: PmaCopyDetail, +) { + detail.warm_ms.push(TimedSample { + idx, + value: copy.warm.elapsed, + ts, + }); + detail.warm_alloc.push(ValueSample { + idx, + value_bytes: (copy.warm.alloc_words as u64) * 8, + ts, + }); + detail.test_jets_ms.push(TimedSample { + idx, + value: copy.test_jets.elapsed, + ts, + }); + detail.test_jets_alloc.push(ValueSample { + idx, + value_bytes: (copy.test_jets.alloc_words as u64) * 8, + ts, + }); + detail.hot_ms.push(TimedSample { + idx, + value: copy.hot.elapsed, + ts, + }); + detail.hot_alloc.push(ValueSample { + idx, + value_bytes: (copy.hot.alloc_words as u64) * 8, + ts, + }); + detail.cache_ms.push(TimedSample { + idx, + value: copy.cache.elapsed, + ts, + }); + detail.cache_alloc.push(ValueSample { + idx, + value_bytes: (copy.cache.alloc_words as u64) * 8, + ts, + }); + detail.cold_ms.push(TimedSample { + idx, + value: copy.cold.elapsed, + ts, + }); + detail.cold_alloc.push(ValueSample { + idx, + value_bytes: (copy.cold.alloc_words as u64) * 8, + ts, + }); + detail.arvo_ms.push(TimedSample { + idx, + value: copy.arvo.elapsed, + ts, + }); + detail.arvo_alloc.push(ValueSample { + idx, + value_bytes: (copy.arvo.alloc_words as u64) * 8, + ts, + }); +} + +fn report_detail_samples(label: &str, detail: &PmaDetailSamples) { + summarize_timed_samples(&format!("{label}_pma_warm_ms"), &detail.warm_ms); + summarize_value_samples(&format!("{label}_pma_warm_alloc_mib"), &detail.warm_alloc); + summarize_timed_samples(&format!("{label}_pma_test_jets_ms"), &detail.test_jets_ms); + summarize_value_samples( + &format!("{label}_pma_test_jets_alloc_mib"), + &detail.test_jets_alloc, + ); + summarize_timed_samples(&format!("{label}_pma_hot_ms"), &detail.hot_ms); + summarize_value_samples(&format!("{label}_pma_hot_alloc_mib"), &detail.hot_alloc); + summarize_timed_samples(&format!("{label}_pma_cache_ms"), &detail.cache_ms); + summarize_value_samples(&format!("{label}_pma_cache_alloc_mib"), &detail.cache_alloc); + summarize_timed_samples(&format!("{label}_pma_cold_ms"), &detail.cold_ms); + summarize_value_samples(&format!("{label}_pma_cold_alloc_mib"), &detail.cold_alloc); + summarize_timed_samples(&format!("{label}_pma_arvo_ms"), &detail.arvo_ms); + summarize_value_samples(&format!("{label}_pma_arvo_alloc_mib"), &detail.arvo_alloc); +} + +fn summarize_timed_samples(label: &str, samples: &[TimedSample]) { + if samples.is_empty() { + info!("bench: {}: no samples", label); + return; + } + let p50 = percentile_sample(samples, 50.0).expect("non-empty samples should have p50"); + let p95 = percentile_sample(samples, 95.0).expect("non-empty samples should have p95"); + let p99 = percentile_sample(samples, 99.0).expect("non-empty samples should have p99"); + let max = top_n_samples(samples, 1)[0]; + info!( + "bench: {}: p50={} p95={} p99={} max={}", + label, + format_sample(p50), + format_sample(p95), + format_sample(p99), + format_sample(max) + ); + + let top3 = top_n_samples(samples, 3); + let top3_fmt = top3 + .iter() + .map(|sample| format_sample(*sample)) + .collect::>() + .join(", "); + info!("bench: {}: top3=[{}]", label, top3_fmt); +} + +fn summarize_value_samples(label: &str, samples: &[ValueSample]) { + if samples.is_empty() { + info!("bench: {}: no samples", label); + return; + } + let p50 = + percentile_value_sample(samples, 50.0).expect("non-empty value samples should have p50"); + let p95 = + percentile_value_sample(samples, 95.0).expect("non-empty value samples should have p95"); + let p99 = + percentile_value_sample(samples, 99.0).expect("non-empty value samples should have p99"); + let max = top_n_value_samples(samples, 1)[0]; + info!( + "bench: {}: p50={} p95={} p99={} max={}", + label, + format_value_sample(p50), + format_value_sample(p95), + format_value_sample(p99), + format_value_sample(max) + ); + + let top3 = top_n_value_samples(samples, 3); + let top3_fmt = top3 + .iter() + .map(|sample| format_value_sample(*sample)) + .collect::>() + .join(", "); + info!("bench: {}: top3=[{}]", label, top3_fmt); +} + +fn percentile_sample(samples: &[TimedSample], pct: f64) -> Option { + if samples.is_empty() { + return None; + } + let mut indices: Vec = (0..samples.len()).collect(); + indices.sort_by(|&a, &b| samples[a].value.cmp(&samples[b].value)); + let rank = ((pct / 100.0) * ((samples.len() - 1) as f64)).ceil() as usize; + Some(samples[indices[rank]]) +} + +fn percentile_value_sample(samples: &[ValueSample], pct: f64) -> Option { + if samples.is_empty() { + return None; + } + let mut indices: Vec = (0..samples.len()).collect(); + indices.sort_by(|&a, &b| samples[a].value_bytes.cmp(&samples[b].value_bytes)); + let rank = ((pct / 100.0) * ((samples.len() - 1) as f64)).ceil() as usize; + Some(samples[indices[rank]]) +} + +fn top_n_samples(samples: &[TimedSample], count: usize) -> Vec { + let mut indices: Vec = (0..samples.len()).collect(); + indices.sort_by(|&a, &b| samples[b].value.cmp(&samples[a].value)); + indices + .into_iter() + .take(count.min(samples.len())) + .map(|idx| samples[idx]) + .collect() +} + +fn top_n_value_samples(samples: &[ValueSample], count: usize) -> Vec { + let mut indices: Vec = (0..samples.len()).collect(); + indices.sort_by(|&a, &b| samples[b].value_bytes.cmp(&samples[a].value_bytes)); + indices + .into_iter() + .take(count.min(samples.len())) + .map(|idx| samples[idx]) + .collect() +} + +fn format_sample(sample: TimedSample) -> String { + format!( + "{:.3}ms@t={:.3}ms(idx={})", + duration_ms(sample.value), + duration_ms(sample.ts), + sample.idx + 1 + ) +} + +fn format_value_sample(sample: ValueSample) -> String { + format!( + "{:.3}MiB@t={:.3}ms(idx={})", + bytes_to_mib(sample.value_bytes), + duration_ms(sample.ts), + sample.idx + 1 + ) +} + +fn duration_ms(d: Duration) -> f64 { + (d.as_micros() as f64) / 1000.0 +} + +fn bytes_to_mib(bytes: u64) -> f64 { + (bytes as f64) / (1024.0 * 1024.0) +} diff --git a/crates/nockchain/src/config.rs b/crates/nockchain/src/config.rs index 363ece915..cee45ca35 100644 --- a/crates/nockchain/src/config.rs +++ b/crates/nockchain/src/config.rs @@ -1,7 +1,9 @@ +use std::ffi::OsString; use std::path::PathBuf; use std::time::Duration; -use clap::{value_parser, ArgAction, Parser}; +use clap::{value_parser, ArgAction, Args, CommandFactory, FromArgMatches, Parser}; +use nockapp::kernel::boot::{NockStackSize, PmaSize}; use nockchain_types::tx_engine::common::Hash; use crate::mining::MiningPkhConfig; @@ -35,6 +37,74 @@ pub const CHAIN_INTERVAL: Duration = Duration::from_secs(20); /// switched to a future block for launch. pub const GENESIS_HEIGHT: u64 = 897767; +/// Validated ASERT fakenet trio. Only constructible via [`FakenetAsertArgs::into_config`]. +#[derive(Debug, Clone, Copy)] +pub struct FakenetAsertConfig { + pub phase: u64, + pub anchor_height: u64, + pub anchor_target_bex: u64, +} + +/// CLI surface for the three ASERT fakenet overrides. All three must be supplied together or not +/// at all; call [`into_config`][FakenetAsertArgs::into_config] to enforce the invariant and obtain +/// a [`FakenetAsertConfig`]. +#[derive(Args, Debug, Clone, Default)] +pub struct FakenetAsertArgs { + #[arg( + long = "fakenet-asert-phase", + help = "Override the asert-phase (aserti3-2d activation height) when running on fakenet. Requires --fakenet.", + requires = "fakenet" + )] + pub phase: Option, + #[arg( + long = "fakenet-asert-anchor-height", + help = "Override the asert-anchor-height when running on fakenet. Must equal asert-phase - 1. Requires --fakenet.", + requires = "fakenet" + )] + pub anchor_height: Option, + #[arg( + long = "fakenet-asert-anchor-target-bex", + help = "Override asert-anchor-target-atom by bex exponent when running on fakenet (target = 2^bex). Requires --fakenet.", + requires = "fakenet" + )] + pub anchor_target_bex: Option, +} + +impl FakenetAsertArgs { + /// Validates the trio invariant and converts to [`FakenetAsertConfig`]. + /// + /// Returns `Ok(None)` when none of the three flags are set. + /// Returns `Err` when only some are set, when `anchor_height + 1 != phase`, or when + /// `anchor_target_bex` exceeds the cap. + pub fn into_config(self) -> Result, String> { + match (self.phase, self.anchor_height, self.anchor_target_bex) { + (None, None, None) => Ok(None), + (Some(phase), Some(anchor_height), Some(bex)) => { + if phase == 0 || Some(phase) != anchor_height.checked_add(1) { + return Err(format!( + "--fakenet-asert-anchor-height ({anchor_height}) must equal \ + --fakenet-asert-phase ({phase}) minus 1" + )); + } + const MAX_BEX: u64 = 512; + if bex > MAX_BEX { + return Err(format!( + "--fakenet-asert-anchor-target-bex ({bex}) must be <= {MAX_BEX}" + )); + } + Ok(Some(FakenetAsertConfig { + phase, + anchor_height, + anchor_target_bex: bex, + })) + } + _ => Err("--fakenet-asert-phase, --fakenet-asert-anchor-height, and \ + --fakenet-asert-anchor-target-bex must all be specified together or not at all" + .to_string()), + } + } +} + /// Command line arguments #[derive(Parser, Debug, Clone)] #[command(name = "nockchain")] @@ -73,6 +143,11 @@ pub struct NockchainCli { default_value = "false" )] pub no_new_peer_id: bool, + #[arg( + long, + help = "Override the path to the libp2p identity key (defaults to .nockchain_identity)" + )] + pub identity_path: Option, #[arg(long, help = "Maximum established incoming connections")] pub max_established_incoming: Option, #[arg(long, help = "Maximum established outgoing connections")] @@ -124,6 +199,8 @@ pub struct NockchainCli { requires = "fakenet" )] pub fakenet_bythos_phase: Option, + #[command(flatten)] + pub fakenet_asert: FakenetAsertArgs, #[arg(long, help = "Path to fake genesis block jam file")] pub fakenet_genesis_jam_path: Option, #[arg(long, help = "Public gRPC binding address (off by default), recommended value = \"127.0.0.1:5555\"", value_parser = clap::value_parser!(std::net::SocketAddr))] @@ -134,6 +211,57 @@ pub struct NockchainCli { pub fast_sync: bool, } +impl NockchainCli { + pub fn parse_with_default_stack_size(default_stack_size: NockStackSize) -> Self { + Self::parse_from_with_default_stack_size(std::env::args_os(), default_stack_size) + } + + fn parse_from_with_default_stack_size(args: I, default_stack_size: NockStackSize) -> Self + where + I: IntoIterator, + T: Into, + { + Self::try_parse_from_with_default_stack_size(args, default_stack_size) + .unwrap_or_else(|err| err.exit()) + } + + fn try_parse_from_with_default_stack_size( + args: I, + default_stack_size: NockStackSize, + ) -> Result + where + I: IntoIterator, + T: Into, + { + let mut matches = Self::command_with_default_stack_size(default_stack_size) + .try_get_matches_from(args.into_iter().map(Into::into))?; + let mut cli = ::from_arg_matches_mut(&mut matches)?; + if cli.nockapp_cli.pma_initial_size.is_none() { + cli.nockapp_cli.pma_initial_size = Some(PmaSize::from_words( + cli.nockapp_cli.stack_size.stack_words(), + )); + } + Ok(cli) + } + + fn command_with_default_stack_size(default_stack_size: NockStackSize) -> clap::Command { + ::command().mut_arg("stack_size", |arg| { + arg.default_value(stack_size_default_arg(default_stack_size)) + }) + } +} + +fn stack_size_default_arg(stack_size: NockStackSize) -> &'static str { + match stack_size { + NockStackSize::Tiny => "tiny", + NockStackSize::Small => "small", + NockStackSize::Normal => "normal", + NockStackSize::Medium => "medium", + NockStackSize::Large => "large", + NockStackSize::Huge => "huge", + } +} + impl NockchainCli { pub fn validate(&self) -> Result<(), String> { if self.mine && !(self.mining_pkh.is_some() || self.mining_pkh_adv.is_some()) { @@ -160,13 +288,16 @@ impl NockchainCli { } } + self.fakenet_asert.clone().into_config().map(|_| ())?; + Ok(()) } } #[cfg(test)] mod tests { - use nockapp::kernel::boot::default_boot_cli; + use nockapp::kernel::boot::{default_boot_cli, NockStackSize}; + use nockapp::utils::{NOCK_STACK_SIZE, NOCK_STACK_SIZE_MEDIUM, NOCK_STACK_SIZE_SMALL}; use super::*; @@ -186,6 +317,7 @@ mod tests { no_default_peers: false, bind: None, no_new_peer_id: false, + identity_path: None, max_established_incoming: None, max_established_outgoing: None, max_pending_incoming: None, @@ -200,6 +332,7 @@ mod tests { fakenet_log_difficulty: 1, fakenet_v1_phase: None, fakenet_bythos_phase: None, + fakenet_asert: FakenetAsertArgs::default(), fakenet_genesis_jam_path: None, bind_public_grpc_addr: Some("127.0.0.1:5555".parse().unwrap()), bind_private_grpc_port: 5555, @@ -207,6 +340,53 @@ mod tests { } } + #[test] + fn default_stack_size_can_be_set_by_binary() { + let cli = + NockchainCli::parse_from_with_default_stack_size(["nockchain"], NockStackSize::Medium); + assert!(matches!(cli.nockapp_cli.stack_size, NockStackSize::Medium)); + assert_eq!( + cli.nockapp_cli.pma_initial_size.unwrap().words(), + NOCK_STACK_SIZE_MEDIUM + ); + } + + #[test] + fn explicit_stack_size_overrides_binary_default() { + let cli = NockchainCli::parse_from_with_default_stack_size( + ["nockchain", "--stack-size", "normal"], + NockStackSize::Medium, + ); + assert!(matches!(cli.nockapp_cli.stack_size, NockStackSize::Normal)); + assert_eq!( + cli.nockapp_cli.pma_initial_size.unwrap().words(), + NOCK_STACK_SIZE + ); + + let cli = NockchainCli::parse_from_with_default_stack_size( + ["nockchain", "--stack-size=small"], + NockStackSize::Medium, + ); + assert!(matches!(cli.nockapp_cli.stack_size, NockStackSize::Small)); + assert_eq!( + cli.nockapp_cli.pma_initial_size.unwrap().words(), + NOCK_STACK_SIZE_SMALL + ); + } + + #[test] + fn explicit_pma_initial_size_overrides_nockchain_stack_default() { + let cli = NockchainCli::parse_from_with_default_stack_size( + ["nockchain", "--stack-size=small", "--pma-initial-size=512MiB"], + NockStackSize::Medium, + ); + assert!(matches!(cli.nockapp_cli.stack_size, NockStackSize::Small)); + assert_eq!( + cli.nockapp_cli.pma_initial_size.unwrap().words(), + 64 * 1024 * 1024 + ); + } + #[test] fn validate_accepts_valid_advanced_configs() { let mut cli = base_cli(); @@ -218,6 +398,76 @@ mod tests { assert!(cli.validate().is_ok()); } + #[test] + fn validate_accepts_all_three_asert_overrides() { + let mut cli = base_cli(); + cli.fakenet = true; + cli.fakenet_asert = FakenetAsertArgs { + phase: Some(10), + anchor_height: Some(9), + anchor_target_bex: Some(4), + }; + assert!(cli.validate().is_ok()); + } + + #[test] + fn validate_accepts_no_asert_overrides() { + let cli = base_cli(); + assert!(cli.validate().is_ok()); + } + + #[test] + fn validate_rejects_partial_asert_overrides() { + let mut cli = base_cli(); + cli.fakenet = true; + cli.fakenet_asert = FakenetAsertArgs { + phase: Some(10), + anchor_height: None, + anchor_target_bex: None, + }; + let err = cli.validate().expect_err("expected partial ASERT error"); + assert!(err.contains("must all be specified together")); + } + + #[test] + fn validate_rejects_anchor_height_not_phase_minus_one() { + let mut cli = base_cli(); + cli.fakenet = true; + cli.fakenet_asert = FakenetAsertArgs { + phase: Some(10), + anchor_height: Some(8), // should be 9 + anchor_target_bex: Some(4), + }; + let err = cli.validate().expect_err("expected anchor invariant error"); + assert!(err.contains("must equal")); + } + + #[test] + fn validate_rejects_asert_phase_zero() { + let mut cli = base_cli(); + cli.fakenet = true; + cli.fakenet_asert = FakenetAsertArgs { + phase: Some(0), + anchor_height: Some(0), + anchor_target_bex: Some(4), + }; + let err = cli.validate().expect_err("expected phase=0 error"); + assert!(err.contains("must equal")); + } + + #[test] + fn validate_rejects_bex_above_cap() { + let mut cli = base_cli(); + cli.fakenet = true; + cli.fakenet_asert = FakenetAsertArgs { + phase: Some(10), + anchor_height: Some(9), + anchor_target_bex: Some(513), + }; + let err = cli.validate().expect_err("expected bex cap error"); + assert!(err.contains("must be <=")); + } + #[test] fn validate_rejects_invalid_mining_pkh_adv_entry() { // We specifically want to catch if users mix up v0 and v1 addresses, because they are both base58-encoded. diff --git a/crates/nockchain/src/lib.rs b/crates/nockchain/src/lib.rs index e7390c4b9..ac16d5555 100644 --- a/crates/nockchain/src/lib.rs +++ b/crates/nockchain/src/lib.rs @@ -11,10 +11,11 @@ pub mod config; pub mod mining; pub mod setup; +pub mod traces; use std::error::Error; use std::fs; -use std::path::Path; +use std::path::{Path, PathBuf}; pub use config::NockchainCli; use libp2p::identity::Keypair; @@ -28,13 +29,13 @@ pub mod colors; use colors::*; use nockapp::noun::slab::{Jammer, NounSlab}; +use nockchain_types::fakenet_blockchain_constants; use nockvm::jets::hot::HotEntry; use nockvm::noun::{D, T, YES}; use nockvm_macros::tas; use tracing::{debug, info, instrument}; use crate::mining::{MiningKeyConfig, MiningPkhConfig}; -use crate::setup::fakenet_blockchain_constants; /// Module for handling driver initialization signals pub mod driver_init { @@ -220,16 +221,16 @@ pub async fn init_with_kernel( cli.validate()?; - let mut nockapp_cli = cli.nockapp_cli.clone(); - nockapp_cli.stack_size = nockapp::kernel::boot::NockStackSize::Medium; + let nockapp_cli = cli.nockapp_cli.clone(); let mut nockapp = boot::setup::(kernel_jam, nockapp_cli, hot_state, "nockchain", None).await?; - let keypair = { - let keypair_path = Path::new(config::IDENTITY_PATH); - load_keypair(keypair_path, cli.no_new_peer_id)? - }; + let identity_path = cli + .identity_path + .clone() + .unwrap_or_else(|| PathBuf::from(config::IDENTITY_PATH)); + let keypair = { load_keypair(identity_path.as_path(), cli.no_new_peer_id)? }; info!("allowed_peers_path: {:?}", cli.allowed_peers_path); let allowed = cli.allowed_peers_path.as_ref().map(|path| { let contents = fs::read_to_string(path).expect("failed to read allowed peers file: {}"); @@ -408,9 +409,15 @@ pub async fn init_with_kernel( if let Some(bythos_phase) = cli.fakenet_bythos_phase { fakenet_constants = fakenet_constants.with_bythos_phase(bythos_phase); } + if let Some(asert) = cli.fakenet_asert.into_config()? { + fakenet_constants = fakenet_constants + .with_asert_phase(asert.phase) + .with_asert_anchor_height(asert.anchor_height) + .with_asert_anchor_target_bex(asert.anchor_target_bex); + } setup::poke( &mut nockapp, - setup::SetupCommand::PokeFakenetConstants(fakenet_constants), + setup::SetupCommand::PokeFakenetConstants(Box::new(fakenet_constants)), ) .await?; if let Some(true) = is_kernel_mainnet { @@ -529,6 +536,7 @@ pub async fn init_with_kernel( )) .await; + nockapp.add_io_driver(crate::traces::traces_driver()).await; nockapp.add_io_driver(nockapp::exit_driver()).await; Ok(nockapp) diff --git a/crates/nockchain/src/main.rs b/crates/nockchain/src/main.rs index 7066e468b..f4ca9adca 100644 --- a/crates/nockchain/src/main.rs +++ b/crates/nockchain/src/main.rs @@ -1,7 +1,6 @@ use std::error::Error; use chaff::Chaff; -use clap::Parser; use kernels_open_dumb::KERNEL; use nockapp::kernel::boot; use nockchain::NockchainAPIConfig; @@ -20,7 +19,7 @@ static ALLOC: tracy_client::ProfiledAllocator = #[tokio::main] async fn main() -> Result<(), Box> { nockvm::check_endian(); - let cli = nockchain::NockchainCli::parse(); + let cli = nockchain::NockchainCli::parse_with_default_stack_size(boot::NockStackSize::Large); boot::init_default_tracing(&cli.nockapp_cli); let prover_hot_state = produce_prover_hot_state(); diff --git a/crates/nockchain/src/mining.rs b/crates/nockchain/src/mining.rs index 76907127a..077cd7ddb 100644 --- a/crates/nockchain/src/mining.rs +++ b/crates/nockchain/src/mining.rs @@ -11,9 +11,8 @@ use nockapp::save::SaveableCheckpoint; use nockapp::utils::NOCK_STACK_SIZE_TINY; use nockapp::CrownError; use nockchain_libp2p_io::tip5_util::tip5_hash_to_base58; -use nockvm::ext::NounExt; use nockvm::interpreter::NockCancelToken; -use nockvm::noun::{Atom, D, NO, T, YES}; +use nockvm::noun::{Atom, NounAllocator, D, NO, T, YES}; use nockvm_macros::tas; use rand::Rng; use tokio::sync::Mutex; @@ -178,64 +177,113 @@ pub fn create_mining_driver( let slab = slab_res.expect("Mining attempt result failed"); let result = unsafe { slab.root() }; - match HoonList::try_from(*result) { - Err(_) => { - start_mining_attempt(serf, mining_data.lock().await, &mut mining_attempts, None, id).await; - } - Ok(effects) => { - let mining_result = - effects.filter_map(|effect| { - if effect.is_atom() { - None - } else { - let Ok(effect_cell) = effect.as_cell() else { - error!("Expected effect to be a cell"); - return None; - }; - let hed = effect_cell.head(); - if hed.eq_bytes("mine-result") { - Some(effect_cell.tail()) - } else { + enum MiningOutcome { + Retry { nonce: Option }, + Mined { poke: NounSlab, nonce: NounSlab }, + } + + let outcome = (|| -> Result { + let space = slab.noun_space(); + match HoonList::try_from(*result, &space) { + Err(_) => Ok(MiningOutcome::Retry { nonce: None }), + Ok(effects) => { + let mining_result = effects + .filter_map(|effect| { + if effect.is_atom() { None + } else { + let Ok(effect_cell) = + effect.in_space(&space).as_cell() + else { + error!("Expected effect to be a cell"); + return None; + }; + let hed = effect_cell.head(); + if hed.eq_bytes("mine-result") { + Some(effect_cell.tail().noun()) + } else { + None + } } - } - }).next(); - match mining_result { - None => { - start_mining_attempt(serf, mining_data.lock().await, &mut mining_attempts, None, id).await; - }, - Some(mine_result) => { - let Ok([res, tail]) = mine_result.uncell() else { - return Err(NockAppError::OtherError(String::from("Expected two elements in mining result"))); - }; - if unsafe { res.raw_equals(&D(0)) } { - // success - // poke main kernel with mined block and start a new attempt - info!("Found block! thread={id}"); - let Ok([hash, poke]) = tail.uncell() else { - error!("Expected two elements in tail"); - return Err(NockAppError::OtherError(String::from("Expected two elements in tail"))); + }) + .next(); + match mining_result { + None => Ok(MiningOutcome::Retry { nonce: None }), + Some(mine_result) => { + let Ok([res, tail]) = + mine_result.uncell(&space) + else { + return Err(NockAppError::OtherError( + String::from( + "Expected two elements in mining result", + ), + )); }; - let mut poke_slab = NounSlab::new(); - poke_slab.copy_into(poke); - handle.poke(MiningWire::Mined.to_wire(), poke_slab).await.expect("Could not poke nockchain with mined PoW"); - - // launch new attempt - let mut nonce_slab = NounSlab::new(); - nonce_slab.copy_into(hash); - start_mining_attempt(serf, mining_data.lock().await, &mut mining_attempts, Some(nonce_slab), id).await; - } else { - // failure - // launch new attempt, using hash as new nonce - // nonce is tail - debug!("didn't find block, starting new attempt. thread={id}"); - let mut nonce_slab = NounSlab::new(); - nonce_slab.copy_into(tail); - start_mining_attempt(serf, mining_data.lock().await, &mut mining_attempts, Some(nonce_slab), id).await; + if unsafe { res.raw_equals(&D(0)) } { + // success + // poke main kernel with mined block and start a new attempt + info!("Found block! thread={id}"); + let Ok([hash, poke]) = tail.uncell(&space) else { + error!("Expected two elements in tail"); + return Err(NockAppError::OtherError( + String::from( + "Expected two elements in tail", + ), + )); + }; + let mut poke_slab = NounSlab::new(); + poke_slab.copy_into(poke, &space); + + let mut nonce_slab = NounSlab::new(); + nonce_slab.copy_into(hash, &space); + Ok(MiningOutcome::Mined { + poke: poke_slab, + nonce: nonce_slab, + }) + } else { + // failure + // launch new attempt, using hash as new nonce + // nonce is tail + debug!( + "didn't find block, starting new attempt. thread={id}" + ); + let mut nonce_slab = NounSlab::new(); + nonce_slab.copy_into(tail, &space); + Ok(MiningOutcome::Retry { + nonce: Some(nonce_slab), + }) + } } } } } + })()?; + + match outcome { + MiningOutcome::Retry { nonce } => { + start_mining_attempt( + serf, + mining_data.lock().await, + &mut mining_attempts, + nonce, + id, + ) + .await; + } + MiningOutcome::Mined { poke, nonce } => { + handle + .poke(MiningWire::Mined.to_wire(), poke) + .await + .expect("Could not poke nockchain with mined PoW"); + start_mining_attempt( + serf, + mining_data.lock().await, + &mut mining_attempts, + Some(nonce), + id, + ) + .await; + } } } @@ -249,29 +297,47 @@ pub fn create_mining_driver( continue; }; - if effect_cell.head().eq_bytes("mine") { - let (version_slab, header_slab, target_slab, pow_len) = { - let [version, commit, target, pow_len_noun] = effect_cell.tail().uncell().expect( - "Expected three elements in %mine effect", - ); - let mut version_slab = NounSlab::new(); - version_slab.copy_into(version); - let mut header_slab = NounSlab::new(); - header_slab.copy_into(commit); - let mut target_slab = NounSlab::new(); - target_slab.copy_into(target); - let pow_len = - pow_len_noun + let candidate = { + let space = effect.noun_space(); + let effect_cell = effect_cell.in_space(&space); + if effect_cell.head().eq_bytes("mine") { + let (version_slab, header_slab, target_slab, pow_len) = { + let [version, commit, target, pow_len_noun] = effect_cell + .tail() + .noun() + .uncell(&space) + .expect("Expected three elements in %mine effect"); + let mut version_slab = NounSlab::new(); + version_slab.copy_into(version, &space); + let mut header_slab = NounSlab::new(); + header_slab.copy_into(commit, &space); + let mut target_slab = NounSlab::new(); + target_slab.copy_into(target, &space); + let pow_len = pow_len_noun + .in_space(&space) .as_atom() .expect("Expected pow-len to be an atom") .as_u64() .expect("Expected pow-len to be a u64"); - (version_slab, header_slab, target_slab, pow_len) - }; - debug!("received new candidate block header: {:?}", - tip5_hash_to_base58(*unsafe { header_slab.root() }) - .expect("Failed to convert header to Base58") - ); + (version_slab, header_slab, target_slab, pow_len) + }; + Some((version_slab, header_slab, target_slab, pow_len)) + } else { + None + } + }; + if let Some((version_slab, header_slab, target_slab, pow_len)) = candidate { + { + let header_space = header_slab.noun_space(); + debug!( + "received new candidate block header: {:?}", + tip5_hash_to_base58( + *unsafe { header_slab.root() }, + &header_space + ) + .expect("Failed to convert header to Base58") + ); + } *(mining_data.lock().await) = Some(MiningData { block_header: header_slab, version: version_slab, @@ -289,6 +355,7 @@ pub fn create_mining_driver( None, hot_state.clone(), NOCK_STACK_SIZE_TINY, + None, test_jets.clone(), Default::default(), ) @@ -318,10 +385,14 @@ pub fn create_mining_driver( fn create_poke(mining_data: &MiningData, nonce: &NounSlab) -> NounSlab { let mut slab = NounSlab::new(); - let header = slab.copy_into(unsafe { *(mining_data.block_header.root()) }); - let version = slab.copy_into(unsafe { *(mining_data.version.root()) }); - let target = slab.copy_into(unsafe { *(mining_data.target.root()) }); - let nonce = slab.copy_into(unsafe { *(nonce.root()) }); + let header_space = mining_data.block_header.noun_space(); + let version_space = mining_data.version.noun_space(); + let target_space = mining_data.target.noun_space(); + let nonce_space = nonce.noun_space(); + let header = slab.copy_into(unsafe { *(mining_data.block_header.root()) }, &header_space); + let version = slab.copy_into(unsafe { *(mining_data.version.root()) }, &version_space); + let target = slab.copy_into(unsafe { *(mining_data.target.root()) }, &target_space); + let nonce = slab.copy_into(unsafe { *(nonce.root()) }, &nonce_space); let poke_noun = T( &mut slab, &[version, header, nonce, target, D(mining_data.pow_len)], @@ -433,12 +504,18 @@ async fn start_mining_attempt( let mining_data_ref = mining_data .as_ref() .expect("Mining data should already be initialized"); + let header_space = mining_data_ref.block_header.noun_space(); + let nonce_space = nonce.noun_space(); debug!( "starting mining attempt on thread {:?} on header {:?}with nonce: {:?}", id, - tip5_hash_to_base58(*unsafe { mining_data_ref.block_header.root() }) - .expect("Failed to convert block header to Base58"), - tip5_hash_to_base58(*unsafe { nonce.root() }).expect("Failed to convert nonce to Base58"), + tip5_hash_to_base58( + *unsafe { mining_data_ref.block_header.root() }, + &header_space + ) + .expect("Failed to convert block header to Base58"), + tip5_hash_to_base58(*unsafe { nonce.root() }, &nonce_space) + .expect("Failed to convert nonce to Base58"), ); let poke_slab = create_poke(mining_data_ref, &nonce); mining_attempts.spawn(async move { diff --git a/crates/nockchain/src/setup.rs b/crates/nockchain/src/setup.rs index f0d253753..7cc13e209 100644 --- a/crates/nockchain/src/setup.rs +++ b/crates/nockchain/src/setup.rs @@ -1,23 +1,14 @@ use std::error::Error; -use std::time::Duration; -use ibig::UBig; -use nockapp::noun::slab::Jammer; -use nockapp::noun::IntoSlab; +use nockapp::noun::slab::{Jammer, NounSlab}; use nockapp::utils::make_tas; use nockapp::wire::Wire; use nockapp::{AtomExt, Bytes, NockApp, NockAppError, ToBytes}; -use nockvm::noun::{Atom, Noun, NounAllocator, D, T}; +pub use nockchain_types::{fakenet_blockchain_constants, BlockchainConstants, Seconds}; +use nockvm::noun::{Atom, D, T}; use nockvm_macros::tas; use noun_serde::NounEncode; -use tracing::info; -use crate::NounSlab; - -#[cfg(feature = "bazel_build")] -pub static FAKENET_GENESIS_BLOCK: &[u8] = include_bytes!(env!("FAKENET_GENESIS_PATH")); - -#[cfg(not(feature = "bazel_build"))] pub static FAKENET_GENESIS_BLOCK: &[u8] = include_bytes!(concat!( env!("CARGO_MANIFEST_DIR"), "/jams/fakenet-genesis-pow-2-bex-1.jam" @@ -29,20 +20,11 @@ pub const FAKENET_GENESIS_MESSAGE: &str = "3WNP3WtcQJYtP5PCvFHDQVEeiZEznsULEY5Lc pub const REALNET_GENESIS_MESSAGE: &str = "2c8Ltbg44dPkEGcNPupcVAtDgD87753M9pG2fg8yC2mTEqg5qAFvvbT"; pub enum SetupCommand { - PokeFakenetConstants(BlockchainConstants), + PokeFakenetConstants(Box), PokeSetGenesisSeal(String), PokeSetBtcData, } -pub fn fakenet_blockchain_constants(pow_len: u64, target_bex: u64) -> BlockchainConstants { - BlockchainConstants::new() - .with_update_candidate_timestamp_interval(Seconds(5 * 60)) - .with_pow_len(pow_len) - .with_genesis_target_atom_bex(target_bex as u128) - .with_first_month_coinbase_min(0) - .with_coinbase_timelock_min(1) -} - pub async fn poke( nockapp: &mut NockApp, command: SetupCommand, @@ -96,7 +78,6 @@ pub fn heard_fake_genesis_block( ) -> Result { let mut poke_slab = NounSlab::new(); let tag = make_tas(&mut poke_slab, "heard-block").as_noun(); - // load the block bytes let block_bytes = if let Some(data) = fake_genesis_data { Bytes::from(data) } else { @@ -107,405 +88,3 @@ pub fn heard_fake_genesis_block( poke_slab.set_root(poke_noun); Ok(poke_slab) } - -#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, NounEncode)] -pub struct Seconds(pub u64); - -impl Seconds { - pub fn new(seconds: u64) -> Self { - Self(seconds) - } - - pub fn as_u64(&self) -> u64 { - self.0 - } - - pub fn to_duration(&self) -> Duration { - Duration::from_secs(self.0) - } -} - -impl From for Seconds { - fn from(seconds: u64) -> Self { - Self(seconds) - } -} - -impl TryFrom for Seconds { - type Error = &'static str; - - fn try_from(duration: Duration) -> Result { - if duration.subsec_nanos() != 0 { - return Err("Duration must be whole seconds only"); - } - Ok(Self(duration.as_secs())) - } -} - -impl From for Duration { - fn from(seconds: Seconds) -> Self { - Duration::from_secs(seconds.0) - } -} - -#[derive(Debug, Clone, Copy, PartialEq, Eq, NounEncode)] -pub struct NoteDataConstraints { - pub max_size: u64, - pub min_fee: u64, -} - -pub struct BlockchainConstants { - // In bits - pub max_block_size: u64, - pub blocks_per_epoch: u64, - // In seconds - pub target_epoch_duration: Seconds, - // in seconds - pub update_candidate_timestamp_interval: Seconds, - // in seconds - pub max_future_timestamp: Seconds, - pub min_past_blocks: u64, - pub genesis_target_atom: UBig, - pub max_target_atom: UBig, - pub check_pow_flag: bool, - pub coinbase_timelock_min: u64, - pub pow_len: u64, - pub max_coinbase_split: u64, - pub first_month_coinbase_min: u64, - pub v1_phase: u64, - pub bythos_phase: u64, - pub note_data: NoteDataConstraints, - pub base_fee: u64, - pub input_fee_divisor: u64, -} - -impl BlockchainConstants { - pub const DEFAULT_MAX_BLOCK_SIZE: u64 = 8000000; - pub const DEFAULT_BLOCKS_PER_EPOCH: u64 = 2016; - pub const DEFAULT_TARGET_EPOCH_DURATION: u64 = 1209600; - pub const DEFAULT_UPDATE_CANDIDATE_TIMESTAMP_INTERVAL_SECS: u64 = 300; - pub const DEFAULT_MAX_FUTURE_TIMESTAMP: u64 = 7200; - // DEFAULT_GENESIS_TARGET_ATOM = MAX_TIP5_ATOM / (2 << 14) - pub const DEFAULT_GENESIS_TARGET_ATOM: &str = - "0x3ffffffec0000003bffffff88000000b3ffffff34000000b3ffffff880000003bffffffec0000"; - // Largest tip5 hash in base-p form - pub const DEFAULT_MAX_TIP5_ATOM: &str = - "0xfffffffb0000000effffffe20000002cffffffcd0000002cffffffe20000000efffffffb00000000"; - pub const DEFAULT_MIN_PAST_BLOCKS: u64 = 11; - pub const DEFAULT_CHECK_POW_FLAG: bool = true; - pub const DEFAULT_COINBASE_TIMELOCK_MIN: u64 = 100; - pub const DEFAULT_POW_LEN: u64 = 64; - pub const DEFAULT_MAX_COINBASE_SPLIT: u64 = 2; - pub const DEFAULT_FIRST_MONTH_COINBASE_MIN: u64 = 4383; - // TODO: update these for FINAL RELEASE - pub const DEFAULT_V1_PHASE: u64 = 40_000; - pub const DEFAULT_BYTHOS_PHASE: u64 = 54_000; - pub const DEFAULT_NOTE_DATA_MAX_SIZE: u64 = 2_048; - pub const DEFAULT_NOTE_DATA_MIN_FEE: u64 = 256; - - // Fakenet-only fee constants. These are poked to the kernel only when --fakenet is set. - // Mainnet uses the Hoon bunt defaults in tx-engine-1.hoon (base-fee=16384, divisor=4). - pub const DEFAULT_BASE_FEE: u64 = 128; - pub const DEFAULT_INPUT_FEE_DIVISOR: u64 = 4; - - pub fn new() -> Self { - let max_tip5_atom = UBig::from_str_with_radix_prefix(Self::DEFAULT_MAX_TIP5_ATOM) - .expect("Failed to parse max tip5 atom"); - let genesis_target_atom = - UBig::from_str_with_radix_prefix(Self::DEFAULT_GENESIS_TARGET_ATOM) - .expect("Failed to parse genesis target atom"); - BlockchainConstants { - max_block_size: Self::DEFAULT_MAX_BLOCK_SIZE, - blocks_per_epoch: Self::DEFAULT_BLOCKS_PER_EPOCH, - target_epoch_duration: Self::DEFAULT_TARGET_EPOCH_DURATION.into(), - update_candidate_timestamp_interval: - Self::DEFAULT_UPDATE_CANDIDATE_TIMESTAMP_INTERVAL_SECS.into(), - max_future_timestamp: Self::DEFAULT_MAX_FUTURE_TIMESTAMP.into(), - min_past_blocks: Self::DEFAULT_MIN_PAST_BLOCKS, - genesis_target_atom: genesis_target_atom, - max_target_atom: max_tip5_atom, - check_pow_flag: Self::DEFAULT_CHECK_POW_FLAG, - coinbase_timelock_min: Self::DEFAULT_COINBASE_TIMELOCK_MIN, - pow_len: Self::DEFAULT_POW_LEN, - max_coinbase_split: Self::DEFAULT_MAX_COINBASE_SPLIT, - first_month_coinbase_min: Self::DEFAULT_FIRST_MONTH_COINBASE_MIN, - v1_phase: Self::DEFAULT_V1_PHASE, - bythos_phase: Self::DEFAULT_BYTHOS_PHASE, - note_data: NoteDataConstraints { - max_size: Self::DEFAULT_NOTE_DATA_MAX_SIZE, - min_fee: Self::DEFAULT_NOTE_DATA_MIN_FEE, - }, - base_fee: Self::DEFAULT_BASE_FEE, - input_fee_divisor: Self::DEFAULT_INPUT_FEE_DIVISOR, - } - } - - pub fn with_genesis_target_atom_bex(mut self, bex: u128) -> Self { - let difficulty = UBig::from((1 << bex) as u128); - self.genesis_target_atom = self.max_target_atom.clone() / difficulty; - info!("Genesis target atom set to {}", self.genesis_target_atom); - self - } - - pub fn with_update_candidate_timestamp_interval(mut self, interval_secs: Seconds) -> Self { - self.update_candidate_timestamp_interval = interval_secs; - self - } - - pub fn with_pow_len(mut self, pow_len: u64) -> Self { - self.pow_len = pow_len; - self - } - - pub fn with_v1_phase(mut self, v1_phase: u64) -> Self { - self.v1_phase = v1_phase; - self - } - - pub fn with_bythos_phase(mut self, bythos_phase: u64) -> Self { - self.bythos_phase = bythos_phase; - self - } - - pub fn with_first_month_coinbase_min(mut self, coinbase_min: u64) -> Self { - self.first_month_coinbase_min = coinbase_min; - self - } - - pub fn with_coinbase_timelock_min(mut self, coinbase_max: u64) -> Self { - self.coinbase_timelock_min = coinbase_max; - self - } - - fn to_blockchain_constants_v0_fields(&self, allocator: &mut A) -> Vec { - let max_block_size = Atom::new(allocator, self.max_block_size).as_noun(); - let blocks_per_epoch = Atom::new(allocator, self.blocks_per_epoch).as_noun(); - let target_epoch_duration = self.target_epoch_duration.to_noun(allocator); - // @dr atoms are seconds in the top 64 bits (kernel default uses ~m5). - let update_candidate_timestamp_interval_atoms = - UBig::from(self.update_candidate_timestamp_interval.0) << 64; - let update_candidate_timestamp_interval = - Atom::from_ubig(allocator, &update_candidate_timestamp_interval_atoms).as_noun(); - let max_future_timestamp = self.max_future_timestamp.to_noun(allocator); - let min_past_blocks = Atom::new(allocator, self.min_past_blocks).as_noun(); - let genesis_target_atom = Atom::from_ubig(allocator, &self.genesis_target_atom).as_noun(); - let max_target_atom = Atom::from_ubig(allocator, &self.max_target_atom).as_noun(); - let check_pow_flag = self.check_pow_flag.to_noun(allocator); - let coinbase_timelock_min = Atom::new(allocator, self.coinbase_timelock_min).as_noun(); - let pow_len = Atom::new(allocator, self.pow_len).as_noun(); - let max_coinbase_split = Atom::new(allocator, self.max_coinbase_split).as_noun(); - let first_month_coinbase_min = - Atom::new(allocator, self.first_month_coinbase_min).as_noun(); - - vec![ - max_block_size, blocks_per_epoch, target_epoch_duration, - update_candidate_timestamp_interval, max_future_timestamp, min_past_blocks, - genesis_target_atom, max_target_atom, check_pow_flag, coinbase_timelock_min, pow_len, - max_coinbase_split, first_month_coinbase_min, - ] - } -} - -impl NounEncode for BlockchainConstants { - fn to_noun(&self, allocator: &mut A) -> Noun { - let v1_phase = Atom::new(allocator, self.v1_phase).as_noun(); - let bythos_phase = Atom::new(allocator, self.bythos_phase).as_noun(); - let note_data = self.note_data.to_noun(allocator); - let base_fee = Atom::new(allocator, self.base_fee).as_noun(); - let input_fee_divisor = Atom::new(allocator, self.input_fee_divisor).as_noun(); - let v0_fields = self.to_blockchain_constants_v0_fields(allocator); - let v0_constants = T(allocator, &v0_fields); - - T( - allocator, - &[v1_phase, bythos_phase, note_data, base_fee, input_fee_divisor, v0_constants], - ) - } -} - -impl IntoSlab for BlockchainConstants { - fn into_slab(self) -> NounSlab { - let mut slab = NounSlab::new(); - let noun = self.to_noun(&mut slab); - slab.set_root(noun); - slab - } -} - -#[cfg(test)] -mod tests { - use ibig::UBig; - - use super::*; - - fn tuple_len(noun: Noun) -> usize { - let mut len = 0; - let mut cur = noun; - loop { - if let Ok(cell) = cur.as_cell() { - len += 1; - cur = cell.tail(); - } else { - len += 1; - break; - } - } - len - } - - #[test] - fn fakenet_blockchain_constants_are_valid() { - let constants = BlockchainConstants::new(); - - assert_eq!( - constants.max_block_size, 8_000_000, - "max-block-size mismatch" - ); - assert_eq!( - constants.blocks_per_epoch, 2_016, - "blocks-per-epoch mismatch" - ); - assert_eq!( - constants.target_epoch_duration, - Seconds::new(14 * 24 * 60 * 60), - "target-epoch-duration mismatch", - ); - assert_eq!( - constants.update_candidate_timestamp_interval, - Seconds::new(5 * 60), - "update-candidate-interval mismatch", - ); - assert_eq!( - constants.max_future_timestamp, - Seconds::new(60 * 120), - "max-future-timestamp mismatch", - ); - assert_eq!(constants.min_past_blocks, 11, "min-past-blocks mismatch"); - - let max_tip5_atom = - UBig::from_str_with_radix_prefix(BlockchainConstants::DEFAULT_MAX_TIP5_ATOM) - .expect("parse max tip5 atom"); - assert_eq!( - constants.max_target_atom, max_tip5_atom, - "max-target-atom mismatch", - ); - - let expected_genesis_target = &max_tip5_atom / (UBig::from(1u64) << 14); - assert_eq!( - constants.genesis_target_atom, expected_genesis_target, - "genesis-target-atom mismatch", - ); - - assert!(constants.check_pow_flag, "check-pow-flag mismatch"); - assert_eq!( - constants.coinbase_timelock_min, 100, - "coinbase-timelock-min mismatch" - ); - assert_eq!(constants.pow_len, 64, "pow-len mismatch"); - assert_eq!( - constants.max_coinbase_split, 2, - "max-coinbase-split mismatch" - ); - assert_eq!( - constants.first_month_coinbase_min, 4_383, - "first-month-coinbase-min mismatch", - ); - assert_eq!(constants.v1_phase, 40_000, "v1-phase mismatch"); - assert_eq!(constants.bythos_phase, 54_000, "bythos-phase mismatch",); - assert_eq!( - constants.note_data, - NoteDataConstraints { - max_size: 2_048, - min_fee: 256, - }, - "note-data mismatch", - ); - assert_eq!(constants.base_fee, 128, "base-fee mismatch"); - assert_eq!(constants.input_fee_divisor, 4, "input-fee-divisor mismatch"); - } - - #[test] - fn with_v1_phase_overrides_default() { - let constants = BlockchainConstants::new().with_v1_phase(54_321); - - assert_eq!(constants.v1_phase, 54_321); - } - - #[test] - fn blockchain_constants_encode_in_new_v1_wrapper() { - let slab = BlockchainConstants::new().into_slab(); - let root = unsafe { *slab.root() }; - - let outer = root.as_cell().expect("outer tuple"); - let v1_phase_atom = outer.head().as_atom().expect("v1-phase should be atom"); - assert_eq!( - v1_phase_atom.as_u64().expect("v1-phase as u64"), - BlockchainConstants::DEFAULT_V1_PHASE - ); - - let rest = outer.tail().as_cell().expect("rest tuple"); - let bythos_phase_atom = rest.head().as_atom().expect("bythos-phase should be atom"); - assert_eq!( - bythos_phase_atom.as_u64().expect("bythos-phase as u64"), - BlockchainConstants::DEFAULT_BYTHOS_PHASE - ); - - let rest = rest.tail().as_cell().expect("note-data and rest tuple"); - let note_data = rest.head().as_cell().expect("note-data tuple"); - let note_data_max_size = note_data - .head() - .as_atom() - .expect("note-data max-size atom") - .as_u64() - .expect("note-data max-size as u64"); - let note_data_min_fee = note_data - .tail() - .as_atom() - .expect("note-data min-fee atom") - .as_u64() - .expect("note-data min-fee as u64"); - assert_eq!( - note_data_max_size, - BlockchainConstants::DEFAULT_NOTE_DATA_MAX_SIZE - ); - assert_eq!( - note_data_min_fee, - BlockchainConstants::DEFAULT_NOTE_DATA_MIN_FEE - ); - - let base_fee_and_rest = rest.tail().as_cell().expect("base-fee and rest tuple"); - let base_fee_atom = base_fee_and_rest.head().as_atom().expect("base-fee atom"); - assert_eq!( - base_fee_atom.as_u64().expect("base-fee as u64"), - BlockchainConstants::DEFAULT_BASE_FEE - ); - - let input_fee_divisor_and_rest = base_fee_and_rest - .tail() - .as_cell() - .expect("input-fee-divisor and rest tuple"); - let input_fee_divisor_atom = input_fee_divisor_and_rest - .head() - .as_atom() - .expect("input-fee-divisor atom"); - assert_eq!( - input_fee_divisor_atom - .as_u64() - .expect("input-fee-divisor as u64"), - BlockchainConstants::DEFAULT_INPUT_FEE_DIVISOR - ); - - let v0_constants = input_fee_divisor_and_rest.tail(); - assert_eq!( - tuple_len(v0_constants), - 13, - "v0 constants should be a 13-tuple" - ); - let v0_cell = v0_constants.as_cell().expect("v0 constants tuple"); - let max_block_size_atom = v0_cell.head().as_atom().expect("max-block-size atom"); - assert_eq!( - max_block_size_atom.as_u64().expect("max-block-size as u64"), - BlockchainConstants::DEFAULT_MAX_BLOCK_SIZE - ); - } -} diff --git a/crates/nockchain/src/traces.rs b/crates/nockchain/src/traces.rs new file mode 100644 index 000000000..d21ccfecf --- /dev/null +++ b/crates/nockchain/src/traces.rs @@ -0,0 +1,139 @@ +//! Translates kernel `%span %new-heaviest-chain` and `%new-heaviest-miner` +//! effects (emitted by inner.hoon's `accept-block` and miner candidate paths) +//! into stdout-visible structured events. Downstream observers (cluster +//! tests, monitoring tools) parse these `new_heaviest_chain` log lines +//! (matching `block_height=`, `heaviest_block_digest=`, `block_target=`) to +//! follow per-node chain state from stdout. + +use std::collections::HashMap; + +use nockapp::driver::{make_driver, IODriverFn}; +use nockchain_math::structs::HoonList; +use nockvm::noun::{Noun, NounAllocator}; +use nockvm_macros::tas; +use tracing::{debug, error, field, info, span, Level}; + +const NEW_HEAVIEST_CHAIN: &str = "new_heaviest_chain"; +const NEW_HEAVIEST_MINER: &str = "new_heaviest_miner"; + +pub fn traces_driver() -> IODriverFn { + make_driver(|handle| async move { + loop { + match handle.next_effect().await { + Ok(effect) => { + let space = effect.noun_space(); + let effect_noun = unsafe { *effect.root() }; + let Ok(effect_cell) = effect_noun.in_space(&space).as_cell() else { + continue; + }; + + if effect_cell.head().as_atom().and_then(|atom| atom.as_u64()) + == Ok(tas!(b"log")) + { + let log_msg = effect_cell.tail().as_atom()?.into_string()?; + info!(log_msg); + } else if effect_cell.head().as_atom().and_then(|atom| atom.as_u64()) + == Ok(tas!(b"span")) + { + let span_eff = effect_cell.tail(); + let name = span_eff.slot(2)?.as_atom()?.into_string()?; + + let raw_fields: Vec = + HoonList::try_from(span_eff.slot(3)?.noun(), &space)? + .into_iter() + .collect(); + + let mut str_fields: HashMap = HashMap::new(); + let mut num_fields: HashMap = HashMap::new(); + let mut parse_ok = true; + for n in raw_fields { + let cell = n.in_space(&space).as_cell()?; + let key = cell.head().as_atom()?.into_string()?; + let raw_val = cell.tail().as_cell()?; + let typ = raw_val.head().as_atom()?.into_string()?; + let val_atom = raw_val.tail().as_atom()?; + if typ == "n" { + num_fields.insert(key, val_atom.as_u64()?); + } else if typ == "s" { + str_fields.insert(key, val_atom.into_string()?); + } else { + error!("Error traces driver: unrecognized field type"); + parse_ok = false; + break; + } + } + if !parse_ok { + continue; + } + + let height = num_fields.get("block_height").copied().unwrap_or(0); + let digest = str_fields + .get("heaviest_block_digest") + .cloned() + .unwrap_or_default(); + let target = str_fields.get("block_target").cloned().unwrap_or_default(); + + match name.as_str() { + "new-heaviest-chain" => { + let span = span!( + Level::INFO, + NEW_HEAVIEST_CHAIN, + block_height = field::Empty, + heaviest_block_digest = field::Empty, + block_target = field::Empty + ); + span.record("block_height", height); + span.record("heaviest_block_digest", digest.as_str()); + span.record("block_target", target.as_str()); + let _g = span.enter(); + info!( + block_height = height, + heaviest_block_digest = digest.as_str(), + block_target = target.as_str(), + "new_heaviest_chain" + ); + } + "new-heaviest-miner" => { + let span = span!( + Level::INFO, + NEW_HEAVIEST_MINER, + block_height = field::Empty, + heaviest_block_digest = field::Empty + ); + span.record("block_height", height); + span.record("heaviest_block_digest", digest.as_str()); + let _g = span.enter(); + info!( + block_height = height, + heaviest_block_digest = digest.as_str(), + "new_heaviest_miner" + ); + } + "orphaned-block" => { + debug!( + block_height = height, + block_digest = digest.as_str(), + "orphaned_block" + ); + } + "chain-reorg" => { + debug!( + block_height = height, + new_tip_digest = digest.as_str(), + "chain_reorg" + ); + } + _ => { + debug!(span_name = name.as_str(), "traces driver: unknown span"); + } + }; + } + } + Err(e) => { + error!("Error in traces driver: {:?}", e); + continue; + } + } + } + }) +} diff --git a/crates/nockchain/tests/h_zoon_checkpoint_migration.rs b/crates/nockchain/tests/h_zoon_checkpoint_migration.rs new file mode 100644 index 000000000..11aed0ea1 --- /dev/null +++ b/crates/nockchain/tests/h_zoon_checkpoint_migration.rs @@ -0,0 +1,223 @@ +use std::error::Error; +use std::path::{Path, PathBuf}; + +use chaff::Chaff; +use nockapp::export::ExportedState; +use nockapp::kernel::boot::{self, NockStackSize, SetupResult}; +use zkvm_jetpack::hot::produce_prover_hot_state; + +const CHECKPOINT_ENV: &str = "NOCKCHAIN_H_ZOON_CHECKPOINT"; +const MIN_EVENT_ENV: &str = "NOCKCHAIN_H_ZOON_CHECKPOINT_MIN_EVENT"; +const TEST_JETS_ENV: &str = "NOCKCHAIN_H_ZOON_TEST_JETS"; +const DEFAULT_MIN_EVENT: u64 = 40_000; + +const H_ZOON_TEST_JETS: &str = concat!( + "k.138/one/two/tri/qua/pen/zeke/ext-field/misc-lib/proof-lib/utils/fri/table-lib/", + "stark-core/fock-core/pow/stark-engine/h-zoon/gor-hip,", + "k.138/one/two/tri/qua/pen/zeke/ext-field/misc-lib/proof-lib/utils/fri/table-lib/", + "stark-core/fock-core/pow/stark-engine/h-zoon/mor-hip,", + "k.138/one/two/tri/qua/pen/zeke/ext-field/misc-lib/proof-lib/utils/fri/table-lib/", + "stark-core/fock-core/pow/stark-engine/h-zoon/zh-molt,", + "k.138/one/two/tri/qua/pen/zeke/ext-field/misc-lib/proof-lib/utils/fri/table-lib/", + "stark-core/fock-core/pow/stark-engine/h-zoon/zh-silt,", + "k.138/one/two/tri/qua/pen/zeke/ext-field/misc-lib/proof-lib/utils/fri/table-lib/", + "stark-core/fock-core/pow/stark-engine/h-zoon/zh-milt,", + "k.138/one/two/tri/qua/pen/zeke/ext-field/misc-lib/proof-lib/utils/fri/table-lib/", + "stark-core/fock-core/pow/stark-engine/h-zoon/zh-balmilt,", + "k.138/one/two/tri/qua/pen/zeke/ext-field/misc-lib/proof-lib/utils/fri/table-lib/", + "stark-core/fock-core/pow/stark-engine/h-zoon/zh-jult,", + // 22 non-gate h-by / h-in container arm jets (open jetpack). + "k.138/one/two/tri/qua/pen/zeke/ext-field/misc-lib/proof-lib/utils/fri/table-lib/", + "stark-core/fock-core/pow/stark-engine/h-zoon/h-by/get,", + "k.138/one/two/tri/qua/pen/zeke/ext-field/misc-lib/proof-lib/utils/fri/table-lib/", + "stark-core/fock-core/pow/stark-engine/h-zoon/h-by/got,", + "k.138/one/two/tri/qua/pen/zeke/ext-field/misc-lib/proof-lib/utils/fri/table-lib/", + "stark-core/fock-core/pow/stark-engine/h-zoon/h-by/gut,", + "k.138/one/two/tri/qua/pen/zeke/ext-field/misc-lib/proof-lib/utils/fri/table-lib/", + "stark-core/fock-core/pow/stark-engine/h-zoon/h-by/has,", + "k.138/one/two/tri/qua/pen/zeke/ext-field/misc-lib/proof-lib/utils/fri/table-lib/", + "stark-core/fock-core/pow/stark-engine/h-zoon/h-by/put,", + "k.138/one/two/tri/qua/pen/zeke/ext-field/misc-lib/proof-lib/utils/fri/table-lib/", + "stark-core/fock-core/pow/stark-engine/h-zoon/h-by/del,", + "k.138/one/two/tri/qua/pen/zeke/ext-field/misc-lib/proof-lib/utils/fri/table-lib/", + "stark-core/fock-core/pow/stark-engine/h-zoon/h-by/mar,", + "k.138/one/two/tri/qua/pen/zeke/ext-field/misc-lib/proof-lib/utils/fri/table-lib/", + "stark-core/fock-core/pow/stark-engine/h-zoon/h-by/gas,", + "k.138/one/two/tri/qua/pen/zeke/ext-field/misc-lib/proof-lib/utils/fri/table-lib/", + "stark-core/fock-core/pow/stark-engine/h-zoon/h-by/uni,", + "k.138/one/two/tri/qua/pen/zeke/ext-field/misc-lib/proof-lib/utils/fri/table-lib/", + "stark-core/fock-core/pow/stark-engine/h-zoon/h-by/int,", + "k.138/one/two/tri/qua/pen/zeke/ext-field/misc-lib/proof-lib/utils/fri/table-lib/", + "stark-core/fock-core/pow/stark-engine/h-zoon/h-by/dif,", + "k.138/one/two/tri/qua/pen/zeke/ext-field/misc-lib/proof-lib/utils/fri/table-lib/", + "stark-core/fock-core/pow/stark-engine/h-zoon/h-by/bif,", + "k.138/one/two/tri/qua/pen/zeke/ext-field/misc-lib/proof-lib/utils/fri/table-lib/", + "stark-core/fock-core/pow/stark-engine/h-zoon/h-by/dig,", + "k.138/one/two/tri/qua/pen/zeke/ext-field/misc-lib/proof-lib/utils/fri/table-lib/", + "stark-core/fock-core/pow/stark-engine/h-zoon/h-in/has,", + "k.138/one/two/tri/qua/pen/zeke/ext-field/misc-lib/proof-lib/utils/fri/table-lib/", + "stark-core/fock-core/pow/stark-engine/h-zoon/h-in/put,", + "k.138/one/two/tri/qua/pen/zeke/ext-field/misc-lib/proof-lib/utils/fri/table-lib/", + "stark-core/fock-core/pow/stark-engine/h-zoon/h-in/del,", + "k.138/one/two/tri/qua/pen/zeke/ext-field/misc-lib/proof-lib/utils/fri/table-lib/", + "stark-core/fock-core/pow/stark-engine/h-zoon/h-in/gas,", + "k.138/one/two/tri/qua/pen/zeke/ext-field/misc-lib/proof-lib/utils/fri/table-lib/", + "stark-core/fock-core/pow/stark-engine/h-zoon/h-in/uni,", + "k.138/one/two/tri/qua/pen/zeke/ext-field/misc-lib/proof-lib/utils/fri/table-lib/", + "stark-core/fock-core/pow/stark-engine/h-zoon/h-in/int,", + "k.138/one/two/tri/qua/pen/zeke/ext-field/misc-lib/proof-lib/utils/fri/table-lib/", + "stark-core/fock-core/pow/stark-engine/h-zoon/h-in/dif,", + "k.138/one/two/tri/qua/pen/zeke/ext-field/misc-lib/proof-lib/utils/fri/table-lib/", + "stark-core/fock-core/pow/stark-engine/h-zoon/h-in/bif,", + "k.138/one/two/tri/qua/pen/zeke/ext-field/misc-lib/proof-lib/utils/fri/table-lib/", + "stark-core/fock-core/pow/stark-engine/h-zoon/h-in/dig", +); + +struct EnvVarGuard { + key: &'static str, + previous: Option, +} + +impl EnvVarGuard { + fn set(key: &'static str, value: &str) -> Self { + let previous = std::env::var(key).ok(); + std::env::set_var(key, value); + Self { key, previous } + } +} + +impl Drop for EnvVarGuard { + fn drop(&mut self) { + match &self.previous { + Some(value) => std::env::set_var(self.key, value), + None => std::env::remove_var(self.key), + } + } +} + +#[tokio::test(flavor = "current_thread")] +#[ignore = "requires NOCKCHAIN_H_ZOON_CHECKPOINT pointing at a large nockchain chkjam"] +async fn state_8_migration_real_checkpoint_exports_migrated_state() -> Result<(), Box> { + // This is a gated checkpoint oracle for the state-8 h-zoon migration. + // Leaving NOCKCHAIN_H_ZOON_TEST_JETS unset exercises the production jet path. + // Set it to `default` to run the expensive jet/Hoon fallback list, or to a + // comma-separated NOCK_TEST_JETS value for focused local differential checks. + let checkpoint = checkpoint_path()?; + let checkpoint = checkpoint.canonicalize().map_err(|err| { + format!( + "failed to canonicalize {} from {CHECKPOINT_ENV}: {err}", + checkpoint.display() + ) + })?; + if !checkpoint.is_file() { + return Err(format!( + "{CHECKPOINT_ENV} must point at a chkjam file: {}", + checkpoint.display() + ) + .into()); + } + + let temp = tempfile::TempDir::new()?; + let app_root = temp.path().join("nockchain"); + let checkpoint_dir = app_root.join("checkpoints"); + std::fs::create_dir_all(&checkpoint_dir)?; + link_checkpoint(&checkpoint, &checkpoint_dir.join("0.chkjam"))?; + + let export_path = temp.path().join("state-8-export.jam"); + let mut cli = boot::default_boot_cli(false); + cli.gc_interval = None; + cli.rotating_snapshot_interval_event_time = None; + cli.disable_fsync = true; + cli.export_state_jam = Some(export_path.to_string_lossy().into_owned()); + cli.stack_size = NockStackSize::Huge; + + let test_jets = test_jets_for_env(); + let _guard = EnvVarGuard::set("NOCK_TEST_JETS", &test_jets); + let hot_state = produce_prover_hot_state(); + + match boot::setup_::( + kernels_open_dumb::KERNEL, + cli, + &hot_state, + "nockchain", + Some(temp.path().to_path_buf()), + ) + .await? + { + SetupResult::ExportedState => {} + SetupResult::App(_) => return Err("checkpoint migration did not export state".into()), + } + + let encoded = tokio::fs::read(&export_path).await?; + let exported = ExportedState::decode(&encoded)?; + let min_event = min_event()?; + if exported.event_num < min_event { + return Err(format!( + "checkpoint event_num {} is below required floor {}", + exported.event_num, min_event + ) + .into()); + } + + let state_hash = blake3::hash(&encoded); + println!( + "h-zoon checkpoint migration oracle passed: input={} event_num={} ker_hash={} exported_state_hash={}", + checkpoint.display(), + exported.event_num, + exported.ker_hash, + state_hash, + ); + + Ok(()) +} + +fn checkpoint_path() -> Result> { + std::env::var_os(CHECKPOINT_ENV) + .map(PathBuf::from) + .ok_or_else(|| { + format!( + "set {CHECKPOINT_ENV} to a large nockchain checkpoint, for example \ + /Users/jake/.data.nockchain/checkpoints/0.chkjam" + ) + .into() + }) +} + +fn min_event() -> Result> { + match std::env::var(MIN_EVENT_ENV) { + Ok(value) => value + .parse() + .map_err(|err| format!("invalid {MIN_EVENT_ENV}={value}: {err}").into()), + Err(_) => Ok(DEFAULT_MIN_EVENT), + } +} + +fn test_jets_for_env() -> String { + match std::env::var(TEST_JETS_ENV) { + Ok(value) if value == "default" => H_ZOON_TEST_JETS.to_owned(), + Ok(value) => value, + Err(_) => String::new(), + } +} + +fn link_checkpoint(source: &Path, destination: &Path) -> Result<(), Box> { + if destination.exists() { + std::fs::remove_file(destination)?; + } + + if std::fs::hard_link(source, destination).is_ok() { + return Ok(()); + } + + #[cfg(unix)] + { + std::os::unix::fs::symlink(source, destination)?; + Ok(()) + } + + #[cfg(not(unix))] + { + std::fs::copy(source, destination)?; + Ok(()) + } +} diff --git a/crates/nockchain/tests/h_zoon_jet_oracle.rs b/crates/nockchain/tests/h_zoon_jet_oracle.rs new file mode 100644 index 000000000..1e9ec54a1 --- /dev/null +++ b/crates/nockchain/tests/h_zoon_jet_oracle.rs @@ -0,0 +1,1021 @@ +//! Always-on jet differential oracle for the seven h-zoon jets. +//! +//! Companion to `h_zoon_checkpoint_migration.rs`. That test runs a full +//! kernel boot/export against a developer-supplied checkpoint and stays +//! `#[ignore]`d because it needs a chkjam pinned to the local node. It can +//! opt into the expensive `NOCK_TEST_JETS` Hoon fallback list for local +//! differential checks. This file fills the "always-on" gap: it constructs +//! the seven h-zoon jet inputs in Rust, calls each jet against an +//! independent Rust oracle, and asserts both produced byte-equivalent +//! nouns. +//! +//! Coverage assertion: an explicit counter tallies the seven jets that +//! ran. If any of the seven is missed -- because the test is silently +//! short-circuiting -- the final assert fails loudly, so we cannot lose +//! differential coverage by accident. +//! +//! When a new h-zoon jet lands, add the call here AND in +//! `h_zoon_checkpoint_migration.rs::H_ZOON_TEST_JETS`. The plan in +//! `docs/H-ZOON-TEST-PLAN.md` (§3) describes the contract; this is its +//! always-on incarnation. + +use nockchain_math::noun_ext::NounMathExt; +use nockchain_math::zoon::common::DefaultTipHasher; +use nockchain_math::zoon::zmap::z_map_put; +use nockchain_math::zoon::zset::z_set_put; +use nockvm::interpreter::Context; +use nockvm::jets::util::test::init_context; +use nockvm::jets::util::BAIL_FAIL; +use nockvm::jets::JetErr; +use nockvm::mem::NockStack; +use nockvm::noun::{Noun, NounAllocator, NounSpace, D, NO, T, YES}; +use nockvm::unifying_equality::unifying_equality; +use zkvm_jetpack::jets::zoon_jets::{ + gor_hip_jet, h_by_bif_jet, h_by_del_jet, h_by_dif_jet, h_by_dig_jet, h_by_gas_jet, + h_by_get_jet, h_by_got_jet, h_by_gut_jet, h_by_has_jet, h_by_int_jet, h_by_mar_jet, + h_by_put_jet, h_by_uni_jet, h_in_bif_jet, h_in_del_jet, h_in_dif_jet, h_in_dig_jet, + h_in_gas_jet, h_in_has_jet, h_in_int_jet, h_in_put_jet, h_in_uni_jet, mor_hip_jet, + zh_balance_milt_jet, zh_jult_jet, zh_milt_jet, zh_molt_jet, zh_silt_jet, +}; + +const GOR_HIP_ORDER: [usize; 5] = [4, 3, 2, 1, 0]; +const MOR_HIP_ORDER: [usize; 5] = [0, 1, 2, 3, 4]; + +fn direct_limb(value: u64) -> u64 { + value & (u64::MAX >> 1) +} + +fn digest(stack: &mut NockStack, limbs: [u64; 5]) -> Noun { + T( + stack, + &[ + D(direct_limb(limbs[0])), + D(direct_limb(limbs[1])), + D(direct_limb(limbs[2])), + D(direct_limb(limbs[3])), + D(direct_limb(limbs[4])), + ], + ) +} + +fn list(stack: &mut NockStack, items: &[Noun]) -> Noun { + items + .iter() + .rev() + .fold(D(0), |tail, item| T(stack, &[*item, tail])) +} + +fn digest_list(stack: &mut NockStack, digests: &[[u64; 5]]) -> Noun { + let items: Vec = digests.iter().map(|limbs| digest(stack, *limbs)).collect(); + list(stack, &items) +} + +fn jet_subject(stack: &mut NockStack, a: Noun, b: Noun) -> Noun { + let sam = T(stack, &[a, b]); + T(stack, &[D(0), sam, D(0)]) +} + +fn unary_jet_subject(stack: &mut NockStack, sample: Noun) -> Noun { + T(stack, &[D(0), sample, D(0)]) +} + +fn run_unary_jet( + ctx: &mut Context, + jet: fn(&mut Context, Noun) -> Result, + sample: Noun, +) -> Result { + let subject = unary_jet_subject(&mut ctx.stack, sample); + jet(ctx, subject) +} + +fn cmp_with_jet( + ctx: &mut Context, + jet: fn(&mut Context, Noun) -> Result, + a: Noun, + b: Noun, +) -> Result { + let subject = jet_subject(&mut ctx.stack, a, b); + let result = jet(ctx, subject)?; + if unsafe { result.raw_equals(&YES) } { + Ok(true) + } else if unsafe { result.raw_equals(&NO) } { + Ok(false) + } else { + panic!("comparator jet did not return %.y/%.n: {result:?}"); + } +} + +fn noun_eq(stack: &mut NockStack, a: Noun, b: Noun) -> bool { + let mut an = a; + let mut bn = b; + unsafe { unifying_equality(stack, &mut an, &mut bn) } +} + +fn z_map_from_entries(stack: &mut A, entries: &[(Noun, Noun)]) -> Noun { + let mut map = D(0); + for (key, value) in entries { + let mut k = *key; + let mut v = *value; + map = z_map_put(stack, &map, &mut k, &mut v, &DefaultTipHasher) + .expect("z-map construction must succeed"); + } + map +} + +fn z_set_from_items(stack: &mut A, items: &[Noun]) -> Noun { + let mut set = D(0); + for item in items { + let mut x = *item; + set = z_set_put(stack, &set, &mut x, &DefaultTipHasher) + .expect("z-set construction must succeed"); + } + set +} + +// Independent oracle: convert a z-map noun to the h-map shape by +// walking the tree, collecting (key, value) entries, then inserting in +// h-order. Mirrors the Hoon arm `zh-molt` exactly. +fn slow_z_map_to_h_map(stack: &mut NockStack, tree: Noun) -> Result { + let mut entries = Vec::new(); + let space = stack.noun_space(); + collect_z_map_entries(tree, &mut entries, &space)?; + let mut map = h_map_empty(); + for (key, value) in entries { + map = h_map_put(stack, map, key, value)?; + } + Ok(map) +} + +fn slow_z_set_to_h_set(stack: &mut NockStack, tree: Noun) -> Result { + let mut items = Vec::new(); + let space = stack.noun_space(); + collect_z_set_items(tree, &mut items, &space)?; + let mut set = h_set_empty(); + for item in items { + set = h_set_put(stack, set, item)?; + } + Ok(set) +} + +fn slow_z_mip_to_h_mip(stack: &mut NockStack, tree: Noun) -> Result { + let mut entries = Vec::new(); + let space = stack.noun_space(); + collect_z_map_entries(tree, &mut entries, &space)?; + let mut map = h_map_empty(); + for (outer_key, inner_map) in entries { + let converted_inner = slow_z_map_to_h_map(stack, inner_map)?; + map = h_map_put(stack, map, outer_key, converted_inner)?; + } + Ok(map) +} + +fn slow_z_jug_to_h_jug(stack: &mut NockStack, tree: Noun) -> Result { + let mut entries = Vec::new(); + let space = stack.noun_space(); + collect_z_map_entries(tree, &mut entries, &space)?; + let mut map = h_map_empty(); + for (outer_key, inner_set) in entries { + let converted_inner = slow_z_set_to_h_set(stack, inner_set)?; + map = h_map_put(stack, map, outer_key, converted_inner)?; + } + Ok(map) +} + +fn collect_z_map_entries( + tree: Noun, + entries: &mut Vec<(Noun, Noun)>, + space: &NounSpace, +) -> Result<(), JetErr> { + if unsafe { tree.raw_equals(&D(0)) } { + return Ok(()); + } + if tree.is_atom() { + return Err(BAIL_FAIL); + } + let [entry, left, right] = tree.uncell(space)?; + let [key, value] = entry.uncell(space)?; + entries.push((key, value)); + collect_z_map_entries(left, entries, space)?; + collect_z_map_entries(right, entries, space) +} + +fn collect_z_set_items(tree: Noun, items: &mut Vec, space: &NounSpace) -> Result<(), JetErr> { + if unsafe { tree.raw_equals(&D(0)) } { + return Ok(()); + } + if tree.is_atom() { + return Err(BAIL_FAIL); + } + let [value, left, right] = tree.uncell(space)?; + items.push(value); + collect_z_set_items(left, items, space)?; + collect_z_set_items(right, items, space) +} + +// Minimal h-tree builders for the oracle: %hmap / %hset empty leaves, +// no balancing here -- only used to compare to the jet output, which +// performs the actual treap construction. The oracle stays naive on +// purpose so it cannot share a balancing bug with the jet. +fn h_map_empty() -> Noun { + D(tas_value(b"hmap")) +} + +fn h_set_empty() -> Noun { + D(tas_value(b"hset")) +} + +fn tas_value(bytes: &[u8]) -> u64 { + let mut acc: u64 = 0; + for (i, byte) in bytes.iter().enumerate() { + acc |= u64::from(*byte) << (8 * i); + } + acc +} + +fn h_map_put(stack: &mut NockStack, tree: Noun, key: Noun, value: Noun) -> Result { + let entry = T(stack, &[key, value]); + let hmap = h_map_empty(); + if unsafe { tree.raw_equals(&hmap) } { + return Ok(T(stack, &[entry, hmap, hmap])); + } + let space = stack.noun_space(); + let [n, l, r] = tree.uncell(&space)?; + let [np, _nq] = n.uncell(&space)?; + if noun_eq(stack, key, np) { + return Ok(T(stack, &[entry, l, r])); + } + if hashed_less(key, np, &space) { + let new_l = h_map_put(stack, l, key, value)?; + rebalance_left(stack, n, new_l, r) + } else { + let new_r = h_map_put(stack, r, key, value)?; + rebalance_right(stack, n, l, new_r) + } +} + +fn h_set_put(stack: &mut NockStack, tree: Noun, item: Noun) -> Result { + let hset = h_set_empty(); + if unsafe { tree.raw_equals(&hset) } { + return Ok(T(stack, &[item, hset, hset])); + } + let space = stack.noun_space(); + let [n, l, r] = tree.uncell(&space)?; + if noun_eq(stack, item, n) { + return Ok(tree); + } + if hashed_less(item, n, &space) { + let new_l = h_set_put(stack, l, item)?; + rebalance_left_set(stack, n, new_l, r) + } else { + let new_r = h_set_put(stack, r, item)?; + rebalance_right_set(stack, n, l, new_r) + } +} + +fn rebalance_left(stack: &mut NockStack, n: Noun, new_l: Noun, r: Noun) -> Result { + let hmap = h_map_empty(); + if unsafe { new_l.raw_equals(&hmap) } { + return Ok(T(stack, &[n, hmap, r])); + } + let space = stack.noun_space(); + let [dn, dl, dr] = new_l.uncell(&space)?; + let [np, _nq] = n.uncell(&space)?; + let [dnp, _dnq] = dn.uncell(&space)?; + if hashed_priority(np, dnp, &space) { + Ok(T(stack, &[n, new_l, r])) + } else { + let inner = T(stack, &[n, dr, r]); + Ok(T(stack, &[dn, dl, inner])) + } +} + +fn rebalance_right(stack: &mut NockStack, n: Noun, l: Noun, new_r: Noun) -> Result { + let hmap = h_map_empty(); + if unsafe { new_r.raw_equals(&hmap) } { + return Ok(T(stack, &[n, l, hmap])); + } + let space = stack.noun_space(); + let [dn, dl, dr] = new_r.uncell(&space)?; + let [np, _nq] = n.uncell(&space)?; + let [dnp, _dnq] = dn.uncell(&space)?; + if hashed_priority(np, dnp, &space) { + Ok(T(stack, &[n, l, new_r])) + } else { + let inner = T(stack, &[n, l, dl]); + Ok(T(stack, &[dn, inner, dr])) + } +} + +fn rebalance_left_set( + stack: &mut NockStack, + n: Noun, + new_l: Noun, + r: Noun, +) -> Result { + let hset = h_set_empty(); + if unsafe { new_l.raw_equals(&hset) } { + return Ok(T(stack, &[n, hset, r])); + } + let space = stack.noun_space(); + let [dn, dl, dr] = new_l.uncell(&space)?; + if hashed_priority(n, dn, &space) { + Ok(T(stack, &[n, new_l, r])) + } else { + let inner = T(stack, &[n, dr, r]); + Ok(T(stack, &[dn, dl, inner])) + } +} + +fn rebalance_right_set( + stack: &mut NockStack, + n: Noun, + l: Noun, + new_r: Noun, +) -> Result { + let hset = h_set_empty(); + if unsafe { new_r.raw_equals(&hset) } { + return Ok(T(stack, &[n, l, hset])); + } + let space = stack.noun_space(); + let [dn, dl, dr] = new_r.uncell(&space)?; + if hashed_priority(n, dn, &space) { + Ok(T(stack, &[n, l, new_r])) + } else { + let inner = T(stack, &[n, l, dl]); + Ok(T(stack, &[dn, inner, dr])) + } +} + +fn hashed_to_limbs(key: Noun, space: &NounSpace) -> Vec<[u64; 5]> { + let mut out = Vec::new(); + if let Ok(limbs) = try_single_digest(key, space) { + out.push(limbs); + return out; + } + let mut cur = key; + while !unsafe { cur.raw_equals(&D(0)) } { + if cur.is_atom() { + return Vec::new(); + } + let Ok([head, tail]) = cur.uncell(space) else { + return Vec::new(); + }; + let Ok(limbs) = try_single_digest(head, space) else { + return Vec::new(); + }; + out.push(limbs); + cur = tail; + } + out +} + +fn try_single_digest(key: Noun, space: &NounSpace) -> Result<[u64; 5], ()> { + let Ok([a, rest1]) = key.uncell(space) else { + return Err(()); + }; + let Ok([b, rest2]) = rest1.uncell(space) else { + return Err(()); + }; + let Ok([c, rest3]) = rest2.uncell(space) else { + return Err(()); + }; + let Ok([d, e]) = rest3.uncell(space) else { + return Err(()); + }; + let limb_of = |n: &Noun| -> Result { + let atom = n.in_space(space).as_atom().map_err(|_| ())?; + atom.as_u64().map_err(|_| ()) + }; + let xs = [limb_of(&a)?, limb_of(&b)?, limb_of(&c)?, limb_of(&d)?, limb_of(&e)?]; + Ok(xs) +} + +fn hashed_less(a: Noun, b: Noun, space: &NounSpace) -> bool { + digest_list_order_oracle( + &hashed_to_limbs(a, space), + &hashed_to_limbs(b, space), + &GOR_HIP_ORDER, + ) +} + +fn hashed_priority(a: Noun, b: Noun, space: &NounSpace) -> bool { + digest_list_order_oracle( + &hashed_to_limbs(a, space), + &hashed_to_limbs(b, space), + &MOR_HIP_ORDER, + ) +} + +fn digest_list_order_oracle(a: &[[u64; 5]], b: &[[u64; 5]], order: &[usize; 5]) -> bool { + let mut idx = 0; + loop { + match (a.get(idx), b.get(idx)) { + (None, _) => return false, + (Some(_), None) => return true, + (Some(av), Some(bv)) => { + if let Some(ordered) = digest_order_oracle(av, bv, order) { + return ordered; + } + } + } + idx += 1; + } +} + +fn digest_order_oracle(a: &[u64; 5], b: &[u64; 5], order: &[usize; 5]) -> Option { + for &i in order { + if a[i] > b[i] { + return Some(true); + } + if a[i] < b[i] { + return Some(false); + } + } + None +} + +fn assert_noun_eq(stack: &mut NockStack, name: &str, mut a: Noun, mut b: Noun) { + let eq = unsafe { unifying_equality(stack, &mut a, &mut b) }; + assert!(eq, "{name}: jet and oracle nouns disagree: {a:?} vs {b:?}"); +} + +// Local page noun, mirroring `local-page:v0:dt` shape: [block 0 parent +// 0 0 0 0 0 0 0 0]. The migration only walks the noun shape; values +// inside the page are opaque to zh-balmilt. +fn local_page_v0(stack: &mut NockStack, block: Noun, parent: Noun) -> Noun { + T( + stack, + &[block, D(0), parent, D(0), D(0), D(0), D(0), D(0), D(0), D(0), D(0)], + ) +} + +#[test] +fn h_zoon_jets_are_byte_equivalent_to_independent_oracles() { + let ctx = &mut init_context(); + let mut jets_ran = 0u32; + + // ---- 1. gor-hip on a single-digest pair ---- + let high = digest(&mut ctx.stack, [0, 0, 0, 0, 2]); + let low = digest(&mut ctx.stack, [99, 99, 99, 99, 1]); + let gor = cmp_with_jet(ctx, gor_hip_jet, high, low).expect("gor-hip"); + assert!( + gor, + "gor-hip must order higher tail-limb first; jet returned %.n" + ); + let oracle = + digest_list_order_oracle(&[[0, 0, 0, 0, 2]], &[[99, 99, 99, 99, 1]], &GOR_HIP_ORDER); + assert_eq!(gor, oracle, "gor-hip jet disagrees with oracle"); + jets_ran += 1; + + // ---- 2. mor-hip on a single-digest pair ---- + let m_high = digest(&mut ctx.stack, [2, 0, 0, 0, 0]); + let m_low = digest(&mut ctx.stack, [1, 99, 99, 99, 99]); + let mor = cmp_with_jet(ctx, mor_hip_jet, m_high, m_low).expect("mor-hip"); + let oracle = + digest_list_order_oracle(&[[2, 0, 0, 0, 0]], &[[1, 99, 99, 99, 99]], &MOR_HIP_ORDER); + assert_eq!(mor, oracle, "mor-hip jet disagrees with oracle"); + jets_ran += 1; + + // build a four-entry z-map with a mix of direct-digest and + // digest-list keys so every conversion jet hits both key shapes + let key_a = digest(&mut ctx.stack, [1, 0, 0, 0, 0]); + let key_b = digest_list(&mut ctx.stack, &[[2, 0, 0, 0, 0], [3, 0, 0, 0, 0]]); + let key_c = digest(&mut ctx.stack, [4, 0, 0, 0, 0]); + let key_d = digest_list(&mut ctx.stack, &[[5, 0, 0, 0, 0], [6, 0, 0, 0, 0]]); + let value_a = D(10); + let value_b = T(&mut ctx.stack, &[D(20), D(21)]); + let value_c = D(30); + let value_d = T(&mut ctx.stack, &[D(40), D(41), D(42)]); + let z_map = z_map_from_entries( + &mut ctx.stack, + &[(key_a, value_a), (key_b, value_b), (key_c, value_c), (key_d, value_d)], + ); + + // ---- 3. zh-molt ---- + let h_from_jet = run_unary_jet(ctx, zh_molt_jet, z_map).expect("zh-molt"); + let h_from_oracle = slow_z_map_to_h_map(&mut ctx.stack, z_map).expect("zh-molt oracle"); + assert_noun_eq(&mut ctx.stack, "zh-molt", h_from_jet, h_from_oracle); + jets_ran += 1; + + // ---- 4. zh-silt ---- + let z_set = z_set_from_items(&mut ctx.stack, &[key_a, key_b, key_c, key_d]); + let s_from_jet = run_unary_jet(ctx, zh_silt_jet, z_set).expect("zh-silt"); + let s_from_oracle = slow_z_set_to_h_set(&mut ctx.stack, z_set).expect("zh-silt oracle"); + assert_noun_eq(&mut ctx.stack, "zh-silt", s_from_jet, s_from_oracle); + jets_ran += 1; + + // ---- 5. zh-milt (nested z-map -> nested h-map) ---- + let inner_key_a = digest(&mut ctx.stack, [0, 1, 0, 0, 0]); + let inner_key_b = digest_list(&mut ctx.stack, &[[0, 2, 0, 0, 0], [0, 3, 0, 0, 0]]); + let inner_map = z_map_from_entries( + &mut ctx.stack, + &[(inner_key_a, D(100)), (inner_key_b, D(200))], + ); + let outer_key_a = digest(&mut ctx.stack, [1, 1, 0, 0, 0]); + let outer_key_b = digest_list(&mut ctx.stack, &[[2, 2, 0, 0, 0], [3, 3, 0, 0, 0]]); + let z_mip = z_map_from_entries( + &mut ctx.stack, + &[(outer_key_a, inner_map), (outer_key_b, inner_map)], + ); + let mip_from_jet = run_unary_jet(ctx, zh_milt_jet, z_mip).expect("zh-milt"); + let mip_from_oracle = slow_z_mip_to_h_mip(&mut ctx.stack, z_mip).expect("zh-milt oracle"); + assert_noun_eq(&mut ctx.stack, "zh-milt", mip_from_jet, mip_from_oracle); + jets_ran += 1; + + // ---- 6. zh-jult (nested z-set inside z-map) ---- + let inner_set = z_set_from_items(&mut ctx.stack, &[inner_key_a, inner_key_b]); + let z_jug = z_map_from_entries( + &mut ctx.stack, + &[(outer_key_a, inner_set), (outer_key_b, inner_set)], + ); + let jug_from_jet = run_unary_jet(ctx, zh_jult_jet, z_jug).expect("zh-jult"); + let jug_from_oracle = slow_z_jug_to_h_jug(&mut ctx.stack, z_jug).expect("zh-jult oracle"); + assert_noun_eq(&mut ctx.stack, "zh-jult", jug_from_jet, jug_from_oracle); + jets_ran += 1; + + // ---- 7. zh-balmilt (parent-chain balance migration) ---- + let parent_block = digest(&mut ctx.stack, [11, 0, 0, 0, 0]); + let child_block = digest(&mut ctx.stack, [12, 0, 0, 0, 0]); + let grandparent = digest(&mut ctx.stack, [10, 0, 0, 0, 0]); + let parent_page = local_page_v0(&mut ctx.stack, parent_block, grandparent); + let child_page = local_page_v0(&mut ctx.stack, child_block, parent_block); + let blocks = z_map_from_entries( + &mut ctx.stack, + &[(parent_block, parent_page), (child_block, child_page)], + ); + let note_key_a = digest(&mut ctx.stack, [21, 0, 0, 0, 0]); + let note_key_b = digest_list(&mut ctx.stack, &[[22, 0, 0, 0, 0], [23, 0, 0, 0, 0]]); + let note_value_a = T(&mut ctx.stack, &[D(100), D(101)]); + let note_value_a2 = T(&mut ctx.stack, &[D(110), D(111)]); + let note_value_b = T(&mut ctx.stack, &[D(200), D(201)]); + let parent_balance = z_map_from_entries(&mut ctx.stack, &[(note_key_a, note_value_a)]); + let child_balance = z_map_from_entries( + &mut ctx.stack, + &[(note_key_a, note_value_a2), (note_key_b, note_value_b)], + ); + let balance = z_map_from_entries( + &mut ctx.stack, + &[(parent_block, parent_balance), (child_block, child_balance)], + ); + let sample = T(&mut ctx.stack, &[blocks, balance]); + let subject = unary_jet_subject(&mut ctx.stack, sample); + let balmilt_from_jet = zh_balance_milt_jet(ctx, subject).expect("zh-balmilt"); + let balmilt_from_oracle = + slow_z_mip_to_h_mip(&mut ctx.stack, balance).expect("zh-balmilt oracle"); + assert_noun_eq( + &mut ctx.stack, "zh-balmilt", balmilt_from_jet, balmilt_from_oracle, + ); + jets_ran += 1; + + // coverage assertion: if any jet was silently skipped we fail. + // Update this constant when a new h-zoon jet lands. + const H_ZOON_JET_COUNT: u32 = 7; + assert_eq!( + jets_ran, H_ZOON_JET_COUNT, + "expected all {H_ZOON_JET_COUNT} h-zoon jets to be exercised, only {jets_ran} ran" + ); +} + +// =========================================================================== +// Randomized differential oracle for the 22 non-gate h-by / h-in arm jets. +// +// Inputs (h-maps / h-sets over single-digest keys) are built with the file's +// naive canonical treap builder. Expected results are computed with plain +// BTreeMap / BTreeSet set logic and re-built with the SAME naive builder. The +// canonical treap is uniquely determined by (key set, mor-hip priorities), so +// any correct jet must produce the byte-identical noun. The oracle's set +// logic is independent of each jet's recursive merge/split/lookup, so an +// algorithmic bug cannot be shared. +// =========================================================================== + +use std::collections::{BTreeMap, BTreeSet}; + +struct Rng(u64); +impl Rng { + fn next(&mut self) -> u64 { + self.0 = self.0.wrapping_add(0x9E37_79B9_7F4A_7C15); + let mut z = self.0; + z = (z ^ (z >> 30)).wrapping_mul(0xBF58_476D_1CE4_E5B9); + z = (z ^ (z >> 27)).wrapping_mul(0x94D0_49BB_1331_11EB); + z ^ (z >> 31) + } + fn upto(&mut self, n: u64) -> u64 { + self.next() % n + } +} + +fn rand_limbs(rng: &mut Rng) -> [u64; 5] { + // small limb domain forces equal keys and gor-hip ordering ties + [rng.upto(3), rng.upto(3), rng.upto(3), rng.upto(3), rng.upto(3)] +} + +fn build_h_map(stack: &mut NockStack, entries: &[([u64; 5], u64)]) -> Noun { + let mut tree = h_map_empty(); + for (limbs, value) in entries { + let k = digest(stack, *limbs); + tree = h_map_put(stack, tree, k, D(*value)).expect("naive h_map_put"); + } + tree +} + +fn build_h_set(stack: &mut NockStack, items: &[[u64; 5]]) -> Noun { + let mut tree = h_set_empty(); + for limbs in items { + let x = digest(stack, *limbs); + tree = h_set_put(stack, tree, x).expect("naive h_set_put"); + } + tree +} + +fn canon_map(stack: &mut NockStack, m: &BTreeMap<[u64; 5], u64>) -> Noun { + let v: Vec<([u64; 5], u64)> = m.iter().map(|(k, val)| (*k, *val)).collect(); + build_h_map(stack, &v) +} + +fn canon_set(stack: &mut NockStack, s: &BTreeSet<[u64; 5]>) -> Noun { + let v: Vec<[u64; 5]> = s.iter().copied().collect(); + build_h_set(stack, &v) +} + +// faithful door-arm subject: arm sample at axis 6, door sample at +// slot(slot(_,7),6) -- same shape the real h-by/h-in call produces. +fn door_subject(stack: &mut NockStack, sample: Noun, container: Noun) -> Noun { + let context = T(stack, &[D(0), container, D(0)]); + T(stack, &[D(0), sample, context]) +} + +fn run_door_jet( + ctx: &mut Context, + jet: fn(&mut Context, Noun) -> Result, + sample: Noun, + container: Noun, +) -> Result { + let subject = door_subject(&mut ctx.stack, sample, container); + jet(ctx, subject) +} + +fn noun_at_axis(mut tree: Noun, axis: u64, space: &NounSpace) -> Option { + if axis == 0 { + return None; + } + let bits = 63 - axis.leading_zeros(); + for i in (0..bits).rev() { + let cell = tree.in_space(space).as_cell().ok()?; + tree = if (axis >> i) & 1 == 0 { + cell.head().noun() + } else { + cell.tail().noun() + }; + } + Some(tree) +} + +fn gor_lt(a: &[u64; 5], b: &[u64; 5]) -> bool { + digest_list_order_oracle(&[*a], &[*b], &GOR_HIP_ORDER) +} + +#[test] +fn h_by_h_in_non_gate_jets_match_independent_oracles_randomized() { + let ctx = &mut init_context(); + let mut covered: BTreeSet<&'static str> = BTreeSet::new(); + + for seed in 0..400u64 { + let mut rng = Rng(seed.wrapping_mul(0x2545F4914F6CDD1D).wrapping_add(1)); + + // ---- build A and B over an overlapping key pool ---- + let pool: Vec<[u64; 5]> = (0..10).map(|_| rand_limbs(&mut rng)).collect(); + let mut amap: BTreeMap<[u64; 5], u64> = BTreeMap::new(); + let mut bmap: BTreeMap<[u64; 5], u64> = BTreeMap::new(); + for _ in 0..(rng.upto(10) + 1) { + let k = pool[rng.upto(pool.len() as u64) as usize]; + amap.insert(k, rng.upto(900) + 1); + } + for _ in 0..(rng.upto(10) + 1) { + let k = pool[rng.upto(pool.len() as u64) as usize]; + bmap.insert(k, rng.upto(900) + 1001); + } + let a_entries: Vec<([u64; 5], u64)> = amap.iter().map(|(k, v)| (*k, *v)).collect(); + let b_entries: Vec<([u64; 5], u64)> = bmap.iter().map(|(k, v)| (*k, *v)).collect(); + let map_a = build_h_map(&mut ctx.stack, &a_entries); + let map_b = build_h_map(&mut ctx.stack, &b_entries); + let aset: BTreeSet<[u64; 5]> = amap.keys().copied().collect(); + let bset: BTreeSet<[u64; 5]> = bmap.keys().copied().collect(); + let set_a = canon_set(&mut ctx.stack, &aset); + let set_b = canon_set(&mut ctx.stack, &bset); + + // probe keys: some present, some absent + let probe = pool[rng.upto(pool.len() as u64) as usize]; + let probe_k = digest(&mut ctx.stack, probe); + + // ---- h-by get / has / got / gut ---- + let got_jet = run_door_jet(ctx, h_by_get_jet, probe_k, map_a).expect("h-by get"); + match amap.get(&probe) { + Some(v) => { + let exp = T(&mut ctx.stack, &[D(0), D(*v)]); + assert_noun_eq(&mut ctx.stack, "h-by/get(some)", got_jet, exp); + } + None => assert!( + got_jet.is_atom() && unsafe { got_jet.raw_equals(&D(0)) }, + "h-by/get(none) must be ~" + ), + } + covered.insert("h-by/get"); + + let has_jet = run_door_jet(ctx, h_by_has_jet, probe_k, map_a).expect("h-by has"); + let exp_has = if amap.contains_key(&probe) { YES } else { NO }; + assert!( + unsafe { has_jet.raw_equals(&exp_has) }, + "h-by/has mismatch seed {seed}" + ); + covered.insert("h-by/has"); + + let got_res = run_door_jet(ctx, h_by_got_jet, probe_k, map_a); + match amap.get(&probe) { + Some(v) => { + let g = got_res.expect("h-by got present"); + let ev = D(*v); + assert_noun_eq(&mut ctx.stack, "h-by/got", g, ev); + } + None => assert!(got_res.is_err(), "h-by/got(absent) must error"), + } + covered.insert("h-by/got"); + + let default = D(424_242); + let gut_sample = T(&mut ctx.stack, &[probe_k, default]); + let gut_jet = run_door_jet(ctx, h_by_gut_jet, gut_sample, map_a).expect("h-by gut"); + let exp_gut = match amap.get(&probe) { + Some(v) => D(*v), + None => D(424_242), + }; + assert_noun_eq(&mut ctx.stack, "h-by/gut", gut_jet, exp_gut); + covered.insert("h-by/gut"); + + // ---- h-by put / del / mar ---- + let pk = pool[rng.upto(pool.len() as u64) as usize]; + let pv = rng.upto(900) + 5000; + let pkn = digest(&mut ctx.stack, pk); + let put_sample = T(&mut ctx.stack, &[pkn, D(pv)]); + let put_jet = run_door_jet(ctx, h_by_put_jet, put_sample, map_a).expect("h-by put"); + let mut put_exp_m = amap.clone(); + put_exp_m.insert(pk, pv); + let put_exp = canon_map(&mut ctx.stack, &put_exp_m); + assert_noun_eq(&mut ctx.stack, "h-by/put", put_jet, put_exp); + covered.insert("h-by/put"); + + let dk = pool[rng.upto(pool.len() as u64) as usize]; + let dkn = digest(&mut ctx.stack, dk); + let del_jet = run_door_jet(ctx, h_by_del_jet, dkn, map_a).expect("h-by del"); + let mut del_exp_m = amap.clone(); + del_exp_m.remove(&dk); + let del_exp = canon_map(&mut ctx.stack, &del_exp_m); + assert_noun_eq(&mut ctx.stack, "h-by/del", del_jet, del_exp); + covered.insert("h-by/del"); + + // mar with ~ -> del ; mar with [~ v] -> put + let mar_none = T(&mut ctx.stack, &[dkn, D(0)]); + let mar_none_jet = run_door_jet(ctx, h_by_mar_jet, mar_none, map_a).expect("h-by mar none"); + assert_noun_eq(&mut ctx.stack, "h-by/mar(none)", mar_none_jet, del_exp); + let some_v = T(&mut ctx.stack, &[D(0), D(pv)]); + let mar_some = T(&mut ctx.stack, &[pkn, some_v]); + let mar_some_jet = run_door_jet(ctx, h_by_mar_jet, mar_some, map_a).expect("h-by mar some"); + assert_noun_eq(&mut ctx.stack, "h-by/mar(some)", mar_some_jet, put_exp); + covered.insert("h-by/mar"); + + // ---- h-by gas ---- + let mut gas_vec: Vec<([u64; 5], u64)> = Vec::new(); + let mut gas_exp_m = amap.clone(); + for _ in 0..rng.upto(5) { + let gk = pool[rng.upto(pool.len() as u64) as usize]; + let gv = rng.upto(900) + 7000; + gas_vec.push((gk, gv)); + gas_exp_m.insert(gk, gv); + } + let gas_items: Vec = gas_vec + .iter() + .map(|(k, v)| { + let kn = digest(&mut ctx.stack, *k); + T(&mut ctx.stack, &[kn, D(*v)]) + }) + .collect(); + let gas_list = list(&mut ctx.stack, &gas_items); + let gas_jet = run_door_jet(ctx, h_by_gas_jet, gas_list, map_a).expect("h-by gas"); + let gas_exp = canon_map(&mut ctx.stack, &gas_exp_m); + assert_noun_eq(&mut ctx.stack, "h-by/gas", gas_jet, gas_exp); + covered.insert("h-by/gas"); + + // ---- h-by uni / int / dif (b wins on equal for uni & int) ---- + let uni_jet = run_door_jet(ctx, h_by_uni_jet, map_b, map_a).expect("h-by uni"); + let mut uni_m = amap.clone(); + for (k, v) in &bmap { + uni_m.insert(*k, *v); + } + let uni_exp = canon_map(&mut ctx.stack, &uni_m); + assert_noun_eq(&mut ctx.stack, "h-by/uni", uni_jet, uni_exp); + covered.insert("h-by/uni"); + + let int_jet = run_door_jet(ctx, h_by_int_jet, map_b, map_a).expect("h-by int"); + let mut int_m: BTreeMap<[u64; 5], u64> = BTreeMap::new(); + for k in amap.keys() { + if let Some(bv) = bmap.get(k) { + int_m.insert(*k, *bv); + } + } + let int_exp = canon_map(&mut ctx.stack, &int_m); + assert_noun_eq(&mut ctx.stack, "h-by/int", int_jet, int_exp); + covered.insert("h-by/int"); + + let dif_jet = run_door_jet(ctx, h_by_dif_jet, map_b, map_a).expect("h-by dif"); + let mut dif_m: BTreeMap<[u64; 5], u64> = BTreeMap::new(); + for (k, v) in &amap { + if !bmap.contains_key(k) { + dif_m.insert(*k, *v); + } + } + let dif_exp = canon_map(&mut ctx.stack, &dif_m); + assert_noun_eq(&mut ctx.stack, "h-by/dif", dif_jet, dif_exp); + covered.insert("h-by/dif"); + + // ---- h-by bif ---- + let bif_jet = run_door_jet(ctx, h_by_bif_jet, probe_k, map_a).expect("h-by bif"); + let space = ctx.stack.noun_space(); + let [bl, br] = bif_jet.uncell(&space).expect("bif pair"); + let mut left_m: BTreeMap<[u64; 5], u64> = BTreeMap::new(); + let mut right_m: BTreeMap<[u64; 5], u64> = BTreeMap::new(); + for (k, v) in &amap { + if *k == probe { + continue; + } + if gor_lt(k, &probe) { + left_m.insert(*k, *v); + } else { + right_m.insert(*k, *v); + } + } + let le = canon_map(&mut ctx.stack, &left_m); + let re = canon_map(&mut ctx.stack, &right_m); + assert_noun_eq(&mut ctx.stack, "h-by/bif.l", bl, le); + assert_noun_eq(&mut ctx.stack, "h-by/bif.r", br, re); + covered.insert("h-by/bif"); + + // ---- h-by dig (independent invariant: axis lands on key) ---- + let dig_jet = run_door_jet(ctx, h_by_dig_jet, probe_k, map_a); + if let Ok(d) = dig_jet { + match amap.get(&probe) { + None => assert!(unsafe { d.raw_equals(&D(0)) }, "h-by/dig(absent) must be ~"), + Some(_) => { + let space = ctx.stack.noun_space(); + let [_z, ax] = d.uncell(&space).expect("dig unit"); + let axv = ax + .in_space(&space) + .as_atom() + .expect("axis atom") + .as_u64() + .expect("axis u64"); + let n = noun_at_axis(map_a, axv, &space).expect("axis node"); + // axis points at the [key value] pair; head is the key + let nk = n.in_space(&space).as_cell().expect("pair").head().noun(); + let mut a = nk; + let mut b = probe_k; + assert!( + unsafe { unifying_equality(&mut ctx.stack, &mut a, &mut b) }, + "h-by/dig axis must address the queried key" + ); + } + } + covered.insert("h-by/dig"); + } // else Punt (deep axis) is an acceptable faithful fallback + + // ---- h-in has / put / del / gas ---- + let s_has = run_door_jet(ctx, h_in_has_jet, probe_k, set_a).expect("h-in has"); + let exp_s_has = if aset.contains(&probe) { YES } else { NO }; + assert!( + unsafe { s_has.raw_equals(&exp_s_has) }, + "h-in/has mismatch seed {seed}" + ); + covered.insert("h-in/has"); + + let spk = digest(&mut ctx.stack, pk); + let s_put = run_door_jet(ctx, h_in_put_jet, spk, set_a).expect("h-in put"); + let mut s_put_set = aset.clone(); + s_put_set.insert(pk); + let s_put_exp = canon_set(&mut ctx.stack, &s_put_set); + assert_noun_eq(&mut ctx.stack, "h-in/put", s_put, s_put_exp); + covered.insert("h-in/put"); + + let sdk = digest(&mut ctx.stack, dk); + let s_del = run_door_jet(ctx, h_in_del_jet, sdk, set_a).expect("h-in del"); + let mut s_del_set = aset.clone(); + s_del_set.remove(&dk); + let s_del_exp = canon_set(&mut ctx.stack, &s_del_set); + assert_noun_eq(&mut ctx.stack, "h-in/del", s_del, s_del_exp); + covered.insert("h-in/del"); + + let mut s_gas_items: Vec<[u64; 5]> = Vec::new(); + let mut s_gas_set = aset.clone(); + for _ in 0..rng.upto(5) { + let gk = pool[rng.upto(pool.len() as u64) as usize]; + s_gas_items.push(gk); + s_gas_set.insert(gk); + } + let s_gas_nouns: Vec = s_gas_items + .iter() + .map(|k| digest(&mut ctx.stack, *k)) + .collect(); + let s_gas_list = list(&mut ctx.stack, &s_gas_nouns); + let s_gas = run_door_jet(ctx, h_in_gas_jet, s_gas_list, set_a).expect("h-in gas"); + let s_gas_exp = canon_set(&mut ctx.stack, &s_gas_set); + assert_noun_eq(&mut ctx.stack, "h-in/gas", s_gas, s_gas_exp); + covered.insert("h-in/gas"); + + // ---- h-in uni / int / dif ---- + let s_uni = run_door_jet(ctx, h_in_uni_jet, set_b, set_a).expect("h-in uni"); + let s_uni_exp = canon_set( + &mut ctx.stack, + &aset.union(&bset).copied().collect::>(), + ); + assert_noun_eq(&mut ctx.stack, "h-in/uni", s_uni, s_uni_exp); + covered.insert("h-in/uni"); + + let s_int = run_door_jet(ctx, h_in_int_jet, set_b, set_a).expect("h-in int"); + let s_int_exp = canon_set( + &mut ctx.stack, + &aset.intersection(&bset).copied().collect::>(), + ); + assert_noun_eq(&mut ctx.stack, "h-in/int", s_int, s_int_exp); + covered.insert("h-in/int"); + + let s_dif = run_door_jet(ctx, h_in_dif_jet, set_b, set_a).expect("h-in dif"); + let s_dif_exp = canon_set( + &mut ctx.stack, + &aset.difference(&bset).copied().collect::>(), + ); + assert_noun_eq(&mut ctx.stack, "h-in/dif", s_dif, s_dif_exp); + covered.insert("h-in/dif"); + + // ---- h-in bif ---- + let s_bif = run_door_jet(ctx, h_in_bif_jet, probe_k, set_a).expect("h-in bif"); + let space = ctx.stack.noun_space(); + let [sbl, sbr] = s_bif.uncell(&space).expect("h-in bif pair"); + let mut sl: BTreeSet<[u64; 5]> = BTreeSet::new(); + let mut sr: BTreeSet<[u64; 5]> = BTreeSet::new(); + for k in &aset { + if *k == probe { + continue; + } + if gor_lt(k, &probe) { + sl.insert(*k); + } else { + sr.insert(*k); + } + } + let sle = canon_set(&mut ctx.stack, &sl); + let sre = canon_set(&mut ctx.stack, &sr); + assert_noun_eq(&mut ctx.stack, "h-in/bif.l", sbl, sle); + assert_noun_eq(&mut ctx.stack, "h-in/bif.r", sbr, sre); + covered.insert("h-in/bif"); + + // ---- h-in dig ---- + let s_dig = run_door_jet(ctx, h_in_dig_jet, probe_k, set_a); + if let Ok(d) = s_dig { + match aset.contains(&probe) { + false => assert!(unsafe { d.raw_equals(&D(0)) }, "h-in/dig(absent) must be ~"), + true => { + let space = ctx.stack.noun_space(); + let [_z, ax] = d.uncell(&space).expect("dig unit"); + let axv = ax + .in_space(&space) + .as_atom() + .expect("axis atom") + .as_u64() + .expect("axis u64"); + let n = noun_at_axis(set_a, axv, &space).expect("axis node"); + let mut a = n; + let mut b = probe_k; + assert!( + unsafe { unifying_equality(&mut ctx.stack, &mut a, &mut b) }, + "h-in/dig axis must address the queried item" + ); + } + } + covered.insert("h-in/dig"); + } + } + + const EXPECTED: &[&str] = &[ + "h-by/get", "h-by/got", "h-by/gut", "h-by/has", "h-by/put", "h-by/del", "h-by/mar", + "h-by/gas", "h-by/uni", "h-by/int", "h-by/dif", "h-by/bif", "h-by/dig", "h-in/has", + "h-in/put", "h-in/del", "h-in/gas", "h-in/uni", "h-in/int", "h-in/dif", "h-in/bif", + "h-in/dig", + ]; + for arm in EXPECTED { + assert!( + covered.contains(arm), + "jet {arm} was never exercised by the randomized differential" + ); + } + assert_eq!( + covered.len(), + EXPECTED.len(), + "unexpected jet coverage set: {covered:?}" + ); +} diff --git a/crates/nockchain/tests/open_prover_bench.rs b/crates/nockchain/tests/open_prover_bench.rs new file mode 100644 index 000000000..cc3e1aa94 --- /dev/null +++ b/crates/nockchain/tests/open_prover_bench.rs @@ -0,0 +1,224 @@ +use std::error::Error; +use std::time::Instant; + +use ibig::UBig; +use kernels_open_miner::KERNEL; +use nockapp::kernel::boot::{parse_test_jets, TraceOpts}; +use nockapp::kernel::form::{PmaConfig, SerfThread}; +use nockapp::noun::slab::NounSlab; +use nockapp::save::SaveableCheckpoint; +use nockapp::utils::NOCK_STACK_SIZE_TINY; +use nockapp::wire::Wire; +use nockapp::AtomExt; +use nockchain::mining::MiningWire; +use nockchain_math::noun_ext::NounMathExtHandle; +use nockchain_math::structs::HoonList; +use nockchain_types::BlockchainConstants; +use nockvm::noun::{Atom, Noun, NounAllocator, D, T, YES}; +use nockvm_macros::tas; +use zkvm_jetpack::hot::produce_prover_hot_state; + +fn tip5_to_noun(slab: &mut NounSlab, values: [u64; 5]) -> Result> { + let mut tuple = Vec::with_capacity(values.len()); + for value in values { + let atom = ::from_value(slab, value) + .map_err(|e| Box::new(e) as Box)?; + tuple.push(atom.as_noun()); + } + Ok(T(slab, &tuple)) +} + +fn bignum_to_noun(slab: &mut NounSlab, value: &UBig) -> Result> { + let mut list = D(0); + let bytes = value.to_le_bytes(); + for chunk in bytes.chunks(4).rev() { + let mut padded = [0u8; 4]; + padded[..chunk.len()].copy_from_slice(chunk); + let chunk = u64::from(u32::from_le_bytes(padded)); + let atom = ::from_value(slab, chunk) + .map_err(|e| Box::new(e) as Box)?; + list = T(slab, &[atom.as_noun(), list]); + } + Ok(T(slab, &[D(tas!(b"bn")), list])) +} + +async fn send_set_mining_key(serf: &SerfThread) -> Result<(), Box> { + let mut slab = NounSlab::new(); + let head = D(tas!(b"command")); + let command = ::from_value(&mut slab, "set-mining-key") + .map_err(|e| Box::new(e) as Box)? + .as_noun(); + let pubkey = ::from_value(&mut slab, "open-prover-test-pubkey") + .map_err(|e| Box::new(e) as Box)? + .as_noun(); + let poke = T(&mut slab, &[head, command, pubkey]); + slab.set_root(poke); + + serf.poke(MiningWire::SetPubKey.to_wire(), slab) + .await + .map_err(|e| Box::new(e) as Box)?; + Ok(()) +} + +async fn send_enable_mining( + serf: &SerfThread, +) -> Result> { + let mut slab = NounSlab::new(); + let head = D(tas!(b"command")); + let command = ::from_value(&mut slab, "enable-mining") + .map_err(|e| Box::new(e) as Box)? + .as_noun(); + let poke = T(&mut slab, &[head, command, YES]); + slab.set_root(poke); + + serf.poke(MiningWire::Enable.to_wire(), slab) + .await + .map_err(|e| Box::new(e) as Box) +} + +fn extract_mine_start(slab: &NounSlab) -> Result<(Noun, Noun, Noun, Noun), Box> { + let space = slab.noun_space(); + let root = unsafe { *slab.root() }; + let effects = HoonList::try_from(root, &space).map_err(|e| Box::new(e) as Box)?; + for effect in effects { + if let Ok(effect_cell) = effect.in_space(&space).as_cell() { + if effect_cell.head().eq_bytes("mine") { + let mine_cell = effect_cell + .tail() + .as_cell() + .map_err(|e| Box::new(e) as Box)?; + let mine_start = mine_cell.head(); + let [version, header, target, pow_len] = mine_start + .uncell::<4>() + .map_err(|e| Box::new(e) as Box)?; + return Ok((version.noun(), header.noun(), target.noun(), pow_len.noun())); + } + } + } + Err(Box::::from( + "kernel did not emit %mine start".to_owned(), + )) +} + +enum CandidateData { + Kernel { + version: Noun, + header: Noun, + target: Noun, + pow_len: u64, + }, + Synthetic, +} + +#[tokio::test(flavor = "current_thread")] +async fn benchmark_open_prover_single_attempt() -> Result<(), Box> { + // Prepare a standalone miner kernel instance with the open-source prover hot state. + let kernel_bytes = Vec::from(KERNEL); + let hot_state = produce_prover_hot_state(); + let test_jets = parse_test_jets(""); + + let serf = SerfThread::::new( + kernel_bytes, + None, + hot_state, + NOCK_STACK_SIZE_TINY, + None::, + test_jets, + TraceOpts::default(), + ) + .await + .map_err(|e| Box::new(e) as Box)?; + + // Initialize mining state so the kernel provides candidate metadata. + send_set_mining_key(&serf).await?; + let enable_result = send_enable_mining(&serf).await?; + let enable_space = enable_result.noun_space(); + let candidate_data = match extract_mine_start(&enable_result) { + Ok((version_noun, header_noun, target_noun, pow_len_noun)) => { + let pow_len_value = pow_len_noun + .in_space(&enable_space) + .as_atom() + .map_err(|e| Box::new(e) as Box)? + .as_u64() + .map_err(|e| Box::new(e) as Box)?; + CandidateData::Kernel { + version: version_noun, + header: header_noun, + target: target_noun, + pow_len: pow_len_value, + } + } + Err(err) => { + println!("WARNING: falling back to synthetic mining candidate: {err}"); + CandidateData::Synthetic + } + }; + + // Build a mining candidate poke: [version header nonce target pow_len]. + let mut poke_slab = NounSlab::new(); + let (version, header, target, pow_len_value) = match candidate_data { + CandidateData::Kernel { + version, + header, + target, + pow_len, + } => ( + poke_slab.copy_into(version, &enable_space), + poke_slab.copy_into(header, &enable_space), + poke_slab.copy_into(target, &enable_space), + pow_len, + ), + CandidateData::Synthetic => { + let max_target = BlockchainConstants::new().max_target_atom; + ( + D(1), + tip5_to_noun(&mut poke_slab, [1, 2, 3, 4, 5])?, + bignum_to_noun(&mut poke_slab, &max_target)?, + BlockchainConstants::DEFAULT_POW_LEN, + ) + } + }; + let nonce = tip5_to_noun(&mut poke_slab, [0, 0, 0, 0, 0])?; + let pow_len = D(pow_len_value); + let poke_noun = T(&mut poke_slab, &[version, header, nonce, target, pow_len]); + poke_slab.set_root(poke_noun); + + // Execute a single proof attempt and record the elapsed time. + let start = Instant::now(); + let poke_result = serf + .poke(MiningWire::Candidate.to_wire(), poke_slab) + .await + .map_err(|e| Box::new(e) as Box)?; + let elapsed = start.elapsed(); + println!( + "Open prover single proof attempt completed in {:.3?}", + elapsed + ); + + // Verify we received a successful %mine-result effect. + let poke_space = poke_result.noun_space(); + let root = unsafe { *poke_result.root() }; + let mut success = false; + let effects = + HoonList::try_from(root, &poke_space).map_err(|e| Box::new(e) as Box)?; + for effect in effects { + if let Ok(effect_cell) = effect.in_space(&poke_space).as_cell() { + if effect_cell.head().eq_bytes("mine-result") { + if let Ok([status, _rest]) = effect_cell.tail().uncell() { + if let Ok(status_atom) = status.as_atom() { + if let Ok(value) = status_atom.as_u64() { + if value == 0 { + success = true; + break; + } + } + } + } + } + } + } + + assert!(success, "open prover mine-result did not succeed"); + serf.cancel_token.cancel(); + Ok(()) +} diff --git a/crates/nockup/Cargo.toml b/crates/nockup/Cargo.toml index 384289aeb..1b6790367 100644 --- a/crates/nockup/Cargo.toml +++ b/crates/nockup/Cargo.toml @@ -11,6 +11,7 @@ documentation = "https://docs.nockchain.org/" readme = "README.md" keywords = ["nockchain", "nockapp", "development", "cli"] categories = ["command-line-utilities", "development-tools"] +exclude = ["templates/*/Cargo.toml"] [package.metadata.docs.rs] all-features = true diff --git a/crates/nockup/src/commands/build/init.rs b/crates/nockup/src/commands/build/init.rs index 72342355a..976038af9 100644 --- a/crates/nockup/src/commands/build/init.rs +++ b/crates/nockup/src/commands/build/init.rs @@ -129,7 +129,11 @@ fn copy_dir_recursive( let entry = entry?; let src_path = entry.path(); let file_name = entry.file_name(); - let dest_path = dest_dir.join(&file_name); + let file_name_str = file_name.to_string_lossy(); + let dest_file_name = file_name_str + .strip_suffix(".template") + .unwrap_or(file_name_str.as_ref()); + let dest_path = dest_dir.join(dest_file_name); if src_path.is_dir() { fs::create_dir_all(&dest_path)?; diff --git a/crates/nockup/src/commands/init.rs b/crates/nockup/src/commands/init.rs index 68673110a..6444b5713 100644 --- a/crates/nockup/src/commands/init.rs +++ b/crates/nockup/src/commands/init.rs @@ -159,7 +159,11 @@ fn copy_dir_recursive( let entry = entry?; let src_path = entry.path(); let file_name = entry.file_name(); - let dest_path = dest_dir.join(&file_name); + let file_name_str = file_name.to_string_lossy(); + let dest_file_name = file_name_str + .strip_suffix(".template") + .unwrap_or(file_name_str.as_ref()); + let dest_path = dest_dir.join(dest_file_name); if src_path.is_dir() { // Create subdirectory and recurse diff --git a/crates/nockup/templates/basic/Cargo.toml b/crates/nockup/templates/basic/Cargo.toml.template similarity index 100% rename from crates/nockup/templates/basic/Cargo.toml rename to crates/nockup/templates/basic/Cargo.toml.template diff --git a/crates/nockvm/rust/ibig/Cargo.toml b/crates/nockvm/rust/ibig/Cargo.toml index d28da5236..e0c3b4f54 100644 --- a/crates/nockvm/rust/ibig/Cargo.toml +++ b/crates/nockvm/rust/ibig/Cargo.toml @@ -31,17 +31,14 @@ std = [] workspace = true [dependencies.num-traits] -default-features = false optional = true version = "0.2" [dependencies.rand] -default-features = false optional = true version = "0.9.2" [dependencies.serde] -default-features = false features = ["derive"] optional = true version = "1.0.217" diff --git a/crates/nockvm/rust/ibig/src/serde.rs b/crates/nockvm/rust/ibig/src/serde.rs index 64ba1990b..a07f98c5e 100644 --- a/crates/nockvm/rust/ibig/src/serde.rs +++ b/crates/nockvm/rust/ibig/src/serde.rs @@ -1,7 +1,7 @@ use alloc::vec::Vec; use core::fmt::{self, Formatter}; -use serde::de::{Deserialize, Deserializer, SeqAccess, Visitor}; +use serde::de::{Deserialize, Deserializer, Error as DeError, SeqAccess, Visitor}; use serde::ser::{Serialize, SerializeSeq, Serializer}; use static_assertions::const_assert; @@ -52,24 +52,18 @@ impl<'de> Visitor<'de> for UBigVisitor { Ok(UBig::from_word(0)) } Some(1) => { - let word_64: u64 = seq.next_element()?.expect(&format!( - "Called `expect()` at {}:{} (git sha: {})", - file!(), - line!(), - env!("GIT_SHA") - )); + let word_64: u64 = seq + .next_element()? + .ok_or_else(|| A::Error::invalid_length(0, &self))?; assert!(seq.next_element::()?.is_none()); Ok(UBig::from(word_64)) } Some(num_words_64) => { let mut buffer = Buffer::allocate(len_64_to_max_len(num_words_64)); - for _ in 0..num_words_64 { - let word_64: u64 = seq.next_element()?.expect(&format!( - "Called `expect()` at {}:{} (git sha: {})", - file!(), - line!(), - env!("GIT_SHA") - )); + for index in 0..num_words_64 { + let word_64: u64 = seq + .next_element()? + .ok_or_else(|| A::Error::invalid_length(index, &self))?; push_word_64(&mut buffer, word_64); } assert!(seq.next_element::()?.is_none()); diff --git a/crates/nockvm/rust/nockvm/Cargo.toml b/crates/nockvm/rust/nockvm/Cargo.toml index 689aa9124..c84e67317 100644 --- a/crates/nockvm/rust/nockvm/Cargo.toml +++ b/crates/nockvm/rust/nockvm/Cargo.toml @@ -21,6 +21,7 @@ check_junior = [] sham_hints = [] stop_for_debug = [] hint_dont = [] +pma-assert = [] # Please keep these alphabetized [dependencies] @@ -42,6 +43,7 @@ num-traits = { workspace = true } rand = { workspace = true } serde = { workspace = true } slotmap = { workspace = true } +smallvec = { workspace = true } static_assertions = { workspace = true } thiserror = { workspace = true } tracing.workspace = true @@ -53,6 +55,8 @@ cc = "1.0" [dev-dependencies] criterion = { workspace = true } +proptest = { workspace = true } +tempfile = { workspace = true } [lints.clippy] missing_safety_doc = "allow" @@ -60,3 +64,11 @@ missing_safety_doc = "allow" [[bench]] name = "hoonc_hotspots" harness = false + +[[bench]] +name = "pma_growth" +harness = false + +[[bench]] +name = "retag_noun_tree" +harness = false diff --git a/crates/nockvm/rust/nockvm/benches/cue_pill.rs b/crates/nockvm/rust/nockvm/benches/cue_pill.rs index 34f0e4624..e9157aa0e 100644 --- a/crates/nockvm/rust/nockvm/benches/cue_pill.rs +++ b/crates/nockvm/rust/nockvm/benches/cue_pill.rs @@ -12,7 +12,7 @@ fn main() -> io::Result<()> { let output_filename = format!("{}.out", filename); let f = File::open(filename)?; let in_len = f.metadata()?.len(); - let mut stack = NockStack::new(1 << 10 << 10 << 10, 0); + let mut stack = NockStack::new(nockvm::mem::NOCK_STACK_SIZE_TINY, 0); let jammed_input = unsafe { let in_map = memmap2::Mmap::map(&f)?; let word_len = (in_len + 7) >> 3; @@ -20,7 +20,7 @@ fn main() -> io::Result<()> { write_bytes(dest.add(word_len as usize - 1), 0, 8); copy_nonoverlapping(in_map.as_ptr(), dest as *mut u8, in_len as usize); mem::drop(in_map); - atom.normalize_as_atom() + atom.normalize_as_atom_stack() }; let now = SystemTime::now(); @@ -52,6 +52,8 @@ fn main() -> io::Result<()> { let nuw = SystemTime::now(); let jammed_output = jam(&mut stack, input); + let space = stack.noun_space(); + let jammed_output = jammed_output.in_space(&space); match nuw.elapsed() { Ok(elapse) => { @@ -69,7 +71,7 @@ fn main() -> io::Result<()> { unsafe { let mut out_map = memmap2::MmapMut::map_mut(&f_out)?; copy_nonoverlapping( - jammed_output.data_pointer() as *mut u8, + jammed_output.data_pointer() as *const u8, out_map.as_mut_ptr(), jammed_output.size() << 3, ); diff --git a/crates/nockvm/rust/nockvm/benches/hoonc_hotspots.rs b/crates/nockvm/rust/nockvm/benches/hoonc_hotspots.rs index be923b9b5..6a4a91095 100644 --- a/crates/nockvm/rust/nockvm/benches/hoonc_hotspots.rs +++ b/crates/nockvm/rust/nockvm/benches/hoonc_hotspots.rs @@ -22,7 +22,8 @@ impl Slogger for BenchSlogger { } fn bench_context() -> Context { - let mut stack = NockStack::new(8 << 20, 0); + let mut stack = NockStack::new(nockvm::mem::NOCK_STACK_SIZE_TINY, 0); + let arena = stack.arena().clone(); let cold = Cold::new(&mut stack); let warm = Warm::new(&mut stack); let hot = Hot::init(&mut stack, URBIT_HOT_STATE); @@ -42,6 +43,7 @@ fn bench_context() -> Context { trace_info: None, running_status: cancel, test_jets, + arena, } } @@ -66,7 +68,7 @@ fn bench_hamt_symbol_table(c: &mut Criterion) { c.bench_function("hamt_symbol_table_hot_path", |b| { b.iter_batched( || { - let mut stack = NockStack::new(16 << 20, 0); + let mut stack = NockStack::new(nockvm::mem::NOCK_STACK_SIZE_TINY, 0); let (table, keys) = build_symbol_table(&mut stack, 256); (stack, table, keys) }, @@ -111,7 +113,7 @@ fn bench_noun_preserve(c: &mut Criterion) { c.bench_function("noun_preserve_deep_core", |b| { b.iter_batched( || { - let mut stack = NockStack::new(24 << 20, 0); + let mut stack = NockStack::new(nockvm::mem::NOCK_STACK_SIZE_TINY, 0); let ast = build_deep_ast(&mut stack, 160, 4); (stack, ast) }, @@ -183,7 +185,7 @@ fn bench_interpret_hint_case(c: &mut Criterion) { let outcome = interpret(&mut ctx, subj, form); black_box(&outcome); }, - BatchSize::SmallInput, + BatchSize::LargeInput, ); }); } @@ -206,7 +208,7 @@ fn bench_unifying_equality(c: &mut Criterion) { c.bench_function("unifying_equality_canopy", |b| { b.iter_batched( || { - let mut stack = NockStack::new(24 << 20, 0); + let mut stack = NockStack::new(nockvm::mem::NOCK_STACK_SIZE_TINY, 0); let mut pairs = Vec::with_capacity(48); for i in 0..48 { let left = build_balanced_tree(&mut stack, 6, i as u64 + 1); @@ -228,7 +230,7 @@ fn bench_unifying_equality(c: &mut Criterion) { black_box((first, second)); } }, - BatchSize::SmallInput, + BatchSize::LargeInput, ); }); } @@ -251,7 +253,7 @@ fn bench_cue_jam_roundtrip(c: &mut Criterion) { c.bench_function("cue_jam_roundtrip", |b| { b.iter_batched( || { - let mut stack = NockStack::new(32 << 20, 0); + let mut stack = NockStack::new(nockvm::mem::NOCK_STACK_SIZE_TINY, 0); let noun = build_serialization_fixture(&mut stack); let jammed = jam(&mut stack, noun); (stack, jammed) @@ -261,7 +263,7 @@ fn bench_cue_jam_roundtrip(c: &mut Criterion) { let rejam = jam(&mut stack, decoded); black_box(rejam); }, - BatchSize::SmallInput, + BatchSize::LargeInput, ); }); } @@ -285,7 +287,7 @@ fn bench_warm_lookup(c: &mut Criterion) { let miss = warm.find_jet(&mut ctx.stack, &mut subject, &mut bogus_formula); black_box((hit, miss)); }, - BatchSize::SmallInput, + BatchSize::LargeInput, ); }); } @@ -320,7 +322,7 @@ fn bench_cache_churn(c: &mut Criterion) { ctx.cache = cache; black_box(ctx.cache.is_null()); }, - BatchSize::SmallInput, + BatchSize::LargeInput, ); }); } diff --git a/crates/nockvm/rust/nockvm/benches/pma_growth.rs b/crates/nockvm/rust/nockvm/benches/pma_growth.rs new file mode 100644 index 000000000..3ffd5a68e --- /dev/null +++ b/crates/nockvm/rust/nockvm/benches/pma_growth.rs @@ -0,0 +1,88 @@ +use criterion::{black_box, criterion_group, criterion_main, BatchSize, Criterion}; +use nockvm::mem::NockStack; +use nockvm::noun::{Cell, Noun, D}; +use nockvm::pma::{Pma, PmaCopy}; +use tempfile::TempDir; + +const CELL_WORDS: usize = 3; +const LIST_CELLS: usize = 4096; +const EXPECTED_WORDS: usize = LIST_CELLS * CELL_WORDS; +const STACK_WORDS: usize = EXPECTED_WORDS * 2 + 1024; + +fn build_direct_atom_list(stack: &mut NockStack, len: usize) -> Noun { + let mut noun = D(0); + for value in (0..len).rev() { + noun = Cell::new(stack, D(value as u64), noun).as_noun(); + } + noun +} + +fn copy_fixture(pma_words: usize, reserved_words: usize) -> (TempDir, NockStack, Pma, Noun) { + let dir = TempDir::new().expect("create PMA benchmark dir"); + let path = dir.path().join("bench.pma"); + let mut stack = NockStack::new(STACK_WORDS, 0); + let noun = build_direct_atom_list(&mut stack, LIST_CELLS); + let pma = Pma::new_with_reserved(pma_words, reserved_words, path).expect("create PMA"); + (dir, stack, pma, noun) +} + +fn populated_pma_fixture() -> (TempDir, std::path::PathBuf) { + let dir = TempDir::new().expect("create PMA open benchmark dir"); + let path = dir.path().join("open.pma"); + { + let mut stack = NockStack::new(STACK_WORDS, 0); + let mut noun = build_direct_atom_list(&mut stack, LIST_CELLS); + let mut pma = Pma::new(EXPECTED_WORDS * 2, path.clone()).expect("create PMA"); + unsafe { + noun.copy_to_pma(&stack, &mut pma); + } + pma.sync_all().expect("sync PMA mapping"); + pma.sync_file().expect("sync PMA file"); + } + (dir, path) +} + +fn bench_pma_copy_no_growth(c: &mut Criterion) { + c.bench_function("pma_copy_direct_list_no_growth", |b| { + b.iter_batched( + || copy_fixture(EXPECTED_WORDS * 2, EXPECTED_WORDS * 4), + |(_dir, stack, mut pma, mut noun)| unsafe { + noun.copy_to_pma(&stack, &mut pma); + black_box(pma.alloc_offset()); + }, + BatchSize::SmallInput, + ); + }); +} + +fn bench_pma_copy_with_growth(c: &mut Criterion) { + c.bench_function("pma_copy_direct_list_with_growth", |b| { + b.iter_batched( + || copy_fixture(1024, EXPECTED_WORDS * 4), + |(_dir, stack, mut pma, mut noun)| unsafe { + noun.copy_to_pma(&stack, &mut pma); + black_box((pma.size_words(), pma.alloc_offset())); + }, + BatchSize::SmallInput, + ); + }); +} + +fn bench_pma_open(c: &mut Criterion) { + let (_dir, path) = populated_pma_fixture(); + c.bench_function("pma_open_populated", |b| { + b.iter(|| { + let pma = Pma::open(black_box(path.clone())).expect("open PMA"); + black_box((pma.size_words(), pma.alloc_offset())); + }); + }); +} + +fn criterion_benchmark(c: &mut Criterion) { + bench_pma_copy_no_growth(c); + bench_pma_copy_with_growth(c); + bench_pma_open(c); +} + +criterion_group!(pma_growth, criterion_benchmark); +criterion_main!(pma_growth); diff --git a/crates/nockvm/rust/nockvm/benches/retag_noun_tree.rs b/crates/nockvm/rust/nockvm/benches/retag_noun_tree.rs new file mode 100644 index 000000000..e8b6d593d --- /dev/null +++ b/crates/nockvm/rust/nockvm/benches/retag_noun_tree.rs @@ -0,0 +1,475 @@ +use std::collections::HashSet; +use std::fs; +use std::path::PathBuf; +use std::time::{Duration, Instant}; + +use criterion::{black_box, criterion_group, criterion_main, BenchmarkId, Criterion}; +use lazy_static::lazy_static; +use nockvm::ext::{IndirectAtomExt, NounExt}; +use nockvm::mem::{NockStack, NOCK_STACK_SIZE_TINY}; +use nockvm::noun::{AllocLocation, Cell, IndirectAtom, Noun, D}; +use nockvm::pma::{Pma, PmaCopy}; +use nockvm::serialization::cue_into_stack_pointer_form; +use rand::rngs::StdRng; +use rand::{Rng, SeedableRng}; +use tempfile::TempDir; + +const PMA_WORDS: usize = 1 << 25; // 256 MiB PMA arena for synthetic cases +const LEAF_COUNT: usize = 5_120; +const RANDOM_SEED_BASE: u64 = 0x5eed_ba11_0000_0000; +const DUMB_JAM_PATH: &str = "../../../../assets/dumb.jam"; + +#[derive(Clone, Copy)] +enum TreeShape { + Balanced, + RightAssoc, + Random(u64), +} + +#[derive(Clone, Copy)] +enum OffsetMix { + Stack100, + Stack50, + Stack90, + Stack10, +} + +fn make_stack() -> NockStack { + NockStack::new(NOCK_STACK_SIZE_TINY, 0) +} + +#[allow(dead_code)] +fn make_kernel_stack() -> NockStack { + NockStack::new(NOCK_STACK_SIZE_TINY, 0) +} + +fn make_pma(size_words: usize) -> (TempDir, Pma) { + let dir = TempDir::new().expect("create temporary PMA directory"); + let path = dir.path().join("bench.pma"); + let pma = Pma::new(size_words, path).expect("create benchmark PMA"); + (dir, pma) +} + +fn small_indirect(stack: &mut NockStack, seed: u64) -> Noun { + let mut buf = [0u8; 24]; + buf[..8].copy_from_slice(&seed.to_le_bytes()); + buf[8..16].copy_from_slice(&seed.wrapping_mul(0x9e37_79b9_7f4a_7c15).to_le_bytes()); + buf[16..24].copy_from_slice(&(seed ^ 0x55aa_55aa_55aa_55aa).to_le_bytes()); + unsafe { IndirectAtom::new_raw_bytes(stack, buf.len(), buf.as_ptr()).as_noun() } +} + +fn large_indirect(stack: &mut NockStack, seed: u64, words: usize) -> Noun { + debug_assert!((5..=1000).contains(&words)); + let mut data = vec![0u64; words]; + for (idx, slot) in data.iter_mut().enumerate() { + *slot = seed + .wrapping_mul(0x9e37_79b9_7f4a_7c15) + .wrapping_add(idx as u64 ^ 0xfeed_face_dead_beef); + } + // Keep the tail non-zero so normalization does not shrink the atom. + data[words - 1] |= 1; + let bytes = unsafe { std::slice::from_raw_parts(data.as_ptr() as *const u8, data.len() << 3) }; + let atom = unsafe { IndirectAtom::new_raw_bytes(stack, bytes.len(), bytes.as_ptr()) }; + atom.as_noun() +} + +fn build_tree_balanced(stack: &mut NockStack, mut leaves: Vec) -> Noun { + assert!(!leaves.is_empty()); + while leaves.len() > 1 { + let mut next = Vec::with_capacity(leaves.len().div_ceil(2)); + for chunk in leaves.chunks(2) { + if chunk.len() == 2 { + let cell = Cell::new(stack, chunk[0], chunk[1]); + next.push(cell.as_noun()); + } else { + next.push(chunk[0]); + } + } + leaves = next; + } + leaves.pop().expect("non-empty tree") +} + +fn build_tree_right_assoc(stack: &mut NockStack, leaves: Vec) -> Noun { + assert!(!leaves.is_empty()); + let mut iter = leaves.into_iter().rev(); + let mut acc = iter + .next() + .expect("at least one element when building right-associated tree"); + for leaf in iter { + acc = Cell::new(stack, leaf, acc).as_noun(); + } + acc +} + +fn build_tree_random(stack: &mut NockStack, leaves: Vec, seed: u64) -> Noun { + assert!(!leaves.is_empty()); + let mut rng = StdRng::seed_from_u64(seed); + let mut nodes = leaves; + while nodes.len() > 1 { + let i = rng.random_range(0..nodes.len()); + let a = nodes.swap_remove(i); + let j = rng.random_range(0..nodes.len()); + let b = nodes.swap_remove(j); + let cell = Cell::new(stack, a, b).as_noun(); + nodes.push(cell); + } + nodes.pop().expect("non-empty tree") +} + +fn build_tree_with_shape(stack: &mut NockStack, leaves: Vec, shape: TreeShape) -> Noun { + match shape { + TreeShape::Balanced => build_tree_balanced(stack, leaves), + TreeShape::RightAssoc => build_tree_right_assoc(stack, leaves), + TreeShape::Random(seed) => build_tree_random(stack, leaves, seed), + } +} + +fn apply_offset_mix(stack: &NockStack, pma: &mut Pma, leaves: &mut [Noun], mix: OffsetMix) { + let offset_target = match mix { + OffsetMix::Stack100 => 0usize, + OffsetMix::Stack50 => 50usize, + OffsetMix::Stack90 => 10usize, + OffsetMix::Stack10 => 90usize, + }; + if offset_target == 0 || leaves.is_empty() { + return; + } + + let leaf_count = (leaves.len() * offset_target) / 100; + for leaf in leaves.iter_mut().take(leaf_count) { + unsafe { leaf.copy_to_pma(stack, pma) }; + } +} + +fn build_unique_direct(_stack: &mut NockStack) -> Vec { + (0..LEAF_COUNT).map(|i| D(i as u64 + 1)).collect() +} + +fn build_unique_small_indirect(stack: &mut NockStack) -> Vec { + (0..LEAF_COUNT) + .map(|i| small_indirect(stack, 0x1000_0000 + i as u64)) + .collect() +} + +fn build_mixed_direct_small_indirect(stack: &mut NockStack) -> Vec { + (0..LEAF_COUNT) + .map(|i| { + if i % 2 == 0 { + D(i as u64 + 1) + } else { + small_indirect(stack, 0x2000_0000 + i as u64) + } + }) + .collect() +} + +fn build_shared_small_indirect(stack: &mut NockStack) -> Vec { + let shared = small_indirect(stack, 0x5eed_baa5); + let uniques = LEAF_COUNT / 10; + let mut leaves = Vec::with_capacity(LEAF_COUNT); + for i in 0..LEAF_COUNT { + if i % 10 == 0 && (i / 10) < uniques { + leaves.push(small_indirect(stack, 0x3000_0000 + (i / 10) as u64)); + } else { + leaves.push(shared); + } + } + leaves +} + +fn build_unique_large_indirect(stack: &mut NockStack) -> Vec { + (0..LEAF_COUNT) + .map(|i| { + let size = 5 + (i % 996); + large_indirect(stack, 0x4000_0000 + i as u64, size) + }) + .collect() +} + +fn build_shared_large_indirect(stack: &mut NockStack) -> Vec { + let shared = large_indirect(stack, 0x5aa5_5aa5, 768); + let mut leaves = Vec::with_capacity(LEAF_COUNT); + for i in 0..LEAF_COUNT { + if i % 10 == 0 { + let size = 5 + (i % 996); + leaves.push(large_indirect(stack, 0x5000_0000 + (i / 10) as u64, size)); + } else { + leaves.push(shared); + } + } + leaves +} + +lazy_static! { + static ref DUMB_JAM_BYTES: Vec = { + let path = PathBuf::from(env!("CARGO_MANIFEST_DIR")).join(DUMB_JAM_PATH); + fs::read(&path).unwrap_or_else(|e| panic!("failed to read {:?}: {}", path, e)) + }; +} + +#[allow(dead_code)] +fn load_kernel_dumb(stack: &mut NockStack) -> Noun { + Noun::cue_bytes_slice(stack, &DUMB_JAM_BYTES).expect("cue dumb.jam") +} + +/// Load the kernel into stack-pointer form (for benchmarking retag_noun_tree) +#[allow(dead_code)] +fn load_kernel_dumb_stack_pointer_form(stack: &mut NockStack) -> Noun { + let buffer = IndirectAtom::from_bytes(stack, &DUMB_JAM_BYTES); + cue_into_stack_pointer_form(stack, buffer).expect("cue dumb.jam into stack-pointer form") +} + +/// Sanity check: returns true if all allocated nouns in the tree are in stack-pointer form +/// (i.e., is_stack_allocated() returns true for all indirect atoms and cells). +/// Direct atoms are ignored since they have no allocation. +/// Returns (is_valid, stack_pointer_count, offset_count) for debugging +#[allow(dead_code)] +fn check_noun_tagging_state( + stack: &NockStack, + pma: Option<&Pma>, + root: Noun, +) -> (bool, usize, usize) { + let space = pma.map_or_else( + || stack.noun_space(), + |pma| nockvm::noun::NounSpace::new(stack, pma), + ); + let mut work: Vec = Vec::with_capacity(32); + let mut visited: HashSet = HashSet::new(); + let mut stack_pointer_count = 0usize; + let mut offset_count = 0usize; + work.push(root); + + while let Some(noun) = work.pop() { + // Direct atoms have no allocation, skip them + if noun.is_direct() { + continue; + } + + // Check for duplicate visits (structural sharing) + let raw = unsafe { noun.as_raw() }; + if !visited.insert(raw) { + continue; + } + + // For allocated nouns (cells and indirect atoms), count their form + if noun.is_allocated() { + if matches!( + noun.in_space(&space).allocated_location(), + Some(AllocLocation::Stack) + ) { + stack_pointer_count += 1; + } else { + offset_count += 1; + } + } + + // If it's a cell, traverse children + if let Ok(cell) = noun.in_space(&space).as_cell() { + let head = cell.head().noun(); + let tail = cell.tail().noun(); + work.push(head); + work.push(tail); + } + } + + let all_stack_pointer = offset_count == 0; + (all_stack_pointer, stack_pointer_count, offset_count) +} + +/// Returns true if all allocated nouns are in stack-pointer form +#[allow(dead_code)] +fn is_entirely_stack_pointer_form(stack: &NockStack, root: Noun) -> bool { + let (all_stack_pointer, _, _) = check_noun_tagging_state(stack, None, root); + all_stack_pointer +} + +/// Sanity check: returns true if all allocated nouns in the tree are in offset form +/// (i.e., is_stack_allocated() returns false for all indirect atoms and cells). +/// Direct atoms are ignored since they have no allocation. +#[allow(dead_code)] +fn is_entirely_offset_form(stack: &NockStack, root: Noun) -> bool { + let space = stack.noun_space(); + let mut work: Vec = Vec::with_capacity(32); + let mut visited: HashSet = HashSet::new(); + work.push(root); + + while let Some(noun) = work.pop() { + // Direct atoms have no allocation, skip them + if noun.is_direct() { + continue; + } + + // Check for duplicate visits (structural sharing) + let raw = unsafe { noun.as_raw() }; + if !visited.insert(raw) { + continue; + } + + // For allocated nouns (cells and indirect atoms), check they're in offset form + if noun.is_allocated() + && matches!( + noun.in_space(&space).allocated_location(), + Some(AllocLocation::Stack) + ) + { + // This noun is in stack-pointer form, not offset form + return false; + } + + // If it's a cell, traverse children + if let Ok(cell) = noun.in_space(&space).as_cell() { + let head = cell.head().noun(); + let tail = cell.tail().noun(); + work.push(head); + work.push(tail); + } + } + + true +} + +struct BenchCase { + _pma_dir: TempDir, + pma: Pma, + stack: NockStack, + root: Noun, +} + +type BaseCase = (&'static str, fn(&mut NockStack) -> Vec); + +fn setup_case(mut make_leaves: F, shape: TreeShape, mix: OffsetMix) -> impl FnMut() -> BenchCase +where + F: FnMut(&mut NockStack) -> Vec, +{ + move || { + let mut stack = make_stack(); + let (pma_dir, mut pma) = make_pma(PMA_WORDS); + let mut leaves = make_leaves(&mut stack); + assert!(leaves.len() >= LEAF_COUNT); + apply_offset_mix(&stack, &mut pma, &mut leaves, mix); + let root = build_tree_with_shape(&mut stack, leaves, shape); + BenchCase { + _pma_dir: pma_dir, + pma, + stack, + root, + } + } +} + +fn bench_retag_noun_tree(c: &mut Criterion) { + let mut group = c.benchmark_group("retag_noun_tree"); + let kernel_only = std::env::var_os("NOCKCHAIN_KERNEL_ONLY").is_some(); + let mut cases: Vec<(String, Box BenchCase>)> = Vec::new(); + + if !kernel_only { + let base_cases: Vec = vec![ + ("direct_unique", build_unique_direct), + ("small_indirect_unique", build_unique_small_indirect), + ( + "mixed_direct_small_indirect", build_mixed_direct_small_indirect, + ), + ("small_indirect_shared", build_shared_small_indirect), + ("large_indirect_unique", build_unique_large_indirect), + ("large_indirect_shared", build_shared_large_indirect), + ]; + + let mixes = [ + (OffsetMix::Stack100, "stack100"), + (OffsetMix::Stack50, "stack50"), + (OffsetMix::Stack90, "stack90"), + (OffsetMix::Stack10, "stack10"), + ]; + + for (case_idx, (label, leaves_fn)) in base_cases.into_iter().enumerate() { + let shapes = [ + (TreeShape::Balanced, "balanced"), + (TreeShape::RightAssoc, "right_assoc"), + ( + TreeShape::Random(RANDOM_SEED_BASE ^ (case_idx as u64)), + "random", + ), + ]; + for (shape, suffix) in shapes { + for (mix, mix_name) in mixes { + let name = format!("{label}_{suffix}_{mix_name}"); + cases.push((name, Box::new(setup_case(leaves_fn, shape, mix)))); + } + } + } + } + + for (label, setup) in cases.iter_mut() { + let name = label.clone(); + group.bench_function(BenchmarkId::new("retag", name), |b| { + // Only measure retag_noun_tree time; setup/build happens outside the timed section. + b.iter_custom(|iters| { + let mut total = Duration::ZERO; + for _ in 0..iters { + let mut case = setup(); + let start = Instant::now(); + unsafe { case.root.copy_to_pma(&case.stack, &mut case.pma) }; + black_box(&case.root); + total += start.elapsed(); + } + total + }); + }); + } + + // Kernel benchmark: retag_noun_tree + // This tests the speed of converting a pointer-form noun to offset form + // using the retag_noun_tree function. + group.bench_function(BenchmarkId::new("kernel", "retag_noun_tree"), |b| { + b.iter_custom(|iters| { + let mut total = Duration::ZERO; + for _ in 0..iters { + // Setup: cue the kernel into stack-pointer form + let mut stack = make_kernel_stack(); + let (_pma_dir, mut pma) = make_pma(NOCK_STACK_SIZE_TINY); + let mut kernel_ptr_form = load_kernel_dumb_stack_pointer_form(&mut stack); + + // Sanity check: kernel should be entirely in stack-pointer form before retagging + let (all_stack, stack_count, offset_count) = + check_noun_tagging_state(&stack, Some(&pma), kernel_ptr_form); + assert!( + all_stack, + "Kernel should be in stack-pointer form before retag_noun_tree. \ + Found {} stack-pointer nouns and {} offset nouns", + stack_count, offset_count + ); + + // Timed section: retag_noun_tree + let start = Instant::now(); + unsafe { kernel_ptr_form.copy_to_pma(&stack, &mut pma) }; + black_box(&kernel_ptr_form); + total += start.elapsed(); + + // Sanity check: kernel should be entirely in offset form after retagging + let (_, stack_count_after, offset_count_after) = + check_noun_tagging_state(&stack, Some(&pma), kernel_ptr_form); + assert!( + stack_count_after == 0, + "Kernel should be in offset form after retag_noun_tree. \ + Found {} stack-pointer nouns and {} offset nouns", + stack_count_after, + offset_count_after + ); + } + total + }); + }); + + group.finish(); +} + +criterion_group! { + name = benches; + config = Criterion::default() + .measurement_time(Duration::from_millis(700)) + .warm_up_time(Duration::from_millis(200)) + .sample_size(10); + targets = bench_retag_noun_tree +} +criterion_main!(benches); diff --git a/crates/nockvm/rust/nockvm/src/ext.rs b/crates/nockvm/rust/nockvm/src/ext.rs index 313dfe147..2f7be2ba5 100644 --- a/crates/nockvm/rust/nockvm/src/ext.rs +++ b/crates/nockvm/rust/nockvm/src/ext.rs @@ -1,49 +1,22 @@ -use std::str; - use bincode::{Decode, Encode}; use bytes::Bytes; +use either::Either::{Left, Right}; +use intmap::IntMap; use crate::interpreter::Error; use crate::mem::NockStack; -use crate::noun::{Atom, IndirectAtom, Noun, NounAllocator, D}; +use crate::noun::{Atom, IndirectAtom, Noun, NounAllocator, NounHandle}; use crate::serialization::{cue, jam}; /// Convenience helpers for working with `Atom`. pub trait AtomExt { fn from_bytes(allocator: &mut A, bytes: &[u8]) -> Atom; - fn eq_bytes>(&self, bytes: B) -> bool; - fn to_bytes_until_nul(&self) -> std::result::Result, str::Utf8Error>; - fn into_string(self) -> std::result::Result; } impl AtomExt for Atom { fn from_bytes(allocator: &mut A, bytes: &[u8]) -> Atom { ::from_bytes(allocator, bytes) } - - fn eq_bytes>(&self, bytes: B) -> bool { - let bytes_ref = bytes.as_ref(); - let atom_bytes = self.as_ne_bytes(); - if bytes_ref.len() > atom_bytes.len() { - return false; - } - if bytes_ref.len() == atom_bytes.len() { - return atom_bytes == bytes_ref; - } - if atom_bytes[bytes_ref.len()..].iter().any(|b| *b != 0) { - return false; - } - &atom_bytes[0..bytes_ref.len()] == bytes_ref - } - - fn to_bytes_until_nul(&self) -> std::result::Result, str::Utf8Error> { - str::from_utf8(self.as_ne_bytes()) - .map(|bytes| bytes.trim_end_matches('\0').as_bytes().to_vec()) - } - - fn into_string(self) -> std::result::Result { - str::from_utf8(self.as_ne_bytes()).map(|string| string.trim_end_matches('\0').to_string()) - } } /// Extension helpers for safely constructing indirect atoms. @@ -67,7 +40,8 @@ impl IndirectAtomExt for IndirectAtom { size: usize, data: *const u8, ) -> Atom { - Self::new_raw_bytes(allocator, size, data).normalize_as_atom() + // Use normalize_as_atom_stack since new_raw_bytes creates stack-pointer form atoms + Self::new_raw_bytes(allocator, size, data).normalize_as_atom_stack() } } @@ -76,8 +50,6 @@ pub trait NounExt { fn cue_bytes(stack: &mut NockStack, bytes: &Bytes) -> std::result::Result; fn cue_bytes_slice(stack: &mut NockStack, bytes: &[u8]) -> std::result::Result; fn jam_self(self, stack: &mut NockStack) -> JammedNoun; - fn list_iter(self) -> NounListIterator; - fn eq_bytes(self, bytes: impl AsRef<[u8]>) -> bool; } impl NounExt for Noun { @@ -94,18 +66,6 @@ impl NounExt for Noun { fn jam_self(self, stack: &mut NockStack) -> JammedNoun { JammedNoun::from_noun(stack, self) } - - fn list_iter(self) -> NounListIterator { - NounListIterator(self) - } - - fn eq_bytes(self, bytes: impl AsRef<[u8]>) -> bool { - if let Ok(atom) = self.as_atom() { - atom.eq_bytes(bytes) - } else { - false - } - } } #[derive(Clone, PartialEq, Debug, Encode, Decode)] @@ -118,7 +78,10 @@ impl JammedNoun { pub fn from_noun(stack: &mut NockStack, noun: Noun) -> Self { let jammed_atom = jam(stack, noun); - JammedNoun(Bytes::copy_from_slice(jammed_atom.as_ne_bytes())) + let space = stack.noun_space(); + JammedNoun(Bytes::copy_from_slice( + jammed_atom.in_space(&space).as_ne_bytes(), + )) } pub fn cue_self(&self, stack: &mut NockStack) -> std::result::Result { @@ -157,23 +120,277 @@ impl Default for JammedNoun { } } -pub struct NounListIterator(Noun); +pub fn make_tas(allocator: &mut A, tas: &str) -> Atom { + ::from_bytes(allocator, tas.as_bytes()) +} + +/// Non-unifying structural equality for nouns. +/// +/// Compares two nouns for structural equality without modifying them +/// (unlike unifying equality which may merge identical substructures). +/// This is suitable for use with allocators that don't support unification +/// (e.g., Pma, NounSlab) since it doesn't require temporary allocations. +/// +/// Uses a worklist algorithm to avoid stack overflow on deep structures. +/// Tracks already-compared pairs to handle structural sharing efficiently. +/// Uses cached mugs (hashes) to quickly reject unequal nouns. +pub fn noun_equality(a: NounHandle, b: NounHandle) -> bool { + let space = a.space(); + // Track pairs we've already determined to be equal + // Key is a_raw << 64 | b_raw (or b_raw << 64 | a_raw for symmetry) + let mut already_equal: IntMap = IntMap::new(); -impl Iterator for NounListIterator { - type Item = Noun; + fn ae_keys(a: Noun, b: Noun) -> (u128, u128) { + let a_raw = unsafe { a.as_raw() } as u128; + let b_raw = unsafe { b.as_raw() } as u128; + (a_raw << 64 | b_raw, b_raw << 64 | a_raw) + } + + fn check_ae(ae: &IntMap, a: Noun, b: Noun) -> bool { + let (key1, key2) = ae_keys(a, b); + ae.contains_key(key1) || ae.contains_key(key2) + } - fn next(&mut self) -> Option { - if let Ok(cell) = self.0.as_cell() { - self.0 = cell.tail(); - Some(cell.head()) - } else if unsafe { self.0.raw_equals(&D(0)) } { - None - } else { - panic!("Improper list terminator: {:?}", self.0); + fn set_ae(ae: &mut IntMap, a: Noun, b: Noun) { + let (key1, _key2) = ae_keys(a, b); + ae.insert(key1, ()); + } + + // Stack entries: either comparing two nouns, or marking a cell pair as equal after children match + enum StackEntry { + Nouns(Noun, Noun), + MarkEqual(Noun, Noun), + } + + let mut stack = vec![StackEntry::Nouns(a.noun(), b.noun())]; + + loop { + let Some(entry) = stack.pop() else { + // Stack empty means all comparisons succeeded + return true; + }; + + match entry { + StackEntry::MarkEqual(a, b) => { + // Children matched, mark this pair as equal + set_ae(&mut already_equal, a, b); + } + StackEntry::Nouns(a, b) => { + // Quick check: identical raw values are equal + if unsafe { a.raw_equals(&b) } { + continue; + } + + // Already compared this pair? + if check_ae(&already_equal, a, b) { + continue; + } + + match ( + a.as_ref_either_direct_allocated(), + b.as_ref_either_direct_allocated(), + ) { + (Right(a_alloc), Right(b_alloc)) => { + // Both allocated - check mugs first for quick rejection + if let Some(a_mug) = a_alloc.get_cached_mug(space) { + if let Some(b_mug) = b_alloc.get_cached_mug(space) { + if a_mug != b_mug { + return false; + } + } + } + + match (a_alloc.as_ref_either(), b_alloc.as_ref_either()) { + (Left(a_indirect), Left(b_indirect)) => { + // Both indirect atoms - compare byte slices + let a_handle = a_indirect.as_atom().in_space(space); + let b_handle = b_indirect.as_atom().in_space(space); + if a_handle.as_ne_bytes() != b_handle.as_ne_bytes() { + return false; + } + set_ae(&mut already_equal, a, b); + } + (Right(a_cell), Right(b_cell)) => { + // Both cells - queue children for comparison + // Mark as equal after children are verified + stack.push(StackEntry::MarkEqual(a, b)); + let a_cell = (*a_cell).in_space(space); + let b_cell = (*b_cell).in_space(space); + stack.push(StackEntry::Nouns( + a_cell.tail().noun(), + b_cell.tail().noun(), + )); + stack.push(StackEntry::Nouns( + a_cell.head().noun(), + b_cell.head().noun(), + )); + } + _ => { + // One indirect, one cell - not equal + return false; + } + } + } + _ => { + // At least one direct atom, and raw_equals failed above + return false; + } + } + } } } } -pub fn make_tas(allocator: &mut A, tas: &str) -> Atom { - ::from_bytes(allocator, tas.as_bytes()) +#[cfg(test)] +mod tests { + use super::noun_equality; + use crate::mem::{NockStack, NOCK_STACK_SIZE_TINY}; + use crate::noun::{Cell, IndirectAtom, D}; + + /// Verifies noun_equality correctly compares nouns for structural equality. + /// + /// Tests: + /// - Direct atoms equal themselves + /// - Different direct atoms are not equal + /// - Indirect atoms equal themselves (same data) + /// - Different indirect atoms are not equal + /// - Cells equal themselves + /// - Cells with different contents are not equal + /// - Nested structures + /// - Structural sharing (same substructure referenced twice) + #[test] + fn test_noun_equality() { + let mut stack = NockStack::new(NOCK_STACK_SIZE_TINY, 0); + + // Direct atoms + let d0 = D(0); + let d1 = D(1); + let d42 = D(42); + let d42_copy = D(42); + + // Indirect atoms + let data1: [u64; 2] = [0xDEADBEEF_CAFEBABE, 0x12345678]; + let data2: [u64; 2] = [0xDEADBEEF_CAFEBABE, 0x12345678]; // same data + let data3: [u64; 2] = [0xDEADBEEF_CAFEBABE, 0x87654321]; // different + + let indirect1 = unsafe { IndirectAtom::new_raw(&mut stack, 2, data1.as_ptr()) }.as_noun(); + let indirect2 = unsafe { IndirectAtom::new_raw(&mut stack, 2, data2.as_ptr()) }.as_noun(); + let indirect3 = unsafe { IndirectAtom::new_raw(&mut stack, 2, data3.as_ptr()) }.as_noun(); + + // Simple cells + let cell1 = Cell::new(&mut stack, D(1), D(2)).as_noun(); + let cell2 = Cell::new(&mut stack, D(1), D(2)).as_noun(); + let cell3 = Cell::new(&mut stack, D(1), D(3)).as_noun(); + let cell4 = Cell::new(&mut stack, D(2), D(2)).as_noun(); + + // Nested cells - build inner cells first to avoid borrow issues + let inner1 = Cell::new(&mut stack, D(1), D(2)).as_noun(); + let nested1 = Cell::new(&mut stack, inner1, D(3)).as_noun(); + let inner2 = Cell::new(&mut stack, D(1), D(2)).as_noun(); + let nested2 = Cell::new(&mut stack, inner2, D(3)).as_noun(); + let inner3 = Cell::new(&mut stack, D(1), D(9)).as_noun(); + let nested3 = Cell::new(&mut stack, inner3, D(3)).as_noun(); + + // Structural sharing + let shared = Cell::new(&mut stack, D(5), D(6)).as_noun(); + let with_sharing = Cell::new(&mut stack, shared, shared).as_noun(); + let inner_a = Cell::new(&mut stack, D(5), D(6)).as_noun(); + let inner_b = Cell::new(&mut stack, D(5), D(6)).as_noun(); + let without_sharing = Cell::new(&mut stack, inner_a, inner_b).as_noun(); + + // Cells containing indirect atoms + let cell_indirect1 = Cell::new(&mut stack, indirect1, D(99)).as_noun(); + let cell_indirect2 = Cell::new(&mut stack, indirect2, D(99)).as_noun(); // same indirect data + let cell_indirect3 = Cell::new(&mut stack, indirect3, D(99)).as_noun(); // different indirect data + + let space = stack.noun_space(); + + assert!( + noun_equality(d0.in_space(&space), d0.in_space(&space)), + "D(0) == D(0)" + ); + assert!( + noun_equality(d42.in_space(&space), d42_copy.in_space(&space)), + "D(42) == D(42)" + ); + assert!( + !noun_equality(d0.in_space(&space), d1.in_space(&space)), + "D(0) != D(1)" + ); + assert!( + !noun_equality(d1.in_space(&space), d42.in_space(&space)), + "D(1) != D(42)" + ); + + assert!( + noun_equality(indirect1.in_space(&space), indirect1.in_space(&space)), + "indirect1 == indirect1 (same ref)" + ); + assert!( + noun_equality(indirect1.in_space(&space), indirect2.in_space(&space)), + "indirect1 == indirect2 (same data)" + ); + assert!( + !noun_equality(indirect1.in_space(&space), indirect3.in_space(&space)), + "indirect1 != indirect3 (different data)" + ); + assert!( + !noun_equality(indirect1.in_space(&space), d42.in_space(&space)), + "indirect != direct" + ); + + assert!( + noun_equality(cell1.in_space(&space), cell1.in_space(&space)), + "[1 2] == [1 2] (same ref)" + ); + assert!( + noun_equality(cell1.in_space(&space), cell2.in_space(&space)), + "[1 2] == [1 2] (different refs)" + ); + assert!( + !noun_equality(cell1.in_space(&space), cell3.in_space(&space)), + "[1 2] != [1 3]" + ); + assert!( + !noun_equality(cell1.in_space(&space), cell4.in_space(&space)), + "[1 2] != [2 2]" + ); + assert!( + !noun_equality(cell1.in_space(&space), d1.in_space(&space)), + "cell != direct atom" + ); + + assert!( + noun_equality(nested1.in_space(&space), nested2.in_space(&space)), + "[[1 2] 3] == [[1 2] 3]" + ); + assert!( + !noun_equality(nested1.in_space(&space), nested3.in_space(&space)), + "[[1 2] 3] != [[1 9] 3]" + ); + + // Both should be equal even though one shares and one doesn't + assert!( + noun_equality( + with_sharing.in_space(&space), + without_sharing.in_space(&space) + ), + "[[5 6] [5 6]] with sharing == without sharing" + ); + + assert!( + noun_equality( + cell_indirect1.in_space(&space), + cell_indirect2.in_space(&space) + ), + "cells with same indirect atoms are equal" + ); + assert!( + !noun_equality( + cell_indirect1.in_space(&space), + cell_indirect3.in_space(&space) + ), + "cells with different indirect atoms are not equal" + ); + } } diff --git a/crates/nockvm/rust/nockvm/src/flog.rs b/crates/nockvm/rust/nockvm/src/flog.rs index e99b6fb53..058fe7302 100644 --- a/crates/nockvm/rust/nockvm/src/flog.rs +++ b/crates/nockvm/rust/nockvm/src/flog.rs @@ -26,7 +26,8 @@ impl<'s> NockWriter<'s, '_> { } unsafe fn finalize(mut self) -> Atom { - self.indirect.normalize_as_atom() + let space = self.stack.noun_space(); + self.indirect.normalize_as_atom(&space) } unsafe fn expand(&mut self) { diff --git a/crates/nockvm/rust/nockvm/src/hamt.rs b/crates/nockvm/rust/nockvm/src/hamt.rs index a9f6d4fb9..5f93a002f 100644 --- a/crates/nockvm/rust/nockvm/src/hamt.rs +++ b/crates/nockvm/rust/nockvm/src/hamt.rs @@ -1,11 +1,14 @@ use std::ptr::{copy_nonoverlapping, null_mut}; use std::slice; +use std::time::Instant; use either::Either::{self, *}; +use tracing::debug; use crate::mem::{NockStack, Preserve}; use crate::mug::mug_u32; -use crate::noun::Noun; +use crate::noun::{Noun, NounAllocator}; +use crate::pma::{Pma, PmaCopy, PmaCopyFrom}; use crate::unifying_equality::unifying_equality; type MutStemEntry = Either<*mut MutStem, Leaf>; @@ -631,6 +634,393 @@ impl Preserve for Hamt { // gen2 + gen3: 257.47 seconds } +impl PmaCopy for Hamt { + #[cfg(feature = "pma-assert")] + fn assert_in_pma(&self, pma: &Pma) { + unsafe { + // Check root stem is in PMA + assert!( + pma.contains_ptr(self.0 as *const u8), + "HAMT root stem should be in PMA" + ); + + // Empty HAMT - nothing else to check + if (*self.0).bitmap == 0 { + return; + } + + // Traverse and check all internal structures + let mut stk: [(Stem, u32, usize); 6] = [( + Stem { + bitmap: 0, + typemap: 0, + buffer: null_mut(), + }, + 0, + 0, + ); 6]; + stk[0] = ((*self.0), (*self.0).bitmap, 0); + let mut depth = 1; + + loop { + if depth == 0 { + break; + } + let (stem, bits, next_idx) = &mut stk[depth - 1]; + + if *bits == 0 { + depth -= 1; + continue; + } + + let bit = bits.trailing_zeros(); + *bits &= *bits - 1; + let idx = *next_idx; + *next_idx = idx + 1; + + let ep = stem.buffer.add(idx); + let is_stem = ((stem.typemap >> bit) & 1) != 0; + + if is_stem { + let child = (*ep).stem; + assert!( + pma.contains_ptr(child.buffer as *const u8), + "HAMT child stem buffer should be in PMA" + ); + stk[depth] = (child, child.bitmap, 0); + depth += 1; + } else { + let leaf = (*ep).leaf; + assert!( + pma.contains_ptr(leaf.buffer as *const u8), + "HAMT leaf buffer should be in PMA" + ); + // Check all nouns in key-value pairs + let mut p = leaf.buffer; + let end = p.add(leaf.len); + while p != end { + (*p).0.assert_in_pma(pma); + (*p).1.assert_in_pma(pma); + p = p.add(1); + } + } + } + } + } + + #[cfg(not(feature = "pma-assert"))] + #[inline(always)] + fn assert_in_pma(&self, _pma: &Pma) {} + + /// Copy this HAMT and all its contents to the PMA. + /// + /// This copies: + /// - The root Stem + /// - All child Stems and their entry buffers + /// - All Leaves and their key-value pair buffers + /// - All Nouns in keys, converted to offset form + /// - All T values in values (via T::copy_to_pma) + /// + /// After this call, self.0 points to the PMA copy and all internal + /// pointers reference PMA locations. + unsafe fn copy_to_pma(&mut self, stack: &NockStack, pma: &mut Pma) { + let trace = std::env::var_os("NOCK_PMA_TRACE").is_some(); + if pma.contains_ptr(self.0 as *const u8) { + if trace { + debug!("pma-copy: hamt skip: root already in PMA"); + } + return; + } + + let trace_start = Instant::now(); + let mut last_progress = trace_start; + let mut stem_copies = 0usize; + let mut leaf_copies = 0usize; + let mut leaf_pairs = 0usize; + let mut stem_skips = 0usize; + let mut leaf_skips = 0usize; + let trace_detail = trace + && std::env::var_os("NOCK_PMA_TRACE_HAMT_DETAIL").is_some() + && std::any::type_name::().contains("WarmEntry"); + if trace { + let root_size = (*self.0).size(); + debug!( + "pma-copy: hamt start: root_ptr={:p} root_size={}", + self.0, root_size + ); + } + + // Copy root stem to PMA + let dest_stem: *mut Stem = pma.alloc_struct(1); + copy_nonoverlapping(self.0, dest_stem, 1); + self.0 = dest_stem; + stem_copies += 1; + + // Copy root's buffer to PMA + let sz = (*dest_stem).size(); + if sz > 0 { + let src_buffer = (*dest_stem).buffer; + if !pma.contains_ptr(src_buffer as *const u8) { + let dest_buffer: *mut Entry = pma.alloc_struct(sz); + copy_nonoverlapping(src_buffer, dest_buffer, sz); + (*dest_stem).buffer = dest_buffer; + } else { + stem_skips += 1; + } + } + + // Worklist: (stem whose buffer is already in PMA, remaining bitmap bits, next buffer index) + let mut stk: [(Stem, u32, usize); 6] = [( + Stem { + bitmap: 0, + typemap: 0, + buffer: null_mut(), + }, + 0, + 0, + ); 6]; + stk[0] = (*dest_stem, (*dest_stem).bitmap, 0); + let mut depth = 1; + + loop { + if depth == 0 { + break; + } + let (stem, bits, next_idx) = &mut stk[depth - 1]; + + if *bits == 0 { + depth -= 1; + continue; + } + + let bit = bits.trailing_zeros(); // 0..31 + *bits &= *bits - 1; // clear lsb set bit + let idx = *next_idx; + *next_idx = idx + 1; + + let ep = stem.buffer.add(idx); + let is_stem = ((stem.typemap >> bit) & 1) != 0; + + if is_stem { + // Child stem - copy its buffer to PMA + let child = (*ep).stem; + if pma.contains_ptr(child.buffer as *const u8) { + stem_skips += 1; + continue; + } + let csz = child.size(); + let db: *mut Entry = pma.alloc_struct(csz); + copy_nonoverlapping(child.buffer, db, csz); + let new_stem = Stem { + bitmap: child.bitmap, + typemap: child.typemap, + buffer: db, + }; + *ep = Entry { stem: new_stem }; + stem_copies += 1; + stk[depth] = (new_stem, new_stem.bitmap, 0); + depth += 1; + } else { + // Leaf - copy its buffer to PMA and evacuate all nouns + let leaf = (*ep).leaf; + if pma.contains_ptr(leaf.buffer as *const u8) { + leaf_skips += 1; + continue; + } + let len = leaf.len; + let db: *mut (Noun, T) = pma.alloc_struct(len); + copy_nonoverlapping(leaf.buffer, db, len); + let new_leaf = Leaf { len, buffer: db }; + leaf_copies += 1; + if trace_detail { + debug!( + "pma-copy: hamt leaf start: len={}, pair_index_start={}", + len, leaf_pairs + ); + } + + // Evacuate all nouns in key-value pairs + let mut p = new_leaf.buffer; + let end = p.add(len); + while p != end { + if trace_detail { + debug!("pma-copy: hamt pair {} key start", leaf_pairs); + } + (*p).0.copy_to_pma(stack, pma); + if trace_detail { + debug!("pma-copy: hamt pair {} key done", leaf_pairs); + debug!("pma-copy: hamt pair {} val start", leaf_pairs); + } + (*p).1.copy_to_pma(stack, pma); + if trace_detail { + debug!("pma-copy: hamt pair {} val done", leaf_pairs); + } + p = p.add(1); + leaf_pairs += 1; + if trace && (leaf_pairs & 0x3fff == 0) { + let now = Instant::now(); + if now.duration_since(last_progress).as_millis() >= 2000 { + debug!( + "pma-copy: hamt progress: stems_copied={}, leaves_copied={}, leaf_pairs={}, stem_skips={}, leaf_skips={}, elapsed_ms={}", + stem_copies, + leaf_copies, + leaf_pairs, + stem_skips, + leaf_skips, + trace_start.elapsed().as_millis() + ); + last_progress = now; + } + } + } + if trace_detail { + debug!( + "pma-copy: hamt leaf done: len={}, pair_index_end={}", + len, leaf_pairs + ); + } + *ep = Entry { leaf: new_leaf }; + } + } + + if trace { + debug!( + "pma-copy: hamt done: stems_copied={}, leaves_copied={}, leaf_pairs={}, stem_skips={}, leaf_skips={}, elapsed_ms={}", + stem_copies, + leaf_copies, + leaf_pairs, + stem_skips, + leaf_skips, + trace_start.elapsed().as_millis() + ); + } + } +} + +impl PmaCopyFrom for Hamt { + unsafe fn copy_from_pma(&mut self, from_pma: &Pma, to_pma: &mut Pma) { + if !from_pma.contains_ptr(self.0 as *const u8) { + return; + } + let trace = std::env::var_os("NOCK_PMA_TRACE").is_some(); + let trace_start = Instant::now(); + let mut last_progress = trace_start; + let mut stem_copies = 0usize; + let mut leaf_copies = 0usize; + let mut leaf_pairs = 0usize; + + if trace { + let root_size = (*self.0).size(); + debug!( + "pma-gc: hamt start: root_ptr={:p} root_size={}", + self.0, root_size + ); + } + + let dest_stem: *mut Stem = to_pma.alloc_struct(1); + copy_nonoverlapping(self.0, dest_stem, 1); + self.0 = dest_stem; + stem_copies += 1; + + let sz = (*dest_stem).size(); + if sz > 0 { + let src_buffer = (*dest_stem).buffer; + let dest_buffer: *mut Entry = to_pma.alloc_struct(sz); + copy_nonoverlapping(src_buffer, dest_buffer, sz); + (*dest_stem).buffer = dest_buffer; + } + + let mut stk: [(Stem, u32, usize); 6] = [( + Stem { + bitmap: 0, + typemap: 0, + buffer: null_mut(), + }, + 0, + 0, + ); 6]; + stk[0] = (*dest_stem, (*dest_stem).bitmap, 0); + let mut depth = 1; + + loop { + if depth == 0 { + break; + } + let (stem, bits, next_idx) = &mut stk[depth - 1]; + + if *bits == 0 { + depth -= 1; + continue; + } + + let bit = bits.trailing_zeros(); + *bits &= *bits - 1; + let idx = *next_idx; + *next_idx = idx + 1; + + let ep = stem.buffer.add(idx); + let is_stem = ((stem.typemap >> bit) & 1) != 0; + + if is_stem { + let child = (*ep).stem; + let csz = child.size(); + let db: *mut Entry = to_pma.alloc_struct(csz); + copy_nonoverlapping(child.buffer, db, csz); + let new_stem = Stem { + bitmap: child.bitmap, + typemap: child.typemap, + buffer: db, + }; + *ep = Entry { stem: new_stem }; + stem_copies += 1; + debug_assert!(depth < 6); + stk[depth] = (new_stem, new_stem.bitmap, 0); + depth += 1; + } else { + let leaf = (*ep).leaf; + let len = leaf.len; + let db: *mut (Noun, T) = to_pma.alloc_struct(len); + copy_nonoverlapping(leaf.buffer, db, len); + let new_leaf = Leaf { len, buffer: db }; + leaf_copies += 1; + + let mut p = new_leaf.buffer; + let end = p.add(len); + while p != end { + (*p).0.copy_from_pma(from_pma, to_pma); + (*p).1.copy_from_pma(from_pma, to_pma); + p = p.add(1); + leaf_pairs += 1; + if trace && (leaf_pairs & 0x3fff == 0) { + let now = Instant::now(); + if now.duration_since(last_progress).as_millis() >= 2000 { + debug!( + "pma-gc: hamt progress: stems_copied={}, leaves_copied={}, leaf_pairs={}, elapsed_ms={}", + stem_copies, + leaf_copies, + leaf_pairs, + trace_start.elapsed().as_millis() + ); + last_progress = now; + } + } + } + *ep = Entry { leaf: new_leaf }; + } + } + + if trace { + debug!( + "pma-gc: hamt done: stems_copied={}, leaves_copied={}, leaf_pairs={}, elapsed_ms={}", + stem_copies, + leaf_copies, + leaf_pairs, + trace_start.elapsed().as_millis() + ); + } + } +} + /// 🐹 /// Humorously named iterator for Hamt, which is a portmanteau of Hamt and iterator. /// Maximum depth of the HAMT is 6, so we can safely use a fixed size array for the traversal stack. @@ -732,6 +1122,7 @@ mod test { } #[test] + #[cfg_attr(miri, ignore = "memfd_create unsupported in Miri")] fn test_hamt_into_iter() { let size = 1 << 27; let top_slots = 100; @@ -753,6 +1144,7 @@ mod test { } #[test] + #[cfg_attr(miri, ignore = "memfd_create unsupported in Miri")] fn test_hamt_iter_big() { let size = 1 << 27; let top_slots = 100; @@ -771,6 +1163,7 @@ mod test { } #[test] + #[cfg_attr(miri, ignore = "memfd_create unsupported in Miri")] fn test_hamt() { let size = 1 << 27; let top_slots = 100; @@ -791,6 +1184,7 @@ mod test { } #[test] + #[cfg_attr(miri, ignore = "memfd_create unsupported in Miri")] fn test_hamt_collision_check() { let size = 1 << 27; let top_slots = 100; @@ -826,6 +1220,7 @@ mod test { } #[test] + #[cfg_attr(miri, ignore = "memfd_create unsupported in Miri")] fn test_hamt_collision_iter() { let size = 1 << 27; let top_slots = 100; diff --git a/crates/nockvm/rust/nockvm/src/interpreter.rs b/crates/nockvm/rust/nockvm/src/interpreter.rs index 838c69d78..73ba07bec 100644 --- a/crates/nockvm/rust/nockvm/src/interpreter.rs +++ b/crates/nockvm/rust/nockvm/src/interpreter.rs @@ -15,11 +15,11 @@ use crate::jets::hot::Hot; use crate::jets::list::util::weld; use crate::jets::warm::Warm; use crate::jets::{cold, JetErr}; -use crate::mem::{NockStack, Preserve}; -use crate::noun::{Atom, Cell, IndirectAtom, Noun, Slots, D, T}; +use crate::mem::{Arena, NockStack, Preserve}; +use crate::noun::{Atom, Cell, CellHandle, IndirectAtom, Noun, NounSpace, Slots, D, T}; use crate::trace::{write_nock_trace, TraceInfo, TraceStack}; use crate::unifying_equality::unifying_equality; -use crate::{assert_acyclic, assert_no_forwarding_pointers, assert_no_junior_pointers, flog, noun}; +use crate::{flog, noun}; // Previous results, not a complete sample but indicative: // nock opcode census: @@ -437,6 +437,7 @@ pub struct Context { pub trace_info: Option, pub running_status: Arc, pub test_jets: Hamt<()>, + pub arena: Arc, } #[derive(Debug, Clone)] @@ -491,6 +492,14 @@ impl Context { } } + pub fn arena(&self) -> &Arc { + &self.arena + } + + pub fn arena_ref(&self) -> &Arena { + &self.arena + } + /** * For jets that need a stack frame internally. * @@ -605,14 +614,30 @@ macro_rules! try_or_bail { }; } -#[allow(unused_variables)] +#[cfg(any( + feature = "check_acyclic", + feature = "check_forwarding", + feature = "check_junior" +))] #[inline(always)] fn debug_assertions(stack: &mut NockStack, noun: Noun) { - assert_acyclic!(noun); - assert_no_forwarding_pointers!(noun); - assert_no_junior_pointers!(stack, noun); + #[cfg(any(feature = "check_acyclic", feature = "check_forwarding"))] + { + let space = stack.fast_noun_space(); + crate::assert_acyclic!(space, noun); + crate::assert_no_forwarding_pointers!(space, noun); + } + crate::assert_no_junior_pointers!(stack, noun); } +#[cfg(not(any( + feature = "check_acyclic", + feature = "check_forwarding", + feature = "check_junior" +)))] +#[inline(always)] +fn debug_assertions(_stack: &mut NockStack, _noun: Noun) {} + /** Interpret nock */ pub fn interpret(context: &mut Context, mut subject: Noun, formula: Noun) -> Result { let orig_subject = subject; // for debugging @@ -667,15 +692,19 @@ pub fn interpret(context: &mut Context, mut subject: Noun, formula: Noun) -> Res // ``` // // (See https://docs.rs/assert_no_alloc/latest/assert_no_alloc/#advanced-use) + let space = context.stack.fast_noun_space(); let nock = unsafe { - try_or_bail!(push_formula(&mut context.stack, formula, true)); + try_or_bail!(push_formula_with_space( + &mut context.stack, formula, true, &space + )); loop { let work_ptr = context.stack.top::(); match &mut *work_ptr { NockWork::Work0(zero) => { opcode_tick!(WORK0); - if let Ok(noun) = subject.slot_atom(zero.axis) { + let noun = subject.slot_atom_trusted_or_checked(zero.axis, &space); + if let Ok(noun) = noun { res = noun; context.stack.pop::(); } else { @@ -688,7 +717,9 @@ pub fn interpret(context: &mut Context, mut subject: Noun, formula: Noun) -> Res opcode_tick!(WORK6); cond.todo = Todo6::ComputeBranch; *work_ptr = NockWork::Work6(cond.clone()); - try_or_bail!(push_formula(&mut context.stack, cond.test, false)); + try_or_bail!(push_formula_with_space( + &mut context.stack, cond.test, false, &space, + )); } Todo6::ComputeBranch => { opcode_tick!(WORK6); @@ -697,9 +728,13 @@ pub fn interpret(context: &mut Context, mut subject: Noun, formula: Noun) -> Res if res.is_direct() { let direct = res.as_direct().unwrap_unchecked(); if direct.data() == 0 { - try_or_bail!(push_formula(stack, cond.zero, cond.tail)); + try_or_bail!(push_formula_with_space( + stack, cond.zero, cond.tail, &space, + )); } else if direct.data() == 1 { - try_or_bail!(push_formula(stack, cond.once, cond.tail)); + try_or_bail!(push_formula_with_space( + stack, cond.once, cond.tail, &space, + )); } else { // Test branch of Nock 6 must return 0 or 1 break BAIL_EXIT; @@ -715,7 +750,9 @@ pub fn interpret(context: &mut Context, mut subject: Noun, formula: Noun) -> Res opcode_tick!(WORK8); pins.todo = Todo8::ComputeResult; *work_ptr = NockWork::Work8(pins.clone()); - try_or_bail!(push_formula(&mut context.stack, pins.pin, false)); + try_or_bail!(push_formula_with_space( + &mut context.stack, pins.pin, false, &space, + )); } Todo8::ComputeResult => { opcode_tick!(WORK8); @@ -723,12 +760,16 @@ pub fn interpret(context: &mut Context, mut subject: Noun, formula: Noun) -> Res if pins.tail { subject = T(stack, &[res, subject]); stack.pop::(); - try_or_bail!(push_formula(stack, pins.formula, true)); + try_or_bail!(push_formula_with_space( + stack, pins.formula, true, &space, + )); } else { pins.todo = Todo8::RestoreSubject; pins.pin = subject; subject = T(stack, &[res, subject]); - try_or_bail!(push_formula(stack, pins.formula, false)); + try_or_bail!(push_formula_with_space( + stack, pins.formula, false, &space, + )); } } Todo8::RestoreSubject => { @@ -742,14 +783,18 @@ pub fn interpret(context: &mut Context, mut subject: Noun, formula: Noun) -> Res opcode_tick!(WORK5); five.todo = Todo5::ComputeRightChild; *work_ptr = NockWork::Work5(five.clone()); - try_or_bail!(push_formula(&mut context.stack, five.left, false)); + try_or_bail!(push_formula_with_space( + &mut context.stack, five.left, false, &space, + )); } Todo5::ComputeRightChild => { opcode_tick!(WORK5); five.todo = Todo5::TestEquals; five.left = res; *work_ptr = NockWork::Work5(five.clone()); - try_or_bail!(push_formula(&mut context.stack, five.right, false)); + try_or_bail!(push_formula_with_space( + &mut context.stack, five.right, false, &space, + )); } Todo5::TestEquals => { opcode_tick!(WORK5); @@ -769,11 +814,14 @@ pub fn interpret(context: &mut Context, mut subject: Noun, formula: Noun) -> Res opcode_tick!(WORK9); kale.todo = Todo9::ComputeResult; *work_ptr = NockWork::Work9(kale.clone()); - try_or_bail!(push_formula(&mut context.stack, kale.core, false)); + try_or_bail!(push_formula_with_space( + &mut context.stack, kale.core, false, &space, + )); } Todo9::ComputeResult => { opcode_tick!(WORK9); - if let Ok(mut formula) = res.slot_atom(kale.axis) { + let formula = res.slot_atom_trusted_or_checked(kale.axis, &space); + if let Ok(mut formula) = formula { if !cfg!(feature = "sham_hints") { if let Some((jet, _path, test)) = context .warm @@ -821,7 +869,9 @@ pub fn interpret(context: &mut Context, mut subject: Noun, formula: Noun) -> Res } subject = res; - try_or_bail!(push_formula(stack, formula, true)); + try_or_bail!(push_formula_with_space( + stack, formula, true, &space, + )); } else { kale.todo = Todo9::RestoreSubject; kale.core = subject; @@ -833,7 +883,9 @@ pub fn interpret(context: &mut Context, mut subject: Noun, formula: Noun) -> Res subject = res; mean_frame_push(stack, 0); *stack.push() = NockWork::Ret; - try_or_bail!(push_formula(stack, formula, true)); + try_or_bail!(push_formula_with_space( + stack, formula, true, &space, + )); // We could trace on 2 as well, but 2 only comes from Hoon via // '.*', so we can assume it's never directly used to invoke @@ -873,13 +925,17 @@ pub fn interpret(context: &mut Context, mut subject: Noun, formula: Noun) -> Res TodoCons::ComputeHead => { opcode_tick!(WORK_CONS); cons.todo = TodoCons::ComputeTail; - try_or_bail!(push_formula(&mut context.stack, cons.head, false)); + try_or_bail!(push_formula_with_space( + &mut context.stack, cons.head, false, &space, + )); } TodoCons::ComputeTail => { opcode_tick!(WORK_CONS); cons.todo = TodoCons::Cons; cons.head = res; - try_or_bail!(push_formula(&mut context.stack, cons.tail, false)); + try_or_bail!(push_formula_with_space( + &mut context.stack, cons.tail, false, &space, + )); } TodoCons::Cons => { opcode_tick!(WORK_CONS); @@ -893,15 +949,21 @@ pub fn interpret(context: &mut Context, mut subject: Noun, formula: Noun) -> Res match diet.todo { Todo10::ComputeTree => { diet.todo = Todo10::ComputePatch; // should we compute patch then tree? - try_or_bail!(push_formula(&mut context.stack, diet.tree, false)); + try_or_bail!(push_formula_with_space( + &mut context.stack, diet.tree, false, &space, + )); } Todo10::ComputePatch => { diet.todo = Todo10::Edit; diet.tree = res; - try_or_bail!(push_formula(&mut context.stack, diet.patch, false)); + try_or_bail!(push_formula_with_space( + &mut context.stack, diet.patch, false, &space, + )); } Todo10::Edit => { - res = edit(&mut context.stack, diet.axis, res, diet.tree); + res = edit_with_space( + &mut context.stack, diet.axis, res, diet.tree, &space, + ); context.stack.pop::(); } } @@ -910,7 +972,9 @@ pub fn interpret(context: &mut Context, mut subject: Noun, formula: Noun) -> Res Todo7::ComputeSubject => { opcode_tick!(WORK7); pose.todo = Todo7::ComputeResult; - try_or_bail!(push_formula(&mut context.stack, pose.subject, false)); + try_or_bail!(push_formula_with_space( + &mut context.stack, pose.subject, false, &space, + )); } Todo7::ComputeResult => { opcode_tick!(WORK7); @@ -918,12 +982,16 @@ pub fn interpret(context: &mut Context, mut subject: Noun, formula: Noun) -> Res if pose.tail { stack.pop::(); subject = res; - try_or_bail!(push_formula(stack, pose.formula, true)); + try_or_bail!(push_formula_with_space( + stack, pose.formula, true, &space, + )); } else { pose.todo = Todo7::RestoreSubject; pose.subject = subject; subject = res; - try_or_bail!(push_formula(stack, pose.formula, false)); + try_or_bail!(push_formula_with_space( + stack, pose.formula, false, &space, + )); } } Todo7::RestoreSubject => { @@ -936,7 +1004,9 @@ pub fn interpret(context: &mut Context, mut subject: Noun, formula: Noun) -> Res Todo3::ComputeChild => { opcode_tick!(WORK3); thee.todo = Todo3::ComputeType; - try_or_bail!(push_formula(&mut context.stack, thee.child, false)); + try_or_bail!(push_formula_with_space( + &mut context.stack, thee.child, false, &space, + )); } Todo3::ComputeType => { opcode_tick!(WORK3); @@ -961,7 +1031,9 @@ pub fn interpret(context: &mut Context, mut subject: Noun, formula: Noun) -> Res } } else { dint.todo = Todo11D::ComputeResult; - try_or_bail!(push_formula(&mut context.stack, dint.hint, false)); + try_or_bail!(push_formula_with_space( + &mut context.stack, dint.hint, false, &space, + )); } } Todo11D::ComputeResult => { @@ -989,7 +1061,9 @@ pub fn interpret(context: &mut Context, mut subject: Noun, formula: Noun) -> Res dint.todo = Todo11D::Done; dint.hint = res; } - try_or_bail!(push_formula(&mut context.stack, dint.body, dint.tail)); + try_or_bail!(push_formula_with_space( + &mut context.stack, dint.body, dint.tail, &space, + )); } } Todo11D::Done => { @@ -1029,12 +1103,14 @@ pub fn interpret(context: &mut Context, mut subject: Noun, formula: Noun) -> Res Todo4::ComputeChild => { opcode_tick!(WORK4); four.todo = Todo4::Increment; - try_or_bail!(push_formula(&mut context.stack, four.child, false)); + try_or_bail!(push_formula_with_space( + &mut context.stack, four.child, false, &space, + )); } Todo4::Increment => { opcode_tick!(WORK4); if let Ok(atom) = res.as_atom() { - res = inc(&mut context.stack, atom).as_noun(); + res = inc_with_space(&mut context.stack, atom, &space).as_noun(); context.stack.pop::(); } else { // Cannot increment (Nock 4) a cell @@ -1063,11 +1139,11 @@ pub fn interpret(context: &mut Context, mut subject: Noun, formula: Noun) -> Res break Ok(res); } NockWork::Work2(ref mut vale) => { - cold_paths::step_work2(context, vale, &mut subject, &mut res)?; + cold_paths::step_work2(context, vale, &mut subject, &mut res, &space)?; continue; } NockWork::Work11S(ref mut sint) => { - match cold_paths::step_work11s(context, sint, &mut subject, &mut res) { + match cold_paths::step_work11s(context, sint, &mut subject, &mut res, &space) { Ok(_) => {} Err(err) => { break Err(err); @@ -1075,7 +1151,7 @@ pub fn interpret(context: &mut Context, mut subject: Noun, formula: Noun) -> Res }; } NockWork::Work12(ref mut scry) => { - cold_paths::step_work12(context, scry, &mut res)?; + cold_paths::step_work12(context, scry, &mut res, &space)?; continue; } }; @@ -1130,19 +1206,24 @@ mod cold_paths { vale: &mut Nock2, subject: &mut Noun, res: &mut Noun, + space: &NounSpace, ) -> Result { match vale.todo { Todo2::ComputeSubject => { opcode_tick!(WORK2); vale.todo = Todo2::ComputeFormula; - try_or_bail!(push_formula(&mut context.stack, vale.subject, false)); + try_or_bail!(push_formula_with_space( + &mut context.stack, vale.subject, false, space, + )); return OK_CONTINUE; } Todo2::ComputeFormula => { opcode_tick!(WORK2); vale.todo = Todo2::ComputeResult; vale.subject = *res; - try_or_bail!(push_formula(&mut context.stack, vale.formula, false)); + try_or_bail!(push_formula_with_space( + &mut context.stack, vale.formula, false, space, + )); return OK_CONTINUE; } Todo2::ComputeResult => { @@ -1151,7 +1232,7 @@ mod cold_paths { if vale.tail { stack.pop::(); *subject = vale.subject; - try_or_bail!(push_formula(stack, *res, true)); + try_or_bail!(push_formula_with_space(stack, *res, true, space)); return OK_CONTINUE; } else { vale.todo = Todo2::RestoreSubject; @@ -1163,7 +1244,7 @@ mod cold_paths { mean_frame_push(stack, 0); *stack.push() = NockWork::Ret; - try_or_bail!(push_formula(stack, *res, true)); + try_or_bail!(push_formula_with_space(stack, *res, true, space)); return OK_CONTINUE; } } @@ -1189,6 +1270,7 @@ mod cold_paths { sint: &mut Nock11S, subject: &mut Noun, res: &mut Noun, + space: &NounSpace, ) -> Result { match sint.todo { Todo11S::ComputeResult => { @@ -1212,7 +1294,7 @@ mod cold_paths { } else { sint.todo = Todo11S::Done; } - push_formula(&mut context.stack, sint.body, sint.tail) + push_formula_with_space(&mut context.stack, sint.body, sint.tail, space) } } Todo11S::Done => { @@ -1235,35 +1317,45 @@ mod cold_paths { scry: &mut Nock12, // subject: &mut Noun, res: &mut Noun, + space: &NounSpace, ) -> Result { match scry.todo { Todo12::ComputeReff => { opcode_tick!(WORK12); scry.todo = Todo12::ComputePath; - try_or_bail!(push_formula(&mut context.stack, scry.reff, false)); + try_or_bail!(push_formula_with_space( + &mut context.stack, scry.reff, false, space, + )); return OK_CONTINUE; } Todo12::ComputePath => { opcode_tick!(WORK12); scry.todo = Todo12::Scry; scry.reff = *res; - try_or_bail!(push_formula(&mut context.stack, scry.path, false)); + try_or_bail!(push_formula_with_space( + &mut context.stack, scry.path, false, space, + )); return OK_CONTINUE; } Todo12::Scry => { if let Some(cell) = context.scry_stack.cell() { scry.path = *res; let scry_stack = context.scry_stack; - let scry_handler = cell.head(); - let scry_gate = scry_handler.as_cell()?; + let cell_handle = cell.in_space(space); + let scry_handler = cell_handle.head().noun(); + let scry_gate = scry_handler.in_space(space).as_cell()?; let payload = T(&mut context.stack, &[scry.reff, *res]); let scry_core = T( &mut context.stack, - &[scry_gate.head(), payload, scry_gate.tail().as_cell()?.tail()], + &[ + scry_gate.head().noun(), + payload, + scry_gate.tail().as_cell()?.tail().noun(), + ], ); let scry_form = T(&mut context.stack, &[D(9), D(2), D(1), scry_core]); - context.scry_stack = cell.tail(); + context.scry_stack = cell_handle.tail().noun(); // Alternately, we could use scry_core as the subject and [9 2 0 1] as // the formula. It's unclear if performance will be better with a purely // static formula. @@ -1276,20 +1368,24 @@ mod cold_paths { return Err(Error::ScryCrashed(D(0))); } } - Right(cell) => match cell.tail().as_either_atom_cell() { - Left(_) => { - let stack = &mut context.stack; - let hunk = T(stack, &[D(tas!(b"hunk")), scry.reff, scry.path]); - mean_push(stack, hunk); - return Err(Error::ScryCrashed(D(0))); - } - Right(cell) => { - *res = cell.tail(); - context.scry_stack = scry_stack; - context.stack.pop::(); - return OK_CONTINUE; + Right(cell) => { + let cell_handle = cell.in_space(space); + match cell_handle.tail().as_either_atom_cell() { + Left(_) => { + let stack = &mut context.stack; + let hunk = + T(stack, &[D(tas!(b"hunk")), scry.reff, scry.path]); + mean_push(stack, hunk); + return Err(Error::ScryCrashed(D(0))); + } + Right(cell) => { + *res = cell.tail().noun(); + context.scry_stack = scry_stack; + context.stack.pop::(); + return OK_CONTINUE; + } } - }, + } }, Err(error) => match error { Error::Deterministic(_, trace) | Error::ScryCrashed(trace) => { @@ -1311,23 +1407,29 @@ mod cold_paths { } } -fn push_formula(stack: &mut NockStack, formula: Noun, tail: bool) -> Result { +fn push_formula_with_space( + stack: &mut NockStack, + formula: Noun, + tail: bool, + space: &NounSpace, +) -> Result { unsafe { if let Ok(formula_cell) = formula.as_cell() { + let (formula_head, formula_tail) = formula_cell.head_tail_trusted(space); // Formula - match formula_cell.head().as_either_atom_cell() { + match formula_head.as_either_atom_cell() { Right(_cell) => { *stack.push() = NockWork::WorkCons(NockCons { todo: TodoCons::ComputeHead, - head: formula_cell.head(), - tail: formula_cell.tail(), + head: formula_head, + tail: formula_tail, }); } Left(atom) => { if let Ok(direct) = atom.as_direct() { match direct.data() { 0 => { - if let Ok(axis_atom) = formula_cell.tail().as_atom() { + if let Ok(axis_atom) = formula_tail.as_atom() { *stack.push() = NockWork::Work0(Nock0 { axis: axis_atom }); } else { // Axis for Nock 0 must be an atom @@ -1335,16 +1437,15 @@ fn push_formula(stack: &mut NockStack, formula: Noun, tail: bool) -> Result { } } 1 => { - *stack.push() = NockWork::Work1(Nock1 { - noun: formula_cell.tail(), - }); + *stack.push() = NockWork::Work1(Nock1 { noun: formula_tail }); } 2 => { - if let Ok(arg_cell) = formula_cell.tail().as_cell() { + if let Ok(arg_cell) = formula_tail.as_cell() { + let (arg_head, arg_tail) = arg_cell.head_tail_trusted(space); *stack.push() = NockWork::Work2(Nock2 { todo: Todo2::ComputeSubject, - subject: arg_cell.head(), - formula: arg_cell.tail(), + subject: arg_head, + formula: arg_tail, tail, }); } else { @@ -1355,21 +1456,22 @@ fn push_formula(stack: &mut NockStack, formula: Noun, tail: bool) -> Result { 3 => { *stack.push() = NockWork::Work3(Nock3 { todo: Todo3::ComputeChild, - child: formula_cell.tail(), + child: formula_tail, }); } 4 => { *stack.push() = NockWork::Work4(Nock4 { todo: Todo4::ComputeChild, - child: formula_cell.tail(), + child: formula_tail, }); } 5 => { - if let Ok(arg_cell) = formula_cell.tail().as_cell() { + if let Ok(arg_cell) = formula_tail.as_cell() { + let (arg_head, arg_tail) = arg_cell.head_tail_trusted(space); *stack.push() = NockWork::Work5(Nock5 { todo: Todo5::ComputeLeftChild, - left: arg_cell.head(), - right: arg_cell.tail(), + left: arg_head, + right: arg_tail, }); } else { // Argument to Nock 5 must be cell @@ -1377,13 +1479,16 @@ fn push_formula(stack: &mut NockStack, formula: Noun, tail: bool) -> Result { }; } 6 => { - if let Ok(arg_cell) = formula_cell.tail().as_cell() { - if let Ok(branch_cell) = arg_cell.tail().as_cell() { + if let Ok(arg_cell) = formula_tail.as_cell() { + let (arg_head, arg_tail) = arg_cell.head_tail_trusted(space); + if let Ok(branch_cell) = arg_tail.as_cell() { + let (branch_head, branch_tail) = + branch_cell.head_tail_trusted(space); *stack.push() = NockWork::Work6(Nock6 { todo: Todo6::ComputeTest, - test: arg_cell.head(), - zero: branch_cell.head(), - once: branch_cell.tail(), + test: arg_head, + zero: branch_head, + once: branch_tail, tail, }); } else { @@ -1396,11 +1501,12 @@ fn push_formula(stack: &mut NockStack, formula: Noun, tail: bool) -> Result { } } 7 => { - if let Ok(arg_cell) = formula_cell.tail().as_cell() { + if let Ok(arg_cell) = formula_tail.as_cell() { + let (arg_head, arg_tail) = arg_cell.head_tail_trusted(space); *stack.push() = NockWork::Work7(Nock7 { todo: Todo7::ComputeSubject, - subject: arg_cell.head(), - formula: arg_cell.tail(), + subject: arg_head, + formula: arg_tail, tail, }); } else { @@ -1409,11 +1515,12 @@ fn push_formula(stack: &mut NockStack, formula: Noun, tail: bool) -> Result { }; } 8 => { - if let Ok(arg_cell) = formula_cell.tail().as_cell() { + if let Ok(arg_cell) = formula_tail.as_cell() { + let (arg_head, arg_tail) = arg_cell.head_tail_trusted(space); *stack.push() = NockWork::Work8(Nock8 { todo: Todo8::ComputeSubject, - pin: arg_cell.head(), - formula: arg_cell.tail(), + pin: arg_head, + formula: arg_tail, tail, }); } else { @@ -1422,13 +1529,14 @@ fn push_formula(stack: &mut NockStack, formula: Noun, tail: bool) -> Result { }; } 9 => { - if let Ok(arg_cell) = formula_cell.tail().as_cell() { - if let Ok(axis_atom) = arg_cell.head().as_atom() { + if let Ok(arg_cell) = formula_tail.as_cell() { + let (arg_head, arg_tail) = arg_cell.head_tail_trusted(space); + if let Ok(axis_atom) = arg_head.as_atom() { let p = stack.push(); *p = NockWork::Work9(Nock9 { todo: Todo9::ComputeCore, axis: axis_atom, - core: arg_cell.tail(), + core: arg_tail, tail, }); } else { @@ -1441,14 +1549,17 @@ fn push_formula(stack: &mut NockStack, formula: Noun, tail: bool) -> Result { }; } 10 => { - if let Ok(arg_cell) = formula_cell.tail().as_cell() { - if let Ok(patch_cell) = arg_cell.head().as_cell() { - if let Ok(axis_atom) = patch_cell.head().as_atom() { + if let Ok(arg_cell) = formula_tail.as_cell() { + let (arg_head, arg_tail) = arg_cell.head_tail_trusted(space); + if let Ok(patch_cell) = arg_head.as_cell() { + let (patch_head, patch_tail) = + patch_cell.head_tail_trusted(space); + if let Ok(axis_atom) = patch_head.as_atom() { *stack.push() = NockWork::Work10(Nock10 { todo: Todo10::ComputeTree, axis: axis_atom, - tree: arg_cell.tail(), - patch: patch_cell.tail(), + tree: arg_tail, + patch: patch_tail, }); } else { // Axis for Nock 10 must be an atom @@ -1464,24 +1575,29 @@ fn push_formula(stack: &mut NockStack, formula: Noun, tail: bool) -> Result { }; } 11 => { - if let Ok(arg_cell) = formula_cell.tail().as_cell() { - match arg_cell.head().as_either_atom_cell() { + if let Ok(arg_cell) = formula_tail.as_cell() { + let (arg_head, arg_tail) = arg_cell.head_tail_trusted(space); + match arg_head.as_either_atom_cell() { Left(tag_atom) => { + let tag = tag_atom; *stack.push() = NockWork::Work11S(Nock11S { todo: Todo11S::ComputeResult, - tag: tag_atom, - body: arg_cell.tail(), - tail: tail && hint::is_tail(tag_atom), + tag, + body: arg_tail, + tail: tail && hint::is_tail(tag), }); } Right(hint_cell) => { - if let Ok(tag_atom) = hint_cell.head().as_atom() { + let (hint_head, hint_tail) = + hint_cell.head_tail_trusted(space); + if let Ok(tag_atom) = hint_head.as_atom() { + let tag = tag_atom; *stack.push() = NockWork::Work11D(Nock11D { todo: Todo11D::ComputeHint, - tag: tag_atom, - hint: hint_cell.tail(), - body: arg_cell.tail(), - tail: tail && hint::is_tail(tag_atom), + tag, + hint: hint_tail, + body: arg_tail, + tail: tail && hint::is_tail(tag), }); } else { // Hint tag must be an atom @@ -1495,11 +1611,12 @@ fn push_formula(stack: &mut NockStack, formula: Noun, tail: bool) -> Result { }; } 12 => { - if let Ok(arg_cell) = formula_cell.tail().as_cell() { + if let Ok(arg_cell) = formula_tail.as_cell() { + let (arg_head, arg_tail) = arg_cell.head_tail_trusted(space); *stack.push() = NockWork::Work12(Nock12 { todo: Todo12::ComputeReff, - reff: arg_cell.head(), - path: arg_cell.tail(), + reff: arg_head, + path: arg_tail, }); } else { // Argument for Nock 12 must be cell @@ -1550,7 +1667,8 @@ fn exit( let h = *(stack.local_noun_pointer(0)); // XX: Small chance of clobbering something important after OOM? // XX: what if we OOM while making a stack trace - match weld(stack, t, h) { + let space = stack.fast_noun_space(); + match weld(stack, t, h, &space) { Ok(trace) => trace, Err(_) => h, } @@ -1595,15 +1713,28 @@ fn mean_push(stack: &mut NockStack, noun: Noun) { /** Pop off of the mean stack. */ fn mean_pop(stack: &mut NockStack) { + let space = stack.fast_noun_space(); + mean_pop_with_space(stack, &space) +} + +fn mean_pop_with_space(stack: &mut NockStack, space: &NounSpace) { unsafe { - *(stack.local_noun_pointer(0)) = (*(stack.local_noun_pointer(0))) + let trace = *(stack.local_noun_pointer(0)); + let trace_cell = trace + .in_space(space) .as_cell() - .expect("serf: unexpected end of mean stack\r") - .tail(); + .expect("serf: unexpected end of mean stack\r"); + *(stack.local_noun_pointer(0)) = trace_cell.tail().noun(); } } -fn edit(stack: &mut NockStack, edit_axis: Atom, patch: Noun, mut tree: Noun) -> Noun { +fn edit_with_space( + stack: &mut NockStack, + edit_axis: Atom, + patch: Noun, + mut tree: Noun, + space: &NounSpace, +) -> Noun { use either::{Left, Right}; use crate::noun::{DirectAxisIterator, IndirectAxisIterator}; @@ -1617,48 +1748,58 @@ fn edit(stack: &mut NockStack, edit_axis: Atom, patch: Noun, mut tree: Noun) -> DirectAxisIterator::new(direct.data()).expect("0 is not allowed as an edit axis"); while let Some(descend_tail) = axis_iter.next() { - let tree_cell = tree.as_cell().expect("Invalid axis for edit"); + let tree_cell = tree + .in_space(space) + .as_cell() + .expect("Invalid axis for edit"); if descend_tail { unsafe { let (cell, cellmem) = Cell::new_raw_mut(stack); *dest = cell.as_noun(); - (*cellmem).head = tree_cell.head(); + (*cellmem).head = tree_cell.head().noun(); dest = &mut ((*cellmem).tail); } - tree = tree_cell.tail(); + tree = tree_cell.tail().noun(); } else { unsafe { let (cell, cellmem) = Cell::new_raw_mut(stack); *dest = cell.as_noun(); - (*cellmem).tail = tree_cell.tail(); + (*cellmem).tail = tree_cell.tail().noun(); dest = &mut ((*cellmem).head); } - tree = tree_cell.head(); + tree = tree_cell.head().noun(); } } } Right(indirect) => { - let mut axis_iter = IndirectAxisIterator::new(indirect.as_slice()) + let indirect_handle = indirect.as_atom().in_space(space); + let indirect_slice = unsafe { + std::slice::from_raw_parts(indirect_handle.data_pointer(), indirect_handle.size()) + }; + let mut axis_iter = IndirectAxisIterator::new(indirect_slice) .expect("0 is not allowed as an edit axis"); while let Some(descend_tail) = axis_iter.next() { - let tree_cell = tree.as_cell().expect("Invalid axis for edit"); + let tree_cell = tree + .in_space(space) + .as_cell() + .expect("Invalid axis for edit"); if descend_tail { unsafe { let (cell, cellmem) = Cell::new_raw_mut(stack); *dest = cell.as_noun(); - (*cellmem).head = tree_cell.head(); + (*cellmem).head = tree_cell.head().noun(); dest = &mut ((*cellmem).tail); } - tree = tree_cell.tail(); + tree = tree_cell.tail().noun(); } else { unsafe { let (cell, cellmem) = Cell::new_raw_mut(stack); *dest = cell.as_noun(); - (*cellmem).tail = tree_cell.tail(); + (*cellmem).tail = tree_cell.tail().noun(); dest = &mut ((*cellmem).head); } - tree = tree_cell.head(); + tree = tree_cell.head().noun(); } } } @@ -1671,21 +1812,29 @@ fn edit(stack: &mut NockStack, edit_axis: Atom, patch: Noun, mut tree: Noun) -> } pub fn inc(stack: &mut NockStack, atom: Atom) -> Atom { + let space = stack.fast_noun_space(); + inc_with_space(stack, atom, &space) +} + +pub fn inc_with_space(stack: &mut NockStack, atom: Atom, space: &NounSpace) -> Atom { match atom.as_either() { Left(direct) => Atom::new(stack, direct.data() + 1), Right(indirect) => { - let indirect_slice = indirect.as_bitslice(); + let indirect_handle = indirect.as_atom().in_space(space); + let indirect_slice = indirect_handle.as_bitslice(); match indirect_slice.first_zero() { None => { // all ones, make an indirect one word bigger - let (new_indirect, new_slice) = - unsafe { IndirectAtom::new_raw_mut_bitslice(stack, indirect.size() + 1) }; + let (new_indirect, new_slice) = unsafe { + IndirectAtom::new_raw_mut_bitslice(stack, indirect_handle.size() + 1) + }; new_slice.set(indirect_slice.len(), true); new_indirect.as_atom() } Some(first_zero) => { - let (new_indirect, new_slice) = - unsafe { IndirectAtom::new_raw_mut_bitslice(stack, indirect.size()) }; + let (new_indirect, new_slice) = unsafe { + IndirectAtom::new_raw_mut_bitslice(stack, indirect_handle.size()) + }; new_slice.set(first_zero, true); new_slice[first_zero + 1..] .copy_from_bitslice(&indirect_slice[first_zero + 1..]); @@ -1746,7 +1895,8 @@ mod hint { if cfg!(feature = "sham_hints") { let jet_formula = hint.cell()?; // XX: what is the head here? - let jet_name = jet_formula.tail(); + let space = context.stack.fast_noun_space(); + let jet_name = jet_formula.in_space(&space).tail().noun(); if let Some(jet) = jets::get_jet(context, jet_name) { match jet(context, subject) { @@ -1850,8 +2000,10 @@ mod hint { let (_form, clue) = hint?; let slog_cell = clue.cell()?; - let pri = slog_cell.head().direct()?.data(); - let tank = slog_cell.tail(); + let space = stack.fast_noun_space(); + let slog_cell = slog_cell.in_space(&space); + let pri = slog_cell.head().noun().direct()?.data(); + let tank = slog_cell.tail().noun(); let s = (*slogger).deref_mut(); s.slog(stack, pri, tank); @@ -1877,23 +2029,25 @@ mod hint { Ok(toon) => { let stack = &mut context.stack; let slogger = &mut context.slogger; + let space = stack.fast_noun_space(); - if unsafe { !toon.head().raw_equals(&D(2)) } { + let toon_cell = CellHandle::new(toon, &space); + if unsafe { !toon_cell.head().noun().raw_equals(&D(2)) } { // +mook will only ever return a $toon with non-%2 head if that's what it was given as // input. Since we control the input for this call exactly, there must exist a programming // error in Ares if this occurs. panic!("serf: %hela: mook returned invalid tone"); } - let mut list = toon.tail(); + let mut list = toon_cell.tail().noun(); loop { if unsafe { list.raw_equals(&D(0)) } { break; } - if let Ok(cell) = list.as_cell() { - slogger.slog(stack, 0, cell.head()); - list = cell.tail(); + if let Ok(cell) = list.in_space(&space).as_cell() { + slogger.slog(stack, 0, cell.head().noun()); + list = cell.tail().noun(); } else { flog!(context, "serf: %hela: list ends without ~"); break; @@ -1926,6 +2080,7 @@ mod hint { let cold = &mut context.cold; let hot = &context.hot; let cache = &mut context.cache; + let space = stack.fast_noun_space(); // XX: handle IndirectAtom tags match tag.direct()?.data() { @@ -1939,12 +2094,12 @@ mod hint { tas!(b"fast") => { if !cfg!(feature = "sham_hints") { if let Some(clue) = hint { - let chum = clue.slot(2).ok()?; - let mut parent = clue.slot(6).ok()?; + let chum = clue.slot(2, &space).ok()?; + let mut parent = clue.slot(6, &space).ok()?; loop { - if let Ok(parent_cell) = parent.as_cell() { - if unsafe { parent_cell.head().raw_equals(&D(11)) } { - match parent.slot(7) { + if let Ok(parent_cell) = parent.in_space(&space).as_cell() { + if unsafe { parent_cell.head().noun().raw_equals(&D(11)) } { + match parent.slot(7, &space) { Ok(noun) => { parent = noun; } @@ -1959,8 +2114,8 @@ mod hint { return None; } } - let parent_formula_op = parent.slot(2).ok()?.atom()?.direct()?; - let parent_formula_ax = parent.slot(3).ok()?.atom()?; + let parent_formula_op = parent.slot(2, &space).ok()?.atom()?.direct()?; + let parent_formula_ax = parent.slot(3, &space).ok()?.atom()?; let cold_res: cold::Result = { if parent_formula_op.data() == 1 { @@ -1985,7 +2140,7 @@ mod hint { context.warm = Warm::init(stack, cold, hot, &context.test_jets) } Err(cold::Error::NoParent) => { - let Ok(chum_atom) = chum.as_atom() else { + let Ok(chum_atom) = chum.in_space(&space).as_atom() else { flog!(context, "serf: cold: register: cell chum"); return None; }; @@ -2023,21 +2178,21 @@ mod hint { mod debug { use either::Either::*; - use crate::noun::Noun; + use crate::noun::{Noun, NounSpace}; #[allow(dead_code)] - pub(super) fn assert_normalized(noun: Noun, path: Noun) { - assert_normalized_helper(noun, path, None); + pub(super) fn assert_normalized(noun: Noun, path: Noun, space: &NounSpace) { + assert_normalized_helper(noun, path, None, space); } #[allow(dead_code)] - pub(super) fn assert_normalized_depth(noun: Noun, path: Noun, depth: usize) { - assert_normalized_helper(noun, path, Some(depth)); + pub(super) fn assert_normalized_depth(noun: Noun, path: Noun, depth: usize, space: &NounSpace) { + assert_normalized_helper(noun, path, Some(depth), space); } #[allow(dead_code)] - fn assert_normalized_helper(noun: Noun, path: Noun, depth: Option) { - match noun.as_either_atom_cell() { + fn assert_normalized_helper(noun: Noun, path: Noun, depth: Option, space: &NounSpace) { + match noun.in_space(space).as_either_atom_cell() { Left(atom) => { if !atom.is_normalized() { if atom.size() == 1 { @@ -2054,8 +2209,8 @@ mod debug { Right(cell) => { if depth.is_none_or(|d| d != 0) { let new_depth = depth.map(|x| x - 1); - assert_normalized_helper(cell.head(), path, new_depth); - assert_normalized_helper(cell.tail(), path, new_depth); + assert_normalized_helper(cell.head().noun(), path, new_depth, space); + assert_normalized_helper(cell.tail().noun(), path, new_depth, space); } } } diff --git a/crates/nockvm/rust/nockvm/src/jets.rs b/crates/nockvm/rust/nockvm/src/jets.rs index 77b4dd102..915225e9c 100644 --- a/crates/nockvm/rust/nockvm/src/jets.rs +++ b/crates/nockvm/rust/nockvm/src/jets.rs @@ -41,7 +41,7 @@ use crate::jets::sort::*; use crate::jets::tree::*; use crate::jets::warm::Warm; use crate::mem::{NockStack, Preserve}; -use crate::noun::{self, Noun, Slots}; +use crate::noun::{self, Noun}; crate::gdb!(); @@ -209,7 +209,7 @@ pub mod util { use super::*; use crate::interpreter::interpret; - use crate::noun::{Noun, D, T}; + use crate::noun::{Noun, NounSpace, D, T}; pub const BAIL_EXIT: JetErr = JetErr::Fail(Error::Deterministic(Mote::Exit, D(0))); pub const BAIL_FAIL: JetErr = JetErr::Fail(Error::NonDeterministic(Mote::Fail, D(0))); @@ -254,8 +254,9 @@ pub mod util { bits_to_word(checked_left_shift(bloq, step)?) } - pub fn slot(noun: Noun, axis: u64) -> Result { - noun.slot(axis).map_err(|_e| BAIL_EXIT) + pub fn slot(noun: Noun, axis: u64, space: &NounSpace) -> Result { + noun.slot_direct_trusted_or_checked(axis, space) + .map_err(|_e| BAIL_EXIT) } /// Extract a bloq and check that it's computable by the current system @@ -269,10 +270,10 @@ pub mod util { } /// Extract the bloq and step from a bite - pub fn bite(a: Noun) -> result::Result<(usize, usize), JetErr> { - if let Ok(cell) = a.as_cell() { - let bloq = bloq(cell.head())?; - let step = cell.tail().as_direct()?.data() as usize; + pub fn bite(a: Noun, space: &NounSpace) -> result::Result<(usize, usize), JetErr> { + if let Ok(cell) = a.in_space(space).as_cell() { + let bloq = bloq(cell.head().noun())?; + let step = cell.tail().noun().as_direct()?.data() as usize; Ok((bloq, step)) } else { bloq(a).map(|x| (x, 1_usize)) @@ -315,10 +316,21 @@ pub mod util { } pub fn slam(context: &mut Context, gate: Noun, sample: Noun) -> result::Result { - let core: Noun = T( - &mut context.stack, - &[gate.as_cell()?.head(), sample, gate.as_cell()?.tail().as_cell()?.tail()], - ); + let space = context.stack.fast_noun_space(); + slam_with_space(context, gate, sample, &space) + } + + pub fn slam_with_space( + context: &mut Context, + gate: Noun, + sample: Noun, + space: &NounSpace, + ) -> result::Result { + let gate_cell = gate.as_cell()?; + let (gate_head, gate_tail) = gate_cell.head_tail_trusted(space); + let gate_tail_cell = gate_tail.as_cell()?; + let (_sample_slot, gate_context_tail) = gate_tail_cell.head_tail_trusted(space); + let core: Noun = T(&mut context.stack, &[gate_head, sample, gate_context_tail]); kick(context, core, D(2)) } @@ -348,7 +360,8 @@ pub mod util { } pub fn init_context() -> Context { - let mut stack = NockStack::new(8 << 10 << 10, 0); + let mut stack = NockStack::new(crate::mem::NOCK_STACK_SIZE_TINY, 0); + let arena = stack.arena().clone(); let cold = Cold::new(&mut stack); let warm = Warm::new(&mut stack); let hot = Hot::init(&mut stack, URBIT_HOT_STATE); @@ -368,6 +381,7 @@ pub mod util { trace_info: None, running_status: cancel, test_jets, + arena, } } @@ -464,6 +478,7 @@ pub mod util { ) }); assert!(res.is_atom(), "jet result not atom"); + let space = context.stack.fast_noun_space(); let res_siz = res .atom() .unwrap_or_else(|| { @@ -474,6 +489,7 @@ pub mod util { option_env!("GIT_SHA") ) }) + .in_space(&space) .size(); assert!(siz == res_siz, "got: {res_siz}, need: {siz}"); } diff --git a/crates/nockvm/rust/nockvm/src/jets/bits.rs b/crates/nockvm/rust/nockvm/src/jets/bits.rs index 088c162d2..2f0ab2ae7 100644 --- a/crates/nockvm/rust/nockvm/src/jets/bits.rs +++ b/crates/nockvm/rust/nockvm/src/jets/bits.rs @@ -7,7 +7,7 @@ use crate::interpreter::Context; use crate::jets::util::*; use crate::jets::Result; use crate::mem::NockStack; -use crate::noun::{IndirectAtom, Noun, D}; +use crate::noun::{IndirectAtom, Noun, NounSpace, D}; crate::gdb!(); @@ -16,26 +16,31 @@ crate::gdb!(); */ pub fn jet_bex(context: &mut Context, subject: Noun) -> Result { - let arg = slot(subject, 6)?.as_direct()?.data() as usize; - Ok(util::bex(&mut context.stack, arg).as_noun()) + let space = context.stack.noun_space(); + let arg = slot(subject, 6, &space)?.as_direct()?.data() as usize; + Ok(util::bex(&mut context.stack, arg, &space).as_noun()) } pub fn jet_can(context: &mut Context, subject: Noun) -> Result { - let arg = slot(subject, 6)?; - let bloq = bloq(slot(arg, 2)?)?; - let original_list = slot(arg, 3)?; + let space = context.stack.noun_space(); + let arg = slot(subject, 6, &space)?; + let bloq = bloq(slot(arg, 2, &space)?)?; + let original_list = slot(arg, 3, &space)?; - util::can(&mut context.stack, bloq, original_list) + util::can(&mut context.stack, bloq, original_list, &space) } pub fn jet_cat(context: &mut Context, subject: Noun) -> Result { - let arg = slot(subject, 6)?; - let bloq = bloq(slot(arg, 2)?)?; - let a = slot(arg, 6)?.as_atom()?; - let b = slot(arg, 7)?.as_atom()?; - - let len_a = util::met(bloq, a); - let len_b = util::met(bloq, b); + let space = context.stack.noun_space(); + let arg = slot(subject, 6, &space)?; + let bloq = bloq(slot(arg, 2, &space)?)?; + let a = slot(arg, 6, &space)?.as_atom()?; + let b = slot(arg, 7, &space)?.as_atom()?; + let a_handle = a.in_space(&space); + let b_handle = b.in_space(&space); + + let len_a = util::met(bloq, a, &space); + let len_b = util::met(bloq, b, &space); let new_len = bite_to_word(bloq, checked_add(len_a, len_b)?)?; if new_len == 0 { Ok(a.as_noun()) @@ -43,19 +48,20 @@ pub fn jet_cat(context: &mut Context, subject: Noun) -> Result { unsafe { let (mut new_indirect, new_slice) = IndirectAtom::new_raw_mut_bitslice(&mut context.stack, new_len); - chop(bloq, 0, len_a, 0, new_slice, a.as_bitslice())?; - chop(bloq, 0, len_b, len_a, new_slice, b.as_bitslice())?; - Ok(new_indirect.normalize_as_atom().as_noun()) + chop(bloq, 0, len_a, 0, new_slice, a_handle.as_bitslice())?; + chop(bloq, 0, len_b, len_a, new_slice, b_handle.as_bitslice())?; + Ok(new_indirect.normalize_as_atom(&space).as_noun()) } } } pub fn jet_cut(context: &mut Context, subject: Noun) -> Result { - let arg = slot(subject, 6)?; - let bloq = bloq(slot(arg, 2)?)?; - let start = slot(arg, 12)?.as_direct()?.data() as usize; - let run = slot(arg, 13)?.as_direct()?.data() as usize; - let atom = slot(arg, 7)?.as_atom()?; + let space = context.stack.noun_space(); + let arg = slot(subject, 6, &space)?; + let bloq = bloq(slot(arg, 2, &space)?)?; + let start = slot(arg, 12, &space)?.as_direct()?.data() as usize; + let run = slot(arg, 13, &space)?.as_direct()?.data() as usize; + let atom = slot(arg, 7, &space)?.as_atom()?; if run == 0 { return Ok(D(0)); @@ -64,101 +70,128 @@ pub fn jet_cut(context: &mut Context, subject: Noun) -> Result { let new_indirect = unsafe { let (mut new_indirect, new_slice) = IndirectAtom::new_raw_mut_bitslice(&mut context.stack, bite_to_word(bloq, run)?); - chop(bloq, start, run, 0, new_slice, atom.as_bitslice())?; - new_indirect.normalize_as_atom() + chop( + bloq, + start, + run, + 0, + new_slice, + atom.in_space(&space).as_bitslice(), + )?; + new_indirect.normalize_as_atom(&space) }; Ok(new_indirect.as_noun()) } pub fn jet_sew(context: &mut Context, subject: Noun) -> Result { - let sam = slot(subject, 6)?; - let bloq = bloq(slot(sam, 2)?)?; - let e = slot(sam, 7)?.as_atom()?; + let space = context.stack.noun_space(); + let sam = slot(subject, 6, &space)?; + let bloq = bloq(slot(sam, 2, &space)?)?; + let e = slot(sam, 7, &space)?.as_atom()?; - let bcd = slot(sam, 6)?; - let offset = slot(bcd, 2)?.as_atom()?.as_u64()? as usize; - let step = slot(bcd, 6)?.as_atom()?.as_u64()? as usize; + let bcd = slot(sam, 6, &space)?; + let offset = slot(bcd, 2, &space)?.in_space(&space).as_atom()?.as_u64()? as usize; + let step = slot(bcd, 6, &space)?.in_space(&space).as_atom()?.as_u64()? as usize; if step == 0 { return Ok(e.as_noun()); } - let donor = slot(bcd, 7)?.as_atom()?; + let donor = slot(bcd, 7, &space)?.as_atom()?; - let len_d = util::met(bloq, donor); - let len_e = util::met(bloq, e); + let len_d = util::met(bloq, donor, &space); + let len_e = util::met(bloq, e, &space); let new_len = max(checked_add(step, offset)?, len_e); unsafe { let (mut dest_indirect, dest) = IndirectAtom::new_raw_mut_bitslice(&mut context.stack, bite_to_word(bloq, new_len)?); - chop(bloq, 0, len_e, 0, dest, e.as_bitslice())?; + chop(bloq, 0, len_e, 0, dest, e.in_space(&space).as_bitslice())?; let (_, lead) = IndirectAtom::new_raw_mut_bitslice(&mut context.stack, bite_to_word(bloq, step)?); - chop(bloq, 0, min(step, len_d), 0, lead, donor.as_bitslice())?; + chop( + bloq, + 0, + min(step, len_d), + 0, + lead, + donor.in_space(&space).as_bitslice(), + )?; chop(bloq, 0, step, offset, dest, lead)?; - Ok(dest_indirect.normalize_as_atom().as_noun()) + Ok(dest_indirect.normalize_as_atom(&space).as_noun()) } } pub fn jet_end(context: &mut Context, subject: Noun) -> Result { - let arg = slot(subject, 6)?; - let (bloq, step) = bite(slot(arg, 2)?)?; - let a = slot(arg, 3)?.as_atom()?; + let space = context.stack.noun_space(); + let arg = slot(subject, 6, &space)?; + let (bloq, step) = bite(slot(arg, 2, &space)?, &space)?; + let a = slot(arg, 3, &space)?.as_atom()?; if step == 0 { Ok(D(0)) - } else if step >= util::met(bloq, a) { + } else if step >= util::met(bloq, a, &space) { Ok(a.as_noun()) } else { unsafe { let (mut new_indirect, new_slice) = IndirectAtom::new_raw_mut_bitslice(&mut context.stack, bite_to_word(bloq, step)?); - chop(bloq, 0, step, 0, new_slice, a.as_bitslice())?; - Ok(new_indirect.normalize_as_atom().as_noun()) + chop( + bloq, + 0, + step, + 0, + new_slice, + a.in_space(&space).as_bitslice(), + )?; + Ok(new_indirect.normalize_as_atom(&space).as_noun()) } } } pub fn jet_lsh(context: &mut Context, subject: Noun) -> Result { - let arg = slot(subject, 6)?; - let (bloq, step) = bite(slot(arg, 2)?)?; - let a = slot(arg, 3)?.as_atom()?; + let space = context.stack.noun_space(); + let arg = slot(subject, 6, &space)?; + let (bloq, step) = bite(slot(arg, 2, &space)?, &space)?; + let a = slot(arg, 3, &space)?.as_atom()?; - util::lsh(&mut context.stack, bloq, step, a) + util::lsh(&mut context.stack, bloq, step, a, &space) } pub fn jet_met(_context: &mut Context, subject: Noun) -> Result { - let arg = slot(subject, 6)?; - let bloq = bloq(slot(arg, 2)?)?; - let a = slot(arg, 3)?.as_atom()?; + let space = _context.stack.noun_space(); + let arg = slot(subject, 6, &space)?; + let bloq = bloq(slot(arg, 2, &space)?)?; + let a = slot(arg, 3, &space)?.as_atom()?; - Ok(D(util::met(bloq, a) as u64)) + Ok(D(util::met(bloq, a, &space) as u64)) } pub fn jet_rap(context: &mut Context, subject: Noun) -> Result { - let arg = slot(subject, 6)?; - let bloq = bloq(slot(arg, 2)?)?; - let original_list = slot(arg, 3)?; - Ok(util::rap(&mut context.stack, bloq, original_list)?.as_noun()) + let space = context.stack.noun_space(); + let arg = slot(subject, 6, &space)?; + let bloq = bloq(slot(arg, 2, &space)?)?; + let original_list = slot(arg, 3, &space)?; + Ok(util::rap(&mut context.stack, bloq, original_list, &space)?.as_noun()) } pub fn jet_rep(context: &mut Context, subject: Noun) -> Result { let stack = &mut context.stack; + let space = stack.noun_space(); - let arg = slot(subject, 6)?; - let arg2 = slot(arg, 2)?; - let arg3 = slot(arg, 3)?; + let arg = slot(subject, 6, &space)?; + let arg2 = slot(arg, 2, &space)?; + let arg3 = slot(arg, 3, &space)?; - rep(stack, arg2, arg3) + rep(stack, arg2, arg3, &space) } -pub fn rep(stack: &mut NockStack, a: Noun, b: Noun) -> Result { - let (bloq, step) = bite(a)?; +pub fn rep(stack: &mut NockStack, a: Noun, b: Noun, space: &NounSpace) -> Result { + let (bloq, step) = bite(a, space)?; let original_list = b; let mut len = 0usize; @@ -168,10 +201,10 @@ pub fn rep(stack: &mut NockStack, a: Noun, b: Noun) -> Result { break; } - let cell = list.as_cell()?; + let cell = list.in_space(space).as_cell()?; len = checked_add(len, step)?; - list = cell.tail(); + list = cell.tail().noun(); } if len == 0 { @@ -187,32 +220,34 @@ pub fn rep(stack: &mut NockStack, a: Noun, b: Noun) -> Result { break; } - let cell = list.as_cell()?; + let cell = list.in_space(space).as_cell()?; let atom = cell.head().as_atom()?; chop(bloq, 0, step, pos, new_slice, atom.as_bitslice())?; pos += step; - list = cell.tail(); + list = cell.tail().noun(); } - Ok(new_indirect.normalize_as_atom().as_noun()) + Ok(new_indirect.normalize_as_atom(space).as_noun()) } } } pub fn jet_rev(context: &mut Context, subject: Noun) -> Result { - let arg = slot(subject, 6)?; - let boz = slot(arg, 2)?.as_atom()?.as_direct()?.data(); + let space = context.stack.noun_space(); + let arg = slot(subject, 6, &space)?; + let boz = slot(arg, 2, &space)?.as_atom()?.as_direct()?.data(); if boz >= 64 { return Err(BAIL_EXIT); } let boz = boz as usize; - let len = slot(arg, 6)?.as_atom()?.as_direct()?.data(); - let dat = slot(arg, 7)?.as_atom()?; + let len = slot(arg, 6, &space)?.as_atom()?.as_direct()?.data(); + let dat = slot(arg, 7, &space)?.as_atom()?; let bits = len << boz; - let src = dat.as_bitslice(); + let dat_handle = dat.in_space(&space); + let src = dat_handle.as_bitslice(); let (mut output, dest) = unsafe { IndirectAtom::new_raw_mut_bitslice(&mut context.stack, bits as usize) }; @@ -226,38 +261,45 @@ pub fn jet_rev(context: &mut Context, subject: Noun) -> Result { dest[start..end].copy_from_bitslice(&src[(total_len - end)..(total_len - start)]); } - Ok(unsafe { output.normalize_as_atom() }.as_noun()) + Ok(unsafe { output.normalize_as_atom(&space) }.as_noun()) } pub fn jet_rip(context: &mut Context, subject: Noun) -> Result { - let arg = slot(subject, 6)?; - let (bloq, step) = bite(slot(arg, 2)?)?; - let atom = slot(arg, 3)?.as_atom()?; - util::rip(&mut context.stack, bloq, step, atom) + let space = context.stack.noun_space(); + let arg = slot(subject, 6, &space)?; + let (bloq, step) = bite(slot(arg, 2, &space)?, &space)?; + let atom = slot(arg, 3, &space)?.as_atom()?; + util::rip(&mut context.stack, bloq, step, atom, &space) } pub fn jet_rsh(context: &mut Context, subject: Noun) -> Result { - let arg = slot(subject, 6)?; - let (bloq, step) = bite(slot(arg, 2)?)?; - let a = slot(arg, 3)?.as_atom()?; + let space = context.stack.noun_space(); + let arg = slot(subject, 6, &space)?; + let (bloq, step) = bite(slot(arg, 2, &space)?, &space)?; + let a = slot(arg, 3, &space)?.as_atom()?; + let a_handle = a.in_space(&space); - let len = util::met(bloq, a); + let len = util::met(bloq, a, &space); if step >= len { return Ok(D(0)); } - let new_size = bits_to_word(checked_sub(a.bit_size(), checked_left_shift(bloq, step)?)?)?; + let new_size = bits_to_word(checked_sub( + a_handle.bit_size(), + checked_left_shift(bloq, step)?, + )?)?; unsafe { let (mut atom, dest) = IndirectAtom::new_raw_mut_bitslice(&mut context.stack, new_size); - chop(bloq, step, len - step, 0, dest, a.as_bitslice())?; - Ok(atom.normalize_as_atom().as_noun()) + chop(bloq, step, len - step, 0, dest, a_handle.as_bitslice())?; + Ok(atom.normalize_as_atom(&space).as_noun()) } } pub fn jet_xeb(_context: &mut Context, subject: Noun) -> Result { - let sam = slot(subject, 6)?; - let a = slot(sam, 1)?.as_atom()?; - Ok(D(util::met(0, a) as u64)) + let space = _context.stack.noun_space(); + let sam = slot(subject, 6, &space)?; + let a = slot(sam, 1, &space)?.as_atom()?; + Ok(D(util::met(0, a, &space) as u64)) } /* @@ -265,42 +307,49 @@ pub fn jet_xeb(_context: &mut Context, subject: Noun) -> Result { */ pub fn jet_con(context: &mut Context, subject: Noun) -> Result { - let arg = slot(subject, 6)?; - let a = slot(arg, 2)?.as_atom()?; - let b = slot(arg, 3)?.as_atom()?; + let space = context.stack.noun_space(); + let arg = slot(subject, 6, &space)?; + let a = slot(arg, 2, &space)?.as_atom()?; + let b = slot(arg, 3, &space)?.as_atom()?; - Ok(util::con(&mut context.stack, a, b).as_noun()) + Ok(util::con(&mut context.stack, a, b, &space).as_noun()) } pub fn jet_dis(context: &mut Context, subject: Noun) -> Result { - let arg = slot(subject, 6)?; - let a = slot(arg, 2)?.as_atom()?; - let b = slot(arg, 3)?.as_atom()?; + let space = context.stack.noun_space(); + let arg = slot(subject, 6, &space)?; + let a = slot(arg, 2, &space)?.as_atom()?; + let b = slot(arg, 3, &space)?.as_atom()?; + let a_handle = a.in_space(&space); + let b_handle = b.in_space(&space); - let new_size = cmp::max(a.size(), b.size()); + let new_size = cmp::max(a_handle.size(), b_handle.size()); unsafe { let (mut atom, dest) = IndirectAtom::new_raw_mut_bitslice(&mut context.stack, new_size); - let a_bit = a.as_bitslice(); + let a_bit = a_handle.as_bitslice(); dest[..a_bit.len()].copy_from_bitslice(a_bit); - *dest &= b.as_bitslice(); - Ok(atom.normalize_as_atom().as_noun()) + *dest &= b_handle.as_bitslice(); + Ok(atom.normalize_as_atom(&space).as_noun()) } } pub fn jet_mix(context: &mut Context, subject: Noun) -> Result { - let arg = slot(subject, 6)?; - let a = slot(arg, 2)?.as_atom()?; - let b = slot(arg, 3)?.as_atom()?; + let space = context.stack.noun_space(); + let arg = slot(subject, 6, &space)?; + let a = slot(arg, 2, &space)?.as_atom()?; + let b = slot(arg, 3, &space)?.as_atom()?; + let a_handle = a.in_space(&space); + let b_handle = b.in_space(&space); - let new_size = cmp::max(a.size(), b.size()); + let new_size = cmp::max(a_handle.size(), b_handle.size()); unsafe { let (mut atom, dest) = IndirectAtom::new_raw_mut_bitslice(&mut context.stack, new_size); - let a_bit = a.as_bitslice(); + let a_bit = a_handle.as_bitslice(); dest[..a_bit.len()].copy_from_bitslice(a_bit); - *dest ^= b.as_bitslice(); - Ok(atom.normalize_as_atom().as_noun()) + *dest ^= b_handle.as_bitslice(); + Ok(atom.normalize_as_atom(&space).as_noun()) } } @@ -310,36 +359,51 @@ pub mod util { use crate::jets::util::*; use crate::jets::{JetErr, Result}; use crate::mem::NockStack; - use crate::noun::{Atom, Cell, DirectAtom, IndirectAtom, Noun, D}; + use crate::noun::{Atom, Cell, DirectAtom, IndirectAtom, Noun, NounSpace, D}; /// Binary exponent - pub fn bex(stack: &mut NockStack, arg: usize) -> Atom { + pub fn bex(stack: &mut NockStack, arg: usize, space: &NounSpace) -> Atom { unsafe { if arg < 63 { DirectAtom::new_unchecked(1 << arg).as_atom() } else { let (mut atom, dest) = IndirectAtom::new_raw_mut_bitslice(stack, (arg + 7) >> 3); dest.set(arg, true); - atom.normalize_as_atom() + atom.normalize_as_atom(space) } } } - pub fn lsh(stack: &mut NockStack, bloq: usize, step: usize, a: Atom) -> Result { - let len = met(bloq, a); + pub fn lsh( + stack: &mut NockStack, + bloq: usize, + step: usize, + a: Atom, + space: &NounSpace, + ) -> Result { + let len = met(bloq, a, space); if len == 0 { return Ok(D(0)); } - let new_size = bits_to_word(checked_add(a.bit_size(), checked_left_shift(bloq, step)?)?)?; + let a_handle = a.in_space(space); + let new_size = bits_to_word(checked_add( + a_handle.bit_size(), + checked_left_shift(bloq, step)?, + )?)?; unsafe { let (mut atom, dest) = IndirectAtom::new_raw_mut_bitslice(stack, new_size); - chop(bloq, 0, len, step, dest, a.as_bitslice())?; - Ok(atom.normalize_as_atom().as_noun()) + chop(bloq, 0, len, step, dest, a_handle.as_bitslice())?; + Ok(atom.normalize_as_atom(space).as_noun()) } } - pub fn can(stack: &mut NockStack, bloq: usize, original_list: Noun) -> Result { + pub fn can( + stack: &mut NockStack, + bloq: usize, + original_list: Noun, + space: &NounSpace, + ) -> Result { let mut len = 0usize; let mut list = original_list; loop { @@ -347,12 +411,12 @@ pub mod util { break; } - let cell = list.as_cell()?; + let cell = list.in_space(space).as_cell()?; let item = cell.head().as_cell()?; - let step = item.head().as_direct()?.data() as usize; + let step = item.head().as_atom()?.atom().as_direct()?.data() as usize; len = checked_add(len, step)?; - list = cell.tail(); + list = cell.tail().noun(); } if len == 0 { @@ -368,41 +432,54 @@ pub mod util { break; } - let cell = list.as_cell()?; + let cell = list.in_space(space).as_cell()?; let item = cell.head().as_cell()?; - let step = item.head().as_direct()?.data() as usize; + let step = item.head().as_atom()?.atom().as_direct()?.data() as usize; let atom = item.tail().as_atom()?; chop(bloq, 0, step, pos, new_slice, atom.as_bitslice())?; pos += step; - list = cell.tail(); + list = cell.tail().noun(); } - Ok(new_indirect.normalize_as_atom().as_noun()) + Ok(new_indirect.normalize_as_atom(space).as_noun()) } } } /// Measure the number of bloqs in an atom - pub fn met(bloq: usize, a: Atom) -> usize { + pub fn met(bloq: usize, a: Atom, space: &NounSpace) -> usize { if unsafe { a.as_noun().raw_equals(&D(0)) } { 0 } else if bloq < 6 { - (a.bit_size() + ((1 << bloq) - 1)) >> bloq + (a.in_space(space).bit_size() + ((1 << bloq) - 1)) >> bloq } else { let bloq_word = bloq - 6; - (a.size() + ((1 << bloq_word) - 1)) >> bloq_word + (a.in_space(space).size() + ((1 << bloq_word) - 1)) >> bloq_word } } - pub fn rip(stack: &mut NockStack, bloq: usize, step: usize, atom: Atom) -> Result { - let len = met(bloq, atom).div_ceil(step); + pub fn rip( + stack: &mut NockStack, + bloq: usize, + step: usize, + atom: Atom, + space: &NounSpace, + ) -> Result { + let len = met(bloq, atom, space).div_ceil(step); let mut list = D(0); for i in (0..len).rev() { let new_atom = unsafe { let (mut new_indirect, new_slice) = IndirectAtom::new_raw_mut_bitslice(stack, step << bloq); - chop(bloq, i * step, step, 0, new_slice, atom.as_bitslice())?; - new_indirect.normalize_as_atom() + chop( + bloq, + i * step, + step, + 0, + new_slice, + atom.in_space(space).as_bitslice(), + )?; + new_indirect.normalize_as_atom(space) }; list = Cell::new(stack, new_atom.as_noun(), list).as_noun(); } @@ -411,15 +488,17 @@ pub mod util { } /// Binary OR - pub fn con(stack: &mut NockStack, a: Atom, b: Atom) -> Atom { - let new_size = cmp::max(a.size(), b.size()); + pub fn con(stack: &mut NockStack, a: Atom, b: Atom, space: &NounSpace) -> Atom { + let a_handle = a.in_space(space); + let b_handle = b.in_space(space); + let new_size = cmp::max(a_handle.size(), b_handle.size()); unsafe { let (mut atom, dest) = IndirectAtom::new_raw_mut_bitslice(stack, new_size); - let a_bit = a.as_bitslice(); + let a_bit = a_handle.as_bitslice(); dest[..a_bit.len()].copy_from_bitslice(a_bit); - *dest |= b.as_bitslice(); - atom.normalize_as_atom() + *dest |= b_handle.as_bitslice(); + atom.normalize_as_atom(space) } } @@ -427,6 +506,7 @@ pub mod util { stack: &mut NockStack, bloq: usize, original_list: Noun, + space: &NounSpace, ) -> result::Result { let mut len = 0usize; let mut list = original_list; @@ -435,10 +515,11 @@ pub mod util { break; } - let cell = list.as_cell()?; + let cell = list.in_space(space).as_cell()?; - len = checked_add(len, met(bloq, cell.head().as_atom()?))?; - list = cell.tail(); + let atom = cell.head().as_atom()?; + len = checked_add(len, met(bloq, atom.atom(), space))?; + list = cell.tail().noun(); } if len == 0 { @@ -455,16 +536,16 @@ pub mod util { break; } - let cell = list.as_cell()?; + let cell = list.in_space(space).as_cell()?; let atom = cell.head().as_atom()?; - let step = met(bloq, atom); + let step = met(bloq, atom.atom(), space); chop(bloq, 0, step, pos, new_slice, atom.as_bitslice())?; pos += step; - list = cell.tail(); + list = cell.tail().noun(); } - Ok(new_indirect.normalize_as_atom()) + Ok(new_indirect.normalize_as_atom(space)) } } } @@ -478,7 +559,7 @@ pub mod util { use crate::noun::D; fn init_stack() -> NockStack { - NockStack::new(8 << 10 << 10, 0) + NockStack::new(crate::mem::NOCK_STACK_SIZE_TINY, 0) } #[test] @@ -496,15 +577,16 @@ pub mod util { option_env!("GIT_SHA") ) }); - assert_eq!(met(0, a), 128); - assert_eq!(met(1, a), 64); - assert_eq!(met(2, a), 32); - assert_eq!(met(3, a), 16); - assert_eq!(met(4, a), 8); - assert_eq!(met(5, a), 4); - assert_eq!(met(6, a), 2); - assert_eq!(met(7, a), 1); - assert_eq!(met(8, a), 1); + let space = s.noun_space(); + assert_eq!(met(0, a, &space), 128); + assert_eq!(met(1, a, &space), 64); + assert_eq!(met(2, a, &space), 32); + assert_eq!(met(3, a, &space), 16); + assert_eq!(met(4, a, &space), 8); + assert_eq!(met(5, a, &space), 4); + assert_eq!(met(6, a, &space), 2); + assert_eq!(met(7, a, &space), 1); + assert_eq!(met(8, a, &space), 1); let a = D(0x7fffffffffffffff).as_atom().unwrap_or_else(|err| { panic!( @@ -514,14 +596,14 @@ pub mod util { option_env!("GIT_SHA") ) }); - assert_eq!(met(0, a), 63); - assert_eq!(met(1, a), 32); - assert_eq!(met(2, a), 16); - assert_eq!(met(3, a), 8); - assert_eq!(met(4, a), 4); - assert_eq!(met(5, a), 2); - assert_eq!(met(6, a), 1); - assert_eq!(met(7, a), 1); + assert_eq!(met(0, a, &space), 63); + assert_eq!(met(1, a, &space), 32); + assert_eq!(met(2, a, &space), 16); + assert_eq!(met(3, a, &space), 8); + assert_eq!(met(4, a, &space), 4); + assert_eq!(met(5, a, &space), 2); + assert_eq!(met(6, a, &space), 1); + assert_eq!(met(7, a, &space), 1); } } } diff --git a/crates/nockvm/rust/nockvm/src/jets/cold.rs b/crates/nockvm/rust/nockvm/src/jets/cold.rs index ff213e95e..21b92a901 100644 --- a/crates/nockvm/rust/nockvm/src/jets/cold.rs +++ b/crates/nockvm/rust/nockvm/src/jets/cold.rs @@ -1,8 +1,15 @@ use std::ptr::{copy_nonoverlapping, null_mut}; +use intmap::IntMap; +use tracing::debug; + use crate::hamt::Hamt; use crate::mem::{self, NockStack, Preserve}; -use crate::noun::{self, Atom, DirectAtom, IndirectAtom, Noun, NounAllocator, Slots, D, T}; +use crate::noun::{ + self, Atom, DirectAtom, IndirectAtom, Noun, NounAllocator, NounSpace, Slots, D, T, +}; +use crate::offset::PmaOffsetWords; +use crate::pma::{Pma, PmaCopy, PmaCopyFrom}; use crate::unifying_equality::unifying_equality; pub enum Error { @@ -22,13 +29,13 @@ pub type Result = std::result::Result; #[derive(Copy, Clone)] pub struct Batteries(*mut BatteriesMem); -const NO_BATTERIES: Batteries = Batteries(null_mut()); +pub(crate) const NO_BATTERIES: Batteries = Batteries(null_mut()); #[derive(Copy, Clone)] -struct BatteriesMem { - battery: Noun, - parent_axis: Atom, - parent_batteries: Batteries, +pub(crate) struct BatteriesMem { + pub(crate) battery: Noun, + pub(crate) parent_axis: Atom, + pub(crate) parent_batteries: Batteries, } impl Preserve for Batteries { @@ -70,6 +77,128 @@ impl Preserve for Batteries { } } +impl PmaCopy for Batteries { + #[cfg(feature = "pma-assert")] + fn assert_in_pma(&self, pma: &Pma) { + if self.0.is_null() { + return; + } + let mut cursor = *self; + loop { + unsafe { + assert!( + pma.contains_ptr(cursor.0 as *const u8), + "Batteries node should be in PMA" + ); + (*cursor.0).battery.assert_in_pma(pma); + (*cursor.0).parent_axis.assert_in_pma(pma); + if (*cursor.0).parent_batteries.0.is_null() { + break; + } + cursor = (*cursor.0).parent_batteries; + } + } + } + + #[cfg(not(feature = "pma-assert"))] + #[inline(always)] + fn assert_in_pma(&self, _pma: &Pma) {} + + unsafe fn copy_to_pma(&mut self, stack: &NockStack, pma: &mut Pma) { + if self.0.is_null() { + return; + } + let trace = std::env::var_os("NOCK_PMA_TRACE_WARM_ENTRY").is_some(); + let start = std::time::Instant::now(); + let mut last_progress = start; + let mut steps = 0usize; + if trace { + debug!("pma-copy: batteries start: head_ptr={:p}", self.0); + } + let mut ptr: *mut Batteries = self; + loop { + if pma.contains_ptr((*ptr).0 as *const u8) { + break; + } + if trace { + let space = stack.noun_space(); + let battery = (*(*ptr).0).battery; + let axis = (*(*ptr).0).parent_axis; + let axis_noun = axis.as_noun(); + debug!( + "pma-copy: batteries node: node_ptr={:p} battery_raw=0x{:x} battery_repr={:?} axis_raw=0x{:x} axis_repr={:?}", + (*ptr).0, + unsafe { battery.as_raw() }, + battery.repr(&space), + unsafe { axis_noun.as_raw() }, + axis_noun.repr(&space) + ); + debug!("pma-copy: batteries battery copy start"); + } + // Copy the battery noun and parent_axis to PMA + (*(*ptr).0).battery.copy_to_pma(stack, pma); + if trace { + debug!("pma-copy: batteries battery copy done"); + debug!("pma-copy: batteries axis copy start"); + } + (*(*ptr).0).parent_axis.copy_to_pma(stack, pma); + if trace { + debug!("pma-copy: batteries axis copy done"); + } + // Allocate new BatteriesMem in PMA and copy + let dest_mem: *mut BatteriesMem = pma.alloc_struct(1); + copy_nonoverlapping((*ptr).0, dest_mem, 1); + // Update pointer to point to PMA copy + *ptr = Batteries(dest_mem); + // Move to next node + ptr = &mut (*dest_mem).parent_batteries; + if (*dest_mem).parent_batteries.0.is_null() { + break; + } + steps += 1; + if trace && (steps & 0x3ff == 0) { + let now = std::time::Instant::now(); + if now.duration_since(last_progress).as_millis() >= 2000 { + debug!( + "pma-copy: batteries progress: steps={}, elapsed_ms={}", + steps, + start.elapsed().as_millis() + ); + last_progress = now; + } + } + } + if trace { + debug!( + "pma-copy: batteries done: steps={}, elapsed_ms={}", + steps, + start.elapsed().as_millis() + ); + } + } +} + +impl PmaCopyFrom for Batteries { + unsafe fn copy_from_pma(&mut self, from_pma: &Pma, to_pma: &mut Pma) { + if self.0.is_null() { + return; + } + let mut ptr: *mut Batteries = self; + loop { + let src_mem = (*ptr).0; + let dest_mem: *mut BatteriesMem = to_pma.alloc_struct(1); + copy_nonoverlapping(src_mem, dest_mem, 1); + *ptr = Batteries(dest_mem); + (*dest_mem).battery.copy_from_pma(from_pma, to_pma); + (*dest_mem).parent_axis.copy_from_pma(from_pma, to_pma); + ptr = &mut (*dest_mem).parent_batteries; + if (*dest_mem).parent_batteries.0.is_null() { + break; + } + } + } +} + impl Iterator for Batteries { type Item = (*mut Noun, Atom); fn next(&mut self) -> Option { @@ -89,7 +218,11 @@ impl Iterator for Batteries { } impl Batteries { - pub fn matches(self, stack: &mut NockStack, mut core: Noun) -> bool { + pub(crate) fn new(ptr: *mut BatteriesMem) -> Self { + Batteries(ptr) + } + + pub fn matches(self, stack: &mut NockStack, mut core: Noun, space: &NounSpace) -> bool { let mut root_found: bool = false; for (battery, parent_axis) in self { @@ -107,11 +240,11 @@ impl Batteries { }; }; }; - if let Ok(mut core_battery) = core.slot(2) { + if let Ok(mut core_battery) = core.slot(2, space) { if unsafe { !unifying_equality(stack, &mut core_battery, battery) } { return false; }; - if let Ok(core_parent) = core.slot_atom(parent_axis) { + if let Ok(core_parent) = core.slot_atom(parent_axis, space) { core = core_parent; continue; } else { @@ -143,6 +276,77 @@ struct BatteriesListMem { next: BatteriesList, } +impl PmaCopy for BatteriesList { + #[cfg(feature = "pma-assert")] + fn assert_in_pma(&self, pma: &Pma) { + if self.0.is_null() { + return; + } + let mut cursor = *self; + loop { + unsafe { + assert!( + pma.contains_ptr(cursor.0 as *const u8), + "BatteriesList node should be in PMA" + ); + (*cursor.0).batteries.assert_in_pma(pma); + if (*cursor.0).next.0.is_null() { + break; + } + cursor = (*cursor.0).next; + } + } + } + + #[cfg(not(feature = "pma-assert"))] + #[inline(always)] + fn assert_in_pma(&self, _pma: &Pma) {} + + unsafe fn copy_to_pma(&mut self, stack: &NockStack, pma: &mut Pma) { + if self.0.is_null() { + return; + } + let mut ptr: *mut BatteriesList = self; + loop { + if pma.contains_ptr((*ptr).0 as *const u8) { + break; + } + // Copy the batteries to PMA + (*(*ptr).0).batteries.copy_to_pma(stack, pma); + // Allocate new BatteriesListMem in PMA and copy + let dest_mem: *mut BatteriesListMem = pma.alloc_struct(1); + copy_nonoverlapping((*ptr).0, dest_mem, 1); + // Update pointer to point to PMA copy + *ptr = BatteriesList(dest_mem); + // Move to next node + ptr = &mut (*dest_mem).next; + if (*dest_mem).next.0.is_null() { + break; + } + } + } +} + +impl PmaCopyFrom for BatteriesList { + unsafe fn copy_from_pma(&mut self, from_pma: &Pma, to_pma: &mut Pma) { + if self.0.is_null() { + return; + } + let mut ptr: *mut BatteriesList = self; + loop { + let src_mem = (*ptr).0; + let dest_mem: *mut BatteriesListMem = to_pma.alloc_struct(1); + copy_nonoverlapping(src_mem, dest_mem, 1); + *ptr = BatteriesList(dest_mem); + (*dest_mem).batteries.copy_from_pma(from_pma, to_pma); + ptr = &mut (*dest_mem).next; + if (*dest_mem).next.0.is_null() { + break; + } + } + } +} + impl Preserve for BatteriesList { unsafe fn assert_in_stack(&self, stack: &NockStack) { if self.0.is_null() { @@ -197,8 +401,13 @@ impl Iterator for BatteriesList { } impl BatteriesList { - fn matches(mut self, stack: &mut NockStack, core: Noun) -> Option { - self.find(|&batteries| batteries.matches(stack, core)) + fn matches( + mut self, + stack: &mut NockStack, + core: Noun, + space: &NounSpace, + ) -> Option { + self.find(|&batteries| batteries.matches(stack, core, space)) } } @@ -252,6 +461,77 @@ impl Preserve for NounList { } } +impl PmaCopy for NounList { + #[cfg(feature = "pma-assert")] + fn assert_in_pma(&self, pma: &Pma) { + if self.0.is_null() { + return; + } + let mut cursor = *self; + loop { + unsafe { + assert!( + pma.contains_ptr(cursor.0 as *const u8), + "NounList node should be in PMA" + ); + (*cursor.0).element.assert_in_pma(pma); + if (*cursor.0).next.0.is_null() { + break; + } + cursor = (*cursor.0).next; + } + } + } + + #[cfg(not(feature = "pma-assert"))] + #[inline(always)] + fn assert_in_pma(&self, _pma: &Pma) {} + + unsafe fn copy_to_pma(&mut self, stack: &NockStack, pma: &mut Pma) { + if self.0.is_null() { + return; + } + let mut ptr: *mut NounList = self; + loop { + if pma.contains_ptr((*ptr).0 as *const u8) { + break; + } + // Copy the element noun to PMA + (*(*ptr).0).element.copy_to_pma(stack, pma); + // Allocate new NounListMem in PMA and copy + let dest_mem: *mut NounListMem = pma.alloc_struct(1); + copy_nonoverlapping((*ptr).0, dest_mem, 1); + // Update pointer to point to PMA copy + *ptr = NounList(dest_mem); + // Move to next node + ptr = &mut (*dest_mem).next; + if (*dest_mem).next.0.is_null() { + break; + } + } + } +} + +impl PmaCopyFrom for NounList { + unsafe fn copy_from_pma(&mut self, from_pma: &Pma, to_pma: &mut Pma) { + if self.0.is_null() { + return; + } + let mut ptr: *mut NounList = self; + loop { + let src_mem = (*ptr).0; + let dest_mem: *mut NounListMem = to_pma.alloc_struct(1); + copy_nonoverlapping(src_mem, dest_mem, 1); + *ptr = NounList(dest_mem); + (*dest_mem).element.copy_from_pma(from_pma, to_pma); + ptr = &mut (*dest_mem).next; + if (*dest_mem).next.0.is_null() { + break; + } + } + } +} + impl Iterator for NounList { type Item = *mut Noun; fn next(&mut self) -> Option { @@ -292,6 +572,57 @@ struct ColdMem { path_to_batteries: Hamt, } +impl PmaCopy for Cold { + #[cfg(feature = "pma-assert")] + fn assert_in_pma(&self, pma: &Pma) { + unsafe { + assert!( + pma.contains_ptr(self.0 as *const u8), + "Cold struct should be in PMA" + ); + (*self.0).battery_to_paths.assert_in_pma(pma); + (*self.0).root_to_paths.assert_in_pma(pma); + (*self.0).path_to_batteries.assert_in_pma(pma); + } + } + + #[cfg(not(feature = "pma-assert"))] + #[inline(always)] + fn assert_in_pma(&self, _pma: &Pma) {} + + unsafe fn copy_to_pma(&mut self, stack: &NockStack, pma: &mut Pma) { + if pma.contains_ptr(self.0 as *const u8) { + return; + } + // Copy each HAMT to PMA + (*self.0).battery_to_paths.copy_to_pma(stack, pma); + (*self.0).root_to_paths.copy_to_pma(stack, pma); + (*self.0).path_to_batteries.copy_to_pma(stack, pma); + // Allocate ColdMem in PMA and copy + let dest_mem: *mut ColdMem = pma.alloc_struct(1); + copy_nonoverlapping(self.0, dest_mem, 1); + // Update pointer to point to PMA copy + self.0 = dest_mem; + } +} + +impl PmaCopyFrom for Cold { + unsafe fn copy_from_pma(&mut self, from_pma: &Pma, to_pma: &mut Pma) { + if self.0.is_null() { + return; + } + let src_mem = self.0; + let dest_mem: *mut ColdMem = to_pma.alloc_struct(1); + copy_nonoverlapping(src_mem, dest_mem, 1); + self.0 = dest_mem; + (*dest_mem).battery_to_paths.copy_from_pma(from_pma, to_pma); + (*dest_mem).root_to_paths.copy_from_pma(from_pma, to_pma); + (*dest_mem) + .path_to_batteries + .copy_from_pma(from_pma, to_pma); + } +} + impl Preserve for Cold { unsafe fn assert_in_stack(&self, stack: &NockStack) { stack.assert_struct_is_in(self.0, 1); @@ -318,6 +649,20 @@ impl Cold { } } + pub fn pma_offset(&self, pma: &Pma) -> Option { + let ptr = self.0 as *const u8; + if pma.contains_ptr(ptr) { + Some(pma.offset_from_ptr(ptr)) + } else { + None + } + } + + pub unsafe fn from_pma_offset(pma: &Pma, offset: PmaOffsetWords) -> Self { + let ptr = pma.ptr_from_offset(offset) as *mut ColdMem; + Cold(ptr) + } + pub fn new(stack: &mut NockStack) -> Self { let battery_to_paths = Hamt::new(stack); let root_to_paths = Hamt::new(stack); @@ -365,14 +710,15 @@ impl Cold { /** Try to match a core directly to the cold state, print the resulting path if found */ pub fn matches(&mut self, stack: &mut NockStack, core: &mut Noun) -> Option { - let mut battery = (*core).slot(2).ok()?; + let space = stack.fast_noun_space(); + let mut battery = (*core).slot(2, &space).ok()?; unsafe { let paths = (*(self.0)).battery_to_paths.lookup(stack, &mut battery)?; for path in paths { if let Some(batteries_list) = (*(self.0)).path_to_batteries.lookup(stack, &mut (*path)) { - if let Some(_batt) = batteries_list.matches(stack, *core) { + if let Some(_batt) = batteries_list.matches(stack, *core, &space) { return Some(*path); } } @@ -393,6 +739,7 @@ impl Cold { parent_axis: Atom, mut chum: Noun, ) -> Result { + let space = stack.fast_noun_space(); unsafe { // Are we registering a root? if let Ok(parent_axis_direct) = parent_axis.as_direct() { @@ -454,17 +801,20 @@ impl Cold { } } - let mut battery = core.slot(2)?; - let mut parent = core.slot_atom(parent_axis)?; + let mut battery = core.slot(2, &space)?; + let mut parent = core.slot_atom(parent_axis, &space)?; // Check if we already registered this core if let Some(paths) = (*(self.0)).battery_to_paths.lookup(stack, &mut battery) { for path in paths { - if let Ok(path_cell) = (*path).as_cell() { - if unifying_equality(stack, &mut path_cell.head(), &mut chum) { + if let Ok(path_cell) = (*path).in_space(&space).as_cell() { + let mut head = path_cell.head().noun(); + if unifying_equality(stack, &mut head, &mut chum) { if let Some(batteries_list) = (*(self.0)).path_to_batteries.lookup(stack, &mut *path) { - if let Some(_batteries) = batteries_list.matches(stack, core) { + if let Some(_batteries) = + batteries_list.matches(stack, core, &space) + { return Ok(false); } } @@ -473,7 +823,7 @@ impl Cold { } } - let mut parent_battery = parent.slot(2)?; + let mut parent_battery = parent.slot(2, &space)?; // err until we actually found a parent let mut ret: Result = Err(Error::NoParent); @@ -488,7 +838,7 @@ impl Cold { let battery_list = path_to_batteries .lookup(stack, &mut *a_path) .unwrap_or(BATTERIES_LIST_NIL); - if let Some(parent_batteries) = battery_list.matches(stack, parent) { + if let Some(parent_batteries) = battery_list.matches(stack, parent, &space) { let mut my_path = T(stack, &[chum, *a_path]); let batteries_mem_ptr: *mut BatteriesMem = stack.struct_alloc(1); @@ -537,7 +887,7 @@ impl Cold { let battery_list = path_to_batteries .lookup(stack, &mut *a_path) .unwrap_or(BATTERIES_LIST_NIL); - if let Some(parent_batteries) = battery_list.matches(stack, parent) { + if let Some(parent_batteries) = battery_list.matches(stack, parent, &space) { let mut my_path = T(stack, &[chum, *a_path]); let batteries_mem_ptr: *mut BatteriesMem = stack.struct_alloc(1); @@ -593,22 +943,101 @@ impl Cold { } } -pub struct NounListIterator(Noun); +pub struct NounListIterator<'a> { + noun: Noun, + space: &'a NounSpace, +} + +impl<'a> NounListIterator<'a> { + fn new(noun: Noun, space: &'a NounSpace) -> Self { + Self { noun, space } + } +} -impl Iterator for NounListIterator { +impl<'a> Iterator for NounListIterator<'a> { type Item = Noun; fn next(&mut self) -> Option { - if let Ok(it) = self.0.as_cell() { - self.0 = it.tail(); - Some(it.head()) - } else if unsafe { self.0.raw_equals(&D(0)) } { + if let Ok(it) = self.noun.as_cell() { + self.noun = it.tail(self.space); + Some(it.head(self.space)) + } else if unsafe { self.noun.raw_equals(&D(0)) } { None } else { - panic!("Improper list terminator: {:?}", self.0) + panic!("Improper list terminator: {:?}", self.noun) } } } +fn copy_noun_into_allocator( + stack: &mut A, + noun: Noun, + space: &NounSpace, +) -> Noun { + let mut copied: IntMap = IntMap::new(); + let mut result = D(0); + let mut copy_stack = vec![(noun, std::ptr::addr_of_mut!(result))]; + + while let Some((noun, dest)) = copy_stack.pop() { + match noun.as_either_direct_allocated() { + either::Either::Left(direct) => unsafe { + *dest = direct.as_noun(); + }, + either::Either::Right(allocated) => match allocated.as_either() { + either::Either::Left(indirect) => { + let atom_handle = indirect.as_atom().in_space(space); + let raw_pointer = unsafe { atom_handle.raw_pointer() }; + let raw_size = atom_handle.raw_size(); + if let Some(copied_noun) = copied.get(raw_pointer as u64) { + unsafe { *dest = *copied_noun }; + continue; + } + + let indirect_mem = unsafe { stack.alloc_indirect(atom_handle.size()) }; + unsafe { + copy_nonoverlapping(raw_pointer, indirect_mem, raw_size); + } + let copied_noun = unsafe { + IndirectAtom::from_raw_pointer(indirect_mem) + .as_atom() + .as_noun() + }; + copied.insert(raw_pointer as u64, copied_noun); + unsafe { *dest = copied_noun }; + } + either::Either::Right(cell) => { + let cell_handle = cell.in_space(space); + let raw_pointer = unsafe { cell_handle.raw_pointer() }; + if let Some(copied_noun) = copied.get(raw_pointer as u64) { + unsafe { *dest = *copied_noun }; + continue; + } + + let cell_mem = unsafe { stack.alloc_cell() }; + unsafe { + copy_nonoverlapping(raw_pointer, cell_mem, 1); + } + let copied_noun = + unsafe { crate::noun::Cell::from_raw_pointer(cell_mem).as_noun() }; + copied.insert(raw_pointer as u64, copied_noun); + unsafe { + *dest = copied_noun; + copy_stack.push(( + cell_handle.tail().noun(), + std::ptr::addr_of_mut!((*cell_mem).tail), + )); + copy_stack.push(( + cell_handle.head().noun(), + std::ptr::addr_of_mut!((*cell_mem).head), + )); + } + } + }, + } + } + + result +} + #[derive(thiserror::Error, Debug)] pub enum FromNounError { #[error("Not an atom")] @@ -630,7 +1059,11 @@ pub trait Nounable { // type Allocator; fn into_noun(self, stack: &mut A) -> Noun; - fn from_noun(stack: &mut A, noun: &Noun) -> NounableResult + fn from_noun( + stack: &mut A, + noun: &Noun, + space: &NounSpace, + ) -> NounableResult where Self: Sized; } @@ -641,8 +1074,13 @@ impl Nounable for Atom { fn into_noun(self, _stack: &mut A) -> Noun { self.as_noun() } - fn from_noun(_stack: &mut A, noun: &Noun) -> NounableResult { - noun.atom().ok_or(FromNounError::NotAtom) + fn from_noun( + stack: &mut A, + noun: &Noun, + space: &NounSpace, + ) -> NounableResult { + let copied = copy_noun_into_allocator(stack, *noun, space); + copied.as_atom().map_err(|_| FromNounError::NotAtom) } } @@ -652,9 +1090,13 @@ impl Nounable for u64 { // Copied from Crown's IntoNoun, not sure why this isn't D(*self) unsafe { Atom::from_raw(self).into_noun(_stack) } } - fn from_noun(_stack: &mut A, noun: &Noun) -> NounableResult { + fn from_noun( + _stack: &mut A, + noun: &Noun, + space: &NounSpace, + ) -> NounableResult { let atom = noun.atom().ok_or(FromNounError::NotAtom)?; - let as_u64 = atom.as_u64()?; + let as_u64 = atom.in_space(space).as_u64()?; Ok(as_u64) } } @@ -665,8 +1107,12 @@ impl Nounable for Noun { self } - fn from_noun(_stack: &mut A, noun: &Self) -> NounableResult { - Ok(*noun) + fn from_noun( + stack: &mut A, + noun: &Self, + space: &NounSpace, + ) -> NounableResult { + Ok(copy_noun_into_allocator(stack, *noun, space)) } } @@ -675,13 +1121,19 @@ impl Nounable for &str { fn into_noun(self, stack: &mut A) -> Noun { let contents_atom = unsafe { let bytes = self.bytes().collect::>(); - IndirectAtom::new_raw_bytes_ref(stack, bytes.as_slice()).normalize_as_atom() + let space = stack.noun_space(); + IndirectAtom::new_raw_bytes_ref(stack, bytes.as_slice()).normalize_as_atom(&space) }; contents_atom.into_noun(stack) } - fn from_noun(_stack: &mut A, noun: &Noun) -> NounableResult { + fn from_noun( + _stack: &mut A, + noun: &Noun, + space: &NounSpace, + ) -> NounableResult { let atom = noun.as_atom()?; - let bytes = atom.as_ne_bytes(); + let atom_handle = atom.in_space(space); + let bytes = atom_handle.as_ne_bytes(); let utf8 = std::str::from_utf8(bytes)?; let allocated = utf8.to_string(); Ok(allocated) @@ -699,10 +1151,14 @@ impl Nounable for &[T] { list } - fn from_noun(_stack: &mut A, noun: &Noun) -> NounableResult { + fn from_noun( + _stack: &mut A, + noun: &Noun, + space: &NounSpace, + ) -> NounableResult { let mut items: Vec<::Target> = vec![]; - for item in NounListIterator(*noun) { - let item = T::from_noun(_stack, &item)?; + for item in NounListIterator::new(*noun, space) { + let item = T::from_noun(_stack, &item, space)?; items.push(item); } Ok(items) @@ -720,15 +1176,19 @@ impl Nounable for (T, U, V) { T(stack, &[a_noun, b_noun, c_noun]) } - fn from_noun(_stack: &mut A, noun: &Noun) -> NounableResult { + fn from_noun( + _stack: &mut A, + noun: &Noun, + space: &NounSpace, + ) -> NounableResult { // it's a three tuple now - let cell = noun.cell().ok_or(FromNounError::NotCell)?; - let head = cell.head(); - let tail = cell.tail(); - let a = T::from_noun(_stack, &head)?; - let cell = tail.as_cell()?; - let b = U::from_noun(_stack, &cell.head())?; - let c = V::from_noun(_stack, &cell.tail())?; + let cell = noun.in_space(space).as_cell()?; + let head = cell.head().noun(); + let tail = cell.tail().noun(); + let a = T::from_noun(_stack, &head, space)?; + let cell = tail.in_space(space).as_cell()?; + let b = U::from_noun(_stack, &cell.head().noun(), space)?; + let c = V::from_noun(_stack, &cell.tail().noun(), space)?; Ok((a, b, c)) } } @@ -742,12 +1202,16 @@ impl Nounable for (T, U) { T(stack, &[a_noun, b_noun]) } - fn from_noun(_stack: &mut A, noun: &Noun) -> NounableResult { - let cell = noun.cell().ok_or(FromNounError::NotCell)?; - let head = cell.head(); - let tail = cell.tail(); - let a = T::from_noun(_stack, &head)?; - let b = U::from_noun(_stack, &tail)?; + fn from_noun( + _stack: &mut A, + noun: &Noun, + space: &NounSpace, + ) -> NounableResult { + let cell = noun.in_space(space).as_cell()?; + let head = cell.head().noun(); + let tail = cell.tail().noun(); + let a = T::from_noun(_stack, &head, space)?; + let b = U::from_noun(_stack, &tail, space)?; Ok((a, b)) } } @@ -762,9 +1226,14 @@ impl Nounable for NounList { list } - fn from_noun(stack: &mut A, noun: &Noun) -> NounableResult { + fn from_noun( + stack: &mut A, + noun: &Noun, + space: &NounSpace, + ) -> NounableResult { let mut result = NOUN_LIST_NIL; - for item in NounListIterator(*noun) { + for item in NounListIterator::new(*noun, space) { + let item = ::from_noun(stack, &item, space)?; let list_mem_ptr: *mut NounListMem = unsafe { stack.alloc_struct(1) }; unsafe { list_mem_ptr.write(NounListMem { @@ -791,12 +1260,16 @@ impl Nounable for Batteries { list } - fn from_noun(stack: &mut A, noun: &Noun) -> NounableResult { + fn from_noun( + stack: &mut A, + noun: &Noun, + space: &NounSpace, + ) -> NounableResult { let mut batteries = NO_BATTERIES; - for item in NounListIterator(*noun) { - let cell = item.cell().ok_or(FromNounError::NotCell)?; - let battery = cell.head(); - let parent_axis = cell.tail().as_atom()?; + for item in NounListIterator::new(*noun, space) { + let cell = item.in_space(space).as_cell()?; + let battery = ::from_noun(stack, &cell.head().noun(), space)?; + let parent_axis = ::from_noun(stack, &cell.tail().noun(), space)?; let batteries_mem: *mut BatteriesMem = unsafe { stack.alloc_struct(1) }; unsafe { batteries_mem.write(BatteriesMem { @@ -822,10 +1295,14 @@ impl Nounable for BatteriesList { list } - fn from_noun(stack: &mut A, noun: &Noun) -> NounableResult { + fn from_noun( + stack: &mut A, + noun: &Noun, + space: &NounSpace, + ) -> NounableResult { let mut batteries_list = BATTERIES_LIST_NIL; - for item in NounListIterator(*noun) { - let batteries = Batteries::from_noun(stack, &item)?; + for item in NounListIterator::new(*noun, space) { + let batteries = Batteries::from_noun(stack, &item, space)?; let batteries_list_mem: *mut BatteriesListMem = unsafe { stack.alloc_struct(1) }; unsafe { batteries_list_mem.write(BatteriesListMem { @@ -859,12 +1336,16 @@ impl Nounable for Hamt { list } - fn from_noun(stack: &mut A, noun: &Noun) -> NounableResult { + fn from_noun( + stack: &mut A, + noun: &Noun, + space: &NounSpace, + ) -> NounableResult { let mut items = Vec::new(); - for item in NounListIterator(*noun) { - let cell = item.cell().ok_or(FromNounError::NotCell)?; - let key = cell.head(); - let value = T::from_noun(stack, &cell.tail())?; + for item in NounListIterator::new(*noun, space) { + let cell = item.in_space(space).as_cell()?; + let key = ::from_noun(stack, &cell.head().noun(), space)?; + let value = T::from_noun(stack, &cell.tail().noun(), space)?; items.push((key, value)); } // items.reverse(); @@ -932,36 +1413,40 @@ impl Nounable for Cold { ) } - fn from_noun(stack: &mut A, noun: &Noun) -> NounableResult { + fn from_noun( + stack: &mut A, + noun: &Noun, + space: &NounSpace, + ) -> NounableResult { let mut battery_to_paths = Vec::new(); let mut root_to_paths = Vec::new(); let mut path_to_batteries = Vec::new(); - let battery_to_paths_noun = noun.slot(2)?; - let root_to_paths_noun = noun.slot(6)?; - let path_to_batteries_noun = noun.slot(7)?; + let battery_to_paths_noun = noun.slot(2, space)?; + let root_to_paths_noun = noun.slot(6, space)?; + let path_to_batteries_noun = noun.slot(7, space)?; // iterate over battery_to_paths_noun - for item in NounListIterator(battery_to_paths_noun) { - let cell = item.cell().ok_or(FromNounError::NotCell)?; - let key = cell.head(); - let value = NounList::from_noun(stack, &cell.tail())?; + for item in NounListIterator::new(battery_to_paths_noun, space) { + let cell = item.in_space(space).as_cell()?; + let key = cell.head().noun(); + let value = NounList::from_noun(stack, &cell.tail().noun(), space)?; battery_to_paths.push((key, value)); } // iterate over root_to_paths_noun - for item in NounListIterator(root_to_paths_noun) { - let cell = item.cell().ok_or(FromNounError::NotCell)?; - let key = cell.head(); - let value = NounList::from_noun(stack, &cell.tail())?; + for item in NounListIterator::new(root_to_paths_noun, space) { + let cell = item.in_space(space).as_cell()?; + let key = cell.head().noun(); + let value = NounList::from_noun(stack, &cell.tail().noun(), space)?; root_to_paths.push((key, value)); } // iterate over path_to_batteries_noun - for item in NounListIterator(path_to_batteries_noun) { - let cell = item.cell().ok_or(FromNounError::NotCell)?; - let key = cell.head(); - let value = BatteriesList::from_noun(stack, &cell.tail())?; + for item in NounListIterator::new(path_to_batteries_noun, space) { + let cell = item.in_space(space).as_cell()?; + let key = cell.head().noun(); + let value = BatteriesList::from_noun(stack, &cell.tail().noun(), space)?; path_to_batteries.push((key, value)); } battery_to_paths.reverse(); @@ -978,11 +1463,12 @@ pub(crate) mod test { use std::iter::FromIterator; use super::*; + use crate::ext::noun_equality; use crate::hamt::Hamt; - use crate::mem::NockStack; - use crate::noun::{Cell, Noun, D}; + use crate::mem::{NockStack, NOCK_STACK_SIZE_TINY}; + use crate::noun::{AllocLocation, Cell, Noun, NounSpace, D}; /// Default stack size for tests where you aren't intending to run out of space - pub(crate) const DEFAULT_STACK_SIZE: usize = 1 << 27; + pub(crate) const DEFAULT_STACK_SIZE: usize = NOCK_STACK_SIZE_TINY; pub(crate) fn make_test_stack(size: usize) -> NockStack { let top_slots = 3; @@ -1021,10 +1507,11 @@ pub(crate) mod test { #[cfg_attr(miri, ignore)] fn cold_bidirectional_conversion() { let mut stack = make_test_stack(DEFAULT_STACK_SIZE); + let space = stack.noun_space(); let cold = make_cold_state(&mut stack); let cold_noun = cold.into_noun(&mut stack); - let new_cold = - Cold::from_noun(&mut stack, &cold_noun).expect("Failed to convert noun to cold"); + let new_cold = Cold::from_noun(&mut stack, &cold_noun, &space) + .expect("Failed to convert noun to cold"); // battery_to_paths let old_battery_to_paths = unsafe { &(*cold.0).battery_to_paths }; @@ -1113,20 +1600,20 @@ pub(crate) mod test { #[cfg_attr(miri, ignore)] fn hamt_bidirectional_conversion() { let mut stack = make_test_stack(DEFAULT_STACK_SIZE); + let space = stack.noun_space(); let items = vec![(D(0), D(1)), (D(2), D(3))]; let hamt = super::hamt_from_vec(&mut stack, items); let noun = hamt.into_noun(&mut stack); - let new_hamt: Vec<(Noun, Noun)> = as Nounable>::from_noun::( - &mut stack, &noun, - ) - .unwrap_or_else(|err| { - panic!( - "Panicked with {err:?} at {}:{} (git sha: {:?})", - file!(), - line!(), - option_env!("GIT_SHA") - ) - }); + let new_hamt: Vec<(Noun, Noun)> = + as Nounable>::from_noun::(&mut stack, &noun, &space) + .unwrap_or_else(|err| { + panic!( + "Panicked with {err:?} at {}:{} (git sha: {:?})", + file!(), + line!(), + option_env!("GIT_SHA") + ) + }); let flat_hamt: Vec<(Noun, Noun)> = hamt.iter().flatten().cloned().collect(); for (a, b) in new_hamt.iter().zip(flat_hamt.iter()) { let key_a = &mut a.0.clone() as *mut Noun; @@ -1148,6 +1635,36 @@ pub(crate) mod test { } } + #[test] + #[cfg_attr(miri, ignore)] + fn hamt_from_noun_rehomes_foreign_keys_and_values() { + let mut source = make_test_stack(DEFAULT_STACK_SIZE); + let source_space = source.noun_space(); + let foreign_key = T(&mut source, &[D(10), D(11)]); + let foreign_value = T(&mut source, &[D(12), D(13)]); + let hamt = super::hamt_from_vec(&mut source, vec![(foreign_key, foreign_value)]); + let noun = hamt.into_noun(&mut source); + + let mut dest = make_test_stack(DEFAULT_STACK_SIZE); + let decoded: Vec<(Noun, Noun)> = + as Nounable>::from_noun::(&mut dest, &noun, &source_space) + .expect("decode hamt from foreign stack"); + + assert_eq!(decoded.len(), 1); + let dest_space = dest.noun_space(); + let (decoded_key, decoded_value) = decoded[0]; + assert!( + !unsafe { decoded_key.raw_equals(&foreign_key) }, + "decoded hamt key should not retain the foreign pointer" + ); + assert!( + !unsafe { decoded_value.raw_equals(&foreign_value) }, + "decoded hamt value should not retain the foreign pointer" + ); + verify_noun_stack_allocated(decoded_key, &dest_space, "decoded hamt key"); + verify_noun_stack_allocated(decoded_value, &dest_space, "decoded hamt value"); + } + fn make_batteries_list(stack: &mut NockStack, v: &[u64]) -> BatteriesList { let mut batteries_list = BATTERIES_LIST_NIL; for &item in v.iter().rev() { @@ -1183,10 +1700,12 @@ pub(crate) mod test { #[cfg_attr(miri, ignore)] fn batteries_list_bidirectional_conversion() { let mut stack = make_test_stack(DEFAULT_STACK_SIZE); + let space = stack.noun_space(); let batteries_list2 = make_batteries_list(&mut stack, &[1, 2]); let batteries_list_noun = batteries_list2.into_noun(&mut stack); - let new_batteries_list2 = BatteriesList::from_noun(&mut stack, &batteries_list_noun) - .expect("Failed to convert noun to batteries list"); + let new_batteries_list2 = + BatteriesList::from_noun(&mut stack, &batteries_list_noun, &space) + .expect("Failed to convert noun to batteries list"); for (a, b) in batteries_list2.zip(new_batteries_list2) { let mut a_noun = a.into_noun(&mut stack); let mut b_noun = b.into_noun(&mut stack); @@ -1239,9 +1758,10 @@ pub(crate) mod test { #[cfg_attr(miri, ignore)] fn batteries_bidirectional_conversion() { let mut stack = make_test_stack(DEFAULT_STACK_SIZE); + let space = stack.noun_space(); let batteries2 = make_batteries(&mut stack); let batteries_noun = batteries2.into_noun(&mut stack); - let new_batteries = Batteries::from_noun(&mut stack, &batteries_noun) + let new_batteries = Batteries::from_noun(&mut stack, &batteries_noun, &space) .expect("Failed to convert noun to batteries"); assert_eq!(new_batteries.count(), 2); assert_eq!(batteries2.count(), 2); @@ -1263,20 +1783,51 @@ pub(crate) mod test { assert!( unsafe { unifying_equality(&mut stack, a_atom_noun_ptr, b_atom_noun_ptr) }, "Parent axes don't match: {:?} {:?}", - a_atom.as_u64(), - b_atom.as_u64() + a_atom.in_space(&space).as_u64(), + b_atom.in_space(&space).as_u64() ); } } + #[test] + #[cfg_attr(miri, ignore)] + fn batteries_from_noun_rehomes_foreign_payloads() { + let mut source = make_test_stack(DEFAULT_STACK_SIZE); + let source_space = source.noun_space(); + let foreign_battery = T(&mut source, &[D(20), D(21)]); + let foreign_axis = Atom::new(&mut source, u64::MAX); + let batteries_item = T(&mut source, &[foreign_battery, foreign_axis.as_noun()]); + let batteries_noun = T(&mut source, &[batteries_item, D(0)]); + + let mut dest = make_test_stack(DEFAULT_STACK_SIZE); + let decoded = Batteries::from_noun(&mut dest, &batteries_noun, &source_space) + .expect("decode batteries from foreign stack"); + unsafe { decoded.assert_in_stack(&dest) }; + + let dest_space = dest.noun_space(); + let (battery_ptr, parent_axis) = decoded.into_iter().next().expect("decoded batteries"); + let battery = unsafe { *battery_ptr }; + assert!( + !unsafe { battery.raw_equals(&foreign_battery) }, + "decoded battery should not retain the foreign pointer" + ); + assert!( + !unsafe { parent_axis.as_noun().raw_equals(&foreign_axis.as_noun()) }, + "decoded parent axis should not retain the foreign pointer" + ); + verify_noun_stack_allocated(battery, &dest_space, "decoded battery"); + verify_noun_stack_allocated(parent_axis.as_noun(), &dest_space, "decoded parent axis"); + } + #[test] #[cfg_attr(miri, ignore)] fn tuple_bidirectional_conversion() { let mut stack = make_test_stack(DEFAULT_STACK_SIZE); + let space = stack.noun_space(); let tup = (D(1), D(2), D(3)); let noun = tup.into_noun(&mut stack); let new_tup: (Noun, Noun, Noun) = - <(Noun, Noun, Noun) as Nounable>::from_noun::(&mut stack, &noun) + <(Noun, Noun, Noun) as Nounable>::from_noun::(&mut stack, &noun, &space) .unwrap_or_else(|err| { panic!( "Panicked with {err:?} at {}:{} (git sha: {:?})", @@ -1318,6 +1869,39 @@ pub(crate) mod test { noun_list } + fn make_noun_list_from_nouns(stack: &mut NockStack, nouns: &[Noun]) -> NounList { + let mut noun_list = NOUN_LIST_NIL; + for &item in nouns.iter().rev() { + let noun_list_mem: *mut NounListMem = unsafe { stack.alloc_struct(1) }; + unsafe { + noun_list_mem.write(NounListMem { + element: item, + next: noun_list, + }); + } + noun_list = NounList(noun_list_mem); + } + noun_list + } + + fn verify_noun_stack_allocated(noun: Noun, space: &NounSpace, context: &str) { + if noun.is_direct() { + return; + } + + let location = noun.in_space(space).allocated_location(); + assert!( + matches!(location, Some(AllocLocation::Stack)), + "{} should be stack-allocated after decode", + context + ); + + if let Ok(cell) = noun.in_space(space).as_cell() { + verify_noun_stack_allocated(cell.head().noun(), space, context); + verify_noun_stack_allocated(cell.tail().noun(), space, context); + } + } + #[test] #[cfg_attr(miri, ignore)] fn noun_list_bidirectional_conversion() { @@ -1328,17 +1912,17 @@ pub(crate) mod test { let slice = vec.as_slice(); let noun_list = make_noun_list(&mut stack, slice); let noun = noun_list.into_noun(&mut stack); - let new_noun_list: NounList = ::from_noun::( - &mut stack, &noun, - ) - .unwrap_or_else(|err| { - panic!( - "Panicked with {err:?} at {}:{} (git sha: {:?})", - file!(), - line!(), - option_env!("GIT_SHA") - ) - }); + let space = stack.noun_space(); + let new_noun_list: NounList = + ::from_noun::(&mut stack, &noun, &space) + .unwrap_or_else(|err| { + panic!( + "Panicked with {err:?} at {}:{} (git sha: {:?})", + file!(), + line!(), + option_env!("GIT_SHA") + ) + }); let mut item_count = 0; for (a, b) in new_noun_list.zip(items.iter()) { let a_ptr = a; @@ -1355,10 +1939,37 @@ pub(crate) mod test { assert_eq!(item_count, ITEM_COUNT as usize); } + #[test] + #[cfg_attr(miri, ignore)] + fn noun_list_from_noun_rehomes_foreign_elements() { + let mut source = make_test_stack(DEFAULT_STACK_SIZE); + let source_space = source.noun_space(); + let foreign_elem = T(&mut source, &[D(1), D(2)]); + let noun = make_noun_list_from_nouns(&mut source, &[foreign_elem]).into_noun(&mut source); + + let mut dest = make_test_stack(DEFAULT_STACK_SIZE); + let decoded = NounList::from_noun(&mut dest, &noun, &source_space) + .expect("decode noun list from foreign stack"); + unsafe { decoded.assert_in_stack(&dest) }; + + let dest_space = dest.noun_space(); + let elem_ptr = decoded + .into_iter() + .next() + .expect("decoded noun list element"); + let elem = unsafe { *elem_ptr }; + assert!( + !unsafe { elem.raw_equals(&foreign_elem) }, + "decoded element should not retain the foreign pointer" + ); + verify_noun_stack_allocated(elem, &dest_space, "decoded noun list element"); + } + #[test] #[cfg_attr(miri, ignore)] fn how_to_noun() { let mut stack = make_test_stack(DEFAULT_STACK_SIZE); + let space = stack.noun_space(); let tup: &[Noun] = &[D(0), D(1)]; let cell = Cell::new_tuple(&mut stack, tup); let noun: Noun = cell.as_noun(); @@ -1372,7 +1983,9 @@ pub(crate) mod test { option_env!("GIT_SHA") ) }) + .in_space(&space) .head() + .noun() .direct() .unwrap_or_else(|| { panic!( @@ -1393,7 +2006,9 @@ pub(crate) mod test { option_env!("GIT_SHA") ) }) + .in_space(&space) .tail() + .noun() .direct() .unwrap_or_else(|| { panic!( @@ -1412,6 +2027,7 @@ pub(crate) mod test { #[cfg_attr(miri, ignore)] fn how_to_noun_but_listy() { let mut stack = make_test_stack(DEFAULT_STACK_SIZE); + let space = stack.noun_space(); let tup: &[Noun] = &[D(0), D(1)]; let cell = Cell::new_tuple(&mut stack, tup); let noun: Noun = cell.as_noun(); @@ -1425,7 +2041,9 @@ pub(crate) mod test { option_env!("GIT_SHA") ) }) + .in_space(&space) .head() + .noun() .direct() .unwrap_or_else(|| { panic!( @@ -1446,7 +2064,9 @@ pub(crate) mod test { option_env!("GIT_SHA") ) }) + .in_space(&space) .tail() + .noun() .direct() .unwrap_or_else(|| { panic!( @@ -1460,4 +2080,538 @@ pub(crate) mod test { assert_eq!(car, 0); assert_eq!(cdr, 1); } + + /// Helper to recursively verify a noun is not stack-allocated + fn verify_noun_not_stack_allocated(noun: Noun, space: &NounSpace, context: &str) { + if noun.is_direct() { + return; + } + + let location = noun.in_space(space).allocated_location(); + assert!( + !matches!(location, Some(AllocLocation::Stack)), + "{} should be in offset form after evacuation", + context + ); + + if let Ok(cell) = noun.in_space(space).as_cell() { + verify_noun_not_stack_allocated(cell.head().noun(), space, context); + verify_noun_not_stack_allocated(cell.tail().noun(), space, context); + } + } + + /// Verifies NounList can be evacuated to PMA and remains functional. + /// + /// This test exercises: + /// - Creating a NounList with multiple elements + /// - Evacuating the NounList to PMA via copy_to_pma + /// - Verifying all elements are still accessible after evacuation + /// - Verifying all nouns are in offset form (not stack-allocated) + /// - Verifying the NounList passes assert_in_pma + /// + /// Note: copy_to_pma sets forwarding pointers in the source nouns, which corrupts + /// them for normal use. We use expected_values (raw u64s) for comparison since + /// those aren't affected by forwarding pointers. + #[test] + #[cfg_attr(miri, ignore)] + fn test_evacuate_noun_list_round_trip() { + use crate::pma::{test_pma_path, Pma, PmaCopy}; + + let mut stack = make_test_stack(DEFAULT_STACK_SIZE); + let mut pma = + Pma::new(100000, test_pma_path("noun_list")).expect("Failed to create test PMA"); + let space = NounSpace::new(&stack, &pma); + + // The expected values - we use these for comparison since the source + // nouns will have forwarding pointers set after evacuation + let expected_values: Vec = vec![10, 20, 30, 40, 50]; + + // Create a NounList with test data + let mut noun_list = make_noun_list(&mut stack, &expected_values); + + // Count elements before evacuation + let count_before: usize = noun_list.into_iter().count(); + assert_eq!(count_before, 5, "Should have 5 elements before evacuation"); + + // Evacuate NounList to PMA + unsafe { + noun_list.copy_to_pma(&stack, &mut pma); + } + + // Count elements and collect values after evacuation + let mut values_after = Vec::new(); + for elem_ptr in noun_list { + let elem = unsafe { *elem_ptr }; + values_after.push(unsafe { elem.as_raw() }); + } + + assert_eq!( + values_after.len(), + expected_values.len(), + "Element count should be preserved" + ); + assert_eq!( + values_after, expected_values, + "Element values should be preserved" + ); + + // Verify all nouns in the list are in offset form + for elem_ptr in noun_list { + let elem = unsafe { *elem_ptr }; + verify_noun_not_stack_allocated(elem, &space, "NounList element"); + } + + // Verify the NounList passes assert_in_pma + noun_list.assert_in_pma(&pma); + } + + /// Verifies NounList with complex nouns (Cells, IndirectAtoms) can be evacuated to PMA. + /// + /// This test exercises evacuation of NounList elements that are not direct atoms, + /// ensuring that Cells and IndirectAtoms are correctly copied to the PMA. + /// + /// Note: copy_to_pma sets forwarding pointers in the source nouns, which corrupts + /// them for normal use. We use a ref_stack to create reference copies for comparison. + #[test] + #[cfg_attr(miri, ignore)] + fn test_evacuate_noun_list_complex_nouns() { + use crate::noun::{Cell, IndirectAtom}; + use crate::pma::{test_pma_path, Pma, PmaCopy}; + + let mut stack = make_test_stack(DEFAULT_STACK_SIZE); + let mut ref_stack = make_test_stack(DEFAULT_STACK_SIZE); + let mut pma = Pma::new(100000, test_pma_path("noun_list_complex")) + .expect("Failed to create test PMA"); + let space = NounSpace::new(&stack, &pma); + + // Create complex nouns on the main stack + // Element 0: A cell [1 2] + let cell1 = Cell::new(&mut stack, D(1), D(2)).as_noun(); + // Element 1: An indirect atom (larger than 63 bits) + let big_data: [u64; 2] = [0xDEADBEEF_CAFEBABE, 0x12345678_9ABCDEF0]; + let indirect1 = + unsafe { IndirectAtom::new_raw(&mut stack, 2, big_data.as_ptr()).as_noun() }; + // Element 2: A nested cell [[3 4] 5] + let inner_cell = Cell::new(&mut stack, D(3), D(4)).as_noun(); + let nested_cell = Cell::new(&mut stack, inner_cell, D(5)).as_noun(); + // Element 3: A direct atom for variety + let direct1 = D(42); + // Element 4: A cell with structural sharing [[a b] [a b]] where a,b are IndirectAtoms + let big_a: [u64; 2] = [0x1111111111111111, 0x2222222222222222]; + let big_b: [u64; 2] = [0x3333333333333333, 0x4444444444444444]; + let indirect_a = unsafe { IndirectAtom::new_raw(&mut stack, 2, big_a.as_ptr()).as_noun() }; + let indirect_b = unsafe { IndirectAtom::new_raw(&mut stack, 2, big_b.as_ptr()).as_noun() }; + let shared_cell = Cell::new(&mut stack, indirect_a, indirect_b).as_noun(); + let structural_sharing = Cell::new(&mut stack, shared_cell, shared_cell).as_noun(); + + // Build the NounList manually with complex nouns + let mut noun_list = NOUN_LIST_NIL; + for noun in [direct1, nested_cell, indirect1, cell1, structural_sharing] + .iter() + .rev() + { + let mem: *mut NounListMem = unsafe { stack.alloc_struct(1) }; + unsafe { + mem.write(NounListMem { + element: *noun, + next: noun_list, + }); + } + noun_list = NounList(mem); + } + + // Create reference copies on ref_stack for comparison after evacuation + let ref_cell1 = Cell::new(&mut ref_stack, D(1), D(2)).as_noun(); + let ref_indirect1 = + unsafe { IndirectAtom::new_raw(&mut ref_stack, 2, big_data.as_ptr()).as_noun() }; + let ref_inner_cell = Cell::new(&mut ref_stack, D(3), D(4)).as_noun(); + let ref_nested_cell = Cell::new(&mut ref_stack, ref_inner_cell, D(5)).as_noun(); + let ref_direct1 = D(42); + let ref_indirect_a = + unsafe { IndirectAtom::new_raw(&mut ref_stack, 2, big_a.as_ptr()).as_noun() }; + let ref_indirect_b = + unsafe { IndirectAtom::new_raw(&mut ref_stack, 2, big_b.as_ptr()).as_noun() }; + let ref_shared_cell = Cell::new(&mut ref_stack, ref_indirect_a, ref_indirect_b).as_noun(); + let ref_structural_sharing = + Cell::new(&mut ref_stack, ref_shared_cell, ref_shared_cell).as_noun(); + // Order must match iteration order of noun_list: direct1, nested_cell, indirect1, cell1, structural_sharing + let ref_nouns = + [ref_direct1, ref_nested_cell, ref_indirect1, ref_cell1, ref_structural_sharing]; + + // Count elements before evacuation + let count_before: usize = noun_list.into_iter().count(); + assert_eq!(count_before, 5, "Should have 5 elements before evacuation"); + + // Evacuate NounList to PMA + unsafe { + noun_list.copy_to_pma(&stack, &mut pma); + } + + // Verify element count after evacuation + let count_after: usize = noun_list.into_iter().count(); + assert_eq!(count_after, 5, "Should have 5 elements after evacuation"); + + // Verify elements match reference copies using unifying_equality + let ref_space = NounSpace::new(&ref_stack, &pma); + for (i, elem_ptr) in noun_list.into_iter().enumerate() { + let elem = unsafe { *elem_ptr }; + let ref_noun = ref_nouns[i]; + assert!( + noun_equality(ref_noun.in_space(&ref_space), elem.in_space(&ref_space),), + "Element {} should match reference after evacuation", + i + ); + } + + // Verify all nouns in the list are in offset form + for elem_ptr in noun_list { + let elem = unsafe { *elem_ptr }; + verify_noun_not_stack_allocated(elem, &space, "NounList complex element"); + } + + // Verify the NounList passes assert_in_pma + noun_list.assert_in_pma(&pma); + } + + #[test] + #[cfg_attr(miri, ignore)] + fn test_copy_from_pma_does_not_mutate_source_noun_list() { + use crate::noun::Cell; + use crate::pma::{test_pma_path, Pma, PmaCopy, PmaCopyFrom}; + + let mut stack = make_test_stack(DEFAULT_STACK_SIZE); + let mut from_pma = Pma::new(100000, test_pma_path("noun_list_copy_from_source")) + .expect("Failed to create source PMA"); + let mut to_pma = Pma::new(100000, test_pma_path("noun_list_copy_from_dest")) + .expect("Failed to create destination PMA"); + + // Deliberately shift destination offsets so source mutation cannot hide + // behind matching allocation offsets in the two slabs. + unsafe { + to_pma.alloc_struct::(17); + } + + let cell = Cell::new(&mut stack, D(1), D(2)).as_noun(); + let mem: *mut NounListMem = unsafe { stack.alloc_struct(1) }; + unsafe { + mem.write(NounListMem { + element: cell, + next: NOUN_LIST_NIL, + }); + } + let mut source = NounList(mem); + unsafe { + source.copy_to_pma(&stack, &mut from_pma); + } + + let source_element_before = unsafe { (*source.0).element.as_raw() }; + let mut copied = source; + unsafe { + copied.copy_from_pma(&from_pma, &mut to_pma); + } + let source_element_after = unsafe { (*source.0).element.as_raw() }; + + assert_eq!( + source_element_after, source_element_before, + "copy_from_pma must not rewrite nouns stored in the source PMA" + ); + } + + /// Verifies Batteries can be evacuated to PMA and remains functional. + /// + /// This test exercises: + /// - Creating a Batteries linked list with multiple entries + /// - Evacuating the Batteries to PMA via copy_to_pma + /// - Verifying all entries are still accessible after evacuation + /// - Verifying all nouns are in offset form (not stack-allocated) + /// - Verifying the Batteries passes assert_in_pma + /// + /// Note: copy_to_pma sets forwarding pointers in the source nouns, which corrupts + /// them for normal use. We use expected values for comparison. + #[test] + #[cfg_attr(miri, ignore)] + fn test_evacuate_batteries_round_trip() { + use crate::pma::{test_pma_path, Pma, PmaCopy}; + + let mut stack = make_test_stack(DEFAULT_STACK_SIZE); + let mut pma = + Pma::new(100000, test_pma_path("batteries")).expect("Failed to create test PMA"); + let space = NounSpace::new(&stack, &pma); + + // Create a Batteries list using the test helper + // This creates: [battery=D(2), axis=D(3)] -> [battery=D(0), axis=D(1)] -> NIL + let mut batteries = make_batteries(&mut stack); + + // Expected values (battery, parent_axis) in iteration order + let expected_values: Vec<(u64, u64)> = vec![(2, 3), (0, 1)]; + + // Evacuate Batteries to PMA + unsafe { + batteries.copy_to_pma(&stack, &mut pma); + } + + // Iterate over evacuated batteries and verify values, count, and offset form + let mut expected_iter = expected_values.iter(); + for (battery_ptr, parent_axis) in batteries { + let (expected_battery, expected_axis) = expected_iter + .next() + .expect("Batteries has more entries than expected"); + + let battery = unsafe { *battery_ptr }; + assert_eq!( + unsafe { battery.as_raw() }, + *expected_battery, + "Battery value should match" + ); + assert_eq!( + parent_axis + .in_space(&space) + .as_u64() + .expect("parent axis should fit in u64"), + *expected_axis, + "Parent axis should match" + ); + + // Verify nouns are in offset form + verify_noun_not_stack_allocated(battery, &space, "Batteries battery"); + verify_noun_not_stack_allocated(parent_axis.as_noun(), &space, "Batteries parent_axis"); + } + assert!( + expected_iter.next().is_none(), + "Batteries has fewer entries than expected" + ); + + // Verify the Batteries passes assert_in_pma + batteries.assert_in_pma(&pma); + } + + /// Verifies BatteriesList can be evacuated to PMA and remains functional. + /// + /// This test exercises: + /// - Creating a BatteriesList with multiple Batteries entries + /// - Evacuating the BatteriesList to PMA via copy_to_pma + /// - Verifying all entries are still accessible after evacuation + /// - Verifying all nouns are in offset form (not stack-allocated) + /// - Verifying the BatteriesList passes assert_in_pma + /// + /// Note: copy_to_pma sets forwarding pointers in the source nouns, which corrupts + /// them for normal use. We use expected values for comparison. + #[test] + #[cfg_attr(miri, ignore)] + fn test_evacuate_batteries_list_round_trip() { + use crate::pma::{test_pma_path, Pma, PmaCopy}; + + let mut stack = make_test_stack(DEFAULT_STACK_SIZE); + let mut pma = + Pma::new(100000, test_pma_path("batteries_list")).expect("Failed to create test PMA"); + let space = NounSpace::new(&stack, &pma); + + // Create a BatteriesList using the test helper + // make_batteries_list(&[7, 8]) creates a list with two Batteries entries, + // each with a single battery noun (D(7) and D(8)) + let mut batteries_list = make_batteries_list(&mut stack, &[7, 8]); + + // Expected battery values in iteration order + let expected_batteries: Vec = vec![7, 8]; + + // Evacuate BatteriesList to PMA + unsafe { + batteries_list.copy_to_pma(&stack, &mut pma); + } + + // Iterate over evacuated batteries_list and verify values, count, and offset form + let mut expected_iter = expected_batteries.iter(); + for batteries in batteries_list { + let expected_battery = expected_iter + .next() + .expect("BatteriesList has more entries than expected"); + + // Each Batteries in this test has a single entry + let mut batteries_iter = batteries.into_iter(); + let (battery_ptr, parent_axis) = batteries_iter + .next() + .expect("Batteries should have at least one entry"); + + let battery = unsafe { *battery_ptr }; + assert_eq!( + unsafe { battery.as_raw() }, + *expected_battery, + "Battery value should match" + ); + assert_eq!( + parent_axis + .in_space(&space) + .as_u64() + .expect("parent axis should fit in u64"), + 0, + "Parent axis should be 0" + ); + + // Verify nouns are in offset form + verify_noun_not_stack_allocated(battery, &space, "BatteriesList battery"); + verify_noun_not_stack_allocated( + parent_axis.as_noun(), + &space, + "BatteriesList parent_axis", + ); + + // Verify no more entries in this Batteries + assert!( + batteries_iter.next().is_none(), + "Batteries should have exactly one entry" + ); + } + assert!( + expected_iter.next().is_none(), + "BatteriesList has fewer entries than expected" + ); + + // Verify the BatteriesList passes assert_in_pma + batteries_list.assert_in_pma(&pma); + } + + /// Verifies Cold jet state can be evacuated to PMA and remains functional. + /// + /// This test exercises: + /// - Creating a Cold state with populated HAMTs (battery_to_paths, root_to_paths, path_to_batteries) + /// - Evacuating the entire Cold structure to PMA via copy_to_pma + /// - Verifying all three HAMTs are accessible after evacuation + /// - Verifying all internal nouns are in offset form (not stack-allocated) + /// - Verifying the Cold structure passes assert_in_pma + /// + /// Cold is a critical component of the jet matching system, storing mappings between: + /// - Batteries (code) -> paths (registered names/hierarchies) + /// - Roots (base cores) -> paths + /// - Paths -> battery lists (for matching cores to jets) + #[test] + #[cfg_attr(miri, ignore)] + fn test_evacuate_cold_round_trip() { + use crate::pma::{test_pma_path, Pma, PmaCopy}; + + let mut stack = make_test_stack(DEFAULT_STACK_SIZE); + let mut pma = Pma::new(100000, test_pma_path("cold")).expect("Failed to create test PMA"); + let space = NounSpace::new(&stack, &pma); + + // Create a Cold state using make_cold_state + let mut cold = make_cold_state(&mut stack); + + // Count entries before evacuation + let count_battery_to_paths_before: usize = + unsafe { (*cold.0).battery_to_paths.iter().map(|e| e.len()).sum() }; + let count_root_to_paths_before: usize = + unsafe { (*cold.0).root_to_paths.iter().map(|e| e.len()).sum() }; + let count_path_to_batteries_before: usize = + unsafe { (*cold.0).path_to_batteries.iter().map(|e| e.len()).sum() }; + + assert_eq!( + count_battery_to_paths_before, 1, + "Should have 1 battery_to_paths entry" + ); + assert_eq!( + count_root_to_paths_before, 2, + "Should have 2 root_to_paths entries" + ); + assert_eq!( + count_path_to_batteries_before, 1, + "Should have 1 path_to_batteries entry" + ); + + // Evacuate Cold to PMA + unsafe { + cold.copy_to_pma(&stack, &mut pma); + } + + // Verify entry counts are preserved after evacuation + let count_battery_to_paths_after: usize = + unsafe { (*cold.0).battery_to_paths.iter().map(|e| e.len()).sum() }; + let count_root_to_paths_after: usize = + unsafe { (*cold.0).root_to_paths.iter().map(|e| e.len()).sum() }; + let count_path_to_batteries_after: usize = + unsafe { (*cold.0).path_to_batteries.iter().map(|e| e.len()).sum() }; + + assert_eq!( + count_battery_to_paths_after, count_battery_to_paths_before, + "battery_to_paths entry count should be preserved" + ); + assert_eq!( + count_root_to_paths_after, count_root_to_paths_before, + "root_to_paths entry count should be preserved" + ); + assert_eq!( + count_path_to_batteries_after, count_path_to_batteries_before, + "path_to_batteries entry count should be preserved" + ); + + // Verify lookups still work after evacuation + // Note: We use fresh D(x) atoms for lookup since the original keys + // may have forwarding pointers set during evacuation + let lookup_battery = unsafe { (*cold.0).battery_to_paths.lookup(&mut stack, &mut D(200)) }; + assert!( + lookup_battery.is_some(), + "battery_to_paths lookup for D(200) should succeed after evacuation" + ); + + let lookup_root1 = unsafe { (*cold.0).root_to_paths.lookup(&mut stack, &mut D(100)) }; + assert!( + lookup_root1.is_some(), + "root_to_paths lookup for D(100) should succeed after evacuation" + ); + + let lookup_root2 = unsafe { (*cold.0).root_to_paths.lookup(&mut stack, &mut D(101)) }; + assert!( + lookup_root2.is_some(), + "root_to_paths lookup for D(101) should succeed after evacuation" + ); + + let lookup_path = unsafe { (*cold.0).path_to_batteries.lookup(&mut stack, &mut D(300)) }; + assert!( + lookup_path.is_some(), + "path_to_batteries lookup for D(300) should succeed after evacuation" + ); + + // Verify all nouns in the Cold HAMTs are in offset form + // Check battery_to_paths + for entries in unsafe { (*cold.0).battery_to_paths.iter() } { + for (key, noun_list) in entries { + verify_noun_not_stack_allocated(*key, &space, "battery_to_paths key"); + // Verify NounList elements + for elem_ptr in *noun_list { + let elem = unsafe { *elem_ptr }; + verify_noun_not_stack_allocated( + elem, &space, "battery_to_paths NounList element", + ); + } + } + } + + // Check root_to_paths + for entries in unsafe { (*cold.0).root_to_paths.iter() } { + for (key, noun_list) in entries { + verify_noun_not_stack_allocated(*key, &space, "root_to_paths key"); + for elem_ptr in *noun_list { + let elem = unsafe { *elem_ptr }; + verify_noun_not_stack_allocated(elem, &space, "root_to_paths NounList element"); + } + } + } + + // Check path_to_batteries + for entries in unsafe { (*cold.0).path_to_batteries.iter() } { + for (key, batteries_list) in entries { + verify_noun_not_stack_allocated(*key, &space, "path_to_batteries key"); + // Verify BatteriesList elements + for batteries in *batteries_list { + for (battery_ptr, _parent_axis) in batteries { + let battery = unsafe { *battery_ptr }; + verify_noun_not_stack_allocated( + battery, &space, "path_to_batteries battery", + ); + } + } + } + } + + // Verify the Cold structure passes assert_in_pma + cold.assert_in_pma(&pma); + } } diff --git a/crates/nockvm/rust/nockvm/src/jets/form.rs b/crates/nockvm/rust/nockvm/src/jets/form.rs index ee7beae7c..029b8e0c0 100644 --- a/crates/nockvm/rust/nockvm/src/jets/form.rs +++ b/crates/nockvm/rust/nockvm/src/jets/form.rs @@ -8,9 +8,10 @@ use crate::noun::Noun; crate::gdb!(); pub fn jet_scow(context: &mut Context, subject: Noun) -> Result { - let aura = slot(subject, 12)?.as_direct()?; - let atom = slot(subject, 13)?.as_atom()?; - util::scow(&mut context.stack, aura, atom) + let space = context.stack.noun_space(); + let aura = slot(subject, 12, &space)?.as_direct()?; + let atom = slot(subject, 13, &space)?.as_atom()?; + util::scow(&mut context.stack, aura, atom, &space) } pub mod util { @@ -20,16 +21,17 @@ pub mod util { use crate::jets; use crate::jets::JetErr; use crate::mem::NockStack; - use crate::noun::{Atom, Cell, DirectAtom, D, T}; + use crate::noun::{Atom, Cell, DirectAtom, NounSpace, D, T}; pub fn scow( stack: &mut NockStack, aura: DirectAtom, // XX: technically this should be Atom? atom: Atom, + space: &NounSpace, ) -> jets::Result { match aura.data() { tas!(b"ud") => { - if atom.as_bitslice().first_one().is_none() { + if atom.in_space(space).as_bitslice().first_one().is_none() { return Ok(T(stack, &[D(b'0' as u64), D(0)])); } @@ -44,7 +46,7 @@ pub mod util { lent += 1; } } else { - let mut n = atom.as_indirect()?.as_ubig(stack); + let mut n = atom.as_indirect()?.as_ubig(stack, space); while !n.is_zero() { root = T(stack, &[D(b'0' as u64 + (&n % 10u64)), root]); @@ -61,11 +63,11 @@ pub mod util { if lent % 3 == 0 { let (cell, memory) = Cell::new_raw_mut(stack); (*memory).head = D(b'.' as u64); - (*memory).tail = list.tail(); - (*(list.to_raw_pointer_mut())).tail = cell.as_noun(); - list = list.tail().as_cell()?; + (*memory).tail = list.in_space(space).tail().noun(); + (*(list.to_raw_pointer_mut(space))).tail = cell.as_noun(); + list = list.in_space(space).tail().as_cell()?.cell(); } - list = list.tail().as_cell()?; + list = list.in_space(space).tail().as_cell()?.cell(); lent -= 1; } diff --git a/crates/nockvm/rust/nockvm/src/jets/hash.rs b/crates/nockvm/rust/nockvm/src/jets/hash.rs index 7923fc6cc..3c9ee20cc 100644 --- a/crates/nockvm/rust/nockvm/src/jets/hash.rs +++ b/crates/nockvm/rust/nockvm/src/jets/hash.rs @@ -9,7 +9,8 @@ use crate::noun::Noun; crate::gdb!(); pub fn jet_mug(context: &mut Context, subject: Noun) -> Result { - let arg = slot(subject, 6)?; + let space = context.stack.noun_space(); + let arg = slot(subject, 6, &space)?; Ok(mug(&mut context.stack, arg).as_noun()) } diff --git a/crates/nockvm/rust/nockvm/src/jets/hot.rs b/crates/nockvm/rust/nockvm/src/jets/hot.rs index 4f6109239..8f9e7a38c 100644 --- a/crates/nockvm/rust/nockvm/src/jets/hot.rs +++ b/crates/nockvm/rust/nockvm/src/jets/hot.rs @@ -4,7 +4,9 @@ use either::Either::{self, Left, Right}; use nockvm_macros::tas; use crate::jets::*; -use crate::noun::{Atom, DirectAtom, IndirectAtom, Noun, D, T}; +use crate::mem::{NockStack, Preserve}; +use crate::noun::{Atom, DirectAtom, IndirectAtom, Noun, NounAllocator, D, T}; +use crate::pma::{Pma, PmaCopy, PmaCopyFrom}; /** Root for Hoon %k.138 */ @@ -838,6 +840,7 @@ pub struct Hot(*mut HotMem); impl Hot { pub fn init(stack: &mut NockStack, constant_hot_state: &[HotEntry]) -> Self { unsafe { + let space = stack.noun_space(); let mut next = Hot(null_mut()); for (htap, axe, jet) in constant_hot_state { let mut a_path = D(0); @@ -845,7 +848,7 @@ impl Hot { match i { Left(tas) => { let chum = IndirectAtom::new_raw_bytes_ref(stack, tas) - .normalize_as_atom() + .normalize_as_atom(&space) .as_noun(); a_path = T(stack, &[chum, a_path]); } @@ -920,3 +923,56 @@ impl Preserve for Hot { } } } + +impl PmaCopy for Hot { + #[cfg(feature = "pma-assert")] + fn assert_in_pma(&self, pma: &Pma) { + let mut it = *self; + while !it.0.is_null() { + assert!( + pma.contains_ptr(it.0 as *const u8), + "Hot node should be in PMA" + ); + unsafe { + (*it.0).a_path.assert_in_pma(pma); + (*it.0).axis.assert_in_pma(pma); + } + it = unsafe { (*it.0).next }; + } + } + + #[cfg(not(feature = "pma-assert"))] + #[inline(always)] + fn assert_in_pma(&self, _pma: &Pma) {} + + unsafe fn copy_to_pma(&mut self, stack: &NockStack, pma: &mut Pma) { + let mut ptr: *mut Hot = self; + while !(*ptr).0.is_null() { + if pma.contains_ptr((*ptr).0 as *const u8) { + break; + } + let src = (*ptr).0; + (*src).a_path.copy_to_pma(stack, pma); + (*src).axis.copy_to_pma(stack, pma); + let dest_mem: *mut HotMem = pma.alloc_struct(1); + copy_nonoverlapping(src, dest_mem, 1); + *ptr = Hot(dest_mem); + ptr = &mut (*dest_mem).next; + } + } +} + +impl PmaCopyFrom for Hot { + unsafe fn copy_from_pma(&mut self, from_pma: &Pma, to_pma: &mut Pma) { + let mut ptr: *mut Hot = self; + while !(*ptr).0.is_null() { + let src = (*ptr).0; + let dest_mem: *mut HotMem = to_pma.alloc_struct(1); + copy_nonoverlapping(src, dest_mem, 1); + *ptr = Hot(dest_mem); + (*dest_mem).a_path.copy_from_pma(from_pma, to_pma); + (*dest_mem).axis.copy_from_pma(from_pma, to_pma); + ptr = &mut (*dest_mem).next; + } + } +} diff --git a/crates/nockvm/rust/nockvm/src/jets/list.rs b/crates/nockvm/rust/nockvm/src/jets/list.rs index 4e71ba3b1..614bd6b9e 100644 --- a/crates/nockvm/rust/nockvm/src/jets/list.rs +++ b/crates/nockvm/rust/nockvm/src/jets/list.rs @@ -9,33 +9,37 @@ use crate::site::{site_slam, Site}; crate::gdb!(); pub fn jet_weld(context: &mut Context, subject: Noun) -> Result { - let sam = slot(subject, 6)?; - let a = slot(sam, 2)?; - let b = slot(sam, 3)?; - util::weld(&mut context.stack, a, b) + let space = context.stack.noun_space(); + let sam = slot(subject, 6, &space)?; + let a = slot(sam, 2, &space)?; + let b = slot(sam, 3, &space)?; + util::weld(&mut context.stack, a, b, &space) } pub fn jet_flop(context: &mut Context, subject: Noun) -> Result { - let sam = slot(subject, 6)?; - util::flop(&mut context.stack, sam) + let space = context.stack.noun_space(); + let sam = slot(subject, 6, &space)?; + util::flop(&mut context.stack, sam, &space) } pub fn jet_lent(_context: &mut Context, subject: Noun) -> Result { - let list = slot(subject, 6)?; - util::lent(list).map(|x| D(x as u64)) + let space = _context.stack.noun_space(); + let list = slot(subject, 6, &space)?; + util::lent(list, &space).map(|x| D(x as u64)) } pub fn jet_roll(context: &mut Context, subject: Noun) -> Result { - let sample = slot(subject, 6)?; - let mut list = slot(sample, 2)?; - let mut gate = slot(sample, 3)?; - let mut prod = slot(gate, 13)?; + let space = context.stack.noun_space(); + let sample = slot(subject, 6, &space)?; + let mut list = slot(sample, 2, &space)?; + let mut gate = slot(sample, 3, &space)?; + let mut prod = slot(gate, 13, &space)?; let site = Site::new(context, &mut gate); loop { - if let Ok(list_cell) = list.as_cell() { - list = list_cell.tail(); - let sam = T(&mut context.stack, &[list_cell.head(), prod]); + if let Ok(list_cell) = list.in_space(&space).as_cell() { + list = list_cell.tail().noun(); + let sam = T(&mut context.stack, &[list_cell.head().noun(), prod]); prod = site_slam(context, &site, sam)?; } else { if unsafe { !list.raw_equals(&D(0)) } { @@ -47,34 +51,36 @@ pub fn jet_roll(context: &mut Context, subject: Noun) -> Result { } pub fn jet_snag(_context: &mut Context, subject: Noun) -> Result { - let sam = slot(subject, 6)?; - let index = slot(sam, 2)?; - let list = slot(sam, 3)?; - - util::snag(list, index) + let space = _context.stack.noun_space(); + let sam = slot(subject, 6, &space)?; + let index = slot(sam, 2, &space)?; + let list = slot(sam, 3, &space)?; + util::snag(list, index, &space) } pub fn jet_snip(context: &mut Context, subject: Noun) -> Result { - let list = slot(subject, 6)?; + let space = context.stack.noun_space(); + let list = slot(subject, 6, &space)?; util::snip(&mut context.stack, list) } pub fn jet_turn(context: &mut Context, subject: Noun) -> Result { - let sample = slot(subject, 6)?; - let mut list = slot(sample, 2)?; - let mut gate = slot(sample, 3)?; let mut res = D(0); let mut dest: *mut Noun = &mut res; // Mutable pointer because we cannot guarantee initialized + let space = context.stack.noun_space(); + let sample = slot(subject, 6, &space)?; + let mut list = slot(sample, 2, &space)?; + let mut gate = slot(sample, 3, &space)?; // Since the gate doesn't change, we can do a single jet check and use that through the whole // loop let site = Site::new(context, &mut gate); loop { - if let Ok(list_cell) = list.as_cell() { - list = list_cell.tail(); + if let Ok(list_cell) = list.in_space(&space).as_cell() { + list = list_cell.tail().noun(); unsafe { let (new_cell, new_mem) = Cell::new_raw_mut(&mut context.stack); - (*new_mem).head = site_slam(context, &site, list_cell.head())?; + (*new_mem).head = site_slam(context, &site, list_cell.head().noun())?; *dest = new_cell.as_noun(); dest = &mut (*new_mem).tail; } @@ -91,41 +97,46 @@ pub fn jet_turn(context: &mut Context, subject: Noun) -> Result { } pub fn jet_zing(context: &mut Context, subject: Noun) -> Result { - let list = slot(subject, 6)?; + let space = context.stack.noun_space(); + let list = slot(subject, 6, &space)?; let stack = &mut context.stack; util::zing(stack, list) } pub fn jet_reap(context: &mut Context, subject: Noun) -> Result { - let sam = slot(subject, 6)?; - let a_noun = slot(sam, 2)?; - let b_noun = slot(sam, 3)?; - - let a = a_noun.as_atom()?.as_u64()?; + let space = context.stack.noun_space(); + let sam = slot(subject, 6, &space)?; + let a_noun = slot(sam, 2, &space)?; + let b_noun = slot(sam, 3, &space)?; + let a = a_noun.in_space(&space).as_atom()?.as_u64()?; util::reap(&mut context.stack, a, b_noun) } pub fn jet_levy(context: &mut Context, subject: Noun) -> Result { - let sam = slot(subject, 6)?; - let a_noun = slot(sam, 2)?; - let b_noun = slot(sam, 3)?; + let space = context.stack.noun_space(); + let sam = slot(subject, 6, &space)?; + let a_noun = slot(sam, 2, &space)?; + let b_noun = slot(sam, 3, &space)?; util::levy(context, a_noun, b_noun) } pub fn jet_find(context: &mut Context, subject: Noun) -> Result { - let sam = slot(subject, 6)?; - let nedl = slot(sam, 2)?; - let hstk = slot(sam, 3)?; + let space = context.stack.noun_space(); + let sam = slot(subject, 6, &space)?; + let nedl = slot(sam, 2, &space)?; + let hstk = slot(sam, 3, &space)?; util::find(context, nedl, hstk) } pub fn jet_scag(context: &mut Context, subject: Noun) -> Result { - let sam = slot(subject, 6)?; - let a = sam.as_cell()?.head().as_atom()?; - let b = sam.as_cell()?.tail(); + let space = context.stack.noun_space(); + let sam = slot(subject, 6, &space)?; + let sam_cell = sam.in_space(&space).as_cell()?; + let a = sam_cell.head().as_atom()?.atom(); + let b = sam_cell.tail().noun(); util::scag(context, a, b) } @@ -137,11 +148,11 @@ pub mod util { use crate::jets::util::BAIL_EXIT; use crate::jets::{JetErr, Result}; use crate::mem::NockStack; - use crate::noun::{Atom, Cell, Noun, NounAllocator, D, NO, T, YES}; + use crate::noun::{Atom, Cell, Noun, NounAllocator, NounSpace, D, NO, T, YES}; use crate::site::{site_slam, Site}; /// Reverse order of list - pub fn flop(alloc: &mut T, noun: Noun) -> Result { + pub fn flop(alloc: &mut T, noun: Noun, space: &NounSpace) -> Result { let mut list = noun; let mut tsil = D(0); loop { @@ -149,69 +160,70 @@ pub mod util { break; } - let cell = list.as_cell()?; - tsil = T(alloc, &[cell.head(), tsil]); - list = cell.tail(); + let cell = list.in_space(space).as_cell()?; + tsil = T(alloc, &[cell.head().noun(), tsil]); + list = cell.tail().noun(); } Ok(tsil) } - pub fn weld(alloc: &mut A, a: Noun, b: Noun) -> Result { + pub fn weld(alloc: &mut A, a: Noun, b: Noun, space: &NounSpace) -> Result { let mut res = D(0); let mut cur = a; loop { if unsafe { cur.raw_equals(&D(0)) } { break; } - let cell = cur.as_cell()?; - res = T(alloc, &[cell.head(), res]); - cur = cell.tail(); + let cell = cur.in_space(space).as_cell()?; + res = T(alloc, &[cell.head().noun(), res]); + cur = cell.tail().noun(); } cur = b; loop { if unsafe { cur.raw_equals(&D(0)) } { break; } - let cell = cur.as_cell()?; - res = T(alloc, &[cell.head(), res]); - cur = cell.tail(); + let cell = cur.in_space(space).as_cell()?; + res = T(alloc, &[cell.head().noun(), res]); + cur = cell.tail().noun(); } - flop(alloc, res) + let out_space = alloc.noun_space(); + flop(alloc, res, &out_space) } - pub fn lent(tape: Noun) -> result::Result { + pub fn lent(tape: Noun, space: &NounSpace) -> result::Result { let mut len = 0usize; let mut list = tape; loop { - if let Some(atom) = list.atom() { + if let Some(atom) = list.in_space(space).atom() { if atom.as_bitslice().first_one().is_none() { break; } else { return Err(BAIL_EXIT); } } - let cell = list.as_cell()?; + let cell = list.in_space(space).as_cell()?; // don't need checked_add or indirect atom result: 2^63-1 atoms would be 64 ebibytes len += 1; - list = cell.tail(); + list = cell.tail().noun(); } Ok(len) } - pub fn snag(tape: Noun, index: Noun) -> Result { + pub fn snag(tape: Noun, index: Noun, space: &NounSpace) -> Result { let mut list = tape; - let mut idx = index.as_atom()?.as_u64()? as usize; + let mut idx = index.in_space(space).as_atom()?.as_u64()? as usize; loop { if unsafe { list.raw_equals(&D(0)) } { return Err(BAIL_EXIT); } - let cell = list.as_cell()?; + let cell = list.in_space(space).as_cell()?; if idx == 0 { - return Ok(cell.head()); + return Ok(cell.head().noun()); } idx -= 1; - list = cell.tail(); + list = cell.tail().noun(); } } @@ -219,15 +231,16 @@ pub mod util { let mut ret = D(0); let mut dest = &mut ret as *mut Noun; let mut list = tape; + let space = stack.noun_space(); - if let Some(atom) = list.atom() { + if let Some(atom) = list.in_space(&space).atom() { if atom.as_bitslice().first_one().is_none() { return Ok(D(0)); } } loop { - let cell = list.as_cell()?; + let cell = list.in_space(&space).as_cell()?; if let Some(atom) = cell.tail().atom() { if atom.as_bitslice().first_one().is_none() { break; @@ -237,11 +250,11 @@ pub mod util { } unsafe { let (new_cell, new_mem) = Cell::new_raw_mut(stack); - (*new_mem).head = cell.head(); + (*new_mem).head = cell.head().noun(); *dest = new_cell.as_noun(); dest = &mut (*new_mem).tail; } - list = cell.tail(); + list = cell.tail().noun(); } unsafe { *dest = D(0) }; Ok(ret) @@ -251,16 +264,17 @@ pub mod util { unsafe { let mut res: Noun = D(0); let mut dest = &mut res as *mut Noun; + let space = stack.noun_space(); while !list.raw_equals(&D(0)) { - let pair = list.as_cell()?; - let mut sublist = pair.head(); - list = pair.tail(); + let pair = list.in_space(&space).as_cell()?; + let mut sublist = pair.head().noun(); + list = pair.tail().noun(); while !sublist.raw_equals(&D(0)) { - let it = sublist.as_cell()?; - let i = it.head(); - sublist = it.tail(); + let it = sublist.in_space(&space).as_cell()?; + let i = it.head().noun(); + sublist = it.tail().noun(); let (new_cell, new_memory) = Cell::new_raw_mut(stack); (*new_memory).head = i; @@ -289,24 +303,26 @@ pub mod util { pub fn levy(context: &mut Context, a_noun: Noun, mut b_noun: Noun) -> Result { let site = Site::new(context, &mut b_noun); let mut list = a_noun; + let space = context.stack.noun_space(); loop { if unsafe { list.raw_equals(&D(0)) } { return Ok(YES); } - let cell = list.as_cell()?; - let b_res = site_slam(context, &site, cell.head())?; + let cell = list.in_space(&space).as_cell()?; + let b_res = site_slam(context, &site, cell.head().noun())?; if unsafe { b_res.raw_equals(&NO) } { return Ok(NO); } - list = cell.tail(); + list = cell.tail().noun(); } } pub fn find(context: &mut Context, nedl: Noun, hstk: Noun) -> Result { let mut hstk = hstk; let mut i = 0; + let space = context.stack.noun_space(); loop { let mut n = nedl; let mut h = hstk; @@ -316,19 +332,31 @@ pub mod util { return Ok(D(0)); // (unit @ud) ~ } - if unsafe { n.as_cell()?.head().raw_equals(&h.as_cell()?.head()) } { - if unsafe { n.as_cell()?.tail().raw_equals(&D(0)) } { + if unsafe { + n.in_space(&space) + .as_cell()? + .head() + .noun() + .raw_equals(&h.in_space(&space).as_cell()?.head().noun()) + } { + if unsafe { + n.in_space(&space) + .as_cell()? + .tail() + .noun() + .raw_equals(&D(0)) + } { // match found return Ok(T(&mut context.stack, &[D(0), D(i)])); // (unit @ud) i } - n = n.as_cell()?.tail(); - h = h.as_cell()?.tail(); + n = n.in_space(&space).as_cell()?.tail().noun(); + h = h.in_space(&space).as_cell()?.tail().noun(); continue; } // try next position - hstk = hstk.as_cell()?.tail(); + hstk = hstk.in_space(&space).as_cell()?.tail().noun(); i += 1; break; } @@ -337,7 +365,8 @@ pub mod util { pub fn scag(context: &mut Context, a: Atom, b: Noun) -> Result { // Accepts an atom a and list b, producing the first a elements of the front of the list. - let a = a.as_u64()?; + let space = context.stack.noun_space(); + let a = a.in_space(&space).as_u64()?; let mut res: Vec = vec![]; let mut list = b; let mut pos = 0; @@ -345,12 +374,12 @@ pub mod util { if unsafe { list.raw_equals(&D(0)) } { break; } - let current_cell = list.as_cell()?; + let current_cell = list.in_space(&space).as_cell()?; if pos >= a { break; } - res.push(current_cell.head()); - list = current_cell.tail(); + res.push(current_cell.head().noun()); + list = current_cell.tail().noun(); pos += 1; } @@ -370,6 +399,7 @@ mod tests { use crate::noun::{D, T}; #[test] + #[cfg_attr(miri, ignore = "memfd_create unsupported in Miri")] fn test_flop() { let c = &mut init_context(); @@ -407,6 +437,7 @@ mod tests { } #[test] + #[cfg_attr(miri, ignore = "memfd_create unsupported in Miri")] fn test_lent() { let c = &mut init_context(); @@ -421,6 +452,7 @@ mod tests { } #[test] + #[cfg_attr(miri, ignore = "memfd_create unsupported in Miri")] fn test_snag() { let c = &mut init_context(); let list1 = T(&mut c.stack, &[D(1), D(2), D(3), D(0)]); @@ -439,6 +471,7 @@ mod tests { } #[test] + #[cfg_attr(miri, ignore = "memfd_create unsupported in Miri")] fn test_snip() { let c = &mut init_context(); @@ -465,6 +498,7 @@ mod tests { } #[test] + #[cfg_attr(miri, ignore = "memfd_create unsupported in Miri")] fn test_zing() { let c = &mut init_context(); @@ -482,6 +516,7 @@ mod tests { } #[test] + #[cfg_attr(miri, ignore = "memfd_create unsupported in Miri")] fn test_weld() { let c = &mut init_context(); let list_1 = T(&mut c.stack, &[D(1), D(2), D(3), D(0)]); @@ -502,6 +537,7 @@ mod tests { } #[test] + #[cfg_attr(miri, ignore = "memfd_create unsupported in Miri")] fn test_reap() { let c = &mut init_context(); @@ -525,6 +561,7 @@ mod tests { } #[test] + #[cfg_attr(miri, ignore = "memfd_create unsupported in Miri")] fn test_find() { let c = &mut init_context(); @@ -577,6 +614,7 @@ mod tests { } #[test] + #[cfg_attr(miri, ignore = "memfd_create unsupported in Miri")] fn test_scag() { let c = &mut init_context(); diff --git a/crates/nockvm/rust/nockvm/src/jets/lock/aes.rs b/crates/nockvm/rust/nockvm/src/jets/lock/aes.rs index b9eb3a0da..d8a3576b8 100644 --- a/crates/nockvm/rust/nockvm/src/jets/lock/aes.rs +++ b/crates/nockvm/rust/nockvm/src/jets/lock/aes.rs @@ -13,103 +13,121 @@ crate::gdb!(); pub fn jet_siva_en(context: &mut Context, subject: Noun) -> Result { let stack = &mut context.stack; - let txt = slot(subject, 6)?.as_atom()?; - let key = slot(subject, 60)?.as_atom()?; - let ads = slot(subject, 61)?; + let space = stack.noun_space(); + let txt = slot(subject, 6, &space)?.as_atom()?; + let key = slot(subject, 60, &space)?.as_atom()?; + let ads = slot(subject, 61, &space)?; - if met(3, key) > 32 { + if met(3, key, &space) > 32 { Err(JetErr::Punt) } else { let key_bytes = &mut [0u8; 32]; - key_bytes[0..key.as_ne_bytes().len()].copy_from_slice(key.as_ne_bytes()); + let key_handle = key.in_space(&space); + let key_ne_bytes = key_handle.as_ne_bytes(); + key_bytes[0..key_ne_bytes.len()].copy_from_slice(key_ne_bytes); - util::_siv_en::<32>(stack, key_bytes, ads, txt) + util::_siv_en::<32>(stack, key_bytes, ads, txt, &space) } } pub fn jet_siva_de(context: &mut Context, subject: Noun) -> Result { let stack = &mut context.stack; - let iv = slot(subject, 12)?.as_atom()?; - let len = slot(subject, 26)?.as_atom()?; - let txt = slot(subject, 27)?.as_atom()?; - let key = slot(subject, 60)?.as_atom()?; - let ads = slot(subject, 61)?; - - if met(3, key) > 32 { + let space = stack.noun_space(); + let iv = slot(subject, 12, &space)?.as_atom()?; + let len = slot(subject, 26, &space)?.as_atom()?; + let txt = slot(subject, 27, &space)?.as_atom()?; + let key = slot(subject, 60, &space)?.as_atom()?; + let ads = slot(subject, 61, &space)?; + + if met(3, key, &space) > 32 { Err(JetErr::Punt) } else { let key_bytes = &mut [0u8; 32]; - key_bytes[0..key.as_ne_bytes().len()].copy_from_slice(key.as_ne_bytes()); + let key_handle = key.in_space(&space); + let key_ne_bytes = key_handle.as_ne_bytes(); + key_bytes[0..key_ne_bytes.len()].copy_from_slice(key_ne_bytes); - util::_siv_de::<32>(stack, key_bytes, ads, iv, len, txt) + util::_siv_de::<32>(stack, key_bytes, ads, iv, len, txt, &space) } } pub fn jet_sivb_en(context: &mut Context, subject: Noun) -> Result { let stack = &mut context.stack; - let txt = slot(subject, 6)?.as_atom()?; - let key = slot(subject, 60)?.as_atom()?; - let ads = slot(subject, 61)?; + let space = stack.noun_space(); + let txt = slot(subject, 6, &space)?.as_atom()?; + let key = slot(subject, 60, &space)?.as_atom()?; + let ads = slot(subject, 61, &space)?; - if met(3, key) > 48 { + if met(3, key, &space) > 48 { Err(JetErr::Punt) } else { let key_bytes = &mut [0u8; 48]; - key_bytes[0..key.as_ne_bytes().len()].copy_from_slice(key.as_ne_bytes()); + let key_handle = key.in_space(&space); + let key_ne_bytes = key_handle.as_ne_bytes(); + key_bytes[0..key_ne_bytes.len()].copy_from_slice(key_ne_bytes); - util::_siv_en::<48>(stack, key_bytes, ads, txt) + util::_siv_en::<48>(stack, key_bytes, ads, txt, &space) } } pub fn jet_sivb_de(context: &mut Context, subject: Noun) -> Result { let stack = &mut context.stack; - let iv = slot(subject, 12)?.as_atom()?; - let len = slot(subject, 26)?.as_atom()?; - let txt = slot(subject, 27)?.as_atom()?; - let key = slot(subject, 60)?.as_atom()?; - let ads = slot(subject, 61)?; - - if met(3, key) > 48 { + let space = stack.noun_space(); + let iv = slot(subject, 12, &space)?.as_atom()?; + let len = slot(subject, 26, &space)?.as_atom()?; + let txt = slot(subject, 27, &space)?.as_atom()?; + let key = slot(subject, 60, &space)?.as_atom()?; + let ads = slot(subject, 61, &space)?; + + if met(3, key, &space) > 48 { Err(JetErr::Punt) } else { let key_bytes = &mut [0u8; 48]; - key_bytes[0..key.as_ne_bytes().len()].copy_from_slice(key.as_ne_bytes()); + let key_handle = key.in_space(&space); + let key_ne_bytes = key_handle.as_ne_bytes(); + key_bytes[0..key_ne_bytes.len()].copy_from_slice(key_ne_bytes); - util::_siv_de::<48>(stack, key_bytes, ads, iv, len, txt) + util::_siv_de::<48>(stack, key_bytes, ads, iv, len, txt, &space) } } pub fn jet_sivc_en(context: &mut Context, subject: Noun) -> Result { let stack = &mut context.stack; - let txt = slot(subject, 6)?.as_atom()?; - let key = slot(subject, 60)?.as_atom()?; - let ads = slot(subject, 61)?; + let space = stack.noun_space(); + let txt = slot(subject, 6, &space)?.as_atom()?; + let key = slot(subject, 60, &space)?.as_atom()?; + let ads = slot(subject, 61, &space)?; - if met(3, key) > 64 { + if met(3, key, &space) > 64 { Err(JetErr::Punt) } else { let key_bytes = &mut [0u8; 64]; - key_bytes[0..key.as_ne_bytes().len()].copy_from_slice(key.as_ne_bytes()); + let key_handle = key.in_space(&space); + let key_ne_bytes = key_handle.as_ne_bytes(); + key_bytes[0..key_ne_bytes.len()].copy_from_slice(key_ne_bytes); - util::_siv_en::<64>(stack, key_bytes, ads, txt) + util::_siv_en::<64>(stack, key_bytes, ads, txt, &space) } } pub fn jet_sivc_de(context: &mut Context, subject: Noun) -> Result { let stack = &mut context.stack; - let iv = slot(subject, 12)?.as_atom()?; - let len = slot(subject, 26)?.as_atom()?; - let txt = slot(subject, 27)?.as_atom()?; - let key = slot(subject, 60)?.as_atom()?; - let ads = slot(subject, 61)?; - - if met(3, key) > 64 { + let space = stack.noun_space(); + let iv = slot(subject, 12, &space)?.as_atom()?; + let len = slot(subject, 26, &space)?.as_atom()?; + let txt = slot(subject, 27, &space)?.as_atom()?; + let key = slot(subject, 60, &space)?.as_atom()?; + let ads = slot(subject, 61, &space)?; + + if met(3, key, &space) > 64 { Err(JetErr::Punt) } else { let key_bytes = &mut [0u8; 64]; - key_bytes[0..key.as_ne_bytes().len()].copy_from_slice(key.as_ne_bytes()); + let key_handle = key.in_space(&space); + let key_ne_bytes = key_handle.as_ne_bytes(); + key_bytes[0..key_ne_bytes.len()].copy_from_slice(key_ne_bytes); - util::_siv_de::<64>(stack, key_bytes, ads, iv, len, txt) + util::_siv_de::<64>(stack, key_bytes, ads, iv, len, txt, &space) } } @@ -122,7 +140,7 @@ mod util { use crate::jets::util::BAIL_FAIL; use crate::jets::{list, JetErr, Result}; use crate::mem::NockStack; - use crate::noun::{Atom, IndirectAtom, Noun, D, T}; + use crate::noun::{Atom, IndirectAtom, Noun, NounSpace, D, T}; /// Associated data for AES-SIV functions. struct AcAesSivData { @@ -135,12 +153,13 @@ mod util { fn _allocate_ads( stack: &mut NockStack, mut ads: Noun, + space: &NounSpace, ) -> result::Result<&'static mut [AcAesSivData], JetErr> { if unsafe { ads.raw_equals(&D(0)) } { return Ok(&mut []); } - let length = list::util::lent(ads)?; + let length = list::util::lent(ads, space)?; let siv_data: &mut [AcAesSivData] = unsafe { let ptr = stack.struct_alloc::(length); @@ -149,18 +168,18 @@ mod util { unsafe { for item in siv_data.iter_mut().take(length) { - let cell = ads.as_cell()?; + let cell = ads.in_space(space).as_cell()?; let head = cell.head().as_atom()?; let bytes = head.as_ne_bytes(); - let len = met(3, head); + let len = met(3, head.atom(), space); let (mut atom, buffer) = IndirectAtom::new_raw_mut_bytes(stack, bytes.len()); buffer[0..len].copy_from_slice(&(bytes[0..len])); item.length = bytes.len(); - item.bytes = atom.data_pointer_mut() as *mut u8; + item.bytes = atom.data_pointer_mut(space) as *mut u8; - ads = cell.tail(); + ads = cell.tail().noun(); } } @@ -172,15 +191,16 @@ mod util { key: &mut [u8; N], ads: Noun, txt: Atom, + space: &NounSpace, ) -> Result { unsafe { - let ac_siv_data = _allocate_ads(stack, ads)?; + let ac_siv_data = _allocate_ads(stack, ads, space)?; let siv_data: &mut [&mut [u8]] = std::slice::from_raw_parts_mut( ac_siv_data.as_mut_ptr() as *mut &mut [u8], ac_siv_data.len(), ); - let txt_len = met(3, txt); + let txt_len = met(3, txt, space); let (mut iv, iv_bytes) = IndirectAtom::new_raw_mut_bytearray::<16, NockStack>(stack); @@ -197,11 +217,14 @@ mod util { option_env!("GIT_SHA") ) }); - Ok(T(stack, &[iv.normalize_as_atom().as_noun(), D(0), D(0)])) + Ok(T( + stack, + &[iv.normalize_as_atom(space).as_noun(), D(0), D(0)], + )) } _ => { let (_txt_ida, txt_bytes) = IndirectAtom::new_raw_mut_bytes(stack, txt_len); - txt_bytes.copy_from_slice(&txt.as_ne_bytes()[0..txt_len]); + txt_bytes.copy_from_slice(&txt.in_space(space).as_ne_bytes()[0..txt_len]); let (mut out_atom, out_bytes) = IndirectAtom::new_raw_mut_bytes(stack, txt_len); ac_aes_siv_en::(key, txt_bytes, siv_data, iv_bytes, out_bytes) .unwrap_or_else(|err| { @@ -215,9 +238,9 @@ mod util { Ok(T( stack, &[ - iv.normalize_as_atom().as_noun(), + iv.normalize_as_atom(space).as_noun(), D(txt_len as u64), - out_atom.normalize_as_atom().as_noun(), + out_atom.normalize_as_atom(space).as_noun(), ], )) } @@ -232,6 +255,7 @@ mod util { iv: Atom, len: Atom, txt: Atom, + space: &NounSpace, ) -> Result { unsafe { let txt_len = match len.as_direct() { @@ -240,9 +264,9 @@ mod util { }; let iv_bytes = &mut [0u8; 16]; - iv_bytes.copy_from_slice(&iv.as_ne_bytes()[0..16]); + iv_bytes.copy_from_slice(&iv.in_space(space).as_ne_bytes()[0..16]); - let ac_siv_data = _allocate_ads(stack, ads)?; + let ac_siv_data = _allocate_ads(stack, ads, space)?; let siv_data: &mut [&mut [u8]] = std::slice::from_raw_parts_mut( ac_siv_data.as_mut_ptr() as *mut &mut [u8], ac_siv_data.len(), @@ -264,7 +288,7 @@ mod util { } _ => { let (_txt_ida, txt_bytes) = IndirectAtom::new_raw_mut_bytes(stack, txt_len); - txt_bytes.copy_from_slice(&txt.as_ne_bytes()[0..txt_len]); + txt_bytes.copy_from_slice(&txt.in_space(space).as_ne_bytes()[0..txt_len]); ac_aes_siv_de::(key, txt_bytes, siv_data, iv_bytes, out_bytes) .unwrap_or_else(|err| { panic!( @@ -277,7 +301,10 @@ mod util { } } - Ok(T(stack, &[D(0), out_atom.normalize_as_atom().as_noun()])) + Ok(T( + stack, + &[D(0), out_atom.normalize_as_atom(space).as_noun()], + )) } } } diff --git a/crates/nockvm/rust/nockvm/src/jets/lock/ed.rs b/crates/nockvm/rust/nockvm/src/jets/lock/ed.rs index a3f604879..63407118c 100644 --- a/crates/nockvm/rust/nockvm/src/jets/lock/ed.rs +++ b/crates/nockvm/rust/nockvm/src/jets/lock/ed.rs @@ -11,34 +11,36 @@ crate::gdb!(); pub fn jet_puck(context: &mut Context, subject: Noun) -> Result { let stack = &mut context.stack; - let sed = slot(subject, 6)?.as_atom()?; + let space = stack.noun_space(); + let sed = slot(subject, 6, &space)?.as_atom()?; - let sed_len = met(3, sed); + let sed_len = met(3, sed, &space); if sed_len > 32 { return Err(BAIL_EXIT); } unsafe { let sed_bytes = &mut [0u8; 32]; - sed_bytes[0..sed_len].copy_from_slice(&(sed.as_ne_bytes())[0..sed_len]); + sed_bytes[0..sed_len].copy_from_slice(&(sed.in_space(&space).as_ne_bytes())[0..sed_len]); let (mut pub_ida, pub_key) = IndirectAtom::new_raw_mut_bytearray::<32, NockStack>(stack); ac_ed_puck(sed_bytes, pub_key); - Ok(pub_ida.normalize_as_atom().as_noun()) + Ok(pub_ida.normalize_as_atom(&space).as_noun()) } } pub fn jet_shar(context: &mut Context, subject: Noun) -> Result { let stack = &mut context.stack; - let pub_key = slot(subject, 12)?.as_atom()?; - let sec_key = slot(subject, 13)?.as_atom()?; + let space = stack.noun_space(); + let pub_key = slot(subject, 12, &space)?.as_atom()?; + let sec_key = slot(subject, 13, &space)?.as_atom()?; - if met(3, sec_key) > 32 { + if met(3, sec_key, &space) > 32 { // sek is size checked by +puck via +suck return Err(BAIL_EXIT); } - if met(3, pub_key) > 32 { + if met(3, pub_key, &space) > 32 { // pub is not size checked in Hoon, but it must be 32 bytes or less for // ucrypt. Therefore, punt on larger values. return Err(JetErr::Punt); @@ -48,8 +50,10 @@ pub fn jet_shar(context: &mut Context, subject: Noun) -> Result { let public = &mut [0u8; 32]; let secret = &mut [0u8; 32]; - let pub_bytes = pub_key.as_ne_bytes(); - let sec_bytes = sec_key.as_ne_bytes(); + let pub_handle = pub_key.in_space(&space); + let sec_handle = sec_key.in_space(&space); + let pub_bytes = pub_handle.as_ne_bytes(); + let sec_bytes = sec_handle.as_ne_bytes(); public[0..pub_bytes.len()].copy_from_slice(pub_bytes); secret[0..sec_bytes.len()].copy_from_slice(sec_bytes); @@ -57,17 +61,19 @@ pub fn jet_shar(context: &mut Context, subject: Noun) -> Result { let (mut shar_ida, shar) = IndirectAtom::new_raw_mut_bytearray::<32, NockStack>(stack); ac_ed_shar(public, secret, shar); - Ok(shar_ida.normalize_as_atom().as_noun()) + Ok(shar_ida.normalize_as_atom(&space).as_noun()) } } pub fn jet_sign(context: &mut Context, subject: Noun) -> Result { let stack = &mut context.stack; - let msg = slot(subject, 12)?.as_atom()?; - let sed = slot(subject, 13)?.as_atom()?; + let space = stack.noun_space(); + let msg = slot(subject, 12, &space)?.as_atom()?; + let sed = slot(subject, 13, &space)?.as_atom()?; unsafe { - let sed_bytes = sed.as_ne_bytes(); + let sed_handle = sed.in_space(&space); + let sed_bytes = sed_handle.as_ne_bytes(); let sed_len = sed_bytes.len(); if sed_len > 32 { return Err(BAIL_EXIT); @@ -77,41 +83,45 @@ pub fn jet_sign(context: &mut Context, subject: Noun) -> Result { let (mut sig_ida, sig) = IndirectAtom::new_raw_mut_bytearray::<64, NockStack>(stack); - let msg_len = met(3, msg); + let msg_len = met(3, msg, &space); if msg_len > 0 { let (_msg_ida, message) = IndirectAtom::new_raw_mut_bytes(stack, msg_len); - message.copy_from_slice(&msg.as_ne_bytes()[0..msg_len]); + message.copy_from_slice(&msg.in_space(&space).as_ne_bytes()[0..msg_len]); ac_ed_sign(message, seed, sig); } else { ac_ed_sign(&[0u8; 0], seed, sig); } sig.reverse(); - Ok(sig_ida.normalize_as_atom().as_noun()) + Ok(sig_ida.normalize_as_atom(&space).as_noun()) } } pub fn jet_veri(_context: &mut Context, subject: Noun) -> Result { - let sig = slot(subject, 12)?.as_atom()?; - let msg = slot(subject, 26)?.as_atom()?; - let puk = slot(subject, 27)?.as_atom()?; + let space = _context.stack.noun_space(); + let sig = slot(subject, 12, &space)?.as_atom()?; + let msg = slot(subject, 26, &space)?.as_atom()?; + let puk = slot(subject, 27, &space)?.as_atom()?; // Both are size checked by Hoon, but without crashing - let sig_bytes = sig.as_ne_bytes(); + let sig_handle = sig.in_space(&space); + let sig_bytes = sig_handle.as_ne_bytes(); if sig_bytes.len() > 64 { return Ok(NO); }; let signature = &mut [0u8; 64]; signature[0..sig_bytes.len()].copy_from_slice(sig_bytes); - let pub_bytes = puk.as_ne_bytes(); + let puk_handle = puk.in_space(&space); + let pub_bytes = puk_handle.as_ne_bytes(); if pub_bytes.len() > 32 { return Ok(NO); }; let public_key = &mut [0u8; 32]; public_key[0..pub_bytes.len()].copy_from_slice(pub_bytes); - let message = &(msg.as_ne_bytes())[0..met(3, msg)]; // drop trailing zeros + let msg_handle = msg.in_space(&space); + let message = &(msg_handle.as_ne_bytes())[0..met(3, msg, &space)]; // drop trailing zeros let valid = ac_ed_veri(message, public_key, signature); diff --git a/crates/nockvm/rust/nockvm/src/jets/lock/sha.rs b/crates/nockvm/rust/nockvm/src/jets/lock/sha.rs index ffd4f80be..458f6674c 100644 --- a/crates/nockvm/rust/nockvm/src/jets/lock/sha.rs +++ b/crates/nockvm/rust/nockvm/src/jets/lock/sha.rs @@ -10,20 +10,23 @@ crate::gdb!(); pub fn jet_shas(context: &mut Context, subject: Noun) -> Result { let stack = &mut context.stack; - let sam = slot(subject, 6)?; - let sal = slot(sam, 2)?.as_atom()?; - let ruz = slot(sam, 3)?.as_atom()?; + let space = stack.noun_space(); + let sam = slot(subject, 6, &space)?; + let sal = slot(sam, 2, &space)?.as_atom()?; + let ruz = slot(sam, 3, &space)?.as_atom()?; unsafe { let (mut out_ida, out) = IndirectAtom::new_raw_mut_bytes(stack, 32); - let sal_bytes = &(sal.as_ne_bytes())[0..met(3, sal)]; // drop trailing zeros + let sal_handle = sal.in_space(&space); + let sal_bytes = &(sal_handle.as_ne_bytes())[0..met(3, sal, &space)]; // drop trailing zeros let (mut _salt_ida, salt) = IndirectAtom::new_raw_mut_bytes(stack, sal_bytes.len()); salt.copy_from_slice(sal_bytes); - let msg_len = met(3, ruz); + let msg_len = met(3, ruz, &space); if msg_len > 0 { - let msg_bytes = &(ruz.as_ne_bytes())[0..msg_len]; + let ruz_handle = ruz.in_space(&space); + let msg_bytes = &(ruz_handle.as_ne_bytes())[0..msg_len]; let (_msg_ida, msg) = IndirectAtom::new_raw_mut_bytes(stack, msg_bytes.len()); msg.copy_from_slice(msg_bytes); ac_shas(msg, salt, out); @@ -31,21 +34,23 @@ pub fn jet_shas(context: &mut Context, subject: Noun) -> Result { ac_shas(&mut [], salt, out); } - Ok(out_ida.normalize_as_atom().as_noun()) + Ok(out_ida.normalize_as_atom(&space).as_noun()) } } pub fn jet_shax(context: &mut Context, subject: Noun) -> Result { let stack = &mut context.stack; - let sam = slot(subject, 6)?; + let space = stack.noun_space(); + let sam = slot(subject, 6, &space)?; let ruz = sam.as_atom()?; - let msg_len = met(3, ruz); + let msg_len = met(3, ruz, &space); unsafe { let (mut ida, out) = IndirectAtom::new_raw_mut_bytes(stack, 32); if msg_len > 0 { - let msg_bytes = &(ruz.as_ne_bytes())[0..msg_len]; + let ruz_handle = ruz.in_space(&space); + let msg_bytes = &(ruz_handle.as_ne_bytes())[0..msg_len]; let (_msg_ida, msg) = IndirectAtom::new_raw_mut_bytes(stack, msg_bytes.len()); msg.copy_from_slice(msg_bytes); ac_shay(msg, out); @@ -53,100 +58,106 @@ pub fn jet_shax(context: &mut Context, subject: Noun) -> Result { ac_shay(&mut [], out); } - Ok(ida.normalize_as_atom().as_noun()) + Ok(ida.normalize_as_atom(&space).as_noun()) } } pub fn jet_shay(context: &mut Context, subject: Noun) -> Result { let stack = &mut context.stack; - let sam = slot(subject, 6)?; - let len = slot(sam, 2)?.as_atom()?; - let ruz = slot(sam, 3)?.as_atom()?; + let space = stack.noun_space(); + let sam = slot(subject, 6, &space)?; + let len = slot(sam, 2, &space)?.as_atom()?; + let ruz = slot(sam, 3, &space)?.as_atom()?; let length = match len.as_direct() { Ok(direct) => direct.data() as usize, Err(_) => return Err(BAIL_FAIL), }; - let msg_len = met(3, ruz); + let msg_len = met(3, ruz, &space); unsafe { let (mut out_ida, out) = IndirectAtom::new_raw_mut_bytes(stack, 32); + let ruz_handle = ruz.in_space(&space); if length == 0 { ac_shay(&mut [], out); } else if msg_len >= length { let (mut _msg_ida, msg) = IndirectAtom::new_raw_mut_bytes(stack, length); - msg.copy_from_slice(&(ruz.as_ne_bytes())[0..length]); + msg.copy_from_slice(&(ruz_handle.as_ne_bytes())[0..length]); ac_shay(msg, out); } else { - let msg_bytes = &(ruz.as_ne_bytes())[0..msg_len]; + let msg_bytes = &(ruz_handle.as_ne_bytes())[0..msg_len]; let (mut _msg_ida, msg) = IndirectAtom::new_raw_mut_bytes(stack, length); msg[0..msg_len].copy_from_slice(msg_bytes); ac_shay(msg, out); } - Ok(out_ida.normalize_as_atom().as_noun()) + Ok(out_ida.normalize_as_atom(&space).as_noun()) } } pub fn jet_shal(context: &mut Context, subject: Noun) -> Result { let stack = &mut context.stack; - let sam = slot(subject, 6)?; - let len = slot(sam, 2)?.as_atom()?; - let ruz = slot(sam, 3)?.as_atom()?; + let space = stack.noun_space(); + let sam = slot(subject, 6, &space)?; + let len = slot(sam, 2, &space)?.as_atom()?; + let ruz = slot(sam, 3, &space)?.as_atom()?; let length = match len.as_direct() { Ok(direct) => direct.data() as usize, Err(_) => return Err(BAIL_FAIL), }; - let msg_len = met(3, ruz); + let msg_len = met(3, ruz, &space); unsafe { let (mut out_ida, out) = IndirectAtom::new_raw_mut_bytes(stack, 64); + let ruz_handle = ruz.in_space(&space); if length == 0 { ac_shal(&mut [], out); } else if msg_len >= length { let (mut _msg_ida, msg) = IndirectAtom::new_raw_mut_bytes(stack, length); - msg.copy_from_slice(&(ruz.as_ne_bytes())[0..length]); + msg.copy_from_slice(&(ruz_handle.as_ne_bytes())[0..length]); ac_shal(msg, out); } else { - let msg_bytes = &(ruz.as_ne_bytes())[0..msg_len]; + let msg_bytes = &(ruz_handle.as_ne_bytes())[0..msg_len]; let (mut _msg_ida, msg) = IndirectAtom::new_raw_mut_bytes(stack, length); msg[0..msg_len].copy_from_slice(msg_bytes); ac_shal(msg, out); } - Ok(out_ida.normalize_as_atom().as_noun()) + Ok(out_ida.normalize_as_atom(&space).as_noun()) } } pub fn jet_sha1(context: &mut Context, subject: Noun) -> Result { let stack = &mut context.stack; - let sam = slot(subject, 6)?; - let len = slot(sam, 2)?.as_atom()?; - let ruz = slot(sam, 3)?.as_atom()?; + let space = stack.noun_space(); + let sam = slot(subject, 6, &space)?; + let len = slot(sam, 2, &space)?.as_atom()?; + let ruz = slot(sam, 3, &space)?.as_atom()?; let length = match len.as_direct() { Ok(direct) => direct.data() as usize, Err(_) => return Err(BAIL_FAIL), }; - let msg_len = met(3, ruz); + let msg_len = met(3, ruz, &space); unsafe { let (mut out_ida, out) = IndirectAtom::new_raw_mut_bytes(stack, 20); + let ruz_handle = ruz.in_space(&space); if length == 0 { ac_sha1(&mut [], out); } else if msg_len >= length { let (mut _msg_ida, msg) = IndirectAtom::new_raw_mut_bytes(stack, length); - msg.copy_from_slice(&(ruz.as_ne_bytes())[0..length]); + msg.copy_from_slice(&(ruz_handle.as_ne_bytes())[0..length]); ac_sha1(msg, out); } else { - let msg_bytes = &(ruz.as_ne_bytes())[0..msg_len]; + let msg_bytes = &(ruz_handle.as_ne_bytes())[0..msg_len]; let (mut _msg_ida, msg) = IndirectAtom::new_raw_mut_bytes(stack, length); msg[0..msg_len].copy_from_slice(msg_bytes); ac_sha1(msg, out); } - Ok(out_ida.normalize_as_atom().as_noun()) + Ok(out_ida.normalize_as_atom(&space).as_noun()) } } diff --git a/crates/nockvm/rust/nockvm/src/jets/lute.rs b/crates/nockvm/rust/nockvm/src/jets/lute.rs index 2ea8aa54a..474ae7b52 100644 --- a/crates/nockvm/rust/nockvm/src/jets/lute.rs +++ b/crates/nockvm/rust/nockvm/src/jets/lute.rs @@ -10,13 +10,14 @@ use crate::noun::{Noun, D, NO, NONE, T, YES}; crate::gdb!(); pub fn jet_ut_crop(context: &mut Context, subject: Noun) -> Result { - let rff = slot(subject, 6)?; - let van = slot(subject, 7)?; + let space = context.stack.fast_noun_space(); + let rff = slot(subject, 6, &space)?; + let van = slot(subject, 7, &space)?; - let bat = slot(van, 2)?; - let sut = slot(van, 6)?; + let bat = slot(van, 2, &space)?; + let sut = slot(van, 6, &space)?; - let flag = if let Ok(noun) = slot(van, 59) { + let flag = if let Ok(noun) = slot(van, 59, &space) { if unsafe { noun.raw_equals(&D(0)) } { 0u64 } else { @@ -31,7 +32,7 @@ pub fn jet_ut_crop(context: &mut Context, subject: Noun) -> Result { match context.cache.lookup(&mut context.stack, &mut key) { Some(pro) => Ok(pro), None => { - let pro = interpret(context, subject, slot(subject, 2)?)?; + let pro = interpret(context, subject, slot(subject, 2, &space)?)?; context.cache = context.cache.insert(&mut context.stack, &mut key, pro); Ok(pro) } @@ -39,14 +40,15 @@ pub fn jet_ut_crop(context: &mut Context, subject: Noun) -> Result { } pub fn jet_ut_fish(context: &mut Context, subject: Noun) -> Result { + let space = context.stack.fast_noun_space(); // axe must be Atom, though we use it as Noun - let axe = slot(subject, 6)?.as_atom()?; - let van = slot(subject, 7)?; + let axe = slot(subject, 6, &space)?.as_atom()?; + let van = slot(subject, 7, &space)?; - let bat = slot(van, 2)?; - let sut = slot(van, 6)?; + let bat = slot(van, 2, &space)?; + let sut = slot(van, 6, &space)?; - let flag = if let Ok(noun) = slot(van, 59) { + let flag = if let Ok(noun) = slot(van, 59, &space) { if unsafe { noun.raw_equals(&D(0)) } { 0u64 } else { @@ -61,7 +63,7 @@ pub fn jet_ut_fish(context: &mut Context, subject: Noun) -> Result { match context.cache.lookup(&mut context.stack, &mut key) { Some(pro) => Ok(pro), None => { - let pro = interpret(context, subject, slot(subject, 2)?)?; + let pro = interpret(context, subject, slot(subject, 2, &space)?)?; context.cache = context.cache.insert(&mut context.stack, &mut key, pro); Ok(pro) } @@ -69,13 +71,14 @@ pub fn jet_ut_fish(context: &mut Context, subject: Noun) -> Result { } pub fn jet_ut_fuse(context: &mut Context, subject: Noun) -> Result { - let rff = slot(subject, 6)?; - let van = slot(subject, 7)?; + let space = context.stack.fast_noun_space(); + let rff = slot(subject, 6, &space)?; + let van = slot(subject, 7, &space)?; - let bat = slot(van, 2)?; - let sut = slot(van, 6)?; + let bat = slot(van, 2, &space)?; + let sut = slot(van, 6, &space)?; - let flag = if let Ok(noun) = slot(van, 59) { + let flag = if let Ok(noun) = slot(van, 59, &space) { if unsafe { noun.raw_equals(&D(0)) } { 0u64 } else { @@ -90,7 +93,7 @@ pub fn jet_ut_fuse(context: &mut Context, subject: Noun) -> Result { match context.cache.lookup(&mut context.stack, &mut key) { Some(pro) => Ok(pro), None => { - let pro = interpret(context, subject, slot(subject, 2)?)?; + let pro = interpret(context, subject, slot(subject, 2, &space)?)?; context.cache = context.cache.insert(&mut context.stack, &mut key, pro); Ok(pro) } @@ -98,21 +101,22 @@ pub fn jet_ut_fuse(context: &mut Context, subject: Noun) -> Result { } pub fn jet_ut_mint(context: &mut Context, subject: Noun) -> Result { - let gol = slot(subject, 12)?; - let gen = slot(subject, 13)?; - let van = slot(subject, 7)?; + let space = context.stack.fast_noun_space(); + let gol = slot(subject, 12, &space)?; + let gen = slot(subject, 13, &space)?; + let van = slot(subject, 7, &space)?; - let bat = slot(van, 2)?; - let sut = slot(van, 6)?; + let bat = slot(van, 2, &space)?; + let sut = slot(van, 6, &space)?; let fun = 141 + tas!(b"mint"); - let vet = slot(van, 59).map_or(NONE, |x| x); + let vet = slot(van, 59, &space).map_or(NONE, |x| x); let mut key = T(&mut context.stack, &[D(fun), vet, sut, gol, gen, bat]); match context.cache.lookup(&mut context.stack, &mut key) { Some(pro) => Ok(pro), None => { - let pro = interpret(context, subject, slot(subject, 2)?)?; + let pro = interpret(context, subject, slot(subject, 2, &space)?)?; context.cache = context.cache.insert(&mut context.stack, &mut key, pro); Ok(pro) } @@ -120,15 +124,16 @@ pub fn jet_ut_mint(context: &mut Context, subject: Noun) -> Result { } pub fn jet_ut_mull(context: &mut Context, subject: Noun) -> Result { - let gol = slot(subject, 12)?; - let dox = slot(subject, 26)?; - let gen = slot(subject, 27)?; - let van = slot(subject, 7)?; + let space = context.stack.fast_noun_space(); + let gol = slot(subject, 12, &space)?; + let dox = slot(subject, 26, &space)?; + let gen = slot(subject, 27, &space)?; + let van = slot(subject, 7, &space)?; - let bat = slot(van, 2)?; - let sut = slot(van, 6)?; + let bat = slot(van, 2, &space)?; + let sut = slot(van, 6, &space)?; - let flag = if let Ok(noun) = slot(van, 59) { + let flag = if let Ok(noun) = slot(van, 59, &space) { if unsafe { noun.raw_equals(&D(0)) } { 0u64 } else { @@ -143,7 +148,7 @@ pub fn jet_ut_mull(context: &mut Context, subject: Noun) -> Result { match context.cache.lookup(&mut context.stack, &mut key) { Some(pro) => Ok(pro), None => { - let pro = interpret(context, subject, slot(subject, 2)?)?; + let pro = interpret(context, subject, slot(subject, 2, &space)?)?; context.cache = context.cache.insert(&mut context.stack, &mut key, pro); Ok(pro) } @@ -151,19 +156,20 @@ pub fn jet_ut_mull(context: &mut Context, subject: Noun) -> Result { } pub fn jet_ut_nest_dext(context: &mut Context, subject: Noun) -> Result { - let nest_in_core = slot(subject, 3)?; + let space = context.stack.fast_noun_space(); + let nest_in_core = slot(subject, 3, &space)?; - let seg = slot(nest_in_core, 12)?; - let reg = slot(nest_in_core, 26)?; - let nest_core = slot(nest_in_core, 7)?; + let seg = slot(nest_in_core, 12, &space)?; + let reg = slot(nest_in_core, 26, &space)?; + let nest_core = slot(nest_in_core, 7, &space)?; - let rff = slot(nest_core, 13)?; - let van = slot(nest_core, 7)?; + let rff = slot(nest_core, 13, &space)?; + let van = slot(nest_core, 7, &space)?; - let bat = slot(van, 2)?; - let sut = slot(van, 6)?; + let bat = slot(van, 2, &space)?; + let sut = slot(van, 6, &space)?; - let flag = if let Ok(noun) = slot(van, 59) { + let flag = if let Ok(noun) = slot(van, 59, &space) { if unsafe { noun.raw_equals(&D(0)) } { 0u64 } else { @@ -178,7 +184,7 @@ pub fn jet_ut_nest_dext(context: &mut Context, subject: Noun) -> Result { match context.cache.lookup(&mut context.stack, &mut key) { Some(pro) => Ok(pro), None => { - let pro = interpret(context, subject, slot(subject, 2)?)?; + let pro = interpret(context, subject, slot(subject, 2, &space)?)?; if unsafe { pro.raw_equals(&YES) && reg.raw_equals(&D(0)) } || unsafe { pro.raw_equals(&NO) && seg.raw_equals(&D(0)) } { @@ -190,12 +196,13 @@ pub fn jet_ut_nest_dext(context: &mut Context, subject: Noun) -> Result { } pub fn jet_ut_redo(context: &mut Context, subject: Noun) -> Result { - let rff = slot(subject, 6)?; - let van = slot(subject, 7)?; - let bat = slot(van, 2)?; - let sut = slot(van, 6)?; + let space = context.stack.fast_noun_space(); + let rff = slot(subject, 6, &space)?; + let van = slot(subject, 7, &space)?; + let bat = slot(van, 2, &space)?; + let sut = slot(van, 6, &space)?; - let flag = if let Ok(noun) = slot(van, 59) { + let flag = if let Ok(noun) = slot(van, 59, &space) { if unsafe { noun.raw_equals(&D(0)) } { 0u64 } else { @@ -210,7 +217,7 @@ pub fn jet_ut_redo(context: &mut Context, subject: Noun) -> Result { match context.cache.lookup(&mut context.stack, &mut key) { Some(pro) => Ok(pro), None => { - let pro = interpret(context, subject, slot(subject, 2)?)?; + let pro = interpret(context, subject, slot(subject, 2, &space)?)?; context.cache = context.cache.insert(&mut context.stack, &mut key, pro); Ok(pro) } @@ -218,13 +225,14 @@ pub fn jet_ut_redo(context: &mut Context, subject: Noun) -> Result { } pub fn jet_ut_rest(context: &mut Context, subject: Noun) -> Result { - let leg = slot(subject, 6)?; - let van = slot(subject, 7)?; + let space = context.stack.fast_noun_space(); + let leg = slot(subject, 6, &space)?; + let van = slot(subject, 7, &space)?; - let bat = slot(van, 2)?; - let sut = slot(van, 6)?; + let bat = slot(van, 2, &space)?; + let sut = slot(van, 6, &space)?; - let flag = if let Ok(noun) = slot(van, 59) { + let flag = if let Ok(noun) = slot(van, 59, &space) { if unsafe { noun.raw_equals(&D(0)) } { 0u64 } else { @@ -239,7 +247,7 @@ pub fn jet_ut_rest(context: &mut Context, subject: Noun) -> Result { match context.cache.lookup(&mut context.stack, &mut key) { Some(pro) => Ok(pro), None => { - let pro = interpret(context, subject, slot(subject, 2)?)?; + let pro = interpret(context, subject, slot(subject, 2, &space)?)?; context.cache = context.cache.insert(&mut context.stack, &mut key, pro); Ok(pro) } diff --git a/crates/nockvm/rust/nockvm/src/jets/math.rs b/crates/nockvm/rust/nockvm/src/jets/math.rs index 1e284a952..fda6ad31c 100644 --- a/crates/nockvm/rust/nockvm/src/jets/math.rs +++ b/crates/nockvm/rust/nockvm/src/jets/math.rs @@ -24,14 +24,16 @@ use crate::noun::{Atom, DirectAtom, IndirectAtom, Noun, D, DIRECT_MAX, T}; crate::gdb!(); pub fn jet_add(context: &mut Context, subject: Noun) -> Result { - let arg = slot(subject, 6)?; - let a = slot(arg, 2)?.as_atom()?; - let b = slot(arg, 3)?.as_atom()?; - Ok(util::add(&mut context.stack, a, b).as_noun()) + let space = context.stack.noun_space(); + let arg = slot(subject, 6, &space)?; + let a = slot(arg, 2, &space)?.as_atom()?; + let b = slot(arg, 3, &space)?.as_atom()?; + Ok(util::add(&mut context.stack, a, b, &space).as_noun()) } pub fn jet_dec(context: &mut Context, subject: Noun) -> Result { - let arg = slot(subject, 6)?; + let space = context.stack.noun_space(); + let arg = slot(subject, 6, &space)?; if let Ok(atom) = arg.as_atom() { match atom.as_either() { Left(direct) => { @@ -42,14 +44,18 @@ pub fn jet_dec(context: &mut Context, subject: Noun) -> Result { } } Right(indirect) => { - let indirect_slice = indirect.as_bitslice(); + let indirect_handle = indirect.as_atom().in_space(&space); + let indirect_slice = indirect_handle.as_bitslice(); match indirect_slice.first_one() { None => { panic!("Decrementing 0 stored as an indirect atom"); } Some(first_one) => { let (mut new_indirect, new_slice) = unsafe { - IndirectAtom::new_raw_mut_bitslice(&mut context.stack, indirect.size()) + IndirectAtom::new_raw_mut_bitslice( + &mut context.stack, + indirect_handle.size(), + ) }; if first_one > 0 { new_slice[..first_one].fill(true); @@ -57,7 +63,7 @@ pub fn jet_dec(context: &mut Context, subject: Noun) -> Result { new_slice.set(first_one, false); new_slice[first_one + 1..] .copy_from_bitslice(&indirect_slice[first_one + 1..]); - let res = unsafe { new_indirect.normalize_as_atom() }; + let res = unsafe { new_indirect.normalize_as_atom(&space) }; Ok(res.as_noun()) } } @@ -70,17 +76,18 @@ pub fn jet_dec(context: &mut Context, subject: Noun) -> Result { pub fn jet_div(context: &mut Context, subject: Noun) -> Result { let stack = &mut context.stack; - let arg = slot(subject, 6)?; - let a = slot(arg, 2)?.as_atom()?; - let b = slot(arg, 3)?.as_atom()?; + let space = stack.noun_space(); + let arg = slot(subject, 6, &space)?; + let a = slot(arg, 2, &space)?.as_atom()?; + let b = slot(arg, 3, &space)?.as_atom()?; if unsafe { b.as_noun().raw_equals(&D(0)) } { Err(BAIL_EXIT) } else if let (Ok(a), Ok(b)) = (a.as_direct(), b.as_direct()) { Ok(unsafe { DirectAtom::new_unchecked(a.data() / b.data()) }.as_noun()) } else { - let a_big = a.as_ubig(stack); - let b_big = b.as_ubig(stack); + let a_big = a.in_space(&space).as_ubig(stack); + let b_big = b.in_space(&space).as_ubig(stack); let res = UBig::div_stack(stack, a_big, b_big); Ok(Atom::from_ubig(stack, &res).as_noun()) } @@ -88,9 +95,10 @@ pub fn jet_div(context: &mut Context, subject: Noun) -> Result { pub fn jet_dvr(context: &mut Context, subject: Noun) -> Result { let stack = &mut context.stack; - let arg = slot(subject, 6)?; - let a = slot(arg, 2)?.as_atom()?; - let b = slot(arg, 3)?.as_atom()?; + let space = stack.noun_space(); + let arg = slot(subject, 6, &space)?; + let a = slot(arg, 2, &space)?.as_atom()?; + let b = slot(arg, 3, &space)?.as_atom()?; if unsafe { b.as_noun().raw_equals(&D(0)) } { Err(BAIL_EXIT) @@ -104,7 +112,10 @@ pub fn jet_dvr(context: &mut Context, subject: Noun) -> Result { ) } } else { - let (div, rem) = a.as_ubig(stack).div_rem(b.as_ubig(stack)); + let (div, rem) = a + .in_space(&space) + .as_ubig(stack) + .div_rem(b.in_space(&space).as_ubig(stack)); ( Atom::from_ubig(stack, &div).as_noun(), Atom::from_ubig(stack, &rem).as_noun(), @@ -117,57 +128,64 @@ pub fn jet_dvr(context: &mut Context, subject: Noun) -> Result { pub fn jet_gte(context: &mut Context, subject: Noun) -> Result { let stack = &mut context.stack; - let arg = slot(subject, 6)?; - let a = slot(arg, 2)?.as_atom()?; - let b = slot(arg, 3)?.as_atom()?; + let space = stack.noun_space(); + let arg = slot(subject, 6, &space)?; + let a = slot(arg, 2, &space)?.as_atom()?; + let b = slot(arg, 3, &space)?.as_atom()?; - Ok(util::gte(stack, a, b)) + Ok(util::gte(stack, a, b, &space)) } pub fn jet_gth(context: &mut Context, subject: Noun) -> Result { let stack = &mut context.stack; - let arg = slot(subject, 6)?; - let a = slot(arg, 2)?.as_atom()?; - let b = slot(arg, 3)?.as_atom()?; + let space = stack.noun_space(); + let arg = slot(subject, 6, &space)?; + let a = slot(arg, 2, &space)?.as_atom()?; + let b = slot(arg, 3, &space)?.as_atom()?; - Ok(util::gth(stack, a, b)) + Ok(util::gth(stack, a, b, &space)) } pub fn jet_lte(context: &mut Context, subject: Noun) -> Result { let stack = &mut context.stack; - let arg = slot(subject, 6)?; - let a = slot(arg, 2)?.as_atom()?; - let b = slot(arg, 3)?.as_atom()?; + let space = stack.noun_space(); + let arg = slot(subject, 6, &space)?; + let a = slot(arg, 2, &space)?.as_atom()?; + let b = slot(arg, 3, &space)?.as_atom()?; - Ok(util::lte(stack, a, b)) + Ok(util::lte(stack, a, b, &space)) } pub fn jet_lth(context: &mut Context, subject: Noun) -> Result { let stack = &mut context.stack; - let arg = slot(subject, 6)?; - let a = slot(arg, 2)?.as_atom()?; - let b = slot(arg, 3)?.as_atom()?; + let space = stack.noun_space(); + let arg = slot(subject, 6, &space)?; + let a = slot(arg, 2, &space)?.as_atom()?; + let b = slot(arg, 3, &space)?.as_atom()?; - Ok(util::lth(stack, a, b)) + Ok(util::lth(stack, a, b, &space)) } pub fn jet_max(context: &mut Context, subject: Noun) -> Result { let stack = &mut context.stack; - let arg = slot(subject, 6)?; - let a = slot(arg, 2)?.as_atom()?; - let b = slot(arg, 3)?.as_atom()?; + let space = stack.noun_space(); + let arg = slot(subject, 6, &space)?; + let a = slot(arg, 2, &space)?.as_atom()?; + let b = slot(arg, 3, &space)?.as_atom()?; + let a_handle = a.in_space(&space); + let b_handle = b.in_space(&space); Ok(if let (Ok(a), Ok(b)) = (a.as_direct(), b.as_direct()) { if a.data() >= b.data() { a.as_noun() } else { b.as_noun() } - } else if a.bit_size() > b.bit_size() { + } else if a_handle.bit_size() > b_handle.bit_size() { a.as_noun() - } else if a.bit_size() < b.bit_size() { + } else if a_handle.bit_size() < b_handle.bit_size() { b.as_noun() - } else if a.as_ubig(stack) >= b.as_ubig(stack) { + } else if a_handle.as_ubig(stack) >= b_handle.as_ubig(stack) { a.as_noun() } else { b.as_noun() @@ -176,21 +194,24 @@ pub fn jet_max(context: &mut Context, subject: Noun) -> Result { pub fn jet_min(context: &mut Context, subject: Noun) -> Result { let stack = &mut context.stack; - let arg = slot(subject, 6)?; - let a = slot(arg, 2)?.as_atom()?; - let b = slot(arg, 3)?.as_atom()?; + let space = stack.noun_space(); + let arg = slot(subject, 6, &space)?; + let a = slot(arg, 2, &space)?.as_atom()?; + let b = slot(arg, 3, &space)?.as_atom()?; + let a_handle = a.in_space(&space); + let b_handle = b.in_space(&space); Ok(if let (Ok(a), Ok(b)) = (a.as_direct(), b.as_direct()) { if a.data() <= b.data() { a.as_noun() } else { b.as_noun() } - } else if a.bit_size() < b.bit_size() { + } else if a_handle.bit_size() < b_handle.bit_size() { a.as_noun() - } else if a.bit_size() > b.bit_size() { + } else if a_handle.bit_size() > b_handle.bit_size() { b.as_noun() - } else if a.as_ubig(stack) <= b.as_ubig(stack) { + } else if a_handle.as_ubig(stack) <= b_handle.as_ubig(stack) { a.as_noun() } else { b.as_noun() @@ -199,25 +220,27 @@ pub fn jet_min(context: &mut Context, subject: Noun) -> Result { pub fn jet_mod(context: &mut Context, subject: Noun) -> Result { let stack = &mut context.stack; - let arg = slot(subject, 6)?; - let a = slot(arg, 2)?.as_atom()?; - let b = slot(arg, 3)?.as_atom()?; + let space = stack.noun_space(); + let arg = slot(subject, 6, &space)?; + let a = slot(arg, 2, &space)?.as_atom()?; + let b = slot(arg, 3, &space)?.as_atom()?; if unsafe { b.as_noun().raw_equals(&D(0)) } { Err(BAIL_EXIT) } else if let (Ok(a), Ok(b)) = (a.as_direct(), b.as_direct()) { Ok(unsafe { DirectAtom::new_unchecked(a.data() % b.data()) }.as_noun()) } else { - let res = a.as_ubig(stack) % b.as_ubig(stack); + let res = a.in_space(&space).as_ubig(stack) % b.in_space(&space).as_ubig(stack); Ok(Atom::from_ubig(stack, &res).as_noun()) } } pub fn jet_mul(context: &mut Context, subject: Noun) -> Result { let stack = &mut context.stack; - let arg = slot(subject, 6)?; - let a = slot(arg, 2)?.as_atom()?; - let b = slot(arg, 3)?.as_atom()?; + let space = stack.noun_space(); + let arg = slot(subject, 6, &space)?; + let a = slot(arg, 2, &space)?.as_atom()?; + let b = slot(arg, 3, &space)?.as_atom()?; if let (Ok(a), Ok(b)) = (a.as_direct(), b.as_direct()) { let res = a.data() as u128 * b.data() as u128; @@ -234,55 +257,60 @@ pub fn jet_mul(context: &mut Context, subject: Noun) -> Result { .as_noun()) } } else { - let a_big = a.as_ubig(stack); - let b_big = b.as_ubig(stack); + let a_big = a.in_space(&space).as_ubig(stack); + let b_big = b.in_space(&space).as_ubig(stack); let res = UBig::mul_stack(stack, a_big, b_big); Ok(Atom::from_ubig(stack, &res).as_noun()) } } pub fn jet_sub(context: &mut Context, subject: Noun) -> Result { - let arg = slot(subject, 6)?; - let a = slot(arg, 2)?.as_atom()?; - let b = slot(arg, 3)?.as_atom()?; + let space = context.stack.noun_space(); + let arg = slot(subject, 6, &space)?; + let a = slot(arg, 2, &space)?.as_atom()?; + let b = slot(arg, 3, &space)?.as_atom()?; - Ok(util::sub(&mut context.stack, a, b)?.as_noun()) + Ok(util::sub(&mut context.stack, a, b, &space)?.as_noun()) } pub mod util { use ibig::UBig; use crate::mem::NockStack; - use crate::noun::{Atom, Error, Noun, NounAllocator, Result, NO, YES}; + use crate::noun::{Atom, Error, Noun, NounAllocator, NounSpace, Result, NO, YES}; /// Addition - pub fn add(stack: &mut NockStack, a: Atom, b: Atom) -> Atom { + pub fn add(stack: &mut NockStack, a: Atom, b: Atom, space: &NounSpace) -> Atom { if let (Ok(a), Ok(b)) = (a.as_direct(), b.as_direct()) { Atom::new(stack, a.data() + b.data()) } else { - let a_big = a.as_ubig(stack); - let b_big = b.as_ubig(stack); + let a_big = a.in_space(space).as_ubig(stack); + let b_big = b.in_space(space).as_ubig(stack); let res = UBig::add_stack(stack, a_big, b_big); Atom::from_ubig(stack, &res) } } /// Greater than or equal to (boolean) - pub fn gte_b(stack: &mut NockStack, a: Atom, b: Atom) -> bool { + pub fn gte_b(stack: &mut NockStack, a: Atom, b: Atom, space: &NounSpace) -> bool { if let (Ok(a), Ok(b)) = (a.as_direct(), b.as_direct()) { a.data() >= b.data() - } else if a.bit_size() > b.bit_size() { - true - } else if a.bit_size() < b.bit_size() { - false } else { - a.as_ubig(stack) >= b.as_ubig(stack) + let a_handle = a.in_space(space); + let b_handle = b.in_space(space); + if a_handle.bit_size() > b_handle.bit_size() { + true + } else if a_handle.bit_size() < b_handle.bit_size() { + false + } else { + a_handle.as_ubig(stack) >= b_handle.as_ubig(stack) + } } } /// Greater than or equal to - pub fn gte(stack: &mut NockStack, a: Atom, b: Atom) -> Noun { - if gte_b(stack, a, b) { + pub fn gte(stack: &mut NockStack, a: Atom, b: Atom, space: &NounSpace) -> Noun { + if gte_b(stack, a, b, space) { YES } else { NO @@ -290,21 +318,25 @@ pub mod util { } /// Greater than (boolean) - pub fn gth_b(stack: &mut NockStack, a: Atom, b: Atom) -> bool { + pub fn gth_b(stack: &mut NockStack, a: Atom, b: Atom, space: &NounSpace) -> bool { if let (Ok(a), Ok(b)) = (a.as_direct(), b.as_direct()) { a.data() > b.data() - } else if a.bit_size() > b.bit_size() { - true - } else if a.bit_size() < b.bit_size() { - false } else { - a.as_ubig(stack) > b.as_ubig(stack) + let a_handle = a.in_space(space); + let b_handle = b.in_space(space); + if a_handle.bit_size() > b_handle.bit_size() { + true + } else if a_handle.bit_size() < b_handle.bit_size() { + false + } else { + a_handle.as_ubig(stack) > b_handle.as_ubig(stack) + } } } /// Greater than - pub fn gth(stack: &mut NockStack, a: Atom, b: Atom) -> Noun { - if gth_b(stack, a, b) { + pub fn gth(stack: &mut NockStack, a: Atom, b: Atom, space: &NounSpace) -> Noun { + if gth_b(stack, a, b, space) { YES } else { NO @@ -312,21 +344,25 @@ pub mod util { } /// Less than or equal to (boolean) - pub fn lte_b(stack: &mut NockStack, a: Atom, b: Atom) -> bool { + pub fn lte_b(stack: &mut NockStack, a: Atom, b: Atom, space: &NounSpace) -> bool { if let (Ok(a), Ok(b)) = (a.as_direct(), b.as_direct()) { a.data() <= b.data() - } else if a.bit_size() < b.bit_size() { - true - } else if a.bit_size() > b.bit_size() { - false } else { - a.as_ubig(stack) <= b.as_ubig(stack) + let a_handle = a.in_space(space); + let b_handle = b.in_space(space); + if a_handle.bit_size() < b_handle.bit_size() { + true + } else if a_handle.bit_size() > b_handle.bit_size() { + false + } else { + a_handle.as_ubig(stack) <= b_handle.as_ubig(stack) + } } } /// Less than or equal to - pub fn lte(stack: &mut NockStack, a: Atom, b: Atom) -> Noun { - if lte_b(stack, a, b) { + pub fn lte(stack: &mut NockStack, a: Atom, b: Atom, space: &NounSpace) -> Noun { + if lte_b(stack, a, b, space) { YES } else { NO @@ -334,21 +370,25 @@ pub mod util { } /// Less than (boolean) - pub fn lth_b(stack: &mut A, a: Atom, b: Atom) -> bool { + pub fn lth_b(stack: &mut A, a: Atom, b: Atom, space: &NounSpace) -> bool { if let (Ok(a), Ok(b)) = (a.as_direct(), b.as_direct()) { a.data() < b.data() - } else if a.bit_size() > b.bit_size() { - false - } else if a.bit_size() < b.bit_size() { - true } else { - a.as_ubig(stack) < b.as_ubig(stack) + let a_handle = a.in_space(space); + let b_handle = b.in_space(space); + if a_handle.bit_size() > b_handle.bit_size() { + false + } else if a_handle.bit_size() < b_handle.bit_size() { + true + } else { + a_handle.as_ubig(stack) < b_handle.as_ubig(stack) + } } } /// Less than - pub fn lth(stack: &mut A, a: Atom, b: Atom) -> Noun { - if lth_b(stack, a, b) { + pub fn lth(stack: &mut A, a: Atom, b: Atom, space: &NounSpace) -> Noun { + if lth_b(stack, a, b, space) { YES } else { NO @@ -356,7 +396,7 @@ pub mod util { } /// Subtraction - pub fn sub(stack: &mut NockStack, a: Atom, b: Atom) -> Result { + pub fn sub(stack: &mut NockStack, a: Atom, b: Atom, space: &NounSpace) -> Result { if let (Ok(a), Ok(b)) = (a.as_direct(), b.as_direct()) { let a_small = a.data(); let b_small = b.data(); @@ -367,14 +407,14 @@ pub mod util { Ok(Atom::new(stack, a_small - b_small)) } } else { - let a_big = a.as_ubig(stack); - let b_big = b.as_ubig(stack); + let a_handle = a.in_space(space); + let b_handle = b.in_space(space); + let a_big = a_handle.as_ubig(stack); + let b_big = b_handle.as_ubig(stack); if a_big < b_big { Err(Error::NotRepresentable) } else { - let a_big = a.as_ubig(stack); - let b_big = b.as_ubig(stack); let res = UBig::sub_stack(stack, a_big, b_big); Ok(Atom::from_ubig(stack, &res)) } diff --git a/crates/nockvm/rust/nockvm/src/jets/nock.rs b/crates/nockvm/rust/nockvm/src/jets/nock.rs index acbe9cd03..3a74deba0 100644 --- a/crates/nockvm/rust/nockvm/src/jets/nock.rs +++ b/crates/nockvm/rust/nockvm/src/jets/nock.rs @@ -3,17 +3,18 @@ use crate::interpreter::Context; use crate::jets::util::slot; use crate::jets::{JetErr, Result}; -use crate::noun::{Noun, D, NO, T}; +use crate::noun::{CellHandle, Noun, D, NO, T}; crate::gdb!(); pub fn jet_mink(context: &mut Context, subject: Noun) -> Result { - let arg = slot(subject, 6)?; + let space = context.stack.noun_space(); + let arg = slot(subject, 6, &space)?; // mink sample = [nock scry_namespace] // = [[subject formula] scry_namespace] - let v_subject = slot(arg, 4)?; - let v_formula = slot(arg, 5)?; - let scry_handler = slot(arg, 3)?; + let v_subject = slot(arg, 4, &space)?; + let v_formula = slot(arg, 5, &space)?; + let scry_handler = slot(arg, 3, &space)?; // Implicit error conversion Ok(util::mink(context, v_subject, v_formula, scry_handler)?) @@ -28,13 +29,15 @@ pub fn jet_mule(context: &mut Context, subject: Noun) -> Result { } pub fn jet_mure(context: &mut Context, subject: Noun) -> Result { - let tap = slot(subject, 6)?; + let space = context.stack.noun_space(); + let tap = slot(subject, 6, &space)?; let fol = util::slam_gate_fol(&mut context.stack); let scry = util::pass_thru_scry(&mut context.stack); match util::mink(context, tap, fol, scry) { Ok(tone) => { - if unsafe { tone.as_cell()?.head().raw_equals(&D(0)) } { + let tone_cell = tone.in_space(&space).as_cell()?; + if unsafe { tone_cell.head().noun().raw_equals(&D(0)) } { Ok(tone) } else { Ok(D(0)) @@ -45,7 +48,8 @@ pub fn jet_mure(context: &mut Context, subject: Noun) -> Result { } pub fn jet_mute(context: &mut Context, subject: Noun) -> Result { - let tap = slot(subject, 6)?; + let space = context.stack.noun_space(); + let tap = slot(subject, 6, &space)?; let fol = util::slam_gate_fol(&mut context.stack); let scry = util::pass_thru_scry(&mut context.stack); @@ -53,16 +57,17 @@ pub fn jet_mute(context: &mut Context, subject: Noun) -> Result { match util::mook(context, tone?.as_cell()?, false) { Ok(toon) => { - match toon.head() { + let toon_handle = CellHandle::new(toon, &space); + match toon_handle.head().noun() { x if unsafe { x.raw_equals(&D(0)) } => Ok(toon.as_noun()), x if unsafe { x.raw_equals(&D(1)) } => { // XX: Need to check that result is actually of type path // return [[%leaf "mute.hunk"] ~] if not - let bon = util::smyt(&mut context.stack, toon.tail())?; + let bon = util::smyt(&mut context.stack, toon_handle.tail().noun(), &space)?; Ok(T(&mut context.stack, &[NO, bon, D(0)])) } x if unsafe { x.raw_equals(&D(2)) } => { - Ok(T(&mut context.stack, &[NO, toon.tail()])) + Ok(T(&mut context.stack, &[NO, toon_handle.tail().noun()])) } _ => panic!("serf: mook: invalid toon"), } @@ -83,7 +88,7 @@ pub mod util { use crate::jets::bits::util::rip; use crate::jets::form::util::scow; use crate::mem::NockStack; - use crate::noun::{tape, Cell, Noun, D, T}; + use crate::noun::{tape, Cell, CellHandle, Noun, NounSpace, D, T}; pub const LEAF: Noun = D(tas!(b"leaf")); pub const ROSE: Noun = D(tas!(b"rose")); @@ -210,8 +215,10 @@ pub mod util { */ // XX: should write a jet_mook wrapper for this function pub fn mook(context: &mut Context, tone: Cell, flop: bool) -> result::Result { - let tag = tone.head().as_direct()?; - let original_list = tone.tail(); + let space = context.stack.noun_space(); + let tone_handle = CellHandle::new(tone, &space); + let tag = tone_handle.head().noun().as_direct()?; + let original_list = tone_handle.tail().noun(); if (tag.data() != 2) | unsafe { original_list.raw_equals(&D(0)) } { return Ok(tone); @@ -227,10 +234,10 @@ pub mod util { let mut list = original_list; while !list.raw_equals(&D(0)) { - let cell = list.as_cell()?; + let cell = list.in_space(&space).as_cell()?; let trace = cell.head().as_cell()?; - let tag = trace.head().as_direct()?; - let dat = trace.tail(); + let tag = trace.head().noun().as_direct()?; + let dat = trace.tail().noun(); let tank: Noun = match tag.data() { tas!(b"hunk") => match dat.as_either_atom_cell() { @@ -242,29 +249,33 @@ pub mod util { Right(cell) => { // XX: need to check that this is actually a path // return leaf+"mook.hunk" if not - let path = cell.tail(); - smyt(&mut context.stack, path)? + let path = CellHandle::new(cell, &space).tail().noun(); + smyt(&mut context.stack, path, &space)? } }, tas!(b"mean") => match dat.as_either_atom_cell() { Left(atom) => { let stack = &mut context.stack; - let tape = rip(stack, 3, 1, atom)?; + let tape = rip(stack, 3, 1, atom, &space)?; T(stack, &[LEAF, tape]) } Right(cell) => { 'tank: { let scry = null_scry(&mut context.stack); // if +mink didn't crash... - if let Ok(tone) = mink(context, dat, cell.head(), scry) { + let cell_handle = CellHandle::new(cell, &space); + if let Ok(tone) = + mink(context, dat, cell_handle.head().noun(), scry) + { if let Some(tonc) = tone.cell() { + let tonc_handle = CellHandle::new(tonc, &space); // ...and +mink didn't fail or block... - if tonc.head().raw_equals(&D(0)) { + if tonc_handle.head().noun().raw_equals(&D(0)) { // ...return $tank from $tone // XX: need to check that this is // actually a tank; // return leaf+"mook.mean" if not - break 'tank tonc.tail(); + break 'tank tonc_handle.tail().noun(); } } else { panic!("+mink in +mook somehow returned atom {tone:?}") @@ -282,7 +293,7 @@ pub mod util { tas!(b"spot") => { let stack = &mut context.stack; - let spot = dat.as_cell()?; + let spot = dat.in_space(&space).as_cell()?; let pint = spot.tail().as_cell()?; let pstr = pint.head().as_cell()?; let pend = pint.tail().as_cell()?; @@ -290,60 +301,64 @@ pub mod util { let colo = T(stack, &[D(b':' as u64), D(0)]); let trel = T(stack, &[colo, D(0), D(0)]); - let smyt = smyt(stack, spot.head())?; + let smyt = smyt(stack, spot.head().noun(), &space)?; let aura = D(tas!(b"ud")).as_direct()?; - let str_lin = scow(stack, aura, pstr.head().as_atom()?)?; - let str_col = scow(stack, aura, pstr.tail().as_atom()?)?; - let end_lin = scow(stack, aura, pend.head().as_atom()?)?; - let end_col = scow(stack, aura, pend.tail().as_atom()?)?; + let str_lin = scow(stack, aura, pstr.head().as_atom()?.atom(), &space)?; + let str_col = scow(stack, aura, pstr.tail().as_atom()?.atom(), &space)?; + let end_lin = scow(stack, aura, pend.head().as_atom()?.atom(), &space)?; + let end_col = scow(stack, aura, pend.tail().as_atom()?.atom(), &space)?; let mut list = end_col.as_cell()?; loop { - if list.tail().atom().is_some() { + let list_handle = CellHandle::new(list, &space); + if list_handle.tail().atom().is_some() { break; } - list = list.tail().as_cell()?; + list = list_handle.tail().noun().as_cell()?; } // "{end_col}]>" let p4 = T(stack, &[D(b']' as u64), D(b'>' as u64), D(0)]); - (*list.tail_as_mut()) = p4; + (*list.tail_as_mut(&space)) = p4; list = end_lin.as_cell()?; loop { - if list.tail().atom().is_some() { + let list_handle = CellHandle::new(list, &space); + if list_handle.tail().atom().is_some() { break; } - list = list.tail().as_cell()?; + list = list_handle.tail().noun().as_cell()?; } // "{end_lin} {end_col}]>" let p3 = T(stack, &[D(b' ' as u64), end_col]); - (*list.tail_as_mut()) = p3; + (*list.tail_as_mut(&space)) = p3; list = str_col.as_cell()?; loop { - if list.tail().atom().is_some() { + let list_handle = CellHandle::new(list, &space); + if list_handle.tail().atom().is_some() { break; } - list = list.tail().as_cell()?; + list = list_handle.tail().noun().as_cell()?; } // "{str_col}].[{end_lin} {end_col}]>" let p2 = T( stack, &[D(b']' as u64), D(b'.' as u64), D(b'[' as u64), end_lin], ); - (*list.tail_as_mut()) = p2; + (*list.tail_as_mut(&space)) = p2; list = str_lin.as_cell()?; loop { - if list.tail().atom().is_some() { + let list_handle = CellHandle::new(list, &space); + if list_handle.tail().atom().is_some() { break; } - list = list.tail().as_cell()?; + list = list_handle.tail().noun().as_cell()?; } // "{str_lin} {str_col}].[{end_lin} {end_col}]>" let p1 = T(stack, &[D(b' ' as u64), str_col]); - (*list.tail_as_mut()) = p1; + (*list.tail_as_mut(&space)) = p1; // "<[{str_lin} {str_col}].[{end_lin} {end_col}]>" let tape = T(stack, &[D(b'<' as u64), D(b'[' as u64), str_lin]); @@ -353,7 +368,7 @@ pub mod util { } _ => { let stack = &mut context.stack; - let tape = rip(stack, 3, 1, tag.as_atom())?; + let tape = rip(stack, 3, 1, tag.as_atom(), &space)?; T( stack, &[ @@ -379,7 +394,7 @@ pub mod util { dest = &mut (*new_memory).tail; } - list = cell.tail(); + list = cell.tail().noun(); } *dest = D(0); @@ -388,26 +403,26 @@ pub mod util { } } - pub fn smyt(stack: &mut NockStack, path: Noun) -> jets::Result { + pub fn smyt(stack: &mut NockStack, path: Noun, space: &NounSpace) -> jets::Result { let lash = D(tas!(b"/")); let zero = D(0); let sep = T(stack, &[lash, zero]); let trel = T(stack, &[sep, sep, zero]); - let tank = smyt_help(stack, path)?; + let tank = smyt_help(stack, path, space)?; Ok(T(stack, &[ROSE, trel, tank])) } - fn smyt_help(stack: &mut NockStack, path: Noun) -> jets::Result { + fn smyt_help(stack: &mut NockStack, path: Noun, space: &NounSpace) -> jets::Result { // XX: switch to using Cell:new_raw_mut if unsafe { path.raw_equals(&D(0)) } { return Ok(D(0)); } - let cell = path.as_cell()?; - let tail = smyt_help(stack, cell.tail())?; - let trip = rip(stack, 3, 1, cell.head().as_atom()?)?; + let cell = path.in_space(space).as_cell()?; + let tail = smyt_help(stack, cell.tail().noun(), space)?; + let trip = rip(stack, 3, 1, cell.head().as_atom()?.atom(), space)?; let head = T(stack, &[LEAF, trip]); Ok(T(stack, &[head, tail])) @@ -434,6 +449,7 @@ mod tests { } #[test] + #[cfg_attr(miri, ignore = "memfd_create unsupported in Miri")] fn test_mink_success() { let context = &mut init_context(); let stack = &mut context.stack; diff --git a/crates/nockvm/rust/nockvm/src/jets/parse.rs b/crates/nockvm/rust/nockvm/src/jets/parse.rs index e55679cdd..670cb8b1c 100644 --- a/crates/nockvm/rust/nockvm/src/jets/parse.rs +++ b/crates/nockvm/rust/nockvm/src/jets/parse.rs @@ -5,9 +5,9 @@ use either::{Left, Right}; use crate::interpreter::Context; use crate::jets::bits::util::met; use crate::jets::math::util::{gte_b, lte_b, lth_b}; -use crate::jets::util::{kick, slam, slot, BAIL_FAIL}; +use crate::jets::util::{kick, slam, slam_with_space, slot, BAIL_FAIL}; use crate::jets::Result; -use crate::noun::{Cell, Noun, D, T}; +use crate::noun::{Cell, CellHandle, Noun, D, T}; crate::gdb!(); @@ -15,13 +15,15 @@ crate::gdb!(); // Text conversion // pub fn jet_trip(context: &mut Context, subject: Noun) -> Result { - let sam = slot(subject, 6)?.as_atom()?; - let chars = met(3, sam); + let space = context.stack.fast_noun_space(); + let sam = slot(subject, 6, &space)?.as_atom()?; + let chars = met(3, sam, &space); if chars == 0 { return Ok(D(0)); }; - let bytes = &sam.as_ne_bytes()[0..chars]; + let sam_handle = sam.in_space(&space); + let bytes = &sam_handle.as_ne_bytes()[0..chars]; let mut result = D(0); let mut dest = &mut result as *mut Noun; @@ -46,11 +48,12 @@ pub fn jet_trip(context: &mut Context, subject: Noun) -> Result { // pub fn jet_last(_context: &mut Context, subject: Noun) -> Result { - let sam = slot(subject, 6)?; - let zyc = slot(sam, 2)?; - let naz = slot(sam, 3)?; + let space = _context.stack.fast_noun_space(); + let sam = slot(subject, 6, &space)?; + let zyc = slot(sam, 2, &space)?; + let naz = slot(sam, 3, &space)?; - util::last(zyc, naz) + util::last(zyc, naz, &space) } // @@ -58,127 +61,138 @@ pub fn jet_last(_context: &mut Context, subject: Noun) -> Result { // pub fn jet_bend(context: &mut Context, subject: Noun) -> Result { - let sam = slot(subject, 6)?; - let vex = slot(sam, 2)?.as_cell()?; - let sab = slot(sam, 3)?; - let van = slot(subject, 7)?; - let raq = slot(van, 6)?; + let space = context.stack.fast_noun_space(); + let sam = slot(subject, 6, &space)?; + let vex = slot(sam, 2, &space)?.in_space(&space).as_cell()?; + let sab = slot(sam, 3, &space)?; + let van = slot(subject, 7, &space)?; + let raq = slot(van, 6, &space)?; - let p_vex = vex.head(); - let q_vex = vex.tail(); + let p_vex = vex.head().noun(); + let q_vex = vex.tail().noun(); if unsafe { q_vex.raw_equals(&D(0)) } { - return Ok(vex.as_noun()); + return Ok(vex.cell().as_noun()); } - let uq_vex = q_vex.as_cell()?.tail().as_cell()?; - let puq_vex = uq_vex.head(); - let quq_vex = uq_vex.tail(); + let uq_vex = q_vex.in_space(&space).as_cell()?.tail().as_cell()?; + let puq_vex = uq_vex.head().noun(); + let quq_vex = uq_vex.tail().noun(); - let yit = slam(context, sab, quq_vex)?.as_cell()?; - let p_yit = yit.head(); - let q_yit = yit.tail(); + let yit = slam_with_space(context, sab, quq_vex, &space)? + .in_space(&space) + .as_cell()?; + let p_yit = yit.head().noun(); + let q_yit = yit.tail().noun(); - let yur = util::last(p_vex, p_yit)?; + let yur = util::last(p_vex, p_yit, &space)?; if unsafe { q_yit.raw_equals(&D(0)) } { Ok(T(&mut context.stack, &[yur, q_vex])) } else { - let uq_yit = q_yit.as_cell()?.tail().as_cell()?; - let puq_yit = uq_yit.head(); - let quq_yit = uq_yit.tail(); + let uq_yit = q_yit.in_space(&space).as_cell()?.tail().as_cell()?; + let puq_yit = uq_yit.head().noun(); + let quq_yit = uq_yit.tail().noun(); let arg = T(&mut context.stack, &[puq_vex, puq_yit]); - let vux = slam(context, raq, arg)?; + let vux = slam_with_space(context, raq, arg, &space)?; if unsafe { vux.raw_equals(&D(0)) } { Ok(T(&mut context.stack, &[yur, q_vex])) } else { - let q_vux = vux.as_cell()?.tail(); + let q_vux = vux.in_space(&space).as_cell()?.tail().noun(); Ok(T(&mut context.stack, &[yur, D(0), q_vux, quq_yit])) } } } pub fn jet_comp(context: &mut Context, subject: Noun) -> Result { - let sam = slot(subject, 6)?; - let vex = slot(sam, 2)?.as_cell()?; - let sab = slot(sam, 3)?; - let van = slot(subject, 7)?; - let raq = slot(van, 6)?; + let space = context.stack.fast_noun_space(); + let sam = slot(subject, 6, &space)?; + let vex = slot(sam, 2, &space)?.in_space(&space).as_cell()?; + let sab = slot(sam, 3, &space)?; + let van = slot(subject, 7, &space)?; + let raq = slot(van, 6, &space)?; - let p_vex = vex.head(); - let q_vex = vex.tail(); + let p_vex = vex.head().noun(); + let q_vex = vex.tail().noun(); if unsafe { q_vex.raw_equals(&D(0)) } { - return Ok(vex.as_noun()); + return Ok(vex.cell().as_noun()); } - let uq_vex = q_vex.as_cell()?.tail().as_cell()?; - let puq_vex = uq_vex.head(); - let quq_vex = uq_vex.tail(); + let uq_vex = q_vex.in_space(&space).as_cell()?.tail().as_cell()?; + let puq_vex = uq_vex.head().noun(); + let quq_vex = uq_vex.tail().noun(); - let yit = slam(context, sab, quq_vex)?.as_cell()?; - let p_yit = yit.head(); - let q_yit = yit.tail(); + let yit = slam_with_space(context, sab, quq_vex, &space)? + .in_space(&space) + .as_cell()?; + let p_yit = yit.head().noun(); + let q_yit = yit.tail().noun(); - let yur = util::last(p_vex, p_yit)?; + let yur = util::last(p_vex, p_yit, &space)?; if unsafe { q_yit.raw_equals(&D(0)) } { Ok(T(&mut context.stack, &[yur, D(0)])) } else { - let uq_yit = q_yit.as_cell()?.tail().as_cell()?; - let puq_yit = uq_yit.head(); - let quq_yit = uq_yit.tail(); + let uq_yit = q_yit.in_space(&space).as_cell()?.tail().as_cell()?; + let puq_yit = uq_yit.head().noun(); + let quq_yit = uq_yit.tail().noun(); let arg = T(&mut context.stack, &[puq_vex, puq_yit]); - let vux = slam(context, raq, arg)?; + let vux = slam_with_space(context, raq, arg, &space)?; Ok(T(&mut context.stack, &[yur, D(0), vux, quq_yit])) } } pub fn jet_glue(context: &mut Context, subject: Noun) -> Result { - let sam = slot(subject, 6)?; - let vex = slot(sam, 2)?.as_cell()?; - let sab = slot(sam, 3)?; - let van = slot(subject, 7)?; - let bus = slot(van, 6)?; + let space = context.stack.fast_noun_space(); + let sam = slot(subject, 6, &space)?; + let vex = slot(sam, 2, &space)?.in_space(&space).as_cell()?; + let sab = slot(sam, 3, &space)?; + let van = slot(subject, 7, &space)?; + let bus = slot(van, 6, &space)?; - let p_vex = vex.head(); - let q_vex = vex.tail(); + let p_vex = vex.head().noun(); + let q_vex = vex.tail().noun(); if unsafe { q_vex.raw_equals(&D(0)) } { - return Ok(vex.as_noun()); + return Ok(vex.cell().as_noun()); } - let uq_vex = q_vex.as_cell()?.tail().as_cell()?; - let puq_vex = uq_vex.head(); - let quq_vex = uq_vex.tail(); + let uq_vex = q_vex.in_space(&space).as_cell()?.tail().as_cell()?; + let puq_vex = uq_vex.head().noun(); + let quq_vex = uq_vex.tail().noun(); - let yit = slam(context, bus, quq_vex)?.as_cell()?; - let p_yit = yit.head(); - let q_yit = yit.tail(); + let yit = slam_with_space(context, bus, quq_vex, &space)? + .in_space(&space) + .as_cell()?; + let p_yit = yit.head().noun(); + let q_yit = yit.tail().noun(); - let yur = util::last(p_vex, p_yit)?; + let yur = util::last(p_vex, p_yit, &space)?; if unsafe { q_yit.raw_equals(&D(0)) } { Ok(T(&mut context.stack, &[yur, D(0)])) } else { - let uq_yit = q_yit.as_cell()?.tail().as_cell()?; - let quq_yit = uq_yit.tail(); + let uq_yit = q_yit.in_space(&space).as_cell()?.tail().as_cell()?; + let quq_yit = uq_yit.tail().noun(); - let wam = slam(context, sab, quq_yit)?.as_cell()?; - let p_wam = wam.head(); - let q_wam = wam.tail(); + let wam = slam_with_space(context, sab, quq_yit, &space)? + .in_space(&space) + .as_cell()?; + let p_wam = wam.head().noun(); + let q_wam = wam.tail().noun(); - let goy = util::last(yur, p_wam)?; + let goy = util::last(yur, p_wam, &space)?; if unsafe { q_wam.raw_equals(&D(0)) } { Ok(T(&mut context.stack, &[goy, D(0)])) } else { - let uq_wam = q_wam.as_cell()?.tail().as_cell()?; - let puq_wam = uq_wam.head(); - let quq_wam = uq_wam.tail(); + let uq_wam = q_wam.in_space(&space).as_cell()?.tail().as_cell()?; + let puq_wam = uq_wam.head().noun(); + let quq_wam = uq_wam.tail().noun(); let puq_arg = T(&mut context.stack, &[puq_vex, puq_wam]); Ok(T(&mut context.stack, &[goy, D(0x0), puq_arg, quq_wam])) @@ -187,55 +201,61 @@ pub fn jet_glue(context: &mut Context, subject: Noun) -> Result { } pub fn jet_pfix(context: &mut Context, subject: Noun) -> Result { - let sam = slot(subject, 6)?; - let vex = slot(sam, 2)?.as_cell()?; - let sab = slot(sam, 3)?; + let space = context.stack.fast_noun_space(); + let sam = slot(subject, 6, &space)?; + let vex = slot(sam, 2, &space)?.in_space(&space).as_cell()?; + let sab = slot(sam, 3, &space)?; - let p_vex = vex.head(); - let q_vex = vex.tail(); + let p_vex = vex.head().noun(); + let q_vex = vex.tail().noun(); if unsafe { q_vex.raw_equals(&D(0)) } { - return Ok(vex.as_noun()); + return Ok(vex.cell().as_noun()); } - let uq_vex = q_vex.as_cell()?.tail().as_cell()?; - let quq_vex = uq_vex.tail(); + let uq_vex = q_vex.in_space(&space).as_cell()?.tail().as_cell()?; + let quq_vex = uq_vex.tail().noun(); - let yit = slam(context, sab, quq_vex)?.as_cell()?; + let yit = slam_with_space(context, sab, quq_vex, &space)? + .in_space(&space) + .as_cell()?; - let p_yit = yit.head(); - let q_yit = yit.tail(); + let p_yit = yit.head().noun(); + let q_yit = yit.tail().noun(); // XX: Why don't we just return yit? When would p_vex ever be the later of the two? - let arg = util::last(p_vex, p_yit)?; + let arg = util::last(p_vex, p_yit, &space)?; Ok(T(&mut context.stack, &[arg, q_yit])) } pub fn jet_plug(context: &mut Context, subject: Noun) -> Result { - let vex = slot(subject, 12)?.as_cell()?; - let sab = slot(subject, 13)?; - let p_vex = vex.head(); - let q_vex = vex.tail(); + let space = context.stack.fast_noun_space(); + let vex = slot(subject, 12, &space)?.in_space(&space).as_cell()?; + let sab = slot(subject, 13, &space)?; + let p_vex = vex.head().noun(); + let q_vex = vex.tail().noun(); if unsafe { q_vex.raw_equals(&D(0)) } { - Ok(vex.as_noun()) + Ok(vex.cell().as_noun()) } else { - let uq_vex = q_vex.as_cell()?.tail().as_cell()?; - let puq_vex = uq_vex.head(); - let quq_vex = uq_vex.tail(); + let uq_vex = q_vex.in_space(&space).as_cell()?.tail().as_cell()?; + let puq_vex = uq_vex.head().noun(); + let quq_vex = uq_vex.tail().noun(); - let yit = slam(context, sab, quq_vex)?.as_cell()?; - let p_yit = yit.head(); - let q_yit = yit.tail(); + let yit = slam_with_space(context, sab, quq_vex, &space)? + .in_space(&space) + .as_cell()?; + let p_yit = yit.head().noun(); + let q_yit = yit.tail().noun(); - let yur = util::last(p_vex, p_yit)?; + let yur = util::last(p_vex, p_yit, &space)?; if unsafe { q_yit.raw_equals(&D(0)) } { Ok(T(&mut context.stack, &[yur, D(0)])) } else { - let uq_yit = q_yit.as_cell()?.tail().as_cell()?; - let puq_yit = uq_yit.head(); - let quq_yit = uq_yit.tail(); + let uq_yit = q_yit.in_space(&space).as_cell()?.tail().as_cell()?; + let puq_yit = uq_yit.head().noun(); + let quq_yit = uq_yit.tail().noun(); let inner = T(&mut context.stack, &[puq_vex, puq_yit]); Ok(T(&mut context.stack, &[yur, D(0), inner, quq_yit])) @@ -244,48 +264,52 @@ pub fn jet_plug(context: &mut Context, subject: Noun) -> Result { } pub fn jet_pose(context: &mut Context, subject: Noun) -> Result { - let vex = slot(subject, 12)?.as_cell()?; - let sab = slot(subject, 13)?; + let space = context.stack.fast_noun_space(); + let vex = slot(subject, 12, &space)?.in_space(&space).as_cell()?; + let sab = slot(subject, 13, &space)?; - let p_vex = vex.head(); - let q_vex = vex.tail(); + let p_vex = vex.head().noun(); + let q_vex = vex.tail().noun(); if unsafe { !q_vex.raw_equals(&D(0)) } { - return Ok(vex.as_noun()); + return Ok(vex.cell().as_noun()); } - let roq = kick(context, sab, D(2))?.as_cell()?; - let yur = util::last(p_vex, roq.head())?; - Ok(T(&mut context.stack, &[yur, roq.tail()])) + let roq = kick(context, sab, D(2))?.in_space(&space).as_cell()?; + let yur = util::last(p_vex, roq.head().noun(), &space)?; + Ok(T(&mut context.stack, &[yur, roq.tail().noun()])) } pub fn jet_sfix(context: &mut Context, subject: Noun) -> Result { - let sam = slot(subject, 6)?; - let vex = slot(sam, 2)?.as_cell()?; - let sab = slot(sam, 3)?; + let space = context.stack.fast_noun_space(); + let sam = slot(subject, 6, &space)?; + let vex = slot(sam, 2, &space)?.in_space(&space).as_cell()?; + let sab = slot(sam, 3, &space)?; - let p_vex = vex.head(); - let q_vex = vex.tail(); + let p_vex = vex.head().noun(); + let q_vex = vex.tail().noun(); if unsafe { q_vex.raw_equals(&D(0)) } { - return Ok(vex.as_noun()); + return Ok(vex.cell().as_noun()); } - let uq_vex = q_vex.as_cell()?.tail().as_cell()?; - let puq_vex = uq_vex.head(); - let quq_vex = uq_vex.tail(); + let uq_vex = q_vex.in_space(&space).as_cell()?.tail().as_cell()?; + let puq_vex = uq_vex.head().noun(); + let quq_vex = uq_vex.tail().noun(); - let yit = slam(context, sab, quq_vex)?.as_cell()?; + let yit = slam_with_space(context, sab, quq_vex, &space)? + .in_space(&space) + .as_cell()?; - let p_yit = yit.head(); - let q_yit = yit.tail(); - let yur = util::last(p_vex, p_yit)?; + let p_yit = yit.head().noun(); + let q_yit = yit.tail().noun(); + let yur = util::last(p_vex, p_yit, &space)?; if unsafe { q_yit.raw_equals(&D(0)) } { Ok(T(&mut context.stack, &[yur, D(0)])) } else { - let uq_yit = q_yit.as_cell()?.tail().as_cell()?; - let quq_yit = uq_yit.tail(); + let uq_yit = q_yit.in_space(&space).as_cell()?.tail().as_cell()?; + let quq_yit = uq_yit.tail().noun(); Ok(T(&mut context.stack, &[yur, D(0), puq_vex, quq_yit])) } @@ -296,144 +320,167 @@ pub fn jet_sfix(context: &mut Context, subject: Noun) -> Result { // pub fn jet_cold(context: &mut Context, subject: Noun) -> Result { - let tub = slot(subject, 6)?; - let van = slot(subject, 7)?; - let cus = slot(van, 12)?; - let sef = slot(van, 13)?; - - let vex = slam(context, sef, tub)?.as_cell()?; - let p_vex = vex.head(); - let q_vex = vex.tail(); + let space = context.stack.fast_noun_space(); + let tub = slot(subject, 6, &space)?; + let van = slot(subject, 7, &space)?; + let cus = slot(van, 12, &space)?; + let sef = slot(van, 13, &space)?; + + let vex = slam_with_space(context, sef, tub, &space)? + .in_space(&space) + .as_cell()?; + let p_vex = vex.head().noun(); + let q_vex = vex.tail().noun(); if unsafe { q_vex.raw_equals(&D(0)) } { - Ok(vex.as_noun()) + Ok(vex.cell().as_noun()) } else { - let quq_vex = q_vex.as_cell()?.tail().as_cell()?.tail(); + let quq_vex = q_vex + .in_space(&space) + .as_cell()? + .tail() + .as_cell()? + .tail() + .noun(); Ok(T(&mut context.stack, &[p_vex, D(0), cus, quq_vex])) } } pub fn jet_cook(context: &mut Context, subject: Noun) -> Result { - let tub = slot(subject, 6)?; - let van = slot(subject, 7)?; - let poq = slot(van, 12)?; - let sef = slot(van, 13)?; - - let vex = slam(context, sef, tub)?.as_cell()?; - let p_vex = vex.head(); - let q_vex = vex.tail(); + let space = context.stack.fast_noun_space(); + let tub = slot(subject, 6, &space)?; + let van = slot(subject, 7, &space)?; + let poq = slot(van, 12, &space)?; + let sef = slot(van, 13, &space)?; + + let vex = slam_with_space(context, sef, tub, &space)? + .in_space(&space) + .as_cell()?; + let p_vex = vex.head().noun(); + let q_vex = vex.tail().noun(); if unsafe { q_vex.raw_equals(&D(0)) } { - Ok(vex.as_noun()) + Ok(vex.cell().as_noun()) } else { - let uq_vex = q_vex.as_cell()?.tail().as_cell()?; - let puq_vex = uq_vex.head(); - let quq_vex = uq_vex.tail(); + let uq_vex = q_vex.in_space(&space).as_cell()?.tail().as_cell()?; + let puq_vex = uq_vex.head().noun(); + let quq_vex = uq_vex.tail().noun(); - let wag = slam(context, poq, puq_vex)?; + let wag = slam_with_space(context, poq, puq_vex, &space)?; Ok(T(&mut context.stack, &[p_vex, D(0), wag, quq_vex])) } } pub fn jet_easy(context: &mut Context, subject: Noun) -> Result { - let tub = slot(subject, 6)?; - let van = slot(subject, 7)?; - let huf = slot(van, 6)?; + let space = context.stack.fast_noun_space(); + let tub = slot(subject, 6, &space)?; + let van = slot(subject, 7, &space)?; + let huf = slot(van, 6, &space)?; Ok(T( &mut context.stack, - &[tub.as_cell()?.head(), D(0), huf, tub], + &[tub.in_space(&space).as_cell()?.head().noun(), D(0), huf, tub], )) } pub fn jet_here(context: &mut Context, subject: Noun) -> Result { - let tub = slot(subject, 6)?; - let van = slot(subject, 7)?; - let hez = slot(van, 12)?; - let sef = slot(van, 13)?; + let space = context.stack.fast_noun_space(); + let tub = slot(subject, 6, &space)?; + let van = slot(subject, 7, &space)?; + let hez = slot(van, 12, &space)?; + let sef = slot(van, 13, &space)?; - let p_tub = tub.as_cell()?.head(); + let p_tub = tub.in_space(&space).as_cell()?.head().noun(); - let vex = slam(context, sef, tub)?.as_cell()?; - let p_vex = vex.head(); - let q_vex = vex.tail(); + let vex = slam_with_space(context, sef, tub, &space)? + .in_space(&space) + .as_cell()?; + let p_vex = vex.head().noun(); + let q_vex = vex.tail().noun(); // XX fixes Vere's jet mismatch with Hoon 139. if unsafe { q_vex.raw_equals(&D(0)) } { - return Ok(vex.as_noun()); + return Ok(vex.cell().as_noun()); } - let uq_vex = q_vex.as_cell()?.tail().as_cell()?; - let puq_vex = uq_vex.head(); - let quq_vex = uq_vex.tail(); - let pquq_vex = quq_vex.as_cell()?.head(); + let uq_vex = q_vex.in_space(&space).as_cell()?.tail().as_cell()?; + let puq_vex = uq_vex.head().noun(); + let quq_vex = uq_vex.tail().noun(); + let pquq_vex = quq_vex.in_space(&space).as_cell()?.head().noun(); let inner_gud = T(&mut context.stack, &[p_tub, pquq_vex]); let gud = T(&mut context.stack, &[inner_gud, puq_vex]); - let wag = slam(context, hez, gud)?; + let wag = slam_with_space(context, hez, gud, &space)?; Ok(T(&mut context.stack, &[p_vex, D(0), wag, quq_vex])) } pub fn jet_just(context: &mut Context, subject: Noun) -> Result { - let tub = slot(subject, 6)?; - let van = slot(subject, 7)?; - let daf = slot(van, 6)?; - - let p_tub = tub.as_cell()?.head(); - let q_tub = tub.as_cell()?.tail(); - - if unsafe { q_tub.raw_equals(&D(0)) || !daf.raw_equals(&q_tub.as_cell()?.head()) } { + let space = context.stack.fast_noun_space(); + let tub = slot(subject, 6, &space)?; + let van = slot(subject, 7, &space)?; + let daf = slot(van, 6, &space)?; + + let tub_cell = tub.in_space(&space).as_cell()?; + let p_tub = tub_cell.head().noun(); + let q_tub = tub_cell.tail().noun(); + + if unsafe { + q_tub.raw_equals(&D(0)) || !daf.raw_equals(&q_tub.in_space(&space).as_cell()?.head().noun()) + } { util::fail(context, p_tub) } else { - util::next(context, tub) + util::next(context, tub, &space) } } pub fn jet_mask(context: &mut Context, subject: Noun) -> Result { - let tub = slot(subject, 6)?; - let van = slot(subject, 7)?; - let mut bud = slot(van, 6)?; + let space = context.stack.fast_noun_space(); + let tub = slot(subject, 6, &space)?; + let van = slot(subject, 7, &space)?; + let mut bud = slot(van, 6, &space)?; - let p_tub = tub.as_cell()?.head(); - let q_tub = tub.as_cell()?.tail(); + let tub_cell = tub.in_space(&space).as_cell()?; + let p_tub = tub_cell.head().noun(); + let q_tub = tub_cell.tail().noun(); if unsafe { q_tub.raw_equals(&D(0)) } { return util::fail(context, p_tub); } - let iq_tub = q_tub.as_cell()?.head(); + let iq_tub = q_tub.in_space(&space).as_cell()?.head().noun(); while unsafe { !bud.raw_equals(&D(0)) } { - let cell = bud.as_cell()?; - if unsafe { cell.head().raw_equals(&iq_tub) } { - return util::next(context, tub); + let cell = bud.in_space(&space).as_cell()?; + if unsafe { cell.head().noun().raw_equals(&iq_tub) } { + return util::next(context, tub, &space); } - bud = cell.tail(); + bud = cell.tail().noun(); } util::fail(context, p_tub) } pub fn jet_shim(context: &mut Context, subject: Noun) -> Result { - let tub = slot(subject, 6)?.as_cell()?; - let van = slot(subject, 7)?; - let zep = slot(van, 6)?.as_cell()?; + let space = context.stack.fast_noun_space(); + let tub = slot(subject, 6, &space)?.in_space(&space).as_cell()?; + let van = slot(subject, 7, &space)?; + let zep = slot(van, 6, &space)?.in_space(&space).as_cell()?; - let p_tub = tub.head(); - let q_tub = tub.tail(); + let p_tub = tub.head().noun(); + let q_tub = tub.tail().noun(); if unsafe { q_tub.raw_equals(&D(0)) } { util::fail(context, p_tub) } else { - let p_zep = zep.head(); - let q_zep = zep.tail(); - let iq_tub = q_tub.as_cell()?.head(); + let p_zep = zep.head().noun(); + let q_zep = zep.tail().noun(); + let iq_tub = q_tub.in_space(&space).as_cell()?.head().noun(); if let (Some(p_zep_d), Some(q_zep_d), Some(iq_tub_d)) = (p_zep.direct(), q_zep.direct(), iq_tub.direct()) { if (iq_tub_d.data() >= p_zep_d.data()) && (iq_tub_d.data() <= q_zep_d.data()) { - util::next(context, tub.as_noun()) + util::next(context, tub.cell().as_noun(), &space) } else { util::fail(context, p_tub) } @@ -444,21 +491,24 @@ pub fn jet_shim(context: &mut Context, subject: Noun) -> Result { } pub fn jet_stag(context: &mut Context, subject: Noun) -> Result { - let tub = slot(subject, 6)?; - let van = slot(subject, 7)?; - let gob = slot(van, 12)?; - let sef = slot(van, 13)?; - - let vex = slam(context, sef, tub)?.as_cell()?; - let p_vex = vex.head(); - let q_vex = vex.tail(); + let space = context.stack.fast_noun_space(); + let tub = slot(subject, 6, &space)?; + let van = slot(subject, 7, &space)?; + let gob = slot(van, 12, &space)?; + let sef = slot(van, 13, &space)?; + + let vex = slam_with_space(context, sef, tub, &space)? + .in_space(&space) + .as_cell()?; + let p_vex = vex.head().noun(); + let q_vex = vex.tail().noun(); if unsafe { q_vex.raw_equals(&D(0)) } { - Ok(vex.as_noun()) + Ok(vex.cell().as_noun()) } else { - let uq_vex = q_vex.as_cell()?.tail().as_cell()?; - let puq_vex = uq_vex.head(); - let quq_vex = uq_vex.tail(); + let uq_vex = q_vex.in_space(&space).as_cell()?.tail().as_cell()?; + let puq_vex = uq_vex.head().noun(); + let quq_vex = uq_vex.tail().noun(); let wag = T(&mut context.stack, &[gob, puq_vex]); Ok(T(&mut context.stack, &[p_vex, D(0), wag, quq_vex])) @@ -466,17 +516,18 @@ pub fn jet_stag(context: &mut Context, subject: Noun) -> Result { } pub fn jet_stew(context: &mut Context, subject: Noun) -> Result { - let tub = slot(subject, 6)?.as_cell()?; - let con = slot(subject, 7)?; - let mut hel = slot(con, 2)?; + let space = context.stack.fast_noun_space(); + let tub = slot(subject, 6, &space)?.in_space(&space).as_cell()?; + let con = slot(subject, 7, &space)?; + let mut hel = slot(con, 2, &space)?; - let p_tub = tub.head(); - let q_tub = tub.tail(); + let p_tub = tub.head().noun(); + let q_tub = tub.tail().noun(); if unsafe { q_tub.raw_equals(&D(0)) } { return util::fail(context, p_tub); } - let iq_tub = q_tub.as_cell()?.head().as_atom()?; + let iq_tub = q_tub.in_space(&space).as_cell()?.head().as_atom()?.atom(); if !iq_tub.is_direct() { // Character cannot be encoded using 8 bytes = computibilty error return Err(BAIL_FAIL); @@ -486,11 +537,11 @@ pub fn jet_stew(context: &mut Context, subject: Noun) -> Result { if unsafe { hel.raw_equals(&D(0)) } { return util::fail(context, p_tub); } else { - let n_hel = slot(hel, 2)?.as_cell()?; - let l_hel = slot(hel, 6)?; - let r_hel = slot(hel, 7)?; - let pn_hel = n_hel.head(); - let qn_hel = n_hel.tail(); + let n_hel = slot(hel, 2, &space)?.in_space(&space).as_cell()?; + let l_hel = slot(hel, 6, &space)?; + let r_hel = slot(hel, 7, &space)?; + let pn_hel = n_hel.head().noun(); + let qn_hel = n_hel.tail().noun(); let bit = match pn_hel.as_either_atom_cell() { Left(atom) => match atom.as_either() { @@ -501,13 +552,14 @@ pub fn jet_stew(context: &mut Context, subject: Noun) -> Result { } }, Right(cell) => { - let hpn_hel = cell.head().as_atom()?; - let tpn_hel = cell.tail().as_atom()?; + let cell_handle = CellHandle::new(cell, &space); + let hpn_hel = cell_handle.head().as_atom()?.atom(); + let tpn_hel = cell_handle.tail().as_atom()?.atom(); match (hpn_hel.as_either(), tpn_hel.as_either()) { (Left(_), Left(_)) => { - gte_b(&mut context.stack, iq_tub, hpn_hel) - && lte_b(&mut context.stack, iq_tub, tpn_hel) + gte_b(&mut context.stack, iq_tub, hpn_hel, &space) + && lte_b(&mut context.stack, iq_tub, tpn_hel, &space) } _ => { // XX: Fixes jet mismatch in Vere @@ -519,14 +571,14 @@ pub fn jet_stew(context: &mut Context, subject: Noun) -> Result { }; if bit { - return slam(context, qn_hel, tub.as_noun()); + return slam(context, qn_hel, tub.cell().as_noun()); } else { let wor = match pn_hel.as_either_atom_cell() { Left(atom) => atom, - Right(cell) => cell.head().as_atom()?, + Right(cell) => CellHandle::new(cell, &space).head().as_atom()?.atom(), }; - if lth_b(&mut context.stack, iq_tub, wor) { + if lth_b(&mut context.stack, iq_tub, wor, &space) { hel = l_hel; } else { hel = r_hel; @@ -544,13 +596,14 @@ struct StirPair { } pub fn jet_stir(context: &mut Context, subject: Noun) -> Result { + let space = context.stack.fast_noun_space(); unsafe { context.with_stack_frame(0, |context| { - let mut tub = slot(subject, 6)?; - let van = slot(subject, 7)?; - let rud = slot(van, 12)?; - let raq = slot(van, 26)?; - let fel = slot(van, 27)?; + let mut tub = slot(subject, 6, &space)?; + let van = slot(subject, 7, &space)?; + let rud = slot(van, 12, &space)?; + let raq = slot(van, 26, &space)?; + let fel = slot(van, 27, &space)?; // initial accumulator (deconstructed) let mut p_wag: Noun; @@ -559,12 +612,14 @@ pub fn jet_stir(context: &mut Context, subject: Noun) -> Result { // push incremental, succesful [fel] parse results onto stack { - let vex = slam(context, fel, tub)?.as_cell()?; - let mut p_vex = vex.head(); - let mut q_vex = vex.tail(); + let vex = slam_with_space(context, fel, tub, &space)? + .in_space(&space) + .as_cell()?; + let mut p_vex = vex.head().noun(); + let mut q_vex = vex.tail().noun(); while !q_vex.raw_equals(&D(0)) { - let puq_vex = slot(q_vex, 6)?; - let quq_vex = slot(q_vex, 7)?; + let puq_vex = slot(q_vex, 6, &space)?; + let quq_vex = slot(q_vex, 7, &space)?; *(context.stack.push::()) = StirPair { har: p_vex, @@ -573,9 +628,11 @@ pub fn jet_stir(context: &mut Context, subject: Noun) -> Result { tub = quq_vex; - let vex = slam(context, fel, tub)?.as_cell()?; - p_vex = vex.head(); - q_vex = vex.tail(); + let vex = slam_with_space(context, fel, tub, &space)? + .in_space(&space) + .as_cell()?; + p_vex = vex.head().noun(); + q_vex = vex.tail().noun(); } p_wag = p_vex; @@ -586,9 +643,9 @@ pub fn jet_stir(context: &mut Context, subject: Noun) -> Result { // unwind the stack, folding parse results into [wag] by way of [raq] while !context.stack.stack_is_empty() { let par_u = *(context.stack.top::()); - p_wag = util::last(par_u.har, p_wag)?; + p_wag = util::last(par_u.har, p_wag, &space)?; let sam = T(&mut context.stack, &[par_u.res, puq_wag]); - puq_wag = slam(context, raq, sam)?; + puq_wag = slam_with_space(context, raq, sam, &space)?; context.stack.pop::(); } @@ -603,16 +660,16 @@ pub mod util { use crate::interpreter::{inc, Context}; use crate::jets::Result; - use crate::noun::{Noun, D, T}; + use crate::noun::{Noun, NounSpace, D, T}; - pub fn last(zyc: Noun, naz: Noun) -> Result { - let zyl = zyc.as_cell()?; - let nal = naz.as_cell()?; + pub fn last(zyc: Noun, naz: Noun, space: &NounSpace) -> Result { + let zyl = zyc.in_space(space).as_cell()?; + let nal = naz.in_space(space).as_cell()?; - let p_zyc = zyl.head().as_direct()?.data(); - let q_zyc = zyl.tail().as_direct()?.data(); - let p_naz = nal.head().as_direct()?.data(); - let q_naz = nal.tail().as_direct()?.data(); + let p_zyc = zyl.head().noun().as_direct()?.data(); + let q_zyc = zyl.tail().noun().as_direct()?.data(); + let p_naz = nal.head().noun().as_direct()?.data(); + let q_naz = nal.tail().noun().as_direct()?.data(); match p_zyc.cmp(&p_naz) { Ordering::Equal => { @@ -628,25 +685,28 @@ pub mod util { } // Passing Noun and doing Cell check inside next is best to keep jet semantics in sync w/ Hoon. - pub fn next(context: &mut Context, tub: Noun) -> Result { - let p_tub = tub.as_cell()?.head(); - let q_tub = tub.as_cell()?.tail(); + pub fn next(context: &mut Context, tub: Noun, space: &NounSpace) -> Result { + let tub_cell = tub.in_space(space).as_cell()?; + let p_tub = tub_cell.head().noun(); + let q_tub = tub_cell.tail().noun(); if unsafe { q_tub.raw_equals(&D(0)) } { return fail(context, p_tub); } - let iq_tub = q_tub.as_cell()?.head(); - let tq_tub = q_tub.as_cell()?.tail(); + let q_tub_cell = q_tub.in_space(space).as_cell()?; + let iq_tub = q_tub_cell.head().noun(); + let tq_tub = q_tub_cell.tail().noun(); - let zac = lust(context, iq_tub, p_tub)?; + let zac = lust(context, iq_tub, p_tub, space)?; Ok(T(&mut context.stack, &[zac, D(0), iq_tub, zac, tq_tub])) } // Passing Noun and doing Cell check inside next is best to keep jet semantics in sync w/ Hoon. - pub fn lust(context: &mut Context, weq: Noun, naz: Noun) -> Result { - let p_naz = naz.as_cell()?.head().as_atom()?; - let q_naz = naz.as_cell()?.tail().as_atom()?; + pub fn lust(context: &mut Context, weq: Noun, naz: Noun, space: &NounSpace) -> Result { + let naz_cell = naz.in_space(space).as_cell()?; + let p_naz = naz_cell.head().as_atom()?.atom(); + let q_naz = naz_cell.tail().as_atom()?.atom(); if unsafe { weq.raw_equals(&D(10)) } { let arg = inc(&mut context.stack, p_naz).as_noun(); diff --git a/crates/nockvm/rust/nockvm/src/jets/serial.rs b/crates/nockvm/rust/nockvm/src/jets/serial.rs index 9bdb4d76f..c0e89b6b3 100644 --- a/crates/nockvm/rust/nockvm/src/jets/serial.rs +++ b/crates/nockvm/rust/nockvm/src/jets/serial.rs @@ -2,16 +2,29 @@ use crate::interpreter::Context; use crate::jets::util::*; use crate::jets::Result; use crate::noun::Noun; -use crate::serialization::{cue, jam}; +use crate::serialization::{cue, cue_into_offset, jam}; crate::gdb!(); pub fn jet_cue(context: &mut Context, subject: Noun) -> Result { - Ok(cue(&mut context.stack, slot(subject, 6)?.as_atom()?)?) + let space = context.stack.noun_space(); + Ok(cue( + &mut context.stack, + slot(subject, 6, &space)?.as_atom()?, + )?) +} + +pub fn jet_cue_into_offset(context: &mut Context, subject: Noun) -> Result { + let space = context.stack.noun_space(); + Ok(cue_into_offset( + &mut context.stack, + slot(subject, 6, &space)?.as_atom()?, + )?) } pub fn jet_jam(context: &mut Context, subject: Noun) -> Result { - Ok(jam(&mut context.stack, slot(subject, 6)?).as_noun()) + let space = context.stack.noun_space(); + Ok(jam(&mut context.stack, slot(subject, 6, &space)?).as_noun()) } #[cfg(test)] @@ -21,6 +34,7 @@ mod tests { use crate::noun::{D, T}; #[test] + #[cfg_attr(miri, ignore = "memfd_create unsupported in Miri")] fn test_jam() { let c = &mut init_context(); @@ -33,6 +47,7 @@ mod tests { } #[test] + #[cfg_attr(miri, ignore = "memfd_create unsupported in Miri")] fn test_cue() { let c = &mut init_context(); @@ -43,4 +58,17 @@ mod tests { let res = T(&mut c.stack, &[D(0x1), D(0x2), D(0x3), D(0x0)]); assert_jet(c, jet_cue, D(0x2d0c871), res); } + + #[test] + #[cfg_attr(miri, ignore = "memfd_create unsupported in Miri")] + fn test_cue_into_offset() { + let c = &mut init_context(); + + assert_jet(c, jet_cue_into_offset, D(0x2), D(0x0)); + assert_jet(c, jet_cue_into_offset, D(0xc), D(0x1)); + let res = T(&mut c.stack, &[D(0x0), D(0x0)]); + assert_jet(c, jet_cue_into_offset, D(0x29), res); + let res = T(&mut c.stack, &[D(0x1), D(0x2), D(0x3), D(0x0)]); + assert_jet(c, jet_cue_into_offset, D(0x2d0c871), res); + } } diff --git a/crates/nockvm/rust/nockvm/src/jets/set.rs b/crates/nockvm/rust/nockvm/src/jets/set.rs index 6d4c3c3e0..95f10dbbd 100644 --- a/crates/nockvm/rust/nockvm/src/jets/set.rs +++ b/crates/nockvm/rust/nockvm/src/jets/set.rs @@ -6,7 +6,7 @@ use crate::jets::util::slot; use crate::jets::{JetErr, Result}; use crate::mem::NockStack; //use crate::mug::mug; -use crate::noun::{Noun, Slots, D, NO, T, YES}; +use crate::noun::{Noun, NounSpace, Slots, D, NO, T, YES}; type JetResult = std::result::Result; @@ -15,10 +15,10 @@ fn is_yes(noun: Noun) -> bool { unsafe { noun.raw_equals(&YES) } } -fn decompose(node: Noun) -> JetResult<(Noun, Noun, Noun)> { - let cell = node.as_cell()?; +fn decompose(node: Noun, space: &NounSpace) -> JetResult<(Noun, Noun, Noun)> { + let cell = node.in_space(space).as_cell()?; let tail = cell.tail().as_cell()?; - Ok((cell.head(), tail.head(), tail.tail())) + Ok((cell.head().noun(), tail.head().noun(), tail.tail().noun())) } fn make_node(stack: &mut NockStack, value: Noun, left: Noun, right: Noun) -> Noun { @@ -28,20 +28,21 @@ fn make_node(stack: &mut NockStack, value: Noun, left: Noun, right: Noun) -> Nou // TODO: fix this jet. identical elements are not being deduplicated pub fn jet_put(context: &mut Context, subject: Noun) -> Result { - let elem = slot(subject, 6)?; - let parent = match slot(subject, 7) { + let space = context.stack.noun_space(); + let elem = slot(subject, 6, &space)?; + let parent = match slot(subject, 7, &space) { Ok(parent) => parent, Err(_) => return Err(JetErr::Punt), }; - let set = match slot(parent, 6) { + let set = match slot(parent, 6, &space) { Ok(set) => set, Err(_) => return Err(JetErr::Punt), }; - put_iter(&mut context.stack, set, elem) + put_iter(&mut context.stack, set, elem, &space) } -fn put_iter(stack: &mut NockStack, root: Noun, elem: Noun) -> JetResult { +fn put_iter(stack: &mut NockStack, root: Noun, elem: Noun, space: &NounSpace) -> JetResult { if unsafe { root.raw_equals(&D(0)) } { return Ok(make_node(stack, elem, D(0), D(0))); } @@ -54,13 +55,13 @@ fn put_iter(stack: &mut NockStack, root: Noun, elem: Noun) -> JetResult { break; } - let (value, left, right) = decompose(current)?; + let (value, left, right) = decompose(current, space)?; if unsafe { elem.raw_equals(&value) } { return Ok(root); } - let go_left = is_yes(gor(stack, elem, value)); + let go_left = is_yes(gor(stack, elem, value, space)); path.push((current, go_left)); current = if go_left { left } else { right }; } @@ -68,17 +69,17 @@ fn put_iter(stack: &mut NockStack, root: Noun, elem: Noun) -> JetResult { let mut new_subtree = make_node(stack, elem, D(0), D(0)); while let Some((node, went_left)) = path.pop() { - let (value, left, right) = decompose(node)?; - let (c_val, c_left, c_right) = decompose(new_subtree)?; + let (value, left, right) = decompose(node, space)?; + let (c_val, c_left, c_right) = decompose(new_subtree, space)?; new_subtree = if went_left { - if is_yes(mor(stack, value, c_val)) { + if is_yes(mor(stack, value, c_val, space)) { make_node(stack, value, new_subtree, right) } else { let new_a = make_node(stack, value, c_right, right); make_node(stack, c_val, c_left, new_a) } - } else if is_yes(mor(stack, value, c_val)) { + } else if is_yes(mor(stack, value, c_val, space)) { make_node(stack, value, left, new_subtree) } else { let new_a = make_node(stack, value, left, c_left); @@ -90,23 +91,28 @@ fn put_iter(stack: &mut NockStack, root: Noun, elem: Noun) -> JetResult { } #[inline(always)] -fn ord_cmp(stack: &mut NockStack, a: Noun, b: Noun) -> Ordering { +fn ord_cmp(stack: &mut NockStack, a: Noun, b: Noun, space: &NounSpace) -> Ordering { unsafe { if a.raw_equals(&b) { return Ordering::Equal; } } - if is_yes(gor(stack, b, a)) { + if is_yes(gor(stack, b, a, space)) { return Ordering::Less; } else { return Ordering::Greater; } } -fn has_loop(stack: &mut NockStack, mut tree: Noun, elem: Noun) -> JetResult { +fn has_loop( + stack: &mut NockStack, + mut tree: Noun, + elem: Noun, + space: &NounSpace, +) -> JetResult { while unsafe { !tree.raw_equals(&D(0)) } { - let (val, left, right) = decompose(tree)?; - match ord_cmp(stack, elem, val) { + let (val, left, right) = decompose(tree, space)?; + match ord_cmp(stack, elem, val, space) { Ordering::Equal => return Ok(true), Ordering::Less => tree = left, Ordering::Greater => tree = right, @@ -117,16 +123,17 @@ fn has_loop(stack: &mut NockStack, mut tree: Noun, elem: Noun) -> JetResult Result { - let elem = subject.slot(6)?; - let parent = match subject.slot(7) { + let space = context.stack.noun_space(); + let elem = subject.slot(6, &space)?; + let parent = match subject.slot(7, &space) { Ok(parent) => parent, Err(_) => return Err(JetErr::Punt), }; - let set = match parent.slot(6) { + let set = match parent.slot(6, &space) { Ok(set) => set, Err(_) => return Err(JetErr::Punt), }; - let present = has_loop(&mut context.stack, set, elem)?; + let present = has_loop(&mut context.stack, set, elem, &space)?; Ok(if present { YES } else { NO }) } @@ -160,15 +167,17 @@ mod tests { // } fn contains(stack: &mut NockStack, mut tree: Noun, elem: Noun) -> bool { + let space = stack.noun_space(); loop { if unsafe { tree.raw_equals(&D(0)) } { return false; } - let (value, left, right) = decompose(tree).expect("tree should be valid set node"); + let (value, left, right) = + decompose(tree, &space).expect("tree should be valid set node"); if unsafe { value.raw_equals(&elem) } { return true; } - tree = if is_yes(gor(stack, elem, value)) { + tree = if is_yes(gor(stack, elem, value, &space)) { left } else { right @@ -177,6 +186,7 @@ mod tests { } #[test] + #[cfg_attr(miri, ignore = "memfd_create unsupported in Miri")] fn insert_into_empty_set() { let context = &mut init_context(); let elem = D(1); @@ -187,6 +197,7 @@ mod tests { } #[test] + #[cfg_attr(miri, ignore = "memfd_create unsupported in Miri")] fn insert_duplicate_retains_tree() { let context = &mut init_context(); let set = node(&mut context.stack, D(5), D(0), D(0)); @@ -196,6 +207,7 @@ mod tests { } #[test] + #[cfg_attr(miri, ignore = "memfd_create unsupported in Miri")] fn insert_distinct_elements() { let context = &mut init_context(); let base = node(&mut context.stack, D(9), D(0), D(0)); @@ -214,21 +226,22 @@ mod tests { } fn put_recursive(stack: &mut NockStack, tree: Noun, elem: Noun) -> JetResult { + let space = stack.noun_space(); if unsafe { tree.raw_equals(&D(0)) } { return Ok(make_node(stack, elem, D(0), D(0))); } - let (value, left, right) = decompose(tree)?; + let (value, left, right) = decompose(tree, &space)?; if unsafe { elem.raw_equals(&value) } { return Ok(tree); } - if is_yes(gor(stack, elem, value)) { + if is_yes(gor(stack, elem, value, &space)) { let c = put_recursive(stack, left, elem)?; - let (c_val, c_left, c_right) = decompose(c)?; + let (c_val, c_left, c_right) = decompose(c, &space)?; - if is_yes(mor(stack, value, c_val)) { + if is_yes(mor(stack, value, c_val, &space)) { Ok(make_node(stack, value, c, right)) } else { let new_a = make_node(stack, value, c_right, right); @@ -236,9 +249,9 @@ mod tests { } } else { let c = put_recursive(stack, right, elem)?; - let (c_val, c_left, c_right) = decompose(c)?; + let (c_val, c_left, c_right) = decompose(c, &space)?; - if is_yes(mor(stack, value, c_val)) { + if is_yes(mor(stack, value, c_val, &space)) { Ok(make_node(stack, value, left, c)) } else { let new_a = make_node(stack, value, left, c_left); @@ -260,7 +273,7 @@ mod tests { } } - fn tree_height(tree: Noun) -> usize { + fn tree_height(tree: Noun, space: &NounSpace) -> usize { let mut max = 0usize; let mut stack_vec = vec![(tree, 0usize)]; @@ -273,7 +286,7 @@ mod tests { max = depth; } - let (value, left, right) = decompose(node).unwrap_or((node, D(0), D(0))); + let (value, left, right) = decompose(node, space).unwrap_or((node, D(0), D(0))); let _ = value; stack_vec.push((left, depth + 1)); stack_vec.push((right, depth + 1)); @@ -283,6 +296,7 @@ mod tests { } #[test] + #[cfg_attr(miri, ignore = "memfd_create unsupported in Miri")] fn put_matches_recursive_small_inputs() { let mut base = [1u64, 2, 3, 4]; let mut perms = Vec::new(); @@ -292,39 +306,42 @@ mod tests { let context = &mut init_context(); let mut jet_tree = D(0); let mut rec_tree = D(0); + let space = context.stack.noun_space(); for val in perm { let noun_val = D(val); - jet_tree = put_iter(&mut context.stack, jet_tree, noun_val) + jet_tree = put_iter(&mut context.stack, jet_tree, noun_val, &space) .expect("put_iter should succeed"); rec_tree = put_recursive(&mut context.stack, rec_tree, noun_val) .expect("put_recursive should succeed"); } - assert_eq!(tree_height(jet_tree), tree_height(rec_tree)); + assert_eq!(tree_height(jet_tree, &space), tree_height(rec_tree, &space)); assert_noun_eq(&mut context.stack, jet_tree, rec_tree); } } #[test] + #[cfg_attr(miri, ignore = "memfd_create unsupported in Miri")] fn put_matches_recursive_random_inputs() { let mut seed = 0xDEADBEEFu64; for _ in 0..20 { let context = &mut init_context(); let mut jet_tree = D(0); let mut rec_tree = D(0); + let space = context.stack.noun_space(); for _ in 0..80 { seed = seed.wrapping_mul(6364136223846793005).wrapping_add(1); let value = (seed >> 32) & 0xFFFF; let noun_val = D(value); - jet_tree = put_iter(&mut context.stack, jet_tree, noun_val) + jet_tree = put_iter(&mut context.stack, jet_tree, noun_val, &space) .expect("put_iter should succeed"); rec_tree = put_recursive(&mut context.stack, rec_tree, noun_val) .expect("put_recursive should succeed"); } - assert_eq!(tree_height(jet_tree), tree_height(rec_tree)); + assert_eq!(tree_height(jet_tree, &space), tree_height(rec_tree, &space)); assert_noun_eq(&mut context.stack, jet_tree, rec_tree); } } diff --git a/crates/nockvm/rust/nockvm/src/jets/sort.rs b/crates/nockvm/rust/nockvm/src/jets/sort.rs index 0ba828bc2..24ffa3ecc 100644 --- a/crates/nockvm/rust/nockvm/src/jets/sort.rs +++ b/crates/nockvm/rust/nockvm/src/jets/sort.rs @@ -8,27 +8,30 @@ use crate::noun::Noun; crate::gdb!(); pub fn jet_dor(context: &mut Context, subject: Noun) -> jets::Result { - let sam = slot(subject, 6)?; - let a = slot(sam, 2)?; - let b = slot(sam, 3)?; + let space = context.stack.noun_space(); + let sam = slot(subject, 6, &space)?; + let a = slot(sam, 2, &space)?; + let b = slot(sam, 3, &space)?; - Ok(util::dor(&mut context.stack, a, b)) + Ok(util::dor(&mut context.stack, a, b, &space)) } pub fn jet_gor(context: &mut Context, subject: Noun) -> jets::Result { - let sam = slot(subject, 6)?; - let a = slot(sam, 2)?; - let b = slot(sam, 3)?; + let space = context.stack.noun_space(); + let sam = slot(subject, 6, &space)?; + let a = slot(sam, 2, &space)?; + let b = slot(sam, 3, &space)?; - Ok(util::gor(&mut context.stack, a, b)) + Ok(util::gor(&mut context.stack, a, b, &space)) } pub fn jet_mor(context: &mut Context, subject: Noun) -> jets::Result { - let sam = slot(subject, 6)?; - let a = slot(sam, 2)?; - let b = slot(sam, 3)?; + let space = context.stack.noun_space(); + let sam = slot(subject, 6, &space)?; + let a = slot(sam, 2, &space)?; + let b = slot(sam, 3, &space)?; - Ok(util::mor(&mut context.stack, a, b)) + Ok(util::mor(&mut context.stack, a, b, &space)) } pub mod util { @@ -40,22 +43,22 @@ pub mod util { use crate::jets::util::slot; use crate::mem::NockStack; use crate::mug::mug; - use crate::noun::{Noun, NO, YES}; + use crate::noun::{Noun, NounSpace, NO, YES}; - pub fn dor(stack: &mut NockStack, a: Noun, b: Noun) -> Noun { + pub fn dor(stack: &mut NockStack, a: Noun, b: Noun, space: &NounSpace) -> Noun { if unsafe { a.raw_equals(&b) } { YES } else { match (a.as_either_atom_cell(), b.as_either_atom_cell()) { - (Left(atom_a), Left(atom_b)) => lth(stack, atom_a, atom_b), + (Left(atom_a), Left(atom_b)) => lth(stack, atom_a, atom_b, space), (Left(_), Right(_)) => YES, (Right(_), Left(_)) => NO, (Right(cell_a), Right(cell_b)) => { - let a_head = match slot(cell_a.as_noun(), 2) { + let a_head = match slot(cell_a.as_noun(), 2, space) { Ok(n) => n, Err(_) => return NO, }; - let b_head = slot(cell_b.as_noun(), 2).unwrap_or_else(|err| { + let b_head = slot(cell_b.as_noun(), 2, space).unwrap_or_else(|err| { panic!( "Panicked with {err:?} at {}:{} (git sha: {:?})", file!(), @@ -63,7 +66,7 @@ pub mod util { option_env!("GIT_SHA") ) }); - let a_tail = slot(cell_a.as_noun(), 3).unwrap_or_else(|err| { + let a_tail = slot(cell_a.as_noun(), 3, space).unwrap_or_else(|err| { panic!( "Panicked with {err:?} at {}:{} (git sha: {:?})", file!(), @@ -71,7 +74,7 @@ pub mod util { option_env!("GIT_SHA") ) }); - let b_tail = slot(cell_b.as_noun(), 3).unwrap_or_else(|err| { + let b_tail = slot(cell_b.as_noun(), 3, space).unwrap_or_else(|err| { panic!( "Panicked with {err:?} at {}:{} (git sha: {:?})", file!(), @@ -80,27 +83,27 @@ pub mod util { ) }); if unsafe { a_head.raw_equals(&b_head) } { - dor(stack, a_tail, b_tail) + dor(stack, a_tail, b_tail, space) } else { - dor(stack, a_head, b_head) + dor(stack, a_head, b_head, space) } } } } } - pub fn gor(stack: &mut NockStack, a: Noun, b: Noun) -> Noun { + pub fn gor(stack: &mut NockStack, a: Noun, b: Noun, space: &NounSpace) -> Noun { let c = mug(stack, a); let d = mug(stack, b); match c.data().cmp(&d.data()) { Ordering::Greater => NO, Ordering::Less => YES, - Ordering::Equal => dor(stack, a, b), + Ordering::Equal => dor(stack, a, b, space), } } - pub fn mor(stack: &mut NockStack, a: Noun, b: Noun) -> Noun { + pub fn mor(stack: &mut NockStack, a: Noun, b: Noun, space: &NounSpace) -> Noun { let c = mug(stack, a); let d = mug(stack, b); @@ -110,7 +113,7 @@ pub mod util { match e.data().cmp(&f.data()) { Ordering::Greater => NO, Ordering::Less => YES, - Ordering::Equal => dor(stack, a, b), + Ordering::Equal => dor(stack, a, b, space), } } } @@ -124,6 +127,7 @@ mod tests { use crate::noun::{D, NO, T, YES}; #[test] + #[cfg_attr(miri, ignore = "memfd_create unsupported in Miri")] fn test_dor() { let c = &mut init_context(); @@ -140,6 +144,7 @@ mod tests { } #[test] + #[cfg_attr(miri, ignore = "memfd_create unsupported in Miri")] fn test_gor() { let c = &mut init_context(); @@ -152,6 +157,7 @@ mod tests { } #[test] + #[cfg_attr(miri, ignore = "memfd_create unsupported in Miri")] fn test_mor() { let c = &mut init_context(); diff --git a/crates/nockvm/rust/nockvm/src/jets/tree.rs b/crates/nockvm/rust/nockvm/src/jets/tree.rs index 1fa10109a..614e1eca1 100644 --- a/crates/nockvm/rust/nockvm/src/jets/tree.rs +++ b/crates/nockvm/rust/nockvm/src/jets/tree.rs @@ -9,14 +9,16 @@ use crate::noun::{IndirectAtom, Noun, D}; crate::gdb!(); pub fn jet_cap(_context: &mut Context, subject: Noun) -> Result { - let arg = slot(subject, 6)?; + let space = _context.stack.noun_space(); + let arg = slot(subject, 6, &space)?; let tom = arg.as_atom()?; - let met = met(0, tom); + let tom_handle = tom.in_space(&space); + let met = met(0, tom, &space); unsafe { if met < 2 { Err(BAIL_EXIT) - } else if *(tom.as_bitslice().get_unchecked(met - 2)) { + } else if *(tom_handle.as_bitslice().get_unchecked(met - 2)) { Ok(D(3)) } else { Ok(D(2)) @@ -26,8 +28,10 @@ pub fn jet_cap(_context: &mut Context, subject: Noun) -> Result { pub fn jet_mas(context: &mut Context, subject: Noun) -> Result { let stack = &mut context.stack; - let tom = slot(subject, 6)?.as_atom()?; - let met = met(0, tom); + let space = stack.noun_space(); + let tom = slot(subject, 6, &space)?.as_atom()?; + let tom_handle = tom.in_space(&space); + let met = met(0, tom, &space); if met < 2 { Err(BAIL_EXIT) @@ -38,17 +42,20 @@ pub fn jet_mas(context: &mut Context, subject: Noun) -> Result { unsafe { IndirectAtom::new_raw_mut_bitslice(stack, out_words) }; out_bs.set(met - 2, true); // Set MSB if met > 2 { - out_bs[0..(met - 2)].copy_from_bitslice(&tom.as_bitslice()[0..(met - 2)]); + out_bs[0..(met - 2)].copy_from_bitslice(&tom_handle.as_bitslice()[0..(met - 2)]); }; - unsafe { Ok(indirect_out.normalize_as_atom().as_noun()) } + unsafe { Ok(indirect_out.normalize_as_atom(&space).as_noun()) } } } pub fn jet_peg(context: &mut Context, subject: Noun) -> Result { let stack = &mut context.stack; - let arg = slot(subject, 6)?; - let a = slot(arg, 2)?.as_atom()?; - let b = slot(arg, 3)?.as_atom()?; + let space = stack.noun_space(); + let arg = slot(subject, 6, &space)?; + let a = slot(arg, 2, &space)?.as_atom()?; + let b = slot(arg, 3, &space)?.as_atom()?; + let a_handle = a.in_space(&space); + let b_handle = b.in_space(&space); unsafe { if a.as_noun().raw_equals(&D(0)) { @@ -60,8 +67,8 @@ pub fn jet_peg(context: &mut Context, subject: Noun) -> Result { }; } - let a_bits = met(0, a); - let b_bits = met(0, b); + let a_bits = met(0, a, &space); + let b_bits = met(0, b, &space); let out_bits = a_bits + b_bits - 1; let out_words = (out_bits + 63) >> 6; // bits to 8-byte words @@ -69,10 +76,10 @@ pub fn jet_peg(context: &mut Context, subject: Noun) -> Result { let (mut indirect_out, out_bs) = unsafe { IndirectAtom::new_raw_mut_bitslice(stack, out_words) }; - out_bs[0..b_bits - 1].copy_from_bitslice(&b.as_bitslice()[0..b_bits - 1]); - out_bs[b_bits - 1..out_bits].copy_from_bitslice(&a.as_bitslice()[0..a_bits]); + out_bs[0..b_bits - 1].copy_from_bitslice(&b_handle.as_bitslice()[0..b_bits - 1]); + out_bs[b_bits - 1..out_bits].copy_from_bitslice(&a_handle.as_bitslice()[0..a_bits]); - unsafe { Ok(indirect_out.normalize_as_atom().as_noun()) } + unsafe { Ok(indirect_out.normalize_as_atom(&space).as_noun()) } } #[cfg(test)] diff --git a/crates/nockvm/rust/nockvm/src/jets/warm.rs b/crates/nockvm/rust/nockvm/src/jets/warm.rs index 2377486e2..ebe6d93ff 100644 --- a/crates/nockvm/rust/nockvm/src/jets/warm.rs +++ b/crates/nockvm/rust/nockvm/src/jets/warm.rs @@ -1,11 +1,15 @@ use std::ptr::{copy_nonoverlapping, null_mut}; +use std::time::Instant; + +use tracing::debug; use crate::hamt::Hamt; use crate::jets::cold::{Batteries, Cold}; use crate::jets::hot::Hot; use crate::jets::Jet; use crate::mem::{NockStack, Preserve}; -use crate::noun::{Noun, Slots}; +use crate::noun::{Noun, NounAllocator, Slots}; +use crate::pma::{Pma, PmaCopy, PmaCopyFrom}; /// key = formula #[derive(Copy, Clone)] @@ -20,6 +24,127 @@ impl Preserve for Warm { } } +impl PmaCopy for Warm { + #[cfg(feature = "pma-assert")] + fn assert_in_pma(&self, pma: &Pma) { + self.0.assert_in_pma(pma); + } + + #[cfg(not(feature = "pma-assert"))] + #[inline(always)] + fn assert_in_pma(&self, _pma: &Pma) {} + + unsafe fn copy_to_pma(&mut self, stack: &NockStack, pma: &mut Pma) { + let trace = std::env::var_os("NOCK_PMA_TRACE").is_some(); + if trace { + debug!("pma-copy: warm stats start"); + let stats_start = Instant::now(); + let mut last_progress = Instant::now(); + let mut key_count = 0usize; + let mut list_count = 0usize; + let mut total_nodes = 0usize; + let mut nodes_stack = 0usize; + let mut nodes_pma = 0usize; + let mut prefix_nodes = 0usize; + let mut lists_all_stack = 0usize; + let mut lists_all_pma = 0usize; + let mut lists_mixed = 0usize; + let mut lists_mixed_after_pma = 0usize; + + for leaf in self.0.iter() { + key_count += leaf.len(); + for (_key, entry) in leaf { + list_count += 1; + let mut cursor = *entry; + let mut seen_pma = false; + let mut list_stack = 0usize; + let mut list_pma = 0usize; + let mut list_prefix = 0usize; + let mut list_mixed_after_pma = false; + while !cursor.0.is_null() { + total_nodes += 1; + if pma.contains_ptr(cursor.0 as *const u8) { + nodes_pma += 1; + list_pma += 1; + seen_pma = true; + } else { + nodes_stack += 1; + list_stack += 1; + if seen_pma { + list_mixed_after_pma = true; + } else { + list_prefix += 1; + } + } + cursor = (*cursor.0).next; + if total_nodes & 0x3fff == 0 { + let now = Instant::now(); + if now.duration_since(last_progress).as_millis() >= 2000 { + debug!( + "pma-copy: warm stats progress: lists={}, nodes={}, stack_nodes={}, pma_nodes={}, elapsed_ms={}", + list_count, + total_nodes, + nodes_stack, + nodes_pma, + stats_start.elapsed().as_millis() + ); + last_progress = now; + } + } + } + prefix_nodes += list_prefix; + if list_pma == 0 { + lists_all_stack += 1; + } else if list_stack == 0 { + lists_all_pma += 1; + } else { + lists_mixed += 1; + } + if list_mixed_after_pma { + lists_mixed_after_pma += 1; + } + } + } + + let stats_ms = stats_start.elapsed().as_millis(); + debug!( + "pma-copy: warm stats done: keys={}, lists={}, nodes={}, stack_nodes={}, pma_nodes={}, prefix_nodes={}, lists_all_stack={}, lists_all_pma={}, lists_mixed={}, lists_mixed_after_pma={}, stats_ms={}", + key_count, + list_count, + total_nodes, + nodes_stack, + nodes_pma, + prefix_nodes, + lists_all_stack, + lists_all_pma, + lists_mixed, + lists_mixed_after_pma, + stats_ms + ); + let alloc_before = pma.alloc_offset(); + debug!("pma-copy: warm copy start"); + let copy_start = Instant::now(); + self.0.copy_to_pma(stack, pma); + let copy_ms = copy_start.elapsed().as_millis(); + let alloc_after = pma.alloc_offset(); + let alloc_words = alloc_after.saturating_sub(alloc_before); + + debug!( + "pma-copy: warm copy done: alloc_words={}, copy_ms={}", + alloc_words, copy_ms + ); + } else { + self.0.copy_to_pma(stack, pma); + } + } +} + +impl PmaCopyFrom for Warm { + unsafe fn copy_from_pma(&mut self, from_pma: &Pma, to_pma: &mut Pma) { + self.0.copy_from_pma(from_pma, to_pma); + } +} + #[derive(Copy, Clone)] struct WarmEntry(*mut WarmEntryMem); @@ -72,6 +197,93 @@ impl Preserve for WarmEntry { } } +impl PmaCopy for WarmEntry { + #[cfg(feature = "pma-assert")] + fn assert_in_pma(&self, pma: &Pma) { + if self.0.is_null() { + return; + } + let mut cursor = *self; + loop { + unsafe { + assert!( + pma.contains_ptr(cursor.0 as *const u8), + "WarmEntry node should be in PMA" + ); + (*cursor.0).batteries.assert_in_pma(pma); + (*cursor.0).path.assert_in_pma(pma); + if (*cursor.0).next.0.is_null() { + break; + } + cursor = (*cursor.0).next; + } + } + } + + #[cfg(not(feature = "pma-assert"))] + #[inline(always)] + fn assert_in_pma(&self, _pma: &Pma) {} + + unsafe fn copy_to_pma(&mut self, stack: &NockStack, pma: &mut Pma) { + if self.0.is_null() { + return; + } + let trace = std::env::var_os("NOCK_PMA_TRACE_WARM_ENTRY").is_some(); + let mut ptr: *mut WarmEntry = self; + loop { + if trace { + debug!("pma-copy: warm entry start: node_ptr={:p}", (*ptr).0); + } + if pma.contains_ptr((*ptr).0 as *const u8) { + break; + } + // Copy batteries and path to PMA + (*(*ptr).0).batteries.copy_to_pma(stack, pma); + if trace { + debug!("pma-copy: warm entry batteries done"); + } + (*(*ptr).0).path.copy_to_pma(stack, pma); + if trace { + debug!("pma-copy: warm entry path done"); + } + // Allocate new WarmEntryMem in PMA and copy + let dest_mem: *mut WarmEntryMem = pma.alloc_struct(1); + copy_nonoverlapping((*ptr).0, dest_mem, 1); + // Update pointer to point to PMA copy + *ptr = WarmEntry(dest_mem); + if trace { + debug!("pma-copy: warm entry done: dest_ptr={:p}", dest_mem); + } + // Move to next node + ptr = &mut (*dest_mem).next; + if (*dest_mem).next.0.is_null() { + break; + } + } + } +} + +impl PmaCopyFrom for WarmEntry { + unsafe fn copy_from_pma(&mut self, from_pma: &Pma, to_pma: &mut Pma) { + if self.0.is_null() { + return; + } + let mut ptr: *mut WarmEntry = self; + loop { + let src_mem = (*ptr).0; + let dest_mem: *mut WarmEntryMem = to_pma.alloc_struct(1); + copy_nonoverlapping(src_mem, dest_mem, 1); + *ptr = WarmEntry(dest_mem); + (*dest_mem).batteries.copy_from_pma(from_pma, to_pma); + (*dest_mem).path.copy_from_pma(from_pma, to_pma); + ptr = &mut (*dest_mem).next; + if (*dest_mem).next.0.is_null() { + break; + } + } + } +} + impl Iterator for WarmEntry { type Item = (Noun, Batteries, Jet, bool); fn next(&mut self) -> Option { @@ -147,6 +359,7 @@ impl Warm { pub fn init(stack: &mut NockStack, cold: &mut Cold, hot: &Hot, test_jets: &Hamt<()>) -> Self { let mut warm = Self::new(stack); + let space = stack.fast_noun_space(); for (mut path, axis, jet) in *hot { let test_path = test_jets.lookup(stack, &mut path).is_some(); let batteries_list = cold.find(stack, &mut path); @@ -155,7 +368,7 @@ impl Warm { let (battery, _parent_axis) = batteries_tmp .next() .expect("IMPOSSIBLE: empty battery entry in cold state"); - if let Ok(mut formula) = unsafe { (*battery).slot_atom(axis) } { + if let Ok(mut formula) = unsafe { (*battery).slot_atom(axis, &space) } { warm.insert(stack, &mut formula, path, batteries, jet, test_path); } else { // XX: need NockStack allocated string interpolation @@ -179,8 +392,9 @@ impl Warm { let Some(warm_it) = self.0.lookup(stack, f) else { return JetLookupResult::NoJet; }; + let space = stack.fast_noun_space(); for (path, batteries, jet, test) in warm_it { - if batteries.matches(stack, *s) { + if batteries.matches(stack, *s, &space) { if test { return JetLookupResult::Test { jet, path }; } else { @@ -191,3 +405,327 @@ impl Warm { JetLookupResult::NoJet } } + +#[cfg(test)] +mod test { + use super::*; + use crate::interpreter::Context; + use crate::jets::cold::{Batteries, BatteriesMem, NO_BATTERIES}; + use crate::jets::JetErr; + use crate::mem::{NockStack, NOCK_STACK_SIZE_TINY}; + use crate::noun::{AllocLocation, NounAllocator, NounSpace, D}; + use crate::pma::{test_pma_path, Pma, PmaCopy}; + + const DEFAULT_STACK_SIZE: usize = NOCK_STACK_SIZE_TINY; + + fn make_test_stack(size: usize) -> NockStack { + NockStack::new(size, 0) + } + + /// Dummy jet function for testing + fn dummy_jet(_ctx: &mut Context, _subj: Noun) -> Result { + Ok(D(42)) + } + + /// Another dummy jet function to differentiate entries + fn dummy_jet_2(_ctx: &mut Context, _subj: Noun) -> Result { + Ok(D(99)) + } + + /// Create a simple Batteries for testing (single entry with given battery value) + fn make_simple_batteries(stack: &mut NockStack, battery_value: u64) -> Batteries { + let batteries_mem: *mut BatteriesMem = unsafe { stack.alloc_struct(1) }; + unsafe { + batteries_mem.write(BatteriesMem { + battery: D(battery_value), + parent_axis: D(0).as_atom().expect("0 is a valid atom"), + parent_batteries: NO_BATTERIES, + }); + } + Batteries::new(batteries_mem) + } + + /// Create a WarmEntry linked list for testing + fn make_warm_entry(stack: &mut NockStack, entries: &[(u64, Jet, u64, bool)]) -> WarmEntry { + let mut warm_entry = WARM_ENTRY_NIL; + for &(battery_value, jet, path_value, test) in entries.iter().rev() { + let batteries = make_simple_batteries(stack, battery_value); + let warm_entry_mem: *mut WarmEntryMem = unsafe { stack.alloc_struct(1) }; + unsafe { + warm_entry_mem.write(WarmEntryMem { + batteries, + jet, + path: D(path_value), + test, + next: warm_entry, + }); + } + warm_entry = WarmEntry(warm_entry_mem); + } + warm_entry + } + + /// Helper to verify a noun is not stack-allocated (is in offset form) + fn verify_noun_not_stack_allocated(noun: Noun, space: &NounSpace, context: &str) { + if noun.is_direct() { + return; + } + let location = noun.in_space(space).allocated_location(); + assert!( + !matches!(location, Some(AllocLocation::Stack)), + "{} should be in offset form after evacuation", + context + ); + if let Ok(cell) = noun.in_space(space).as_cell() { + verify_noun_not_stack_allocated(cell.head().noun(), space, context); + verify_noun_not_stack_allocated(cell.tail().noun(), space, context); + } + } + + /// Verifies WarmEntry can be evacuated to PMA and remains functional. + /// + /// This test exercises: + /// - Creating a WarmEntry linked list with multiple entries + /// - Evacuating the WarmEntry to PMA via copy_to_pma + /// - Verifying all entries are still accessible after evacuation + /// - Verifying all nouns are in offset form (not stack-allocated) + /// - Verifying the WarmEntry passes assert_in_pma + /// + /// Note: copy_to_pma sets forwarding pointers in the source nouns, which corrupts + /// them for normal use. We use expected values for comparison. + #[test] + #[cfg_attr(miri, ignore)] + fn test_evacuate_warm_entry_round_trip() { + let mut stack = make_test_stack(DEFAULT_STACK_SIZE); + let mut pma = + Pma::new(100000, test_pma_path("warm_entry")).expect("Failed to create test PMA"); + let space = NounSpace::new(&stack, &pma); + + // Create WarmEntry linked list with two entries + // (battery_value, jet, path_value, test) + let entries: Vec<(u64, Jet, u64, bool)> = + vec![(10, dummy_jet, 100, false), (20, dummy_jet_2, 200, true)]; + let mut warm_entry = make_warm_entry(&mut stack, &entries); + + // Evacuate WarmEntry to PMA + unsafe { + warm_entry.copy_to_pma(&stack, &mut pma); + } + + // Iterate over evacuated warm_entry and verify values + let mut expected_iter = entries.iter(); + for (path, batteries, jet, test) in warm_entry { + let (expected_battery, expected_jet, expected_path, expected_test) = expected_iter + .next() + .expect("WarmEntry has more entries than expected"); + + // Verify path + assert_eq!( + unsafe { path.as_raw() }, + *expected_path, + "Path should match" + ); + + // Verify jet function pointer + assert!( + std::ptr::fn_addr_eq(jet, *expected_jet), + "Jet function pointer should match" + ); + + // Verify test flag + assert_eq!(test, *expected_test, "Test flag should match"); + + // Verify batteries (first entry only for simplicity) + let mut batteries_iter = batteries.into_iter(); + let (battery_ptr, parent_axis) = batteries_iter + .next() + .expect("Batteries should have at least one entry"); + let battery = unsafe { *battery_ptr }; + assert_eq!( + unsafe { battery.as_raw() }, + *expected_battery, + "Battery value should match" + ); + assert_eq!( + parent_axis + .in_space(&space) + .as_u64() + .expect("parent axis should fit in u64"), + 0, + "Parent axis should be 0" + ); + + // Verify nouns are in offset form + verify_noun_not_stack_allocated(path, &space, "WarmEntry path"); + verify_noun_not_stack_allocated(battery, &space, "WarmEntry battery"); + } + + assert!( + expected_iter.next().is_none(), + "WarmEntry has fewer entries than expected" + ); + + // Verify the WarmEntry passes assert_in_pma + warm_entry.assert_in_pma(&pma); + } + + /// Verifies Warm state can be evacuated to PMA and remains functional. + /// + /// This test exercises: + /// - Creating a Warm HAMT with multiple formula->WarmEntry mappings + /// - Evacuating the Warm to PMA via copy_to_pma + /// - Verifying all entries are still accessible via lookup after evacuation + /// - Verifying all nouns are in offset form (not stack-allocated) + /// - Verifying the Warm passes assert_in_pma + /// + /// Note: copy_to_pma sets forwarding pointers in the source nouns, which corrupts + /// them for normal use. We use expected values for comparison. + #[test] + #[cfg_attr(miri, ignore)] + fn test_evacuate_warm_round_trip() { + let mut stack = make_test_stack(DEFAULT_STACK_SIZE); + let mut pma = Pma::new(100000, test_pma_path("warm")).expect("Failed to create test PMA"); + let space = NounSpace::new(&stack, &pma); + + // Create a Warm and insert some entries + let mut warm = Warm::new(&mut stack); + + // Insert entry 1: formula D(100) -> (battery=10, jet=dummy_jet, path=1000, test=false) + let batteries1 = make_simple_batteries(&mut stack, 10); + let mut formula1 = D(100); + warm.insert( + &mut stack, + &mut formula1, + D(1000), + batteries1, + dummy_jet, + false, + ); + + // Insert entry 2: formula D(200) -> (battery=20, jet=dummy_jet_2, path=2000, test=true) + let batteries2 = make_simple_batteries(&mut stack, 20); + let mut formula2 = D(200); + warm.insert( + &mut stack, + &mut formula2, + D(2000), + batteries2, + dummy_jet_2, + true, + ); + + // Insert entry 3: same formula as entry 1, different jet (creates linked list) + let batteries3 = make_simple_batteries(&mut stack, 30); + let mut formula3 = D(100); + warm.insert( + &mut stack, + &mut formula3, + D(3000), + batteries3, + dummy_jet_2, + true, + ); + + // Expected values for verification + // formula D(100) should have two entries: (30, dummy_jet_2, 3000, true) -> (10, dummy_jet, 1000, false) + // formula D(200) should have one entry: (20, dummy_jet_2, 2000, true) + let expected_formula_100: Vec<(u64, Jet, u64, bool)> = + vec![(30, dummy_jet_2, 3000, true), (10, dummy_jet, 1000, false)]; + let expected_formula_200: Vec<(u64, Jet, u64, bool)> = vec![(20, dummy_jet_2, 2000, true)]; + + // Evacuate Warm to PMA + unsafe { + warm.copy_to_pma(&stack, &mut pma); + } + + // Verify lookup for formula D(100) + let mut lookup_key1 = D(100); + let warm_entry1 = warm + .0 + .lookup(&mut stack, &mut lookup_key1) + .expect("Should find entry for formula D(100)"); + + let mut expected_iter1 = expected_formula_100.iter(); + for (path, batteries, jet, test) in warm_entry1 { + let (expected_battery, expected_jet, expected_path, expected_test) = expected_iter1 + .next() + .expect("WarmEntry has more entries than expected"); + + assert_eq!( + unsafe { path.as_raw() }, + *expected_path, + "Path should match" + ); + assert!(std::ptr::fn_addr_eq(jet, *expected_jet), "Jet should match"); + assert_eq!(test, *expected_test, "Test flag should match"); + + // Verify battery + let mut batteries_iter = batteries.into_iter(); + let (battery_ptr, _) = batteries_iter.next().expect("Batteries should have entry"); + let battery = unsafe { *battery_ptr }; + assert_eq!( + unsafe { battery.as_raw() }, + *expected_battery, + "Battery should match" + ); + + // Verify nouns are in offset form + verify_noun_not_stack_allocated(path, &space, "Warm path"); + verify_noun_not_stack_allocated(battery, &space, "Warm battery"); + } + assert!( + expected_iter1.next().is_none(), + "Missing entries for formula D(100)" + ); + + // Verify lookup for formula D(200) + let mut lookup_key2 = D(200); + let warm_entry2 = warm + .0 + .lookup(&mut stack, &mut lookup_key2) + .expect("Should find entry for formula D(200)"); + + let mut expected_iter2 = expected_formula_200.iter(); + for (path, batteries, jet, test) in warm_entry2 { + let (expected_battery, expected_jet, expected_path, expected_test) = expected_iter2 + .next() + .expect("WarmEntry has more entries than expected"); + + assert_eq!( + unsafe { path.as_raw() }, + *expected_path, + "Path should match" + ); + assert!(std::ptr::fn_addr_eq(jet, *expected_jet), "Jet should match"); + assert_eq!(test, *expected_test, "Test flag should match"); + + // Verify battery + let mut batteries_iter = batteries.into_iter(); + let (battery_ptr, _) = batteries_iter.next().expect("Batteries should have entry"); + let battery = unsafe { *battery_ptr }; + assert_eq!( + unsafe { battery.as_raw() }, + *expected_battery, + "Battery should match" + ); + + // Verify nouns are in offset form + verify_noun_not_stack_allocated(path, &space, "Warm path"); + verify_noun_not_stack_allocated(battery, &space, "Warm battery"); + } + assert!( + expected_iter2.next().is_none(), + "Missing entries for formula D(200)" + ); + + // Verify non-existent lookup returns None + let mut lookup_key3 = D(999); + assert!( + warm.0.lookup(&mut stack, &mut lookup_key3).is_none(), + "Lookup for non-existent formula should return None" + ); + + // Verify the Warm passes assert_in_pma + warm.assert_in_pma(&pma); + } +} diff --git a/crates/nockvm/rust/nockvm/src/lib.rs b/crates/nockvm/rust/nockvm/src/lib.rs index b1b9b4879..5c5b77282 100644 --- a/crates/nockvm/rust/nockvm/src/lib.rs +++ b/crates/nockvm/rust/nockvm/src/lib.rs @@ -15,9 +15,10 @@ pub mod jets; pub mod mem; pub mod mug; pub mod noun; +pub mod offset; +pub mod pma; pub mod serialization; pub mod site; -pub mod substantive; pub mod trace; pub mod unifying_equality; @@ -71,7 +72,7 @@ mod tests { use crate::mem::NockStack; use crate::noun::*; use crate::serialization::jam; - let mut stack = NockStack::new(8 << 10 << 10, 0); + let mut stack = NockStack::new(crate::mem::NOCK_STACK_SIZE_TINY, 0); let head = Atom::new(&mut stack, 0).as_noun(); let tail = Atom::new(&mut stack, 1).as_noun(); let cell = Cell::new(&mut stack, head, tail).as_noun(); diff --git a/crates/nockvm/rust/nockvm/src/mem.rs b/crates/nockvm/rust/nockvm/src/mem.rs index 0f71dc7e4..435b334bc 100644 --- a/crates/nockvm/rust/nockvm/src/mem.rs +++ b/crates/nockvm/rust/nockvm/src/mem.rs @@ -1,21 +1,40 @@ // TODO: fix stack push in PC -use std::alloc::Layout; -use std::ops::{Deref, DerefMut}; +use std::alloc::{alloc, dealloc, Layout}; +use std::fs::{File, OpenOptions}; +#[cfg(unix)] +use std::os::unix::io::AsRawFd; +#[cfg(not(feature = "no_check_oom"))] use std::panic::panic_any; +use std::path::Path; use std::ptr::copy_nonoverlapping; +use std::sync::atomic::{AtomicU64, AtomicUsize, Ordering}; +use std::sync::Arc; use std::vec::Vec; -use std::{mem, ptr}; +use std::{io, mem, ptr}; use either::Either::{self, Left, Right}; use ibig::Stack; -use memmap2::MmapMut; +use memmap2::{Mmap, MmapMut, MmapOptions}; use thiserror::Error; -use crate::noun::{Atom, Cell, CellMemory, IndirectAtom, Noun, NounAllocator}; -use crate::{assert_acyclic, assert_no_forwarding_pointers, assert_no_junior_pointers}; +use crate::noun::{Atom, Cell, CellMemory, IndirectAtom, Noun, NounAllocator, NounSpace}; +use crate::offset::{StackOffsetWords, WordOffset}; crate::gdb!(); +#[cfg(all(feature = "mmap", feature = "malloc"))] +compile_error!("nockvm features \"mmap\" and \"malloc\" are mutually exclusive"); +#[cfg(not(any(feature = "mmap", feature = "malloc")))] +compile_error!("nockvm requires either the \"mmap\" or \"malloc\" feature"); + +pub const NOCK_STACK_1KB: usize = 1 << 7; +pub const NOCK_STACK_SIZE_TINY: usize = (NOCK_STACK_1KB << 10 << 10) * 2; // 2GB +pub const NOCK_STACK_SIZE_SMALL: usize = (NOCK_STACK_1KB << 10 << 10) * 4; // 4GB +pub const NOCK_STACK_SIZE: usize = (NOCK_STACK_1KB << 10 << 10) * 8; // 8GB +pub const NOCK_STACK_SIZE_MEDIUM: usize = (NOCK_STACK_1KB << 10 << 10) * 16; // 16GB +pub const NOCK_STACK_SIZE_LARGE: usize = (NOCK_STACK_1KB << 10 << 10) * 32; // 32GB +pub const NOCK_STACK_SIZE_HUGE: usize = (NOCK_STACK_1KB << 10 << 10) * 64; // 64GB + /** Number of reserved slots for alloc_pointer and frame_pointer in each frame */ pub(crate) const RESERVED: usize = 3; @@ -31,9 +50,16 @@ pub(crate) const fn word_size_of() -> usize { } /** Utility function to compute the raw memory usage of an [IndirectAtom] */ -fn indirect_raw_size(atom: IndirectAtom) -> usize { - debug_assert!(atom.size() > 0); - atom.size() + 2 +fn indirect_raw_size(atom: IndirectAtom, space: &NounSpace) -> usize { + let atom_handle = atom.as_atom().in_space(space); + debug_assert!(atom_handle.size() > 0); + atom_handle.size() + 2 +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct NockStackFreeGapAdvice { + pub free_gap_words: usize, + pub advised_bytes: usize, } #[derive(Debug, Clone)] @@ -75,7 +101,11 @@ pub enum NewStackError { #[error("stack too small")] StackTooSmall, #[error("Failed to map memory for stack: {0}")] - MmapFailed(#[from] std::io::Error), + MmapFailed(std::io::Error), + #[error("Failed to open arena file: {0}")] + FileOpenFailed(std::io::Error), + #[error("Failed to resize arena file: {0}")] + FileResizeFailed(std::io::Error), } #[derive(Debug, Clone, Copy)] @@ -156,80 +186,461 @@ pub enum Direction { IncreasingDeref, } -pub enum AllocType { - Mmap, - Malloc, +#[derive(Debug)] +pub struct Arena { + base: *mut u8, + words: AtomicUsize, + mapped_bytes: AtomicUsize, + reserved_words: usize, + reserved_mapped_bytes: usize, + fd: Option>, + mapping: MappingKind, } -pub enum Memory { - Mmap(MmapMut), - Malloc(*mut u8, usize), +#[derive(Debug)] +enum MappingKind { + ReadWrite(MmapMut), + ReadOnly(Mmap), + Malloc(MallocMapping), + #[cfg(unix)] + GrowableFile(GrowableFileMapping), } -impl Deref for Memory { - type Target = [u8]; +#[derive(Debug)] +struct MallocMapping { + ptr: *mut u8, + layout: Layout, +} - #[inline] - fn deref(&self) -> &[u8] { - match self { - Memory::Mmap(mmap) => mmap.deref(), - Memory::Malloc(ptr, size) => unsafe { core::slice::from_raw_parts(*ptr, *size) }, +impl MallocMapping { + fn new(bytes: usize) -> Self { + let layout = Layout::from_size_align(bytes, mem::size_of::()) + .expect("Invalid layout for stack allocation"); + let ptr = unsafe { alloc(layout) }; + if ptr.is_null() { + std::alloc::handle_alloc_error(layout); } + Self { ptr, layout } } } -impl DerefMut for Memory { - #[inline] - fn deref_mut(&mut self) -> &mut [u8] { - match self { - Memory::Mmap(mmap) => mmap.deref_mut(), - Memory::Malloc(ptr, size) => unsafe { core::slice::from_raw_parts_mut(*ptr, *size) }, +impl Drop for MallocMapping { + fn drop(&mut self) { + unsafe { + dealloc(self.ptr, self.layout); } } } -impl AsRef<[u8]> for Memory { - #[inline] - fn as_ref(&self) -> &[u8] { - self.deref() - } +#[cfg(unix)] +#[derive(Debug)] +struct GrowableFileMapping { + ptr: *mut u8, + len: usize, } -impl AsMut<[u8]> for Memory { - #[inline] - fn as_mut(&mut self) -> &mut [u8] { - self.deref_mut() +#[cfg(unix)] +impl Drop for GrowableFileMapping { + fn drop(&mut self) { + if self.len == 0 || self.ptr.is_null() { + return; + } + unsafe { + libc::munmap(self.ptr as *mut libc::c_void, self.len); + } } } -impl Memory { - /// Layout and MmapMut::map_anon take their sizes/lengths in bytes but we speak in terms - /// of machine words which are u64 for our purposes so we're 8x'ing them with a cutesy shift. - pub(crate) fn allocate(alloc_type: AllocType, size: usize) -> Result { - let memory = match alloc_type { - AllocType::Mmap => { - let mmap_mut = MmapMut::map_anon(size << 3)?; - Self::Mmap(mmap_mut) +impl Arena { + #[allow(clippy::arc_with_non_send_sync)] + pub fn allocate(words: usize) -> Result, NewStackError> { + let bytes = words.checked_shl(3).ok_or(NewStackError::StackTooSmall)?; + #[cfg(feature = "mmap")] + { + let mut mapping = MmapMut::map_anon(bytes).map_err(NewStackError::MmapFailed)?; + let base = mapping.as_mut_ptr(); + return Ok(Arc::new(Self { + base, + words: AtomicUsize::new(words), + mapped_bytes: AtomicUsize::new(bytes), + reserved_words: words, + reserved_mapped_bytes: bytes, + fd: None, + mapping: MappingKind::ReadWrite(mapping), + })); + } + #[cfg(feature = "malloc")] + { + let mapping = MallocMapping::new(bytes); + let base = mapping.ptr; + return Ok(Arc::new(Self { + base, + words: AtomicUsize::new(words), + mapped_bytes: AtomicUsize::new(bytes), + reserved_words: words, + reserved_mapped_bytes: bytes, + fd: None, + mapping: MappingKind::Malloc(mapping), + })); + } + } + + pub fn allocate_file( + path: &Path, + words: usize, + tail_bytes: usize, + ) -> Result, NewStackError> { + let bytes = words.checked_shl(3).ok_or(NewStackError::StackTooSmall)?; + let file_bytes = bytes + .checked_add(tail_bytes) + .ok_or(NewStackError::StackTooSmall)?; + let file = OpenOptions::new() + .read(true) + .write(true) + .create(true) + .truncate(true) + .open(path) + .map_err(NewStackError::FileOpenFailed)?; + file.set_len(file_bytes as u64) + .map_err(NewStackError::FileResizeFailed)?; + let file = Arc::new(file); + let mut mapping = unsafe { MmapMut::map_mut(&*file).map_err(NewStackError::MmapFailed)? }; + let base = mapping.as_mut_ptr(); + let mapped_bytes = mapping.len(); + #[allow(clippy::arc_with_non_send_sync)] + Ok(Arc::new(Self { + base, + words: AtomicUsize::new(words), + mapped_bytes: AtomicUsize::new(mapped_bytes), + reserved_words: words, + reserved_mapped_bytes: mapped_bytes, + fd: Some(file), + mapping: MappingKind::ReadWrite(mapping), + })) + } + + pub fn allocate_growable_file( + path: &Path, + capacity_words: usize, + reserved_words: usize, + tail_bytes: usize, + ) -> Result, NewStackError> { + let file_bytes = file_len_for_words(capacity_words, tail_bytes)?; + let reserved_file_bytes = file_len_for_words(reserved_words, tail_bytes)?; + let file = OpenOptions::new() + .read(true) + .write(true) + .create(true) + .truncate(true) + .open(path) + .map_err(NewStackError::FileOpenFailed)?; + file.set_len(file_bytes as u64) + .map_err(NewStackError::FileResizeFailed)?; + Self::map_growable_file( + file, capacity_words, reserved_words, file_bytes, reserved_file_bytes, + ) + } + + pub fn open_growable_file( + path: &Path, + capacity_words: usize, + reserved_words: usize, + tail_bytes: usize, + ) -> Result, NewStackError> { + let file_bytes = file_len_for_words(capacity_words, tail_bytes)?; + let reserved_file_bytes = file_len_for_words(reserved_words, tail_bytes)?; + let file = OpenOptions::new() + .read(true) + .write(true) + .open(path) + .map_err(NewStackError::FileOpenFailed)?; + Self::map_growable_file( + file, capacity_words, reserved_words, file_bytes, reserved_file_bytes, + ) + } + + #[cfg(unix)] + fn map_growable_file( + file: File, + capacity_words: usize, + reserved_words: usize, + file_bytes: usize, + reserved_file_bytes: usize, + ) -> Result, NewStackError> { + if reserved_words < capacity_words || reserved_file_bytes < file_bytes { + return Err(NewStackError::StackTooSmall); + } + let reserved_mapping_bytes = page_align_len(reserved_file_bytes)?; + let file_mapping_bytes = page_align_len(file_bytes)?; + let file = Arc::new(file); + let reservation = unsafe { + libc::mmap( + ptr::null_mut(), + reserved_mapping_bytes, + libc::PROT_NONE, + libc::MAP_PRIVATE | libc::MAP_ANON, + -1, + 0, + ) + }; + if reservation == libc::MAP_FAILED { + return Err(NewStackError::MmapFailed(io::Error::last_os_error())); + } + let mapped = unsafe { + libc::mmap( + reservation, + file_mapping_bytes, + libc::PROT_READ | libc::PROT_WRITE, + libc::MAP_SHARED | libc::MAP_FIXED, + file.as_raw_fd(), + 0, + ) + }; + if mapped == libc::MAP_FAILED { + let err = io::Error::last_os_error(); + unsafe { + libc::munmap(reservation, reserved_mapping_bytes); } - AllocType::Malloc => { - // Align is in terms of bytes so I'm aligning it to 64-bits / 8 bytes, word size. - let layout = Layout::from_size_align(size << 3, std::mem::size_of::()) - .expect("Invalid layout"); - let alloc = unsafe { std::alloc::alloc(layout) }; - if alloc.is_null() { - // std promises that std::alloc::handle_alloc_error will diverge - std::alloc::handle_alloc_error(layout); + return Err(NewStackError::MmapFailed(err)); + } + debug_assert_eq!(mapped, reservation); + let base = reservation as *mut u8; + #[allow(clippy::arc_with_non_send_sync)] + Ok(Arc::new(Self { + base, + words: AtomicUsize::new(capacity_words), + mapped_bytes: AtomicUsize::new(file_bytes), + reserved_words, + reserved_mapped_bytes: reserved_file_bytes, + fd: Some(file), + mapping: MappingKind::GrowableFile(GrowableFileMapping { + ptr: base, + len: reserved_mapping_bytes, + }), + })) + } + + #[cfg(not(unix))] + fn map_growable_file( + _file: File, + _capacity_words: usize, + _reserved_words: usize, + _file_bytes: usize, + _reserved_file_bytes: usize, + ) -> Result, NewStackError> { + Err(NewStackError::MmapFailed(io::Error::new( + io::ErrorKind::Unsupported, + "growable PMA file mappings require Unix mmap", + ))) + } + + pub fn open_file(path: &Path, words: usize) -> Result, NewStackError> { + let file = OpenOptions::new() + .read(true) + .write(true) + .open(path) + .map_err(NewStackError::FileOpenFailed)?; + let file = Arc::new(file); + let mut mapping = unsafe { MmapMut::map_mut(&*file).map_err(NewStackError::MmapFailed)? }; + let base = mapping.as_mut_ptr(); + let mapped_bytes = mapping.len(); + #[allow(clippy::arc_with_non_send_sync)] + Ok(Arc::new(Self { + base, + words: AtomicUsize::new(words), + mapped_bytes: AtomicUsize::new(mapped_bytes), + reserved_words: words, + reserved_mapped_bytes: mapped_bytes, + fd: Some(file), + mapping: MappingKind::ReadWrite(mapping), + })) + } + + #[inline] + pub fn words(&self) -> usize { + self.words.load(Ordering::Acquire) + } + + #[inline] + pub fn reserved_words(&self) -> usize { + self.reserved_words + } + + #[inline] + pub fn len_bytes(&self) -> usize { + self.words() + .checked_mul(8) + .expect("arena length in bytes exceeds usize") + } + + #[inline] + pub fn mapped_len_bytes(&self) -> usize { + self.mapped_bytes.load(Ordering::Acquire) + } + + #[inline] + pub fn reserved_len_bytes(&self) -> usize { + self.reserved_mapped_bytes + } + + #[inline] + pub fn base_ptr(&self) -> *mut u8 { + self.base + } + + #[inline] + pub fn ptr_from_offset(&self, offset_words: O) -> *mut u8 { + let offset_bytes = offset_words + .checked_bytes_usize() + .expect("offset in words exceeds usize byte capacity"); + unsafe { self.base.add(offset_bytes) } + } + + #[inline] + pub fn offset_from_ptr(&self, ptr: *const u8) -> O { + let base = self.base as usize; + let ptr_usize = ptr as usize; + debug_assert!( + ptr_usize >= base, + "pointer {ptr:p} is below arena base {:p}", + self.base + ); + let offset_bytes = ptr_usize + .checked_sub(base) + .expect("pointer is below arena base"); + debug_assert!( + offset_bytes.is_multiple_of(8), + "unaligned pointer passed to offset_from_ptr: {ptr:p}" + ); + let offset_words = u64::try_from(offset_bytes >> 3) + .expect("offset in words exceeds u64 addressable range"); + O::from_words(offset_words) + } + + pub fn map_copy_read_only(&self) -> io::Result { + let Some(fd) = &self.fd else { + return Err(io::Error::new( + io::ErrorKind::Unsupported, + "arena has no file backing", + )); + }; + unsafe { + MmapOptions::new() + .len(self.mapped_len_bytes()) + .map_copy_read_only(&**fd) + } + } + + pub fn clone_read_only(&self) -> io::Result> { + let Some(fd) = &self.fd else { + return Err(io::Error::new( + io::ErrorKind::Unsupported, + "arena has no file backing", + )); + }; + let mapping = unsafe { + MmapOptions::new() + .len(self.mapped_len_bytes()) + .map_copy_read_only(&**fd)? + }; + let base = mapping.as_ptr() as *mut u8; + let words = self.words(); + let mapped_bytes = self.mapped_len_bytes(); + #[allow(clippy::arc_with_non_send_sync)] + Ok(Arc::new(Self { + base, + words: AtomicUsize::new(words), + mapped_bytes: AtomicUsize::new(mapped_bytes), + reserved_words: words, + reserved_mapped_bytes: mapped_bytes, + fd: Some(Arc::clone(fd)), + mapping: MappingKind::ReadOnly(mapping), + })) + } + + pub fn grow_file_capacity( + &self, + new_words: usize, + tail_bytes: usize, + ) -> Result<(), NewStackError> { + if new_words > self.reserved_words { + return Err(NewStackError::StackTooSmall); + } + let old_words = self.words(); + if new_words <= old_words { + return Ok(()); + } + let file_bytes = file_len_for_words(new_words, tail_bytes)?; + if file_bytes > self.reserved_mapped_bytes { + return Err(NewStackError::StackTooSmall); + } + let Some(fd) = &self.fd else { + return Err(NewStackError::FileOpenFailed(io::Error::new( + io::ErrorKind::Unsupported, + "arena has no file backing", + ))); + }; + fd.set_len(file_bytes as u64) + .map_err(NewStackError::FileResizeFailed)?; + #[cfg(unix)] + if let MappingKind::GrowableFile(_) = &self.mapping { + let old_file_bytes = self.mapped_len_bytes(); + let old_mapping_bytes = page_align_len(old_file_bytes)?; + let new_mapping_bytes = page_align_len(file_bytes)?; + if new_mapping_bytes > old_mapping_bytes { + let map_len = new_mapping_bytes + .checked_sub(old_mapping_bytes) + .ok_or(NewStackError::StackTooSmall)?; + let map_addr = unsafe { self.base.add(old_mapping_bytes) }; + let map_offset = i64::try_from(old_mapping_bytes) + .map_err(|_| NewStackError::StackTooSmall)? + as libc::off_t; + let mapped = unsafe { + libc::mmap( + map_addr as *mut libc::c_void, + map_len, + libc::PROT_READ | libc::PROT_WRITE, + libc::MAP_SHARED | libc::MAP_FIXED, + fd.as_raw_fd(), + map_offset, + ) + }; + if mapped == libc::MAP_FAILED { + return Err(NewStackError::MmapFailed(io::Error::last_os_error())); } - Self::Malloc(alloc, size) + debug_assert_eq!(mapped, map_addr as *mut libc::c_void); } - }; - Ok(memory) + } + self.words.store(new_words, Ordering::Release); + self.mapped_bytes.store(file_bytes, Ordering::Release); + Ok(()) } } +fn file_len_for_words(words: usize, tail_bytes: usize) -> Result { + let bytes = words.checked_shl(3).ok_or(NewStackError::StackTooSmall)?; + bytes + .checked_add(tail_bytes) + .ok_or(NewStackError::StackTooSmall) +} + +#[cfg(unix)] +fn page_align_len(len: usize) -> Result { + let page_size = unsafe { libc::sysconf(libc::_SC_PAGESIZE) }; + if page_size <= 0 { + return Err(NewStackError::MmapFailed(io::Error::last_os_error())); + } + let page_size = usize::try_from(page_size).map_err(|_| NewStackError::StackTooSmall)?; + let mask = page_size + .checked_sub(1) + .ok_or(NewStackError::StackTooSmall)?; + len.checked_add(mask) + .map(|value| value & !mask) + .ok_or(NewStackError::StackTooSmall) +} + /// A stack for Nock computation, which supports stack allocation and delimited copying collection /// for returned nouns -#[allow(dead_code)] // We need the memory field to keep our memory from being unmapped pub struct NockStack { /// The base pointer from the original allocation start: *const u64, @@ -243,8 +654,12 @@ pub struct NockStack { alloc_offset: usize, /// The least amount of space between the stack and alloc pointers since last reset least_space: usize, - /// The underlying memory allocation which must be kept alive - memory: Memory, + /// Shared arena metadata / backing allocation + arena: Arc, + /// Optional PMA arena for offset noun resolution + pma: Option>, + /// Epoch that increments on reset/flip to invalidate stack-pointer nouns. + stack_epoch: Arc, /// Whether or not [`Self::pre_copy()`] has been called on the current stack frame. pc: bool, } @@ -292,6 +707,66 @@ impl NockStack { self.derive_ptr(self.alloc_offset) } + #[inline] + pub fn arena(&self) -> &Arc { + &self.arena + } + + #[inline] + pub fn arena_ref(&self) -> &Arena { + &self.arena + } + + #[inline] + pub fn noun_space(&self) -> NounSpace { + NounSpace::from_stack(self, self.pma.clone()) + } + + #[inline] + pub(crate) fn fast_noun_space(&self) -> NounSpace { + NounSpace::from_stack_ephemeral(self) + } + + #[inline] + pub fn stack_epoch(&self) -> Arc { + Arc::clone(&self.stack_epoch) + } + + #[inline] + pub(crate) fn stack_epoch_ref(&self) -> &AtomicU64 { + self.stack_epoch.as_ref() + } + + #[inline] + pub fn stack_epoch_snapshot(&self) -> u64 { + self.stack_epoch.load(Ordering::Relaxed) + } + + #[inline] + pub(crate) fn pma_ref(&self) -> Option<&Arena> { + self.pma.as_deref() + } + + #[inline] + pub fn install_pma_arena(&mut self, pma: Arc) { + self.pma = Some(pma); + } + + #[inline] + pub fn clear_pma_arena(&mut self) { + self.pma = None; + } + + #[inline] + pub fn ptr_from_offset(&self, offset_words: StackOffsetWords) -> *mut u8 { + self.arena.ptr_from_offset(offset_words) + } + + #[inline] + pub fn offset_from_ptr(&self, ptr: *const u8) -> StackOffsetWords { + self.arena.offset_from_ptr(ptr) + } + /** Initialization: * The initial frame is a west frame. When the stack is initialized, a number of slots is given. * We add three extra slots to store the “previous” frame, stack, and allocation pointer. For the @@ -312,31 +787,43 @@ impl NockStack { if top_slots + RESERVED > size { return Err(NewStackError::StackTooSmall); } + let arena = Arena::allocate(size)?; + Self::from_arena_internal(arena, top_slots) + } + + pub fn from_arena( + arena: Arc, + top_slots: usize, + ) -> Result<(NockStack, usize), NewStackError> { + if top_slots + RESERVED > arena.words() { + return Err(NewStackError::StackTooSmall); + } + Self::from_arena_internal(arena, top_slots) + } + + fn from_arena_internal( + arena: Arc, + top_slots: usize, + ) -> Result<(NockStack, usize), NewStackError> { + let size = arena.words(); let free = size - (top_slots + RESERVED); - #[cfg(feature = "mmap")] - let mut memory = Memory::allocate(AllocType::Mmap, size)?; - #[cfg(feature = "malloc")] - let mut memory = Memory::allocate(AllocType::Malloc, size)?; - let start = memory.as_mut_ptr() as *mut u64; + let start = arena.base_ptr() as *mut u64; - // Here, frame_offset < alloc_offset, so the initial frame is West let frame_offset = RESERVED + top_slots; let stack_offset = frame_offset; - // FIXME: This was alloc_offset = size; why? let alloc_offset = size; let least_space = alloc_offset .checked_sub(stack_offset) .expect("Stack too small to create"); unsafe { - // Store previous frame/stack/alloc info in reserved slots let prev_frame_slot = frame_offset - (FRAME + 1); let prev_stack_slot = frame_offset - (STACK + 1); let prev_alloc_slot = frame_offset - (ALLOC + 1); - *(start.add(prev_frame_slot)) = ptr::null::() as u64; // "frame pointer" from "previous" frame - *(start.add(prev_stack_slot)) = ptr::null::() as u64; // "stack pointer" from "previous" frame - *(start.add(prev_alloc_slot)) = start as u64; // "alloc pointer" from "previous" frame + *(start.add(prev_frame_slot)) = ptr::null::() as u64; + *(start.add(prev_stack_slot)) = ptr::null::() as u64; + *(start.add(prev_alloc_slot)) = start as u64; }; assert_eq!(alloc_offset - stack_offset, free); @@ -348,13 +835,17 @@ impl NockStack { stack_offset, alloc_offset, least_space, - memory, + arena, + pma: None, + stack_epoch: Arc::new(AtomicU64::new(0)), pc: false, }, free, )) } + #[cold] + #[inline(never)] fn memory_state(&self, words: Option) -> MemoryState { unsafe { MemoryState { @@ -370,14 +861,30 @@ impl NockStack { } } + #[cold] + #[inline(never)] fn cannot_alloc_in_pc(&self, size: Option) -> AllocationError { AllocationError::CannotAllocateInPreCopy(self.memory_state(size)) } + #[cold] + #[inline(never)] fn out_of_memory(&self, alloc: Allocation, words: Option) -> AllocationError { AllocationError::OutOfMemory(OutOfMemoryError(self.memory_state(words), alloc)) } + #[cold] + #[inline(never)] + fn panic_cannot_alloc_in_pc(&self, words: usize) -> ! { + panic_any(self.cannot_alloc_in_pc(Some(words))) + } + + #[cold] + #[inline(never)] + fn panic_out_of_memory_for(&self, alloc_type: AllocationType, words: usize) -> ! { + panic_any(self.out_of_memory(self.get_alloc_config(alloc_type), Some(words))) + } + pub(crate) fn get_alloc_config(&self, alloc_type: AllocationType) -> Allocation { Allocation { orientation: if self.is_west() { @@ -390,6 +897,15 @@ impl NockStack { } } + #[inline] + unsafe fn push_limit_offset(&self) -> usize { + if self.pc { + self.prev_alloc_offset() + } else { + self.alloc_offset + } + } + // When frame_pointer < alloc_pointer, the frame is West // West frame layout: // - start @@ -422,166 +938,160 @@ impl NockStack { // Directionality parameters: (East/West), (Stack/Alloc), (pc: true/false) // Types of size: word (words: usize) /// Check if an allocation or pointer retrieval indicates an invalid request or an invalid state - pub(crate) fn alloc_would_oom_(&self, alloc: Allocation, words: usize) { - // When the fast path is enabled, make the parameters count as used to avoid - // unused-variable warnings while still short-circuiting. + pub(crate) fn alloc_would_oom_(&self, _alloc: Allocation, _words: usize) { #[cfg(feature = "no_check_oom")] + return; + #[cfg(not(feature = "no_check_oom"))] { - let _ = (&alloc, words); - } - - if cfg!(feature = "no_check_oom") { - return; - } - let _memory_state = self.memory_state(Some(words)); - if self.pc && !alloc.alloc_type.allowed_when_pc() { - panic_any(self.cannot_alloc_in_pc(Some(words))); - } - - // Convert words to byte count (for compatibility with old code) - let _bytes = words * 8; - - // Check space availability based on offsets - let (target_offset, limit_offset, direction) = match (alloc.alloc_type, alloc.orientation) { - // West + Alloc, alloc is decreasing - (AllocationType::Alloc, ArenaOrientation::West) => { - let start_offset = self.alloc_offset; - let limit_offset = self.stack_offset; - let target_offset = if start_offset >= words { - start_offset - words - } else { - panic!("Alloc would underflow in West+Alloc"); - }; - (target_offset, limit_offset, Direction::Decreasing) - } - // East + Alloc, alloc is increasing - (AllocationType::Alloc, ArenaOrientation::East) => { - let start_offset = self.alloc_offset; - let limit_offset = self.stack_offset; - let target_offset = start_offset + words; - (target_offset, limit_offset, Direction::Increasing) + let alloc = _alloc; + let words = _words; + if self.pc && !alloc.alloc_type.allowed_when_pc() { + panic_any(self.cannot_alloc_in_pc(Some(words))); } - // West + Push, stack is increasing - (AllocationType::Push, ArenaOrientation::West) => { - let start_offset = self.stack_offset; - let limit_offset = if self.pc { - unsafe { self.prev_alloc_offset() } - } else { - self.alloc_offset - }; - let target_offset = start_offset + words; - (target_offset, limit_offset, Direction::Increasing) - } - // East + Push, stack is decreasing - (AllocationType::Push, ArenaOrientation::East) => { - let start_offset = self.stack_offset; - let limit_offset = if self.pc { - unsafe { self.prev_alloc_offset() } - } else { - self.alloc_offset - }; - let target_offset = if start_offset >= words { - start_offset - words - } else { - panic!("Push would underflow in East+Push"); - }; - (target_offset, limit_offset, Direction::Decreasing) - } - // West + FramePush, alloc is decreasing - (AllocationType::FramePush, ArenaOrientation::West) => { - let start_offset = self.alloc_offset; - let limit_offset = self.stack_offset; - let target_offset = if start_offset >= words { - start_offset - words - } else { - panic!("FramePush would underflow in West+FramePush"); - }; - (target_offset, limit_offset, Direction::Decreasing) - } - // East + FramePush, alloc is increasing - (AllocationType::FramePush, ArenaOrientation::East) => { - let start_offset = self.alloc_offset; - let limit_offset = self.stack_offset; - let target_offset = start_offset + words; - (target_offset, limit_offset, Direction::Increasing) - } - // West + SlotPointer, polarity is reversed because we're getting the prev pointer - (AllocationType::SlotPointer, ArenaOrientation::West) => { - let _slots_available = unsafe { - self.slots_available() - .expect("No slots available on slot_pointer alloc check") - }; - let start_offset = self.frame_offset; - let limit_offset = unsafe { self.prev_alloc_offset() }; - let target_offset = if start_offset > words + 1 { - start_offset - words - 1 - } else { - panic!("SlotPointer would underflow in West+SlotPointer"); - }; - (target_offset, limit_offset, Direction::Decreasing) - } - // East + SlotPointer, polarity is reversed because we're getting the prev pointer - (AllocationType::SlotPointer, ArenaOrientation::East) => { - let _slots_available = unsafe { - self.slots_available() - .expect("No slots available on slot_pointer alloc check") - }; - let start_offset = self.frame_offset; - let limit_offset = unsafe { self.prev_alloc_offset() }; - let target_offset = start_offset + words; - (target_offset, limit_offset, Direction::IncreasingDeref) - } - // The alloc previous frame stuff is like doing a normal alloc but start offset is prev alloc and limit offset is stack offset - // polarity is reversed because we're getting the prev pointer - (AllocationType::AllocPreviousFrame, ArenaOrientation::West) => { - let start_offset = unsafe { self.prev_alloc_offset() }; - let limit_offset = self.stack_offset; - let target_offset = start_offset + words; - (target_offset, limit_offset, Direction::Increasing) - } - // polarity is reversed because we're getting the prev pointer - (AllocationType::AllocPreviousFrame, ArenaOrientation::East) => { - let start_offset = unsafe { self.prev_alloc_offset() }; - let limit_offset = self.stack_offset; - let target_offset = if start_offset >= words { - start_offset - words - } else { - panic!("AllocPreviousFrame would underflow in East+AllocPreviousFrame"); - }; - (target_offset, limit_offset, Direction::Decreasing) - } - (AllocationType::FlipTopFrame, ArenaOrientation::West) => { - let start_offset = self.size; // End of the memory region - let limit_offset = unsafe { self.prev_alloc_offset() }; - let target_offset = if start_offset >= words { - start_offset - words - } else { - panic!("FlipTopFrame would underflow in West+FlipTopFrame"); + + // Check space availability based on offsets + let (target_offset, limit_offset, direction) = + match (alloc.alloc_type, alloc.orientation) { + // West + Alloc, alloc is decreasing + (AllocationType::Alloc, ArenaOrientation::West) => { + let start_offset = self.alloc_offset; + let limit_offset = self.stack_offset; + let target_offset = if start_offset >= words { + start_offset - words + } else { + panic!("Alloc would underflow in West+Alloc"); + }; + (target_offset, limit_offset, Direction::Decreasing) + } + // East + Alloc, alloc is increasing + (AllocationType::Alloc, ArenaOrientation::East) => { + let start_offset = self.alloc_offset; + let limit_offset = self.stack_offset; + let target_offset = start_offset + words; + (target_offset, limit_offset, Direction::Increasing) + } + // West + Push, stack is increasing + (AllocationType::Push, ArenaOrientation::West) => { + let start_offset = self.stack_offset; + let limit_offset = if self.pc { + unsafe { self.prev_alloc_offset() } + } else { + self.alloc_offset + }; + let target_offset = start_offset + words; + (target_offset, limit_offset, Direction::Increasing) + } + // East + Push, stack is decreasing + (AllocationType::Push, ArenaOrientation::East) => { + let start_offset = self.stack_offset; + let limit_offset = if self.pc { + unsafe { self.prev_alloc_offset() } + } else { + self.alloc_offset + }; + let target_offset = if start_offset >= words { + start_offset - words + } else { + panic!("Push would underflow in East+Push"); + }; + (target_offset, limit_offset, Direction::Decreasing) + } + // West + FramePush, alloc is decreasing + (AllocationType::FramePush, ArenaOrientation::West) => { + let start_offset = self.alloc_offset; + let limit_offset = self.stack_offset; + let target_offset = if start_offset >= words { + start_offset - words + } else { + panic!("FramePush would underflow in West+FramePush"); + }; + (target_offset, limit_offset, Direction::Decreasing) + } + // East + FramePush, alloc is increasing + (AllocationType::FramePush, ArenaOrientation::East) => { + let start_offset = self.alloc_offset; + let limit_offset = self.stack_offset; + let target_offset = start_offset + words; + (target_offset, limit_offset, Direction::Increasing) + } + // West + SlotPointer, polarity is reversed because we're getting the prev pointer + (AllocationType::SlotPointer, ArenaOrientation::West) => { + let _slots_available = unsafe { + self.slots_available() + .expect("No slots available on slot_pointer alloc check") + }; + let start_offset = self.frame_offset; + let limit_offset = unsafe { self.prev_alloc_offset() }; + let target_offset = if start_offset > words + 1 { + start_offset - words - 1 + } else { + panic!("SlotPointer would underflow in West+SlotPointer"); + }; + (target_offset, limit_offset, Direction::Decreasing) + } + // East + SlotPointer, polarity is reversed because we're getting the prev pointer + (AllocationType::SlotPointer, ArenaOrientation::East) => { + let _slots_available = unsafe { + self.slots_available() + .expect("No slots available on slot_pointer alloc check") + }; + let start_offset = self.frame_offset; + let limit_offset = unsafe { self.prev_alloc_offset() }; + let target_offset = start_offset + words; + (target_offset, limit_offset, Direction::IncreasingDeref) + } + // The alloc previous frame stuff is like doing a normal alloc but start offset is prev alloc and limit offset is stack offset + // polarity is reversed because we're getting the prev pointer + (AllocationType::AllocPreviousFrame, ArenaOrientation::West) => { + let start_offset = unsafe { self.prev_alloc_offset() }; + let limit_offset = self.stack_offset; + let target_offset = start_offset + words; + (target_offset, limit_offset, Direction::Increasing) + } + // polarity is reversed because we're getting the prev pointer + (AllocationType::AllocPreviousFrame, ArenaOrientation::East) => { + let start_offset = unsafe { self.prev_alloc_offset() }; + let limit_offset = self.stack_offset; + let target_offset = if start_offset >= words { + start_offset - words + } else { + panic!("AllocPreviousFrame would underflow in East+AllocPreviousFrame"); + }; + (target_offset, limit_offset, Direction::Decreasing) + } + (AllocationType::FlipTopFrame, ArenaOrientation::West) => { + let start_offset = self.size; // End of the memory region + let limit_offset = unsafe { self.prev_alloc_offset() }; + let target_offset = if start_offset >= words { + start_offset - words + } else { + panic!("FlipTopFrame would underflow in West+FlipTopFrame"); + }; + (target_offset, limit_offset, Direction::Decreasing) + } + (AllocationType::FlipTopFrame, ArenaOrientation::East) => { + let start_offset = 0; // Beginning of the memory region + let limit_offset = unsafe { self.prev_alloc_offset() }; + let target_offset = start_offset + words; + (target_offset, limit_offset, Direction::Increasing) + } }; - (target_offset, limit_offset, Direction::Decreasing) - } - (AllocationType::FlipTopFrame, ArenaOrientation::East) => { - let start_offset = 0; // Beginning of the memory region - let limit_offset = unsafe { self.prev_alloc_offset() }; - let target_offset = start_offset + words; - (target_offset, limit_offset, Direction::Increasing) - } - }; - match direction { - Direction::Increasing => { - if target_offset > limit_offset { - panic_any(self.out_of_memory(alloc, Some(words))) + match direction { + Direction::Increasing => { + if target_offset > limit_offset { + panic_any(self.out_of_memory(alloc, Some(words))) + } } - } - Direction::Decreasing => { - if target_offset < limit_offset { - panic_any(self.out_of_memory(alloc, Some(words))) + Direction::Decreasing => { + if target_offset < limit_offset { + panic_any(self.out_of_memory(alloc, Some(words))) + } } - } - // TODO this check is imprecise and should take into account the size of the pointer! - Direction::IncreasingDeref => { - if target_offset >= limit_offset { - panic_any(self.out_of_memory(alloc, Some(words))) + // TODO this check is imprecise and should take into account the size of the pointer! + Direction::IncreasingDeref => { + if target_offset >= limit_offset { + panic_any(self.out_of_memory(alloc, Some(words))) + } } } } @@ -684,11 +1194,12 @@ impl NockStack { assert!(self.is_west()); }; + self.stack_epoch.fetch_add(1, Ordering::Relaxed); } /// Resets the NockStack. The top frame is west as in the initial creation of the NockStack. // Doesn't need an OOM check, pop analogue - pub(crate) fn reset(&mut self, top_slots: usize) { + pub unsafe fn reset(&mut self, top_slots: usize) { // Set offsets for west frame layout self.frame_offset = RESERVED + top_slots; self.stack_offset = self.frame_offset; @@ -699,19 +1210,18 @@ impl NockStack { .expect("Resetting a stack too small (should never happen)"); self.pc = false; - unsafe { - // Calculate slot offsets for previous pointers - let prev_frame_slot = self.frame_offset - (FRAME + 1); - let prev_stack_slot = self.frame_offset - (STACK + 1); - let prev_alloc_slot = self.frame_offset - (ALLOC + 1); + // Calculate slot offsets for previous pointers + let prev_frame_slot = self.frame_offset - (FRAME + 1); + let prev_stack_slot = self.frame_offset - (STACK + 1); + let prev_alloc_slot = self.frame_offset - (ALLOC + 1); - // Store null pointers for previous frame/stack and base pointer for previous alloc - *(self.derive_ptr(prev_frame_slot)) = ptr::null::() as u64; // "frame pointer" from "previous" frame - *(self.derive_ptr(prev_stack_slot)) = ptr::null::() as u64; // "stack pointer" from "previous" frame - *(self.derive_ptr(prev_alloc_slot)) = self.start as u64; // "alloc pointer" from "previous" frame + // Store null pointers for previous frame/stack and base pointer for previous alloc + *(self.derive_ptr(prev_frame_slot)) = ptr::null::() as u64; // "frame pointer" from "previous" frame + *(self.derive_ptr(prev_stack_slot)) = ptr::null::() as u64; // "stack pointer" from "previous" frame + *(self.derive_ptr(prev_alloc_slot)) = self.start as u64; // "alloc pointer" from "previous" frame - assert!(self.is_west()); - }; + assert!(self.is_west()); + self.stack_epoch.fetch_add(1, Ordering::Relaxed); } pub(crate) fn copying(&self) -> bool { @@ -774,6 +1284,53 @@ impl NockStack { self.least_space } + /// Advise the OS that the currently free top-frame gap does not need to remain resident. + /// + /// The advised range is page-aligned inward so live frame metadata and preserved allocations are + /// not included. This is intended for anonymous mmap-backed NockStacks after a top-frame flip. + pub fn advise_free_gap(&self, threshold_words: usize) -> io::Result { + if self.pc { + return Err(io::Error::new( + io::ErrorKind::InvalidInput, + "cannot advise NockStack free gap during pre-copy", + )); + } + + let (start_words, end_words) = self.free_gap_word_range(); + let free_gap_words = end_words.saturating_sub(start_words); + if free_gap_words < threshold_words || !self.arena_supports_anonymous_dontneed() { + return Ok(NockStackFreeGapAdvice { + free_gap_words, + advised_bytes: 0, + }); + } + + let start_bytes = start_words + .checked_mul(mem::size_of::()) + .ok_or_else(|| io::Error::other("NockStack free-gap start offset overflow"))?; + let end_bytes = end_words + .checked_mul(mem::size_of::()) + .ok_or_else(|| io::Error::other("NockStack free-gap end offset overflow"))?; + let advised_bytes = + madvise_dontneed_byte_range(self.start as usize, start_bytes, end_bytes)?; + Ok(NockStackFreeGapAdvice { + free_gap_words, + advised_bytes, + }) + } + + fn free_gap_word_range(&self) -> (usize, usize) { + if self.stack_offset <= self.alloc_offset { + (self.stack_offset, self.alloc_offset) + } else { + (self.alloc_offset, self.stack_offset) + } + } + + fn arena_supports_anonymous_dontneed(&self) -> bool { + self.arena.fd.is_none() && matches!(&self.arena.mapping, MappingKind::ReadWrite(_)) + } + /** Check to see if an allocation is in frame */ #[inline] pub(crate) unsafe fn is_in_frame(&self, ptr: *const T) -> bool { @@ -953,6 +1510,20 @@ impl NockStack { } } + #[inline] + unsafe fn reserved_slot_offset(&self, slot: usize) -> usize { + if self.pc { + self.free_slot_offset(slot) + } else { + self.slot_offset(slot) + } + } + + #[inline] + unsafe fn reserved_slot_pointer(&self, slot: usize) -> *mut u64 { + self.derive_ptr(self.reserved_slot_offset(slot)) + } + /** Mutable pointer into a slot in free space east of allocation pointer */ unsafe fn free_slot_east(&self, slot: usize) -> *mut u64 { self.derive_ptr(self.free_slot_east_offset(slot)) @@ -1001,34 +1572,19 @@ impl NockStack { /** Pointer to where the previous frame pointer is saved in a frame */ unsafe fn prev_frame_pointer_pointer(&self) -> *mut *mut u64 { - let res = if !self.pc { - self.slot_pointer(FRAME) - } else { - self.free_slot(FRAME) - }; - res as *mut *mut u64 + self.reserved_slot_pointer(FRAME) as *mut *mut u64 } /** Pointer to where the previous stack pointer is saved in a frame */ pub(crate) unsafe fn prev_stack_pointer_pointer(&self) -> *mut *mut u64 { - let res = if !self.pc { - self.slot_pointer(STACK) - } else { - self.free_slot(STACK) - }; - res as *mut *mut u64 + self.reserved_slot_pointer(STACK) as *mut *mut u64 } // Removed prev_alloc_offset_offset - it was using undefined functions /** Pointer to where the previous alloc pointer is saved in a frame */ unsafe fn prev_alloc_pointer_pointer(&self) -> *mut *mut u64 { - let res = if !self.pc { - self.slot_pointer(ALLOC) - } else { - self.free_slot(ALLOC) - }; - res as *mut *mut u64 + self.reserved_slot_pointer(ALLOC) as *mut *mut u64 } /** Allocation @@ -1042,73 +1598,45 @@ impl NockStack { * */ // Bump the alloc pointer for a west frame to make space for an allocation unsafe fn raw_alloc_west(&mut self, words: usize) -> *mut u64 { - self.alloc_would_oom(AllocationType::Alloc, words); if self.pc { - panic!("Allocation during cleanup phase is prohibited."); + self.panic_cannot_alloc_in_pc(words); } - // Calculate new offset with safe subtraction let new_alloc_offset = match self.alloc_offset.checked_sub(words) { Some(offset) => offset, - None => panic!("Alloc offset underflow in West frame"), + None => self.panic_out_of_memory_for(AllocationType::Alloc, words), }; + if new_alloc_offset < self.stack_offset { + self.panic_out_of_memory_for(AllocationType::Alloc, words); + } - // Update the space low-water-mark - let new_space = match new_alloc_offset.checked_sub(self.stack_offset) { - Some(space) => space, - None => panic_any( - self.out_of_memory(self.get_alloc_config(AllocationType::Alloc), Some(words)), - ), - }; + let new_space = new_alloc_offset - self.stack_offset; self.least_space = new_space.min(self.least_space); - // Derive pointer from the new offset let alloc_ptr = self.derive_ptr(new_alloc_offset); - - // Update the alloc offset self.alloc_offset = new_alloc_offset; debug_assert!(self.alloc_offset <= self.size, "Alloc offset out of bounds"); - - // Return the pointer to the allocated space alloc_ptr } /** Bump the alloc pointer for an east frame to make space for an allocation */ unsafe fn raw_alloc_east(&mut self, words: usize) -> *mut u64 { - self.alloc_would_oom(AllocationType::Alloc, words); if self.pc { - panic!("Allocation during cleanup phase is prohibited."); + self.panic_cannot_alloc_in_pc(words); } - // Get the pointer for the current allocation let alloc_ptr = self.derive_ptr(self.alloc_offset); - - // Calculate new offset with safe addition let new_alloc_offset = match self.alloc_offset.checked_add(words) { Some(offset) => offset, - None => panic!("Alloc offset overflow in East frame"), - }; - - let new_space = match self.stack_offset.checked_sub(new_alloc_offset) { - Some(space) => space, - None => panic_any( - self.out_of_memory(self.get_alloc_config(AllocationType::Alloc), Some(words)), - ), + None => self.panic_out_of_memory_for(AllocationType::Alloc, words), }; - self.least_space = new_space.min(self.least_space); - - // Check that the new offset is within bounds - if new_alloc_offset > self.size { - panic!( - "New allocation offset out of bounds: {} > {}", - new_alloc_offset, self.size - ); + if new_alloc_offset > self.stack_offset || new_alloc_offset > self.size { + self.panic_out_of_memory_for(AllocationType::Alloc, words); } - // Update the alloc offset + let new_space = self.stack_offset - new_alloc_offset; + self.least_space = new_space.min(self.least_space); self.alloc_offset = new_alloc_offset; - - // Return the pointer to the allocated space alloc_ptr } @@ -1366,6 +1894,7 @@ impl NockStack { unsafe fn assert_noun_in(&self, noun: Noun) { let mut dbg_stack = Vec::new(); dbg_stack.push(noun); + let space = self.fast_noun_space(); // Get the appropriate offsets based on pre-copy status let alloc_offset = if self.pc { @@ -1397,7 +1926,7 @@ impl NockStack { if let Some(subnoun) = dbg_stack.pop() { if let Ok(a) = subnoun.as_allocated() { // Get the pointer address - let np = a.to_raw_pointer() as usize; + let np = a.to_raw_pointer(&space) as usize; // Check if the noun is in the free space (which would be an error) if np >= low && np < hi { @@ -1406,8 +1935,9 @@ impl NockStack { // If it's a cell, check its head and tail too if let Right(c) = a.as_either() { - dbg_stack.push(c.tail()); - dbg_stack.push(c.head()); + let c_handle = c.in_space(&space); + dbg_stack.push(c_handle.tail().noun()); + dbg_stack.push(c_handle.head().noun()); } } } else { @@ -1450,27 +1980,43 @@ impl NockStack { /// This computation for num_locals is done in the east/west variants, but roughly speaking it's the input n words + 3 for prev frame alloc/stack/frame pointers pub fn frame_push(&mut self, num_locals: usize) { if self.pc { - panic!("frame_push during cleanup phase is prohibited."); + self.panic_cannot_alloc_in_pc(0); } - let words = num_locals + RESERVED; - self.alloc_would_oom(AllocationType::FramePush, words); + let words = match num_locals.checked_add(RESERVED) { + Some(words) => words, + None => self.panic_out_of_memory_for(AllocationType::FramePush, num_locals), + }; // Save current offsets let current_frame_offset = self.frame_offset; let current_stack_offset = self.stack_offset; let current_alloc_offset = self.alloc_offset; - - unsafe { - // Calculate new offsets - if self.is_west() { - self.frame_offset = self.alloc_offset - words; - } else { - self.frame_offset = self.alloc_offset + words; + let is_west = self.is_west(); + let new_frame_offset = if is_west { + let new_frame_offset = match current_alloc_offset.checked_sub(words) { + Some(offset) => offset, + None => self.panic_out_of_memory_for(AllocationType::FramePush, words), + }; + if new_frame_offset < current_stack_offset { + self.panic_out_of_memory_for(AllocationType::FramePush, words); } + new_frame_offset + } else { + let new_frame_offset = match current_alloc_offset.checked_add(words) { + Some(offset) => offset, + None => self.panic_out_of_memory_for(AllocationType::FramePush, words), + }; + if new_frame_offset > current_stack_offset || new_frame_offset > self.size { + self.panic_out_of_memory_for(AllocationType::FramePush, words); + } + new_frame_offset + }; + unsafe { + self.frame_offset = new_frame_offset; // Update stack and alloc offsets - self.alloc_offset = self.stack_offset; - self.stack_offset = self.frame_offset; + self.alloc_offset = current_stack_offset; + self.stack_offset = new_frame_offset; // Store pointers to previous frame in reserved slots let current_frame_ptr = self.derive_ptr(current_frame_offset); @@ -1525,7 +2071,7 @@ impl NockStack { * a west frame when pc == false has a west-oriented lightweight stack, * but when pc == true it becomes east-oriented.*/ pub(crate) unsafe fn push(&mut self) -> *mut T { - if self.is_west() && !self.pc || !self.is_west() && self.pc { + if self.is_west() ^ self.pc { self.push_west::() } else { self.push_east::() @@ -1535,82 +2081,35 @@ impl NockStack { /// Push onto a west-oriented lightweight stack, moving the stack_pointer. unsafe fn push_west(&mut self) -> *mut T { let words = word_size_of::(); - self.alloc_would_oom_( - Allocation { - orientation: ArenaOrientation::West, - alloc_type: AllocationType::Push, - pc: self.pc, - }, - words, - ); - - // Get the appropriate limit offset - let limit_offset = if self.pc { - self.prev_alloc_offset() - } else { - self.alloc_offset + let limit_offset = self.push_limit_offset(); + let new_stack_offset = match self.stack_offset.checked_add(words) { + Some(offset) => offset, + None => self.panic_out_of_memory_for(AllocationType::Push, words), }; - - // Get the current pointer at stack_offset (before we move it) - let alloc_ptr = self.derive_ptr(self.stack_offset); - - // Calculate the new stack offset - let new_stack_offset = self.stack_offset + words; - - // Check if we've gone past the limit if new_stack_offset > limit_offset { - panic!( - "Out of memory, alloc_would_oom didn't catch it. memory_state: {:#?}", - self.memory_state(Some(words)) - ); - } else { - // Update stack offset and return the original pointer - self.stack_offset = new_stack_offset; - alloc_ptr as *mut T + self.panic_out_of_memory_for(AllocationType::Push, words); } + + let alloc_ptr = self.derive_ptr(self.stack_offset); + self.stack_offset = new_stack_offset; + alloc_ptr as *mut T } /// Push onto an east-oriented ligthweight stack, moving the stack_pointer unsafe fn push_east(&mut self) -> *mut T { let words = word_size_of::(); - self.alloc_would_oom_( - Allocation { - orientation: ArenaOrientation::East, - alloc_type: AllocationType::Push, - pc: self.pc, - }, - words, - ); - - // Get the appropriate limit offset - let limit_offset = if self.pc { - self.prev_alloc_offset() - } else { - self.alloc_offset + let limit_offset = self.push_limit_offset(); + let new_stack_offset = match self.stack_offset.checked_sub(words) { + Some(offset) => offset, + None => self.panic_out_of_memory_for(AllocationType::Push, words), }; - - // Calculate the new stack offset - if self.stack_offset < words { - panic!("Stack offset underflow during push_east"); - } - let new_stack_offset = self.stack_offset - words; - - // Check if we've gone below the limit if new_stack_offset < limit_offset { - panic!( - "Out of memory, alloc_would_oom didn't catch it. memory_state: {:#?}", - self.memory_state(Some(words)) - ); - } else { - // Get the pointer at the new offset - let alloc_ptr = self.derive_ptr(new_stack_offset); - - // Update stack offset - self.stack_offset = new_stack_offset; - - // Return the pointer at the new offset - alloc_ptr as *mut T + self.panic_out_of_memory_for(AllocationType::Push, words); } + + let alloc_ptr = self.derive_ptr(new_stack_offset); + self.stack_offset = new_stack_offset; + alloc_ptr as *mut T } /** Pop a west-oriented lightweight stack, moving the stack pointer. */ @@ -1638,7 +2137,7 @@ impl NockStack { // Re: #684: We don't need OOM checks on pop #[inline] pub(crate) unsafe fn pop(&mut self) { - if self.is_west() && !self.pc || !self.is_west() && self.pc { + if self.is_west() ^ self.pc { self.pop_west::(); } else { self.pop_east::(); @@ -1651,7 +2150,7 @@ impl NockStack { * but when pc == true it becomes east-oriented.*/ #[inline] pub(crate) unsafe fn top(&mut self) -> *mut T { - if self.is_west() && !self.pc || !self.is_west() && self.pc { + if self.is_west() ^ self.pc { self.top_west() } else { self.top_east() @@ -1694,6 +2193,7 @@ impl NockStack { pub(crate) fn no_junior_pointers(&self, noun: Noun) -> bool { unsafe { if let Ok(c) = noun.as_cell() { + let space = self.fast_noun_space(); let mut dbg_stack = Vec::new(); // Start with the current frame's offsets @@ -1719,7 +2219,7 @@ impl NockStack { }; // Determine the cell pointer offset - let cell_ptr_offset = (c.to_raw_pointer() as usize - self.start as usize) / 8; + let cell_ptr_offset = (c.to_raw_pointer(&space) as usize - self.start as usize) / 8; // Determine range for the cell's frame let (range_lo_offset, range_hi_offset) = loop { @@ -1791,11 +2291,12 @@ impl NockStack { let range_hi_ptr = self.derive_ptr(range_hi_offset); // Check all nouns in the tree - dbg_stack.push(c.head()); - dbg_stack.push(c.tail()); + let c_handle = c.in_space(&space); + dbg_stack.push(c_handle.head().noun()); + dbg_stack.push(c_handle.tail().noun()); while let Some(n) = dbg_stack.pop() { if let Ok(a) = n.as_allocated() { - let ptr = a.to_raw_pointer(); + let ptr = a.to_raw_pointer(&space); // Calculate pointer offset let ptr_offset = (ptr as usize - self.start as usize) / 8; @@ -1813,8 +2314,9 @@ impl NockStack { // Continue traversing if it's a cell if let Some(c) = a.cell() { - dbg_stack.push(c.tail()); - dbg_stack.push(c.head()); + let c_handle = c.in_space(&space); + dbg_stack.push(c_handle.tail().noun()); + dbg_stack.push(c_handle.head().noun()); } } } @@ -1930,6 +2432,69 @@ impl NockStack { } } +#[cfg(unix)] +fn madvise_dontneed_byte_range( + base_addr: usize, + start_bytes: usize, + end_bytes: usize, +) -> io::Result { + if start_bytes >= end_bytes { + return Ok(0); + } + let page = page_size()?; + let start_addr = base_addr + .checked_add(start_bytes) + .ok_or_else(|| io::Error::other("NockStack madvise start address overflow"))?; + let end_addr = base_addr + .checked_add(end_bytes) + .ok_or_else(|| io::Error::other("NockStack madvise end address overflow"))?; + let aligned_start = align_up(start_addr, page) + .ok_or_else(|| io::Error::other("NockStack madvise aligned start overflow"))?; + let aligned_end = align_down(end_addr, page); + if aligned_start >= aligned_end { + return Ok(0); + } + let len = aligned_end - aligned_start; + let ret = + unsafe { libc::madvise(aligned_start as *mut libc::c_void, len, libc::MADV_DONTNEED) }; + if ret != 0 { + return Err(io::Error::last_os_error()); + } + Ok(len) +} + +#[cfg(not(unix))] +fn madvise_dontneed_byte_range( + _base_addr: usize, + _start_bytes: usize, + _end_bytes: usize, +) -> io::Result { + Ok(0) +} + +#[cfg(unix)] +fn page_size() -> io::Result { + let page = unsafe { libc::sysconf(libc::_SC_PAGESIZE) }; + if page <= 0 { + return Err(io::Error::last_os_error()); + } + Ok(page as usize) +} + +#[cfg(unix)] +fn align_up(value: usize, align: usize) -> Option { + debug_assert!(align.is_power_of_two()); + value + .checked_add(align - 1) + .map(|value| align_down(value, align)) +} + +#[cfg(unix)] +fn align_down(value: usize, align: usize) -> usize { + debug_assert!(align.is_power_of_two()); + value & !(align - 1) +} + impl NounAllocator for NockStack { unsafe fn alloc_indirect(&mut self, words: usize) -> *mut u64 { self.indirect_alloc(words) @@ -1946,6 +2511,10 @@ impl NounAllocator for NockStack { unsafe fn equals(&mut self, a: *mut Noun, b: *mut Noun) -> bool { crate::unifying_equality::unifying_equality(self, a, b) } + + fn noun_space(&self) -> NounSpace { + NockStack::noun_space(self) + } } /// Immutable, acyclic objects which may be copied up the stack @@ -1963,9 +2532,10 @@ impl Preserve for () { impl Preserve for IndirectAtom { unsafe fn preserve(&mut self, stack: &mut NockStack) { - let size = indirect_raw_size(*self); + let space = stack.fast_noun_space(); + let size = indirect_raw_size(*self, &space); let buf = stack.struct_alloc_in_previous_frame::(size); - copy_nonoverlapping(self.to_raw_pointer(), buf, size); + copy_nonoverlapping(self.to_raw_pointer(&space), buf, size); *self = IndirectAtom::from_raw_pointer(buf); } unsafe fn assert_in_stack(&self, stack: &NockStack) { @@ -2001,21 +2571,25 @@ impl Preserve for Noun { /// This version tries to bail earlier than the old one and we're using a Vec /// for the worklist. unsafe fn noun_preserve(stack: &mut NockStack, noun: &mut Noun) { - assert_acyclic!(*noun); - assert_no_forwarding_pointers!(*noun); - assert_no_junior_pointers!(stack, *noun); - let root_allocated = match noun.as_either_direct_allocated() { Either::Left(_direct) => return, Either::Right(allocated) => allocated, }; - if let Some(new_allocated) = root_allocated.forwarding_pointer() { + let space = stack.fast_noun_space(); + #[cfg(any(feature = "check_acyclic", feature = "check_forwarding"))] + { + crate::assert_acyclic!(space, *noun); + crate::assert_no_forwarding_pointers!(space, *noun); + } + crate::assert_no_junior_pointers!(stack, *noun); + + if let Some(new_allocated) = root_allocated.forwarding_pointer(&space) { *noun = new_allocated.as_noun(); return; } - if !stack.is_in_frame(root_allocated.to_raw_pointer()) { + if !stack.is_in_frame(root_allocated.to_raw_pointer(&space)) { return; } @@ -2029,35 +2603,37 @@ unsafe fn noun_preserve(stack: &mut NockStack, noun: &mut Noun) { *dest_ptr = value; }, Either::Right(allocated) => unsafe { - if let Some(new_allocated) = allocated.forwarding_pointer() { + if let Some(new_allocated) = allocated.forwarding_pointer(&space) { *dest_ptr = new_allocated.as_noun(); continue; } - if !stack.is_in_frame(allocated.to_raw_pointer()) { + if !stack.is_in_frame(allocated.to_raw_pointer(&space)) { *dest_ptr = value; continue; } match allocated.as_either() { Either::Left(mut indirect) => { - let alloc = stack.indirect_alloc_in_previous_frame(indirect.size()); + let indirect_handle = indirect.as_atom().in_space(&space); + let alloc = stack.indirect_alloc_in_previous_frame(indirect_handle.size()); copy_nonoverlapping( - indirect.to_raw_pointer(), + indirect_handle.raw_pointer(), alloc, - indirect_raw_size(indirect), + indirect_raw_size(indirect, &space), ); - indirect.set_forwarding_pointer(alloc); + indirect.set_forwarding_pointer(alloc, &space); *dest_ptr = IndirectAtom::from_raw_pointer(alloc).as_noun(); } Either::Right(mut cell) => { let alloc = stack.struct_alloc_in_previous_frame::(1); - (*alloc).metadata = (*cell.to_raw_pointer()).metadata; + (*alloc).metadata = (*cell.to_raw_pointer(&space)).metadata; - let tail = cell.tail(); - let head = cell.head(); + let cell_handle = cell.in_space(&space); + let tail = cell_handle.tail().noun(); + let head = cell_handle.head().noun(); - cell.set_forwarding_pointer(alloc); + cell.set_forwarding_pointer(alloc, &space); work.push((tail, &mut (*alloc).tail)); work.push((head, &mut (*alloc).head)); @@ -2069,9 +2645,12 @@ unsafe fn noun_preserve(stack: &mut NockStack, noun: &mut Noun) { } } - assert_acyclic!(*noun); - assert_no_forwarding_pointers!(*noun); - assert_no_junior_pointers!(stack, *noun); + #[cfg(any(feature = "check_acyclic", feature = "check_forwarding"))] + { + crate::assert_acyclic!(space, *noun); + crate::assert_no_forwarding_pointers!(space, *noun); + } + crate::assert_no_junior_pointers!(stack, *noun); } impl Stack for NockStack { @@ -2123,13 +2702,16 @@ impl Preserve for AllocationError { #[cfg(test)] mod test { use std::iter::FromIterator; + #[cfg(debug_assertions)] use std::panic::{catch_unwind, AssertUnwindSafe}; + use proptest::prelude::*; + use super::*; use crate::jets::cold::test::{make_noun_list, make_test_stack}; use crate::jets::cold::{NounList, Nounable}; use crate::mem::NockStack; - use crate::noun::D; + use crate::noun::{Noun, D}; fn test_noun_list_alloc_fn( stack_size: usize, @@ -2139,6 +2721,7 @@ mod test { // const STACK_SIZE: usize = 1; // println!("TEST_SIZE: {}", STACK_SIZE); let mut stack = make_test_stack(stack_size); + let space = stack.fast_noun_space(); // Stack size 1 works until 15 elements, 14 passes, 15 fails. // const ITEM_COUNT: u64 = 15; let vec = Vec::from_iter(0..item_count); @@ -2148,7 +2731,7 @@ mod test { assert!(!noun_list.0.is_null()); let noun = noun_list.into_noun(&mut stack); let new_noun_list: NounList = - ::from_noun::(&mut stack, &noun)?; + ::from_noun::(&mut stack, &noun, &space)?; let mut tracking_item_count = 0; println!("items: {:?}", items); for (a, b) in new_noun_list.zip(items.iter()) { @@ -2168,6 +2751,7 @@ mod test { } // cargo test -p nockvm test_noun_list_alloc -- --nocapture + #[cfg(debug_assertions)] #[test] #[cfg_attr(miri, ignore)] fn test_noun_list_alloc() { @@ -2184,7 +2768,9 @@ mod test { } // cargo test -p nockvm test_frame_push -- --nocapture + #[cfg(debug_assertions)] #[test] + #[cfg_attr(miri, ignore = "memfd_create unsupported in Miri")] fn test_frame_push() { // fails at 100, passes at 99, top_slots default to 100? const PASSES: usize = 503; @@ -2201,7 +2787,9 @@ mod test { } // cargo test -p nockvm test_stack_push -- --nocapture + #[cfg(debug_assertions)] #[test] + #[cfg_attr(miri, ignore = "memfd_create unsupported in Miri")] fn test_stack_push() { const PASSES: usize = 506; const STACK_SIZE: usize = 512; @@ -2220,7 +2808,9 @@ mod test { } // cargo test -p nockvm test_frame_and_stack_push -- --nocapture + #[cfg(debug_assertions)] #[test] + #[cfg_attr(miri, ignore = "memfd_create unsupported in Miri")] fn test_frame_and_stack_push() { const STACK_SIZE: usize = 514; // to make sure of an odd space for the stack push const SUCCESS_PUSHES: usize = 101; @@ -2256,7 +2846,9 @@ mod test { // cargo test -p nockvm test_slot_pointer -- --nocapture // Test the slot_pointer checking by pushing frames and slots until we run out of space + #[cfg(debug_assertions)] #[test] + #[cfg_attr(miri, ignore = "memfd_create unsupported in Miri")] fn test_slot_pointer() { const STACK_SIZE: usize = 512; const SLOT_POINTERS: usize = 32; @@ -2286,7 +2878,9 @@ mod test { // cargo test -p nockvm test_prev_alloc -- --nocapture // Test the alloc in previous frame checking by pushing a frame and then allocating in the previous frame until we run out of space + #[cfg(debug_assertions)] #[test] + #[cfg_attr(miri, ignore = "memfd_create unsupported in Miri")] fn test_prev_alloc() { const STACK_SIZE: usize = 512; const SUCCESS_ALLOCS: usize = 503; @@ -2321,4 +2915,59 @@ mod test { "Didn't get expected alloc error", ); } + + #[test] + #[cfg_attr(miri, ignore = "madvise unsupported in Miri")] + fn advise_free_gap_respects_threshold() { + let stack = NockStack::new(NOCK_STACK_1KB * 64, 0); + let advice = stack + .advise_free_gap(usize::MAX) + .expect("advise below threshold"); + assert!(advice.free_gap_words > 0); + assert_eq!(advice.advised_bytes, 0); + } + + #[cfg(unix)] + #[test] + #[cfg_attr(miri, ignore = "madvise unsupported in Miri")] + fn advise_free_gap_page_aligns_and_advises_anonymous_mmap() { + let stack = NockStack::new(NOCK_STACK_1KB * 1024, 0); + let advice = stack.advise_free_gap(0).expect("advise free gap"); + assert!(advice.free_gap_words > 0); + assert!(advice.advised_bytes > 0); + assert_eq!(advice.advised_bytes % page_size().expect("page size"), 0); + } + + proptest! { + #[test] + #[cfg_attr(miri, ignore = "memfd_create unsupported in Miri")] + fn arena_offset_round_trip(words in 1usize..256, offset in 0usize..2048) { + let arena = Arena::allocate(words).expect("arena allocation failed"); + let off = offset % words; + let ptr = unsafe { arena.base_ptr().add(off << 3) }; + let round: StackOffsetWords = arena.offset_from_ptr(ptr); + prop_assert_eq!(round.words() as usize, off); + let resolved = arena.ptr_from_offset(round); + prop_assert_eq!(resolved, ptr); + } + + #[test] + #[cfg_attr(miri, ignore = "memfd_create unsupported in Miri")] + fn stack_offset_round_trip_across_orientations(words in (RESERVED + 8)..512usize, offset in 0usize..4096) { + let mut stack = NockStack::new(words, 0); + let total_words = stack.arena().words(); + let off = offset % total_words; + let ptr = unsafe { stack.arena().base_ptr().add(off << 3) }; + let round = stack.offset_from_ptr(ptr); + prop_assert_eq!(round.words() as usize, off); + prop_assert_eq!(stack.ptr_from_offset(round), ptr); + + unsafe { stack.flip_top_frame(0); } + let off2 = (offset + 1) % total_words; + let ptr2 = unsafe { stack.arena().base_ptr().add(off2 << 3) }; + let round2 = stack.offset_from_ptr(ptr2); + prop_assert_eq!(round2.words() as usize, off2); + prop_assert_eq!(stack.ptr_from_offset(round2), ptr2); + } + } } diff --git a/crates/nockvm/rust/nockvm/src/mug.rs b/crates/nockvm/rust/nockvm/src/mug.rs index 6ef9e2f06..35c16f3f5 100644 --- a/crates/nockvm/rust/nockvm/src/mug.rs +++ b/crates/nockvm/rust/nockvm/src/mug.rs @@ -2,42 +2,40 @@ use either::Either::*; use murmur3::murmur3_32_of_slice; use crate::mem::*; -use crate::noun::{Allocated, Atom, DirectAtom, Noun}; -use crate::{assert_acyclic, assert_no_forwarding_pointers, assert_no_junior_pointers}; +use crate::noun::{Allocated, Atom, DirectAtom, Noun, NounSpace}; crate::gdb!(); // Murmur3 hash an atom with a given padded length -fn muk_u32(syd: u32, len: usize, key: Atom) -> u32 { - match key.as_either() { - Left(direct) => murmur3_32_of_slice(&direct.data().to_le_bytes()[0..len], syd), - Right(indirect) => murmur3_32_of_slice(&indirect.as_ne_bytes()[..len], syd), - } +fn muk_u32(syd: u32, len: usize, key: Atom, space: &NounSpace) -> u32 { + murmur3_32_of_slice(&key.in_space(space).as_ne_bytes()[..len], syd) } /** Byte size of an atom. * * Assumes atom is normalized */ -pub fn met3_usize(atom: Atom) -> usize { +pub fn met3_usize(atom: Atom, space: &NounSpace) -> usize { match atom.as_either() { Left(direct) => (64 - (direct.data().leading_zeros() as usize) + 7) >> 3, Right(indirect) => { - let last_word = unsafe { *(indirect.data_pointer().add(indirect.size() - 1)) }; + let indirect_handle = indirect.as_atom().in_space(space); + let size = indirect_handle.size(); + let last_word = unsafe { *(indirect_handle.data_pointer().add(size - 1)) }; let last_word_bytes = (64 - (last_word.leading_zeros() as usize) + 7) >> 3; - ((indirect.size() - 1) << 3) + last_word_bytes + ((size - 1) << 3) + last_word_bytes } } } -fn mum_u32(syd: u32, fal: u32, key: Atom) -> u32 { - let wyd = met3_usize(key); +fn mum_u32(syd: u32, fal: u32, key: Atom, space: &NounSpace) -> u32 { + let wyd = met3_usize(key, space); let mut i = 0; loop { if i == 8 { break fal; } else { - let haz = muk_u32(syd, wyd, key); + let haz = muk_u32(syd, wyd, key, space); let ham = (haz >> 31) ^ (haz & !(1 << 31)); if ham == 0 { i += 1; @@ -49,8 +47,8 @@ fn mum_u32(syd: u32, fal: u32, key: Atom) -> u32 { } } -pub fn calc_atom_mug_u32(atom: Atom) -> u32 { - mum_u32(0xCAFEBABE, 0x7FFF, atom) +pub fn calc_atom_mug_u32(atom: Atom, space: &NounSpace) -> u32 { + mum_u32(0xCAFEBABE, 0x7FFF, atom, space) } /** Unsafe because this passes a direct atom to mum_u32 made by concatenating the two mugs, @@ -59,19 +57,20 @@ pub fn calc_atom_mug_u32(atom: Atom) -> u32 { * # Safety * head_mug and tail_mug both have msb 0. */ -pub unsafe fn calc_cell_mug_u32(head_mug: u32, tail_mug: u32) -> u32 { +pub unsafe fn calc_cell_mug_u32(head_mug: u32, tail_mug: u32, space: &NounSpace) -> u32 { let cat_mugs = (head_mug as u64) | ((tail_mug as u64) << 32); mum_u32( 0xDEADBEEF, 0xFFFE, DirectAtom::new_unchecked(cat_mugs).as_atom(), + space, ) // this is safe on mugs since mugs are 31 bits } -pub fn get_mug(noun: Noun) -> Option { +pub fn get_mug(noun: Noun, space: &NounSpace) -> Option { match noun.as_either_direct_allocated() { - Left(direct) => Some(calc_atom_mug_u32(direct.as_atom())), - Right(allocated) => allocated.get_cached_mug(), + Left(direct) => Some(calc_atom_mug_u32(direct.as_atom(), space)), + Right(allocated) => allocated.get_cached_mug(space), } } @@ -85,9 +84,9 @@ const MASK_OUT_MUG: u64 = !(u32::MAX as u64); * Ensure the calculated mug is correct or this will result in incorrect mugs being returned. * This could cause jet mismatches. */ -pub unsafe fn set_mug(allocated: &mut Allocated, mug: u32) { - let metadata = allocated.get_metadata(); - allocated.set_metadata((metadata & MASK_OUT_MUG) | (mug as u64)); +pub unsafe fn set_mug(allocated: &mut Allocated, mug: u32, space: &NounSpace) { + let metadata = allocated.get_metadata(space); + allocated.set_metadata((metadata & MASK_OUT_MUG) | (mug as u64), space); } /** Calculate and cache the mug for a noun, but do *not* recursively calculate cache mugs for @@ -95,46 +94,59 @@ pub unsafe fn set_mug(allocated: &mut Allocated, mug: u32) { * * If called on a cell with no mug cached for the head or tail, this function will return `None`. */ -pub fn allocated_mug_u32_one(allocated: &mut Allocated) -> Option { - match allocated.get_cached_mug() { +pub fn allocated_mug_u32_one(allocated: &mut Allocated, space: &NounSpace) -> Option { + match allocated.get_cached_mug(space) { Some(mug) => Some(mug), None => match allocated.as_either() { Left(indirect) => { - let mug = calc_atom_mug_u32(indirect.as_atom()); + let mug = calc_atom_mug_u32(indirect.as_atom(), space); unsafe { - set_mug(allocated, mug); + set_mug(allocated, mug, space); } Some(mug) } - Right(cell) => match (get_mug(cell.head()), get_mug(cell.tail())) { - (Some(head_mug), Some(tail_mug)) => { - let mug = unsafe { calc_cell_mug_u32(head_mug, tail_mug) }; - unsafe { - set_mug(allocated, mug); + Right(cell) => { + let cell = cell.in_space(space); + match ( + get_mug(cell.head().noun(), space), + get_mug(cell.tail().noun(), space), + ) { + (Some(head_mug), Some(tail_mug)) => { + let mug = unsafe { calc_cell_mug_u32(head_mug, tail_mug, space) }; + unsafe { + set_mug(allocated, mug, space); + } + Some(mug) } - Some(mug) + _ => None, } - _ => None, - }, + } }, } } -pub fn mug_u32_one(mut noun: Noun) -> Option { +pub fn mug_u32_one(mut noun: Noun, space: &NounSpace) -> Option { match noun.as_ref_mut_either_direct_allocated() { - Left(direct) => Some(calc_atom_mug_u32(direct.as_atom())), - Right(allocated) => allocated_mug_u32_one(allocated), + Left(direct) => Some(calc_atom_mug_u32(direct.as_atom(), space)), + Right(allocated) => allocated_mug_u32_one(allocated, space), } } pub fn mug_u32(stack: &mut NockStack, noun: Noun) -> u32 { - if let Some(mug) = get_mug(noun) { - return mug; + { + let space = stack.fast_noun_space(); + if let Some(mug) = get_mug(noun, &space) { + return mug; + } } - assert_acyclic!(noun); - assert_no_forwarding_pointers!(noun); - assert_no_junior_pointers!(stack, noun); + #[cfg(any(feature = "check_acyclic", feature = "check_forwarding"))] + { + let _space = stack.fast_noun_space(); + crate::assert_acyclic!(_space, noun); + crate::assert_no_forwarding_pointers!(_space, noun); + } + crate::assert_no_junior_pointers!(stack, noun); stack.frame_push(0); unsafe { @@ -152,35 +164,60 @@ pub fn mug_u32(stack: &mut NockStack, noun: Noun) -> u32 { } continue; } // no point in calculating a direct mug here as we wont cache it - Right(mut allocated) => match allocated.get_cached_mug() { - Some(_mug) => { - unsafe { - stack.pop::(); - } - continue; - } - None => match allocated.as_either() { - Left(indirect) => unsafe { - set_mug(&mut allocated, calc_atom_mug_u32(indirect.as_atom())); - stack.pop::(); + Right(mut allocated) => { + let cached = { + let space = stack.fast_noun_space(); + allocated.get_cached_mug(&space) + }; + match cached { + Some(_mug) => { + unsafe { + stack.pop::(); + } continue; - }, - Right(cell) => unsafe { - match (get_mug(cell.head()), get_mug(cell.tail())) { - (Some(head_mug), Some(tail_mug)) => { - set_mug(&mut allocated, calc_cell_mug_u32(head_mug, tail_mug)); - stack.pop::(); - continue; - } - _ => { - *(stack.push()) = cell.tail(); - *(stack.push()) = cell.head(); - continue; + } + None => match allocated.as_either() { + Left(indirect) => unsafe { + let space = stack.fast_noun_space(); + set_mug( + &mut allocated, + calc_atom_mug_u32(indirect.as_atom(), &space), + &space, + ); + stack.pop::(); + continue; + }, + Right(cell) => { + let (head, tail, cached) = { + let space = stack.fast_noun_space(); + let cell = cell.in_space(&space); + let head = cell.head().noun(); + let tail = cell.tail().noun(); + (head, tail, (get_mug(head, &space), get_mug(tail, &space))) + }; + match cached { + (Some(head_mug), Some(tail_mug)) => unsafe { + let space = stack.fast_noun_space(); + set_mug( + &mut allocated, + calc_cell_mug_u32(head_mug, tail_mug, &space), + &space, + ); + stack.pop::(); + continue; + }, + _ => { + unsafe { + *(stack.push()) = tail; + *(stack.push()) = head; + } + continue; + } } } }, - }, - }, + } + } } } } @@ -188,11 +225,16 @@ pub fn mug_u32(stack: &mut NockStack, noun: Noun) -> u32 { stack.frame_pop(); } - assert_acyclic!(noun); - assert_no_forwarding_pointers!(noun); - assert_no_junior_pointers!(stack, noun); + #[cfg(any(feature = "check_acyclic", feature = "check_forwarding"))] + { + let _space = stack.fast_noun_space(); + crate::assert_acyclic!(_space, noun); + crate::assert_no_forwarding_pointers!(_space, noun); + } + crate::assert_no_junior_pointers!(stack, noun); - get_mug(noun).expect("Noun should have a mug once it is mugged.") + let space = stack.fast_noun_space(); + get_mug(noun, &space).expect("Noun should have a mug once it is mugged.") } pub fn mug(stack: &mut NockStack, noun: Noun) -> DirectAtom { diff --git a/crates/nockvm/rust/nockvm/src/noun.rs b/crates/nockvm/rust/nockvm/src/noun.rs index abf12f8ea..28947c229 100644 --- a/crates/nockvm/rust/nockvm/src/noun.rs +++ b/crates/nockvm/rust/nockvm/src/noun.rs @@ -1,5 +1,7 @@ use std::slice::{from_raw_parts, from_raw_parts_mut}; -use std::{error, fmt, ptr}; +use std::sync::atomic::{AtomicU64, Ordering}; +use std::sync::Arc; +use std::{error, fmt, ptr, str}; use bitvec::prelude::{BitSlice, Lsb0}; use either::{Either, Left, Right}; @@ -8,7 +10,9 @@ use intmap::IntMap; use nockvm_macros::tas; use static_assertions::assert_cfg; -use crate::mem::{word_size_of, NockStack}; +use crate::mem::{word_size_of, Arena, NockStack}; +use crate::offset::{PmaOffsetWords, WordOffset}; +use crate::pma::Pma; crate::gdb!(); @@ -38,6 +42,905 @@ pub(crate) const CELL_TAG: u64 = INDIRECT_MASK; /** Tag mask for a cell. */ pub(crate) const CELL_MASK: u64 = !(u64::MAX >> 3); +pub(crate) const LOCATION_BIT: u64 = 1 << 60; + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum PtrLocation { + Stack, + Offset, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum AllocLocation { + Stack, + PmaPtr, + PmaOffset, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum NounRepr { + Direct, + Indirect(AllocLocation), + Cell(AllocLocation), + Forwarding(AllocLocation), +} + +impl AllocLocation { + pub fn is_stack(self) -> bool { + matches!(self, AllocLocation::Stack) + } + + pub fn is_pma(self) -> bool { + matches!(self, AllocLocation::PmaPtr | AllocLocation::PmaOffset) + } + + pub fn is_offset(self) -> bool { + matches!(self, AllocLocation::PmaOffset) + } +} + +impl NounRepr { + pub fn is_allocated(self) -> bool { + !matches!(self, NounRepr::Direct) + } + + pub fn location(self) -> Option { + match self { + NounRepr::Direct => None, + NounRepr::Indirect(loc) => Some(loc), + NounRepr::Cell(loc) => Some(loc), + NounRepr::Forwarding(loc) => Some(loc), + } + } +} + +#[derive(Debug, Clone, Copy)] +struct TaggedPtr(u64); + +impl TaggedPtr { + #[inline(always)] + fn from_raw(raw: u64) -> Self { + Self(raw) + } + + #[inline(always)] + unsafe fn from_stack_ptr(ptr: *const u8, tag: u64) -> Self { + debug_assert!( + (ptr as usize) & 0x7 == 0, + "Stack pointer {:p} not 8-byte aligned", + ptr + ); + Self(((ptr as u64) >> 3) | tag) + } + + #[inline(always)] + fn from_offset(words: O, tag: u64) -> Self { + assert!( + words.words() < LOCATION_BIT, + "offset {} exceeds payload capacity", + words.words() + ); + Self(words.words() | LOCATION_BIT | tag) + } + + #[inline(always)] + fn location(self) -> PtrLocation { + if self.0 & LOCATION_BIT == 0 { + PtrLocation::Stack + } else { + PtrLocation::Offset + } + } + + #[inline(always)] + fn payload(self, mask: u64) -> u64 { + self.0 & !(mask | LOCATION_BIT) + } + + #[inline(always)] + fn resolve_const_trusted(self, mask: u64, space: &NounSpace) -> *const u8 { + let payload = self.0 & !(mask | LOCATION_BIT); + if self.0 & LOCATION_BIT == 0 { + let payload = usize::try_from( + payload + .checked_shl(3) + .expect("stack pointer payload exceeds addressable range"), + ) + .expect("stack pointer payload exceeds usize addressable range"); + payload as *const u8 + } else { + let base = unsafe { space.pma_base_unchecked() }; + let offset = PmaOffsetWords::from_words(payload) + .checked_bytes_usize() + .expect("PMA offset exceeds addressable range"); + base.checked_add(offset) + .expect("PMA offset exceeds addressable range") as *const u8 + } + } + + fn resolve_const(self, mask: u64, space: &NounSpace) -> *const u8 { + match self.location() { + PtrLocation::Stack => space.resolve_stack_ptr(self.payload(mask)), + PtrLocation::Offset => { + space.resolve_pma_ptr(PmaOffsetWords::from_words(self.payload(mask))) + } + } + } + + #[inline(always)] + fn resolve_mut(self, mask: u64, space: &NounSpace) -> *mut u8 { + self.resolve_const(mask, space) as *mut u8 + } + + #[inline(always)] + fn raw(self) -> u64 { + self.0 + } +} + +pub struct NounSpace { + stack: Option>, + pma: Option>, + stack_epoch: Option>, + stack_epoch_ptr: Option<*const AtomicU64>, + stack_epoch_snapshot: Option, + stack_base: Option, + stack_end: Option, + pma_base: Option, + pma_end: Option, + pma_words: Option, + extra_ptr_ranges: Vec<(usize, usize)>, +} + +impl NounSpace { + #[inline] + fn arena_cache(arena: Option<&Arena>) -> (Option, Option, Option) { + if let Some(arena) = arena { + let base = arena.base_ptr() as usize; + let end = base + .checked_add(arena.len_bytes()) + .expect("arena bounds exceed usize address space"); + (Some(base), Some(end), Some(arena.words())) + } else { + (None, None, None) + } + } + + #[inline] + fn pma_arena_cache(arena: Option<&Arena>) -> (Option, Option, Option) { + if let Some(arena) = arena { + let base = arena.base_ptr() as usize; + let end = base + .checked_add(arena.reserved_len_bytes()) + .expect("PMA reserved bounds exceed usize address space"); + (Some(base), Some(end), Some(arena.words())) + } else { + (None, None, None) + } + } + + pub fn from_arenas(stack: Option>, pma: Option>) -> Self { + let (stack_base, stack_end, _) = Self::arena_cache(stack.as_deref()); + let (pma_base, pma_end, pma_words) = Self::pma_arena_cache(pma.as_deref()); + Self { + stack, + pma, + stack_epoch: None, + stack_epoch_ptr: None, + stack_epoch_snapshot: None, + stack_base, + stack_end, + pma_base, + pma_end, + pma_words, + extra_ptr_ranges: Vec::new(), + } + } + + pub fn from_stack(stack: &NockStack, pma: Option>) -> Self { + let stack_arena = Arc::clone(stack.arena()); + let stack_epoch = stack.stack_epoch(); + let (stack_base, stack_end, _) = Self::arena_cache(Some(stack.arena_ref())); + let (pma_base, pma_end, pma_words) = Self::pma_arena_cache(pma.as_deref()); + Self { + stack: Some(stack_arena), + pma, + stack_epoch: Some(stack_epoch), + stack_epoch_ptr: Some(stack.stack_epoch_ref() as *const AtomicU64), + stack_epoch_snapshot: Some(stack.stack_epoch_snapshot()), + stack_base, + stack_end, + pma_base, + pma_end, + pma_words, + extra_ptr_ranges: Vec::new(), + } + } + + pub(crate) fn from_stack_ephemeral(stack: &NockStack) -> Self { + let (stack_base, stack_end, _) = Self::arena_cache(Some(stack.arena_ref())); + let (pma_base, pma_end, pma_words) = Self::pma_arena_cache(stack.pma_ref()); + Self { + stack: None, + pma: None, + stack_epoch: None, + stack_epoch_ptr: Some(stack.stack_epoch_ref() as *const AtomicU64), + stack_epoch_snapshot: Some(stack.stack_epoch_snapshot()), + stack_base, + stack_end, + pma_base, + pma_end, + pma_words, + extra_ptr_ranges: Vec::new(), + } + } + + #[inline(always)] + unsafe fn pma_base_unchecked(&self) -> usize { + debug_assert!( + self.pma_base.is_some(), + "PMA arena is required to resolve offset nouns" + ); + self.pma_base.unwrap_unchecked() + } + + pub fn new(stack: &NockStack, pma: &Pma) -> Self { + Self::from_stack(stack, Some(Arc::clone(pma.arena()))) + } + + pub fn stack_only(stack: &NockStack) -> Self { + Self::from_stack(stack, None) + } + + pub fn pma_only(pma: &Pma) -> Self { + Self::from_arenas(None, Some(Arc::clone(pma.arena()))) + } + + pub fn empty() -> Self { + Self::from_arenas(None, None) + } + + pub fn with_extra_ptr_ranges(mut self, ranges: Vec<(usize, usize)>) -> Self { + self.extra_ptr_ranges = ranges; + self + } + + pub fn handle<'a>(&'a self, noun: Noun) -> NounHandle<'a> { + NounHandle::new(noun, self) + } + + /// Introduce a fresh generative brand tied to this exact `NounSpace` for the duration of `f`. + /// + /// This is an experimental proof-of-concept layer that sits alongside the existing unbranded + /// `Noun`/`NounHandle` APIs. Code inside the closure can only combine branded handles that came + /// from the same branded `NounSpace`. + /// + /// Valid code continues to type-check within one branded scope: + /// + /// ``` + /// use nockvm::mem::{NockStack, NOCK_STACK_SIZE_TINY}; + /// use nockvm::noun::{Cell, D, NounSpace}; + /// + /// let mut stack = NockStack::new(NOCK_STACK_SIZE_TINY, 0); + /// let noun = Cell::new(&mut stack, D(1), D(2)).as_noun(); + /// let space = NounSpace::stack_only(&stack); + /// + /// space.with_brand(|space| { + /// let cell = space.handle(noun).as_cell().unwrap(); + /// assert_eq!(cell.head().as_atom().unwrap().as_u64().unwrap(), 1); + /// assert_eq!(cell.tail().as_atom().unwrap().as_u64().unwrap(), 2); + /// }); + /// ``` + /// + /// Handles from different branded spaces do not type-check together: + /// + /// ```compile_fail + /// use nockvm::mem::{NockStack, NOCK_STACK_SIZE_TINY}; + /// use nockvm::noun::{BrandedNounHandle, BrandedNounSpace, Cell, D, NounSpace}; + /// + /// fn same_space<'space, 'id>( + /// _space: BrandedNounSpace<'space, 'id>, + /// _noun: BrandedNounHandle<'space, 'id>, + /// ) { + /// } + /// + /// let mut stack_a = NockStack::new(NOCK_STACK_SIZE_TINY, 0); + /// let mut stack_b = NockStack::new(NOCK_STACK_SIZE_TINY, 0); + /// let noun_a = Cell::new(&mut stack_a, D(1), D(2)).as_noun(); + /// let space_a = NounSpace::stack_only(&stack_a); + /// let space_b = NounSpace::stack_only(&stack_b); + /// + /// space_a.with_brand(|space_a| { + /// space_b.with_brand(|space_b| { + /// same_space(space_b, space_a.handle(noun_a)); + /// }); + /// }); + /// ``` + /// + /// Branded handles also cannot escape the generative scope: + /// + /// ```compile_fail + /// use nockvm::mem::{NockStack, NOCK_STACK_SIZE_TINY}; + /// use nockvm::noun::{Cell, D, NounSpace}; + /// + /// let mut stack = NockStack::new(NOCK_STACK_SIZE_TINY, 0); + /// let noun = Cell::new(&mut stack, D(1), D(2)).as_noun(); + /// let space = NounSpace::stack_only(&stack); + /// + /// let _escaped = space.with_brand(|space| space.handle(noun)); + /// ``` + pub fn with_brand(&self, f: impl for<'id> FnOnce(BrandedNounSpace<'_, 'id>) -> R) -> R { + f(BrandedNounSpace::new(self)) + } + + fn assert_stack_epoch(&self) { + let Some(epoch_ptr) = self.stack_epoch_ptr else { + return; + }; + let snapshot = self + .stack_epoch_snapshot + .expect("stack epoch snapshot missing"); + let current = unsafe { (*epoch_ptr).load(Ordering::Relaxed) }; + assert!( + current == snapshot, + "NounSpace used after NockStack reset/flip (current epoch {current}, snapshot {snapshot})" + ); + } + + fn resolve_stack_ptr(&self, payload: u64) -> *const u8 { + let ptr = usize::try_from( + payload + .checked_shl(3) + .expect("stack pointer payload exceeds addressable range"), + ) + .expect("stack pointer payload exceeds usize addressable range") + as *const u8; + let addr = ptr as usize; + if let (Some(base), Some(end)) = (self.stack_base, self.stack_end) { + if addr >= base && addr < end { + self.assert_stack_epoch(); + return ptr; + } + } + if let (Some(base), Some(end)) = (self.pma_base, self.pma_end) { + if addr >= base && addr < end { + return ptr; + } + } + for (base, end) in &self.extra_ptr_ranges { + if addr >= *base && addr < *end { + return ptr; + } + } + panic!( + "pointer-form noun {:p} is not within stack or PMA arenas", + ptr + ); + } + + fn classify_ptr(&self, ptr: *const u8) -> AllocLocation { + let addr = ptr as usize; + if let (Some(base), Some(end)) = (self.stack_base, self.stack_end) { + if addr >= base && addr < end { + self.assert_stack_epoch(); + return AllocLocation::Stack; + } + } + if let (Some(base), Some(end)) = (self.pma_base, self.pma_end) { + if addr >= base && addr < end { + return AllocLocation::PmaPtr; + } + } + for (base, end) in &self.extra_ptr_ranges { + if addr >= *base && addr < *end { + return AllocLocation::Stack; + } + } + panic!( + "pointer-form noun {:p} is not within stack or PMA arenas", + ptr + ); + } + + fn resolve_pma_ptr(&self, offset: PmaOffsetWords) -> *const u8 { + let offset = offset + .try_into_usize() + .expect("PMA offset exceeds addressable range"); + let arena_words = self + .pma + .as_ref() + .map(|arena| arena.words()) + .or(self.pma_words) + .expect("PMA arena is required to resolve offset nouns"); + assert!( + offset < arena_words, + "PMA offset {} out of bounds (size words {})", + offset, + arena_words + ); + let base = self + .pma_base + .expect("PMA arena is required to resolve offset nouns"); + let offset_bytes = offset + .checked_mul(8) + .expect("PMA offset exceeds addressable range"); + let ptr = base + .checked_add(offset_bytes) + .expect("PMA offset exceeds addressable range") as *const u8; + assert!( + { + let end = self + .pma_end + .expect("PMA arena is required to resolve offset nouns"); + let addr = ptr as usize; + addr >= base && addr < end + }, + "PMA offset {} resolves outside the PMA arena", + offset + ); + ptr + } +} + +mod generative_brand { + pub enum Id {} +} + +type Brand<'id> = + std::marker::PhantomData &'id generative_brand::Id>; + +#[doc(hidden)] +#[derive(Copy, Clone)] +pub struct BrandedNounSpace<'space, 'id> { + space: &'space NounSpace, + _brand: Brand<'id>, +} + +impl<'space, 'id> BrandedNounSpace<'space, 'id> { + fn new(space: &'space NounSpace) -> Self { + Self { + space, + _brand: std::marker::PhantomData, + } + } + + pub fn handle(self, noun: Noun) -> BrandedNounHandle<'space, 'id> { + BrandedNounHandle::from_unbranded(NounHandle::new(noun, self.space)) + } +} + +#[derive(Copy, Clone)] +pub struct NounHandle<'a> { + noun: Noun, + space: &'a NounSpace, +} + +impl<'a> NounHandle<'a> { + pub fn new(noun: Noun, space: &'a NounSpace) -> Self { + Self { noun, space } + } + + pub fn noun(self) -> Noun { + self.noun + } + + pub fn space(self) -> &'a NounSpace { + self.space + } + + pub fn repr(self) -> NounRepr { + self.noun.repr(self.space) + } + + pub fn allocated_location(self) -> Option { + self.noun.allocated_location(self.space) + } + + pub fn is_direct(self) -> bool { + self.noun.is_direct() + } + + pub fn is_atom(self) -> bool { + self.noun.is_atom() + } + + pub fn is_cell(self) -> bool { + self.noun.is_cell() + } + + pub fn is_allocated(self) -> bool { + self.noun.is_allocated() + } + + pub fn as_atom(self) -> Result> { + self.noun + .as_atom() + .map(|atom| AtomHandle::new(atom, self.space)) + } + + pub fn as_cell(self) -> Result> { + self.noun + .as_cell() + .map(|cell| CellHandle::new(cell, self.space)) + } + + pub fn atom(self) -> Option> { + self.noun + .atom() + .map(|atom| AtomHandle::new(atom, self.space)) + } + + pub fn cell(self) -> Option> { + self.noun + .cell() + .map(|cell| CellHandle::new(cell, self.space)) + } + + pub fn as_either_atom_cell(self) -> Either, CellHandle<'a>> { + match self.noun.as_either_atom_cell() { + Left(atom) => Left(AtomHandle::new(atom, self.space)), + Right(cell) => Right(CellHandle::new(cell, self.space)), + } + } + + pub fn slot(self, axis: u64) -> Result> { + self.noun + .slot(axis, self.space) + .map(|noun| NounHandle::new(noun, self.space)) + } + + pub fn slot_atom(self, atom: Atom) -> Result> { + self.noun + .slot_atom(atom, self.space) + .map(|noun| NounHandle::new(noun, self.space)) + } + + pub fn list_iter(self) -> NounHandleListIterator<'a> { + NounHandleListIterator { noun: self } + } + + pub fn eq_bytes(self, bytes: impl AsRef<[u8]>) -> bool { + if let Ok(atom) = self.noun.as_atom() { + AtomHandle::new(atom, self.space).eq_bytes(bytes) + } else { + false + } + } + + pub fn mass(self) -> usize { + self.noun.mass(self.space) + } + + pub fn mass_frame(self, stack: &NockStack) -> usize { + self.noun.mass_frame(stack, self.space) + } + + pub unsafe fn forwarding_pointer(self) -> Option> { + let allocated = self.noun.as_allocated().ok()?; + allocated + .forwarding_pointer(self.space) + .map(|forwarded| NounHandle::new(forwarded.as_noun(), self.space)) + } +} + +#[doc(hidden)] +#[derive(Copy, Clone)] +pub struct BrandedNounHandle<'space, 'id> { + handle: NounHandle<'space>, + _brand: Brand<'id>, +} + +impl<'space, 'id> BrandedNounHandle<'space, 'id> { + fn from_unbranded(handle: NounHandle<'space>) -> Self { + Self { + handle, + _brand: std::marker::PhantomData, + } + } + + pub fn repr(self) -> NounRepr { + self.handle.repr() + } + + pub fn as_atom(self) -> Result> { + self.handle.as_atom().map(BrandedAtomHandle::from_unbranded) + } + + pub fn as_cell(self) -> Result> { + self.handle.as_cell().map(BrandedCellHandle::from_unbranded) + } + + pub fn slot(self, axis: u64) -> Result { + self.handle.slot(axis).map(Self::from_unbranded) + } +} + +#[derive(Copy, Clone)] +pub struct AtomHandle<'a> { + atom: Atom, + space: &'a NounSpace, +} + +impl<'a> AtomHandle<'a> { + pub fn new(atom: Atom, space: &'a NounSpace) -> Self { + Self { atom, space } + } + + pub fn atom(self) -> Atom { + self.atom + } + + pub fn space(self) -> &'a NounSpace { + self.space + } + + pub fn as_noun(self) -> NounHandle<'a> { + NounHandle::new(self.atom.as_noun(), self.space) + } + + pub fn is_direct(self) -> bool { + self.atom.is_direct() + } + + pub fn is_indirect(self) -> bool { + self.atom.is_indirect() + } + + pub fn is_normalized(&self) -> bool { + self.atom.is_normalized(self.space) + } + + pub fn as_ne_bytes(&self) -> &[u8] { + self.atom.as_ne_bytes(self.space) + } + + pub fn eq_bytes>(&self, bytes: B) -> bool { + let bytes_ref = bytes.as_ref(); + let atom_bytes = self.as_ne_bytes(); + if bytes_ref.len() > atom_bytes.len() { + return false; + } + if bytes_ref.len() == atom_bytes.len() { + return atom_bytes == bytes_ref; + } + if atom_bytes[bytes_ref.len()..].iter().any(|b| *b != 0) { + return false; + } + &atom_bytes[0..bytes_ref.len()] == bytes_ref + } + + pub fn to_bytes_until_nul(&self) -> std::result::Result, str::Utf8Error> { + str::from_utf8(self.as_ne_bytes()) + .map(|bytes| bytes.trim_end_matches('\0').as_bytes().to_vec()) + } + + pub fn into_string(self) -> std::result::Result { + str::from_utf8(self.as_ne_bytes()).map(|string| string.trim_end_matches('\0').to_string()) + } + + pub fn to_ne_bytes(self) -> Vec { + self.atom.to_ne_bytes(self.space) + } + + pub fn to_be_bytes(self) -> Vec { + self.atom.to_be_bytes(self.space) + } + + pub fn to_le_bytes(self) -> Vec { + self.atom.to_le_bytes(self.space) + } + + pub fn as_u64(self) -> Result { + self.atom.as_u64(self.space) + } + + pub fn as_u64_pair(self) -> Result<[u64; 2]> { + unsafe { self.atom.as_u64_pair(self.space) } + } + + pub fn as_bitslice(&self) -> &BitSlice { + self.atom.as_bitslice(self.space) + } + + pub fn as_bitslice_mut(&mut self) -> &mut BitSlice { + self.atom.as_bitslice_mut(self.space) + } + + pub fn as_ubig(self, stack: &mut S) -> UBig { + self.atom.as_ubig(stack, self.space) + } + + pub fn size(self) -> usize { + self.atom.size(self.space) + } + + pub fn bit_size(self) -> usize { + self.atom.bit_size(self.space) + } + + pub fn data_pointer(&self) -> *const u64 { + self.atom.data_pointer(self.space) + } + + pub fn raw_size(self) -> usize { + match self.atom.as_either() { + Left(_direct) => 1, + Right(indirect) => indirect.raw_size(self.space), + } + } + + pub unsafe fn raw_pointer(self) -> *const u64 { + let indirect = self + .atom + .as_indirect() + .expect("expected indirect atom for raw_pointer"); + indirect.to_raw_pointer(self.space) + } + + pub unsafe fn raw_pointer_mut(self) -> *mut u64 { + let mut indirect = self + .atom + .as_indirect() + .expect("expected indirect atom for raw_pointer_mut"); + indirect.to_raw_pointer_mut(self.space) + } + + pub unsafe fn set_forwarding_pointer(self, new_me: *const u64) { + let mut indirect = self + .atom + .as_indirect() + .expect("expected indirect atom for set_forwarding_pointer"); + indirect.set_forwarding_pointer(new_me, self.space); + } + + pub unsafe fn normalize(self) -> AtomHandle<'a> { + let mut atom = self.atom; + let normalized = atom.normalize(self.space); + AtomHandle::new(normalized, self.space) + } + pub fn as_direct(&self) -> Result { + if self.is_direct() { + unsafe { Ok(self.atom.direct) } + } else { + Err(Error::NotDirectAtom) + } + } + + pub fn as_indirect(&self) -> Result { + if self.is_indirect() { + unsafe { Ok(self.atom.indirect) } + } else { + Err(Error::NotIndirectAtom) + } + } + + pub fn as_either(&self) -> Either { + if self.is_indirect() { + unsafe { Right(self.atom.indirect) } + } else { + unsafe { Left(self.atom.direct) } + } + } +} + +#[doc(hidden)] +#[derive(Copy, Clone)] +pub struct BrandedAtomHandle<'space, 'id> { + handle: AtomHandle<'space>, + _brand: Brand<'id>, +} + +impl<'space, 'id> BrandedAtomHandle<'space, 'id> { + fn from_unbranded(handle: AtomHandle<'space>) -> Self { + Self { + handle, + _brand: std::marker::PhantomData, + } + } + + pub fn as_noun(self) -> BrandedNounHandle<'space, 'id> { + BrandedNounHandle::from_unbranded(self.handle.as_noun()) + } + + pub fn as_u64(self) -> Result { + self.handle.as_u64() + } +} + +#[derive(Copy, Clone)] +pub struct CellHandle<'a> { + cell: Cell, + space: &'a NounSpace, +} + +impl<'a> CellHandle<'a> { + pub fn new(cell: Cell, space: &'a NounSpace) -> Self { + Self { cell, space } + } + + pub fn cell(self) -> Cell { + self.cell + } + + pub fn space(self) -> &'a NounSpace { + self.space + } + + pub fn as_noun(self) -> NounHandle<'a> { + NounHandle::new(self.cell.as_noun(), self.space) + } + + pub fn head(self) -> NounHandle<'a> { + NounHandle::new(self.cell.head(self.space), self.space) + } + + pub fn tail(self) -> NounHandle<'a> { + NounHandle::new(self.cell.tail(self.space), self.space) + } + + pub unsafe fn raw_pointer(self) -> *const CellMemory { + self.cell.to_raw_pointer(self.space) + } + + pub unsafe fn raw_pointer_mut(self) -> *mut CellMemory { + let mut cell = self.cell; + cell.to_raw_pointer_mut(self.space) + } + + pub unsafe fn set_forwarding_pointer(self, new_me: *const CellMemory) { + let mut cell = self.cell; + cell.set_forwarding_pointer(new_me, self.space); + } +} + +#[doc(hidden)] +#[derive(Copy, Clone)] +pub struct BrandedCellHandle<'space, 'id> { + handle: CellHandle<'space>, + _brand: Brand<'id>, +} + +impl<'space, 'id> BrandedCellHandle<'space, 'id> { + fn from_unbranded(handle: CellHandle<'space>) -> Self { + Self { + handle, + _brand: std::marker::PhantomData, + } + } + + pub fn as_noun(self) -> BrandedNounHandle<'space, 'id> { + BrandedNounHandle::from_unbranded(self.handle.as_noun()) + } + + pub fn head(self) -> BrandedNounHandle<'space, 'id> { + BrandedNounHandle::from_unbranded(self.handle.head()) + } + + pub fn tail(self) -> BrandedNounHandle<'space, 'id> { + BrandedNounHandle::from_unbranded(self.handle.tail()) + } +} + +pub struct NounHandleListIterator<'a> { + noun: NounHandle<'a>, +} + +impl<'a> Iterator for NounHandleListIterator<'a> { + type Item = NounHandle<'a>; + + fn next(&mut self) -> Option { + if let Ok(cell) = self.noun.as_cell() { + let head = cell.head(); + self.noun = cell.tail(); + Some(head) + } else if unsafe { self.noun.noun().raw_equals(&D(0)) } { + None + } else { + panic!("Improper list terminator: {:?}", self.noun.noun()); + } + } +} + /* A note on forwarding pointers: * * Forwarding pointers are only used temporarily during copies between NockStack frames and between @@ -53,7 +956,7 @@ pub(crate) const CELL_MASK: u64 = !(u64::MAX >> 3); * 1. The current frame must be immediately popped after preserving data, when * copying from a junior NockStack frame to a senior NockStack frame. * 2. All persistent derived state (e.g. Hot state, Warm state) must be preserved - * and the root NockStack frame flipped after saving data to the PMA. + * and the NockStack reset after saving data to the PMA. */ /** Tag for a forwarding pointer */ @@ -73,9 +976,9 @@ pub const NONE: Noun = unsafe { DirectAtom::new_unchecked(tas!(b"MORMAGIC")).as_ #[cfg(feature = "check_acyclic")] #[macro_export] macro_rules! assert_acyclic { - ( $x:expr ) => { + ( $space:expr, $x:expr ) => { assert_no_alloc::permit_alloc(|| { - assert!(crate::noun::acyclic_noun($x)); + assert!(crate::noun::acyclic_noun(($x).in_space($space))); }) }; } @@ -83,15 +986,15 @@ macro_rules! assert_acyclic { #[cfg(not(feature = "check_acyclic"))] #[macro_export] macro_rules! assert_acyclic { - ( $x:expr ) => {}; + ( $space:expr, $x:expr ) => {}; } -pub fn acyclic_noun(noun: Noun) -> bool { +pub(crate) fn acyclic_noun(noun: NounHandle) -> bool { let mut seen = IntMap::new(); - acyclic_noun_go(noun, &mut seen) + acyclic_noun_go(noun.noun(), &mut seen, noun.space()) } -fn acyclic_noun_go(noun: Noun, seen: &mut IntMap) -> bool { +fn acyclic_noun_go(noun: Noun, seen: &mut IntMap, space: &NounSpace) -> bool { match noun.as_either_atom_cell() { Left(_atom) => true, Right(cell) => { @@ -99,8 +1002,9 @@ fn acyclic_noun_go(noun: Noun, seen: &mut IntMap) -> bool { false } else { seen.insert(cell.0, ()); - if acyclic_noun_go(cell.head(), seen) { - if acyclic_noun_go(cell.tail(), seen) { + let cell_handle = cell.in_space(space); + if acyclic_noun_go(cell_handle.head().noun(), seen, space) { + if acyclic_noun_go(cell_handle.tail().noun(), seen, space) { seen.remove(cell.0); true } else { @@ -117,9 +1021,9 @@ fn acyclic_noun_go(noun: Noun, seen: &mut IntMap) -> bool { #[cfg(feature = "check_forwarding")] #[macro_export] macro_rules! assert_no_forwarding_pointers { - ( $x:expr ) => { + ( $space:expr, $x:expr ) => { assert_no_alloc::permit_alloc(|| { - assert!(crate::noun::no_forwarding_pointers($x)); + assert!(crate::noun::no_forwarding_pointers(($x).in_space($space))); }) }; } @@ -127,20 +1031,21 @@ macro_rules! assert_no_forwarding_pointers { #[cfg(not(feature = "check_forwarding"))] #[macro_export] macro_rules! assert_no_forwarding_pointers { - ( $x:expr ) => {}; + ( $space:expr, $x:expr ) => {}; } -pub fn no_forwarding_pointers(noun: Noun) -> bool { +pub(crate) fn no_forwarding_pointers(noun: NounHandle) -> bool { let mut dbg_stack = Vec::new(); - dbg_stack.push(noun); + let space = noun.space(); + dbg_stack.push(noun.noun()); while !dbg_stack.is_empty() { if let Some(noun) = dbg_stack.pop() { if unsafe { noun.raw & FORWARDING_MASK == FORWARDING_TAG } { return false; - } else if let Ok(cell) = noun.as_cell() { - dbg_stack.push(cell.tail()); - dbg_stack.push(cell.head()); + } else if let Ok(cell) = noun.in_space(space).as_cell() { + dbg_stack.push(cell.tail().noun()); + dbg_stack.push(cell.head().noun()); } } else { break; @@ -160,9 +1065,19 @@ fn is_indirect_atom(noun: u64) -> bool { noun & INDIRECT_MASK == INDIRECT_TAG } -/** Test if a noun is a cell. */ -fn is_cell(noun: u64) -> bool { - noun & CELL_MASK == CELL_TAG +/** Test if a noun is a cell. */ +fn is_cell(noun: u64) -> bool { + noun & CELL_MASK == CELL_TAG +} + +#[inline(always)] +unsafe fn noun_from_raw(raw: u64) -> Noun { + Noun { raw } +} + +#[inline(always)] +unsafe fn cell_ptr_from_raw_trusted(raw: u64, space: &NounSpace) -> *const CellMemory { + TaggedPtr::from_raw(raw).resolve_const_trusted(CELL_MASK, space) as *const CellMemory } /** A noun-related error. */ @@ -357,32 +1272,65 @@ pub struct IndirectAtom(u64); impl IndirectAtom { /** Tag the pointer and type it as an indirect atom. */ pub unsafe fn from_raw_pointer(ptr: *const u64) -> Self { - IndirectAtom(((ptr as u64) >> 3) | INDIRECT_TAG) + IndirectAtom(TaggedPtr::from_stack_ptr(ptr as *const u8, INDIRECT_TAG).raw()) + } + + pub fn from_offset_words(words: O) -> Self { + IndirectAtom(TaggedPtr::from_offset(words, INDIRECT_TAG).raw()) } /** Strip the tag from an indirect atom and return it as a mutable pointer to its memory buffer. */ - unsafe fn to_raw_pointer_mut(&mut self) -> *mut u64 { - (self.0 << 3) as *mut u64 + unsafe fn to_raw_pointer_mut(&mut self, space: &NounSpace) -> *mut u64 { + TaggedPtr::from_raw(self.0).resolve_mut(INDIRECT_MASK, space) as *mut u64 } /** Strip the tag from an indirect atom and return it as a pointer to its memory buffer. */ - pub unsafe fn to_raw_pointer(&self) -> *const u64 { - (self.0 << 3) as *const u64 + #[allow(clippy::wrong_self_convention)] + pub(crate) unsafe fn to_raw_pointer(&self, space: &NounSpace) -> *const u64 { + TaggedPtr::from_raw(self.0).resolve_const(INDIRECT_MASK, space) as *const u64 + } + + /** Strip the tag with no location/epoch checks. Trusted local fast path only. */ + #[inline(always)] + #[allow(clippy::wrong_self_convention)] + pub(crate) unsafe fn to_raw_pointer_trusted(&self, space: &NounSpace) -> *const u64 { + TaggedPtr::from_raw(self.0).resolve_const_trusted(INDIRECT_MASK, space) as *const u64 + } + + /// Get raw pointer for stack-pointer form atoms only + pub unsafe fn to_raw_pointer_stack(&self) -> *const u64 { + let tagged = TaggedPtr::from_raw(self.0); + if tagged.location() == PtrLocation::Stack { + ((tagged.payload(INDIRECT_MASK)) << 3) as *const u64 + } else { + panic!("expected stack-pointer Noun, got offset instead"); + } + } + + /// Get mutable raw pointer for stack-pointer form atoms only + pub fn to_raw_pointer_mut_stack(&mut self) -> *mut u64 { + let tagged = TaggedPtr::from_raw(self.0); + if tagged.location() == PtrLocation::Stack { + ((tagged.payload(INDIRECT_MASK)) << 3) as *mut u64 + } else { + panic!("expected stack-pointer Noun, got offset instead"); + } } - pub unsafe fn set_forwarding_pointer(&mut self, new_me: *const u64) { + pub unsafe fn set_forwarding_pointer(&mut self, new_me: *const u64, space: &NounSpace) { // This is OK because the size is stored as 64 bit words, not bytes. // Thus, a true size value will never be larger than U64::MAX >> 3, and so // any of the high bits set as an MSB - *self.to_raw_pointer_mut().add(1) = ((new_me as u64) >> 3) | FORWARDING_TAG; + *self.to_raw_pointer_mut(space).add(1) = + TaggedPtr::from_stack_ptr(new_me as *const u8, FORWARDING_TAG).raw(); } - pub unsafe fn forwarding_pointer(&self) -> Option { - let size_raw = *self.to_raw_pointer().add(1); + pub(crate) unsafe fn forwarding_pointer(&self, space: &NounSpace) -> Option { + let size_raw = *self.to_raw_pointer(space).add(1); if size_raw & FORWARDING_MASK == FORWARDING_TAG { - // we can replace this by masking out the forwarding pointer and putting in the - // indirect tag - Some(Self::from_raw_pointer((size_raw << 3) as *const u64)) + let ptr = + TaggedPtr::from_raw(size_raw).resolve_const(FORWARDING_MASK, space) as *const u64; + Some(Self::from_raw_pointer(ptr)) } else { None } @@ -399,7 +1347,8 @@ impl IndirectAtom { ) -> Self { let (mut indirect, buffer) = Self::new_raw_mut(allocator, size); ptr::copy_nonoverlapping(data, buffer, size); - *(indirect.normalize()) + // Use normalize_stack since new_raw_mut creates stack-pointer form atoms + *(indirect.normalize_stack()) } /** Make an indirect atom by copying from other memory. @@ -413,7 +1362,8 @@ impl IndirectAtom { ) -> Self { let (mut indirect, buffer) = Self::new_raw_mut_bytes(allocator, size); ptr::copy_nonoverlapping(data, buffer.as_mut_ptr(), size); - *(indirect.normalize()) + // Use normalize_stack since new_raw_mut_bytes creates stack-pointer form atoms + *(indirect.normalize_stack()) } pub unsafe fn new_raw_bytes_ref(allocator: &mut A, data: &[u8]) -> Self { @@ -485,55 +1435,82 @@ impl IndirectAtom { (noun, &mut *(ptr as *mut [u8; N])) } - /** Size of an indirect atom in 64-bit words (not bytes) */ - pub fn size(&self) -> usize { - unsafe { *(self.to_raw_pointer().add(1)) as usize } + /** Size of an indirect atom in 64-bit words */ + pub(crate) fn size(&self, space: &NounSpace) -> usize { + unsafe { *(self.to_raw_pointer(space).add(1)) as usize } } /** Memory size of an indirect atom (including size + metadata fields) in 64-bit words */ - pub fn raw_size(&self) -> usize { - self.size() + 2 + pub(crate) fn raw_size(&self, space: &NounSpace) -> usize { + self.size(space) + 2 } - pub fn bit_size(&self) -> usize { + pub(crate) fn bit_size(&self, space: &NounSpace) -> usize { unsafe { - ((self.size() - 1) << 6) + 64 - - (*(self.to_raw_pointer().add(2 + self.size() - 1))).leading_zeros() as usize + ((self.size(space) - 1) << 6) + 64 + - (*(self.to_raw_pointer(space).add(2 + self.size(space) - 1))).leading_zeros() + as usize } } /** Pointer to data for indirect atom */ - pub fn data_pointer(&self) -> *const u64 { - unsafe { self.to_raw_pointer().add(2) } + pub(crate) fn data_pointer(&self, space: &NounSpace) -> *const u64 { + unsafe { self.to_raw_pointer(space).add(2) } } - pub fn data_pointer_mut(&mut self) -> *mut u64 { - unsafe { self.to_raw_pointer_mut().add(2) } + pub(crate) fn data_pointer_mut(&mut self, space: &NounSpace) -> *mut u64 { + unsafe { self.to_raw_pointer_mut(space).add(2) } } - pub fn as_slice(&self) -> &[u64] { - unsafe { from_raw_parts(self.data_pointer(), self.size()) } + pub fn data_pointer_stack(&self) -> Option<*const u64> { + let tagged = TaggedPtr::from_raw(self.0); + if tagged.location() == PtrLocation::Stack { + Some(((tagged.payload(INDIRECT_MASK)) << 3) as *const u64) + } else { + None + } } - pub fn as_mut_slice(&mut self) -> &mut [u64] { - unsafe { from_raw_parts_mut(self.data_pointer_mut(), self.size()) } + pub(crate) fn as_slice(&self, space: &NounSpace) -> &[u64] { + unsafe { from_raw_parts(self.data_pointer(space), self.size(space)) } } - pub fn as_ne_bytes(&self) -> &[u8] { - unsafe { from_raw_parts(self.data_pointer() as *const u8, self.size() << 3) } + #[inline(always)] + pub(crate) fn size_trusted(&self, space: &NounSpace) -> usize { + unsafe { *(self.to_raw_pointer_trusted(space).add(1)) as usize } } - pub fn to_ne_bytes(&self) -> Vec { - self.as_ne_bytes().to_vec() + #[inline(always)] + pub(crate) fn data_pointer_trusted(&self, space: &NounSpace) -> *const u64 { + unsafe { self.to_raw_pointer_trusted(space).add(2) } + } + + #[inline(always)] + pub(crate) fn as_slice_trusted(&self, space: &NounSpace) -> &[u64] { + unsafe { from_raw_parts(self.data_pointer_trusted(space), self.size_trusted(space)) } + } + + pub(crate) fn as_mut_slice(&mut self, space: &NounSpace) -> &mut [u64] { + unsafe { from_raw_parts_mut(self.data_pointer_mut(space), self.size(space)) } + } + + pub(crate) fn as_ne_bytes(&self, space: &NounSpace) -> &[u8] { + unsafe { from_raw_parts(self.data_pointer(space) as *const u8, self.size(space) << 3) } + } + + #[allow(clippy::wrong_self_convention)] + pub(crate) fn to_ne_bytes(&self, space: &NounSpace) -> Vec { + self.as_ne_bytes(space).to_vec() } #[allow(unused)] - pub fn to_be_bytes(&self) -> Vec { - if self.size() == 1 { - let num = unsafe { *(self.data_pointer()) }; + #[allow(clippy::wrong_self_convention)] + pub(crate) fn to_be_bytes(&self, space: &NounSpace) -> Vec { + if self.size(space) == 1 { + let num = unsafe { *(self.data_pointer(space)) }; num.to_be_bytes().to_vec() } else { - let mut bytes_ne = self.to_ne_bytes(); + let mut bytes_ne = self.to_ne_bytes(space); #[cfg(target_endian = "little")] { bytes_ne.reverse() @@ -543,12 +1520,13 @@ impl IndirectAtom { } #[allow(unused)] - pub fn to_le_bytes(&self) -> Vec { - if self.size() == 1 { - let num = unsafe { *(self.data_pointer()) }; + #[allow(clippy::wrong_self_convention)] + pub(crate) fn to_le_bytes(&self, space: &NounSpace) -> Vec { + if self.size(space) == 1 { + let num = unsafe { *(self.data_pointer(space)) }; num.to_le_bytes().to_vec() } else { - let mut bytes_ne = self.to_ne_bytes(); + let mut bytes_ne = self.to_ne_bytes(space); #[cfg(target_endian = "big")] { bytes_ne.reverse() @@ -558,17 +1536,18 @@ impl IndirectAtom { } } + #[allow(unused)] /** BitSlice view on an indirect atom, with lifetime tied to reference to indirect atom. */ - pub fn as_bitslice(&self) -> &BitSlice { - BitSlice::from_slice(self.as_slice()) + pub(crate) fn as_bitslice(&self, space: &NounSpace) -> &BitSlice { + BitSlice::from_slice(self.as_slice(space)) } - pub fn as_bitslice_mut(&mut self) -> &mut BitSlice { - BitSlice::from_slice_mut(self.as_mut_slice()) + pub(crate) fn as_bitslice_mut(&mut self, space: &NounSpace) -> &mut BitSlice { + BitSlice::from_slice_mut(self.as_mut_slice(space)) } - pub fn as_ubig(&self, stack: &mut S) -> UBig { - let bytes_mem_repr = self.as_ne_bytes(); + pub(crate) fn as_ubig(&self, stack: &mut S, space: &NounSpace) -> UBig { + let bytes_mem_repr = self.as_ne_bytes(space); #[cfg(target_endian = "little")] { @@ -580,19 +1559,19 @@ impl IndirectAtom { } } - pub unsafe fn as_u64(self) -> Result { - if self.size() == 1 { - Ok(*(self.data_pointer())) + pub(crate) unsafe fn as_u64(self, space: &NounSpace) -> Result { + if self.size(space) == 1 { + Ok(*(self.data_pointer(space))) } else { Err(Error::NotRepresentable) } } /** Produce a SoftFloat-compatible ordered pair of 64-bit words */ - pub fn as_u64_pair(self) -> Result<[u64; 2]> { - if self.size() <= 2 { + pub(crate) fn as_u64_pair(self, space: &NounSpace) -> Result<[u64; 2]> { + if self.size(space) <= 2 { let u128_array = &mut [0u64; 2]; - u128_array.copy_from_slice(&(self.as_slice()[0..2])); + u128_array.copy_from_slice(&(self.as_slice(space)[0..2])); Ok(*u128_array) } else { Err(Error::NotRepresentable) @@ -600,25 +1579,57 @@ impl IndirectAtom { } /** Ensure that the size does not contain any trailing 0 words */ - pub unsafe fn normalize(&mut self) -> &Self { - let mut index = self.size() - 1; - let data = self.data_pointer(); + pub(crate) unsafe fn normalize(&mut self, space: &NounSpace) -> &Self { + let mut index = self.size(space) - 1; + let data = self.data_pointer(space); + loop { + if index == 0 || *(data.add(index)) != 0 { + break; + } + index -= 1; + } + *(self.to_raw_pointer_mut(space).add(1)) = (index + 1) as u64; + self + } + + /// Normalize a stack-pointer form indirect atom (no arena needed). + /// Panics if the atom is in offset form. + pub unsafe fn normalize_stack(&mut self) -> &Self { + let ptr = self.to_raw_pointer_mut_stack(); + let mut index = (*(ptr.add(1)) as usize) - 1; // size is at offset 1 + let data = ptr.add(2); // data starts at offset 2 loop { if index == 0 || *(data.add(index)) != 0 { break; } index -= 1; } - *(self.to_raw_pointer_mut().add(1)) = (index + 1) as u64; + *(ptr.add(1)) = (index + 1) as u64; self } /** Normalize, but convert to direct atom if it will fit */ - pub unsafe fn normalize_as_atom(&mut self) -> Atom { - self.normalize(); - if self.size() == 1 && *(self.data_pointer()) <= DIRECT_MAX { + pub unsafe fn normalize_as_atom(&mut self, space: &NounSpace) -> Atom { + self.normalize(space); + if self.size(space) == 1 && *(self.data_pointer(space)) <= DIRECT_MAX { + Atom { + direct: DirectAtom(*(self.data_pointer(space))), + } + } else { + Atom { indirect: *self } + } + } + + /// Normalize a stack-pointer form atom, converting to direct if it fits. + /// Panics if the atom is in offset form. + pub unsafe fn normalize_as_atom_stack(&mut self) -> Atom { + self.normalize_stack(); + let ptr = self.to_raw_pointer_stack(); + let size = *(ptr.add(1)) as usize; + let data = ptr.add(2); + if size == 1 && *data <= DIRECT_MAX { Atom { - direct: DirectAtom(*(self.data_pointer())), + direct: DirectAtom(*data), } } else { Atom { indirect: *self } @@ -643,16 +1654,17 @@ impl IndirectAtom { // b) disables no-allocation, creates a string, utilitzes it (eprintf or generate tape), and then deallocates impl fmt::Debug for IndirectAtom { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - write!(f, "0x")?; - let mut i = self.size() - 1; - loop { - write!(f, "_{:016x}", unsafe { *(self.data_pointer().add(i)) })?; - if i == 0 { - break; + let tagged = TaggedPtr::from_raw(self.0); + match tagged.location() { + PtrLocation::Stack => { + let ptr = ((tagged.payload(INDIRECT_MASK)) << 3) as *const u8; + write!(f, "IndirectAtom(StackPtr={ptr:p})") + } + PtrLocation::Offset => { + let offset = tagged.payload(INDIRECT_MASK); + write!(f, "IndirectAtom(PmaOffset={offset})") } - i -= 1; } - Ok(()) } } @@ -670,37 +1682,58 @@ pub struct Cell(u64); impl Cell { pub unsafe fn from_raw_pointer(ptr: *const CellMemory) -> Self { - Cell(((ptr as u64) >> 3) | CELL_TAG) + Cell(TaggedPtr::from_stack_ptr(ptr as *const u8, CELL_TAG).raw()) + } + + pub fn from_offset_words(words: O) -> Self { + Cell(TaggedPtr::from_offset(words, CELL_TAG).raw()) } - pub unsafe fn to_raw_pointer(&self) -> *const CellMemory { - (self.0 << 3) as *const CellMemory + #[allow(clippy::wrong_self_convention)] + pub(crate) unsafe fn to_raw_pointer(&self, space: &NounSpace) -> *const CellMemory { + TaggedPtr::from_raw(self.0).resolve_const(CELL_MASK, space) as *const CellMemory } - pub unsafe fn to_raw_pointer_mut(&mut self) -> *mut CellMemory { - (self.0 << 3) as *mut CellMemory + #[inline(always)] + #[allow(clippy::wrong_self_convention)] + pub(crate) unsafe fn to_raw_pointer_trusted(&self, space: &NounSpace) -> *const CellMemory { + TaggedPtr::from_raw(self.0).resolve_const_trusted(CELL_MASK, space) as *const CellMemory + } + + pub(crate) unsafe fn to_raw_pointer_mut(&mut self, space: &NounSpace) -> *mut CellMemory { + TaggedPtr::from_raw(self.0).resolve_mut(CELL_MASK, space) as *mut CellMemory + } + + #[inline(always)] + pub fn stack_memory_pointer(&self) -> Option<*const CellMemory> { + let tagged = TaggedPtr::from_raw(self.0); + if tagged.location() == PtrLocation::Stack { + Some(((tagged.payload(CELL_MASK)) << 3) as *const CellMemory) + } else { + None + } } - pub unsafe fn head_as_mut(mut self) -> *mut Noun { - &mut (*self.to_raw_pointer_mut()).head as *mut Noun + pub(crate) unsafe fn head_as_mut(mut self, space: &NounSpace) -> *mut Noun { + &mut (*self.to_raw_pointer_mut(space)).head as *mut Noun } - pub unsafe fn tail_as_mut(mut self) -> *mut Noun { - &mut (*self.to_raw_pointer_mut()).tail as *mut Noun + pub(crate) unsafe fn tail_as_mut(mut self, space: &NounSpace) -> *mut Noun { + &mut (*self.to_raw_pointer_mut(space)).tail as *mut Noun } - pub unsafe fn set_forwarding_pointer(&mut self, new_me: *const CellMemory) { - (*self.to_raw_pointer_mut()).head = Noun { - raw: ((new_me as u64) >> 3) | FORWARDING_TAG, + pub unsafe fn set_forwarding_pointer(&mut self, new_me: *const CellMemory, space: &NounSpace) { + (*self.to_raw_pointer_mut(space)).head = Noun { + raw: TaggedPtr::from_stack_ptr(new_me as *const u8, FORWARDING_TAG).raw(), } } - pub unsafe fn forwarding_pointer(&self) -> Option { - let head_raw = (*self.to_raw_pointer()).head.raw; + pub(crate) unsafe fn forwarding_pointer(&self, space: &NounSpace) -> Option { + let head_raw = (*self.to_raw_pointer(space)).head.raw; if head_raw & FORWARDING_MASK == FORWARDING_TAG { - // we can replace this by masking out the forwarding pointer and putting in the cell - // tag - Some(Self::from_raw_pointer((head_raw << 3) as *const CellMemory)) + let ptr = TaggedPtr::from_raw(head_raw).resolve_const(FORWARDING_MASK, space) + as *const CellMemory; + Some(Self::from_raw_pointer(ptr)) } else { None } @@ -741,18 +1774,34 @@ impl Cell { } // TODO: idk about making these owned independently of their parent - pub fn head(&self) -> Noun { - unsafe { (*(self.to_raw_pointer())).head } + pub(crate) fn head(&self, space: &NounSpace) -> Noun { + unsafe { (*(self.to_raw_pointer(space))).head } } // TODO: Ditto, etc. - pub fn tail(&self) -> Noun { - unsafe { (*(self.to_raw_pointer())).tail } + pub(crate) fn tail(&self, space: &NounSpace) -> Noun { + unsafe { (*(self.to_raw_pointer(space))).tail } + } + + #[inline(always)] + pub(crate) fn head_tail(&self, space: &NounSpace) -> (Noun, Noun) { + unsafe { + let cell = self.to_raw_pointer(space); + ((*cell).head, (*cell).tail) + } + } + + #[inline(always)] + pub(crate) fn head_tail_trusted(&self, space: &NounSpace) -> (Noun, Noun) { + unsafe { + let cell = self.to_raw_pointer_trusted(space); + ((*cell).head, (*cell).tail) + } } - pub fn head_ref(&self) -> &Noun { + pub(crate) fn head_ref<'a>(&'a self, space: &'a NounSpace) -> &'a Noun { unsafe { - self.to_raw_pointer() + self.to_raw_pointer(space) .as_ref() .map(|cell| &cell.head) .unwrap_or_else(|| panic!("head_ref: invalid pointer")) @@ -760,9 +1809,9 @@ impl Cell { } // TODO: Ditto, etc. - pub fn tail_ref(&self) -> &Noun { + pub(crate) fn tail_ref<'a>(&'a self, space: &'a NounSpace) -> &'a Noun { unsafe { - self.to_raw_pointer() + self.to_raw_pointer(space) .as_ref() .map(|cell| &cell.tail) .unwrap_or_else(|| panic!("head_ref: invalid pointer")) @@ -776,41 +1825,60 @@ impl Cell { pub fn as_noun(&self) -> Noun { Noun { cell: *self } } + + pub fn in_space<'a>(self, space: &'a NounSpace) -> CellHandle<'a> { + CellHandle::new(self, space) + } } impl fmt::Debug for Cell { fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { - write!(f, "[")?; - let cell = *self; - write!(f, "{:?},", cell.head())?; - write!(f, " {:?}]", unsafe { cell.tail().raw })?; - Ok(()) + let tagged = TaggedPtr::from_raw(self.0); + match tagged.location() { + PtrLocation::Stack => { + let ptr = ((tagged.payload(CELL_MASK)) << 3) as *const u8; + write!(f, "Cell(StackPtr={ptr:p})") + } + PtrLocation::Offset => { + let offset = tagged.payload(CELL_MASK); + write!(f, "Cell(PmaOffset={offset})") + } + } } } -pub struct FullDebugCell<'a>(pub &'a Cell); +pub struct FullDebugCell<'a, 'b> { + pub cell: &'a Cell, + pub space: &'b NounSpace, +} -impl fmt::Debug for FullDebugCell<'_> { +impl fmt::Debug for FullDebugCell<'_, '_> { fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { - fn do_fmt(cell: &Cell, brackets: bool, f: &mut fmt::Formatter) -> fmt::Result { + fn do_fmt( + cell: &Cell, + space: &NounSpace, + brackets: bool, + f: &mut fmt::Formatter, + ) -> fmt::Result { if brackets { write!(f, "[")?; } - match cell.head().as_cell() { + let cell_handle = (*cell).in_space(space); + match cell_handle.head().as_cell() { Ok(head_cell) => { - do_fmt(&head_cell, true, f)?; + do_fmt(&head_cell.cell(), space, true, f)?; write!(f, " ")?; } Err(_) => { - write!(f, "{:?} ", cell.head())?; + write!(f, "{:?} ", cell_handle.head().noun())?; } } - match cell.tail().as_cell() { + match cell_handle.tail().as_cell() { Ok(next_cell) => { - do_fmt(&next_cell, false, f)?; + do_fmt(&next_cell.cell(), space, false, f)?; } Err(_) => { - write!(f, "{:?}", cell.tail())?; + write!(f, "{:?}", cell_handle.tail().noun())?; } } if brackets { @@ -819,20 +1887,23 @@ impl fmt::Debug for FullDebugCell<'_> { Ok(()) } - do_fmt(self.0, true, f)?; + do_fmt(self.cell, self.space, true, f)?; Ok(()) } } // Render a path which is a linked-list of cells of of atoms (direct and indirect strings) -pub struct DebugPath<'a>(pub &'a Cell); +pub struct DebugPath<'a, 'b> { + pub cell: &'a Cell, + pub space: &'b NounSpace, +} -impl fmt::Debug for DebugPath<'_> { +impl fmt::Debug for DebugPath<'_, '_> { fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { write!(f, "[")?; - let mut cell = *self.0; + let mut cell = *self.cell; loop { - let head = cell.head().as_atom(); + let head = cell.head(self.space).as_atom(); match head { Ok(atom) => { if atom.is_direct() { @@ -847,13 +1918,13 @@ impl fmt::Debug for DebugPath<'_> { write!(f, "ERR, not atom")?; } } - match cell.tail().as_cell() { + match cell.tail(self.space).as_cell() { Ok(next_cell) => { write!(f, " ")?; cell = next_cell; } Err(_) => { - write!(f, " {:?}]", cell.tail())?; + write!(f, " {:?}]", cell.tail(self.space))?; break; } } @@ -939,7 +2010,7 @@ impl<'a> IndirectAxisIterator<'a> { // Direct axis traversal without bitvec - for u64 axes #[inline(always)] -fn slot_direct(cell: &Cell, axis: u64) -> Result { +fn slot_direct(cell: &Cell, axis: u64, space: &NounSpace) -> Result { if axis == 0 { return Err(Error::NotRepresentable); } @@ -953,7 +2024,7 @@ fn slot_direct(cell: &Cell, axis: u64) -> Result { for idx in (0..highest).rev() { let descend_tail = ((axis >> idx) & 1) != 0; - let memory = unsafe { current.to_raw_pointer() }; + let memory = unsafe { current.to_raw_pointer(space) }; noun = unsafe { if descend_tail { (*memory).tail @@ -974,11 +2045,50 @@ fn slot_direct(cell: &Cell, axis: u64) -> Result { Ok(noun) } +// Trusted local axis traversal for the interpreter hot path. +// This skips location and epoch checks and assumes all traversed nouns are +// stack-local or PMA-local. +#[inline(always)] +fn slot_direct_trusted(cell: &Cell, axis: u64, space: &NounSpace) -> Result { + if axis == 0 { + return Err(Error::NotRepresentable); + } + if axis == 1 { + return Ok(cell.as_noun()); + } + + let highest = 63 - axis.leading_zeros() as usize; + let mut current_raw = cell.0; + let mut noun_raw = current_raw; + + for idx in (0..highest).rev() { + let descend_tail = ((axis >> idx) & 1) != 0; + let memory = unsafe { cell_ptr_from_raw_trusted(current_raw, space) }; + noun_raw = unsafe { + if descend_tail { + (*memory).tail.raw + } else { + (*memory).head.raw + } + }; + + if idx != 0 { + if is_cell(noun_raw) { + current_raw = noun_raw; + } else { + return Err(Error::NotRepresentable); + } + } + } + + Ok(unsafe { noun_from_raw(noun_raw) }) +} + impl Slots for Cell {} // Indirect axis traversal - for large axes stored in word slices #[inline(always)] -fn slot_indirect(cell: &Cell, words: &[u64]) -> Result { +fn slot_indirect(cell: &Cell, words: &[u64], space: &NounSpace) -> Result { if words.is_empty() { return Err(Error::NotRepresentable); } @@ -1011,7 +2121,7 @@ fn slot_indirect(cell: &Cell, words: &[u64]) -> Result { let bit_idx = idx & 63; let descend_tail = ((words[word_idx] >> bit_idx) & 1) != 0; - let memory = unsafe { current.to_raw_pointer() }; + let memory = unsafe { current.to_raw_pointer(space) }; noun = unsafe { if descend_tail { (*memory).tail @@ -1032,15 +2142,69 @@ fn slot_indirect(cell: &Cell, words: &[u64]) -> Result { Ok(noun) } +#[inline(always)] +fn slot_indirect_trusted(cell: &Cell, words: &[u64], space: &NounSpace) -> Result { + if words.is_empty() { + return Err(Error::NotRepresentable); + } + + let mut highest_word_idx = words.len() - 1; + while highest_word_idx > 0 && words[highest_word_idx] == 0 { + highest_word_idx -= 1; + } + + let highest_word = words[highest_word_idx]; + if highest_word == 0 { + return Err(Error::NotRepresentable); + } + + let highest_bit_in_word = 63 - highest_word.leading_zeros() as usize; + let highest = (highest_word_idx << 6) + highest_bit_in_word; + + if highest == 0 { + return Ok(cell.as_noun()); + } + + let mut current_raw = cell.0; + let mut noun_raw = current_raw; + let mut idx = highest; + + while idx != 0 { + idx -= 1; + let word_idx = idx >> 6; + let bit_idx = idx & 63; + let descend_tail = ((words[word_idx] >> bit_idx) & 1) != 0; + + let memory = unsafe { cell_ptr_from_raw_trusted(current_raw, space) }; + noun_raw = unsafe { + if descend_tail { + (*memory).tail.raw + } else { + (*memory).head.raw + } + }; + + if idx != 0 { + if is_cell(noun_raw) { + current_raw = noun_raw; + } else { + return Err(Error::NotRepresentable); + } + } + } + + Ok(unsafe { noun_from_raw(noun_raw) }) +} + impl private::RawSlots for Cell { #[inline(always)] - fn raw_slot_direct(&self, axis: u64) -> Result { - slot_direct(self, axis) + fn raw_slot_direct(&self, axis: u64, space: &NounSpace) -> Result { + slot_direct(self, axis, space) } #[inline(always)] - fn raw_slot_indirect(&self, axis: &[u64]) -> Result { - slot_indirect(self, axis) + fn raw_slot_indirect(&self, axis: &[u64], space: &NounSpace) -> Result { + slot_indirect(self, axis, space) } } @@ -1099,11 +2263,11 @@ impl Atom { unsafe { is_indirect_atom(self.raw) } } - pub fn is_normalized(&self) -> bool { + pub(crate) fn is_normalized(&self, space: &NounSpace) -> bool { unsafe { if let Some(indirect) = self.indirect() { - if (indirect.size() == 1 && *indirect.data_pointer() <= DIRECT_MAX) - || *indirect.data_pointer().add(indirect.size() - 1) == 0 + if (indirect.size(space) == 1 && *indirect.data_pointer(space) <= DIRECT_MAX) + || *indirect.data_pointer(space).add(indirect.size(space) - 1) == 0 { return false; } @@ -1141,48 +2305,53 @@ impl Atom { Noun { atom: self } } + pub fn in_space<'a>(self, space: &'a NounSpace) -> AtomHandle<'a> { + AtomHandle::new(self, space) + } + /// Returns a slice of bytes in native-endian order. Currently, Sword only supports /// little-endian machines, so this will return little-endian. - pub fn as_ne_bytes(&self) -> &[u8] { + pub(crate) fn as_ne_bytes(&self, space: &NounSpace) -> &[u8] { if self.is_direct() { unsafe { self.direct.as_ne_bytes() } } else { - unsafe { self.indirect.as_ne_bytes() } + unsafe { self.indirect.as_ne_bytes(space) } } } /// Returns Vec in native-endian order - pub fn to_ne_bytes(&self) -> Vec { + #[allow(clippy::wrong_self_convention)] + pub(crate) fn to_ne_bytes(&self, space: &NounSpace) -> Vec { if self.is_direct() { unsafe { self.direct.to_ne_bytes() } } else { - unsafe { self.indirect.to_ne_bytes() } + unsafe { self.indirect.to_ne_bytes(space) } } } /// Returns Vec in big-endian order - pub fn to_be_bytes(self) -> Vec { + pub(crate) fn to_be_bytes(self, space: &NounSpace) -> Vec { if self.is_direct() { unsafe { self.direct.to_be_bytes() } } else { - unsafe { self.indirect.to_be_bytes() } + unsafe { self.indirect.to_be_bytes(space) } } } /// Returns Vec in little-endian order - pub fn to_le_bytes(self) -> Vec { + pub(crate) fn to_le_bytes(self, space: &NounSpace) -> Vec { if self.is_direct() { unsafe { self.direct.to_le_bytes() } } else { - unsafe { self.indirect.to_le_bytes() } + unsafe { self.indirect.to_le_bytes(space) } } } - pub fn as_u64(self) -> Result { + pub(crate) fn as_u64(self, space: &NounSpace) -> Result { if self.is_direct() { Ok(unsafe { self.direct.data() }) } else { - unsafe { self.indirect.as_u64() } + unsafe { self.indirect.as_u64(space) } } } @@ -1195,36 +2364,36 @@ impl Atom { } /** Produce a SoftFloat-compatible ordered pair of 64-bit words */ - pub unsafe fn as_u64_pair(self) -> Result<[u64; 2]> { + pub(crate) unsafe fn as_u64_pair(self, space: &NounSpace) -> Result<[u64; 2]> { if self.is_direct() { let u128_array = &mut [0u64; 2]; u128_array[0] = self.as_direct()?.data(); u128_array[1] = 0x0_u64; Ok(*u128_array) } else { - unsafe { self.indirect.as_u64_pair() } + unsafe { self.indirect.as_u64_pair(space) } } } - pub fn as_bitslice(&self) -> &BitSlice { + pub(crate) fn as_bitslice(&self, space: &NounSpace) -> &BitSlice { if self.is_indirect() { - unsafe { self.indirect.as_bitslice() } + unsafe { self.indirect.as_bitslice(space) } } else { unsafe { self.direct.as_bitslice() } } } - pub fn as_bitslice_mut(&mut self) -> &mut BitSlice { + pub(crate) fn as_bitslice_mut(&mut self, space: &NounSpace) -> &mut BitSlice { if self.is_indirect() { - unsafe { self.indirect.as_bitslice_mut() } + unsafe { self.indirect.as_bitslice_mut(space) } } else { unsafe { self.direct.as_bitslice_mut() } } } - pub fn as_ubig(self, stack: &mut S) -> UBig { + pub(crate) fn as_ubig(self, stack: &mut S, space: &NounSpace) -> UBig { if self.is_indirect() { - unsafe { self.indirect.as_ubig(stack) } + unsafe { self.indirect.as_ubig(stack, space) } } else { unsafe { self.direct.as_ubig(stack) } } @@ -1246,31 +2415,30 @@ impl Atom { } } - /// Size of the atom in 64-bit words (not bytes). Direct atoms return 1. - pub fn size(&self) -> usize { + pub(crate) fn size(&self, space: &NounSpace) -> usize { match self.as_either() { Left(_direct) => 1, - Right(indirect) => indirect.size(), + Right(indirect) => indirect.size(space), } } - pub fn bit_size(&self) -> usize { + pub(crate) fn bit_size(&self, space: &NounSpace) -> usize { match self.as_either() { Left(direct) => direct.bit_size(), - Right(indirect) => indirect.bit_size(), + Right(indirect) => indirect.bit_size(space), } } - pub fn data_pointer(&self) -> *const u64 { + pub(crate) fn data_pointer(&self, space: &NounSpace) -> *const u64 { match self.as_either() { Left(_direct) => (self as *const Atom) as *const u64, - Right(indirect) => indirect.data_pointer(), + Right(indirect) => indirect.data_pointer(space), } } - pub unsafe fn normalize(&mut self) -> Atom { + pub(crate) unsafe fn normalize(&mut self, space: &NounSpace) -> Atom { if self.is_indirect() { - self.indirect.normalize_as_atom() + self.indirect.normalize_as_atom(space) } else { *self } @@ -1316,31 +2484,47 @@ impl Allocated { unsafe { is_cell(self.raw) } } - pub unsafe fn to_raw_pointer(&self) -> *const u64 { - (self.raw << 3) as *const u64 + #[allow(clippy::wrong_self_convention)] + pub(crate) unsafe fn to_raw_pointer(&self, space: &NounSpace) -> *const u64 { + let tagged = TaggedPtr::from_raw(self.raw); + if self.is_indirect() { + tagged.resolve_const(INDIRECT_MASK, space) as *const u64 + } else { + tagged.resolve_const(CELL_MASK, space) as *const u64 + } } - pub unsafe fn to_raw_pointer_mut(&mut self) -> *mut u64 { - (self.raw << 3) as *mut u64 + pub(crate) unsafe fn to_raw_pointer_mut(&mut self, space: &NounSpace) -> *mut u64 { + let tagged = TaggedPtr::from_raw(self.raw); + if self.is_indirect() { + tagged.resolve_mut(INDIRECT_MASK, space) as *mut u64 + } else { + tagged.resolve_mut(CELL_MASK, space) as *mut u64 + } } - unsafe fn const_to_raw_pointer_mut(self) -> *mut u64 { - (self.raw << 3) as *mut u64 + pub(crate) unsafe fn const_to_raw_pointer_mut(self, space: &NounSpace) -> *mut u64 { + let tagged = TaggedPtr::from_raw(self.raw); + if self.is_indirect() { + tagged.resolve_mut(INDIRECT_MASK, space) as *mut u64 + } else { + tagged.resolve_mut(CELL_MASK, space) as *mut u64 + } } - pub unsafe fn forwarding_pointer(&self) -> Option { + pub(crate) unsafe fn forwarding_pointer(&self, space: &NounSpace) -> Option { match self.as_either() { - Left(indirect) => indirect.forwarding_pointer().map(|i| i.as_allocated()), - Right(cell) => cell.forwarding_pointer().map(|c| c.as_allocated()), + Left(indirect) => indirect.forwarding_pointer(space).map(|i| i.as_allocated()), + Right(cell) => cell.forwarding_pointer(space).map(|c| c.as_allocated()), } } - pub unsafe fn get_metadata(&self) -> u64 { - *(self.to_raw_pointer()) + pub(crate) unsafe fn get_metadata(&self, space: &NounSpace) -> u64 { + *(self.to_raw_pointer(space)) } - pub unsafe fn set_metadata(&mut self, metadata: u64) { - *(self.const_to_raw_pointer_mut()) = metadata; + pub(crate) unsafe fn set_metadata(&mut self, metadata: u64, space: &NounSpace) { + *(self.const_to_raw_pointer_mut(space)) = metadata; } pub fn as_either(&self) -> Either { @@ -1371,9 +2555,9 @@ impl Allocated { Noun { allocated: *self } } - pub fn get_cached_mug(self: Allocated) -> Option { + pub(crate) fn get_cached_mug(self: Allocated, space: &NounSpace) -> Option { unsafe { - let bottom_metadata = self.get_metadata() as u32 & 0x7FFFFFFF; // magic number: LS 31 bits + let bottom_metadata = self.get_metadata(space) as u32 & 0x7FFFFFFF; // magic number: LS 31 bits if bottom_metadata > 0 { Some(bottom_metadata) } else { @@ -1406,6 +2590,10 @@ impl Noun { unsafe { self.raw == u64::MAX } } + pub fn in_space<'a>(self, space: &'a NounSpace) -> NounHandle<'a> { + NounHandle::new(self, space) + } + pub fn is_direct(&self) -> bool { unsafe { is_direct_atom(self.raw) } } @@ -1422,6 +2610,58 @@ impl Noun { self.is_indirect() || self.is_cell() } + pub(crate) fn repr(&self, space: &NounSpace) -> NounRepr { + let raw = unsafe { self.as_raw() }; + if is_direct_atom(raw) { + return NounRepr::Direct; + } + + enum AllocKind { + Indirect, + Cell, + Forwarding, + } + + let (mask, kind) = if is_indirect_atom(raw) { + (INDIRECT_MASK, AllocKind::Indirect) + } else if is_cell(raw) { + (CELL_MASK, AllocKind::Cell) + } else if raw & FORWARDING_MASK == FORWARDING_TAG { + (FORWARDING_MASK, AllocKind::Forwarding) + } else { + unreachable!("unknown noun tag for raw {:#x}", raw); + }; + + let tagged = TaggedPtr::from_raw(raw); + let location = match tagged.location() { + PtrLocation::Offset => { + if matches!(kind, AllocKind::Forwarding) { + panic!("forwarding pointers cannot be offset-form"); + } + AllocLocation::PmaOffset + } + PtrLocation::Stack => { + let ptr = (tagged.payload(mask) << 3) as *const u8; + space.classify_ptr(ptr) + } + }; + + match kind { + AllocKind::Indirect => NounRepr::Indirect(location), + AllocKind::Cell => NounRepr::Cell(location), + AllocKind::Forwarding => NounRepr::Forwarding(location), + } + } + + pub(crate) fn allocated_location(&self, space: &NounSpace) -> Option { + self.repr(space).location() + } + + #[inline] + pub(crate) fn is_stack_allocated(&self, space: &NounSpace) -> bool { + matches!(self.allocated_location(space), Some(AllocLocation::Stack)) + } + pub fn is_cell(&self) -> bool { unsafe { is_cell(self.raw) } } @@ -1557,19 +2797,19 @@ impl Noun { * * This counts the total size, see mass_frame() to count the size in the current frame. */ - pub fn mass(self) -> usize { + pub(crate) fn mass(self, space: &NounSpace) -> usize { unsafe { - let res = self.mass_wind(&|_| true); - self.mass_unwind(&|_| true); + let res = self.mass_wind(space, &|_| true); + self.mass_unwind(space, &|_| true); res } } /** Produce the size of a noun in the current frame, in words */ - pub fn mass_frame(self, stack: &NockStack) -> usize { + pub(crate) fn mass_frame(self, stack: &NockStack, space: &NounSpace) -> usize { unsafe { - let res = self.mass_wind(&|p| stack.is_in_frame(p)); - self.mass_unwind(&|p| stack.is_in_frame(p)); + let res = self.mass_wind(space, &|p| stack.is_in_frame(p)); + self.mass_unwind(space, &|p| stack.is_in_frame(p)); res } } @@ -1588,17 +2828,22 @@ impl Noun { * the first noun, and the second will be the mass of the second noun minus the overlap with * the first noun. */ - pub unsafe fn mass_wind(self, inside: &impl Fn(*const u64) -> bool) -> usize { + pub(crate) unsafe fn mass_wind( + self, + space: &NounSpace, + inside: &impl Fn(*const u64) -> bool, + ) -> usize { if let Ok(mut allocated) = self.as_allocated() { - if inside(allocated.to_raw_pointer()) { - if allocated.get_metadata() & (1 << 32) == 0 { - allocated.set_metadata(allocated.get_metadata() | (1 << 32)); + if inside(allocated.to_raw_pointer(space)) { + if allocated.get_metadata(space) & (1 << 32) == 0 { + allocated.set_metadata(allocated.get_metadata(space) | (1 << 32), space); match allocated.as_either() { - Left(indirect) => indirect.size() + 2, + Left(indirect) => indirect.size(space) + 2, Right(cell) => { + let cell_handle = cell.in_space(space); word_size_of::() - + cell.head().mass_wind(inside) - + cell.tail().mass_wind(inside) + + cell_handle.head().noun().mass_wind(space, inside) + + cell_handle.tail().noun().mass_wind(space, inside) } } } else { @@ -1613,13 +2858,18 @@ impl Noun { } /** See mass_wind() */ - pub unsafe fn mass_unwind(self, inside: &impl Fn(*const u64) -> bool) { + pub(crate) unsafe fn mass_unwind( + self, + space: &NounSpace, + inside: &impl Fn(*const u64) -> bool, + ) { if let Ok(mut allocated) = self.as_allocated() { - if inside(allocated.to_raw_pointer()) { - allocated.set_metadata(allocated.get_metadata() & !(1 << 32)); + if inside(allocated.to_raw_pointer(space)) { + allocated.set_metadata(allocated.get_metadata(space) & !(1 << 32), space); if let Right(cell) = allocated.as_either() { - cell.head().mass_unwind(inside); - cell.tail().mass_unwind(inside); + let cell_handle = cell.in_space(space); + cell_handle.head().noun().mass_unwind(space, inside); + cell_handle.tail().noun().mass_unwind(space, inside); } } } @@ -1635,19 +2885,6 @@ impl fmt::Debug for Noun { write!(f, "{:?}", self.indirect) } else if self.is_cell() { write!(f, "{:?}", self.cell) - } else if self.allocated.forwarding_pointer().is_some() { - write!( - f, - "Noun::Forwarding({:?})", - self.allocated - .forwarding_pointer() - .unwrap_or_else(|| panic!( - "Panicked at {}:{} (git sha: {:?})", - file!(), - line!(), - option_env!("GIT_SHA") - )) - ) } else { write!(f, "Noun::Unknown({:x})", self.raw) } @@ -1658,9 +2895,9 @@ impl fmt::Debug for Noun { impl Slots for Noun {} impl private::RawSlots for Noun { #[inline(always)] - fn raw_slot_direct(&self, axis: u64) -> Result { + fn raw_slot_direct(&self, axis: u64, space: &NounSpace) -> Result { match self.as_either_atom_cell() { - Right(cell) => cell.raw_slot_direct(axis), + Right(cell) => cell.raw_slot_direct(axis, space), Left(_atom) => { if axis == 1 { Ok(*self) @@ -1673,9 +2910,9 @@ impl private::RawSlots for Noun { } #[inline(always)] - fn raw_slot_indirect(&self, axis: &[u64]) -> Result { + fn raw_slot_indirect(&self, axis: &[u64], space: &NounSpace) -> Result { match self.as_either_atom_cell() { - Right(cell) => cell.raw_slot_indirect(axis), + Right(cell) => cell.raw_slot_indirect(axis, space), Left(_atom) => { // Check if axis is 1 (all words are 0 except word[0] & 1 == 1) if axis.len() == 1 && axis[0] == 1 { @@ -1691,6 +2928,72 @@ impl private::RawSlots for Noun { } } +impl Noun { + #[inline(always)] + fn raw_slot_direct_trusted(self, axis: u64, space: &NounSpace) -> Result { + match self.as_either_atom_cell() { + Right(cell) => slot_direct_trusted(&cell, axis, space), + Left(_atom) => { + if axis == 1 { + Ok(self) + } else if axis == 0 { + Err(Error::NotRepresentable) + } else { + Err(Error::NotCell) + } + } + } + } + + #[inline(always)] + fn raw_slot_indirect_trusted(self, axis: &[u64], space: &NounSpace) -> Result { + match self.as_either_atom_cell() { + Right(cell) => slot_indirect_trusted(&cell, axis, space), + Left(_atom) => { + if axis.len() == 1 && axis[0] == 1 { + Ok(self) + } else if axis.is_empty() || (axis.len() == 1 && axis[0] == 0) { + Err(Error::NotRepresentable) + } else { + Err(Error::NotCell) + } + } + } + } + + #[inline(always)] + pub(crate) fn slot_direct_trusted_or_checked( + self, + axis: u64, + space: &NounSpace, + ) -> Result { + self.raw_slot_direct_trusted(axis, space) + } + + #[inline(always)] + pub(crate) fn slot_indirect_trusted_or_checked( + self, + axis: &[u64], + space: &NounSpace, + ) -> Result { + self.raw_slot_indirect_trusted(axis, space) + } + + #[inline(always)] + pub(crate) fn slot_atom_trusted_or_checked( + self, + atom: Atom, + space: &NounSpace, + ) -> Result { + match atom.as_either() { + Left(direct) => self.slot_direct_trusted_or_checked(direct.data(), space), + Right(indirect) => { + self.slot_indirect_trusted_or_checked(indirect.as_slice_trusted(space), space) + } + } + } +} + /** * An allocation object (probably a mem::NockStack) which can allocate a memory buffer sized to * a certain number of nouns @@ -1710,26 +3013,28 @@ pub trait NounAllocator: Sized + Stack { /** Check if two allocated nouns are equal **/ unsafe fn equals(&mut self, a: *mut Noun, b: *mut Noun) -> bool; + + fn noun_space(&self) -> NounSpace; } /** * Implementing types allow component Nouns to be retreived by numeric axis */ -pub trait Slots: private::RawSlots { +pub(crate) trait Slots: private::RawSlots { /** * Retrieve component Noun at given axis, or fail with descriptive error */ - fn slot(&self, axis: u64) -> Result { - self.raw_slot_direct(axis) + fn slot(&self, axis: u64, space: &NounSpace) -> Result { + self.raw_slot_direct(axis, space) } /** * Retrieve component Noun at axis given as Atom, or fail with descriptive error */ - fn slot_atom(&self, atom: Atom) -> Result { + fn slot_atom(&self, atom: Atom, space: &NounSpace) -> Result { match atom.as_either() { - Left(direct) => self.raw_slot_direct(direct.data()), - Right(indirect) => self.raw_slot_indirect(indirect.as_slice()), + Left(direct) => self.raw_slot_direct(direct.data(), space), + Right(indirect) => self.raw_slot_indirect(indirect.as_slice(space), space), } } } @@ -1738,7 +3043,7 @@ pub trait Slots: private::RawSlots { * Implementation methods that should not be made available to derived crates */ mod private { - use crate::noun::{Noun, Result}; + use crate::noun::{Noun, NounSpace, Result}; /** * Implementation of the Slots trait @@ -1747,89 +3052,127 @@ mod private { /** * Actual logic of retreiving Noun object at some axis (direct) */ - fn raw_slot_direct(&self, axis: u64) -> Result; + fn raw_slot_direct(&self, axis: u64, space: &NounSpace) -> Result; /** * Actual logic of retreiving Noun object at some axis (indirect) */ - fn raw_slot_indirect(&self, axis: &[u64]) -> Result; + fn raw_slot_indirect(&self, axis: &[u64], space: &NounSpace) -> Result; } } #[cfg(test)] mod tests { use crate::jets::util::test::init_context; - use crate::noun::{Cell, Slots, D}; + use crate::noun::{Cell, NounSpace, Slots, D}; #[test] + #[cfg_attr(miri, ignore = "memfd_create unsupported in Miri")] fn test_slot_direct_simple() { let mut context = init_context(); + let space = NounSpace::stack_only(&context.stack); let cell = Cell::new(&mut context.stack, D(1), D(2)); // axis 1 returns the whole cell assert!(unsafe { - cell.slot(1) - .expect("slot(1) should exist") + cell.slot(1, &space) + .expect("axis 1 should resolve to the whole cell") .raw_equals(&cell.as_noun()) }); // axis 2 returns head assert!(unsafe { - cell.slot(2) - .expect("slot(2) should exist") + cell.slot(2, &space) + .expect("axis 2 should resolve to the head") .raw_equals(&D(1)) }); // axis 3 returns tail assert!(unsafe { - cell.slot(3) - .expect("slot(3) should exist") + cell.slot(3, &space) + .expect("axis 3 should resolve to the tail") .raw_equals(&D(2)) }); } #[test] + #[cfg_attr(miri, ignore = "memfd_create unsupported in Miri")] fn test_slot_direct_nested() { let mut context = init_context(); + let space = NounSpace::stack_only(&context.stack); let inner = Cell::new(&mut context.stack, D(3), D(4)); // cell = [1 [3 4]] let cell = Cell::new(&mut context.stack, D(1), inner.as_noun()); // axis 6 = 110 binary = tail then head = head of tail = 3 assert!(unsafe { - cell.slot(6) - .expect("slot(6) should exist") + cell.slot(6, &space) + .expect("axis 6 should resolve to tail/head") .raw_equals(&D(3)) }); // axis 7 = 111 binary = tail then tail = tail of tail = 4 assert!(unsafe { - cell.slot(7) - .expect("slot(7) should exist") + cell.slot(7, &space) + .expect("axis 7 should resolve to tail/tail") .raw_equals(&D(4)) }); // axis 4 = 100 binary = head then stop = should fail (head is atom) - assert!(cell.slot(4).is_err()); + assert!(cell.slot(4, &space).is_err()); // cell2 = [[3 4] 2] let cell2 = Cell::new(&mut context.stack, inner.as_noun(), D(2)); // axis 5 = 101 binary = head then tail = tail of head = 4 assert!(unsafe { cell2 - .slot(5) - .expect("slot(5) should exist") + .slot(5, &space) + .expect("axis 5 should resolve to head/tail") .raw_equals(&D(4)) }); } #[test] + #[cfg_attr(miri, ignore = "memfd_create unsupported in Miri")] fn test_slot_zero_axis() { let mut context = init_context(); + let space = NounSpace::stack_only(&context.stack); let cell = Cell::new(&mut context.stack, D(1), D(2)); // axis 0 should fail - assert!(cell.slot(0).is_err()); + assert!(cell.slot(0, &space).is_err()); + } + + #[test] + #[cfg_attr(miri, ignore = "memfd_create unsupported in Miri")] + fn test_with_brand_valid_usage() { + let mut context = init_context(); + let space = NounSpace::stack_only(&context.stack); + let cell = Cell::new(&mut context.stack, D(10), D(20)).as_noun(); + + space.with_brand(|space| { + let cell = space.handle(cell).as_cell().expect("cell"); + assert_eq!( + cell.head() + .as_atom() + .expect("head atom") + .as_u64() + .expect("head atom should fit in u64"), + 10 + ); + assert_eq!( + cell.tail() + .as_atom() + .expect("tail atom") + .as_u64() + .expect("tail atom should fit in u64"), + 20 + ); + assert!(matches!( + cell.as_noun().repr(), + crate::noun::NounRepr::Cell(_) + )); + }); } } @@ -1845,9 +3188,10 @@ mod test { #[cfg_attr(miri, ignore)] fn test_to_ne_bytes_direct() { let mut context = init_context(); + let space = context.stack.noun_space(); let big = ubig!(0x1234567890abcdefa0); let atom = Atom::from_ubig(&mut context.stack, &big); - let bytes = atom.to_ne_bytes(); + let bytes = atom.in_space(&space).to_ne_bytes(); #[cfg(target_endian = "little")] { assert_eq!( @@ -1875,8 +3219,9 @@ mod test { #[cfg_attr(miri, ignore)] fn test_to_ne_bytes_indirect() { let mut context = init_context(); + let space = context.stack.noun_space(); let atom = Atom::new(&mut context.stack, 0x1234); - let bytes = atom.to_ne_bytes(); + let bytes = atom.in_space(&space).to_ne_bytes(); #[cfg(target_endian = "little")] { assert_eq!(bytes, vec![0x34, 0x12, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00]); @@ -1892,14 +3237,15 @@ mod test { #[cfg_attr(miri, ignore)] fn test_to_x_bytes_direct() { let mut context = init_context(); + let space = context.stack.noun_space(); let atom = Atom::new(&mut context.stack, 0x1234); - let bytes_le = atom.to_le_bytes(); + let bytes_le = atom.in_space(&space).to_le_bytes(); assert_eq!( bytes_le, vec![0x34, 0x12, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00] ); - let bytes_be = atom.to_be_bytes(); + let bytes_be = atom.in_space(&space).to_be_bytes(); assert_eq!( bytes_be, vec![0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x12, 0x34] @@ -1911,14 +3257,15 @@ mod test { #[cfg_attr(miri, ignore)] fn test_to_le_bytes_indirect() { let mut context = init_context(); + let space = context.stack.noun_space(); let big = ubig!(0x1234567890abcd); let atom = Atom::from_ubig(&mut context.stack, &big); - let bytes = atom.to_le_bytes(); + let bytes = atom.in_space(&space).to_le_bytes(); assert_eq!(bytes, vec![0xcd, 0xab, 0x90, 0x78, 0x56, 0x34, 0x12, 0x00]); // let big = ubig!(0x1234567890abcdefa0); let atom = Atom::from_ubig(&mut context.stack, &big); - let bytes = atom.to_le_bytes(); + let bytes = atom.in_space(&space).to_le_bytes(); assert_eq!( bytes, vec![ @@ -1933,14 +3280,15 @@ mod test { #[cfg_attr(miri, ignore)] fn test_to_be_bytes_indirect() { let mut context = init_context(); + let space = context.stack.noun_space(); let big = ubig!(0x34567890abcdef); let atom = Atom::from_ubig(&mut context.stack, &big); - let bytes = atom.to_be_bytes(); + let bytes = atom.in_space(&space).to_be_bytes(); assert_eq!(bytes, vec![0x00, 0x34, 0x56, 0x78, 0x90, 0xab, 0xcd, 0xef]); // let big = ubig!(0x1234567890abcdefa0); let atom = Atom::from_ubig(&mut context.stack, &big); - let bytes = atom.to_be_bytes(); + let bytes = atom.in_space(&space).to_be_bytes(); assert_eq!( bytes, vec![ diff --git a/crates/nockvm/rust/nockvm/src/offset.rs b/crates/nockvm/rust/nockvm/src/offset.rs new file mode 100644 index 000000000..19974d473 --- /dev/null +++ b/crates/nockvm/rust/nockvm/src/offset.rs @@ -0,0 +1,142 @@ +use std::fmt; + +use bincode::{Decode, Encode}; +use thiserror::Error; + +#[derive(Debug, Error, Clone, Copy, PartialEq, Eq)] +pub enum OffsetError { + #[error("word offset {words} does not fit in u64")] + TooLargeForU64 { words: usize }, + #[error("word offset {words} does not fit in usize")] + TooLargeForUsize { words: u64 }, +} + +pub trait WordOffset: Copy + Eq + Ord + fmt::Debug + Sized { + fn from_words(words: u64) -> Self; + fn words(self) -> u64; + + fn zero() -> Self { + Self::from_words(0) + } + + fn checked_add(self, rhs: Self) -> Option { + self.words().checked_add(rhs.words()).map(Self::from_words) + } + + fn checked_add_words(self, rhs: u64) -> Option { + self.words().checked_add(rhs).map(Self::from_words) + } + + fn checked_sub(self, rhs: Self) -> Option { + self.words().checked_sub(rhs.words()).map(Self::from_words) + } + + fn checked_sub_words(self, rhs: u64) -> Option { + self.words().checked_sub(rhs).map(Self::from_words) + } + + fn checked_bytes(self) -> Option { + self.words().checked_mul(8) + } + + fn checked_bytes_usize(self) -> Option { + self.checked_bytes() + .and_then(|bytes| usize::try_from(bytes).ok()) + } + + fn try_from_usize(words: usize) -> Result { + let words = u64::try_from(words).map_err(|_| OffsetError::TooLargeForU64 { words })?; + Ok(Self::from_words(words)) + } + + fn try_into_usize(self) -> Result { + usize::try_from(self.words()).map_err(|_| OffsetError::TooLargeForUsize { + words: self.words(), + }) + } +} + +macro_rules! define_offset_words { + ($name:ident) => { + #[repr(transparent)] + #[derive( + Clone, Copy, Debug, Default, Encode, Decode, PartialEq, Eq, PartialOrd, Ord, Hash, + )] + pub struct $name(u64); + + impl $name { + pub const fn from_words(words: u64) -> Self { + Self(words) + } + + pub const fn words(self) -> u64 { + self.0 + } + + pub fn try_from_usize(words: usize) -> Result { + ::try_from_usize(words) + } + + pub fn try_into_usize(self) -> Result { + ::try_into_usize(self) + } + + pub fn checked_add(self, rhs: Self) -> Option { + ::checked_add(self, rhs) + } + + pub fn checked_add_words(self, rhs: u64) -> Option { + ::checked_add_words(self, rhs) + } + + pub fn checked_sub(self, rhs: Self) -> Option { + ::checked_sub(self, rhs) + } + + pub fn checked_sub_words(self, rhs: u64) -> Option { + ::checked_sub_words(self, rhs) + } + + pub fn checked_bytes(self) -> Option { + ::checked_bytes(self) + } + + pub fn checked_bytes_usize(self) -> Option { + ::checked_bytes_usize(self) + } + } + + impl WordOffset for $name { + fn from_words(words: u64) -> Self { + Self(words) + } + + fn words(self) -> u64 { + self.0 + } + } + + impl TryFrom for $name { + type Error = OffsetError; + + fn try_from(words: usize) -> Result { + Self::try_from_usize(words) + } + } + + impl From<$name> for u64 { + fn from(value: $name) -> Self { + value.0 + } + } + + impl fmt::Display for $name { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "{}", self.0) + } + } + }; +} + +define_offset_words!(StackOffsetWords); +define_offset_words!(PmaOffsetWords); diff --git a/crates/nockvm/rust/nockvm/src/pma.rs b/crates/nockvm/rust/nockvm/src/pma.rs new file mode 100644 index 000000000..1802a3f9d --- /dev/null +++ b/crates/nockvm/rust/nockvm/src/pma.rs @@ -0,0 +1,4004 @@ +//! Persistent Memory Arena (PMA) +//! +//! The PMA is a file-backed memory region for storing long-lived Nouns. +//! It uses bump allocation and stores nouns in offset form. + +use std::fs; +use std::io::{self, Read, Seek, SeekFrom, Write}; +#[cfg(unix)] +use std::os::unix::fs::MetadataExt; +use std::path::{Path, PathBuf}; +use std::ptr::copy_nonoverlapping; +use std::sync::Arc; +use std::time::{Duration, Instant}; + +use either::Either::{Left, Right}; +#[cfg(feature = "pma-assert")] +use intmap::IntMap; +#[cfg(unix)] +use libc; +use smallvec::SmallVec; +use thiserror::Error; +use tracing::{debug, info}; + +use crate::ext::noun_equality; +use crate::mem::{word_size_of, Arena, NewStackError, NockStack}; +use crate::noun::{ + AllocLocation, Atom, Cell, CellMemory, IndirectAtom, Noun, NounAllocator, NounRepr, NounSpace, +}; +use crate::offset::PmaOffsetWords; + +mod stream; +pub use stream::{ + classify_pma_noun, PmaDirectJamConfig, PmaDirectJamError, PmaDirectReader, PmaRawNounKind, +}; + +const PMA_MAGIC: u64 = u64::from_le_bytes(*b"NOCKPMA1"); +const PMA_VERSION_V1: u64 = 1; +const PMA_VERSION: u64 = 2; +const PMA_V2_TRAILER_MAGIC: u64 = u64::from_le_bytes(*b"NOCKPM2!"); +const PMA_GROWTH_JOURNAL_MAGIC: u64 = u64::from_le_bytes(*b"PMAGROW1"); +const PMA_MIGRATION_JOURNAL_MAGIC: u64 = u64::from_le_bytes(*b"PMAMIGR2"); +const DEFAULT_PMA_RESERVED_BYTES: usize = 1 << 40; // 1 TiB virtual reservation. +const NOCK_PMA_RESERVED_WORDS_ENV: &str = "NOCK_PMA_RESERVED_WORDS"; +const NOCK_PMA_GROWTH_EVENTS_PATH_ENV: &str = "NOCK_PMA_GROWTH_EVENTS_PATH"; +const NOCK_PMA_RESIZE_FAIL_AT_ENV: &str = "NOCK_PMA_RESIZE_FAIL_AT"; +const NOCK_PMA_MIGRATION_FAIL_AT_ENV: &str = "NOCK_PMA_MIGRATION_FAIL_AT"; +const NOCK_PMA_DISABLE_RESIZE_ENV: &str = "NOCK_PMA_DISABLE_RESIZE_FOR_REGRESSION"; + +#[cfg(test)] +thread_local! { + static TEST_GROWTH_FAILURE_POINT: std::cell::RefCell> = + const { std::cell::RefCell::new(None) }; + static TEST_MIGRATION_FAILURE_POINT: std::cell::RefCell> = + const { std::cell::RefCell::new(None) }; +} + +/// The metadata for the PMA is a trailer or footer because otherwise the base + offset pointer derivations would need +/// to account for the footer size. With this design it's just base pointer + offset. +#[repr(C)] +#[derive(Clone, Copy, Debug)] +struct PmaLegacyTrailer { + magic: u64, + version: u64, + data_words: u64, + alloc_offset: u64, +} + +const PMA_LEGACY_TRAILER_BYTES: usize = std::mem::size_of::(); +const PMA_TRAILER_BYTES: usize = 64; + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct PmaFileMetadata { + pub magic: u64, + pub version: u64, + pub data_words: u64, + pub alloc_words: u64, + pub capacity_words: u64, + pub free_words: u64, + pub reserved_words: u64, + pub file_bytes: u64, + pub apparent_file_bytes: u64, + pub physical_file_bytes: Option, +} + +impl PmaLegacyTrailer { + fn to_bytes(self) -> [u8; PMA_LEGACY_TRAILER_BYTES] { + let mut buf = [0u8; PMA_LEGACY_TRAILER_BYTES]; + buf[0..8].copy_from_slice(&self.magic.to_le_bytes()); + buf[8..16].copy_from_slice(&self.version.to_le_bytes()); + buf[16..24].copy_from_slice(&self.data_words.to_le_bytes()); + buf[24..32].copy_from_slice(&self.alloc_offset.to_le_bytes()); + buf + } + + fn from_bytes(buf: [u8; PMA_LEGACY_TRAILER_BYTES]) -> Self { + let magic = u64::from_le_bytes(buf[0..8].try_into().expect("magic slice")); + let version = u64::from_le_bytes(buf[8..16].try_into().expect("version slice")); + let data_words = u64::from_le_bytes(buf[16..24].try_into().expect("data_words slice")); + let alloc_offset = u64::from_le_bytes(buf[24..32].try_into().expect("alloc_offset slice")); + Self { + magic, + version, + data_words, + alloc_offset, + } + } +} + +#[derive(Clone, Copy, Debug)] +struct PmaTrailerV2 { + reserved_words: u64, + footer: PmaLegacyTrailer, +} + +impl PmaTrailerV2 { + fn new(capacity_words: u64, alloc_words: u64, reserved_words: u64) -> Self { + Self { + reserved_words, + footer: PmaLegacyTrailer { + magic: PMA_MAGIC, + version: PMA_VERSION, + data_words: capacity_words, + alloc_offset: alloc_words, + }, + } + } + + fn checksum(version: u64, reserved_words: u64, capacity_words: u64, alloc_words: u64) -> u64 { + PMA_V2_TRAILER_MAGIC + ^ version.rotate_left(5) + ^ reserved_words.rotate_left(13) + ^ capacity_words.rotate_left(29) + ^ alloc_words.rotate_left(43) + } + + fn to_bytes(self) -> [u8; PMA_TRAILER_BYTES] { + let mut buf = [0u8; PMA_TRAILER_BYTES]; + let checksum = Self::checksum( + PMA_VERSION, self.reserved_words, self.footer.data_words, self.footer.alloc_offset, + ); + buf[0..8].copy_from_slice(&PMA_V2_TRAILER_MAGIC.to_le_bytes()); + buf[8..16].copy_from_slice(&PMA_VERSION.to_le_bytes()); + buf[16..24].copy_from_slice(&self.reserved_words.to_le_bytes()); + buf[24..32].copy_from_slice(&checksum.to_le_bytes()); + buf[32..64].copy_from_slice(&self.footer.to_bytes()); + buf + } + + fn from_bytes(buf: [u8; PMA_TRAILER_BYTES]) -> Option { + let extension_magic = u64::from_le_bytes(buf[0..8].try_into().ok()?); + let version = u64::from_le_bytes(buf[8..16].try_into().ok()?); + let reserved_words = u64::from_le_bytes(buf[16..24].try_into().ok()?); + let checksum = u64::from_le_bytes(buf[24..32].try_into().ok()?); + let mut footer_buf = [0u8; PMA_LEGACY_TRAILER_BYTES]; + footer_buf.copy_from_slice(&buf[32..64]); + let footer = PmaLegacyTrailer::from_bytes(footer_buf); + let expected_checksum = Self::checksum( + version, reserved_words, footer.data_words, footer.alloc_offset, + ); + (extension_magic == PMA_V2_TRAILER_MAGIC + && version == PMA_VERSION + && footer.magic == PMA_MAGIC + && footer.version == PMA_VERSION + && footer.alloc_offset <= footer.data_words + && footer.data_words <= reserved_words + && checksum == expected_checksum) + .then_some(Self { + reserved_words, + footer, + }) + } +} + +#[derive(Clone, Copy, Debug)] +struct PmaGrowthJournal { + magic: u64, + old_data_words: u64, + old_alloc_words: u64, + new_data_words: u64, + reserved_words: u64, + checksum: u64, +} + +#[derive(Clone, Copy, Debug)] +struct PmaGrowthEvent { + old_words: usize, + new_words: usize, + alloc_words: usize, + required_words: usize, + request_words: usize, + context: &'static str, + elapsed: Duration, +} + +impl PmaGrowthJournal { + const BYTES: usize = 48; + const LEGACY_BYTES: usize = 40; + + fn new( + old_data_words: u64, + old_alloc_words: u64, + new_data_words: u64, + reserved_words: u64, + ) -> Self { + let checksum = Self::checksum( + old_data_words, old_alloc_words, new_data_words, reserved_words, + ); + Self { + magic: PMA_GROWTH_JOURNAL_MAGIC, + old_data_words, + old_alloc_words, + new_data_words, + reserved_words, + checksum, + } + } + + fn checksum( + old_data_words: u64, + old_alloc_words: u64, + new_data_words: u64, + reserved_words: u64, + ) -> u64 { + PMA_GROWTH_JOURNAL_MAGIC + ^ old_data_words.rotate_left(7) + ^ old_alloc_words.rotate_left(17) + ^ new_data_words.rotate_left(31) + ^ reserved_words.rotate_left(47) + } + + fn to_bytes(self) -> [u8; Self::BYTES] { + let mut buf = [0u8; Self::BYTES]; + buf[0..8].copy_from_slice(&self.magic.to_le_bytes()); + buf[8..16].copy_from_slice(&self.old_data_words.to_le_bytes()); + buf[16..24].copy_from_slice(&self.old_alloc_words.to_le_bytes()); + buf[24..32].copy_from_slice(&self.new_data_words.to_le_bytes()); + buf[32..40].copy_from_slice(&self.reserved_words.to_le_bytes()); + buf[40..48].copy_from_slice(&self.checksum.to_le_bytes()); + buf + } + + fn from_bytes(buf: [u8; Self::BYTES]) -> Option { + let magic = u64::from_le_bytes(buf[0..8].try_into().ok()?); + let old_data_words = u64::from_le_bytes(buf[8..16].try_into().ok()?); + let old_alloc_words = u64::from_le_bytes(buf[16..24].try_into().ok()?); + let new_data_words = u64::from_le_bytes(buf[24..32].try_into().ok()?); + let reserved_words = u64::from_le_bytes(buf[32..40].try_into().ok()?); + let checksum = u64::from_le_bytes(buf[40..48].try_into().ok()?); + let journal = Self { + magic, + old_data_words, + old_alloc_words, + new_data_words, + reserved_words, + checksum, + }; + journal.validate().then_some(journal) + } + + fn from_legacy_bytes(buf: [u8; Self::LEGACY_BYTES]) -> Option { + let magic = u64::from_le_bytes(buf[0..8].try_into().ok()?); + let old_data_words = u64::from_le_bytes(buf[8..16].try_into().ok()?); + let old_alloc_words = u64::from_le_bytes(buf[16..24].try_into().ok()?); + let new_data_words = u64::from_le_bytes(buf[24..32].try_into().ok()?); + let legacy_checksum = u64::from_le_bytes(buf[32..40].try_into().ok()?); + let expected_legacy_checksum = PMA_GROWTH_JOURNAL_MAGIC + ^ old_data_words.rotate_left(7) + ^ old_alloc_words.rotate_left(17) + ^ new_data_words.rotate_left(31); + if magic != PMA_GROWTH_JOURNAL_MAGIC + || old_alloc_words > old_data_words + || old_data_words > new_data_words + || legacy_checksum != expected_legacy_checksum + { + return None; + } + Some(Self::new( + old_data_words, old_alloc_words, new_data_words, new_data_words, + )) + } + + fn validate(self) -> bool { + self.magic == PMA_GROWTH_JOURNAL_MAGIC + && self.old_alloc_words <= self.old_data_words + && self.old_data_words <= self.new_data_words + && self.new_data_words <= self.reserved_words + && self.checksum + == Self::checksum( + self.old_data_words, self.old_alloc_words, self.new_data_words, + self.reserved_words, + ) + } +} + +#[derive(Clone, Copy, Debug)] +struct PmaMigrationJournal { + magic: u64, + capacity_words: u64, + alloc_words: u64, + reserved_words: u64, + checksum: u64, +} + +impl PmaMigrationJournal { + const BYTES: usize = 40; + + fn new(capacity_words: u64, alloc_words: u64, reserved_words: u64) -> Self { + let checksum = Self::checksum(capacity_words, alloc_words, reserved_words); + Self { + magic: PMA_MIGRATION_JOURNAL_MAGIC, + capacity_words, + alloc_words, + reserved_words, + checksum, + } + } + + fn checksum(capacity_words: u64, alloc_words: u64, reserved_words: u64) -> u64 { + PMA_MIGRATION_JOURNAL_MAGIC + ^ capacity_words.rotate_left(11) + ^ alloc_words.rotate_left(23) + ^ reserved_words.rotate_left(37) + } + + fn to_bytes(self) -> [u8; Self::BYTES] { + let mut buf = [0u8; Self::BYTES]; + buf[0..8].copy_from_slice(&self.magic.to_le_bytes()); + buf[8..16].copy_from_slice(&self.capacity_words.to_le_bytes()); + buf[16..24].copy_from_slice(&self.alloc_words.to_le_bytes()); + buf[24..32].copy_from_slice(&self.reserved_words.to_le_bytes()); + buf[32..40].copy_from_slice(&self.checksum.to_le_bytes()); + buf + } + + fn from_bytes(buf: [u8; Self::BYTES]) -> Option { + let magic = u64::from_le_bytes(buf[0..8].try_into().ok()?); + let capacity_words = u64::from_le_bytes(buf[8..16].try_into().ok()?); + let alloc_words = u64::from_le_bytes(buf[16..24].try_into().ok()?); + let reserved_words = u64::from_le_bytes(buf[24..32].try_into().ok()?); + let checksum = u64::from_le_bytes(buf[32..40].try_into().ok()?); + let journal = Self { + magic, + capacity_words, + alloc_words, + reserved_words, + checksum, + }; + journal.validate().then_some(journal) + } + + fn validate(self) -> bool { + self.magic == PMA_MIGRATION_JOURNAL_MAGIC + && self.alloc_words <= self.capacity_words + && self.capacity_words <= self.reserved_words + && self.checksum + == Self::checksum(self.capacity_words, self.alloc_words, self.reserved_words) + } +} + +/// Errors that can occur during PMA operations +#[derive(Debug, Error)] +pub enum PmaError { + #[error("PMA is full, cannot allocate {requested} words (available: {available})")] + OutOfMemory { requested: usize, available: usize }, + + #[error("Failed to create arena: {0}")] + ArenaError(#[from] NewStackError), + + #[error("PMA metadata IO failed: {0}")] + MetadataIo(#[from] std::io::Error), + + #[error("Invalid PMA metadata: {0}")] + InvalidMetadata(String), + + #[error("PMA growth failed: {0}")] + GrowthFailed(String), + + #[error("PMA metadata migration failed: {0}")] + MigrationFailed(String), +} + +/// The Persistent Memory Arena +/// +/// A bump-allocated memory region for storing nouns in offset form. +/// The PMA is backed by a file and can persist across program restarts. +/// +/// "Bump-allocated" means allocation simply increments the `alloc_offset` +/// pointer by the requested size—there is no free list, no compaction, and +/// no mechanism to reclaim memory once allocated. This makes allocation +/// extremely fast (just a pointer bump) but means the PMA grows monotonically +/// until explicitly reset. +/// +/// When a Noun that lives in the PMA needs to be modified, the workflow is: +/// 1. The Noun is read from the PMA (already in offset form) +/// 2. Modifications happen in the NockStack (ephemeral working memory) +/// 3. The modified Noun is copied back to the PMA via `copy_to_pma()` +/// +/// Step 3 only allocates space for the Allocated subtrees that changed. For +/// example, if `[2 3]` becomes `[4 3]`: +/// - The Cell is Allocated, so a NEW cell is allocated in the PMA with head=4, +/// tail=3 with new DirectAtoms for the 4 and 3 since they are not Allocated. +/// - The old `[2 3]` cell remains in the PMA, untouched but now unreachable +/// +/// For Allocated structures, unchanged subtrees that are already in PMA (offset +/// form) are reused without copying. If `[[1 2] 3]` becomes `[[1 2] 4]`: +/// - A NEW outer cell is allocated with tail=4 +/// - The head still points to the existing `[1 2]` in PMA (no copy needed) +/// - Only the old outer cell becomes garbage; `[1 2]` is shared +/// +/// This copy allocates fresh space in the PMA for the new version—the old +/// version is not overwritten or freed, it simply becomes unreachable garbage. +/// Garbage collection (Milestone 4) will eventually reclaim this dead space. +/// +/// Currently Pma is only suitable for a single reader/writer. In the future, +/// `alloc_offset` will be changed to `AtomicUsize` to allow multiple readers. +/// +/// For more information, see `open/docs/pma/DESIGN.md`. +pub struct Pma { + /// The underlying arena for memory management and pointer resolution + arena: Arc, + /// Current allocation offset in words (bump pointer) + alloc_offset: usize, + /// Path to the backing file (for future file-backed persistence) + path: PathBuf, + /// Set after a growth failure may have partially changed the file/mapping. + growth_poisoned: bool, +} + +impl Pma { + pub fn read_file_metadata(path: &Path) -> Result { + let mut file = std::fs::File::open(path)?; + match Self::read_file_metadata_from_reader(&mut file) { + Ok(metadata) => Ok(metadata), + Err(err) => { + if let Some(metadata) = Self::recover_metadata_from_migration_journal(path)? { + return Ok(metadata); + } + if let Some(metadata) = Self::recover_metadata_from_growth_journal(path)? { + return Ok(metadata); + } + Err(err) + } + } + } + + fn read_file_metadata_from_reader( + file: &mut std::fs::File, + ) -> Result { + let os_metadata = file.metadata()?; + let file_len_u64 = os_metadata.len(); + let file_len = usize::try_from(file_len_u64) + .map_err(|_| PmaError::InvalidMetadata("file is too large to map".to_string()))?; + if file_len < PMA_LEGACY_TRAILER_BYTES { + return Err(PmaError::InvalidMetadata(format!( + "file too small: {file_len} bytes" + ))); + } + + if file_len >= PMA_TRAILER_BYTES { + let data_bytes = file_len - PMA_TRAILER_BYTES; + if data_bytes.is_multiple_of(8) { + let mut trailer_bytes = [0u8; PMA_TRAILER_BYTES]; + file.seek(SeekFrom::End(-(PMA_TRAILER_BYTES as i64)))?; + file.read_exact(&mut trailer_bytes)?; + if let Some(trailer) = PmaTrailerV2::from_bytes(trailer_bytes) { + let capacity_words = data_bytes >> 3; + return Self::metadata_from_v2_trailer( + trailer, capacity_words, file_len_u64, &os_metadata, + ); + } + } + } + + let data_bytes = file_len - PMA_LEGACY_TRAILER_BYTES; + if data_bytes.is_multiple_of(8) { + let mut trailer_bytes = [0u8; PMA_LEGACY_TRAILER_BYTES]; + file.seek(SeekFrom::End(-(PMA_LEGACY_TRAILER_BYTES as i64)))?; + file.read_exact(&mut trailer_bytes)?; + let trailer = PmaLegacyTrailer::from_bytes(trailer_bytes); + if let Ok(metadata) = Self::metadata_from_legacy_trailer( + trailer, + data_bytes >> 3, + file_len_u64, + &os_metadata, + ) { + return Ok(metadata); + } + } + + // If a v1-to-v2 migration extended the file but crashed before publishing a valid + // v2 trailer, the old v1 trailer is still at the old EOF: file_len - PMA_TRAILER_BYTES. + if file_len >= PMA_TRAILER_BYTES { + let data_bytes = file_len - PMA_TRAILER_BYTES; + if data_bytes.is_multiple_of(8) { + let mut trailer_bytes = [0u8; PMA_LEGACY_TRAILER_BYTES]; + file.seek(SeekFrom::Start(data_bytes as u64))?; + file.read_exact(&mut trailer_bytes)?; + let trailer = PmaLegacyTrailer::from_bytes(trailer_bytes); + if let Ok(metadata) = Self::metadata_from_legacy_trailer( + trailer, + data_bytes >> 3, + file_len_u64, + &os_metadata, + ) { + return Ok(metadata); + } + } + } + + Err(PmaError::InvalidMetadata( + "no valid PMA v2 or legacy trailer found".to_string(), + )) + } + + fn metadata_from_v2_trailer( + trailer: PmaTrailerV2, + data_words: usize, + file_len_u64: u64, + os_metadata: &fs::Metadata, + ) -> Result { + if trailer.footer.data_words as usize != data_words { + return Err(PmaError::InvalidMetadata(format!( + "metadata capacity_words {} does not match file ({data_words})", + trailer.footer.data_words + ))); + } + Ok(PmaFileMetadata { + magic: trailer.footer.magic, + version: trailer.footer.version, + data_words: trailer.footer.data_words, + alloc_words: trailer.footer.alloc_offset, + capacity_words: trailer.footer.data_words, + free_words: trailer + .footer + .data_words + .saturating_sub(trailer.footer.alloc_offset), + reserved_words: trailer.reserved_words, + file_bytes: file_len_u64, + apparent_file_bytes: file_len_u64, + physical_file_bytes: physical_file_bytes(os_metadata), + }) + } + + fn metadata_from_legacy_trailer( + trailer: PmaLegacyTrailer, + data_words: usize, + file_len_u64: u64, + os_metadata: &fs::Metadata, + ) -> Result { + if trailer.magic != PMA_MAGIC { + return Err(PmaError::InvalidMetadata("bad PMA magic".to_string())); + } + if trailer.version != PMA_VERSION_V1 { + return Err(PmaError::InvalidMetadata(format!( + "unsupported PMA version {}", + trailer.version + ))); + } + if trailer.data_words as usize != data_words { + return Err(PmaError::InvalidMetadata(format!( + "metadata data_words {} does not match file ({data_words})", + trailer.data_words + ))); + } + if trailer.alloc_offset > trailer.data_words { + return Err(PmaError::InvalidMetadata(format!( + "alloc_offset {} exceeds data_words {}", + trailer.alloc_offset, trailer.data_words + ))); + } + Ok(PmaFileMetadata { + magic: trailer.magic, + version: trailer.version, + data_words: trailer.data_words, + alloc_words: trailer.alloc_offset, + capacity_words: trailer.data_words, + free_words: trailer.data_words.saturating_sub(trailer.alloc_offset), + reserved_words: trailer.data_words, + file_bytes: file_len_u64, + apparent_file_bytes: file_len_u64, + physical_file_bytes: physical_file_bytes(os_metadata), + }) + } + + /// Return the default virtual reservation for a PMA with `capacity_words` current capacity. + pub fn default_reserved_words_for_capacity(capacity_words: usize) -> usize { + default_reserved_words(capacity_words) + } + + /// Create a new PMA with the given size in words + pub fn new(size_words: usize, path: PathBuf) -> Result { + let reserved_words = Self::default_reserved_words_for_capacity(size_words); + Self::new_with_reserved(size_words, reserved_words, path) + } + + /// Create a new PMA with explicit current capacity and virtual reservation. + pub fn new_with_reserved( + capacity_words: usize, + reserved_words: usize, + path: PathBuf, + ) -> Result { + let reserved_words = reserved_words.max(capacity_words); + let arena = Arena::allocate_growable_file( + &path, capacity_words, reserved_words, PMA_TRAILER_BYTES, + )?; + let pma = Self { + arena, + alloc_offset: 0, + path, + growth_poisoned: false, + }; + pma.persist_metadata(); + Ok(pma) + } + + /// Open an existing PMA file without truncating it or raising its virtual reservation. + pub fn open(path: PathBuf) -> Result { + Self::open_with_min_preserving_reservation(path, 0) + } + + /// Open an existing PMA and ensure its current capacity is at least `min_words`, preserving + /// the existing virtual reservation unless `min_words` requires raising it. + pub fn open_with_min(path: PathBuf, min_words: usize) -> Result { + Self::open_with_min_preserving_reservation(path, min_words) + } + + /// Open an existing PMA with an explicit minimum current capacity and virtual reservation. + pub fn open_with_min_and_reserved( + path: PathBuf, + min_words: usize, + reserved_words: usize, + ) -> Result { + Self::open_with_min_inner(path, min_words, Some(reserved_words)) + } + + fn open_with_min_preserving_reservation( + path: PathBuf, + min_words: usize, + ) -> Result { + Self::open_with_min_inner(path, min_words, None) + } + + fn open_with_min_inner( + path: PathBuf, + min_words: usize, + requested_reserved_words: Option, + ) -> Result { + let metadata = Self::read_file_metadata(&path)?; + let data_words = usize::try_from(metadata.data_words).map_err(|_| { + PmaError::InvalidMetadata("PMA data_words exceeds usize addressable range".to_string()) + })?; + let alloc_offset = usize::try_from(metadata.alloc_words).map_err(|_| { + PmaError::InvalidMetadata( + "PMA alloc_offset exceeds usize addressable range".to_string(), + ) + })?; + let existing_reserved_words = + usize::try_from(metadata.reserved_words).unwrap_or(usize::MAX); + let reserved_words = requested_reserved_words + .unwrap_or(existing_reserved_words) + .max(existing_reserved_words) + .max(data_words) + .max(min_words); + if metadata.version == PMA_VERSION_V1 { + Self::migrate_legacy_metadata(&path, data_words, alloc_offset, reserved_words)?; + } + let arena = Arena::open_growable_file( + &path, + data_words, + reserved_words.max(data_words), + PMA_TRAILER_BYTES, + )?; + let mut pma = Self { + arena, + alloc_offset, + path, + growth_poisoned: false, + }; + if pma.size_words() < min_words && !pma_growth_disabled_for_regression() { + pma.grow_to_capacity(min_words)?; + } else { + pma.persist_metadata(); + } + pma.clear_growth_journal_best_effort(); + pma.clear_migration_journal_best_effort(); + Ok(pma) + } + + fn recover_metadata_from_growth_journal( + path: &Path, + ) -> Result, PmaError> { + let Some(journal) = read_growth_journal(path)? else { + return Ok(None); + }; + let os_metadata = fs::metadata(path)?; + let file_len = os_metadata.len(); + let new_len = file_len_for_words_u64(journal.new_data_words)?; + let old_len = file_len_for_words_u64(journal.old_data_words)?; + if file_len == new_len { + return Ok(Some(PmaFileMetadata { + magic: PMA_MAGIC, + version: PMA_VERSION, + data_words: journal.new_data_words, + alloc_words: journal.old_alloc_words, + capacity_words: journal.new_data_words, + free_words: journal + .new_data_words + .saturating_sub(journal.old_alloc_words), + reserved_words: journal.reserved_words, + file_bytes: file_len, + apparent_file_bytes: file_len, + physical_file_bytes: physical_file_bytes(&os_metadata), + })); + } + if file_len == old_len { + return Ok(Some(PmaFileMetadata { + magic: PMA_MAGIC, + version: PMA_VERSION, + data_words: journal.old_data_words, + alloc_words: journal.old_alloc_words, + capacity_words: journal.old_data_words, + free_words: journal + .old_data_words + .saturating_sub(journal.old_alloc_words), + reserved_words: journal.reserved_words, + file_bytes: file_len, + apparent_file_bytes: file_len, + physical_file_bytes: physical_file_bytes(&os_metadata), + })); + } + Ok(None) + } + + fn recover_metadata_from_migration_journal( + path: &Path, + ) -> Result, PmaError> { + let Some(journal) = read_migration_journal(path)? else { + return Ok(None); + }; + let os_metadata = fs::metadata(path)?; + let file_len = os_metadata.len(); + let old_len = + file_len_for_words_u64_with_tail(journal.capacity_words, PMA_LEGACY_TRAILER_BYTES)?; + let new_len = file_len_for_words_u64(journal.capacity_words)?; + if file_len != old_len && file_len != new_len { + return Ok(None); + } + Ok(Some(PmaFileMetadata { + magic: PMA_MAGIC, + version: PMA_VERSION_V1, + data_words: journal.capacity_words, + alloc_words: journal.alloc_words, + capacity_words: journal.capacity_words, + free_words: journal.capacity_words.saturating_sub(journal.alloc_words), + reserved_words: journal.reserved_words, + file_bytes: file_len, + apparent_file_bytes: file_len, + physical_file_bytes: physical_file_bytes(&os_metadata), + })) + } + + fn migrate_legacy_metadata( + path: &Path, + capacity_words: usize, + alloc_words: usize, + reserved_words: usize, + ) -> Result<(), PmaError> { + let capacity_words_u64 = u64::try_from(capacity_words) + .map_err(|_| PmaError::InvalidMetadata("PMA capacity exceeds u64".to_string()))?; + let alloc_words_u64 = u64::try_from(alloc_words) + .map_err(|_| PmaError::InvalidMetadata("PMA allocation exceeds u64".to_string()))?; + let reserved_words_u64 = u64::try_from(reserved_words) + .map_err(|_| PmaError::InvalidMetadata("PMA reservation exceeds u64".to_string()))?; + let journal = + PmaMigrationJournal::new(capacity_words_u64, alloc_words_u64, reserved_words_u64); + write_migration_journal(path, journal)?; + if should_inject_migration_failure("before_new_metadata_write") { + return Err(PmaError::MigrationFailed( + "injected PMA migration failure before new metadata write".to_string(), + )); + } + + let mut file = fs::OpenOptions::new().read(true).write(true).open(path)?; + file.set_len(file_len_for_words_u64(capacity_words_u64)?)?; + file.seek(SeekFrom::Start(capacity_words_u64 * 8))?; + let trailer = PmaTrailerV2::new(capacity_words_u64, alloc_words_u64, reserved_words_u64); + file.write_all(&trailer.to_bytes())?; + if should_inject_migration_failure("after_new_metadata_write_before_fsync") { + return Err(PmaError::MigrationFailed( + "injected PMA migration failure after new metadata write before fsync".to_string(), + )); + } + file.sync_data()?; + if should_inject_migration_failure("after_metadata_fsync_before_parent_directory_sync") { + return Err(PmaError::MigrationFailed( + "injected PMA migration failure after metadata fsync before parent directory sync" + .to_string(), + )); + } + if let Some(parent) = path.parent() { + fs::File::open(parent)?.sync_all()?; + } + if should_inject_migration_failure("after_parent_directory_sync") { + return Err(PmaError::MigrationFailed( + "injected PMA migration failure after parent directory sync".to_string(), + )); + } + fs::remove_file(migration_journal_path(path))?; + if let Some(parent) = path.parent() { + fs::File::open(parent)?.sync_all()?; + } + if should_inject_migration_failure("after_marker_declares_upgraded") { + return Err(PmaError::MigrationFailed( + "injected PMA migration failure after marker declares upgraded".to_string(), + )); + } + info!( + path = %path.display(), + capacity_words, + alloc_words, + reserved_words, + "PMA legacy metadata migrated to v2" + ); + Ok(()) + } + + fn clear_growth_journal_best_effort(&self) { + let _ = fs::remove_file(growth_journal_path(&self.path)); + } + + fn clear_migration_journal_best_effort(&self) { + let _ = fs::remove_file(migration_journal_path(&self.path)); + } + + fn growth_poisoned_error() -> PmaError { + PmaError::GrowthFailed( + "previous PMA growth failed after possibly changing the backing file; restart the node or reopen the PMA so growth journal recovery can complete".to_string(), + ) + } + + fn ensure_not_growth_poisoned(&self) -> Result<(), PmaError> { + if self.growth_poisoned { + Err(Self::growth_poisoned_error()) + } else { + Ok(()) + } + } + + fn panic_if_growth_poisoned(&self) { + if let Err(err) = self.ensure_not_growth_poisoned() { + panic!("{err}"); + } + } + + fn poison_growth(&mut self, message: String) -> Result { + self.growth_poisoned = true; + Err(PmaError::GrowthFailed(format!( + "{message}; PMA handle is poisoned, restart the node or reopen the PMA so growth journal recovery can complete" + ))) + } + + pub fn is_growth_poisoned(&self) -> bool { + self.growth_poisoned + } + + pub fn file_metadata(&self) -> Result { + let mut metadata = Self::read_file_metadata(&self.path)?; + metadata.reserved_words = + u64::try_from(self.reserved_words()).expect("PMA reserved words exceed u64"); + metadata.capacity_words = + u64::try_from(self.size_words()).expect("PMA capacity words exceed u64"); + metadata.data_words = metadata.capacity_words; + metadata.alloc_words = self.alloc_offset_words().into(); + metadata.free_words = metadata.capacity_words.saturating_sub(metadata.alloc_words); + Ok(metadata) + } + + /// Flush the entire PMA mapping to storage. + pub fn sync_all(&self) -> io::Result<()> { + #[cfg(unix)] + { + let len_bytes = self.arena.mapped_len_bytes(); + if len_bytes == 0 { + return Ok(()); + } + let ret = unsafe { + libc::msync( + self.arena.base_ptr() as *mut libc::c_void, + len_bytes, + libc::MS_SYNC, + ) + }; + if ret != 0 { + return Err(io::Error::last_os_error()); + } + } + Ok(()) + } + + pub fn sync_used_data(&self) -> io::Result<()> { + let used_words = self.alloc_offset().min(self.size_words()); + let used_bytes = PmaOffsetWords::try_from_usize(used_words) + .expect("PMA used word count exceeds u64 addressable range") + .checked_bytes_usize() + .expect("PMA used byte count exceeds usize addressable range"); + self.sync_mapped_range(0, used_bytes) + } + + pub fn sync_trailer(&self) -> io::Result<()> { + self.sync_mapped_range(self.arena.len_bytes(), PMA_TRAILER_BYTES) + } + + pub fn sync_file(&self) -> io::Result<()> { + std::fs::OpenOptions::new() + .read(true) + .write(true) + .open(&self.path)? + .sync_data() + } + + fn sync_mapped_range(&self, offset_bytes: usize, len_bytes: usize) -> io::Result<()> { + #[cfg(unix)] + { + if len_bytes == 0 { + return Ok(()); + } + let mapped_len = self.arena.mapped_len_bytes(); + if mapped_len == 0 { + return Ok(()); + } + let page = unsafe { libc::sysconf(libc::_SC_PAGESIZE) }; + if page <= 0 { + return Err(io::Error::last_os_error()); + } + let page = page as usize; + let start = offset_bytes.min(mapped_len); + let end = offset_bytes.saturating_add(len_bytes).min(mapped_len); + if start >= end { + return Ok(()); + } + let start_aligned = (start / page) * page; + let end_aligned = end + .checked_add(page - 1) + .map(|value| (value / page) * page) + .unwrap_or(mapped_len) + .min(mapped_len); + let sync_len = end_aligned.saturating_sub(start_aligned); + if sync_len == 0 { + return Ok(()); + } + let ret = unsafe { + libc::msync( + self.arena.base_ptr().add(start_aligned) as *mut libc::c_void, + sync_len, + libc::MS_SYNC, + ) + }; + if ret != 0 { + return Err(io::Error::last_os_error()); + } + } + Ok(()) + } + + /// Get the backing file path. + pub fn path(&self) -> &PathBuf { + &self.path + } + + /// Get the underlying arena + pub fn arena(&self) -> &Arc { + &self.arena + } + + /// Get the current allocation offset in words + pub fn alloc_offset(&self) -> usize { + self.alloc_offset + } + + /// Get the current allocation offset as a typed PMA word offset. + pub fn alloc_offset_words(&self) -> PmaOffsetWords { + PmaOffsetWords::try_from_usize(self.alloc_offset) + .expect("PMA allocation offset exceeds u64 addressable range") + } + + /// Get the total size of the PMA in words + pub fn size_words(&self) -> usize { + self.arena.words() + } + + /// Get the maximum reserved PMA offset range in words. + pub fn reserved_words(&self) -> usize { + self.arena.reserved_words() + } + + /// Get the number of free words remaining + pub fn free_words(&self) -> usize { + self.size_words().saturating_sub(self.alloc_offset()) + } + + /// Convert a pointer within the PMA to an offset in words + pub fn offset_from_ptr(&self, ptr: *const u8) -> PmaOffsetWords { + self.arena.offset_from_ptr(ptr) + } + + /// Convert an offset in words to a pointer + pub fn ptr_from_offset(&self, offset_words: PmaOffsetWords) -> *mut u8 { + self.arena.ptr_from_offset(offset_words) + } + + /// Check if a pointer is within the PMA's memory region + pub fn contains_ptr(&self, ptr: *const u8) -> bool { + let base = self.arena.base_ptr() as usize; + let end = base + .checked_add(self.arena.reserved_len_bytes()) + .expect("PMA bounds exceed usize address space"); + let ptr_addr = ptr as usize; + ptr_addr >= base && ptr_addr < end + } + + /// Reset the allocation pointer to zero + pub fn reset(&mut self) { + self.panic_if_growth_poisoned(); + self.alloc_offset = 0; + self.persist_metadata(); + } + + /// Reset the allocation pointer to a specific offset + /// + /// # Panics + /// Panics if `offset` is greater than the PMA size. + pub fn reset_to(&mut self, offset: PmaOffsetWords) { + self.panic_if_growth_poisoned(); + let offset = offset + .try_into_usize() + .expect("PMA reset offset exceeds usize addressable range"); + assert!( + offset <= self.size_words(), + "reset_to offset {} exceeds PMA size {}", + offset, + self.size_words() + ); + self.alloc_offset = offset; + self.persist_metadata(); + } + + /// Check if an allocation of `words` would exceed available space. + /// + /// # Panics + /// Panics with `PmaError::OutOfMemory` if there isn't enough space. + pub fn alloc_would_oom(&self, words: usize) { + if words > self.free_words() { + panic!( + "{}", + PmaError::OutOfMemory { + requested: words, + available: self.free_words(), + } + ); + } + } + + /// Ensure at least `min_free_words` are available in the current PMA capacity. + pub fn ensure_free_words(&mut self, min_free_words: usize) -> Result<(), PmaError> { + self.ensure_not_growth_poisoned()?; + let required = self + .alloc_offset + .checked_add(min_free_words) + .ok_or_else(|| PmaError::GrowthFailed("required PMA words overflowed".to_string()))?; + self.grow_to_fit_with_context(required, min_free_words, "ensure_free_words") + } + + /// Allocate `words` from the PMA, returning a pointer to the allocation. + /// + /// # Panics + /// Panics if there isn't enough space in the PMA. + unsafe fn raw_alloc(&mut self, words: usize) -> *mut u64 { + self.panic_if_growth_poisoned(); + let required = self + .alloc_offset + .checked_add(words) + .unwrap_or_else(|| panic!("PMA allocation offset overflow for {words} words")); + if required > self.size_words() { + if pma_growth_disabled_for_regression() { + self.alloc_would_oom(words); + } + if let Err(err) = self.grow_to_fit_with_context(required, words, "raw_alloc") { + panic!("{err}"); + } + } + self.alloc_would_oom(words); + let ptr = self.arena.ptr_from_offset(self.alloc_offset_words()) as *mut u64; + self.alloc_offset += words; + self.persist_metadata(); + ptr + } + + pub fn grow_to_fit(&mut self, required_words: usize) -> Result<(), PmaError> { + self.ensure_not_growth_poisoned()?; + let request_words = required_words.saturating_sub(self.alloc_offset); + self.grow_to_fit_with_context(required_words, request_words, "grow_to_fit") + } + + fn grow_to_fit_with_context( + &mut self, + required_words: usize, + request_words: usize, + context: &'static str, + ) -> Result<(), PmaError> { + self.ensure_not_growth_poisoned()?; + if required_words <= self.size_words() { + return Ok(()); + } + let mut new_words = self.size_words().max(1); + while new_words < required_words { + new_words = new_words.checked_mul(2).ok_or_else(|| { + PmaError::GrowthFailed("PMA growth capacity overflowed".to_string()) + })?; + } + new_words = new_words.min(self.reserved_words()); + if new_words < required_words { + return Err(PmaError::OutOfMemory { + requested: request_words, + available: self.free_words(), + }); + } + self.grow_to_capacity_with_context(new_words, required_words, request_words, context) + } + + pub fn grow_to_capacity(&mut self, new_words: usize) -> Result<(), PmaError> { + self.ensure_not_growth_poisoned()?; + self.grow_to_capacity_with_context(new_words, new_words, 0, "grow_to_capacity") + } + + fn grow_to_capacity_with_context( + &mut self, + new_words: usize, + required_words: usize, + request_words: usize, + context: &'static str, + ) -> Result<(), PmaError> { + self.ensure_not_growth_poisoned()?; + let old_words = self.size_words(); + if new_words <= old_words { + return Ok(()); + } + if should_inject_growth_failure("create_destination") + || should_inject_growth_failure("before_file_extension") + { + return Err(PmaError::GrowthFailed( + "injected PMA growth failure before file extension".to_string(), + )); + } + if new_words > self.reserved_words() { + return Err(PmaError::OutOfMemory { + requested: request_words.max(new_words.saturating_sub(self.alloc_offset)), + available: self.free_words(), + }); + } + let started = Instant::now(); + let old_alloc = self.alloc_offset; + let journal = PmaGrowthJournal::new( + u64::try_from(old_words).expect("PMA capacity exceeds u64"), + u64::try_from(old_alloc).expect("PMA alloc exceeds u64"), + u64::try_from(new_words).expect("PMA capacity exceeds u64"), + u64::try_from(self.reserved_words()).expect("PMA reservation exceeds u64"), + ); + write_growth_journal(&self.path, journal)?; + if should_inject_growth_failure("after_journal") { + return Err(PmaError::GrowthFailed( + "injected PMA growth failure after journal".to_string(), + )); + } + if let Err(err) = self.arena.grow_file_capacity(new_words, PMA_TRAILER_BYTES) { + return self.poison_growth(format!("PMA growth failed during file extension: {err}")); + } + if should_inject_growth_failure("after_file_extension") { + return self + .poison_growth("injected PMA growth failure after file extension".to_string()); + } + let old_trailer = unsafe { self.arena.base_ptr().add(old_words * 8) }; + unsafe { + std::ptr::write_bytes(old_trailer, 0, PMA_TRAILER_BYTES); + } + if should_inject_growth_failure("after_zero_old_trailer") { + return self.poison_growth( + "injected PMA growth failure after zeroing old trailer".to_string(), + ); + } + self.persist_metadata(); + if should_inject_growth_failure("after_new_trailer_write") { + return self + .poison_growth("injected PMA growth failure after new trailer write".to_string()); + } + if let Err(err) = self.sync_trailer() { + return self.poison_growth(format!("PMA growth failed syncing metadata: {err}")); + } + if should_inject_growth_failure("after_metadata_fsync") { + return self + .poison_growth("injected PMA growth failure after metadata sync".to_string()); + } + if let Err(err) = self.sync_file() { + return self.poison_growth(format!("PMA growth failed syncing backing file: {err}")); + } + if should_inject_growth_failure("after_file_fsync") { + return self.poison_growth("injected PMA growth failure after file fsync".to_string()); + } + self.clear_growth_journal_best_effort(); + if should_inject_growth_failure("after_metadata_publication") { + info!( + path = %self.path.display(), + "ignoring injected PMA growth failure after metadata publication because growth is already durable" + ); + } + self.record_growth_event(PmaGrowthEvent { + old_words, + new_words, + alloc_words: old_alloc, + required_words, + request_words, + context, + elapsed: started.elapsed(), + }); + Ok(()) + } + + fn record_growth_event(&self, event: PmaGrowthEvent) { + info!( + path = %self.path.display(), + old_capacity_words = event.old_words, + new_capacity_words = event.new_words, + required_words = event.required_words, + allocation_request_words = event.request_words, + alloc_words = event.alloc_words, + reserved_words = self.reserved_words(), + context = event.context, + elapsed_ms = event.elapsed.as_secs_f64() * 1000.0, + "PMA automatic growth complete" + ); + let Some(events_path) = std::env::var_os(NOCK_PMA_GROWTH_EVENTS_PATH_ENV) else { + return; + }; + let line = format!( + "path={} old_capacity_words={} new_capacity_words={} required_words={} allocation_request_words={} alloc_words={} reserved_words={} context={} elapsed_ms={:.3}\n", + self.path.display(), + event.old_words, + event.new_words, + event.required_words, + event.request_words, + event.alloc_words, + self.reserved_words(), + event.context, + event.elapsed.as_secs_f64() * 1000.0 + ); + if let Ok(mut file) = fs::OpenOptions::new() + .create(true) + .append(true) + .open(events_path) + { + let _ = file.write_all(line.as_bytes()); + let _ = file.sync_data(); + } + } + + pub fn persist_metadata(&self) { + self.panic_if_growth_poisoned(); + debug_assert!( + self.arena.mapped_len_bytes() + >= self + .arena + .len_bytes() + .checked_add(PMA_TRAILER_BYTES) + .expect("PMA trailer exceeds usize address space"), + "PMA arena mapping is too small for metadata trailer" + ); + let trailer = PmaTrailerV2::new( + u64::try_from(self.arena.words()) + .expect("PMA arena size exceeds u64 addressable range"), + self.alloc_offset_words().into(), + u64::try_from(self.reserved_words()).expect("PMA reservation exceeds u64"), + ); + let bytes = trailer.to_bytes(); + let dst = unsafe { self.arena.base_ptr().add(self.arena.len_bytes()) }; + unsafe { + std::ptr::copy_nonoverlapping(bytes.as_ptr(), dst, bytes.len()); + } + } + + /// Hint the OS to drop the first `numerator/denominator` of allocated PMA data. + pub fn advise_drop_allocated_prefix( + &self, + numerator: usize, + denominator: usize, + ) -> io::Result { + if denominator == 0 { + return Err(io::Error::new( + io::ErrorKind::InvalidInput, + "denominator must be non-zero", + )); + } + + let alloc_words = self.alloc_offset().min(self.size_words()); + let advise_words = alloc_words.saturating_mul(numerator) / denominator; + if advise_words == 0 { + return Ok(0); + } + let mut len_bytes = advise_words.saturating_mul(8); + len_bytes = len_bytes.min(self.arena.len_bytes()); + + let page = unsafe { libc::sysconf(libc::_SC_PAGESIZE) }; + if page <= 0 { + return Err(io::Error::other("failed to read page size")); + } + let page = page as usize; + let len_aligned = (len_bytes / page) * page; + if len_aligned == 0 { + return Ok(0); + } + madvise_drop_file_backed_pages(self.arena.base_ptr() as *mut libc::c_void, len_aligned)?; + Ok(len_aligned) + } +} + +fn default_reserved_words(capacity_words: usize) -> usize { + if let Ok(value) = std::env::var(NOCK_PMA_RESERVED_WORDS_ENV) { + if let Ok(words) = value.parse::() { + return words.max(capacity_words); + } + } + let default_reserved_words = DEFAULT_PMA_RESERVED_BYTES / std::mem::size_of::(); + default_reserved_words.max(capacity_words) +} + +#[cfg(unix)] +fn physical_file_bytes(metadata: &fs::Metadata) -> Option { + metadata.blocks().checked_mul(512) +} + +#[cfg(not(unix))] +fn physical_file_bytes(_metadata: &fs::Metadata) -> Option { + None +} + +fn pma_growth_disabled_for_regression() -> bool { + std::env::var_os(NOCK_PMA_DISABLE_RESIZE_ENV).is_some() +} + +fn should_inject_growth_failure(point: &str) -> bool { + #[cfg(test)] + if TEST_GROWTH_FAILURE_POINT.with(|configured| { + matches!(*configured.borrow(), Some(configured) if configured == point || configured == "any") + }) { + return true; + } + std::env::var(NOCK_PMA_RESIZE_FAIL_AT_ENV) + .map(|value| value == point || value == "any") + .unwrap_or(false) +} + +fn should_inject_migration_failure(point: &str) -> bool { + #[cfg(test)] + if TEST_MIGRATION_FAILURE_POINT.with(|configured| { + matches!(*configured.borrow(), Some(configured) if configured == point || configured == "any") + }) { + return true; + } + std::env::var(NOCK_PMA_MIGRATION_FAIL_AT_ENV) + .map(|value| value == point || value == "any") + .unwrap_or(false) +} + +#[cfg(test)] +fn set_test_growth_failure_point(point: Option<&'static str>) { + TEST_GROWTH_FAILURE_POINT.with(|configured| *configured.borrow_mut() = point); +} + +#[cfg(test)] +fn set_test_migration_failure_point(point: Option<&'static str>) { + TEST_MIGRATION_FAILURE_POINT.with(|configured| *configured.borrow_mut() = point); +} + +fn file_len_for_words_u64(words: u64) -> Result { + file_len_for_words_u64_with_tail(words, PMA_TRAILER_BYTES) +} + +fn file_len_for_words_u64_with_tail(words: u64, tail_bytes: usize) -> Result { + words + .checked_mul(8) + .and_then(|bytes| bytes.checked_add(tail_bytes as u64)) + .ok_or_else(|| PmaError::InvalidMetadata("PMA file length overflowed".to_string())) +} + +fn growth_journal_path(path: &Path) -> PathBuf { + path.with_extension("grow") +} + +fn migration_journal_path(path: &Path) -> PathBuf { + path.with_extension("migrate") +} + +fn read_growth_journal(path: &Path) -> Result, PmaError> { + let path = growth_journal_path(path); + let bytes = match fs::read(&path) { + Ok(bytes) => bytes, + Err(err) if err.kind() == io::ErrorKind::NotFound => return Ok(None), + Err(err) => return Err(PmaError::MetadataIo(err)), + }; + if bytes.len() == PmaGrowthJournal::BYTES { + let mut buf = [0u8; PmaGrowthJournal::BYTES]; + buf.copy_from_slice(&bytes); + return Ok(PmaGrowthJournal::from_bytes(buf)); + } + if bytes.len() == PmaGrowthJournal::LEGACY_BYTES { + let mut buf = [0u8; PmaGrowthJournal::LEGACY_BYTES]; + buf.copy_from_slice(&bytes); + return Ok(PmaGrowthJournal::from_legacy_bytes(buf)); + } + Ok(None) +} + +fn write_growth_journal(path: &Path, journal: PmaGrowthJournal) -> Result<(), PmaError> { + let journal_path = growth_journal_path(path); + let tmp_path = journal_path.with_extension("grow.tmp"); + { + let mut file = fs::OpenOptions::new() + .create(true) + .truncate(true) + .write(true) + .open(&tmp_path)?; + file.write_all(&journal.to_bytes())?; + file.sync_all()?; + } + fs::rename(&tmp_path, &journal_path)?; + if let Some(parent) = journal_path.parent() { + let dir = fs::File::open(parent)?; + dir.sync_all()?; + } + Ok(()) +} + +fn read_migration_journal(path: &Path) -> Result, PmaError> { + let path = migration_journal_path(path); + let bytes = match fs::read(&path) { + Ok(bytes) => bytes, + Err(err) if err.kind() == io::ErrorKind::NotFound => return Ok(None), + Err(err) => return Err(PmaError::MetadataIo(err)), + }; + if bytes.len() != PmaMigrationJournal::BYTES { + return Ok(None); + } + let mut buf = [0u8; PmaMigrationJournal::BYTES]; + buf.copy_from_slice(&bytes); + Ok(PmaMigrationJournal::from_bytes(buf)) +} + +fn write_migration_journal(path: &Path, journal: PmaMigrationJournal) -> Result<(), PmaError> { + let journal_path = migration_journal_path(path); + let tmp_path = journal_path.with_extension("migrate.tmp"); + { + let mut file = fs::OpenOptions::new() + .create(true) + .truncate(true) + .write(true) + .open(&tmp_path)?; + file.write_all(&journal.to_bytes())?; + file.sync_all()?; + } + fs::rename(&tmp_path, &journal_path)?; + if let Some(parent) = journal_path.parent() { + fs::File::open(parent)?.sync_all()?; + } + Ok(()) +} + +#[cfg(unix)] +fn madvise_drop_file_backed_pages(ptr: *mut libc::c_void, len: usize) -> io::Result<()> { + #[cfg(target_os = "linux")] + { + let ret = unsafe { libc::madvise(ptr, len, libc::MADV_PAGEOUT) }; + if ret == 0 { + return Ok(()); + } + + let err = io::Error::last_os_error(); + match err.raw_os_error() { + Some(libc::EINVAL) | Some(libc::ENOSYS) => { + let fallback = unsafe { libc::madvise(ptr, len, libc::MADV_DONTNEED) }; + if fallback == 0 { + return Ok(()); + } + return Err(io::Error::last_os_error()); + } + _ => return Err(err), + } + } + + #[cfg(not(target_os = "linux"))] + { + let ret = unsafe { libc::madvise(ptr, len, libc::MADV_DONTNEED) }; + if ret != 0 { + return Err(io::Error::last_os_error()); + } + Ok(()) + } +} + +impl ibig::Stack for Pma { + unsafe fn alloc_layout(&mut self, layout: std::alloc::Layout) -> *mut u64 { + // Convert bytes to words, rounding up + let words = (layout.size() + 7) >> 3; + self.raw_alloc(words) + } +} + +impl NounAllocator for Pma { + unsafe fn alloc_indirect(&mut self, words: usize) -> *mut u64 { + self.raw_alloc(words + 2) + } + + unsafe fn alloc_cell(&mut self) -> *mut CellMemory { + self.raw_alloc(word_size_of::()) as *mut CellMemory + } + + unsafe fn alloc_struct(&mut self, count: usize) -> *mut T { + self.raw_alloc(word_size_of::() * count) as *mut T + } + + unsafe fn equals(&mut self, a: *mut Noun, b: *mut Noun) -> bool { + let a = &*a; + let b = &*b; + let space = NounSpace::pma_only(self); + noun_equality(a.in_space(&space), b.in_space(&space)) + } + + fn noun_space(&self) -> NounSpace { + NounSpace::pma_only(self) + } +} + +/// Trait for types that can be copied into the PMA. +/// +/// This is used to evacuate nouns from the NockStack to the PMA for persistence. +pub trait PmaCopy { + /// Copy this value into the PMA. + /// + /// For nouns, this evacuates allocated data (indirect atoms, cells) to the PMA + /// and converts pointers to offset form. Direct atoms are unchanged since they + /// fit in a single word. + /// + /// # Safety + /// The caller must ensure `stack` and `pma` describe the arenas that own the + /// nouns being copied; pointer-form nouns are resolved via `NounSpace::new`. + unsafe fn copy_to_pma(&mut self, stack: &NockStack, pma: &mut Pma); + + /// Assert that this value is fully contained within the PMA. + /// + /// For nouns, this verifies that all allocated data (indirect atoms, cells) + /// resides in the PMA. Direct atoms always pass since they have no allocations. + /// + /// # Panics + /// Panics if any part of this value is not in the PMA. + fn assert_in_pma(&self, pma: &Pma); +} + +/// Trait for types that can be copied from one PMA to another. +/// +/// This is used for PMA compaction, copying reachable data from a from-space +/// PMA into a to-space PMA. +pub trait PmaCopyFrom { + /// Copy this value from `from_pma` into `to_pma`, updating any internal + /// pointers to reference the new PMA. + /// + /// # Safety + /// The caller must ensure the value currently resides in `from_pma`. + unsafe fn copy_from_pma(&mut self, from_pma: &Pma, to_pma: &mut Pma); +} + +impl PmaCopy for () { + unsafe fn copy_to_pma(&mut self, _stack: &NockStack, _pma: &mut Pma) {} + + fn assert_in_pma(&self, _pma: &Pma) {} +} + +impl PmaCopyFrom for () { + unsafe fn copy_from_pma(&mut self, _from_pma: &Pma, _to_pma: &mut Pma) {} +} + +impl PmaCopy for Atom { + unsafe fn copy_to_pma(&mut self, stack: &NockStack, pma: &mut Pma) { + let mut noun = self.as_noun(); + noun.copy_to_pma(stack, pma); + *self = noun.as_atom().expect("Atom remains atom after copy_to_pma"); + } + + #[cfg(feature = "pma-assert")] + fn assert_in_pma(&self, pma: &Pma) { + self.as_noun().assert_in_pma(pma); + } + + #[cfg(not(feature = "pma-assert"))] + #[inline(always)] + fn assert_in_pma(&self, _pma: &Pma) {} +} + +impl PmaCopyFrom for Atom { + unsafe fn copy_from_pma(&mut self, from_pma: &Pma, to_pma: &mut Pma) { + let mut noun = self.as_noun(); + noun.copy_from_pma(from_pma, to_pma); + if let Ok(atom) = noun.as_atom() { + *self = atom; + } + } +} + +impl PmaCopy for Noun { + /// Copy a noun and all its allocated substructure to the PMA. + /// + /// Uses a worklist algorithm to avoid stack overflow on deep structures. + /// Structural sharing is preserved via forwarding pointers: if the same + /// substructure is referenced multiple times, it's only copied once. + /// + /// # Algorithm + /// 1. Push (noun, destination_ptr) onto worklist + /// 2. Pop and process each item: + /// - Direct atoms: write directly to destination + /// - Already in PMA (offset form): write directly to destination + /// - Has forwarding pointer: write forwarded offset-form to destination + /// - Indirect atom: copy to PMA, set forwarding pointer, write offset-form + /// - Cell: copy metadata to PMA, set forwarding pointer, queue head/tail + /// + /// # Safety + /// - Source nouns will have forwarding pointers set (corrupting the stack data) + unsafe fn copy_to_pma(&mut self, stack: &NockStack, pma: &mut Pma) { + if self.is_direct() { + return; + } + + let trace_noun = std::env::var_os("NOCK_PMA_TRACE_NOUN").is_some(); + let trace_start = Instant::now(); + let mut last_progress = trace_start; + let mut steps = 0usize; + + let space = NounSpace::new(stack, &*pma); + let root_repr = self.repr(&space); + match root_repr { + NounRepr::Indirect(AllocLocation::PmaOffset) + | NounRepr::Cell(AllocLocation::PmaOffset) => { + self.assert_in_pma(pma); + return; + } + NounRepr::Indirect(AllocLocation::PmaPtr) | NounRepr::Cell(AllocLocation::PmaPtr) => { + let offset_noun = { + let allocated = self.as_allocated().expect("repr said allocated"); + let ptr = allocated.to_raw_pointer(&space); + assert!( + pma.contains_ptr(ptr as *const u8), + "noun claims PMA pointer but is outside PMA" + ); + let offset = pma.offset_from_ptr(ptr as *const u8); + if allocated.is_indirect() { + IndirectAtom::from_offset_words(offset).as_noun() + } else { + Cell::from_offset_words(offset).as_noun() + } + }; + *self = offset_noun; + self.assert_in_pma(pma); + return; + } + NounRepr::Forwarding(_) => { + panic!("forwarding-pointer noun encountered during PMA copy"); + } + _ => {} + } + + let mut work: SmallVec<[(Noun, *mut Noun); 64]> = SmallVec::new(); + work.push((*self, self as *mut Noun)); + + while let Some((noun, dest_ptr)) = work.pop() { + steps += 1; + if trace_noun && (steps & 0x3fff == 0) { + let now = Instant::now(); + if now.duration_since(last_progress).as_millis() >= 2000 { + debug!( + "pma-copy: noun progress: steps={}, elapsed_ms={}", + steps, + trace_start.elapsed().as_millis() + ); + last_progress = now; + } + } + match noun.as_either_direct_allocated() { + Left(_direct) => { + *dest_ptr = noun; + } + Right(allocated) => { + let forwarded = allocated.forwarding_pointer(&space); + if let Some(forwarded) = forwarded { + let offset_noun = { + let ptr = forwarded.to_raw_pointer(&space); + assert!( + pma.contains_ptr(ptr as *const u8), + "forwarding pointer escapes PMA" + ); + let offset = pma.offset_from_ptr(ptr as *const u8); + if forwarded.is_indirect() { + IndirectAtom::from_offset_words(offset).as_noun() + } else { + Cell::from_offset_words(offset).as_noun() + } + }; + *dest_ptr = offset_noun; + continue; + } + + let repr = noun.repr(&space); + + match repr { + NounRepr::Indirect(AllocLocation::PmaOffset) + | NounRepr::Cell(AllocLocation::PmaOffset) => { + noun.assert_in_pma(pma); + *dest_ptr = noun; + continue; + } + NounRepr::Indirect(AllocLocation::PmaPtr) + | NounRepr::Cell(AllocLocation::PmaPtr) => { + let offset_noun = { + let ptr = allocated.to_raw_pointer(&space); + assert!( + pma.contains_ptr(ptr as *const u8), + "noun claims PMA pointer but is outside PMA" + ); + let offset = pma.offset_from_ptr(ptr as *const u8); + if allocated.is_indirect() { + IndirectAtom::from_offset_words(offset).as_noun() + } else { + Cell::from_offset_words(offset).as_noun() + } + }; + noun.assert_in_pma(pma); + *dest_ptr = offset_noun; + continue; + } + NounRepr::Forwarding(_) => { + panic!("forwarding-pointer noun encountered during PMA copy"); + } + NounRepr::Direct => { + *dest_ptr = noun; + continue; + } + NounRepr::Indirect(AllocLocation::Stack) + | NounRepr::Cell(AllocLocation::Stack) => {} + } + + match allocated.as_either() { + Left(mut indirect) => { + let (raw_size, src_ptr) = + { (indirect.raw_size(&space), indirect.to_raw_pointer(&space)) }; + + let pma_ptr = pma.raw_alloc(raw_size); + copy_nonoverlapping(src_ptr, pma_ptr, raw_size); + + indirect.set_forwarding_pointer(pma_ptr, &space); + + let offset = pma.offset_from_ptr(pma_ptr as *const u8); + *dest_ptr = IndirectAtom::from_offset_words(offset).as_noun(); + } + Right(mut cell) => { + let (src_cell, head, tail) = { + let src_cell = cell.to_raw_pointer(&space); + let head = (*src_cell).head; + let tail = (*src_cell).tail; + (src_cell, head, tail) + }; + + let pma_ptr = pma.raw_alloc(word_size_of::()); + let pma_cell = pma_ptr as *mut CellMemory; + (*pma_cell).metadata = (*src_cell).metadata; + + cell.set_forwarding_pointer(pma_cell, &space); + + work.push((tail, &mut (*pma_cell).tail)); + work.push((head, &mut (*pma_cell).head)); + + let offset = pma.offset_from_ptr(pma_ptr as *const u8); + *dest_ptr = Cell::from_offset_words(offset).as_noun(); + } + } + } + } + } + + if trace_noun { + debug!( + "pma-copy: noun done: steps={}, elapsed_ms={}", + steps, + trace_start.elapsed().as_millis() + ); + } + } + + /// Assert that this noun and all its substructure is in the PMA. + /// + #[cfg(feature = "pma-assert")] + fn assert_in_pma(&self, pma: &Pma) { + if self.is_direct() { + return; + } + + let space = NounSpace::pma_only(pma); + let mut seen = IntMap::new(); + let mut work = vec![*self]; + + while let Some(noun) = work.pop() { + if noun.is_direct() { + continue; + } + + match noun.repr(&space) { + NounRepr::Indirect(AllocLocation::Stack) | NounRepr::Cell(AllocLocation::Stack) => { + panic!("noun is stack-allocated, not in PMA"); + } + NounRepr::Forwarding(_) => { + panic!("forwarding pointer is not valid PMA state"); + } + NounRepr::Indirect(_) | NounRepr::Direct => {} + NounRepr::Cell(_) => { + let cell = noun.in_space(&space).as_cell().expect("checked is_cell"); + let ptr = unsafe { cell.raw_pointer() } as usize as u64; + if seen.get(ptr).is_some() { + continue; + } + seen.insert(ptr, ()); + work.push(cell.head().noun()); + work.push(cell.tail().noun()); + } + } + } + } + + #[cfg(not(feature = "pma-assert"))] + #[inline(always)] + fn assert_in_pma(&self, _pma: &Pma) {} +} + +impl PmaCopyFrom for Noun { + unsafe fn copy_from_pma(&mut self, from_pma: &Pma, to_pma: &mut Pma) { + if self.is_direct() { + return; + } + let to_base = to_pma.arena().base_ptr() as usize; + let to_end = to_base + .checked_add(to_pma.arena().reserved_len_bytes()) + .expect("PMA bounds exceed usize address space"); + let space = NounSpace::pma_only(from_pma).with_extra_ptr_ranges(vec![(to_base, to_end)]); + let mut work: SmallVec<[(Noun, *mut Noun); 64]> = SmallVec::new(); + work.push((*self, self as *mut Noun)); + + while let Some((noun, dest_ptr)) = work.pop() { + match noun.as_either_direct_allocated() { + Left(_direct) => { + *dest_ptr = noun; + } + Right(allocated) => { + if let Some(forwarded) = allocated.forwarding_pointer(&space) { + let ptr = forwarded.to_raw_pointer(&space) as *const u8; + let offset = to_pma.offset_from_ptr(ptr); + *dest_ptr = if forwarded.is_indirect() { + IndirectAtom::from_offset_words(offset).as_noun() + } else { + Cell::from_offset_words(offset).as_noun() + }; + continue; + } + + match allocated.as_either() { + Left(mut indirect) => { + let raw_size = indirect.raw_size(&space); + let src_ptr = indirect.to_raw_pointer(&space); + let pma_ptr = to_pma.raw_alloc(raw_size); + copy_nonoverlapping(src_ptr, pma_ptr, raw_size); + + indirect.set_forwarding_pointer(pma_ptr, &space); + + let offset = to_pma.offset_from_ptr(pma_ptr as *const u8); + *dest_ptr = IndirectAtom::from_offset_words(offset).as_noun(); + } + Right(mut cell) => { + let src_cell = cell.to_raw_pointer(&space); + let head = (*src_cell).head; + let tail = (*src_cell).tail; + + let pma_ptr = to_pma.raw_alloc(word_size_of::()); + let pma_cell = pma_ptr as *mut CellMemory; + (*pma_cell).metadata = (*src_cell).metadata; + + cell.set_forwarding_pointer(pma_cell, &space); + + work.push((tail, &mut (*pma_cell).tail)); + work.push((head, &mut (*pma_cell).head)); + + let offset = to_pma.offset_from_ptr(pma_ptr as *const u8); + *dest_ptr = Cell::from_offset_words(offset).as_noun(); + } + } + } + } + } + } +} + +#[cfg(test)] +pub(crate) fn test_pma_path(label: &str) -> PathBuf { + use std::sync::atomic::{AtomicUsize, Ordering}; + + static COUNTER: AtomicUsize = AtomicUsize::new(0); + let id = COUNTER.fetch_add(1, Ordering::Relaxed); + let pid = std::process::id(); + let mut path = std::env::temp_dir(); + path.push(format!("nockvm_pma_{label}_{pid}_{id}.mmap")); + path +} + +#[cfg(test)] +mod tests { + use std::alloc::Layout; + use std::fs; + use std::sync::Mutex; + + use ibig::Stack; + + use super::*; + use crate::hamt::Hamt; + use crate::jets::cold::NounListMem; + use crate::mem::{word_size_of, NockStack, NOCK_STACK_SIZE_TINY}; + use crate::noun::{AllocLocation, D, DIRECT_MAX}; + + static PMA_ENV_LOCK: Mutex<()> = Mutex::new(()); + + /// Helper to create a test PMA with a given size + fn test_pma(size_words: usize) -> Pma { + let path = test_pma_path("pma"); + Pma::new(size_words, path).expect("Failed to create test PMA") + } + + /// Verifies bump allocation returns sequential offsets and correctly tracks free space. + /// + /// This test exercises: + /// - Pma::new creates a valid PMA + /// - alloc_offset() starts at 0 + /// - free_words() equals size initially + /// - NounAllocator::alloc_indirect bumps the offset correctly + /// - NounAllocator::alloc_cell allocates CellMemory + /// - NounAllocator::alloc_struct allocates arbitrary structs + /// - Sequential allocations don't overlap + #[test] + #[cfg_attr(miri, ignore = "memfd_create unsupported in Miri")] + fn test_pma_allocation() { + let mut pma = test_pma(1000); + + // Initial state: nothing allocated yet + assert_eq!(pma.alloc_offset(), 0, "Initial alloc_offset should be 0"); + assert_eq!( + pma.free_words(), + 1000, + "Initial free_words should equal size" + ); + + // First allocation: alloc_indirect(10) allocates 10 + 2 = 12 words (data + metadata + size) + let ptr1 = unsafe { pma.alloc_indirect(10) }; + assert!( + !ptr1.is_null(), + "First allocation should return non-null pointer" + ); + assert_eq!( + pma.alloc_offset(), + 12, + "After alloc_indirect(10), offset should be 12" + ); + assert_eq!( + pma.free_words(), + 988, + "After alloc_indirect(10), free should be 988" + ); + + // Second allocation: alloc_indirect(20) allocates 20 + 2 = 22 words + let ptr2 = unsafe { pma.alloc_indirect(20) }; + assert!( + !ptr2.is_null(), + "Second allocation should return non-null pointer" + ); + assert_eq!( + pma.alloc_offset(), + 34, + "After second alloc, offset should be 34" + ); + assert_eq!( + pma.free_words(), + 966, + "After second alloc, free should be 966" + ); + + // Third allocation: alloc_cell allocates word_size_of::() words + let ptr3 = unsafe { pma.alloc_cell() }; + assert!( + !ptr3.is_null(), + "Cell allocation should return non-null pointer" + ); + let cell_words = word_size_of::(); + let offset_after_cell = 34 + cell_words; + assert_eq!( + pma.alloc_offset(), + offset_after_cell, + "After cell alloc, offset should increase by CellMemory size" + ); + + // Fourth allocation: alloc_struct for NounListMem + let struct_words = word_size_of::(); + let ptr4: *mut NounListMem = unsafe { pma.alloc_struct(1) }; + assert!( + !ptr4.is_null(), + "Struct allocation should return non-null pointer" + ); + let offset_after_struct = offset_after_cell + struct_words; + assert_eq!( + pma.alloc_offset(), + offset_after_struct, + "After struct alloc, offset should increase by struct size in words" + ); + + // Fifth allocation: alloc_struct with count > 1 (allocate array of 3 NounListMem) + let ptr5: *mut NounListMem = unsafe { pma.alloc_struct(3) }; + assert!( + !ptr5.is_null(), + "Array struct allocation should return non-null pointer" + ); + let offset_after_array = offset_after_struct + (struct_words * 3); + assert_eq!( + pma.alloc_offset(), + offset_after_array, + "After array alloc, offset should increase by struct_size * count" + ); + + // Sixth allocation: alloc_layout for ibig::Stack trait (allocate 8 u64s) + let layout_words = 8usize; + let layout = Layout::array::(layout_words).expect("valid layout"); + let ptr6 = unsafe { pma.alloc_layout(layout) }; + assert!( + !ptr6.is_null(), + "Layout allocation should return non-null pointer" + ); + assert_eq!( + pma.alloc_offset(), + offset_after_array + layout_words, + "After layout alloc, offset should increase by layout size in words" + ); + + // Verify all allocations are sequential and non-overlapping + // For a bump allocator, each pointer should be at or after the end of the previous allocation + let ptr1_end = unsafe { ptr1.add(12) }; // 12 words for alloc_indirect(10) + let ptr2_end = unsafe { ptr2.add(22) }; // 22 words for alloc_indirect(20) + let ptr3_end = unsafe { (ptr3 as *mut u64).add(cell_words) }; + let ptr4_end = unsafe { (ptr4 as *mut u64).add(struct_words) }; + let ptr5_end = unsafe { (ptr5 as *mut u64).add(struct_words * 3) }; + + assert!(ptr2 >= ptr1_end, "ptr2 should start at or after ptr1's end"); + assert!( + ptr3 as *mut u64 >= ptr2_end, + "ptr3 should start at or after ptr2's end" + ); + assert!( + ptr4 as *mut u64 >= ptr3_end, + "ptr4 should start at or after ptr3's end" + ); + assert!( + ptr5 as *mut u64 >= ptr4_end, + "ptr5 should start at or after ptr4's end" + ); + assert!(ptr6 >= ptr5_end, "ptr6 should start at or after ptr5's end"); + } + + /// Verifies offset-to-pointer and pointer-to-offset conversions are inverses. + /// + /// This test exercises: + /// - ptr_from_offset converts word offset to pointer + /// - offset_from_ptr converts pointer back to word offset + /// - Round-trip: offset -> ptr -> offset gives same offset + /// - Round-trip: ptr -> offset -> ptr gives same ptr + #[test] + #[cfg_attr(miri, ignore = "memfd_create unsupported in Miri")] + fn test_pma_offset_round_trip() { + let mut pma = test_pma(1000); + + // Test with offset 0 (base of PMA) + let ptr_at_0 = pma.ptr_from_offset(PmaOffsetWords::from_words(0)); + let offset_from_0 = pma.offset_from_ptr(ptr_at_0); + assert_eq!(offset_from_0.words(), 0, "Offset at base should be 0"); + + // Test with a known offset + let test_offset = PmaOffsetWords::from_words(42); + let ptr = pma.ptr_from_offset(test_offset); + let recovered_offset = pma.offset_from_ptr(ptr); + assert_eq!( + recovered_offset, test_offset, + "Round-trip offset -> ptr -> offset should return same offset" + ); + + // Test with pointer from an allocation + let alloc_ptr = unsafe { pma.alloc_indirect(10) }; + let alloc_offset = pma.offset_from_ptr(alloc_ptr as *const u8); + let recovered_ptr = pma.ptr_from_offset(alloc_offset); + assert_eq!( + recovered_ptr, alloc_ptr as *mut u8, + "Round-trip ptr -> offset -> ptr should return same pointer" + ); + + // Test multiple allocations have distinct offsets + let ptr1 = unsafe { pma.alloc_indirect(5) }; + let ptr2 = unsafe { pma.alloc_indirect(5) }; + let offset1 = pma.offset_from_ptr(ptr1 as *const u8); + let offset2 = pma.offset_from_ptr(ptr2 as *const u8); + assert_ne!( + offset1, offset2, + "Different allocations should have different offsets" + ); + + // Verify the offsets differ by the expected amount (5 + 2 = 7 words) + assert_eq!( + offset2 + .checked_sub(offset1) + .expect("offsets should be ordered") + .words(), + 7, + "Second allocation offset should be 7 words after first" + ); + } + + /// Verifies contains_ptr correctly identifies pointers inside vs outside the PMA. + #[test] + #[cfg_attr(miri, ignore = "memfd_create unsupported in Miri")] + fn test_pma_contains_ptr() { + let path = test_pma_path("pma_contains_ptr"); + let mut pma = Pma::new_with_reserved(1000, 2000, path).expect("Failed to create test PMA"); + + // Get base pointer and compute some test pointers + let base = pma.arena().base_ptr(); + let len_bytes = pma.arena().len_bytes(); + let reserved_len_bytes = pma.arena().reserved_len_bytes(); + + // Base pointer should be in PMA + assert!(pma.contains_ptr(base), "Base pointer should be in PMA"); + + // Pointer at offset 0 should be in PMA + let ptr_at_0 = pma.ptr_from_offset(PmaOffsetWords::from_words(0)); + assert!( + pma.contains_ptr(ptr_at_0), + "Pointer at offset 0 should be in PMA" + ); + + // Pointer in the middle should be in PMA + let middle_offset = PmaOffsetWords::from_words(500); + let ptr_middle = pma.ptr_from_offset(middle_offset); + assert!( + pma.contains_ptr(ptr_middle), + "Pointer in middle should be in PMA" + ); + + // Last valid byte should be in PMA + let last_byte = unsafe { base.add(len_bytes - 1) }; + assert!(pma.contains_ptr(last_byte), "Last byte should be in PMA"); + + // Pointer just past the current file capacity should still be in the reserved PMA range + let past_current_capacity = unsafe { base.add(len_bytes) }; + assert!( + pma.contains_ptr(past_current_capacity), + "Pointer past current capacity should be in reserved PMA range" + ); + + // Last reserved byte should be in PMA + let last_reserved_byte = unsafe { base.add(reserved_len_bytes - 1) }; + assert!( + pma.contains_ptr(last_reserved_byte), + "Last reserved byte should be in PMA" + ); + + // Pointer just past the reserved range should NOT be in PMA + let past_reserved_end = unsafe { base.add(reserved_len_bytes) }; + assert!( + !pma.contains_ptr(past_reserved_end), + "Pointer past reserved range should not be in PMA" + ); + + // Pointer well past the reserved range should NOT be in PMA + let way_past_end = unsafe { base.add(reserved_len_bytes + 1000) }; + assert!( + !pma.contains_ptr(way_past_end), + "Pointer way past reserved range should not be in PMA" + ); + + // Pointer before the base should NOT be in PMA (if base > 0) + if base as usize > 0 { + let before_base = unsafe { base.sub(1) }; + assert!( + !pma.contains_ptr(before_base), + "Pointer before base should not be in PMA" + ); + } + + // Null pointer should NOT be in PMA + assert!( + !pma.contains_ptr(std::ptr::null()), + "Null pointer should not be in PMA" + ); + + // Allocated pointer should be in PMA + let alloc_ptr = unsafe { pma.alloc_indirect(10) }; + assert!( + pma.contains_ptr(alloc_ptr as *const u8), + "Allocated pointer should be in PMA" + ); + } + + /// Verifies allocation fails gracefully when PMA is full. + /// + /// This test exercises: + /// - alloc_would_oom() does not panic when there's space + /// - alloc_would_oom() panics when there isn't enough space + /// - Exact-fit allocations succeed + #[test] + #[cfg_attr(miri, ignore = "memfd_create unsupported in Miri")] + fn test_pma_out_of_memory() { + use std::panic::{catch_unwind, AssertUnwindSafe}; + + let mut pma = test_pma(100); // Small PMA: 100 words + + // alloc_would_oom should not panic when there's space + pma.alloc_would_oom(50); // Should not panic + pma.alloc_would_oom(100); // Should not panic (exact fit) + + // alloc_would_oom should panic when there isn't space + let result = catch_unwind(AssertUnwindSafe(|| { + pma.alloc_would_oom(101); + })); + assert!( + result.is_err(), + "alloc_would_oom(101) should panic with 100 free" + ); + + // Allocate some space + unsafe { pma.alloc_indirect(10) }; // 12 words (10 + 2 for metadata/size) + assert_eq!(pma.alloc_offset(), 12); + assert_eq!(pma.free_words(), 88); + + // alloc_would_oom should reflect remaining space + pma.alloc_would_oom(88); // Should not panic + let result = catch_unwind(AssertUnwindSafe(|| { + pma.alloc_would_oom(89); + })); + assert!( + result.is_err(), + "alloc_would_oom(89) should panic with 88 free" + ); + + // Fill the rest + unsafe { pma.alloc_struct::(88) }; + assert_eq!(pma.alloc_offset(), 100); + assert_eq!(pma.free_words(), 0); + + // alloc_would_oom should panic for any non-zero allocation when full + let result = catch_unwind(AssertUnwindSafe(|| { + pma.alloc_would_oom(1); + })); + assert!(result.is_err(), "alloc_would_oom(1) should panic when full"); + + // But 0 words should not panic + pma.alloc_would_oom(0); // Should not panic + + // Reset and verify we can allocate again + pma.reset(); + assert_eq!(pma.free_words(), 100); + pma.alloc_would_oom(100); // Should not panic after reset + + // Verify exact-fit allocation works + unsafe { pma.alloc_struct::(100) }; + assert_eq!(pma.alloc_offset(), 100); + assert_eq!(pma.free_words(), 0); + } + + /// Verifies reset() and reset_to() correctly manage the allocation pointer. + /// + /// This test exercises: + /// - reset() sets alloc_offset back to 0 + /// - reset_to(offset) sets alloc_offset to a specific value + /// - After reset, free_words equals size again + /// - Allocations after reset start from the reset point + #[test] + #[cfg_attr(miri, ignore = "memfd_create unsupported in Miri")] + fn test_pma_reset() { + let mut pma = test_pma(1000); + + // Allocate some space + unsafe { pma.alloc_indirect(10) }; // 12 words + unsafe { pma.alloc_indirect(20) }; // 22 words + assert_eq!(pma.alloc_offset(), 34); + assert_eq!(pma.free_words(), 966); + + // Reset to zero + pma.reset(); + assert_eq!(pma.alloc_offset(), 0, "reset() should set offset to 0"); + assert_eq!( + pma.free_words(), + 1000, + "reset() should restore all free space" + ); + + // Allocations after reset should start from 0 + let ptr_after_reset = unsafe { pma.alloc_indirect(5) }; // 7 words + assert_eq!(pma.alloc_offset(), 7); + let offset_after_reset = pma.offset_from_ptr(ptr_after_reset as *const u8); + assert_eq!( + offset_after_reset.words(), + 0, + "First allocation after reset should be at offset 0" + ); + + // Allocate more to create a checkpoint + unsafe { pma.alloc_indirect(10) }; // 12 more words + let checkpoint = pma.alloc_offset(); + assert_eq!(checkpoint, 19); // 7 + 12 + + // Allocate even more + unsafe { pma.alloc_indirect(30) }; // 32 more words + assert_eq!(pma.alloc_offset(), 51); // 19 + 32 + + // Reset to checkpoint + pma.reset_to(PmaOffsetWords::try_from_usize(checkpoint).expect("checkpoint fits")); + assert_eq!( + pma.alloc_offset(), + 19, + "reset_to() should set offset to checkpoint" + ); + assert_eq!( + pma.free_words(), + 981, + "reset_to() should restore free space from checkpoint" + ); + + // Next allocation should start at the checkpoint + let ptr_after_reset_to = unsafe { pma.alloc_indirect(3) }; // 5 words + let offset_after_reset_to = pma.offset_from_ptr(ptr_after_reset_to as *const u8); + assert_eq!( + offset_after_reset_to.words(), + 19, + "Allocation after reset_to should start at checkpoint" + ); + assert_eq!(pma.alloc_offset(), 24); // 19 + 5 + } + + /// Verifies reset_to panics when given an offset outside the PMA bounds. + #[test] + #[should_panic(expected = "reset_to offset")] + #[cfg_attr(miri, ignore = "memfd_create unsupported in Miri")] + fn test_pma_reset_to_out_of_bounds() { + let mut pma = test_pma(1000); + pma.reset_to(PmaOffsetWords::from_words(1001)); // Should panic: offset exceeds PMA size + } + + #[test] + fn test_pma_open_restores_alloc_offset() { + let path = test_pma_path("open_restore"); + { + let mut pma = Pma::new(1000, path.clone()).expect("Failed to create test PMA"); + unsafe { pma.alloc_indirect(10) }; + unsafe { pma.alloc_cell() }; + assert!( + pma.alloc_offset() > 0, + "Expected allocations to advance offset" + ); + } + + let pma = Pma::open(path).expect("Failed to open PMA"); + assert!( + pma.alloc_offset() > 0, + "alloc_offset should be restored on open" + ); + } + + #[test] + #[cfg_attr(miri, ignore = "file-backed PMA unsupported in Miri")] + fn test_growable_pma_reports_capacity_separately_from_reservation() { + let path = test_pma_path("reporting"); + let pma = + Pma::new_with_reserved(1024, 16 * 1024, path.clone()).expect("create growable PMA"); + let file_len = fs::metadata(&path).expect("pma metadata").len(); + let expected_len = 1024_u64 * 8 + PMA_TRAILER_BYTES as u64; + assert_eq!( + file_len, expected_len, + "apparent file size should track current capacity, not reserved words" + ); + + let metadata = pma.file_metadata().expect("runtime PMA metadata"); + assert_eq!(metadata.capacity_words, 1024); + assert_eq!(metadata.data_words, 1024); + assert_eq!(metadata.alloc_words, 0); + assert_eq!(metadata.free_words, 1024); + assert_eq!(metadata.reserved_words, 16 * 1024); + assert_eq!(metadata.apparent_file_bytes, expected_len); + assert_eq!(metadata.file_bytes, expected_len); + #[cfg(unix)] + assert!( + metadata.physical_file_bytes.is_some(), + "unix metadata should expose physical allocated bytes" + ); + } + + #[test] + #[cfg_attr(miri, ignore = "file-backed PMA unsupported in Miri")] + fn test_legacy_pma_migrates_to_v2_and_preserves_content() { + let path = test_pma_path("legacy_migration"); + let mut stack = NockStack::new(128, 0); + let mut root = Cell::new(&mut stack, D(30), D(31)).as_noun(); + let alloc_words; + { + let mut pma = Pma::new_with_reserved(32, 128, path.clone()).expect("create PMA"); + unsafe { + root.copy_to_pma(&stack, &mut pma); + } + alloc_words = pma.alloc_offset(); + pma.sync_all().expect("sync PMA"); + pma.sync_file().expect("sync PMA file"); + } + { + let mut file = fs::OpenOptions::new() + .read(true) + .write(true) + .open(&path) + .expect("open PMA file"); + file.set_len(32 * 8 + PMA_LEGACY_TRAILER_BYTES as u64) + .expect("truncate to legacy trailer"); + file.seek(SeekFrom::Start(32 * 8)).expect("seek trailer"); + let trailer = PmaLegacyTrailer { + magic: PMA_MAGIC, + version: PMA_VERSION_V1, + data_words: 32, + alloc_offset: alloc_words as u64, + }; + file.write_all(&trailer.to_bytes()) + .expect("write legacy trailer"); + file.sync_all().expect("sync legacy PMA file"); + } + + let legacy_metadata = Pma::read_file_metadata(&path).expect("read legacy metadata"); + assert_eq!(legacy_metadata.version, PMA_VERSION_V1); + assert_eq!(legacy_metadata.capacity_words, 32); + assert_eq!(legacy_metadata.alloc_words, alloc_words as u64); + + let reopened = Pma::open_with_min(path.clone(), 64).expect("migrate and grow legacy PMA"); + assert_eq!(reopened.size_words(), 64); + assert!(reopened.reserved_words() >= 64); + assert_eq!(reopened.alloc_offset(), alloc_words); + let migrated_metadata = Pma::read_file_metadata(&path).expect("read migrated metadata"); + assert_eq!(migrated_metadata.version, PMA_VERSION); + assert_eq!(migrated_metadata.capacity_words, 64); + assert!(migrated_metadata.reserved_words >= 64); + assert_eq!(migrated_metadata.alloc_words, alloc_words as u64); + + let space = NounSpace::pma_only(&reopened); + let cell = root.in_space(&space).as_cell().expect("root cell"); + assert_eq!( + cell.head() + .as_atom() + .expect("head atom") + .as_u64() + .expect("head atom should fit in u64"), + 30 + ); + assert_eq!( + cell.tail() + .as_atom() + .expect("tail atom") + .as_u64() + .expect("tail atom should fit in u64"), + 31 + ); + } + + #[test] + #[cfg_attr(miri, ignore = "file-backed PMA unsupported in Miri")] + fn test_legacy_migration_journal_recovers_after_injected_failures() { + let _env_lock = PMA_ENV_LOCK.lock().expect("PMA env lock"); + let fail_points = [ + "before_new_metadata_write", "after_new_metadata_write_before_fsync", + "after_metadata_fsync_before_parent_directory_sync", "after_parent_directory_sync", + "after_marker_declares_upgraded", + ]; + for fail_point in fail_points { + let path = test_pma_path(&format!("legacy_migration_{fail_point}")); + let mut stack = NockStack::new(128, 0); + let mut root = Cell::new(&mut stack, D(40), D(41)).as_noun(); + let alloc_words; + { + let mut pma = Pma::new_with_reserved(32, 128, path.clone()).expect("create PMA"); + unsafe { + root.copy_to_pma(&stack, &mut pma); + } + alloc_words = pma.alloc_offset(); + pma.sync_all().expect("sync PMA"); + pma.sync_file().expect("sync PMA file"); + } + { + let mut file = fs::OpenOptions::new() + .read(true) + .write(true) + .open(&path) + .expect("open PMA file"); + file.set_len(32 * 8 + PMA_LEGACY_TRAILER_BYTES as u64) + .expect("truncate to legacy trailer"); + file.seek(SeekFrom::Start(32 * 8)).expect("seek trailer"); + let trailer = PmaLegacyTrailer { + magic: PMA_MAGIC, + version: PMA_VERSION_V1, + data_words: 32, + alloc_offset: alloc_words as u64, + }; + file.write_all(&trailer.to_bytes()) + .expect("write legacy trailer"); + file.sync_all().expect("sync legacy PMA file"); + } + + set_test_migration_failure_point(Some(fail_point)); + let err = match Pma::open(path.clone()) { + Ok(_) => panic!("injected migration failure should fail at {fail_point}"), + Err(err) => err, + }; + set_test_migration_failure_point(None); + assert!( + err.to_string().contains("injected PMA migration failure"), + "unexpected injected migration error at {fail_point}: {err}" + ); + + let reopened = Pma::open(path.clone()).expect("reopen should complete migration"); + assert_eq!(reopened.alloc_offset(), alloc_words); + let migrated_metadata = Pma::read_file_metadata(&path).expect("read migrated metadata"); + assert_eq!(migrated_metadata.version, PMA_VERSION); + assert_eq!(migrated_metadata.capacity_words, 32); + assert!( + !migration_journal_path(&path).exists(), + "successful reopen should clear migration journal for {fail_point}" + ); + let space = NounSpace::pma_only(&reopened); + let cell = root.in_space(&space).as_cell().expect("root cell"); + assert_eq!( + cell.head() + .as_atom() + .expect("head atom") + .as_u64() + .expect("head atom should fit in u64"), + 40 + ); + assert_eq!( + cell.tail() + .as_atom() + .expect("tail atom") + .as_u64() + .expect("tail atom should fit in u64"), + 41 + ); + } + } + + #[test] + #[cfg_attr(miri, ignore = "file-backed PMA unsupported in Miri")] + fn test_growth_journal_recovers_after_injected_failures() { + let _env_lock = PMA_ENV_LOCK.lock().expect("PMA env lock"); + let fail_points = [ + "after_journal", "after_file_extension", "after_zero_old_trailer", + "after_new_trailer_write", "after_metadata_fsync", "after_file_fsync", + ]; + for fail_point in fail_points { + let path = test_pma_path(&format!("growth_journal_recovery_{fail_point}")); + let mut stack = NockStack::new(128, 0); + let mut root = Cell::new(&mut stack, D(10), D(11)).as_noun(); + { + let mut pma = Pma::new_with_reserved(32, 1024, path.clone()).expect("create PMA"); + unsafe { + root.copy_to_pma(&stack, &mut pma); + } + assert_eq!(pma.alloc_offset(), word_size_of::()); + set_test_growth_failure_point(Some(fail_point)); + let err = pma + .grow_to_capacity(64) + .expect_err("injected growth failure should fail"); + set_test_growth_failure_point(None); + assert!( + err.to_string().contains("injected PMA growth failure"), + "unexpected injected failure error at {fail_point}: {err}" + ); + let should_poison = fail_point != "after_journal"; + assert_eq!( + pma.is_growth_poisoned(), + should_poison, + "PMA poison state mismatch after {fail_point}" + ); + if should_poison { + let followup_err = pma + .ensure_free_words(1) + .expect_err("poisoned PMA handle should reject further mutation"); + assert!( + followup_err + .to_string() + .contains("previous PMA growth failed"), + "unexpected poisoned-handle error after {fail_point}: {followup_err}" + ); + } + } + + let mut reopened = Pma::open(path.clone()).unwrap_or_else(|err| { + panic!("reopen should recover growth failure at {fail_point}: {err}") + }); + assert!( + (32..=64).contains(&reopened.size_words()), + "recovered PMA capacity should be old or new after {fail_point}, got {}", + reopened.size_words() + ); + assert_eq!( + reopened.reserved_words(), + 1024, + "growth journal recovery should preserve virtual reservation after {fail_point}" + ); + reopened.grow_to_capacity(512).unwrap_or_else(|err| { + panic!("reserved PMA should grow again after {fail_point}: {err}") + }); + assert_eq!(reopened.reserved_words(), 1024); + assert_eq!(reopened.alloc_offset(), word_size_of::()); + assert!( + !growth_journal_path(&path).exists(), + "successful reopen should clear growth journal after {fail_point}" + ); + let space = NounSpace::pma_only(&reopened); + let cell = root.in_space(&space).as_cell().expect("root cell"); + assert_eq!( + cell.head() + .as_atom() + .expect("head atom") + .as_u64() + .expect("head atom should fit in u64"), + 10 + ); + assert_eq!( + cell.tail() + .as_atom() + .expect("tail atom") + .as_u64() + .expect("tail atom should fit in u64"), + 11 + ); + } + } + + #[test] + #[cfg_attr(miri, ignore = "file-backed PMA unsupported in Miri")] + fn test_growth_failure_after_publication_is_committed_success() { + let _env_lock = PMA_ENV_LOCK.lock().expect("PMA env lock"); + let path = test_pma_path("growth_after_publication_success"); + let mut pma = Pma::new_with_reserved(32, 1024, path.clone()).expect("create PMA"); + + set_test_growth_failure_point(Some("after_metadata_publication")); + pma.grow_to_capacity(64) + .expect("post-publication injection should not report failed growth"); + set_test_growth_failure_point(None); + + assert_eq!(pma.size_words(), 64); + assert_eq!(pma.reserved_words(), 1024); + assert!(!pma.is_growth_poisoned()); + assert!( + !growth_journal_path(&path).exists(), + "successful published growth should clear growth journal" + ); + + let reopened = Pma::open(path).expect("reopen published growth"); + assert_eq!(reopened.size_words(), 64); + assert_eq!(reopened.reserved_words(), 1024); + } + + /// Verifies direct atoms are unchanged by evacuation since they fit in a single word. + /// + /// Direct atoms don't require any allocation - they're just 64-bit values with + /// the MSB = 0. Evacuation should leave them completely unchanged. + #[test] + #[cfg_attr(miri, ignore = "memfd_create unsupported in Miri")] + fn test_evacuate_direct_atom() { + let stack = NockStack::new(NOCK_STACK_SIZE_TINY, 0); + let mut pma = test_pma(1000); + + // Test several direct atom values + let test_values: [u64; 5] = [0, 1, 42, 12345, DIRECT_MAX]; + + for &val in &test_values { + let mut noun = D(val); + let original_raw = unsafe { noun.as_raw() }; + + // Evacuate to PMA + unsafe { noun.copy_to_pma(&stack, &mut pma) }; + + // Direct atoms should be completely unchanged + let after_raw = unsafe { noun.as_raw() }; + assert_eq!( + original_raw, after_raw, + "Direct atom {} should be unchanged after evacuation", + val + ); + + // Verify it's still a direct atom + assert!( + noun.is_direct(), + "Should still be a direct atom after evacuation" + ); + + // Direct atoms should trivially pass assert_in_pma (no allocations to check) + noun.assert_in_pma(&pma); + } + + // PMA should have no allocations (direct atoms don't need space) + assert_eq!( + pma.alloc_offset(), + 0, + "No allocations should be made for direct atoms" + ); + } + + /// Verifies indirect atoms (too large for direct representation) are copied to PMA + /// and converted to offset form. + /// + /// This test exercises: + /// - Creating an indirect atom on the NockStack + /// - Evacuating it to the PMA via copy_to_pma + /// - Verifying the atom is now in offset form (LOCATION_BIT set) + /// - Verifying the data can be read correctly via the PMA arena + /// - Verifying PMA allocations were made + #[test] + #[cfg_attr(miri, ignore = "memfd_create unsupported in Miri")] + fn test_evacuate_indirect_atom() { + let mut stack = NockStack::new(NOCK_STACK_SIZE_TINY, 0); + let mut pma = test_pma(1000); + let space = NounSpace::new(&stack, &pma); + + // Create an indirect atom on the stack (value > DIRECT_MAX requires indirect storage) + // We'll use a 2-word value to ensure it's indirect + let data: [u64; 2] = [0xDEADBEEF_CAFEBABE, 0x12345678_9ABCDEF0]; + let indirect = unsafe { IndirectAtom::new_raw(&mut stack, 2, data.as_ptr()) }; + let mut noun = indirect.as_noun(); + + // Verify it's an indirect atom on the stack + assert!(noun.is_indirect(), "Should be an indirect atom"); + assert!(!noun.is_direct(), "Should not be a direct atom"); + assert!( + matches!( + noun.in_space(&space).allocated_location(), + Some(AllocLocation::Stack) + ), + "Should be stack-allocated before evacuation" + ); + + // Record the initial PMA offset + let initial_offset = pma.alloc_offset(); + assert_eq!(initial_offset, 0, "PMA should start empty"); + + // Evacuate to PMA + unsafe { noun.copy_to_pma(&stack, &mut pma) }; + + // Verify PMA allocation was made + // Indirect atom needs: metadata (1) + size (1) + data (2) = 4 words + assert!( + pma.alloc_offset() > initial_offset, + "PMA should have allocations after evacuation" + ); + assert_eq!( + pma.alloc_offset(), + 4, // metadata + size + 2 data words + "Indirect atom should allocate 4 words in PMA" + ); + + // Verify the noun is now in offset form (not stack-allocated) + assert!( + !matches!( + noun.in_space(&space).allocated_location(), + Some(AllocLocation::Stack) + ), + "Should be in offset form after evacuation" + ); + assert!(noun.is_indirect(), "Should still be an indirect atom"); + + // Verify data is readable and correct via PMA arena + let atom = noun.as_atom().expect("Should be an atom"); + let read_indirect = atom.as_indirect().expect("Should be indirect"); + + // Read the size - should be 2 words + let read_handle = read_indirect.as_atom().in_space(&space); + let size = read_handle.size(); + assert_eq!(size, 2, "Indirect atom should have size 2"); + + // Read the data back and verify it matches + let data_ptr = read_handle.data_pointer(); + let read_data = unsafe { std::slice::from_raw_parts(data_ptr, 2) }; + assert_eq!(read_data[0], data[0], "First data word should match"); + assert_eq!(read_data[1], data[1], "Second data word should match"); + + // Verify assert_in_pma passes + noun.assert_in_pma(&pma); + } + + /// Verifies a simple cell with direct atom contents is evacuated and readable from PMA. + /// + /// This test exercises: + /// - Creating a cell [head tail] on the NockStack + /// - Evacuating it to the PMA + /// - Verifying the cell is in offset form + /// - Verifying head and tail are readable and correct + #[test] + #[cfg_attr(miri, ignore = "memfd_create unsupported in Miri")] + fn test_evacuate_simple_cell() { + let mut stack = NockStack::new(NOCK_STACK_SIZE_TINY, 0); + let mut pma = test_pma(1000); + let space = NounSpace::new(&stack, &pma); + + // Create a simple cell [42 123] with direct atoms + let mut noun = Cell::new(&mut stack, D(42), D(123)).as_noun(); + + // Verify it's a cell on the stack + assert!(noun.is_cell(), "Should be a cell"); + assert!( + matches!( + noun.in_space(&space).allocated_location(), + Some(AllocLocation::Stack) + ), + "Should be stack-allocated before evacuation" + ); + + unsafe { noun.copy_to_pma(&stack, &mut pma) }; + + // Verify PMA allocation was made (CellMemory size) + let cell_words = word_size_of::(); + assert_eq!( + pma.alloc_offset(), + cell_words, + "Cell should allocate {} words", + cell_words + ); + + // Verify the noun is now in offset form + assert!( + !matches!( + noun.in_space(&space).allocated_location(), + Some(AllocLocation::Stack) + ), + "Should be in offset form after evacuation" + ); + assert!(noun.is_cell(), "Should still be a cell"); + + // Read head and tail + let cell = noun.in_space(&space).as_cell().expect("Should be a cell"); + let head = cell.head().noun(); + let tail = cell.tail().noun(); + + // Verify head and tail are correct direct atoms + assert!(head.is_direct(), "Head should be direct"); + assert!(tail.is_direct(), "Tail should be direct"); + assert_eq!( + head.as_direct().expect("head is direct").data(), + 42, + "Head should be 42" + ); + assert_eq!( + tail.as_direct().expect("tail is direct").data(), + 123, + "Tail should be 123" + ); + + // Verify assert_in_pma passes + noun.assert_in_pma(&pma); + } + + /// Verifies nested cell structures are fully evacuated with all sub-cells in offset form. + /// + /// This test exercises: + /// - Creating nested cells [[1 2] [3 4]] + /// - Evacuating the entire structure + /// - Verifying all cells are in offset form + /// - Verifying all values are readable + #[test] + #[cfg_attr(miri, ignore = "memfd_create unsupported in Miri")] + fn test_evacuate_nested_cells() { + let mut stack = NockStack::new(NOCK_STACK_SIZE_TINY, 0); + let mut pma = test_pma(1000); + let space = NounSpace::new(&stack, &pma); + + // Create nested cells: [[1 2] [3 4]] + let left = Cell::new(&mut stack, D(1), D(2)).as_noun(); + let right = Cell::new(&mut stack, D(3), D(4)).as_noun(); + let mut noun = Cell::new(&mut stack, left, right).as_noun(); + + // Verify structure before evacuation + assert!(noun.is_cell(), "Root should be a cell"); + assert!( + matches!( + noun.in_space(&space).allocated_location(), + Some(AllocLocation::Stack) + ), + "Root should be stack-allocated" + ); + + unsafe { noun.copy_to_pma(&stack, &mut pma) }; + + // Should allocate 3 cells worth of space + let cell_words = word_size_of::(); + assert_eq!( + pma.alloc_offset(), + cell_words * 3, + "Should allocate 3 cells" + ); + + // Verify root is in offset form + assert!( + !matches!( + noun.in_space(&space).allocated_location(), + Some(AllocLocation::Stack) + ), + "Root should be in offset form" + ); + + // Navigate and verify structure + let root = noun.in_space(&space).as_cell().expect("root is cell"); + let left_cell = root.head().as_cell().expect("left is cell"); + let right_cell = root.tail().as_cell().expect("right is cell"); + + // Verify left cell [1 2] + assert!( + !matches!(root.head().allocated_location(), Some(AllocLocation::Stack)), + "Left should be in offset form" + ); + assert_eq!(left_cell.head().noun().as_direct().expect("1").data(), 1); + assert_eq!(left_cell.tail().noun().as_direct().expect("2").data(), 2); + + // Verify right cell [3 4] + assert!( + !matches!(root.tail().allocated_location(), Some(AllocLocation::Stack)), + "Right should be in offset form" + ); + assert_eq!(right_cell.head().noun().as_direct().expect("3").data(), 3); + assert_eq!(right_cell.tail().noun().as_direct().expect("4").data(), 4); + + // Verify assert_in_pma passes for entire structure + noun.assert_in_pma(&pma); + } + + /// Verifies cells containing indirect atoms have both the cell and atoms correctly evacuated. + #[test] + #[cfg_attr(miri, ignore = "memfd_create unsupported in Miri")] + fn test_evacuate_cell_with_indirect_atoms() { + let mut stack = NockStack::new(NOCK_STACK_SIZE_TINY, 0); + let mut pma = test_pma(1000); + let space = NounSpace::new(&stack, &pma); + + // Create indirect atoms + let data1: [u64; 2] = [0xAAAAAAAA_BBBBBBBB, 0xCCCCCCCC_DDDDDDDD]; + let data2: [u64; 2] = [0x11111111_22222222, 0x33333333_44444444]; + let indirect1 = unsafe { IndirectAtom::new_raw(&mut stack, 2, data1.as_ptr()) }; + let indirect2 = unsafe { IndirectAtom::new_raw(&mut stack, 2, data2.as_ptr()) }; + + // Create cell with indirect atoms + let mut noun = Cell::new(&mut stack, indirect1.as_noun(), indirect2.as_noun()).as_noun(); + + assert!( + matches!( + noun.in_space(&space).allocated_location(), + Some(AllocLocation::Stack) + ), + "Should be stack-allocated" + ); + + unsafe { noun.copy_to_pma(&stack, &mut pma) }; + + // Should allocate: 1 cell + 2 indirect atoms (4 words each) + let cell_words = word_size_of::(); + let indirect_words = 4; // metadata + size + 2 data words + assert_eq!( + pma.alloc_offset(), + cell_words + indirect_words * 2, + "Should allocate cell + 2 indirect atoms" + ); + + // Verify structure + assert!( + !matches!( + noun.in_space(&space).allocated_location(), + Some(AllocLocation::Stack) + ), + "Root should be in offset form" + ); + + let cell = noun.in_space(&space).as_cell().expect("is cell"); + let head = cell.head().noun(); + let tail = cell.tail().noun(); + + // Verify head is indirect atom with correct data + assert!(head.is_indirect(), "Head should be indirect"); + assert!( + !matches!( + head.in_space(&space).allocated_location(), + Some(AllocLocation::Stack) + ), + "Head should be in offset form" + ); + let head_indirect = head.as_indirect().expect("head indirect"); + let head_handle = head_indirect.as_atom().in_space(&space); + let head_data = unsafe { std::slice::from_raw_parts(head_handle.data_pointer(), 2) }; + assert_eq!(head_data[0], data1[0]); + assert_eq!(head_data[1], data1[1]); + + // Verify tail is indirect atom with correct data + assert!(tail.is_indirect(), "Tail should be indirect"); + assert!( + !matches!( + tail.in_space(&space).allocated_location(), + Some(AllocLocation::Stack) + ), + "Tail should be in offset form" + ); + let tail_indirect = tail.as_indirect().expect("tail indirect"); + let tail_handle = tail_indirect.as_atom().in_space(&space); + let tail_data = unsafe { std::slice::from_raw_parts(tail_handle.data_pointer(), 2) }; + assert_eq!(tail_data[0], data2[0]); + assert_eq!(tail_data[1], data2[1]); + + noun.assert_in_pma(&pma); + } + + /// Verifies structural sharing is preserved: [x x] evacuates x only once. + /// + /// When the same noun is referenced multiple times, the forwarding pointer + /// mechanism ensures it's only copied once, and both references point to + /// the same PMA location. + #[test] + #[cfg_attr(miri, ignore = "memfd_create unsupported in Miri")] + fn test_evacuate_shared_structure() { + let mut stack = NockStack::new(NOCK_STACK_SIZE_TINY, 0); + let mut pma = test_pma(1000); + let space = NounSpace::new(&stack, &pma); + + // Create a shared subcell + let shared = Cell::new(&mut stack, D(1), D(2)).as_noun(); + + // Create [shared shared] - both head and tail point to same cell + let mut noun = Cell::new(&mut stack, shared, shared).as_noun(); + + unsafe { noun.copy_to_pma(&stack, &mut pma) }; + + // Should allocate only 2 cells: the root and the shared subcell (not 3!) + let cell_words = word_size_of::(); + assert_eq!( + pma.alloc_offset(), + cell_words * 2, + "Should allocate only 2 cells due to sharing" + ); + + // Verify both head and tail point to the same PMA location + let root = noun.in_space(&space).as_cell().expect("is cell"); + let head_raw = unsafe { root.head().noun().as_raw() }; + let tail_raw = unsafe { root.tail().noun().as_raw() }; + assert_eq!( + head_raw, tail_raw, + "Head and tail should point to same location (sharing preserved)" + ); + + // Verify the shared cell is correct + let shared_cell = root.head().as_cell().expect("shared is cell"); + assert_eq!(shared_cell.head().noun().as_direct().expect("1").data(), 1); + assert_eq!(shared_cell.tail().noun().as_direct().expect("2").data(), 2); + + noun.assert_in_pma(&pma); + } + + /// Verifies evacuating an already-evacuated noun is a no-op that allocates nothing. + #[test] + #[cfg_attr(miri, ignore = "memfd_create unsupported in Miri")] + fn test_evacuate_already_evacuated() { + let mut stack = NockStack::new(NOCK_STACK_SIZE_TINY, 0); + let mut pma = test_pma(1000); + + // Create and evacuate a cell + let mut noun = Cell::new(&mut stack, D(1), D(2)).as_noun(); + unsafe { noun.copy_to_pma(&stack, &mut pma) }; + + let offset_after_first = pma.alloc_offset(); + assert!(offset_after_first > 0, "Should have allocated something"); + + // Evacuate again - should be a no-op + unsafe { noun.copy_to_pma(&stack, &mut pma) }; + + assert_eq!( + pma.alloc_offset(), + offset_after_first, + "Second evacuation should not allocate anything" + ); + + noun.assert_in_pma(&pma); + } + + /// Verifies deeply nested structures are fully evacuated and traversable after evacuation. + /// + /// This test exercises the worklist algorithm's ability to handle deep trees + /// without stack overflow (since we use iteration, not recursion). + #[test] + #[cfg_attr(miri, ignore = "memfd_create unsupported in Miri")] + fn test_evacuate_deep_tree() { + let mut stack = NockStack::new(NOCK_STACK_SIZE_TINY, 0); + let mut pma = test_pma(10000); + let space = NounSpace::new(&stack, &pma); + + // Create a deeply nested structure: [1 [2 [3 [4 ... [999 1000]]]]] + const DEPTH: u64 = 500; + + // Build from the inside out + let mut noun = D(DEPTH); + for i in (1..DEPTH).rev() { + noun = Cell::new(&mut stack, D(i), noun).as_noun(); + } + + // Verify it's deeply nested and stack-allocated + assert!(noun.is_cell(), "Root should be a cell"); + assert!( + matches!( + noun.in_space(&space).allocated_location(), + Some(AllocLocation::Stack) + ), + "Should be stack-allocated" + ); + + // Count the depth before evacuation + let mut depth_before = 0u64; + let mut current = noun; + while current.is_cell() { + depth_before += 1; + current = current + .in_space(&space) + .as_cell() + .expect("depth walk should find a cell") + .tail() + .noun(); + } + assert_eq!( + depth_before, + DEPTH - 1, + "Should have correct depth before evacuation" + ); + + // Evacuate + unsafe { noun.copy_to_pma(&stack, &mut pma) }; + + // Should allocate (DEPTH - 1) cells + let cell_words = word_size_of::(); + assert_eq!( + pma.alloc_offset(), + cell_words * (DEPTH as usize - 1), + "Should allocate {} cells", + DEPTH - 1 + ); + + // Verify root is in offset form + assert!( + !matches!( + noun.in_space(&space).allocated_location(), + Some(AllocLocation::Stack) + ), + "Root should be in offset form" + ); + + // Traverse the entire structure and verify values + let mut current = noun; + for expected in 1..DEPTH { + assert!(current.is_cell(), "Should be cell at depth {}", expected); + let cell = current.in_space(&space).as_cell().expect("is cell"); + + // Verify head value + let head = cell.head().noun(); + assert!( + head.is_direct(), + "Head at depth {} should be direct", + expected + ); + assert_eq!( + head.as_direct().expect("direct").data(), + expected, + "Head at depth {} should be {}", + expected, + expected + ); + + // Verify this cell is in offset form + assert!( + !matches!( + current.in_space(&space).allocated_location(), + Some(AllocLocation::Stack) + ), + "Cell at depth {} should be in offset form", + expected + ); + + current = cell.tail().noun(); + } + + // Final element should be direct atom DEPTH + assert!(current.is_direct(), "Leaf should be direct atom"); + assert_eq!( + current.as_direct().expect("direct").data(), + DEPTH, + "Leaf should be {}", + DEPTH + ); + + // Verify assert_in_pma passes for entire structure + noun.assert_in_pma(&pma); + } + + /// Verifies deeply nested structures with variable-sized indirect atoms are fully evacuated. + /// + /// Similar to test_evacuate_deep_tree, but each value is an IndirectAtom with + /// data size varying from 2 to 10 words. This tests the evacuation of mixed + /// cell/indirect-atom structures with variable allocation sizes. + #[test] + #[cfg_attr(miri, ignore = "memfd_create unsupported in Miri")] + fn test_evacuate_deep_tree_indirect_atoms() { + let mut stack = NockStack::new(NOCK_STACK_SIZE_TINY, 0); + let mut pma = test_pma(100000); // Larger PMA for indirect atoms + let space = NounSpace::new(&stack, &pma); + + const DEPTH: usize = 200; + + // Helper to create an indirect atom with `word_count` words of data + // Data pattern: first word is the index, remaining words are index + word_position + let make_indirect = |stack: &mut NockStack, index: usize, word_count: usize| -> Noun { + let mut data = vec![0u64; word_count]; + for (i, word) in data.iter_mut().enumerate() { + *word = (index as u64) << 32 | (i as u64); + } + unsafe { IndirectAtom::new_raw(stack, word_count, data.as_ptr()).as_noun() } + }; + + // Helper to compute word count for index (varies 2-10) + let word_count_for_index = |index: usize| -> usize { (index % 9) + 2 }; + + // Build from inside out: [indirect_1 [indirect_2 [indirect_3 ... indirect_DEPTH]]] + let mut noun = make_indirect(&mut stack, DEPTH, word_count_for_index(DEPTH)); + for i in (1..DEPTH).rev() { + let head = make_indirect(&mut stack, i, word_count_for_index(i)); + noun = Cell::new(&mut stack, head, noun).as_noun(); + } + + // Verify structure before evacuation + assert!(noun.is_cell(), "Root should be a cell"); + assert!( + matches!( + noun.in_space(&space).allocated_location(), + Some(AllocLocation::Stack) + ), + "Should be stack-allocated" + ); + + // Count expected allocations: + // - (DEPTH - 1) cells + // - DEPTH indirect atoms, each with (word_count + 2) words (metadata + size + data) + let cell_words = word_size_of::(); + let mut expected_indirect_words = 0usize; + for i in 1..=DEPTH { + expected_indirect_words += word_count_for_index(i) + 2; // +2 for metadata and size + } + let expected_total = (cell_words * (DEPTH - 1)) + expected_indirect_words; + + // Evacuate + unsafe { noun.copy_to_pma(&stack, &mut pma) }; + + // Verify allocation size + assert_eq!( + pma.alloc_offset(), + expected_total, + "Should allocate {} words total ({} cells + {} indirect atom words)", + expected_total, + DEPTH - 1, + expected_indirect_words + ); + + // Verify root is in offset form + assert!( + !matches!( + noun.in_space(&space).allocated_location(), + Some(AllocLocation::Stack) + ), + "Root should be in offset form" + ); + + // Traverse and verify all values + let mut current = noun; + for expected_index in 1..DEPTH { + assert!( + current.is_cell(), + "Should be cell at depth {}", + expected_index + ); + let cell = current.in_space(&space).as_cell().expect("is cell"); + + // Verify head is an indirect atom with correct data + let head = cell.head().noun(); + assert!( + head.is_indirect(), + "Head at depth {} should be indirect", + expected_index + ); + assert!( + !matches!( + head.in_space(&space).allocated_location(), + Some(AllocLocation::Stack) + ), + "Head at depth {} should be in offset form", + expected_index + ); + + let head_indirect = head.as_indirect().expect("indirect"); + let head_handle = head_indirect.as_atom().in_space(&space); + let expected_word_count = word_count_for_index(expected_index); + assert_eq!( + head_handle.size(), + expected_word_count, + "Indirect atom at depth {} should have {} words", + expected_index, + expected_word_count + ); + + // Verify data pattern + let data_ptr = head_handle.data_pointer(); + for word_idx in 0..expected_word_count { + let expected_value = (expected_index as u64) << 32 | (word_idx as u64); + let actual_value = unsafe { *data_ptr.add(word_idx) }; + assert_eq!( + actual_value, expected_value, + "Data mismatch at depth {}, word {}", + expected_index, word_idx + ); + } + + current = cell.tail().noun(); + } + + // Final element should be indirect atom for index DEPTH + assert!(current.is_indirect(), "Leaf should be indirect atom"); + assert!( + !matches!( + current.in_space(&space).allocated_location(), + Some(AllocLocation::Stack) + ), + "Leaf should be in offset form" + ); + + let leaf_indirect = current.as_indirect().expect("indirect"); + let leaf_handle = leaf_indirect.as_atom().in_space(&space); + let expected_leaf_words = word_count_for_index(DEPTH); + assert_eq!( + leaf_handle.size(), + expected_leaf_words, + "Leaf indirect atom should have {} words", + expected_leaf_words + ); + + // Verify leaf data pattern + let leaf_data_ptr = leaf_handle.data_pointer(); + for word_idx in 0..expected_leaf_words { + let expected_value = (DEPTH as u64) << 32 | (word_idx as u64); + let actual_value = unsafe { *leaf_data_ptr.add(word_idx) }; + assert_eq!( + actual_value, expected_value, + "Leaf data mismatch at word {}", + word_idx + ); + } + + // Verify assert_in_pma passes for entire structure + noun.assert_in_pma(&pma); + } + + /// Verifies NounAllocator::equals works through the Pma interface. + #[test] + #[cfg_attr(miri, ignore = "memfd_create unsupported in Miri")] + fn test_pma_noun_allocator_equals() { + let mut stack = NockStack::new(NOCK_STACK_SIZE_TINY, 0); + let mut pma = test_pma(1000); + + let mut noun1 = Cell::new(&mut stack, D(1), D(2)).as_noun(); + let mut noun2 = Cell::new(&mut stack, D(1), D(2)).as_noun(); + let mut noun3 = Cell::new(&mut stack, D(1), D(3)).as_noun(); + + unsafe { + noun1.copy_to_pma(&stack, &mut pma); + noun2.copy_to_pma(&stack, &mut pma); + noun3.copy_to_pma(&stack, &mut pma); + } + + // Test through NounAllocator trait + assert!( + unsafe { pma.equals(&mut noun1 as *mut Noun, &mut noun2 as *mut Noun) }, + "NounAllocator::equals should return true for equal nouns" + ); + assert!( + !unsafe { pma.equals(&mut noun1 as *mut Noun, &mut noun3 as *mut Noun) }, + "NounAllocator::equals should return false for unequal nouns" + ); + } + + /// Verifies that a HAMT can be evacuated to PMA and lookups still work. + /// + /// This test exercises: + /// - Creating a HAMT with multiple entries (direct atoms as keys/values) + /// - Evacuating the entire HAMT structure to PMA + /// - Verifying all entries are still retrievable via lookup + /// - Verifying all internal pointers are in offset form (not stack-allocated) + #[test] + #[cfg_attr(miri, ignore = "memfd_create unsupported in Miri")] + fn test_evacuate_hamt_round_trip() { + let mut stack = NockStack::new(NOCK_STACK_SIZE_TINY, 0); + let mut pma = test_pma(10000); + let space = NounSpace::new(&stack, &pma); + + // Create a HAMT with several entries + let mut hamt: Hamt = Hamt::new(&mut stack); + + // Insert 10 key-value pairs + for i in 0u64..10 { + let mut key = D(i); + let value = D(i * 100); + hamt = hamt.insert(&mut stack, &mut key, value); + } + + // Verify lookups work before evacuation + for i in 0u64..10 { + let mut key = D(i); + let result = hamt.lookup(&mut stack, &mut key); + assert!( + result.is_some(), + "Lookup for key {} should succeed before evacuation", + i + ); + let value = result.expect("lookup should return a value before evacuation"); + assert!(value.is_direct(), "Value should be direct atom"); + assert_eq!( + value + .as_direct() + .expect("lookup value should be a direct atom") + .data(), + i * 100, + "Value for key {} should be {}", + i, + i * 100 + ); + } + + // Evacuate the HAMT to PMA + unsafe { + hamt.copy_to_pma(&stack, &mut pma); + } + + // Verify entries are still present after evacuation + let mut found = [false; 10]; + for entries in hamt.iter() { + for (key, value) in entries { + let key_direct = key.as_direct().expect("key should be direct"); + let value_direct = value.as_direct().expect("value should be direct"); + let idx = key_direct.data() as usize; + assert!( + idx < found.len(), + "Key {} should be within expected range", + idx + ); + assert_eq!( + value_direct.data(), + (idx as u64) * 100, + "Value for key {} should still be {} after evacuation", + idx, + (idx as u64) * 100 + ); + found[idx] = true; + } + } + assert!( + found.iter().all(|present| *present), + "All keys should be present after evacuation" + ); + + // Verify internal structure is in PMA (offset form) + // Iterate over the HAMT and check all nouns are not stack-allocated + for entries in hamt.iter() { + for (key, value) in entries { + if !key.is_direct() { + assert!( + !matches!( + key.in_space(&space).allocated_location(), + Some(AllocLocation::Stack) + ), + "HAMT key should be in offset form after evacuation" + ); + } + if !value.is_direct() { + assert!( + !matches!( + value.in_space(&space).allocated_location(), + Some(AllocLocation::Stack) + ), + "HAMT value should be in offset form after evacuation" + ); + } + } + } + } + + /// Test that copy_to_pma correctly copies nouns to PMA and produces valid offset-form nouns. + /// + /// Note: copy_to_pma sets forwarding pointers in the source nouns, which corrupts + /// them for normal use. This is by design for structural sharing. Therefore, we + /// cannot compare source vs PMA copy directly. Instead, we verify the PMA copy + /// contains the expected data. + /// + /// This test may look superfluous, but it helped debug test_evacuate_hamt_complex_nouns so + /// that's why its in here. + #[test] + #[cfg_attr(miri, ignore = "memfd_create unsupported in Miri")] + fn test_copy_to_pma_preserves_data() { + use crate::noun::{Cell, IndirectAtom}; + + let mut stack = NockStack::new(NOCK_STACK_SIZE_TINY, 0); + let mut pma = test_pma(10000); + let space = NounSpace::new(&stack, &pma); + + // Test with indirect atom + let data: [u64; 2] = [0xDEADBEEF_CAFEBABE, 0x12345678_9ABCDEF0]; + let stack_indirect = + unsafe { IndirectAtom::new_raw(&mut stack, 2, data.as_ptr()) }.as_noun(); + + // Copy to PMA + let mut pma_indirect = stack_indirect; + unsafe { pma_indirect.copy_to_pma(&stack, &mut pma) }; + + // Verify the PMA copy is in offset form + assert!( + !matches!( + pma_indirect.in_space(&space).allocated_location(), + Some(AllocLocation::Stack) + ), + "PMA copy should be in offset form" + ); + + // Verify the PMA copy contains correct data + let pma_ia = pma_indirect + .as_indirect() + .expect("PMA copy should be an indirect atom"); + let pma_handle = pma_ia.as_atom().in_space(&space); + let pma_size = pma_handle.size(); + assert_eq!(pma_size, 2, "PMA indirect atom should have size 2"); + + let pma_bytes = pma_handle.as_ne_bytes(); + assert_eq!( + pma_bytes.len(), + 16, + "PMA indirect should have 16 bytes of data" + ); + + // Verify actual data values + let pma_slice = + unsafe { std::slice::from_raw_parts(pma_handle.data_pointer(), pma_handle.size()) }; + assert_eq!(pma_slice[0], 0xDEADBEEF_CAFEBABE, "First word should match"); + assert_eq!( + pma_slice[1], 0x12345678_9ABCDEF0, + "Second word should match" + ); + + // Test with cell containing direct atoms + let stack_cell = Cell::new(&mut stack, D(42), D(99)).as_noun(); + let mut pma_cell = stack_cell; + unsafe { pma_cell.copy_to_pma(&stack, &mut pma) }; + + assert!( + !matches!( + pma_cell.in_space(&space).allocated_location(), + Some(AllocLocation::Stack) + ), + "PMA cell should be in offset form" + ); + let cell = pma_cell + .in_space(&space) + .as_cell() + .expect("PMA noun should be a cell"); + assert_eq!( + cell.head() + .noun() + .as_direct() + .expect("cell head should be direct") + .data(), + 42, + "Cell head should be 42" + ); + assert_eq!( + cell.tail() + .noun() + .as_direct() + .expect("cell tail should be direct") + .data(), + 99, + "Cell tail should be 99" + ); + + // Test with nested structure + let inner = Cell::new(&mut stack, D(1), D(2)).as_noun(); + let stack_nested = Cell::new(&mut stack, inner, D(3)).as_noun(); + let mut pma_nested = stack_nested; + unsafe { pma_nested.copy_to_pma(&stack, &mut pma) }; + + assert!( + !matches!( + pma_nested.in_space(&space).allocated_location(), + Some(AllocLocation::Stack) + ), + "PMA nested should be in offset form" + ); + let outer = pma_nested + .in_space(&space) + .as_cell() + .expect("PMA nested noun should be an outer cell"); + assert_eq!( + outer + .tail() + .noun() + .as_direct() + .expect("outer tail should be direct") + .data(), + 3, + "Outer tail should be 3" + ); + let inner_cell = outer.head().as_cell().expect("outer head should be a cell"); + assert_eq!( + inner_cell + .head() + .noun() + .as_direct() + .expect("inner head should be direct") + .data(), + 1, + "Inner head should be 1" + ); + assert_eq!( + inner_cell + .tail() + .noun() + .as_direct() + .expect("inner tail should be direct") + .data(), + 2, + "Inner tail should be 2" + ); + } + + /// Test HAMT evacuation with complex noun types: Cells and IndirectAtoms. + /// + /// This test exercises: + /// - HAMT with indirect atoms as keys (large numbers) + /// - HAMT with cells as values (nested structures) + /// - Deep cell nesting to test recursive evacuation + /// - Structural equality verification using a reference copy on a separate stack + /// + /// Note: copy_to_pma sets forwarding pointers in source nouns, corrupting them. + /// To verify values, we create a second NockStack with fresh copies of the same + /// data and compare those against the PMA copy using noun_equality. + #[test] + #[cfg_attr(miri, ignore = "memfd_create unsupported in Miri")] + fn test_evacuate_hamt_complex_nouns() { + use crate::ext::noun_equality; + use crate::noun::{Cell, IndirectAtom}; + + let mut stack = NockStack::new(NOCK_STACK_SIZE_TINY, 0); + let mut pma = test_pma(100000); + + // Create a second stack with reference copies of keys/values for comparison + // This stack won't be corrupted by forwarding pointers + let mut ref_stack = NockStack::new(NOCK_STACK_SIZE_TINY, 0); + let space = NounSpace::new(&stack, &pma); + let ref_space = NounSpace::new(&ref_stack, &pma); + + let mut hamt: Hamt = Hamt::new(&mut stack); + + // Store reference keys/values on the separate stack + let mut ref_keys: Vec = Vec::new(); + let mut ref_values: Vec = Vec::new(); + + // Insert entries with indirect atom keys and cell values + for i in 0u64..5 { + let key_data: [u64; 2] = [0xDEADBEEF_CAFEBABE + i, 0x12345678_9ABCDEF0 + i]; + + // Create on main stack for HAMT + let key_atom = + unsafe { IndirectAtom::new_raw(&mut stack, 2, key_data.as_ptr()) }.as_noun(); + let inner = Cell::new(&mut stack, D(i + 100), D(i + 200)).as_noun(); + let value = Cell::new(&mut stack, D(i), inner).as_noun(); + + // Create identical copies on reference stack + let ref_key = + unsafe { IndirectAtom::new_raw(&mut ref_stack, 2, key_data.as_ptr()) }.as_noun(); + let ref_inner = Cell::new(&mut ref_stack, D(i + 100), D(i + 200)).as_noun(); + let ref_value = Cell::new(&mut ref_stack, D(i), ref_inner).as_noun(); + ref_keys.push(ref_key); + ref_values.push(ref_value); + + let mut key_copy = key_atom; + hamt = hamt.insert(&mut stack, &mut key_copy, value); + } + + // Insert entries with cell keys and indirect atom values + for i in 5u64..10 { + let val_data: [u64; 2] = [i * 1000, i * 2000]; + + // Create on main stack for HAMT + let key = Cell::new(&mut stack, D(i), D(i + 1)).as_noun(); + let value = + unsafe { IndirectAtom::new_raw(&mut stack, 2, val_data.as_ptr()) }.as_noun(); + + // Create identical copies on reference stack + let ref_key = Cell::new(&mut ref_stack, D(i), D(i + 1)).as_noun(); + let ref_value = + unsafe { IndirectAtom::new_raw(&mut ref_stack, 2, val_data.as_ptr()) }.as_noun(); + ref_keys.push(ref_key); + ref_values.push(ref_value); + + let mut key_copy = key; + hamt = hamt.insert(&mut stack, &mut key_copy, value); + } + + // Insert entries with deeply nested cells + for i in 10u64..12 { + // Create on main stack for HAMT + let ab = Cell::new(&mut stack, D(i), D(i + 1)).as_noun(); + let abc = Cell::new(&mut stack, ab, D(i + 2)).as_noun(); + let key = Cell::new(&mut stack, abc, D(i + 3)).as_noun(); + let zw = Cell::new(&mut stack, D(i + 10), D(i + 11)).as_noun(); + let yzw = Cell::new(&mut stack, D(i + 9), zw).as_noun(); + let value = Cell::new(&mut stack, D(i + 8), yzw).as_noun(); + + // Create identical copies on reference stack + let ref_ab = Cell::new(&mut ref_stack, D(i), D(i + 1)).as_noun(); + let ref_abc = Cell::new(&mut ref_stack, ref_ab, D(i + 2)).as_noun(); + let ref_key = Cell::new(&mut ref_stack, ref_abc, D(i + 3)).as_noun(); + let ref_zw = Cell::new(&mut ref_stack, D(i + 10), D(i + 11)).as_noun(); + let ref_yzw = Cell::new(&mut ref_stack, D(i + 9), ref_zw).as_noun(); + let ref_value = Cell::new(&mut ref_stack, D(i + 8), ref_yzw).as_noun(); + ref_keys.push(ref_key); + ref_values.push(ref_value); + + let mut key_copy = key; + hamt = hamt.insert(&mut stack, &mut key_copy, value); + } + + // Count entries before evacuation + let count_before: usize = hamt.iter().map(|entries| entries.len()).sum(); + assert_eq!(count_before, 12, "Should have 12 entries before evacuation"); + + // Evacuate the HAMT to PMA + unsafe { + hamt.copy_to_pma(&stack, &mut pma); + } + + // Count entries after evacuation + let count_after: usize = hamt.iter().map(|entries| entries.len()).sum(); + assert_eq!( + count_after, count_before, + "Entry count should be preserved after evacuation" + ); + + // Verify all values match by comparing PMA nouns to reference stack nouns + let mut found_count = 0; + for entries in hamt.iter() { + for (pma_key, pma_value) in entries { + // Find matching reference key and verify value matches + let mut found = false; + for (idx, ref_key) in ref_keys.iter().enumerate() { + if noun_equality( + (*pma_key).in_space(&ref_space), + (*ref_key).in_space(&ref_space), + ) { + assert!( + noun_equality( + (*pma_value).in_space(&ref_space), + ref_values[idx].in_space(&ref_space), + ), + "Value for key {} should match reference after evacuation", + idx + ); + found = true; + found_count += 1; + break; + } + } + assert!(found, "Every PMA key should match a reference key"); + } + } + assert_eq!( + found_count, + ref_keys.len(), + "Should find all {} entries in HAMT after evacuation", + ref_keys.len() + ); + + // Verify all nouns in the HAMT are in offset form + for entries in hamt.iter() { + for (key, value) in entries { + verify_noun_not_stack_allocated(*key, &space, "HAMT key"); + verify_noun_not_stack_allocated(*value, &space, "HAMT value"); + } + } + + // Verify the HAMT structure itself is in PMA + hamt.assert_in_pma(&pma); + } + + /// Helper to recursively verify a noun is not stack-allocated + fn verify_noun_not_stack_allocated(noun: Noun, space: &NounSpace, context: &str) { + if noun.is_direct() { + return; + } + + let location = noun.in_space(space).allocated_location(); + assert!( + !matches!(location, Some(AllocLocation::Stack)), + "{} should be in offset form after evacuation", + context + ); + + if let Ok(cell) = noun.in_space(space).as_cell() { + verify_noun_not_stack_allocated(cell.head().noun(), space, context); + verify_noun_not_stack_allocated(cell.tail().noun(), space, context); + } + } + + /// Verifies that PmaCopy for () is a no-op that allocates nothing. + /// + /// The unit type has no data, so copy_to_pma should not allocate anything + /// and assert_in_pma should trivially pass. + #[test] + #[cfg_attr(miri, ignore = "memfd_create unsupported in Miri")] + fn test_evacuate_unit() { + let stack = NockStack::new(NOCK_STACK_SIZE_TINY, 0); + let mut pma = test_pma(1000); + + let mut unit = (); + let initial_offset = pma.alloc_offset(); + + // Copy to PMA - should be a no-op + unsafe { unit.copy_to_pma(&stack, &mut pma) }; + + // Verify no allocations were made + assert_eq!( + pma.alloc_offset(), + initial_offset, + "No allocations should be made for unit type" + ); + + // assert_in_pma should not panic + unit.assert_in_pma(&pma); + } +} + +#[cfg(all(test, any(target_os = "linux", target_os = "macos")))] +mod paging_tests { + use super::{madvise_drop_file_backed_pages, test_pma_path, Pma}; + + const SLAB_BYTES: usize = 64 * 1024 * 1024; + const TOUCH_PAGES: usize = 64; + + #[test] + #[cfg_attr(miri, ignore = "mincore/madvise unsupported in Miri")] + fn pma_file_backed_pages_out_and_faults_lazily() { + let words = SLAB_BYTES >> 3; + let path = test_pma_path("paging"); + let pma = Pma::new(words, path).expect("failed to create PMA"); + let base = pma.arena().base_ptr(); + let len = pma.arena().len_bytes(); + let page = page_size(); + + assert_eq!(len, SLAB_BYTES, "unexpected PMA length"); + assert_eq!( + len % page, + 0, + "PMA length must be page sized, len={len}, page={page}" + ); + + touch_entire_region(base, len, page); + let resident_bitmap = mincore_bitmap(base, len); + let initial_ratio = residency_ratio(&resident_bitmap); + println!("[pma-paging] initial residency ratio {:.3}", initial_ratio); + assert!( + resident_bitmap.iter().all(|b| b & 1 == 1), + "expected fully resident slab after touching every page" + ); + + drop_all_pages(base, len); + let after_drop = mincore_bitmap(base, len); + let post_drop_ratio = residency_ratio(&after_drop); + println!( + "[pma-paging] post-drop residency ratio {:.3}", + post_drop_ratio + ); + if post_drop_ratio > 0.9 { + println!( + "[pma-paging] paging did not drop pages; skipping remainder (ratio={post_drop_ratio:.3})" + ); + return; + } + assert!( + post_drop_ratio < 0.1, + "expected paging to drop most pages, ratio={post_drop_ratio}" + ); + + let total_pages = len / page; + let touched_pages = fault_sparse(base, len, page, TOUCH_PAGES); + assert!(touched_pages > 0, "expected to fault at least one page"); + + let post_fault = mincore_bitmap(base, len); + let post_fault_ratio = residency_ratio(&post_fault); + let expected_ratio = touched_pages as f64 / total_pages.max(1) as f64; + println!( + "[pma-paging] post-fault residency ratio {:.4} (expected {:.4}, touched {} pages)", + post_fault_ratio, expected_ratio, touched_pages + ); + assert!( + post_fault_ratio >= expected_ratio * 0.5 && post_fault_ratio <= expected_ratio * 2.0, + "faulted pages should roughly match touched subset (ratio {} expected {})", + post_fault_ratio, + expected_ratio + ); + } + + fn touch_entire_region(ptr: *mut u8, len: usize, page: usize) { + for offset in (0..len).step_by(page) { + unsafe { + std::ptr::write_volatile(ptr.add(offset), (offset / page % 255) as u8); + } + } + } + + fn fault_sparse(ptr: *mut u8, len: usize, page: usize, desired_pages: usize) -> usize { + let total_pages = len / page; + if total_pages == 0 { + return 0; + } + let touches = desired_pages.min(total_pages.max(1)); + let stride = (total_pages / touches).max(1); + let mut touched = 0; + let mut page_idx = 0; + while touched < touches && page_idx < total_pages { + unsafe { + std::ptr::read_volatile(ptr.add(page_idx * page)); + } + touched += 1; + page_idx = page_idx.saturating_add(stride); + } + touched + } + + fn drop_all_pages(ptr: *mut u8, len: usize) { + madvise_drop_file_backed_pages(ptr as *mut libc::c_void, len) + .expect("failed to advise file-backed PMA pages out"); + std::thread::sleep(std::time::Duration::from_millis(50)); + } + + fn mincore_bitmap(ptr: *mut u8, len: usize) -> Vec { + let page = page_size(); + assert_eq!( + len % page, + 0, + "mincore requires len to be page sized, len={len}, page={page}" + ); + let pages = len / page; + let mut vec = vec![0u8; pages]; + let ret = unsafe { libc::mincore(ptr as *mut libc::c_void, len, vec.as_mut_ptr().cast()) }; + if ret != 0 { + panic!("mincore failed: {}", std::io::Error::last_os_error()); + } + vec + } + + fn residency_ratio(bitmap: &[u8]) -> f64 { + if bitmap.is_empty() { + return 0.0; + } + let resident = bitmap.iter().filter(|b| **b & 1 == 1).count(); + resident as f64 / bitmap.len() as f64 + } + + fn page_size() -> usize { + unsafe { libc::sysconf(libc::_SC_PAGESIZE) as usize } + } +} diff --git a/crates/nockvm/rust/nockvm/src/pma/stream.rs b/crates/nockvm/rust/nockvm/src/pma/stream.rs new file mode 100644 index 000000000..3c75892a5 --- /dev/null +++ b/crates/nockvm/rust/nockvm/src/pma/stream.rs @@ -0,0 +1,406 @@ +use std::alloc::{alloc, dealloc, Layout}; +use std::fs::{File, OpenOptions}; +use std::io; +#[cfg(unix)] +use std::os::unix::fs::FileExt; +#[cfg(target_os = "linux")] +use std::os::unix::fs::OpenOptionsExt; +#[cfg(target_os = "macos")] +use std::os::unix::io::AsRawFd; +use std::path::Path; +use std::ptr::NonNull; + +#[cfg(unix)] +use libc; +use thiserror::Error; + +use super::Pma; +use crate::noun::{ + CELL_MASK, CELL_TAG, DIRECT_MASK, DIRECT_TAG, INDIRECT_MASK, INDIRECT_TAG, LOCATION_BIT, +}; +use crate::offset::PmaOffsetWords; + +const DEFAULT_CACHE_PAGES: usize = 64; + +#[derive(Clone, Copy, Debug)] +pub struct PmaDirectJamConfig { + pub cache_pages: usize, + pub require_direct_io: bool, +} + +impl Default for PmaDirectJamConfig { + fn default() -> Self { + Self { + cache_pages: DEFAULT_CACHE_PAGES, + require_direct_io: true, + } + } +} + +#[derive(Debug, Error)] +pub enum PmaDirectJamError { + #[error("direct IO open failed: {0}")] + DirectIoOpen(#[source] io::Error), + #[error("io error: {0}")] + Io(#[from] io::Error), + #[error("invalid noun raw {raw:#x}: {reason}")] + InvalidNoun { raw: u64, reason: &'static str }, + #[error("offset {offset} out of bounds (limit {limit})")] + OffsetOutOfBounds { offset: u64, limit: u64 }, + #[error("indirect atom size {size} at offset {offset} exceeds limit {limit}")] + IndirectSizeOutOfBounds { offset: u64, size: u64, limit: u64 }, + #[error("aligned allocation failed")] + AllocationFailed, + #[error("invalid alignment: {0}")] + BadAlignment(String), + #[error("direct IO unsupported on this platform")] + UnsupportedPlatform, +} + +struct AlignedBuffer { + ptr: NonNull, + len: usize, + layout: Layout, +} + +impl AlignedBuffer { + fn new(len: usize, align: usize) -> Result { + let layout = Layout::from_size_align(len, align) + .map_err(|err| PmaDirectJamError::BadAlignment(err.to_string()))?; + let ptr = unsafe { alloc(layout) }; + let ptr = NonNull::new(ptr).ok_or(PmaDirectJamError::AllocationFailed)?; + Ok(Self { ptr, len, layout }) + } + + fn as_slice(&self) -> &[u8] { + unsafe { std::slice::from_raw_parts(self.ptr.as_ptr(), self.len) } + } + + fn as_mut_slice(&mut self) -> &mut [u8] { + unsafe { std::slice::from_raw_parts_mut(self.ptr.as_ptr(), self.len) } + } +} + +impl Drop for AlignedBuffer { + fn drop(&mut self) { + unsafe { + dealloc(self.ptr.as_ptr(), self.layout); + } + } +} + +struct CachedPage { + index: u64, + last_used: u64, + data: AlignedBuffer, +} + +pub struct PmaDirectReader { + file: File, + page_size: usize, + data_words: u64, + alloc_words: u64, + cache_capacity: usize, + cache: Vec, + tick: u64, +} + +impl PmaDirectReader { + pub fn new(pma: &Pma, config: PmaDirectJamConfig) -> Result { + Self::from_path( + pma.path(), + u64::try_from(pma.size_words()).expect("PMA data words exceed u64 addressable range"), + pma.alloc_offset_words().into(), + config, + ) + } + + pub fn from_path( + path: &Path, + data_words: u64, + alloc_words: u64, + config: PmaDirectJamConfig, + ) -> Result { + let file = open_direct_file(path, config.require_direct_io)?; + let page_size = page_size()?; + let cache_capacity = config.cache_pages.max(1); + Ok(Self { + file, + page_size, + data_words, + alloc_words, + cache_capacity, + cache: Vec::with_capacity(cache_capacity), + tick: 0, + }) + } + + pub fn alloc_words(&self) -> u64 { + self.alloc_words + } + + pub fn read_u64(&mut self, offset_words: u64) -> Result { + if offset_words >= self.alloc_words { + return Err(PmaDirectJamError::OffsetOutOfBounds { + offset: offset_words, + limit: self.alloc_words, + }); + } + let byte_offset = + offset_words + .checked_mul(8) + .ok_or(PmaDirectJamError::OffsetOutOfBounds { + offset: offset_words, + limit: self.alloc_words, + })?; + let page_size = self.page_size as u64; + let page_index = byte_offset / page_size; + let page_offset = (byte_offset % page_size) as usize; + let cache_index = self.ensure_page(page_index)?; + let page = self.cache[cache_index].data.as_slice(); + if page_offset + 8 <= self.page_size { + let mut bytes = [0u8; 8]; + bytes.copy_from_slice(&page[page_offset..page_offset + 8]); + return Ok(u64::from_ne_bytes(bytes)); + } + + let mut bytes = [0u8; 8]; + let first_len = self.page_size - page_offset; + bytes[..first_len].copy_from_slice(&page[page_offset..]); + let next_index = self.ensure_page(page_index + 1)?; + let next_page = self.cache[next_index].data.as_slice(); + bytes[first_len..].copy_from_slice(&next_page[..8 - first_len]); + Ok(u64::from_ne_bytes(bytes)) + } + + fn ensure_page(&mut self, index: u64) -> Result { + self.tick = self.tick.wrapping_add(1); + if let Some((idx, entry)) = self + .cache + .iter_mut() + .enumerate() + .find(|(_, entry)| entry.index == index) + { + entry.last_used = self.tick; + return Ok(idx); + } + + let slot = if self.cache.len() < self.cache_capacity { + let data = AlignedBuffer::new(self.page_size, self.page_size)?; + self.cache.push(CachedPage { + index, + last_used: self.tick, + data, + }); + self.cache.len() - 1 + } else { + let (idx, _) = self + .cache + .iter() + .enumerate() + .min_by_key(|(_, entry)| entry.last_used) + .ok_or(PmaDirectJamError::AllocationFailed)?; + self.cache[idx].index = index; + self.cache[idx].last_used = self.tick; + idx + }; + + let data = self.cache[slot].data.as_mut_slice(); + Self::read_page(&self.file, self.page_size, self.data_words, index, data)?; + Ok(slot) + } + + fn read_page( + file: &File, + page_size: usize, + data_words: u64, + index: u64, + buffer: &mut [u8], + ) -> Result<(), PmaDirectJamError> { + let offset_bytes = + index + .checked_mul(page_size as u64) + .ok_or(PmaDirectJamError::OffsetOutOfBounds { + offset: index, + limit: data_words, + })?; + let mut total = 0usize; + while total < buffer.len() { + let read = file.read_at(&mut buffer[total..], offset_bytes + total as u64)?; + if read == 0 { + if total == 0 { + return Err(PmaDirectJamError::Io(io::Error::new( + io::ErrorKind::UnexpectedEof, + "short read while reading PMA page", + ))); + } + break; + } + total += read; + } + if total < buffer.len() { + buffer[total..].fill(0); + } + Ok(()) + } + + pub fn read_cell(&mut self, offset: u64) -> Result<(u64, u64), PmaDirectJamError> { + let limit = self.alloc_words; + if offset + 3 > limit { + return Err(PmaDirectJamError::OffsetOutOfBounds { offset, limit }); + } + let head = self.read_u64(offset + 1)?; + let tail = self.read_u64(offset + 2)?; + Ok((head, tail)) + } + + pub fn indirect_atom_words(&mut self, offset: u64) -> Result { + let size_raw = self.read_u64(offset + 1)?; + if size_raw & CELL_MASK == CELL_MASK { + return Err(PmaDirectJamError::InvalidNoun { + raw: size_raw, + reason: "forwarding pointer in indirect atom header", + }); + } + let size_words = + usize::try_from(size_raw).map_err(|_| PmaDirectJamError::IndirectSizeOutOfBounds { + offset, + size: size_raw, + limit: self.alloc_words, + })?; + if size_words == 0 { + return Err(PmaDirectJamError::InvalidNoun { + raw: size_raw, + reason: "zero-length indirect atom", + }); + } + let limit = self.alloc_words; + let end = offset + .checked_add(2) + .and_then(|base| base.checked_add(size_words as u64)) + .ok_or(PmaDirectJamError::IndirectSizeOutOfBounds { + offset, + size: size_raw, + limit, + })?; + if end > limit { + return Err(PmaDirectJamError::IndirectSizeOutOfBounds { + offset, + size: size_raw, + limit, + }); + } + Ok(size_words) + } + + pub fn indirect_atom_bits(&mut self, offset: u64) -> Result { + let size_words = self.indirect_atom_words(offset)?; + let last_word = self.read_u64(offset + 2 + (size_words as u64 - 1))?; + let last_bits = 64usize.saturating_sub(last_word.leading_zeros() as usize); + Ok((size_words - 1).saturating_mul(64) + last_bits) + } +} + +#[derive(Clone, Copy, Debug)] +pub enum PmaRawNounKind { + Direct(u64), + Indirect { offset: PmaOffsetWords }, + Cell { offset: PmaOffsetWords }, +} + +pub fn classify_pma_noun(raw: u64) -> Result { + if raw & DIRECT_MASK == DIRECT_TAG { + return Ok(PmaRawNounKind::Direct(raw)); + } + if raw & INDIRECT_MASK == INDIRECT_TAG { + return offset_noun(raw, INDIRECT_MASK, "indirect"); + } + if raw & CELL_MASK == CELL_TAG { + return offset_noun(raw, CELL_MASK, "cell"); + } + if raw & CELL_MASK == CELL_MASK { + return Err(PmaDirectJamError::InvalidNoun { + raw, + reason: "forwarding pointer not supported", + }); + } + Err(PmaDirectJamError::InvalidNoun { + raw, + reason: "unknown noun tag", + }) +} + +fn offset_noun( + raw: u64, + mask: u64, + kind: &'static str, +) -> Result { + if raw & LOCATION_BIT == 0 { + return Err(PmaDirectJamError::InvalidNoun { + raw, + reason: "expected offset-form noun", + }); + } + let offset = raw & !(mask | LOCATION_BIT); + let offset = PmaOffsetWords::from_words(offset); + match kind { + "indirect" => Ok(PmaRawNounKind::Indirect { offset }), + "cell" => Ok(PmaRawNounKind::Cell { offset }), + _ => Err(PmaDirectJamError::InvalidNoun { + raw, + reason: "unknown offset noun type", + }), + } +} + +fn page_size() -> Result { + #[cfg(unix)] + { + let page = unsafe { libc::sysconf(libc::_SC_PAGESIZE) }; + if page <= 0 { + return Err(PmaDirectJamError::Io(io::Error::last_os_error())); + } + Ok(page as usize) + } + #[cfg(not(unix))] + { + Ok(4096) + } +} + +fn open_direct_file(path: &Path, require_direct_io: bool) -> Result { + let mut options = OpenOptions::new(); + options.read(true); + #[cfg(target_os = "linux")] + { + if require_direct_io { + options.custom_flags(libc::O_DIRECT); + } + } + let file = options.open(path).map_err(|err| { + if require_direct_io { + PmaDirectJamError::DirectIoOpen(err) + } else { + PmaDirectJamError::Io(err) + } + })?; + + #[cfg(target_os = "macos")] + { + if require_direct_io { + let rc = unsafe { libc::fcntl(file.as_raw_fd(), libc::F_NOCACHE, 1) }; + if rc != 0 { + return Err(PmaDirectJamError::DirectIoOpen(io::Error::last_os_error())); + } + } + } + + #[cfg(not(any(target_os = "linux", target_os = "macos")))] + { + if require_direct_io { + return Err(PmaDirectJamError::UnsupportedPlatform); + } + } + + Ok(file) +} diff --git a/crates/nockvm/rust/nockvm/src/serialization.rs b/crates/nockvm/rust/nockvm/src/serialization.rs index aab773119..6d5d8aad5 100644 --- a/crates/nockvm/rust/nockvm/src/serialization.rs +++ b/crates/nockvm/rust/nockvm/src/serialization.rs @@ -1,3 +1,5 @@ +use std::sync::Arc; + use bitvec::prelude::{BitSlice, Lsb0}; use either::Either::{Left, Right}; @@ -5,13 +7,14 @@ use crate::hamt::MutHamt; use crate::interpreter::Error::{self, *}; use crate::interpreter::Mote::*; use crate::mem::NockStack; -use crate::noun::{Atom, Cell, DirectAtom, IndirectAtom, Noun, D}; +use crate::noun::{Atom, Cell, CellHandle, DirectAtom, IndirectAtom, Noun, NounSpace, D}; crate::gdb!(); /// Calculate the number of bits needed to represent an atom -pub fn met0_usize(atom: Atom) -> usize { - let atom_bitslice = atom.as_bitslice(); +pub fn met0_usize(atom: Atom, space: &NounSpace) -> usize { + let atom_handle = atom.in_space(space); + let atom_bitslice = atom_handle.as_bitslice(); atom_bitslice.last_one().map_or(0, |last_one| last_one + 1) } @@ -100,14 +103,43 @@ enum CueStackEntry { /// # Arguments /// * `stack` - A mutable reference to the NockStack /// * `buffer` - A reference to a BitSlice containing the serialized noun +/// * `use_offset_tags` - If true, create nouns in offset form during deserialization. +/// When false, nouns are created in stack-pointer form. The returned noun is preserved into +/// the previous frame and returned in stack-pointer form either way. /// /// # Returns /// A Result containing either the deserialized Noun or an Error -pub fn cue_bitslice(stack: &mut NockStack, buffer: &BitSlice) -> Result { +/// +/// # Output form +/// +/// The return value is preserved into the previous frame using stack-pointer tags, so the +/// returned noun is in stack-pointer form. `use_offset_tags` only affects the temporary +/// representation during deserialization inside the current frame. +/// +/// If you need stack-pointer-form nouns without preserving into the previous frame, use +/// `cue_into_stack_pointer_form()`, which manually manages the frame. +/// +/// **Additional caveat with backrefs:** When the serialized data contains backreferences +/// (structural sharing), the result may have *mixed* tagging (some stack-pointer, some offset) +/// due to interactions between `unifying_equality` in the HAMT and the preserve step. +fn cue_bitslice_with_mode( + stack: &mut NockStack, + buffer: &BitSlice, + use_offset_tags: bool, +) -> Result { let backref_map = MutHamt::::new(stack); let mut result = D(0); let mut cursor = 0; + let space = if use_offset_tags { + let arena = Arc::clone(stack.arena()); + NounSpace::from_arenas(Some(Arc::clone(&arena)), Some(arena)) + } else { + stack.noun_space() + }; + // NOTE: with_frame() preserves into the previous frame using stack-pointer tags. The + // `use_offset_tags` parameter controls whether nouns are created in offset form or + // stack-pointer form during deserialization, before preservation. unsafe { stack.with_frame(0, |stack: &mut NockStack| { *(stack.push::()) = @@ -133,7 +165,13 @@ pub fn cue_bitslice(stack: &mut NockStack, buffer: &BitSlice) -> Resu } else { // 10 tag: cell let (cell, cell_mem_ptr) = Cell::new_raw_mut(stack); - *dest_ptr = cell.as_noun(); + let cell_noun = if use_offset_tags { + let offset = stack.offset_from_ptr(cell_mem_ptr as *const u8); + Cell::from_offset_words(offset).as_noun() + } else { + cell.as_noun() + }; + *dest_ptr = cell_noun; let mut backref_atom = Atom::new(stack, (cursor - 2) as u64).as_noun(); backref_map.insert(stack, &mut backref_atom, *dest_ptr); @@ -149,7 +187,10 @@ pub fn cue_bitslice(stack: &mut NockStack, buffer: &BitSlice) -> Resu } else { // 0 tag: atom let backref: u64 = (cursor - 1) as u64; - *dest_ptr = rub_atom(stack, &mut cursor, buffer)?.as_noun(); + *dest_ptr = rub_atom_internal( + stack, &mut cursor, buffer, use_offset_tags, &space, + )? + .as_noun(); let mut backref_atom = Atom::new(stack, backref).as_noun(); backref_map.insert(stack, &mut backref_atom, *dest_ptr); } @@ -164,6 +205,10 @@ pub fn cue_bitslice(stack: &mut NockStack, buffer: &BitSlice) -> Resu } } +pub fn cue_bitslice(stack: &mut NockStack, buffer: &BitSlice) -> Result { + cue_bitslice_with_mode(stack, buffer, false) +} + /// Deserialize a noun from an Atom /// /// This function is a wrapper around cue_bitslice that takes an Atom as input. @@ -175,8 +220,103 @@ pub fn cue_bitslice(stack: &mut NockStack, buffer: &BitSlice) -> Resu /// # Returns /// A Result containing either the deserialized Noun or an Error pub fn cue(stack: &mut NockStack, buffer: Atom) -> Result { - let buffer_bitslice = buffer.as_bitslice(); - cue_bitslice(stack, buffer_bitslice) + let space = stack.noun_space(); + let buffer_handle = buffer.in_space(&space); + let buffer_bitslice = buffer_handle.as_bitslice(); + cue_bitslice_with_mode(stack, buffer_bitslice, false) +} + +/// Deserialize a noun from a BitSlice. +/// +/// Offset tagging is reserved for PMA copy at event boundaries; cue_into_offset +/// returns stack-pointer nouns for now. +pub fn cue_bitslice_into_offset( + stack: &mut NockStack, + buffer: &BitSlice, +) -> Result { + cue_bitslice_with_mode(stack, buffer, false) +} + +/// Deserialize a noun from an Atom into offset-tagged form. +pub fn cue_into_offset(stack: &mut NockStack, buffer: Atom) -> Result { + let space = stack.noun_space(); + let buffer_handle = buffer.in_space(&space); + let buffer_bitslice = buffer_handle.as_bitslice(); + cue_bitslice_into_offset(stack, buffer_bitslice) +} + +/// Deserialize a noun from a BitSlice, keeping allocations in stack-pointer form. +/// +/// Unlike the regular `cue` function, this does NOT preserve the result to the +/// previous frame, so all nouns remain in stack-pointer form. This is useful +/// for benchmarking `retag_noun_tree` which expects stack-pointer form input. +/// +/// WARNING: The result is allocated in the current frame and will be invalid +/// after the frame is popped. Use this only when you need stack-pointer form +/// nouns and will not pop the frame. +pub fn cue_into_stack_pointer_form(stack: &mut NockStack, buffer: Atom) -> Result { + let backref_map = MutHamt::::new(stack); + let mut result = D(0); + let mut cursor = 0; + let space = stack.noun_space(); + let buffer_handle = buffer.in_space(&space); + let buffer_bitslice = buffer_handle.as_bitslice(); + + // Manually manage frame without the preserve step + unsafe { + stack.frame_push(0); + *(stack.push::()) = + CueStackEntry::DestinationPointer(&mut result as *mut Noun); + let res = loop { + if stack.stack_is_empty() { + break Ok(result); + } + let stack_entry = *stack.top::(); + stack.pop::(); + match stack_entry { + CueStackEntry::DestinationPointer(dest_ptr) => { + if next_bit(&mut cursor, buffer_bitslice) { + if next_bit(&mut cursor, buffer_bitslice) { + // 11 tag: backref + let mut backref_noun = + Atom::new(stack, rub_backref(&mut cursor, buffer_bitslice)?) + .as_noun(); + *dest_ptr = backref_map + .lookup(stack, &mut backref_noun) + .ok_or(Deterministic(Exit, D(0)))?; + } else { + // 10 tag: cell - always use stack-pointer form + let (cell, cell_mem_ptr) = Cell::new_raw_mut(stack); + *dest_ptr = cell.as_noun(); + let mut backref_atom = Atom::new(stack, (cursor - 2) as u64).as_noun(); + backref_map.insert(stack, &mut backref_atom, *dest_ptr); + *(stack.push()) = + CueStackEntry::BackRef(cursor as u64 - 2, dest_ptr as *const Noun); + *(stack.push()) = + CueStackEntry::DestinationPointer(&mut (*cell_mem_ptr).tail); + *(stack.push()) = + CueStackEntry::DestinationPointer(&mut (*cell_mem_ptr).head); + } + } else { + // 0 tag: atom - always use stack-pointer form + let backref: u64 = (cursor - 1) as u64; + *dest_ptr = + rub_atom_internal(stack, &mut cursor, buffer_bitslice, false, &space)? + .as_noun(); + let mut backref_atom = Atom::new(stack, backref).as_noun(); + backref_map.insert(stack, &mut backref_atom, *dest_ptr); + } + } + CueStackEntry::BackRef(backref, noun_ptr) => { + let mut backref_atom = Atom::new(stack, backref).as_noun(); + backref_map.insert(stack, &mut backref_atom, *noun_ptr) + } + } + }; + // Pop frame without preserve - nouns stay in stack-pointer form + stack.frame_pop(); + res + } } /// Get the size in bits of an encoded atom or backref @@ -223,6 +363,17 @@ fn rub_atom( stack: &mut NockStack, cursor: &mut usize, buffer: &BitSlice, +) -> Result { + let space = stack.noun_space(); + rub_atom_internal(stack, cursor, buffer, false, &space) +} + +fn rub_atom_internal( + stack: &mut NockStack, + cursor: &mut usize, + buffer: &BitSlice, + use_offset_tags: bool, + space: &NounSpace, ) -> Result { let size = get_size(cursor, buffer)?; let bits = next_up_to_n_bits(cursor, buffer, size); @@ -238,8 +389,12 @@ fn rub_atom( let wordsize = (size + 63) >> 6; let (mut atom, slice) = unsafe { IndirectAtom::new_raw_mut_bitslice(stack, wordsize) }; slice[0..bits.len()].copy_from_bitslice(bits); - debug_assert!(atom.size() > 0); - unsafe { Ok(atom.normalize_as_atom()) } + debug_assert!(atom.as_atom().in_space(space).size() > 0); + if use_offset_tags { + let offset = stack.offset_from_ptr(unsafe { atom.to_raw_pointer(space) } as *const u8); + atom = IndirectAtom::from_offset_words(offset); + } + unsafe { Ok(atom.normalize_as_atom(space)) } } } @@ -275,6 +430,7 @@ struct JamState<'a> { /// Implements a compact encoding scheme for nouns, with backreferences for shared structures. pub fn jam(stack: &mut NockStack, noun: Noun) -> Atom { let backref_map = MutHamt::new(stack); + let space = stack.noun_space(); let size = 8; let (atom, slice) = unsafe { IndirectAtom::new_raw_mut_bitslice(stack, size) }; let mut state = JamState { @@ -295,16 +451,16 @@ pub fn jam(stack: &mut NockStack, noun: Noun) -> Atom { if let Some(backref) = backref_map.lookup(stack, &mut noun) { match noun.as_either_atom_cell() { Left(atom) => { - let atom_size = met0_usize(atom); + let atom_size = met0_usize(atom, &space); let backref_size = met0_u64_to_usize(backref); if atom_size <= backref_size { - jam_atom(stack, &mut state, atom); + jam_atom(stack, &mut state, atom, &space); } else { - jam_backref(stack, &mut state, backref); + jam_backref(stack, &mut state, backref, &space); } } Right(_cell) => { - jam_backref(stack, &mut state, backref); + jam_backref(stack, &mut state, backref, &space); } } unsafe { @@ -315,7 +471,7 @@ pub fn jam(stack: &mut NockStack, noun: Noun) -> Atom { backref_map.insert(stack, &mut noun, state.cursor as u64); match noun.as_either_atom_cell() { Left(atom) => { - jam_atom(stack, &mut state, atom); + jam_atom(stack, &mut state, atom, &space); unsafe { stack.pop::(); }; @@ -324,9 +480,10 @@ pub fn jam(stack: &mut NockStack, noun: Noun) -> Atom { Right(cell) => { jam_cell(stack, &mut state); unsafe { + let cell_handle = CellHandle::new(cell, &space); stack.pop::(); - *(stack.push::()) = cell.tail(); - *(stack.push::()) = cell.head(); + *(stack.push::()) = cell_handle.tail().noun(); + *(stack.push::()) = cell_handle.head().noun(); }; continue; } @@ -334,7 +491,7 @@ pub fn jam(stack: &mut NockStack, noun: Noun) -> Atom { } } unsafe { - let mut result = state.atom.normalize_as_atom(); + let mut result = state.atom.normalize_as_atom(&space); stack.preserve(&mut result); stack.frame_pop(); result @@ -342,7 +499,7 @@ pub fn jam(stack: &mut NockStack, noun: Noun) -> Atom { } /// Serialize an atom into the jam state -fn jam_atom(traversal: &mut NockStack, state: &mut JamState, atom: Atom) { +fn jam_atom(traversal: &mut NockStack, state: &mut JamState, atom: Atom, space: &NounSpace) { loop { if state.cursor + 1 > state.slice.len() { double_atom_size(traversal, state); @@ -353,7 +510,7 @@ fn jam_atom(traversal: &mut NockStack, state: &mut JamState, atom: Atom) { state.slice.set(state.cursor, false); // 0 tag for atom state.cursor += 1; loop { - if let Ok(()) = mat(traversal, state, atom) { + if let Ok(()) = mat(traversal, state, atom, space) { break; } else { double_atom_size(traversal, state); @@ -376,7 +533,7 @@ fn jam_cell(traversal: &mut NockStack, state: &mut JamState) { } /// Serialize a backreference into the jam state -fn jam_backref(traversal: &mut NockStack, state: &mut JamState, backref: u64) { +fn jam_backref(traversal: &mut NockStack, state: &mut JamState, backref: u64, space: &NounSpace) { loop { if state.cursor + 2 > state.slice.len() { double_atom_size(traversal, state); @@ -389,7 +546,7 @@ fn jam_backref(traversal: &mut NockStack, state: &mut JamState, backref: u64) { state.cursor += 2; let backref_atom = Atom::new(traversal, backref); loop { - if let Ok(()) = mat(traversal, state, backref_atom) { + if let Ok(()) = mat(traversal, state, backref_atom, space) { break; } else { double_atom_size(traversal, state); @@ -410,8 +567,13 @@ fn double_atom_size(traversal: &mut NockStack, state: &mut JamState) { /// Encode an atom's size and value into the jam state /// /// INVARIANT: mat must not modify state.cursor unless it will also return `Ok(())` -fn mat(traversal: &mut NockStack, state: &mut JamState, atom: Atom) -> Result<(), ()> { - let b_atom_size = met0_usize(atom); +fn mat( + traversal: &mut NockStack, + state: &mut JamState, + atom: Atom, + space: &NounSpace, +) -> Result<(), ()> { + let b_atom_size = met0_usize(atom, space); let b_atom_size_atom = Atom::new(traversal, b_atom_size as u64); if b_atom_size == 0 { if state.cursor + 1 > state.slice.len() { @@ -422,17 +584,19 @@ fn mat(traversal: &mut NockStack, state: &mut JamState, atom: Atom) -> Result<() Ok(()) } } else { - let c_b_size = met0_usize(b_atom_size_atom); + let c_b_size = met0_usize(b_atom_size_atom, space); if state.cursor + c_b_size + c_b_size + b_atom_size > state.slice.len() { Err(()) } else { state.slice[state.cursor..state.cursor + c_b_size].fill(false); // a 0 bit for each bit in the atom size state.slice.set(state.cursor + c_b_size, true); // a terminating 1 bit state.slice[state.cursor + c_b_size + 1..state.cursor + c_b_size + c_b_size] - .copy_from_bitslice(&b_atom_size_atom.as_bitslice()[0..c_b_size - 1]); // the atom size excepting the most significant 1 (since we know where that is from the size-of-the-size) + .copy_from_bitslice( + &b_atom_size_atom.in_space(space).as_bitslice()[0..c_b_size - 1], + ); // the atom size excepting the most significant 1 (since we know where that is from the size-of-the-size) state.slice[state.cursor + c_b_size + c_b_size ..state.cursor + c_b_size + c_b_size + b_atom_size] - .copy_from_bitslice(&atom.as_bitslice()[0..b_atom_size]); + .copy_from_bitslice(&atom.in_space(space).as_bitslice()[0..b_atom_size]); state.cursor += c_b_size + c_b_size + b_atom_size; Ok(()) } @@ -448,10 +612,10 @@ mod tests { use super::*; use crate::jets::util::test::assert_noun_eq; - use crate::mem::NockStack; - use crate::noun::{Atom, Cell, CellMemory, Noun}; + use crate::mem::{NockStack, NOCK_STACK_SIZE, NOCK_STACK_SIZE_TINY}; + use crate::noun::{AllocLocation, Atom, Cell, CellMemory, Noun}; fn setup_stack() -> NockStack { - NockStack::new(1 << 30, 0) + NockStack::new(NOCK_STACK_SIZE, 0) } #[test] @@ -471,6 +635,23 @@ mod tests { assert_noun_eq(&mut stack, cued, atom.as_noun()); } + #[test] + #[cfg_attr(miri, ignore)] + fn test_jam_cue_into_offset_atom() { + let mut stack = setup_stack(); + let atom = Atom::new(&mut stack, 42); + let jammed = jam(&mut stack, atom.as_noun()); + let cued = cue_into_offset(&mut stack, jammed).unwrap_or_else(|err| { + panic!( + "Panicked with {err:?} at {}:{} (git sha: {:?})", + file!(), + line!(), + option_env!("GIT_SHA") + ) + }); + assert_noun_eq(&mut stack, cued, atom.as_noun()); + } + #[test] #[cfg_attr(miri, ignore)] fn test_jam_cue_cell() { @@ -490,6 +671,25 @@ mod tests { assert_noun_eq(&mut stack, cued, cell); } + #[test] + #[cfg_attr(miri, ignore)] + fn test_jam_cue_into_offset_cell() { + let mut stack = setup_stack(); + let n1 = Atom::new(&mut stack, 1).as_noun(); + let n2 = Atom::new(&mut stack, 2).as_noun(); + let cell = Cell::new(&mut stack, n1, n2).as_noun(); + let jammed = jam(&mut stack, cell); + let cued = cue_into_offset(&mut stack, jammed).unwrap_or_else(|err| { + panic!( + "Panicked with {err:?} at {}:{} (git sha: {:?})", + file!(), + line!(), + option_env!("GIT_SHA") + ) + }); + assert_noun_eq(&mut stack, cued, cell); + } + #[test] #[cfg_attr(miri, ignore)] fn test_jam_cue_nested_cell() { @@ -511,6 +711,27 @@ mod tests { assert_noun_eq(&mut stack, cued, outer_cell.as_noun()); } + #[test] + #[cfg_attr(miri, ignore)] + fn test_jam_cue_into_offset_nested_cell() { + let mut stack = setup_stack(); + let n3 = Atom::new(&mut stack, 3).as_noun(); + let n4 = Atom::new(&mut stack, 4).as_noun(); + let inner_cell = Cell::new(&mut stack, n3, n4); + let n1 = Atom::new(&mut stack, 1).as_noun(); + let outer_cell = Cell::new(&mut stack, n1, inner_cell.as_noun()); + let jammed = jam(&mut stack, outer_cell.as_noun()); + let cued = cue_into_offset(&mut stack, jammed).unwrap_or_else(|err| { + panic!( + "Panicked with {err:?} at {}:{} (git sha: {:?})", + file!(), + line!(), + option_env!("GIT_SHA") + ) + }); + assert_noun_eq(&mut stack, cued, outer_cell.as_noun()); + } + #[test] #[cfg_attr(miri, ignore)] fn test_jam_cue_shared_structure() { @@ -529,6 +750,24 @@ mod tests { assert_noun_eq(&mut stack, cued, cell.as_noun()); } + #[test] + #[cfg_attr(miri, ignore)] + fn test_jam_cue_into_offset_shared_structure() { + let mut stack = setup_stack(); + let shared_atom = Atom::new(&mut stack, 42); + let cell = Cell::new(&mut stack, shared_atom.as_noun(), shared_atom.as_noun()); + let jammed = jam(&mut stack, cell.as_noun()); + let cued = cue_into_offset(&mut stack, jammed).unwrap_or_else(|err| { + panic!( + "Panicked with {err:?} at {}:{} (git sha: {:?})", + file!(), + line!(), + option_env!("GIT_SHA") + ) + }); + assert_noun_eq(&mut stack, cued, cell.as_noun()); + } + #[test] #[cfg_attr(miri, ignore)] fn test_jam_cue_large_atom() { @@ -546,6 +785,23 @@ mod tests { assert_noun_eq(&mut stack, cued, large_atom.as_noun()); } + #[test] + #[cfg_attr(miri, ignore)] + fn test_jam_cue_into_offset_large_atom() { + let mut stack = setup_stack(); + let large_atom = Atom::new(&mut stack, u64::MAX); + let jammed = jam(&mut stack, large_atom.as_noun()); + let cued = cue_into_offset(&mut stack, jammed).unwrap_or_else(|err| { + panic!( + "Panicked with {err:?} at {}:{} (git sha: {:?})", + file!(), + line!(), + option_env!("GIT_SHA") + ) + }); + assert_noun_eq(&mut stack, cued, large_atom.as_noun()); + } + #[test] #[cfg_attr(miri, ignore)] fn test_jam_cue_empty_atom() { @@ -563,6 +819,23 @@ mod tests { assert_noun_eq(&mut stack, cued, empty_atom.as_noun()); } + #[test] + #[cfg_attr(miri, ignore)] + fn test_jam_cue_into_offset_empty_atom() { + let mut stack = setup_stack(); + let empty_atom = Atom::new(&mut stack, 0); + let jammed = jam(&mut stack, empty_atom.as_noun()); + let cued = cue_into_offset(&mut stack, jammed).unwrap_or_else(|err| { + panic!( + "Panicked with {err:?} at {}:{} (git sha: {:?})", + file!(), + line!(), + option_env!("GIT_SHA") + ) + }); + assert_noun_eq(&mut stack, cued, empty_atom.as_noun()); + } + #[test] #[cfg_attr(miri, ignore)] fn test_jam_cue_complex_structure() { @@ -584,6 +857,27 @@ mod tests { assert_noun_eq(&mut stack, cued, cell3.as_noun()); } + #[test] + #[cfg_attr(miri, ignore)] + fn test_jam_cue_into_offset_complex_structure() { + let mut stack = setup_stack(); + let atom1 = Atom::new(&mut stack, 1); + let atom2 = Atom::new(&mut stack, 2); + let cell1 = Cell::new(&mut stack, atom1.as_noun(), atom2.as_noun()); + let cell2 = Cell::new(&mut stack, cell1.as_noun(), atom2.as_noun()); + let cell3 = Cell::new(&mut stack, cell2.as_noun(), cell1.as_noun()); + let jammed = jam(&mut stack, cell3.as_noun()); + let cued = cue_into_offset(&mut stack, jammed).unwrap_or_else(|err| { + panic!( + "Panicked with {err:?} at {}:{} (git sha: {:?})", + file!(), + line!(), + option_env!("GIT_SHA") + ) + }); + assert_noun_eq(&mut stack, cued, cell3.as_noun()); + } + #[test] #[cfg_attr(miri, ignore)] fn test_cue_invalid_input() { @@ -593,6 +887,15 @@ mod tests { assert!(result.is_err()); } + #[test] + #[cfg_attr(miri, ignore)] + fn test_cue_into_offset_invalid_input() { + let mut stack = setup_stack(); + let invalid_atom = Atom::new(&mut stack, 0b11); // Invalid tag + let result = cue_into_offset(&mut stack, invalid_atom); + assert!(result.is_err()); + } + #[test] #[cfg_attr(miri, ignore)] fn test_jam_cue_roundtrip_property() { @@ -601,6 +904,7 @@ mod tests { println!("Testing noun with depth: {}", depth); let mut stack = setup_stack(); + let space = stack.noun_space(); let mut rng_clone = rng.clone(); let (original, total_size) = generate_deeply_nested_noun(&mut stack, depth, &mut rng_clone); @@ -608,11 +912,14 @@ mod tests { "Total size of all generated nouns: {:.2} KB", total_size as f64 / 1024.0 ); - println!("Original size: {:.2} KB", original.mass() as f64 / 1024.0); + println!( + "Original size: {:.2} KB", + original.mass(&space) as f64 / 1024.0 + ); let jammed = jam(&mut stack, original); println!( "Jammed size: {:.2} KB", - jammed.as_noun().mass() as f64 / 1024.0 + jammed.as_noun().mass(&space) as f64 / 1024.0 ); let cued = cue(&mut stack, jammed).unwrap_or_else(|err| { panic!( @@ -622,7 +929,45 @@ mod tests { option_env!("GIT_SHA") ) }); - println!("Cued size: {:.2} KB", cued.mass() as f64 / 1024.0); + println!("Cued size: {:.2} KB", cued.mass(&space) as f64 / 1024.0); + + assert_noun_eq(&mut stack, cued, original); + } + + #[test] + #[cfg_attr(miri, ignore)] + fn test_jam_cue_into_offset_roundtrip_property() { + let rng = StdRng::seed_from_u64(1); + let depth = 9; + println!("Testing noun with depth: {}", depth); + + let mut stack = setup_stack(); + let space = stack.noun_space(); + let mut rng_clone = rng.clone(); + let (original, total_size) = generate_deeply_nested_noun(&mut stack, depth, &mut rng_clone); + + println!( + "Total size of all generated nouns: {:.2} KB", + total_size as f64 / 1024.0 + ); + println!( + "Original size: {:.2} KB", + original.mass(&space) as f64 / 1024.0 + ); + let jammed = jam(&mut stack, original); + println!( + "Jammed size: {:.2} KB", + jammed.as_noun().mass(&space) as f64 / 1024.0 + ); + let cued = cue_into_offset(&mut stack, jammed).unwrap_or_else(|err| { + panic!( + "Panicked with {err:?} at {}:{} (git sha: {:?})", + file!(), + line!(), + option_env!("GIT_SHA") + ) + }); + println!("Cued size: {:.2} KB", cued.mass(&space) as f64 / 1024.0); assert_noun_eq(&mut stack, cued, original); } @@ -638,6 +983,7 @@ mod tests { depth: usize, accumulated_size: usize, ) -> (Noun, usize) { + let space = stack.noun_space(); let mut done = false; if depth >= MAX_DEPTH || stack.size() < 1024 || accumulated_size > stack.size() - 1024 { // println!("Done at depth and size: {} {:.2} KB", depth, accumulated_size as f64 / 1024.0); @@ -648,20 +994,20 @@ mod tests { let value = rng.random::(); let atom = Atom::new(stack, value); let noun = atom.as_noun(); - (noun, accumulated_size + noun.mass()) + (noun, accumulated_size + noun.mass(&space)) } else { let (left, left_size) = inner(stack, bits / 2, rng, depth + 1, accumulated_size); let (right, _) = inner(stack, bits / 2, rng, depth + 1, left_size); let cell = Cell::new(stack, left, right); let noun = cell.as_noun(); - (noun, noun.mass()) + (noun, noun.mass(&space)) }; if space_needed_noun(result.0, stack) > stack.size() { eprintln!( "Stack size exceeded with noun size {:.2} KB", - result.0.mass() as f64 / 1024.0 + result.0.mass(&space) as f64 / 1024.0 ); unsafe { let top_noun = *stack.top::(); @@ -679,6 +1025,7 @@ mod tests { unsafe { stack.with_frame(0, |stack| { *(stack.push::()) = noun; + let space = stack.noun_space(); let mut size = 0; while !stack.stack_is_empty() { let noun = *(stack.top::()); @@ -687,13 +1034,14 @@ mod tests { Left(atom) => match atom.as_either() { Left(_) => {} Right(indirect) => { - size += indirect.raw_size(); + size += indirect.raw_size(&space); } }, Right(cell) => { size += size_of::(); - *(stack.push::()) = cell.tail(); - *(stack.push::()) = cell.head(); + let cell_handle = CellHandle::new(cell, &space); + *(stack.push::()) = cell_handle.tail().noun(); + *(stack.push::()) = cell_handle.head().noun(); } } } @@ -715,20 +1063,21 @@ mod tests { let (right, right_size) = generate_deeply_nested_noun(stack, depth - 1, rng); let cell = Cell::new(stack, left, right); let noun = cell.as_noun(); - let total_size = left_size + right_size + noun.mass(); + let space = stack.noun_space(); + let total_size = left_size + right_size + noun.mass(&space); if { space_needed_noun(noun, stack) } > stack.size() { eprintln!( "Stack size exceeded at depth {} with noun size {:.2} KB", depth, - noun.mass() as f64 / 1024.0 + noun.mass(&space) as f64 / 1024.0 ); unsafe { let top_noun = *stack.top::(); (top_noun, total_size) } } else { - // println!("Size: {:.2} KB, depth: {}", noun.mass() as f64 / 1024.0, depth); + // println!("Size: {:.2} KB, depth: {}", noun.mass(&space) as f64 / 1024.0, depth); (noun, total_size) } } @@ -750,10 +1099,26 @@ mod tests { } } + #[test] + #[cfg_attr(miri, ignore)] + fn test_cue_into_offset_invalid_backreference() { + std::env::set_var("RUST_BACKTRACE", "full"); + + let mut stack = setup_stack(); + let invalid_atom = Atom::new(&mut stack, 0b11); // Invalid atom representation + let result = cue_into_offset(&mut stack, invalid_atom); + + assert!(result.is_err()); + if let Err(e) = result { + println!("Error: {:?}", e); + assert!(matches!(e, Error::Deterministic(_, _))); + } + } + #[ignore] // We will put this back when we have proper error catching #[test] fn test_cue_nondeterministic_error() { - let mut big_stack = NockStack::new(1 << 30, 0); + let mut big_stack = NockStack::new(NOCK_STACK_SIZE, 0); let mut rng = StdRng::seed_from_u64(1); @@ -762,10 +1127,11 @@ mod tests { // Attempt to jam and then cue the large atom in the big stack let jammed = jam(&mut big_stack, large_atom); + let jam_space = big_stack.noun_space(); // make a smaller stack to try to cause a nondeterministic error // NOTE: if the stack is big enough to fit the jammed atom, cue panics - let mut stack = NockStack::new(jammed.as_noun().mass() / 2_usize, 0); + let mut stack = NockStack::new(jammed.as_noun().mass(&jam_space) / 2_usize, 0); // Attempt to cue the jammed noun with limited stack space let result: Result<(), Error> = match cue(&mut stack, jammed) { @@ -784,19 +1150,300 @@ mod tests { } } + #[ignore] // We will put this back when we have proper error catching + #[test] + fn test_cue_into_offset_nondeterministic_error() { + let mut big_stack = NockStack::new(NOCK_STACK_SIZE, 0); + + let mut rng = StdRng::seed_from_u64(1); + + // Create an atom with a very large value to potentially cause overflow + let (large_atom, _) = generate_deeply_nested_noun(&mut big_stack, 5, &mut rng); + + // Attempt to jam and then cue the large atom in the big stack + let jammed = jam(&mut big_stack, large_atom); + let jam_space = big_stack.noun_space(); + + // make a smaller stack to try to cause a nondeterministic error + // NOTE: if the stack is big enough to fit the jammed atom, cue panics + let mut stack = NockStack::new(jammed.as_noun().mass(&jam_space) / 2_usize, 0); + + // Attempt to cue the jammed noun with limited stack space + let result: Result<(), Error> = match cue_into_offset(&mut stack, jammed) { + Ok(_res) => { + panic!("Unexpected success: cue operation did not fail"); + } + Err(e) => Err(e), + }; + + // Check if we got a nondeterministic error + println!("Result: {:?}", result); + assert!(result.is_err()); + if let Err(e) = result { + assert!(matches!(e, Error::NonDeterministic(_, _))); + println!("got expected error: {:?}", e); + } + } + #[test] #[cfg_attr(miri, ignore)] fn test_cell_construction() { let mut stack = setup_stack(); + let space = stack.noun_space(); let (cell, cell_mem_ptr) = unsafe { Cell::new_raw_mut(&mut stack) }; - unsafe { assert!(std::ptr::eq(cell_mem_ptr, cell.to_raw_pointer())) }; + unsafe { assert!(std::ptr::eq(cell_mem_ptr, cell.to_raw_pointer(&space))) }; + } + + /// Helper to check if a noun tree is entirely in stack-pointer form + fn is_entirely_stack_pointer_form(stack: &NockStack, root: Noun) -> bool { + use std::collections::HashSet; + let space = stack.noun_space(); + let mut work: Vec = Vec::with_capacity(32); + let mut visited: HashSet = HashSet::new(); + work.push(root); + + while let Some(noun) = work.pop() { + if noun.is_direct() { + continue; + } + let raw = unsafe { noun.as_raw() }; + if !visited.insert(raw) { + continue; + } + if noun.is_allocated() + && !matches!( + noun.in_space(&space).allocated_location(), + Some(AllocLocation::Stack) + ) + { + return false; + } + if let Ok(cell) = noun.in_space(&space).as_cell() { + work.push(cell.head().noun()); + work.push(cell.tail().noun()); + } + } + true + } + + /// Helper to check if a noun tree is entirely in offset form + fn is_entirely_offset_form(stack: &NockStack, root: Noun) -> bool { + use std::collections::HashSet; + let space = stack.noun_space(); + let mut work: Vec = Vec::with_capacity(32); + let mut visited: HashSet = HashSet::new(); + work.push(root); + + while let Some(noun) = work.pop() { + if noun.is_direct() { + continue; + } + let raw = unsafe { noun.as_raw() }; + if !visited.insert(raw) { + continue; + } + if noun.is_allocated() + && matches!( + noun.in_space(&space).allocated_location(), + Some(AllocLocation::Stack) + ) + { + return false; + } + if let Ok(cell) = noun.in_space(&space).as_cell() { + work.push(cell.head().noun()); + work.push(cell.tail().noun()); + } + } + true + } + + /// Test that cue() produces stack-pointer-form nouns. + #[test] + #[cfg_attr(miri, ignore)] + fn test_cue_produces_stack_pointer_form() { + let mut stack = setup_stack(); + + // Create a simple cell [1 2] + let head = D(1); + let tail = D(2); + let cell = Cell::new(&mut stack, head, tail); + + // Jam it + let jammed = jam(&mut stack, cell.as_noun()); + + // Cue it back using regular cue (which internally uses use_offset_tags=false) + let cued = cue(&mut stack, jammed).expect("cue should succeed"); + + // The result should be in stack-pointer form + assert!( + is_entirely_stack_pointer_form(&stack, cued), + "cue() should produce stack-pointer-form nouns" + ); + } + + /// Test that cue_into_offset() returns stack-pointer-form nouns after preserve + #[test] + #[cfg_attr(miri, ignore)] + fn test_cue_into_offset_produces_stack_pointer_form() { + let mut stack = setup_stack(); + + // Create a simple cell [1 2] + let head = D(1); + let tail = D(2); + let cell = Cell::new(&mut stack, head, tail); + + // Jam it + let jammed = jam(&mut stack, cell.as_noun()); + + // Cue it back using cue_into_offset + let cued = cue_into_offset(&mut stack, jammed).expect("cue_into_offset should succeed"); + + // The result should be in stack-pointer form + assert!( + is_entirely_stack_pointer_form(&stack, cued), + "cue_into_offset() should produce stack-pointer-form nouns after preserve" + ); + } + + /// Test that cue_into_stack_pointer_form() produces stack-pointer-form nouns + #[test] + #[cfg_attr(miri, ignore)] + fn test_cue_into_stack_pointer_form_produces_stack_pointer_form() { + let mut stack = setup_stack(); + + // Create a simple cell [1 2] + let head = D(1); + let tail = D(2); + let cell = Cell::new(&mut stack, head, tail); + + // Jam it + let jammed = jam(&mut stack, cell.as_noun()); + + // Cue it back using cue_into_stack_pointer_form + let cued = cue_into_stack_pointer_form(&mut stack, jammed) + .expect("cue_into_stack_pointer_form should succeed"); + + // The result should be in stack-pointer form + assert!( + is_entirely_stack_pointer_form(&stack, cued), + "cue_into_stack_pointer_form() should produce stack-pointer-form nouns" + ); + } + + /// Test with a more complex structure including indirect atoms + #[test] + #[cfg_attr(miri, ignore)] + fn test_cue_tagging_with_indirect_atoms() { + let mut stack = setup_stack(); + + // Create a structure with an indirect atom (value > DIRECT_MAX) + let big_value: u64 = 0x8000_0000_0000_0000; // This requires indirect representation + let indirect_atom = unsafe { IndirectAtom::new_raw(&mut stack, 1, &big_value) }; + let cell = Cell::new(&mut stack, indirect_atom.as_noun(), D(42)); + + // Jam it + let jammed = jam(&mut stack, cell.as_noun()); + + // Test cue() produces stack-pointer form + let cued_stack = cue(&mut stack, jammed).expect("cue should succeed"); + assert!( + is_entirely_stack_pointer_form(&stack, cued_stack), + "cue() should produce stack-pointer-form nouns even with indirect atoms" + ); + + // Test cue_into_stack_pointer_form() produces stack-pointer form + let cued_stack = cue_into_stack_pointer_form(&mut stack, jammed) + .expect("cue_into_stack_pointer_form should succeed"); + assert!( + is_entirely_stack_pointer_form(&stack, cued_stack), + "cue_into_stack_pointer_form() should produce stack-pointer-form nouns with indirect atoms" + ); + } + + /// Helper to count stack-pointer and offset form nouns + fn count_noun_tagging(stack: &NockStack, root: Noun) -> (usize, usize) { + use std::collections::HashSet; + let space = stack.noun_space(); + let mut work: Vec = Vec::with_capacity(32); + let mut visited: HashSet = HashSet::new(); + let mut stack_pointer_count = 0usize; + let mut offset_count = 0usize; + work.push(root); + + while let Some(noun) = work.pop() { + if noun.is_direct() { + continue; + } + let raw = unsafe { noun.as_raw() }; + if !visited.insert(raw) { + continue; + } + if noun.is_allocated() { + if matches!( + noun.in_space(&space).allocated_location(), + Some(AllocLocation::Stack) + ) { + stack_pointer_count += 1; + } else { + offset_count += 1; + } + } + if let Ok(cell) = noun.in_space(&space).as_cell() { + work.push(cell.head().noun()); + work.push(cell.tail().noun()); + } + } + (stack_pointer_count, offset_count) + } + + /// Test with structural sharing (backrefs) + #[test] + #[cfg_attr(miri, ignore)] + fn test_cue_tagging_with_backrefs() { + let mut stack = setup_stack(); + + // Create a structure with sharing: [[1 2] [1 2]] + // The inner [1 2] should be shared via backref + let inner = Cell::new(&mut stack, D(1), D(2)); + let outer = Cell::new(&mut stack, inner.as_noun(), inner.as_noun()); + + // Jam it (this will use backrefs for the shared inner cell) + let jammed = jam(&mut stack, outer.as_noun()); + + // Test cue() - check what tagging we get + let cued_offset = cue(&mut stack, jammed).expect("cue should succeed"); + let (stack_count, offset_count) = count_noun_tagging(&stack, cued_offset); + println!( + "cue() with backrefs: {} stack-pointer, {} offset", + stack_count, offset_count + ); + // Note: Due to how preserve works with backrefs, we might get mixed results + // The important thing is that the structure is valid and can be traversed + + // Test cue_into_stack_pointer_form() produces stack-pointer form + let cued_stack = cue_into_stack_pointer_form(&mut stack, jammed) + .expect("cue_into_stack_pointer_form should succeed"); + let (stack_count2, offset_count2) = count_noun_tagging(&stack, cued_stack); + println!( + "cue_into_stack_pointer_form() with backrefs: {} stack-pointer, {} offset", + stack_count2, offset_count2 + ); + assert!( + is_entirely_stack_pointer_form(&stack, cued_stack), + "cue_into_stack_pointer_form() should produce stack-pointer-form nouns with backrefs, \ + but got {} stack-pointer and {} offset", + stack_count2, + offset_count2 + ); } #[test] + #[cfg_attr(miri, ignore = "memfd_create unsupported in Miri")] fn test_cyclic_structure() { use bitvec::prelude::*; - let mut stack = NockStack::new(1000, 0); + let mut stack = NockStack::new(NOCK_STACK_SIZE_TINY, 0); // Create a jammed representation of a cyclic structure // [0 *] where * refers back to the entire cell, i.e. 0b11110001 diff --git a/crates/nockvm/rust/nockvm/src/site.rs b/crates/nockvm/rust/nockvm/src/site.rs index da8cd70bd..4e0779826 100644 --- a/crates/nockvm/rust/nockvm/src/site.rs +++ b/crates/nockvm/rust/nockvm/src/site.rs @@ -21,7 +21,8 @@ pub struct Site { impl Site { /// Prepare a locally cached gate to call repeatedly. pub fn new(ctx: &mut Context, core: &mut Noun) -> Site { - let mut battery = slot(*core, 2).unwrap_or_else(|err| { + let space = ctx.stack.noun_space(); + let mut battery = slot(*core, 2, &space).unwrap_or_else(|err| { panic!( "Panicked with {err:?} at {}:{} (git sha: {:?})", file!(), @@ -29,7 +30,7 @@ impl Site { option_env!("GIT_SHA") ) }); - let context = slot(*core, 7).unwrap_or_else(|err| { + let context = slot(*core, 7, &space).unwrap_or_else(|err| { panic!( "Panicked with {err:?} at {}:{} (git sha: {:?})", file!(), @@ -53,7 +54,8 @@ impl Site { let mut ret = true; for mut batteries in batteries_list { if let Some((_battery, parent_axis)) = batteries.next() { - let parent_axis_prefix_bits = &parent_axis.as_bitslice()[0..3]; + let parent_axis_handle = parent_axis.in_space(&space); + let parent_axis_prefix_bits = &parent_axis_handle.as_bitslice()[0..3]; if parent_axis_prefix_bits == axis_7_bits { continue; } else { diff --git a/crates/nockvm/rust/nockvm/src/substantive/convert.rs b/crates/nockvm/rust/nockvm/src/substantive/convert.rs deleted file mode 100644 index f7326cc4d..000000000 --- a/crates/nockvm/rust/nockvm/src/substantive/convert.rs +++ /dev/null @@ -1,388 +0,0 @@ -use super::*; -use crate::noun::{ - Atom, Cell, IndirectAtom, Noun, NounAllocator, DIRECT_MASK, DIRECT_TAG, INDIRECT_TAG, -}; - -pub trait IntoArena { - fn into_arena(self, arena: &mut NounArena) -> NounKey; -} - -pub trait FromArena { - fn from_arena(arena: &NounArena, key: NounKey, alloc: &mut impl NounAllocator) -> Self; -} - -impl IntoArena for Noun { - fn into_arena(self, arena: &mut NounArena) -> NounKey { - // Use raw value to check type since we need to preserve tag bits - unsafe { - if self.raw & DIRECT_MASK == DIRECT_TAG { - // Direct atom - arena.add_direct(self.raw) // Keep raw value with tag bits - } else if self.raw & !(u64::MAX >> 2) == INDIRECT_TAG { - // Indirect atom - let indirect = self.indirect().unwrap_or_else(|| { - panic!( - "Panicked at {}:{} (git sha: {:?})", - file!(), - line!(), - option_env!("GIT_SHA") - ) - }); - let size = indirect.size(); - let mut words = Vec::with_capacity(size); - let src = indirect.data_pointer(); - for i in 0..size { - words.push(*src.add(i)); - } - arena.add_indirect(words) - } else { - // Cell - let cell = self.cell().unwrap_or_else(|| { - panic!( - "Panicked at {}:{} (git sha: {:?})", - file!(), - line!(), - option_env!("GIT_SHA") - ) - }); - let head = cell.head().into_arena(arena); - let tail = cell.tail().into_arena(arena); - arena.add_cell(head, tail) - } - } - } -} - -impl IntoArena for Atom { - fn into_arena(self, arena: &mut NounArena) -> NounKey { - unsafe { - if self.raw & DIRECT_MASK == DIRECT_TAG { - // Preserve raw value including tag bits - arena.add_direct(self.raw) - } else { - // Indirect atom - let indirect = self.indirect().unwrap_or_else(|| { - panic!( - "Panicked at {}:{} (git sha: {:?})", - file!(), - line!(), - option_env!("GIT_SHA") - ) - }); - let size = indirect.size(); - let mut words = Vec::with_capacity(size); - let src = indirect.data_pointer(); - for i in 0..size { - words.push(*src.add(i)); - } - arena.add_indirect(words) - } - } - } -} - -impl FromArena for Noun { - fn from_arena(arena: &NounArena, key: NounKey, alloc: &mut impl NounAllocator) -> Self { - match arena.get_node(key) { - Node::Direct(value) => { - // Direct atoms come with their tag bits preserved - Noun { raw: *value } - } - Node::Indirect(words) => { - // Need to create a proper indirect atom with correct tag - unsafe { IndirectAtom::new_raw(alloc, words.len(), words.as_ptr()).as_noun() } - } - Node::Cell(head, tail) => { - // Create cell with proper tag bits - let head_noun = Noun::from_arena(arena, *head, alloc); - let tail_noun = Noun::from_arena(arena, *tail, alloc); - Cell::new(alloc, head_noun, tail_noun).as_noun() - } - } - } -} - -impl FromArena for Atom { - fn from_arena(arena: &NounArena, key: NounKey, alloc: &mut impl NounAllocator) -> Self { - match arena.get_node(key) { - Node::Direct(value) => { - // Direct atoms preserve their tag bits - Atom { raw: *value } - } - Node::Indirect(words) => unsafe { - IndirectAtom::new_raw(alloc, words.len(), words.as_ptr()).as_atom() - }, - Node::Cell(..) => panic!("Cannot convert Cell to Atom"), - } - } -} - -impl NounArena { - pub fn import_noun(&mut self, noun: Noun) -> NounKey { - noun.into_arena(self) - } - - pub fn export_noun(&self, key: NounKey, alloc: &mut impl NounAllocator) -> Noun { - Noun::from_arena(self, key, alloc) - } - - pub fn import_atom(&mut self, atom: Atom) -> NounKey { - atom.into_arena(self) - } - - pub fn export_atom(&self, key: NounKey, alloc: &mut impl NounAllocator) -> Result { - match self.get_node(key) { - Node::Direct(_) | Node::Indirect(_) => Ok(Atom::from_arena(self, key, alloc)), - Node::Cell(..) => Err(NounError::NotCell), - } - } -} - -unsafe fn noun_semantically_equal(a: &Noun, b: &Noun) -> bool { - // If they're exactly equal (including pointer bits), great - if a.raw_equals(b) { - return true; - } - - // For direct atoms, only value matters - if a.is_direct() && b.is_direct() { - return a - .direct() - .unwrap_or_else(|| { - panic!( - "Panicked at {}:{} (git sha: {:?})", - file!(), - line!(), - option_env!("GIT_SHA") - ) - }) - .data() - == b.direct() - .unwrap_or_else(|| { - panic!( - "Panicked at {}:{} (git sha: {:?})", - file!(), - line!(), - option_env!("GIT_SHA") - ) - }) - .data(); - } - - // For indirect atoms, compare content - if a.is_indirect() && b.is_indirect() { - let a_ind = a.indirect().unwrap_or_else(|| { - panic!( - "Panicked at {}:{} (git sha: {:?})", - file!(), - line!(), - option_env!("GIT_SHA") - ) - }); - let b_ind = b.indirect().unwrap_or_else(|| { - panic!( - "Panicked at {}:{} (git sha: {:?})", - file!(), - line!(), - option_env!("GIT_SHA") - ) - }); - if a_ind.size() != b_ind.size() { - return false; - } - let a_slice = a_ind.as_slice(); - let b_slice = b_ind.as_slice(); - return a_slice == b_slice; - } - - // For cells, recursively compare head and tail - if a.is_cell() && b.is_cell() { - let a_cell = a.cell().unwrap_or_else(|| { - panic!( - "Panicked at {}:{} (git sha: {:?})", - file!(), - line!(), - option_env!("GIT_SHA") - ) - }); - let b_cell = b.cell().unwrap_or_else(|| { - panic!( - "Panicked at {}:{} (git sha: {:?})", - file!(), - line!(), - option_env!("GIT_SHA") - ) - }); - return noun_semantically_equal(&a_cell.head(), &b_cell.head()) - && noun_semantically_equal(&a_cell.tail(), &b_cell.tail()); - } - - false -} - -#[cfg(test)] -mod tests { - use either::Either; - - use super::*; - use crate::mem::NockStack; - use crate::noun::DirectAtom; - - fn print_noun_debug(noun: &Noun) -> String { - match noun.as_either_atom_cell() { - Either::Left(atom) => match atom.as_either() { - Either::Left(direct) => format!("Direct({:#x})", direct.data()), - Either::Right(indirect) => { - let mut s = String::from("Indirect["); - unsafe { - for i in 0..indirect.size() { - if i > 0 { - s.push_str(", "); - } - s.push_str(&format!("{:#x}", *indirect.data_pointer().add(i))); - } - } - s.push(']'); - s - } - }, - Either::Right(cell) => { - format!( - "[{} {}]", - print_noun_debug(&cell.head()), - print_noun_debug(&cell.tail()) - ) - } - } - } - - fn print_raw_debug(noun: &Noun) -> String { - format!("raw: {:#x}", unsafe { noun.raw }) - } - - #[test] - fn test_direct_atom_conversion() { - let mut arena = NounArena::new(); - let mut stack = NockStack::new(1024, 100); - - let small = unsafe { DirectAtom::new_unchecked(42).as_noun() }; - println!( - "Original: {} ({})", - print_noun_debug(&small), - print_raw_debug(&small) - ); - - let key = small.into_arena(&mut arena); - println!("Arena node: {:?}", arena.get_node(key)); - - let roundtrip = Noun::from_arena(&arena, key, &mut stack); - println!( - "Roundtrip: {} ({})", - print_noun_debug(&roundtrip), - print_raw_debug(&roundtrip) - ); - - assert!(unsafe { noun_semantically_equal(&small, &roundtrip) }); - } - - #[test] - fn test_indirect_atom_conversion() { - let mut arena = NounArena::new(); - let mut stack = NockStack::new(1024, 100); - - let large_value = DIRECT_MAX + 1; - let large = unsafe { IndirectAtom::new_raw(&mut stack, 1, &large_value).as_noun() }; - println!( - "Original: {} ({})", - print_noun_debug(&large), - print_raw_debug(&large) - ); - - let key = large.into_arena(&mut arena); - println!("Arena node: {:?}", arena.get_node(key)); - - let roundtrip = Noun::from_arena(&arena, key, &mut stack); - println!( - "Roundtrip: {} ({})", - print_noun_debug(&roundtrip), - print_raw_debug(&roundtrip) - ); - - assert!(unsafe { noun_semantically_equal(&large, &roundtrip) }); - } - - #[test] - fn test_cell_conversion() { - let mut arena = NounArena::new(); - let mut stack = NockStack::new(1024, 100); - - let small = unsafe { DirectAtom::new_unchecked(42).as_noun() }; - let large_value = DIRECT_MAX + 1; - let large = unsafe { IndirectAtom::new_raw(&mut stack, 1, &large_value).as_noun() }; - let cell = Cell::new(&mut stack, small, large).as_noun(); - println!( - "Original: {} ({})", - print_noun_debug(&cell), - print_raw_debug(&cell) - ); - - let key = cell.into_arena(&mut arena); - println!("Arena node: {:?}", arena.get_node(key)); - - let roundtrip = Noun::from_arena(&arena, key, &mut stack); - println!( - "Roundtrip: {} ({})", - print_noun_debug(&roundtrip), - print_raw_debug(&roundtrip) - ); - - assert!(unsafe { noun_semantically_equal(&cell, &roundtrip) }); - } - - #[test] - fn test_complex_structure() { - let mut arena = NounArena::new(); - let mut stack = NockStack::new(1024, 100); - - // Create [1 [2 3]] - let n1 = unsafe { DirectAtom::new_unchecked(1).as_noun() }; - let n2 = unsafe { DirectAtom::new_unchecked(2).as_noun() }; - let n3 = unsafe { DirectAtom::new_unchecked(3).as_noun() }; - let inner = Cell::new(&mut stack, n2, n3).as_noun(); - let outer = Cell::new(&mut stack, n1, inner).as_noun(); - println!( - "Original: {} ({})", - print_noun_debug(&outer), - print_raw_debug(&outer) - ); - - let key = outer.into_arena(&mut arena); - println!("Arena structure:"); - println!("Root: {:?}", arena.get_node(key)); - - let (head, tail) = arena.as_cell(key).unwrap_or_else(|err| { - panic!( - "Panicked with {err:?} at {}:{} (git sha: {:?})", - file!(), - line!(), - option_env!("GIT_SHA") - ) - }); - println!("Head: {:?}", arena.get_node(head)); - println!("Tail: {:?}", arena.get_node(tail)); - - if let Node::Cell(inner_head, inner_tail) = arena.get_node(tail) { - println!("Inner head: {:?}", arena.get_node(*inner_head)); - println!("Inner tail: {:?}", arena.get_node(*inner_tail)); - } - - let roundtrip = Noun::from_arena(&arena, key, &mut stack); - println!( - "Roundtrip: {} ({})", - print_noun_debug(&roundtrip), - print_raw_debug(&roundtrip) - ); - - assert!(unsafe { noun_semantically_equal(&outer, &roundtrip) }); - } -} diff --git a/crates/nockvm/rust/nockvm/src/substantive/mod.rs b/crates/nockvm/rust/nockvm/src/substantive/mod.rs deleted file mode 100644 index 65fc85d82..000000000 --- a/crates/nockvm/rust/nockvm/src/substantive/mod.rs +++ /dev/null @@ -1,388 +0,0 @@ -pub mod convert; - -use std::collections::HashSet; - -use ibig::UBig; -use slotmap::{new_key_type, SlotMap}; -// use std::fmt; - -/// The maximum value of a “direct” atom (63 bits). -pub const DIRECT_MAX: u64 = u64::MAX >> 1; - -/// An enum storing the various possible node contents: -/// - Direct(u64): small atoms up to DIRECT_MAX. -/// - Indirect(Vec): larger atoms represented as one or more u64 words (little-endian). -/// - Cell(NounKey, NounKey): cons cell linking two Nouns (by keys). -#[derive(Debug, PartialEq, Eq)] -pub enum Node { - Direct(u64), - Indirect(Vec), - Cell(NounKey, NounKey), -} - -// A key that references one of the above `Node`s in the `SlotMap`. -new_key_type! { pub struct NounKey; } - -/// Errors we might run into when manipulating Nouns. -#[derive(Debug)] -pub enum NounError { - AxisDescendsThroughAtom, - NotRepresentable, - NotCell, -} - -impl std::fmt::Display for NounError { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - match self { - NounError::AxisDescendsThroughAtom => { - write!(f, "Axis tried to descend through an atom") - } - NounError::NotRepresentable => write!(f, "Not representable"), - NounError::NotCell => write!(f, "Not a cell"), - } - } -} - -impl std::error::Error for NounError {} - -pub type Result = std::result::Result; - -/// The arena that owns all Nouns (Node entries) in a SlotMap. -pub struct NounArena { - nodes: SlotMap, -} - -impl Default for NounArena { - fn default() -> Self { - Self::new() - } -} - -impl NounArena { - /// Create a new, empty `NounArena`. - pub fn new() -> Self { - NounArena { - nodes: SlotMap::with_key(), // or just SlotMap::new() - } - } - - /// Add a direct (small) atom to the arena, returning its key. - pub fn add_direct(&mut self, value: u64) -> NounKey { - self.nodes.insert(Node::Direct(value)) - } - - /// Add an indirect (large) atom to the arena, returning its key. - pub fn add_indirect(&mut self, data: Vec) -> NounKey { - self.nodes.insert(Node::Indirect(data)) - } - - /// Add a cell (head, tail) to the arena, returning its key. - pub fn add_cell(&mut self, head: NounKey, tail: NounKey) -> NounKey { - self.nodes.insert(Node::Cell(head, tail)) - } - - /// Look up the underlying `Node` for a given key. - pub fn get_node(&self, nk: NounKey) -> &Node { - self.nodes.get(nk).expect("Invalid NounKey") - } - - /// Determine if the given key refers to an atom (direct or indirect). - pub fn is_atom(&self, nk: NounKey) -> bool { - match self.get_node(nk) { - Node::Direct(_) | Node::Indirect(_) => true, - Node::Cell(..) => false, - } - } - - /// Determine if the given key refers to a cell. - pub fn is_cell(&self, nk: NounKey) -> bool { - matches!(self.get_node(nk), Node::Cell(..)) - } - - /// Create a noun from a u64, deciding direct vs. indirect automatically. - pub fn from_u64(&mut self, value: u64) -> NounKey { - if value <= DIRECT_MAX { - self.add_direct(value) - } else { - // store as a one-element vector - self.add_indirect(vec![value]) - } - } - - // to_le_bytes and new_raw are copies. We should be able to do this completely without copies - // if we integrate with ibig properly. - // pub fn from_ubig(allocator: &mut A, big: &UBig) -> Atom { - // let bit_size = big.bit_len(); - // let buffer = big.to_le_bytes_stack(); - // if bit_size < 64 { - // let mut value = 0u64; - // for i in (0..bit_size).step_by(8) { - // value |= (buffer[i / 8] as u64) << i; - // } - // unsafe { DirectAtom::new_unchecked(value).as_atom() } - // } else { - // let byte_size = (big.bit_len() + 7) >> 3; - // unsafe { IndirectAtom::new_raw_bytes(allocator, byte_size, buffer.as_ptr()).as_atom() } - // } - // } - - /// Create a noun from an `ibig::UBig`. - /// If it fits into DIRECT_MAX, store as direct; otherwise store in a Vec. - pub fn from_ubig(&mut self, big: &UBig) -> NounKey { - let bit_size = big.bit_len(); - let buffer = big.to_le_bytes_stack(); - if bit_size <= 63 { - // safely fits in a u64 - let mut value = 0u64; - for i in (0..bit_size).step_by(8) { - value |= (buffer[i / 8] as u64) << i; - } - self.add_direct(value) - } else { - // store as little-endian words - let bytes_le = big.to_le_bytes(); - let mut words = Vec::with_capacity(bytes_le.len().div_ceil(8)); - for chunk in bytes_le.chunks(8) { - let mut arr = [0u8; 8]; - arr[..chunk.len()].copy_from_slice(chunk); - words.push(u64::from_le_bytes(arr)); - } - // remove trailing zeros if any (but leave at least one word) - while words.len() > 1 - && *words.last().unwrap_or_else(|| { - panic!( - "Panicked at {}:{} (git sha: {:?})", - file!(), - line!(), - option_env!("GIT_SHA") - ) - }) == 0 - { - words.pop(); - } - self.add_indirect(words) - } - } - - /// Retrieve a sub-noun by axis navigation (like `slot(axis)` in Nock). - /// - axis 1 -> the noun itself - /// - axis 2 -> head of noun - /// - axis 3 -> tail of noun - /// - etc., interpreting the axis bits from LSB to MSB - pub fn slot(&self, root: NounKey, axis: u64) -> Result { - if axis == 0 { - return Err(NounError::AxisDescendsThroughAtom); - } - if axis == 1 { - // "root" axis - return Ok(root); - } - - let mut path = axis; - let mut current = root; - // descend - while path > 1 { - let last_bit = path & 1; - path >>= 1; - - match self.get_node(current) { - Node::Cell(head, tail) => { - current = if last_bit == 0 { *head } else { *tail }; - } - // tried to descend through an atom - Node::Direct(_) | Node::Indirect(_) => { - return Err(NounError::AxisDescendsThroughAtom); - } - } - } - Ok(current) - } - - /// Retrieve (head, tail) if this is a cell, else error. - pub fn as_cell(&self, noun: NounKey) -> Result<(NounKey, NounKey)> { - match self.get_node(noun) { - Node::Cell(h, t) => Ok((*h, *t)), - _ => Err(NounError::NotCell), - } - } - - /// Simple check for cycles by pointer equality in the slotmap. - /// Usually, `SlotMap` cannot form cycles in typical usage unless you store references - /// back into the same map, but we can do a DFS to confirm for demonstration. - pub fn is_acyclic(&self, root: NounKey) -> bool { - let mut visited = HashSet::new(); - self.acyclic_dfs(root, &mut visited) - } - - fn acyclic_dfs(&self, current: NounKey, visited: &mut HashSet) -> bool { - if !visited.insert(current) { - // we've seen this node, so there's a cycle - return false; - } - match self.get_node(current) { - Node::Cell(h, t) => { - // check sub-nodes - if self.acyclic_dfs(*h, visited) && self.acyclic_dfs(*t, visited) { - // remove from visited for safe backtracking, or we can keep for global cycle detection - visited.remove(¤t); - true - } else { - false - } - } - Node::Direct(_) | Node::Indirect(_) => { - // atoms can't form cycles - visited.remove(¤t); - true - } - } - } -} - -/// Convenience function to create a cell in the arena. -pub fn new_cell(arena: &mut NounArena, head: NounKey, tail: NounKey) -> NounKey { - arena.add_cell(head, tail) -} - -/// Convenience function to create a “tape” from a string: a linked list of bytes. -pub fn tape(arena: &mut NounArena, text: &str) -> NounKey { - // Build from the end - let mut list = arena.add_direct(0); // empty list - for &byte in text.as_bytes().iter().rev() { - let atom = arena.add_direct(byte as u64); - list = arena.add_cell(atom, list); - } - list -} - -/// Implement a custom Debug that prints out the entire structure of a noun, -/// for demonstration (not a cycle-safe impl). -pub fn debug_print(arena: &NounArena, key: NounKey) -> String { - match arena.get_node(key) { - Node::Direct(x) => format!("{x}"), - Node::Indirect(words) => { - if words.is_empty() { - "0x".to_string() - } else { - let mut s = String::from("0x"); - for w in words.iter().rev() { - s.push_str(&format!("_{w:016x}")); - } - s - } - } - Node::Cell(h, t) => { - let hd = debug_print(arena, *h); - let tl = debug_print(arena, *t); - format!("[{hd} {tl}]") - } - } -} - -#[cfg(test)] -mod test { - use super::*; - - // TODO: I ripped ibig::UBig out of this test because it was tripping a memory leak under Miri - #[test] - fn test_noun_arena() { - let mut arena = NounArena::new(); - - // Make a small direct atom - let d1 = arena.from_u64(123); - - // Make a large indirect atom explicitly (simulating what we did with ibig) - // Using a large number that would be stored indirectly: [0xFFFFFFFFFFFFFFFF, 0x1] - // This represents 2^128 - 1 - let a2 = arena.add_indirect(vec![0xFFFFFFFFFFFFFFFF, 0x1]); - - // Make a cell - let cell = arena.add_cell(d1, a2); - - // Print it - println!("Cell is {:?}", debug_print(&arena, cell)); - - // Access sub-part by axis. Axis=2 => the head - println!( - "slot(cell,2) => {:?}", - debug_print(&arena, arena.slot(cell, 2).expect("Failed to get slot")) - ); - // Axis=3 => the tail - println!( - "slot(cell,3) => {:?}", - debug_print( - &arena, - arena.slot(cell, 3).unwrap_or_else(|err| panic!( - "Panicked with {err:?} at {}:{} (git sha: {:?})", - file!(), - line!(), - option_env!("GIT_SHA") - )) - ) - ); - - // Test that we can actually get the expected values - let head = arena.slot(cell, 2).unwrap_or_else(|err| { - panic!( - "Panicked with {err:?} at {}:{} (git sha: {:?})", - file!(), - line!(), - option_env!("GIT_SHA") - ) - }); - match arena.get_node(head) { - Node::Direct(n) => assert_eq!(*n, 123), - _ => panic!("Expected Direct(123)"), - } - - let tail = arena.slot(cell, 3).unwrap_or_else(|err| { - panic!( - "Panicked with {err:?} at {}:{} (git sha: {:?})", - file!(), - line!(), - option_env!("GIT_SHA") - ) - }); - match arena.get_node(tail) { - Node::Indirect(words) => { - assert_eq!(words.len(), 2); - assert_eq!(words[0], 0xFFFFFFFFFFFFFFFF); - assert_eq!(words[1], 0x1); - } - _ => panic!("Expected Indirect atom"), - } - - // Tape example - let txt = "hello"; - let tape_key = tape(&mut arena, txt); - println!("Tape for '{}' => {:?}", txt, debug_print(&arena, tape_key)); - - // Verify the tape contents - let mut curr = tape_key; - let chars: Vec<_> = txt.bytes().collect(); // Get chars in original order - for &expected_char in chars.iter() { - match arena.as_cell(curr) { - Ok((head, tail)) => { - // Check the character - match arena.get_node(head) { - Node::Direct(n) => { - assert_eq!(*n as u8, expected_char); - } - _ => panic!("Expected Direct atom in tape"), - } - curr = tail; - } - Err(_) => panic!("Expected Cell in tape"), - } - } - // Should be at the terminating 0 - match arena.get_node(curr) { - Node::Direct(n) => assert_eq!(*n, 0), - _ => panic!("Expected Direct(0) at end of tape"), - } - - // Cycle check - println!("Is cell acyclic? {}", arena.is_acyclic(cell)); - assert!(arena.is_acyclic(cell)); - } -} diff --git a/crates/nockvm/rust/nockvm/src/trace/filter.rs b/crates/nockvm/rust/nockvm/src/trace/filter.rs index ee6cfc4fc..56c10f64f 100644 --- a/crates/nockvm/rust/nockvm/src/trace/filter.rs +++ b/crates/nockvm/rust/nockvm/src/trace/filter.rs @@ -1,7 +1,8 @@ use super::*; +use crate::noun::NounSpace; pub trait TraceFilter: Send { - fn should_trace(&mut self, path: Noun) -> bool; + fn should_trace(&mut self, path: Noun, space: &NounSpace) -> bool; fn compose bool>( self, @@ -41,9 +42,9 @@ pub struct ComposeFilter bool>(pub A, pub B, pub F); impl bool + Send> TraceFilter for ComposeFilter { - fn should_trace(&mut self, path: Noun) -> bool { - let a = self.0.should_trace(path); - let b = self.1.should_trace(path); + fn should_trace(&mut self, path: Noun, space: &NounSpace) -> bool { + let a = self.0.should_trace(path, space); + let b = self.1.should_trace(path, space); (self.2)(a, b) } } @@ -53,23 +54,24 @@ pub struct KeywordFilter { } impl + Send> TraceFilter for KeywordFilter { - fn should_trace(&mut self, path: Noun) -> bool { - fn has_keywords(n: Noun, cnt: usize, kw: &[impl AsRef]) -> bool { + fn should_trace(&mut self, path: Noun, space: &NounSpace) -> bool { + fn has_keywords(n: Noun, cnt: usize, kw: &[impl AsRef], space: &NounSpace) -> bool { if cnt == 0 { return false; } - if let Ok(n) = n.as_atom() { + if let Ok(n) = n.in_space(space).as_atom() { let b = n.as_ne_bytes(); let b = &b[..b.iter().rposition(|&b| b != 0).map_or(0, |i| i + 1)]; return kw.iter().map(|v| v.as_ref()).any(|v| v.as_bytes() == b); } - if let Ok(c) = n.as_cell() { - return has_keywords(c.head(), cnt - 1, kw) || has_keywords(c.tail(), cnt - 1, kw); + if let Ok(c) = n.in_space(space).as_cell() { + return has_keywords(c.head().noun(), cnt - 1, kw, space) + || has_keywords(c.tail().noun(), cnt - 1, kw, space); } false } - has_keywords(path, 10, &self.keywords) + has_keywords(path, 10, &self.keywords, space) } } @@ -79,7 +81,7 @@ pub struct IntervalFilter { } impl TraceFilter for IntervalFilter { - fn should_trace(&mut self, _: Noun) -> bool { + fn should_trace(&mut self, _: Noun, _: &NounSpace) -> bool { let c = self.cnt; self.cnt += 1; c.is_multiple_of(self.interval) diff --git a/crates/nockvm/rust/nockvm/src/trace/mod.rs b/crates/nockvm/rust/nockvm/src/trace/mod.rs index a7f136b5e..2cc011397 100644 --- a/crates/nockvm/rust/nockvm/src/trace/mod.rs +++ b/crates/nockvm/rust/nockvm/src/trace/mod.rs @@ -84,7 +84,8 @@ pub struct TraceInfo { impl TraceInfo { pub fn append_trace(&mut self, stack: &mut NockStack, path: Noun) { if let Some(filter) = self.filter.as_mut() { - if !filter.should_trace(path) { + let space = stack.noun_space(); + if !filter.should_trace(path, &space) { return; } } @@ -136,31 +137,37 @@ pub unsafe fn write_nock_trace( pub fn path_to_cord(stack: &mut NockStack, path: Noun) -> Atom { let mut cursor = path; let mut length = 0usize; + let space = stack.noun_space(); // count how much size we need - while let Ok(c) = cursor.as_cell() { + while let Ok(c) = cursor.in_space(&space).as_cell() { unsafe { match c.head().as_either_atom_cell() { Left(a) => { length += 1; - length += met3_usize(a); + length += met3_usize(a.atom(), &space); } Right(ch) => { if let Ok(nm) = ch.head().as_atom() { if let Ok(kv) = ch.tail().as_atom() { - let kvt = scow(stack, DirectAtom::new_unchecked(tas!(b"ud")), kv) - .expect("scow should succeed in path_to_cord"); - let kvc = - rap(stack, 3, kvt).expect("rap should succeed in path_to_cord"); + let kvt = scow( + stack, + DirectAtom::new_unchecked(tas!(b"ud")), + kv.atom(), + &space, + ) + .expect("scow should succeed in path_to_cord"); + let kvc = rap(stack, 3, kvt, &space) + .expect("rap should succeed in path_to_cord"); length += 1; - length += met3_usize(nm); - length += met3_usize(kvc); + length += met3_usize(nm.atom(), &space); + length += met3_usize(kvc, &space); } } } } } - cursor = c.tail(); + cursor = c.tail().noun(); } // reset cursor, then actually write the path @@ -169,39 +176,44 @@ pub fn path_to_cord(stack: &mut NockStack, path: Noun) -> Atom { let (mut deres, buffer) = unsafe { IndirectAtom::new_raw_mut_bytes(stack, length) }; let slash = (b"/")[0]; - while let Ok(c) = cursor.as_cell() { + while let Ok(c) = cursor.in_space(&space).as_cell() { unsafe { match c.head().as_either_atom_cell() { Left(a) => { buffer[idx] = slash; idx += 1; - let bytelen = met3_usize(a); + let bytelen = met3_usize(a.atom(), &space); buffer[idx..idx + bytelen].copy_from_slice(&a.as_ne_bytes()[0..bytelen]); idx += bytelen; } Right(ch) => { if let Ok(nm) = ch.head().as_atom() { if let Ok(kv) = ch.tail().as_atom() { - let kvt = scow(stack, DirectAtom::new_unchecked(tas!(b"ud")), kv) - .expect("scow should succeed in path_to_cord"); - let kvc = - rap(stack, 3, kvt).expect("rap should succeed in path_to_cord"); + let kvt = scow( + stack, + DirectAtom::new_unchecked(tas!(b"ud")), + kv.atom(), + &space, + ) + .expect("scow should succeed in path_to_cord"); + let kvc = rap(stack, 3, kvt, &space) + .expect("rap should succeed in path_to_cord"); buffer[idx] = slash; idx += 1; - let nmlen = met3_usize(nm); + let nmlen = met3_usize(nm.atom(), &space); buffer[idx..idx + nmlen].copy_from_slice(&nm.as_ne_bytes()[0..nmlen]); idx += nmlen; - let kvclen = met3_usize(kvc); + let kvclen = met3_usize(kvc, &space); buffer[idx..idx + kvclen] - .copy_from_slice(&kvc.as_ne_bytes()[0..kvclen]); + .copy_from_slice(&kvc.in_space(&space).as_ne_bytes()[0..kvclen]); idx += kvclen; } } } } } - cursor = c.tail(); + cursor = c.tail().noun(); } - unsafe { deres.normalize_as_atom() } + unsafe { deres.normalize_as_atom(&space) } } diff --git a/crates/nockvm/rust/nockvm/src/trace/tracing_backend.rs b/crates/nockvm/rust/nockvm/src/trace/tracing_backend.rs index f5ed949e8..e45ce51fc 100644 --- a/crates/nockvm/rust/nockvm/src/trace/tracing_backend.rs +++ b/crates/nockvm/rust/nockvm/src/trace/tracing_backend.rs @@ -132,22 +132,25 @@ impl Drop for TracingBackend { impl TraceBackend for TracingBackend { fn append_trace(&mut self, stack: &mut NockStack, path: Noun) { let mut tmp = path; + let space = stack.noun_space(); let chum = loop { match tmp.as_either_atom_cell() { Either::Left(atom) => break atom, - Either::Right(cell) => tmp = cell.head(), + Either::Right(cell) => tmp = cell.in_space(&space).head().noun(), } }; - let Ok(chum) = std::str::from_utf8(chum.as_ne_bytes()) else { + let chum_handle = chum.in_space(&space); + let Ok(chum) = std::str::from_utf8(chum_handle.as_ne_bytes()) else { return; }; let chum = chum.trim_end_matches('\0'); let path = path_to_cord(stack, path); - let path = std::str::from_utf8(path.as_ne_bytes()).unwrap_or(""); + let path_handle = path.in_space(&space); + let path = std::str::from_utf8(path_handle.as_ne_bytes()).unwrap_or(""); if self.subscriber.is_none() { self.subscriber = Some(dispatcher::get_default(Clone::clone)); diff --git a/crates/nockvm/rust/nockvm/src/unifying_equality.rs b/crates/nockvm/rust/nockvm/src/unifying_equality.rs index d0dafd329..bb9e892e3 100644 --- a/crates/nockvm/rust/nockvm/src/unifying_equality.rs +++ b/crates/nockvm/rust/nockvm/src/unifying_equality.rs @@ -3,7 +3,6 @@ use libc::{c_void, memcmp}; use crate::mem::{NockStack, ALLOC, FRAME, STACK}; use crate::noun::Noun; -use crate::{assert_acyclic, assert_no_forwarding_pointers}; #[cfg(feature = "check_junior")] #[macro_export] @@ -295,21 +294,28 @@ unsafe fn x_is_junior( /// senior noun, *never vice versa*, to avoid introducing references from more senior frames /// into more junior frames, which would result in incorrect operation of the copier. pub unsafe fn unifying_equality(stack: &mut NockStack, a: *mut Noun, b: *mut Noun) -> bool { - assert_acyclic!(*a); - assert_acyclic!(*b); - assert_no_forwarding_pointers!(*a); - assert_no_forwarding_pointers!(*b); - assert_no_junior_pointers!(stack, *a); - assert_no_junior_pointers!(stack, *b); + #[cfg(any(feature = "check_acyclic", feature = "check_forwarding"))] + { + let _space = stack.fast_noun_space(); + crate::assert_acyclic!(_space, *a); + crate::assert_acyclic!(_space, *b); + crate::assert_no_forwarding_pointers!(_space, *a); + crate::assert_no_forwarding_pointers!(_space, *b); + } + crate::assert_no_junior_pointers!(stack, *a); + crate::assert_no_junior_pointers!(stack, *b); if raw_word_eq(a, b) { return true; } - if let (Ok(aa), Ok(bb)) = ((*a).as_allocated(), (*b).as_allocated()) { - if let (Some(am), Some(bm)) = (aa.get_cached_mug(), bb.get_cached_mug()) { - if am != bm { - return false; + { + let space = stack.fast_noun_space(); + if let (Ok(aa), Ok(bb)) = ((*a).as_allocated(), (*b).as_allocated()) { + if let (Some(am), Some(bm)) = (aa.get_cached_mug(&space), bb.get_cached_mug(&space)) { + if am != bm { + return false; + } } } } @@ -331,7 +337,11 @@ pub unsafe fn unifying_equality(stack: &mut NockStack, a: *mut Noun, b: *mut Nou } if let (Ok(xa), Ok(ya)) = ((*x).as_allocated(), (*y).as_allocated()) { - if let (Some(xm), Some(ym)) = (xa.get_cached_mug(), ya.get_cached_mug()) { + let cached = { + let space = stack.fast_noun_space(); + (xa.get_cached_mug(&space), ya.get_cached_mug(&space)) + }; + if let (Some(xm), Some(ym)) = cached { if xm != ym { break; } @@ -339,16 +349,41 @@ pub unsafe fn unifying_equality(stack: &mut NockStack, a: *mut Noun, b: *mut Nou match (xa.as_either(), ya.as_either()) { (Left(xi), Left(yi)) => { - if xi.size() == yi.size() - && memcmp( - xi.data_pointer() as *const c_void, - yi.data_pointer() as *const c_void, - xi.size() << 3, - ) == 0 - { - let xptr = xi.to_raw_pointer(); - let yptr = yi.to_raw_pointer(); - if x_is_junior(stack, current_bounds, &mut bounds_state, xptr, yptr) { + let (data_equal, xptr, yptr) = { + let space = stack.fast_noun_space(); + let xi_handle = xi.as_atom().in_space(&space); + let yi_handle = yi.as_atom().in_space(&space); + let xsize = xi_handle.size(); + let ysize = yi_handle.size(); + if xsize != ysize { + (false, std::ptr::null(), std::ptr::null()) + } else { + let equal = memcmp( + xi_handle.data_pointer() as *const c_void, + yi_handle.data_pointer() as *const c_void, + xsize << 3, + ) == 0; + (equal, unsafe { xi_handle.raw_pointer() }, unsafe { + yi_handle.raw_pointer() + }) + } + }; + + if data_equal { + let replace_x = { + let space = stack.fast_noun_space(); + let x_loc = (*x).allocated_location(&space); + let y_loc = (*y).allocated_location(&space); + match (x_loc, y_loc) { + (Some(xl), Some(yl)) if xl.is_pma() && yl.is_stack() => false, + (Some(xl), Some(yl)) if xl.is_stack() && yl.is_pma() => true, + (Some(xl), Some(yl)) if xl.is_pma() && yl.is_pma() => xptr > yptr, + _ => x_is_junior( + stack, current_bounds, &mut bounds_state, xptr, yptr, + ), + } + }; + if replace_x { *x = *y; } else { *y = *x; @@ -361,16 +396,37 @@ pub unsafe fn unifying_equality(stack: &mut NockStack, a: *mut Noun, b: *mut Nou } (Right(xc), Right(yc)) => { + let (xh, yh, xt, yt, xptr, yptr) = { + let space = stack.fast_noun_space(); + ( + xc.head_as_mut(&space), + yc.head_as_mut(&space), + xc.tail_as_mut(&space), + yc.tail_as_mut(&space), + xc.to_raw_pointer(&space) as *const u64, + yc.to_raw_pointer(&space) as *const u64, + ) + }; + // check head; only compute tail eq if needed; push only unequal sides - let xh = xc.head_as_mut(); - let yh = yc.head_as_mut(); if raw_word_eq(xh, yh) { - let xt = xc.tail_as_mut(); - let yt = yc.tail_as_mut(); if raw_word_eq(xt, yt) { - let xptr = xc.to_raw_pointer() as *const u64; - let yptr = yc.to_raw_pointer() as *const u64; - if x_is_junior(stack, current_bounds, &mut bounds_state, xptr, yptr) { + let replace_x = { + let space = stack.fast_noun_space(); + let x_loc = (*x).allocated_location(&space); + let y_loc = (*y).allocated_location(&space); + match (x_loc, y_loc) { + (Some(xl), Some(yl)) if xl.is_pma() && yl.is_stack() => false, + (Some(xl), Some(yl)) if xl.is_stack() && yl.is_pma() => true, + (Some(xl), Some(yl)) if xl.is_pma() && yl.is_pma() => { + xptr > yptr + } + _ => x_is_junior( + stack, current_bounds, &mut bounds_state, xptr, yptr, + ), + } + }; + if replace_x { *x = *y; } else { *y = *x; @@ -383,8 +439,6 @@ pub unsafe fn unifying_equality(stack: &mut NockStack, a: *mut Noun, b: *mut Nou } } else { // head unequal; only push tail if it is also unequal - let xt = xc.tail_as_mut(); - let yt = yc.tail_as_mut(); if !raw_word_eq(xt, yt) { *(stack.push::<(*mut Noun, *mut Noun)>()) = (xt, yt); } @@ -404,12 +458,16 @@ pub unsafe fn unifying_equality(stack: &mut NockStack, a: *mut Noun, b: *mut Nou stack.frame_pop(); - assert_acyclic!(*a); - assert_acyclic!(*b); - assert_no_forwarding_pointers!(*a); - assert_no_forwarding_pointers!(*b); - assert_no_junior_pointers!(stack, *a); - assert_no_junior_pointers!(stack, *b); + #[cfg(any(feature = "check_acyclic", feature = "check_forwarding"))] + { + let _space = stack.fast_noun_space(); + crate::assert_acyclic!(_space, *a); + crate::assert_acyclic!(_space, *b); + crate::assert_no_forwarding_pointers!(_space, *a); + crate::assert_no_forwarding_pointers!(_space, *b); + } + crate::assert_no_junior_pointers!(stack, *a); + crate::assert_no_junior_pointers!(stack, *b); raw_word_eq(a, b) } diff --git a/crates/nockvm/rust/nockvm/tests/pma_regressions.rs b/crates/nockvm/rust/nockvm/tests/pma_regressions.rs new file mode 100644 index 000000000..66a5ad5d5 --- /dev/null +++ b/crates/nockvm/rust/nockvm/tests/pma_regressions.rs @@ -0,0 +1,28 @@ +use std::error::Error; +use std::sync::Mutex; + +mod pma_regressions { + pub(crate) mod multi_resize_bootstrap; + pub(crate) mod resize_exhaustion; +} + +type TestResult = Result<(), Box>; + +static PMA_REGRESSION_LOCK: Mutex<()> = Mutex::new(()); + +fn run_serialized(test: fn() -> TestResult) -> TestResult { + let _guard = PMA_REGRESSION_LOCK + .lock() + .expect("PMA regression test lock poisoned"); + test() +} + +#[test] +fn pma_multi_resize_bootstrap_regression() -> TestResult { + run_serialized(pma_regressions::multi_resize_bootstrap::run_regression) +} + +#[test] +fn pma_resize_exhaustion_regression() -> TestResult { + run_serialized(pma_regressions::resize_exhaustion::run_regression) +} diff --git a/crates/nockvm/rust/nockvm/tests/pma_regressions/multi_resize_bootstrap.rs b/crates/nockvm/rust/nockvm/tests/pma_regressions/multi_resize_bootstrap.rs new file mode 100644 index 000000000..3933ff224 --- /dev/null +++ b/crates/nockvm/rust/nockvm/tests/pma_regressions/multi_resize_bootstrap.rs @@ -0,0 +1,283 @@ +use std::any::Any; +use std::error::Error; +use std::ffi::OsString; +use std::fs; +use std::panic::{self, AssertUnwindSafe}; +use std::path::{Path, PathBuf}; +use std::sync::{Arc, Mutex}; +use std::time::{SystemTime, UNIX_EPOCH}; + +use nockvm::mem::NockStack; +use nockvm::noun::{Cell, Noun, NounSpace, D}; +use nockvm::pma::{Pma, PmaCopy}; + +const INITIAL_PMA_WORDS: usize = 1024; +const CELL_WORDS: usize = 3; +const TARGET_PMA_WORDS: usize = INITIAL_PMA_WORDS * 10; +const LIST_CELLS: usize = (TARGET_PMA_WORDS / CELL_WORDS) + 1; +const EXPECTED_ALLOC_WORDS: usize = LIST_CELLS * CELL_WORDS; +const STACK_WORDS: usize = EXPECTED_ALLOC_WORDS * 2 + 1024; +const MIN_GROWTH_EVENTS: usize = 2; +const MAX_REASONABLE_FINAL_WORDS: usize = INITIAL_PMA_WORDS * 32; +const GROWTH_EVENTS_ENV: &str = "NOCK_PMA_GROWTH_EVENTS_PATH"; + +pub(crate) fn run_regression() -> Result<(), Box> { + nockvm::check_endian(); + run() +} + +fn run() -> Result<(), Box> { + assert_eq!( + std::mem::size_of::(), + CELL_WORDS * std::mem::size_of::(), + "regression assumes one direct-atom list cell copies to exactly {CELL_WORDS} PMA words" + ); + + let workspace = Workspace::new("pma-multi-resize-bootstrap-regression")?; + let pma_path = workspace.path().join("bootstrap.pma"); + let growth_events_path = workspace.path().join("growth-events.log"); + let _growth_events_guard = EnvVarGuard::set(GROWTH_EVENTS_ENV, growth_events_path.as_os_str()); + + println!( + "building bootstrap noun: initial_pma_words={INITIAL_PMA_WORDS} target_words={TARGET_PMA_WORDS} list_cells={LIST_CELLS} expected_alloc_words={EXPECTED_ALLOC_WORDS} stack_words={STACK_WORDS}" + ); + let mut stack = NockStack::new(STACK_WORDS, 0); + let source_root = build_direct_atom_list(&mut stack, LIST_CELLS); + + let mut pma = Pma::new(INITIAL_PMA_WORDS, pma_path.clone())?; + let mut pma_root = source_root; + println!( + "copying bootstrap noun into PMA: path={} size_words={} alloc_words={} free_words={}", + pma_path.display(), + pma.size_words(), + pma.alloc_offset(), + pma.free_words() + ); + + let copy_result = catch_unwind_message(|| unsafe { + pma_root.copy_to_pma(&stack, &mut pma); + }); + if let Err(message) = copy_result { + return Err(std::io::Error::other(format!( + "bootstrap copy failed before automatic multi-growth could complete: {message}" + )) + .into()); + } + + let growth_events = read_growth_events(&growth_events_path)?; + println!( + "bootstrap copy completed: final_size_words={} alloc_words={} free_words={} growth_events={}", + pma.size_words(), + pma.alloc_offset(), + pma.free_words(), + growth_events.len() + ); + for event in &growth_events { + println!("pma growth event: {event}"); + } + + if growth_events.len() < MIN_GROWTH_EVENTS { + return Err(std::io::Error::other(format!( + "expected at least {MIN_GROWTH_EVENTS} automatic PMA growth events while copying a noun more than 10x the initial capacity, got {}. Implement growable PMA instrumentation by appending one line per growth to ${GROWTH_EVENTS_ENV}.", + growth_events.len() + )) + .into()); + } + if pma.size_words() < EXPECTED_ALLOC_WORDS { + return Err(std::io::Error::other(format!( + "PMA did not grow enough for copied noun: final_size_words={} expected_alloc_words={EXPECTED_ALLOC_WORDS}", + pma.size_words() + )) + .into()); + } + if pma.size_words() > MAX_REASONABLE_FINAL_WORDS { + return Err(std::io::Error::other(format!( + "PMA grew to an unreasonable current capacity: final_size_words={} max_reasonable_words={MAX_REASONABLE_FINAL_WORDS}. The regression expects on-demand growth, not materializing a huge max reservation as current file capacity.", + pma.size_words() + )) + .into()); + } + if pma.alloc_offset() != EXPECTED_ALLOC_WORDS { + return Err(std::io::Error::other(format!( + "copied noun used unexpected PMA words: alloc_words={} expected_alloc_words={EXPECTED_ALLOC_WORDS}", + pma.alloc_offset() + )) + .into()); + } + + verify_pma_list(pma_root, &pma, LIST_CELLS)?; + pma.sync_all()?; + pma.sync_file()?; + assert_reasonable_file_len(&pma_path, pma.size_words())?; + drop(pma); + + let reopened = Pma::open(pma_path.clone())?; + println!( + "reopened PMA: path={} size_words={} alloc_words={} free_words={}", + pma_path.display(), + reopened.size_words(), + reopened.alloc_offset(), + reopened.free_words() + ); + verify_pma_list(pma_root, &reopened, LIST_CELLS)?; + assert_eq!( + reopened.alloc_offset(), + EXPECTED_ALLOC_WORDS, + "reopened PMA should preserve allocation cursor" + ); + assert_reasonable_file_len(&pma_path, reopened.size_words())?; + + println!( + "PMA automatically grew multiple times while bootstrapping a noun over 10x initial capacity, and final PMA noun content verified" + ); + Ok(()) +} + +fn build_direct_atom_list(stack: &mut NockStack, len: usize) -> Noun { + let mut noun = D(0); + for value in (0..len).rev() { + noun = Cell::new(stack, D(value as u64), noun).as_noun(); + } + noun +} + +fn verify_pma_list(root: Noun, pma: &Pma, expected_len: usize) -> Result<(), Box> { + let space = NounSpace::pma_only(pma); + let mut cursor = root.in_space(&space); + for expected in 0..expected_len { + let cell = cursor.as_cell().map_err(|err| { + std::io::Error::other(format!( + "expected PMA list cell at index {expected}, got non-cell: {err:?}" + )) + })?; + let actual = cell.head().as_atom()?.as_u64()?; + if actual != expected as u64 { + return Err(std::io::Error::other(format!( + "PMA list content mismatch at index {expected}: actual={actual} expected={expected}" + )) + .into()); + } + cursor = cell.tail(); + } + let terminator = cursor.as_atom()?.as_u64()?; + if terminator != 0 { + return Err(std::io::Error::other(format!( + "PMA list terminator mismatch: actual={terminator} expected=0" + )) + .into()); + } + Ok(()) +} + +fn read_growth_events(path: &Path) -> Result, Box> { + if !path.exists() { + return Ok(Vec::new()); + } + Ok(fs::read_to_string(path)? + .lines() + .map(str::trim) + .filter(|line| !line.is_empty()) + .map(str::to_owned) + .collect()) +} + +fn assert_reasonable_file_len(path: &Path, current_words: usize) -> Result<(), Box> { + let file_len = fs::metadata(path)?.len(); + let expected_capacity_bytes = (current_words as u64) + .checked_mul(8) + .ok_or_else(|| std::io::Error::other("current PMA words overflow bytes"))?; + let max_reasonable_len = expected_capacity_bytes + .checked_add(1024 * 1024) + .ok_or_else(|| std::io::Error::other("reasonable PMA file length overflowed"))?; + if file_len > max_reasonable_len { + return Err(std::io::Error::other(format!( + "PMA file apparent length is too large for current capacity: file_len_bytes={file_len} current_words={current_words} expected_capacity_bytes={expected_capacity_bytes}. The slab file should materialize current capacity, not maximum virtual reservation." + )) + .into()); + } + Ok(()) +} + +fn catch_unwind_message(f: F) -> Result<(), String> +where + F: FnOnce(), +{ + let hook = panic::take_hook(); + let captured = Arc::new(Mutex::new(None)); + let captured_for_hook = Arc::clone(&captured); + panic::set_hook(Box::new(move |info| { + let message = info + .payload_as_str() + .map(str::to_owned) + .unwrap_or_else(|| panic_message(info.payload())); + *captured_for_hook.lock().expect("panic hook lock poisoned") = Some(message); + })); + + let result = panic::catch_unwind(AssertUnwindSafe(f)); + panic::set_hook(hook); + + match result { + Ok(()) => Ok(()), + Err(payload) => Err(captured + .lock() + .expect("panic hook lock poisoned") + .take() + .unwrap_or_else(|| panic_message(&*payload))), + } +} + +fn panic_message(payload: &(dyn Any + Send)) -> String { + if let Some(message) = payload.downcast_ref::() { + message.clone() + } else if let Some(message) = payload.downcast_ref::<&'static str>() { + (*message).to_string() + } else { + "".to_string() + } +} + +struct EnvVarGuard { + key: &'static str, + previous: Option, +} + +impl EnvVarGuard { + fn set(key: &'static str, value: &std::ffi::OsStr) -> Self { + let previous = std::env::var_os(key); + std::env::set_var(key, value); + Self { key, previous } + } +} + +impl Drop for EnvVarGuard { + fn drop(&mut self) { + if let Some(previous) = self.previous.take() { + std::env::set_var(self.key, previous); + } else { + std::env::remove_var(self.key); + } + } +} + +struct Workspace { + path: PathBuf, +} + +impl Workspace { + fn new(label: &str) -> Result> { + let nanos = SystemTime::now().duration_since(UNIX_EPOCH)?.as_nanos(); + let path = std::env::temp_dir().join(format!("{label}-{}-{nanos}", std::process::id())); + fs::create_dir_all(&path)?; + Ok(Self { path }) + } + + fn path(&self) -> &Path { + &self.path + } +} + +impl Drop for Workspace { + fn drop(&mut self) { + let _ = fs::remove_dir_all(&self.path); + } +} diff --git a/crates/nockvm/rust/nockvm/tests/pma_regressions/resize_exhaustion.rs b/crates/nockvm/rust/nockvm/tests/pma_regressions/resize_exhaustion.rs new file mode 100644 index 000000000..3532280f2 --- /dev/null +++ b/crates/nockvm/rust/nockvm/tests/pma_regressions/resize_exhaustion.rs @@ -0,0 +1,280 @@ +use std::any::Any; +use std::error::Error; +use std::ffi::{OsStr, OsString}; +use std::fs; +use std::io::{Seek, SeekFrom, Write}; +use std::panic::{self, AssertUnwindSafe}; +use std::path::{Path, PathBuf}; +use std::sync::{Arc, Mutex}; +use std::time::{SystemTime, UNIX_EPOCH}; + +use nockvm::mem::NockStack; +use nockvm::noun::{Cell, D}; +use nockvm::pma::{Pma, PmaCopy, PmaError}; + +const OLD_WORDS: usize = 32; +const NEW_MIN_WORDS: usize = 64; +const FILL_CELLS: u64 = 10; +const CELL_WORDS: usize = 3; +const STACK_WORDS: usize = 128; +const DISABLE_GROWTH_ENV: &str = "NOCK_PMA_DISABLE_RESIZE_FOR_REGRESSION"; +const PMA_MAGIC: u64 = u64::from_le_bytes(*b"NOCKPMA1"); +const PMA_LEGACY_VERSION: u64 = 1; +const PMA_LEGACY_TRAILER_BYTES: u64 = 32; + +pub(crate) fn run_regression() -> Result<(), Box> { + nockvm::check_endian(); + let workspace = Workspace::new("pma-resize-exhaustion-regression")?; + run(workspace.path()) +} + +fn run(workspace: &Path) -> Result<(), Box> { + let fixture = workspace.join("fixture.pma"); + let raw_canary = workspace.join("raw-canary.pma"); + let resized_candidate = workspace.join("resized-candidate.pma"); + + println!( + "creating exhausted fixture: old_words={OLD_WORDS} fill_cells={FILL_CELLS} cell_words={CELL_WORDS}" + ); + make_exhausted_fixture(&fixture)?; + print_file_metadata("fixture", &fixture)?; + assert_eq!( + Pma::read_file_metadata(&fixture)?.version, + PMA_LEGACY_VERSION, + "fixture must exercise the shipped fixed-size v1 trailer" + ); + + fs::copy(&fixture, &raw_canary)?; + let _disable_growth = EnvVarGuard::set(DISABLE_GROWTH_ENV, OsStr::new("1")); + assert_raw_open_still_reproduces_oom(&raw_canary)?; + drop(_disable_growth); + + fs::copy(&fixture, &resized_candidate)?; + assert_resized_open_accepts_next_cell(&resized_candidate)?; + + println!("resized-open accepted next cell"); + Ok(()) +} + +fn copy_one_cell_to_pma(pma: &mut Pma, head: u64, tail: u64) { + let mut stack = NockStack::new(STACK_WORDS, 0); + let mut noun = Cell::new(&mut stack, D(head), D(tail)).as_noun(); + unsafe { noun.copy_to_pma(&stack, pma) }; +} + +fn make_exhausted_fixture(path: &Path) -> Result<(), Box> { + let mut pma = Pma::new(OLD_WORDS, path.to_path_buf())?; + for i in 0..FILL_CELLS { + copy_one_cell_to_pma(&mut pma, i, i + 1); + } + assert_eq!( + pma.alloc_offset(), + FILL_CELLS as usize * CELL_WORDS, + "fixture should consume exactly ten CellMemory allocations" + ); + assert_eq!( + pma.free_words(), + 2, + "fixture must have exactly two free words to reproduce the incident-class OOM" + ); + pma.sync_all()?; + pma.sync_file()?; + write_legacy_v1_trailer(path, OLD_WORDS as u64, pma.alloc_offset() as u64)?; + Ok(()) +} + +fn write_legacy_v1_trailer( + path: &Path, + data_words: u64, + alloc_words: u64, +) -> Result<(), Box> { + let mut file = fs::OpenOptions::new().read(true).write(true).open(path)?; + file.set_len(data_words * 8 + PMA_LEGACY_TRAILER_BYTES)?; + file.seek(SeekFrom::Start(data_words * 8))?; + for word in [PMA_MAGIC, PMA_LEGACY_VERSION, data_words, alloc_words] { + file.write_all(&word.to_le_bytes())?; + } + file.sync_all()?; + Ok(()) +} + +fn assert_raw_open_still_reproduces_oom(path: &Path) -> Result<(), Box> { + let mut pma = Pma::open(path.to_path_buf())?; + print_pma_state("raw-open before canary", &pma); + assert_eq!(pma.size_words(), OLD_WORDS); + assert_eq!(pma.alloc_offset(), FILL_CELLS as usize * CELL_WORDS); + assert_eq!(pma.free_words(), 2); + + let panic = catch_unwind_message(|| copy_one_cell_to_pma(&mut pma, 100, 101)); + match panic { + Ok(()) => Err(std::io::Error::other( + "raw Pma::open unexpectedly allowed the exhausted old slab to accept another cell", + ) + .into()), + Err(message) => { + if !message.contains("PMA is full") || !message.contains("available: 2") { + return Err(std::io::Error::other(format!( + "raw-open canary panicked with the wrong message: {message}" + )) + .into()); + } + println!("raw-open canary reproduced PMA OOM: {message}"); + Ok(()) + } + } +} + +fn assert_resized_open_accepts_next_cell(path: &Path) -> Result<(), Box> { + let mut pma = open_or_resize_pma(path.to_path_buf(), NEW_MIN_WORDS)?; + print_pma_state("production resize/open before next cell", &pma); + assert_eq!( + pma.alloc_offset(), + FILL_CELLS as usize * CELL_WORDS, + "resize/open must preserve the used prefix and allocation cursor" + ); + + if pma.size_words() < NEW_MIN_WORDS { + let panic = catch_unwind_message(|| copy_one_cell_to_pma(&mut pma, 200, 201)); + let message = match panic { + Ok(()) => "undersized PMA unexpectedly accepted the next cell".to_string(), + Err(message) => message, + }; + return Err(std::io::Error::other(format!( + "production resize/open path returned an undersized PMA: requested_min_words={NEW_MIN_WORDS} actual_words={} next_cell_result={message}", + pma.size_words() + )) + .into()); + } + + assert!( + pma.free_words() >= NEW_MIN_WORDS - (FILL_CELLS as usize * CELL_WORDS), + "resized PMA should have enough free words for future allocations" + ); + + let copy = catch_unwind_message(|| copy_one_cell_to_pma(&mut pma, 200, 201)); + if let Err(message) = copy { + return Err(std::io::Error::other(format!( + "resized-open still panicked while copying the next cell: {message}" + )) + .into()); + } + + assert_eq!( + pma.alloc_offset(), + (FILL_CELLS as usize + 1) * CELL_WORDS, + "next cell should allocate at the preserved cursor" + ); + print_pma_state("production resize/open after next cell", &pma); + Ok(()) +} + +fn open_or_resize_pma(path: PathBuf, min_words: usize) -> Result { + Pma::open_with_min(path, min_words) +} + +fn print_file_metadata(label: &str, path: &Path) -> Result<(), Box> { + let metadata = Pma::read_file_metadata(path)?; + println!( + "{label}: path={} version={} data_words={} alloc_words={} free_words={} reserved_words={} apparent_file_bytes={}", + path.display(), + metadata.version, + metadata.data_words, + metadata.alloc_words, + metadata.data_words.saturating_sub(metadata.alloc_words), + metadata.reserved_words, + metadata.apparent_file_bytes + ); + Ok(()) +} + +fn print_pma_state(label: &str, pma: &Pma) { + println!( + "{label}: size_words={} alloc_offset={} free_words={}", + pma.size_words(), + pma.alloc_offset(), + pma.free_words() + ); +} + +fn catch_unwind_message(f: F) -> Result<(), String> +where + F: FnOnce(), +{ + let hook = panic::take_hook(); + let captured = Arc::new(Mutex::new(None)); + let captured_for_hook = Arc::clone(&captured); + panic::set_hook(Box::new(move |info| { + let message = info + .payload_as_str() + .map(str::to_owned) + .unwrap_or_else(|| panic_message(info.payload())); + *captured_for_hook.lock().expect("panic hook lock poisoned") = Some(message); + })); + + let result = panic::catch_unwind(AssertUnwindSafe(f)); + panic::set_hook(hook); + + match result { + Ok(()) => Ok(()), + Err(payload) => Err(captured + .lock() + .expect("panic hook lock poisoned") + .take() + .unwrap_or_else(|| panic_message(&*payload))), + } +} + +fn panic_message(payload: &(dyn Any + Send)) -> String { + if let Some(message) = payload.downcast_ref::() { + message.clone() + } else if let Some(message) = payload.downcast_ref::<&'static str>() { + (*message).to_string() + } else { + "".to_string() + } +} + +struct Workspace { + path: PathBuf, +} + +struct EnvVarGuard { + key: &'static str, + previous: Option, +} + +impl EnvVarGuard { + fn set(key: &'static str, value: &OsStr) -> Self { + let previous = std::env::var_os(key); + std::env::set_var(key, value); + Self { key, previous } + } +} + +impl Drop for EnvVarGuard { + fn drop(&mut self) { + match &self.previous { + Some(value) => std::env::set_var(self.key, value), + None => std::env::remove_var(self.key), + } + } +} + +impl Workspace { + fn new(label: &str) -> Result> { + let nanos = SystemTime::now().duration_since(UNIX_EPOCH)?.as_nanos(); + let path = std::env::temp_dir().join(format!("{label}-{}-{nanos}", std::process::id())); + fs::create_dir_all(&path)?; + Ok(Self { path }) + } + + fn path(&self) -> &Path { + &self.path + } +} + +impl Drop for Workspace { + fn drop(&mut self) { + let _ = fs::remove_dir_all(&self.path); + } +} diff --git a/crates/noun-serde-derive/src/lib.rs b/crates/noun-serde-derive/src/lib.rs index 9e0cd245b..c1b09b985 100644 --- a/crates/noun-serde-derive/src/lib.rs +++ b/crates/noun-serde-derive/src/lib.rs @@ -1,9 +1,18 @@ extern crate proc_macro; +use std::collections::HashSet; + use proc_macro::TokenStream; +use proc_macro2::TokenStream as TokenStream2; use quote::{format_ident, quote}; use syn::{parse_macro_input, Attribute, Data, DeriveInput, Fields, LitBool, Token}; +#[derive(Clone, Debug)] +enum TagValue { + Text(String), + Numeric(u64), +} + /// Parses the `#[noun(tagged = bool)]` attribute from a list of attributes. /// /// Used to determine whether an enum should be encoded with tags. @@ -62,6 +71,82 @@ fn parse_untagged_attr(attrs: &[Attribute]) -> Option { parse_noun_bool_attr(attrs, "untagged") } +fn parse_tag_attr(attrs: &[Attribute]) -> Option { + let mut value = None; + for attr in attrs { + if !attr.path().is_ident("noun") { + continue; + } + let _ = attr.parse_nested_meta(|meta| { + if meta.path.is_ident("tag") { + let parser = meta.value()?; + let expr = parser.parse::()?; + if let syn::Expr::Lit(syn::ExprLit { + lit: syn::Lit::Str(s), + .. + }) = expr + { + value = Some(TagValue::Text(s.value())); + } else if let syn::Expr::Lit(syn::ExprLit { + lit: syn::Lit::Int(i), + .. + }) = expr + { + if let Ok(parsed) = i.base10_parse::() { + value = Some(TagValue::Numeric(parsed)); + } + } + } + Ok(()) + }); + } + value +} + +fn text_tag_atom_value(tag: &str) -> Option { + let bytes = tag.as_bytes(); + if bytes.len() > 8 { + return None; + } + + let mut atom_bytes = [0u8; 8]; + atom_bytes[..bytes.len()].copy_from_slice(bytes); + Some(u64::from_le_bytes(atom_bytes)) +} + +fn tag_value_matches_key(tag: &TagValue) -> String { + match tag { + TagValue::Text(value) => text_tag_atom_value(value) + .map(|atom| format!("a:{atom}")) + .unwrap_or_else(|| format!("s:{value}")), + TagValue::Numeric(value) => format!("a:{value}"), + } +} + +fn resolve_variant_tag(attrs: &[Attribute], variant_name: &proc_macro2::Ident) -> TagValue { + parse_tag_attr(attrs).unwrap_or_else(|| TagValue::Text(variant_name.to_string().to_lowercase())) +} + +fn encode_tag_expr(tag: &TagValue) -> TokenStream2 { + match tag { + TagValue::Text(tag) => quote! { ::nockvm::ext::make_tas(allocator, #tag).as_noun() }, + TagValue::Numeric(tag) => quote! { ::nockvm::noun::D(#tag) }, + } +} + +fn decode_tag_match_expr(tag: &TagValue) -> TokenStream2 { + match tag { + TagValue::Text(tag) => quote! { string_tag.as_deref() == Some(#tag) }, + TagValue::Numeric(tag) => quote! { + tag_noun_handle + .as_atom() + .ok() + .and_then(|atom| atom.as_u64().ok()) + == Some(#tag) + }, + } +} + /// Parses the `#[noun(axis = u64)]` attribute from a list of attributes. /// /// Used to specify the axis of a field in a struct or tuple. @@ -122,7 +207,8 @@ fn parse_axis_attr(attrs: &[Attribute]) -> Option { /// /// # Attributes /// -/// - `#[noun(tag = "string")]`: Specifies a custom tag for enum variants (defaults to lowercase variant name) +/// - `#[noun(tag = "string")]`: Specifies a textual tag for enum variants (defaults to lowercase variant name) +/// - `#[noun(tag = 1)]`: Specifies an atom tag for enum variants, matching Hoon `%1` /// - `#[noun(tagged = bool)]`: Controls whether fields are tagged with their names (enum-level or variant-level) /// - `#[noun(untagged)]`: Encode enum variants without tags and try variants in order when decoding /// @@ -154,7 +240,7 @@ fn parse_axis_attr(attrs: &[Attribute]) -> Option { /// /// // When encoded: [42 43] /// let point = Point { x: 42, y: 43 }; -/// let mut allocator = NockStack::new(8 << 10 << 10, 0); +/// let mut allocator = NockStack::new(nockvm::mem::NOCK_STACK_SIZE_TINY, 0); /// let noun = point.to_noun(&mut allocator); /// /// #[derive(NounEncode)] @@ -166,7 +252,7 @@ fn parse_axis_attr(attrs: &[Attribute]) -> Option { /// /// // When encoded: [[%x 42] [%y 43]] /// let tagged_point = TaggedPoint { x: 42, y: 43 }; -/// let mut allocator = NockStack::new(8 << 10 << 10, 0); +/// let mut allocator = NockStack::new(nockvm::mem::NOCK_STACK_SIZE_TINY, 0); /// let noun = tagged_point.to_noun(&mut allocator); /// /// #[derive(NounEncode)] @@ -179,12 +265,12 @@ fn parse_axis_attr(attrs: &[Attribute]) -> Option { /// /// // When encoded: [%move [42 43]] /// let cmd = Command::Move { point: Point { x: 42, y: 43 } }; -/// let mut allocator = NockStack::new(8 << 10 << 10, 0); +/// let mut allocator = NockStack::new(nockvm::mem::NOCK_STACK_SIZE_TINY, 0); /// let noun = cmd.to_noun(&mut allocator); /// /// // When encoded: %stop /// let stop = Command::Stop; -/// let mut allocator = NockStack::new(8 << 10 << 10, 0); +/// let mut allocator = NockStack::new(nockvm::mem::NOCK_STACK_SIZE_TINY, 0); /// let noun = stop.to_noun(&mut allocator); /// ``` pub fn derive_noun_encode(input: TokenStream) -> TokenStream { @@ -473,129 +559,195 @@ pub fn derive_noun_encode(input: TokenStream) -> TokenStream { } } } else { - let cases: Vec<_> = data - .variants - .iter() - .map(|variant| { - let variant_name = &variant.ident; - let tag = variant - .attrs - .iter() - .find_map(|attr| { - if attr.path().is_ident("noun") { - attr.parse_args::().ok().and_then(|nv| { - if nv.path.is_ident("tag") { - if let syn::Expr::Lit(syn::ExprLit { - lit: syn::Lit::Str(s), - .. - }) = nv.value - { - Some(s.value()) - } else { - None - } - } else { - None - } - }) - } else { - None - } - }) - .unwrap_or_else(|| variant_name.to_string().to_lowercase()); + let mut seen_tags = HashSet::new(); + for variant in data.variants.iter() { + let variant_untagged = parse_untagged_attr(&variant.attrs).unwrap_or(false); + if variant_untagged { + continue; + } + let tag = resolve_variant_tag(&variant.attrs, &variant.ident); + let key = tag_value_matches_key(&tag); + if !seen_tags.insert(key) { + return syn::Error::new_spanned( + &variant.ident, "duplicate enum tag in #[derive(NounEncode)]", + ) + .to_compile_error() + .into(); + } + } - // Check variant-level tagged attribute, fallback to enum-level - let is_tagged = - parse_tagged_attr(&variant.attrs).unwrap_or(enum_tagged.unwrap_or(false)); + let mut cases = Vec::new(); + for variant in data.variants.iter() { + let variant_name = &variant.ident; + let tag = resolve_variant_tag(&variant.attrs, variant_name); + let tag_expr = encode_tag_expr(&tag); - match &variant.fields { - Fields::Named(fields) => { - let field_names: Vec<_> = fields - .named - .iter() - .map(|f| f.ident.as_ref().expect("named field must have ident")) - .collect(); + // Check variant-level tagged attribute, fallback to enum-level + let is_tagged = + parse_tagged_attr(&variant.attrs).unwrap_or(enum_tagged.unwrap_or(false)); + let variant_untagged = parse_untagged_attr(&variant.attrs).unwrap_or(false); - if is_tagged { - // Tagged encoding: [%tag [[%field1 value1] [%field2 value2] ...]] + let case = match &variant.fields { + Fields::Named(fields) => { + let field_names: Vec<_> = fields + .named + .iter() + .map(|f| f.ident.as_ref().expect("named field must have ident")) + .collect(); + + if variant_untagged { + if fields.named.is_empty() { quote! { - #name::#variant_name { #(#field_names),* } => { - let tag = ::nockvm::ext::make_tas(allocator, #tag).as_noun(); - let mut field_nouns = Vec::new(); - #( - let field_tag = ::nockvm::ext::make_tas(allocator, stringify!(#field_names)).as_noun(); - let field_value = ::noun_serde::NounEncode::to_noun(#field_names, allocator); - field_nouns.push(::nockvm::noun::T(allocator, &[field_tag, field_value])); - )* - // Fold field pairs into a list: [[k1 v1] [[k2 v2] ... 0]] - let data = field_nouns.into_iter().rev().fold(::nockvm::noun::D(0), |acc, pair_noun| { - if acc.is_atom() && acc.as_atom().map_or(false, |a| a.as_u64() == Ok(0)) { - ::nockvm::noun::T(allocator, &[pair_noun, ::nockvm::noun::D(0)]) // Base case: [last_pair 0] - } else { - ::nockvm::noun::T(allocator, &[pair_noun, acc]) - } - }); - ::nockvm::noun::T(allocator, &[tag, data]) + #name::#variant_name { } => { + ::nockvm::noun::D(0) + } + } + } else if fields.named.len() == 1 { + let field_name = field_names[0]; + quote! { + #name::#variant_name { #field_name } => { + ::noun_serde::NounEncode::to_noun(#field_name, allocator) } } } else { - // Untagged encoding: [%tag [value1 value2 ...]] + let field_encoders = fields.named.iter().enumerate().map(|(i, field)| { + let field_name = field.ident.as_ref().expect("named field must have ident"); + let field_var = format_ident!("encoded_field_{}", i); + quote! { + let #field_var = ::noun_serde::NounEncode::to_noun(#field_name, allocator); + encoded_fields.push(#field_var); + } + }); quote! { #name::#variant_name { #(#field_names),* } => { - let tag = ::nockvm::ext::make_tas(allocator, #tag).as_noun(); - let mut field_nouns = vec![tag]; - #( - let field_noun = ::noun_serde::NounEncode::to_noun(#field_names, allocator); - field_nouns.push(field_noun); - )* - ::nockvm::noun::T(allocator, &field_nouns) + let mut encoded_fields = Vec::new(); + #(#field_encoders)* + let mut result = encoded_fields.pop().unwrap(); + for noun in encoded_fields.into_iter().rev() { + result = ::nockvm::noun::T(allocator, &[noun, result]); + } + result } } } + } else if is_tagged { + quote! { + #name::#variant_name { #(#field_names),* } => { + let tag = #tag_expr; + let mut field_nouns = Vec::new(); + #( + let field_tag = ::nockvm::ext::make_tas(allocator, stringify!(#field_names)).as_noun(); + let field_value = ::noun_serde::NounEncode::to_noun(#field_names, allocator); + field_nouns.push(::nockvm::noun::T(allocator, &[field_tag, field_value])); + )* + let data = field_nouns.into_iter().rev().fold(::nockvm::noun::D(0), |acc, pair_noun| { + if unsafe { acc.raw_equals(&::nockvm::noun::D(0)) } { + ::nockvm::noun::T(allocator, &[pair_noun, ::nockvm::noun::D(0)]) + } else { + ::nockvm::noun::T(allocator, &[pair_noun, acc]) + } + }); + ::nockvm::noun::T(allocator, &[tag, data]) + } + } + } else { + quote! { + #name::#variant_name { #(#field_names),* } => { + let tag = #tag_expr; + let mut field_nouns = vec![tag]; + #( + let field_noun = ::noun_serde::NounEncode::to_noun(#field_names, allocator); + field_nouns.push(field_noun); + )* + ::nockvm::noun::T(allocator, &field_nouns) + } + } } - Fields::Unnamed(fields) => { - let field_count = fields.unnamed.len(); - let field_idents: Vec<_> = (0..field_count) - .map(|i| format_ident!("field_{}", i)) - .collect(); + } + Fields::Unnamed(fields) => { + let field_count = fields.unnamed.len(); + let field_idents: Vec<_> = (0..field_count) + .map(|i| format_ident!("field_{}", i)) + .collect(); - if field_count == 1 { - let _ty = &fields.unnamed[0].ty; + if variant_untagged { + if field_count == 0 { + quote! { + #name::#variant_name => { + ::nockvm::noun::D(0) + } + } + } else if field_count == 1 { quote! { #name::#variant_name(value) => { - let tag = ::nockvm::ext::make_tas(allocator, #tag).as_noun(); - let data = ::noun_serde::NounEncode::to_noun(value, allocator); - ::nockvm::noun::T(allocator, &[tag, data]) + ::noun_serde::NounEncode::to_noun(value, allocator) } } } else { - let field_idents_rev = field_idents.iter().rev().collect::>(); - let first_field = field_idents_rev[0]; - let rest_fields = &field_idents_rev[1..]; - + let field_encoders = field_idents.iter().enumerate().map(|(i, ident)| { + let field_var = format_ident!("encoded_field_{}", i); + quote! { + let #field_var = ::noun_serde::NounEncode::to_noun(#ident, allocator); + encoded_fields.push(#field_var); + } + }); quote! { #name::#variant_name(#(#field_idents),*) => { - let tag = ::nockvm::ext::make_tas(allocator, #tag).as_noun(); - // Build nested cell structure right-to-left - let mut data = ::noun_serde::NounEncode::to_noun(#first_field, allocator); - #( - data = ::nockvm::noun::T(allocator, &[::noun_serde::NounEncode::to_noun(#rest_fields, allocator), data]); - )* - ::nockvm::noun::T(allocator, &[tag, data]) + let mut encoded_fields = Vec::new(); + #(#field_encoders)* + let mut result = encoded_fields.pop().unwrap(); + for noun in encoded_fields.into_iter().rev() { + result = ::nockvm::noun::T(allocator, &[noun, result]); + } + result } } } + } else if field_count == 1 { + quote! { + #name::#variant_name(value) => { + let tag = #tag_expr; + let data = ::noun_serde::NounEncode::to_noun(value, allocator); + ::nockvm::noun::T(allocator, &[tag, data]) + } + } + } else { + let field_idents_rev = + field_idents.iter().rev().collect::>(); + let first_field = field_idents_rev[0]; + let rest_fields = &field_idents_rev[1..]; + + quote! { + #name::#variant_name(#(#field_idents),*) => { + let tag = #tag_expr; + let mut data = ::noun_serde::NounEncode::to_noun(#first_field, allocator); + #( + let next = ::noun_serde::NounEncode::to_noun(#rest_fields, allocator); + data = ::nockvm::noun::T(allocator, &[next, data]); + )* + ::nockvm::noun::T(allocator, &[tag, data]) + } + } } - Fields::Unit => { + } + Fields::Unit => { + if variant_untagged { quote! { #name::#variant_name => { - ::nockvm::ext::make_tas(allocator, #tag).as_noun() + ::nockvm::noun::D(0) + } + } + } else { + quote! { + #name::#variant_name => { + #tag_expr } } } } - }) - .collect(); + }; + cases.push(case); + } quote! { match self { @@ -657,7 +809,7 @@ pub fn derive_noun_decode(input: TokenStream) -> TokenStream { quote! { ::tracing::trace!(target: "noun_serde_decode", "Decoding {} (single field struct), is_atom={}, is_cell={}", #name_str, noun.is_atom(), noun.is_cell()); ::tracing::trace!(target: "noun_serde_decode", " field={} type={}", #field_name_str, stringify!(#field_type)); - let #field_name = <#field_type as ::noun_serde::NounDecode>::from_noun(noun) + let #field_name = <#field_type as ::noun_serde::NounDecode>::from_noun_handle(&noun) .map_err(|e| { ::tracing::trace!(target: "noun_serde_decode", " FAILED decoding field {} in {}: {:?}", #field_name_str, #name_str, e); e @@ -714,13 +866,13 @@ pub fn derive_noun_decode(input: TokenStream) -> TokenStream { let field_name_str = name.to_string(); quote! { ::tracing::trace!(target: "noun_serde_decode", " field={} type={} axis={}", #field_name_str, stringify!(#ty), #axis); - let field_noun = ::nockvm::noun::Slots::slot(&cell, #axis) + let field_noun = cell.as_noun().slot(#axis) .map_err(|e| { ::tracing::trace!(target: "noun_serde_decode", " FAILED to get slot {} for field {} in {}: {:?}", #axis, #field_name_str, #name_str, e); ::noun_serde::NounDecodeError::ExpectedCell })?; ::tracing::trace!(target: "noun_serde_decode", " field={} is_atom={} is_cell={}", #field_name_str, field_noun.is_atom(), field_noun.is_cell()); - let #name = <#ty as ::noun_serde::NounDecode>::from_noun(&field_noun) + let #name = <#ty as ::noun_serde::NounDecode>::from_noun_handle(&field_noun) .map_err(|e| { ::tracing::trace!(target: "noun_serde_decode", " FAILED decoding field {} in {}: {:?}", #field_name_str, #name_str, e); e @@ -750,7 +902,7 @@ pub fn derive_noun_decode(input: TokenStream) -> TokenStream { quote! { ::tracing::trace!(target: "noun_serde_decode", "Decoding {} (single field tuple), is_atom={}, is_cell={}", #name_str, noun.is_atom(), noun.is_cell()); ::tracing::trace!(target: "noun_serde_decode", " field=0 type={}", stringify!(#field_type)); - let field_0 = <#field_type as ::noun_serde::NounDecode>::from_noun(noun) + let field_0 = <#field_type as ::noun_serde::NounDecode>::from_noun_handle(&noun) .map_err(|e| { ::tracing::trace!(target: "noun_serde_decode", " FAILED decoding field 0 in {}: {:?}", #name_str, e); e @@ -815,13 +967,13 @@ pub fn derive_noun_decode(input: TokenStream) -> TokenStream { quote! { ::tracing::trace!(target: "noun_serde_decode", " field={} type={} axis={}", #field_num_str, stringify!(#field_type), #axis); - let field_noun = ::nockvm::noun::Slots::slot(&cell, #axis) + let field_noun = cell.as_noun().slot(#axis) .map_err(|e| { ::tracing::trace!(target: "noun_serde_decode", " FAILED to get slot {} for field {} in {}: {:?}", #axis, #field_num_str, #name_str, e); ::noun_serde::NounDecodeError::FieldError(stringify!(#field_ident).to_string(), "Missing field".into()) })?; ::tracing::trace!(target: "noun_serde_decode", " field={} is_atom={} is_cell={}", #field_num_str, field_noun.is_atom(), field_noun.is_cell()); - let #field_ident = <#field_type as ::noun_serde::NounDecode>::from_noun(&field_noun) + let #field_ident = <#field_type as ::noun_serde::NounDecode>::from_noun_handle(&field_noun) .map_err(|e| { ::tracing::trace!(target: "noun_serde_decode", " FAILED decoding field {} in {}: {:?}", #field_num_str, #name_str, e); ::noun_serde::NounDecodeError::FieldError(stringify!(#field_ident).to_string(), e.to_string()) @@ -888,7 +1040,7 @@ pub fn derive_noun_decode(input: TokenStream) -> TokenStream { let field_name = field_names[0]; let field_type = field_types[0]; quote! { - let #field_name = <#field_type as ::noun_serde::NounDecode>::from_noun(noun)?; + let #field_name = <#field_type as ::noun_serde::NounDecode>::from_noun_handle(&noun)?; Ok(Self::#variant_name { #field_name }) } } else { @@ -926,9 +1078,9 @@ pub fn derive_noun_decode(input: TokenStream) -> TokenStream { }; let axis = custom_axis.unwrap_or(default_axis); quote! { - let field_noun = ::nockvm::noun::Slots::slot(&cell, #axis) + let field_noun = cell.as_noun().slot(#axis) .map_err(|_| ::noun_serde::NounDecodeError::ExpectedCell)?; - let #name = <#ty as ::noun_serde::NounDecode>::from_noun(&field_noun)?; + let #name = <#ty as ::noun_serde::NounDecode>::from_noun_handle(&field_noun)?; } }); quote! { @@ -965,7 +1117,7 @@ pub fn derive_noun_decode(input: TokenStream) -> TokenStream { } else if field_count == 1 { let field_type = field_types[0]; quote! { - let value = <#field_type as ::noun_serde::NounDecode>::from_noun(noun)?; + let value = <#field_type as ::noun_serde::NounDecode>::from_noun_handle(&noun)?; Ok(Self::#variant_name(value)) } } else { @@ -993,9 +1145,9 @@ pub fn derive_noun_decode(input: TokenStream) -> TokenStream { }; let axis = custom_axis.unwrap_or(default_axis); quote! { - let field_noun = ::nockvm::noun::Slots::slot(&cell, #axis) + let field_noun = cell.as_noun().slot(#axis) .map_err(|_| ::noun_serde::NounDecodeError::ExpectedCell)?; - let #name = <#ty as ::noun_serde::NounDecode>::from_noun(&field_noun)?; + let #name = <#ty as ::noun_serde::NounDecode>::from_noun_handle(&field_noun)?; } }); quote! { @@ -1042,57 +1194,240 @@ pub fn derive_noun_decode(input: TokenStream) -> TokenStream { Err(last_err.unwrap_or(::noun_serde::NounDecodeError::InvalidEnumVariant)) } } else { - let cases: Vec<_> = data.variants.iter().map(|variant| { - let variant_name = &variant.ident; - let tag = variant.attrs.iter() - .find_map(|attr| { - if attr.path().is_ident("noun") { - attr.parse_args::().ok() - .and_then(|nv| if nv.path.is_ident("tag") { - if let syn::Expr::Lit(syn::ExprLit { lit: syn::Lit::Str(s), .. }) = nv.value { - Some(s.value()) + let mut seen_tags = HashSet::new(); + for variant in data.variants.iter() { + let variant_untagged = parse_untagged_attr(&variant.attrs).unwrap_or(false); + if variant_untagged { + continue; + } + let tag = resolve_variant_tag(&variant.attrs, &variant.ident); + let key = tag_value_matches_key(&tag); + if !seen_tags.insert(key) { + return syn::Error::new_spanned( + &variant.ident, "duplicate enum tag in #[derive(NounDecode)]", + ) + .to_compile_error() + .into(); + } + } + + let untagged_attempts: Vec<_> = data + .variants + .iter() + .filter(|variant| parse_untagged_attr(&variant.attrs).unwrap_or(false)) + .map(|variant| { + let variant_name = &variant.ident; + let attempt = match &variant.fields { + Fields::Named(fields) => { + let field_names: Vec<_> = fields + .named + .iter() + .map(|f| f.ident.as_ref().expect("named field must have ident")) + .collect(); + let field_types: Vec<_> = fields.named.iter().map(|f| &f.ty).collect(); + + if field_names.is_empty() { + quote! { + let atom = noun + .as_atom() + .map_err(|_| ::noun_serde::NounDecodeError::ExpectedAtom)?; + let atom_u64 = atom + .as_u64() + .map_err(|_| ::noun_serde::NounDecodeError::InvalidEnumVariant)?; + if atom_u64 == 0 { + Ok(Self::#variant_name { }) } else { - None + Err(::noun_serde::NounDecodeError::InvalidEnumVariant) } + } + } else if field_names.len() == 1 { + let field_name = field_names[0]; + let field_type = field_types[0]; + quote! { + let #field_name = <#field_type as ::noun_serde::NounDecode>::from_noun_handle(&noun)?; + Ok(Self::#variant_name { #field_name }) + } + } else { + let num_fields = field_names.len(); + let field_decoders = field_names + .iter() + .zip(field_types.iter()) + .enumerate() + .map(|(i, (name, ty))| { + let field = fields + .named + .iter() + .find(|f| { + f.ident + .as_ref() + .expect("named field must have ident") + == *name + }) + .expect("field must exist"); + let custom_axis = parse_axis_attr(&field.attrs); + let default_axis = if i == 0 { + 2 + } else if i == num_fields - 1 { + let mut axis = 2; + for _ in 1..i { + axis = 2 * axis + 2; + } + axis + 1 + } else { + let mut axis = 2; + for _ in 1..=i { + axis = 2 * axis + 2; + } + axis + }; + let axis = custom_axis.unwrap_or(default_axis); + quote! { + let field_noun = cell.as_noun().slot(#axis) + .map_err(|_| ::noun_serde::NounDecodeError::ExpectedCell)?; + let #name = <#ty as ::noun_serde::NounDecode>::from_noun_handle(&field_noun)?; + } + }); + quote! { + let cell = noun + .as_cell() + .map_err(|_| ::noun_serde::NounDecodeError::ExpectedCell)?; + #(#field_decoders)* + Ok(Self::#variant_name { #(#field_names),* }) + } + } + } + Fields::Unnamed(fields) => { + let field_count = fields.unnamed.len(); + let field_names: Vec<_> = (0..field_count) + .map(|i| format_ident!("field_{}", i)) + .collect(); + let field_types: Vec<_> = + fields.unnamed.iter().map(|f| &f.ty).collect(); + + if field_count == 0 { + quote! { + let atom = noun + .as_atom() + .map_err(|_| ::noun_serde::NounDecodeError::ExpectedAtom)?; + let atom_u64 = atom + .as_u64() + .map_err(|_| ::noun_serde::NounDecodeError::InvalidEnumVariant)?; + if atom_u64 == 0 { + Ok(Self::#variant_name) + } else { + Err(::noun_serde::NounDecodeError::InvalidEnumVariant) + } + } + } else if field_count == 1 { + let field_type = field_types[0]; + quote! { + let value = <#field_type as ::noun_serde::NounDecode>::from_noun_handle(&noun)?; + Ok(Self::#variant_name(value)) + } + } else { + let field_decoders = field_names + .iter() + .zip(field_types.iter()) + .enumerate() + .map(|(i, (name, ty))| { + let field = &fields.unnamed[i]; + let custom_axis = parse_axis_attr(&field.attrs); + let default_axis = if i == 0 { + 2 + } else if i == field_count - 1 { + let mut axis = 2; + for _ in 1..i { + axis = 2 * axis + 2; + } + axis + 1 + } else { + let mut axis = 2; + for _ in 1..=i { + axis = 2 * axis + 2; + } + axis + }; + let axis = custom_axis.unwrap_or(default_axis); + quote! { + let field_noun = cell.as_noun().slot(#axis) + .map_err(|_| ::noun_serde::NounDecodeError::ExpectedCell)?; + let #name = <#ty as ::noun_serde::NounDecode>::from_noun_handle(&field_noun)?; + } + }); + quote! { + let cell = noun + .as_cell() + .map_err(|_| ::noun_serde::NounDecodeError::ExpectedCell)?; + #(#field_decoders)* + Ok(Self::#variant_name(#(#field_names),*)) + } + } + } + Fields::Unit => { + quote! { + let atom = noun + .as_atom() + .map_err(|_| ::noun_serde::NounDecodeError::ExpectedAtom)?; + let atom_u64 = atom + .as_u64() + .map_err(|_| ::noun_serde::NounDecodeError::InvalidEnumVariant)?; + if atom_u64 == 0 { + Ok(Self::#variant_name) } else { - None - }) - } else { - None + Err(::noun_serde::NounDecodeError::InvalidEnumVariant) + } + } } - }) - .unwrap_or_else(|| variant_name.to_string().to_lowercase()); + }; + quote! { + if let Ok(value) = (|| -> Result { + #attempt + })() { + return Ok(value); + } + } + }) + .collect(); - // Check variant-level tagged attribute, fallback to enum-level - let is_tagged = parse_tagged_attr(&variant.attrs).unwrap_or(enum_tagged.unwrap_or(false)); + let mut cases = Vec::new(); + for variant in data + .variants + .iter() + .filter(|variant| !parse_untagged_attr(&variant.attrs).unwrap_or(false)) + { + let variant_name = &variant.ident; + let tag = resolve_variant_tag(&variant.attrs, variant_name); + let tag_match_expr = decode_tag_match_expr(&tag); - match &variant.fields { + let is_tagged = + parse_tagged_attr(&variant.attrs).unwrap_or(enum_tagged.unwrap_or(false)); + + let case = match &variant.fields { Fields::Named(fields) => { - let field_names: Vec<_> = fields.named.iter() + let field_names: Vec<_> = fields + .named + .iter() .map(|f| f.ident.as_ref().expect("named field must have ident")) .collect(); - - let field_types: Vec<_> = fields.named.iter() - .map(|f| &f.ty) - .collect(); - - let variant_name_str = variant_name.to_string(); + let field_types: Vec<_> = fields.named.iter().map(|f| &f.ty).collect(); if is_tagged { - // Tagged decoding: [%tag [[%field1 value1] [%field2 value2] ...]] - let variant_name_str = variant_name.to_string(); - let field_decoders = field_names.iter().zip(field_types.iter()).enumerate() + let field_decoders = field_names + .iter() + .zip(field_types.iter()) + .enumerate() .map(|(i, (name, ty))| { - let field_name_str = name.to_string(); - // Get the corresponding field - let field = fields.named.iter().find(|f| { - f.ident.as_ref().expect("named field must have ident") == *name - }).expect("field must exist"); - - // Check for custom axis + let field = fields + .named + .iter() + .find(|f| { + f.ident + .as_ref() + .expect("named field must have ident") + == *name + }) + .expect("field must exist"); let custom_axis = parse_axis_attr(&field.attrs); - - // Calculate the axis for right-branching binary tree let default_axis = if i == 0 { 2 } else { @@ -1102,55 +1437,48 @@ pub fn derive_noun_decode(input: TokenStream) -> TokenStream { } axis * 2 }; - let axis = custom_axis.unwrap_or(default_axis); quote! { - ::tracing::trace!(target: "noun_serde_decode", " variant={} field={} type={} axis={}", #variant_name_str, #field_name_str, stringify!(#ty), #axis); - let field_cell = ::nockvm::noun::Slots::slot(&data, #axis) - .map_err(|e| { - ::tracing::trace!(target: "noun_serde_decode", " FAILED to get slot {} for field {} in variant {}: {:?}", #axis, #field_name_str, #variant_name_str, e); - e - })? + let field_cell = data.slot(#axis) + .map_err(|_| ::noun_serde::NounDecodeError::ExpectedCell)? .as_cell() - .map_err(|e| { - ::tracing::trace!(target: "noun_serde_decode", " FAILED field {} in variant {} expected cell: {:?}", #field_name_str, #variant_name_str, e); - e - })?; - let #name = <#ty as ::noun_serde::NounDecode>::from_noun(&field_cell.tail()) - .map_err(|e| { - ::tracing::trace!(target: "noun_serde_decode", " FAILED decoding field {} in variant {}: {:?}", #field_name_str, #variant_name_str, e); - e - })?; - ::tracing::trace!(target: "noun_serde_decode", " SUCCESS decoded field {} in variant {}", #field_name_str, #variant_name_str); + .map_err(|_| ::noun_serde::NounDecodeError::ExpectedCell)?; + let #name = <#ty as ::noun_serde::NounDecode>::from_noun_handle(&field_cell.tail())?; } }); quote! { - tag if tag == #tag => { - ::tracing::trace!(target: "noun_serde_decode", "Matched variant {} (tagged named fields)", #variant_name_str); - if let Ok(cell) = noun.as_cell() { + if #tag_match_expr { + return (|| -> Result { + let cell = noun + .as_cell() + .map_err(|_| ::noun_serde::NounDecodeError::ExpectedCell)?; let data = cell.tail(); #(#field_decoders)* - ::tracing::trace!(target: "noun_serde_decode", "SUCCESS decoded variant {}", #variant_name_str); Ok(Self::#variant_name { #(#field_names),* }) - } else { - ::tracing::trace!(target: "noun_serde_decode", "FAILED variant {} expected cell", #variant_name_str); - Err(::noun_serde::NounDecodeError::ExpectedCell) - } + })(); } } } else { let num_fields = field_names.len(); - let field_decoders = field_names.iter().zip(field_types.iter()).enumerate() + let field_decoders = field_names + .iter() + .zip(field_types.iter()) + .enumerate() .map(|(i, (name, ty))| { - let field = fields.named.iter().find(|f| { - f.ident.as_ref().expect("named field must have ident") == *name - }).expect("field must exist"); - + let field = fields + .named + .iter() + .find(|f| { + f.ident + .as_ref() + .expect("named field must have ident") + == *name + }) + .expect("field must exist"); let custom_axis = parse_axis_attr(&field.attrs); - let default_axis = if i == 0 { - 2 // first field at axis 2 + 2 } else if i == num_fields - 1 { let mut axis = 2; for _ in 1..i { @@ -1164,64 +1492,42 @@ pub fn derive_noun_decode(input: TokenStream) -> TokenStream { } axis }; - let axis = custom_axis.unwrap_or(default_axis); - let field_name_str = name.to_string(); quote! { - ::tracing::trace!(target: "noun_serde_decode", " variant={} field={} type={} axis={}", #variant_name_str, #field_name_str, stringify!(#ty), #axis); - let field_noun = ::nockvm::noun::Slots::slot(&data_cell, #axis) - .map_err(|e| { - ::tracing::trace!(target: "noun_serde_decode", " FAILED to get slot {} for field {} in variant {}: {:?}", #axis, #field_name_str, #variant_name_str, e); - ::noun_serde::NounDecodeError::ExpectedCell - })?; - ::tracing::trace!(target: "noun_serde_decode", " variant={} field={} is_atom={} is_cell={}", #variant_name_str, #field_name_str, field_noun.is_atom(), field_noun.is_cell()); - let #name = <#ty as ::noun_serde::NounDecode>::from_noun(&field_noun) - .map_err(|e| { - ::tracing::trace!(target: "noun_serde_decode", " FAILED decoding field {} in variant {}: {:?}", #field_name_str, #variant_name_str, e); - e - })?; - ::tracing::trace!(target: "noun_serde_decode", " SUCCESS decoded field {} in variant {}", #field_name_str, #variant_name_str); + let field_noun = data_cell.slot(#axis) + .map_err(|_| ::noun_serde::NounDecodeError::ExpectedCell)?; + let #name = <#ty as ::noun_serde::NounDecode>::from_noun_handle(&field_noun)?; } }); let payload_atom_handler = if num_fields == 1 { let field_name = field_names[0]; let field_type = field_types[0]; - let field_name_str = field_name.to_string(); quote! { - ::tracing::trace!(target: "noun_serde_decode", "Matched variant {} (untagged named fields, payload atom)", #variant_name_str); - let #field_name = <#field_type as ::noun_serde::NounDecode>::from_noun(&payload) - .map_err(|e| { - ::tracing::trace!(target: "noun_serde_decode", " FAILED decoding field {} in variant {}: {:?}", #field_name_str, #variant_name_str, e); - e - })?; - ::tracing::trace!(target: "noun_serde_decode", "SUCCESS decoded variant {}", #variant_name_str); + let #field_name = <#field_type as ::noun_serde::NounDecode>::from_noun_handle(&payload)?; Ok(Self::#variant_name { #field_name }) } } else { quote! { - ::tracing::trace!(target: "noun_serde_decode", "FAILED variant {} payload atom but multiple fields expected", #variant_name_str); Err(::noun_serde::NounDecodeError::ExpectedCell) } }; quote! { - tag if tag == #tag => { - ::tracing::trace!(target: "noun_serde_decode", "Matched variant {} (untagged named fields)", #variant_name_str); - if let Ok(cell) = noun.as_cell() { + if #tag_match_expr { + return (|| -> Result { + let cell = noun + .as_cell() + .map_err(|_| ::noun_serde::NounDecodeError::ExpectedCell)?; let payload = cell.tail(); if let Ok(payload_cell) = payload.as_cell() { - let data_cell = payload_cell; + let data_cell = payload_cell.as_noun(); #(#field_decoders)* - ::tracing::trace!(target: "noun_serde_decode", "SUCCESS decoded variant {}", #variant_name_str); Ok(Self::#variant_name { #(#field_names),* }) } else { #payload_atom_handler } - } else { - ::tracing::trace!(target: "noun_serde_decode", "FAILED variant {} expected cell", #variant_name_str); - Err(::noun_serde::NounDecodeError::ExpectedCell) - } + })(); } } } @@ -1231,91 +1537,105 @@ pub fn derive_noun_decode(input: TokenStream) -> TokenStream { let field_names: Vec<_> = (0..field_count) .map(|i| format_ident!("field_{}", i)) .collect(); + let field_types: Vec<_> = + fields.unnamed.iter().map(|f| &f.ty).collect(); - let field_types: Vec<_> = fields.unnamed.iter() - .map(|f| &f.ty) - .collect(); - - if field_count == 1 { - let ty = &field_types[0]; + if field_count == 0 { quote! { - tag if tag == #tag => { - if let Ok(cell) = noun.as_cell() { - let value = <#ty as ::noun_serde::NounDecode>::from_noun(&cell.tail())?; + if #tag_match_expr { + return Ok(Self::#variant_name); + } + } + } else if field_count == 1 { + let ty = field_types[0]; + quote! { + if #tag_match_expr { + return (|| -> Result { + let cell = noun + .as_cell() + .map_err(|_| ::noun_serde::NounDecodeError::ExpectedCell)?; + let value = <#ty as ::noun_serde::NounDecode>::from_noun_handle(&cell.tail())?; Ok(Self::#variant_name(value)) - } else { - Err(::noun_serde::NounDecodeError::ExpectedCell) - } + })(); } } } else { - let field_idents_rev = field_names.iter().rev().collect::>(); - let first_field = field_idents_rev[0]; - let rest_fields = &field_idents_rev[1..]; + let field_decoders = field_names + .iter() + .zip(field_types.iter()) + .enumerate() + .map(|(i, (name, ty))| { + let field = &fields.unnamed[i]; + let custom_axis = parse_axis_attr(&field.attrs); + let default_axis = if i == 0 { + 2 + } else if i == field_count - 1 { + let mut axis = 2; + for _ in 1..i { + axis = 2 * axis + 2; + } + axis + 1 + } else { + let mut axis = 2; + for _ in 1..=i { + axis = 2 * axis + 2; + } + axis + }; + let axis = custom_axis.unwrap_or(default_axis); + quote! { + let field_noun = data_cell.slot(#axis) + .map_err(|_| ::noun_serde::NounDecodeError::ExpectedCell)?; + let #name = <#ty as ::noun_serde::NounDecode>::from_noun_handle(&field_noun)?; + } + }); quote! { - #name::#variant_name(#(#field_names),*) => { - let tag = ::nockvm::ext::make_tas(allocator, #tag).as_noun(); - // Build nested cell structure right-to-left - let mut data = ::noun_serde::NounEncode::to_noun(#first_field, allocator); - #( - data = ::nockvm::noun::T(allocator, &[::noun_serde::NounEncode::to_noun(#rest_fields, allocator), data]); - )* - ::nockvm::noun::T(allocator, &[tag, data]) + if #tag_match_expr { + return (|| -> Result { + let cell = noun + .as_cell() + .map_err(|_| ::noun_serde::NounDecodeError::ExpectedCell)?; + let data_cell = cell.tail(); + #(#field_decoders)* + Ok(Self::#variant_name(#(#field_names),*)) + })(); } } } } Fields::Unit => { quote! { - tag if tag == #tag => Ok(Self::#variant_name) + if #tag_match_expr { + return Ok(Self::#variant_name); + } } } - } - }).collect(); + }; + cases.push(case); + } quote! { - ::tracing::trace!(target: "noun_serde_decode", "Decoding enum {}, is_atom={}, is_cell={}", #name_str, noun.is_atom(), noun.is_cell()); - let tag = if let Ok(atom) = noun.as_atom() { - let bytes = atom.as_ne_bytes(); - let tag_str = ::std::str::from_utf8(bytes) - .map_err(|e| { - ::tracing::trace!(target: "noun_serde_decode", "FAILED to decode tag for {} as UTF-8: {:?}", #name_str, e); - ::noun_serde::NounDecodeError::InvalidTag - })? - .trim_end_matches('\0') - .to_string(); - ::tracing::trace!(target: "noun_serde_decode", "Decoded tag for {} (from atom): {:?}", #name_str, tag_str); - tag_str + #(#untagged_attempts)* + + let tag_noun_handle = if noun.is_atom() { + noun } else if let Ok(cell) = noun.as_cell() { - let atom = cell.head().as_atom() - .map_err(|e| { - ::tracing::trace!(target: "noun_serde_decode", "FAILED to decode tag for {}, head is not atom", #name_str); - ::noun_serde::NounDecodeError::InvalidTag - })?; - let bytes = atom.as_ne_bytes(); - let tag_str = ::std::str::from_utf8(bytes) - .map_err(|e| { - ::tracing::trace!(target: "noun_serde_decode", "FAILED to decode tag for {} as UTF-8: {:?}", #name_str, e); - ::noun_serde::NounDecodeError::InvalidTag - })? - .trim_end_matches('\0') - .to_string(); - ::tracing::trace!(target: "noun_serde_decode", "Decoded tag for {} (from cell head): {:?}", #name_str, tag_str); - tag_str + cell.head() } else { - ::tracing::trace!(target: "noun_serde_decode", "FAILED to decode tag for {}, neither atom nor cell", #name_str); return Err(::noun_serde::NounDecodeError::InvalidEnumData); }; + let string_tag = tag_noun_handle + .as_atom() + .ok() + .and_then(|atom| { + ::std::str::from_utf8(atom.as_ne_bytes()) + .ok() + .map(|tag| tag.trim_end_matches('\0').to_string()) + }); - ::tracing::trace!(target: "noun_serde_decode", "Matching enum {} with tag {:?}", #name_str, tag); - match tag.as_str() { - #(#cases,)* - _ => { - ::tracing::trace!(target: "noun_serde_decode", "FAILED to match variant for {} with tag {:?}", #name_str, tag); - Err(::noun_serde::NounDecodeError::InvalidEnumVariant) - } - } + #(#cases)* + Err(::noun_serde::NounDecodeError::InvalidEnumVariant) } } } @@ -1327,7 +1647,11 @@ pub fn derive_noun_decode(input: TokenStream) -> TokenStream { // Generate the impl block let expanded = quote! { impl ::noun_serde::NounDecode for #name { - fn from_noun(noun: &::nockvm::noun::Noun) -> Result { + fn from_noun( + noun: &::nockvm::noun::Noun, + space: &::nockvm::noun::NounSpace, + ) -> Result { + let noun = noun.in_space(space); #decode_impl } } @@ -1335,4 +1659,3 @@ pub fn derive_noun_decode(input: TokenStream) -> TokenStream { TokenStream::from(expanded) } - diff --git a/crates/noun-serde-derive/tests/struct_encoding_test.rs b/crates/noun-serde-derive/tests/struct_encoding_test.rs index 887b8ad44..358ae1579 100644 --- a/crates/noun-serde-derive/tests/struct_encoding_test.rs +++ b/crates/noun-serde-derive/tests/struct_encoding_test.rs @@ -2,6 +2,31 @@ use nockvm::mem::NockStack; use noun_serde::{NounDecode, NounEncode}; +struct StackGuard { + stack: NockStack, +} + +impl StackGuard { + fn new(words: usize) -> Self { + let stack = NockStack::new(words, 0); + Self { stack } + } +} + +impl std::ops::Deref for StackGuard { + type Target = NockStack; + + fn deref(&self) -> &Self::Target { + &self.stack + } +} + +impl std::ops::DerefMut for StackGuard { + fn deref_mut(&mut self) -> &mut Self::Target { + &mut self.stack + } +} + #[derive(Debug, Clone, PartialEq, NounEncode, NounDecode)] struct SingleField { x: u64, @@ -51,13 +76,14 @@ struct FiveFields { #[test] fn test_struct_encoding_no_terminator() { - let mut stack = NockStack::new(8 << 10 << 10, 0); + let mut stack = StackGuard::new(8 << 10 << 10); + let space = stack.noun_space(); // Test single field - should encode as just the field value let single = SingleField { x: 42 }; - let encoded = single.to_noun(&mut stack); + let encoded = single.to_noun(&mut *stack); // Test that it decodes correctly - this is the actual test - let decoded = SingleField::from_noun(&encoded).unwrap(); + let decoded = SingleField::from_noun(&encoded, &space).unwrap(); assert_eq!(single, decoded); // Also verify it's a single atom (not wrapped in a cell) assert!( @@ -67,15 +93,31 @@ fn test_struct_encoding_no_terminator() { // Test two fields - should encode as [x y] let two = TwoFields { x: 42, y: 43 }; - let encoded = two.to_noun(&mut stack); - let decoded = TwoFields::from_noun(&encoded).unwrap(); + let encoded = two.to_noun(&mut *stack); + let decoded = TwoFields::from_noun(&encoded, &space).unwrap(); assert_eq!(two, decoded); // Verify it's a cell with two atoms let cell = encoded .as_cell() .expect("Two fields should encode as a cell"); - assert_eq!(cell.head().as_atom().unwrap().as_u64().unwrap(), 42); - assert_eq!(cell.tail().as_atom().unwrap().as_u64().unwrap(), 43); + assert_eq!( + cell.in_space(&space) + .head() + .as_atom() + .unwrap() + .as_u64() + .unwrap(), + 42 + ); + assert_eq!( + cell.in_space(&space) + .tail() + .as_atom() + .unwrap() + .as_u64() + .unwrap(), + 43 + ); // Test three fields - should encode as [x [y z]] let three = ThreeFields { @@ -83,25 +125,45 @@ fn test_struct_encoding_no_terminator() { y: 43, z: 44, }; - let encoded = three.to_noun(&mut stack); - let decoded = ThreeFields::from_noun(&encoded).unwrap(); + let encoded = three.to_noun(&mut *stack); + let decoded = ThreeFields::from_noun(&encoded, &space).unwrap(); assert_eq!(three, decoded); // Verify structure: [42 [43 44]] let cell = encoded .as_cell() .expect("Three fields should encode as a cell"); - assert_eq!(cell.head().as_atom().unwrap().as_u64().unwrap(), 42); - let tail_cell = cell.tail().as_cell().expect("Tail should be a cell"); + assert_eq!( + cell.in_space(&space) + .head() + .as_atom() + .unwrap() + .as_u64() + .unwrap(), + 42 + ); + let tail_cell = cell + .in_space(&space) + .tail() + .as_cell() + .expect("Tail should be a cell"); assert_eq!(tail_cell.head().as_atom().unwrap().as_u64().unwrap(), 43); assert_eq!(tail_cell.tail().as_atom().unwrap().as_u64().unwrap(), 44); // Test empty struct - should encode as 0 let empty = EmptyStruct; - let encoded = empty.to_noun(&mut stack); - let decoded = EmptyStruct::from_noun(&encoded).unwrap(); + let encoded = empty.to_noun(&mut *stack); + let decoded = EmptyStruct::from_noun(&encoded, &space).unwrap(); assert_eq!(empty, decoded); // Verify it's atom 0 - assert_eq!(encoded.as_atom().unwrap().as_u64().unwrap(), 0); + assert_eq!( + encoded + .in_space(&space) + .as_atom() + .unwrap() + .as_u64() + .unwrap(), + 0 + ); // Test four fields - should encode as [a [b [c d]]] let four = FourFields { @@ -110,15 +172,27 @@ fn test_struct_encoding_no_terminator() { c: 102, d: 103, }; - let encoded = four.to_noun(&mut stack); - let decoded = FourFields::from_noun(&encoded).unwrap(); + let encoded = four.to_noun(&mut *stack); + let decoded = FourFields::from_noun(&encoded, &space).unwrap(); assert_eq!(four, decoded); // Verify structure: [100 [101 [102 103]]] let cell = encoded .as_cell() .expect("Four fields should encode as a cell"); - assert_eq!(cell.head().as_atom().unwrap().as_u64().unwrap(), 100); - let tail1 = cell.tail().as_cell().expect("Tail should be a cell"); + assert_eq!( + cell.in_space(&space) + .head() + .as_atom() + .unwrap() + .as_u64() + .unwrap(), + 100 + ); + let tail1 = cell + .in_space(&space) + .tail() + .as_cell() + .expect("Tail should be a cell"); assert_eq!(tail1.head().as_atom().unwrap().as_u64().unwrap(), 101); let tail2 = tail1.tail().as_cell().expect("Tail should be a cell"); assert_eq!(tail2.head().as_atom().unwrap().as_u64().unwrap(), 102); @@ -132,15 +206,27 @@ fn test_struct_encoding_no_terminator() { y: 203, z: 204, }; - let encoded = five.to_noun(&mut stack); - let decoded = FiveFields::from_noun(&encoded).unwrap(); + let encoded = five.to_noun(&mut *stack); + let decoded = FiveFields::from_noun(&encoded, &space).unwrap(); assert_eq!(five, decoded); // Verify structure: [200 [201 [202 [203 204]]]] let cell = encoded .as_cell() .expect("Five fields should encode as a cell"); - assert_eq!(cell.head().as_atom().unwrap().as_u64().unwrap(), 200); - let tail1 = cell.tail().as_cell().expect("Tail should be a cell"); + assert_eq!( + cell.in_space(&space) + .head() + .as_atom() + .unwrap() + .as_u64() + .unwrap(), + 200 + ); + let tail1 = cell + .in_space(&space) + .tail() + .as_cell() + .expect("Tail should be a cell"); assert_eq!(tail1.head().as_atom().unwrap().as_u64().unwrap(), 201); let tail2 = tail1.tail().as_cell().expect("Tail should be a cell"); assert_eq!(tail2.head().as_atom().unwrap().as_u64().unwrap(), 202); @@ -151,43 +237,80 @@ fn test_struct_encoding_no_terminator() { #[test] fn test_tuple_struct_encoding_no_terminator() { - let mut stack = NockStack::new(8 << 10 << 10, 0); + let mut stack = StackGuard::new(8 << 10 << 10); + let space = stack.noun_space(); // Test single tuple field - should encode as just the field value let single = TupleSingle(42); - let encoded = single.to_noun(&mut stack); - let decoded = TupleSingle::from_noun(&encoded).unwrap(); + let encoded = single.to_noun(&mut *stack); + let decoded = TupleSingle::from_noun(&encoded, &space).unwrap(); assert_eq!(single, decoded); // Verify it's a single atom (not wrapped in a cell) assert!( encoded.as_atom().is_ok(), "Single tuple field should encode as an atom" ); - assert_eq!(encoded.as_atom().unwrap().as_u64().unwrap(), 42); + assert_eq!( + encoded + .in_space(&space) + .as_atom() + .unwrap() + .as_u64() + .unwrap(), + 42 + ); // Test two tuple fields - should encode as [first second] let double = TupleDouble(42, 43); - let encoded = double.to_noun(&mut stack); - let decoded = TupleDouble::from_noun(&encoded).unwrap(); + let encoded = double.to_noun(&mut *stack); + let decoded = TupleDouble::from_noun(&encoded, &space).unwrap(); assert_eq!(double, decoded); // Verify it's a cell with two atoms let cell = encoded .as_cell() .expect("Two tuple fields should encode as a cell"); - assert_eq!(cell.head().as_atom().unwrap().as_u64().unwrap(), 42); - assert_eq!(cell.tail().as_atom().unwrap().as_u64().unwrap(), 43); + assert_eq!( + cell.in_space(&space) + .head() + .as_atom() + .unwrap() + .as_u64() + .unwrap(), + 42 + ); + assert_eq!( + cell.in_space(&space) + .tail() + .as_atom() + .unwrap() + .as_u64() + .unwrap(), + 43 + ); // Test three tuple fields - should encode as [first [second third]] let triple = TupleTriple(42, 43, 44); - let encoded = triple.to_noun(&mut stack); - let decoded = TupleTriple::from_noun(&encoded).unwrap(); + let encoded = triple.to_noun(&mut *stack); + let decoded = TupleTriple::from_noun(&encoded, &space).unwrap(); assert_eq!(triple, decoded); // Verify structure: [42 [43 44]] let cell = encoded .as_cell() .expect("Three tuple fields should encode as a cell"); - assert_eq!(cell.head().as_atom().unwrap().as_u64().unwrap(), 42); - let tail_cell = cell.tail().as_cell().expect("Tail should be a cell"); + assert_eq!( + cell.in_space(&space) + .head() + .as_atom() + .unwrap() + .as_u64() + .unwrap(), + 42 + ); + let tail_cell = cell + .in_space(&space) + .tail() + .as_cell() + .expect("Tail should be a cell"); assert_eq!(tail_cell.head().as_atom().unwrap().as_u64().unwrap(), 43); assert_eq!(tail_cell.tail().as_atom().unwrap().as_u64().unwrap(), 44); } diff --git a/crates/noun-serde-derive/tests/tagged_union_lock.rs b/crates/noun-serde-derive/tests/tagged_union_lock.rs new file mode 100644 index 000000000..55b0169d5 --- /dev/null +++ b/crates/noun-serde-derive/tests/tagged_union_lock.rs @@ -0,0 +1,116 @@ +use nockvm::mem::NockStack; +use nockvm::noun::{NounSpace, D, T}; +use noun_serde::{NounDecode, NounEncode}; + +#[derive(Debug, Clone, PartialEq, Eq, NounEncode, NounDecode)] +enum DerivedLock { + #[noun(untagged)] + SpendCondition(u64), + #[noun(tag = 2)] + V2((u64, u64)), + #[noun(tag = 4)] + V4(((u64, u64), (u64, u64))), +} + +#[derive(Debug, Clone, PartialEq, Eq, NounEncode, NounDecode)] +enum MixedTags { + #[noun(tag = 0)] + Zero(u64), + #[noun(tag = "ok")] + Ok, +} + +#[derive(Debug, Clone, PartialEq, Eq, NounEncode, NounDecode)] +enum MultiFieldNumericTag { + #[noun(tag = 0)] + Payload(String, [u64; 3]), +} + +#[test] +fn lock_leaf_roundtrip_uses_untagged_variant() { + let mut stack = NockStack::new(nockvm::mem::NOCK_STACK_SIZE_TINY, 0); + let lock = DerivedLock::SpendCondition(42); + let noun = lock.to_noun(&mut stack); + let space = NounSpace::stack_only(&stack); + let atom = noun + .in_space(&space) + .as_atom() + .expect("leaf lock should encode as atom"); + assert_eq!(atom.as_u64().expect("leaf lock atom should fit"), 42); + + let decoded = DerivedLock::from_noun(&noun, &space).expect("decode lock leaf"); + assert_eq!(decoded, lock); +} + +#[test] +fn lock_tree_roundtrip_uses_integer_atom_tag_variants() { + let mut stack = NockStack::new(nockvm::mem::NOCK_STACK_SIZE_TINY, 0); + let lock = DerivedLock::V4(((11, 12), (13, 14))); + let noun = lock.to_noun(&mut stack); + let space = NounSpace::stack_only(&stack); + + let cell = noun + .in_space(&space) + .as_cell() + .expect("v4 lock encodes as tagged cell"); + let tag_atom = cell.head().as_atom().expect("tag must be atom"); + assert_eq!(tag_atom.as_u64().expect("tag should fit"), 4); + + let decoded = DerivedLock::from_noun(&noun, &space).expect("decode lock v4"); + assert_eq!(decoded, lock); +} + +#[test] +fn lock_tree_decodes_from_manual_tagged_noun() { + let mut stack = NockStack::new(nockvm::mem::NOCK_STACK_SIZE_TINY, 0); + let payload = T(&mut stack, &[D(7), D(8)]); + let noun = T(&mut stack, &[D(2), payload]); + let space = NounSpace::stack_only(&stack); + + let decoded = DerivedLock::from_noun(&noun, &space).expect("decode lock v2 from noun"); + assert_eq!(decoded, DerivedLock::V2((7, 8))); +} + +#[test] +fn mixed_string_and_integer_tags_roundtrip() { + let mut stack = NockStack::new(nockvm::mem::NOCK_STACK_SIZE_TINY, 0); + let space = NounSpace::stack_only(&stack); + + let zero = MixedTags::Zero(9); + let zero_noun = zero.to_noun(&mut stack); + assert_eq!( + MixedTags::from_noun(&zero_noun, &space).expect("decode zero"), + zero + ); + + let ok = MixedTags::Ok; + let ok_noun = ok.to_noun(&mut stack); + assert_eq!( + MixedTags::from_noun(&ok_noun, &space).expect("decode ok"), + ok + ); +} + +#[test] +fn multi_field_integer_tag_roundtrip_uses_numeric_atom() { + let mut stack = NockStack::new(nockvm::mem::NOCK_STACK_SIZE_TINY, 0); + let value = MultiFieldNumericTag::Payload("base".to_string(), [11, 22, 33]); + let noun = value.to_noun(&mut stack); + let space = NounSpace::stack_only(&stack); + + let cell = noun + .in_space(&space) + .as_cell() + .expect("payload should encode as tagged cell"); + let tag = cell + .head() + .as_atom() + .expect("payload tag should be atom") + .as_u64() + .expect("payload tag should fit"); + assert_eq!(tag, 0); + + let decoded = + MultiFieldNumericTag::from_noun(&noun, &space).expect("decode multi-field payload"); + assert_eq!(decoded, value); +} diff --git a/crates/noun-serde-derive/tests/untagged_enum.rs b/crates/noun-serde-derive/tests/untagged_enum.rs index abd355d55..0408b6950 100644 --- a/crates/noun-serde-derive/tests/untagged_enum.rs +++ b/crates/noun-serde-derive/tests/untagged_enum.rs @@ -33,9 +33,14 @@ enum UntaggedWithUnit { Pair(u64, u64), } +fn decode_in_stack(noun: &nockvm::noun::Noun, stack: &NockStack) -> T { + let space = stack.noun_space(); + T::from_noun(noun, &space).expect("decode noun") +} + #[test] fn untagged_enum_roundtrip_full() { - let mut stack = NockStack::new(8 << 10 << 10, 0); + let mut stack = NockStack::new(nockvm::mem::NOCK_STACK_SIZE_TINY, 0); let proof = LockMerkleProof::Full(LockMerkleProofFull { spend_condition: 1, axis: 2, @@ -43,41 +48,41 @@ fn untagged_enum_roundtrip_full() { version: tas!(b"full"), }); let noun = proof.to_noun(&mut stack); - let decoded = LockMerkleProof::from_noun(&noun).expect("decode full"); + let decoded: LockMerkleProof = decode_in_stack(&noun, &stack); assert_eq!(decoded, proof); } #[test] fn untagged_enum_decodes_stub() { - let mut stack = NockStack::new(8 << 10 << 10, 0); + let mut stack = NockStack::new(nockvm::mem::NOCK_STACK_SIZE_TINY, 0); let proof = LockMerkleProofStub { spend_condition: 7, axis: 1, proof: 9, }; let noun = proof.to_noun(&mut stack); - let decoded = LockMerkleProof::from_noun(&noun).expect("decode stub"); + let decoded: LockMerkleProof = decode_in_stack(&noun, &stack); assert_eq!(decoded, LockMerkleProof::Stub(proof)); } #[test] fn untagged_enum_stub_roundtrip_via_wrapper() { // Encode Stub via wrapper, decode back - should stay Stub - let mut stack = NockStack::new(8 << 10 << 10, 0); + let mut stack = NockStack::new(nockvm::mem::NOCK_STACK_SIZE_TINY, 0); let proof = LockMerkleProof::Stub(LockMerkleProofStub { spend_condition: 100, axis: 200, proof: 300, }); let noun = proof.to_noun(&mut stack); - let decoded = LockMerkleProof::from_noun(&noun).expect("decode stub via wrapper"); + let decoded: LockMerkleProof = decode_in_stack(&noun, &stack); assert_eq!(decoded, proof); } #[test] fn untagged_enum_full_has_correct_version() { // Verify that decoded Full has version = tas!(b"full") - let mut stack = NockStack::new(8 << 10 << 10, 0); + let mut stack = NockStack::new(nockvm::mem::NOCK_STACK_SIZE_TINY, 0); let proof = LockMerkleProof::Full(LockMerkleProofFull { spend_condition: 1, axis: 2, @@ -85,7 +90,7 @@ fn untagged_enum_full_has_correct_version() { version: tas!(b"full"), }); let noun = proof.to_noun(&mut stack); - let decoded = LockMerkleProof::from_noun(&noun).expect("decode full"); + let decoded = decode_in_stack(&noun, &stack); match decoded { LockMerkleProof::Full(full) => { assert_eq!(full.version, tas!(b"full")); @@ -97,16 +102,16 @@ fn untagged_enum_full_has_correct_version() { #[test] fn untagged_enum_discriminates_by_structure() { // 3-tuple decodes as Stub, 4-tuple decodes as Full - let mut stack = NockStack::new(8 << 10 << 10, 0); + let mut stack = NockStack::new(nockvm::mem::NOCK_STACK_SIZE_TINY, 0); // Build a 3-tuple manually: [spend_condition axis proof] let three_tuple = T(&mut stack, &[D(10), D(20), D(30)]); - let decoded_3 = LockMerkleProof::from_noun(&three_tuple).expect("decode 3-tuple"); + let decoded_3 = decode_in_stack(&three_tuple, &stack); assert!(matches!(decoded_3, LockMerkleProof::Stub(_))); // Build a 4-tuple manually: [spend_condition axis proof version] let four_tuple = T(&mut stack, &[D(10), D(20), D(30), D(tas!(b"full"))]); - let decoded_4 = LockMerkleProof::from_noun(&four_tuple).expect("decode 4-tuple"); + let decoded_4 = decode_in_stack(&four_tuple, &stack); assert!(matches!(decoded_4, LockMerkleProof::Full(_))); } @@ -114,11 +119,10 @@ fn untagged_enum_discriminates_by_structure() { fn untagged_enum_four_tuple_with_wrong_version_still_decodes_as_full() { // A 4-tuple with version != %full still decodes as Full (structure-based discrimination) // The version field is just data - discrimination is by arity - let mut stack = NockStack::new(8 << 10 << 10, 0); + let mut stack = NockStack::new(nockvm::mem::NOCK_STACK_SIZE_TINY, 0); let four_tuple_wrong_version = T(&mut stack, &[D(10), D(20), D(30), D(999)]); - let decoded = LockMerkleProof::from_noun(&four_tuple_wrong_version) - .expect("decode 4-tuple with wrong version"); + let decoded = decode_in_stack(&four_tuple_wrong_version, &stack); match decoded { LockMerkleProof::Full(full) => { // It decoded as Full, but version is wrong @@ -132,7 +136,7 @@ fn untagged_enum_four_tuple_with_wrong_version_still_decodes_as_full() { #[test] fn untagged_enum_full_and_stub_decode_correctly_with_same_prefix() { // Full and Stub with same first 3 fields should still decode correctly - let mut stack = NockStack::new(8 << 10 << 10, 0); + let mut stack = NockStack::new(nockvm::mem::NOCK_STACK_SIZE_TINY, 0); let stub = LockMerkleProofStub { spend_condition: 1, @@ -150,8 +154,8 @@ fn untagged_enum_full_and_stub_decode_correctly_with_same_prefix() { let stub_noun = stub.to_noun(&mut stack); let full_noun = full.to_noun(&mut stack); - let decoded_stub = LockMerkleProof::from_noun(&stub_noun).expect("decode stub"); - let decoded_full = LockMerkleProof::from_noun(&full_noun).expect("decode full"); + let decoded_stub = decode_in_stack(&stub_noun, &stack); + let decoded_full = decode_in_stack(&full_noun, &stack); // Stub decodes as Stub, Full decodes as Full assert!(matches!(decoded_stub, LockMerkleProof::Stub(_))); @@ -179,17 +183,17 @@ fn untagged_enum_full_and_stub_decode_correctly_with_same_prefix() { #[test] fn untagged_enum_unit_variant_requires_zero_atom() { - let mut stack = NockStack::new(8 << 10 << 10, 0); + let mut stack = NockStack::new(nockvm::mem::NOCK_STACK_SIZE_TINY, 0); let atom_one = D(1); - let decoded_atom = UntaggedWithUnit::from_noun(&atom_one).expect("decode atom"); + let decoded_atom: UntaggedWithUnit = decode_in_stack(&atom_one, &stack); assert_eq!(decoded_atom, UntaggedWithUnit::Value(1)); let cell = T(&mut stack, &[D(2), D(3)]); - let decoded_cell = UntaggedWithUnit::from_noun(&cell).expect("decode cell"); + let decoded_cell: UntaggedWithUnit = decode_in_stack(&cell, &stack); assert_eq!(decoded_cell, UntaggedWithUnit::Pair(2, 3)); let zero = D(0); - let decoded_zero = UntaggedWithUnit::from_noun(&zero).expect("decode zero atom"); + let decoded_zero: UntaggedWithUnit = decode_in_stack(&zero, &stack); assert_eq!(decoded_zero, UntaggedWithUnit::Empty); } diff --git a/crates/noun-serde-derive/tests/untagged_enum_regression.rs b/crates/noun-serde-derive/tests/untagged_enum_regression.rs index 92ac917e1..d7d3f58e6 100644 --- a/crates/noun-serde-derive/tests/untagged_enum_regression.rs +++ b/crates/noun-serde-derive/tests/untagged_enum_regression.rs @@ -14,26 +14,28 @@ enum RecipientSpec { #[test] fn untagged_named_variant_decodes_all_fields() { - let mut stack = NockStack::new(8 << 10 << 10, 0); + let mut stack = NockStack::new(nockvm::mem::NOCK_STACK_SIZE_TINY, 0); + let space = stack.noun_space(); let expected = RecipientSpec::Pkh { hash: Hash([0x1234, 0x5678]), amount: 42, }; let noun = expected.to_noun(&mut stack); - let decoded = RecipientSpec::from_noun(&noun).expect("recipient spec decodes"); + let decoded = RecipientSpec::from_noun(&noun, &space).expect("recipient spec decodes"); assert_eq!(decoded, expected); } #[test] fn untagged_named_variant_with_multiple_fields_decodes_all_fields() { - let mut stack = NockStack::new(8 << 10 << 10, 0); + let mut stack = NockStack::new(nockvm::mem::NOCK_STACK_SIZE_TINY, 0); + let space = stack.noun_space(); let expected = RecipientSpec::Multi { first: 0xaaaa, second: 0xbbbb, }; let noun = expected.to_noun(&mut stack); - let decoded = RecipientSpec::from_noun(&noun).expect("multi recipient spec decodes"); + let decoded = RecipientSpec::from_noun(&noun, &space).expect("multi recipient spec decodes"); assert_eq!(decoded, expected); } diff --git a/crates/noun-serde-derive/tests/wallet_types_derived.rs b/crates/noun-serde-derive/tests/wallet_types_derived.rs index 2a8b58215..e7ef323ff 100644 --- a/crates/noun-serde-derive/tests/wallet_types_derived.rs +++ b/crates/noun-serde-derive/tests/wallet_types_derived.rs @@ -3,9 +3,35 @@ use std::collections::{HashMap, HashSet}; #[allow(unused)] use nockapp::utils::make_tas; +use nockvm::mem::NockStack; use nockvm::noun::FullDebugCell; use noun_serde::{NounDecode, NounEncode}; +struct StackGuard { + stack: NockStack, +} + +impl StackGuard { + fn new(words: usize) -> Self { + let stack = NockStack::new(words, 0); + Self { stack } + } +} + +impl std::ops::Deref for StackGuard { + type Target = NockStack; + + fn deref(&self) -> &Self::Target { + &self.stack + } +} + +impl std::ops::DerefMut for StackGuard { + fn deref_mut(&mut self) -> &mut Self::Target { + &mut self.stack + } +} + #[derive(Debug, Clone, PartialEq, NounEncode, NounDecode)] pub enum Key { Pub(u64), @@ -235,74 +261,80 @@ struct Spend { #[cfg(test)] mod tests { - use nockvm::mem::NockStack; - use super::*; // Expected Atom error, shouldn't this be a cell list? // #[test] // fn test_trek_encoding() { - // let mut stack = NockStack::new(8 << 10 << 10, 0); + // let mut stack = StackGuard::new(8 << 10 << 10); // let trek = Trek(vec![ // "path".to_string(), // "to".to_string(), // "key".to_string(), // ]); - // let encoded = trek.to_noun(&mut stack); + // let encoded = trek.to_noun(&mut *stack); // let decoded = Trek::from_noun(&mut stack, &encoded).unwrap(); // assert_eq!(trek, decoded); // } #[test] fn test_source_encoding() { - let mut stack = NockStack::new(8 << 10 << 10, 0); + let mut stack = StackGuard::new(8 << 10 << 10); + let space = stack.noun_space(); let hash = Source::Hash(0x1234); - let encoded = hash.to_noun(&mut stack); - let decoded = Source::from_noun(&encoded).unwrap(); + let encoded = hash.to_noun(&mut *stack); + let decoded = Source::from_noun(&encoded, &space).unwrap(); assert_eq!(hash, decoded); let coinbase = Source::Coinbase; - let encoded = coinbase.to_noun(&mut stack); - let decoded = Source::from_noun(&encoded).unwrap(); + let encoded = coinbase.to_noun(&mut *stack); + let decoded = Source::from_noun(&encoded, &space).unwrap(); assert_eq!(coinbase, decoded); } #[test] fn test_lock_encoding() { - let mut stack = NockStack::new(8 << 10 << 10, 0); + let mut stack = StackGuard::new(8 << 10 << 10); + let space = stack.noun_space(); let mut pubkeys = HashSet::new(); pubkeys.insert(0x1234); pubkeys.insert(0x5678); let lock = Lock { m: 2, pubkeys }; - let encoded = lock.to_noun(&mut stack); - let decoded = Lock::from_noun(&encoded).unwrap(); + let encoded = lock.to_noun(&mut *stack); + let decoded = Lock::from_noun(&encoded, &space).unwrap(); assert_eq!(lock, decoded); } #[test] fn test_timelock_encoding() { - let mut stack = NockStack::new(8 << 10 << 10, 0); + let mut stack = StackGuard::new(8 << 10 << 10); + let space = stack.noun_space(); let timelock = Timelock { block: 0x1234, intent: TimelockIntent::After, }; - let encoded = timelock.to_noun(&mut stack); + let encoded = timelock.to_noun(&mut *stack); + let encoded_cell = encoded.as_cell().unwrap(); println!( "Encoded timelock: {:?}", - FullDebugCell(&encoded.as_cell().unwrap()) + FullDebugCell { + cell: &encoded_cell, + space: &space + } ); - let decoded = Timelock::from_noun(&encoded).unwrap(); + let decoded = Timelock::from_noun(&encoded, &space).unwrap(); assert_eq!(timelock, decoded); } #[test] fn test_seed_encoding() { - let mut stack = NockStack::new(8 << 10 << 10, 0); + let mut stack = StackGuard::new(8 << 10 << 10); + let space = stack.noun_space(); let mut pubkeys = HashSet::new(); pubkeys.insert(0x1234); @@ -314,14 +346,15 @@ mod tests { gift: 100, parent_hash: 0x9abc, }; - let encoded = seed.to_noun(&mut stack); - let decoded = Seed::from_noun(&encoded).unwrap(); + let encoded = seed.to_noun(&mut *stack); + let decoded = Seed::from_noun(&encoded, &space).unwrap(); assert_eq!(seed, decoded); } #[test] fn test_preseed_encoding() { - let mut stack = NockStack::new(8 << 10 << 10, 0); + let mut stack = StackGuard::new(8 << 10 << 10); + let space = stack.noun_space(); let preseed = PreSeed { name: "test_seed".to_string(), @@ -334,14 +367,15 @@ mod tests { parent_hash: true, }, }; - let encoded = preseed.to_noun(&mut stack); - let decoded = PreSeed::from_noun(&encoded).unwrap(); + let encoded = preseed.to_noun(&mut *stack); + let decoded = PreSeed::from_noun(&encoded, &space).unwrap(); assert_eq!(preseed, decoded); } #[test] fn test_spend_encoding() { - let mut stack = NockStack::new(8 << 10 << 10, 0); + let mut stack = StackGuard::new(8 << 10 << 10); + let space = stack.noun_space(); let mut signatures = HashMap::new(); signatures.insert(0x1234, 0x5678); @@ -364,14 +398,15 @@ mod tests { seeds, fee: 10, }; - let encoded = spend.to_noun(&mut stack); - let decoded = Spend::from_noun(&encoded).unwrap(); + let encoded = spend.to_noun(&mut *stack); + let decoded = Spend::from_noun(&encoded, &space).unwrap(); assert_eq!(spend, decoded); } #[test] fn test_preinput_encoding() { - let mut stack = NockStack::new(8 << 10 << 10, 0); + let mut stack = StackGuard::new(8 << 10 << 10); + let space = stack.noun_space(); let preinput = PreInput { name: "test_input".to_string(), @@ -385,44 +420,47 @@ mod tests { }, }, }; - let encoded = preinput.to_noun(&mut stack); - let decoded = PreInput::from_noun(&encoded).unwrap(); + let encoded = preinput.to_noun(&mut *stack); + let decoded = PreInput::from_noun(&encoded, &space).unwrap(); assert_eq!(preinput, decoded); } #[test] fn test_draft_encoding() { - let mut stack = NockStack::new(8 << 10 << 10, 0); + let mut stack = StackGuard::new(8 << 10 << 10); + let space = stack.noun_space(); let draft = Draft { name: "test_draft".to_string(), inputs: 0x1234, }; - let encoded = draft.to_noun(&mut stack); - let decoded = Draft::from_noun(&encoded).unwrap(); + let encoded = draft.to_noun(&mut *stack); + let decoded = Draft::from_noun(&encoded, &space).unwrap(); assert_eq!(draft, decoded); } #[test] fn test_seed_tuple_encoding() { - let mut stack = NockStack::new(8 << 10 << 10, 0); + let mut stack = StackGuard::new(8 << 10 << 10); + let space = stack.noun_space(); // Test with Some value let seed_tuple = SeedTuple(42, Some("seed-phrase".to_string())); - let encoded = seed_tuple.to_noun(&mut stack); - let decoded = SeedTuple::from_noun(&encoded).unwrap(); + let encoded = seed_tuple.to_noun(&mut *stack); + let decoded = SeedTuple::from_noun(&encoded, &space).unwrap(); assert_eq!(seed_tuple, decoded); // Test with None value let seed_tuple = SeedTuple(42, None); - let encoded = seed_tuple.to_noun(&mut stack); - let decoded = SeedTuple::from_noun(&encoded).unwrap(); + let encoded = seed_tuple.to_noun(&mut *stack); + let decoded = SeedTuple::from_noun(&encoded, &space).unwrap(); assert_eq!(seed_tuple, decoded); } #[test] fn test_wallet_state_encoding() { - let mut stack = NockStack::new(8 << 10 << 10, 0); + let mut stack = StackGuard::new(8 << 10 << 10); + let space = stack.noun_space(); let mut hash_to_name = HashMap::new(); hash_to_name.insert(0x1234, "note1".to_string()); @@ -493,15 +531,16 @@ mod tests { }; println!("Encoding wallet state..."); - let encoded = wallet_state.to_noun(&mut stack); + let encoded = wallet_state.to_noun(&mut *stack); println!("Encoded noun: {:?}", encoded); - let decoded = WalletState::from_noun(&encoded).unwrap(); + let decoded = WalletState::from_noun(&encoded, &space).unwrap(); assert_eq!(wallet_state, decoded); } #[test] fn test_draft_entity_kind_encoding() { - let mut stack = NockStack::new(8 << 10 << 10, 0); + let mut stack = StackGuard::new(8 << 10 << 10); + let space = stack.noun_space(); // Test Draft variant let draft_kind = DraftEntityKind::Draft { @@ -512,25 +551,25 @@ mod tests { }, }; println!("Encoding draft kind..."); - let encoded = draft_kind.to_noun(&mut stack); + let encoded = draft_kind.to_noun(&mut *stack); println!("Encoded draft kind: {:?}", encoded); println!( "Encoded draft kind head: {:?}", - encoded.as_cell().unwrap().head() + encoded.in_space(&space).as_cell().unwrap().head().noun() ); println!( "Encoded draft kind tail: {:?}", - encoded.as_cell().unwrap().tail() + encoded.in_space(&space).as_cell().unwrap().tail().noun() ); - if let Ok(tail_cell) = encoded.as_cell().unwrap().tail().as_cell() { - println!("Tail head: {:?}", tail_cell.head()); - println!("Tail tail: {:?}", tail_cell.tail()); + if let Ok(tail_cell) = encoded.in_space(&space).as_cell().unwrap().tail().as_cell() { + println!("Tail head: {:?}", tail_cell.head().noun()); + println!("Tail tail: {:?}", tail_cell.tail().noun()); if let Ok(tail_tail_cell) = tail_cell.tail().as_cell() { - println!("Tail tail head: {:?}", tail_tail_cell.head()); - println!("Tail tail tail: {:?}", tail_tail_cell.tail()); + println!("Tail tail head: {:?}", tail_tail_cell.head().noun()); + println!("Tail tail tail: {:?}", tail_tail_cell.tail().noun()); } } - let decoded = DraftEntityKind::from_noun(&encoded).unwrap(); + let decoded = DraftEntityKind::from_noun(&encoded, &space).unwrap(); assert_eq!(draft_kind, decoded); // Test Input variant @@ -543,9 +582,9 @@ mod tests { }, }; println!("Encoding input kind..."); - let encoded = input_kind.to_noun(&mut stack); + let encoded = input_kind.to_noun(&mut *stack); println!("Encoded input kind: {:?}", encoded); - let decoded = DraftEntityKind::from_noun(&encoded).unwrap(); + let decoded = DraftEntityKind::from_noun(&encoded, &space).unwrap(); assert_eq!(input_kind, decoded); // Test Seed variant @@ -558,9 +597,9 @@ mod tests { }, }; println!("Encoding seed kind..."); - let encoded = seed_kind.to_noun(&mut stack); + let encoded = seed_kind.to_noun(&mut *stack); println!("Encoded seed kind: {:?}", encoded); - let decoded = DraftEntityKind::from_noun(&encoded).unwrap(); + let decoded = DraftEntityKind::from_noun(&encoded, &space).unwrap(); assert_eq!(seed_kind, decoded); } } diff --git a/crates/noun-serde/src/lib.rs b/crates/noun-serde/src/lib.rs index a3055fc28..ef686e09b 100644 --- a/crates/noun-serde/src/lib.rs +++ b/crates/noun-serde/src/lib.rs @@ -1,3 +1,4 @@ +#![feature(negative_impls)] // Allow unwrap in test code - standard practice for test assertions #![cfg_attr(test, allow(clippy::unwrap_used))] @@ -12,8 +13,8 @@ use nockvm::ext::{make_tas, AtomExt}; use nockvm::jets::util::BAIL_FAIL; use nockvm::jets::JetErr; #[allow(unused_imports)] -use nockvm::noun::{Atom, FullDebugCell, Noun, NounAllocator, Slots, D, T}; -use nockvm::noun::{NO, YES}; +use nockvm::noun::{Atom, FullDebugCell, Noun, NounAllocator, NounSpace, D, T}; +use nockvm::noun::{NounHandle, NO, YES}; pub use noun_serde_derive::{NounDecode, NounEncode}; use tracing::trace; @@ -23,12 +24,12 @@ pub mod prelude { // Trait extensions for Noun pub trait NounSerdeDecodeExt { - fn decode(&self) -> Result; + fn decode(&self, space: &NounSpace) -> Result; } impl NounSerdeDecodeExt for nockvm::noun::Noun { - fn decode(&self) -> Result { - T::from_noun(self) + fn decode(&self, space: &NounSpace) -> Result { + T::from_noun(self, space) } } @@ -48,8 +49,13 @@ pub trait NounEncode { /// Trait for types that can be decoded from a Noun pub trait NounDecode: Sized { - /// Try to decode this value from a Noun - fn from_noun(noun: &Noun) -> Result; + /// Try to decode this value from a Noun with a separate NounSpace argument + fn from_noun(noun: &Noun, space: &NounSpace) -> Result; + fn from_noun_handle(noun: &NounHandle) -> Result { + let space = noun.space(); + let noun = noun.noun(); + Self::from_noun(&noun, space) + } } /// Error that can occur during Noun decoding @@ -84,6 +90,9 @@ pub enum NounDecodeError { #[error("Failed to decode Constraints")] ConstraintsDecodeError, + + #[error("Failed to decode noun")] + DecodeError, } impl From for JetErr { @@ -104,18 +113,10 @@ impl From for NounDecodeError { } } -// Base no-nop implementations for Noun -impl NounEncode for Noun { - fn to_noun(&self, _allocator: &mut A) -> Noun { - *self - } -} - -impl NounDecode for Noun { - fn from_noun(noun: &Noun) -> Result { - Ok(*noun) - } -} +// Raw nouns do not carry arena provenance, so noun-serde intentionally does +// not support identity encode/decode for `Noun`. +impl !NounEncode for Noun {} +impl !NounDecode for Noun {} // Implementations for primitive types impl NounEncode for u64 { @@ -126,8 +127,8 @@ impl NounEncode for u64 { } impl NounDecode for u64 { - fn from_noun(noun: &Noun) -> Result { - match noun.as_atom() { + fn from_noun(noun: &Noun, space: &NounSpace) -> Result { + match noun.in_space(space).as_atom() { Ok(atom) => atom .as_u64() .map_err(|_| NounDecodeError::Custom("Atom too large for u64".into())), @@ -150,8 +151,8 @@ impl NounEncode for UBig { } impl NounDecode for u32 { - fn from_noun(noun: &Noun) -> Result { - match noun.as_atom() { + fn from_noun(noun: &Noun, space: &NounSpace) -> Result { + match noun.in_space(space).as_atom() { Ok(atom) => atom .as_u64() .map(|x| x as u32) @@ -169,8 +170,8 @@ impl NounEncode for String { } impl NounDecode for String { - fn from_noun(noun: &Noun) -> Result { - match noun.as_atom() { + fn from_noun(noun: &Noun, space: &NounSpace) -> Result { + match noun.in_space(space).as_atom() { Ok(atom) => atom .into_string() .map_err(|err| NounDecodeError::Custom(format!("Invalid string atom: {:?}", err))), @@ -180,10 +181,15 @@ impl NounDecode for String { } impl NounDecode for (X, Y) { - fn from_noun(noun: &Noun) -> Result { - let cell = noun.as_cell().map_err(|_| NounDecodeError::ExpectedCell)?; - let a = X::from_noun(&cell.slot(2)?)?; - let b = Y::from_noun(&cell.slot(3)?)?; + fn from_noun(noun: &Noun, space: &NounSpace) -> Result { + let cell = noun + .in_space(space) + .as_cell() + .map_err(|_| NounDecodeError::ExpectedCell)?; + let head = cell.as_noun().slot(2)?.noun(); + let tail = cell.as_noun().slot(3)?.noun(); + let a = X::from_noun(&head, space)?; + let b = Y::from_noun(&tail, space)?; Ok((a, b)) } } @@ -198,11 +204,17 @@ impl NounEncode for (X, Y) { } impl NounDecode for (X, Y, Z) { - fn from_noun(noun: &Noun) -> Result { - let cell = noun.as_cell().map_err(|_| NounDecodeError::ExpectedCell)?; - let a = X::from_noun(&cell.slot(2)?)?; - let b = Y::from_noun(&cell.slot(6)?)?; - let c = Z::from_noun(&cell.slot(7)?)?; + fn from_noun(noun: &Noun, space: &NounSpace) -> Result { + let cell = noun + .in_space(space) + .as_cell() + .map_err(|_| NounDecodeError::ExpectedCell)?; + let a_slot = cell.as_noun().slot(2)?.noun(); + let b_slot = cell.as_noun().slot(6)?.noun(); + let c_slot = cell.as_noun().slot(7)?.noun(); + let a = X::from_noun(&a_slot, space)?; + let b = Y::from_noun(&b_slot, space)?; + let c = Z::from_noun(&c_slot, space)?; Ok((a, b, c)) } } @@ -219,12 +231,19 @@ impl NounEncode for (X, Y, Z) { } impl NounDecode for (W, X, Y, Z) { - fn from_noun(noun: &Noun) -> Result { - let cell = noun.as_cell().map_err(|_| NounDecodeError::ExpectedCell)?; - let a = W::from_noun(&cell.slot(2)?)?; - let b = X::from_noun(&cell.slot(6)?)?; - let c = Y::from_noun(&cell.slot(14)?)?; - let d = Z::from_noun(&cell.slot(15)?)?; + fn from_noun(noun: &Noun, space: &NounSpace) -> Result { + let cell = noun + .in_space(space) + .as_cell() + .map_err(|_| NounDecodeError::ExpectedCell)?; + let a_slot = cell.as_noun().slot(2)?.noun(); + let b_slot = cell.as_noun().slot(6)?.noun(); + let c_slot = cell.as_noun().slot(14)?.noun(); + let d_slot = cell.as_noun().slot(15)?.noun(); + let a = W::from_noun(&a_slot, space)?; + let b = X::from_noun(&b_slot, space)?; + let c = Y::from_noun(&c_slot, space)?; + let d = Z::from_noun(&d_slot, space)?; Ok((a, b, c, d)) } } @@ -245,13 +264,21 @@ impl NounEncode for impl NounDecode for (V, W, X, Y, Z) { - fn from_noun(noun: &Noun) -> Result { - let cell = noun.as_cell().map_err(|_| NounDecodeError::ExpectedCell)?; - let a = V::from_noun(&cell.slot(2)?)?; - let b = W::from_noun(&cell.slot(6)?)?; - let c = X::from_noun(&cell.slot(14)?)?; - let d = Y::from_noun(&cell.slot(30)?)?; - let e = Z::from_noun(&cell.slot(31)?)?; + fn from_noun(noun: &Noun, space: &NounSpace) -> Result { + let cell = noun + .in_space(space) + .as_cell() + .map_err(|_| NounDecodeError::ExpectedCell)?; + let a_slot = cell.as_noun().slot(2)?.noun(); + let b_slot = cell.as_noun().slot(6)?.noun(); + let c_slot = cell.as_noun().slot(14)?.noun(); + let d_slot = cell.as_noun().slot(30)?.noun(); + let e_slot = cell.as_noun().slot(31)?.noun(); + let a = V::from_noun(&a_slot, space)?; + let b = W::from_noun(&b_slot, space)?; + let c = X::from_noun(&c_slot, space)?; + let d = Y::from_noun(&d_slot, space)?; + let e = Z::from_noun(&e_slot, space)?; Ok((a, b, c, d, e)) } } @@ -276,14 +303,23 @@ impl impl NounDecode for (U, V, W, X, Y, Z) { - fn from_noun(noun: &Noun) -> Result { - let cell = noun.as_cell().map_err(|_| NounDecodeError::ExpectedCell)?; - let a = U::from_noun(&cell.slot(2)?)?; - let b = V::from_noun(&cell.slot(6)?)?; - let c = W::from_noun(&cell.slot(14)?)?; - let d = X::from_noun(&cell.slot(30)?)?; - let e = Y::from_noun(&cell.slot(62)?)?; - let f = Z::from_noun(&cell.slot(63)?)?; + fn from_noun(noun: &Noun, space: &NounSpace) -> Result { + let cell = noun + .in_space(space) + .as_cell() + .map_err(|_| NounDecodeError::ExpectedCell)?; + let a_slot = cell.as_noun().slot(2)?.noun(); + let b_slot = cell.as_noun().slot(6)?.noun(); + let c_slot = cell.as_noun().slot(14)?.noun(); + let d_slot = cell.as_noun().slot(30)?.noun(); + let e_slot = cell.as_noun().slot(62)?.noun(); + let f_slot = cell.as_noun().slot(63)?.noun(); + let a = U::from_noun(&a_slot, space)?; + let b = V::from_noun(&b_slot, space)?; + let c = W::from_noun(&c_slot, space)?; + let d = X::from_noun(&d_slot, space)?; + let e = Y::from_noun(&e_slot, space)?; + let f = Z::from_noun(&f_slot, space)?; Ok((a, b, c, d, e, f)) } } @@ -318,11 +354,11 @@ impl NounEncode for bool { } impl NounDecode for bool { - fn from_noun(noun: &Noun) -> Result { + fn from_noun(noun: &Noun, space: &NounSpace) -> Result { trace!("Decoding bool from noun: {:?}", noun); - match noun.as_atom() { + match noun.in_space(space).as_atom() { Ok(atom) => { - trace!("Successfully decoded as atom: {:?}", atom); + trace!("Successfully decoded as atom: {:?}", atom.atom()); match atom.as_u64() { Ok(0) => { trace!("Decoded as 0 -> true (%.y)"); @@ -358,9 +394,9 @@ impl NounEncode for Option { } impl NounDecode for Option { - fn from_noun(noun: &Noun) -> Result { + fn from_noun(noun: &Noun, space: &NounSpace) -> Result { // First check if it's an atom 0 (None) - if let Ok(atom) = noun.as_atom() { + if let Ok(atom) = noun.in_space(space).as_atom() { match atom.as_u64() { Ok(0) => { trace!("Found ~ (0), returning None"); @@ -371,7 +407,10 @@ impl NounDecode for Option { } // Otherwise it must be a cell [~ value] - let cell = noun.as_cell().map_err(|_| NounDecodeError::ExpectedCell)?; + let cell = noun + .in_space(space) + .as_cell() + .map_err(|_| NounDecodeError::ExpectedCell)?; let head = cell .head() .as_atom() @@ -383,7 +422,8 @@ impl NounDecode for Option { )); } - let value = T::from_noun(&cell.tail())?; + let value_noun = cell.tail().noun(); + let value = T::from_noun(&value_noun, space)?; Ok(Some(value)) } } @@ -398,17 +438,15 @@ impl NounEncode for Vec { } impl NounDecode for Vec { - fn from_noun(noun: &Noun) -> Result { + fn from_noun(noun: &Noun, space: &NounSpace) -> Result { let mut result = Vec::new(); - let mut current = noun; - #[allow(unused_assignments)] - let mut current_tail = None; + let mut current = noun.in_space(space); while let Ok(cell) = current.as_cell() { - let item = T::from_noun(&cell.head())?; + let head = cell.head().noun(); + let item = T::from_noun(&head, space)?; result.push(item); - current_tail = Some(cell.tail()); - current = current_tail.as_ref().expect("current_tail was just set"); + current = cell.tail(); } if let Ok(atom) = current.as_atom() { @@ -435,8 +473,8 @@ impl NounEncode for Vec { } impl NounDecode for Vec { - fn from_noun(noun: &Noun) -> Result { - let nums = Vec::::from_noun(noun)?; + fn from_noun(noun: &Noun, space: &NounSpace) -> Result { + let nums = Vec::::from_noun(noun, space)?; Ok(nums.into_iter().map(|x| x as u8).collect()) } } @@ -473,9 +511,10 @@ impl NounDecode for Vec { /// map.insert("key1".to_string(), 42u64); /// map.insert("key2".to_string(), 43u64); /// -/// let mut stack = NockStack::new(8 << 10 << 10, 0); +/// let mut stack = NockStack::new(nockvm::mem::NOCK_STACK_SIZE_TINY, 0); /// let encoded = map.to_noun(&mut stack); -/// let decoded = HashMap::::from_noun(&encoded).unwrap(); +/// let space = stack.noun_space(); +/// let decoded = HashMap::::from_noun(&encoded, &space).unwrap(); /// assert_eq!(map, decoded); /// ``` impl NounEncode for HashMap @@ -558,11 +597,11 @@ where K: std::hash::Hash + Eq + std::fmt::Debug, V: std::fmt::Debug, { - fn from_noun(noun: &Noun) -> Result { + fn from_noun(noun: &Noun, space: &NounSpace) -> Result { trace!("\nDecoding HashMap from noun: {:?}", noun); // Handle empty tree case - if let Ok(atom) = noun.as_atom() { - trace!("Got atom: {:?}", atom); + if let Ok(atom) = noun.in_space(space).as_atom() { + trace!("Got atom: {:?}", atom.atom()); if atom.as_u64()? == 0 { return Ok(HashMap::new()); } @@ -578,16 +617,17 @@ where >( node: &Noun, map: &mut HashMap, + space: &NounSpace, ) -> Result<(), NounDecodeError> { // Base case: empty branch - if let Ok(atom) = node.as_atom() { + if let Ok(atom) = node.in_space(space).as_atom() { if atom.as_u64()? == 0 { return Ok(()); } return Err(NounDecodeError::ExpectedCell); } - let cell = node.as_cell()?; + let cell = node.in_space(space).as_cell()?; // Get the key-value pair from the node let pair = cell.head().as_cell().map_err(|e| { @@ -595,32 +635,30 @@ where NounDecodeError::ExpectedCell })?; - trace!( - "Got node - key: {:?}, value: {:?}", - pair.head(), - pair.tail() - ); + let pair_head = pair.head().noun(); + let pair_tail = pair.tail().noun(); + trace!("Got node - key: {:?}, value: {:?}", pair_head, pair_tail); trace!("Key type: {:?}", std::any::type_name::()); trace!("Value type: {:?}", std::any::type_name::()); - let key = K::from_noun(&pair.head())?; - let value = V::from_noun(&pair.tail())?; + let key = K::from_noun(&pair_head, space)?; + let value = V::from_noun(&pair_tail, space)?; trace!("Key: {:?}, Value: {:?}", key, value); map.insert(key, value); // Get left and right branches let rest = cell.tail().as_cell()?; - let left = &rest.head(); - let right = &rest.tail(); + let left = rest.head().noun(); + let right = rest.tail().noun(); // Recursively process left and right branches - traverse_tree(left, map)?; - traverse_tree(right, map)?; + traverse_tree(&left, map, space)?; + traverse_tree(&right, map, space)?; Ok(()) } - traverse_tree(noun, &mut map)?; + traverse_tree(noun, &mut map, space)?; Ok(map) } } @@ -648,9 +686,10 @@ where /// # use nockvm::mem::NockStack; /// let result: Result = Ok(42); /// -/// let mut stack = NockStack::new(8 << 10 << 10, 0); +/// let mut stack = NockStack::new(nockvm::mem::NOCK_STACK_SIZE_TINY, 0); /// let encoded = result.to_noun(&mut stack); -/// let decoded = Result::::from_noun(&encoded).unwrap(); +/// let space = stack.noun_space(); +/// let decoded = Result::::from_noun(&encoded, &space).unwrap(); /// assert_eq!(result, decoded); /// ``` impl NounEncode for Result { @@ -698,11 +737,16 @@ impl NounEncode for Result { /// 3. Decodes the tail into the appropriate type /// 4. Wraps in Ok/Err accordingly impl NounDecode for Result { - fn from_noun(noun: &Noun) -> Result { + fn from_noun(noun: &Noun, space: &NounSpace) -> Result { trace!("\nDecoding Result from noun: {:?}", noun); - let cell = noun.as_cell().map_err(|_| NounDecodeError::ExpectedCell)?; - trace!("Result cell head: {:?}", cell.head()); - trace!("Result cell tail: {:?}", cell.tail()); + let cell = noun + .in_space(space) + .as_cell() + .map_err(|_| NounDecodeError::ExpectedCell)?; + let head = cell.head().noun(); + let tail = cell.tail().noun(); + trace!("Result cell head: {:?}", head); + trace!("Result cell tail: {:?}", tail); let tag = cell .head() @@ -715,11 +759,11 @@ impl NounDecode for Result { match tag.as_str() { "ok" => { trace!("Decoding Ok variant"); - Ok(Ok(T::from_noun(&cell.tail())?)) + Ok(Ok(T::from_noun(&tail, space)?)) } "err" => { trace!("Decoding Err variant"); - Ok(Err(E::from_noun(&cell.tail())?)) + Ok(Err(E::from_noun(&tail, space)?)) } _ => { trace!("Invalid Result tag: {}", tag); @@ -738,11 +782,11 @@ pub fn encode_bool(value: bool) -> Noun { } } -pub fn decode_bool(noun: &Noun) -> Result { +pub fn decode_bool(noun: &Noun, space: &NounSpace) -> Result { trace!("Decoding bool from noun: {:?}", noun); - match noun.as_atom() { + match noun.in_space(space).as_atom() { Ok(atom) => { - trace!("Successfully decoded as atom: {:?}", atom); + trace!("Successfully decoded as atom: {:?}", atom.atom()); match atom.as_u64() { Ok(0) => { trace!("Decoded as 0 -> true (%.y)"); @@ -792,9 +836,10 @@ pub fn decode_bool(noun: &Noun) -> Result { /// set.insert("key1".to_string()); /// set.insert("key2".to_string()); /// -/// let mut stack = NockStack::new(8 << 10 << 10, 0); +/// let mut stack = NockStack::new(nockvm::mem::NOCK_STACK_SIZE_TINY, 0); /// let encoded = set.to_noun(&mut stack); -/// let decoded = HashSet::::from_noun(&encoded).unwrap(); +/// let space = stack.noun_space(); +/// let decoded = HashSet::::from_noun(&encoded, &space).unwrap(); /// assert_eq!(set, decoded); /// ``` impl NounEncode for HashSet @@ -853,9 +898,9 @@ impl NounDecode for HashSet where T: std::hash::Hash + Eq, { - fn from_noun(noun: &Noun) -> Result { + fn from_noun(noun: &Noun, space: &NounSpace) -> Result { // Handle empty tree case - if let Ok(atom) = noun.as_atom() { + if let Ok(atom) = noun.in_space(space).as_atom() { if atom.as_u64()? == 0 { return Ok(HashSet::new()); } @@ -868,34 +913,36 @@ where fn traverse_tree( node: &Noun, set: &mut HashSet, + space: &NounSpace, ) -> Result<(), NounDecodeError> { // Base case: empty branch - if let Ok(atom) = node.as_atom() { + if let Ok(atom) = node.in_space(space).as_atom() { if atom.as_u64()? == 0 { return Ok(()); } return Err(NounDecodeError::ExpectedCell); } - let cell = node.as_cell()?; + let cell = node.in_space(space).as_cell()?; // Insert the node value - let value = T::from_noun(&cell.head())?; + let value_noun = cell.head().noun(); + let value = T::from_noun(&value_noun, space)?; set.insert(value); // Get left and right branches let rest = cell.tail().as_cell()?; - let left = &rest.head(); - let right = &rest.tail(); + let left = rest.head().noun(); + let right = rest.tail().noun(); // Recursively process left and right branches - traverse_tree(left, set)?; - traverse_tree(right, set)?; + traverse_tree(&left, set, space)?; + traverse_tree(&right, set, space)?; Ok(()) } - traverse_tree(noun, &mut set)?; + traverse_tree(noun, &mut set, space)?; Ok(set) } } @@ -925,9 +972,10 @@ where /// map.insert(1u64, "value1".to_string()); /// map.insert(2u64, "value2".to_string()); /// -/// let mut stack = NockStack::new(8 << 10 << 10, 0); +/// let mut stack = NockStack::new(nockvm::mem::NOCK_STACK_SIZE_TINY, 0); /// let encoded = map.to_noun(&mut stack); -/// let decoded = BTreeMap::::from_noun(&encoded).unwrap(); +/// let space = stack.noun_space(); +/// let decoded = BTreeMap::::from_noun(&encoded, &space).unwrap(); /// assert_eq!(map, decoded); /// ``` impl NounEncode for BTreeMap { @@ -996,42 +1044,45 @@ impl NounEncode for BTreeMap { /// 3. Recursively processes all subtrees /// 4. Atom 0 represents empty subtrees impl NounDecode for BTreeMap { - fn from_noun(noun: &Noun) -> Result { + fn from_noun(noun: &Noun, space: &NounSpace) -> Result { let mut map = BTreeMap::new(); fn traverse_tree( node: &Noun, map: &mut BTreeMap, + space: &NounSpace, ) -> Result<(), NounDecodeError> { // Base case: empty tree (atom 0) - if let Ok(atom) = node.as_atom() { + if let Ok(atom) = node.in_space(space).as_atom() { if atom.as_u64()? == 0 { return Ok(()); } return Err(NounDecodeError::ExpectedCell); } - let cell = node.as_cell()?; + let cell = node.in_space(space).as_cell()?; // Get the [key value] pair from the node let pair = cell.head().as_cell()?; - let key = K::from_noun(&pair.head())?; - let value = V::from_noun(&pair.tail())?; + let key_noun = pair.head().noun(); + let value_noun = pair.tail().noun(); + let key = K::from_noun(&key_noun, space)?; + let value = V::from_noun(&value_noun, space)?; map.insert(key, value); // Get left and right subtrees let rest = cell.tail().as_cell()?; - let left = &rest.head(); - let right = &rest.tail(); + let left = rest.head().noun(); + let right = rest.tail().noun(); // Recursively process left and right subtrees - traverse_tree(left, map)?; - traverse_tree(right, map)?; + traverse_tree(&left, map, space)?; + traverse_tree(&right, map, space)?; Ok(()) } - traverse_tree(noun, &mut map)?; + traverse_tree(noun, &mut map, space)?; Ok(map) } } @@ -1051,8 +1102,8 @@ impl NounEncode for usize { /// usize values are decoded from u64 atoms. On 32-bit systems, values larger /// than u32::MAX will cause a decode error. impl NounDecode for usize { - fn from_noun(noun: &Noun) -> Result { - let value = u64::from_noun(noun)?; + fn from_noun(noun: &Noun, space: &NounSpace) -> Result { + let value = u64::from_noun(noun, space)?; // Check if the value fits in usize for the current architecture if value > usize::MAX as u64 { @@ -1067,25 +1118,26 @@ impl NounDecode for usize { } impl NounDecode for [T; N] { - fn from_noun(noun: &Noun) -> Result { + fn from_noun(noun: &Noun, space: &NounSpace) -> Result { let mut result = Vec::with_capacity(N); if N > 0 { - let mut current = *noun; + let mut current = noun.in_space(space); // Process all elements except the last one for _ in 0..N - 1 { let cell = current .as_cell() .map_err(|_| NounDecodeError::ExpectedCell)?; - let head = cell.head(); - let item = T::from_noun(&head)?; + let head = cell.head().noun(); + let item = T::from_noun(&head, space)?; result.push(item); current = cell.tail(); } // Process the last element - let last_item = T::from_noun(¤t)?; + let last_noun = current.noun(); + let last_item = T::from_noun(&last_noun, space)?; result.push(last_item); } @@ -1126,15 +1178,19 @@ mod array_tests { fn test_empty_array() { let mut context = init_context(); let stack = &mut context.stack; + let space = stack.noun_space(); let empty_array: [u64; 0] = []; // Test encoding let noun = empty_array.to_noun(stack); assert!(noun.is_atom()); - assert_eq!(noun.as_atom().unwrap().as_u64().unwrap(), 0); + assert_eq!( + noun.in_space(&space).as_atom().unwrap().as_u64().unwrap(), + 0 + ); // Test decoding - let decoded: [u64; 0] = NounDecode::from_noun(&noun).unwrap(); + let decoded: [u64; 0] = NounDecode::from_noun(&noun, &space).unwrap(); assert_eq!(decoded.len(), 0); } @@ -1142,6 +1198,7 @@ mod array_tests { fn test_single_element_array() { let mut context = init_context(); let stack = &mut context.stack; + let space = stack.noun_space(); let array = [42u64]; // Test encoding @@ -1149,10 +1206,13 @@ mod array_tests { // Encoding should produce just the single element as an atom assert!(noun.is_atom()); - assert_eq!(noun.as_atom().unwrap().as_u64().unwrap(), 42); + assert_eq!( + noun.in_space(&space).as_atom().unwrap().as_u64().unwrap(), + 42 + ); // Test decoding - let decoded: [u64; 1] = NounDecode::from_noun(&noun).unwrap(); + let decoded: [u64; 1] = NounDecode::from_noun(&noun, &space).unwrap(); assert_eq!(decoded, [42]); } @@ -1160,6 +1220,7 @@ mod array_tests { fn test_multi_element_array() { let mut context = init_context(); let stack = &mut context.stack; + let space = stack.noun_space(); let array = [1u64, 2, 3, 4, 5]; // Test encoding @@ -1167,32 +1228,35 @@ mod array_tests { // It should create a right-associative chain of cells assert!(noun.is_cell()); - let cell = noun.as_cell().unwrap(); + let cell = noun.in_space(&space).as_cell().unwrap(); assert_eq!(cell.head().as_atom().unwrap().as_u64().unwrap(), 1); // Recursively check the structure - let tail1 = cell.tail(); + let tail1 = cell.tail().noun(); assert!(tail1.is_cell()); - let cell1 = tail1.as_cell().unwrap(); + let cell1 = tail1.in_space(&space).as_cell().unwrap(); assert_eq!(cell1.head().as_atom().unwrap().as_u64().unwrap(), 2); - let tail2 = cell1.tail(); + let tail2 = cell1.tail().noun(); assert!(tail2.is_cell()); - let cell2 = tail2.as_cell().unwrap(); + let cell2 = tail2.in_space(&space).as_cell().unwrap(); assert_eq!(cell2.head().as_atom().unwrap().as_u64().unwrap(), 3); - let tail3 = cell2.tail(); + let tail3 = cell2.tail().noun(); assert!(tail3.is_cell()); - let cell3 = tail3.as_cell().unwrap(); + let cell3 = tail3.in_space(&space).as_cell().unwrap(); assert_eq!(cell3.head().as_atom().unwrap().as_u64().unwrap(), 4); // Last element should be an atom, not a cell - let tail4 = cell3.tail(); + let tail4 = cell3.tail().noun(); assert!(tail4.is_atom()); - assert_eq!(tail4.as_atom().unwrap().as_u64().unwrap(), 5); + assert_eq!( + tail4.in_space(&space).as_atom().unwrap().as_u64().unwrap(), + 5 + ); // Test decoding - let decoded: [u64; 5] = NounDecode::from_noun(&noun).unwrap(); + let decoded: [u64; 5] = NounDecode::from_noun(&noun, &space).unwrap(); assert_eq!(decoded, [1, 2, 3, 4, 5]); } @@ -1200,39 +1264,42 @@ mod array_tests { fn test_string_array() { let mut context = init_context(); let stack = &mut context.stack; + let space = stack.noun_space(); let array = ["hello".to_string(), "world".to_string()]; // Test encoding let noun = array.to_noun(stack); // Test decoding - let decoded: [String; 2] = NounDecode::from_noun(&noun).unwrap(); + let decoded: [String; 2] = NounDecode::from_noun(&noun, &space).unwrap(); assert_eq!(decoded, ["hello".to_string(), "world".to_string()]); } #[test] fn test_bool_array() { let mut context = init_context(); let stack = &mut context.stack; + let space = stack.noun_space(); let array = [true, false, true]; // Test encoding let noun = array.to_noun(stack); // Test decoding - let decoded: [bool; 3] = NounDecode::from_noun(&noun).unwrap(); + let decoded: [bool; 3] = NounDecode::from_noun(&noun, &space).unwrap(); assert_eq!(decoded, [true, false, true]); } #[test] fn test_nested_array() { let mut context = init_context(); let stack = &mut context.stack; + let space = stack.noun_space(); let array = [[1u64, 2u64], [3u64, 4u64]]; // Test encoding let noun = array.to_noun(stack); // Test decoding - let decoded: [[u64; 2]; 2] = NounDecode::from_noun(&noun).unwrap(); + let decoded: [[u64; 2]; 2] = NounDecode::from_noun(&noun, &space).unwrap(); assert_eq!(decoded, [[1, 2], [3, 4]]); } @@ -1240,40 +1307,70 @@ mod array_tests { fn test_decode_error() { let mut context = init_context(); let stack = &mut context.stack; + let space = stack.noun_space(); // Create an atom (not a proper cell chain for array) let noun = D(42); // Try to decode as a multi-element array (should fail) - let result = <[u64; 3]>::from_noun(&noun); + let result = <[u64; 3]>::from_noun(&noun, &space); assert!(result.is_err()); // Create a cell with wrong structure let bad_noun = T(stack, &[D(1), D(2)]); // Last element should be a cell for a 3-element array - let result = <[u64; 3]>::from_noun(&bad_noun); + let result = <[u64; 3]>::from_noun(&bad_noun, &space); assert!(result.is_err()); } } #[cfg(test)] mod tip5_tests { + use nockvm::mem::NockStack; + use super::*; + struct StackGuard { + stack: NockStack, + } + + impl StackGuard { + fn new(words: usize) -> Self { + let stack = NockStack::new(words, 0); + Self { stack } + } + } + + impl std::ops::Deref for StackGuard { + type Target = NockStack; + + fn deref(&self) -> &Self::Target { + &self.stack + } + } + + impl std::ops::DerefMut for StackGuard { + fn deref_mut(&mut self) -> &mut Self::Target { + &mut self.stack + } + } + #[test] fn test_tip5_encode_decode_array() { - let mut stack = nockvm::mem::NockStack::new(1024 * 1024, 0); - let noun = T(&mut stack, &[D(1), D(2), D(3), D(4), D(5)]); - let decoded = noun.decode::<[u64; 5]>().unwrap(); + let mut stack = StackGuard::new(1024 * 1024); + let space = stack.noun_space(); + let noun = T(&mut *stack, &[D(1), D(2), D(3), D(4), D(5)]); + let decoded = noun.decode::<[u64; 5]>(&space).unwrap(); assert_eq!([1, 2, 3, 4, 5], decoded); } #[test] fn test_tip5_encode_decode_tuple() { - let mut stack = nockvm::mem::NockStack::new(1024 * 1024, 0); - let noun = T(&mut stack, &[D(1), D(2), D(3), D(4), D(5)]); - let decoded = noun.decode::<(u64, u64, u64, u64, u64)>().unwrap(); + let mut stack = StackGuard::new(1024 * 1024); + let space = stack.noun_space(); + let noun = T(&mut *stack, &[D(1), D(2), D(3), D(4), D(5)]); + let decoded = noun.decode::<(u64, u64, u64, u64, u64)>(&space).unwrap(); assert_eq!((1, 2, 3, 4, 5), decoded); } } @@ -1286,42 +1383,73 @@ mod btreemap_tests { use super::*; + struct StackGuard { + stack: NockStack, + } + + impl StackGuard { + fn new(words: usize) -> Self { + let stack = NockStack::new(words, 0); + Self { stack } + } + } + + impl std::ops::Deref for StackGuard { + type Target = NockStack; + + fn deref(&self) -> &Self::Target { + &self.stack + } + } + + impl std::ops::DerefMut for StackGuard { + fn deref_mut(&mut self) -> &mut Self::Target { + &mut self.stack + } + } + #[test] fn test_btreemap_empty() { - let mut stack = NockStack::new(1024 * 1024, 0); + let mut stack = StackGuard::new(1024 * 1024); + let space = stack.noun_space(); let map: BTreeMap = BTreeMap::new(); - let noun = map.to_noun(&mut stack); + let noun = map.to_noun(&mut *stack); // Empty map should encode as atom 0 assert!(noun.as_atom().is_ok()); - assert_eq!(noun.as_atom().unwrap().as_u64().unwrap(), 0); + assert_eq!( + noun.in_space(&space).as_atom().unwrap().as_u64().unwrap(), + 0 + ); // Round-trip test - let decoded = BTreeMap::::from_noun(&noun).unwrap(); + let decoded = BTreeMap::::from_noun(&noun, &space).unwrap(); assert_eq!(map, decoded); } #[test] fn test_btreemap_single_entry() { - let mut stack = NockStack::new(1024 * 1024, 0); + let mut stack = StackGuard::new(1024 * 1024); + let space = stack.noun_space(); let mut map = BTreeMap::new(); map.insert(42u64, "hello".to_string()); - let noun = map.to_noun(&mut stack); + let noun = map.to_noun(&mut *stack); // Should be a cell structure [node left right] assert!(noun.as_cell().is_ok()); // Round-trip test - let decoded = BTreeMap::::from_noun(&noun).unwrap(); + let decoded = BTreeMap::::from_noun(&noun, &space).unwrap(); assert_eq!(map, decoded); assert_eq!(decoded.get(&42), Some(&"hello".to_string())); } #[test] fn test_btreemap_multiple_entries() { - let mut stack = NockStack::new(1024 * 1024, 0); + let mut stack = StackGuard::new(1024 * 1024); + let space = stack.noun_space(); let mut map = BTreeMap::new(); map.insert(1u64, "one".to_string()); map.insert(2u64, "two".to_string()); @@ -1329,10 +1457,10 @@ mod btreemap_tests { map.insert(10u64, "ten".to_string()); map.insert(5u64, "five".to_string()); - let noun = map.to_noun(&mut stack); + let noun = map.to_noun(&mut *stack); // Round-trip test - let decoded = BTreeMap::::from_noun(&noun).unwrap(); + let decoded = BTreeMap::::from_noun(&noun, &space).unwrap(); assert_eq!(map, decoded); // Check all values are preserved @@ -1346,16 +1474,17 @@ mod btreemap_tests { #[test] fn test_btreemap_u64_u64() { - let mut stack = NockStack::new(1024 * 1024, 0); + let mut stack = StackGuard::new(1024 * 1024); + let space = stack.noun_space(); let mut map = BTreeMap::new(); map.insert(1u64, 100u64); map.insert(2u64, 200u64); map.insert(3u64, 300u64); - let noun = map.to_noun(&mut stack); + let noun = map.to_noun(&mut *stack); // Round-trip test - let decoded = BTreeMap::::from_noun(&noun).unwrap(); + let decoded = BTreeMap::::from_noun(&noun, &space).unwrap(); assert_eq!(map, decoded); // Check specific values @@ -1366,7 +1495,8 @@ mod btreemap_tests { #[test] fn test_btreemap_nested_structure() { - let mut stack = NockStack::new(1024 * 1024, 0); + let mut stack = StackGuard::new(1024 * 1024); + let space = stack.noun_space(); // Create a map of maps let mut inner1 = BTreeMap::new(); @@ -1381,10 +1511,10 @@ mod btreemap_tests { outer.insert(1u64, inner1.clone()); outer.insert(2u64, inner2.clone()); - let noun = outer.to_noun(&mut stack); + let noun = outer.to_noun(&mut *stack); // Round-trip test - let decoded = BTreeMap::>::from_noun(&noun).unwrap(); + let decoded = BTreeMap::>::from_noun(&noun, &space).unwrap(); assert_eq!(outer, decoded); // Check nested values @@ -1396,29 +1526,31 @@ mod btreemap_tests { #[test] fn test_btreemap_invalid_noun_structure() { - let mut stack = NockStack::new(1024 * 1024, 0); + let mut stack = StackGuard::new(1024 * 1024); + let space = stack.noun_space(); // Test with invalid atom (not 0) - let invalid_atom = nockvm::noun::Atom::new(&mut stack, 42).as_noun(); - let result = BTreeMap::::from_noun(&invalid_atom); + let invalid_atom = nockvm::noun::Atom::new(&mut *stack, 42).as_noun(); + let result = BTreeMap::::from_noun(&invalid_atom, &space); assert!(result.is_err()); // Test with malformed cell (missing key-value structure) let malformed = nockvm::noun::T( - &mut stack, + &mut *stack, &[ nockvm::noun::D(1), // Should be [key value] pair nockvm::noun::D(0), nockvm::noun::D(0), ], ); - let result = BTreeMap::::from_noun(&malformed); + let result = BTreeMap::::from_noun(&malformed, &space); assert!(result.is_err()); } #[test] fn test_btreemap_ordering_preserved() { - let mut stack = NockStack::new(1024 * 1024, 0); + let mut stack = StackGuard::new(1024 * 1024); + let space = stack.noun_space(); // Insert in random order let mut map = BTreeMap::new(); @@ -1427,8 +1559,8 @@ mod btreemap_tests { map.insert(val, format!("value_{}", i)); } - let noun = map.to_noun(&mut stack); - let decoded = BTreeMap::::from_noun(&noun).unwrap(); + let noun = map.to_noun(&mut *stack); + let decoded = BTreeMap::::from_noun(&noun, &space).unwrap(); // BTreeMap should maintain sorted order let original_keys: Vec<_> = map.keys().collect(); @@ -1444,30 +1576,32 @@ mod btreemap_tests { #[test] fn test_usize_encoding() { - let mut stack = NockStack::new(1024 * 1024, 0); + let mut stack = StackGuard::new(1024 * 1024); + let space = stack.noun_space(); // Test various usize values let values = vec![0usize, 1, 42, 1000, usize::MAX]; for &value in &values { - let noun = value.to_noun(&mut stack); - let decoded = usize::from_noun(&noun).unwrap(); + let noun = value.to_noun(&mut *stack); + let decoded = usize::from_noun(&noun, &space).unwrap(); assert_eq!(value, decoded); } } #[test] fn test_btreemap_usize_belt() { - let mut stack = NockStack::new(1024 * 1024, 0); + let mut stack = StackGuard::new(1024 * 1024); + let space = stack.noun_space(); let mut map = BTreeMap::new(); map.insert(0usize, 100u64); map.insert(1usize, 200u64); map.insert(2usize, 300u64); - let noun = map.to_noun(&mut stack); + let noun = map.to_noun(&mut *stack); // Round-trip test - let decoded = BTreeMap::::from_noun(&noun).unwrap(); + let decoded = BTreeMap::::from_noun(&noun, &space).unwrap(); assert_eq!(map, decoded); // Check specific values @@ -1483,8 +1617,11 @@ impl NounEncode for Bytes { } impl NounDecode for Bytes { - fn from_noun(noun: &Noun) -> Result { - let atom = noun.as_atom().map_err(|_| NounDecodeError::ExpectedAtom)?; + fn from_noun(noun: &Noun, space: &NounSpace) -> Result { + let atom = noun + .in_space(space) + .as_atom() + .map_err(|_| NounDecodeError::ExpectedAtom)?; let bytes = atom.as_ne_bytes().to_vec(); Ok(Bytes::from(bytes)) } diff --git a/crates/noun-serde/src/wallet.rs b/crates/noun-serde/src/wallet.rs index 265c0fd60..96f83fb3c 100644 --- a/crates/noun-serde/src/wallet.rs +++ b/crates/noun-serde/src/wallet.rs @@ -1,7 +1,7 @@ use std::collections::{HashMap, HashSet}; -use nockvm::ext::{make_tas, AtomExt}; -use nockvm::noun::{Noun, NounAllocator, D, T}; +use nockvm::ext::make_tas; +use nockvm::noun::{Noun, NounAllocator, NounSpace, D, T}; use crate::{NounDecode, NounDecodeError, NounEncode}; @@ -31,8 +31,11 @@ impl NounEncode for Key { impl NounDecode for Key { #[allow(unused_variables)] - fn from_noun(noun: &Noun) -> Result { - let cell = noun.as_cell().map_err(|_| NounDecodeError::ExpectedCell)?; + fn from_noun(noun: &Noun, space: &NounSpace) -> Result { + let cell = noun + .in_space(space) + .as_cell() + .map_err(|_| NounDecodeError::ExpectedCell)?; let tag = cell.head().as_atom()?.into_string()?; let value = cell.tail().as_atom()?.as_u64()?; @@ -65,15 +68,19 @@ impl NounEncode for Coil { } impl NounDecode for Coil { - fn from_noun(noun: &Noun) -> Result { - let cell = noun.as_cell().map_err(|_| NounDecodeError::ExpectedCell)?; + fn from_noun(noun: &Noun, space: &NounSpace) -> Result { + let cell = noun + .in_space(space) + .as_cell() + .map_err(|_| NounDecodeError::ExpectedCell)?; let tag = cell.head().as_atom()?.into_string()?; if tag != "coil" { return Err(NounDecodeError::InvalidTag); } let data = cell.tail().as_cell()?; - let key = Key::from_noun(&data.head())?; + let key_noun = data.head().noun(); + let key = Key::from_noun(&key_noun, space)?; let knot = data.tail().as_atom()?.as_u64()?; Ok(Coil { key, knot }) @@ -107,12 +114,15 @@ impl NounEncode for Meta { } impl NounDecode for Meta { - fn from_noun(noun: &Noun) -> Result { - let cell = noun.as_cell().map_err(|_| NounDecodeError::ExpectedCell)?; + fn from_noun(noun: &Noun, space: &NounSpace) -> Result { + let cell = noun + .in_space(space) + .as_cell() + .map_err(|_| NounDecodeError::ExpectedCell)?; let tag = cell.head().as_atom()?.into_string()?; match tag.as_str() { - "coil" => Ok(Meta::Coil(Coil::from_noun(noun)?)), + "coil" => Ok(Meta::Coil(Coil::from_noun(noun, space)?)), "label" => { let value = cell.tail().as_atom()?.into_string()?; Ok(Meta::Label(value)) @@ -152,13 +162,17 @@ impl NounEncode for Transaction { } impl NounDecode for Transaction { - fn from_noun(noun: &Noun) -> Result { - let cell = noun.as_cell().map_err(|_| NounDecodeError::ExpectedCell)?; + fn from_noun(noun: &Noun, space: &NounSpace) -> Result { + let cell = noun + .in_space(space) + .as_cell() + .map_err(|_| NounDecodeError::ExpectedCell)?; let recipient = cell.head().as_atom()?.as_u64()?; let tail = cell.tail().as_cell()?; let amount = tail.head().as_atom()?.as_u64()?; - let status = TransactionStatus::from_noun(&tail.tail())?; + let status_noun = tail.tail().noun(); + let status = TransactionStatus::from_noun(&status_noun, space)?; Ok(Transaction { recipient, @@ -180,8 +194,8 @@ impl NounEncode for TransactionStatus { } impl NounDecode for TransactionStatus { - fn from_noun(noun: &Noun) -> Result { - let tag = noun.as_atom()?.into_string()?; + fn from_noun(noun: &Noun, space: &NounSpace) -> Result { + let tag = noun.in_space(space).as_atom()?.into_string()?; match tag.as_str() { "unsigned" => Ok(TransactionStatus::Unsigned), "signed" => Ok(TransactionStatus::Signed), @@ -221,8 +235,11 @@ impl NounEncode for FileEffect { } impl NounDecode for FileEffect { - fn from_noun(noun: &Noun) -> Result { - let cell = noun.as_cell().map_err(|_| NounDecodeError::ExpectedCell)?; + fn from_noun(noun: &Noun, space: &NounSpace) -> Result { + let cell = noun + .in_space(space) + .as_cell() + .map_err(|_| NounDecodeError::ExpectedCell)?; let file_tag = cell.head().as_atom()?.into_string()?; if file_tag != "file" { return Err(NounDecodeError::InvalidTag); @@ -273,12 +290,15 @@ impl NounEncode for Effect { } impl NounDecode for Effect { - fn from_noun(noun: &Noun) -> Result { - let cell = noun.as_cell().map_err(|_| NounDecodeError::ExpectedCell)?; + fn from_noun(noun: &Noun, space: &NounSpace) -> Result { + let cell = noun + .in_space(space) + .as_cell() + .map_err(|_| NounDecodeError::ExpectedCell)?; let tag = cell.head().as_atom()?.into_string()?; match tag.as_str() { - "file" => Ok(Effect::File(FileEffect::from_noun(noun)?)), + "file" => Ok(Effect::File(FileEffect::from_noun(noun, space)?)), "markdown" => { let text = cell.tail().as_atom()?.into_string()?; Ok(Effect::Markdown(text)) @@ -311,8 +331,11 @@ impl NounEncode for SpendMask { } impl NounDecode for SpendMask { - fn from_noun(noun: &Noun) -> Result { - let cell = noun.as_cell().map_err(|_| NounDecodeError::ExpectedCell)?; + fn from_noun(noun: &Noun, space: &NounSpace) -> Result { + let cell = noun + .in_space(space) + .as_cell() + .map_err(|_| NounDecodeError::ExpectedCell)?; let signature = cell.head().as_atom()?.as_u64()? != 0; let rest = cell.tail().as_cell()?; @@ -343,10 +366,14 @@ impl NounEncode for InputMask { } impl NounDecode for InputMask { - fn from_noun(noun: &Noun) -> Result { - let cell = noun.as_cell().map_err(|_| NounDecodeError::ExpectedCell)?; + fn from_noun(noun: &Noun, space: &NounSpace) -> Result { + let cell = noun + .in_space(space) + .as_cell() + .map_err(|_| NounDecodeError::ExpectedCell)?; let note = cell.head().as_atom()?.as_u64()? != 0; - let spend = SpendMask::from_noun(&cell.tail())?; + let spend_noun = cell.tail().noun(); + let spend = SpendMask::from_noun(&spend_noun, space)?; Ok(InputMask { note, spend }) } } @@ -382,30 +409,33 @@ impl NounEncode for SeedMask { } impl NounDecode for SeedMask { - fn from_noun(noun: &Noun) -> Result { + fn from_noun(noun: &Noun, space: &NounSpace) -> Result { let mut current = noun; let next_cell = |n: &Noun| -> Result<(Noun, Noun), NounDecodeError> { - let cell = n.as_cell().map_err(|_| NounDecodeError::ExpectedCell)?; - Ok((cell.head(), cell.tail())) + let cell = n + .in_space(space) + .as_cell() + .map_err(|_| NounDecodeError::ExpectedCell)?; + Ok((cell.head().noun(), cell.tail().noun())) }; let (output_source_noun, current_) = next_cell(current)?; - let output_source = output_source_noun.as_atom()?.as_u64()? != 0; + let output_source = output_source_noun.in_space(space).as_atom()?.as_u64()? != 0; current = ¤t_; let (recipient_noun, current_) = next_cell(current)?; - let recipient = recipient_noun.as_atom()?.as_u64()? != 0; + let recipient = recipient_noun.in_space(space).as_atom()?.as_u64()? != 0; current = ¤t_; let (timelock_intent_noun, current_) = next_cell(current)?; - let timelock_intent = timelock_intent_noun.as_atom()?.as_u64()? != 0; + let timelock_intent = timelock_intent_noun.in_space(space).as_atom()?.as_u64()? != 0; current = ¤t_; let (gift_noun, current_) = next_cell(current)?; - let gift = gift_noun.as_atom()?.as_u64()? != 0; + let gift = gift_noun.in_space(space).as_atom()?.as_u64()? != 0; current = ¤t_; - let parent_hash = current.as_atom()?.as_u64()? != 0; + let parent_hash = current.in_space(space).as_atom()?.as_u64()? != 0; Ok(SeedMask { output_source, @@ -428,13 +458,17 @@ impl NounEncode for PreSeed { } impl NounDecode for PreSeed { - fn from_noun(noun: &Noun) -> Result { - let cell = noun.as_cell().map_err(|_| NounDecodeError::ExpectedCell)?; + fn from_noun(noun: &Noun, space: &NounSpace) -> Result { + let cell = noun + .in_space(space) + .as_cell() + .map_err(|_| NounDecodeError::ExpectedCell)?; let name = cell.head().as_atom()?.into_string()?; let data = cell.tail().as_cell()?; let seed = data.head().as_atom()?.as_u64()?; - let mask = SeedMask::from_noun(&data.tail())?; + let mask_noun = data.tail().noun(); + let mask = SeedMask::from_noun(&mask_noun, space)?; Ok(PreSeed { name, seed, mask }) } @@ -459,13 +493,17 @@ impl NounEncode for PreInput { } impl NounDecode for PreInput { - fn from_noun(noun: &Noun) -> Result { - let cell = noun.as_cell().map_err(|_| NounDecodeError::ExpectedCell)?; + fn from_noun(noun: &Noun, space: &NounSpace) -> Result { + let cell = noun + .in_space(space) + .as_cell() + .map_err(|_| NounDecodeError::ExpectedCell)?; let name = cell.head().as_atom()?.into_string()?; let data = cell.tail().as_cell()?; let input = data.head().as_atom()?.as_u64()?; - let mask = InputMask::from_noun(&data.tail())?; + let mask_noun = data.tail().noun(); + let mask = InputMask::from_noun(&mask_noun, space)?; Ok(PreInput { name, input, mask }) } @@ -487,8 +525,11 @@ impl NounEncode for Draft { } impl NounDecode for Draft { - fn from_noun(noun: &Noun) -> Result { - let cell = noun.as_cell().map_err(|_| NounDecodeError::ExpectedCell)?; + fn from_noun(noun: &Noun, space: &NounSpace) -> Result { + let cell = noun + .in_space(space) + .as_cell() + .map_err(|_| NounDecodeError::ExpectedCell)?; let name = cell.head().as_atom()?.into_string()?; let inputs = cell.tail().as_atom()?.as_u64()?; @@ -538,8 +579,11 @@ impl NounEncode for DraftEntity { } impl NounDecode for DraftEntity { - fn from_noun(noun: &Noun) -> Result { - let cell = noun.as_cell().map_err(|_| NounDecodeError::ExpectedCell)?; + fn from_noun(noun: &Noun, space: &NounSpace) -> Result { + let cell = noun + .in_space(space) + .as_cell() + .map_err(|_| NounDecodeError::ExpectedCell)?; let tag = cell.head().as_atom()?.into_string()?; let data = cell.tail().as_cell()?; @@ -547,15 +591,18 @@ impl NounDecode for DraftEntity { let kind = match tag.as_str() { "draft" => { - let draft = Draft::from_noun(&data.tail())?; + let draft_noun = data.tail().noun(); + let draft = Draft::from_noun(&draft_noun, space)?; DraftEntityKind::Draft { name, draft } } "input" => { - let input = PreInput::from_noun(&data.tail())?; + let input_noun = data.tail().noun(); + let input = PreInput::from_noun(&input_noun, space)?; DraftEntityKind::Input { name, input } } "seed" => { - let seed = PreSeed::from_noun(&data.tail())?; + let seed_noun = data.tail().noun(); + let seed = PreSeed::from_noun(&seed_noun, space)?; DraftEntityKind::Seed { name, seed } } _ => return Err(NounDecodeError::InvalidEnumVariant), @@ -581,10 +628,15 @@ impl NounEncode for Master { } impl NounDecode for Master { - fn from_noun(noun: &Noun) -> Result { - let cell = noun.as_cell().map_err(|_| NounDecodeError::ExpectedCell)?; - let pub_key = Coil::from_noun(&cell.head())?; - let prv_key = Coil::from_noun(&cell.tail())?; + fn from_noun(noun: &Noun, space: &NounSpace) -> Result { + let cell = noun + .in_space(space) + .as_cell() + .map_err(|_| NounDecodeError::ExpectedCell)?; + let pub_key_noun = cell.head().noun(); + let prv_key_noun = cell.tail().noun(); + let pub_key = Coil::from_noun(&pub_key_noun, space)?; + let prv_key = Coil::from_noun(&prv_key_noun, space)?; Ok(Master { pub_key, prv_key }) } @@ -612,8 +664,8 @@ impl NounEncode for Balance { } impl NounDecode for Balance { - fn from_noun(noun: &Noun) -> Result { - let notes = HashMap::from_noun(noun)?; + fn from_noun(noun: &Noun, space: &NounSpace) -> Result { + let notes = HashMap::from_noun(noun, space)?; Ok(Balance { notes }) } } @@ -636,8 +688,8 @@ impl NounEncode for Network { } impl NounDecode for Network { - fn from_noun(noun: &Noun) -> Result { - let tag = noun.as_atom()?.into_string()?; + fn from_noun(noun: &Noun, space: &NounSpace) -> Result { + let tag = noun.in_space(space).as_atom()?.into_string()?; match tag.as_str() { "mainnet" => Ok(Network::Mainnet), "testnet" => Ok(Network::Testnet), @@ -664,8 +716,8 @@ impl NounEncode for PeekRequest { } impl NounDecode for PeekRequest { - fn from_noun(noun: &Noun) -> Result { - let tag = noun.as_atom()?.into_string()?; + fn from_noun(noun: &Noun, space: &NounSpace) -> Result { + let tag = noun.in_space(space).as_atom()?.into_string()?; match tag.as_str() { "balance" => Ok(PeekRequest::Balance), "block" => Ok(PeekRequest::Block), @@ -776,8 +828,8 @@ impl NounEncode for Trek { } impl NounDecode for Trek { - fn from_noun(noun: &Noun) -> Result { - let mut current = *noun; + fn from_noun(noun: &Noun, space: &NounSpace) -> Result { + let mut current = noun.in_space(space); let mut parts = Vec::new(); while let Ok(cell) = current.as_cell() { let part = cell.head().as_atom()?.into_string()?; @@ -805,8 +857,11 @@ impl NounEncode for Source { } impl NounDecode for Source { - fn from_noun(noun: &Noun) -> Result { - let cell = noun.as_cell().map_err(|_| NounDecodeError::ExpectedCell)?; + fn from_noun(noun: &Noun, space: &NounSpace) -> Result { + let cell = noun + .in_space(space) + .as_cell() + .map_err(|_| NounDecodeError::ExpectedCell)?; let head = cell.head().as_atom()?.as_u64()?; let tail = cell.tail().as_atom()?.as_u64()?; match (head, tail) { @@ -844,10 +899,14 @@ impl NounEncode for Lock { } impl NounDecode for Lock { - fn from_noun(noun: &Noun) -> Result { - let cell = noun.as_cell().map_err(|_| NounDecodeError::ExpectedCell)?; + fn from_noun(noun: &Noun, space: &NounSpace) -> Result { + let cell = noun + .in_space(space) + .as_cell() + .map_err(|_| NounDecodeError::ExpectedCell)?; let m = cell.head().as_atom()?.as_u64()?; - let pubkeys = HashSet::from_noun(&cell.tail())?; + let pubkeys_noun = cell.tail().noun(); + let pubkeys = HashSet::from_noun(&pubkeys_noun, space)?; Ok(Lock { m, pubkeys }) } } @@ -878,8 +937,11 @@ impl NounEncode for Timelock { } impl NounDecode for Timelock { - fn from_noun(noun: &Noun) -> Result { - let cell = noun.as_cell().map_err(|_| NounDecodeError::ExpectedCell)?; + fn from_noun(noun: &Noun, space: &NounSpace) -> Result { + let cell = noun + .in_space(space) + .as_cell() + .map_err(|_| NounDecodeError::ExpectedCell)?; let block = cell.head().as_atom()?.as_u64()?; let intent = match cell.tail().as_atom()?.as_u64()? { 0 => TimelockIntent::None, @@ -947,21 +1009,24 @@ impl NounEncode for Seed { } impl NounDecode for Seed { - fn from_noun(noun: &Noun) -> Result { + fn from_noun(noun: &Noun, space: &NounSpace) -> Result { println!("\nDecoding Seed from noun: {:?}", noun); - let cell = noun.as_cell().map_err(|_| NounDecodeError::ExpectedCell)?; + let cell = noun + .in_space(space) + .as_cell() + .map_err(|_| NounDecodeError::ExpectedCell)?; println!( "Root cell - head: {:?}, tail: {:?}", - cell.head(), - cell.tail() + cell.head().noun(), + cell.tail().noun() ); // Decode output_source - let source_noun = cell.head(); + let source_noun = cell.head().noun(); println!("Source noun: {:?}", source_noun); - let output_source = if let Ok(atom) = source_noun.as_atom() { + let output_source = if let Ok(atom) = source_noun.in_space(space).as_atom() { if atom.as_u64()? == 0 { println!("Found atom 0, decoding as None"); None @@ -970,11 +1035,11 @@ impl NounDecode for Seed { return Err(NounDecodeError::InvalidEnumVariant); } } else { - let source_cell = source_noun.as_cell()?; + let source_cell = source_noun.in_space(space).as_cell()?; println!( "Source cell - head: {:?}, tail: {:?}", - source_cell.head(), - source_cell.tail() + source_cell.head().noun(), + source_cell.tail().noun() ); if source_cell.head().as_atom()?.as_u64()? != 0 { @@ -983,25 +1048,27 @@ impl NounDecode for Seed { } println!("Decoding Some(Source)"); - Some(Source::from_noun(&source_cell.tail())?) + let source_tail = source_cell.tail().noun(); + Some(Source::from_noun(&source_tail, space)?) }; println!("Decoded output_source: {:?}", output_source); let rest = cell.tail().as_cell()?; println!( "First rest cell - head: {:?}, tail: {:?}", - rest.head(), - rest.tail() + rest.head().noun(), + rest.tail().noun() ); - let recipient = Lock::from_noun(&rest.head())?; + let recipient_noun = rest.head().noun(); + let recipient = Lock::from_noun(&recipient_noun, space)?; println!("Decoded recipient: {:?}", recipient); let rest = rest.tail().as_cell()?; println!( "Second rest cell - head: {:?}, tail: {:?}", - rest.head(), - rest.tail() + rest.head().noun(), + rest.tail().noun() ); let timelock_intent = match rest.head().as_atom()?.as_u64()? { @@ -1018,8 +1085,8 @@ impl NounDecode for Seed { let rest = rest.tail().as_cell()?; println!( "Third rest cell - head: {:?}, tail: {:?}", - rest.head(), - rest.tail() + rest.head().noun(), + rest.tail().noun() ); let gift = rest.head().as_atom()?.as_u64()?; @@ -1081,21 +1148,24 @@ impl NounEncode for Spend { } impl NounDecode for Spend { - fn from_noun(noun: &Noun) -> Result { + fn from_noun(noun: &Noun, space: &NounSpace) -> Result { println!("\nDecoding Spend from noun: {:?}", noun); - let cell = noun.as_cell().map_err(|_| NounDecodeError::ExpectedCell)?; + let cell = noun + .in_space(space) + .as_cell() + .map_err(|_| NounDecodeError::ExpectedCell)?; println!( "Root cell - head: {:?}, tail: {:?}", - cell.head(), - cell.tail() + cell.head().noun(), + cell.tail().noun() ); // Decode signature Option - let sig_noun = cell.head(); + let sig_noun = cell.head().noun(); println!("Signature noun: {:?}", sig_noun); - let signature = if let Ok(atom) = sig_noun.as_atom() { + let signature = if let Ok(atom) = sig_noun.in_space(space).as_atom() { if atom.as_u64()? == 0 { println!("Found atom 0, decoding as None"); None @@ -1104,11 +1174,11 @@ impl NounDecode for Spend { return Err(NounDecodeError::InvalidEnumVariant); } } else { - let sig_cell = sig_noun.as_cell()?; + let sig_cell = sig_noun.in_space(space).as_cell()?; println!( "Signature cell - head: {:?}, tail: {:?}", - sig_cell.head(), - sig_cell.tail() + sig_cell.head().noun(), + sig_cell.tail().noun() ); if sig_cell.head().as_atom()?.as_u64()? != 0 { @@ -1117,18 +1187,20 @@ impl NounDecode for Spend { } println!("Decoding Some(HashMap)"); - Some(HashMap::from_noun(&sig_cell.tail())?) + let sig_tail = sig_cell.tail().noun(); + Some(HashMap::from_noun(&sig_tail, space)?) }; println!("Decoded signature: {:?}", signature); let data = cell.tail().as_cell()?; println!( "Data cell - head: {:?}, tail: {:?}", - data.head(), - data.tail() + data.head().noun(), + data.tail().noun() ); - let seeds = HashSet::from_noun(&data.head())?; + let seeds_noun = data.head().noun(); + let seeds = HashSet::from_noun(&seeds_noun, space)?; println!("Decoded seeds: {:?}", seeds); let fee = data.tail().as_atom()?.as_u64()?; @@ -1151,65 +1223,95 @@ mod tests { use super::*; + struct StackGuard { + stack: NockStack, + } + + impl StackGuard { + fn new(words: usize) -> Self { + let stack = NockStack::new(words, 0); + Self { stack } + } + } + + impl std::ops::Deref for StackGuard { + type Target = NockStack; + + fn deref(&self) -> &Self::Target { + &self.stack + } + } + + impl std::ops::DerefMut for StackGuard { + fn deref_mut(&mut self) -> &mut Self::Target { + &mut self.stack + } + } + #[test] fn test_trek_encoding() { - let mut stack = NockStack::new(8 << 10 << 10, 0); + let mut stack = StackGuard::new(8 << 10 << 10); + let space = stack.noun_space(); let trek = Trek(vec![ "path".to_string(), "to".to_string(), "key".to_string(), ]); - let encoded = trek.to_noun(&mut stack); - let decoded = Trek::from_noun(&encoded).unwrap(); + let encoded = trek.to_noun(&mut *stack); + let decoded = Trek::from_noun(&encoded, &space).unwrap(); assert_eq!(trek, decoded); } #[test] fn test_source_encoding() { - let mut stack = NockStack::new(8 << 10 << 10, 0); + let mut stack = StackGuard::new(8 << 10 << 10); + let space = stack.noun_space(); let hash = Source::Hash(0x1234); - let encoded = hash.to_noun(&mut stack); - let decoded = Source::from_noun(&encoded).unwrap(); + let encoded = hash.to_noun(&mut *stack); + let decoded = Source::from_noun(&encoded, &space).unwrap(); assert_eq!(hash, decoded); let coinbase = Source::Coinbase; - let encoded = coinbase.to_noun(&mut stack); - let decoded = Source::from_noun(&encoded).unwrap(); + let encoded = coinbase.to_noun(&mut *stack); + let decoded = Source::from_noun(&encoded, &space).unwrap(); assert_eq!(coinbase, decoded); } #[test] fn test_lock_encoding() { - let mut stack = NockStack::new(8 << 10 << 10, 0); + let mut stack = StackGuard::new(8 << 10 << 10); + let space = stack.noun_space(); let mut pubkeys = HashSet::new(); pubkeys.insert(0x1234); pubkeys.insert(0x5678); let lock = Lock { m: 2, pubkeys }; - let encoded = lock.to_noun(&mut stack); - let decoded = Lock::from_noun(&encoded).unwrap(); + let encoded = lock.to_noun(&mut *stack); + let decoded = Lock::from_noun(&encoded, &space).unwrap(); assert_eq!(lock, decoded); } #[test] fn test_timelock_encoding() { - let mut stack = NockStack::new(8 << 10 << 10, 0); + let mut stack = StackGuard::new(8 << 10 << 10); + let space = stack.noun_space(); let timelock = Timelock { block: 0x1234, intent: TimelockIntent::After, }; - let encoded = timelock.to_noun(&mut stack); - let decoded = Timelock::from_noun(&encoded).unwrap(); + let encoded = timelock.to_noun(&mut *stack); + let decoded = Timelock::from_noun(&encoded, &space).unwrap(); assert_eq!(timelock, decoded); } #[test] fn test_seed_encoding() { - let mut stack = NockStack::new(8 << 10 << 10, 0); + let mut stack = StackGuard::new(8 << 10 << 10); + let space = stack.noun_space(); let mut pubkeys = HashSet::new(); pubkeys.insert(0x1234); @@ -1221,14 +1323,15 @@ mod tests { gift: 100, parent_hash: 0x9abc, }; - let encoded = seed.to_noun(&mut stack); - let decoded = Seed::from_noun(&encoded).unwrap(); + let encoded = seed.to_noun(&mut *stack); + let decoded = Seed::from_noun(&encoded, &space).unwrap(); assert_eq!(seed, decoded); } #[test] fn test_preseed_encoding() { - let mut stack = NockStack::new(8 << 10 << 10, 0); + let mut stack = StackGuard::new(8 << 10 << 10); + let space = stack.noun_space(); let preseed = PreSeed { name: "test_seed".to_string(), @@ -1241,14 +1344,15 @@ mod tests { parent_hash: true, }, }; - let encoded = preseed.to_noun(&mut stack); - let decoded = PreSeed::from_noun(&encoded).unwrap(); + let encoded = preseed.to_noun(&mut *stack); + let decoded = PreSeed::from_noun(&encoded, &space).unwrap(); assert_eq!(preseed, decoded); } #[test] fn test_spend_encoding() { - let mut stack = NockStack::new(8 << 10 << 10, 0); + let mut stack = StackGuard::new(8 << 10 << 10); + let space = stack.noun_space(); let mut signatures = HashMap::new(); signatures.insert(0x1234, 0x5678); @@ -1271,14 +1375,15 @@ mod tests { seeds, fee: 10, }; - let encoded = spend.to_noun(&mut stack); - let decoded = Spend::from_noun(&encoded).unwrap(); + let encoded = spend.to_noun(&mut *stack); + let decoded = Spend::from_noun(&encoded, &space).unwrap(); assert_eq!(spend, decoded); } #[test] fn test_preinput_encoding() { - let mut stack = NockStack::new(8 << 10 << 10, 0); + let mut stack = StackGuard::new(8 << 10 << 10); + let space = stack.noun_space(); let preinput = PreInput { name: "test_input".to_string(), @@ -1292,21 +1397,22 @@ mod tests { }, }, }; - let encoded = preinput.to_noun(&mut stack); - let decoded = PreInput::from_noun(&encoded).unwrap(); + let encoded = preinput.to_noun(&mut *stack); + let decoded = PreInput::from_noun(&encoded, &space).unwrap(); assert_eq!(preinput, decoded); } #[test] fn test_draft_encoding() { - let mut stack = NockStack::new(8 << 10 << 10, 0); + let mut stack = StackGuard::new(8 << 10 << 10); + let space = stack.noun_space(); let draft = Draft { name: "test_draft".to_string(), inputs: 0x1234, // Using u64 as specified in struct }; - let encoded = draft.to_noun(&mut stack); - let decoded = Draft::from_noun(&encoded).unwrap(); + let encoded = draft.to_noun(&mut *stack); + let decoded = Draft::from_noun(&encoded, &space).unwrap(); assert_eq!(draft, decoded); } } diff --git a/crates/noun-serde/tests/serde.rs b/crates/noun-serde/tests/serde.rs index e35345db4..909ebd6e5 100644 --- a/crates/noun-serde/tests/serde.rs +++ b/crates/noun-serde/tests/serde.rs @@ -1,11 +1,46 @@ #![allow(clippy::unwrap_used)] +#[cfg(test)] +use std::ops::{Deref, DerefMut}; + +#[cfg(test)] +struct TestStackGuard { + stack: nockvm::mem::NockStack, +} + +#[cfg(test)] +impl TestStackGuard { + fn new(words: usize) -> Self { + let stack = nockvm::mem::NockStack::new(words, 0); + Self { stack } + } +} + +#[cfg(test)] +impl Deref for TestStackGuard { + type Target = nockvm::mem::NockStack; + + fn deref(&self) -> &Self::Target { + &self.stack + } +} + +#[cfg(test)] +impl DerefMut for TestStackGuard { + fn deref_mut(&mut self) -> &mut Self::Target { + &mut self.stack + } +} + #[cfg(test)] mod tests { use std::collections::HashSet; + use nockvm::noun::NounSpace; use noun_serde::{decode_bool, encode_bool, NounDecode, NounEncode}; + use super::TestStackGuard; + // Helper struct for testing #[derive(Debug, PartialEq, NounEncode, NounDecode)] struct TestStruct { @@ -26,90 +61,98 @@ mod tests { // Test primitive type encoding/decoding #[test] fn test_u64_encoding() { - let mut stack = nockvm::mem::NockStack::new(8 << 10 << 10, 0); + let mut stack = TestStackGuard::new(8 << 10 << 10); + let space = stack.noun_space(); let original = 42u64; - let encoded = original.to_noun(&mut stack); - let decoded = u64::from_noun(&encoded).unwrap(); + let encoded = original.to_noun(&mut *stack); + let decoded = u64::from_noun(&encoded, &space).unwrap(); assert_eq!(original, decoded); } #[test] fn test_string_encoding() { - let mut stack = nockvm::mem::NockStack::new(8 << 10 << 10, 0); + let mut stack = TestStackGuard::new(8 << 10 << 10); + let space = stack.noun_space(); let original = String::from("test"); - let encoded = original.to_noun(&mut stack); - let decoded = String::from_noun(&encoded).unwrap(); + let encoded = original.to_noun(&mut *stack); + let decoded = String::from_noun(&encoded, &space).unwrap(); assert_eq!(original, decoded); } #[test] fn test_option_encoding() { - let mut stack = nockvm::mem::NockStack::new(8 << 10 << 10, 0); + let mut stack = TestStackGuard::new(8 << 10 << 10); + let space = stack.noun_space(); // Test Some let original = Some(42u64); - let encoded = original.to_noun(&mut stack); - let decoded = Option::::from_noun(&encoded).unwrap(); + let encoded = original.to_noun(&mut *stack); + let decoded = Option::::from_noun(&encoded, &space).unwrap(); assert_eq!(original, decoded); // Test None let original: Option = None; - let encoded = original.to_noun(&mut stack); - let decoded = Option::::from_noun(&encoded).unwrap(); + let encoded = original.to_noun(&mut *stack); + let decoded = Option::::from_noun(&encoded, &space).unwrap(); assert_eq!(original, decoded); } #[test] fn test_vec_encoding() { - let mut stack = nockvm::mem::NockStack::new(8 << 10 << 10, 0); + let mut stack = TestStackGuard::new(8 << 10 << 10); + let space = stack.noun_space(); let original = vec![1u64, 2, 3, 4, 5]; - let encoded = original.to_noun(&mut stack); - let decoded = Vec::::from_noun(&encoded).unwrap(); + let encoded = original.to_noun(&mut *stack); + let decoded = Vec::::from_noun(&encoded, &space).unwrap(); assert_eq!(original, decoded); } #[test] fn test_bool_encoding() { + let space = NounSpace::empty(); // Test true let encoded = encode_bool(true); - let decoded = decode_bool(&encoded).unwrap(); + let decoded = decode_bool(&encoded, &space).unwrap(); assert!(decoded); // Test false let encoded = encode_bool(false); - let decoded = decode_bool(&encoded).unwrap(); + let decoded = decode_bool(&encoded, &space).unwrap(); assert!(!decoded); } #[test] fn test_struct_encoding() { - let mut stack = nockvm::mem::NockStack::new(8 << 10 << 10, 0); + let mut stack = TestStackGuard::new(8 << 10 << 10); + let space = stack.noun_space(); let original = TestStruct { a: 42, b: "test".to_string(), c: Some(123), }; - let encoded = original.to_noun(&mut stack); + let encoded = original.to_noun(&mut *stack); println!("encoded TestStruct: {:?}", encoded); - let decoded = TestStruct::from_noun(&encoded).unwrap(); + let decoded = TestStruct::from_noun(&encoded, &space).unwrap(); assert_eq!(original, decoded); } #[test] fn test_enum_variants() { - let mut stack = nockvm::mem::NockStack::new(8 << 10 << 10, 0); + let mut stack = TestStackGuard::new(8 << 10 << 10); + let space = stack.noun_space(); // Test variant A (single field) let original = TestEnum::A(42); println!("\nTesting variant A:"); println!("original: {:?}", original); - let encoded = original.to_noun(&mut stack); + let encoded = original.to_noun(&mut *stack); println!("encoded raw: {:?}", encoded); if let Ok(cell) = encoded.as_cell() { - println!("head: {:?}", cell.head()); - println!("tail: {:?}", cell.tail()); + let cell = cell.in_space(&space); + println!("head: {:?}", cell.head().noun()); + println!("tail: {:?}", cell.tail().noun()); } - let decoded = TestEnum::from_noun(&encoded).unwrap(); + let decoded = TestEnum::from_noun(&encoded, &space).unwrap(); println!("decoded: {:?}", decoded); assert_eq!(original, decoded); @@ -120,17 +163,18 @@ mod tests { }; println!("\nTesting variant B:"); println!("original: {:?}", original); - let encoded = original.to_noun(&mut stack); + let encoded = original.to_noun(&mut *stack); println!("encoded raw: {:?}", encoded); if let Ok(cell) = encoded.as_cell() { - println!("head: {:?}", cell.head()); - println!("tail: {:?}", cell.tail()); + let cell = cell.in_space(&space); + println!("head: {:?}", cell.head().noun()); + println!("tail: {:?}", cell.tail().noun()); if let Ok(tail_cell) = cell.tail().as_cell() { - println!("tail.head: {:?}", tail_cell.head()); - println!("tail.tail: {:?}", tail_cell.tail()); + println!("tail.head: {:?}", tail_cell.head().noun()); + println!("tail.tail: {:?}", tail_cell.tail().noun()); } } - let decoded = TestEnum::from_noun(&encoded).unwrap(); + let decoded = TestEnum::from_noun(&encoded, &space).unwrap(); println!("decoded: {:?}", decoded); assert_eq!(original, decoded); @@ -138,19 +182,21 @@ mod tests { let original = TestEnum::C; println!("\nTesting variant C:"); println!("original: {:?}", original); - let encoded = original.to_noun(&mut stack); + let encoded = original.to_noun(&mut *stack); println!("encoded raw: {:?}", encoded); if let Ok(cell) = encoded.as_cell() { - println!("head: {:?}", cell.head()); - println!("tail: {:?}", cell.tail()); + let cell = cell.in_space(&space); + println!("head: {:?}", cell.head().noun()); + println!("tail: {:?}", cell.tail().noun()); } - let decoded = TestEnum::from_noun(&encoded).unwrap(); + let decoded = TestEnum::from_noun(&encoded, &space).unwrap(); assert_eq!(original, decoded); } #[test] fn test_hashset_encoding() { - let mut stack = nockvm::mem::NockStack::new(8 << 10 << 10, 0); + let mut stack = TestStackGuard::new(8 << 10 << 10); + let space = stack.noun_space(); // Create a test set let mut set = HashSet::new(); @@ -159,16 +205,58 @@ mod tests { set.insert(3u64); // Test encoding and decoding - let encoded = set.to_noun(&mut stack); - let decoded = HashSet::::from_noun(&encoded).unwrap(); + let encoded = set.to_noun(&mut *stack); + let decoded = HashSet::::from_noun(&encoded, &space).unwrap(); assert_eq!(set, decoded); // Test empty set let empty_set: HashSet = HashSet::new(); - let encoded_empty = empty_set.to_noun(&mut stack); - let decoded_empty = HashSet::::from_noun(&encoded_empty).unwrap(); + let encoded_empty = empty_set.to_noun(&mut *stack); + let decoded_empty = HashSet::::from_noun(&encoded_empty, &space).unwrap(); assert_eq!(empty_set, decoded_empty); } + + #[test] + fn test_nested_option_representation() { + let mut stack = nockvm::mem::NockStack::new(nockvm::mem::NOCK_STACK_SIZE_TINY, 0); + let proof = "dummy-proof".to_string(); + + let nested = Some(Some(proof)); + let encoded = nested.to_noun(&mut stack); + let space = stack.noun_space(); + + let outer_cell = encoded + .in_space(&space) + .as_cell() + .expect("outer option should be cell"); + assert_eq!( + outer_cell + .head() + .as_atom() + .expect("outer head should be atom") + .as_u64() + .expect("outer head should be 0"), + 0 + ); + + let inner_cell = outer_cell + .tail() + .as_cell() + .expect("inner option should be cell"); + assert_eq!( + inner_cell + .head() + .as_atom() + .expect("inner head should be atom") + .as_u64() + .expect("inner head should be 0"), + 0 + ); + assert_eq!( + String::from_noun(&inner_cell.tail().noun(), &space).expect("decode proof"), + "dummy-proof" + ); + } } #[cfg(test)] @@ -176,10 +264,12 @@ mod complex_tests { use std::collections::HashMap; use std::fmt::Debug; - use nockvm::ext::{make_tas, AtomExt}; - use nockvm::noun::{FullDebugCell, Noun, NounAllocator, Slots, T}; + use nockvm::ext::make_tas; + use nockvm::noun::{FullDebugCell, Noun, NounAllocator, NounSpace, T}; use noun_serde::{NounDecode, NounDecodeError, NounEncode}; + use super::TestStackGuard; + // Complex recursive tree structure #[derive(Debug, PartialEq, Clone)] enum Tree @@ -226,17 +316,23 @@ mod complex_tests { where T: NounEncode + NounDecode + Debug + PartialEq + Clone, { - fn from_noun(noun: &Noun) -> Result { - let cell = noun.as_cell().map_err(|_| NounDecodeError::ExpectedCell)?; + fn from_noun(noun: &Noun, space: &NounSpace) -> Result { + let cell = noun + .in_space(space) + .as_cell() + .map_err(|_| NounDecodeError::ExpectedCell)?; let tag = cell.head().as_atom()?.into_string()?; match tag.as_str() { "branch" => { let data = cell.tail().as_cell()?; - let left = Box::new(Tree::from_noun(&data.head())?); + let left_noun = data.head().noun(); + let left = Box::new(Tree::from_noun(&left_noun, space)?); let rest = data.tail().as_cell()?; - let right = Box::new(Tree::from_noun(&rest.head())?); - let metadata = HashMap::from_noun(&rest.tail())?; + let right_noun = rest.head().noun(); + let right = Box::new(Tree::from_noun(&right_noun, space)?); + let metadata_noun = rest.tail().noun(); + let metadata = HashMap::from_noun(&metadata_noun, space)?; Ok(Tree::Branch { left, right, @@ -244,7 +340,8 @@ mod complex_tests { }) } "leaf" => { - let value = T::from_noun(&cell.tail())?; + let value_noun = cell.tail().noun(); + let value = T::from_noun(&value_noun, space)?; Ok(Tree::Leaf(value)) } _ => Err(NounDecodeError::InvalidEnumVariant), @@ -281,7 +378,8 @@ mod complex_tests { #[test] fn test_recursive_tree() { - let mut stack = nockvm::mem::NockStack::new(8 << 10 << 10, 0); + let mut stack = TestStackGuard::new(8 << 10 << 10); + let space = stack.noun_space(); // Create a complex tree let mut metadata1 = HashMap::new(); @@ -300,19 +398,24 @@ mod complex_tests { metadata: metadata2, }; - let encoded = tree.to_noun(&mut stack); + let encoded = tree.to_noun(&mut *stack); + let encoded_cell = encoded.as_cell().unwrap(); println!( "Encoded tree: {:?}", - FullDebugCell(&encoded.as_cell().unwrap()) + FullDebugCell { + cell: &encoded_cell, + space: &space + } ); - let decoded = Tree::from_noun(&encoded).unwrap(); + let decoded = Tree::from_noun(&encoded, &space).unwrap(); assert_eq!(tree, decoded); } #[test] fn test_transaction_status() { - let mut stack = nockvm::mem::NockStack::new(8 << 10 << 10, 0); + let mut stack = TestStackGuard::new(8 << 10 << 10); + let space = stack.noun_space(); let status = TransactionStatus::Pending { retries: 3, @@ -320,33 +423,35 @@ mod complex_tests { }; println!("\nEncoding status: {:?}", status); - let encoded = status.to_noun(&mut stack); + let encoded = status.to_noun(&mut *stack); println!("Encoded status noun: {:?}", encoded); if let Ok(cell) = encoded.as_cell() { + let cell = cell.in_space(&space); println!("Status cell structure:"); - println!("Head: {:?}", cell.head()); + println!("Head: {:?}", cell.head().noun()); if let Ok(head_atom) = cell.head().as_atom() { if let Ok(tag) = head_atom.into_string() { println!("Tag string: {}", tag); } } - println!("Tail: {:?}", cell.tail()); + println!("Tail: {:?}", cell.tail().noun()); if let Ok(tail_cell) = cell.tail().as_cell() { println!("Tail structure:"); - println!(" Head: {:?}", tail_cell.head()); - println!(" Tail: {:?}", tail_cell.tail()); + println!(" Head: {:?}", tail_cell.head().noun()); + println!(" Tail: {:?}", tail_cell.tail().noun()); } } println!("\nDecoding status..."); - let decoded = TransactionStatus::from_noun(&encoded).unwrap(); + let decoded = TransactionStatus::from_noun(&encoded, &space).unwrap(); println!("Decoded status: {:?}", decoded); assert_eq!(status, decoded); } #[test] fn test_transaction_data_decoding() { - let mut stack = nockvm::mem::NockStack::new(8 << 10 << 10, 0); + let mut stack = TestStackGuard::new(8 << 10 << 10); + let space = stack.noun_space(); let original = TransactionData { sender: 0x1234, @@ -357,30 +462,74 @@ mod complex_tests { }; println!("\nEncoding TransactionData: {:?}", original); - let encoded = original.to_noun(&mut stack); + let encoded = original.to_noun(&mut *stack); println!("Encoded noun: {:?}", encoded); // Print the binary tree structure if let Ok(cell) = encoded.as_cell() { println!("\nBinary tree structure:"); - println!("Root cell: {:?}", FullDebugCell(&cell)); - println!("At axis 2 (sender): {:?}", cell.slot(2)); - println!("At axis 3: {:?}", cell.slot(3)); - println!("At axis 6 (receiver): {:?}", cell.slot(6)); - println!("At axis 7: {:?}", cell.slot(7)); - println!("At axis 14 (amount): {:?}", cell.slot(14)); - println!("At axis 15: {:?}", cell.slot(15)); + println!( + "Root cell: {:?}", + FullDebugCell { + cell: &cell, + space: &space + } + ); + let cell_noun = cell.as_noun(); + println!( + "At axis 2 (sender): {:?}", + cell_noun + .in_space(&space) + .slot(2) + .map(|handle| handle.noun()) + ); + println!( + "At axis 3: {:?}", + cell_noun + .in_space(&space) + .slot(3) + .map(|handle| handle.noun()) + ); + println!( + "At axis 6 (receiver): {:?}", + cell_noun + .in_space(&space) + .slot(6) + .map(|handle| handle.noun()) + ); + println!( + "At axis 7: {:?}", + cell_noun + .in_space(&space) + .slot(7) + .map(|handle| handle.noun()) + ); + println!( + "At axis 14 (amount): {:?}", + cell_noun + .in_space(&space) + .slot(14) + .map(|handle| handle.noun()) + ); + println!( + "At axis 15: {:?}", + cell_noun + .in_space(&space) + .slot(15) + .map(|handle| handle.noun()) + ); } println!("\nDecoding TransactionData..."); - let decoded = TransactionData::from_noun(&encoded).unwrap(); + let decoded = TransactionData::from_noun(&encoded, &space).unwrap(); println!("Decoded result: {:?}", decoded); assert_eq!(original, decoded); } #[test] fn test_complex_transaction() { - let mut stack = nockvm::mem::NockStack::new(8 << 10 << 10, 0); + let mut stack = TestStackGuard::new(8 << 10 << 10); + let space = stack.noun_space(); let transaction = Transaction { id: 1, @@ -400,19 +549,20 @@ mod complex_tests { }; println!("\nEncoding transaction: {:?}", transaction); - let encoded = transaction.to_noun(&mut stack); + let encoded = transaction.to_noun(&mut *stack); println!("\nEncoded transaction noun: {:?}", encoded); - if let Ok(cell) = encoded.as_cell() { - println!("Transaction cell head: {:?}", cell.head()); - println!("Transaction cell tail: {:?}", cell.tail()); - if let Ok(status_cell) = cell.slot(7).unwrap().as_cell() { - println!("Status tag: {:?}", status_cell.head()); - println!("Status data: {:?}", status_cell.tail()); + let encoded_handle = encoded.in_space(&space); + if let Ok(cell) = encoded_handle.as_cell() { + println!("Transaction cell head: {:?}", cell.head().noun()); + println!("Transaction cell tail: {:?}", cell.tail().noun()); + if let Ok(status_cell) = encoded_handle.slot(7).and_then(|h| h.as_cell()) { + println!("Status tag: {:?}", status_cell.head().noun()); + println!("Status data: {:?}", status_cell.tail().noun()); } } println!("\nDecoding transaction..."); - let decoded = Transaction::from_noun(&encoded).unwrap(); + let decoded = Transaction::from_noun(&encoded, &space).unwrap(); println!("Successfully decoded transaction: {:?}", decoded); assert_eq!(transaction, decoded); @@ -421,8 +571,8 @@ mod complex_tests { transaction2.status = TransactionStatus::Complete { result: Ok(vec![1, 2, 3]), }; - let encoded2 = transaction2.to_noun(&mut stack); - let decoded2 = Transaction::from_noun(&encoded2).unwrap(); + let encoded2 = transaction2.to_noun(&mut *stack); + let decoded2 = Transaction::from_noun(&encoded2, &space).unwrap(); assert_eq!(transaction2, decoded2); let mut transaction3 = transaction; @@ -430,46 +580,53 @@ mod complex_tests { reason: "Test failure".to_string(), trace: vec![404, 500], }; - let encoded3 = transaction3.to_noun(&mut stack); - let decoded3 = Transaction::from_noun(&encoded3).unwrap(); + let encoded3 = transaction3.to_noun(&mut *stack); + let decoded3 = Transaction::from_noun(&encoded3, &space).unwrap(); assert_eq!(transaction3, decoded3); } #[test] fn test_nested_options_and_results() { - let mut stack = nockvm::mem::NockStack::new(8 << 10 << 10, 0); + let mut stack = TestStackGuard::new(8 << 10 << 10); + let space = stack.noun_space(); // Test deeply nested Option>> let nested_data: Option>, String>> = Some(Ok(Some(vec![1, 2, 3]))); - let encoded = nested_data.to_noun(&mut stack); + let encoded = nested_data.to_noun(&mut *stack); + let encoded_cell = encoded.as_cell().unwrap(); println!( "Encoded nested data: {:?}", - FullDebugCell(&encoded.as_cell().unwrap()) + FullDebugCell { + cell: &encoded_cell, + space: &space + } ); - let decoded = Option::>, String>>::from_noun(&encoded).unwrap(); + let decoded = + Option::>, String>>::from_noun(&encoded, &space).unwrap(); assert_eq!(nested_data, decoded); // Test None case let none_data: Option>, String>> = None; - let encoded_none = none_data.to_noun(&mut stack); + let encoded_none = none_data.to_noun(&mut *stack); let decoded_none = - Option::>, String>>::from_noun(&encoded_none).unwrap(); + Option::>, String>>::from_noun(&encoded_none, &space).unwrap(); assert_eq!(none_data, decoded_none); // Test Error case let err_data: Option>, String>> = Some(Err("test error".to_string())); - let encoded_err = err_data.to_noun(&mut stack); + let encoded_err = err_data.to_noun(&mut *stack); let decoded_err = - Option::>, String>>::from_noun(&encoded_err).unwrap(); + Option::>, String>>::from_noun(&encoded_err, &space).unwrap(); assert_eq!(err_data, decoded_err); } #[test] fn test_complex_collections() { - let mut stack = nockvm::mem::NockStack::new(8 << 10 << 10, 0); + let mut stack = TestStackGuard::new(8 << 10 << 10); + let space = stack.noun_space(); // Test Vec>>> let mut map1 = HashMap::new(); @@ -481,13 +638,18 @@ mod complex_tests { let complex_collection = vec![map1, map2]; - let encoded = complex_collection.to_noun(&mut stack); + let encoded = complex_collection.to_noun(&mut *stack); + let encoded_cell = encoded.as_cell().unwrap(); println!( "Encoded collection: {:?}", - FullDebugCell(&encoded.as_cell().unwrap()) + FullDebugCell { + cell: &encoded_cell, + space: &space + } ); - let decoded = Vec::>>>::from_noun(&encoded).unwrap(); + let decoded = + Vec::>>>::from_noun(&encoded, &space).unwrap(); assert_eq!(complex_collection, decoded); } } diff --git a/crates/raw-tx-checker/src/main.rs b/crates/raw-tx-checker/src/main.rs index 0651c0250..4ed4cc298 100644 --- a/crates/raw-tx-checker/src/main.rs +++ b/crates/raw-tx-checker/src/main.rs @@ -4,14 +4,12 @@ use std::path::PathBuf; use anyhow::{anyhow, Context, Result}; use clap::Parser; use nockchain_types::tx_engine::common::Hash; -use nockvm::mem::NockStack; +use nockvm::mem::{NockStack, NOCK_STACK_SIZE_TINY}; use nockvm::noun::IndirectAtom; use nockvm::serialization; use noun_serde::prelude::*; use zkvm_jetpack::jets::tip5_jets::hash_hashable; -const DEFAULT_STACK_WORDS: usize = 8 << 10 << 10; - #[derive(Parser)] #[command(author, version, about = "Inspect raw transaction hashable jams")] struct Args { @@ -26,18 +24,19 @@ fn main() -> Result<()> { let jam_bytes = fs::read(&args.input) .with_context(|| format!("failed to read {}", args.input.display()))?; - let mut stack = NockStack::new(DEFAULT_STACK_WORDS, 0); + let mut stack = NockStack::new(NOCK_STACK_SIZE_TINY, 0); + let space = stack.noun_space(); let jam_atom = unsafe { - IndirectAtom::new_raw_bytes(&mut stack, jam_bytes.len(), jam_bytes.as_ptr()) - .normalize_as_atom() + let mut atom = IndirectAtom::new_raw_bytes(&mut stack, jam_bytes.len(), jam_bytes.as_ptr()); + atom.normalize_as_atom_stack() }; let hashable = serialization::cue(&mut stack, jam_atom) .map_err(|err| anyhow!("failed to cue jammed noun: {err:?}"))?; - let digest_noun = hash_hashable(&mut stack, hashable) + let digest_noun = hash_hashable(&mut stack, hashable, &space) .map_err(|err| anyhow!("hash_hashable jet failed: {err:?}"))?; - let tip5_hash = Hash::from_noun(&digest_noun)?; + let tip5_hash = Hash::from_noun(&digest_noun, &space)?; println!("Tip5 limbs (hex):"); for (idx, limb) in tip5_hash.0.iter().enumerate() { let belt_u64 = limb.0; diff --git a/crates/wallet-tx-builder/Cargo.toml b/crates/wallet-tx-builder/Cargo.toml new file mode 100644 index 000000000..97a7fa13d --- /dev/null +++ b/crates/wallet-tx-builder/Cargo.toml @@ -0,0 +1,15 @@ +[package] +name = "wallet-tx-builder" +version.workspace = true +edition.workspace = true +publish = false + +[dependencies] +bytes.workspace = true +nockapp.workspace = true +nockchain-math.workspace = true +nockchain-types.workspace = true +nockvm.workspace = true +noun-serde.workspace = true +thiserror.workspace = true +tracing.workspace = true diff --git a/crates/wallet-tx-builder/src/adapter.rs b/crates/wallet-tx-builder/src/adapter.rs new file mode 100644 index 000000000..d12bcec0c --- /dev/null +++ b/crates/wallet-tx-builder/src/adapter.rs @@ -0,0 +1,325 @@ +use std::collections::BTreeMap; +use std::fmt::Debug; + +use nockchain_types::tx_engine::common::{BlockHeight, Hash, Name}; +use nockchain_types::tx_engine::v1::note::BalanceUpdate; +use thiserror::Error; + +use crate::determinism::compare_names_lex; +use crate::types::CandidateNote; + +#[derive(Debug, Clone, PartialEq, Eq)] +/// Snapshot identity metadata shared by all pages in one consistent balance view. +pub struct SnapshotMetadata { + pub height: BlockHeight, + pub block_id: Hash, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +/// Deduplicated, normalized snapshot used as planner candidate input. +pub struct NormalizedSnapshot { + pub metadata: SnapshotMetadata, + pub candidates: Vec, +} + +#[derive(Debug, Clone, PartialEq, Eq, Error)] +/// Errors indicating that fetched pages do not represent one consistent snapshot. +pub enum SnapshotConsistencyError { + #[error("missing snapshot metadata")] + MissingMetadata, + #[error("snapshot height drifted across pages")] + HeightDrift, + #[error("snapshot block-id drifted across pages")] + BlockIdDrift, +} + +#[derive(Debug, Clone, PartialEq, Eq, Error)] +/// Errors raised while deduplicating notes from snapshot pages. +pub enum CandidateNormalizationError { + #[error("duplicate note has non-identical payload for name {first}/{last}")] + DuplicateNameMismatch { first: String, last: String }, +} + +#[derive(Debug, Clone, PartialEq, Eq, Error)] +/// Combined normalization error for metadata consistency and candidate payloads. +pub enum NormalizeSnapshotError { + #[error(transparent)] + Snapshot(#[from] SnapshotConsistencyError), + #[error(transparent)] + Candidate(#[from] CandidateNormalizationError), +} + +#[derive(Debug, Error)] +/// End-to-end collection failure while fetching and normalizing balance snapshots. +pub enum CollectSnapshotError +where + E: std::fmt::Display + Debug, +{ + #[error("snapshot page fetch failed: {0}")] + Fetch(E), + #[error(transparent)] + Normalize(#[from] NormalizeSnapshotError), + #[error("snapshot drift retries exhausted after {attempts} attempts: {last_error}")] + SnapshotDriftRetriesExhausted { + attempts: usize, + last_error: SnapshotConsistencyError, + }, +} + +/// Source capable of fetching a full set of balance pages for one snapshot attempt. +pub trait SnapshotPageFetcher { + type Error: std::fmt::Display + Debug; + + fn fetch_pages(&mut self) -> Result, Self::Error>; +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)] +struct NoteNameKey { + first: [u64; 5], + last: [u64; 5], +} + +/// Fetches pages and retries drift errors until a stable normalized snapshot is produced. +pub fn collect_snapshot_with_retry( + fetcher: &mut F, + max_retries: usize, +) -> Result> +where + F: SnapshotPageFetcher, +{ + let mut attempts = 0usize; + loop { + attempts += 1; + let pages = fetcher.fetch_pages().map_err(CollectSnapshotError::Fetch)?; + match normalize_balance_pages(&pages) { + Ok(snapshot) => return Ok(snapshot), + Err(NormalizeSnapshotError::Snapshot(err)) + if matches!( + err, + SnapshotConsistencyError::HeightDrift | SnapshotConsistencyError::BlockIdDrift + ) => + { + if attempts <= max_retries { + continue; + } + return Err(CollectSnapshotError::SnapshotDriftRetriesExhausted { + attempts, + last_error: err, + }); + } + Err(err) => return Err(CollectSnapshotError::Normalize(err)), + } + } +} + +/// Normalizes balance pages into one metadata stamp and deduplicated candidate set. +pub fn normalize_balance_pages( + pages: &[BalanceUpdate], +) -> Result { + let metadata = enforce_single_snapshot( + &pages + .iter() + .map(|page| SnapshotMetadata { + height: page.height.clone(), + block_id: page.block_id.clone(), + }) + .collect::>(), + )?; + + let mut deduped = BTreeMap::::new(); + for page in pages { + for (name, note) in &page.notes.0 { + let candidate = CandidateNote::from_note(name, note); + let key = note_name_key(name); + if let Some(existing) = deduped.get(&key) { + if existing != &candidate { + return Err(CandidateNormalizationError::DuplicateNameMismatch { + first: name.first.to_base58(), + last: name.last.to_base58(), + } + .into()); + } + continue; + } + deduped.insert(key, candidate); + } + } + + let mut candidates = deduped.into_values().collect::>(); + candidates.sort_by(|a, b| compare_names_lex(&a.identity().name, &b.identity().name)); + Ok(NormalizedSnapshot { + metadata, + candidates, + }) +} + +/// Validates all provided pages were produced at one height and block id. +pub fn enforce_single_snapshot( + pages: &[SnapshotMetadata], +) -> Result { + let first = pages + .first() + .ok_or(SnapshotConsistencyError::MissingMetadata)?; + for page in pages.iter().skip(1) { + if page.height != first.height { + return Err(SnapshotConsistencyError::HeightDrift); + } + if page.block_id != first.block_id { + return Err(SnapshotConsistencyError::BlockIdDrift); + } + } + Ok(first.clone()) +} + +fn note_name_key(name: &Name) -> NoteNameKey { + NoteNameKey { + first: name.first.to_array(), + last: name.last.to_array(), + } +} + +#[cfg(test)] +mod tests { + use bytes::Bytes; + use nockapp::noun::slab::{NockJammer, NounSlab}; + use nockchain_math::belt::Belt; + use nockchain_types::tx_engine::common::Nicks; + use nockchain_types::tx_engine::v1::note::{Balance, Note, NoteData, NoteDataEntry, NoteV1}; + use noun_serde::NounEncode; + + use super::*; + + fn hash(v: u64) -> Hash { + Hash::from_limbs(&[v, 0, 0, 0, 0]) + } + + fn name(v: u64) -> Name { + Name::new(hash(v), hash(v + 100)) + } + + fn jam(value: &T) -> Bytes { + let mut slab: NounSlab = NounSlab::new(); + let noun = value.to_noun(&mut slab); + slab.set_root(noun); + slab.jam() + } + + fn note_v1(name: Name, origin_page: u64, assets: u64, key: &str, value: u64) -> Note { + let note_data = NoteData::new(vec![NoteDataEntry::new(key.to_string(), jam(&value))]); + Note::V1(NoteV1::new( + BlockHeight(Belt(origin_page)), + name, + note_data, + Nicks(assets as usize), + )) + } + + fn page(height: u64, block_id: u64, notes: Vec<(Name, Note)>) -> BalanceUpdate { + BalanceUpdate { + height: BlockHeight(Belt(height)), + block_id: hash(block_id), + notes: Balance(notes), + } + } + + #[test] + fn normalize_balance_pages_dedupes_identical_entries() { + let n = name(1); + let p1 = page(10, 99, vec![(n.clone(), note_v1(n.clone(), 10, 5, "k", 1))]); + let p2 = page(10, 99, vec![(n.clone(), note_v1(n.clone(), 10, 5, "k", 1))]); + + let normalized = normalize_balance_pages(&[p1, p2]).expect("normalized"); + assert_eq!(normalized.candidates.len(), 1); + assert_eq!(normalized.metadata.height, BlockHeight(Belt(10))); + assert_eq!(normalized.metadata.block_id, hash(99)); + } + + #[test] + fn normalize_balance_pages_rejects_duplicate_payload_mismatch() { + let n = name(2); + let p1 = page(10, 99, vec![(n.clone(), note_v1(n.clone(), 10, 5, "k", 1))]); + let p2 = page(10, 99, vec![(n.clone(), note_v1(n.clone(), 10, 6, "k", 1))]); + + let error = normalize_balance_pages(&[p1, p2]).expect_err("expected mismatch"); + assert!(matches!( + error, + NormalizeSnapshotError::Candidate( + CandidateNormalizationError::DuplicateNameMismatch { .. } + ) + )); + } + + struct SequenceFetcher { + responses: Vec, &'static str>>, + idx: usize, + } + + impl SnapshotPageFetcher for SequenceFetcher { + type Error = &'static str; + + fn fetch_pages(&mut self) -> Result, Self::Error> { + let response = self + .responses + .get(self.idx) + .cloned() + .expect("missing response"); + self.idx += 1; + response + } + } + + #[test] + fn collect_snapshot_with_retry_recovers_from_drift() { + let n = name(1); + let drift = vec![ + page(10, 99, vec![(n.clone(), note_v1(n.clone(), 10, 1, "k", 1))]), + page(10, 100, vec![(name(2), note_v1(name(2), 10, 2, "k", 2))]), + ]; + let stable = vec![page( + 10, + 99, + vec![ + (n.clone(), note_v1(n.clone(), 10, 1, "k", 1)), + (name(2), note_v1(name(2), 10, 2, "k", 2)), + ], + )]; + + let mut fetcher = SequenceFetcher { + responses: vec![Ok(drift), Ok(stable)], + idx: 0, + }; + + let snapshot = collect_snapshot_with_retry(&mut fetcher, 2).expect("retry succeeded"); + assert_eq!(snapshot.candidates.len(), 2); + assert_eq!(fetcher.idx, 2); + } + + #[test] + fn collect_snapshot_with_retry_exhausts_on_persistent_drift() { + let n = name(1); + let drift_a = vec![ + page(10, 99, vec![(n.clone(), note_v1(n.clone(), 10, 1, "k", 1))]), + page(10, 100, vec![(name(2), note_v1(name(2), 10, 2, "k", 2))]), + ]; + let drift_b = vec![ + page( + 10, + 200, + vec![(n.clone(), note_v1(n.clone(), 10, 1, "k", 1))], + ), + page(10, 201, vec![(name(2), note_v1(name(2), 10, 2, "k", 2))]), + ]; + + let mut fetcher = SequenceFetcher { + responses: vec![Ok(drift_a), Ok(drift_b)], + idx: 0, + }; + + let error = collect_snapshot_with_retry(&mut fetcher, 1).expect_err("expected drift"); + assert!(matches!( + error, + CollectSnapshotError::SnapshotDriftRetriesExhausted { .. } + )); + assert_eq!(fetcher.idx, 2); + } +} diff --git a/crates/wallet-tx-builder/src/determinism.rs b/crates/wallet-tx-builder/src/determinism.rs new file mode 100644 index 000000000..1b19d439e --- /dev/null +++ b/crates/wallet-tx-builder/src/determinism.rs @@ -0,0 +1,163 @@ +use std::cmp::Ordering; + +use nockapp::noun::slab::{NockJammer, NounSlab}; +use nockchain_math::zoon::common::{tip, DefaultTipHasher}; +use nockchain_types::tx_engine::common::Name; +use nockchain_types::tx_engine::v1::tx::{LockPrimitive, SpendCondition}; +use noun_serde::NounEncode; + +use crate::types::{CandidateNote, SelectionOrder}; + +/// Converts a note name into a lexicographically comparable base58 tuple key. +pub fn note_name_lex_key(name: &Name) -> (String, String) { + (name.first.to_base58(), name.last.to_base58()) +} + +/// Compares note names using stable lexical ordering over base58 display forms. +pub fn compare_names_lex(a: &Name, b: &Name) -> Ordering { + note_name_lex_key(a).cmp(¬e_name_lex_key(b)) +} + +/// Sorts candidates by assets then name, preserving legacy deterministic behavior. +pub fn sort_candidates(candidates: &mut [CandidateNote], direction: SelectionOrder) { + candidates.sort_by(|a, b| { + let assets_cmp = a.assets().0.cmp(&b.assets().0); + let assets_cmp = match direction { + SelectionOrder::Ascending => assets_cmp, + SelectionOrder::Descending => assets_cmp.reverse(), + }; + assets_cmp.then_with(|| compare_names_lex(&a.identity().name, &b.identity().name)) + }); +} + +/// Canonicalizes spend-condition primitives and hash lists for stable matching. +pub fn canonicalize_spend_condition(spend_condition: &SpendCondition) -> SpendCondition { + let mut primitives = spend_condition + .iter() + .cloned() + .map(canonicalize_lock_primitive) + .collect::>(); + primitives.sort_by_key(lock_primitive_sort_key); + SpendCondition::new(primitives) +} + +fn canonicalize_lock_primitive(primitive: LockPrimitive) -> LockPrimitive { + primitive +} + +fn lock_primitive_sort_key(primitive: &LockPrimitive) -> [u64; 5] { + tip5_sort_key(primitive) +} + +fn tip5_sort_key(value: &T) -> [u64; 5] { + let mut slab: NounSlab = NounSlab::new(); + let noun = value.to_noun(&mut slab); + tip(&mut slab, noun, &DefaultTipHasher) + .expect("tip5 hash should always succeed for noun-encodable primitives") +} + +#[cfg(test)] +mod tests { + use nockchain_math::belt::Belt; + use nockchain_types::tx_engine::common::{BlockHeight, Hash, Nicks}; + use nockchain_types::tx_engine::v1::tx::{Hax, LockPrimitive, Pkh}; + + use super::*; + use crate::note_data::DecodedNoteData; + use crate::types::{CandidateIdentity, CandidateNote, CandidateV1Note, RawNoteDataEntry}; + + fn hash(v: u64) -> Hash { + Hash::from_limbs(&[v, 0, 0, 0, 0]) + } + + fn candidate(assets: usize, first: u64, last: u64) -> CandidateNote { + CandidateNote::V1(CandidateV1Note { + identity: CandidateIdentity { + name: Name::new(hash(first), hash(last)), + origin_page: BlockHeight(Belt(1)), + }, + assets: Nicks(assets), + raw_note_data: Vec::::new(), + decoded_note_data: DecodedNoteData(Vec::new()), + }) + } + + #[test] + fn legacy_sort_orders_by_assets_then_name() { + let mut candidates = vec![candidate(10, 2, 1), candidate(5, 3, 1), candidate(10, 1, 1)]; + + sort_candidates(&mut candidates, SelectionOrder::Ascending); + + assert_eq!(candidates[0].assets().0, 5); + assert!(compare_names_lex( + &candidates[1].identity().name, + &candidates[2].identity().name + ) + .is_le()); + } + + #[test] + fn legacy_sort_descending_reverses_assets_only() { + let mut candidates = vec![candidate(10, 2, 1), candidate(5, 3, 1), candidate(10, 1, 1)]; + + sort_candidates(&mut candidates, SelectionOrder::Descending); + + assert_eq!(candidates[0].assets().0, 10); + assert_eq!(candidates[1].assets().0, 10); + assert!(compare_names_lex( + &candidates[0].identity().name, + &candidates[1].identity().name + ) + .is_le()); + } + + #[test] + fn canonicalize_spend_condition_dedups_and_sorts_hashes() { + let h1 = hash(1); + let h2 = hash(2); + let h3 = hash(3); + let sc = SpendCondition::new(vec![ + LockPrimitive::Hax(Hax::new(vec![h3.clone(), h2.clone(), h3.clone()])), + LockPrimitive::Pkh(Pkh::new(2, vec![h2.clone(), h1.clone(), h2.clone()])), + LockPrimitive::Burn, + ]); + + let canonical = canonicalize_spend_condition(&sc); + let canonical_primitives = canonical.iter().cloned().collect::>(); + assert_eq!(canonical_primitives.len(), 3); + + let pkh = canonical_primitives + .iter() + .find_map(|primitive| match primitive { + LockPrimitive::Pkh(pkh) => Some(pkh), + _ => None, + }) + .expect("pkh primitive should be present"); + assert_eq!(pkh.hashes.len(), 2); + assert!(pkh.hashes.contains(&h1)); + assert!(pkh.hashes.contains(&h2)); + + let hax = canonical_primitives + .iter() + .find_map(|primitive| match primitive { + LockPrimitive::Hax(Hax(hashes)) => Some(hashes), + _ => None, + }) + .expect("hax primitive should be present"); + assert_eq!(hax.len(), 2); + assert!(hax.contains(&hash(2))); + assert!(hax.contains(&h3)); + + assert!(canonical_primitives + .iter() + .any(|primitive| matches!(primitive, LockPrimitive::Burn))); + + let sort_keys = canonical_primitives + .iter() + .map(lock_primitive_sort_key) + .collect::>(); + let mut sorted_keys = sort_keys.clone(); + sorted_keys.sort(); + assert_eq!(sort_keys, sorted_keys); + } +} diff --git a/crates/wallet-tx-builder/src/fee.rs b/crates/wallet-tx-builder/src/fee.rs new file mode 100644 index 000000000..27f9e261f --- /dev/null +++ b/crates/wallet-tx-builder/src/fee.rs @@ -0,0 +1,156 @@ +use nockchain_types::tx_engine::common::BlockHeight; + +const DEFAULT_INPUT_FEE_DIVISOR: u64 = 1; + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct FeeInputs { + pub seed_words: u64, + pub witness_words: u64, + pub base_fee: u64, + pub input_fee_divisor: u64, + pub min_fee: u64, + pub height: BlockHeight, + pub bythos_phase: BlockHeight, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct FeeBreakdown { + pub seed_fee: u64, + pub witness_fee: u64, + pub word_fee: u64, + pub minimum_fee: u64, + pub witness_divisor: u64, +} + +pub fn compute_minimum_fee(inputs: FeeInputs) -> FeeBreakdown { + let witness_divisor = + witness_divisor(inputs.height, inputs.bythos_phase, inputs.input_fee_divisor); + let seed_fee = inputs.seed_words.saturating_mul(inputs.base_fee); + let witness_fee = if witness_divisor == 0 { + 0 + } else { + inputs + .witness_words + .saturating_mul(inputs.base_fee) + .saturating_div(witness_divisor) + }; + let word_fee = seed_fee.saturating_add(witness_fee); + let minimum_fee = word_fee.max(inputs.min_fee); + + FeeBreakdown { + seed_fee, + witness_fee, + word_fee, + minimum_fee, + witness_divisor, + } +} + +pub fn witness_divisor( + height: BlockHeight, + bythos_phase: BlockHeight, + input_fee_divisor: u64, +) -> u64 { + if (height.0).0 >= (bythos_phase.0).0 { + input_fee_divisor.max(DEFAULT_INPUT_FEE_DIVISOR) + } else { + DEFAULT_INPUT_FEE_DIVISOR + } +} + +#[cfg(test)] +mod tests { + use nockchain_math::belt::Belt; + + use super::*; + + fn fee_inputs() -> FeeInputs { + FeeInputs { + seed_words: 0, + witness_words: 0, + base_fee: 0, + input_fee_divisor: DEFAULT_INPUT_FEE_DIVISOR, + min_fee: 0, + height: BlockHeight(Belt(0)), + bythos_phase: BlockHeight(Belt(0)), + } + } + + #[test] + fn pre_bythos_uses_divisor_one() { + let bythos_divisor = 8; + let divisor = witness_divisor(BlockHeight(Belt(10)), BlockHeight(Belt(20)), bythos_divisor); + assert_eq!(divisor, DEFAULT_INPUT_FEE_DIVISOR); + } + + #[test] + fn post_bythos_uses_input_divisor() { + let bythos_divisor = 8; + let divisor = witness_divisor(BlockHeight(Belt(20)), BlockHeight(Belt(20)), bythos_divisor); + assert_eq!(divisor, bythos_divisor); + } + + #[test] + fn compute_minimum_fee_uses_pre_bythos_divisor_one() { + let mut inputs = fee_inputs(); + inputs.seed_words = 3; + inputs.witness_words = 8; + inputs.base_fee = 5; + inputs.input_fee_divisor = 4; + inputs.height = BlockHeight(Belt(9)); + inputs.bythos_phase = BlockHeight(Belt(10)); + + let fee = compute_minimum_fee(inputs); + assert_eq!(fee.seed_fee, 15); + assert_eq!(fee.witness_divisor, 1); + assert_eq!(fee.witness_fee, 40); + assert_eq!(fee.word_fee, 55); + assert_eq!(fee.minimum_fee, 55); + } + + #[test] + fn compute_minimum_fee_uses_post_bythos_divisor() { + let mut inputs = fee_inputs(); + inputs.seed_words = 3; + inputs.witness_words = 8; + inputs.base_fee = 5; + inputs.input_fee_divisor = 4; + inputs.height = BlockHeight(Belt(10)); + inputs.bythos_phase = BlockHeight(Belt(10)); + + let fee = compute_minimum_fee(inputs); + assert_eq!(fee.seed_fee, 15); + assert_eq!(fee.witness_divisor, 4); + assert_eq!(fee.witness_fee, 10); + assert_eq!(fee.word_fee, 25); + assert_eq!(fee.minimum_fee, 25); + } + + #[test] + fn compute_minimum_fee_applies_min_fee_floor() { + let mut inputs = fee_inputs(); + inputs.seed_words = 1; + inputs.witness_words = 1; + inputs.base_fee = 2; + inputs.min_fee = 99; + + let fee = compute_minimum_fee(inputs); + assert_eq!(fee.word_fee, 4); + assert_eq!(fee.minimum_fee, 99); + } + + #[test] + fn compute_minimum_fee_uses_saturating_math() { + let mut inputs = fee_inputs(); + inputs.seed_words = u64::MAX; + inputs.witness_words = u64::MAX; + inputs.base_fee = 2; + inputs.input_fee_divisor = 1; + + let fee = compute_minimum_fee(inputs); + assert_eq!(fee.seed_fee, u64::MAX); + assert_eq!(fee.witness_fee, u64::MAX); + assert_eq!(fee.word_fee, u64::MAX); + assert_eq!(fee.minimum_fee, u64::MAX); + } +} diff --git a/crates/wallet-tx-builder/src/lib.rs b/crates/wallet-tx-builder/src/lib.rs new file mode 100644 index 000000000..8ec7c2abd --- /dev/null +++ b/crates/wallet-tx-builder/src/lib.rs @@ -0,0 +1,8 @@ +pub mod adapter; +pub mod determinism; +pub mod fee; +pub mod lock_resolver; +pub mod note_data; +pub mod planner; +pub mod types; +pub mod word_count; diff --git a/crates/wallet-tx-builder/src/lock_resolver.rs b/crates/wallet-tx-builder/src/lock_resolver.rs new file mode 100644 index 000000000..640017011 --- /dev/null +++ b/crates/wallet-tx-builder/src/lock_resolver.rs @@ -0,0 +1,422 @@ +use nockchain_types::tx_engine::common::Hash; +use nockchain_types::tx_engine::v1::tx::SpendCondition; + +use crate::note_data::DecodedNoteData; + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum LockResolutionSource { + NoteData, + ReconstructedSimplePkh, + ReconstructedCoinbasePkh, + Unknown, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct LockResolution { + pub source: LockResolutionSource, + pub spend_condition: Option, + pub spend_condition_count: Option, +} + +impl LockResolution { + /// Builds an unresolved lock result. + pub fn unknown() -> Self { + Self { + source: LockResolutionSource::Unknown, + spend_condition: None, + spend_condition_count: None, + } + } +} + +/// Input context used by a matcher while resolving the effective spend lock. +pub struct ResolveLockRequest<'a> { + pub note_first_name: &'a Hash, + pub decoded_note_data: &'a DecodedNoteData, + pub signer_pkh: Option<&'a Hash>, + pub coinbase_relative_min: Option, +} + +/// Matcher that owns lock-selection policy for spendability. +/// +/// The matcher carries any local signing/unlock context and is responsible for +/// resolving which lock should be used for a candidate note. Matchers may +/// override `resolve_lock` to compose or layer additional strategies. +pub trait LockMatcher { + /// Returns true when this matcher can satisfy the provided spend condition + /// for the target note first-name. + fn matches(&self, note_first_name: &Hash, spend_condition: &SpendCondition) -> bool; + + /// Resolves the effective lock for a note using matcher-specific policy. + /// + /// The default strategy is: + /// 1. Use a decoded single-leaf `%lock` note-data payload when it is + /// spendable by this matcher for the target first-name. + /// 2. Otherwise, reconstruct a simple PKH lock from the local signer hash and require an + /// exact first-name match via matcher policy. + /// 3. If simple reconstruction does not match, try reconstructed coinbase-style PKH using + /// the provided relative-min constant. + /// 4. The reconstruction uses exactly the provided relative-min constant. + fn resolve_lock(&self, request: ResolveLockRequest<'_>) -> LockResolution { + if let Some(lock_data) = request.decoded_note_data.first_decoded_lock() { + // TODO(wallet-tx-builder): support multi-leaf `%lock` note-data resolution by matching + // against the full lock tree root and selecting a concrete leaf/index pair. + // Today the matcher contract is leaf-oriented (`SpendCondition` only), so a `%lock` + // payload with multiple spend conditions is ambiguous and is intentionally ignored. + if lock_data.spend_conditions.len() == 1 { + let spend_condition = &lock_data.spend_conditions[0]; + if self.matches(request.note_first_name, spend_condition) { + return LockResolution { + source: LockResolutionSource::NoteData, + spend_condition: Some(spend_condition.clone()), + spend_condition_count: None, + }; + } + } + } + + if let Some(pkh) = request.signer_pkh { + let simple = SpendCondition::simple_pkh(pkh.clone()); + if self.matches(request.note_first_name, &simple) { + return LockResolution { + source: LockResolutionSource::ReconstructedSimplePkh, + spend_condition: Some(simple), + spend_condition_count: None, + }; + } + } + + if let (Some(pkh), Some(relative_min)) = (request.signer_pkh, request.coinbase_relative_min) + { + let coinbase = SpendCondition::coinbase_pkh(pkh.clone(), relative_min); + if self.matches(request.note_first_name, &coinbase) { + return LockResolution { + source: LockResolutionSource::ReconstructedCoinbasePkh, + spend_condition: Some(coinbase), + spend_condition_count: None, + }; + } + } + + LockResolution::unknown() + } +} + +#[derive(Debug, Default)] +pub struct NeverMatches; + +impl LockMatcher for NeverMatches { + fn matches(&self, _note_first_name: &Hash, _spend_condition: &SpendCondition) -> bool { + false + } +} + +#[cfg(test)] +mod tests { + use bytes::Bytes; + + use super::*; + use crate::note_data::{ + DecodedNoteDataEntry, DecodedNoteDataPayload, LockDataPayload, NormalizedNoteDataKey, + }; + + fn hash(v: u64) -> Hash { + Hash::from_limbs(&[v, 0, 0, 0, 0]) + } + + struct MatchExpected { + expected: SpendCondition, + } + + impl LockMatcher for MatchExpected { + fn matches(&self, _note_first_name: &Hash, spend_condition: &SpendCondition) -> bool { + spend_condition == &self.expected + } + } + + fn lock_entry(lock: SpendCondition) -> DecodedNoteDataEntry { + DecodedNoteDataEntry { + raw_key: "lock".to_string(), + normalized_key: NormalizedNoteDataKey::Lock, + raw_blob: Bytes::new(), + payload: DecodedNoteDataPayload::Lock(LockDataPayload { + version: 0, + spend_conditions: vec![lock], + }), + decode_error: None, + } + } + + fn decoded_note_data(entries: Vec) -> DecodedNoteData { + DecodedNoteData(entries) + } + + struct AlwaysMatches; + + impl LockMatcher for AlwaysMatches { + fn matches(&self, _note_first_name: &Hash, _spend_condition: &SpendCondition) -> bool { + true + } + } + + fn first_name_for_lock(spend_condition: &SpendCondition) -> Hash { + spend_condition + .first_name() + .expect("first-name should compute") + .into_hash() + } + + #[test] + fn note_data_lock_has_priority_over_reconstruction() { + let note_lock = SpendCondition::simple_pkh(hash(9)); + let matcher = MatchExpected { + expected: note_lock.clone(), + }; + let decoded = decoded_note_data(vec![lock_entry(note_lock.clone())]); + let result = matcher.resolve_lock(ResolveLockRequest { + note_first_name: &first_name_for_lock(¬e_lock), + decoded_note_data: &decoded, + signer_pkh: Some(&hash(7)), + coinbase_relative_min: Some(5), + }); + + assert_eq!(result.source, LockResolutionSource::NoteData); + assert_eq!(result.spend_condition, Some(note_lock)); + assert_eq!(result.spend_condition_count, None); + } + + #[test] + fn note_data_lock_tree_is_not_treated_as_a_single_spend_condition() { + let first = SpendCondition::simple_pkh(hash(9)); + let second = SpendCondition::simple_pkh(hash(10)); + let matcher = MatchExpected { + expected: second.clone(), + }; + let entry = DecodedNoteDataEntry { + raw_key: "lock".to_string(), + normalized_key: NormalizedNoteDataKey::Lock, + raw_blob: Bytes::new(), + payload: DecodedNoteDataPayload::Lock(LockDataPayload { + version: 0, + spend_conditions: vec![first, second.clone()], + }), + decode_error: None, + }; + let decoded = decoded_note_data(vec![entry]); + let second_first_name = first_name_for_lock(&second); + + let result = matcher.resolve_lock(ResolveLockRequest { + note_first_name: &second_first_name, + decoded_note_data: &decoded, + signer_pkh: None, + coinbase_relative_min: None, + }); + + assert_eq!(result.source, LockResolutionSource::Unknown); + assert_eq!(result.spend_condition, None); + assert_eq!(result.spend_condition_count, None); + } + + #[test] + fn note_data_lock_reports_leaf_count_for_larger_lock_trees() { + let first = SpendCondition::simple_pkh(hash(9)); + let second = SpendCondition::simple_pkh(hash(10)); + let third = SpendCondition::simple_pkh(hash(11)); + let fourth = SpendCondition::simple_pkh(hash(12)); + let matcher = MatchExpected { + expected: third.clone(), + }; + let entry = DecodedNoteDataEntry { + raw_key: "lock".to_string(), + normalized_key: NormalizedNoteDataKey::Lock, + raw_blob: Bytes::new(), + payload: DecodedNoteDataPayload::Lock(LockDataPayload { + version: 0, + spend_conditions: vec![first, second, third.clone(), fourth], + }), + decode_error: None, + }; + let decoded = decoded_note_data(vec![entry]); + let third_first_name = first_name_for_lock(&third); + + let result = matcher.resolve_lock(ResolveLockRequest { + note_first_name: &third_first_name, + decoded_note_data: &decoded, + signer_pkh: None, + coinbase_relative_min: None, + }); + + assert_eq!(result.source, LockResolutionSource::Unknown); + assert_eq!(result.spend_condition, None); + assert_eq!(result.spend_condition_count, None); + } + + #[test] + fn note_data_lock_is_ignored_when_matcher_rejects_it() { + let note_lock = SpendCondition::simple_pkh(hash(3)); + let decoded = decoded_note_data(vec![lock_entry(note_lock)]); + let result = NeverMatches.resolve_lock(ResolveLockRequest { + note_first_name: &hash(1), + decoded_note_data: &decoded, + signer_pkh: None, + coinbase_relative_min: None, + }); + + assert_eq!(result.source, LockResolutionSource::Unknown); + assert_eq!(result.spend_condition, None); + assert_eq!(result.spend_condition_count, None); + } + + #[test] + fn simple_reconstruction_is_attempted() { + let pkh = hash(42); + let expected_simple = SpendCondition::simple_pkh(pkh.clone()); + let matcher = MatchExpected { + expected: expected_simple.clone(), + }; + let decoded = decoded_note_data(Vec::new()); + let note_first_name = first_name_for_lock(&expected_simple); + + let result = matcher.resolve_lock(ResolveLockRequest { + note_first_name: ¬e_first_name, + decoded_note_data: &decoded, + signer_pkh: Some(&pkh), + coinbase_relative_min: Some(20), + }); + + assert_eq!(result.source, LockResolutionSource::ReconstructedSimplePkh); + assert_eq!(result.spend_condition, Some(expected_simple)); + assert_eq!(result.spend_condition_count, None); + } + + #[test] + fn coinbase_reconstruction_is_attempted_after_simple() { + let pkh = hash(42); + let expected_coinbase = SpendCondition::coinbase_pkh(pkh.clone(), 20); + let matcher = MatchExpected { + expected: expected_coinbase.clone(), + }; + let decoded = decoded_note_data(Vec::new()); + let note_first_name = first_name_for_lock(&expected_coinbase); + + let result = matcher.resolve_lock(ResolveLockRequest { + note_first_name: ¬e_first_name, + decoded_note_data: &decoded, + signer_pkh: Some(&pkh), + coinbase_relative_min: Some(20), + }); + + assert_eq!( + result.source, + LockResolutionSource::ReconstructedCoinbasePkh + ); + assert_eq!(result.spend_condition, Some(expected_coinbase)); + assert_eq!(result.spend_condition_count, None); + } + + #[test] + fn unresolved_locks_return_unknown() { + let decoded = decoded_note_data(Vec::new()); + let result = NeverMatches.resolve_lock(ResolveLockRequest { + note_first_name: &hash(1), + decoded_note_data: &decoded, + signer_pkh: None, + coinbase_relative_min: None, + }); + + assert_eq!(result.source, LockResolutionSource::Unknown); + assert_eq!(result.spend_condition, None); + assert_eq!(result.spend_condition_count, None); + } + + #[test] + fn coinbase_first_name_does_not_take_simple_reconstruction_path() { + let pkh = hash(42); + let expected_coinbase = SpendCondition::coinbase_pkh(pkh.clone(), 20); + let note_first_name = first_name_for_lock(&expected_coinbase); + let decoded = decoded_note_data(Vec::new()); + let matcher = MatchExpected { + expected: expected_coinbase.clone(), + }; + + let result = matcher.resolve_lock(ResolveLockRequest { + note_first_name: ¬e_first_name, + decoded_note_data: &decoded, + signer_pkh: Some(&pkh), + coinbase_relative_min: Some(20), + }); + + assert_eq!( + result.source, + LockResolutionSource::ReconstructedCoinbasePkh + ); + assert_eq!(result.spend_condition, Some(expected_coinbase)); + } + + #[test] + fn malformed_lock_entry_falls_back_to_reconstruction() { + let pkh = hash(42); + let expected_simple = SpendCondition::simple_pkh(pkh.clone()); + let decoded = decoded_note_data(vec![DecodedNoteDataEntry { + raw_key: "lock".to_string(), + normalized_key: NormalizedNoteDataKey::Lock, + raw_blob: Bytes::new(), + payload: DecodedNoteDataPayload::Raw, + decode_error: Some("bad lock".to_string()), + }]); + + let result = AlwaysMatches.resolve_lock(ResolveLockRequest { + note_first_name: &first_name_for_lock(&expected_simple), + decoded_note_data: &decoded, + signer_pkh: Some(&pkh), + coinbase_relative_min: Some(20), + }); + + assert_eq!(result.source, LockResolutionSource::ReconstructedSimplePkh); + assert_eq!(result.spend_condition, Some(expected_simple)); + } + + #[test] + fn decoded_lock_mismatch_falls_back_to_coinbase_reconstruction() { + let pkh = hash(42); + let simple = SpendCondition::simple_pkh(pkh.clone()); + let expected_coinbase = SpendCondition::coinbase_pkh(pkh.clone(), 20); + let decoded = decoded_note_data(vec![lock_entry(simple)]); + let matcher = MatchExpected { + expected: expected_coinbase.clone(), + }; + + let result = matcher.resolve_lock(ResolveLockRequest { + note_first_name: &first_name_for_lock(&expected_coinbase), + decoded_note_data: &decoded, + signer_pkh: Some(&pkh), + coinbase_relative_min: Some(20), + }); + + assert_eq!( + result.source, + LockResolutionSource::ReconstructedCoinbasePkh + ); + assert_eq!(result.spend_condition, Some(expected_coinbase)); + } + + #[test] + fn coinbase_reconstruction_uses_configured_relative_min_only() { + let pkh = hash(42); + let expected_legacy_coinbase = SpendCondition::coinbase_pkh(pkh.clone(), 100); + let decoded = decoded_note_data(Vec::new()); + let matcher = MatchExpected { + expected: expected_legacy_coinbase.clone(), + }; + + let result = matcher.resolve_lock(ResolveLockRequest { + note_first_name: &first_name_for_lock(&expected_legacy_coinbase), + decoded_note_data: &decoded, + signer_pkh: Some(&pkh), + coinbase_relative_min: Some(1), + }); + + assert_eq!(result.source, LockResolutionSource::Unknown); + assert_eq!(result.spend_condition, None); + } +} diff --git a/crates/wallet-tx-builder/src/note_data.rs b/crates/wallet-tx-builder/src/note_data.rs new file mode 100644 index 000000000..471bf1f85 --- /dev/null +++ b/crates/wallet-tx-builder/src/note_data.rs @@ -0,0 +1,908 @@ +use bytes::Bytes; +use nockapp::noun::slab::{NockJammer, NounSlab}; +use nockapp::Noun; +use nockchain_types::tx_engine::common::Hash; +use nockchain_types::tx_engine::v1::note::{NoteData, NoteDataEntry}; +use nockchain_types::tx_engine::v1::tx::{Lock, SpendCondition}; +use nockvm::noun::{NounAllocator, NounSpace}; +use noun_serde::{NounDecode, NounDecodeError, NounEncode}; +use thiserror::Error; + +use crate::types::RawNoteDataEntry; + +/// Canonical note-data key for lock payloads. +pub const NOTE_DATA_KEY_LOCK: &str = "lock"; +/// Canonical note-data key for bridge deposit payloads. +pub const NOTE_DATA_KEY_BRIDGE_DEPOSIT: &str = "bridge"; +/// Canonical note-data key for bridge withdrawal payloads. +pub const NOTE_DATA_KEY_BRIDGE_WITHDRAWAL: &str = "bridge-w"; + +#[derive(Debug, Clone, PartialEq, Eq, NounEncode, NounDecode)] +/// Internal noun parser for `%lock` payload shape `[%0 lock]`. +enum LockPayloadNoun { + #[noun(tag = 0)] + V0(Lock), +} + +#[derive(Debug, Clone, PartialEq, Eq, NounEncode, NounDecode)] +/// Internal noun parser for `%bridge` payload shape `[%0 %base [a b c]]`. +enum BridgeDepositPayloadNoun { + #[noun(tag = 0)] + V0(String, [u64; 3]), +} + +#[derive(Debug, Clone, PartialEq, Eq, NounEncode, NounDecode)] +/// Internal noun parser for `%bridge-w` payload shape +/// `[%0 base-event-id base-hash lock-root base-batch-end]`. +enum BridgeWithdrawalPayloadNoun { + #[noun(tag = 0)] + V0(Vec, Hash, Hash, u64), +} + +/// Typed note-data entry constructors for canonical wallet/planner payloads. +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum TypedNoteDataEntry { + /// `%lock` => jam([%0 lock]) + Lock { lock: Box }, + /// `%bridge` => jam([%0 %base [a b c]]) + BridgeDeposit { evm_address_based: [u64; 3] }, + /// `%bridge-w` => jam([%0 base-event-id base-hash lock-root base-batch-end]) + BridgeWithdrawal { + base_event_id: Vec, + base_hash: Hash, + lock_root: Hash, + base_batch_end: u64, + }, +} + +impl TypedNoteDataEntry { + /// Constructs a typed lock note-data entry. + pub fn lock(lock: Lock) -> Self { + Self::Lock { + lock: Box::new(lock), + } + } + + /// Constructs a typed bridge deposit note-data entry. + pub fn bridge_deposit(evm_address_based: [u64; 3]) -> Self { + Self::BridgeDeposit { evm_address_based } + } + + /// Constructs a typed bridge withdrawal note-data entry. + pub fn bridge_withdrawal( + base_event_id: Vec, + base_hash: Hash, + lock_root: Hash, + base_batch_end: u64, + ) -> Self { + Self::BridgeWithdrawal { + base_event_id, + base_hash, + lock_root, + base_batch_end, + } + } + + /// Returns the canonical note-data key for this typed entry. + pub fn key(&self) -> &'static str { + match self { + Self::Lock { .. } => NOTE_DATA_KEY_LOCK, + Self::BridgeDeposit { .. } => NOTE_DATA_KEY_BRIDGE_DEPOSIT, + Self::BridgeWithdrawal { .. } => NOTE_DATA_KEY_BRIDGE_WITHDRAWAL, + } + } + + /// Encodes this typed entry into raw key/blob form for tx-engine note-data. + pub fn to_raw_entry(&self) -> RawNoteDataEntry { + let blob = match self { + Self::Lock { lock } => jam_payload(&LockPayloadNoun::V0(lock.as_ref().clone())), + Self::BridgeDeposit { evm_address_based } => jam_payload( + &BridgeDepositPayloadNoun::V0("base".to_string(), *evm_address_based), + ), + Self::BridgeWithdrawal { + base_event_id, + base_hash, + lock_root, + base_batch_end, + } => jam_payload(&BridgeWithdrawalPayloadNoun::V0( + base_event_id.clone(), + base_hash.clone(), + lock_root.clone(), + *base_batch_end, + )), + }; + RawNoteDataEntry { + key: self.key().to_string(), + blob, + } + } +} + +impl RawNoteDataEntry { + /// Encodes a typed `%lock` note-data entry. + pub fn from_lock(lock: Lock) -> Self { + TypedNoteDataEntry::lock(lock).to_raw_entry() + } + + /// Encodes a typed `%bridge` note-data entry for Base deposits. + pub fn from_bridge_deposit(evm_address_based: [u64; 3]) -> Self { + TypedNoteDataEntry::bridge_deposit(evm_address_based).to_raw_entry() + } + + /// Encodes a typed `%bridge-w` note-data entry for bridge withdrawals. + pub fn from_bridge_withdrawal( + base_event_id: Vec, + base_hash: Hash, + lock_root: Hash, + base_batch_end: u64, + ) -> Self { + TypedNoteDataEntry::bridge_withdrawal(base_event_id, base_hash, lock_root, base_batch_end) + .to_raw_entry() + } +} + +#[derive(Debug, Clone, PartialEq, Eq)] +/// Canonicalized view of known note-data keys. +pub enum NormalizedNoteDataKey { + /// `%lock` payload. + Lock, + /// `%bridge` payload. + Bridge, + /// `%bridge-w` payload. + BridgeWithdrawal, + /// Any unrecognized key preserved verbatim. + Other(String), +} + +impl NormalizedNoteDataKey { + /// Normalizes a raw key into known variants while preserving unknown keys. + pub fn from_raw(raw: &str) -> Self { + let normalized = raw.trim(); + match normalized { + NOTE_DATA_KEY_LOCK => Self::Lock, + NOTE_DATA_KEY_BRIDGE_DEPOSIT => Self::Bridge, + NOTE_DATA_KEY_BRIDGE_WITHDRAWAL => Self::BridgeWithdrawal, + other => Self::Other(other.to_string()), + } + } +} + +#[derive(Debug, Clone, PartialEq, Eq)] +/// Decoded `%lock` note-data payload. +pub struct LockDataPayload { + /// Payload schema version. + pub version: u64, + /// Flattened lock spend-condition leaves in deterministic left-to-right order. + pub spend_conditions: Vec, +} + +impl LockDataPayload { + /// Parses a `%lock` payload noun with shape `[version lock]`. + pub fn from_noun(noun: &Noun, space: &NounSpace) -> Result { + let spend_conditions = + match LockPayloadNoun::from_noun(noun, space).map_err(NoteDataDecodeError::from)? { + LockPayloadNoun::V0(lock) => lock.flatten_spend_conditions(), + }; + Ok(Self { + version: 0, + spend_conditions, + }) + } + + /// Cues and parses a `%lock` payload blob. + pub fn from_blob(blob: &Bytes) -> Result { + let mut slab: NounSlab = NounSlab::new(); + let noun = slab + .cue_into(blob.clone()) + .map_err(|error| NoteDataDecodeError::InvalidJam(error.to_string()))?; + let space = slab.noun_space(); + Self::from_noun(&noun, &space) + } +} + +#[derive(Debug, Clone, PartialEq, Eq)] +/// Parsed lock form extracted from tracked lock nouns and note-data payloads. +pub struct ParsedLockForm { + /// Flattened lock spend-condition leaves in deterministic left-to-right order. + pub spend_conditions: Vec, + /// Number of spend-condition leaves represented by this lock tree. + pub spend_condition_count: u64, +} + +impl ParsedLockForm { + /// Parses a lock noun with tx-engine's canonical decoder and exposes flattened leaves. + pub fn from_noun(noun: &Noun, space: &NounSpace) -> Result { + let lock = Lock::from_noun(noun, space).map_err(NoteDataDecodeError::from)?; + Ok(Self { + spend_conditions: lock.flatten_spend_conditions(), + spend_condition_count: lock.spend_condition_count(), + }) + } +} + +#[derive(Debug, Clone, PartialEq, Eq)] +/// Supported bridge network tags for typed bridge note-data payloads. +pub enum BridgeNetwork { + /// Ethereum Base network. + Base, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +/// Decoded `%bridge` note-data payload. +pub struct BridgeDepositDataPayload { + /// Payload schema version. + pub version: u64, + /// Bridge network identifier. + pub network: BridgeNetwork, + /// Encoded EVM address split into three words. + pub evm_address_based: [u64; 3], +} + +impl BridgeDepositDataPayload { + /// Parses a `%bridge` payload noun with shape `[version network evm-address-based]`. + pub fn from_noun(noun: &Noun, space: &NounSpace) -> Result { + let (network, evm_address_based) = match BridgeDepositPayloadNoun::from_noun(noun, space) + .map_err(NoteDataDecodeError::from)? + { + BridgeDepositPayloadNoun::V0(network, evm_address_based) => { + (network, evm_address_based) + } + }; + if network != "base" { + return Err(NoteDataDecodeError::UnsupportedBridgeNetwork(network)); + } + Ok(Self { + version: 0, + network: BridgeNetwork::Base, + evm_address_based, + }) + } + + /// Cues and parses a `%bridge` payload blob. + pub fn from_blob(blob: &Bytes) -> Result { + let mut slab: NounSlab = NounSlab::new(); + let noun = slab + .cue_into(blob.clone()) + .map_err(|error| NoteDataDecodeError::InvalidJam(error.to_string()))?; + let space = slab.noun_space(); + Self::from_noun(&noun, &space) + } +} + +#[derive(Debug, Clone, PartialEq, Eq)] +/// Decoded `%bridge-w` note-data payload. +pub struct BridgeWithdrawalDataPayload { + /// Payload schema version. + pub version: u64, + /// Source bridge event identifier segments. + pub base_event_id: Vec, + /// Source bridge batch hash. + pub base_hash: Hash, + /// Destination lock root used for withdrawal output. + pub lock_root: Hash, + /// Source bridge batch end height. + pub base_batch_end: u64, +} + +impl BridgeWithdrawalDataPayload { + /// Parses a `%bridge-w` payload noun with shape + /// `[version base-event-id base-hash lock-root base-batch-end]`. + pub fn from_noun(noun: &Noun, space: &NounSpace) -> Result { + let (base_event_id, base_hash, lock_root, base_batch_end) = + match BridgeWithdrawalPayloadNoun::from_noun(noun, space) + .map_err(NoteDataDecodeError::from)? + { + BridgeWithdrawalPayloadNoun::V0( + base_event_id, + base_hash, + lock_root, + base_batch_end, + ) => (base_event_id, base_hash, lock_root, base_batch_end), + }; + Ok(Self { + version: 0, + base_event_id, + base_hash, + lock_root, + base_batch_end, + }) + } + + /// Cues and parses a `%bridge-w` payload blob. + pub fn from_blob(blob: &Bytes) -> Result { + let mut slab: NounSlab = NounSlab::new(); + let noun = slab + .cue_into(blob.clone()) + .map_err(|error| NoteDataDecodeError::InvalidJam(error.to_string()))?; + let space = slab.noun_space(); + Self::from_noun(&noun, &space) + } +} + +#[derive(Debug, Clone, PartialEq, Eq)] +/// Best-effort typed decode of a note-data blob. +pub enum DecodedNoteDataPayload { + /// Successfully decoded `%lock`. + Lock(LockDataPayload), + /// Successfully decoded `%bridge`. + BridgeDeposit(BridgeDepositDataPayload), + /// Successfully decoded `%bridge-w`. + BridgeWithdrawal(BridgeWithdrawalDataPayload), + /// Raw or failed-to-decode payload. + Raw, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +/// Fully decoded note-data list with raw and typed per-entry views. +pub struct DecodedNoteData(pub Vec); + +impl From<&NoteData> for DecodedNoteData { + /// Decodes an entire note-data list into typed entries with error retention. + fn from(note_data: &NoteData) -> Self { + Self( + note_data + .iter() + .map(DecodedNoteDataEntry::from_entry) + .collect(), + ) + } +} + +impl From for DecodedNoteData { + /// Decodes an owned note-data list into typed entries with error retention. + fn from(note_data: NoteData) -> Self { + Self( + note_data + .0 + .into_iter() + .map(|entry| DecodedNoteDataEntry::from_entry(&entry)) + .collect(), + ) + } +} + +impl DecodedNoteData { + /// Returns the first decoded lock payload entry in this decoded note-data set. + pub fn first_decoded_lock(&self) -> Option<&LockDataPayload> { + self.0.iter().find_map(|entry| match &entry.payload { + DecodedNoteDataPayload::Lock(lock) => Some(lock), + _ => None, + }) + } +} + +#[derive(Debug, Clone, PartialEq, Eq)] +/// One decoded note-data entry with both raw and typed views. +pub struct DecodedNoteDataEntry { + /// Original key as stored on chain. + pub raw_key: String, + /// Canonicalized key used for dispatch. + pub normalized_key: NormalizedNoteDataKey, + /// Original jammed payload blob. + pub raw_blob: Bytes, + /// Typed payload view when decode succeeds. + pub payload: DecodedNoteDataPayload, + /// Decode error message when known-key decode fails. + pub decode_error: Option, +} + +impl DecodedNoteDataEntry { + /// Decodes one raw note-data entry and preserves payload on failures. + pub fn from_raw_entry(entry: &RawNoteDataEntry) -> Self { + let normalized_key = NormalizedNoteDataKey::from_raw(&entry.key); + let decode_result = match normalized_key { + NormalizedNoteDataKey::Lock => { + LockDataPayload::from_blob(&entry.blob).map(DecodedNoteDataPayload::Lock) + } + NormalizedNoteDataKey::Bridge => BridgeDepositDataPayload::from_blob(&entry.blob) + .map(DecodedNoteDataPayload::BridgeDeposit), + NormalizedNoteDataKey::BridgeWithdrawal => { + BridgeWithdrawalDataPayload::from_blob(&entry.blob) + .map(DecodedNoteDataPayload::BridgeWithdrawal) + } + NormalizedNoteDataKey::Other(_) => Ok(DecodedNoteDataPayload::Raw), + }; + + match decode_result { + Ok(payload) => Self { + raw_key: entry.key.clone(), + normalized_key, + raw_blob: entry.blob.clone(), + payload, + decode_error: None, + }, + Err(error) => Self { + raw_key: entry.key.clone(), + normalized_key, + raw_blob: entry.blob.clone(), + payload: DecodedNoteDataPayload::Raw, + decode_error: Some(error.to_string()), + }, + } + } + + /// Decodes one tx-engine note-data entry by key and preserves payload on failures. + pub fn from_entry(entry: &NoteDataEntry) -> Self { + Self::from_raw_entry(&RawNoteDataEntry { + key: entry.key.clone(), + blob: entry.blob.clone(), + }) + } +} + +#[derive(Debug, Error)] +/// Errors surfaced by note-data payload decoding. +pub enum NoteDataDecodeError { + #[error("invalid jam payload: {0}")] + InvalidJam(String), + #[error("noun decode failed: {0}")] + NounDecode(String), + #[error("unsupported bridge network tag: {0}")] + UnsupportedBridgeNetwork(String), +} + +impl From for NoteDataDecodeError { + /// Maps noun-serde failures into note-data decode errors. + fn from(value: NounDecodeError) -> Self { + Self::NounDecode(value.to_string()) + } +} + +/// TODO(grpc): Extend balance note-data messages to include decoded/typed +/// `DecodedNoteData` alongside raw blobs so wallet/planner paths can consume +/// typed note-data without re-decoding jam payloads per consumer. +/// Jams a noun-encodable value into a byte payload suitable for note-data blobs. +fn jam_payload(value: &T) -> Bytes { + let mut slab: NounSlab = NounSlab::new(); + let noun = value.to_noun(&mut slab); + slab.set_root(noun); + slab.jam() +} + +#[cfg(test)] +mod tests { + use std::collections::BTreeSet; + + use nockchain_types::tx_engine::v1::tx::{Lock, LockPrimitive, Pkh}; + use nockvm::ext::NounExt; + use nockvm::mem::NockStack; + use nockvm::noun::{Noun, NounAllocator, D, T}; + use noun_serde::{NounDecode, NounEncode}; + + use super::*; + + #[derive(Debug, Clone, PartialEq, Eq, NounDecode)] + struct FixtureEntry { + case: String, + note_data: NoteData, + } + + fn hash(v: u64) -> Hash { + Hash::from_limbs(&[v, 0, 0, 0, 0]) + } + + fn hash_from_limbs(a: u64, b: u64, c: u64, d: u64, e: u64) -> Hash { + Hash::from_limbs(&[a, b, c, d, e]) + } + + fn jam(value: &T) -> Bytes { + let mut slab: NounSlab = NounSlab::new(); + let noun = value.to_noun(&mut slab); + slab.set_root(noun); + slab.jam() + } + + fn decode_fixtures() -> Vec { + let fixture_bytes = include_bytes!("../tests/fixtures/note_data_fixtures.jam"); + let mut stack = NockStack::new(nockapp::utils::NOCK_STACK_SIZE, 0); + let noun = Noun::cue_bytes_slice(&mut stack, fixture_bytes).expect("fixture jam must cue"); + let space = stack.noun_space(); + Vec::::from_noun(&noun, &space).expect("fixture noun must decode") + } + + fn normalize_case_tag(tag: &str) -> &str { + tag.strip_prefix('%').unwrap_or(tag) + } + + fn fixture_note_data(case: &str) -> NoteData { + decode_fixtures() + .into_iter() + .find(|fixture| normalize_case_tag(&fixture.case) == case) + .unwrap_or_else(|| panic!("missing fixture case: {case}")) + .note_data + } + + #[test] + fn normalize_note_data_keys_handles_known_bridge_keys() { + assert_eq!( + NormalizedNoteDataKey::from_raw(NOTE_DATA_KEY_LOCK), + NormalizedNoteDataKey::Lock + ); + assert_eq!( + NormalizedNoteDataKey::from_raw(NOTE_DATA_KEY_BRIDGE_DEPOSIT), + NormalizedNoteDataKey::Bridge + ); + assert_eq!( + NormalizedNoteDataKey::from_raw(NOTE_DATA_KEY_BRIDGE_WITHDRAWAL), + NormalizedNoteDataKey::BridgeWithdrawal + ); + } + + #[test] + fn lock_payload_noun_decodes_tagged_zero_atom() { + let spend = SpendCondition::new(vec![LockPrimitive::Burn]); + let mut slab: NounSlab = NounSlab::new(); + let noun = LockPayloadNoun::V0(Lock::SpendCondition(spend.clone())).to_noun(&mut slab); + let space = slab.noun_space(); + + let version = noun + .in_space(&space) + .as_cell() + .expect("tagged payload should be a cell") + .head() + .noun(); + + assert_eq!( + u64::from_noun(&version, &space).expect("tag atom should be numeric"), + 0 + ); + + assert_eq!( + LockPayloadNoun::from_noun(&noun, &space).expect("tagged zero atom should decode"), + LockPayloadNoun::V0(Lock::SpendCondition(spend)) + ); + } + + #[test] + fn typed_lock_note_data_entry_encodes_and_decodes() { + let spend = SpendCondition::new(vec![LockPrimitive::Burn]); + let entry = RawNoteDataEntry::from_lock(Lock::SpendCondition(spend.clone())); + assert_eq!(entry.key, NOTE_DATA_KEY_LOCK); + + let decoded = DecodedNoteDataEntry::from_raw_entry(&entry); + match decoded.payload { + DecodedNoteDataPayload::Lock(lock_payload) => { + assert_eq!(lock_payload.version, 0); + assert_eq!(lock_payload.spend_conditions, vec![spend]); + } + other => panic!("expected typed lock payload, got {other:?}"), + } + assert!(decoded.decode_error.is_none()); + } + + #[test] + fn typed_bridge_deposit_note_data_entry_encodes_and_decodes() { + let entry = RawNoteDataEntry::from_bridge_deposit([11, 22, 33]); + assert_eq!(entry.key, NOTE_DATA_KEY_BRIDGE_DEPOSIT); + + let decoded = DecodedNoteDataEntry::from_raw_entry(&entry); + match decoded.payload { + DecodedNoteDataPayload::BridgeDeposit(bridge_payload) => { + assert_eq!(bridge_payload.version, 0); + assert_eq!(bridge_payload.network, BridgeNetwork::Base); + assert_eq!(bridge_payload.evm_address_based, [11, 22, 33]); + } + other => panic!("expected typed bridge deposit payload, got {other:?}"), + } + assert!(decoded.decode_error.is_none()); + } + + #[test] + fn typed_bridge_withdrawal_note_data_entry_encodes_and_decodes() { + let entry = RawNoteDataEntry::from_bridge_withdrawal( + vec![1, 2, 3, 4], + hash_from_limbs(1, 2, 3, 4, 5), + hash_from_limbs(6, 7, 8, 9, 10), + 57_600, + ); + assert_eq!(entry.key, NOTE_DATA_KEY_BRIDGE_WITHDRAWAL); + + let decoded = DecodedNoteDataEntry::from_raw_entry(&entry); + match decoded.payload { + DecodedNoteDataPayload::BridgeWithdrawal(bridge_payload) => { + assert_eq!(bridge_payload.version, 0); + assert_eq!(bridge_payload.base_event_id, vec![1, 2, 3, 4]); + assert_eq!(bridge_payload.base_hash, hash_from_limbs(1, 2, 3, 4, 5)); + assert_eq!(bridge_payload.lock_root, hash_from_limbs(6, 7, 8, 9, 10)); + assert_eq!(bridge_payload.base_batch_end, 57_600); + } + other => panic!("expected typed bridge withdrawal payload, got {other:?}"), + } + assert!(decoded.decode_error.is_none()); + } + + #[test] + fn fixture_jam_contains_expected_cases() { + let cases = decode_fixtures() + .into_iter() + .map(|fixture| normalize_case_tag(&fixture.case).to_string()) + .collect::>(); + let expected = [ + "all-keys", "lock-single", "lock-v2", "lock-v4", "bridge-deposit", "lock-v8", + "lock-v16", "lock-unsupported-version", "bridge-deposit-large", + "bridge-unsupported-network", "bridge-unsupported-version", "bridge-withdrawal", + "bridge-withdrawal-unsupported-version", "bridge-withdrawal-long-event", "wildcard", + ] + .into_iter() + .map(str::to_string) + .collect::>(); + assert_eq!(cases, expected); + } + + #[test] + fn decode_lock_single_note_data_payload() { + let note_data = fixture_note_data("lock-single"); + let decoded = DecodedNoteData::from(¬e_data).0; + let entry = &decoded[0]; + match &entry.payload { + DecodedNoteDataPayload::Lock(lock) => { + let burn = SpendCondition::new(vec![LockPrimitive::Burn]); + assert_eq!(lock.version, 0); + assert_eq!(lock.spend_conditions, vec![burn]); + } + other => panic!( + "expected lock payload, got {:?}, error={:?}", + other, entry.decode_error + ), + } + } + + #[test] + fn decode_lock_v2_note_data_payload() { + let note_data = fixture_note_data("lock-v2"); + let decoded = DecodedNoteData::from(¬e_data).0; + let entry = &decoded[0]; + match &entry.payload { + DecodedNoteDataPayload::Lock(lock) => { + let burn = SpendCondition::new(vec![LockPrimitive::Burn]); + assert_eq!(lock.version, 0); + assert_eq!(lock.spend_conditions, vec![burn.clone(), burn]); + } + other => panic!( + "expected lock payload, got {:?}, error={:?}", + other, entry.decode_error + ), + } + } + + #[test] + fn decode_lock_note_data_payload() { + let note_data = fixture_note_data("lock-v4"); + let decoded = DecodedNoteData::from(¬e_data).0; + let entry = &decoded[0]; + match &entry.payload { + DecodedNoteDataPayload::Lock(lock) => { + let burn = SpendCondition::new(vec![LockPrimitive::Burn]); + assert_eq!(lock.version, 0); + assert_eq!( + lock.spend_conditions, + vec![burn.clone(), burn.clone(), burn.clone(), burn] + ); + } + other => panic!( + "expected lock payload, got {:?}, error={:?}", + other, entry.decode_error + ), + } + } + + #[test] + fn decode_lock_v8_note_data_payload() { + let note_data = fixture_note_data("lock-v8"); + let decoded = DecodedNoteData::from(¬e_data).0; + let entry = &decoded[0]; + match &entry.payload { + DecodedNoteDataPayload::Lock(lock) => { + let burn = SpendCondition::new(vec![LockPrimitive::Burn]); + assert_eq!(lock.version, 0); + assert_eq!(lock.spend_conditions.len(), 8); + assert!(lock.spend_conditions.iter().all(|sc| sc == &burn)); + } + other => panic!( + "expected lock payload, got {:?}, error={:?}", + other, entry.decode_error + ), + } + } + + #[test] + fn decode_lock_v16_note_data_payload() { + let note_data = fixture_note_data("lock-v16"); + let decoded = DecodedNoteData::from(¬e_data).0; + let entry = &decoded[0]; + match &entry.payload { + DecodedNoteDataPayload::Lock(lock) => { + let burn = SpendCondition::new(vec![LockPrimitive::Burn]); + assert_eq!(lock.version, 0); + assert_eq!(lock.spend_conditions.len(), 16); + assert!(lock.spend_conditions.iter().all(|sc| sc == &burn)); + } + other => panic!( + "expected lock payload, got {:?}, error={:?}", + other, entry.decode_error + ), + } + } + + #[test] + fn decode_lock_form_noun_reports_tree_leaf_count() { + fn simple_sc(v: u64) -> SpendCondition { + SpendCondition::new(vec![LockPrimitive::Pkh(Pkh::new(1, vec![hash(v)]))]) + } + + let sc1 = simple_sc(11); + let sc2 = simple_sc(12); + let sc3 = simple_sc(13); + let sc4 = simple_sc(14); + + let mut slab: NounSlab = NounSlab::new(); + let sc1_n = sc1.to_noun(&mut slab); + let sc2_n = sc2.to_noun(&mut slab); + let sc3_n = sc3.to_noun(&mut slab); + let sc4_n = sc4.to_noun(&mut slab); + + let v2_left_pair = T(&mut slab, &[sc1_n, sc2_n]); + let v2_right_pair = T(&mut slab, &[sc3_n, sc4_n]); + let v4_pair = T(&mut slab, &[v2_left_pair, v2_right_pair]); + let v4_lock = T(&mut slab, &[D(4), v4_pair]); + let space = slab.noun_space(); + + let parsed = ParsedLockForm::from_noun(&v4_lock, &space).expect("decode lock form"); + assert_eq!(parsed.spend_condition_count, 4); + assert_eq!(parsed.spend_conditions, vec![sc1, sc2, sc3, sc4]); + } + + #[test] + fn recognized_keys_fallback_to_raw_on_decode_errors() { + let malformed = jam(&42_u64); + let note_data = NoteData::new(vec![NoteDataEntry::new("bridge".to_string(), malformed)]); + + let decoded = DecodedNoteData::from(¬e_data).0; + let entry = &decoded[0]; + assert!(matches!(entry.payload, DecodedNoteDataPayload::Raw)); + assert!(entry.decode_error.is_some()); + } + + #[test] + fn decode_bridge_deposit_payload() { + let note_data = fixture_note_data("bridge-deposit"); + let decoded = DecodedNoteData::from(¬e_data).0; + let entry = &decoded[0]; + match &entry.payload { + DecodedNoteDataPayload::BridgeDeposit(bridge) => { + assert_eq!(bridge.version, 0); + assert_eq!(bridge.network, BridgeNetwork::Base); + assert_eq!(bridge.evm_address_based, [11, 22, 33]); + } + other => panic!( + "expected bridge payload, got {:?}, error={:?}", + other, entry.decode_error + ), + } + } + + #[test] + fn decode_bridge_deposit_large_payload() { + let note_data = fixture_note_data("bridge-deposit-large"); + let decoded = DecodedNoteData::from(¬e_data).0; + let entry = &decoded[0]; + match &entry.payload { + DecodedNoteDataPayload::BridgeDeposit(bridge) => { + assert_eq!(bridge.version, 0); + assert_eq!(bridge.network, BridgeNetwork::Base); + assert_eq!( + bridge.evm_address_based, + [4_200_001, 98_765_432, 1_234_567_890] + ); + } + other => panic!( + "expected bridge payload, got {:?}, error={:?}", + other, entry.decode_error + ), + } + } + + #[test] + fn decode_bridge_withdrawal_payload() { + let note_data = fixture_note_data("bridge-withdrawal"); + let decoded = DecodedNoteData::from(¬e_data).0; + let entry = &decoded[0]; + match &entry.payload { + DecodedNoteDataPayload::BridgeWithdrawal(bridge_w) => { + assert_eq!(bridge_w.version, 0); + assert_eq!(bridge_w.base_event_id, vec![1, 2, 3, 4]); + assert_eq!(bridge_w.base_hash, hash_from_limbs(1, 2, 3, 4, 5)); + assert_eq!(bridge_w.lock_root, hash_from_limbs(6, 7, 8, 9, 10)); + assert_eq!(bridge_w.base_batch_end, 57_600); + } + other => panic!( + "expected bridge withdrawal payload, got {:?}, error={:?}", + other, entry.decode_error + ), + } + } + + #[test] + fn decode_bridge_withdrawal_long_event_payload() { + let note_data = fixture_note_data("bridge-withdrawal-long-event"); + let decoded = DecodedNoteData::from(¬e_data).0; + let entry = &decoded[0]; + match &entry.payload { + DecodedNoteDataPayload::BridgeWithdrawal(bridge_w) => { + assert_eq!(bridge_w.version, 0); + assert_eq!(bridge_w.base_event_id, vec![10, 20, 30, 40, 50, 60, 70]); + assert_eq!(bridge_w.base_hash, hash_from_limbs(90, 80, 70, 60, 50)); + assert_eq!(bridge_w.lock_root, hash_from_limbs(15, 25, 35, 45, 55)); + assert_eq!(bridge_w.base_batch_end, 88_001); + } + other => panic!( + "expected bridge withdrawal payload, got {:?}, error={:?}", + other, entry.decode_error + ), + } + } + + #[test] + fn malformed_recognized_keys_fallback_to_raw_with_decode_error() { + for case in [ + "lock-unsupported-version", "bridge-unsupported-network", "bridge-unsupported-version", + "bridge-withdrawal-unsupported-version", + ] { + let note_data = fixture_note_data(case); + let decoded = DecodedNoteData::from(¬e_data).0; + let entry = &decoded[0]; + assert!( + matches!(entry.payload, DecodedNoteDataPayload::Raw), + "case {case} should fallback to raw payload" + ); + assert!( + entry.decode_error.is_some(), + "case {case} should include decode error" + ); + } + } + + #[test] + fn decode_all_keys_fixture_includes_wildcard_as_raw() { + let note_data = fixture_note_data("all-keys"); + let decoded = DecodedNoteData::from(¬e_data).0; + assert_eq!(decoded.len(), 4); + assert!(decoded.iter().all(|entry| entry.decode_error.is_none())); + + let lock_entry = decoded + .iter() + .find(|entry| entry.raw_key == "lock") + .expect("lock entry"); + match &lock_entry.payload { + DecodedNoteDataPayload::Lock(lock) => assert_eq!(lock.spend_conditions.len(), 4), + other => panic!("expected lock payload, got {other:?}"), + } + + let bridge_entry = decoded + .iter() + .find(|entry| entry.raw_key == "bridge") + .expect("bridge entry"); + assert!(matches!( + bridge_entry.payload, + DecodedNoteDataPayload::BridgeDeposit(_) + )); + + let bridge_withdrawal_entry = decoded + .iter() + .find(|entry| entry.raw_key == "bridge-w") + .expect("bridge withdrawal entry"); + assert!(matches!( + bridge_withdrawal_entry.payload, + DecodedNoteDataPayload::BridgeWithdrawal(_) + )); + + let wildcard_entry = decoded + .iter() + .find(|entry| entry.raw_key == "memo") + .expect("wildcard entry"); + assert!(matches!( + wildcard_entry.payload, + DecodedNoteDataPayload::Raw + )); + } +} diff --git a/crates/wallet-tx-builder/src/planner.rs b/crates/wallet-tx-builder/src/planner.rs new file mode 100644 index 000000000..33352d55f --- /dev/null +++ b/crates/wallet-tx-builder/src/planner.rs @@ -0,0 +1,1614 @@ +use std::collections::BTreeSet; + +use nockchain_types::tx_engine::common::{BlockHeight, SchnorrPubkey, Version}; +use nockchain_types::tx_engine::v0::TimelockIntent as V0TimelockIntent; +use nockchain_types::tx_engine::v1::tx::{LockPrimitive, LockTim, SpendCondition}; +use thiserror::Error; + +use crate::determinism::sort_candidates; +use crate::fee::{compute_minimum_fee, FeeInputs}; +use crate::lock_resolver::{LockMatcher, LockResolutionSource, ResolveLockRequest}; +use crate::types::{ + CandidateNote, CandidateV0Note, CandidateVersionPolicy, PlanRequest, PlanResult, PlannedOutput, + PlanningMode, SelectionMode, WordCountBreakdown, +}; +use crate::word_count::{WitnessWordInput, WordCountEstimator}; + +/// Planner failures for candidate admission, lock resolution, and fee conservation. +#[derive(Debug, Error)] +pub enum PlanError { + #[error("plan request must include at least one recipient output")] + NoRecipients, + #[error("manual mode requires at least one note name")] + ManualNamesMissing, + #[error("manual mode references unknown note {first}/{last}")] + ManualNoteMissing { first: String, last: String }, + #[error("manual mode contains duplicate note name {first}/{last}")] + DuplicateManualName { first: String, last: String }, + #[error( + "unable to resolve effective lock for note {first}/{last}; source={resolution_source:?}" + )] + UnknownLock { + first: String, + last: String, + resolution_source: LockResolutionSource, + }, + #[error("insufficient funds: selected_total={selected_total} required={required}")] + InsufficientFunds { selected_total: u64, required: u64 }, + #[error("conservation failed for selected transaction")] + ConservationFailed, + #[error( + "v0 migration sweep leaves no spendable output after fees: selected_total={selected_total} fee={fee}" + )] + V0MigrationProducesZeroValue { selected_total: u64, fee: u64 }, + #[error( + "candidate note {first}/{last} has version {version:?}, but selector policy is {policy:?}" + )] + CandidateVersionDisabled { + version: Version, + policy: CandidateVersionPolicy, + first: String, + last: String, + }, +} + +#[derive(Debug, Clone)] +/// One selected candidate note tracked in planner state. +struct SelectedInput { + /// Candidate note accepted into the current plan. + candidate: CandidateNote, +} + +#[derive(Debug, Clone)] +/// Recomputed fee/output state for standard planner output selection. +struct StandardRecomputeState { + /// Fee chosen for the current selected-input set. + final_fee: u64, + /// Minimum fee implied by current seed/witness words. + minimum_fee: u64, + /// Seed words recomputed from current output set. + seed_words: u64, + /// Witness words recomputed from selected input locks. + witness_words: u64, + /// Output set corresponding to `final_fee` (refund included when present). + outputs: Vec, +} + +#[derive(Debug, Clone)] +/// Recomputed state for a finalizable v0 migration sweep. +struct V0MigrationReadyState { + /// Fee chosen for the current selected-input set. + final_fee: u64, + /// Minimum fee implied by current seed/witness words. + minimum_fee: u64, + /// Seed words recomputed from the destination output. + seed_words: u64, + /// Witness words recomputed from selected input locks. + witness_words: u64, + /// Single migration destination output. + destination_output: PlannedOutput, +} + +#[derive(Debug, Clone)] +/// Explicit migration recompute state used while accumulating legacy inputs. +enum V0MigrationRecomputeState { + /// Current selection still cannot fund a positive migration output after fees. + NeedsMoreInputs { + final_fee: u64, + minimum_fee: u64, + seed_words: u64, + witness_words: u64, + }, + /// Current selection can fund a positive migration destination output. + Ready(V0MigrationReadyState), +} + +impl V0MigrationRecomputeState { + fn final_fee(&self) -> u64 { + match self { + Self::NeedsMoreInputs { final_fee, .. } => *final_fee, + Self::Ready(state) => state.final_fee, + } + } + + fn minimum_fee(&self) -> u64 { + match self { + Self::NeedsMoreInputs { minimum_fee, .. } => *minimum_fee, + Self::Ready(state) => state.minimum_fee, + } + } + + fn seed_words(&self) -> u64 { + match self { + Self::NeedsMoreInputs { seed_words, .. } => *seed_words, + Self::Ready(state) => state.seed_words, + } + } + + fn witness_words(&self) -> u64 { + match self { + Self::NeedsMoreInputs { witness_words, .. } => *witness_words, + Self::Ready(state) => state.witness_words, + } + } +} + +#[derive(Debug)] +/// Mutable planner session carrying running selection and fee state. +struct PlanSession<'a> { + /// Immutable request/configuration for this planning run. + request: &'a PlanRequest, + /// Word-count estimator bound to request chain context. + word_count_estimator: WordCountEstimator<'a>, + /// Sum of recipient gift amounts (target transfer value). + gift_total: u64, + /// Seed-word baseline for recipient outputs only (no refund output). + /// This is used as the first lower-bound fee check before considering refund shape. + seed_words_without_refund: u64, + /// Running total of witness words for all currently selected inputs. + witness_words_total: u64, + /// Selected inputs in deterministic order. + selected: Vec, + /// Running sum of selected input assets. + selected_total: u64, + /// Human-readable decision trace emitted in plan result. + debug_trace: Vec, +} + +impl<'a> PlanSession<'a> { + /// Initializes a planning session with immutable request context and + /// precomputed recipient-side seed word baseline. + fn new(request: &'a PlanRequest) -> Self { + let word_count_estimator = WordCountEstimator::new(&request.chain_context); + let (gift_total, seed_words_without_refund) = match &request.planning_mode { + PlanningMode::Standard => { + let gift_total = request + .recipient_outputs + .iter() + .fold(0u64, |acc, output| acc.saturating_add(output.amount)); + let seed_words_without_refund = + word_count_estimator.estimate_seed_words(&request.recipient_outputs); + (gift_total, seed_words_without_refund) + } + PlanningMode::V0MigrationSweep { destination_output } => ( + 0, + word_count_estimator.estimate_seed_words(std::slice::from_ref(destination_output)), + ), + }; + Self { + request, + word_count_estimator, + gift_total, + seed_words_without_refund, + witness_words_total: 0, + selected: Vec::new(), + selected_total: 0, + debug_trace: Vec::new(), + } + } + + /// Returns the total amount needed to satisfy gifts plus the provided fee. + fn required_total(&self, fee: u64) -> u64 { + self.gift_total.saturating_add(fee) + } + + /// Applies the request's candidate-version policy and emits the usual + /// manual/auto-mode version mismatch diagnostics. + fn candidate_version_allowed(&mut self, candidate: &CandidateNote) -> Result { + let candidate_version = candidate.version(); + let allowed_version = match self.request.candidate_version_policy { + CandidateVersionPolicy::V1Only => Version::V1, + CandidateVersionPolicy::V0Only => Version::V0, + }; + if candidate_version != allowed_version { + let (first, last) = candidate.note_name_display(); + if matches!(&self.request.selection_mode, SelectionMode::Manual { .. }) { + return Err(PlanError::CandidateVersionDisabled { + version: candidate_version, + policy: self.request.candidate_version_policy, + first, + last, + }); + } + self.debug_trace.push(format!( + "skipped note {first}/{last}: version {candidate_version:?} disabled by selector policy {policy:?}", + policy = self.request.candidate_version_policy + )); + return Ok(false); + } + + Ok(true) + } + + /// Attempts to add one standard candidate note, resolving spendability and + /// updating running witness/asset totals when accepted. + fn try_select_standard_candidate( + &mut self, + candidate: CandidateNote, + matcher: &M, + ) -> Result, PlanError> { + if !self.candidate_version_allowed(&candidate)? { + return Ok(None); + } + + if let CandidateNote::V0(note) = candidate { + return self.try_select_standard_v0_candidate(note); + } + + let resolution = matcher.resolve_lock(ResolveLockRequest { + note_first_name: &candidate.identity().name.first, + decoded_note_data: candidate.decoded_note_data(), + signer_pkh: self.request.signer_pkh.as_ref(), + coinbase_relative_min: self.request.coinbase_relative_min, + }); + let Some(spend_condition) = resolution.spend_condition else { + let (first, last) = candidate.note_name_display(); + if matches!(&self.request.selection_mode, SelectionMode::Manual { .. }) { + return Err(PlanError::UnknownLock { + first, + last, + resolution_source: resolution.source, + }); + } + self.debug_trace.push(format!( + "skipped note {first}/{last}: unresolved lock source={:?}", + resolution.source + )); + return Ok(None); + }; + if !timelock_satisfied( + &spend_condition, + &candidate.identity().origin_page, + &self.request.chain_context.height, + ) { + let (first, last) = candidate.note_name_display(); + self.debug_trace.push(format!( + "skipped note {first}/{last}: timelock not satisfied at height={}", + height_value(&self.request.chain_context.height) + )); + return Ok(None); + } + + let candidate_assets = candidate.assets().0 as u64; + let (first, last) = candidate.note_name_display(); + let witness_words_for_input = + self.word_count_estimator + .estimate_witness_words_for_input(&WitnessWordInput { + spend_condition: spend_condition.clone(), + input_origin_page: candidate.identity().origin_page.clone(), + spend_condition_count: resolution.spend_condition_count, + }); + self.selected_total = self.selected_total.saturating_add(candidate_assets); + self.witness_words_total = self + .witness_words_total + .saturating_add(witness_words_for_input); + self.selected.push(SelectedInput { candidate }); + + let recompute = self.recompute_standard_fee(); + let required = self.required_total(recompute.final_fee); + self.debug_trace.push(format!( + "selected note {first}/{last} assets={} selected_total={} seed_words={} witness_words={} min_fee={} final_fee={} required={}", + candidate_assets, + self.selected_total, + recompute.seed_words, + recompute.witness_words, + recompute.minimum_fee, + recompute.final_fee, + required + )); + + Ok(Some(recompute)) + } + + /// Attempts to admit one legacy v0 note under the standard create-tx flow. + fn try_select_standard_v0_candidate( + &mut self, + note: CandidateV0Note, + ) -> Result, PlanError> { + let (first, last) = ( + note.identity.name.first.to_base58(), + note.identity.name.last.to_base58(), + ); + if !v0_note_spendable_by_signers(¬e, &self.request.v0_migration_signer_pubkeys) { + self.debug_trace.push(format!( + "skipped note {first}/{last}: legacy lock is not spendable by planner signer pubkeys", + )); + return Ok(None); + } + if !v0_timelock_satisfied( + note.timelock.as_ref(), + ¬e.identity.origin_page, + &self.request.chain_context.height, + ) { + self.debug_trace.push(format!( + "skipped note {first}/{last}: v0 timelock not satisfied at height={}", + height_value(&self.request.chain_context.height) + )); + return Ok(None); + } + + let candidate_assets = note.assets.0 as u64; + let witness_words_for_input = self + .word_count_estimator + .estimate_v0_witness_words(note.lock.keys_required); + self.selected_total = self.selected_total.saturating_add(candidate_assets); + self.witness_words_total = self + .witness_words_total + .saturating_add(witness_words_for_input); + self.selected.push(SelectedInput { + candidate: CandidateNote::V0(note), + }); + + let recompute = self.recompute_standard_fee(); + let required = self.required_total(recompute.final_fee); + self.debug_trace.push(format!( + "selected note {first}/{last} assets={} selected_total={} seed_words={} witness_words={} min_fee={} final_fee={} required={}", + candidate_assets, + self.selected_total, + recompute.seed_words, + recompute.witness_words, + recompute.minimum_fee, + recompute.final_fee, + required + )); + + Ok(Some(recompute)) + } + + /// Attempts to admit one legacy v0 note under the v0 migration sweep rules. + fn try_select_v0_migration_candidate( + &mut self, + candidate: CandidateNote, + ) -> Result, PlanError> { + if !self.candidate_version_allowed(&candidate)? { + return Ok(None); + } + + let CandidateNote::V0(note) = candidate else { + return Ok(None); + }; + let (first, last) = ( + note.identity.name.first.to_base58(), + note.identity.name.last.to_base58(), + ); + if !v0_note_spendable_by_signers(¬e, &self.request.v0_migration_signer_pubkeys) { + self.debug_trace.push(format!( + "skipped note {first}/{last}: legacy lock is not spendable by migration signer pubkeys", + )); + return Ok(None); + } + if !v0_timelock_satisfied( + note.timelock.as_ref(), + ¬e.identity.origin_page, + &self.request.chain_context.height, + ) { + self.debug_trace.push(format!( + "skipped note {first}/{last}: v0 timelock not satisfied at height={}", + height_value(&self.request.chain_context.height) + )); + return Ok(None); + } + + let candidate_assets = note.assets.0 as u64; + let witness_words_for_input = self + .word_count_estimator + .estimate_v0_witness_words(note.lock.keys_required); + self.selected_total = self.selected_total.saturating_add(candidate_assets); + self.witness_words_total = self + .witness_words_total + .saturating_add(witness_words_for_input); + self.selected.push(SelectedInput { + candidate: CandidateNote::V0(note), + }); + + let recompute = self.recompute_v0_migration(); + self.debug_trace.push(format!( + "selected migration note {first}/{last} assets={} selected_total={} seed_words={} witness_words={} min_fee={} final_fee={}", + candidate_assets, + self.selected_total, + recompute.seed_words(), + recompute.witness_words(), + recompute.minimum_fee(), + recompute.final_fee(), + )); + + Ok(Some(recompute)) + } + + /// Recomputes fee and output shape from the current standard selected-input state. + fn recompute_standard_fee(&self) -> StandardRecomputeState { + let witness_words = self.witness_words_total; + let fee_capacity = self.selected_total.saturating_sub(self.gift_total); + let minimum_without_refund = + self.minimum_fee(self.seed_words_without_refund, witness_words); + let mut final_fee = minimum_without_refund; + if fee_capacity > minimum_without_refund { + let refund_if_min_without = fee_capacity.saturating_sub(minimum_without_refund); + let outputs_with_refund = outputs_with_refund(self.request, refund_if_min_without); + let seed_words_with_refund = self + .word_count_estimator + .estimate_seed_words(&outputs_with_refund); + let minimum_with_refund = self.minimum_fee(seed_words_with_refund, witness_words); + if fee_capacity > minimum_with_refund { + final_fee = minimum_with_refund; + } else { + // If we cannot fit min-fee-with-refund, consume the remainder as fee and + // emit no refund output to preserve conservation without iterative toggling. + // This never increases gifts: recipient outputs are fixed by + // `request.recipient_outputs`; only the leftover split between refund and + // fee changes in this branch. + final_fee = fee_capacity; + } + } + + let refund = self.refund_amount(final_fee); + let outputs = outputs_with_refund(self.request, refund); + let seed_words = self.word_count_estimator.estimate_seed_words(&outputs); + let minimum_fee = self.minimum_fee(seed_words, witness_words); + StandardRecomputeState { + final_fee, + minimum_fee, + seed_words, + witness_words, + outputs, + } + } + + /// Recomputes fee and destination output shape for the current v0 migration selection. + fn recompute_v0_migration(&self) -> V0MigrationRecomputeState { + let PlanningMode::V0MigrationSweep { destination_output } = &self.request.planning_mode + else { + unreachable!("v0 migration recompute should only run in V0MigrationSweep mode"); + }; + let witness_words = self.witness_words_total; + let seed_words = self + .word_count_estimator + .estimate_seed_words(std::slice::from_ref(destination_output)); + let minimum_fee = self.minimum_fee(seed_words, witness_words); + let final_fee = minimum_fee; + let Some(sweep_amount) = self.selected_total.checked_sub(final_fee) else { + return V0MigrationRecomputeState::NeedsMoreInputs { + final_fee, + minimum_fee, + seed_words, + witness_words, + }; + }; + if sweep_amount == 0 { + return V0MigrationRecomputeState::NeedsMoreInputs { + final_fee, + minimum_fee, + seed_words, + witness_words, + }; + } + + V0MigrationRecomputeState::Ready(V0MigrationReadyState { + final_fee, + minimum_fee, + seed_words, + witness_words, + destination_output: PlannedOutput { + lock_root: destination_output.lock_root.clone(), + amount: sweep_amount, + note_data: destination_output.note_data.clone(), + }, + }) + } + + /// Computes minimum fee from seed/witness word counts under current chain rules. + fn minimum_fee(&self, seed_words: u64, witness_words: u64) -> u64 { + compute_minimum_fee(FeeInputs { + seed_words, + witness_words, + base_fee: self.request.chain_context.base_fee, + input_fee_divisor: self.request.chain_context.input_fee_divisor, + min_fee: self.request.chain_context.min_fee, + height: self.request.chain_context.height.clone(), + bythos_phase: self.request.chain_context.bythos_phase.clone(), + }) + .minimum_fee + } + + /// Computes refundable remainder for a candidate final fee. + fn refund_amount(&self, fee: u64) -> u64 { + let required = self.gift_total.saturating_add(fee); + self.selected_total.saturating_sub(required) + } +} + +/// Plans input selection, fee, and outputs for create-tx using deterministic +/// ordering and lock/timelock spendability checks. +pub fn plan_create_tx( + request: &PlanRequest, + matcher: &M, +) -> Result { + if matches!(&request.planning_mode, PlanningMode::Standard) + && request.recipient_outputs.is_empty() + { + return Err(PlanError::NoRecipients); + } + + let candidates = request.ordered_candidates()?; + let mut session = PlanSession::new(request); + + match &request.planning_mode { + PlanningMode::Standard => { + for candidate in candidates { + let Some(recompute) = session.try_select_standard_candidate(candidate, matcher)? + else { + continue; + }; + if matches!(&request.selection_mode, SelectionMode::Auto) + && session.selected_total >= session.required_total(recompute.final_fee) + { + break; + } + } + + let recompute = session.recompute_standard_fee(); + let required = session.required_total(recompute.final_fee); + if session.selected_total < required { + return Err(PlanError::InsufficientFunds { + selected_total: session.selected_total, + required, + }); + } + + let allocation = allocate_inputs( + session.selected_total, session.gift_total, recompute.final_fee, + ) + .expect("required <= selected_total should always allocate"); + let conservation = ConservationCheck { + input_total: session.selected_total, + gift_total: allocation.gift_total, + refund_total: allocation.refund, + fee: allocation.fee, + }; + if !conservation.is_balanced() { + return Err(PlanError::ConservationFailed); + } + + Ok(PlanResult { + selected: session + .selected + .iter() + .map(|input| input.candidate.identity().clone()) + .collect(), + selected_total: session.selected_total, + outputs: recompute.outputs, + final_fee: recompute.final_fee, + word_counts: WordCountBreakdown { + seed_words: recompute.seed_words, + witness_words: recompute.witness_words, + }, + debug_trace: session.debug_trace, + }) + } + PlanningMode::V0MigrationSweep { .. } => { + for candidate in candidates { + let _ = session.try_select_v0_migration_candidate(candidate)?; + } + + let recompute = session.recompute_v0_migration(); + let ready = match recompute { + V0MigrationRecomputeState::NeedsMoreInputs { final_fee, .. } => { + return Err(PlanError::V0MigrationProducesZeroValue { + selected_total: session.selected_total, + fee: final_fee, + }); + } + V0MigrationRecomputeState::Ready(ready) => ready, + }; + + if session.selected_total + != ready + .destination_output + .amount + .saturating_add(ready.final_fee) + { + return Err(PlanError::ConservationFailed); + } + + Ok(PlanResult { + selected: session + .selected + .iter() + .map(|input| input.candidate.identity().clone()) + .collect(), + selected_total: session.selected_total, + outputs: vec![ready.destination_output], + final_fee: ready.final_fee, + word_counts: WordCountBreakdown { + seed_words: ready.seed_words, + witness_words: ready.witness_words, + }, + debug_trace: session.debug_trace, + }) + } + } +} + +impl PlanRequest { + /// Produces candidate ordering for the selected mode: + /// deterministic sort by `SelectionOrder` for both auto and manual candidate sets. + fn ordered_candidates(&self) -> Result, PlanError> { + match &self.selection_mode { + SelectionMode::Auto => { + let mut out = self.candidates.clone(); + sort_candidates(&mut out, self.order_direction); + Ok(out) + } + SelectionMode::Manual { note_names } => { + if note_names.is_empty() { + return Err(PlanError::ManualNamesMissing); + } + let mut seen = BTreeSet::<([u64; 5], [u64; 5])>::new(); + let mut out = Vec::::new(); + for name in note_names { + let key = (name.first.to_array(), name.last.to_array()); + if !seen.insert(key) { + return Err(PlanError::DuplicateManualName { + first: name.first.to_base58(), + last: name.last.to_base58(), + }); + } + let Some(candidate) = self + .candidates + .iter() + .find(|candidate| candidate.identity().name == *name) + .cloned() + else { + return Err(PlanError::ManualNoteMissing { + first: name.first.to_base58(), + last: name.last.to_base58(), + }); + }; + out.push(candidate); + } + sort_candidates(&mut out, self.order_direction); + Ok(out) + } + } + } +} + +/// Builds the output set for fee accounting and final result emission. +/// Recipient outputs are copied as-is; refund is optional and appended only when +/// `refund > 0`. Omitting refund does not alter gift amounts. +fn outputs_with_refund(request: &PlanRequest, refund: u64) -> Vec { + let mut outputs = request.recipient_outputs.clone(); + if refund > 0 { + outputs.push(PlannedOutput { + amount: refund, + ..request.refund_output.clone() + }); + } + outputs +} + +fn v0_note_spendable_by_signers(note: &CandidateV0Note, signer_pubkeys: &[SchnorrPubkey]) -> bool { + signer_pubkeys + .iter() + .any(|signer| v0_note_spendable_by_signer(note, signer)) +} + +fn v0_note_spendable_by_signer(note: &CandidateV0Note, signer_pubkey: &SchnorrPubkey) -> bool { + (note.lock.pubkeys.len() == 1 && note.lock.pubkeys.first() == Some(signer_pubkey)) + || (note.lock.keys_required == 1 && note.lock.pubkeys.iter().any(|pk| pk == signer_pubkey)) +} + +fn v0_timelock_satisfied( + timelock: Option<&V0TimelockIntent>, + note_origin_page: &BlockHeight, + current_height: &BlockHeight, +) -> bool { + let Some(timelock) = timelock else { + return true; + }; + + let now = height_value(current_height); + let since = height_value(note_origin_page); + let rel_min_ok = timelock.relative.min.as_ref().is_none_or(|min| { + since + .checked_add((min.0).0) + .is_some_and(|required| now >= required) + }); + let rel_max_ok = timelock.relative.max.as_ref().is_none_or(|max| { + since + .checked_add((max.0).0) + .is_some_and(|required| now <= required) + }); + let abs_min_ok = timelock + .absolute + .min + .as_ref() + .is_none_or(|min| now >= height_value(min)); + let abs_max_ok = timelock + .absolute + .max + .as_ref() + .is_none_or(|max| now <= height_value(max)); + + rel_min_ok && rel_max_ok && abs_min_ok && abs_max_ok +} + +#[derive(Debug, Clone, PartialEq, Eq)] +struct AllocationResult { + gift_total: u64, + fee: u64, + refund: u64, +} + +/// Splits selected inputs into gifts, fee, and refund while preserving conservation. +/// `gift_total` is caller-provided and never increased here; any leftover after +/// `gift_total + fee` is assigned to `refund`. +fn allocate_inputs(total_inputs: u64, gift_total: u64, fee: u64) -> Option { + let required = gift_total.checked_add(fee)?; + if total_inputs < required { + return None; + } + Some(AllocationResult { + gift_total, + fee, + refund: total_inputs - required, + }) +} + +#[derive(Debug, Clone, PartialEq, Eq)] +struct ConservationCheck { + input_total: u64, + gift_total: u64, + refund_total: u64, + fee: u64, +} + +impl ConservationCheck { + fn is_balanced(&self) -> bool { + self.input_total + == self + .gift_total + .saturating_add(self.refund_total) + .saturating_add(self.fee) + } +} + +/// Extracts raw numeric block height from the tx-engine wrapper type. +fn height_value(height: &BlockHeight) -> u64 { + (height.0).0 +} + +/// Returns true when every timelock primitive in the spend condition is +/// currently satisfiable. +fn timelock_satisfied( + spend_condition: &SpendCondition, + note_origin_page: &BlockHeight, + current_height: &BlockHeight, +) -> bool { + spend_condition.iter().all(|primitive| match primitive { + LockPrimitive::Tim(tim) => { + timelock_primitive_satisfied(tim, note_origin_page, current_height) + } + _ => true, + }) +} + +/// Evaluates a single timelock primitive against note origin height and +/// current chain height. +fn timelock_primitive_satisfied( + tim: &LockTim, + note_origin_page: &BlockHeight, + current_height: &BlockHeight, +) -> bool { + let now = height_value(current_height); + let since = height_value(note_origin_page); + let rel_min_ok = tim.rel.min.as_ref().is_none_or(|min| { + since + .checked_add((min.0).0) + .is_some_and(|required| now >= required) + }); + let rel_max_ok = tim.rel.max.as_ref().is_none_or(|max| { + since + .checked_add((max.0).0) + .is_some_and(|required| now <= required) + }); + let abs_min_ok = tim + .abs + .min + .as_ref() + .is_none_or(|min| now >= height_value(min)); + let abs_max_ok = tim + .abs + .max + .as_ref() + .is_none_or(|max| now <= height_value(max)); + rel_min_ok && rel_max_ok && abs_min_ok && abs_max_ok +} + +#[cfg(test)] +mod tests { + use bytes::Bytes; + use nockapp::noun::slab::{NockJammer, NounSlab}; + use nockchain_math::belt::Belt; + use nockchain_math::crypto::cheetah::{ch_scal, A_GEN}; + use nockchain_types::tx_engine::common::{ + BlockHeight, BlockHeightDelta, Hash, Name, Nicks, SchnorrPubkey, TimelockRangeAbsolute, + TimelockRangeRelative, Version, + }; + use nockchain_types::tx_engine::v0::{Lock as V0Lock, TimelockIntent as V0TimelockIntent}; + use nockchain_types::tx_engine::v1::tx::{Lock, LockPrimitive, LockTim, Pkh, SpendCondition}; + use noun_serde::NounEncode; + + use super::*; + use crate::lock_resolver::LockMatcher; + use crate::note_data::{ + DecodedNoteData, DecodedNoteDataEntry, DecodedNoteDataPayload, LockDataPayload, + NormalizedNoteDataKey, + }; + use crate::types::{ + CandidateIdentity, CandidateNote, CandidateV0Note, CandidateV1Note, CandidateVersionPolicy, + ChainContext, PlanningMode, RawNoteDataEntry, SelectionOrder, + }; + + /// Constructs a deterministic hash value from a single test limb. + fn hash(v: u64) -> Hash { + Hash::from_limbs(&[v, 0, 0, 0, 0]) + } + + /// Builds a deterministic note name pair for tests. + fn name(v: u64) -> Name { + Name::new(hash(v), hash(v + 100)) + } + + /// Builds a minimal candidate note with the provided asset amount. + fn candidate(v: u64, assets: u64) -> CandidateNote { + CandidateNote::V1(CandidateV1Note { + identity: CandidateIdentity { + name: name(v), + origin_page: BlockHeight(Belt(10)), + }, + assets: Nicks(assets as usize), + raw_note_data: Vec::::new(), + decoded_note_data: DecodedNoteData(Vec::new()), + }) + } + + /// Builds a minimal candidate note with one decoded `%lock` entry. + fn candidate_with_lock( + v: u64, + assets: u64, + spend_conditions: Vec, + ) -> CandidateNote { + CandidateNote::V1(CandidateV1Note { + identity: CandidateIdentity { + name: name(v), + origin_page: BlockHeight(Belt(10)), + }, + assets: Nicks(assets as usize), + raw_note_data: Vec::::new(), + decoded_note_data: DecodedNoteData(vec![DecodedNoteDataEntry { + raw_key: "lock".to_string(), + normalized_key: NormalizedNoteDataKey::Lock, + raw_blob: Bytes::new(), + payload: DecodedNoteDataPayload::Lock(LockDataPayload { + version: 0, + spend_conditions, + }), + decode_error: None, + }]), + }) + } + + /// Builds a minimal v0 candidate note with the provided asset amount. + fn candidate_v0(v: u64, assets: u64) -> CandidateNote { + CandidateNote::V0(CandidateV0Note { + identity: CandidateIdentity { + name: name(v), + origin_page: BlockHeight(Belt(10)), + }, + assets: Nicks(assets as usize), + lock: V0Lock { + keys_required: 1, + pubkeys: vec![signer_pubkey(1)], + }, + timelock: None, + }) + } + + fn candidate_v0_with_lock( + v: u64, + assets: u64, + lock: V0Lock, + timelock: Option, + ) -> CandidateNote { + CandidateNote::V0(CandidateV0Note { + identity: CandidateIdentity { + name: name(v), + origin_page: BlockHeight(Belt(10)), + }, + assets: Nicks(assets as usize), + lock, + timelock, + }) + } + + /// Builds an output with note-data so seed-word accounting exercises metadata paths. + fn output(lock_root: u64, amount: u64) -> PlannedOutput { + PlannedOutput { + lock_root: hash(lock_root), + amount, + note_data: vec![RawNoteDataEntry { + key: "meta".to_string(), + blob: jam(&0_u64), + }], + } + } + + /// Builds an output with no note-data for tests that isolate refund behavior. + fn output_without_note_data(lock_root: u64, amount: u64) -> PlannedOutput { + PlannedOutput { + lock_root: hash(lock_root), + amount, + note_data: Vec::new(), + } + } + + fn p2pkh_output(lock_root: u64, amount: u64) -> PlannedOutput { + PlannedOutput { + lock_root: hash(lock_root), + amount, + note_data: vec![RawNoteDataEntry::from_lock(Lock::SpendCondition(simple_pkh_lock( + hash(lock_root), + )))], + } + } + + fn signer_pubkey(multiplier: u64) -> SchnorrPubkey { + if multiplier == 1 { + SchnorrPubkey(A_GEN) + } else { + SchnorrPubkey(ch_scal(multiplier, &A_GEN).expect("scaled test pubkey")) + } + } + + fn v0_timelock_relative_min(min: u64) -> Option { + Some(V0TimelockIntent { + absolute: TimelockRangeAbsolute::none(), + relative: TimelockRangeRelative::new(Some(BlockHeightDelta(Belt(min))), None), + }) + } + + /// Creates a simple single-signer PKH spend condition. + fn simple_pkh_lock(pkh: Hash) -> SpendCondition { + SpendCondition::new(vec![LockPrimitive::Pkh(Pkh::new(1, vec![pkh]))]) + } + + /// Creates a coinbase-style lock containing PKH + relative timelock. + fn coinbase_like_lock(pkh: Hash, relative_min: u64) -> SpendCondition { + SpendCondition::new(vec![ + LockPrimitive::Pkh(Pkh::new(1, vec![pkh])), + LockPrimitive::Tim(LockTim { + rel: TimelockRangeRelative::new(Some(BlockHeightDelta(Belt(relative_min))), None), + abs: TimelockRangeAbsolute::none(), + }), + ]) + } + + /// Jam-encodes an arbitrary noun-encodable test value into bytes. + fn jam(value: &T) -> Bytes { + let mut slab: NounSlab = NounSlab::new(); + let noun = value.to_noun(&mut slab); + slab.set_root(noun); + slab.jam() + } + + /// Creates a baseline plan request used by planner unit tests. + fn base_request() -> PlanRequest { + PlanRequest { + planning_mode: PlanningMode::Standard, + selection_mode: SelectionMode::Auto, + order_direction: SelectionOrder::Ascending, + include_data: true, + chain_context: ChainContext { + height: BlockHeight(Belt(10)), + bythos_phase: BlockHeight(Belt(10)), + base_fee: 0, + input_fee_divisor: 4, + min_fee: 0, + }, + signer_pkh: Some(hash(999)), + candidate_version_policy: CandidateVersionPolicy::V1Only, + candidates: vec![candidate(1, 8), candidate(2, 3), candidate(3, 20)], + recipient_outputs: vec![output(42, 10)], + refund_output: output(43, 0), + coinbase_relative_min: Some(5), + v0_migration_signer_pubkeys: Vec::new(), + } + } + + struct AlwaysMatches; + impl LockMatcher for AlwaysMatches { + /// Test matcher that accepts every first-name/lock combination. + fn matches(&self, _note_first_name: &Hash, _spend_condition: &SpendCondition) -> bool { + true + } + } + + struct MatchSingleSignerPkh { + signer_pkh: Hash, + } + + impl LockMatcher for MatchSingleSignerPkh { + /// Test matcher that only accepts locks whose PKH primitive can be + /// satisfied by `signer_pkh` with m=1. + fn matches(&self, _note_first_name: &Hash, spend_condition: &SpendCondition) -> bool { + let mut saw_pkh = false; + for primitive in spend_condition.iter() { + match primitive { + LockPrimitive::Pkh(pkh) => { + saw_pkh = true; + if pkh.m != 1 { + return false; + } + if !pkh.hashes.iter().any(|hash| hash == &self.signer_pkh) { + return false; + } + } + LockPrimitive::Tim(_) => {} + _ => return false, + } + } + saw_pkh + } + } + + #[test] + /// Verifies planner rejects empty recipient output lists. + fn no_recipients_returns_error() { + let mut request = base_request(); + request.recipient_outputs = Vec::new(); + let error = plan_create_tx(&request, &AlwaysMatches).expect_err("expected no recipients"); + assert!(matches!(error, PlanError::NoRecipients)); + } + + #[test] + /// Verifies v0 migration sweep consumes all admissible legacy notes and ignores v1 notes. + fn v0_migration_sweep_selects_all_admissible_v0_candidates() { + let mut request = base_request(); + request.planning_mode = PlanningMode::V0MigrationSweep { + destination_output: output_without_note_data(42, 0), + }; + request.candidate_version_policy = CandidateVersionPolicy::V0Only; + request.candidates = vec![candidate_v0(1, 8), candidate_v0(2, 3), candidate(3, 99)]; + request.recipient_outputs = Vec::new(); + request.v0_migration_signer_pubkeys = vec![signer_pubkey(1)]; + + let result = plan_create_tx(&request, &AlwaysMatches).expect("migration plan"); + assert_eq!(result.selected.len(), 2); + assert_eq!(result.selected_total, 11); + assert_eq!(result.outputs.len(), 1); + assert_eq!(result.outputs[0].amount, 11); + assert_eq!(result.final_fee, 0); + } + + #[test] + /// Verifies v0 migration sweep skips legacy notes whose lock is not spendable by the migration signer. + fn v0_migration_sweep_skips_unspendable_v0_candidates() { + let mut request = base_request(); + request.planning_mode = PlanningMode::V0MigrationSweep { + destination_output: output_without_note_data(42, 0), + }; + request.candidate_version_policy = CandidateVersionPolicy::V0Only; + request.candidates = vec![ + candidate_v0(1, 8), + candidate_v0_with_lock( + 2, + 13, + V0Lock { + keys_required: 1, + pubkeys: vec![signer_pubkey(2)], + }, + None, + ), + ]; + request.recipient_outputs = Vec::new(); + request.v0_migration_signer_pubkeys = vec![signer_pubkey(1)]; + + let result = plan_create_tx(&request, &AlwaysMatches).expect("migration plan"); + assert_eq!(result.selected.len(), 1); + assert_eq!(result.selected[0].name, name(1)); + assert_eq!(result.outputs[0].amount, 8); + } + + #[test] + /// Verifies v0 migration sweep skips legacy notes whose v0 timelock is not yet spendable. + fn v0_migration_sweep_skips_unmatured_v0_timelocked_candidates() { + let mut request = base_request(); + request.planning_mode = PlanningMode::V0MigrationSweep { + destination_output: output_without_note_data(42, 0), + }; + request.candidate_version_policy = CandidateVersionPolicy::V0Only; + request.candidates = vec![ + candidate_v0(1, 8), + candidate_v0_with_lock( + 2, + 13, + V0Lock { + keys_required: 1, + pubkeys: vec![signer_pubkey(1)], + }, + v0_timelock_relative_min(5), + ), + ]; + request.recipient_outputs = Vec::new(); + request.v0_migration_signer_pubkeys = vec![signer_pubkey(1)]; + + let result = plan_create_tx(&request, &AlwaysMatches).expect("migration plan"); + assert_eq!(result.selected.len(), 1); + assert_eq!(result.selected[0].name, name(1)); + assert_eq!(result.outputs[0].amount, 8); + } + + #[test] + /// Verifies v0 migration sweep errors when fees consume the full selected value. + fn v0_migration_sweep_errors_when_fee_consumes_all_selected_value() { + let mut request = base_request(); + request.planning_mode = PlanningMode::V0MigrationSweep { + destination_output: output_without_note_data(42, 0), + }; + request.chain_context.min_fee = 10; + request.candidate_version_policy = CandidateVersionPolicy::V0Only; + request.candidates = vec![candidate_v0(1, 10)]; + request.recipient_outputs = Vec::new(); + request.v0_migration_signer_pubkeys = vec![signer_pubkey(1)]; + + let error = + plan_create_tx(&request, &AlwaysMatches).expect_err("expected zero-value sweep error"); + assert!(matches!( + error, + PlanError::V0MigrationProducesZeroValue { + selected_total: 10, + fee: 10, + } + )); + } + + #[test] + /// Verifies v0 migration uses the expected legacy witness words and fee under fakenet-style constants. + fn v0_migration_sweep_matches_expected_word_and_fee_counts() { + let mut request = base_request(); + request.planning_mode = PlanningMode::V0MigrationSweep { + destination_output: p2pkh_output(42, 0), + }; + request.chain_context = ChainContext { + height: BlockHeight(Belt(1)), + bythos_phase: BlockHeight(Belt(1)), + base_fee: 128, + input_fee_divisor: 4, + min_fee: 256, + }; + request.candidate_version_policy = CandidateVersionPolicy::V0Only; + request.candidates = vec![candidate_v0(1, 25_000)]; + request.recipient_outputs = Vec::new(); + request.v0_migration_signer_pubkeys = vec![signer_pubkey(1)]; + + let result = plan_create_tx(&request, &AlwaysMatches).expect("migration plan"); + assert_eq!(result.selected_total, 25_000); + assert_eq!(result.word_counts.seed_words, 14); + assert_eq!(result.word_counts.witness_words, 31); + assert_eq!(result.final_fee, 2_784); + assert_eq!(result.outputs.len(), 1); + assert_eq!(result.outputs[0].amount, 22_216); + } + + #[test] + /// Verifies manual mode requires at least one provided note name. + fn manual_mode_requires_at_least_one_name() { + let mut request = base_request(); + request.selection_mode = SelectionMode::Manual { + note_names: Vec::new(), + }; + let error = + plan_create_tx(&request, &AlwaysMatches).expect_err("expected missing manual names"); + assert!(matches!(error, PlanError::ManualNamesMissing)); + } + + #[test] + /// Verifies manual mode returns a structured error for unknown note names. + fn manual_mode_unknown_note_name_returns_error() { + let mut request = base_request(); + request.selection_mode = SelectionMode::Manual { + note_names: vec![name(999)], + }; + let error = plan_create_tx(&request, &AlwaysMatches).expect_err("expected missing note"); + assert!(matches!(error, PlanError::ManualNoteMissing { .. })); + } + + #[test] + /// Verifies manual mode rejects duplicate note names. + fn manual_mode_duplicate_note_name_returns_error() { + let mut request = base_request(); + request.selection_mode = SelectionMode::Manual { + note_names: vec![name(1), name(1)], + }; + let error = + plan_create_tx(&request, &AlwaysMatches).expect_err("expected duplicate manual note"); + assert!(matches!(error, PlanError::DuplicateManualName { .. })); + } + + #[test] + /// Verifies auto mode consumes candidates in deterministic order until coverage. + fn auto_mode_selects_ordered_notes_until_cover() { + let request = base_request(); + let result = plan_create_tx(&request, &AlwaysMatches).expect("plan"); + + assert_eq!(result.selected.len(), 2); + assert_eq!(result.selected_total, 11); + assert_eq!(result.final_fee, 0); + assert_eq!(result.selected[0].name, name(2)); + assert_eq!(result.selected[1].name, name(1)); + } + + #[test] + /// Verifies manual mode applies `SelectionOrder` after filtering to manual note names. + fn manual_mode_orders_selected_candidates_by_selection_order() { + let mut request = base_request(); + request.selection_mode = SelectionMode::Manual { + note_names: vec![name(1), name(2)], + }; + request.recipient_outputs = vec![output(42, 0)]; + + let result = plan_create_tx(&request, &AlwaysMatches).expect("plan"); + assert_eq!(result.selected.len(), 2); + assert_eq!(result.selected[0].name, name(2)); + assert_eq!(result.selected[1].name, name(1)); + } + + #[test] + /// Verifies manual mode descending order reverses assets ordering for selected candidates. + fn manual_mode_descending_orders_selected_candidates_by_selection_order() { + let mut request = base_request(); + request.selection_mode = SelectionMode::Manual { + note_names: vec![name(1), name(2)], + }; + request.order_direction = SelectionOrder::Descending; + request.recipient_outputs = vec![output(42, 0)]; + + let result = plan_create_tx(&request, &AlwaysMatches).expect("plan"); + assert_eq!(result.selected.len(), 2); + assert_eq!(result.selected[0].name, name(1)); + assert_eq!(result.selected[1].name, name(2)); + } + + #[test] + /// Verifies v0-only selector policy rejects manual v1 candidates. + fn manual_mode_v0_only_policy_rejects_v1_candidates() { + let mut request = base_request(); + request.selection_mode = SelectionMode::Manual { + note_names: vec![name(1)], + }; + request.candidate_version_policy = CandidateVersionPolicy::V0Only; + request.candidates = vec![candidate(1, 8)]; + request.recipient_outputs = vec![output(42, 10)]; + + let error = plan_create_tx(&request, &AlwaysMatches) + .expect_err("expected v0-only policy rejection for v1 manual selection"); + assert!(matches!( + error, + PlanError::CandidateVersionDisabled { + version: Version::V1, + policy: CandidateVersionPolicy::V0Only, + .. + } + )); + } + + #[test] + /// Verifies planner consumes fee capacity when adding a refund output would + /// increase the minimum fee beyond available capacity. + fn auto_mode_consumes_capacity_as_fee_when_refund_output_is_not_fee_viable() { + let signer = hash(999); + let mut request = base_request(); + request.chain_context.base_fee = 1; + request.chain_context.input_fee_divisor = 1_000_000_000; + request.candidates = vec![candidate(1, 8), candidate(2, 4)]; + request.recipient_outputs = vec![output_without_note_data(42, 10)]; + request.refund_output = output_without_note_data(43, 0); + request.signer_pkh = Some(signer); + request.coinbase_relative_min = None; + + let result = plan_create_tx(&request, &AlwaysMatches).expect("plan"); + assert_eq!(result.selected_total, 12); + assert_eq!(result.final_fee, 2); + assert_eq!( + result.outputs.len(), + 1, + "no refund output should be emitted" + ); + } + + #[test] + /// Verifies output assembly appends refund output when refund amount is positive. + fn outputs_with_refund_appends_refund_output_when_positive() { + let request = base_request(); + let outputs = outputs_with_refund(&request, 1); + assert_eq!(outputs.len(), 2); + assert_eq!(outputs[1].amount, 1); + } + + #[test] + /// Verifies notes rejected by a signer-aware matcher are skipped and do not + /// block selection of later spendable notes. + fn auto_mode_skips_notes_unmatched_by_signer_matcher() { + let signer = hash(999); + let unmatched_lock = simple_pkh_lock(hash(111)); + let matched_lock = simple_pkh_lock(signer.clone()); + let mut request = base_request(); + request.candidates = vec![ + candidate_with_lock(1, 5, vec![unmatched_lock.clone()]), + candidate_with_lock(2, 8, vec![matched_lock.clone()]), + ]; + request.candidates[0].identity_mut().name.first = + unmatched_lock.first_name().expect("first-name").into_hash(); + request.candidates[1].identity_mut().name.first = + matched_lock.first_name().expect("first-name").into_hash(); + let expected_selected_name = request.candidates[1].identity().name.clone(); + request.recipient_outputs = vec![output(42, 8)]; + request.signer_pkh = None; + request.coinbase_relative_min = None; + + let matcher = MatchSingleSignerPkh { signer_pkh: signer }; + let result = plan_create_tx(&request, &matcher).expect("plan"); + assert_eq!(result.selected.len(), 1); + assert_eq!(result.selected[0].name, expected_selected_name); + } + + #[test] + /// Verifies unresolved locks are skipped and eventually surface as insufficient funds. + fn unresolved_locks_are_skipped_until_insufficient_funds() { + struct NeverMatches; + impl LockMatcher for NeverMatches { + /// Test matcher that rejects every first-name/lock combination. + fn matches(&self, _note_first_name: &Hash, _spend_condition: &SpendCondition) -> bool { + false + } + } + + let mut request = base_request(); + request.signer_pkh = None; + request.coinbase_relative_min = None; + let error = plan_create_tx(&request, &NeverMatches).expect_err("expected lock error"); + assert!(matches!(error, PlanError::InsufficientFunds { .. })); + } + + #[test] + /// Verifies v0-only selector policy skips v1 notes in auto mode. + fn auto_mode_v0_only_policy_skips_v1_candidates() { + let mut request = base_request(); + request.candidate_version_policy = CandidateVersionPolicy::V0Only; + request.candidates = vec![candidate(1, 8), candidate_v0(2, 8)]; + request.recipient_outputs = vec![output(42, 12)]; + + let error = plan_create_tx(&request, &AlwaysMatches) + .expect_err("expected insufficient funds after v1 skip in v0-only mode"); + assert!(matches!(error, PlanError::InsufficientFunds { .. })); + } + + #[test] + /// Verifies v0-only selector policy can select v0 candidates. + fn auto_mode_v0_only_policy_selects_v0_candidates() { + struct NeverMatches; + impl LockMatcher for NeverMatches { + fn matches(&self, _note_first_name: &Hash, _spend_condition: &SpendCondition) -> bool { + false + } + } + + let mut request = base_request(); + request.candidate_version_policy = CandidateVersionPolicy::V0Only; + request.candidates = vec![candidate(1, 8), candidate_v0(2, 12)]; + request.recipient_outputs = vec![output(42, 10)]; + request.v0_migration_signer_pubkeys = vec![signer_pubkey(1)]; + + let result = plan_create_tx(&request, &NeverMatches).expect("plan"); + assert_eq!(result.selected.len(), 1); + assert_eq!(result.selected[0].name, name(2)); + } + + #[test] + /// Verifies v1-only selector policy skips v0 notes in auto mode. + fn auto_mode_v1_only_policy_skips_v0_candidates() { + let mut request = base_request(); + request.candidates = vec![candidate_v0(1, 100)]; + request.recipient_outputs = vec![output(42, 10)]; + + let error = plan_create_tx(&request, &AlwaysMatches) + .expect_err("expected insufficient funds with v0 filtered out"); + assert!(matches!(error, PlanError::InsufficientFunds { .. })); + } + + #[test] + /// Verifies v1-only selector policy rejects manual v0 candidates. + fn manual_mode_v1_only_policy_rejects_v0_candidates() { + let mut request = base_request(); + request.selection_mode = SelectionMode::Manual { + note_names: vec![name(1)], + }; + request.candidates = vec![candidate_v0(1, 100)]; + request.recipient_outputs = vec![output(42, 10)]; + + let error = plan_create_tx(&request, &AlwaysMatches) + .expect_err("expected v1-only policy rejection for v0 manual selection"); + assert!(matches!( + error, + PlanError::CandidateVersionDisabled { + version: Version::V0, + policy: CandidateVersionPolicy::V1Only, + .. + } + )); + } + + #[test] + /// Verifies auto mode skips notes gated by unsatisfied timelocks. + fn auto_mode_skips_timelocked_notes_that_are_not_spendable_yet() { + let signer = hash(999); + let timelocked = coinbase_like_lock(signer.clone(), 5); + let spendable = simple_pkh_lock(signer); + + let mut request = base_request(); + request.candidates = vec![ + candidate_with_lock(1, 8, vec![timelocked.clone()]), + candidate_with_lock(2, 8, vec![spendable.clone()]), + ]; + request.candidates[0].identity_mut().name.first = + timelocked.first_name().expect("first-name").into_hash(); + request.candidates[1].identity_mut().name.first = + spendable.first_name().expect("first-name").into_hash(); + let expected_selected_name = request.candidates[1].identity().name.clone(); + request.candidates[0].identity_mut().origin_page = BlockHeight(Belt(8)); + request.candidates[1].identity_mut().origin_page = BlockHeight(Belt(2)); + request.recipient_outputs = vec![output(42, 8)]; + request.signer_pkh = None; + request.coinbase_relative_min = None; + + let result = plan_create_tx(&request, &AlwaysMatches).expect("plan"); + assert_eq!(result.selected.len(), 1); + assert_eq!(result.selected[0].name, expected_selected_name); + assert!( + result + .debug_trace + .iter() + .any(|entry| entry.contains("timelock not satisfied")), + "expected debug trace to mention timelock filtering" + ); + } + + #[test] + /// Verifies manual mode also skips notes gated by unsatisfied timelocks. + fn manual_mode_skips_timelocked_notes_that_are_not_spendable_yet() { + let signer = hash(999); + let timelocked = coinbase_like_lock(signer.clone(), 5); + let spendable = simple_pkh_lock(signer); + + let mut request = base_request(); + request.candidates = vec![ + candidate_with_lock(1, 8, vec![timelocked.clone()]), + candidate_with_lock(2, 8, vec![spendable.clone()]), + ]; + request.candidates[0].identity_mut().name.first = + timelocked.first_name().expect("first-name").into_hash(); + request.candidates[1].identity_mut().name.first = + spendable.first_name().expect("first-name").into_hash(); + let selected_name = request.candidates[1].identity().name.clone(); + request.selection_mode = SelectionMode::Manual { + note_names: request + .candidates + .iter() + .map(|candidate| candidate.identity().name.clone()) + .collect(), + }; + request.candidates[0].identity_mut().origin_page = BlockHeight(Belt(8)); + request.candidates[1].identity_mut().origin_page = BlockHeight(Belt(2)); + request.recipient_outputs = vec![output(42, 8)]; + request.signer_pkh = None; + request.coinbase_relative_min = None; + + let result = plan_create_tx(&request, &AlwaysMatches).expect("plan"); + assert_eq!(result.selected.len(), 1); + assert_eq!(result.selected[0].name, selected_name); + assert!( + result + .debug_trace + .iter() + .any(|entry| entry.contains("timelock not satisfied")), + "expected debug trace to mention timelock filtering" + ); + } + + #[test] + /// Verifies relative timelock min/max are inclusive at boundaries and + /// fail outside those bounds. + fn timelock_relative_bounds_apply_at_edges() { + let tim = LockTim { + rel: TimelockRangeRelative::new( + Some(BlockHeightDelta(Belt(5))), + Some(BlockHeightDelta(Belt(7))), + ), + abs: TimelockRangeAbsolute::none(), + }; + let origin = BlockHeight(Belt(100)); + + assert!(!timelock_primitive_satisfied( + &tim, + &origin, + &BlockHeight(Belt(104)) + )); + assert!(timelock_primitive_satisfied( + &tim, + &origin, + &BlockHeight(Belt(105)) + )); + assert!(timelock_primitive_satisfied( + &tim, + &origin, + &BlockHeight(Belt(107)) + )); + assert!(!timelock_primitive_satisfied( + &tim, + &origin, + &BlockHeight(Belt(108)) + )); + } + + #[test] + /// Verifies absolute timelock min/max are inclusive at boundaries and + /// fail outside those bounds. + fn timelock_absolute_bounds_apply_at_edges() { + let tim = LockTim { + rel: TimelockRangeRelative::none(), + abs: TimelockRangeAbsolute::new( + Some(BlockHeight(Belt(200))), + Some(BlockHeight(Belt(202))), + ), + }; + let origin = BlockHeight(Belt(0)); + + assert!(!timelock_primitive_satisfied( + &tim, + &origin, + &BlockHeight(Belt(199)) + )); + assert!(timelock_primitive_satisfied( + &tim, + &origin, + &BlockHeight(Belt(200)) + )); + assert!(timelock_primitive_satisfied( + &tim, + &origin, + &BlockHeight(Belt(202)) + )); + assert!(!timelock_primitive_satisfied( + &tim, + &origin, + &BlockHeight(Belt(203)) + )); + } + + #[test] + /// Verifies overflow in relative timelock arithmetic is treated as + /// unsatisfied rather than wrapping. + fn timelock_relative_overflow_is_unsatisfied() { + let tim = LockTim { + rel: TimelockRangeRelative::new(Some(BlockHeightDelta(Belt(10))), None), + abs: TimelockRangeAbsolute::none(), + }; + let origin = BlockHeight(Belt(u64::MAX - 1)); + assert!(!timelock_primitive_satisfied( + &tim, + &origin, + &BlockHeight(Belt(u64::MAX)) + )); + } +} diff --git a/crates/wallet-tx-builder/src/types.rs b/crates/wallet-tx-builder/src/types.rs new file mode 100644 index 000000000..b8e35bb78 --- /dev/null +++ b/crates/wallet-tx-builder/src/types.rs @@ -0,0 +1,295 @@ +use bytes::Bytes; +use nockchain_types::tx_engine::common::{BlockHeight, Hash, Name, Nicks, SchnorrPubkey, Version}; +use nockchain_types::tx_engine::v0::{Lock as V0Lock, TimelockIntent as V0TimelockIntent}; +use nockchain_types::tx_engine::v1::note::Note; +use nockchain_types::tx_engine::v1::tx::SpendCondition; + +use crate::note_data::DecodedNoteData; + +/// Candidate-selection mode for planner input admission and ordering. +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum SelectionMode { + /// Manual candidate mode with an explicit candidate set provided by name. + /// Ordering is still governed by `SelectionOrder`. + Manual { note_names: Vec }, + /// Automatic candidate mode that orders from the normalized candidate set. + Auto, +} + +/// Refers to the order of the `assets` in each note when selecting candidates. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum SelectionOrder { + Ascending, + Descending, +} + +/// Controls which note versions the selector is allowed to consume. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum CandidateVersionPolicy { + /// Default mode: selector chooses only v1 notes. + V1Only, + /// Special v0 fan-in mode: selector chooses only v0 notes. + V0Only, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +/// Chain-level fee and height context used for planner decisions. +pub struct ChainContext { + /// Current chain height used for fee rules and timelock checks. + pub height: BlockHeight, + /// Activation height for Bythos-era fee/seed accounting behavior. + pub bythos_phase: BlockHeight, + /// Per-word fee unit. + pub base_fee: u64, + /// Witness fee discount divisor applied post-Bythos. + pub input_fee_divisor: u64, + /// Global minimum fee floor. + pub min_fee: u64, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +/// Stable identity for a note candidate independent of decode/render details. +pub struct CandidateIdentity { + /// Full note name `[first last]`. + pub name: Name, + /// Block/page where this note originated. + pub origin_page: BlockHeight, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +/// Raw key/value note-data entry preserved for round-tripping and fee accounting. +pub struct RawNoteDataEntry { + /// Note-data key (e.g. `lock`, `bridge`, `bridge-w`). + pub key: String, + /// Jammed payload bytes for this key. + pub blob: Bytes, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +/// Candidate payload for a legacy v0 note. +pub struct CandidateV0Note { + /// Stable identity metadata for this candidate note. + pub identity: CandidateIdentity, + /// Spendable asset amount carried by this note. + pub assets: Nicks, + /// Legacy v0 signing lock used for migration spendability checks. + pub lock: V0Lock, + /// Legacy v0 timelock constraints applied to this note. + pub timelock: Option, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +/// Candidate payload for a v1 note including decoded note-data views. +pub struct CandidateV1Note { + /// Stable identity metadata for this candidate note. + pub identity: CandidateIdentity, + /// Spendable asset amount carried by this note. + pub assets: Nicks, + /// Raw note-data entries preserved from wallet snapshot. + pub raw_note_data: Vec, + /// Best-effort decoded note-data view used by lock/timelock logic. + pub decoded_note_data: DecodedNoteData, +} + +/// Shared empty decoded note-data view for v0 candidates. +static EMPTY_DECODED_NOTE_DATA: DecodedNoteData = DecodedNoteData(Vec::new()); + +#[derive(Debug, Clone, PartialEq, Eq)] +/// Version-dispatched candidate note used by planner selection and filtering. +pub enum CandidateNote { + V0(CandidateV0Note), + V1(CandidateV1Note), +} + +impl CandidateNote { + /// Builds a candidate note from a decoded tx-engine note value. + pub fn from_note(name: &Name, note: &Note) -> Self { + match note { + Note::V0(note_v0) => CandidateNote::V0(CandidateV0Note { + identity: CandidateIdentity { + name: name.clone(), + origin_page: note_v0.head.origin_page.clone(), + }, + assets: note_v0.tail.assets.clone(), + lock: note_v0.tail.lock.clone(), + timelock: note_v0.head.timelock.0.clone(), + }), + Note::V1(note_v1) => { + let raw_note_data = note_v1 + .note_data + .iter() + .map(|entry| RawNoteDataEntry { + key: entry.key.clone(), + blob: entry.blob.clone(), + }) + .collect::>(); + let decoded_note_data = DecodedNoteData::from(¬e_v1.note_data); + CandidateNote::V1(CandidateV1Note { + identity: CandidateIdentity { + name: name.clone(), + origin_page: note_v1.origin_page.clone(), + }, + assets: note_v1.assets.clone(), + raw_note_data, + decoded_note_data, + }) + } + } + } + + /// Returns the stable identity metadata shared across candidate versions. + pub fn identity(&self) -> &CandidateIdentity { + match self { + CandidateNote::V0(note) => ¬e.identity, + CandidateNote::V1(note) => ¬e.identity, + } + } + + /// Returns mutable identity metadata used by planner tests and transforms. + pub fn identity_mut(&mut self) -> &mut CandidateIdentity { + match self { + CandidateNote::V0(note) => &mut note.identity, + CandidateNote::V1(note) => &mut note.identity, + } + } + + /// Returns this candidate's spendable asset amount. + pub fn assets(&self) -> &Nicks { + match self { + CandidateNote::V0(note) => ¬e.assets, + CandidateNote::V1(note) => ¬e.assets, + } + } + + /// Returns decoded note-data entries; v0 notes expose an empty decoded view. + pub fn decoded_note_data(&self) -> &DecodedNoteData { + match self { + CandidateNote::V0(_) => &EMPTY_DECODED_NOTE_DATA, + CandidateNote::V1(note) => ¬e.decoded_note_data, + } + } + + /// Returns raw note-data entries; v0 notes expose an empty slice. + pub fn raw_note_data(&self) -> &[RawNoteDataEntry] { + match self { + CandidateNote::V0(_) => &[], + CandidateNote::V1(note) => ¬e.raw_note_data, + } + } + + /// Returns this candidate note version. + pub fn version(&self) -> Version { + match self { + CandidateNote::V0(_) => Version::V0, + CandidateNote::V1(_) => Version::V1, + } + } + + /// Returns this note name as base58 `(first, last)` for logging and errors. + pub fn note_name_display(&self) -> (String, String) { + ( + self.identity().name.first.to_base58(), + self.identity().name.last.to_base58(), + ) + } +} + +#[derive(Debug, Clone, PartialEq, Eq)] +/// Planner output target with amount and optional note-data payloads. +pub struct PlannedOutput { + /// Destination lock root for this output. + pub lock_root: Hash, + /// Output amount assigned by planner/allocation. + pub amount: u64, + /// Output note-data that contributes to seed-word accounting. + pub note_data: Vec, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +/// Planner mode selects between fixed-recipient create-tx and legacy v0 sweep migration. +pub enum PlanningMode { + /// Standard create-tx planner behavior with fixed recipient outputs and optional refund. + Standard, + /// Full-sweep migration mode that spends all admissible v0 notes into one destination output. + V0MigrationSweep { + /// Destination output template used for fee accounting and final emission. + destination_output: PlannedOutput, + }, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +/// Assembled transaction input pairing selected note identity and spend lock. +pub struct AssembledInput { + pub note: CandidateIdentity, + pub lock: SpendCondition, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +/// Assembled transaction output containing lock root and assigned amount. +pub struct AssembledOutput { + pub lock_root: Hash, + pub amount: u64, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +/// Assembled transaction shape used by downstream tx construction steps. +pub struct AssembledTransaction { + pub inputs: Vec, + pub outputs: Vec, + pub fee: u64, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +/// Full planner input bundle for deterministic create-tx planning. +pub struct PlanRequest { + /// Planner mode controls output/funding semantics. + pub planning_mode: PlanningMode, + /// Selection mode: caller-specified manual candidate set (by note names) or automatic candidate search. + pub selection_mode: SelectionMode, + /// Order used when sorting candidate notes by their `assets` value. + pub order_direction: SelectionOrder, + /// Whether output note-data should be included in assembled outputs. + pub include_data: bool, + /// Chain fee/timelock context. + pub chain_context: ChainContext, + /// Primary signer PKH used for spendability filtering. + pub signer_pkh: Option, + /// Candidate note version policy used by selector admission checks. + pub candidate_version_policy: CandidateVersionPolicy, + /// Normalized candidate notes from a single wallet snapshot. + pub candidates: Vec, + /// Recipient outputs requested by the command. + pub recipient_outputs: Vec, + /// Refund output template; amount is set by planner and must always exist. + pub refund_output: PlannedOutput, + /// Relative timelock minimum used to classify coinbase-style locks. + pub coinbase_relative_min: Option, + /// Signer pubkeys used by v0 migration admission checks. + pub v0_migration_signer_pubkeys: Vec, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +/// Seed/witness word-count totals used to justify final fee decisions. +pub struct WordCountBreakdown { + /// Seed-side word count used for fee computation. + pub seed_words: u64, + /// Witness-side word count used for fee computation. + pub witness_words: u64, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +/// Final planner output including selected inputs, outputs, and fee details. +pub struct PlanResult { + /// Final selected input identities in deterministic spend order. + pub selected: Vec, + /// Total assets represented by selected inputs. + pub selected_total: u64, + /// Final output set (recipients plus optional refund). + pub outputs: Vec, + /// Planner-computed final fee. + pub final_fee: u64, + /// Word-count components backing final fee. + pub word_counts: WordCountBreakdown, + /// Human-readable planner decisions for debugging. + pub debug_trace: Vec, +} diff --git a/crates/wallet-tx-builder/src/word_count.rs b/crates/wallet-tx-builder/src/word_count.rs new file mode 100644 index 000000000..d7bb3b6f8 --- /dev/null +++ b/crates/wallet-tx-builder/src/word_count.rs @@ -0,0 +1,635 @@ +use std::collections::BTreeMap; + +use bytes::Bytes; +use nockchain_types::tx_engine::common::BlockHeight; +use nockchain_types::tx_engine::v1::tx::{LockPrimitive, SpendCondition}; + +use crate::note_data::{DecodedNoteDataEntry, DecodedNoteDataPayload, LockDataPayload}; +use crate::types::{ChainContext, PlannedOutput, RawNoteDataEntry}; + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct WitnessWordInput { + pub spend_condition: SpendCondition, + pub input_origin_page: BlockHeight, + // Optional lock-level hint used for lock-merkle-path estimation. + // When absent we conservatively assume a single spend-condition lock. + pub spend_condition_count: Option, +} + +#[derive(Debug, Clone, Copy)] +pub struct WordCountEstimator<'a> { + chain_context: &'a ChainContext, +} + +impl<'a> WordCountEstimator<'a> { + pub fn new(chain_context: &'a ChainContext) -> Self { + Self { chain_context } + } + + pub fn estimate_seed_words(&self, outputs: &[PlannedOutput]) -> u64 { + if (self.chain_context.height.0).0 >= (self.chain_context.bythos_phase.0).0 { + Self::estimate_seed_words_merged(outputs) + } else { + Self::estimate_seed_words_legacy(outputs) + } + } + + pub fn estimate_seed_words_legacy(outputs: &[PlannedOutput]) -> u64 { + outputs + .iter() + .map(|output| Self::estimate_note_data_words(&output.note_data)) + .sum() + } + + pub fn estimate_seed_words_merged(outputs: &[PlannedOutput]) -> u64 { + let mut merged_by_lock_root = BTreeMap::<[u64; 5], BTreeMap>::new(); + for output in outputs { + let entry = merged_by_lock_root + .entry(output.lock_root.to_array()) + .or_default(); + for note_data_entry in &output.note_data { + entry.insert(note_data_entry.key.clone(), note_data_entry.blob.clone()); + } + } + + merged_by_lock_root + .into_values() + .map(|merged| { + let entries = merged + .into_iter() + .map(|(key, blob)| RawNoteDataEntry { key, blob }) + .collect::>(); + Self::estimate_note_data_words(&entries) + }) + .sum() + } + + pub fn estimate_witness_words(&self, inputs: &[WitnessWordInput]) -> u64 { + inputs + .iter() + .map(|input| self.estimate_witness_words_for_input(input)) + .sum() + } + + pub fn estimate_witness_words_for_input(&self, input: &WitnessWordInput) -> u64 { + let bythos_active = (input.input_origin_page.0).0 >= (self.chain_context.bythos_phase.0).0; + Self::witness_words_for_lock( + &input.spend_condition, bythos_active, input.spend_condition_count, + ) + } + + pub fn estimate_v0_witness_words(&self, signatures_required: u64) -> u64 { + // Legacy spend-0 witnesses only carry the signature map. + // 13 words for the schnorr pubkey, + // 16 words for the schnorr signature consisting of 2 8-tuples + Self::map_words(signatures_required, 13, 16) + } + + fn witness_words_for_lock( + spend_condition: &SpendCondition, + bythos_active: bool, + spend_condition_count: Option, + ) -> u64 { + // mirror wallet +estimate-fee structure: + // witness_words = lmp_words + pkh_signature_words + tim_words + hax_words + let lmp_words = + Self::estimate_lmp_words(spend_condition, bythos_active, spend_condition_count); + let pkh_signature_words = Self::estimate_pkh_signature_words(spend_condition); + let tim_words = 1; // witness.tim is null in current create-tx path + let hax_words = 1; // witness.hax is empty map in current create-tx path + lmp_words + .saturating_add(pkh_signature_words) + .saturating_add(tim_words) + .saturating_add(hax_words) + } + + fn estimate_lmp_words( + spend_condition: &SpendCondition, + bythos_active: bool, + spend_condition_count: Option, + ) -> u64 { + // lock-merkle-proof: + // stub: [spend_condition axis proof] + // full: [%full spend_condition axis proof] + // proof: [root path] + // path length ~= log2(number of spend conditions). Use 1 (empty path) when + // lock-level cardinality is unknown. + let version_words: u64 = if bythos_active { 1 } else { 0 }; + let spend_condition_words = Self::estimate_spend_condition_words(spend_condition); + let axis_words = 1; + let proof_words = Self::estimate_merkle_proof_words(spend_condition_count); + version_words + .saturating_add(spend_condition_words) + .saturating_add(axis_words) + .saturating_add(proof_words) + } + + fn estimate_merkle_proof_words(spend_condition_count_hint: Option) -> u64 { + // proof: [root path], where root is one noun-digest (5 atoms) and path is + // a list of noun-digests. For a single spend-condition lock, path is empty. + let count = Self::normalize_spend_condition_count(spend_condition_count_hint.unwrap_or(1)); + let path_len = count.ilog2() as u64; + let path_words = Self::estimate_list_words_len(path_len, 5); + 5 + path_words + } + + fn normalize_spend_condition_count(raw_count: u64) -> u64 { + if raw_count <= 1 { + return 1; + } + if raw_count.is_power_of_two() { + return raw_count; + } + raw_count + .checked_next_power_of_two() + .unwrap_or(1_u64 << (u64::BITS - 1)) + } + + fn estimate_pkh_signature_words(spend_condition: &SpendCondition) -> u64 { + let num_sigs_required = spend_condition + .iter() + .map(|primitive| match primitive { + LockPrimitive::Pkh(pkh) => pkh.m, + _ => 0, + }) + .sum::(); + // from wallet +estimate-fee: + // map-words entries key=hash(5 leaves) value=(pubkey + signature) => 13 + 16 + Self::map_words(num_sigs_required, 5, 13 + 16) + } + + fn estimate_note_data_words(entries: &[RawNoteDataEntry]) -> u64 { + if entries.is_empty() { + return 1; + } + // z-map tree has n+1 null branches. Sum key/value leaves per entry + null branches. + let kv_words = entries + .iter() + .map(|entry| { + let key_words = 1; // @tas key atom + let value_words = Self::estimate_note_data_value_words(entry); + key_words + value_words + }) + .sum::(); + kv_words.saturating_add(entries.len() as u64 + 1) + } + + fn estimate_note_data_value_words(entry: &RawNoteDataEntry) -> u64 { + let decoded = DecodedNoteDataEntry::from_raw_entry(entry); + match decoded.payload { + DecodedNoteDataPayload::Lock(LockDataPayload { + version, + spend_conditions, + }) => { + // [%0 lock] + let version_words: u64 = if version == 0 { 1 } else { 2 }; + version_words.saturating_add(Self::estimate_lock_words(&spend_conditions)) + } + DecodedNoteDataPayload::BridgeDeposit(bridge) => { + // [%0 %base [a b c]] + let network_words = match bridge.network { + crate::note_data::BridgeNetwork::Base => 1, + }; + 1 + network_words + 3 + } + DecodedNoteDataPayload::BridgeWithdrawal(bridge_w) => { + // [%0 beid base-hash lock-root base-batch-end] + let beid_words = + Self::estimate_list_words_len(bridge_w.base_event_id.len() as u64, 1); + 1 + beid_words + 5 + 5 + 1 + } + DecodedNoteDataPayload::Raw => Self::estimate_raw_blob_words(&entry.blob), + } + } + + fn estimate_raw_blob_words(blob: &Bytes) -> u64 { + // Conservative content-size proxy when payload is opaque. + // 8 bytes per word baseline, always at least 1 leaf. + let byte_len = blob.len() as u64; + byte_len.saturating_add(7).saturating_div(8).max(1) + } + + fn estimate_spend_condition_words(spend_condition: &SpendCondition) -> u64 { + let primitive_words = spend_condition + .iter() + .map(Self::estimate_lock_primitive_words) + .sum::(); + // list terminator + primitive_words + 1 + } + + fn estimate_lock_words(spend_conditions: &[SpendCondition]) -> u64 { + if spend_conditions.is_empty() { + return 1; + } + + let spend_condition_words = spend_conditions + .iter() + .map(Self::estimate_spend_condition_words) + .sum::(); + if spend_conditions.len() == 1 { + return spend_condition_words; + } + + let normalized_len = Self::normalize_spend_condition_count(spend_conditions.len() as u64); + let branch_nodes = normalized_len.saturating_sub(1); + // Each branch contributes a lock tag and one branch-pair cell. + spend_condition_words.saturating_add(branch_nodes.saturating_mul(2)) + } + + fn estimate_lock_primitive_words(primitive: &LockPrimitive) -> u64 { + match primitive { + LockPrimitive::Pkh(pkh) => { + // [%pkh [m hashes]] + let hash_set_words = Self::estimate_set_words_len(pkh.hashes.len() as u64, 5); + 1 + 1 + hash_set_words + } + LockPrimitive::Tim(tim) => { + // [%tim [rel abs]], each range: [min max], each bound option: None=>1, Some=>2. + let rel_words = Self::estimate_range_words( + tim.rel.min.as_ref().map(|_| 1_u64), + tim.rel.max.as_ref().map(|_| 1_u64), + ); + let abs_words = Self::estimate_range_words( + tim.abs.min.as_ref().map(|_| 1_u64), + tim.abs.max.as_ref().map(|_| 1_u64), + ); + 1 + rel_words + abs_words + } + LockPrimitive::Hax(hax) => { + // [%hax hashes] + let hash_set_words = Self::estimate_set_words_len(hax.0.len() as u64, 5); + 1 + hash_set_words + } + LockPrimitive::Burn => { + // [%brn ~] + 1 + 1 + } + } + } + + fn estimate_range_words(min: Option, max: Option) -> u64 { + Self::estimate_option_words(min) + Self::estimate_option_words(max) + } + + fn estimate_option_words(payload_words: Option) -> u64 { + match payload_words { + Some(words) => 1 + words, // [~ value] + None => 1, // ~ + } + } + + fn estimate_set_words_len(entries: u64, key_words: u64) -> u64 { + // set has key payload + binary-branch null count. + entries + .saturating_mul(key_words) + .saturating_add(entries.saturating_add(1)) + } + + fn map_words(entries: u64, key_leaves: u64, val_leaves: u64) -> u64 { + let per_node_count = key_leaves.saturating_add(val_leaves); + entries + .saturating_mul(per_node_count) + .saturating_add(entries.saturating_add(1)) + } + + fn estimate_list_words_len(entries: u64, item_words: u64) -> u64 { + entries.saturating_mul(item_words).saturating_add(1) + } +} + +pub fn estimate_seed_words(outputs: &[PlannedOutput], chain_context: &ChainContext) -> u64 { + WordCountEstimator::new(chain_context).estimate_seed_words(outputs) +} + +pub fn estimate_seed_words_legacy(outputs: &[PlannedOutput]) -> u64 { + WordCountEstimator::estimate_seed_words_legacy(outputs) +} + +pub fn estimate_seed_words_merged(outputs: &[PlannedOutput]) -> u64 { + WordCountEstimator::estimate_seed_words_merged(outputs) +} + +pub fn estimate_witness_words(inputs: &[WitnessWordInput], chain_context: &ChainContext) -> u64 { + WordCountEstimator::new(chain_context).estimate_witness_words(inputs) +} + +#[cfg(test)] +mod tests { + use nockapp::noun::slab::{NockJammer, NounSlab}; + use nockapp::utils::NOCK_STACK_SIZE; + use nockchain_math::belt::Belt; + use nockchain_types::tx_engine::common::Hash; + use nockchain_types::tx_engine::v1::note::NoteData; + use nockchain_types::tx_engine::v1::tx::{LockPrimitive, Pkh}; + use nockvm::ext::NounExt; + use nockvm::mem::NockStack; + use nockvm::noun::Noun; + use noun_serde::{NounDecode, NounEncode}; + + use super::*; + use crate::types::RawNoteDataEntry; + + #[derive(Debug, Clone, PartialEq, Eq, NounDecode)] + struct FixtureEntry { + case: String, + note_data: NoteData, + } + + fn hash(v: u64) -> Hash { + Hash::from_limbs(&[v, 0, 0, 0, 0]) + } + + fn jam(value: &T) -> Bytes { + let mut slab: NounSlab = NounSlab::new(); + let noun = value.to_noun(&mut slab); + slab.set_root(noun); + slab.jam() + } + + fn output(lock_root: u64, key: &str, value: u64) -> PlannedOutput { + PlannedOutput { + lock_root: hash(lock_root), + amount: 1, + note_data: vec![RawNoteDataEntry { + key: key.to_string(), + blob: jam(&value), + }], + } + } + + fn decode_note_data_fixtures() -> Vec { + let fixture_bytes = include_bytes!("../tests/fixtures/note_data_fixtures.jam"); + let mut stack = NockStack::new(NOCK_STACK_SIZE, 0); + let noun = Noun::cue_bytes_slice(&mut stack, fixture_bytes).expect("fixture jam must cue"); + let space = stack.noun_space(); + Vec::::from_noun(&noun, &space).expect("fixture noun must decode") + } + + fn normalize_case_tag(tag: &str) -> &str { + tag.strip_prefix('%').unwrap_or(tag) + } + + fn fixture_note_data(case: &str) -> NoteData { + decode_note_data_fixtures() + .into_iter() + .find(|fixture| normalize_case_tag(&fixture.case) == case) + .unwrap_or_else(|| panic!("missing fixture case: {case}")) + .note_data + } + + fn output_from_fixture(case: &str) -> PlannedOutput { + let note_data = fixture_note_data(case); + let note_data = note_data + .iter() + .map(|entry| RawNoteDataEntry { + key: entry.key.clone(), + blob: entry.blob.clone(), + }) + .collect::>(); + PlannedOutput { + lock_root: hash(99), + amount: 1, + note_data, + } + } + + fn output_from_fixture_with_lock_root(case: &str, lock_root: u64) -> PlannedOutput { + let note_data = fixture_note_data(case); + let note_data = note_data + .iter() + .map(|entry| RawNoteDataEntry { + key: entry.key.clone(), + blob: entry.blob.clone(), + }) + .collect::>(); + PlannedOutput { + lock_root: hash(lock_root), + amount: 1, + note_data, + } + } + + #[test] + fn seed_estimate_counts_all_supported_note_data_payload_cases() { + let lock_single = estimate_seed_words_legacy(&[output_from_fixture("lock-single")]); + let lock_v2 = estimate_seed_words_legacy(&[output_from_fixture("lock-v2")]); + let lock_v4 = estimate_seed_words_legacy(&[output_from_fixture("lock-v4")]); + let lock_v8 = estimate_seed_words_legacy(&[output_from_fixture("lock-v8")]); + let lock_v16 = estimate_seed_words_legacy(&[output_from_fixture("lock-v16")]); + let bridge_deposit = estimate_seed_words_legacy(&[output_from_fixture("bridge-deposit")]); + let bridge_deposit_large = + estimate_seed_words_legacy(&[output_from_fixture("bridge-deposit-large")]); + let bridge_withdrawal = + estimate_seed_words_legacy(&[output_from_fixture("bridge-withdrawal")]); + let bridge_withdrawal_long = + estimate_seed_words_legacy(&[output_from_fixture("bridge-withdrawal-long-event")]); + + assert_eq!(lock_single, 7); + assert_eq!(lock_v2, 12); + assert_eq!(lock_v4, 22); + assert_eq!(lock_v8, 42); + assert_eq!(lock_v16, 82); + assert_eq!(bridge_deposit, 8); + assert_eq!(bridge_deposit_large, 8); + assert_eq!(bridge_withdrawal, 20); + assert_eq!(bridge_withdrawal_long, 23); + } + + #[test] + fn seed_estimate_counts_wildcard_note_data_with_raw_blob_proxy() { + let note_data = fixture_note_data("wildcard"); + let raw_entry = note_data.iter().next().expect("wildcard fixture entry"); + let expected_raw_blob_words = (raw_entry.blob.len() as u64).saturating_add(7) / 8; + let expected_value_words = expected_raw_blob_words.max(1); + // one key/value entry: key (1) + value + z-map null branches (2) + let expected_total_words = 1 + expected_value_words + 2; + + let actual = estimate_seed_words_legacy(&[output_from_fixture("wildcard")]); + assert_eq!(actual, expected_total_words); + } + + #[test] + fn malformed_recognized_keys_use_raw_blob_word_proxy() { + for case in [ + "lock-unsupported-version", "bridge-unsupported-network", "bridge-unsupported-version", + "bridge-withdrawal-unsupported-version", + ] { + let note_data = fixture_note_data(case); + let raw_entry = note_data.iter().next().expect("fixture entry"); + let expected_raw_blob_words = (raw_entry.blob.len() as u64).saturating_add(7) / 8; + let expected_value_words = expected_raw_blob_words.max(1); + let expected_total_words = 1 + expected_value_words + 2; + + let actual = estimate_seed_words_legacy(&[output_from_fixture(case)]); + assert_eq!(actual, expected_total_words, "case {case}"); + } + } + + #[test] + fn merged_seed_estimate_overwrites_duplicate_keys_per_lock_root() { + let first = output_from_fixture_with_lock_root("bridge-deposit", 111); + let second = output_from_fixture_with_lock_root("bridge-deposit-large", 111); + let merged = estimate_seed_words_merged(&[first.clone(), second.clone()]); + let expected = estimate_seed_words_legacy(std::slice::from_ref(&second)); + let legacy_total = estimate_seed_words_legacy(&[first, second]); + + assert_eq!(merged, expected); + assert!(merged < legacy_total); + } + + #[test] + fn seed_estimate_switches_to_merged_at_bythos() { + let outputs = vec![output(1, "k", 1), output(1, "k", 2), output(2, "k", 3)]; + let legacy = estimate_seed_words_legacy(&outputs); + let merged = estimate_seed_words_merged(&outputs); + assert!(merged <= legacy); + + let pre = estimate_seed_words( + &outputs, + &ChainContext { + height: BlockHeight(Belt(9)), + bythos_phase: BlockHeight(Belt(10)), + base_fee: 1, + input_fee_divisor: 4, + min_fee: 0, + }, + ); + assert_eq!(pre, legacy); + + let post = estimate_seed_words( + &outputs, + &ChainContext { + height: BlockHeight(Belt(10)), + bythos_phase: BlockHeight(Belt(10)), + base_fee: 1, + input_fee_divisor: 4, + min_fee: 0, + }, + ); + assert_eq!(post, merged); + } + + #[test] + fn witness_estimate_depends_on_required_signatures() { + let sc_one = SpendCondition::new(vec![LockPrimitive::Pkh(Pkh::new(1, vec![hash(7)]))]); + let sc_three = SpendCondition::new(vec![LockPrimitive::Pkh(Pkh::new( + 3, + vec![hash(7), hash(8), hash(9)], + ))]); + + let context = ChainContext { + height: BlockHeight(Belt(11)), + bythos_phase: BlockHeight(Belt(10)), + base_fee: 1, + input_fee_divisor: 4, + min_fee: 0, + }; + let one = estimate_witness_words( + &[WitnessWordInput { + spend_condition: sc_one, + input_origin_page: BlockHeight(Belt(11)), + spend_condition_count: None, + }], + &context, + ); + let three = estimate_witness_words( + &[WitnessWordInput { + spend_condition: sc_three, + input_origin_page: BlockHeight(Belt(11)), + spend_condition_count: None, + }], + &context, + ); + + assert!(three > one); + } + + #[test] + fn witness_estimate_grows_with_merkle_path_hint() { + let spend_condition = + SpendCondition::new(vec![LockPrimitive::Pkh(Pkh::new(1, vec![hash(7)]))]); + let context = ChainContext { + height: BlockHeight(Belt(11)), + bythos_phase: BlockHeight(Belt(10)), + base_fee: 1, + input_fee_divisor: 4, + min_fee: 0, + }; + + let one = estimate_witness_words( + &[WitnessWordInput { + spend_condition: spend_condition.clone(), + input_origin_page: BlockHeight(Belt(11)), + spend_condition_count: Some(1), + }], + &context, + ); + let four = estimate_witness_words( + &[WitnessWordInput { + spend_condition: spend_condition.clone(), + input_origin_page: BlockHeight(Belt(11)), + spend_condition_count: Some(4), + }], + &context, + ); + let eight = estimate_witness_words( + &[WitnessWordInput { + spend_condition, + input_origin_page: BlockHeight(Belt(11)), + spend_condition_count: Some(8), + }], + &context, + ); + + assert!(four > one); + assert!(eight > four); + } + + #[test] + fn witness_estimate_normalizes_non_power_of_two_count_hint() { + let spend_condition = + SpendCondition::new(vec![LockPrimitive::Pkh(Pkh::new(1, vec![hash(1)]))]); + let context = ChainContext { + height: BlockHeight(Belt(11)), + bythos_phase: BlockHeight(Belt(10)), + base_fee: 1, + input_fee_divisor: 4, + min_fee: 0, + }; + + let three = estimate_witness_words( + &[WitnessWordInput { + spend_condition: spend_condition.clone(), + input_origin_page: BlockHeight(Belt(11)), + spend_condition_count: Some(3), + }], + &context, + ); + let four = estimate_witness_words( + &[WitnessWordInput { + spend_condition, + input_origin_page: BlockHeight(Belt(11)), + spend_condition_count: Some(4), + }], + &context, + ); + assert_eq!(three, four); + } + + #[test] + fn v0_witness_estimate_matches_legacy_signature_map_shape() { + let context = ChainContext { + height: BlockHeight(Belt(1)), + bythos_phase: BlockHeight(Belt(1)), + base_fee: 128, + input_fee_divisor: 4, + min_fee: 256, + }; + let estimator = WordCountEstimator::new(&context); + + assert_eq!(estimator.estimate_v0_witness_words(1), 31); + assert_eq!(estimator.estimate_v0_witness_words(2), 61); + assert_eq!(estimator.estimate_v0_witness_words(3), 91); + } +} diff --git a/crates/wallet-tx-builder/tests/fixtures/README.md b/crates/wallet-tx-builder/tests/fixtures/README.md new file mode 100644 index 000000000..b3c7e33b3 --- /dev/null +++ b/crates/wallet-tx-builder/tests/fixtures/README.md @@ -0,0 +1,17 @@ +This directory stores checked-in fixtures used by `wallet-tx-builder` tests. + +Compatibility note +- If `tx_engine` noun formats change without a version bump, regenerate `note_data_fixtures.jam`. +- Otherwise note-data decoding and word-count tests can drift from the real chain encoding. + +Fixture map +- `open/crates/wallet-tx-builder/tests/fixtures/note_data_fixtures.jam` + - generator script: `closed/hoon/scripts/fixtures/v1/generate-note-data-fixtures.hoon` + - payload source: `closed/hoon/tests/wallet/mod/note-data-fixtures.hoon` + - regeneration command: +```bash +cargo run --profile release --bin hoon-closed -- \ + closed/hoon/scripts/fixtures/v1/generate-note-data-fixtures.hoon \ + closed/hoon +cp out.jam open/crates/wallet-tx-builder/tests/fixtures/note_data_fixtures.jam +``` diff --git a/crates/wallet-tx-builder/tests/fixtures/note_data_fixtures.jam b/crates/wallet-tx-builder/tests/fixtures/note_data_fixtures.jam new file mode 100644 index 000000000..af8527b80 Binary files /dev/null and b/crates/wallet-tx-builder/tests/fixtures/note_data_fixtures.jam differ diff --git a/crates/zkvm-jetpack/Cargo.toml b/crates/zkvm-jetpack/Cargo.toml index 25488cec9..be3a6e7b0 100644 --- a/crates/zkvm-jetpack/Cargo.toml +++ b/crates/zkvm-jetpack/Cargo.toml @@ -13,6 +13,7 @@ either.workspace = true hex-literal.workspace = true ibig.workspace = true nockchain-math.workspace = true +nockchain-types.workspace = true nockvm.workspace = true nockvm_macros.workspace = true noun-serde.workspace = true diff --git a/crates/zkvm-jetpack/src/form/math/gen_trace.rs b/crates/zkvm-jetpack/src/form/math/gen_trace.rs index fc9fed406..af3581d8e 100644 --- a/crates/zkvm-jetpack/src/form/math/gen_trace.rs +++ b/crates/zkvm-jetpack/src/form/math/gen_trace.rs @@ -1,7 +1,7 @@ use either::{Left, Right}; use nockvm::interpreter::Context; use nockvm::jets::JetErr; -use nockvm::noun::{Noun, D}; +use nockvm::noun::{Noun, NounSpace, D}; use crate::form::belt::Belt; use crate::form::felt::{fmul_, Felt}; @@ -37,7 +37,7 @@ enum Dyck { Noun(Noun), } -pub fn build_tree_data(noun: Noun, alf: &Felt) -> Result { +pub fn build_tree_data(noun: Noun, alf: &Felt, space: &NounSpace) -> Result { let mut stack: Vec = Vec::::new(); stack.push(Dyck::Noun(noun)); @@ -62,15 +62,16 @@ pub fn build_tree_data(noun: Noun, alf: &Felt) -> Result { } Dyck::Noun(noun) => match noun.as_either_atom_cell() { Right(cell) => { - stack.push(Dyck::Noun(cell.tail())); + let cell = cell.in_space(space); + stack.push(Dyck::Noun(cell.tail().noun())); stack.push(Dyck::One); - stack.push(Dyck::Noun(cell.head())); + stack.push(Dyck::Noun(cell.head().noun())); dyck = fmul_(&dyck, alf); } Left(atom) => { size = fmul_(&size, alf); leaf = fmul_(&leaf, alf); - leaf.0[0] = leaf.0[0] + Belt(atom.as_u64()?); + leaf.0[0] = leaf.0[0] + Belt(atom.in_space(space).as_u64()?); } }, } @@ -84,19 +85,22 @@ pub fn build_tree_data(noun: Noun, alf: &Felt) -> Result { } pub fn leaf_sequence(context: &mut Context, sample: Noun) -> Result { + let space = context.stack.noun_space(); let mut leaf: Vec = Vec::::new(); - do_leaf_sequence(sample, &mut leaf)?; + do_leaf_sequence(sample, &mut leaf, &space)?; Ok(vec_to_hoon_list(&mut context.stack, &leaf)) } -fn do_leaf_sequence(noun: Noun, vec: &mut Vec) -> Result<(), JetErr> { +fn do_leaf_sequence(noun: Noun, vec: &mut Vec, space: &NounSpace) -> Result<(), JetErr> { if noun.is_atom() { - vec.push(noun.as_atom()?.as_u64()?); + vec.push(noun.in_space(space).as_atom()?.as_u64()?); Ok(()) } else { - let cell = noun.as_cell()?; - do_leaf_sequence(cell.head(), vec)?; - do_leaf_sequence(cell.tail(), vec)?; + let cell = noun.in_space(space).as_cell()?; + let head = cell.head().noun(); + let tail = cell.tail().noun(); + do_leaf_sequence(head, vec, space)?; + do_leaf_sequence(tail, vec, space)?; Ok(()) } } diff --git a/crates/zkvm-jetpack/src/form/math/prover.rs b/crates/zkvm-jetpack/src/form/math/prover.rs index 444f6248c..a5390bb3c 100644 --- a/crates/zkvm-jetpack/src/form/math/prover.rs +++ b/crates/zkvm-jetpack/src/form/math/prover.rs @@ -1,3 +1,5 @@ +use nockvm::noun::NounSpace; + use crate::form::belt::*; use crate::form::bpoly::{bp_fft, bpoly_zero_extend}; use crate::form::felt::{fpow, Felt}; @@ -34,11 +36,12 @@ pub fn compute_deep( omicrons: &[Felt], deep_challenge: &Felt, comp_eval_point: &Felt, + space: &NounSpace, ) -> Vec { let composition_pieces = composition_pieces .into_iter() .map(|x| { - FPolySlice::try_from(x) + FPolySlice::try_from(x, space) .unwrap_or_else(|err| { panic!( "Panicked with {err:?} at {}:{} (git sha: {:?})", @@ -56,7 +59,7 @@ pub fn compute_deep( let mut fps: Vec<(Vec>, &Felt)> = vec![]; for (trace_poly, omicron) in trace_polys.into_iter().zip(omicrons.iter()) { - let Ok(trace_poly) = MarySlice::try_from(trace_poly) else { + let Ok(trace_poly) = MarySlice::try_from(trace_poly, space) else { panic!("trace_poly in trace_polys is not a valid FPolySlice"); }; diff --git a/crates/zkvm-jetpack/src/hot.rs b/crates/zkvm-jetpack/src/hot.rs index f3eaf764a..7506d0fa4 100644 --- a/crates/zkvm-jetpack/src/hot.rs +++ b/crates/zkvm-jetpack/src/hot.rs @@ -1364,6 +1364,753 @@ pub const ZOON_JETS: &[HotEntry] = &[ 1, mor_tip_jet, ), + ( + &[ + K_138, + Left(b"one"), + Left(b"two"), + Left(b"tri"), + Left(b"qua"), + Left(b"pen"), + Left(b"zeke"), + Left(b"ext-field"), + Left(b"misc-lib"), + Left(b"proof-lib"), + Left(b"utils"), + Left(b"fri"), + Left(b"table-lib"), + Left(b"stark-core"), + Left(b"fock-core"), + Left(b"pow"), + Left(b"stark-engine"), + Left(b"h-zoon"), + Left(b"gor-hip"), + ], + 1, + gor_hip_jet, + ), + ( + &[ + K_138, + Left(b"one"), + Left(b"two"), + Left(b"tri"), + Left(b"qua"), + Left(b"pen"), + Left(b"zeke"), + Left(b"ext-field"), + Left(b"misc-lib"), + Left(b"proof-lib"), + Left(b"utils"), + Left(b"fri"), + Left(b"table-lib"), + Left(b"stark-core"), + Left(b"fock-core"), + Left(b"pow"), + Left(b"stark-engine"), + Left(b"h-zoon"), + Left(b"mor-hip"), + ], + 1, + mor_hip_jet, + ), + ( + &[ + K_138, + Left(b"one"), + Left(b"two"), + Left(b"tri"), + Left(b"qua"), + Left(b"pen"), + Left(b"zeke"), + Left(b"ext-field"), + Left(b"misc-lib"), + Left(b"proof-lib"), + Left(b"utils"), + Left(b"fri"), + Left(b"table-lib"), + Left(b"stark-core"), + Left(b"fock-core"), + Left(b"pow"), + Left(b"stark-engine"), + Left(b"h-zoon"), + Left(b"zh-molt"), + ], + 1, + zh_molt_jet, + ), + ( + &[ + K_138, + Left(b"one"), + Left(b"two"), + Left(b"tri"), + Left(b"qua"), + Left(b"pen"), + Left(b"zeke"), + Left(b"ext-field"), + Left(b"misc-lib"), + Left(b"proof-lib"), + Left(b"utils"), + Left(b"fri"), + Left(b"table-lib"), + Left(b"stark-core"), + Left(b"fock-core"), + Left(b"pow"), + Left(b"stark-engine"), + Left(b"h-zoon"), + Left(b"zh-silt"), + ], + 1, + zh_silt_jet, + ), + ( + &[ + K_138, + Left(b"one"), + Left(b"two"), + Left(b"tri"), + Left(b"qua"), + Left(b"pen"), + Left(b"zeke"), + Left(b"ext-field"), + Left(b"misc-lib"), + Left(b"proof-lib"), + Left(b"utils"), + Left(b"fri"), + Left(b"table-lib"), + Left(b"stark-core"), + Left(b"fock-core"), + Left(b"pow"), + Left(b"stark-engine"), + Left(b"h-zoon"), + Left(b"zh-milt"), + ], + 1, + zh_milt_jet, + ), + ( + &[ + K_138, + Left(b"one"), + Left(b"two"), + Left(b"tri"), + Left(b"qua"), + Left(b"pen"), + Left(b"zeke"), + Left(b"ext-field"), + Left(b"misc-lib"), + Left(b"proof-lib"), + Left(b"utils"), + Left(b"fri"), + Left(b"table-lib"), + Left(b"stark-core"), + Left(b"fock-core"), + Left(b"pow"), + Left(b"stark-engine"), + Left(b"h-zoon"), + Left(b"zh-balmilt"), + ], + 1, + zh_balance_milt_jet, + ), + ( + &[ + K_138, + Left(b"one"), + Left(b"two"), + Left(b"tri"), + Left(b"qua"), + Left(b"pen"), + Left(b"zeke"), + Left(b"ext-field"), + Left(b"misc-lib"), + Left(b"proof-lib"), + Left(b"utils"), + Left(b"fri"), + Left(b"table-lib"), + Left(b"stark-core"), + Left(b"fock-core"), + Left(b"pow"), + Left(b"stark-engine"), + Left(b"h-zoon"), + Left(b"zh-jult"), + ], + 1, + zh_jult_jet, + ), + ( + &[ + K_138, + Left(b"one"), + Left(b"two"), + Left(b"tri"), + Left(b"qua"), + Left(b"pen"), + Left(b"zeke"), + Left(b"ext-field"), + Left(b"misc-lib"), + Left(b"proof-lib"), + Left(b"utils"), + Left(b"fri"), + Left(b"table-lib"), + Left(b"stark-core"), + Left(b"fock-core"), + Left(b"pow"), + Left(b"stark-engine"), + Left(b"h-zoon"), + Left(b"h-by"), + Left(b"get"), + ], + 1, + h_by_get_jet, + ), + ( + &[ + K_138, + Left(b"one"), + Left(b"two"), + Left(b"tri"), + Left(b"qua"), + Left(b"pen"), + Left(b"zeke"), + Left(b"ext-field"), + Left(b"misc-lib"), + Left(b"proof-lib"), + Left(b"utils"), + Left(b"fri"), + Left(b"table-lib"), + Left(b"stark-core"), + Left(b"fock-core"), + Left(b"pow"), + Left(b"stark-engine"), + Left(b"h-zoon"), + Left(b"h-by"), + Left(b"got"), + ], + 1, + h_by_got_jet, + ), + ( + &[ + K_138, + Left(b"one"), + Left(b"two"), + Left(b"tri"), + Left(b"qua"), + Left(b"pen"), + Left(b"zeke"), + Left(b"ext-field"), + Left(b"misc-lib"), + Left(b"proof-lib"), + Left(b"utils"), + Left(b"fri"), + Left(b"table-lib"), + Left(b"stark-core"), + Left(b"fock-core"), + Left(b"pow"), + Left(b"stark-engine"), + Left(b"h-zoon"), + Left(b"h-by"), + Left(b"gut"), + ], + 1, + h_by_gut_jet, + ), + ( + &[ + K_138, + Left(b"one"), + Left(b"two"), + Left(b"tri"), + Left(b"qua"), + Left(b"pen"), + Left(b"zeke"), + Left(b"ext-field"), + Left(b"misc-lib"), + Left(b"proof-lib"), + Left(b"utils"), + Left(b"fri"), + Left(b"table-lib"), + Left(b"stark-core"), + Left(b"fock-core"), + Left(b"pow"), + Left(b"stark-engine"), + Left(b"h-zoon"), + Left(b"h-by"), + Left(b"has"), + ], + 1, + h_by_has_jet, + ), + ( + &[ + K_138, + Left(b"one"), + Left(b"two"), + Left(b"tri"), + Left(b"qua"), + Left(b"pen"), + Left(b"zeke"), + Left(b"ext-field"), + Left(b"misc-lib"), + Left(b"proof-lib"), + Left(b"utils"), + Left(b"fri"), + Left(b"table-lib"), + Left(b"stark-core"), + Left(b"fock-core"), + Left(b"pow"), + Left(b"stark-engine"), + Left(b"h-zoon"), + Left(b"h-by"), + Left(b"put"), + ], + 1, + h_by_put_jet, + ), + ( + &[ + K_138, + Left(b"one"), + Left(b"two"), + Left(b"tri"), + Left(b"qua"), + Left(b"pen"), + Left(b"zeke"), + Left(b"ext-field"), + Left(b"misc-lib"), + Left(b"proof-lib"), + Left(b"utils"), + Left(b"fri"), + Left(b"table-lib"), + Left(b"stark-core"), + Left(b"fock-core"), + Left(b"pow"), + Left(b"stark-engine"), + Left(b"h-zoon"), + Left(b"h-by"), + Left(b"del"), + ], + 1, + h_by_del_jet, + ), + ( + &[ + K_138, + Left(b"one"), + Left(b"two"), + Left(b"tri"), + Left(b"qua"), + Left(b"pen"), + Left(b"zeke"), + Left(b"ext-field"), + Left(b"misc-lib"), + Left(b"proof-lib"), + Left(b"utils"), + Left(b"fri"), + Left(b"table-lib"), + Left(b"stark-core"), + Left(b"fock-core"), + Left(b"pow"), + Left(b"stark-engine"), + Left(b"h-zoon"), + Left(b"h-by"), + Left(b"mar"), + ], + 1, + h_by_mar_jet, + ), + ( + &[ + K_138, + Left(b"one"), + Left(b"two"), + Left(b"tri"), + Left(b"qua"), + Left(b"pen"), + Left(b"zeke"), + Left(b"ext-field"), + Left(b"misc-lib"), + Left(b"proof-lib"), + Left(b"utils"), + Left(b"fri"), + Left(b"table-lib"), + Left(b"stark-core"), + Left(b"fock-core"), + Left(b"pow"), + Left(b"stark-engine"), + Left(b"h-zoon"), + Left(b"h-by"), + Left(b"gas"), + ], + 1, + h_by_gas_jet, + ), + ( + &[ + K_138, + Left(b"one"), + Left(b"two"), + Left(b"tri"), + Left(b"qua"), + Left(b"pen"), + Left(b"zeke"), + Left(b"ext-field"), + Left(b"misc-lib"), + Left(b"proof-lib"), + Left(b"utils"), + Left(b"fri"), + Left(b"table-lib"), + Left(b"stark-core"), + Left(b"fock-core"), + Left(b"pow"), + Left(b"stark-engine"), + Left(b"h-zoon"), + Left(b"h-by"), + Left(b"uni"), + ], + 1, + h_by_uni_jet, + ), + ( + &[ + K_138, + Left(b"one"), + Left(b"two"), + Left(b"tri"), + Left(b"qua"), + Left(b"pen"), + Left(b"zeke"), + Left(b"ext-field"), + Left(b"misc-lib"), + Left(b"proof-lib"), + Left(b"utils"), + Left(b"fri"), + Left(b"table-lib"), + Left(b"stark-core"), + Left(b"fock-core"), + Left(b"pow"), + Left(b"stark-engine"), + Left(b"h-zoon"), + Left(b"h-by"), + Left(b"int"), + ], + 1, + h_by_int_jet, + ), + ( + &[ + K_138, + Left(b"one"), + Left(b"two"), + Left(b"tri"), + Left(b"qua"), + Left(b"pen"), + Left(b"zeke"), + Left(b"ext-field"), + Left(b"misc-lib"), + Left(b"proof-lib"), + Left(b"utils"), + Left(b"fri"), + Left(b"table-lib"), + Left(b"stark-core"), + Left(b"fock-core"), + Left(b"pow"), + Left(b"stark-engine"), + Left(b"h-zoon"), + Left(b"h-by"), + Left(b"dif"), + ], + 1, + h_by_dif_jet, + ), + ( + &[ + K_138, + Left(b"one"), + Left(b"two"), + Left(b"tri"), + Left(b"qua"), + Left(b"pen"), + Left(b"zeke"), + Left(b"ext-field"), + Left(b"misc-lib"), + Left(b"proof-lib"), + Left(b"utils"), + Left(b"fri"), + Left(b"table-lib"), + Left(b"stark-core"), + Left(b"fock-core"), + Left(b"pow"), + Left(b"stark-engine"), + Left(b"h-zoon"), + Left(b"h-by"), + Left(b"bif"), + ], + 1, + h_by_bif_jet, + ), + ( + &[ + K_138, + Left(b"one"), + Left(b"two"), + Left(b"tri"), + Left(b"qua"), + Left(b"pen"), + Left(b"zeke"), + Left(b"ext-field"), + Left(b"misc-lib"), + Left(b"proof-lib"), + Left(b"utils"), + Left(b"fri"), + Left(b"table-lib"), + Left(b"stark-core"), + Left(b"fock-core"), + Left(b"pow"), + Left(b"stark-engine"), + Left(b"h-zoon"), + Left(b"h-by"), + Left(b"dig"), + ], + 1, + h_by_dig_jet, + ), + ( + &[ + K_138, + Left(b"one"), + Left(b"two"), + Left(b"tri"), + Left(b"qua"), + Left(b"pen"), + Left(b"zeke"), + Left(b"ext-field"), + Left(b"misc-lib"), + Left(b"proof-lib"), + Left(b"utils"), + Left(b"fri"), + Left(b"table-lib"), + Left(b"stark-core"), + Left(b"fock-core"), + Left(b"pow"), + Left(b"stark-engine"), + Left(b"h-zoon"), + Left(b"h-in"), + Left(b"has"), + ], + 1, + h_in_has_jet, + ), + ( + &[ + K_138, + Left(b"one"), + Left(b"two"), + Left(b"tri"), + Left(b"qua"), + Left(b"pen"), + Left(b"zeke"), + Left(b"ext-field"), + Left(b"misc-lib"), + Left(b"proof-lib"), + Left(b"utils"), + Left(b"fri"), + Left(b"table-lib"), + Left(b"stark-core"), + Left(b"fock-core"), + Left(b"pow"), + Left(b"stark-engine"), + Left(b"h-zoon"), + Left(b"h-in"), + Left(b"put"), + ], + 1, + h_in_put_jet, + ), + ( + &[ + K_138, + Left(b"one"), + Left(b"two"), + Left(b"tri"), + Left(b"qua"), + Left(b"pen"), + Left(b"zeke"), + Left(b"ext-field"), + Left(b"misc-lib"), + Left(b"proof-lib"), + Left(b"utils"), + Left(b"fri"), + Left(b"table-lib"), + Left(b"stark-core"), + Left(b"fock-core"), + Left(b"pow"), + Left(b"stark-engine"), + Left(b"h-zoon"), + Left(b"h-in"), + Left(b"del"), + ], + 1, + h_in_del_jet, + ), + ( + &[ + K_138, + Left(b"one"), + Left(b"two"), + Left(b"tri"), + Left(b"qua"), + Left(b"pen"), + Left(b"zeke"), + Left(b"ext-field"), + Left(b"misc-lib"), + Left(b"proof-lib"), + Left(b"utils"), + Left(b"fri"), + Left(b"table-lib"), + Left(b"stark-core"), + Left(b"fock-core"), + Left(b"pow"), + Left(b"stark-engine"), + Left(b"h-zoon"), + Left(b"h-in"), + Left(b"gas"), + ], + 1, + h_in_gas_jet, + ), + ( + &[ + K_138, + Left(b"one"), + Left(b"two"), + Left(b"tri"), + Left(b"qua"), + Left(b"pen"), + Left(b"zeke"), + Left(b"ext-field"), + Left(b"misc-lib"), + Left(b"proof-lib"), + Left(b"utils"), + Left(b"fri"), + Left(b"table-lib"), + Left(b"stark-core"), + Left(b"fock-core"), + Left(b"pow"), + Left(b"stark-engine"), + Left(b"h-zoon"), + Left(b"h-in"), + Left(b"uni"), + ], + 1, + h_in_uni_jet, + ), + ( + &[ + K_138, + Left(b"one"), + Left(b"two"), + Left(b"tri"), + Left(b"qua"), + Left(b"pen"), + Left(b"zeke"), + Left(b"ext-field"), + Left(b"misc-lib"), + Left(b"proof-lib"), + Left(b"utils"), + Left(b"fri"), + Left(b"table-lib"), + Left(b"stark-core"), + Left(b"fock-core"), + Left(b"pow"), + Left(b"stark-engine"), + Left(b"h-zoon"), + Left(b"h-in"), + Left(b"int"), + ], + 1, + h_in_int_jet, + ), + ( + &[ + K_138, + Left(b"one"), + Left(b"two"), + Left(b"tri"), + Left(b"qua"), + Left(b"pen"), + Left(b"zeke"), + Left(b"ext-field"), + Left(b"misc-lib"), + Left(b"proof-lib"), + Left(b"utils"), + Left(b"fri"), + Left(b"table-lib"), + Left(b"stark-core"), + Left(b"fock-core"), + Left(b"pow"), + Left(b"stark-engine"), + Left(b"h-zoon"), + Left(b"h-in"), + Left(b"dif"), + ], + 1, + h_in_dif_jet, + ), + ( + &[ + K_138, + Left(b"one"), + Left(b"two"), + Left(b"tri"), + Left(b"qua"), + Left(b"pen"), + Left(b"zeke"), + Left(b"ext-field"), + Left(b"misc-lib"), + Left(b"proof-lib"), + Left(b"utils"), + Left(b"fri"), + Left(b"table-lib"), + Left(b"stark-core"), + Left(b"fock-core"), + Left(b"pow"), + Left(b"stark-engine"), + Left(b"h-zoon"), + Left(b"h-in"), + Left(b"bif"), + ], + 1, + h_in_bif_jet, + ), + ( + &[ + K_138, + Left(b"one"), + Left(b"two"), + Left(b"tri"), + Left(b"qua"), + Left(b"pen"), + Left(b"zeke"), + Left(b"ext-field"), + Left(b"misc-lib"), + Left(b"proof-lib"), + Left(b"utils"), + Left(b"fri"), + Left(b"table-lib"), + Left(b"stark-core"), + Left(b"fock-core"), + Left(b"pow"), + Left(b"stark-engine"), + Left(b"h-zoon"), + Left(b"h-in"), + Left(b"dig"), + ], + 1, + h_in_dig_jet, + ), ]; pub const CURVE_JETS: &[HotEntry] = &[ diff --git a/crates/zkvm-jetpack/src/jets/base_jets.rs b/crates/zkvm-jetpack/src/jets/base_jets.rs index 6f4c193da..6c56fe59f 100644 --- a/crates/zkvm-jetpack/src/jets/base_jets.rs +++ b/crates/zkvm-jetpack/src/jets/base_jets.rs @@ -4,7 +4,7 @@ use nockvm::jets::bits::util::rip; use nockvm::jets::util::{bite, slot, BAIL_FAIL}; use nockvm::jets::Result; use nockvm::mem::NockStack; -use nockvm::noun::{Atom, Noun, D, NO, T, YES}; +use nockvm::noun::{Atom, Noun, NounSpace, D, NO, T, YES}; use tracing::debug; // base field jets @@ -21,129 +21,155 @@ use tracing::debug; // inefficient, it makes division of labor clear. pub fn badd_jet(context: &mut Context, subject: Noun) -> Result { - let sam = slot(subject, 6)?; - let a = slot(sam, 2)?; - let b = slot(sam, 3)?; + let space = context.stack.noun_space(); + let sam = slot(subject, 6, &space)?; + let a = slot(sam, 2, &space)?; + let b = slot(sam, 3, &space)?; let (Ok(a_atom), Ok(b_atom)) = (a.as_atom(), b.as_atom()) else { debug!("a or b was not an atom"); return Err(BAIL_FAIL); }; - let (a_belt, b_belt): (Belt, Belt) = (a_atom.as_u64()?.into(), b_atom.as_u64()?.into()); + let (a_belt, b_belt): (Belt, Belt) = ( + a_atom.in_space(&space).as_u64()?.into(), + b_atom.in_space(&space).as_u64()?.into(), + ); Ok(Atom::new(&mut context.stack, (a_belt + b_belt).into()).as_noun()) } pub fn bsub_jet(context: &mut Context, subject: Noun) -> Result { - let sam = slot(subject, 6)?; - let a = slot(sam, 2)?; - let b = slot(sam, 3)?; + let space = context.stack.noun_space(); + let sam = slot(subject, 6, &space)?; + let a = slot(sam, 2, &space)?; + let b = slot(sam, 3, &space)?; let (Ok(a_atom), Ok(b_atom)) = (a.as_atom(), b.as_atom()) else { debug!("a or b was not an atom"); return Err(BAIL_FAIL); }; - let (a_belt, b_belt): (Belt, Belt) = (a_atom.as_u64()?.into(), b_atom.as_u64()?.into()); + let (a_belt, b_belt): (Belt, Belt) = ( + a_atom.in_space(&space).as_u64()?.into(), + b_atom.in_space(&space).as_u64()?.into(), + ); Ok(Atom::new(&mut context.stack, (a_belt - b_belt).into()).as_noun()) } pub fn bneg_jet(context: &mut Context, subject: Noun) -> Result { - let a = slot(subject, 6)?; + let space = context.stack.noun_space(); + let a = slot(subject, 6, &space)?; let Ok(a_atom) = a.as_atom() else { debug!("a was not an atom"); return Err(BAIL_FAIL); }; - let a_belt: Belt = a_atom.as_u64()?.into(); + let a_belt: Belt = a_atom.in_space(&space).as_u64()?.into(); Ok(Atom::new(&mut context.stack, (-a_belt).into()).as_noun()) } pub fn bmul_jet(context: &mut Context, subject: Noun) -> Result { - let sam = slot(subject, 6)?; - let a = slot(sam, 2)?; - let b = slot(sam, 3)?; + let space = context.stack.noun_space(); + let sam = slot(subject, 6, &space)?; + let a = slot(sam, 2, &space)?; + let b = slot(sam, 3, &space)?; let (Ok(a_atom), Ok(b_atom)) = (a.as_atom(), b.as_atom()) else { debug!("a or b was not an atom"); return Err(BAIL_FAIL); }; - let (a_belt, b_belt): (Belt, Belt) = (a_atom.as_u64()?.into(), b_atom.as_u64()?.into()); + let (a_belt, b_belt): (Belt, Belt) = ( + a_atom.in_space(&space).as_u64()?.into(), + b_atom.in_space(&space).as_u64()?.into(), + ); Ok(Atom::new(&mut context.stack, (a_belt * b_belt).into()).as_noun()) } pub fn ordered_root_jet(context: &mut Context, subject: Noun) -> Result { - let n = slot(subject, 6)?; + let space = context.stack.noun_space(); + let n = slot(subject, 6, &space)?; let Ok(n_atom) = n.as_atom() else { debug!("n was not an atom"); return Err(BAIL_FAIL); }; - let n_u64 = Belt(n_atom.as_u64()?); + let n_u64 = Belt(n_atom.in_space(&space).as_u64()?); // TODO: clean this up let res_atom = Atom::new(&mut context.stack, n_u64.ordered_root()?.into()); Ok(res_atom.as_noun()) } pub fn bpow_jet(context: &mut Context, subject: Noun) -> Result { - let sam = slot(subject, 6)?; - let x = slot(sam, 2)?; - let n = slot(sam, 3)?; + let space = context.stack.noun_space(); + let sam = slot(subject, 6, &space)?; + let x = slot(sam, 2, &space)?; + let n = slot(sam, 3, &space)?; let (Ok(x_atom), Ok(n_atom)) = (x.as_atom(), n.as_atom()) else { debug!("x or n was not an atom"); return Err(BAIL_FAIL); }; - let (x_belt, n_belt) = (x_atom.as_u64()?, n_atom.as_u64()?); + let (x_belt, n_belt) = ( + x_atom.in_space(&space).as_u64()?, + n_atom.in_space(&space).as_u64()?, + ); Ok(Atom::new(&mut context.stack, bpow(x_belt, n_belt)).as_noun()) } pub fn rip_correct_jet(context: &mut Context, subject: Noun) -> Result { + let space = context.stack.noun_space(); let stack = &mut context.stack; - let sam = slot(subject, 6)?; - let a_noun = slot(sam, 2)?; - let b_noun = slot(sam, 3)?; + let sam = slot(subject, 6, &space)?; + let a_noun = slot(sam, 2, &space)?; + let b_noun = slot(sam, 3, &space)?; - let b = b_noun.as_atom()?; - let (bloq, step) = bite(a_noun)?; - rip_correct(stack, bloq, step, b) + let b = b_noun.in_space(&space).as_atom()?.atom(); + let (bloq, step) = bite(a_noun, &space)?; + rip_correct(stack, bloq, step, b, &space) } -pub fn rip_correct(stack: &mut NockStack, bloq: usize, step: usize, b: Atom) -> Result { - if b.is_direct() && b.as_u64()? == 0 { +pub fn rip_correct( + stack: &mut NockStack, + bloq: usize, + step: usize, + b: Atom, + space: &NounSpace, +) -> Result { + if b.is_direct() && b.in_space(space).as_u64()? == 0 { return Ok(T(stack, &[D(0), D(0)])); } - rip(stack, bloq, step, b) + rip(stack, bloq, step, b, space) } -pub fn levy_based(a_noun: Noun) -> bool { +pub fn levy_based(a_noun: Noun, space: &NounSpace) -> bool { let mut list = a_noun; loop { if unsafe { list.raw_equals(&D(0)) } { return true; } - let cell = list.as_cell().expect("cell not found"); - let based_res = based(cell.head()); + let cell = list.in_space(space).as_cell().expect("cell not found"); + let based_res = based(cell.head().noun(), space); if !based_res { return false; } - list = cell.tail(); + list = cell.tail().noun(); } } pub fn based_jet(_context: &mut Context, subject: Noun) -> Result { - let sam = slot(subject, 6)?; - if based(sam) { + let space = _context.stack.noun_space(); + let sam = slot(subject, 6, &space)?; + if based(sam, &space) { Ok(YES) } else { Ok(NO) } } -fn based(a_noun: Noun) -> bool { - let Ok(a_atom) = a_noun.as_atom() else { +fn based(a_noun: Noun, space: &NounSpace) -> bool { + let Ok(a_atom) = a_noun.in_space(space).as_atom() else { return false; // no atom }; let Ok(a_u64) = a_atom.as_u64() else { @@ -154,26 +180,28 @@ fn based(a_noun: Noun) -> bool { } pub fn based_noun_jet(_context: &mut Context, subject: Noun) -> Result { - let n = slot(subject, 6)?; - if based_noun(n) { + let space = _context.stack.noun_space(); + let n = slot(subject, 6, &space)?; + if based_noun(n, &space) { Ok(YES) } else { Ok(NO) } } -pub fn based_noun(n: Noun) -> bool { +pub fn based_noun(n: Noun, space: &NounSpace) -> bool { if n.is_atom() { - return based(n); + return based(n, space); } let n_cell = n + .in_space(space) .as_cell() .expect("n should be a cell since it's not an atom"); - let res1 = based_noun(n_cell.head()); + let res1 = based_noun(n_cell.head().noun(), space); if !res1 { return false; } - based_noun(n_cell.tail()) + based_noun(n_cell.tail().noun(), space) } diff --git a/crates/zkvm-jetpack/src/jets/bp_jets.rs b/crates/zkvm-jetpack/src/jets/bp_jets.rs index 515aa67bf..faa46abf6 100644 --- a/crates/zkvm-jetpack/src/jets/bp_jets.rs +++ b/crates/zkvm-jetpack/src/jets/bp_jets.rs @@ -3,7 +3,7 @@ use nockvm::jets::list::util::flop; use nockvm::jets::util::{slot, BAIL_FAIL}; use nockvm::jets::{JetErr, Result}; use nockvm::mem::NockStack; -use nockvm::noun::{Atom, Cell, IndirectAtom, Noun, D, NO, T, YES}; +use nockvm::noun::{Atom, Cell, IndirectAtom, Noun, NounSpace, D, NO, T, YES}; use tracing::debug; use crate::form::belt::*; @@ -18,13 +18,14 @@ use crate::jets::mary_jets::{mary_to_list_fields, snag_one_fields}; use crate::utils::is_hoon_list_end; pub fn bpoly_to_list_jet(context: &mut Context, subject: Noun) -> Result { + let space = context.stack.noun_space(); let stack = &mut context.stack; - let sam = slot(subject, 6)?; - bpoly_to_list(stack, sam) + let sam = slot(subject, 6, &space)?; + bpoly_to_list(stack, sam, &space) } -pub fn bpoly_to_list(stack: &mut NockStack, sam: Noun) -> Result { - let Ok(sam_bpoly) = BPolySlice::try_from(sam) else { +pub fn bpoly_to_list(stack: &mut NockStack, sam: Noun, space: &NounSpace) -> Result { + let Ok(sam_bpoly) = BPolySlice::try_from(sam, space) else { return Err(BAIL_FAIL); }; @@ -46,11 +47,15 @@ pub fn bpoly_to_list(stack: &mut NockStack, sam: Noun) -> Result { } pub fn bpadd_jet(context: &mut Context, subject: Noun) -> Result { - let sam = slot(subject, 6)?; - let bp = slot(sam, 2)?; - let bq = slot(sam, 3)?; - - let (Ok(bp_poly), Ok(bq_poly)) = (BPolySlice::try_from(bp), BPolySlice::try_from(bq)) else { + let space = context.stack.noun_space(); + let sam = slot(subject, 6, &space)?; + let bp = slot(sam, 2, &space)?; + let bq = slot(sam, 3, &space)?; + + let (Ok(bp_poly), Ok(bq_poly)) = ( + BPolySlice::try_from(bp, &space), + BPolySlice::try_from(bq, &space), + ) else { return Err(BAIL_FAIL); }; @@ -65,9 +70,10 @@ pub fn bpadd_jet(context: &mut Context, subject: Noun) -> Result { } pub fn bpneg_jet(context: &mut Context, subject: Noun) -> Result { - let bp = slot(subject, 6)?; + let space = context.stack.noun_space(); + let bp = slot(subject, 6, &space)?; - let Ok(bp_poly) = BPolySlice::try_from(bp) else { + let Ok(bp_poly) = BPolySlice::try_from(bp, &space) else { return Err(BAIL_FAIL); }; @@ -81,11 +87,15 @@ pub fn bpneg_jet(context: &mut Context, subject: Noun) -> Result { } pub fn bpsub_jet(context: &mut Context, subject: Noun) -> Result { - let sam = slot(subject, 6)?; - let p = slot(sam, 2)?; - let q = slot(sam, 3)?; - - let (Ok(p_poly), Ok(q_poly)) = (BPolySlice::try_from(p), BPolySlice::try_from(q)) else { + let space = context.stack.noun_space(); + let sam = slot(subject, 6, &space)?; + let p = slot(sam, 2, &space)?; + let q = slot(sam, 3, &space)?; + + let (Ok(p_poly), Ok(q_poly)) = ( + BPolySlice::try_from(p, &space), + BPolySlice::try_from(q, &space), + ) else { return Err(BAIL_FAIL); }; @@ -100,13 +110,14 @@ pub fn bpsub_jet(context: &mut Context, subject: Noun) -> Result { } pub fn bpscal_jet(context: &mut Context, subject: Noun) -> Result { - let sam = slot(subject, 6)?; - let c = slot(sam, 2)?; - let bp = slot(sam, 3)?; - let (Ok(c_atom), Ok(bp_poly)) = (c.as_atom(), BPolySlice::try_from(bp)) else { + let space = context.stack.noun_space(); + let sam = slot(subject, 6, &space)?; + let c = slot(sam, 2, &space)?; + let bp = slot(sam, 3, &space)?; + let (Ok(c_atom), Ok(bp_poly)) = (c.as_atom(), BPolySlice::try_from(bp, &space)) else { return Err(BAIL_FAIL); }; - let c_64 = c_atom.as_u64()?; + let c_64 = c_atom.in_space(&space).as_u64()?; let (res, res_poly): (IndirectAtom, &mut [Belt]) = new_handle_mut_slice(&mut context.stack, Some(bp_poly.len())); @@ -118,11 +129,15 @@ pub fn bpscal_jet(context: &mut Context, subject: Noun) -> Result { } pub fn bpmul_jet(context: &mut Context, subject: Noun) -> Result { - let sam = slot(subject, 6)?; - let bp = slot(sam, 2)?; - let bq = slot(sam, 3)?; - - let (Ok(bp_poly), Ok(bq_poly)) = (BPolySlice::try_from(bp), BPolySlice::try_from(bq)) else { + let space = context.stack.noun_space(); + let sam = slot(subject, 6, &space)?; + let bp = slot(sam, 2, &space)?; + let bq = slot(sam, 3, &space)?; + + let (Ok(bp_poly), Ok(bq_poly)) = ( + BPolySlice::try_from(bp, &space), + BPolySlice::try_from(bq, &space), + ) else { return Err(BAIL_FAIL); }; @@ -142,11 +157,15 @@ pub fn bpmul_jet(context: &mut Context, subject: Noun) -> Result { } pub fn bp_hadamard_jet(context: &mut Context, subject: Noun) -> Result { - let sam = slot(subject, 6)?; - let bp = slot(sam, 2)?; - let bq = slot(sam, 3)?; - - let (Ok(bp_poly), Ok(bq_poly)) = (BPolySlice::try_from(bp), BPolySlice::try_from(bq)) else { + let space = context.stack.noun_space(); + let sam = slot(subject, 6, &space)?; + let bp = slot(sam, 2, &space)?; + let bq = slot(sam, 3, &space)?; + + let (Ok(bp_poly), Ok(bq_poly)) = ( + BPolySlice::try_from(bp, &space), + BPolySlice::try_from(bq, &space), + ) else { return Err(BAIL_FAIL); }; assert_eq!(bp_poly.len(), bq_poly.len()); @@ -161,14 +180,15 @@ pub fn bp_hadamard_jet(context: &mut Context, subject: Noun) -> Result { } pub fn bp_ntt_jet(context: &mut Context, subject: Noun) -> Result { - let sam = slot(subject, 6)?; - let bp = slot(sam, 2)?; - let root = slot(sam, 3)?; + let space = context.stack.noun_space(); + let sam = slot(subject, 6, &space)?; + let bp = slot(sam, 2, &space)?; + let root = slot(sam, 3, &space)?; - let (Ok(bp_poly), Ok(root_atom)) = (BPolySlice::try_from(bp), root.as_atom()) else { + let (Ok(bp_poly), Ok(root_atom)) = (BPolySlice::try_from(bp, &space), root.as_atom()) else { return Err(BAIL_FAIL); }; - let root_64 = root_atom.as_u64()?; + let root_64 = root_atom.in_space(&space).as_u64()?; let returned_bpoly = bp_ntt(bp_poly.0, &Belt(root_64)); // TODO: preallocate and pass res buffer into bp_ntt? let (res_atom, res_poly): (IndirectAtom, &mut [Belt]) = @@ -181,9 +201,10 @@ pub fn bp_ntt_jet(context: &mut Context, subject: Noun) -> Result { } pub fn bp_fft_jet(context: &mut Context, subject: Noun) -> Result { - let p = slot(subject, 6)?; + let space = context.stack.noun_space(); + let p = slot(subject, 6, &space)?; - let Ok(p_poly) = BPolySlice::try_from(p) else { + let Ok(p_poly) = BPolySlice::try_from(p, &space) else { return Err(BAIL_FAIL); }; let returned_bpoly = bp_fft(p_poly.0)?; @@ -198,11 +219,12 @@ pub fn bp_fft_jet(context: &mut Context, subject: Noun) -> Result { } pub fn bp_shift_jet(context: &mut Context, subject: Noun) -> Result { - let sam = slot(subject, 6)?; - let bp = slot(sam, 2)?; - let c = slot(sam, 3)?; + let space = context.stack.noun_space(); + let sam = slot(subject, 6, &space)?; + let bp = slot(sam, 2, &space)?; + let c = slot(sam, 3, &space)?; - let (Ok(bp_poly), Ok(c_belt)) = (BPolySlice::try_from(bp), c.as_belt()) else { + let (Ok(bp_poly), Ok(c_belt)) = (BPolySlice::try_from(bp, &space), c.as_belt(&space)) else { return Err(BAIL_FAIL); }; let (res_atom, res_poly): (IndirectAtom, &mut [Belt]) = @@ -215,14 +237,17 @@ pub fn bp_shift_jet(context: &mut Context, subject: Noun) -> Result { } pub fn bp_coseword_jet(context: &mut Context, subject: Noun) -> Result { - let sam = slot(subject, 6)?; - let p = slot(sam, 2)?; - let offset = slot(sam, 6)?; - let order = slot(sam, 7)?; - - let (Ok(p_poly), Ok(offset_belt), Ok(order_atom)) = - (BPolySlice::try_from(p), offset.as_belt(), order.as_atom()) - else { + let space = context.stack.noun_space(); + let sam = slot(subject, 6, &space)?; + let p = slot(sam, 2, &space)?; + let offset = slot(sam, 6, &space)?; + let order = slot(sam, 7, &space)?; + + let (Ok(p_poly), Ok(offset_belt), Ok(order_atom)) = ( + BPolySlice::try_from(p, &space), + offset.as_belt(&space), + order.as_atom(), + ) else { return Err(BAIL_FAIL); }; let order_32: u32 = order_atom.as_u32()?; @@ -237,21 +262,22 @@ pub fn bp_coseword_jet(context: &mut Context, subject: Noun) -> Result { } pub fn init_bpoly_jet(context: &mut Context, subject: Noun) -> Result { + let space = context.stack.noun_space(); let stack = &mut context.stack; - let poly = slot(subject, 6)?; + let poly = slot(subject, 6, &space)?; - let list_belt = HoonList::try_from(poly)?.into_iter(); + let list_belt = HoonList::try_from(poly, &space)?.into_iter(); let count = list_belt.count(); let (res, res_poly): (IndirectAtom, &mut [Belt]) = new_handle_mut_slice(stack, Some(count)); - init_bpoly(list_belt, res_poly); + init_bpoly(list_belt, res_poly, &space); let res_cell = finalize_poly(stack, Some(res_poly.len()), res); Ok(res_cell) } -pub fn init_bpoly(list_belt: HoonList, res_poly: &mut [Belt]) { +pub fn init_bpoly(list_belt: HoonList<'_>, res_poly: &mut [Belt], space: &NounSpace) { for (i, belt_noun) in list_belt.enumerate() { - let belt = belt_noun.as_belt().expect("error at as_belt"); + let belt = belt_noun.as_belt(space).expect("error at as_belt"); res_poly[i] = belt; } } @@ -260,17 +286,18 @@ pub fn init_bpoly(list_belt: HoonList, res_poly: &mut [Belt]) { // pub fn bp_is_zero_jet(_context: &mut Context, subject: Noun) -> Result { - let p = slot(subject, 6)?; + let space = _context.stack.noun_space(); + let p = slot(subject, 6, &space)?; - if bp_is_zero(p) { + if bp_is_zero(p, &space) { Ok(YES) } else { Ok(NO) } } -pub fn bp_is_zero(p: Noun) -> bool { - let p_slice = BPolySlice::try_from(p).expect("invalid p"); +pub fn bp_is_zero(p: Noun, space: &NounSpace) -> bool { + let p_slice = BPolySlice::try_from(p, space).expect("invalid p"); p_slice.is_zero() } @@ -279,60 +306,68 @@ fn lift(belt: Belt) -> Felt { felt_from_u64s(belt.0, 0, 0) } -pub fn get_bpoly_fields(bpoly: Noun) -> std::result::Result<(Atom, Atom), JetErr> { - let [bpoly_len, bpoly_dat] = bpoly.uncell()?; // +$ bpoly [len=@ dat=@ux] +pub fn get_bpoly_fields( + bpoly: Noun, + space: &NounSpace, +) -> std::result::Result<(Atom, Atom), JetErr> { + let [bpoly_len, bpoly_dat] = bpoly.uncell(space)?; // +$ bpoly [len=@ dat=@ux] Ok((bpoly_len.as_atom()?, bpoly_dat.as_atom()?)) } // bpeval-lift: evaluate a bpoly at a felt pub fn bpeval_lift_jet(context: &mut Context, subject: Noun) -> Result { + let space = context.stack.noun_space(); let stack = &mut context.stack; - let sam = slot(subject, 6)?; - let [bp, x_noun] = sam.uncell()?; // TODO defaults? [bp=`bpoly`one-bpoly x=`felt`(lift 1)] - let x = x_noun.as_felt()?; + let sam = slot(subject, 6, &space)?; + let [bp, x_noun] = sam.uncell(&space)?; // TODO defaults? [bp=`bpoly`one-bpoly x=`felt`(lift 1)] + let x = x_noun.as_felt(&space)?; let lift0 = lift(Belt(0)); - if bp_is_zero(bp) { + if bp_is_zero(bp, &space) { return felt_as_noun(context, lift0); } - let (bp_len, bp_dat) = get_bpoly_fields(bp)?; + let (bp_len, bp_dat) = get_bpoly_fields(bp, &space)?; - if bp_len.as_u64()? == 1 { - return snag_one_fields(stack, 0, 1, bp_dat); + if bp_len.in_space(&space).as_u64()? == 1 { + return snag_one_fields(stack, 0, 1, bp_dat, &space); } - let p = mary_to_list_fields(stack, bp_len, bp_dat.as_noun(), 1)?; - let mut p = flop(stack, p)?; + let p = mary_to_list_fields(stack, bp_len, bp_dat.as_noun(), 1, &space)?; + let mut p = flop(stack, p, &space)?; let mut res = lift0; loop { if is_hoon_list_end(&p) { return Err(BAIL_FAIL); } - let p_cell = p.as_cell()?; - let res_lift = lift(p_cell.head().as_belt()?); + let p_cell = p.in_space(&space).as_cell()?; + let res_lift = lift(p_cell.head().noun().as_belt(&space)?); let mut res_fmul = Felt::zero(); fmul(&res, x, &mut res_fmul); let mut res_add = Felt::zero(); fadd(&res_fmul, &res_lift, &mut res_add); - if is_hoon_list_end(&p_cell.tail()) { + if is_hoon_list_end(&p_cell.tail().noun()) { return felt_as_noun(context, res_add); } res = res_add; - p = p_cell.tail(); + p = p_cell.tail().noun(); } } pub fn bpdvr_jet(context: &mut Context, subject: Noun) -> Result { - let sam = slot(subject, 6)?; - let ba = slot(sam, 2)?; - let bb = slot(sam, 3)?; - - let (Ok(ba_poly), Ok(bb_poly)) = (BPolySlice::try_from(ba), BPolySlice::try_from(bb)) else { + let space = context.stack.noun_space(); + let sam = slot(subject, 6, &space)?; + let ba = slot(sam, 2, &space)?; + let bb = slot(sam, 3, &space)?; + + let (Ok(ba_poly), Ok(bb_poly)) = ( + BPolySlice::try_from(ba, &space), + BPolySlice::try_from(bb, &space), + ) else { debug!("ba or bb was not a bpoly"); return Err(BAIL_FAIL); }; diff --git a/crates/zkvm-jetpack/src/jets/cheetah_jets.rs b/crates/zkvm-jetpack/src/jets/cheetah_jets.rs index 0792c06e4..04ff9ca0f 100644 --- a/crates/zkvm-jetpack/src/jets/cheetah_jets.rs +++ b/crates/zkvm-jetpack/src/jets/cheetah_jets.rs @@ -1,9 +1,8 @@ use ibig::UBig; -use nockvm::ext::NounExt; use nockvm::interpreter::Context; -use nockvm::jets::util::BAIL_FAIL; +use nockvm::jets::util::{slot, BAIL_FAIL}; use nockvm::jets::JetErr; -use nockvm::noun::{Noun, Slots}; +use nockvm::noun::Noun; use noun_serde::{NounDecode, NounEncode}; use rayon::iter::{IntoParallelRefIterator, ParallelIterator}; @@ -13,11 +12,12 @@ use crate::form::tip5; #[inline(always)] pub fn ch_scal_jet(context: &mut Context, subject: Noun) -> Result { - let sam = subject.slot(6)?; - let n_atom = sam.slot(2)?.as_atom()?; + let space = context.stack.noun_space(); + let sam = slot(subject, 6, &space)?; + let n_atom = slot(sam, 2, &space)?.in_space(&space).as_atom()?; - let p = sam.slot(3)?; - let a_pt = CheetahPoint::from_noun(&p).map_err(|_| BAIL_FAIL)?; + let p = slot(sam, 3, &space)?; + let a_pt = CheetahPoint::from_noun(&p, &space).map_err(|_| BAIL_FAIL)?; let res = if let Ok(n) = n_atom.as_u64() { ch_scal(n, &a_pt)? @@ -32,16 +32,23 @@ pub fn ch_scal_jet(context: &mut Context, subject: Noun) -> Result } pub fn verify_affine_jet(context: &mut Context, subject: Noun) -> Result { - let sam = subject.slot(6)?; - let pubkey = sam.slot(2)?; - let m = sam.slot(6)?; - let chal = sam.slot(14)?.as_atom()?.as_ubig(&mut context.stack); - let sig = sam.slot(15)?.as_atom()?.as_ubig(&mut context.stack); - - let pubkey: CheetahPoint = CheetahPoint::from_noun(&pubkey).map_err(|_| BAIL_FAIL)?; - let m = <[Belt; 5]>::from_noun(&m).map_err(|_| BAIL_FAIL)?; - - let res = verify_affine(pubkey, &m, &chal, &sig)?; + let space = context.stack.noun_space(); + let sam = slot(subject, 6, &space)?; + let pubkey = slot(sam, 2, &space)?; + let m = slot(sam, 6, &space)?; + let chal = slot(sam, 14, &space)? + .in_space(&space) + .as_atom()? + .as_ubig(&mut context.stack); + let sig = slot(sam, 15, &space)? + .in_space(&space) + .as_atom()? + .as_ubig(&mut context.stack); + + let pubkey: CheetahPoint = CheetahPoint::from_noun(&pubkey, &space).map_err(|_| BAIL_FAIL)?; + let m = <[Belt; 5]>::from_noun(&m, &space).map_err(|_| BAIL_FAIL)?; + + let res = verify_affine(&pubkey, &m, &chal, &sig)?; Ok(res.to_noun(&mut context.stack)) } @@ -70,12 +77,15 @@ pub(crate) struct ValidateArgs { //} pub fn batch_verify_affine_jet(context: &mut Context, subject: Noun) -> Result { - let list = subject.slot(6)?; + let space = context.stack.noun_space(); + let list = slot(subject, 6, &space)?; let args = list + .in_space(&space) .list_iter() .map(|arg| { - let pubkey = CheetahPoint::from_noun(&arg.slot(2)?).map_err(|_| BAIL_FAIL)?; - let m = <[Belt; 5]>::from_noun(&arg.slot(6)?).map_err(|_| BAIL_FAIL)?; + let pubkey = + CheetahPoint::from_noun(&arg.slot(2)?.noun(), &space).map_err(|_| BAIL_FAIL)?; + let m = <[Belt; 5]>::from_noun(&arg.slot(6)?.noun(), &space).map_err(|_| BAIL_FAIL)?; let chal = arg.slot(14)?.as_atom()?.as_ubig(&mut context.stack); let sig = arg.slot(15)?.as_atom()?.as_ubig(&mut context.stack); Ok(ValidateArgs { @@ -96,7 +106,7 @@ pub fn batch_verify_affine_jet(context: &mut Context, subject: Noun) -> Result Result Result { let left = ch_scal_big(sig, &A_GEN)?; - let right = ch_neg(&ch_scal_big(chal, &pubkey)?); + let right = ch_neg(&ch_scal_big(chal, pubkey)?); let sum = ch_add(&left, &right)?; if sum.x == F6_ZERO { return Err(BAIL_FAIL); @@ -379,7 +389,7 @@ mod tests { }; let m = [Belt(1), Belt(2), Belt(3), Belt(4), Belt(5)]; - assert!(verify_affine(pubkey, &m, &chal, &sig)?); + assert!(verify_affine(&pubkey, &m, &chal, &sig)?); Ok(()) } @@ -414,7 +424,7 @@ mod tests { inf: false, }; let m = [Belt(8), Belt(9), Belt(10), Belt(11), Belt(12)]; - assert!(verify_affine(pubkey, &m, &chal, &sig)?); + assert!(verify_affine(&pubkey, &m, &chal, &sig)?); Ok(()) } diff --git a/crates/zkvm-jetpack/src/jets/compute_table_jets_v2.rs b/crates/zkvm-jetpack/src/jets/compute_table_jets_v2.rs index 69a98def0..24746abd8 100644 --- a/crates/zkvm-jetpack/src/jets/compute_table_jets_v2.rs +++ b/crates/zkvm-jetpack/src/jets/compute_table_jets_v2.rs @@ -1,7 +1,7 @@ use nockvm::interpreter::Context; use nockvm::jets::util::{slot, BAIL_EXIT, BAIL_FAIL}; use nockvm::jets::JetErr; -use nockvm::noun::{Atom, IndirectAtom, Noun, D, T}; +use nockvm::noun::{Atom, IndirectAtom, Noun, NounSpace, D, T}; use nockvm_macros::tas; use tracing::debug; @@ -14,18 +14,19 @@ use crate::form::structs::HoonList; use crate::jets::table_utils::*; pub fn compute_v2_mega_extend_jet(context: &mut Context, subject: Noun) -> Result { - let sam = slot(subject, 6)?; - let table_mary = slot(sam, 2)?; - let all_chals = slot(sam, 6)?; - let _fock_ret = slot(sam, 7)?; + let space = context.stack.noun_space(); + let sam = slot(subject, 6, &space)?; + let table_mary = slot(sam, 2, &space)?; + let all_chals = slot(sam, 6, &space)?; + let _fock_ret = slot(sam, 7, &space)?; - let chals: MegaExtChals = init_mega_ext_chals(all_chals)?; + let chals: MegaExtChals = init_mega_ext_chals(all_chals, &space)?; let z2: Felt = fmul_(&chals.z, &chals.z); let z3: Felt = fmul_(&z2, &chals.z); let z_inv: Felt = finv_(&chals.z); - let table_noun = slot(table_mary, 3)?; - let Ok(table) = MarySlice::try_from(table_noun) else { + let table_noun = slot(table_mary, 3, &space)?; + let Ok(table) = MarySlice::try_from(table_noun, &space) else { debug!("cannot convert mary arg to mary"); return Err(BAIL_FAIL); }; @@ -497,16 +498,17 @@ fn compress_ion(ion: &Ion, a: &Felt, b: &Felt, c: &Felt) -> Felt { } pub fn compute_v2_extend_jet(context: &mut Context, subject: Noun) -> Result { - let sam = slot(subject, 6)?; - let table_mary = slot(sam, 2)?; - let chals_rd1 = slot(sam, 6)?; - let fock_ret = slot(sam, 7)?; - let queue = slot(fock_ret, 2)?; + let space = context.stack.noun_space(); + let sam = slot(subject, 6, &space)?; + let table_mary = slot(sam, 2, &space)?; + let chals_rd1 = slot(sam, 6, &space)?; + let fock_ret = slot(sam, 7, &space)?; + let queue = slot(fock_ret, 2, &space)?; - let chals: ExtChals = init_ext_chals(chals_rd1)?; + let chals: ExtChals = init_ext_chals(chals_rd1, &space)?; - let table_noun = slot(table_mary, 3)?; - let Ok(table) = MarySlice::try_from(table_noun) else { + let table_noun = slot(table_mary, 3, &space)?; + let Ok(table) = MarySlice::try_from(table_noun, &space) else { debug!("cannot convert mary arg to mary"); return Err(BAIL_FAIL); }; @@ -515,7 +517,7 @@ pub fn compute_v2_extend_jet(context: &mut Context, subject: Noun) -> Result = build_compute_queue(queue, &chals.alf)?; + let stack: Vec = build_compute_queue(queue, &chals.alf, &space)?; let mut stack_idx: usize = 0; let mut row_idx: usize = 0; @@ -527,26 +529,30 @@ pub fn compute_v2_extend_jet(context: &mut Context, subject: Noun) -> Result Result, JetErr> { +fn build_compute_queue(list: Noun, alf: &Felt, space: &NounSpace) -> Result, JetErr> { let mut res: Vec = Vec::::new(); - for n in HoonList::try_from(list)?.into_iter() { - let tree_data = build_tree_data(n, alf)?; + for n in HoonList::try_from(list, space)?.into_iter() { + let tree_data = build_tree_data(n, alf, space)?; res.push(tree_data) } Ok(res) diff --git a/crates/zkvm-jetpack/src/jets/crypto_jets.rs b/crates/zkvm-jetpack/src/jets/crypto_jets.rs index 5474b279d..87faf43a1 100644 --- a/crates/zkvm-jetpack/src/jets/crypto_jets.rs +++ b/crates/zkvm-jetpack/src/jets/crypto_jets.rs @@ -10,15 +10,16 @@ use nockvm::noun::{Atom, Noun}; use crate::form::crypto::argon2::{argon2_hook, Argon2Args}; pub fn argon2_jet(context: &mut Context, subject: Noun) -> Result { - let parent_core = slot(subject, 7)?; - let params = slot(parent_core, 6)?; + let space = context.stack.noun_space(); + let parent_core = slot(subject, 7, &space)?; + let params = slot(parent_core, 6, &space)?; // prepare parameters - let args = Argon2Args::from_noun(&mut context.stack, ¶ms)?; + let args = Argon2Args::from_noun(&mut context.stack, ¶ms, &space)?; - let sam = slot(subject, 6)?; - let msg: Byts = Byts::from_noun(&mut context.stack, &slot(sam, 2)?)?; - let sat: Byts = Byts::from_noun(&mut context.stack, &slot(sam, 3)?)?; + let sam = slot(subject, 6, &space)?; + let msg: Byts = Byts::from_noun(&mut context.stack, &slot(sam, 2, &space)?, &space)?; + let sat: Byts = Byts::from_noun(&mut context.stack, &slot(sam, 3, &space)?, &space)?; let mut res = vec![0; args.out]; argon2_hook(args, &msg.0, &sat.0, &mut res).map_err(|_| BAIL_EXIT)?; @@ -113,14 +114,16 @@ pub mod test { let salt = salt_byts.clone().into_noun(&mut context.stack); let inner = T(&mut context.stack, &[password, salt]); - let args = Argon2Args::from_noun(&mut context.stack, ¶ms).unwrap_or_else(|err| { - panic!( - "Panicked with {err:?} at {}:{} (git sha: {:?})", - file!(), - line!(), - option_env!("GIT_SHA") - ) - }); + let space = context.stack.noun_space(); + let args = + Argon2Args::from_noun(&mut context.stack, ¶ms, &space).unwrap_or_else(|err| { + panic!( + "Panicked with {err:?} at {}:{} (git sha: {:?})", + file!(), + line!(), + option_env!("GIT_SHA") + ) + }); let expected_tag = &mut vec![0u8; args.out]; argon2_hook(args, &password_byts.0, &salt_byts.0, expected_tag).unwrap_or_else(|err| { diff --git a/crates/zkvm-jetpack/src/jets/fext_jets.rs b/crates/zkvm-jetpack/src/jets/fext_jets.rs index a0e4255db..e99b6c973 100644 --- a/crates/zkvm-jetpack/src/jets/fext_jets.rs +++ b/crates/zkvm-jetpack/src/jets/fext_jets.rs @@ -11,21 +11,22 @@ use crate::form::noun_ext::NounMathExt; use crate::utils::*; pub fn zip_roll_jet(context: &mut Context, subject: Noun) -> Result { - let sample = slot(subject, 6)?; - let mut list_a = slot(sample, 2)?; - let mut list_b = slot(sample, 6)?; - let mut gate = slot(sample, 7)?; - let mut prod = slot(gate, 13)?; + let space = context.stack.noun_space(); + let sample = slot(subject, 6, &space)?; + let mut list_a = slot(sample, 2, &space)?; + let mut list_b = slot(sample, 6, &space)?; + let mut gate = slot(sample, 7, &space)?; + let mut prod = slot(gate, 13, &space)?; let site = Site::new(context, &mut gate); loop { - if let Ok(list_a_cell) = list_a.as_cell() { - if let Ok(list_b_cell) = list_b.as_cell() { - list_a = list_a_cell.tail(); - list_b = list_b_cell.tail(); + if let Ok(list_a_cell) = list_a.in_space(&space).as_cell() { + if let Ok(list_b_cell) = list_b.in_space(&space).as_cell() { + list_a = list_a_cell.tail().noun(); + list_b = list_b_cell.tail().noun(); let left_sam = T( &mut context.stack, - &[list_a_cell.head(), list_b_cell.head()], + &[list_a_cell.head().noun(), list_b_cell.head().noun()], ); let sam = T(&mut context.stack, &[left_sam, prod]); prod = site_slam(context, &site, sam)?; @@ -46,103 +47,110 @@ pub fn zip_roll_jet(context: &mut Context, subject: Noun) -> Result Result { - let sam = slot(subject, 6)?; - let a = slot(sam, 2)?; - let b = slot(sam, 3)?; + let space = context.stack.noun_space(); + let sam = slot(subject, 6, &space)?; + let a = slot(sam, 2, &space)?; + let b = slot(sam, 3, &space)?; - let (Ok(a_felt), Ok(b_felt)) = (a.as_felt(), b.as_felt()) else { + let (Ok(a_felt), Ok(b_felt)) = (a.as_felt(&space), b.as_felt(&space)) else { debug!("a or b not a felt"); return Err(BAIL_FAIL); }; let (res_atom, res_felt): (IndirectAtom, &mut Felt) = new_handle_mut_felt(&mut context.stack); fadd(a_felt, b_felt, res_felt); - assert!(felt_atom_is_valid(res_atom)); + assert!(felt_atom_is_valid(res_atom, &space)); Ok(res_atom.as_noun()) } pub fn fsub_jet(context: &mut Context, subject: Noun) -> Result { - let sam = slot(subject, 6)?; - let a = slot(sam, 2)?; - let b = slot(sam, 3)?; + let space = context.stack.noun_space(); + let sam = slot(subject, 6, &space)?; + let a = slot(sam, 2, &space)?; + let b = slot(sam, 3, &space)?; - let (Ok(a_felt), Ok(b_felt)) = (a.as_felt(), b.as_felt()) else { + let (Ok(a_felt), Ok(b_felt)) = (a.as_felt(&space), b.as_felt(&space)) else { debug!("a or b not a felt"); return Err(BAIL_FAIL); }; let (res_atom, res_felt): (IndirectAtom, &mut Felt) = new_handle_mut_felt(&mut context.stack); fsub(a_felt, b_felt, res_felt); - assert!(felt_atom_is_valid(res_atom)); + assert!(felt_atom_is_valid(res_atom, &space)); Ok(res_atom.as_noun()) } pub fn fneg_jet(context: &mut Context, subject: Noun) -> Result { - let a = slot(subject, 6)?; + let space = context.stack.noun_space(); + let a = slot(subject, 6, &space)?; - let Ok(a_felt) = a.as_felt() else { + let Ok(a_felt) = a.as_felt(&space) else { debug!("a not a felt"); return Err(BAIL_FAIL); }; let (res_atom, res_felt): (IndirectAtom, &mut Felt) = new_handle_mut_felt(&mut context.stack); fneg(a_felt, res_felt); - assert!(felt_atom_is_valid(res_atom)); + assert!(felt_atom_is_valid(res_atom, &space)); Ok(res_atom.as_noun()) } pub fn fmul_jet(context: &mut Context, subject: Noun) -> Result { - let sam = slot(subject, 6)?; - let a = slot(sam, 2)?; - let b = slot(sam, 3)?; + let space = context.stack.noun_space(); + let sam = slot(subject, 6, &space)?; + let a = slot(sam, 2, &space)?; + let b = slot(sam, 3, &space)?; - let (Ok(a_felt), Ok(b_felt)) = (a.as_felt(), b.as_felt()) else { + let (Ok(a_felt), Ok(b_felt)) = (a.as_felt(&space), b.as_felt(&space)) else { debug!("a or b not a felt"); return Err(BAIL_FAIL); }; let (res_atom, res_felt): (IndirectAtom, &mut Felt) = new_handle_mut_felt(&mut context.stack); fmul(a_felt, b_felt, res_felt); - assert!(felt_atom_is_valid(res_atom)); + assert!(felt_atom_is_valid(res_atom, &space)); Ok(res_atom.as_noun()) } pub fn finv_jet(context: &mut Context, subject: Noun) -> Result { - let a = slot(subject, 6)?; + let space = context.stack.noun_space(); + let a = slot(subject, 6, &space)?; - let Ok(a_felt) = a.as_felt() else { + let Ok(a_felt) = a.as_felt(&space) else { debug!("a is not a felt"); return Err(BAIL_FAIL); }; let (res_atom, res_felt): (IndirectAtom, &mut Felt) = new_handle_mut_felt(&mut context.stack); finv(a_felt, res_felt); - assert!(felt_atom_is_valid(res_atom)); + assert!(felt_atom_is_valid(res_atom, &space)); Ok(res_atom.as_noun()) } pub fn fdiv_jet(context: &mut Context, subject: Noun) -> Result { - let sam = slot(subject, 6)?; - let a = slot(sam, 2)?; - let b = slot(sam, 3)?; + let space = context.stack.noun_space(); + let sam = slot(subject, 6, &space)?; + let a = slot(sam, 2, &space)?; + let b = slot(sam, 3, &space)?; - let (Ok(a_felt), Ok(b_felt)) = (a.as_felt(), b.as_felt()) else { + let (Ok(a_felt), Ok(b_felt)) = (a.as_felt(&space), b.as_felt(&space)) else { debug!("a or b not felts"); return Err(BAIL_FAIL); }; let (res_atom, res_felt): (IndirectAtom, &mut Felt) = new_handle_mut_felt(&mut context.stack); fdiv(a_felt, b_felt, res_felt); - assert!(felt_atom_is_valid(res_atom)); + assert!(felt_atom_is_valid(res_atom, &space)); Ok(res_atom.as_noun()) } pub fn fpow_jet(context: &mut Context, subject: Noun) -> Result { - let sam = slot(subject, 6)?; - let x = slot(sam, 2)?; - let n = slot(sam, 3)?; + let space = context.stack.noun_space(); + let sam = slot(subject, 6, &space)?; + let x = slot(sam, 2, &space)?; + let n = slot(sam, 3, &space)?; - let (Ok(x_felt), Ok(n_atom)) = (x.as_felt(), n.as_atom()) else { + let (Ok(x_felt), Ok(n_atom)) = (x.as_felt(&space), n.in_space(&space).as_atom()) else { debug!("x not a felt or n not an atom"); return Err(BAIL_FAIL); }; @@ -150,6 +158,6 @@ pub fn fpow_jet(context: &mut Context, subject: Noun) -> Result { let (res_atom, res_felt): (IndirectAtom, &mut Felt) = new_handle_mut_felt(&mut context.stack); fpow(x_felt, n_64, res_felt); - assert!(felt_atom_is_valid(res_atom)); + assert!(felt_atom_is_valid(res_atom, &space)); Ok(res_atom.as_noun()) } diff --git a/crates/zkvm-jetpack/src/jets/fp_jets.rs b/crates/zkvm-jetpack/src/jets/fp_jets.rs index 6519dd5e1..02fddf99a 100644 --- a/crates/zkvm-jetpack/src/jets/fp_jets.rs +++ b/crates/zkvm-jetpack/src/jets/fp_jets.rs @@ -12,14 +12,17 @@ use crate::form::poly::FPolySlice; use crate::form::structs::HoonList; pub fn fp_coseword_jet(context: &mut Context, subject: Noun) -> Result { - let sam = slot(subject, 6)?; - let p = slot(sam, 2)?; - let offset = slot(sam, 6)?; - let order = slot(sam, 7)?; - - let (Ok(p_poly), Ok(offset_felt), Ok(order_atom)) = - (FPolySlice::try_from(p), offset.as_felt(), order.as_atom()) - else { + let space = context.stack.noun_space(); + let sam = slot(subject, 6, &space)?; + let p = slot(sam, 2, &space)?; + let offset = slot(sam, 6, &space)?; + let order = slot(sam, 7, &space)?; + + let (Ok(p_poly), Ok(offset_felt), Ok(order_atom)) = ( + FPolySlice::try_from(p, &space), + offset.as_felt(&space), + order.as_atom(), + ) else { debug!("p not an fpoly, offset not a felt, or order not an atom"); return Err(BAIL_FAIL); }; @@ -36,15 +39,16 @@ pub fn fp_coseword_jet(context: &mut Context, subject: Noun) -> Result { } pub fn init_fpoly_jet(context: &mut Context, subject: Noun) -> Result { - let poly = slot(subject, 6)?; + let space = context.stack.noun_space(); + let poly = slot(subject, 6, &space)?; - let list_felt = HoonList::try_from(poly)?.into_iter(); + let list_felt = HoonList::try_from(poly, &space)?.into_iter(); let count = list_felt.count(); let (res, res_poly): (IndirectAtom, &mut [Felt]) = new_handle_mut_slice(&mut context.stack, Some(count)); for (i, felt_noun) in list_felt.enumerate() { - let Ok(felt) = felt_noun.as_felt() else { + let Ok(felt) = felt_noun.as_felt(&space) else { debug!("list element not a felt"); return Err(BAIL_FAIL); }; @@ -56,10 +60,11 @@ pub fn init_fpoly_jet(context: &mut Context, subject: Noun) -> Result { Ok(res_cell) } pub fn fpeval_jet(context: &mut Context, subject: Noun) -> Result { - let sam = slot(subject, 6)?; - let fp = slot(sam, 2)?; - let felt = slot(sam, 3)?; - let (Ok(fp_poly), Ok(felt)) = (FPolySlice::try_from(fp), felt.as_felt()) else { + let space = context.stack.noun_space(); + let sam = slot(subject, 6, &space)?; + let fp = slot(sam, 2, &space)?; + let felt = slot(sam, 3, &space)?; + let (Ok(fp_poly), Ok(felt)) = (FPolySlice::try_from(fp, &space), felt.as_felt(&space)) else { debug!("fp or fq not an fpoly"); return Err(BAIL_FAIL); }; @@ -71,9 +76,10 @@ pub fn fpeval_jet(context: &mut Context, subject: Noun) -> Result { } pub fn lift_to_fpoly_jet(context: &mut Context, subject: Noun) -> Result { - let belt = slot(subject, 6)?; + let space = context.stack.noun_space(); + let belt = slot(subject, 6, &space)?; - let Ok(belts) = HoonList::try_from(belt) else { + let Ok(belts) = HoonList::try_from(belt, &space) else { debug!("belts not a list"); return Err(BAIL_FAIL); }; @@ -83,7 +89,7 @@ pub fn lift_to_fpoly_jet(context: &mut Context, subject: Noun) -> Result { let (res, res_poly): (IndirectAtom, &mut [Felt]) = new_handle_mut_slice(&mut context.stack, Some(count)); - lift_to_fpoly(belts, res_poly); + lift_to_fpoly(belts, res_poly, &space); let res_cell = finalize_poly(&mut context.stack, Some(res_poly.len()), res); @@ -91,7 +97,8 @@ pub fn lift_to_fpoly_jet(context: &mut Context, subject: Noun) -> Result { } pub fn range_jet(context: &mut Context, subject: Noun) -> Result { - let sample = slot(subject, 6)?; + let space = context.stack.noun_space(); + let sample = slot(subject, 6, &space)?; let mut res = D(0); let mut dest: *mut Noun = &mut res; @@ -99,13 +106,13 @@ pub fn range_jet(context: &mut Context, subject: Noun) -> Result { let start: u64; let end: u64; - if let Ok(atom) = sample.as_atom() { + if let Ok(atom) = sample.in_space(&space).as_atom() { start = 0; - end = atom.as_direct()?.data(); + end = atom.atom().as_direct()?.data(); } else { - let cell = sample.as_cell()?; - start = cell.head().as_atom()?.as_direct()?.data(); - end = cell.tail().as_atom()?.as_direct()?.data(); + let cell = sample.in_space(&space).as_cell()?; + start = cell.head().as_atom()?.atom().as_direct()?.data(); + end = cell.tail().as_atom()?.atom().as_direct()?.data(); } for idx in start..end { diff --git a/crates/zkvm-jetpack/src/jets/fpntt_jets.rs b/crates/zkvm-jetpack/src/jets/fpntt_jets.rs index 191656b6a..cd673357d 100644 --- a/crates/zkvm-jetpack/src/jets/fpntt_jets.rs +++ b/crates/zkvm-jetpack/src/jets/fpntt_jets.rs @@ -54,17 +54,22 @@ pub fn felt_as_noun(context: &mut Context, felt: Felt) -> Result { // frep_jet pub fn frep_jet(context: &mut Context, subject: Noun) -> Result { - let sample = slot(subject, 6)?; - let x = hoon_list_to_vecbelt(sample)?; + let space = context.stack.noun_space(); + let sample = slot(subject, 6, &space)?; + let x = hoon_list_to_vecbelt(sample, &space)?; let felt = frep(x)?; felt_as_noun(context, felt) } pub fn fp_ntt_jet(context: &mut Context, subject: Noun) -> Result { - let sample = slot(subject, 6)?; - let [fp_noun, root_noun] = sample.uncell()?; - - let (Ok(fp), Ok(root)) = (FPolySlice::try_from(fp_noun), root_noun.as_felt()) else { + let space = context.stack.noun_space(); + let sample = slot(subject, 6, &space)?; + let [fp_noun, root_noun] = sample.uncell(&space)?; + + let (Ok(fp), Ok(root)) = ( + FPolySlice::try_from(fp_noun, &space), + root_noun.as_felt(&space), + ) else { return Err(BAIL_FAIL); }; diff --git a/crates/zkvm-jetpack/src/jets/mary_jets.rs b/crates/zkvm-jetpack/src/jets/mary_jets.rs index 4b9150640..11829ed32 100644 --- a/crates/zkvm-jetpack/src/jets/mary_jets.rs +++ b/crates/zkvm-jetpack/src/jets/mary_jets.rs @@ -6,7 +6,7 @@ use nockvm::jets::math::util::add; use nockvm::jets::util::{bite_to_word, chop, slot, BAIL_FAIL}; use nockvm::jets::JetErr; use nockvm::mem::NockStack; -use nockvm::noun::{Atom, IndirectAtom, Noun, D, NO, T, YES}; +use nockvm::noun::{Atom, IndirectAtom, Noun, NounSpace, D, NO, T, YES}; use nockvm_macros::tas; use tracing::{debug, error}; @@ -25,13 +25,15 @@ use crate::jets::tip5_jets::{digest_to_noundigest, hash_hashable, hash_pairs}; use crate::utils::vecnoun_to_hoon_list; pub fn mary_swag_jet(context: &mut Context, subject: Noun) -> Result { - let door = slot(subject, 7)?; - let ma = slot(door, 6)?; - let sam = slot(subject, 6)?; - let i = sam.as_cell()?.head().as_direct()?.data() as usize; - let j = sam.as_cell()?.tail().as_direct()?.data() as usize; - - let Ok(mary) = MarySlice::try_from(ma) else { + let space = context.stack.noun_space(); + let door = slot(subject, 7, &space)?; + let ma = slot(door, 6, &space)?; + let sam = slot(subject, 6, &space)?; + let sam_cell = sam.in_space(&space).as_cell()?; + let i = sam_cell.head().noun().as_direct()?.data() as usize; + let j = sam_cell.tail().noun().as_direct()?.data() as usize; + + let Ok(mary) = MarySlice::try_from(ma, &space) else { debug!("cannot convert mary arg to mary"); return Err(BAIL_FAIL); }; @@ -49,18 +51,34 @@ pub fn mary_swag_jet(context: &mut Context, subject: Noun) -> Result Result { - let door = slot(subject, 7)?; - let ma = slot(door, 6)?; - let ma2 = slot(subject, 6)?; - - let step = ma.as_cell()?.head().as_direct()?.data() as u32; - let step2 = ma2.as_cell()?.head().as_direct()?.data() as u32; + let space = context.stack.noun_space(); + let door = slot(subject, 7, &space)?; + let ma = slot(door, 6, &space)?; + let ma2 = slot(subject, 6, &space)?; + + let step = ma + .in_space(&space) + .as_cell()? + .head() + .noun() + .as_direct()? + .data() as u32; + let step2 = ma2 + .in_space(&space) + .as_cell()? + .head() + .noun() + .as_direct()? + .data() as u32; if step != step2 { debug!("can only weld marys of same step"); return Err(BAIL_FAIL); } - let (Ok(mary1), Ok(mary2)) = (MarySlice::try_from(ma), MarySlice::try_from(ma2)) else { + let (Ok(mary1), Ok(mary2)) = ( + MarySlice::try_from(ma, &space), + MarySlice::try_from(ma2, &space), + ) else { debug!("mary1 or mary2 is not an fpoly"); return Err(BAIL_FAIL); }; @@ -74,11 +92,15 @@ pub fn mary_weld_jet(context: &mut Context, subject: Noun) -> Result Result { - let door = slot(subject, 7)?; - let ma = slot(door, 6)?; - let offset = slot(subject, 6)?; - - let (Ok(mary), Ok(offset)) = (MarySlice::try_from(ma), offset.as_atom()?.as_u64()) else { + let space = context.stack.noun_space(); + let door = slot(subject, 7, &space)?; + let ma = slot(door, 6, &space)?; + let offset = slot(subject, 6, &space)?; + + let (Ok(mary), Ok(offset)) = ( + MarySlice::try_from(ma, &space), + offset.in_space(&space).as_atom()?.as_u64(), + ) else { debug!("fp is not an fpoly or n is not an atom"); return Err(BAIL_FAIL); }; @@ -101,38 +123,46 @@ pub fn mary_transpose_jet(context: &mut Context, subject: Noun) -> Result Result { + let space = context.stack.noun_space(); let stack = &mut context.stack; - let door = slot(subject, 7)?; - let step = slot(door, 6)?.as_atom()?.as_u64()?; - let a = slot(subject, 6)?; + let door = slot(subject, 7, &space)?; + let step = slot(door, 6, &space)? + .in_space(&space) + .as_atom()? + .as_u64()?; + let a = slot(subject, 6, &space)?; if step == 1u64 { Ok(a) } else { let reap_res = reap(stack, step - 1, D(0))?; let init_bpoly_arg = T(stack, &[a, reap_res]); - let init_bpoly_arg_list = HoonList::try_from(init_bpoly_arg)?; + let init_bpoly_arg_list = HoonList::try_from(init_bpoly_arg, &space)?; let count = init_bpoly_arg_list.count(); let (res, res_poly): (IndirectAtom, &mut [Belt]) = new_handle_mut_slice(stack, Some(count)); - init_bpoly(init_bpoly_arg_list, res_poly); + init_bpoly(init_bpoly_arg_list, res_poly, &space); let res_cell = finalize_poly(stack, Some(res_poly.len()), res); - Ok(res_cell.as_cell()?.tail()) + Ok(res_cell.in_space(&space).as_cell()?.tail().noun()) } } pub fn fet_jet(context: &mut Context, subject: Noun) -> Result { + let space = context.stack.noun_space(); let stack = &mut context.stack; - let door = slot(subject, 7)?; - let step = slot(door, 6)?.as_atom()?.as_u64()?; - let a = slot(subject, 6)?.as_atom()?; + let door = slot(subject, 7, &space)?; + let step = slot(door, 6, &space)? + .in_space(&space) + .as_atom()? + .as_u64()?; + let a = slot(subject, 6, &space)?.as_atom()?; - let v = rip_correct(stack, 6, 1, a)?; + let v = rip_correct(stack, 6, 1, a, &space)?; - let lent_v = lent(v)? as u64; + let lent_v = lent(v, &space)? as u64; - if ((lent_v == 1) && (step == 1)) || (lent_v == (step + 1)) && levy_based(v) { + if ((lent_v == 1) && (step == 1)) || (lent_v == (step + 1)) && levy_based(v, &space) { Ok(YES) } else { Ok(NO) @@ -140,8 +170,9 @@ pub fn fet_jet(context: &mut Context, subject: Noun) -> Result { } pub fn transpose_bpolys_jet(context: &mut Context, subject: Noun) -> Result { - let sam = slot(subject, 6)?; - let bpolys = MarySlice::try_from(sam).expect("cannot convert bpolys arg"); + let space = context.stack.noun_space(); + let sam = slot(subject, 6, &space)?; + let bpolys = MarySlice::try_from(sam, &space).expect("cannot convert bpolys arg"); transpose_bpolys(context, bpolys) } @@ -164,23 +195,35 @@ fn transpose_bpolys(context: &mut Context, bpolys: MarySlice) -> Result Result { + let space = context.stack.noun_space(); let stack = &mut context.stack; - let door = slot(subject, 7)?; - let mary_noun = slot(door, 6)?; - let i = slot(subject, 6)?.as_direct()?.data() as usize; + let door = slot(subject, 7, &space)?; + let mary_noun = slot(door, 6, &space)?; + let i = slot(subject, 6, &space)?.as_direct()?.data() as usize; - snag_one(stack, mary_noun, i) + snag_one(stack, mary_noun, i, &space) } // snag from ave -pub fn snag_one(stack: &mut NockStack, mary_noun: Noun, i: usize) -> Result { - let mary_cell = mary_noun.as_cell()?; - let ma_step = mary_cell.head().as_atom()?.as_u32()?; - let ma_len = mary_cell.tail().as_cell()?.head().as_atom()?.as_u32()?; - let ma_dat: Atom = mary_cell.tail().as_cell()?.tail().as_atom()?; +pub fn snag_one( + stack: &mut NockStack, + mary_noun: Noun, + i: usize, + space: &NounSpace, +) -> Result { + let mary_cell = mary_noun.in_space(space).as_cell()?; + let ma_step = mary_cell.head().as_atom()?.atom().as_u32()?; + let ma_len = mary_cell + .tail() + .as_cell()? + .head() + .as_atom()? + .atom() + .as_u32()?; + let ma_dat: Atom = mary_cell.tail().as_cell()?.tail().as_atom()?.atom(); assert!(i < ma_len as usize); - snag_one_fields(stack, i, ma_step, ma_dat) + snag_one_fields(stack, i, ma_step, ma_dat, space) } // snag from ave with separate fields @@ -189,14 +232,22 @@ pub fn snag_one_fields( i: usize, ma_step: u32, ma_dat: Atom, + space: &NounSpace, ) -> Result { - let res = cut(stack, 6, i * ma_step as usize, ma_step as usize, ma_dat)?; + let res = cut( + stack, + 6, + i * ma_step as usize, + ma_step as usize, + ma_dat, + space, + )?; if ma_step == 1 { return Ok(res); } - let high_bit = lsh(stack, 0, bex(6) * ma_step as usize, D(1).as_atom()?)?; + let high_bit = lsh(stack, 0, bex(6) * ma_step as usize, D(1).as_atom()?, space)?; - Ok(add(stack, high_bit.as_atom()?, res.as_atom()?).as_noun()) + Ok(add(stack, high_bit.as_atom()?, res.as_atom()?, space).as_noun()) } // cut from hoon-138 @@ -206,6 +257,7 @@ fn cut( start: usize, run: usize, atom: Atom, + space: &NounSpace, ) -> Result { if run == 0 { return Ok(D(0)); @@ -214,8 +266,15 @@ fn cut( let new_indirect = unsafe { let (mut new_indirect, new_slice) = IndirectAtom::new_raw_mut_bitslice(stack, bite_to_word(bloq, run)?); - chop(bloq, start, run, 0, new_slice, atom.as_bitslice())?; - new_indirect.normalize_as_atom() + chop( + bloq, + start, + run, + 0, + new_slice, + atom.in_space(space).as_bitslice(), + )?; + new_indirect.normalize_as_atom_stack() }; Ok(new_indirect.as_noun()) } @@ -227,24 +286,30 @@ fn bex(arg: usize) -> usize { } pub fn snag_as_bpoly_jet(context: &mut Context, subject: Noun) -> Result { + let space = context.stack.noun_space(); let stack = &mut context.stack; - let door = slot(subject, 7)?; - let mary_noun = slot(door, 6)?; - let i = slot(subject, 6)?.as_direct()?.data() as usize; + let door = slot(subject, 7, &space)?; + let mary_noun = slot(door, 6, &space)?; + let i = slot(subject, 6, &space)?.as_direct()?.data() as usize; - snag_as_bpoly(stack, mary_noun, i) + snag_as_bpoly(stack, mary_noun, i, &space) } -pub fn snag_as_bpoly(stack: &mut NockStack, mary_noun: Noun, i: usize) -> Result { - let mary_cell = mary_noun.as_cell()?; - let ma_step = mary_cell.head().as_atom()?.as_u32()?; +pub fn snag_as_bpoly( + stack: &mut NockStack, + mary_noun: Noun, + i: usize, + space: &NounSpace, +) -> Result { + let mary_cell = mary_noun.in_space(space).as_cell()?; + let ma_step = mary_cell.head().as_atom()?.atom().as_u32()?; - let dat = snag_one(stack, mary_noun, i)?; + let dat = snag_one(stack, mary_noun, i, space)?; if ma_step == 1 { let step = bex(6) * ma_step as usize; - let high_bit = lsh(stack, 0, step, D(1).as_atom()?)?; - let res_add = add(stack, high_bit.as_atom()?, dat.as_atom()?).as_noun(); + let high_bit = lsh(stack, 0, step, D(1).as_atom()?, space)?; + let res_add = add(stack, high_bit.as_atom()?, dat.as_atom()?, space).as_noun(); return Ok(T(stack, &[D(ma_step as u64), res_add])); } @@ -252,26 +317,28 @@ pub fn snag_as_bpoly(stack: &mut NockStack, mary_noun: Noun, i: usize) -> Result } pub fn change_step_jet(context: &mut Context, subject: Noun) -> Result { + let space = context.stack.noun_space(); let stack = &mut context.stack; - let door = slot(subject, 7)?; - let ma_noun = slot(door, 6)?; - let new_step_noun = slot(subject, 6)?; + let door = slot(subject, 7, &space)?; + let ma_noun = slot(door, 6, &space)?; + let new_step_noun = slot(subject, 6, &space)?; - change_step(stack, ma_noun, new_step_noun) + change_step(stack, ma_noun, new_step_noun, &space) } pub fn change_step( stack: &mut NockStack, ma_noun: Noun, new_step_noun: Noun, + space: &NounSpace, ) -> Result { - let new_step = new_step_noun.as_atom()?.as_u64()?; // |= [new-step=@] ?? + let new_step = new_step_noun.in_space(space).as_atom()?.as_u64()?; // |= [new-step=@] ?? - let [ma_step_noun, ma_array] = ma_noun.uncell()?; // +$ mary [step=@ =array] - let [array_len_noun, array_dat] = ma_array.uncell()?; // +$ array [len=@ dat=@ux] + let [ma_step_noun, ma_array] = ma_noun.uncell(space)?; // +$ mary [step=@ =array] + let [array_len_noun, array_dat] = ma_array.uncell(space)?; // +$ array [len=@ dat=@ux] - let ma_step = ma_step_noun.as_atom()?.as_u64()?; - let array_len = array_len_noun.as_atom()?.as_u64()?; + let ma_step = ma_step_noun.in_space(space).as_atom()?.as_u64()?; + let array_len = array_len_noun.in_space(space).as_atom()?.as_u64()?; if ma_step == new_step { return Ok(ma_noun); @@ -284,14 +351,15 @@ pub fn change_step( } pub fn bp_build_merk_heap_jet(context: &mut Context, subject: Noun) -> Result { + let space = context.stack.noun_space(); let stack = &mut context.stack; - let mary_noun = slot(subject, 6)?; + let mary_noun = slot(subject, 6, &space)?; - let (_ma_step, ma_array_len, _ma_array_dat) = get_mary_fields(mary_noun)?; - let heap_mary = heapify_mary(stack, mary_noun)?; - let xeb_m = simple_xeb(ma_array_len.as_u64()? as usize); + let (_ma_step, ma_array_len, _ma_array_dat) = get_mary_fields(mary_noun, &space)?; + let heap_mary = heapify_mary(stack, mary_noun, &space)?; + let xeb_m = simple_xeb(ma_array_len.in_space(&space).as_u64()? as usize); - let snag_digest = snag_as_digest(stack, heap_mary, 0)?; + let snag_digest = snag_as_digest(stack, heap_mary, 0, &space)?; let res1 = T(stack, &[snag_digest, heap_mary]); let res = T(stack, &[D(xeb_m as u64), res1]); @@ -306,84 +374,110 @@ fn simple_xeb(n: usize) -> usize { } } -pub fn get_mary_fields(p: Noun) -> Result<(Atom, Atom, Noun), JetErr> { - let [ma_step, ma_array] = p.uncell()?; // +$ mary [step=@ =array] - let [ma_array_len, ma_array_dat] = ma_array.uncell()?; // +$ array [len=@ dat=@ux] +pub fn get_mary_fields(p: Noun, space: &NounSpace) -> Result<(Atom, Atom, Noun), JetErr> { + let [ma_step, ma_array] = p.uncell(space)?; // +$ mary [step=@ =array] + let [ma_array_len, ma_array_dat] = ma_array.uncell(space)?; // +$ array [len=@ dat=@ux] Ok((ma_step.as_atom()?, ma_array_len.as_atom()?, ma_array_dat)) } -fn heapify_mary(stack: &mut NockStack, m_noun: Noun) -> Result { - let (_ma_step, ma_array_len, _ma_array_dat) = get_mary_fields(m_noun)?; - let size = bex(simple_xeb(ma_array_len.as_u64()? as usize)) - 1; +fn heapify_mary(stack: &mut NockStack, m_noun: Noun, space: &NounSpace) -> Result { + let (_ma_step, ma_array_len, _ma_array_dat) = get_mary_fields(m_noun, space)?; + let size = bex(simple_xeb(ma_array_len.in_space(space).as_u64()? as usize)) - 1; // calc high-bit - let high_bit = lsh(stack, 6, size * 5, D(1).as_atom()?)?.as_atom()?; + let high_bit = lsh(stack, 6, size * 5, D(1).as_atom()?, space)?.as_atom()?; // make leaves let mut res_vec: Vec = Vec::new(); - for i in 0..ma_array_len.as_u64()? { - let t = snag_as_bpoly(stack, m_noun, i as usize)?; + for i in 0..ma_array_len.in_space(space).as_u64()? { + let t = snag_as_bpoly(stack, m_noun, i as usize, space)?; let hashable_bpoly = T(stack, &[D(tas!(b"mary")), D(1), t]); - let hash = hash_hashable(stack, hashable_bpoly)?; - let leafs = leaf_sequence(stack, hash)?; + let hash = hash_hashable(stack, hashable_bpoly, space)?; + let leafs = leaf_sequence(stack, hash, space)?; res_vec.push(leafs); } - let mut res = vecnoun_to_hoon_list(stack, res_vec.as_slice()); + let mut res = vecnoun_to_hoon_list(stack, res_vec.as_slice(), space); let mut curr = res; loop { - let lent_curr = lent(curr)?; + let lent_curr = lent(curr, space)?; if lent_curr == 1 { break; } else { - let pairs = hash_pairs(stack, curr)?; - res = weld(stack, pairs, res)?; + let pairs = hash_pairs(stack, curr, space)?; + res = weld(stack, pairs, res, space)?; curr = pairs; } } let a = zing(stack, res)?; - let b = rep(stack, D(6), a)?; - let c = add(stack, high_bit, b.as_atom()?); + let b = rep(stack, D(6), a, space)?; + let c = add(stack, high_bit, b.as_atom()?, space); let res = T(stack, &[D(5), D(size as u64), c.as_noun()]); Ok(res) } pub fn snag_as_digest_jet(context: &mut Context, subject: Noun) -> Result { + let space = context.stack.noun_space(); let stack = &mut context.stack; - let sam = slot(subject, 6)?; - let m_noun = slot(sam, 2)?; - let i_noun = slot(sam, 3)?; + let sam = slot(subject, 6, &space)?; + let m_noun = slot(sam, 2, &space)?; + let i_noun = slot(sam, 3, &space)?; - let i = i_noun.as_atom()?.as_u64()? as usize; - snag_as_digest(stack, m_noun, i) + let i = i_noun.in_space(&space).as_atom()?.as_u64()? as usize; + snag_as_digest(stack, m_noun, i, &space) } -fn snag_as_digest(stack: &mut NockStack, m_noun: Noun, i: usize) -> Result { - let buf = snag_one(stack, m_noun, i)?.as_atom()?; +fn snag_as_digest( + stack: &mut NockStack, + m_noun: Noun, + i: usize, + space: &NounSpace, +) -> Result { + let buf = snag_one(stack, m_noun, i, space)?.as_atom()?; let mut digest = [0u64; DIGEST_LENGTH]; - digest[0] = cut(stack, 6, 0, 1, buf)?.as_atom()?.as_u64()?; - digest[1] = cut(stack, 6, 1, 1, buf)?.as_atom()?.as_u64()?; - digest[2] = cut(stack, 6, 2, 1, buf)?.as_atom()?.as_u64()?; - digest[3] = cut(stack, 6, 3, 1, buf)?.as_atom()?.as_u64()?; - digest[4] = cut(stack, 6, 4, 1, buf)?.as_atom()?.as_u64()?; + digest[0] = cut(stack, 6, 0, 1, buf, space)? + .in_space(space) + .as_atom()? + .as_u64()?; + digest[1] = cut(stack, 6, 1, 1, buf, space)? + .in_space(space) + .as_atom()? + .as_u64()?; + digest[2] = cut(stack, 6, 2, 1, buf, space)? + .in_space(space) + .as_atom()? + .as_u64()?; + digest[3] = cut(stack, 6, 3, 1, buf, space)? + .in_space(space) + .as_atom()? + .as_u64()?; + digest[4] = cut(stack, 6, 4, 1, buf, space)? + .in_space(space) + .as_atom()? + .as_u64()?; Ok(digest_to_noundigest(stack, digest)) } pub fn mary_to_list_jet(context: &mut Context, subject: Noun) -> Result { + let space = context.stack.noun_space(); let stack = &mut context.stack; - let ma_noun = slot(subject, 6)?; - mary_to_list(stack, ma_noun) + let ma_noun = slot(subject, 6, &space)?; + mary_to_list(stack, ma_noun, &space) } -pub fn mary_to_list(stack: &mut NockStack, ma_noun: Noun) -> Result { - let (ma_step, ma_array_len, ma_array_dat) = get_mary_fields(ma_noun)?; - let ma_step = ma_step.as_u64()? as usize; +pub fn mary_to_list( + stack: &mut NockStack, + ma_noun: Noun, + space: &NounSpace, +) -> Result { + let (ma_step, ma_array_len, ma_array_dat) = get_mary_fields(ma_noun, space)?; + let ma_step = ma_step.in_space(space).as_u64()? as usize; - mary_to_list_fields(stack, ma_array_len, ma_array_dat, ma_step) + mary_to_list_fields(stack, ma_array_len, ma_array_dat, ma_step, space) } pub fn mary_to_list_fields( @@ -391,27 +485,28 @@ pub fn mary_to_list_fields( ma_array_len: Atom, ma_array_dat: Noun, ma_step: usize, + space: &NounSpace, ) -> Result { - if ma_array_len.as_u64()? == 0 { + if ma_array_len.in_space(space).as_u64()? == 0 { return Ok(D(0)); } - let res_rip = rip(stack, 6, ma_step, ma_array_dat.as_atom()?)?; + let res_rip = rip(stack, 6, ma_step, ma_array_dat.as_atom()?, space)?; let res_snip = snip(stack, res_rip)?; let mut res_turn: Vec = Vec::new(); - for elem in HoonList::try_from(res_snip)?.into_iter() { + for elem in HoonList::try_from(res_snip, space)?.into_iter() { //%+ add elem //let x = elem + let res_wutcol = if ma_step == 1 { D(0) } else { - lsh(stack, 6, ma_step, D(1).as_atom()?)? + lsh(stack, 6, ma_step, D(1).as_atom()?, space)? }; - let res_add = add(stack, elem.as_atom()?, res_wutcol.as_atom()?); + let res_add = add(stack, elem.as_atom()?, res_wutcol.as_atom()?, space); res_turn.push(res_add.as_noun()); } - Ok(vecnoun_to_hoon_list(stack, res_turn.as_slice())) + Ok(vecnoun_to_hoon_list(stack, res_turn.as_slice(), space)) } diff --git a/crates/zkvm-jetpack/src/jets/mega_jets.rs b/crates/zkvm-jetpack/src/jets/mega_jets.rs index 48172f42c..cc2d48f0c 100644 --- a/crates/zkvm-jetpack/src/jets/mega_jets.rs +++ b/crates/zkvm-jetpack/src/jets/mega_jets.rs @@ -7,7 +7,7 @@ use crate::form::belt::*; use crate::form::bpoly::*; use crate::form::handle::*; use crate::form::mega::{brek, MegaTyp}; -use crate::form::noun_ext::NounMathExt; +use crate::form::noun_ext::{NounMathExt, NounMathExtHandle}; use crate::form::poly::*; use crate::form::structs::{HoonMap, HoonMapIter}; @@ -20,28 +20,29 @@ fn lagrange_one_bpoly(len: usize) -> BPolyVec { } pub fn mp_substitute_mega_jet(context: &mut Context, subject: Noun) -> Result { - let sam = slot(subject, 6)?; + let space = context.stack.noun_space(); + let sam = slot(subject, 6, &space)?; let stack = &mut context.stack; let [p_noun, trace_evals_noun, height_noun, chals_noun, dyns_noun, com_map_noun] = - sam.uncell()?; + sam.uncell(&space)?; - let Ok(trace_evals) = BPolySlice::try_from(trace_evals_noun) else { + let Ok(trace_evals) = BPolySlice::try_from(trace_evals_noun, &space) else { return Err(BAIL_FAIL); }; let Ok(height_atom) = height_noun.as_atom() else { return Err(BAIL_FAIL); }; - let Ok(height) = height_atom.as_u64() else { + let Ok(height) = height_atom.in_space(&space).as_u64() else { return Err(BAIL_FAIL); }; let height_usize = height as usize; - let Ok(dyns) = BPolySlice::try_from(dyns_noun) else { + let Ok(dyns) = BPolySlice::try_from(dyns_noun, &space) else { return Err(BAIL_FAIL); }; - let Ok(challenges) = BPolySlice::try_from(chals_noun) else { + let Ok(challenges) = BPolySlice::try_from(chals_noun, &space) else { return Err(BAIL_FAIL); }; @@ -49,16 +50,16 @@ pub fn mp_substitute_mega_jet(context: &mut Context, subject: Noun) -> Result { if com_map_noun.raw_equals(&D(0)) { None } else { - HoonMap::try_from(com_map_noun).ok() + HoonMap::try_from(com_map_noun.in_space(&space)).ok() } }; let mut acc_vec = zero_bpoly(); - - let mut p_iter = HoonMapIter::from(p_noun); + let p_noun_handle = p_noun.in_space(&space); + let mut p_iter = HoonMapIter::new(&p_noun_handle); p_iter.try_fold((), |_, n| { let [k_noun, v_noun] = n.uncell()?; - let Ok(k) = BPolySlice::try_from(k_noun) else { + let Ok(k) = BPolySlice::try_from(k_noun.noun(), &space) else { return Err(BAIL_FAIL); }; let Ok(v) = v_noun.as_belt() else { @@ -142,7 +143,7 @@ pub fn mp_substitute_mega_jet(context: &mut Context, subject: Noun) -> Result { .as_ref() .and_then(|m| m.get(stack, D(idx as u64))) .ok_or(BAIL_EXIT)?; - let Ok(com_slice) = BPolySlice::try_from(com_noun) else { + let Ok(com_slice) = BPolySlice::try_from(com_noun, &space) else { return Err(BAIL_FAIL); }; diff --git a/crates/zkvm-jetpack/src/jets/memory_table_jets_v2.rs b/crates/zkvm-jetpack/src/jets/memory_table_jets_v2.rs index ab599fd0a..27cab23fc 100644 --- a/crates/zkvm-jetpack/src/jets/memory_table_jets_v2.rs +++ b/crates/zkvm-jetpack/src/jets/memory_table_jets_v2.rs @@ -5,7 +5,7 @@ use nockvm::interpreter::Context; use nockvm::jets::util::{slot, BAIL_FAIL}; use nockvm::jets::JetErr; use nockvm::mem::NockStack; -use nockvm::noun::{Atom, IndirectAtom, Noun, D, T}; +use nockvm::noun::{Atom, IndirectAtom, Noun, NounSpace, D, T}; use nockvm_macros::tas; use tracing::debug; @@ -17,18 +17,19 @@ use crate::form::mary::*; use crate::jets::table_utils::*; pub fn memory_v2_extend_jet(context: &mut Context, subject: Noun) -> Result { - let sam = slot(subject, 6)?; - let table_mary = slot(sam, 2)?; - let chals_rd1 = slot(sam, 6)?; - let fock_ret = slot(sam, 7)?; - let sf = slot(fock_ret, 15)?; - let subject = slot(sf, 2)?; - let formula = slot(sf, 3)?; - - let chals: ExtChals = init_ext_chals(chals_rd1)?; - - let table_noun = slot(table_mary, 3)?; - let Ok(table) = MarySlice::try_from(table_noun) else { + let space = context.stack.noun_space(); + let sam = slot(subject, 6, &space)?; + let table_mary = slot(sam, 2, &space)?; + let chals_rd1 = slot(sam, 6, &space)?; + let fock_ret = slot(sam, 7, &space)?; + let sf = slot(fock_ret, 15, &space)?; + let subject = slot(sf, 2, &space)?; + let formula = slot(sf, 3, &space)?; + + let chals: ExtChals = init_ext_chals(chals_rd1, &space)?; + + let table_noun = slot(table_mary, 3, &space)?; + let Ok(table) = MarySlice::try_from(table_noun, &space) else { debug!("cannot convert mary arg to mary"); return Err(BAIL_FAIL); }; @@ -39,8 +40,9 @@ pub fn memory_v2_extend_jet(context: &mut Context, subject: Noun) -> Result Result) -> Result, JetErr> { +fn rna_bfta(tres: Vec<(Noun, bool)>, space: &NounSpace) -> Result, JetErr> { let mut queue: VecDeque<(Noun, Belt)> = tres .iter() .filter(|(n, _b)| n.is_cell()) @@ -145,8 +147,9 @@ fn rna_bfta(tres: Vec<(Noun, bool)>) -> Result, JetErr> { option_env!("GIT_SHA") ) }); - let head = noun.as_cell()?.head(); - let tail = noun.as_cell()?.tail(); + let cell = noun.in_space(space).as_cell()?; + let head = cell.head().noun(); + let tail = cell.tail().noun(); match (head.is_atom(), tail.is_atom()) { (true, true) => {} @@ -173,17 +176,19 @@ fn add_ions( stack: &mut NockStack, lst: &mut Vec, chals: &ExtChals, + space: &NounSpace, ) -> Result, JetErr> { let mut res = Vec::::new(); let cache = MutHamt::::new(stack); for noun in lst { - let mut head = noun.as_cell()?.head(); - let mut tail = noun.as_cell()?.tail(); + let cell = noun.in_space(space).as_cell()?; + let mut head = cell.head().noun(); + let mut tail = cell.tail().noun(); let left: Ion = if head.is_atom() { - atom_ion(head.as_atom()?, &chals.alf)? + atom_ion(head.as_atom()?, &chals.alf, space)? } else { cache.lookup(stack, &mut head).unwrap_or_else(|| { panic!( @@ -196,7 +201,7 @@ fn add_ions( }; let right: Ion = if tail.is_atom() { - atom_ion(tail.as_atom()?, &chals.alf)? + atom_ion(tail.as_atom()?, &chals.alf, space)? } else { cache.lookup(stack, &mut tail).unwrap_or_else(|| { panic!( @@ -234,11 +239,11 @@ fn cons_ion(alf: &Felt, left: &Ion, right: &Ion) -> Ion { Ion { size, dyck, leaf } } -fn atom_ion(atom: Atom, alf: &Felt) -> Result { +fn atom_ion(atom: Atom, alf: &Felt, space: &NounSpace) -> Result { Ok(Ion { size: *alf, dyck: Felt::zero(), - leaf: Felt::lift(Belt(atom.as_u64()?)), + leaf: Felt::lift(Belt(atom.in_space(space).as_u64()?)), }) } @@ -249,19 +254,20 @@ struct MemoryBankEx { } pub fn memory_v2_mega_extend_jet(context: &mut Context, subject: Noun) -> Result { - let sam = slot(subject, 6)?; - let table_mary = slot(sam, 2)?; - let all_chals = slot(sam, 6)?; - let fock_ret = slot(sam, 7)?; - let sf = slot(fock_ret, 15)?; - let subject = slot(sf, 2)?; + let space = context.stack.noun_space(); + let sam = slot(subject, 6, &space)?; + let table_mary = slot(sam, 2, &space)?; + let all_chals = slot(sam, 6, &space)?; + let fock_ret = slot(sam, 7, &space)?; + let sf = slot(fock_ret, 15, &space)?; + let subject = slot(sf, 2, &space)?; let subject_is_atom: bool = subject.is_atom(); - let chals: MegaExtChals = init_mega_ext_chals(all_chals)?; + let chals: MegaExtChals = init_mega_ext_chals(all_chals, &space)?; let z2: Felt = fmul_(&chals.z, &chals.z); - let table_noun = slot(table_mary, 3)?; - let Ok(table) = MarySlice::try_from(table_noun) else { + let table_noun = slot(table_mary, 3, &space)?; + let Ok(table) = MarySlice::try_from(table_noun, &space) else { debug!("cannot convert mary arg to mary"); return Err(BAIL_FAIL); }; @@ -273,7 +279,7 @@ pub fn memory_v2_mega_extend_jet(context: &mut Context, subject: Noun) -> Result let first_row = get_row(&table, 0); let second_row = get_row(&table, 1); - let subj_fp_triple: TreeData = build_tree_data(subject, &chals.alf)?; + let subj_fp_triple: TreeData = build_tree_data(subject, &chals.alf, &space)?; let input: Felt = ifp_compress( &Ion { size: subj_fp_triple.size, diff --git a/crates/zkvm-jetpack/src/jets/proof_gen_jets.rs b/crates/zkvm-jetpack/src/jets/proof_gen_jets.rs index 5c05e1120..64963fb3f 100644 --- a/crates/zkvm-jetpack/src/jets/proof_gen_jets.rs +++ b/crates/zkvm-jetpack/src/jets/proof_gen_jets.rs @@ -4,7 +4,7 @@ use std::collections::BTreeMap; use nockvm::interpreter::Context; use nockvm::jets::util::{slot, BAIL_FAIL}; use nockvm::jets::JetErr; -use nockvm::noun::{IndirectAtom, Noun, D}; +use nockvm::noun::{IndirectAtom, Noun, NounSpace, D}; use nockvm_macros::tas; use noun_serde::NounDecode; use tracing::debug; @@ -29,12 +29,10 @@ pub struct MPComp { pub com: Vec, } -impl TryFrom for MPComp { - type Error = JetErr; - - fn try_from(noun: Noun) -> Result { - let dep_list = HoonList::try_from(slot(noun, 2)?)?; - let com_list = HoonList::try_from(slot(noun, 3)?)?; +impl MPComp { + pub fn try_from(noun: Noun, space: &NounSpace) -> Result { + let dep_list = HoonList::try_from(slot(noun, 2, space)?, space)?; + let com_list = HoonList::try_from(slot(noun, 3, space)?, space)?; let mut dep = Vec::with_capacity(dep_list.count()); let mut com = Vec::with_capacity(com_list.count()); @@ -51,11 +49,9 @@ impl TryFrom for MPComp { } } -impl TryFrom for MPUltra { - type Error = JetErr; - - fn try_from(mp_ultra: Noun) -> Result { - let mp_ultra_cell = mp_ultra.as_cell().unwrap_or_else(|err| { +impl MPUltra { + pub fn try_from(mp_ultra: Noun, space: &NounSpace) -> Result { + let mp_ultra_cell = mp_ultra.in_space(space).as_cell().unwrap_or_else(|err| { panic!( "Panicked with {err:?} at {}:{} (git sha: {:?})", file!(), @@ -83,8 +79,11 @@ impl TryFrom for MPUltra { option_env!("GIT_SHA") ) }) { - tas!(b"mega") => Ok(MPUltra::Mega(mp_ultra_cell.tail())), - tas!(b"comp") => Ok(MPUltra::Comp(MPComp::try_from(mp_ultra_cell.tail())?)), + tas!(b"mega") => Ok(MPUltra::Mega(mp_ultra_cell.tail().noun())), + tas!(b"comp") => Ok(MPUltra::Comp(MPComp::try_from( + mp_ultra_cell.tail().noun(), + space, + )?)), _ => panic!("Invalid MPUltra type"), } } @@ -115,11 +114,10 @@ pub struct ConstraintData { pub degs: Vec, } -impl TryFrom for CountMap { - type Error = JetErr; - - fn try_from(noun: Noun) -> Result { - let counts = HoonMapIter::from(noun); +impl CountMap { + pub fn try_from(noun: Noun, space: &NounSpace) -> Result { + let noun_handle = noun.in_space(space); + let counts = HoonMapIter::new(&noun_handle); let mut outer = ProofMap::::new(); @@ -134,13 +132,16 @@ impl TryFrom for CountMap { ) }); (term_cell.head().as_atom()?.as_u64()? as usize, { - let tail = term_cell.tail(); + let tail = term_cell.tail().noun(); Counts { - boundary: slot(tail, 2)?.as_atom()?.as_u64()? as usize, - row: slot(tail, 6)?.as_atom()?.as_u64()? as usize, - transition: slot(tail, 14)?.as_atom()?.as_u64()? as usize, - terminal: slot(tail, 30)?.as_atom()?.as_u64()? as usize, - extra: slot(tail, 31)?.as_atom()?.as_u64()? as usize, + boundary: slot(tail, 2, space)?.in_space(space).as_atom()?.as_u64()? + as usize, + row: slot(tail, 6, space)?.in_space(space).as_atom()?.as_u64()? as usize, + transition: slot(tail, 14, space)?.in_space(space).as_atom()?.as_u64()? + as usize, + terminal: slot(tail, 30, space)?.in_space(space).as_atom()?.as_u64()? + as usize, + extra: slot(tail, 31, space)?.in_space(space).as_atom()?.as_u64()? as usize, } }) }; @@ -150,12 +151,11 @@ impl TryFrom for CountMap { } } -impl TryFrom for IndexBPolyMap<'_> { - type Error = JetErr; - - fn try_from(hoon_map: Noun) -> Result { +impl IndexBPolyMap<'_> { + pub fn try_from(hoon_map: Noun, space: &NounSpace) -> Result { let mut composition_chals = ProofMap::::new(); - let hoon_map = HoonMapIter::from(hoon_map); + let hoon_map_handle = hoon_map.in_space(space); + let hoon_map = HoonMapIter::new(&hoon_map_handle); for term_noun in hoon_map.into_iter() { let (k, v): (usize, &[Belt]) = { @@ -169,7 +169,7 @@ impl TryFrom for IndexBPolyMap<'_> { }); ( term_cell.head().as_atom()?.as_u64()? as usize, - BPolySlice::try_from(term_cell.tail()) + BPolySlice::try_from(term_cell.tail().noun(), space) .unwrap_or_else(|err| { panic!( "Panicked with {err:?} at {}:{} (git sha: {:?})", @@ -187,11 +187,10 @@ impl TryFrom for IndexBPolyMap<'_> { } } -impl TryFrom for Constraints { - type Error = JetErr; - - fn try_from(hoon_map: Noun) -> Result { - let hoon_map = HoonMapIter::from(hoon_map); +impl Constraints { + pub fn try_from(hoon_map: Noun, space: &NounSpace) -> Result { + let hoon_map_handle = hoon_map.in_space(space); + let hoon_map = HoonMapIter::new(&hoon_map_handle); let mut constraints = ProofMap::new(); for term_noun in hoon_map.into_iter() { @@ -206,7 +205,7 @@ impl TryFrom for Constraints { }); ( term_cell.head().as_atom()?.as_u64()? as usize, - MPDenseConstraints::try_from(term_cell.tail())?, + MPDenseConstraints::try_from(term_cell.tail().noun(), space)?, ) }; @@ -216,26 +215,23 @@ impl TryFrom for Constraints { } } -impl TryFrom for MPDenseConstraints { - type Error = JetErr; - - fn try_from(noun: Noun) -> Result { - let [boundary, row, transition, terminal, extra] = noun.uncell()?; - - let boundary: Vec = HoonList::try_from(boundary)? - .map(ConstraintData::try_from) +impl MPDenseConstraints { + pub fn try_from(noun: Noun, space: &NounSpace) -> Result { + let [boundary, row, transition, terminal, extra] = noun.uncell(space)?; + let boundary: Vec = HoonList::try_from(boundary, space)? + .map(|x| ConstraintData::try_from(x, space)) .collect::, _>>()?; - let row: Vec = HoonList::try_from(row)? - .map(ConstraintData::try_from) + let row: Vec = HoonList::try_from(row, space)? + .map(|x| ConstraintData::try_from(x, space)) .collect::, _>>()?; - let transition: Vec = HoonList::try_from(transition)? - .map(ConstraintData::try_from) + let transition: Vec = HoonList::try_from(transition, space)? + .map(|x| ConstraintData::try_from(x, space)) .collect::, _>>()?; - let terminal: Vec = HoonList::try_from(terminal)? - .map(ConstraintData::try_from) + let terminal: Vec = HoonList::try_from(terminal, space)? + .map(|x| ConstraintData::try_from(x, space)) .collect::, _>>()?; - let extra: Vec = HoonList::try_from(extra)? - .map(ConstraintData::try_from) + let extra: Vec = HoonList::try_from(extra, space)? + .map(|x| ConstraintData::try_from(x, space)) .collect::, _>>()?; Ok(MPDenseConstraints { @@ -248,14 +244,12 @@ impl TryFrom for MPDenseConstraints { } } -impl TryFrom for ConstraintData { - type Error = JetErr; - - fn try_from(noun: Noun) -> Result { - let cell = noun.as_cell()?; - let cs = MPUltra::try_from(cell.head())?; - let degs: Vec = HoonList::try_from(cell.tail())? - .map(|n| n.as_atom()?.as_u64()) +impl ConstraintData { + pub fn try_from(noun: Noun, space: &NounSpace) -> Result { + let cell = noun.in_space(space).as_cell()?; + let cs = MPUltra::try_from(cell.head().noun(), space)?; + let degs: Vec = HoonList::try_from(cell.tail().noun(), space)? + .map(|n| n.in_space(space).as_atom()?.as_u64()) .collect::, _>>()?; Ok(ConstraintData { constraint: cs, @@ -265,12 +259,13 @@ impl TryFrom for ConstraintData { } pub fn precompute_ntts_jet(context: &mut Context, subject: Noun) -> Result { - let sam = slot(subject, 6)?; - let polys = slot(sam, 2)?; - let height = slot(sam, 6)?.as_atom()?.as_u64()? as usize; - let max_ntt_len = slot(sam, 7)?.as_atom()?.as_u64()? as usize; + let space = context.stack.noun_space(); + let sam = slot(subject, 6, &space)?; + let polys = slot(sam, 2, &space)?; + let height = slot(sam, 6, &space)?.in_space(&space).as_atom()?.as_u64()? as usize; + let max_ntt_len = slot(sam, 7, &space)?.in_space(&space).as_atom()?.as_u64()? as usize; - let polys = MarySlice::try_from(polys).unwrap_or_else(|err| { + let polys = MarySlice::try_from(polys, &space).unwrap_or_else(|err| { panic!( "Panicked with {err:?} at {}:{} (git sha: {:?})", file!(), @@ -290,32 +285,34 @@ pub fn precompute_ntts_jet(context: &mut Context, subject: Noun) -> Result Result { - let sam = slot(subject, 6)?; + let space = context.stack.noun_space(); + let sam = slot(subject, 6, &space)?; let [trace_evaluations, heights, constraint_map, counts_map, dyn_list, weights_map, challenges, deep_challange, table_full_widths, is_extra] = - sam.uncell()?; + sam.uncell(&space)?; - let Ok(trace_evaluations) = FPolySlice::try_from(trace_evaluations) else { + let Ok(trace_evaluations) = FPolySlice::try_from(trace_evaluations, &space) else { debug!("trace_evaluations is not a valid FPolySlice"); return Err(BAIL_FAIL); }; - let Ok(heights) = Vec::::from_noun(&heights) else { + let Ok(heights) = Vec::::from_noun(&heights, &space) else { debug!("heights decode failed"); return Err(BAIL_FAIL); }; - let constraint_map = Constraints::try_from(constraint_map)?; - let counts_map = CountMap::try_from(counts_map)?; + let constraint_map = Constraints::try_from(constraint_map, &space)?; + let counts_map = CountMap::try_from(counts_map, &space)?; - let dyn_list: Vec> = HoonList::try_from(dyn_list)? + let dyn_list: Vec> = HoonList::try_from(dyn_list, &space)? .into_iter() - .map(BPolySlice::try_from) + .map(|x| BPolySlice::try_from(x, &space)) .collect::>, _>>()?; - let weights_map = IndexBPolyMap::try_from(weights_map)?; - let challenges = BPolySlice::try_from(challenges)?; - let deep_challenge = deep_challange.as_felt()?; - let table_full_widths: Vec = HoonList::try_from(table_full_widths)? + let weights_map = IndexBPolyMap::try_from(weights_map, &space)?; + let challenges = BPolySlice::try_from(challenges, &space)?; + let deep_challenge = deep_challange.as_felt(&space)?; + let table_full_widths: Vec = HoonList::try_from(table_full_widths, &space)? .into_iter() .map(|x| { - x.as_atom() + x.in_space(&space) + .as_atom() .expect("table_full_widths element should be an atom") .as_u64() .expect("table_full_widths element should be a u64") @@ -325,7 +322,7 @@ pub fn eval_composition_poly_jet(context: &mut Context, subject: Noun) -> Result let res = eval_composition_poly( &trace_evaluations, &heights, &constraint_map, &counts_map, &dyn_list, &weights_map, - &challenges, deep_challenge, &table_full_widths, is_extra, + &challenges, deep_challenge, &table_full_widths, is_extra, &space, )?; let (res_atom, res_felt): (IndirectAtom, &mut Felt) = new_handle_mut_felt(&mut context.stack); @@ -345,6 +342,7 @@ fn eval_composition_poly( deep_challenge: &Felt, table_full_widths: &[u64], is_extra: bool, + space: &NounSpace, ) -> Result { let dp = degree_processing(heights, is_extra, constraint_map); @@ -388,6 +386,7 @@ fn eval_composition_poly( challenges.0, &dp.fri_degree_bound, deep_challenge, + space, )?; acc = fadd_(&acc, &fmul_(&boundary_zerofier, &boundary_eval)); @@ -400,6 +399,7 @@ fn eval_composition_poly( challenges.0, &dp.fri_degree_bound, deep_challenge, + space, )?; acc = fadd_(&acc, &fmul_(&row_zerofier, &row_eval)); @@ -412,6 +412,7 @@ fn eval_composition_poly( challenges.0, &dp.fri_degree_bound, deep_challenge, + space, )?; acc = fadd_(&acc, &fmul_(&transition_zerofier, &trans_eval)); @@ -424,6 +425,7 @@ fn eval_composition_poly( challenges.0, &dp.fri_degree_bound, deep_challenge, + space, )?; acc = fadd_(&acc, &fmul_(&terminal_zerofier, &term_eval)); @@ -437,6 +439,7 @@ fn eval_composition_poly( challenges.0, &dp.fri_degree_bound, deep_challenge, + space, )?; acc = fadd_(&acc, &fmul_(&row_zerofier, &extra_eval)); } @@ -445,6 +448,7 @@ fn eval_composition_poly( Ok(acc) } +#[allow(clippy::too_many_arguments)] fn evaluate_constraints( constraints: &[PolyWithDegreeFudges<'_>], dyns: &BPolySlice<'_>, @@ -453,12 +457,13 @@ fn evaluate_constraints( challenges: &[Belt], fri_degree_bound: &u64, deep_challenge: &Felt, + space: &NounSpace, ) -> Result { let mut acc = Felt::zero(); let mut idx = 0; for constraint in constraints { - let evaled = mpeval_ultra_felt(constraint.poly, evals, challenges, dyns.0)?; + let evaled = mpeval_ultra_felt(constraint.poly, evals, challenges, dyns.0, space)?; for (deg, eval) in constraint.degrees.iter().zip(evaled.iter()) { let alpha = Felt::lift(weights[2 * idx]); let beta = Felt::lift(weights[2 * idx + 1]); @@ -622,35 +627,36 @@ fn compute_degree(typ: &ConstraintType, height: u64, deg: u64) -> u64 { } pub fn compute_deep_jet(context: &mut Context, subject: Noun) -> Result { - let sam = slot(subject, 6)?; - let trace_polys = slot(sam, 2)?; - let trace_openings = slot(sam, 6)?; - let composition_pieces = slot(sam, 14)?; - let composition_piece_openings = slot(sam, 30)?; - let weights = slot(sam, 62)?; - let omicrons = slot(sam, 126)?; - let deep_challenge = slot(sam, 254)?; - let comp_eval_point = slot(sam, 255)?; + let space = context.stack.noun_space(); + let sam = slot(subject, 6, &space)?; + let trace_polys = slot(sam, 2, &space)?; + let trace_openings = slot(sam, 6, &space)?; + let composition_pieces = slot(sam, 14, &space)?; + let composition_piece_openings = slot(sam, 30, &space)?; + let weights = slot(sam, 62, &space)?; + let omicrons = slot(sam, 126, &space)?; + let deep_challenge = slot(sam, 254, &space)?; + let comp_eval_point = slot(sam, 255, &space)?; // TODO: implement conversion from NounError to JetErr let (Ok(trace_openings), Ok(composition_piece_openings), Ok(weights), Ok(omicrons)) = ( - FPolySlice::try_from(trace_openings), - FPolySlice::try_from(composition_piece_openings), - FPolySlice::try_from(weights), - FPolySlice::try_from(omicrons), + FPolySlice::try_from(trace_openings, &space), + FPolySlice::try_from(composition_piece_openings, &space), + FPolySlice::try_from(weights, &space), + FPolySlice::try_from(omicrons, &space), ) else { debug!("one of trace_openings, composition_piece_openings, weights, or omicrons is not a valid FPolySlice"); return Err(BAIL_FAIL); }; - let trace_polys = HoonList::try_from(trace_polys)?; - let composition_pieces = HoonList::try_from(composition_pieces)?; - let deep_challenge = deep_challenge.as_felt()?; - let comp_eval_point = comp_eval_point.as_felt()?; + let trace_polys = HoonList::try_from(trace_polys, &space)?; + let composition_pieces = HoonList::try_from(composition_pieces, &space)?; + let deep_challenge = deep_challenge.as_felt(&space)?; + let comp_eval_point = comp_eval_point.as_felt(&space)?; let compute_deep_res = compute_deep( trace_polys, trace_openings.0, composition_pieces, composition_piece_openings.0, weights.0, - omicrons.0, deep_challenge, comp_eval_point, + omicrons.0, deep_challenge, comp_eval_point, &space, ); let (res, res_poly): (IndirectAtom, &mut [Felt]) = diff --git a/crates/zkvm-jetpack/src/jets/shape_jets.rs b/crates/zkvm-jetpack/src/jets/shape_jets.rs index 3b82a0b4e..b43451955 100644 --- a/crates/zkvm-jetpack/src/jets/shape_jets.rs +++ b/crates/zkvm-jetpack/src/jets/shape_jets.rs @@ -6,14 +6,16 @@ use nockvm::noun::Noun; use crate::form::shape::{dyck, leaf_sequence}; pub fn leaf_sequence_jet(context: &mut Context, subject: Noun) -> Result { - let t = slot(subject, 6)?; - leaf_sequence(&mut context.stack, t) + let space = context.stack.noun_space(); + let t = slot(subject, 6, &space)?; + leaf_sequence(&mut context.stack, t, &space) } pub fn dyck_jet(context: &mut Context, subject: Noun) -> Result { + let space = context.stack.noun_space(); let stack = &mut context.stack; - let t = slot(subject, 6)?; - dyck(stack, t) + let t = slot(subject, 6, &space)?; + dyck(stack, t, &space) } #[cfg(test)] diff --git a/crates/zkvm-jetpack/src/jets/table_utils.rs b/crates/zkvm-jetpack/src/jets/table_utils.rs index 7fb3138dd..3af1e6f52 100644 --- a/crates/zkvm-jetpack/src/jets/table_utils.rs +++ b/crates/zkvm-jetpack/src/jets/table_utils.rs @@ -1,5 +1,5 @@ use nockvm::jets::JetErr; -use nockvm::noun::Noun; +use nockvm::noun::{Noun, NounSpace}; use crate::form::belt::Belt; use crate::form::felt::*; @@ -53,10 +53,10 @@ pub struct MegaExtChals { pub gam: Felt, } -pub fn init_ext_chals(chals: Noun) -> Result { +pub fn init_ext_chals(chals: Noun, space: &NounSpace) -> Result { let mut belts: Vec = Vec::::with_capacity(100); - for b in HoonList::try_from(chals)?.into_iter() { - belts.push(b.as_atom()?.as_u64()?); + for b in HoonList::try_from(chals, space)?.into_iter() { + belts.push(b.in_space(space).as_atom()?.as_u64()?); } let mut felts: Vec = Vec::::with_capacity(30); for trip in belts.chunks(3) { @@ -89,10 +89,10 @@ pub fn init_ext_chals(chals: Noun) -> Result { }) } -pub fn init_mega_ext_chals(chals: Noun) -> Result { +pub fn init_mega_ext_chals(chals: Noun, space: &NounSpace) -> Result { let mut belts: Vec = Vec::::with_capacity(100); - for b in HoonList::try_from(chals)?.into_iter() { - belts.push(b.as_atom()?.as_u64()?); + for b in HoonList::try_from(chals, space)?.into_iter() { + belts.push(b.in_space(space).as_atom()?.as_u64()?); } let mut felts: Vec = Vec::::with_capacity(30); for trip in belts.chunks(3) { diff --git a/crates/zkvm-jetpack/src/jets/tip5_jets.rs b/crates/zkvm-jetpack/src/jets/tip5_jets.rs index cfa37bb7f..149de4cf8 100644 --- a/crates/zkvm-jetpack/src/jets/tip5_jets.rs +++ b/crates/zkvm-jetpack/src/jets/tip5_jets.rs @@ -3,7 +3,7 @@ use nockvm::interpreter::Context; use nockvm::jets::util::{slot, BAIL_FAIL}; use nockvm::jets::JetErr; use nockvm::mem::NockStack; -use nockvm::noun::{Atom, Noun, D, T}; +use nockvm::noun::{Atom, Noun, NounSpace, D, T}; use nockvm_macros::tas; use crate::form::belt::{mont_reduction, montify, montiply, Belt}; @@ -17,7 +17,10 @@ use crate::utils::{ vec_to_hoon_list, vecnoun_to_hoon_list, }; -pub fn hoon_list_to_sponge(list: Noun) -> Result<[u64; tip5::STATE_SIZE], JetErr> { +pub fn hoon_list_to_sponge( + list: Noun, + space: &NounSpace, +) -> Result<[u64; tip5::STATE_SIZE], JetErr> { if list.is_atom() { return Err(BAIL_FAIL); } @@ -27,9 +30,9 @@ pub fn hoon_list_to_sponge(list: Noun) -> Result<[u64; tip5::STATE_SIZE], JetErr let mut i = 0; while current.is_cell() { - let cell = current.as_cell()?; + let cell = current.in_space(space).as_cell()?; sponge[i] = cell.head().as_atom()?.as_u64()?; - current = cell.tail(); + current = cell.tail().noun(); i += 1; } @@ -41,9 +44,10 @@ pub fn hoon_list_to_sponge(list: Noun) -> Result<[u64; tip5::STATE_SIZE], JetErr } pub fn permutation_jet(context: &mut Context, subject: Noun) -> Result { + let space = context.stack.noun_space(); let stack = &mut context.stack; - let sample = slot(subject, 6)?; - let mut sponge = hoon_list_to_sponge(sample)?; + let sample = slot(subject, 6, &space)?; + let mut sponge = hoon_list_to_sponge(sample, &space)?; tip5::permute(&mut sponge); let new_sponge = vec_to_hoon_list(stack, &sponge); @@ -52,9 +56,10 @@ pub fn permutation_jet(context: &mut Context, subject: Noun) -> Result Result { + let space = context.stack.noun_space(); let stack = &mut context.stack; - let input = slot(subject, 6)?; - let mut input_vec = hoon_list_to_vecbelt(input)?; + let input = slot(subject, 6, &space)?; + let mut input_vec = hoon_list_to_vecbelt(input, &space)?; let digest = tip5::hash::hash_varlen(&mut input_vec); @@ -62,9 +67,10 @@ pub fn hash_varlen_jet(context: &mut Context, subject: Noun) -> Result Result { + let space = context.stack.noun_space(); let stack = &mut context.stack; - let sam = slot(subject, 6)?; - let x = sam.as_atom()?.as_u64()?; + let sam = slot(subject, 6, &space)?; + let x = sam.in_space(&space).as_atom()?.as_u64()?; let res = montify(x); @@ -72,24 +78,27 @@ pub fn montify_jet(context: &mut Context, subject: Noun) -> Result } pub fn montiply_jet(context: &mut Context, subject: Noun) -> Result { - let sam = slot(subject, 6)?; - let a = sam.as_cell()?.head().as_atom()?.as_u64()?; - let b = sam.as_cell()?.tail().as_atom()?.as_u64()?; + let space = context.stack.noun_space(); + let sam = slot(subject, 6, &space)?; + let sam_cell = sam.in_space(&space).as_cell()?; + let a = sam_cell.head().as_atom()?.as_u64()?; + let b = sam_cell.tail().as_atom()?.as_u64()?; Ok(belt_as_noun(&mut context.stack, Belt(montiply(a, b)))) } pub fn mont_reduction_jet(context: &mut Context, subject: Noun) -> Result { - let sam = slot(subject, 6)?; - let x_atom = sam.as_atom()?; + let space = context.stack.noun_space(); + let sam = slot(subject, 6, &space)?; + let x_atom = sam.in_space(&space).as_atom()?; let x_u128: u128 = if x_atom.is_indirect() { - if x_atom.as_indirect()?.size() > 2 { + if x_atom.size() > 2 { // mont_reduction asserts that x < RP, so u128 should be sufficient anyway??!! let x_bitslice = x_atom.as_bitslice(); assert!(fits_in_u128(x_bitslice)); bitslice_to_u128(x_bitslice) - } else if x_atom.as_indirect()?.size() == 2 { - let x = unsafe { x_atom.as_u64_pair()? }; + } else if x_atom.size() == 2 { + let x = x_atom.as_u64_pair()?; ((x[1] as u128) << 64u128) + (x[0] as u128) } else { x_atom.as_u64()? as u128 @@ -105,9 +114,10 @@ pub fn mont_reduction_jet(context: &mut Context, subject: Noun) -> Result Result { + let space = context.stack.noun_space(); let stack = &mut context.stack; - let input = slot(subject, 6)?; - tip5::hash::hash_belts_list(stack, input) + let input = slot(subject, 6, &space)?; + tip5::hash::hash_belts_list(stack, input, &space) } pub fn digest_to_noundigest(stack: &mut NockStack, digest: [u64; 5]) -> Noun { @@ -122,9 +132,10 @@ pub fn digest_to_noundigest(stack: &mut NockStack, digest: [u64; 5]) -> Noun { //hash-10: hash list of 10 belts into a list of 5 belts pub fn hash_10_jet(context: &mut Context, subject: Noun) -> Result { + let space = context.stack.noun_space(); let stack = &mut context.stack; - let input = slot(subject, 6)?; - let mut input_vec = hoon_list_to_vecbelt(input)?; + let input = slot(subject, 6, &space)?; + let mut input_vec = hoon_list_to_vecbelt(input, &space)?; let digest = tip5::hash::hash_10(&mut input_vec); @@ -132,14 +143,19 @@ pub fn hash_10_jet(context: &mut Context, subject: Noun) -> Result } pub fn hash_pairs_jet(context: &mut Context, subject: Noun) -> Result { + let space = context.stack.noun_space(); let stack = &mut context.stack; - let lis_noun = slot(subject, 6)?; // (list (list @)) + let lis_noun = slot(subject, 6, &space)?; // (list (list @)) - hash_pairs(stack, lis_noun) + hash_pairs(stack, lis_noun, &space) } -pub fn hash_pairs(stack: &mut NockStack, lis_noun: Noun) -> Result { - let lis = hoon_list_to_vecnoun(lis_noun)?; +pub fn hash_pairs( + stack: &mut NockStack, + lis_noun: Noun, + space: &NounSpace, +) -> Result { + let lis = hoon_list_to_vecnoun(lis_noun, space)?; let lent_lis = lis.len(); assert!(lent_lis > 0); @@ -150,8 +166,8 @@ pub fn hash_pairs(stack: &mut NockStack, lis_noun: Noun) -> Result if (b + 1) == lent_lis { res.push(lis[b]); } else { - let b0 = hoon_list_to_vecbelt(lis[b])?; - let mut b1 = hoon_list_to_vecbelt(lis[b + 1])?; + let b0 = hoon_list_to_vecbelt(lis[b], space)?; + let mut b1 = hoon_list_to_vecbelt(lis[b + 1], space)?; let mut pair = b0; pair.append(&mut b1); let digest = tip5::hash::hash_10(&mut pair); @@ -160,19 +176,20 @@ pub fn hash_pairs(stack: &mut NockStack, lis_noun: Noun) -> Result } } - Ok(vecnoun_to_hoon_list(stack, res.as_slice())) + Ok(vecnoun_to_hoon_list(stack, res.as_slice(), space)) } pub fn hash_ten_cell_jet(context: &mut Context, subject: Noun) -> Result { + let space = context.stack.noun_space(); let stack = &mut context.stack; - let ten_cell = slot(subject, 6)?; // [noun-digest noun-digest] - hash_ten_cell(stack, ten_cell) + let ten_cell = slot(subject, 6, &space)?; // [noun-digest noun-digest] + hash_ten_cell(stack, ten_cell, &space) } -fn hash_ten_cell(stack: &mut NockStack, ten_cell: Noun) -> Result { +fn hash_ten_cell(stack: &mut NockStack, ten_cell: Noun, space: &NounSpace) -> Result { // leaf_sequence(ten-cell) let mut leaf: Vec = Vec::::new(); - crate::form::shape::do_leaf_sequence(ten_cell, &mut leaf)?; + crate::form::shape::do_leaf_sequence(ten_cell, &mut leaf, space)?; let mut leaf_belt = leaf.into_iter().map(Belt).collect(); // list-to-tuple hash10 @@ -181,90 +198,101 @@ fn hash_ten_cell(stack: &mut NockStack, ten_cell: Noun) -> Result } pub fn hash_noun_varlen_jet(context: &mut Context, subject: Noun) -> Result { + let space = context.stack.noun_space(); let stack = &mut context.stack; - let n = slot(subject, 6)?; - tip5::hash::hash_noun_varlen(stack, n) + let n = slot(subject, 6, &space)?; + tip5::hash::hash_noun_varlen(stack, n, &space) } pub fn hash_hashable_jet(context: &mut Context, subject: Noun) -> Result { + let space = context.stack.noun_space(); let stack = &mut context.stack; - let h = slot(subject, 6)?; + let h = slot(subject, 6, &space)?; - hash_hashable(stack, h) + hash_hashable(stack, h, &space) } -pub fn hash_hashable(stack: &mut NockStack, h: Noun) -> Result { +pub fn hash_hashable(stack: &mut NockStack, h: Noun, space: &NounSpace) -> Result { if !h.is_cell() { return Err(BAIL_FAIL); } - let h_head = h.as_cell()?.head(); - let h_tail = h.as_cell()?.tail(); + let h_cell = h.in_space(space).as_cell()?; + let h_head = h_cell.head().noun(); + let h_tail = h_cell.tail().noun(); if h_head.is_direct() { let tag = h_head.as_direct()?; match tag.data() { tas!(b"hash") => hash_hashable_hash(stack, h_tail), - tas!(b"leaf") => hash_hashable_leaf(stack, h_tail), - tas!(b"list") => hash_hashable_list(stack, h_tail), - tas!(b"mary") => hash_hashable_mary(stack, h_tail), - _ => hash_hashable_other(stack, h_head, h_tail), + tas!(b"leaf") => hash_hashable_leaf(stack, h_tail, space), + tas!(b"list") => hash_hashable_list(stack, h_tail, space), + tas!(b"mary") => hash_hashable_mary(stack, h_tail, space), + _ => hash_hashable_other(stack, h_head, h_tail, space), } } else { - hash_hashable_other(stack, h_head, h_tail) + hash_hashable_other(stack, h_head, h_tail, space) } } fn hash_hashable_hash(_stack: &mut NockStack, p: Noun) -> Result { Ok(p) } -fn hash_hashable_leaf(stack: &mut NockStack, p: Noun) -> Result { - tip5::hash::hash_noun_varlen(stack, p) +fn hash_hashable_leaf(stack: &mut NockStack, p: Noun, space: &NounSpace) -> Result { + tip5::hash::hash_noun_varlen(stack, p, space) } -fn hash_hashable_list(stack: &mut NockStack, p: Noun) -> Result { - let turn: Vec = HoonList::try_from(p)? +fn hash_hashable_list(stack: &mut NockStack, p: Noun, space: &NounSpace) -> Result { + let turn: Vec = HoonList::try_from(p, space)? .into_iter() - .map(|x| hash_hashable(stack, x).expect("hash_hashable should succeed for list element")) + .map(|x| { + hash_hashable(stack, x, space).expect("hash_hashable should succeed for list element") + }) .collect(); - let turn_list = vecnoun_to_hoon_list(stack, &turn); - tip5::hash::hash_noun_varlen(stack, turn_list) + let turn_list = vecnoun_to_hoon_list(stack, &turn, space); + tip5::hash::hash_noun_varlen(stack, turn_list, space) } -fn hash_hashable_mary(stack: &mut NockStack, p: Noun) -> Result { - let (ma_step, ma_array_len, _ma_array_dat) = get_mary_fields(p)?; +fn hash_hashable_mary(stack: &mut NockStack, p: Noun, space: &NounSpace) -> Result { + let (ma_step, ma_array_len, _ma_array_dat) = get_mary_fields(p, space)?; - let ma_changed = change_step(stack, p, D(1))?; - let [_ma_changed_step, ma_changed_array] = ma_changed.uncell()?; // +$ mary [step=@ =array] - let bpoly_list = bpoly_to_list(stack, ma_changed_array)?; - let hash_belts_list = tip5::hash::hash_belts_list(stack, bpoly_list)?; + let ma_changed = change_step(stack, p, D(1), space)?; + let [_ma_changed_step, ma_changed_array] = ma_changed.uncell(space)?; // +$ mary [step=@ =array] + let bpoly_list = bpoly_to_list(stack, ma_changed_array, space)?; + let hash_belts_list = tip5::hash::hash_belts_list(stack, bpoly_list, space)?; let leaf_step = T(stack, &[D(tas!(b"leaf")), ma_step.as_noun()]); let leaf_len = T(stack, &[D(tas!(b"leaf")), ma_array_len.as_noun()]); let hash = T(stack, &[D(tas!(b"hash")), hash_belts_list]); let arg = T(stack, &[leaf_step, leaf_len, hash]); - hash_hashable(stack, arg) + hash_hashable(stack, arg, space) } -fn hash_hashable_other(stack: &mut NockStack, p: Noun, q: Noun) -> Result { - let ph = hash_hashable(stack, p)?; - let qh = hash_hashable(stack, q)?; +fn hash_hashable_other( + stack: &mut NockStack, + p: Noun, + q: Noun, + space: &NounSpace, +) -> Result { + let ph = hash_hashable(stack, p, space)?; + let qh = hash_hashable(stack, q, space)?; let cell = T(stack, &[ph, qh]); - hash_ten_cell(stack, cell) + hash_ten_cell(stack, cell, space) } pub fn digest_to_atom_jet(context: &mut Context, subject: Noun) -> Result { + let space = context.stack.noun_space(); let stack = &mut context.stack; - let cells = slot(subject, 6)?; - let [a, b, c, d, e] = cells.uncell()?; - - let a_big = a.as_atom()?.as_ubig(stack); - let b_big = b.as_atom()?.as_ubig(stack); - let c_big = c.as_atom()?.as_ubig(stack); - let d_big = d.as_atom()?.as_ubig(stack); - let e_big = e.as_atom()?.as_ubig(stack); + let cells = slot(subject, 6, &space)?; + let [a, b, c, d, e] = cells.uncell(&space)?; + + let a_big = a.in_space(&space).as_atom()?.as_ubig(stack); + let b_big = b.in_space(&space).as_atom()?.as_ubig(stack); + let c_big = c.in_space(&space).as_atom()?.as_ubig(stack); + let d_big = d.in_space(&space).as_atom()?.as_ubig(stack); + let e_big = e.in_space(&space).as_atom()?.as_ubig(stack); // Use stack-aware operations for pow and multiplication let p_ubig = UBig::from(crate::form::belt::PRIME); diff --git a/crates/zkvm-jetpack/src/jets/tip5_sponge.rs b/crates/zkvm-jetpack/src/jets/tip5_sponge.rs index 9357fa803..3995c6de2 100644 --- a/crates/zkvm-jetpack/src/jets/tip5_sponge.rs +++ b/crates/zkvm-jetpack/src/jets/tip5_sponge.rs @@ -3,7 +3,7 @@ use nockvm::interpreter::Context; use nockvm::jets::util::slot; use nockvm::jets::JetErr; use nockvm::mem::NockStack; -use nockvm::noun::{Cell, Noun, T}; +use nockvm::noun::{Cell, Noun, NounSpace, T}; use crate::form::belt::mont_reduction; use crate::form::tip5; @@ -11,7 +11,13 @@ use crate::jets::tip5_jets::*; use crate::utils::*; // edit door values -fn door_edit(stack: &mut NockStack, edit_axis_path: u64, patch: Noun, mut tree: Noun) -> Noun { +fn door_edit( + stack: &mut NockStack, + edit_axis_path: u64, + patch: Noun, + mut tree: Noun, + space: &NounSpace, +) -> Noun { let edit_axis = BitSlice::::from_element(&edit_axis_path); let mut res = patch; @@ -26,24 +32,24 @@ fn door_edit(stack: &mut NockStack, edit_axis_path: u64, patch: Noun, mut tree: } break; }; - if let Ok(tree_cell) = tree.as_cell() { + if let Ok(tree_cell) = tree.in_space(space).as_cell() { cursor -= 1; if edit_axis[cursor] { unsafe { let (cell, cellmem) = Cell::new_raw_mut(stack); *dest = cell.as_noun(); - (*cellmem).head = tree_cell.head(); + (*cellmem).head = tree_cell.head().noun(); dest = &mut ((*cellmem).tail); } - tree = tree_cell.tail(); + tree = tree_cell.tail().noun(); } else { unsafe { let (cell, cellmem) = Cell::new_raw_mut(stack); *dest = cell.as_noun(); - (*cellmem).tail = tree_cell.tail(); + (*cellmem).tail = tree_cell.tail().noun(); dest = &mut ((*cellmem).head); } - tree = tree_cell.head(); + tree = tree_cell.head().noun(); } } else { panic!("Invalid axis for edit"); @@ -53,13 +59,14 @@ fn door_edit(stack: &mut NockStack, edit_axis_path: u64, patch: Noun, mut tree: } pub fn sponge_absorb_jet(context: &mut Context, subject: Noun) -> Result { + let space = context.stack.noun_space(); let stack = &mut context.stack; - let input_noun = slot(subject, 6)?; - let door = slot(subject, 7)?; - let sponge_noun = slot(door, 6)?; + let input_noun = slot(subject, 6, &space)?; + let door = slot(subject, 7, &space)?; + let sponge_noun = slot(door, 6, &space)?; - let mut input_vec = hoon_list_to_vecbelt(input_noun)?; - let mut sponge = hoon_list_to_sponge(sponge_noun)?; + let mut input_vec = hoon_list_to_vecbelt(input_noun, &space)?; + let mut sponge = hoon_list_to_sponge(sponge_noun, &space)?; // assert that input is made of base field elements tip5::hash::assert_all_based(&input_vec); @@ -76,7 +83,7 @@ pub fn sponge_absorb_jet(context: &mut Context, subject: Noun) -> Result Result Result { + let space = context.stack.noun_space(); let stack = &mut context.stack; - let door = slot(subject, 3)?; - let sponge_noun = slot(door, 6)?; - let mut sponge = hoon_list_to_sponge(sponge_noun)?; + let door = slot(subject, 3, &space)?; + let sponge_noun = slot(door, 6, &space)?; + let mut sponge = hoon_list_to_sponge(sponge_noun, &space)?; let mut output = [0u64; tip5::RATE]; for i in 0..tip5::RATE { @@ -116,7 +124,7 @@ pub fn sponge_squeeze_jet(context: &mut Context, subject: Noun) -> Result Result { - let sam = slot(subject, 6)?; - let t = slot(sam, 2)?; - let alf_noun = slot(sam, 3)?; - let Ok(alf) = alf_noun.as_felt() else { + let space = context.stack.noun_space(); + let sam = slot(subject, 6, &space)?; + let t = slot(sam, 2, &space)?; + let alf_noun = slot(sam, 3, &space)?; + let Ok(alf) = alf_noun.as_felt(&space) else { error!("alf not a felt"); return Err(BAIL_FAIL); }; - let tree_data: TreeData = build_tree_data(t, alf)?; + let tree_data: TreeData = build_tree_data(t, alf, &space)?; let (leaf_atom, leaf_res): (IndirectAtom, &mut Felt) = new_handle_mut_felt(&mut context.stack); let (dyck_atom, dyck_res): (IndirectAtom, &mut Felt) = new_handle_mut_felt(&mut context.stack); diff --git a/crates/zkvm-jetpack/src/jets/verifier_jets.rs b/crates/zkvm-jetpack/src/jets/verifier_jets.rs index 78d46333f..57d4ef2a1 100644 --- a/crates/zkvm-jetpack/src/jets/verifier_jets.rs +++ b/crates/zkvm-jetpack/src/jets/verifier_jets.rs @@ -2,25 +2,24 @@ use nockvm::interpreter::Context; use nockvm::jets::util::{slot, BAIL_FAIL}; use nockvm::jets::JetErr; use nockvm::mem::NockStack; -use nockvm::noun::{Atom, Cell, IndirectAtom, Noun}; +use nockvm::noun::{Atom, IndirectAtom, Noun, NounSpace}; use nockvm_macros::tas; use tracing::debug; use crate::form::belt::{bpow, Belt}; use crate::form::felt::*; use crate::form::handle::new_handle_mut_felt; -use crate::form::noun_ext::{AtomMathExt, NounMathExt}; -use crate::form::poly::{BPolySlice, Element, FPolySlice, Poly, PolySlice}; +use crate::form::noun_ext::{AtomMathExt, NounMathExt, NounMathExtHandle}; +use crate::form::poly::{BPolySlice, Element, FPolySlice, Poly}; use crate::form::structs::{HoonList, HoonMapIter}; use crate::jets::proof_gen_jets::{MPUltra, ProofMap}; pub struct IndexFeltMap(pub ProofMap); pub struct IndexBeltMap(pub ProofMap); -impl TryFrom for IndexFeltMap { - type Error = JetErr; - - fn try_from(hoon_map: Noun) -> Result { - let hoon_map = HoonMapIter::from(hoon_map); +impl IndexFeltMap { + pub fn try_from(hoon_map: Noun, space: &NounSpace) -> Result { + let hoon_map_handle = hoon_map.in_space(space); + let hoon_map = HoonMapIter::new(&hoon_map_handle); let mut map = ProofMap::::new(); for term_noun in hoon_map.into_iter() { @@ -35,7 +34,7 @@ impl TryFrom for IndexFeltMap { }); ( term_cell.head().as_atom()?.as_u64()? as usize, - *term_cell.tail().as_atom()?.as_felt()?, + *term_cell.tail().as_atom()?.atom().as_felt(space)?, ) }; @@ -45,11 +44,10 @@ impl TryFrom for IndexFeltMap { } } -impl TryFrom for IndexBeltMap { - type Error = JetErr; - - fn try_from(hoon_map: Noun) -> Result { - let hoon_map = HoonMapIter::from(hoon_map); +impl IndexBeltMap { + pub fn try_from(hoon_map: Noun, space: &NounSpace) -> Result { + let hoon_map_handle = hoon_map.in_space(space); + let hoon_map = HoonMapIter::new(&hoon_map_handle); let mut map = ProofMap::::new(); for term_noun in hoon_map.into_iter() { @@ -64,7 +62,7 @@ impl TryFrom for IndexBeltMap { }); ( term_cell.head().as_atom()?.as_u64()? as usize, - term_cell.tail().as_atom()?.as_belt()?, + term_cell.tail().as_atom()?.atom().as_belt(space)?, ) }; @@ -75,89 +73,94 @@ impl TryFrom for IndexBeltMap { } pub fn evaluate_deep_jet(context: &mut Context, subject: Noun) -> Result { - let sam = slot(subject, 6)?; - let mut sam_cur: Cell = sam.as_cell()?; + let space = context.stack.noun_space(); + let sam = slot(subject, 6, &space)?; + let mut sam_cur = sam.in_space(&space).as_cell()?; // Extract all parameters from the subject - let trace_evaluations = sam_cur.head(); + let trace_evaluations = sam_cur.head().noun(); sam_cur = sam_cur.tail().as_cell()?; - let comp_evaluations = sam_cur.head(); + let comp_evaluations = sam_cur.head().noun(); sam_cur = sam_cur.tail().as_cell()?; - let trace_elems = sam_cur.head(); + let trace_elems = sam_cur.head().noun(); sam_cur = sam_cur.tail().as_cell()?; - let comp_elems = sam_cur.head(); + let comp_elems = sam_cur.head().noun(); sam_cur = sam_cur.tail().as_cell()?; - let num_comp_pieces = sam_cur.head(); + let num_comp_pieces = sam_cur.head().noun(); sam_cur = sam_cur.tail().as_cell()?; - let weights = sam_cur.head(); + let weights = sam_cur.head().noun(); sam_cur = sam_cur.tail().as_cell()?; - let heights = sam_cur.head(); + let heights = sam_cur.head().noun(); sam_cur = sam_cur.tail().as_cell()?; - let full_widths = sam_cur.head(); + let full_widths = sam_cur.head().noun(); sam_cur = sam_cur.tail().as_cell()?; - let omega = sam_cur.head(); + let omega = sam_cur.head().noun(); sam_cur = sam_cur.tail().as_cell()?; - let index = sam_cur.head(); + let index = sam_cur.head().noun(); sam_cur = sam_cur.tail().as_cell()?; - let deep_challenge = sam_cur.head(); - let new_comp_eval = sam_cur.tail(); + let deep_challenge = sam_cur.head().noun(); + let new_comp_eval = sam_cur.tail().noun(); // Convert nouns to appropriate types - let Ok(trace_evaluations) = FPolySlice::try_from(trace_evaluations) else { + let Ok(trace_evaluations) = FPolySlice::try_from(trace_evaluations, &space) else { debug!("trace_evaluations is not a valid FPolySlice"); return Err(BAIL_FAIL); }; - let Ok(comp_evaluations) = FPolySlice::try_from(comp_evaluations) else { + let Ok(comp_evaluations) = FPolySlice::try_from(comp_evaluations, &space) else { debug!("comp_evaluations is not a valid FPolySlice"); return Err(BAIL_FAIL); }; - let trace_elems: Vec = HoonList::try_from(trace_elems)? + let trace_elems: Vec = HoonList::try_from(trace_elems, &space)? .into_iter() .map(|x| { - x.as_atom() + x.in_space(&space) + .as_atom() .expect("trace_elems element should be an atom") .as_u64() .expect("trace_elems element should be a u64") }) .map(Belt) .collect(); - let comp_elems: Vec = HoonList::try_from(comp_elems)? + let comp_elems: Vec = HoonList::try_from(comp_elems, &space)? .into_iter() .map(|x| { - x.as_atom() + x.in_space(&space) + .as_atom() .expect("comp_elems element should be an atom") .as_u64() .expect("comp_elems element should be a u64") }) .map(Belt) .collect(); - let num_comp_pieces = num_comp_pieces.as_atom()?.as_u64()?; - let Ok(weights) = FPolySlice::try_from(weights) else { + let num_comp_pieces = num_comp_pieces.in_space(&space).as_atom()?.as_u64()?; + let Ok(weights) = FPolySlice::try_from(weights, &space) else { debug!("weights is not a valid FPolySlice"); return Err(BAIL_FAIL); }; - let heights: Vec = HoonList::try_from(heights)? + let heights: Vec = HoonList::try_from(heights, &space)? .into_iter() .map(|x| { - x.as_atom() + x.in_space(&space) + .as_atom() .expect("heights element should be an atom") .as_u64() .expect("heights element should be a u64") }) .collect(); - let full_widths: Vec = HoonList::try_from(full_widths)? + let full_widths: Vec = HoonList::try_from(full_widths, &space)? .into_iter() .map(|x| { - x.as_atom() + x.in_space(&space) + .as_atom() .expect("full_widths element should be an atom") .as_u64() .expect("full_widths element should be a u64") }) .collect(); - let omega = omega.as_felt()?; - let index = index.as_atom()?.as_u64()?; - let deep_challenge = deep_challenge.as_felt()?; - let new_comp_eval = new_comp_eval.as_felt()?; + let omega = omega.as_felt(&space)?; + let index = index.in_space(&space).as_atom()?.as_u64()?; + let deep_challenge = deep_challenge.as_felt(&space)?; + let new_comp_eval = new_comp_eval.as_felt(&space)?; // TODO use g defined wherever it is let g = Felt::lift(Belt(7)); @@ -268,7 +271,7 @@ trait Fops: Element + Copy + core::ops::Add + core::ops::Mul + PartialEq + Eq { fn to_noun(self, stack: &mut NockStack) -> Noun; - fn from_noun(noun: Noun) -> Result; + fn from_noun(noun: Noun, space: &NounSpace) -> Result; // =/ pow-op ?:(=(field %base) bpow fpow) fn pow(&self, exp: u64) -> Self; // =/ lift-op ?:(=(field %base) |=(v=@ `@ux`v) lift) @@ -280,8 +283,8 @@ impl Fops for Belt { Atom::new(stack, self.0).as_noun() } - fn from_noun(noun: Noun) -> Result { - Ok(Belt(noun.as_atom()?.as_u64()?)) + fn from_noun(noun: Noun, space: &NounSpace) -> Result { + Ok(Belt(noun.in_space(space).as_atom()?.as_u64()?)) } fn pow(&self, exp: u64) -> Self { @@ -300,8 +303,8 @@ impl Fops for Felt { a.as_noun() } - fn from_noun(noun: Noun) -> Result { - if let Ok(r) = noun.as_felt() { + fn from_noun(noun: Noun, space: &NounSpace) -> Result { + if let Ok(r) = noun.as_felt(space) { Ok(*r) } else { Err(BAIL_FAIL) @@ -326,29 +329,30 @@ pub fn mpeval_jet(context: &mut Context, subject: Noun) -> Result // com-map=(map @ elt) // == // ^- elt - let sam = slot(subject, 6)?; + let space = context.stack.noun_space(); + let sam = slot(subject, 6, &space)?; let stack = &mut context.stack; - let [field, mp, args, chal_map, dyns, com_map] = sam.uncell()?; - let chals = BPolySlice::try_from(chal_map)?.0; + let [field, mp, args, chal_map, dyns, com_map] = sam.uncell(&space)?; + let chals = BPolySlice::try_from(chal_map, &space)?.0; - let Ok(dyns) = BPolySlice::try_from(dyns) else { + let Ok(dyns) = BPolySlice::try_from(dyns, &space) else { return Err(BAIL_FAIL); }; let ret = match field.as_direct()?.data() { tas!(b"ext") => { - let Ok(com_map) = IndexFeltMap::try_from(com_map) else { + let Ok(com_map) = IndexFeltMap::try_from(com_map, &space) else { return Err(BAIL_FAIL); }; - let args = FPolySlice::try_from(args)?.0; - mpeval::(mp, args, chals, dyns.0, Some(&com_map.0))?.to_noun(stack) + let args = FPolySlice::try_from(args, &space)?.0; + mpeval::(mp, args, chals, dyns.0, Some(&com_map.0), &space)?.to_noun(stack) } tas!(b"base") => { - let Ok(com_map) = IndexBeltMap::try_from(com_map) else { + let Ok(com_map) = IndexBeltMap::try_from(com_map, &space) else { return Err(BAIL_FAIL); }; - let args = BPolySlice::try_from(args)?.0; - mpeval::(mp, args, chals, dyns.0, Some(&com_map.0))?.to_noun(stack) + let args = BPolySlice::try_from(args, &space)?.0; + mpeval::(mp, args, chals, dyns.0, Some(&com_map.0), &space)?.to_noun(stack) } _ => return Err(BAIL_FAIL), }; @@ -362,11 +366,9 @@ fn mpeval( chals: &[Belt], dyns: &[Belt], com_map: Option<&ProofMap>, + space: &NounSpace, //com_map: Noun, -) -> Result -where - for<'a> PolySlice<'a, F>: TryFrom, -{ +) -> Result { /* let Ok(args) = PolySlice::try_from(args) else { return jet_err(); @@ -390,15 +392,16 @@ where // %+ roll ~(tap by mp) // |= [[k=bpoly v=belt] acc=_init-zero] - let mut mp = HoonMapIter::from(mp); + let mp_handle = mp.in_space(space); + let mut mp = HoonMapIter::new(&mp_handle); mp.try_fold(F::zero(), |acc, n| { let [k, v] = n.uncell()?; - let Ok(k) = BPolySlice::try_from(k) else { + let Ok(k) = BPolySlice::try_from(k.noun(), space) else { return Err(BAIL_FAIL); }; - let v = Belt::from_noun(v)?; + let v = Belt::from_noun(v.noun(), space)?; // =/ coeff=@ux (lift-op v) let coeff = F::lift(v); // ?: =(init-zero coeff) @@ -486,25 +489,26 @@ pub fn mpeval_ultra_felt( args: &[Felt], chals: &[Belt], dyns: &[Belt], + space: &NounSpace, ) -> Result, JetErr> { match mp { MPUltra::Mega(mp_mega) => { let mut vec: Vec = Vec::new(); - let eval = mpeval::(*mp_mega, args, chals, dyns, None)?; + let eval = mpeval::(*mp_mega, args, chals, dyns, None, space)?; vec.push(eval); Ok(vec) } MPUltra::Comp(mp_comp) => { let mut deps = ProofMap::new(); for (i, dep) in mp_comp.dep.iter().enumerate() { - let eval = mpeval::(*dep, args, chals, dyns, None)?; + let eval = mpeval::(*dep, args, chals, dyns, None, space)?; deps.insert(i, eval); } mp_comp .com .iter() - .map(|com| mpeval::(*com, args, chals, dyns, Some(&deps))) + .map(|com| mpeval::(*com, args, chals, dyns, Some(&deps), space)) .collect() } } diff --git a/crates/zkvm-jetpack/src/jets/zoon_jets.rs b/crates/zkvm-jetpack/src/jets/zoon_jets.rs index e89aafd87..1d5e10442 100644 --- a/crates/zkvm-jetpack/src/jets/zoon_jets.rs +++ b/crates/zkvm-jetpack/src/jets/zoon_jets.rs @@ -1,30 +1,77 @@ +//! Zoon and h-zoon jets, including the balance state-migration fast path. +//! +//! The slow consensus migration is not ordinary z-map to h-map conversion. The +//! balance field is a z-mip whose outer key is a block id and whose value is a +//! full note-balance z-map snapshot for that block: +//! +//! ```text +//! blocks: block-id -> page(parent-id, ...) +//! balance: block-id -> full note-balance snapshot +//! ``` +//! +//! The Hoon arm `+zh-balmilt` still means `(zh-milt balance)`. That simple +//! fallback is the protocol oracle. The Rust jet gets `blocks` only as an +//! evaluation hint, then uses it to recover parent order across snapshots. +//! +//! The speedup comes from exploiting persistent z-map structure. Adjacent blocks +//! usually spend or create a few notes, while the child snapshot shares most of +//! its source tree cells with the parent snapshot. Generic `zh-milt` cannot see +//! those parent links, which makes it repeatedly walk nearly identical full +//! balance maps. The balance jet converts parents first, memoizes exact +//! repeated z-map roots by noun identity, and converts a child from the +//! already-converted parent by collecting z-map deltas. +//! +//! Delta conversion is deliberately conservative: +//! +//! - identical z-map roots reuse the parent h-map; +//! - matching treap nodes recurse only through changed subtrees; +//! - small local rotations flatten and compare by compact hashed key; +//! - duplicate compact keys, malformed trees, missing parents, cycles, and large +//! rotation regions bail to generic conversion; +//! - deletes run before puts, preserving changes where two key nouns have the +//! same compact digest. +//! +//! The fast path is permitted to reduce work only. Outer entries are rebuilt +//! with the original key nouns, snapshot-level fallback still shares the generic +//! memo, and test-jet mode compares the Rust noun against the Hoon oracle. + +use std::cmp::Ordering; +use std::collections::HashMap; + use nockchain_math::belt::Belt; +use nockchain_math::noun_ext::NounMathExt; use nockchain_math::tip5::hash::{hash_10, hash_noun_varlen_digest}; use nockchain_math::zoon::common::{dor_tip, lth_tip}; use nockvm::interpreter::Context; -use nockvm::jets::util::slot; +use nockvm::jets::util::{slot, BAIL_FAIL}; use nockvm::jets::JetErr; -use nockvm::noun::{Noun, D, NO, T, YES}; +use nockvm::noun::{Cell, CellMemory, Noun, NounAllocator, NounSpace, D, NO, T, YES}; use nockvm_macros::tas; use noun_serde::NounDecode; +use rayon::slice::ParallelSliceMut; use crate::jets::tip5_jets::digest_to_noundigest; const TIP_CACHE_TAG: u64 = tas!(b"zntip"); const DOUBLE_TIP_CACHE_TAG: u64 = tas!(b"zndtip"); +const PARALLEL_SORT_THRESHOLD: usize = 4096; +const SMALL_HASHED_KEY_LIMIT: usize = 2; +const Z_DIFF_ROTATION_ENTRY_LIMIT: usize = 512; pub fn dor_tip_jet(context: &mut Context, subject: Noun) -> Result { - let sam = slot(subject, 6)?; - let mut a = slot(sam, 2)?; - let mut b = slot(sam, 3)?; + let space = context.stack.noun_space(); + let sam = slot(subject, 6, &space)?; + let mut a = slot(sam, 2, &space)?; + let mut b = slot(sam, 3, &space)?; Ok(bool_to_noun(dor_tip(&mut context.stack, &mut a, &mut b)?)) } pub fn gor_tip_jet(context: &mut Context, subject: Noun) -> Result { - let sam = slot(subject, 6)?; - let mut a = slot(sam, 2)?; - let mut b = slot(sam, 3)?; + let space = context.stack.noun_space(); + let sam = slot(subject, 6, &space)?; + let mut a = slot(sam, 2, &space)?; + let mut b = slot(sam, 3, &space)?; let a_tip = get_tip_digest(context, a)?; let b_tip = get_tip_digest(context, b)?; @@ -39,9 +86,10 @@ pub fn gor_tip_jet(context: &mut Context, subject: Noun) -> Result } pub fn mor_tip_jet(context: &mut Context, subject: Noun) -> Result { - let sam = slot(subject, 6)?; - let mut a = slot(sam, 2)?; - let mut b = slot(sam, 3)?; + let space = context.stack.noun_space(); + let sam = slot(subject, 6, &space)?; + let mut a = slot(sam, 2, &space)?; + let mut b = slot(sam, 3, &space)?; let a_tip = get_double_tip_digest(context, a)?; let b_tip = get_double_tip_digest(context, b)?; @@ -55,283 +103,3674 @@ pub fn mor_tip_jet(context: &mut Context, subject: Noun) -> Result Ok(bool_to_noun(ordered)) } -fn bool_to_noun(value: bool) -> Noun { - if value { - YES - } else { - NO - } +/// Jet for `+gor-hip`, the h-zoon key ordering over existing digest limbs. +/// +/// Keys are decoded as a digest list. A direct digest is a one-item list. Each +/// digest compares limbs `[4, 3, 2, 1, 0]`; equal digests continue to the next +/// list item. Equal lists return false, and an equal-prefix shorter list loses. +/// There is no `dor` fallback because h-zoon keys are already consensus digest +/// commitments. +pub fn gor_hip_jet(context: &mut Context, subject: Noun) -> Result { + let space = context.stack.noun_space(); + let sam = slot(subject, 6, &space)?; + let a = slot(sam, 2, &space)?; + let b = slot(sam, 3, &space)?; + + Ok(bool_to_noun(gor_hip(a, b, &space)?)) } -fn cache_lookup_digest( - context: &mut Context, - tag: u64, - noun: Noun, -) -> Result, JetErr> { - let mut key = T(&mut context.stack, &[D(tag), noun]); - match context.cache.lookup(&mut context.stack, &mut key) { - Some(cached) => Ok(Some(<[u64; 5]>::from_noun(&cached)?)), - None => Ok(None), - } +/// Jet for `+mor-hip`, the h-zoon priority ordering over existing digest limbs. +/// +/// This uses the same digest-list rules as `+gor-hip`, but compares digest +/// limbs `[0, 1, 2, 3, 4]`. That is the priority order used to keep the h-tree +/// balanced without hashing the key noun again. +pub fn mor_hip_jet(context: &mut Context, subject: Noun) -> Result { + let space = context.stack.noun_space(); + let sam = slot(subject, 6, &space)?; + let a = slot(sam, 2, &space)?; + let b = slot(sam, 3, &space)?; + + Ok(bool_to_noun(mor_hip(a, b, &space)?)) } -fn cache_insert_digest(context: &mut Context, tag: u64, noun: Noun, digest: [u64; 5]) { - let mut key = T(&mut context.stack, &[D(tag), noun]); - let value = digest_to_noundigest(&mut context.stack, digest); - context.cache = context.cache.insert(&mut context.stack, &mut key, value); +/// Jet for `+zh-molt`, converting a legacy z-map tree into an h-map tree. +pub fn zh_molt_jet(context: &mut Context, subject: Noun) -> Result { + let mut memo = HashMap::new(); + let space = context.stack.noun_space(); + let a = slot(subject, 6, &space)?; + z_map_to_h_map(&mut context.stack, a, &space, &mut memo) } -fn get_tip_digest(context: &mut Context, noun: Noun) -> Result<[u64; 5], JetErr> { - if let Some(cached) = cache_lookup_digest(context, TIP_CACHE_TAG, noun)? { - return Ok(cached); - } +/// Jet for `+zh-silt`, converting a legacy z-set tree into an h-set tree. +pub fn zh_silt_jet(context: &mut Context, subject: Noun) -> Result { + let mut memo = HashMap::new(); + let space = context.stack.noun_space(); + let a = slot(subject, 6, &space)?; + z_set_to_h_set(&mut context.stack, a, &space, &mut memo) +} - let digest = hash_noun_varlen_digest(&mut context.stack, noun)?; - cache_insert_digest(context, TIP_CACHE_TAG, noun, digest); - Ok(digest) +/// Jet for `+zh-milt`, converting a legacy z-mip tree into an h-mip tree. +pub fn zh_milt_jet(context: &mut Context, subject: Noun) -> Result { + let mut memo = ZHConversionMemo::default(); + let space = context.stack.noun_space(); + let a = slot(subject, 6, &space)?; + z_mip_to_h_mip(&mut context.stack, a, &space, &mut memo) } -fn get_double_tip_digest(context: &mut Context, noun: Noun) -> Result<[u64; 5], JetErr> { - if let Some(cached) = cache_lookup_digest(context, DOUBLE_TIP_CACHE_TAG, noun)? { - return Ok(cached); - } +/// Jet for `+zh-balmilt`, the balance-specific state migration converter. +/// +/// This arm exists because balance history has a shape the generic converter +/// cannot see: +/// +/// ```text +/// blocks: block-id -> page(parent-id, ...) +/// balance: block-id -> full note-balance snapshot +/// ``` +/// +/// Each balance snapshot is a complete z-map of spendable notes at that block. +/// Adjacent blocks usually change only a few notes, and the source z-maps share +/// most of their underlying tree cells. Generic `zh-milt` is still perfectly +/// correct, but it only receives `balance`; it does not know which snapshot is +/// the parent of another snapshot. Its memo catches exactly repeated roots, but +/// a changed root with shared subtrees is still walked as a full independent +/// map. High checkpoints turned that into billion-scale logical tree visits. +/// +/// Consensus rule for this jet: +/// +/// - the Hoon fallback is `(zh-milt balance)`, which remains the protocol oracle; +/// - `blocks` is only an optimization hint for choosing parent-first order; +/// - the fast path may reuse a converted parent only by applying z-map diffs; +/// - snapshots without a known converted parent use generic conversion; +/// - outer h-map entries are rebuilt from the original key nouns; +/// - exact key nouns matter, not just compact digest identity; +/// - anything malformed, cyclic, unsupported, or too wide falls back to the +/// generic converter. +/// +/// In test-jet mode the interpreter runs this Rust arm and then evaluates the +/// Hoon fallback on the same sample. Any noun mismatch becomes a `%jest` bail. +pub fn zh_balance_milt_jet(context: &mut Context, subject: Noun) -> Result { + let space = context.stack.noun_space(); + let sam = slot(subject, 6, &space)?; + let blocks = slot(sam, 2, &space)?; + let balance = slot(sam, 3, &space)?; + z_balance_mip_to_h_mip_with_blocks(&mut context.stack, blocks, balance, &space) +} - let tip_digest = get_tip_digest(context, noun)?; - let mut input: Vec = Vec::with_capacity(10); - input.extend(tip_digest.into_iter().map(Belt)); - input.extend(tip_digest.into_iter().map(Belt)); - let digest = hash_10(&mut input); +/// Jet for `+zh-jult`, converting a legacy z-jug tree into an h-jug tree. +pub fn zh_jult_jet(context: &mut Context, subject: Noun) -> Result { + let mut memo = ZHConversionMemo::default(); + let space = context.stack.noun_space(); + let a = slot(subject, 6, &space)?; + z_jug_to_h_jug(&mut context.stack, a, &space, &mut memo) +} - cache_insert_digest(context, DOUBLE_TIP_CACHE_TAG, noun, digest); - Ok(digest) +#[derive(Default)] +struct ZHConversionMemo { + maps: HashMap, + sets: HashMap, + mip_outer: HashMap, + jug_outer: HashMap, } -#[cfg(test)] -mod tests { - use ibig::UBig; - use nockvm::interpreter::Context; - use nockvm::jets::util::test::{init_context, A}; - use nockvm::mem::NockStack; - use nockvm::noun::{Noun, D, T}; - use nockvm::unifying_equality::unifying_equality; - use quickcheck::{Arbitrary, Gen, QuickCheck}; +fn z_map_to_h_map( + stack: &mut A, + tree: Noun, + space: &NounSpace, + memo: &mut HashMap, +) -> Result { + if unsafe { tree.raw_equals(&D(0)) } { + return Ok(h_map_empty()); + } + if tree.is_atom() { + return Err(BAIL_FAIL); + } - use super::*; + let tree_key = noun_identity(tree); + if let Some(converted) = memo.get(&tree_key) { + return Ok(*converted); + } - #[test] - fn dor_tip_matches_hoon_for_mixed_atom_cell_inputs() { - let c = &mut init_context(); - let atom = D(7); - let cell = T(&mut c.stack, &[D(1), D(2)]); + let mut convert_value = |_stack: &mut A, value: Noun| Ok(value); + let converted = z_map_to_h_map_with(stack, tree, space, memo, &mut convert_value)?; + memo.insert(tree_key, converted); + Ok(converted) +} - assert!( - cmp_with_jet(c, dor_tip_jet, atom, cell).expect("dor-tip jet should succeed"), - "expected dor-tip atom( + stack: &mut A, + tree: Noun, + space: &NounSpace, + memo: &mut HashMap, +) -> Result { + if unsafe { tree.raw_equals(&D(0)) } { + return Ok(h_set_empty()); + } + if tree.is_atom() { + return Err(BAIL_FAIL); } - #[test] - fn gor_tip_reuses_tip_cache() { - let c = &mut init_context(); - let a = T(&mut c.stack, &[D(1), D(2)]); - let b = T(&mut c.stack, &[D(3), D(4)]); - let subject = jet_subject(&mut c.stack, a, b); + let tree_key = noun_identity(tree); + if let Some(converted) = memo.get(&tree_key) { + return Ok(*converted); + } - assert_eq!(cache_entries(c), 0); - let _first = gor_tip_jet(c, subject).expect("gor-tip should succeed"); - let after_first = cache_entries(c); - assert!( - after_first >= 2, - "expected tip cache entries for both nouns, found {after_first}" - ); + let converted = z_set_to_h_set_with(stack, tree, space, memo)?; + memo.insert(tree_key, converted); + Ok(converted) +} - let _second = gor_tip_jet(c, subject).expect("gor-tip should succeed"); - let after_second = cache_entries(c); - assert_eq!(after_second, after_first); +fn z_mip_to_h_mip( + stack: &mut A, + tree: Noun, + space: &NounSpace, + memo: &mut ZHConversionMemo, +) -> Result { + if unsafe { tree.raw_equals(&D(0)) } { + return Ok(h_map_empty()); + } + if tree.is_atom() { + return Err(BAIL_FAIL); } - #[test] - fn mor_tip_reuses_tip_and_double_tip_cache() { - let c = &mut init_context(); - let a = T(&mut c.stack, &[D(17), D(23)]); - let b = T(&mut c.stack, &[D(19), D(29)]); - let subject = jet_subject(&mut c.stack, a, b); + let tree_key = noun_identity(tree); + if let Some(converted) = memo.mip_outer.get(&tree_key) { + return Ok(*converted); + } - assert_eq!(cache_entries(c), 0); - let _first = mor_tip_jet(c, subject).expect("mor-tip should succeed"); - let after_mor = cache_entries(c); - assert!( - after_mor >= 4, - "expected tip + double-tip cache entries for both nouns, found {after_mor}" - ); + let mut outer_memo = std::mem::take(&mut memo.mip_outer); + let mut convert_value = + |stack: &mut A, value: Noun| z_map_to_hashed_h_map(stack, value, space, memo); + let converted = z_map_to_h_map_with(stack, tree, space, &mut outer_memo, &mut convert_value); + memo.mip_outer = outer_memo; + let converted = converted?; + memo.mip_outer.insert(tree_key, converted); + Ok(converted) +} - let _second = mor_tip_jet(c, subject).expect("mor-tip should succeed"); - let after_second_mor = cache_entries(c); - assert_eq!(after_second_mor, after_mor); +fn z_map_to_hashed_h_map( + stack: &mut A, + tree: Noun, + space: &NounSpace, + memo: &mut ZHConversionMemo, +) -> Result { + if unsafe { tree.raw_equals(&D(0)) } { + return Ok(h_map_empty()); + } + if tree.is_atom() { + return Err(BAIL_FAIL); + } - let _gor = gor_tip_jet(c, subject).expect("gor-tip should succeed"); - let after_gor = cache_entries(c); - assert_eq!(after_gor, after_mor); + let tree_key = noun_identity(tree); + if let Some(converted) = memo.maps.get(&tree_key) { + return Ok(*converted); } - #[test] - fn jets_error_on_malformed_sample_shape() { - let c = &mut init_context(); - let malformed_subject = T(&mut c.stack, &[D(0), D(42), D(0)]); + let mut convert_value = |_stack: &mut A, value: Noun| Ok(value); + let converted = z_map_to_h_map_with(stack, tree, space, &mut memo.maps, &mut convert_value)?; + memo.maps.insert(tree_key, converted); + Ok(converted) +} - assert!(dor_tip_jet(c, malformed_subject).is_err()); - assert!(gor_tip_jet(c, malformed_subject).is_err()); - assert!(mor_tip_jet(c, malformed_subject).is_err()); +fn z_balance_mip_to_h_mip_with_blocks( + stack: &mut A, + blocks: Noun, + balance: Noun, + space: &NounSpace, +) -> Result { + if unsafe { balance.raw_equals(&D(0)) } { + return Ok(h_map_empty()); + } + if balance.is_atom() { + return Err(BAIL_FAIL); } - #[test] - fn gor_tip_errors_on_non_decodable_tip_cache_entry() { - let c = &mut init_context(); - let a = T(&mut c.stack, &[D(1), D(2)]); - let b = T(&mut c.stack, &[D(3), D(4)]); - inject_bad_cache_value(c, TIP_CACHE_TAG, a); + // Parent links decide only the fast-path order. They do not define the + // result. If the block map is not shaped like consensus state, convert the + // whole balance through the generic `zh-milt` path. + let parents = match collect_block_parents(blocks, space) { + Ok(parents) => parents, + Err(_) => { + let mut memo = ZHConversionMemo::default(); + return z_mip_to_h_mip(stack, balance, space, &mut memo); + } + }; - let subject = jet_subject(&mut c.stack, a, b); - assert!(gor_tip_jet(c, subject).is_err()); + let mut entries = Vec::new(); + collect_z_map_entries(balance, space, &mut entries)?; + let mut snapshots = Vec::with_capacity(entries.len()); + let mut index_by_block = HashMap::with_capacity(entries.len()); + for entry in entries { + let block_digest = match digest_from_noun(entry.key, space) { + Ok(block_digest) => block_digest, + Err(_) => { + // Generic h-zoon accepts every valid hashed key noun. The + // balance fast path needs direct block digests to join a + // balance snapshot to `blocks`, and it must also preserve the + // original outer key noun when rebuilding the output. If those + // two facts diverge, the whole balance goes through the oracle + // path. + let mut memo = ZHConversionMemo::default(); + return z_mip_to_h_mip(stack, balance, space, &mut memo); + } + }; + let index = snapshots.len(); + snapshots.push(BalanceSnapshot { + block_id: entry.key, + block_digest, + z_balance: entry.value, + }); + index_by_block.insert(block_digest, index); } - #[test] - fn mor_tip_errors_on_non_decodable_double_tip_cache_entry() { - let c = &mut init_context(); - let a = T(&mut c.stack, &[D(5), D(6)]); - let b = T(&mut c.stack, &[D(7), D(8)]); - inject_bad_cache_value(c, DOUBLE_TIP_CACHE_TAG, a); + let mut state = BalanceConversionState { + stack, + snapshots, + index_by_block, + parents, + space, + converted: Vec::new(), + visiting: Vec::new(), + memo: ZHConversionMemo::default(), + }; + state.converted.resize(state.snapshots.len(), None); + state.visiting.resize(state.snapshots.len(), false); - let subject = jet_subject(&mut c.stack, a, b); - assert!(mor_tip_jet(c, subject).is_err()); + // Convert every snapshot. `memo.maps` catches exact repeated balance roots. + // Parent-derived diffs handle the common persistent-tree case: a new root + // with mostly shared descendants and a small note delta. A missing parent + // only disables this optimization for the affected snapshot. + for index in 0..state.snapshots.len() { + state.convert_index(index)?; } - #[test] - fn gor_and_mor_error_on_non_u64_atom_inputs() { - let c = &mut init_context(); - let huge_atom = A(&mut c.stack, &(UBig::from(1u128) << 64)); - let other = D(1); - - let subject = jet_subject(&mut c.stack, huge_atom, other); - assert!(gor_tip_jet(c, subject).is_err()); - assert!(mor_tip_jet(c, subject).is_err()); + let mut outer_entries = Vec::with_capacity(state.snapshots.len()); + for index in 0..state.snapshots.len() { + let key = state.snapshots[index].block_id; + let value = state.converted[index].ok_or(BAIL_FAIL)?; + outer_entries.push(HMapEntry { + noun: T(state.stack, &[key, value]), + key, + }); } + h_map_from_entries(state.stack, outer_entries, state.space) +} - #[test] - fn quickcheck_order_laws_for_dor_gor_mor() { - fn prop(a: BoundedNounInput, b: BoundedNounInput, c_input: BoundedNounInput) -> bool { - let context = &mut init_context(); - let an = bounded_noun_from_input(&mut context.stack, &a); - let bn = bounded_noun_from_input(&mut context.stack, &b); - let cn = bounded_noun_from_input(&mut context.stack, &c_input); +struct BalanceSnapshot { + block_id: Noun, + block_digest: [u64; 5], + z_balance: Noun, +} - let dor_ab = - cmp_with_jet(context, dor_tip_jet, an, bn).expect("dor comparison should succeed"); - let dor_ba = - cmp_with_jet(context, dor_tip_jet, bn, an).expect("dor comparison should succeed"); - let dor_bc = - cmp_with_jet(context, dor_tip_jet, bn, cn).expect("dor comparison should succeed"); - let dor_ac = - cmp_with_jet(context, dor_tip_jet, an, cn).expect("dor comparison should succeed"); +// Migration-local state for one balance field. +// +// `converted` is indexed by the flattened outer balance entries and stores the +// h-map for each block's note balance. `memo.maps` is the same cache used by +// generic z-map conversion, which keeps repeated roots identical across the +// oracle path and the balance-aware path. +struct BalanceConversionState<'a, A: NounAllocator> { + stack: &'a mut A, + snapshots: Vec, + index_by_block: HashMap<[u64; 5], usize>, + parents: HashMap<[u64; 5], [u64; 5]>, + space: &'a NounSpace, + converted: Vec>, + visiting: Vec, + memo: ZHConversionMemo, +} - if !dor_ab && !dor_ba { - return false; +impl BalanceConversionState<'_, A> { + fn convert_index(&mut self, index: usize) -> Result { + if let Some(converted) = self.cached_conversion(index) { + return Ok(converted); + } + + let mut path = Vec::new(); + let mut current = index; + + // Build the ancestor path iteratively. Earlier recursive code worked on + // the sample checkpoint, but it spent stack on a migration already close + // to the allocator floor. The loop stops at the nearest converted, + // missing, self-parented, or cyclic ancestor. Any snapshot still lacking + // a converted parent then uses generic conversion. + loop { + if self.cached_conversion(current).is_some() { + break; } - if dor_ab && dor_ba && !noun_eq(context, an, bn) { - return false; + if self.visiting[current] { + break; } - if dor_ab && dor_bc && !dor_ac { - return false; + + self.visiting[current] = true; + path.push(current); + + let Some(parent_index) = self.parent_index(current) else { + break; + }; + if parent_index == current || self.visiting[parent_index] { + break; + } + if self.cached_conversion(parent_index).is_some() { + break; } - for jet in [gor_tip_jet, mor_tip_jet] { - let ab = cmp_with_jet(context, jet, an, bn).expect("comparison should succeed"); - let ba = cmp_with_jet(context, jet, bn, an).expect("comparison should succeed"); - let bc = cmp_with_jet(context, jet, bn, cn).expect("comparison should succeed"); - let ac = cmp_with_jet(context, jet, an, cn).expect("comparison should succeed"); + current = parent_index; + } - if !ab && !ba { - return false; // totality - } - if ab && ba && !noun_eq(context, an, bn) { - return false; // antisymmetry - } - if ab && bc && !ac { - return false; // transitivity - } + for &path_index in &path { + self.visiting[path_index] = false; + } + + for path_index in path.into_iter().rev() { + if self.cached_conversion(path_index).is_some() { + continue; } - true + // Try the delta path only when the parent h-map already exists. If + // the source trees do not prove a bounded local change, this single + // snapshot falls back to generic conversion. + let converted = match self.try_convert_from_converted_parent(path_index) { + Ok(Some(converted)) => converted, + Ok(None) | Err(_) => self.convert_generic(path_index)?, + }; + self.store_conversion(path_index, converted); } - QuickCheck::new() - .tests(256) - .quickcheck(prop as fn(BoundedNounInput, BoundedNounInput, BoundedNounInput) -> bool); + self.converted[index].ok_or(BAIL_FAIL) } - #[test] - fn quickcheck_cold_vs_warm_cache_equivalence() { - fn prop(a: BoundedNounInput, b: BoundedNounInput) -> bool { - let mut cold = init_context(); - let cold_a = bounded_noun_from_input(&mut cold.stack, &a); - let cold_b = bounded_noun_from_input(&mut cold.stack, &b); + fn cached_conversion(&mut self, index: usize) -> Option { + if let Some(converted) = self.converted[index] { + return Some(converted); + } - let cold_dor = - cmp_with_jet(&mut cold, dor_tip_jet, cold_a, cold_b).expect("dor should succeed"); - let cold_gor = - cmp_with_jet(&mut cold, gor_tip_jet, cold_a, cold_b).expect("gor should succeed"); - let cold_mor = - cmp_with_jet(&mut cold, mor_tip_jet, cold_a, cold_b).expect("mor should succeed"); + let z_balance = self.snapshots[index].z_balance; + if let Some(converted) = self.memo.maps.get(&noun_identity(z_balance)).copied() { + self.converted[index] = Some(converted); + return Some(converted); + } - let mut warm = init_context(); - let warm_a = bounded_noun_from_input(&mut warm.stack, &a); - let warm_b = bounded_noun_from_input(&mut warm.stack, &b); + None + } - let warm_first_dor = - cmp_with_jet(&mut warm, dor_tip_jet, warm_a, warm_b).expect("dor should succeed"); - let warm_first_gor = - cmp_with_jet(&mut warm, gor_tip_jet, warm_a, warm_b).expect("gor should succeed"); - let warm_first_mor = - cmp_with_jet(&mut warm, mor_tip_jet, warm_a, warm_b).expect("mor should succeed"); + fn store_conversion(&mut self, index: usize, converted: Noun) { + let z_balance = self.snapshots[index].z_balance; + self.memo.maps.insert(noun_identity(z_balance), converted); + self.converted[index] = Some(converted); + } - let warm_second_dor = - cmp_with_jet(&mut warm, dor_tip_jet, warm_a, warm_b).expect("dor should succeed"); - let warm_second_gor = - cmp_with_jet(&mut warm, gor_tip_jet, warm_a, warm_b).expect("gor should succeed"); - let warm_second_mor = - cmp_with_jet(&mut warm, mor_tip_jet, warm_a, warm_b).expect("mor should succeed"); + fn parent_index(&self, index: usize) -> Option { + let block_digest = self.snapshots[index].block_digest; + let parent_digest = self.parents.get(&block_digest)?; + self.index_by_block.get(parent_digest).copied() + } - cold_dor == warm_first_dor - && cold_gor == warm_first_gor - && cold_mor == warm_first_mor - && warm_first_dor == warm_second_dor - && warm_first_gor == warm_second_gor - && warm_first_mor == warm_second_mor - } + fn try_convert_from_converted_parent(&mut self, index: usize) -> Result, JetErr> { + let Some(parent_index) = self.parent_index(index) else { + return Ok(None); + }; - QuickCheck::new() - .tests(256) - .quickcheck(prop as fn(BoundedNounInput, BoundedNounInput) -> bool); + let Some(parent_h_balance) = self.converted[parent_index] else { + return Ok(None); + }; + let parent_z_balance = self.snapshots[parent_index].z_balance; + let child_z_balance = self.snapshots[index].z_balance; + h_map_from_parent_z_diff( + self.stack, parent_z_balance, parent_h_balance, child_z_balance, self.space, + ) + .map(Some) } - fn cache_entries(context: &Context) -> usize { - context.cache.iter().map(|pairs| pairs.len()).sum() + fn convert_generic(&mut self, index: usize) -> Result { + z_map_to_hashed_h_map( + self.stack, self.snapshots[index].z_balance, self.space, &mut self.memo, + ) } +} - fn inject_bad_cache_value(context: &mut Context, tag: u64, noun: Noun) { - let mut key = T(&mut context.stack, &[D(tag), noun]); - context.cache = context.cache.insert(&mut context.stack, &mut key, D(7)); +fn collect_block_parents( + blocks: Noun, + space: &NounSpace, +) -> Result, JetErr> { + let mut entries = Vec::new(); + collect_z_map_entries(blocks, space, &mut entries)?; + let mut parents = HashMap::with_capacity(entries.len()); + for entry in entries { + let block_id = digest_from_noun(entry.key, space)?; + let parent = local_page_parent(entry.value, space)?; + parents.insert(block_id, digest_from_noun(parent, space)?); } + Ok(parents) +} - fn cmp_with_jet( - context: &mut Context, - jet: fn(&mut Context, Noun) -> Result, - a: Noun, - b: Noun, +fn local_page_parent(page: Noun, space: &NounSpace) -> Result { + let page_cell = page.in_space(space).as_cell().map_err(|_| BAIL_FAIL)?; + if page_cell.head().noun().is_atom() { + slot(page, 30, space) + } else { + slot(page, 14, space) + } +} + +fn z_jug_to_h_jug( + stack: &mut A, + tree: Noun, + space: &NounSpace, + memo: &mut ZHConversionMemo, +) -> Result { + if unsafe { tree.raw_equals(&D(0)) } { + return Ok(h_map_empty()); + } + if tree.is_atom() { + return Err(BAIL_FAIL); + } + + let tree_key = noun_identity(tree); + if let Some(converted) = memo.jug_outer.get(&tree_key) { + return Ok(*converted); + } + + let mut outer_memo = std::mem::take(&mut memo.jug_outer); + let mut convert_value = + |stack: &mut A, value: Noun| z_set_to_hashed_h_set(stack, value, space, memo); + let converted = z_map_to_h_map_with(stack, tree, space, &mut outer_memo, &mut convert_value); + memo.jug_outer = outer_memo; + let converted = converted?; + memo.jug_outer.insert(tree_key, converted); + Ok(converted) +} + +fn z_set_to_hashed_h_set( + stack: &mut A, + tree: Noun, + space: &NounSpace, + memo: &mut ZHConversionMemo, +) -> Result { + if unsafe { tree.raw_equals(&D(0)) } { + return Ok(h_set_empty()); + } + if tree.is_atom() { + return Err(BAIL_FAIL); + } + + let tree_key = noun_identity(tree); + if let Some(converted) = memo.sets.get(&tree_key) { + return Ok(*converted); + } + + let converted = z_set_to_h_set_with(stack, tree, space, &mut memo.sets)?; + memo.sets.insert(tree_key, converted); + Ok(converted) +} + +fn z_map_to_h_map_with( + stack: &mut A, + tree: Noun, + space: &NounSpace, + memo: &mut HashMap, + convert_value: &mut F, +) -> Result +where + A: NounAllocator, + F: FnMut(&mut A, Noun) -> Result, +{ + if unsafe { tree.raw_equals(&D(0)) } { + return Ok(h_map_empty()); + } + if tree.is_atom() { + return Err(BAIL_FAIL); + } + + let tree_key = noun_identity(tree); + if let Some(converted) = memo.get(&tree_key) { + return Ok(*converted); + } + + let mut entries = Vec::new(); + collect_z_map_entries(tree, space, &mut entries)?; + let mut converted_entries = Vec::with_capacity(entries.len()); + for entry in entries { + let key = entry.key; + let value = entry.value; + let converted_value = convert_value(stack, value)?; + let converted_entry = if unsafe { converted_value.raw_equals(&value) } { + entry.noun + } else { + T(stack, &[key, converted_value]) + }; + converted_entries.push(HMapEntry { + noun: converted_entry, + key, + }); + } + let converted = h_map_from_entries(stack, converted_entries, space)?; + memo.insert(tree_key, converted); + Ok(converted) +} + +fn z_set_to_h_set_with( + stack: &mut A, + tree: Noun, + space: &NounSpace, + memo: &mut HashMap, +) -> Result { + if unsafe { tree.raw_equals(&D(0)) } { + return Ok(h_set_empty()); + } + if tree.is_atom() { + return Err(BAIL_FAIL); + } + + let tree_key = noun_identity(tree); + if let Some(converted) = memo.get(&tree_key) { + return Ok(*converted); + } + + let mut items = Vec::new(); + collect_z_set_items(tree, space, &mut items)?; + let converted = h_set_from_items(stack, items, space)?; + memo.insert(tree_key, converted); + Ok(converted) +} + +struct HMapEntry { + noun: Noun, + key: Noun, +} + +struct ZMapEntry { + noun: Noun, + key: Noun, + value: Noun, +} + +fn collect_z_map_entries( + tree: Noun, + space: &NounSpace, + entries: &mut Vec, +) -> Result<(), JetErr> { + if unsafe { tree.raw_equals(&D(0)) } { + return Ok(()); + } + if tree.is_atom() { + return Err(BAIL_FAIL); + } + + let [entry, left, right] = tree.uncell(space)?; + let [key, value] = entry.uncell(space)?; + entries.push(ZMapEntry { + noun: entry, + key, + value, + }); + collect_z_map_entries(left, space, entries)?; + collect_z_map_entries(right, space, entries) +} + +fn collect_z_set_items(tree: Noun, space: &NounSpace, items: &mut Vec) -> Result<(), JetErr> { + if unsafe { tree.raw_equals(&D(0)) } { + return Ok(()); + } + if tree.is_atom() { + return Err(BAIL_FAIL); + } + + let [value, left, right] = tree.uncell(space)?; + items.push(value); + collect_z_set_items(left, space, items)?; + collect_z_set_items(right, space, items) +} + +enum HMapDiffAction { + Put { key: Noun, value: Noun }, + Del { key: Noun }, +} + +struct ZMapDiffEntry { + key_hash: SmallHashedKey, + key: Noun, + value: Noun, +} + +fn h_map_from_parent_z_diff( + stack: &mut A, + parent_z: Noun, + parent_h: Noun, + child_z: Noun, + space: &NounSpace, +) -> Result { + if unsafe { parent_z.raw_equals(&child_z) } { + return Ok(parent_h); + } + + let mut actions = Vec::new(); + collect_z_map_diff(stack, parent_z, child_z, space, &mut actions)?; + + let mut result = parent_h; + // Apply deletions first. Generic `zh-milt` preserves exact key nouns, and + // the diff path needs to model replacements as delete old key noun, then put + // new key noun. + for action in &actions { + if let HMapDiffAction::Del { key } = action { + result = h_map_del(stack, result, *key, space)?; + } + } + for action in actions { + if let HMapDiffAction::Put { key, value } = action { + result = h_map_put(stack, result, key, value, space)?; + } + } + Ok(result) +} + +fn collect_z_map_diff( + stack: &mut A, + parent_z: Noun, + child_z: Noun, + space: &NounSpace, + actions: &mut Vec, +) -> Result<(), JetErr> { + if unsafe { parent_z.raw_equals(&child_z) } { + return Ok(()); + } + + if unsafe { parent_z.raw_equals(&D(0)) } { + collect_z_map_put_actions(child_z, space, actions)?; + return Ok(()); + } + if unsafe { child_z.raw_equals(&D(0)) } { + collect_z_map_del_actions(parent_z, space, actions)?; + return Ok(()); + } + if parent_z.is_atom() || child_z.is_atom() { + return Err(BAIL_FAIL); + } + + let [parent_entry, parent_left, parent_right] = parent_z.uncell(space)?; + let [parent_key, parent_value] = parent_entry.uncell(space)?; + let [child_entry, child_left, child_right] = child_z.uncell(space)?; + let [child_key, child_value] = child_entry.uncell(space)?; + + if noun_equal(stack, parent_key, child_key) { + if !noun_equal(stack, parent_value, child_value) { + actions.push(HMapDiffAction::Put { + key: child_key, + value: child_value, + }); + } + collect_z_map_diff(stack, parent_left, child_left, space, actions)?; + collect_z_map_diff(stack, parent_right, child_right, space, actions) + } else { + collect_rotated_z_map_diff(stack, parent_z, child_z, space, actions) + } +} + +fn collect_rotated_z_map_diff( + stack: &mut A, + parent_z: Noun, + child_z: Noun, + space: &NounSpace, + actions: &mut Vec, +) -> Result<(), JetErr> { + // Zoon maps are treaps. A small logical update can rotate the local root, + // making the root keys differ even when most of the two subtrees match. For + // small regions we flatten and compare by hashed key. Duplicate compact keys + // make that proof ambiguous, and large regions would cost the work this jet + // is avoiding; both cases bail to generic conversion. + let parent_len = count_z_map_entries_limited(parent_z, Z_DIFF_ROTATION_ENTRY_LIMIT + 1, space)?; + let child_len = count_z_map_entries_limited(child_z, Z_DIFF_ROTATION_ENTRY_LIMIT + 1, space)?; + if parent_len + child_len > Z_DIFF_ROTATION_ENTRY_LIMIT { + return Err(BAIL_FAIL); + } + + let mut parent_entries = Vec::with_capacity(parent_len); + let mut child_entries = Vec::with_capacity(child_len); + collect_z_map_diff_entries(parent_z, space, &mut parent_entries)?; + collect_z_map_diff_entries(child_z, space, &mut child_entries)?; + + let mut parent_by_key = HashMap::with_capacity(parent_entries.len()); + for entry in parent_entries { + if parent_by_key + .insert(entry.key_hash, (entry.key, entry.value)) + .is_some() + { + return Err(BAIL_FAIL); + } + } + + let mut child_by_key = HashMap::with_capacity(child_entries.len()); + for entry in child_entries { + if child_by_key + .insert(entry.key_hash, (entry.key, entry.value)) + .is_some() + { + return Err(BAIL_FAIL); + } + } + + for (key_hash, (parent_key, _value)) in &parent_by_key { + match child_by_key.get(key_hash) { + Some((child_key, _value)) if noun_equal(stack, *parent_key, *child_key) => {} + _ => actions.push(HMapDiffAction::Del { key: *parent_key }), + } + } + + for (key_hash, (child_key, child_value)) in child_by_key { + match parent_by_key.get(&key_hash) { + Some((parent_key, parent_value)) + if noun_equal(stack, *parent_key, child_key) + && noun_equal(stack, *parent_value, child_value) => {} + _ => actions.push(HMapDiffAction::Put { + key: child_key, + value: child_value, + }), + } + } + + Ok(()) +} + +fn collect_z_map_put_actions( + tree: Noun, + space: &NounSpace, + actions: &mut Vec, +) -> Result<(), JetErr> { + if unsafe { tree.raw_equals(&D(0)) } { + return Ok(()); + } + if tree.is_atom() { + return Err(BAIL_FAIL); + } + + let [entry, left, right] = tree.uncell(space)?; + let [key, value] = entry.uncell(space)?; + small_hashed_key_from_noun(key, space).ok_or(BAIL_FAIL)?; + actions.push(HMapDiffAction::Put { key, value }); + collect_z_map_put_actions(left, space, actions)?; + collect_z_map_put_actions(right, space, actions) +} + +fn collect_z_map_del_actions( + tree: Noun, + space: &NounSpace, + actions: &mut Vec, +) -> Result<(), JetErr> { + if unsafe { tree.raw_equals(&D(0)) } { + return Ok(()); + } + if tree.is_atom() { + return Err(BAIL_FAIL); + } + + let [entry, left, right] = tree.uncell(space)?; + let [key, _value] = entry.uncell(space)?; + small_hashed_key_from_noun(key, space).ok_or(BAIL_FAIL)?; + actions.push(HMapDiffAction::Del { key }); + collect_z_map_del_actions(left, space, actions)?; + collect_z_map_del_actions(right, space, actions) +} + +fn collect_z_map_diff_entries( + tree: Noun, + space: &NounSpace, + entries: &mut Vec, +) -> Result<(), JetErr> { + if unsafe { tree.raw_equals(&D(0)) } { + return Ok(()); + } + if tree.is_atom() { + return Err(BAIL_FAIL); + } + + let [entry, left, right] = tree.uncell(space)?; + let [key, value] = entry.uncell(space)?; + entries.push(ZMapDiffEntry { + key_hash: small_hashed_key_from_noun(key, space).ok_or(BAIL_FAIL)?, + key, + value, + }); + collect_z_map_diff_entries(left, space, entries)?; + collect_z_map_diff_entries(right, space, entries) +} + +fn count_z_map_entries_limited( + tree: Noun, + limit: usize, + space: &NounSpace, +) -> Result { + if unsafe { tree.raw_equals(&D(0)) } { + return Ok(0); + } + if tree.is_atom() { + return Err(BAIL_FAIL); + } + if limit == 0 { + return Ok(1); + } + + let [_entry, left, right] = tree.uncell(space)?; + let left_count = count_z_map_entries_limited(left, limit, space)?; + if left_count > limit { + return Ok(left_count); + } + let remaining = limit.saturating_sub(left_count + 1); + let right_count = count_z_map_entries_limited(right, remaining, space)?; + Ok(left_count + 1 + right_count) +} + +struct HMapBuildNode { + entry: Noun, + digests: HashedKey, + original_index: usize, + left: Option, + right: Option, +} + +struct SmallHMapBuildNode { + entry: Noun, + key: SmallHashedKey, + original_index: usize, + left: Option, + right: Option, +} + +struct HSetBuildNode { + value: Noun, + digests: HashedKey, + original_index: usize, + left: Option, + right: Option, +} + +struct SmallHSetBuildNode { + value: Noun, + key: SmallHashedKey, + original_index: usize, + left: Option, + right: Option, +} + +#[derive(Clone, Copy, Debug, Eq, Hash, PartialEq)] +struct SmallHashedKey { + len: usize, + digests: [[u64; 5]; SMALL_HASHED_KEY_LIMIT], +} + +impl SmallHashedKey { + fn empty() -> Self { + Self { + len: 0, + digests: [[0; 5]; SMALL_HASHED_KEY_LIMIT], + } + } + + fn single(digest: [u64; 5]) -> Self { + Self { + len: 1, + digests: [digest, [0; 5]], + } + } + + fn pair(first: [u64; 5], second: [u64; 5]) -> Self { + Self { + len: 2, + digests: [first, second], + } + } +} + +enum HashedKey { + Single([u64; 5]), + List(Vec<[u64; 5]>), +} + +impl HashedKey { + fn as_slice(&self) -> &[[u64; 5]] { + match self { + HashedKey::Single(digest) => std::slice::from_ref(digest), + HashedKey::List(digests) => digests.as_slice(), + } + } +} + +fn h_map_from_entries( + stack: &mut A, + entries: Vec, + space: &NounSpace, +) -> Result { + if entries.is_empty() { + return Ok(h_map_empty()); + } + + if let Some(mut nodes) = small_h_map_build_nodes(&entries, space) { + sort_small_h_map_build_nodes(&mut nodes); + ensure_strict_small_h_map_order(&nodes)?; + let root = link_small_h_map_nodes_by_priority(&mut nodes); + return build_small_h_map_node(stack, &nodes, root); + } + + let mut nodes = Vec::with_capacity(entries.len()); + for (original_index, entry) in entries.into_iter().enumerate() { + nodes.push(HMapBuildNode { + entry: entry.noun, + digests: hashed_to_digests(entry.key, space)?, + original_index, + left: None, + right: None, + }); + } + sort_h_map_build_nodes(&mut nodes); + ensure_strict_h_map_order(&nodes)?; + let root = link_h_map_nodes_by_priority(&mut nodes); + build_h_map_node(stack, &nodes, root) +} + +fn h_set_from_items( + stack: &mut A, + items: Vec, + space: &NounSpace, +) -> Result { + if items.is_empty() { + return Ok(h_set_empty()); + } + + if let Some(mut nodes) = small_h_set_build_nodes(&items, space) { + sort_small_h_set_build_nodes(&mut nodes); + ensure_strict_small_h_set_order(&nodes)?; + let root = link_small_h_set_nodes_by_priority(&mut nodes); + return build_small_h_set_node(stack, &nodes, root); + } + + let mut nodes = Vec::with_capacity(items.len()); + for (original_index, value) in items.into_iter().enumerate() { + nodes.push(HSetBuildNode { + value, + digests: hashed_to_digests(value, space)?, + original_index, + left: None, + right: None, + }); + } + sort_h_set_build_nodes(&mut nodes); + ensure_strict_h_set_order(&nodes)?; + let root = link_h_set_nodes_by_priority(&mut nodes); + build_h_set_node(stack, &nodes, root) +} + +fn small_h_map_build_nodes( + entries: &[HMapEntry], + space: &NounSpace, +) -> Option> { + let mut nodes = Vec::with_capacity(entries.len()); + for (original_index, entry) in entries.iter().enumerate() { + let key = small_hashed_key_from_noun(entry.key, space)?; + nodes.push(SmallHMapBuildNode { + entry: entry.noun, + key, + original_index, + left: None, + right: None, + }); + } + Some(nodes) +} + +fn small_h_set_build_nodes(items: &[Noun], space: &NounSpace) -> Option> { + let mut nodes = Vec::with_capacity(items.len()); + for (original_index, value) in items.iter().enumerate() { + let key = small_hashed_key_from_noun(*value, space)?; + nodes.push(SmallHSetBuildNode { + value: *value, + key, + original_index, + left: None, + right: None, + }); + } + Some(nodes) +} + +fn small_hashed_key_from_noun(noun: Noun, space: &NounSpace) -> Option { + if unsafe { noun.raw_equals(&D(0)) } { + return Some(SmallHashedKey::empty()); + } + + let cell = noun.in_space(space).as_cell().ok()?; + let head = cell.head().noun(); + let tail = cell.tail().noun(); + if head.is_atom() { + return small_digest_from_noun(noun, space).map(SmallHashedKey::single); + } + + let first = small_digest_from_noun(head, space)?; + if unsafe { tail.raw_equals(&D(0)) } { + return None; + } + + let second_cell = tail.in_space(space).as_cell().ok()?; + let second_noun = second_cell.head().noun(); + let rest = second_cell.tail().noun(); + let second = small_digest_from_noun(second_noun, space)?; + if unsafe { !rest.raw_equals(&D(0)) } { + return None; + } + + Some(SmallHashedKey::pair(first, second)) +} + +fn small_digest_from_noun(noun: Noun, space: &NounSpace) -> Option<[u64; 5]> { + let first = noun.in_space(space).as_cell().ok()?; + let first_head = first.head().noun(); + let first_tail = first.tail().noun(); + let second = first_tail.in_space(space).as_cell().ok()?; + let second_head = second.head().noun(); + let second_tail = second.tail().noun(); + let third = second_tail.in_space(space).as_cell().ok()?; + let third_head = third.head().noun(); + let third_tail = third.tail().noun(); + let fourth = third_tail.in_space(space).as_cell().ok()?; + let fourth_head = fourth.head().noun(); + let fourth_tail = fourth.tail().noun(); + Some([ + atom_to_u64_opt(first_head, space)?, + atom_to_u64_opt(second_head, space)?, + atom_to_u64_opt(third_head, space)?, + atom_to_u64_opt(fourth_head, space)?, + atom_to_u64_opt(fourth_tail, space)?, + ]) +} + +fn atom_to_u64_opt(noun: Noun, space: &NounSpace) -> Option { + noun.in_space(space).as_atom().ok()?.as_u64().ok() +} + +fn compare_h_map_build_nodes(a: &HMapBuildNode, b: &HMapBuildNode) -> Ordering { + descending_gor_order(a.digests.as_slice(), b.digests.as_slice()) + .then_with(|| a.original_index.cmp(&b.original_index)) +} + +fn compare_h_set_build_nodes(a: &HSetBuildNode, b: &HSetBuildNode) -> Ordering { + descending_gor_order(a.digests.as_slice(), b.digests.as_slice()) + .then_with(|| a.original_index.cmp(&b.original_index)) +} + +fn compare_small_h_map_build_nodes(a: &SmallHMapBuildNode, b: &SmallHMapBuildNode) -> Ordering { + descending_small_gor_order(&a.key, &b.key).then_with(|| a.original_index.cmp(&b.original_index)) +} + +fn compare_small_h_set_build_nodes(a: &SmallHSetBuildNode, b: &SmallHSetBuildNode) -> Ordering { + descending_small_gor_order(&a.key, &b.key).then_with(|| a.original_index.cmp(&b.original_index)) +} + +fn sort_h_map_build_nodes(nodes: &mut [HMapBuildNode]) { + if nodes.len() >= PARALLEL_SORT_THRESHOLD { + nodes.par_sort_unstable_by(compare_h_map_build_nodes); + } else { + nodes.sort_unstable_by(compare_h_map_build_nodes); + } +} + +fn sort_h_set_build_nodes(nodes: &mut [HSetBuildNode]) { + if nodes.len() >= PARALLEL_SORT_THRESHOLD { + nodes.par_sort_unstable_by(compare_h_set_build_nodes); + } else { + nodes.sort_unstable_by(compare_h_set_build_nodes); + } +} + +fn sort_small_h_map_build_nodes(nodes: &mut [SmallHMapBuildNode]) { + if nodes.len() >= PARALLEL_SORT_THRESHOLD { + nodes.par_sort_unstable_by(compare_small_h_map_build_nodes); + } else { + nodes.sort_unstable_by(compare_small_h_map_build_nodes); + } +} + +fn sort_small_h_set_build_nodes(nodes: &mut [SmallHSetBuildNode]) { + if nodes.len() >= PARALLEL_SORT_THRESHOLD { + nodes.par_sort_unstable_by(compare_small_h_set_build_nodes); + } else { + nodes.sort_unstable_by(compare_small_h_set_build_nodes); + } +} + +fn ensure_strict_h_map_order(nodes: &[HMapBuildNode]) -> Result<(), JetErr> { + for pair in nodes.windows(2) { + if descending_gor_order(pair[0].digests.as_slice(), pair[1].digests.as_slice()) + == Ordering::Equal + { + return Err(BAIL_FAIL); + } + } + Ok(()) +} + +fn ensure_strict_h_set_order(nodes: &[HSetBuildNode]) -> Result<(), JetErr> { + for pair in nodes.windows(2) { + if descending_gor_order(pair[0].digests.as_slice(), pair[1].digests.as_slice()) + == Ordering::Equal + { + return Err(BAIL_FAIL); + } + } + Ok(()) +} + +fn ensure_strict_small_h_map_order(nodes: &[SmallHMapBuildNode]) -> Result<(), JetErr> { + for pair in nodes.windows(2) { + if pair[0].key == pair[1].key { + return Err(BAIL_FAIL); + } + } + Ok(()) +} + +fn ensure_strict_small_h_set_order(nodes: &[SmallHSetBuildNode]) -> Result<(), JetErr> { + for pair in nodes.windows(2) { + if pair[0].key == pair[1].key { + return Err(BAIL_FAIL); + } + } + Ok(()) +} + +fn link_h_map_nodes_by_priority(nodes: &mut [HMapBuildNode]) -> usize { + let mut stack: Vec = Vec::new(); + for index in 0..nodes.len() { + let mut left = None; + while let Some(&top) = stack.last() { + if hashed_order_low_to_high( + nodes[top].digests.as_slice(), + nodes[index].digests.as_slice(), + ) == Ordering::Greater + { + break; + } + left = stack.pop(); + } + nodes[index].left = left; + if let Some(&top) = stack.last() { + nodes[top].right = Some(index); + } + stack.push(index); + } + stack[0] +} + +fn link_h_set_nodes_by_priority(nodes: &mut [HSetBuildNode]) -> usize { + let mut stack: Vec = Vec::new(); + for index in 0..nodes.len() { + let mut left = None; + while let Some(&top) = stack.last() { + if hashed_order_low_to_high( + nodes[top].digests.as_slice(), + nodes[index].digests.as_slice(), + ) == Ordering::Greater + { + break; + } + left = stack.pop(); + } + nodes[index].left = left; + if let Some(&top) = stack.last() { + nodes[top].right = Some(index); + } + stack.push(index); + } + stack[0] +} + +fn link_small_h_map_nodes_by_priority(nodes: &mut [SmallHMapBuildNode]) -> usize { + let mut stack: Vec = Vec::new(); + for index in 0..nodes.len() { + let mut left = None; + while let Some(&top) = stack.last() { + if small_mor_order(&nodes[top].key, &nodes[index].key) == Ordering::Greater { + break; + } + left = stack.pop(); + } + nodes[index].left = left; + if let Some(&top) = stack.last() { + nodes[top].right = Some(index); + } + stack.push(index); + } + stack[0] +} + +fn link_small_h_set_nodes_by_priority(nodes: &mut [SmallHSetBuildNode]) -> usize { + let mut stack: Vec = Vec::new(); + for index in 0..nodes.len() { + let mut left = None; + while let Some(&top) = stack.last() { + if small_mor_order(&nodes[top].key, &nodes[index].key) == Ordering::Greater { + break; + } + left = stack.pop(); + } + nodes[index].left = left; + if let Some(&top) = stack.last() { + nodes[top].right = Some(index); + } + stack.push(index); + } + stack[0] +} + +fn build_h_map_node( + stack: &mut A, + nodes: &[HMapBuildNode], + index: usize, +) -> Result { + let cells = allocate_tree_cells(stack, nodes.len()); + for (node_index, node) in nodes.iter().enumerate() { + fill_tree_cell( + cells[node_index], + node.entry, + node.left + .map_or_else(h_map_empty, |left| cells[left].outer.as_noun()), + node.right + .map_or_else(h_map_empty, |right| cells[right].outer.as_noun()), + ); + } + Ok(cells[index].outer.as_noun()) +} + +fn build_h_set_node( + stack: &mut A, + nodes: &[HSetBuildNode], + index: usize, +) -> Result { + let cells = allocate_tree_cells(stack, nodes.len()); + for (node_index, node) in nodes.iter().enumerate() { + fill_tree_cell( + cells[node_index], + node.value, + node.left + .map_or_else(h_set_empty, |left| cells[left].outer.as_noun()), + node.right + .map_or_else(h_set_empty, |right| cells[right].outer.as_noun()), + ); + } + Ok(cells[index].outer.as_noun()) +} + +fn build_small_h_map_node( + stack: &mut A, + nodes: &[SmallHMapBuildNode], + index: usize, +) -> Result { + let cells = allocate_tree_cells(stack, nodes.len()); + for (node_index, node) in nodes.iter().enumerate() { + fill_tree_cell( + cells[node_index], + node.entry, + node.left + .map_or_else(h_map_empty, |left| cells[left].outer.as_noun()), + node.right + .map_or_else(h_map_empty, |right| cells[right].outer.as_noun()), + ); + } + Ok(cells[index].outer.as_noun()) +} + +fn build_small_h_set_node( + stack: &mut A, + nodes: &[SmallHSetBuildNode], + index: usize, +) -> Result { + let cells = allocate_tree_cells(stack, nodes.len()); + for (node_index, node) in nodes.iter().enumerate() { + fill_tree_cell( + cells[node_index], + node.value, + node.left + .map_or_else(h_set_empty, |left| cells[left].outer.as_noun()), + node.right + .map_or_else(h_set_empty, |right| cells[right].outer.as_noun()), + ); + } + Ok(cells[index].outer.as_noun()) +} + +#[derive(Clone, Copy)] +struct TreeCellPair { + outer: Cell, + outer_memory: *mut CellMemory, + inner: Cell, + inner_memory: *mut CellMemory, +} + +fn allocate_tree_cells(stack: &mut A, len: usize) -> Vec { + let mut cells = Vec::with_capacity(len); + for _ in 0..len { + let outer = unsafe { Cell::new_raw_mut(stack) }; + let inner = unsafe { Cell::new_raw_mut(stack) }; + cells.push(TreeCellPair { + outer: outer.0, + outer_memory: outer.1, + inner: inner.0, + inner_memory: inner.1, + }); + } + cells +} + +fn fill_tree_cell(cells: TreeCellPair, node: Noun, left: Noun, right: Noun) { + unsafe { + (*cells.outer_memory).head = node; + (*cells.outer_memory).tail = cells.inner.as_noun(); + (*cells.inner_memory).head = left; + (*cells.inner_memory).tail = right; + } +} + +fn descending_gor_order(a: &[[u64; 5]], b: &[[u64; 5]]) -> Ordering { + hashed_order_high_to_low(a, b).reverse() +} + +fn descending_small_gor_order(a: &SmallHashedKey, b: &SmallHashedKey) -> Ordering { + small_gor_order(a, b).reverse() +} + +fn small_gor_order(a: &SmallHashedKey, b: &SmallHashedKey) -> Ordering { + let shared_len = a.len.min(b.len); + for digest_index in 0..shared_len { + for limb_index in (0..5).rev() { + let ordering = + a.digests[digest_index][limb_index].cmp(&b.digests[digest_index][limb_index]); + if ordering != Ordering::Equal { + return ordering; + } + } + } + a.len.cmp(&b.len) +} + +fn small_mor_order(a: &SmallHashedKey, b: &SmallHashedKey) -> Ordering { + let shared_len = a.len.min(b.len); + for digest_index in 0..shared_len { + for limb_index in 0..5 { + let ordering = + a.digests[digest_index][limb_index].cmp(&b.digests[digest_index][limb_index]); + if ordering != Ordering::Equal { + return ordering; + } + } + } + a.len.cmp(&b.len) +} + +fn hashed_order_high_to_low(a: &[[u64; 5]], b: &[[u64; 5]]) -> Ordering { + compare_digest_lists_ordering(a, b, compare_digest_high_to_low) +} + +fn hashed_order_low_to_high(a: &[[u64; 5]], b: &[[u64; 5]]) -> Ordering { + compare_digest_lists_ordering(a, b, compare_digest_low_to_high) +} + +fn h_map_put( + stack: &mut A, + tree: Noun, + key: Noun, + value: Noun, + space: &NounSpace, +) -> Result { + hashed_to_digests(key, space)?; + if tree.is_atom() { + let entry = T(stack, &[key, value]); + return Ok(T(stack, &[entry, h_map_empty(), h_map_empty()])); + } + + let [entry, left, right] = tree.uncell(space)?; + let [node_key, node_value] = entry.uncell(space)?; + hashed_to_digests(node_key, space)?; + if noun_equal(stack, key, node_key) { + if noun_equal(stack, value, node_value) { + return Ok(tree); + } + let entry = T(stack, &[key, value]); + return Ok(T(stack, &[entry, left, right])); + } + + if gor_hip(key, node_key, space)? { + let child = h_map_put(stack, left, key, value, space)?; + let [child_entry, child_left, child_right] = child.uncell(space)?; + let [child_key, _child_value] = child_entry.uncell(space)?; + if mor_hip(node_key, child_key, space)? { + Ok(T(stack, &[entry, child, right])) + } else { + let demoted = T(stack, &[entry, child_right, right]); + Ok(T(stack, &[child_entry, child_left, demoted])) + } + } else { + let child = h_map_put(stack, right, key, value, space)?; + let [child_entry, child_left, child_right] = child.uncell(space)?; + let [child_key, _child_value] = child_entry.uncell(space)?; + if mor_hip(node_key, child_key, space)? { + Ok(T(stack, &[entry, left, child])) + } else { + let demoted = T(stack, &[entry, left, child_left]); + Ok(T(stack, &[child_entry, demoted, child_right])) + } + } +} + +fn h_map_del( + stack: &mut A, + tree: Noun, + key: Noun, + space: &NounSpace, +) -> Result { + hashed_to_digests(key, space)?; + if tree.is_atom() { + return Ok(h_map_empty()); + } + + let [entry, left, right] = tree.uncell(space)?; + let [node_key, _node_value] = entry.uncell(space)?; + hashed_to_digests(node_key, space)?; + if noun_equal(stack, key, node_key) { + return h_map_join(stack, left, right, space); + } + + if gor_hip(key, node_key, space)? { + let child = h_map_del(stack, left, key, space)?; + if unsafe { child.raw_equals(&left) } { + Ok(tree) + } else { + Ok(T(stack, &[entry, child, right])) + } + } else { + let child = h_map_del(stack, right, key, space)?; + if unsafe { child.raw_equals(&right) } { + Ok(tree) + } else { + Ok(T(stack, &[entry, left, child])) + } + } +} + +fn h_map_join( + stack: &mut A, + left: Noun, + right: Noun, + space: &NounSpace, +) -> Result { + if left.is_atom() { + return Ok(right); + } + if right.is_atom() { + return Ok(left); + } + + let [left_entry, left_left, left_right] = left.uncell(space)?; + let [left_key, _left_value] = left_entry.uncell(space)?; + let [right_entry, right_left, right_right] = right.uncell(space)?; + let [right_key, _right_value] = right_entry.uncell(space)?; + hashed_to_digests(left_key, space)?; + hashed_to_digests(right_key, space)?; + + if mor_hip(left_key, right_key, space)? { + let joined = h_map_join(stack, left_right, right, space)?; + Ok(T(stack, &[left_entry, left_left, joined])) + } else { + let joined = h_map_join(stack, left, right_left, space)?; + Ok(T(stack, &[right_entry, joined, right_right])) + } +} + +fn h_set_put( + stack: &mut A, + tree: Noun, + value: Noun, + space: &NounSpace, +) -> Result { + hashed_to_digests(value, space)?; + if tree.is_atom() { + return Ok(T(stack, &[value, h_set_empty(), h_set_empty()])); + } + + let [node_value, left, right] = tree.uncell(space)?; + hashed_to_digests(node_value, space)?; + if noun_equal(stack, value, node_value) { + return Ok(tree); + } + + if gor_hip(value, node_value, space)? { + let child = h_set_put(stack, left, value, space)?; + let [child_value, child_left, child_right] = child.uncell(space)?; + if mor_hip(node_value, child_value, space)? { + Ok(T(stack, &[node_value, child, right])) + } else { + let demoted = T(stack, &[node_value, child_right, right]); + Ok(T(stack, &[child_value, child_left, demoted])) + } + } else { + let child = h_set_put(stack, right, value, space)?; + let [child_value, child_left, child_right] = child.uncell(space)?; + if mor_hip(node_value, child_value, space)? { + Ok(T(stack, &[node_value, left, child])) + } else { + let demoted = T(stack, &[node_value, left, child_left]); + Ok(T(stack, &[child_value, demoted, child_right])) + } + } +} + +fn h_map_empty() -> Noun { + D(tas!(b"hmap")) +} + +fn h_set_empty() -> Noun { + D(tas!(b"hset")) +} + +fn noun_identity(noun: Noun) -> u64 { + unsafe { noun.as_raw() } +} + +fn noun_equal(stack: &mut A, mut a: Noun, mut b: Noun) -> bool { + unsafe { stack.equals(&mut a, &mut b) } +} + +fn bool_to_noun(value: bool) -> Noun { + if value { + YES + } else { + NO + } +} + +fn gor_hip(a: Noun, b: Noun, space: &NounSpace) -> Result { + let a_digests = hashed_to_digests(a, space)?; + let b_digests = hashed_to_digests(b, space)?; + Ok(compare_digest_lists( + a_digests.as_slice(), + b_digests.as_slice(), + compare_digest_high_to_low, + )) +} + +fn mor_hip(a: Noun, b: Noun, space: &NounSpace) -> Result { + let a_digests = hashed_to_digests(a, space)?; + let b_digests = hashed_to_digests(b, space)?; + Ok(compare_digest_lists( + a_digests.as_slice(), + b_digests.as_slice(), + compare_digest_low_to_high, + )) +} + +fn hashed_to_digests(noun: Noun, space: &NounSpace) -> Result { + if let Ok(digest) = digest_from_noun(noun, space) { + return Ok(HashedKey::Single(digest)); + } + + let mut rest = noun; + let mut digests = Vec::new(); + loop { + if unsafe { rest.raw_equals(&D(0)) } { + if digests.len() == 1 { + return Err(BAIL_FAIL); + } + return Ok(HashedKey::List(digests)); + } + let cell = rest.in_space(space).as_cell().map_err(|_| BAIL_FAIL)?; + let head = cell.head().noun(); + digests.push(digest_from_noun(head, space)?); + rest = cell.tail().noun(); + } +} + +fn digest_from_noun(noun: Noun, space: &NounSpace) -> Result<[u64; 5], JetErr> { + let first = noun.in_space(space).as_cell().map_err(|_| BAIL_FAIL)?; + let first_head = first.head().noun(); + let first_tail = first.tail().noun(); + let second = first_tail + .in_space(space) + .as_cell() + .map_err(|_| BAIL_FAIL)?; + let second_head = second.head().noun(); + let second_tail = second.tail().noun(); + let third = second_tail + .in_space(space) + .as_cell() + .map_err(|_| BAIL_FAIL)?; + let third_head = third.head().noun(); + let third_tail = third.tail().noun(); + let fourth = third_tail + .in_space(space) + .as_cell() + .map_err(|_| BAIL_FAIL)?; + let fourth_head = fourth.head().noun(); + let fourth_tail = fourth.tail().noun(); + Ok([ + atom_to_u64(first_head, space)?, + atom_to_u64(second_head, space)?, + atom_to_u64(third_head, space)?, + atom_to_u64(fourth_head, space)?, + atom_to_u64(fourth_tail, space)?, + ]) +} + +fn atom_to_u64(noun: Noun, space: &NounSpace) -> Result { + noun.in_space(space) + .as_atom() + .map_err(|_| BAIL_FAIL)? + .as_u64() + .map_err(|_| BAIL_FAIL) +} + +fn compare_digest_lists( + a: &[[u64; 5]], + b: &[[u64; 5]], + compare_digest: fn(&[u64; 5], &[u64; 5]) -> Option, +) -> bool { + let mut index = 0; + loop { + match (a.get(index), b.get(index)) { + (None, _) => return false, + (Some(_), None) => return true, + (Some(a_digest), Some(b_digest)) => { + if let Some(ordered) = compare_digest(a_digest, b_digest) { + return ordered; + } + } + } + index += 1; + } +} + +fn compare_digest_lists_ordering( + a: &[[u64; 5]], + b: &[[u64; 5]], + compare_digest: fn(&[u64; 5], &[u64; 5]) -> Option, +) -> Ordering { + let mut index = 0; + loop { + match (a.get(index), b.get(index)) { + (None, None) => return Ordering::Equal, + (None, Some(_)) => return Ordering::Less, + (Some(_), None) => return Ordering::Greater, + (Some(a_digest), Some(b_digest)) => { + if let Some(ordered) = compare_digest(a_digest, b_digest) { + return if ordered { + Ordering::Greater + } else { + Ordering::Less + }; + } + } + } + index += 1; + } +} + +fn compare_digest_high_to_low(a: &[u64; 5], b: &[u64; 5]) -> Option { + for index in (0..5).rev() { + if a[index] > b[index] { + return Some(true); + } + if a[index] < b[index] { + return Some(false); + } + } + None +} + +fn compare_digest_low_to_high(a: &[u64; 5], b: &[u64; 5]) -> Option { + for index in 0..5 { + if a[index] > b[index] { + return Some(true); + } + if a[index] < b[index] { + return Some(false); + } + } + None +} + +fn cache_lookup_digest( + context: &mut Context, + tag: u64, + noun: Noun, +) -> Result, JetErr> { + let space = context.stack.noun_space(); + let mut key = T(&mut context.stack, &[D(tag), noun]); + match context.cache.lookup(&mut context.stack, &mut key) { + Some(cached) => Ok(Some(<[u64; 5]>::from_noun(&cached, &space)?)), + None => Ok(None), + } +} + +fn cache_insert_digest(context: &mut Context, tag: u64, noun: Noun, digest: [u64; 5]) { + let mut key = T(&mut context.stack, &[D(tag), noun]); + let value = digest_to_noundigest(&mut context.stack, digest); + context.cache = context.cache.insert(&mut context.stack, &mut key, value); +} + +fn get_tip_digest(context: &mut Context, noun: Noun) -> Result<[u64; 5], JetErr> { + if let Some(cached) = cache_lookup_digest(context, TIP_CACHE_TAG, noun)? { + return Ok(cached); + } + let space = context.stack.noun_space(); + let digest = hash_noun_varlen_digest(&mut context.stack, noun, &space)?; + cache_insert_digest(context, TIP_CACHE_TAG, noun, digest); + Ok(digest) +} + +fn get_double_tip_digest(context: &mut Context, noun: Noun) -> Result<[u64; 5], JetErr> { + if let Some(cached) = cache_lookup_digest(context, DOUBLE_TIP_CACHE_TAG, noun)? { + return Ok(cached); + } + + let tip_digest = get_tip_digest(context, noun)?; + let mut input: Vec = Vec::with_capacity(10); + input.extend(tip_digest.into_iter().map(Belt)); + input.extend(tip_digest.into_iter().map(Belt)); + let digest = hash_10(&mut input); + + cache_insert_digest(context, DOUBLE_TIP_CACHE_TAG, noun, digest); + Ok(digest) +} + +// =========================================================================== +// h-by / h-in container arm jets (non-gate arms only). +// +// Each jet is a faithful reimplementation of the corresponding `~/`-hinted +// arm in open/hoon/common/h-zoon.hoon. Door arms: the arm sample is at +// `slot(subject, 6)`; the door sample (the map/set) is reached via the arm +// context `slot(subject, 7)` then `slot(door, 6)` (same convention as the +// nockvm +by/+in jets). Empty markers are %hmap / %hset; key ordering is +// gor-hip (descent) and mor-hip (priority), never `gor`/`mor`. Reuses the +// production-proven h_map_put / h_map_del / h_map_join / h_set_put helpers. +// =========================================================================== + +fn h_door_sample(subject: Noun, space: &NounSpace) -> Result { + let door = slot(subject, 7, space)?; + slot(door, 6, space) +} + +// ---- h-map (h-by) helpers ---- + +// +get: `(unit value)` — D(0) for ~, [0 value] for [~ u]. +fn h_map_get_unit( + stack: &mut A, + tree: Noun, + key: Noun, + space: &NounSpace, +) -> Result { + let mut node = tree; + loop { + if node.is_atom() { + return Ok(D(0)); + } + let [entry, left, right] = node.uncell(space)?; + let [node_key, node_value] = entry.uncell(space)?; + if noun_equal(stack, key, node_key) { + return Ok(T(stack, &[D(0), node_value])); + } + node = if gor_hip(key, node_key, space)? { + left + } else { + right + }; + } +} + +// +uni: treap merge; on equal key the right operand (b) wins. +fn h_map_uni( + stack: &mut A, + a: Noun, + b: Noun, + space: &NounSpace, +) -> Result { + if b.is_atom() { + return Ok(a); + } + if a.is_atom() { + return Ok(b); + } + let [ae, al, ar] = a.uncell(space)?; + let [ak, _av] = ae.uncell(space)?; + let [be, bl, br] = b.uncell(space)?; + let [bk, _bv] = be.uncell(space)?; + if noun_equal(stack, bk, ak) { + let nl = h_map_uni(stack, al, bl, space)?; + let nr = h_map_uni(stack, ar, br, space)?; + return Ok(T(stack, &[be, nl, nr])); + } + if mor_hip(ak, bk, space)? { + if gor_hip(bk, ak, space)? { + let bmod = T(stack, &[be, bl, h_map_empty()]); + let inner = h_map_uni(stack, al, bmod, space)?; + let a2 = T(stack, &[ae, inner, ar]); + h_map_uni(stack, a2, br, space) + } else { + let bmod = T(stack, &[be, h_map_empty(), br]); + let inner = h_map_uni(stack, ar, bmod, space)?; + let a2 = T(stack, &[ae, al, inner]); + h_map_uni(stack, a2, bl, space) + } + } else if gor_hip(ak, bk, space)? { + let amod = T(stack, &[ae, al, h_map_empty()]); + let inner = h_map_uni(stack, amod, bl, space)?; + let b2 = T(stack, &[be, inner, br]); + h_map_uni(stack, ar, b2, space) + } else { + let amod = T(stack, &[ae, h_map_empty(), ar]); + let inner = h_map_uni(stack, amod, br, space)?; + let b2 = T(stack, &[be, bl, inner]); + h_map_uni(stack, al, b2, space) + } +} + +// +bif: split `tree` at `key`; returns (left, right), key dropped. +fn h_map_bif( + stack: &mut A, + tree: Noun, + key: Noun, + space: &NounSpace, +) -> Result<(Noun, Noun), JetErr> { + if tree.is_atom() { + return Ok((h_map_empty(), h_map_empty())); + } + let [entry, left, right] = tree.uncell(space)?; + let [node_key, _node_value] = entry.uncell(space)?; + if noun_equal(stack, key, node_key) { + return Ok((left, right)); + } + if gor_hip(key, node_key, space)? { + let (dl, dr) = h_map_bif(stack, left, key, space)?; + Ok((dl, T(stack, &[entry, dr, right]))) + } else { + let (dl, dr) = h_map_bif(stack, right, key, space)?; + Ok((T(stack, &[entry, left, dl]), dr)) + } +} + +// +int: treap intersection. +fn h_map_int( + stack: &mut A, + a: Noun, + b: Noun, + space: &NounSpace, +) -> Result { + if b.is_atom() || a.is_atom() { + return Ok(h_map_empty()); + } + let [ae, al, ar] = a.uncell(space)?; + let [ak, _av] = ae.uncell(space)?; + let [be, bl, br] = b.uncell(space)?; + let [bk, _bv] = be.uncell(space)?; + if mor_hip(ak, bk, space)? { + if noun_equal(stack, bk, ak) { + let nl = h_map_int(stack, al, bl, space)?; + let nr = h_map_int(stack, ar, br, space)?; + Ok(T(stack, &[be, nl, nr])) + } else if gor_hip(bk, ak, space)? { + let bmod = T(stack, &[be, bl, h_map_empty()]); + let x = h_map_int(stack, al, bmod, space)?; + let y = h_map_int(stack, a, br, space)?; + h_map_uni(stack, x, y, space) + } else { + let bmod = T(stack, &[be, h_map_empty(), br]); + let x = h_map_int(stack, ar, bmod, space)?; + let y = h_map_int(stack, a, bl, space)?; + h_map_uni(stack, x, y, space) + } + } else if noun_equal(stack, ak, bk) { + let nl = h_map_int(stack, al, bl, space)?; + let nr = h_map_int(stack, ar, br, space)?; + Ok(T(stack, &[be, nl, nr])) + } else if gor_hip(ak, bk, space)? { + let amod = T(stack, &[ae, al, h_map_empty()]); + let x = h_map_int(stack, amod, bl, space)?; + let y = h_map_int(stack, ar, b, space)?; + h_map_uni(stack, x, y, space) + } else { + let amod = T(stack, &[ae, h_map_empty(), ar]); + let x = h_map_int(stack, amod, br, space)?; + let y = h_map_int(stack, al, b, space)?; + h_map_uni(stack, x, y, space) + } +} + +// +dif: a without any key in b. +fn h_map_dif( + stack: &mut A, + a: Noun, + b: Noun, + space: &NounSpace, +) -> Result { + if b.is_atom() { + return Ok(a); + } + let [be, bl, br] = b.uncell(space)?; + let [bk, _bv] = be.uncell(space)?; + let (cl, cr) = h_map_bif(stack, a, bk, space)?; + let d = h_map_dif(stack, cl, bl, space)?; + let e = h_map_dif(stack, cr, br, space)?; + h_map_join(stack, d, e, space) +} + +// peg axis composition; None on u128 overflow (jet then punts to hoon). +fn peg(a: u128, b: u128) -> Option { + if b == 0 { + return None; + } + if b == 1 { + return Some(a); + } + let d = 127u32 - b.leading_zeros(); + let span = 1u128.checked_shl(d)?; + let low = b - span; + a.checked_mul(span)?.checked_add(low) +} + +fn axis_to_u64(axis: u128) -> Result { + if axis >= (1u128 << 62) { + return Err(JetErr::Punt); + } + Ok(axis as u64) +} + +// +dig: axis of key as `(unit @)`. +fn h_map_dig( + stack: &mut A, + tree: Noun, + key: Noun, + space: &NounSpace, +) -> Result { + let mut node = tree; + let mut axis: u128 = 1; + loop { + if node.is_atom() { + return Ok(D(0)); + } + let [entry, left, right] = node.uncell(space)?; + let [node_key, _node_value] = entry.uncell(space)?; + if noun_equal(stack, key, node_key) { + let ax = peg(axis, 2).ok_or(JetErr::Punt)?; + let ax = axis_to_u64(ax)?; + return Ok(T(stack, &[D(0), D(ax)])); + } + if gor_hip(key, node_key, space)? { + axis = peg(axis, 6).ok_or(JetErr::Punt)?; + node = left; + } else { + axis = peg(axis, 7).ok_or(JetErr::Punt)?; + node = right; + } + } +} + +// fold +put over a `(list [p q])`. +fn h_map_gas( + stack: &mut A, + tree: Noun, + list: Noun, + space: &NounSpace, +) -> Result { + let mut acc = tree; + let mut rest = list; + loop { + if rest.is_atom() { + return Ok(acc); + } + let [item, tail] = rest.uncell(space)?; + let [pair_key, pair_value] = item.uncell(space)?; + acc = h_map_put(stack, acc, pair_key, pair_value, space)?; + rest = tail; + } +} + +// ---- h-set (h-in) helpers ---- + +fn h_set_has( + stack: &mut A, + tree: Noun, + item: Noun, + space: &NounSpace, +) -> Result { + let mut node = tree; + loop { + if node.is_atom() { + return Ok(false); + } + let [node_item, left, right] = node.uncell(space)?; + if noun_equal(stack, item, node_item) { + return Ok(true); + } + node = if gor_hip(item, node_item, space)? { + left + } else { + right + }; + } +} + +fn h_set_join( + stack: &mut A, + left: Noun, + right: Noun, + space: &NounSpace, +) -> Result { + if left.is_atom() { + return Ok(right); + } + if right.is_atom() { + return Ok(left); + } + let [li, ll, lr] = left.uncell(space)?; + let [ri, rl, rr] = right.uncell(space)?; + if mor_hip(li, ri, space)? { + let joined = h_set_join(stack, lr, right, space)?; + Ok(T(stack, &[li, ll, joined])) + } else { + let joined = h_set_join(stack, left, rl, space)?; + Ok(T(stack, &[ri, joined, rr])) + } +} + +fn h_set_del( + stack: &mut A, + tree: Noun, + item: Noun, + space: &NounSpace, +) -> Result { + if tree.is_atom() { + return Ok(h_set_empty()); + } + let [node_item, left, right] = tree.uncell(space)?; + if noun_equal(stack, item, node_item) { + return h_set_join(stack, left, right, space); + } + if gor_hip(item, node_item, space)? { + let child = h_set_del(stack, left, item, space)?; + Ok(T(stack, &[node_item, child, right])) + } else { + let child = h_set_del(stack, right, item, space)?; + Ok(T(stack, &[node_item, left, child])) + } +} + +fn h_set_uni( + stack: &mut A, + a: Noun, + b: Noun, + space: &NounSpace, +) -> Result { + if noun_equal(stack, a, b) { + return Ok(a); + } + if b.is_atom() { + return Ok(a); + } + if a.is_atom() { + return Ok(b); + } + let [an, al, ar] = a.uncell(space)?; + let [bn, bl, br] = b.uncell(space)?; + if noun_equal(stack, bn, an) { + let nl = h_set_uni(stack, al, bl, space)?; + let nr = h_set_uni(stack, ar, br, space)?; + return Ok(T(stack, &[bn, nl, nr])); + } + if mor_hip(an, bn, space)? { + if gor_hip(bn, an, space)? { + let bmod = T(stack, &[bn, bl, h_set_empty()]); + let inner = h_set_uni(stack, al, bmod, space)?; + let a2 = T(stack, &[an, inner, ar]); + h_set_uni(stack, a2, br, space) + } else { + let bmod = T(stack, &[bn, h_set_empty(), br]); + let inner = h_set_uni(stack, ar, bmod, space)?; + let a2 = T(stack, &[an, al, inner]); + h_set_uni(stack, a2, bl, space) + } + } else if gor_hip(an, bn, space)? { + let amod = T(stack, &[an, al, h_set_empty()]); + let inner = h_set_uni(stack, amod, bl, space)?; + let b2 = T(stack, &[bn, inner, br]); + h_set_uni(stack, ar, b2, space) + } else { + let amod = T(stack, &[an, h_set_empty(), ar]); + let inner = h_set_uni(stack, amod, br, space)?; + let b2 = T(stack, &[bn, bl, inner]); + h_set_uni(stack, al, b2, space) + } +} + +fn h_set_int( + stack: &mut A, + a: Noun, + b: Noun, + space: &NounSpace, +) -> Result { + if b.is_atom() || a.is_atom() { + return Ok(h_set_empty()); + } + let [an, al, ar] = a.uncell(space)?; + let [bn, bl, br] = b.uncell(space)?; + if noun_equal(stack, bn, an) { + let nl = h_set_int(stack, al, bl, space)?; + let nr = h_set_int(stack, ar, br, space)?; + return Ok(T(stack, &[an, nl, nr])); + } + if !mor_hip(an, bn, space)? { + return h_set_int(stack, b, a, space); + } + if gor_hip(bn, an, space)? { + let bmod = T(stack, &[bn, bl, h_set_empty()]); + let x = h_set_int(stack, al, bmod, space)?; + let y = h_set_int(stack, a, br, space)?; + h_set_uni(stack, x, y, space) + } else { + let bmod = T(stack, &[bn, h_set_empty(), br]); + let x = h_set_int(stack, ar, bmod, space)?; + let y = h_set_int(stack, a, bl, space)?; + h_set_uni(stack, x, y, space) + } +} + +// +bif (set): mirrors the `=< +` node-builder; tail is (left, right). +fn h_set_bif_node( + stack: &mut A, + tree: Noun, + item: Noun, + space: &NounSpace, +) -> Result { + if tree.is_atom() { + return Ok(T(stack, &[item, h_set_empty(), h_set_empty()])); + } + let [n, l, r] = tree.uncell(space)?; + if noun_equal(stack, item, n) { + return Ok(tree); + } + if gor_hip(item, n, space)? { + let c = h_set_bif_node(stack, l, item, space)?; + let [cn, cl, cr] = c.uncell(space)?; + let a2 = T(stack, &[n, cr, r]); + Ok(T(stack, &[cn, cl, a2])) + } else { + let c = h_set_bif_node(stack, r, item, space)?; + let [cn, cl, cr] = c.uncell(space)?; + let a2 = T(stack, &[n, l, cl]); + Ok(T(stack, &[cn, a2, cr])) + } +} + +fn h_set_bif( + stack: &mut A, + tree: Noun, + item: Noun, + space: &NounSpace, +) -> Result<(Noun, Noun), JetErr> { + let node = h_set_bif_node(stack, tree, item, space)?; + let [_n, l, r] = node.uncell(space)?; + Ok((l, r)) +} + +fn h_set_dif( + stack: &mut A, + a: Noun, + b: Noun, + space: &NounSpace, +) -> Result { + if b.is_atom() { + return Ok(a); + } + let [bn, bl, br] = b.uncell(space)?; + let (cl, cr) = h_set_bif(stack, a, bn, space)?; + let d = h_set_dif(stack, cl, bl, space)?; + let e = h_set_dif(stack, cr, br, space)?; + h_set_join(stack, d, e, space) +} + +fn h_set_dig( + stack: &mut A, + tree: Noun, + item: Noun, + space: &NounSpace, +) -> Result { + let mut node = tree; + let mut axis: u128 = 1; + loop { + if node.is_atom() { + return Ok(D(0)); + } + let [node_item, left, right] = node.uncell(space)?; + if noun_equal(stack, item, node_item) { + let ax = peg(axis, 2).ok_or(JetErr::Punt)?; + let ax = axis_to_u64(ax)?; + return Ok(T(stack, &[D(0), D(ax)])); + } + if gor_hip(item, node_item, space)? { + axis = peg(axis, 6).ok_or(JetErr::Punt)?; + node = left; + } else { + axis = peg(axis, 7).ok_or(JetErr::Punt)?; + node = right; + } + } +} + +fn h_set_gas( + stack: &mut A, + tree: Noun, + list: Noun, + space: &NounSpace, +) -> Result { + let mut acc = tree; + let mut rest = list; + loop { + if rest.is_atom() { + return Ok(acc); + } + let [item, tail] = rest.uncell(space)?; + acc = h_set_put(stack, acc, item, space)?; + rest = tail; + } +} + +// ---- h-by jet entrypoints ---- + +pub fn h_by_get_jet(context: &mut Context, subject: Noun) -> Result { + let space = context.stack.noun_space(); + let key = slot(subject, 6, &space)?; + let map = h_door_sample(subject, &space)?; + h_map_get_unit(&mut context.stack, map, key, &space) +} + +pub fn h_by_got_jet(context: &mut Context, subject: Noun) -> Result { + let space = context.stack.noun_space(); + let key = slot(subject, 6, &space)?; + let map = h_door_sample(subject, &space)?; + let unit = h_map_get_unit(&mut context.stack, map, key, &space)?; + if unit.is_atom() { + return Err(BAIL_FAIL); + } + slot(unit, 3, &space) +} + +pub fn h_by_gut_jet(context: &mut Context, subject: Noun) -> Result { + let space = context.stack.noun_space(); + let sam = slot(subject, 6, &space)?; + let key = slot(sam, 2, &space)?; + let default = slot(sam, 3, &space)?; + let map = h_door_sample(subject, &space)?; + let unit = h_map_get_unit(&mut context.stack, map, key, &space)?; + if unit.is_atom() { + Ok(default) + } else { + slot(unit, 3, &space) + } +} + +pub fn h_by_has_jet(context: &mut Context, subject: Noun) -> Result { + let space = context.stack.noun_space(); + let key = slot(subject, 6, &space)?; + let map = h_door_sample(subject, &space)?; + let unit = h_map_get_unit(&mut context.stack, map, key, &space)?; + Ok(bool_to_noun(!unit.is_atom())) +} + +pub fn h_by_put_jet(context: &mut Context, subject: Noun) -> Result { + let space = context.stack.noun_space(); + let sam = slot(subject, 6, &space)?; + let key = slot(sam, 2, &space)?; + let value = slot(sam, 3, &space)?; + let map = h_door_sample(subject, &space)?; + h_map_put(&mut context.stack, map, key, value, &space) +} + +pub fn h_by_del_jet(context: &mut Context, subject: Noun) -> Result { + let space = context.stack.noun_space(); + let key = slot(subject, 6, &space)?; + let map = h_door_sample(subject, &space)?; + h_map_del(&mut context.stack, map, key, &space) +} + +pub fn h_by_mar_jet(context: &mut Context, subject: Noun) -> Result { + let space = context.stack.noun_space(); + let sam = slot(subject, 6, &space)?; + let key = slot(sam, 2, &space)?; + let unit = slot(sam, 3, &space)?; + let map = h_door_sample(subject, &space)?; + if unit.is_atom() { + h_map_del(&mut context.stack, map, key, &space) + } else { + let value = slot(unit, 3, &space)?; + h_map_put(&mut context.stack, map, key, value, &space) + } +} + +pub fn h_by_gas_jet(context: &mut Context, subject: Noun) -> Result { + let space = context.stack.noun_space(); + let list = slot(subject, 6, &space)?; + let map = h_door_sample(subject, &space)?; + h_map_gas(&mut context.stack, map, list, &space) +} + +pub fn h_by_uni_jet(context: &mut Context, subject: Noun) -> Result { + let space = context.stack.noun_space(); + let b = slot(subject, 6, &space)?; + let a = h_door_sample(subject, &space)?; + h_map_uni(&mut context.stack, a, b, &space) +} + +pub fn h_by_int_jet(context: &mut Context, subject: Noun) -> Result { + let space = context.stack.noun_space(); + let b = slot(subject, 6, &space)?; + let a = h_door_sample(subject, &space)?; + h_map_int(&mut context.stack, a, b, &space) +} + +pub fn h_by_dif_jet(context: &mut Context, subject: Noun) -> Result { + let space = context.stack.noun_space(); + let b = slot(subject, 6, &space)?; + let a = h_door_sample(subject, &space)?; + h_map_dif(&mut context.stack, a, b, &space) +} + +pub fn h_by_bif_jet(context: &mut Context, subject: Noun) -> Result { + let space = context.stack.noun_space(); + let key = slot(subject, 6, &space)?; + let map = h_door_sample(subject, &space)?; + let (left, right) = h_map_bif(&mut context.stack, map, key, &space)?; + Ok(T(&mut context.stack, &[left, right])) +} + +pub fn h_by_dig_jet(context: &mut Context, subject: Noun) -> Result { + let space = context.stack.noun_space(); + let key = slot(subject, 6, &space)?; + let map = h_door_sample(subject, &space)?; + h_map_dig(&mut context.stack, map, key, &space) +} + +// ---- h-in jet entrypoints ---- + +pub fn h_in_has_jet(context: &mut Context, subject: Noun) -> Result { + let space = context.stack.noun_space(); + let item = slot(subject, 6, &space)?; + let set = h_door_sample(subject, &space)?; + Ok(bool_to_noun(h_set_has( + &mut context.stack, set, item, &space, + )?)) +} + +pub fn h_in_put_jet(context: &mut Context, subject: Noun) -> Result { + let space = context.stack.noun_space(); + let item = slot(subject, 6, &space)?; + let set = h_door_sample(subject, &space)?; + h_set_put(&mut context.stack, set, item, &space) +} + +pub fn h_in_del_jet(context: &mut Context, subject: Noun) -> Result { + let space = context.stack.noun_space(); + let item = slot(subject, 6, &space)?; + let set = h_door_sample(subject, &space)?; + h_set_del(&mut context.stack, set, item, &space) +} + +pub fn h_in_gas_jet(context: &mut Context, subject: Noun) -> Result { + let space = context.stack.noun_space(); + let list = slot(subject, 6, &space)?; + let set = h_door_sample(subject, &space)?; + h_set_gas(&mut context.stack, set, list, &space) +} + +pub fn h_in_uni_jet(context: &mut Context, subject: Noun) -> Result { + let space = context.stack.noun_space(); + let b = slot(subject, 6, &space)?; + let a = h_door_sample(subject, &space)?; + h_set_uni(&mut context.stack, a, b, &space) +} + +pub fn h_in_int_jet(context: &mut Context, subject: Noun) -> Result { + let space = context.stack.noun_space(); + let b = slot(subject, 6, &space)?; + let a = h_door_sample(subject, &space)?; + h_set_int(&mut context.stack, a, b, &space) +} + +pub fn h_in_dif_jet(context: &mut Context, subject: Noun) -> Result { + let space = context.stack.noun_space(); + let b = slot(subject, 6, &space)?; + let a = h_door_sample(subject, &space)?; + h_set_dif(&mut context.stack, a, b, &space) +} + +pub fn h_in_bif_jet(context: &mut Context, subject: Noun) -> Result { + let space = context.stack.noun_space(); + let item = slot(subject, 6, &space)?; + let set = h_door_sample(subject, &space)?; + let (left, right) = h_set_bif(&mut context.stack, set, item, &space)?; + Ok(T(&mut context.stack, &[left, right])) +} + +pub fn h_in_dig_jet(context: &mut Context, subject: Noun) -> Result { + let space = context.stack.noun_space(); + let item = slot(subject, 6, &space)?; + let set = h_door_sample(subject, &space)?; + h_set_dig(&mut context.stack, set, item, &space) +} + +#[cfg(test)] +mod tests { + use ibig::UBig; + use nockchain_math::zoon::common::DefaultTipHasher; + use nockchain_math::zoon::zmap::z_map_put; + use nockchain_math::zoon::zset::z_set_put; + use nockvm::interpreter::Context; + use nockvm::jets::util::test::{init_context, A}; + use nockvm::jets::util::BAIL_FAIL; + use nockvm::mem::NockStack; + use nockvm::noun::{Noun, D, T}; + use nockvm::unifying_equality::unifying_equality; + use quickcheck::{Arbitrary, Gen, QuickCheck}; + + use super::*; + + #[test] + fn dor_tip_matches_hoon_for_mixed_atom_cell_inputs() { + let c = &mut init_context(); + let atom = D(7); + let cell = T(&mut c.stack, &[D(1), D(2)]); + + assert!( + cmp_with_jet(c, dor_tip_jet, atom, cell).expect("dor-tip jet should succeed"), + "expected dor-tip atom= 2, + "expected tip cache entries for both nouns, found {after_first}" + ); + + let _second = gor_tip_jet(c, subject).expect("gor-tip should succeed"); + let after_second = cache_entries(c); + assert_eq!(after_second, after_first); + } + + #[test] + fn mor_tip_reuses_tip_and_double_tip_cache() { + let c = &mut init_context(); + let a = T(&mut c.stack, &[D(17), D(23)]); + let b = T(&mut c.stack, &[D(19), D(29)]); + let subject = jet_subject(&mut c.stack, a, b); + + assert_eq!(cache_entries(c), 0); + let _first = mor_tip_jet(c, subject).expect("mor-tip should succeed"); + let after_mor = cache_entries(c); + assert!( + after_mor >= 4, + "expected tip + double-tip cache entries for both nouns, found {after_mor}" + ); + + let _second = mor_tip_jet(c, subject).expect("mor-tip should succeed"); + let after_second_mor = cache_entries(c); + assert_eq!(after_second_mor, after_mor); + + let _gor = gor_tip_jet(c, subject).expect("gor-tip should succeed"); + let after_gor = cache_entries(c); + assert_eq!(after_gor, after_mor); + } + + #[test] + fn jets_error_on_malformed_sample_shape() { + let c = &mut init_context(); + let malformed_subject = T(&mut c.stack, &[D(0), D(42), D(0)]); + + assert!(dor_tip_jet(c, malformed_subject).is_err()); + assert!(gor_tip_jet(c, malformed_subject).is_err()); + assert!(mor_tip_jet(c, malformed_subject).is_err()); + assert!(gor_hip_jet(c, malformed_subject).is_err()); + assert!(mor_hip_jet(c, malformed_subject).is_err()); + } + + #[test] + fn gor_hip_orders_digest_limbs_high_to_low() { + let c = &mut init_context(); + let high = digest(&mut c.stack, [0, 0, 0, 0, 2]); + let low = digest(&mut c.stack, [99, 99, 99, 99, 1]); + + assert!( + cmp_with_jet(c, gor_hip_jet, high, low).expect("gor-hip should succeed"), + "gor-hip must compare the most significant digest limb first" + ); + assert!( + !cmp_with_jet(c, gor_hip_jet, low, high).expect("gor-hip should succeed"), + "gor-hip reverse order should be false" + ); + } + + #[test] + fn mor_hip_orders_digest_limbs_low_to_high() { + let c = &mut init_context(); + let high = digest(&mut c.stack, [2, 0, 0, 0, 0]); + let low = digest(&mut c.stack, [1, 99, 99, 99, 99]); + + assert!( + cmp_with_jet(c, mor_hip_jet, high, low).expect("mor-hip should succeed"), + "mor-hip must compare the reversed digest limb order" + ); + assert!( + !cmp_with_jet(c, mor_hip_jet, low, high).expect("mor-hip should succeed"), + "mor-hip reverse order should be false" + ); + } + + #[test] + fn hip_comparators_accept_digest_lists() { + let c = &mut init_context(); + let prefix = digest(&mut c.stack, [1, 2, 3, 4, 5]); + let second_high = digest(&mut c.stack, [0, 0, 0, 0, 7]); + let second_low = digest(&mut c.stack, [0, 0, 0, 0, 6]); + let a = list(&mut c.stack, &[prefix, second_high]); + let b = list(&mut c.stack, &[prefix, second_low]); + + assert!( + cmp_with_jet(c, gor_hip_jet, a, b).expect("gor-hip should succeed"), + "gor-hip should walk digest lists after an equal prefix" + ); + assert!( + cmp_with_jet(c, mor_hip_jet, a, b).expect("mor-hip should succeed"), + "mor-hip should walk digest lists after an equal prefix" + ); + } + + #[test] + fn hip_comparators_match_independent_digest_oracle() { + const DIGEST_CASES: [([u64; 5], [u64; 5]); 6] = [ + ([0, 0, 0, 0, 2], [99, 99, 99, 99, 1]), + ([2, 0, 0, 0, 0], [1, 99, 99, 99, 99]), + ([7, 7, 7, 7, 7], [7, 7, 7, 7, 7]), + ([0, 1, 2, 3, 4], [0, 1, 2, 3, 5]), + ([9, 1, 2, 3, 4], [8, 1, 2, 3, 4]), + ([3, 4, 5, 6, 7], [3, 4, 5, 6, 6]), + ]; + let c = &mut init_context(); + + for (left, right) in DIGEST_CASES { + let a = digest(&mut c.stack, left); + let b = digest(&mut c.stack, right); + assert_eq!( + cmp_with_jet(c, gor_hip_jet, a, b).expect("gor-hip should succeed"), + digest_list_order_oracle(&[left], &[right], &GOR_HIP_ORDER), + "gor-hip oracle mismatch for {left:?} vs {right:?}" + ); + assert_eq!( + cmp_with_jet(c, mor_hip_jet, a, b).expect("mor-hip should succeed"), + digest_list_order_oracle(&[left], &[right], &MOR_HIP_ORDER), + "mor-hip oracle mismatch for {left:?} vs {right:?}" + ); + } + + let list_cases = [ + ( + vec![[1, 2, 3, 4, 5], [0, 0, 0, 0, 9]], + vec![[1, 2, 3, 4, 5], [0, 0, 0, 0, 8]], + ), + ( + vec![[1, 2, 3, 4, 5], [8, 8, 8, 8, 8]], + vec![[1, 2, 3, 4, 5], [9, 9, 9, 9, 9]], + ), + ( + vec![[1, 2, 3, 4, 5], [9, 9, 9, 9, 9]], + vec![[1, 2, 3, 4, 5], [8, 8, 8, 8, 8]], + ), + ( + vec![[4, 4, 4, 4, 4], [5, 5, 5, 5, 5]], + vec![[4, 4, 4, 4, 4], [5, 5, 5, 5, 5]], + ), + ]; + + for (left, right) in list_cases { + let a = digest_list(&mut c.stack, &left); + let b = digest_list(&mut c.stack, &right); + assert_eq!( + cmp_with_jet(c, gor_hip_jet, a, b).expect("gor-hip should succeed"), + digest_list_order_oracle(&left, &right, &GOR_HIP_ORDER), + "gor-hip list oracle mismatch for {left:?} vs {right:?}" + ); + assert_eq!( + cmp_with_jet(c, mor_hip_jet, a, b).expect("mor-hip should succeed"), + digest_list_order_oracle(&left, &right, &MOR_HIP_ORDER), + "mor-hip list oracle mismatch for {left:?} vs {right:?}" + ); + } + } + + #[test] + fn hip_comparators_pin_empty_digest_list_boundaries() { + let c = &mut init_context(); + let empty = D(0); + let value = digest(&mut c.stack, [1, 2, 3, 4, 5]); + + assert!(!cmp_with_jet(c, gor_hip_jet, empty, value).expect("gor-hip should succeed")); + assert!(cmp_with_jet(c, gor_hip_jet, value, empty).expect("gor-hip should succeed")); + assert!(!cmp_with_jet(c, gor_hip_jet, empty, empty).expect("gor-hip should succeed")); + assert!(!cmp_with_jet(c, mor_hip_jet, empty, value).expect("mor-hip should succeed")); + assert!(cmp_with_jet(c, mor_hip_jet, value, empty).expect("mor-hip should succeed")); + assert!(!cmp_with_jet(c, mor_hip_jet, empty, empty).expect("mor-hip should succeed")); + } + + #[test] + fn hip_jets_error_on_malformed_keys() { + let c = &mut init_context(); + let good = digest(&mut c.stack, [1, 2, 3, 4, 5]); + let malformed_atom = D(42); + let malformed_middle = list(&mut c.stack, &[good, D(42)]); + let improper_tail = T(&mut c.stack, &[good, D(7)]); + let singleton = digest_list(&mut c.stack, &[[1, 2, 3, 4, 5]]); + + for bad in [malformed_atom, malformed_middle, improper_tail, singleton] { + assert!( + cmp_with_jet(c, gor_hip_jet, bad, good).is_err(), + "gor-hip accepted malformed key {bad:?}" + ); + assert!( + cmp_with_jet(c, mor_hip_jet, good, bad).is_err(), + "mor-hip accepted malformed key {bad:?}" + ); + } + } + + #[test] + fn zh_conversion_jets_reject_malformed_z_trees() { + let c = &mut init_context(); + let key = digest(&mut c.stack, [1, 2, 3, 4, 5]); + let value = digest(&mut c.stack, [6, 7, 8, 9, 10]); + let malformed_key = D(42); + let malformed_map_entry = T(&mut c.stack, &[malformed_key, value]); + let malformed_map = T(&mut c.stack, &[malformed_map_entry, D(0), D(0)]); + let improper_map_entry = T(&mut c.stack, &[key, value]); + let improper_map = T(&mut c.stack, &[improper_map_entry, D(0)]); + let malformed_set = T(&mut c.stack, &[malformed_key, D(0), D(0)]); + let improper_set = T(&mut c.stack, &[key, D(0)]); + + assert!(run_unary_jet(c, zh_molt_jet, D(42)).is_err()); + assert!(run_unary_jet(c, zh_molt_jet, malformed_map).is_err()); + assert!(run_unary_jet(c, zh_molt_jet, improper_map).is_err()); + assert!(run_unary_jet(c, zh_silt_jet, D(42)).is_err()); + assert!(run_unary_jet(c, zh_silt_jet, malformed_set).is_err()); + assert!(run_unary_jet(c, zh_silt_jet, improper_set).is_err()); + } + + #[test] + fn zh_conversion_jets_reject_non_strict_h_order_trees() { + let c = &mut init_context(); + let limbs = [1, 2, 3, 4, 5]; + let digest_key = digest(&mut c.stack, limbs); + let singleton_list_key = digest_list(&mut c.stack, &[limbs]); + + let map_entry_a = T(&mut c.stack, &[digest_key, D(10)]); + let map_entry_b = T(&mut c.stack, &[singleton_list_key, D(20)]); + let map_child = T(&mut c.stack, &[map_entry_b, D(0), D(0)]); + let map = T(&mut c.stack, &[map_entry_a, D(0), map_child]); + + let set_child = T(&mut c.stack, &[singleton_list_key, D(0), D(0)]); + let set = T(&mut c.stack, &[digest_key, D(0), set_child]); + + assert!(run_unary_jet(c, zh_molt_jet, map).is_err()); + assert!(run_unary_jet(c, zh_silt_jet, set).is_err()); + } + + #[test] + fn small_digest_fast_path_rejects_singleton_digest_list_keys() { + let c = &mut init_context(); + let limbs = [1, 2, 3, 4, 5]; + let second = [6, 7, 8, 9, 10]; + let direct = digest(&mut c.stack, limbs); + let singleton = digest_list(&mut c.stack, &[limbs]); + let pair = digest_list(&mut c.stack, &[limbs, second]); + let too_long = digest_list(&mut c.stack, &[limbs, second, [11, 12, 13, 14, 15]]); + let improper = T(&mut c.stack, &[direct, D(7)]); + let malformed_singleton = T(&mut c.stack, &[D(42), D(0)]); + let pair_key = SmallHashedKey::pair(limbs, second); + let space = c.stack.noun_space(); + + assert_eq!( + small_hashed_key_from_noun(direct, &space), + Some(SmallHashedKey::single(limbs)) + ); + assert_eq!(small_hashed_key_from_noun(singleton, &space), None); + assert_eq!(small_hashed_key_from_noun(pair, &space), Some(pair_key)); + assert_eq!( + small_hashed_key_from_noun(D(0), &space), + Some(SmallHashedKey::empty()) + ); + assert_eq!(small_hashed_key_from_noun(too_long, &space), None); + assert_eq!(small_hashed_key_from_noun(improper, &space), None); + assert_eq!( + small_hashed_key_from_noun(malformed_singleton, &space), + None + ); + } + + #[test] + fn zh_conversion_jets_match_slow_oracles_for_short_digest_lists() { + let c = &mut init_context(); + let key_a = digest_list(&mut c.stack, &[[1, 0, 0, 0, 0], [1, 1, 0, 0, 0]]); + let key_b = digest_list(&mut c.stack, &[[0, 2, 0, 0, 0], [0, 2, 1, 0, 0]]); + let key_c = digest(&mut c.stack, [0, 0, 3, 0, 0]); + let key_d = digest_list(&mut c.stack, &[[0, 0, 0, 4, 0], [0, 0, 0, 0, 5]]); + let map = z_map_from_entries( + &mut c.stack, + &[(key_a, D(10)), (key_b, D(20)), (key_c, D(30)), (key_d, D(40))], + ); + let entry_a = T(&mut c.stack, &[key_a, D(10)]); + let entry_b = T(&mut c.stack, &[key_b, D(20)]); + let entry_c = T(&mut c.stack, &[key_c, D(30)]); + let entry_d = T(&mut c.stack, &[key_d, D(40)]); + let h_entries = vec![ + HMapEntry { + noun: entry_a, + key: key_a, + }, + HMapEntry { + noun: entry_b, + key: key_b, + }, + HMapEntry { + noun: entry_c, + key: key_c, + }, + HMapEntry { + noun: entry_d, + key: key_d, + }, + ]; + let space = c.stack.noun_space(); + assert!(small_h_map_build_nodes(&h_entries, &space).is_some()); + + let converted_map = run_unary_jet(c, zh_molt_jet, map).expect("zh-molt should convert"); + let expected_map = slow_z_map_to_h_map(&mut c.stack, map).expect("oracle should convert"); + assert_noun_eq(&mut c.stack, converted_map, expected_map); + + let set = z_set_from_items(&mut c.stack, &[key_a, key_b, key_c, key_d]); + let set_items = vec![key_a, key_b, key_c, key_d]; + assert!(small_h_set_build_nodes(&set_items, &space).is_some()); + + let converted_set = run_unary_jet(c, zh_silt_jet, set).expect("zh-silt should convert"); + let expected_set = slow_z_set_to_h_set(&mut c.stack, set).expect("oracle should convert"); + assert_noun_eq(&mut c.stack, converted_set, expected_set); + + let inner_key_a = digest_list(&mut c.stack, &[[4, 0, 0, 0, 0], [4, 1, 0, 0, 0]]); + let inner_key_b = digest_list(&mut c.stack, &[[0, 5, 0, 0, 0], [0, 0, 6, 0, 0]]); + let inner_map = + z_map_from_entries(&mut c.stack, &[(inner_key_a, D(40)), (inner_key_b, D(50))]); + let outer_key_a = digest_list(&mut c.stack, &[[0, 0, 6, 0, 0], [0, 0, 6, 1, 0]]); + let outer_key_b = digest_list(&mut c.stack, &[[0, 0, 0, 7, 0], [0, 0, 0, 7, 1]]); + let mip = z_map_from_entries( + &mut c.stack, + &[(outer_key_a, inner_map), (outer_key_b, inner_map)], + ); + let converted_mip = run_unary_jet(c, zh_milt_jet, mip).expect("zh-milt should convert"); + let expected_mip = slow_z_mip_to_h_mip(&mut c.stack, mip).expect("oracle should convert"); + assert_noun_eq(&mut c.stack, converted_mip, expected_mip); + + let inner_set = z_set_from_items(&mut c.stack, &[inner_key_a, inner_key_b]); + let jug = z_map_from_entries( + &mut c.stack, + &[(outer_key_a, inner_set), (outer_key_b, inner_set)], + ); + let converted_jug = run_unary_jet(c, zh_jult_jet, jug).expect("zh-jult should convert"); + let expected_jug = slow_z_jug_to_h_jug(&mut c.stack, jug).expect("oracle should convert"); + assert_noun_eq(&mut c.stack, converted_jug, expected_jug); + } + + #[test] + fn zh_milt_jet_preserves_arbitrary_inner_values() { + let c = &mut init_context(); + let outer_key = digest(&mut c.stack, [1, 1, 1, 1, 1]); + let inner_key = digest(&mut c.stack, [2, 2, 2, 2, 2]); + let value_tail = T(&mut c.stack, &[D(99), D(100)]); + let arbitrary_value = T(&mut c.stack, &[D(42), value_tail]); + let inner = z_map_from_entries(&mut c.stack, &[(inner_key, arbitrary_value)]); + let mip = z_map_from_entries(&mut c.stack, &[(outer_key, inner)]); + + let converted = run_unary_jet(c, zh_milt_jet, mip).expect("zh-milt should convert"); + let expected = slow_z_mip_to_h_mip(&mut c.stack, mip).expect("oracle should convert"); + + assert_noun_eq(&mut c.stack, converted, expected); + } + + #[test] + fn balance_parent_diff_matches_generic_for_put_update_delete() { + let c = &mut init_context(); + let key_a = digest(&mut c.stack, [1, 0, 0, 0, 0]); + let key_b = digest_list(&mut c.stack, &[[2, 0, 0, 0, 0], [3, 0, 0, 0, 0]]); + let key_c = digest(&mut c.stack, [4, 0, 0, 0, 0]); + let key_d = digest_list(&mut c.stack, &[[5, 0, 0, 0, 0], [6, 0, 0, 0, 0]]); + let note_a = T(&mut c.stack, &[D(10), D(11)]); + let note_b = T(&mut c.stack, &[D(20), D(21)]); + let note_c = T(&mut c.stack, &[D(30), D(31)]); + let note_b_updated = T(&mut c.stack, &[D(22), D(23)]); + let note_d = T(&mut c.stack, &[D(40), D(41)]); + let parent_z = z_map_from_entries( + &mut c.stack, + &[(key_a, note_a), (key_b, note_b), (key_c, note_c)], + ); + let child_z = z_map_from_entries( + &mut c.stack, + &[(key_a, note_a), (key_b, note_b_updated), (key_d, note_d)], + ); + let parent_h = slow_z_map_to_h_map(&mut c.stack, parent_z).expect("parent converts"); + let space = c.stack.noun_space(); + + let derived = h_map_from_parent_z_diff(&mut c.stack, parent_z, parent_h, child_z, &space) + .expect("diff works"); + let expected = slow_z_map_to_h_map(&mut c.stack, child_z).expect("child converts"); + + assert_noun_eq(&mut c.stack, derived, expected); + } + + #[test] + fn balance_parent_diff_rejects_singleton_key_shape_change() { + let c = &mut init_context(); + let key_direct = digest(&mut c.stack, [8, 0, 0, 0, 0]); + let key_list = digest_list(&mut c.stack, &[[8, 0, 0, 0, 0]]); + let note = T(&mut c.stack, &[D(80), D(81)]); + let parent_z = z_map_from_entries(&mut c.stack, &[(key_direct, note)]); + let child_z = z_map_from_entries(&mut c.stack, &[(key_list, note)]); + let parent_h = slow_z_map_to_h_map(&mut c.stack, parent_z).expect("parent converts"); + let space = c.stack.noun_space(); + + assert!( + h_map_from_parent_z_diff(&mut c.stack, parent_z, parent_h, child_z, &space).is_err() + ); + assert!(slow_z_map_to_h_map(&mut c.stack, child_z).is_err()); + } + + #[test] + fn zh_balance_milt_jet_matches_generic_milt_for_parent_chain() { + let c = &mut init_context(); + let parent_block = digest(&mut c.stack, [11, 0, 0, 0, 0]); + let child_block = digest(&mut c.stack, [12, 0, 0, 0, 0]); + let missing_genesis_parent = digest(&mut c.stack, [10, 0, 0, 0, 0]); + let parent_page = local_page_v0(&mut c.stack, parent_block, missing_genesis_parent); + let child_page = local_page_v0(&mut c.stack, child_block, parent_block); + let blocks = z_map_from_entries( + &mut c.stack, + &[(parent_block, parent_page), (child_block, child_page)], + ); + + let key_a = digest(&mut c.stack, [1, 1, 0, 0, 0]); + let key_b = digest_list(&mut c.stack, &[[2, 1, 0, 0, 0], [3, 1, 0, 0, 0]]); + let key_c = digest(&mut c.stack, [4, 1, 0, 0, 0]); + let note_a = T(&mut c.stack, &[D(100), D(101)]); + let note_b = T(&mut c.stack, &[D(200), D(201)]); + let note_c = T(&mut c.stack, &[D(300), D(301)]); + let parent_balance = z_map_from_entries(&mut c.stack, &[(key_a, note_a), (key_b, note_b)]); + let child_balance = z_map_from_entries(&mut c.stack, &[(key_b, note_b), (key_c, note_c)]); + let balance = z_map_from_entries( + &mut c.stack, + &[(parent_block, parent_balance), (child_block, child_balance)], + ); + + let sample = T(&mut c.stack, &[blocks, balance]); + let converted = + run_unary_jet(c, zh_balance_milt_jet, sample).expect("zh-balmilt should convert"); + let expected = slow_z_mip_to_h_mip(&mut c.stack, balance).expect("generic oracle converts"); + + assert_noun_eq(&mut c.stack, converted, expected); + } + + #[test] + fn zh_balance_milt_jet_falls_back_for_non_digest_outer_balance_keys() { + let c = &mut init_context(); + let block = digest(&mut c.stack, [41, 0, 0, 0, 0]); + let parent = digest(&mut c.stack, [40, 0, 0, 0, 0]); + let page = local_page_v0(&mut c.stack, block, parent); + let blocks = z_map_from_entries(&mut c.stack, &[(block, page)]); + + let outer_key = digest_list(&mut c.stack, &[[42, 0, 0, 0, 0], [43, 0, 0, 0, 0]]); + let inner_key = digest(&mut c.stack, [44, 0, 0, 0, 0]); + let inner_value = T(&mut c.stack, &[D(440), D(441)]); + let inner = z_map_from_entries(&mut c.stack, &[(inner_key, inner_value)]); + let balance = z_map_from_entries(&mut c.stack, &[(outer_key, inner)]); + let sample = T(&mut c.stack, &[blocks, balance]); + + let converted = + run_unary_jet(c, zh_balance_milt_jet, sample).expect("zh-balmilt should convert"); + let expected = slow_z_mip_to_h_mip(&mut c.stack, balance).expect("generic oracle converts"); + + assert_noun_eq(&mut c.stack, converted, expected); + } + + #[test] + fn zh_balance_milt_jet_matches_generic_milt_for_parent_cycle() { + let c = &mut init_context(); + let block_a = digest(&mut c.stack, [21, 0, 0, 0, 0]); + let block_b = digest(&mut c.stack, [22, 0, 0, 0, 0]); + let page_a = local_page_v0(&mut c.stack, block_a, block_b); + let page_b = local_page_v0(&mut c.stack, block_b, block_a); + let blocks = z_map_from_entries(&mut c.stack, &[(block_a, page_a), (block_b, page_b)]); + + let key_a = digest(&mut c.stack, [31, 0, 0, 0, 0]); + let key_b = digest_list(&mut c.stack, &[[32, 0, 0, 0, 0], [33, 0, 0, 0, 0]]); + let key_c = digest(&mut c.stack, [34, 0, 0, 0, 0]); + let note_a = T(&mut c.stack, &[D(410), D(411)]); + let note_b = T(&mut c.stack, &[D(420), D(421)]); + let note_c = T(&mut c.stack, &[D(430), D(431)]); + let balance_a = z_map_from_entries(&mut c.stack, &[(key_a, note_a), (key_b, note_b)]); + let balance_b = z_map_from_entries(&mut c.stack, &[(key_b, note_b), (key_c, note_c)]); + let balance = + z_map_from_entries(&mut c.stack, &[(block_a, balance_a), (block_b, balance_b)]); + + let sample = T(&mut c.stack, &[blocks, balance]); + let converted = + run_unary_jet(c, zh_balance_milt_jet, sample).expect("zh-balmilt should convert"); + let expected = slow_z_mip_to_h_mip(&mut c.stack, balance).expect("generic oracle converts"); + + assert_noun_eq(&mut c.stack, converted, expected); + } + + #[test] + fn zh_balance_milt_jet_matches_generic_milt_for_long_mixed_history() { + let c = &mut init_context(); + let mut keys = Vec::new(); + for index in 0..18 { + let limb = index as u64 + 1; + let key = match index % 3 { + 0 => digest(&mut c.stack, [limb, 0, 0, 0, 0]), + 1 => digest_list(&mut c.stack, &[[0, limb, 0, 0, 0], [0, limb, 1, 0, 0]]), + _ => digest_list(&mut c.stack, &[[0, 0, limb, 0, 0], [0, 0, 0, limb, 0]]), + }; + keys.push(key); + } + + let mut block_entries: Vec<(Noun, Noun)> = Vec::new(); + let mut balance_entries: Vec<(Noun, Noun)> = Vec::new(); + let mut live_values = vec![None; keys.len()]; + let mut parent = digest(&mut c.stack, [900, 0, 0, 0, 0]); + + for height in 0..48 { + let block = digest( + &mut c.stack, + [1_000 + height as u64, height as u64, 0, 0, 0], + ); + let page = if height % 5 == 0 { + local_page_v1(&mut c.stack, block, parent) + } else { + local_page_v0(&mut c.stack, block, parent) + }; + block_entries.push((block, page)); + + let primary = height % keys.len(); + if height % 11 == 5 { + live_values[primary] = None; + } else { + live_values[primary] = Some(T( + &mut c.stack, + &[D(10_000 + height as u64), D(primary as u64)], + )); + } + + let secondary = (height * 7 + 3) % keys.len(); + if height % 4 == 0 { + live_values[secondary] = Some(T( + &mut c.stack, + &[D(20_000 + height as u64), D(secondary as u64)], + )); + } + + if live_values.iter().all(Option::is_none) { + live_values[0] = Some(T(&mut c.stack, &[D(30_000 + height as u64), D(0)])); + } + + let mut entries = Vec::new(); + for (key, value) in keys.iter().zip(live_values.iter()) { + if let Some(value) = value { + entries.push((*key, *value)); + } + } + + let balance = if height % 13 == 0 && !balance_entries.is_empty() { + balance_entries + .last() + .expect("previous balance should exist") + .1 + } else { + z_map_from_entries(&mut c.stack, &entries) + }; + balance_entries.push((block, balance)); + parent = block; + } + + let blocks = z_map_from_entries(&mut c.stack, &block_entries); + let balance = z_map_from_entries(&mut c.stack, &balance_entries); + let sample = T(&mut c.stack, &[blocks, balance]); + + let converted = + run_unary_jet(c, zh_balance_milt_jet, sample).expect("zh-balmilt should convert"); + let expected = slow_z_mip_to_h_mip(&mut c.stack, balance).expect("generic oracle converts"); + + assert_noun_eq(&mut c.stack, converted, expected); + } + + #[test] + fn quickcheck_parent_diff_matches_generic_for_generated_note_maps() { + fn prop(input: NoteMapDiffInput) -> bool { + let c = &mut init_context(); + let (parent_z, child_z) = generated_note_map_pair(&mut c.stack, &input); + let parent_h = match slow_z_map_to_h_map(&mut c.stack, parent_z) { + Ok(parent_h) => parent_h, + Err(_) => return false, + }; + let space = c.stack.noun_space(); + let derived = + match h_map_from_parent_z_diff(&mut c.stack, parent_z, parent_h, child_z, &space) { + Ok(derived) => derived, + Err(_) => return false, + }; + let expected = match slow_z_map_to_h_map(&mut c.stack, child_z) { + Ok(expected) => expected, + Err(_) => return false, + }; + + noun_eq(c, derived, expected) + } + + QuickCheck::new() + .tests(256) + .quickcheck(prop as fn(NoteMapDiffInput) -> bool); + } + + #[test] + fn quickcheck_zh_balance_milt_matches_generic_for_generated_histories() { + fn prop(input: BalanceHistoryInput) -> bool { + let c = &mut init_context(); + let (blocks, balance, _block_ids) = generated_balance_history(&mut c.stack, &input); + let sample = T(&mut c.stack, &[blocks, balance]); + let converted = match run_unary_jet(c, zh_balance_milt_jet, sample) { + Ok(converted) => converted, + Err(_) => return false, + }; + let expected = match slow_z_mip_to_h_mip(&mut c.stack, balance) { + Ok(expected) => expected, + Err(_) => return false, + }; + + noun_eq(c, converted, expected) + } + + QuickCheck::new() + .tests(128) + .quickcheck(prop as fn(BalanceHistoryInput) -> bool); + } + + #[test] + fn quickcheck_zh_balance_milt_bad_block_hints_match_generic() { + fn prop(input: BalanceHistoryInput, atom_hint: bool) -> bool { + let c = &mut init_context(); + let (_blocks, balance, block_ids) = generated_balance_history(&mut c.stack, &input); + let bad_blocks = if atom_hint { + D(42) + } else { + let key = block_ids + .first() + .copied() + .unwrap_or_else(|| digest(&mut c.stack, [1; 5])); + z_map_from_entries(&mut c.stack, &[(key, D(7))]) + }; + let sample = T(&mut c.stack, &[bad_blocks, balance]); + let converted = match run_unary_jet(c, zh_balance_milt_jet, sample) { + Ok(converted) => converted, + Err(_) => return false, + }; + let expected = match slow_z_mip_to_h_mip(&mut c.stack, balance) { + Ok(expected) => expected, + Err(_) => return false, + }; + + noun_eq(c, converted, expected) + } + + QuickCheck::new() + .tests(128) + .quickcheck(prop as fn(BalanceHistoryInput, bool) -> bool); + } + + #[test] + fn zh_balance_milt_falls_back_for_large_rotation_region() { + let c = &mut init_context(); + let parent_block = digest(&mut c.stack, [71, 0, 0, 0, 0]); + let child_block = digest(&mut c.stack, [72, 0, 0, 0, 0]); + let grandparent_block = digest(&mut c.stack, [70; 5]); + let parent_page = local_page_v0(&mut c.stack, parent_block, grandparent_block); + let child_page = local_page_v0(&mut c.stack, child_block, parent_block); + let blocks = z_map_from_entries( + &mut c.stack, + &[(parent_block, parent_page), (child_block, child_page)], + ); + let parent_balance = generated_dense_note_map(&mut c.stack, 0x7100, 270); + let child_balance = generated_dense_note_map(&mut c.stack, 0x7200, 270); + let balance = z_map_from_entries( + &mut c.stack, + &[(parent_block, parent_balance), (child_block, child_balance)], + ); + let parent_h = slow_z_map_to_h_map(&mut c.stack, parent_balance).expect("parent converts"); + let space = c.stack.noun_space(); + + assert!( + h_map_from_parent_z_diff( + &mut c.stack, parent_balance, parent_h, child_balance, &space, + ) + .is_err(), + "large unmatched rotation regions should use generic conversion" + ); + + let sample = T(&mut c.stack, &[blocks, balance]); + let converted = + run_unary_jet(c, zh_balance_milt_jet, sample).expect("zh-balmilt should convert"); + let expected = slow_z_mip_to_h_mip(&mut c.stack, balance).expect("generic oracle converts"); + + assert_noun_eq(&mut c.stack, converted, expected); + } + + #[test] + fn balance_parent_diff_bails_on_singleton_digest_list_keys() { + let c = &mut init_context(); + let limbs = [88, 0, 0, 0, 0]; + let direct = digest(&mut c.stack, limbs); + let singleton = digest_list(&mut c.stack, &[limbs]); + let parent_key = digest(&mut c.stack, [89, 0, 0, 0, 0]); + let parent_z = z_map_from_entries(&mut c.stack, &[(parent_key, D(1))]); + let child_z = z_map_from_entries(&mut c.stack, &[(direct, D(2)), (singleton, D(3))]); + let mut actions: Vec = Vec::new(); + let space = c.stack.noun_space(); + + assert!( + collect_rotated_z_map_diff(&mut c.stack, parent_z, child_z, &space, &mut actions) + .is_err(), + "singleton digest-list keys should reject before diff optimization" + ); + } + + #[test] + fn local_page_parent_reads_v0_and_v1_storage_shapes() { + let c = &mut init_context(); + let block = digest(&mut c.stack, [7, 7, 7, 7, 7]); + let parent = digest(&mut c.stack, [8, 8, 8, 8, 8]); + let v0 = local_page_v0(&mut c.stack, block, parent); + let v1 = local_page_v1(&mut c.stack, block, parent); + let space = c.stack.noun_space(); + + let v0_parent = local_page_parent(v0, &space).expect("v0 parent should parse"); + let v1_parent = local_page_parent(v1, &space).expect("v1 parent should parse"); + + assert_noun_eq(&mut c.stack, v0_parent, parent); + assert_noun_eq(&mut c.stack, v1_parent, parent); + } + + #[test] + fn zh_nested_conversion_jets_validate_inner_container_keys() { + let c = &mut init_context(); + let outer_key = digest(&mut c.stack, [1, 1, 1, 1, 1]); + let malformed_map_entry = T(&mut c.stack, &[D(42), D(7)]); + let bad_inner_map = T(&mut c.stack, &[malformed_map_entry, D(0), D(0)]); + let bad_mip = z_map_from_entries(&mut c.stack, &[(outer_key, bad_inner_map)]); + + let jug_key = digest(&mut c.stack, [3, 3, 3, 3, 3]); + let bad_set = T(&mut c.stack, &[D(42), D(0), D(0)]); + let bad_jug = z_map_from_entries(&mut c.stack, &[(jug_key, bad_set)]); + + assert!(run_unary_jet(c, zh_milt_jet, bad_mip).is_err()); + assert!(run_unary_jet(c, zh_jult_jet, bad_jug).is_err()); + } + + #[test] + fn zh_conversion_jets_match_slow_put_oracles_for_mixed_shapes() { + let c = &mut init_context(); + let key_a = digest(&mut c.stack, [1, 0, 0, 0, 0]); + let key_b = digest_list(&mut c.stack, &[[2, 0, 0, 0, 0], [3, 0, 0, 0, 0]]); + let key_c = digest(&mut c.stack, [4, 0, 0, 0, 0]); + let value_a = D(42); + let value_b_tail = T(&mut c.stack, &[D(7), D(8)]); + let value_b = T(&mut c.stack, &[D(6), value_b_tail]); + let value_c = digest(&mut c.stack, [9, 0, 0, 0, 0]); + let map = z_map_from_entries( + &mut c.stack, + &[(key_a, value_a), (key_b, value_b), (key_c, value_c)], + ); + let converted_map = run_unary_jet(c, zh_molt_jet, map).expect("zh-molt should convert"); + let expected_map = slow_z_map_to_h_map(&mut c.stack, map).expect("oracle should convert"); + assert_noun_eq(&mut c.stack, converted_map, expected_map); + + let set = z_set_from_items(&mut c.stack, &[key_a, key_b, key_c]); + let converted_set = run_unary_jet(c, zh_silt_jet, set).expect("zh-silt should convert"); + let expected_set = slow_z_set_to_h_set(&mut c.stack, set).expect("oracle should convert"); + assert_noun_eq(&mut c.stack, converted_set, expected_set); + + let inner_key_a = digest(&mut c.stack, [0, 1, 0, 0, 0]); + let inner_key_b = digest_list(&mut c.stack, &[[0, 2, 0, 0, 0], [0, 3, 0, 0, 0]]); + let first_value = T(&mut c.stack, &[D(1), D(2)]); + let second_value = T(&mut c.stack, &[D(3), D(4)]); + let replacement_value = T(&mut c.stack, &[D(5), D(6)]); + let inner_map_a = z_map_from_entries( + &mut c.stack, + &[ + (inner_key_a, first_value), + (inner_key_b, second_value), + (inner_key_a, replacement_value), + ], + ); + let inner_map_b = z_map_from_entries( + &mut c.stack, + &[(inner_key_b, value_c), (inner_key_a, value_b)], + ); + let outer_key_a = digest(&mut c.stack, [5, 0, 0, 0, 0]); + let outer_key_b = digest(&mut c.stack, [6, 0, 0, 0, 0]); + let mip = z_map_from_entries( + &mut c.stack, + &[(outer_key_a, inner_map_a), (outer_key_b, inner_map_b)], + ); + let converted_mip = run_unary_jet(c, zh_milt_jet, mip).expect("zh-milt should convert"); + let expected_mip = slow_z_mip_to_h_mip(&mut c.stack, mip).expect("oracle should convert"); + assert_noun_eq(&mut c.stack, converted_mip, expected_mip); + + let inner_set_a = z_set_from_items(&mut c.stack, &[inner_key_a, inner_key_b]); + let inner_set_b = z_set_from_items(&mut c.stack, &[inner_key_b, key_c]); + let jug = z_map_from_entries( + &mut c.stack, + &[(outer_key_a, inner_set_a), (outer_key_b, inner_set_b)], + ); + let converted_jug = run_unary_jet(c, zh_jult_jet, jug).expect("zh-jult should convert"); + let expected_jug = slow_z_jug_to_h_jug(&mut c.stack, jug).expect("oracle should convert"); + assert_noun_eq(&mut c.stack, converted_jug, expected_jug); + } + + #[test] + fn gor_tip_errors_on_non_decodable_tip_cache_entry() { + let c = &mut init_context(); + let a = T(&mut c.stack, &[D(1), D(2)]); + let b = T(&mut c.stack, &[D(3), D(4)]); + inject_bad_cache_value(c, TIP_CACHE_TAG, a); + + let subject = jet_subject(&mut c.stack, a, b); + assert!(gor_tip_jet(c, subject).is_err()); + } + + #[test] + fn mor_tip_errors_on_non_decodable_double_tip_cache_entry() { + let c = &mut init_context(); + let a = T(&mut c.stack, &[D(5), D(6)]); + let b = T(&mut c.stack, &[D(7), D(8)]); + inject_bad_cache_value(c, DOUBLE_TIP_CACHE_TAG, a); + + let subject = jet_subject(&mut c.stack, a, b); + assert!(mor_tip_jet(c, subject).is_err()); + } + + #[test] + fn gor_and_mor_error_on_non_u64_atom_inputs() { + let c = &mut init_context(); + let huge_atom = A(&mut c.stack, &(UBig::from(1u128) << 64)); + let other = D(1); + + let subject = jet_subject(&mut c.stack, huge_atom, other); + assert!(gor_tip_jet(c, subject).is_err()); + assert!(mor_tip_jet(c, subject).is_err()); + } + + #[test] + fn quickcheck_order_laws_for_dor_gor_mor() { + fn prop(a: BoundedNounInput, b: BoundedNounInput, c_input: BoundedNounInput) -> bool { + let context = &mut init_context(); + let an = bounded_noun_from_input(&mut context.stack, &a); + let bn = bounded_noun_from_input(&mut context.stack, &b); + let cn = bounded_noun_from_input(&mut context.stack, &c_input); + + let dor_ab = + cmp_with_jet(context, dor_tip_jet, an, bn).expect("dor comparison should succeed"); + let dor_ba = + cmp_with_jet(context, dor_tip_jet, bn, an).expect("dor comparison should succeed"); + let dor_bc = + cmp_with_jet(context, dor_tip_jet, bn, cn).expect("dor comparison should succeed"); + let dor_ac = + cmp_with_jet(context, dor_tip_jet, an, cn).expect("dor comparison should succeed"); + + if !dor_ab && !dor_ba { + return false; + } + if dor_ab && dor_ba && !noun_eq(context, an, bn) { + return false; + } + if dor_ab && dor_bc && !dor_ac { + return false; + } + + for jet in [gor_tip_jet, mor_tip_jet] { + let ab = cmp_with_jet(context, jet, an, bn).expect("comparison should succeed"); + let ba = cmp_with_jet(context, jet, bn, an).expect("comparison should succeed"); + let bc = cmp_with_jet(context, jet, bn, cn).expect("comparison should succeed"); + let ac = cmp_with_jet(context, jet, an, cn).expect("comparison should succeed"); + + if !ab && !ba { + return false; // totality + } + if ab && ba && !noun_eq(context, an, bn) { + return false; // antisymmetry + } + if ab && bc && !ac { + return false; // transitivity + } + } + + true + } + + QuickCheck::new() + .tests(256) + .quickcheck(prop as fn(BoundedNounInput, BoundedNounInput, BoundedNounInput) -> bool); + } + + #[test] + fn quickcheck_cold_vs_warm_cache_equivalence() { + fn prop(a: BoundedNounInput, b: BoundedNounInput) -> bool { + let mut cold = init_context(); + let cold_a = bounded_noun_from_input(&mut cold.stack, &a); + let cold_b = bounded_noun_from_input(&mut cold.stack, &b); + + let cold_dor = + cmp_with_jet(&mut cold, dor_tip_jet, cold_a, cold_b).expect("dor should succeed"); + let cold_gor = + cmp_with_jet(&mut cold, gor_tip_jet, cold_a, cold_b).expect("gor should succeed"); + let cold_mor = + cmp_with_jet(&mut cold, mor_tip_jet, cold_a, cold_b).expect("mor should succeed"); + + let mut warm = init_context(); + let warm_a = bounded_noun_from_input(&mut warm.stack, &a); + let warm_b = bounded_noun_from_input(&mut warm.stack, &b); + + let warm_first_dor = + cmp_with_jet(&mut warm, dor_tip_jet, warm_a, warm_b).expect("dor should succeed"); + let warm_first_gor = + cmp_with_jet(&mut warm, gor_tip_jet, warm_a, warm_b).expect("gor should succeed"); + let warm_first_mor = + cmp_with_jet(&mut warm, mor_tip_jet, warm_a, warm_b).expect("mor should succeed"); + + let warm_second_dor = + cmp_with_jet(&mut warm, dor_tip_jet, warm_a, warm_b).expect("dor should succeed"); + let warm_second_gor = + cmp_with_jet(&mut warm, gor_tip_jet, warm_a, warm_b).expect("gor should succeed"); + let warm_second_mor = + cmp_with_jet(&mut warm, mor_tip_jet, warm_a, warm_b).expect("mor should succeed"); + + cold_dor == warm_first_dor + && cold_gor == warm_first_gor + && cold_mor == warm_first_mor + && warm_first_dor == warm_second_dor + && warm_first_gor == warm_second_gor + && warm_first_mor == warm_second_mor + } + + QuickCheck::new() + .tests(256) + .quickcheck(prop as fn(BoundedNounInput, BoundedNounInput) -> bool); + } + + // The +gor-hip / +mor-hip total-order properties must hold over the entire + // digest input space; the migration's +apt invariant on h-* containers is a + // direct consequence. Random triples exercise both the single-digest fast + // path and the digest-list lex path, including equal-prefix list ties. + #[test] + fn quickcheck_hip_total_order_laws() { + fn prop(a: HipKeyInput, b: HipKeyInput, c_input: HipKeyInput) -> bool { + let context = &mut init_context(); + let an = noun_for_hip_key(&mut context.stack, &a); + let bn = noun_for_hip_key(&mut context.stack, &b); + let cn = noun_for_hip_key(&mut context.stack, &c_input); + + for jet in [gor_hip_jet, mor_hip_jet] { + let ab = cmp_with_jet(context, jet, an, bn).expect("hip jet should succeed"); + let ba = cmp_with_jet(context, jet, bn, an).expect("hip jet should succeed"); + let bc = cmp_with_jet(context, jet, bn, cn).expect("hip jet should succeed"); + let ac = cmp_with_jet(context, jet, an, cn).expect("hip jet should succeed"); + + let eq_ab = noun_eq(context, an, bn); + let eq_bc = noun_eq(context, bn, cn); + + // totality: gor(a, b) || gor(b, a) || a == b + if !ab && !ba && !eq_ab { + return false; + } + // antisymmetry: not both ab and ba unless a == b + if ab && ba && !eq_ab { + return false; + } + // transitivity through a "strict-less" lifting + let strict_ab = ab && !eq_ab; + let strict_bc = bc && !eq_bc; + if strict_ab && strict_bc && !ac { + return false; + } + } + + // mor-hip relates to gor-hip by reversing each digest's limb order + // (gor compares limbs 4..0, mor compares 0..4). + let reversed_a = reverse_limbs_for_key(&a); + let reversed_b = reverse_limbs_for_key(&b); + let an_rev = noun_for_hip_key(&mut context.stack, &reversed_a); + let bn_rev = noun_for_hip_key(&mut context.stack, &reversed_b); + + let mor_ab = cmp_with_jet(context, mor_hip_jet, an, bn).expect("mor-hip"); + let gor_rev = + cmp_with_jet(context, gor_hip_jet, an_rev, bn_rev).expect("gor-hip reversed"); + if mor_ab != gor_rev { + return false; + } + + // gor-hip on digest lists must agree with an independent lex oracle. + let limbs_a = limbs_for_key(&a); + let limbs_b = limbs_for_key(&b); + let oracle_gor = digest_list_order_oracle(&limbs_a, &limbs_b, &GOR_HIP_ORDER); + let oracle_mor = digest_list_order_oracle(&limbs_a, &limbs_b, &MOR_HIP_ORDER); + let jet_gor = cmp_with_jet(context, gor_hip_jet, an, bn).expect("gor-hip"); + let jet_mor = cmp_with_jet(context, mor_hip_jet, an, bn).expect("mor-hip"); + if oracle_gor != jet_gor { + return false; + } + if oracle_mor != jet_mor { + return false; + } + + true + } + + QuickCheck::new() + .tests(1024) + .quickcheck(prop as fn(HipKeyInput, HipKeyInput, HipKeyInput) -> bool); + } + + // Adversarial fuzz for the h-zoon migration jets. Anywhere a noun crosses + // from peer/disk into the migration we model the noun as malicious and + // require either correct acceptance + round-trip identity, or clean + // rejection via JetErr. No panic, no silent type confusion. + // + // The legitimate construction first feeds a small z-map / z-set built by + // `z_map_put` / `z_set_put`; the mutated variants then derive adversarial + // inputs by: + // - swapping a digest key for a non-digest atom (tag confusion) + // - truncating the digest list to 4 limbs (shape confusion) + // - injecting a %ztree marker (0x6565727a74) into an arm position + // - flipping the left/right children of an internal z-tree node so + // `apt` no longer holds + // + // Property: + // conversion(mutated) is Ok => round-trip back to z is identical + // conversion(mutated) is Err => no panic, JetErr is observed + #[test] + fn quickcheck_zh_molt_rejects_mutated_z_maps() { + fn prop(input: SoftFuzzInput) -> bool { + let c = &mut init_context(); + let map = build_seed_z_map(&mut c.stack, &input); + let mutated = mutate_z_tree(&mut c.stack, map, &input); + + // The jet must never panic on the mutated noun, but it may + // return either Ok (when the mutation preserved the validity + // constraints) or Err (when validity broke). On Ok we check + // the inverse oracle agrees. + match run_unary_jet(c, zh_molt_jet, mutated) { + Ok(converted) => { + // If we got Ok, the noun is a well-formed z-map (the + // jet validates) and conversion must agree with the + // independent slow oracle. + match slow_z_map_to_h_map(&mut c.stack, mutated) { + Ok(expected) => noun_eq(c, converted, expected), + Err(_) => { + // jet accepted but oracle rejected — divergence + false + } + } + } + Err(_) => true, + } + } + + QuickCheck::new() + .tests(2048) + .quickcheck(prop as fn(SoftFuzzInput) -> bool); + } + + #[test] + fn quickcheck_zh_silt_rejects_mutated_z_sets() { + fn prop(input: SoftFuzzInput) -> bool { + let c = &mut init_context(); + let set = build_seed_z_set(&mut c.stack, &input); + let mutated = mutate_z_tree(&mut c.stack, set, &input); + + match run_unary_jet(c, zh_silt_jet, mutated) { + Ok(converted) => match slow_z_set_to_h_set(&mut c.stack, mutated) { + Ok(expected) => noun_eq(c, converted, expected), + Err(_) => false, + }, + Err(_) => true, + } + } + + QuickCheck::new() + .tests(2048) + .quickcheck(prop as fn(SoftFuzzInput) -> bool); + } + + // Marker non-confusion: %ztree (the atom 0x7a74726565) is not a valid + // runtime leaf for any z-container. We verify the conversion jets reject + // any tree that contains %ztree at any internal position, and that the + // comparator jets reject %ztree-as-key likewise. + #[test] + fn ztree_marker_in_z_tree_is_rejected_by_conversion_jets() { + let c = &mut init_context(); + let key = digest(&mut c.stack, [1, 2, 3, 4, 5]); + let value = D(99); + let entry = T(&mut c.stack, &[key, value]); + let ztree_marker = D(0x6565727a74); // %ztree as a direct atom + let with_marker_left = T(&mut c.stack, &[entry, ztree_marker, D(0)]); + let with_marker_right = T(&mut c.stack, &[entry, D(0), ztree_marker]); + let just_marker = ztree_marker; + + for bad in [with_marker_left, with_marker_right, just_marker] { + assert!( + run_unary_jet(c, zh_molt_jet, bad).is_err(), + "zh-molt accepted %ztree marker as z-map" + ); + // For zh-silt we need a singular entry shape, not a pair shape. + let set_with_marker_left = T(&mut c.stack, &[key, ztree_marker, D(0)]); + let set_with_marker_right = T(&mut c.stack, &[key, D(0), ztree_marker]); + for set_bad in [set_with_marker_left, set_with_marker_right] { + assert!( + run_unary_jet(c, zh_silt_jet, set_bad).is_err(), + "zh-silt accepted %ztree marker as z-set" + ); + } + } + } + + // hset / hmap tag confusion: %hmap / %hset markers must not be accepted + // by the z-side conversion jets either. + #[test] + fn h_markers_in_z_input_are_rejected_by_conversion_jets() { + let c = &mut init_context(); + let hmap_marker = D(0x70616d68); // %hmap + let hset_marker = D(0x74657368); // %hset + let key = digest(&mut c.stack, [1, 2, 3, 4, 5]); + let value = D(11); + let entry = T(&mut c.stack, &[key, value]); + + // empty leaf where a z-tree expects ~ (i.e. D(0)) + assert!(run_unary_jet(c, zh_molt_jet, hmap_marker).is_err()); + assert!(run_unary_jet(c, zh_silt_jet, hset_marker).is_err()); + + // empty leaf in a child position + let with_hmap_l = T(&mut c.stack, &[entry, hmap_marker, D(0)]); + let with_hset_r = T(&mut c.stack, &[key, D(0), hset_marker]); + assert!(run_unary_jet(c, zh_molt_jet, with_hmap_l).is_err()); + assert!(run_unary_jet(c, zh_silt_jet, with_hset_r).is_err()); + } + + // Empty-list and equal-prefix edge cases are pinned points already + // covered by `test-hip-ordering-prefix-and-empty-boundaries`. This block + // confirms the same invariants survive random pairings drawn around the + // empty-list and equal-prefix neighborhoods. + #[test] + fn quickcheck_hip_empty_and_prefix_boundaries() { + fn prop(a: HipBoundaryInput, b: HipBoundaryInput) -> bool { + let context = &mut init_context(); + let an = noun_for_boundary(&mut context.stack, &a); + let bn = noun_for_boundary(&mut context.stack, &b); + + // Comparators must succeed on every valid boundary noun without panic. + let _ = cmp_with_jet(context, gor_hip_jet, an, bn).expect("gor-hip boundary"); + let _ = cmp_with_jet(context, mor_hip_jet, an, bn).expect("mor-hip boundary"); + + // The independent lex oracle must agree. + let la = limbs_for_boundary(&a); + let lb = limbs_for_boundary(&b); + let jet_g = cmp_with_jet(context, gor_hip_jet, an, bn).expect("gor-hip"); + let jet_m = cmp_with_jet(context, mor_hip_jet, an, bn).expect("mor-hip"); + jet_g == digest_list_order_oracle(&la, &lb, &GOR_HIP_ORDER) + && jet_m == digest_list_order_oracle(&la, &lb, &MOR_HIP_ORDER) + } + + QuickCheck::new() + .tests(512) + .quickcheck(prop as fn(HipBoundaryInput, HipBoundaryInput) -> bool); + } + + fn cache_entries(context: &Context) -> usize { + context.cache.iter().map(|pairs| pairs.len()).sum() + } + + fn inject_bad_cache_value(context: &mut Context, tag: u64, noun: Noun) { + let mut key = T(&mut context.stack, &[D(tag), noun]); + context.cache = context.cache.insert(&mut context.stack, &mut key, D(7)); + } + + fn cmp_with_jet( + context: &mut Context, + jet: fn(&mut Context, Noun) -> Result, + a: Noun, + b: Noun, ) -> Result { let subject = jet_subject(&mut context.stack, a, b); let result = jet(context, subject)?; @@ -355,6 +3794,200 @@ mod tests { T(stack, &[D(0), sam, D(0)]) } + fn digest(stack: &mut NockStack, limbs: [u64; 5]) -> Noun { + T( + stack, + &[ + D(direct_limb(limbs[0])), + D(direct_limb(limbs[1])), + D(direct_limb(limbs[2])), + D(direct_limb(limbs[3])), + D(direct_limb(limbs[4])), + ], + ) + } + + fn list(stack: &mut NockStack, items: &[Noun]) -> Noun { + items + .iter() + .rev() + .fold(D(0), |tail, item| T(stack, &[*item, tail])) + } + + fn run_unary_jet( + context: &mut Context, + jet: fn(&mut Context, Noun) -> Result, + sample: Noun, + ) -> Result { + let subject = unary_jet_subject(&mut context.stack, sample); + jet(context, subject) + } + + fn unary_jet_subject(stack: &mut NockStack, sample: Noun) -> Noun { + T(stack, &[D(0), sample, D(0)]) + } + + fn z_map_from_entries(stack: &mut NockStack, entries: &[(Noun, Noun)]) -> Noun { + let mut map = D(0); + for (key, value) in entries { + let mut key = *key; + let mut value = *value; + map = z_map_put(stack, &map, &mut key, &mut value, &DefaultTipHasher) + .expect("z-map construction should succeed"); + } + map + } + + fn z_set_from_items(stack: &mut NockStack, items: &[Noun]) -> Noun { + let mut set = D(0); + for item in items { + let mut item = *item; + set = z_set_put(stack, &set, &mut item, &DefaultTipHasher) + .expect("z-set construction should succeed"); + } + set + } + + fn slow_z_map_to_h_map(stack: &mut NockStack, tree: Noun) -> Result { + let space = stack.noun_space(); + let mut entries = Vec::new(); + collect_z_map_entries(tree, &space, &mut entries)?; + let mut map = h_map_empty(); + for (key, value) in entries { + map = h_map_put(stack, map, key, value, &space)?; + } + Ok(map) + } + + fn slow_z_set_to_h_set(stack: &mut NockStack, tree: Noun) -> Result { + let space = stack.noun_space(); + let mut items = Vec::new(); + collect_z_set_items(tree, &space, &mut items)?; + let mut set = h_set_empty(); + for item in items { + set = h_set_put(stack, set, item, &space)?; + } + Ok(set) + } + + fn slow_z_mip_to_h_mip(stack: &mut NockStack, tree: Noun) -> Result { + let space = stack.noun_space(); + let mut entries = Vec::new(); + collect_z_map_entries(tree, &space, &mut entries)?; + let mut map = h_map_empty(); + for (outer_key, inner_map) in entries { + let converted_inner = slow_z_map_to_h_map(stack, inner_map)?; + map = h_map_put(stack, map, outer_key, converted_inner, &space)?; + } + Ok(map) + } + + fn slow_z_jug_to_h_jug(stack: &mut NockStack, tree: Noun) -> Result { + let space = stack.noun_space(); + let mut entries = Vec::new(); + collect_z_map_entries(tree, &space, &mut entries)?; + let mut map = h_map_empty(); + for (outer_key, inner_set) in entries { + let converted_inner = slow_z_set_to_h_set(stack, inner_set)?; + map = h_map_put(stack, map, outer_key, converted_inner, &space)?; + } + Ok(map) + } + + fn collect_z_map_entries( + tree: Noun, + space: &NounSpace, + entries: &mut Vec<(Noun, Noun)>, + ) -> Result<(), JetErr> { + if unsafe { tree.raw_equals(&D(0)) } { + return Ok(()); + } + if tree.is_atom() { + return Err(BAIL_FAIL); + } + + let [entry, left, right] = tree.uncell(space)?; + let [key, value] = entry.uncell(space)?; + entries.push((key, value)); + collect_z_map_entries(left, space, entries)?; + collect_z_map_entries(right, space, entries) + } + + fn collect_z_set_items( + tree: Noun, + space: &NounSpace, + items: &mut Vec, + ) -> Result<(), JetErr> { + if unsafe { tree.raw_equals(&D(0)) } { + return Ok(()); + } + if tree.is_atom() { + return Err(BAIL_FAIL); + } + + let [value, left, right] = tree.uncell(space)?; + items.push(value); + collect_z_set_items(left, space, items)?; + collect_z_set_items(right, space, items) + } + + fn assert_noun_eq(stack: &mut NockStack, mut left: Noun, mut right: Noun) { + assert!( + unsafe { stack.equals(&mut left, &mut right) }, + "nouns are not structurally equal: {left:?} != {right:?}" + ); + } + + const GOR_HIP_ORDER: [usize; 5] = [4, 3, 2, 1, 0]; + const MOR_HIP_ORDER: [usize; 5] = [0, 1, 2, 3, 4]; + + fn digest_list(stack: &mut NockStack, digests: &[[u64; 5]]) -> Noun { + let items: Vec = digests.iter().map(|limbs| digest(stack, *limbs)).collect(); + list(stack, &items) + } + + fn local_page_v0(stack: &mut NockStack, block: Noun, parent: Noun) -> Noun { + T( + stack, + &[block, D(0), parent, D(0), D(0), D(0), D(0), D(0), D(0), D(0), D(0)], + ) + } + + fn local_page_v1(stack: &mut NockStack, block: Noun, parent: Noun) -> Noun { + T( + stack, + &[D(1), block, D(0), parent, D(0), D(0), D(0), D(0), D(0), D(0), D(0), D(0)], + ) + } + + fn digest_list_order_oracle(a: &[[u64; 5]], b: &[[u64; 5]], order: &[usize; 5]) -> bool { + let mut index = 0; + loop { + match (a.get(index), b.get(index)) { + (None, _) => return false, + (Some(_), None) => return true, + (Some(a_digest), Some(b_digest)) => { + if let Some(ordered) = digest_order_oracle(a_digest, b_digest, order) { + return ordered; + } + } + } + index += 1; + } + } + + fn digest_order_oracle(a: &[u64; 5], b: &[u64; 5], order: &[usize; 5]) -> Option { + for index in order { + if a[*index] > b[*index] { + return Some(true); + } + if a[*index] < b[*index] { + return Some(false); + } + } + None + } + #[derive(Clone, Debug)] struct BoundedNounInput { seed: u64, @@ -370,6 +4003,542 @@ mod tests { } } + #[derive(Clone, Debug)] + struct NoteMapDiffInput { + seed: u64, + notes: u8, + } + + impl Arbitrary for NoteMapDiffInput { + fn arbitrary(g: &mut Gen) -> Self { + Self { + seed: nonzero_seed(u64::arbitrary(g)), + notes: (u8::arbitrary(g) % 32) + 1, + } + } + } + + #[derive(Clone, Debug)] + struct BalanceHistoryInput { + seed: u64, + blocks: u8, + notes: u8, + } + + impl Arbitrary for BalanceHistoryInput { + fn arbitrary(g: &mut Gen) -> Self { + Self { + seed: nonzero_seed(u64::arbitrary(g)), + blocks: (u8::arbitrary(g) % 48) + 1, + notes: (u8::arbitrary(g) % 32) + 1, + } + } + } + + // A `hashed` key for the +gor-hip / +mor-hip comparators. + // + // Generated keys must be either a single noun-digest:tip5 (a 5-tuple of + // u63-bounded limbs) or a digest list with length 0 or at least 2. We bias the generator + // toward shared prefixes so equal-prefix list comparisons stress the + // "ties on prefix, decide on later limb" branch of the comparator. + #[derive(Clone, Debug)] + struct HipKeyInput { + single: bool, + digests: Vec<[u64; 5]>, + } + + impl Arbitrary for HipKeyInput { + fn arbitrary(g: &mut Gen) -> Self { + let single = bool::arbitrary(g); + let length = if single { + 1 + } else { + (u8::arbitrary(g) % 3 + 2) as usize + }; + + // Keep limbs small to maximise equality-on-prefix collisions while + // still spanning the comparator's interesting limb-difference cases. + let bias = u8::arbitrary(g) % 8; + let bound: u64 = if bias < 4 { + 4 + } else if bias < 7 { + 64 + } else { + u64::MAX >> 1 + }; + + let mut digests = Vec::with_capacity(length); + for _ in 0..length { + let mut limbs = [0u64; 5]; + for slot in limbs.iter_mut() { + *slot = (u64::arbitrary(g) & (u64::MAX >> 1)) % bound.max(1); + } + digests.push(limbs); + } + Self { single, digests } + } + } + + fn limbs_for_key(key: &HipKeyInput) -> Vec<[u64; 5]> { + key.digests.clone() + } + + fn noun_for_hip_key(stack: &mut NockStack, key: &HipKeyInput) -> Noun { + if key.single { + digest(stack, key.digests[0]) + } else { + digest_list(stack, &key.digests) + } + } + + fn reverse_limbs_for_key(key: &HipKeyInput) -> HipKeyInput { + let reversed = key + .digests + .iter() + .map(|limbs| { + let mut r = *limbs; + r.reverse(); + r + }) + .collect(); + HipKeyInput { + single: key.single, + digests: reversed, + } + } + + // Boundary input: includes the empty digest list (which the comparators + // treat as the minimum) and short digest lists in its neighborhood. + #[derive(Clone, Debug)] + enum HipBoundaryInput { + Empty, + Single([u64; 5]), + Pair([u64; 5], [u64; 5]), + } + + impl Arbitrary for HipBoundaryInput { + fn arbitrary(g: &mut Gen) -> Self { + let limbs_for = |g: &mut Gen| { + let bound = (u8::arbitrary(g) % 4 + 1) as u64; + let mut limbs = [0u64; 5]; + for slot in limbs.iter_mut() { + *slot = u64::arbitrary(g) % bound; + } + limbs + }; + match u8::arbitrary(g) % 3 { + 0 => HipBoundaryInput::Empty, + 1 => HipBoundaryInput::Single(limbs_for(g)), + _ => HipBoundaryInput::Pair(limbs_for(g), limbs_for(g)), + } + } + } + + fn noun_for_boundary(stack: &mut NockStack, input: &HipBoundaryInput) -> Noun { + match input { + HipBoundaryInput::Empty => D(0), + HipBoundaryInput::Single(limbs) => digest(stack, *limbs), + HipBoundaryInput::Pair(a, b) => digest_list(stack, &[*a, *b]), + } + } + + fn limbs_for_boundary(input: &HipBoundaryInput) -> Vec<[u64; 5]> { + match input { + HipBoundaryInput::Empty => Vec::new(), + HipBoundaryInput::Single(limbs) => vec![*limbs], + HipBoundaryInput::Pair(a, b) => vec![*a, *b], + } + } + + // Input drawing for adversarial soft-cast fuzz. + // + // `size` controls how many entries the seed z-* container has before + // mutation. We keep it small (1..=4) because the mutation alone is the + // adversary; what matters is breadth of mutation kinds and seed shape, + // not container size. + #[derive(Clone, Debug)] + struct SoftFuzzInput { + seed: u64, + size: u8, + mutation: u8, + } + + impl Arbitrary for SoftFuzzInput { + fn arbitrary(g: &mut Gen) -> Self { + Self { + seed: nonzero_seed(u64::arbitrary(g)), + size: (u8::arbitrary(g) % 4) + 1, + mutation: u8::arbitrary(g) % 8, + } + } + } + + fn build_seed_z_map(stack: &mut NockStack, input: &SoftFuzzInput) -> Noun { + let mut state = input.seed; + let mut entries = Vec::new(); + for index in 0..(input.size as usize) { + let limbs = [ + next_u64(&mut state) & (u64::MAX >> 1), + next_u64(&mut state) & (u64::MAX >> 1), + next_u64(&mut state) & (u64::MAX >> 1), + next_u64(&mut state) & (u64::MAX >> 1), + next_u64(&mut state) & (u64::MAX >> 1), + ]; + let key = if index % 2 == 0 { + digest(stack, limbs) + } else { + let second = [ + limbs[0].wrapping_add(1) & (u64::MAX >> 1), + limbs[1], + limbs[2], + limbs[3], + limbs[4], + ]; + digest_list(stack, &[limbs, second]) + }; + entries.push((key, D((index as u64).wrapping_add(1)))); + } + z_map_from_entries(stack, &entries) + } + + fn build_seed_z_set(stack: &mut NockStack, input: &SoftFuzzInput) -> Noun { + let mut state = input.seed; + let mut items = Vec::new(); + for index in 0..(input.size as usize) { + let limbs = [ + next_u64(&mut state) & (u64::MAX >> 1), + next_u64(&mut state) & (u64::MAX >> 1), + next_u64(&mut state) & (u64::MAX >> 1), + next_u64(&mut state) & (u64::MAX >> 1), + next_u64(&mut state) & (u64::MAX >> 1), + ]; + let item = if index % 2 == 0 { + digest(stack, limbs) + } else { + let second = [ + limbs[0].wrapping_add(1) & (u64::MAX >> 1), + limbs[1], + limbs[2], + limbs[3], + limbs[4], + ]; + digest_list(stack, &[limbs, second]) + }; + items.push(item); + } + z_set_from_items(stack, &items) + } + + fn mutate_z_tree(stack: &mut NockStack, tree: Noun, input: &SoftFuzzInput) -> Noun { + let space = stack.noun_space(); + if unsafe { tree.raw_equals(&D(0)) } { + // mutate the empty tree by replacing the leaf with a marker + return D(0x6565727a74); // %ztree + } + match input.mutation % 8 { + 0 => tree, // identity (control: must succeed) + 1 => { + // Swap left/right at the root (likely breaks h-order + // invariant; the jet must detect this). + let Ok([entry, left, right]) = tree.uncell(&space) else { + return tree; + }; + T(stack, &[entry, right, left]) + } + 2 => { + // Replace the root's left child with a %ztree marker. + let Ok([entry, _left, right]) = tree.uncell(&space) else { + return tree; + }; + T(stack, &[entry, D(0x6565727a74), right]) + } + 3 => { + // Replace the root's right child with a %hset marker. + let Ok([entry, left, _right]) = tree.uncell(&space) else { + return tree; + }; + T(stack, &[entry, left, D(0x74657368)]) + } + 4 => { + // Replace the root entry's key with a non-digest atom (forces + // the jet's small-key parser into the slow oracle path, + // which then rejects). + let Ok([entry, left, right]) = tree.uncell(&space) else { + return tree; + }; + if let Ok([_key, value]) = entry.uncell(&space) { + let bad_key = D(42); + let new_entry = T(stack, &[bad_key, value]); + T(stack, &[new_entry, left, right]) + } else { + tree + } + } + 5 => { + // Force the root entry to be a single-atom cell (improper + // pair, breaks z-map shape). + let Ok([entry, left, right]) = tree.uncell(&space) else { + return tree; + }; + T(stack, &[entry, left, right]) // structural identity + // (no-op when valid; for sets + // adds zero coverage) + } + 6 => { + // Truncate a single-digest key to 4 limbs (improper pair). + let Ok([entry, left, right]) = tree.uncell(&space) else { + return tree; + }; + if let Ok([key, value_or_l_set]) = entry.uncell(&space) { + if let Ok([a, b, c_, d, _e]) = key.uncell_chain_5(&space) { + let bad_key = T(stack, &[a, b, c_, d]); + let new_entry = T(stack, &[bad_key, value_or_l_set]); + return T(stack, &[new_entry, left, right]); + } + } + tree + } + _ => { + // Append spurious data after the right subtree (improper). + let Ok([entry, left, right]) = tree.uncell(&space) else { + return tree; + }; + let bad_tail = T(stack, &[right, D(0xdeadbeef)]); + T(stack, &[entry, left, bad_tail]) + } + } + } + + // Helper to destructure a flat 5-tuple of atoms. + trait UncellChain5 { + fn uncell_chain_5(self, space: &NounSpace) -> Result<[Noun; 5], JetErr>; + } + + impl UncellChain5 for Noun { + fn uncell_chain_5(self, space: &NounSpace) -> Result<[Noun; 5], JetErr> { + let [a, rest1] = self.uncell(space)?; + let [b, rest2] = rest1.uncell(space)?; + let [c_, rest3] = rest2.uncell(space)?; + let [d, e] = rest3.uncell(space)?; + Ok([a, b, c_, d, e]) + } + } + + fn generated_note_map_pair(stack: &mut NockStack, input: &NoteMapDiffInput) -> (Noun, Noun) { + let note_count = input.notes as usize; + let mut parent_entries = Vec::new(); + let mut child_entries = Vec::new(); + let mut state = input.seed; + + for index in 0..note_count { + let token = next_u64(&mut state); + let parent_present = !token.is_multiple_of(5); + let child_present = match token % 7 { + 0 => false, + 1..=3 => true, + _ => parent_present, + }; + let parent_variant = (token % 3) as u8; + let parent_key = generated_note_key(stack, input.seed, index, parent_variant); + let child_variant = if token.is_multiple_of(11) { + ((parent_variant + 1) % 3) as u8 + } else { + parent_variant + }; + let child_key = generated_note_key(stack, input.seed, index, child_variant); + let parent_value = generated_note_value(stack, input.seed, index, 0); + let child_value = generated_note_value(stack, input.seed ^ token, index, 1); + + if parent_present { + parent_entries.push((parent_key, parent_value)); + } + if child_present { + child_entries.push((child_key, child_value)); + } + } + + ( + z_map_from_entries(stack, &parent_entries), + z_map_from_entries(stack, &child_entries), + ) + } + + fn generated_balance_history( + stack: &mut NockStack, + input: &BalanceHistoryInput, + ) -> (Noun, Noun, Vec) { + let block_count = input.blocks as usize; + let note_count = input.notes as usize; + let mut state = input.seed; + let mut block_ids = Vec::with_capacity(block_count); + for height in 0..block_count { + block_ids.push(generated_block_id(stack, input.seed, height)); + } + + let mut block_entries = Vec::with_capacity(block_count); + let mut balance_entries = Vec::with_capacity(block_count); + let mut live_values = vec![None; note_count]; + let mut last_balance = D(0); + + for height in 0..block_count { + let block = block_ids[height]; + let parent = generated_parent_id(stack, input.seed, height, &block_ids, &mut state); + let page = if next_u64(&mut state) & 1 == 0 { + local_page_v0(stack, block, parent) + } else { + local_page_v1(stack, block, parent) + }; + block_entries.push((block, page)); + + let reuse_previous = height > 0 && next_u64(&mut state).is_multiple_of(13); + let z_balance = if reuse_previous { + last_balance + } else { + mutate_live_notes(stack, input.seed, height, &mut state, &mut live_values); + let mut note_entries = Vec::new(); + for (index, value) in live_values.iter().enumerate() { + if let Some(value) = value { + let variant = ((input.seed >> (index % 8)) as u8) + .wrapping_add(height as u8) + .wrapping_add(index as u8); + let key = generated_note_key(stack, input.seed, index, variant); + note_entries.push((key, *value)); + } + } + z_map_from_entries(stack, ¬e_entries) + }; + balance_entries.push((block, z_balance)); + last_balance = z_balance; + } + + ( + z_map_from_entries(stack, &block_entries), + z_map_from_entries(stack, &balance_entries), + block_ids, + ) + } + + fn mutate_live_notes( + stack: &mut NockStack, + seed: u64, + height: usize, + state: &mut u64, + live_values: &mut [Option], + ) { + let mutations = (next_u64(state) % 3) + 1; + for mutation in 0..mutations { + let index = (next_u64(state) as usize) % live_values.len(); + if next_u64(state).is_multiple_of(7) { + live_values[index] = None; + } else { + live_values[index] = Some(generated_note_value( + stack, + seed, + index, + height + mutation as usize, + )); + } + } + if live_values.iter().all(Option::is_none) { + live_values[0] = Some(generated_note_value(stack, seed, 0, height)); + } + } + + fn generated_parent_id( + stack: &mut NockStack, + seed: u64, + height: usize, + block_ids: &[Noun], + state: &mut u64, + ) -> Noun { + if height == 0 || next_u64(state).is_multiple_of(17) { + return digest(stack, [seed, height as u64, 0xdead, 0xbeef, 0]); + } + if height > 2 && next_u64(state).is_multiple_of(7) { + block_ids[height - 2] + } else { + block_ids[height - 1] + } + } + + fn generated_dense_note_map(stack: &mut NockStack, seed: u64, count: usize) -> Noun { + let mut entries = Vec::with_capacity(count); + for index in 0..count { + let key = generated_note_key(stack, seed, index, 0); + let value = generated_note_value(stack, seed, index, 0); + entries.push((key, value)); + } + z_map_from_entries(stack, &entries) + } + + fn generated_block_id(stack: &mut NockStack, seed: u64, height: usize) -> Noun { + digest( + stack, + [ + seed.wrapping_add(0x1000), + height as u64, + seed.rotate_left(11), + height as u64 ^ 0x5555, + seed.wrapping_mul(31).wrapping_add(height as u64), + ], + ) + } + + fn generated_note_key(stack: &mut NockStack, seed: u64, index: usize, variant: u8) -> Noun { + let first = [ + seed.wrapping_add(index as u64).wrapping_add(1), + seed.rotate_left(7).wrapping_add(index as u64 * 3), + index as u64 ^ 0xa5a5, + seed.rotate_right(9), + seed.wrapping_mul(17).wrapping_add(index as u64), + ]; + let second = [ + seed.wrapping_add(index as u64).wrapping_add(0x100), + seed.rotate_left(13), + index as u64 ^ 0x5a5a, + seed.rotate_right(3).wrapping_add(index as u64), + seed.wrapping_mul(19), + ]; + let third = [ + seed.wrapping_add(index as u64).wrapping_add(0x200), + seed.rotate_left(17), + index as u64 ^ 0x3333, + seed.rotate_right(5), + seed.wrapping_mul(23).wrapping_add(index as u64), + ]; + + match variant % 4 { + 0 => digest(stack, first), + 1 => digest_list(stack, &[first, third]), + 2 => digest_list(stack, &[first, second]), + _ => digest_list(stack, &[first, second, third]), + } + } + + fn generated_note_value( + stack: &mut NockStack, + seed: u64, + index: usize, + version: usize, + ) -> Noun { + T( + stack, + &[ + D(direct_limb(seed.wrapping_add(version as u64))), + D(index as u64), + D((version as u64) << 32 | index as u64), + ], + ) + } + + fn nonzero_seed(seed: u64) -> u64 { + if seed == 0 { + 1 + } else { + seed + } + } + fn bounded_noun_from_input(stack: &mut NockStack, input: &BoundedNounInput) -> Noun { let mut state = if input.seed == 0 { 1 } else { input.seed }; bounded_noun_from_state(stack, &mut state, input.depth) @@ -396,4 +4565,8 @@ mod tests { *state = x; x } + + fn direct_limb(value: u64) -> u64 { + value & (u64::MAX >> 1) + } } diff --git a/crates/zkvm-jetpack/src/utils.rs b/crates/zkvm-jetpack/src/utils.rs index 8b813c27c..18fd111d6 100644 --- a/crates/zkvm-jetpack/src/utils.rs +++ b/crates/zkvm-jetpack/src/utils.rs @@ -1,24 +1,39 @@ use bitvec::prelude::{BitSlice, Lsb0}; use ibig::UBig; use nockvm::interpreter::Context; +use nockvm::jets::cold::Nounable; use nockvm::jets::JetErr; use nockvm::mem::NockStack; -use nockvm::noun::{Atom, IndirectAtom, Noun, D, DIRECT_MAX, NONE, T}; +#[cfg(test)] +use nockvm::mem::NOCK_STACK_SIZE_TINY; +use nockvm::noun::{Atom, IndirectAtom, Noun, NounSpace, D, DIRECT_MAX, NONE, T}; pub use tracing::{debug, trace}; use crate::form::belt::*; // tests whether a felt atom has the leading 1. we cannot actually test // Felt, because it doesn't include the leading 1. -pub fn felt_atom_is_valid(felt_atom: IndirectAtom) -> bool { - let dat_ptr = felt_atom.data_pointer(); +pub fn felt_atom_is_valid(felt_atom: IndirectAtom, space: &NounSpace) -> bool { + let dat_ptr = felt_atom.as_atom().in_space(space).data_pointer(); unsafe { *(dat_ptr.add(3)) == 0x1 } } -pub fn vecnoun_to_hoon_list(stack: &mut NockStack, vec: &[Noun]) -> Noun { +fn copy_noun_into_stack(stack: &mut NockStack, noun: Noun, space: &NounSpace) -> Noun { + ::from_noun(stack, &noun, space).unwrap_or_else(|err| { + panic!( + "Panicked with {err:?} at {}:{} (git sha: {:?})", + file!(), + line!(), + option_env!("GIT_SHA") + ) + }) +} + +pub fn vecnoun_to_hoon_list(stack: &mut NockStack, vec: &[Noun], space: &NounSpace) -> Noun { let mut list = D(0); for n in vec.iter().rev() { - list = T(stack, &[*n, list]); + let copied = copy_noun_into_stack(stack, *n, space); + list = T(stack, &[copied, list]); } list } @@ -46,14 +61,15 @@ pub fn vec_to_hoon_tuple(stack: &mut NockStack, vec: &[u64]) -> Noun { list } -pub fn vecnoun_to_hoon_tuple(stack: &mut NockStack, vec: &[Noun]) -> Noun { +pub fn vecnoun_to_hoon_tuple(stack: &mut NockStack, vec: &[Noun], space: &NounSpace) -> Noun { assert!(vec.len() >= 2); let mut list = NONE; for n in vec.iter().rev() { + let copied = copy_noun_into_stack(stack, *n, space); list = if list.is_none() { - *n + copied } else { - T(stack, &[*n, list]) + T(stack, &[copied, list]) } } list @@ -99,27 +115,27 @@ pub fn u128_as_noun(stack: &mut NockStack, res: u128) -> Noun { } } -pub fn hoon_list_to_vecbelt(list: Noun) -> Result, JetErr> { +pub fn hoon_list_to_vecbelt(list: Noun, space: &NounSpace) -> Result, JetErr> { let mut input_iterate = list; let mut input_vec: Vec = Vec::new(); while !is_hoon_list_end(&input_iterate) { - let input_cell = input_iterate.as_cell()?; + let input_cell = input_iterate.in_space(space).as_cell()?; let head_belt = Belt(input_cell.head().as_atom()?.as_u64()?); input_vec.push(head_belt); - input_iterate = input_cell.tail(); + input_iterate = input_cell.tail().noun(); } Ok(input_vec) } -pub fn hoon_list_to_vecnoun(list: Noun) -> Result, JetErr> { +pub fn hoon_list_to_vecnoun(list: Noun, space: &NounSpace) -> Result, JetErr> { let mut input_iterate = list; let mut input_vec: Vec = Vec::new(); while !is_hoon_list_end(&input_iterate) { - let input_cell = input_iterate.as_cell()?; - let head_belt = input_cell.head(); - input_vec.push(head_belt); - input_iterate = input_cell.tail(); + let input_cell = input_iterate.in_space(space).as_cell()?; + let head = input_cell.head().noun(); + input_vec.push(head); + input_iterate = input_cell.tail().noun(); } Ok(input_vec) @@ -139,3 +155,34 @@ pub fn make_cell_hash(context: &mut Context, hash: &[u64]) -> Noun { } res_cell } + +#[cfg(test)] +mod tests { + use nockvm::ext::noun_equality; + + use super::*; + + #[test] + fn vecnoun_to_hoon_list_copies_foreign_nouns_into_destination_stack() { + let mut source = NockStack::new(NOCK_STACK_SIZE_TINY, 0); + let left = T(&mut source, &[D(1), D(2)]); + let right = T(&mut source, &[D(3), D(4)]); + let source_tail = T(&mut source, &[right, D(0)]); + let source_list = T(&mut source, &[left, source_tail]); + let source_space = source.noun_space(); + let source_items = hoon_list_to_vecnoun(source_list, &source_space).expect("decode list"); + + let mut dest = NockStack::new(NOCK_STACK_SIZE_TINY, 0); + let copied_list = vecnoun_to_hoon_list(&mut dest, &source_items, &source_space); + let dest_space = dest.noun_space(); + let expected_left = T(&mut dest, &[D(1), D(2)]); + let expected_right = T(&mut dest, &[D(3), D(4)]); + let expected_tail = T(&mut dest, &[expected_right, D(0)]); + let expected_list = T(&mut dest, &[expected_left, expected_tail]); + + assert!(noun_equality( + copied_list.in_space(&dest_space), + expected_list.in_space(&dest_space), + )); + } +} diff --git a/docker-compose.metrics.yml b/docker-compose.metrics.yml new file mode 100644 index 000000000..b6961f069 --- /dev/null +++ b/docker-compose.metrics.yml @@ -0,0 +1,41 @@ +services: + influxdb: + image: influxdb:${INFLUXDB_VERSION:-2.7} + container_name: nockchain-influxdb + ports: + - "8086:8086" + volumes: + - influxdb-data:/var/lib/influxdb2 + environment: + DOCKER_INFLUXDB_INIT_MODE: setup + DOCKER_INFLUXDB_INIT_USERNAME: ${INFLUXDB_USERNAME:-admin} + DOCKER_INFLUXDB_INIT_PASSWORD: ${INFLUXDB_PASSWORD:-adminpassword} + DOCKER_INFLUXDB_INIT_ORG: ${INFLUXDB_ORG:-nockchain} + DOCKER_INFLUXDB_INIT_BUCKET: ${INFLUXDB_BUCKET:-metrics} + DOCKER_INFLUXDB_INIT_ADMIN_TOKEN: ${INFLUXDB_ADMIN_TOKEN:-nockchain-local-token} + networks: + - metrics + + telegraf: + image: telegraf:${TELEGRAF_VERSION:-1.30} + container_name: nockchain-telegraf + ports: + - "8125:8125/udp" + volumes: + - ./telegraf.conf:/etc/telegraf/telegraf.conf:ro + environment: + INFLUX_TOKEN: ${INFLUXDB_ADMIN_TOKEN:-nockchain-local-token} + INFLUX_ORG: ${INFLUXDB_ORG:-nockchain} + INFLUX_BUCKET: ${INFLUXDB_BUCKET:-metrics} + depends_on: + - influxdb + networks: + - metrics + +volumes: + influxdb-data: + +networks: + metrics: + name: ${DOCKER_METRICS_NETWORK:-nockchain-metrics} + external: true diff --git a/docs/architecture/tx-engine/00-overview.md b/docs/architecture/tx-engine/00-overview.md new file mode 100644 index 000000000..d372e32b4 --- /dev/null +++ b/docs/architecture/tx-engine/00-overview.md @@ -0,0 +1,115 @@ +# Nockchain Transaction Engine: Architecture Overview + +## Purpose + +The Nockchain transaction engine (tx-engine) is the consensus-critical subsystem that defines how value is created, transferred, and validated on the Nockchain network. It implements a UTXO-based model with distinct design inspirations drawn from three pillars of blockchain innovation: + +- **Bitcoin Segregated Witness (SegWit)** — witness data separation from transaction structure +- **Bitcoin Taproot (BIP 341/342)** — Merkelized script trees with selective branch revelation +- **Extended UTXO (eUTXO)** — arbitrary data attached to UTXOs, as pioneered by Cardano + +The engine is distinctive in that it spans two languages and runtimes: **Hoon** (running on the Nock VM) is the **source of truth** for all type definitions and consensus validation logic, while **Rust** provides serialization/deserialization of Hoon nouns for networking, APIs, and wallet applications. + +## Design Lineage + +| Nockchain Concept | Bitcoin Equivalent | Cardano eUTXO Equivalent | +|---|---|---| +| Note | UTXO (TxOut) | UTxO | +| Name (first, last) | Outpoint (txid:vout) | TxOutRef | +| Lock (v0: M-of-N keys) | P2SH / P2PKH | — | +| Lock tree (v1) | Taproot script tree (MAST) | — | +| LockMerkleProof | Taproot control block + script | — | +| Spend0 (legacy bridge) | SegWit P2SH-wrapped spend | — | +| Spend1 (witness-based) | SegWit native spend | — | +| Witness struct | SegWit witness field | Redeemer | +| NoteData | — | Datum | +| SpendCondition (AND list) | Script execution | Validator | +| Lock primitives (Pkh/Tim/Hax/Burn) | OP_CHECKSIG / OP_CLTV / OP_HASH / OP_RETURN | — | +| Seed | TxOut (output) | TxOut | +| Nicks / Nocks | Satoshis / BTC | Lovelace / ADA | +| Tip5 hash | SHA-256d | Blake2b | +| z-map / z-set | — | — (Nock-native) | + +## Codebase Map + +### Hoon: Source of Truth (Type Definitions + Validation) + +``` +hoon/common/ +├── tx-engine.hoon # Versioned facade: dispatches to tx-engine-0 or tx-engine-1 +├── tx-engine-0.hoon # V0 engine: type definitions, validation, balance updates (~2471 lines) +└── tx-engine-1.hoon # V1 engine: type definitions, witness checks, lock merkle proofs, + # fee calculation, context-aware validation (~1969 lines) +``` + +### Rust: Serialization/Deserialization Mirror (for Networking & Applications) + +``` +crates/nockchain-types/src/tx_engine/ +├── common/ +│ ├── mod.rs # Hash, Name, Nicks, Version, Source, SchnorrPubkey/Signature, timelocks +│ └── page.rs # Page (block), BlockId, CoinbaseSplit, PageMsg +├── v0/ +│ ├── tx.rs # V0 RawTx, Input, Spend, Seeds, Seed +│ └── note.rs # NoteV0, Lock (M-of-N), Balance, Timelock +├── v1/ +│ ├── tx.rs # V1 RawTx, Spend enum (Legacy/Witness), Spend0, Spend1, Witness, +│ │ # LockMerkleProof, SpendCondition, LockPrimitive, Pkh, Tim, Hax, Seeds, Seed +│ └── note.rs # NoteV1, NoteData, NoteDataEntry, Balance, Note enum (V0/V1) +└── mod.rs # Module root +``` + +### Protocol Specifications + +``` +changelog/protocol/ +├── 009-legacy-segwit-cutover-initial.md # V0→V1 split (block 37350) +├── 010-legacy-v1-phase-39000.md # V1 phase finalization +├── 011-legacy-lmp-axis-hotfix.md # Lock merkle proof axis fix +├── 012-bythos.md # LMP versioning + fee rebalancing (block 54000) +└── 013-nous.md # Networking upgrade (non-tx-engine) +``` + +## Version History + +``` +Genesis ──── Block 0 + │ V0 engine only: M-of-N multisig, sig-keyed coinbase + │ +Protocol 009 ── Block 37350 (v1-phase) + │ Dual engine: V0 rejected, V1 required + │ Witness separation (Spend0 bridge, Spend1 native) + │ Lock trees with Merkle proofs + │ Note data (eUTXO datum) + │ Fee floor with word-count pricing + │ +Protocol 010 ── Block 39000 + │ V1-phase boundary finalized + │ +Protocol 012 (Bythos) ── Block 54000 + │ Lock merkle proof versioning (stub → full) + │ Fee rebalancing: witness discount (1/4 rate) + │ Context-aware mempool admission + │ +Present +``` + +## Analysis Index + +| Document | Topic | +|---|---| +| [01 — UTXO Model](01-utxo-model.md) | Notes, Names, Locks, balances, and asset denomination | +| [02 — SegWit Witness Separation](02-segwit-witness-separation.md) | V1 witness separation, Spend0/Spend1, bridge spends | +| [03 — Taproot Lock Merkle Proofs](03-taproot-lock-merkle-proofs.md) | MAST-like lock trees, selective branch revelation | +| [04 — eUTXO Note Data](04-eutxo-note-data.md) | On-chain datum via NoteData | +| [05 — Lock Primitives](05-lock-primitives-script-model.md) | Pkh, Tim, Hax, Burn and composition model | +| [06 — Fee Structure](06-fee-structure.md) | SegWit-inspired witness weight discount | +| [07 — Validation Pipeline](07-transaction-validation-pipeline.md) | End-to-end transaction validation | +| [08 — Protocol Evolution](08-protocol-evolution.md) | Upgrade mechanics and history | +| [09 — Noun Encoding](09-noun-encoding-data-layer.md) | Rust↔Hoon bridge via noun serialization | +| [10 — Zoon Persistent Data Structures](10-zoon-persistent-data-structures.md) | Hash-ordered z-maps and z-sets (cryptographic treaps) | +| [11 — Tip5 Hash Function](11-tip5-hash-function.md) | Algebraic sponge hash over Goldilocks field | +| [12 — Schnorr Signatures & Cheetah Curve](12-schnorr-signatures-cheetah-curve.md) | STARK-friendly signatures over Fp^6 extension | +| [13 — Merkle Trees & Commitments](13-merkle-trees-and-commitments.md) | Merkle proof construction and hashable commitment trees | +| [14 — Goldilocks Field Arithmetic](14-goldilocks-field-arithmetic.md) | Base and extension field types (ztd one and two) | +| [15 — ZTD STARK Proof Stack](15-ztd-stark-proof-stack.md) | Full STARK prover hierarchy (ztd three through eight) | diff --git a/docs/architecture/tx-engine/01-utxo-model.md b/docs/architecture/tx-engine/01-utxo-model.md new file mode 100644 index 000000000..44baba790 --- /dev/null +++ b/docs/architecture/tx-engine/01-utxo-model.md @@ -0,0 +1,224 @@ +# The UTXO Model: Notes, Names, and Balances + +## Overview + +Nockchain uses an unspent transaction output (UTXO) model, where all on-chain value exists as discrete, immutable "notes" that are created by transactions and consumed (spent) exactly once. This is the same fundamental model used by Bitcoin, in contrast to Ethereum's account-based model. + +## Notes = UTXOs + +A **note** is Nockchain's UTXO. It represents a discrete unit of value locked under specific spending conditions. Notes exist in two versions, reflecting the protocol's evolution. + +All note types are defined authoritatively in Hoon (`hoon/common/tx-engine-0.hoon` and `tx-engine-1.hoon`). The Rust structs in `crates/nockchain-types/` are serialization mirrors that encode/decode Hoon nouns for networking and applications. + +### NoteV0 (Genesis) + +The Hoon type definition (authoritative): + +```hoon +:: from hoon/common/tx-engine-0.hoon (++nnote, line 1621) ++$ form + $: $: version=%0 + origin-page=page-number + =timelock + == + name=nname + =sig :: M-of-N multisig (Hoon type is ++sig, Rust mirrors as Lock) + =source + assets=coins + == +``` + +Rust serialization mirror: + +```rust +// crates/nockchain-types/src/tx_engine/v0/note.rs:99-102 +pub struct NoteV0 { + pub head: NoteHead, // version, origin_page, timelock + pub tail: NoteTail, // name, lock, source, assets +} +``` + +A V0 note carries: +- **version**: always `V0` (encoded as `0`) +- **origin_page**: the block height at which this note was added to the balance +- **timelock**: optional absolute/relative constraints on when the note can be spent +- **name**: the unique identifier (see below) +- **sig**: M-of-N multisig condition (Hoon `++sig` type: `m` + z-set of Schnorr public keys; Rust mirrors as `Lock`) +- **source**: provenance tracking (hash + coinbase flag) +- **assets**: value in nicks + +### NoteV1 (Post-SegWit) + +Hoon type definition (authoritative, from `tx-engine-1.hoon`): + +```hoon ++$ nnote-1 + $: version=%1 + origin-page=page-number + name=nname + note-data=(z-map @tas *) + assets=coins + == +``` + +Rust serialization mirror: + +```rust +// crates/nockchain-types/src/tx_engine/v1/note.rs:58-65 +pub struct NoteV1 { + pub version: Version, // always V1 + pub origin_page: BlockHeight, + pub name: Name, + pub note_data: NoteData, // NEW: arbitrary key-value data (eUTXO datum) + pub assets: Nicks, +} +``` + +V1 notes replace the explicit `Lock` and `Source` fields with: +- **note_data**: a key-value map of arbitrary data (the eUTXO-inspired datum) +- The lock information is now embedded in the `Name` itself (via the lock root hash) + +The lock is no longer stored in the note directly — instead, the note's `Name` commits to the lock root, and the spender must provide a `LockMerkleProof` in the witness to demonstrate they know a valid spending path. + +## Name: UTXO Identifier + +A **Name** (`nname` in Hoon) uniquely identifies a note within the UTXO set. Defined in Hoon as a pair of hashes plus a null terminator: + +```hoon ++$ nname [first=hash last=hash ~] +``` + +Rust serialization mirror: + +```rust +// crates/nockchain-types/src/tx_engine/common/mod.rs:266-270 +pub struct Name { + pub first: Hash, // lock root hash + pub last: Hash, // source-derived hash (parent hash of transaction) + null: usize, // always 0 (Hoon noun alignment) +} +``` + +### Comparison with Bitcoin's Outpoint + +| Property | Bitcoin Outpoint | Nockchain Name | +|---|---|---| +| Structure | `(txid, vout_index)` | `(lock_root_hash, source_last_hash)` | +| References | Transaction + output index | Lock commitment + source commitment | +| Size | 36 bytes (32 + 4) | 80 bytes (2 × Tip5 hash) | +| Commits to lock? | No (lock is in the output) | Yes (first = lock root hash) | + +The key difference is that a Nockchain Name **commits to the spending conditions** in its identity. The `first` field is the hash of the lock tree root, meaning the UTXO's identity is intrinsically bound to how it can be spent. This is closer to Bitcoin's P2SH/P2WSH model (where the address commits to the script hash) than to bare P2PK. + +The `last` field derives from the source transaction's hash, ensuring uniqueness even when two outputs share the same lock. + +## Lock: Spending Conditions + +### V0 Lock (Simple M-of-N) + +Authoritative Hoon type (`tx-engine-0.hoon:1389`, named `++sig`): + +```hoon ++$ form + $~ [m=1 pubkeys=*(z-set schnorr-pubkey)] + [m=@udD pubkeys=(z-set schnorr-pubkey)] +``` + +Rust serialization mirror: + +```rust +// crates/nockchain-types/src/tx_engine/v0/note.rs:39 +pub struct Lock { + pub keys_required: u64, // M (Hoon: m=@udD, max 255) + pub pubkeys: Vec, // N public keys (Hoon: z-set) +} +``` + +This is analogous to Bitcoin's `OP_CHECKMULTISIG` — a simple threshold scheme requiring M of N Schnorr signatures. The Hoon type uses a `z-set` (Tip5-ordered persistent set) for pubkeys, ensuring deterministic ordering; the Rust mirror serializes this as a `Vec`. + +### V1 Lock (Lock Tree) + +In V1, the lock concept was generalized to a **tree of spend conditions** (see [03-taproot-lock-merkle-proofs.md](03-taproot-lock-merkle-proofs.md) and [05-lock-primitives-script-model.md](05-lock-primitives-script-model.md)). The lock is no longer stored on the note; instead, the note's Name commits to the lock root hash, and spenders provide a Merkle proof to a specific branch. + +## Asset Denomination + +```rust +// crates/nockchain-types/src/tx_engine/common/mod.rs:82 +pub struct Nicks(pub usize); +``` + +Nockchain uses a two-tier denomination: +- **Nicks**: the base unit (analogous to satoshis) +- **Nocks**: the display unit (analogous to bitcoin), where `1 nock = 65536 nicks` (2^16) + +The choice of 2^16 as the subdivision factor (vs Bitcoin's 10^8) reflects Nockchain's preference for power-of-two arithmetic, consistent with the Nock VM's binary tree orientation. + +## Balance: The UTXO Set + +The full UTXO set is called the **balance** — a persistent sorted map (`z-map`) from Names to Notes: + +```rust +// crates/nockchain-types/src/tx_engine/v1/note.rs:23 +pub struct Balance(pub Vec<(Name, Note)>); +``` + +Where `Note` is a version-discriminated enum: + +```rust +// crates/nockchain-types/src/tx_engine/v1/note.rs:53-56 +pub enum Note { + V0(NoteV0), + V1(NoteV1), +} +``` + +Discrimination is structural: V0 notes have a cell (pair) as their head element; V1 notes have an atom (the version number). This allows the balance to contain both V0 and V1 notes simultaneously during the transition period. + +## Source: Provenance Tracking + +```rust +// crates/nockchain-types/src/tx_engine/common/mod.rs:134-137 +pub struct Source { + pub hash: Hash, + pub is_coinbase: bool, +} +``` + +Every note tracks its origin: whether it was created by a coinbase (mining reward) or a regular transaction, plus a hash linking to the originating transaction. This is used in the Name computation and for validation rules that may treat coinbase outputs differently (e.g., maturation requirements). + +## Transaction Structure + +### V0 Transaction + +```rust +// crates/nockchain-types/src/tx_engine/v0/tx.rs:95-100 +pub struct RawTx { + pub id: TxId, + pub inputs: Inputs, // z-map of (Name → Input) + pub timelock_range: TimelockRangeAbsolute, + pub total_fees: Nicks, +} +``` + +V0 transactions embed the full note in each input (`Input { note, spend }`), and the spend carries an optional signature directly. + +### V1 Transaction + +```rust +// crates/nockchain-types/src/tx_engine/v1/tx.rs:19-23 +pub struct RawTx { + pub version: Version, + pub id: TxId, + pub spends: Spends, // z-map of (Name → Spend) +} +``` + +V1 transactions are leaner: they reference notes by Name rather than embedding them, and carry a version tag. The `Spends` map associates each consumed note (by Name) with its spending proof. See [02-segwit-witness-separation.md](02-segwit-witness-separation.md) for the Spend structure. + +## Key Design Observations + +1. **Identity-committed locks**: Unlike Bitcoin where an outpoint is `(txid, index)` and the lock script lives in the output, Nockchain bakes the lock hash into the Name itself. This means you cannot construct a valid Name without knowing the lock, providing an inherent binding between identity and spending authority. + +2. **Structural version discrimination**: Rather than using explicit version tags for note discrimination, the system uses structural patterns (cell head vs atom head). This is idiomatic Hoon — types are discriminated by shape, not by tags. + +3. **Persistent balanced trees**: The balance (UTXO set) is stored as a z-map (a persistent balanced binary tree with Tip5-based ordering), which allows efficient functional updates — inserting and removing notes without mutating the existing tree. diff --git a/docs/architecture/tx-engine/02-segwit-witness-separation.md b/docs/architecture/tx-engine/02-segwit-witness-separation.md new file mode 100644 index 000000000..159fcb632 --- /dev/null +++ b/docs/architecture/tx-engine/02-segwit-witness-separation.md @@ -0,0 +1,217 @@ +# SegWit-Inspired Witness Separation + +## Bitcoin SegWit Recap + +Bitcoin's Segregated Witness (BIP 141, activated August 2017) introduced a fundamental structural change: **signature (witness) data was moved out of the transaction body** into a separate witness structure. This achieved three goals: + +1. **Transaction malleability fix**: Transaction IDs no longer depend on signatures, preventing third parties from modifying txids by tweaking signatures. +2. **Fee discount**: Witness data is counted at 1/4 weight, incentivizing efficient use of block space. +3. **Script versioning**: A new `witness_version` byte enabled future soft-fork upgrades (which led to Taproot). + +## Nockchain's Witness Separation + +Nockchain's V1 transaction engine (Protocol 009, activated at block 37350) implements an analogous separation. The design is documented as the "segwit cutover" in `changelog/protocol/009-legacy-segwit-cutover-initial.md`. + +### V0 Model: Embedded Signatures + +In V0, spending proofs were embedded directly in the input. The authoritative Hoon type definition: + +```hoon +:: from hoon/common/tx-engine-0.hoon +++ input [note=nnote =spend] +++ spend $: signature=(unit signature) + =seeds + fee=coins + == +``` + +Rust serialization mirror: + +```rust +// crates/nockchain-types/src/tx_engine/v0/tx.rs:51-54 +pub struct Input { + pub note: NoteV0, // The full UTXO being spent (embedded!) + pub spend: Spend, // Contains signature directly +} +``` + +Problems with this model: +- The full note is embedded in each input, duplicating data already in the UTXO set +- Signatures are part of the core transaction structure, complicating ID computation +- No extensibility path for new authentication mechanisms + +### V1 Model: Separated Witness + +V1 introduces a clean separation. The authoritative Hoon type definitions: + +```hoon +:: from hoon/common/tx-engine-1.hoon +++ spend + $% [%0 =signature =seeds fee=coins] + [%1 =witness =seeds fee=coins] + == +``` + +Rust serialization mirror: + +```rust +// crates/nockchain-types/src/tx_engine/v1/tx.rs:90-94 +pub enum Spend { + Legacy(Spend0), // Tag %0: bridge spend (v0 note → v1 output) + Witness(Spend1), // Tag %1: native v1 spend with separate witness +} +``` + +#### Spend0: Legacy Bridge + +`Spend0` (tag `%0`) exists solely to allow v0 notes (created before the cutover) to be spent under the v1 engine. It carries a direct signature, just like v0, but produces v1 outputs. It is a **bridge mechanism** — once all v0 notes are spent, Spend0 becomes unnecessary. + +This is analogous to Bitcoin's P2SH-wrapped SegWit outputs, which allowed SegWit transactions to be sent from non-SegWit-aware wallets during the transition. + +#### Spend1: Native Witness + +`Spend1` (tag `%1`) is the native V1 spend with a separated witness. The Hoon type: + +```hoon +:: The witness type (authoritative definition from tx-engine-1.hoon) +++ witness + $: =lock-merkle-proof :: which branch of the lock tree + =pkh-signature :: Schnorr signature proofs + hax=(z-map hash *) :: hash preimage reveals + tim=@ :: reserved (always 0) + == +``` + +Rust serialization mirror: + +```rust +// crates/nockchain-types/src/tx_engine/v1/tx.rs:167-174 +pub struct Witness { + pub lock_merkle_proof: LockMerkleProof, + pub pkh_signature: PkhSignature, + pub hax: Vec, + pub tim: usize, +} +``` + +Each witness field corresponds to a lock primitive type: +- `lock_merkle_proof` → proves which spend condition branch is being exercised +- `pkh_signature` → satisfies `Pkh` (public-key-hash) lock primitives +- `hax` → satisfies `Hax` (hash preimage) lock primitives +- `tim` → reserved for `Tim` (timelock) witness data (currently unused; timelocks are checked against block context) + +## Structural Comparison + +### V0 Transaction Layout + +``` +RawTx +├── id: TxId +├── inputs: z-map +│ └── (Name → Input) +│ ├── note: NoteV0 ← full UTXO embedded +│ └── spend +│ ├── signature ← auth proof embedded in spend +│ ├── seeds ← outputs +│ └── fee +├── timelock_range +└── total_fees +``` + +### V1 Transaction Layout + +``` +RawTx +├── version: 1 +├── id: TxId +└── spends: z-map + └── (Name → Spend) + └── Spend::Witness(Spend1) + ├── witness ← SEPARATED authentication proof + │ ├── lock_merkle_proof + │ ├── pkh_signature + │ ├── hax + │ └── tim + ├── seeds ← outputs + └── fee +``` + +Key structural differences: +1. **No embedded note**: V1 references notes by Name only; the validator looks them up in the balance +2. **Witness separated**: Authentication data has its own structured container +3. **Extensible witness**: The Witness struct has dedicated fields per lock primitive type, allowing independent evolution +4. **Version tagged**: The RawTx carries an explicit version number + +## Height-Gated Activation + +The cutover is enforced by a strict height-based rule matrix: + +| Block Height | V0 Transactions | V1 Transactions | Coinbase Format | +|---|---|---|---| +| `< v1-phase` (37350) | Allowed | Rejected | V0 (sig-keyed) | +| `≥ v1-phase` (37350) | Rejected | Required | V1 (lock-hash-rooted) | + +From `changelog/protocol/009-legacy-segwit-cutover-initial.md`: +> At `height >= v1-phase`: v1 raw transactions are required. v0 raw transactions are rejected (`%v0-tx-after-cutoff`). + +This is a **hard cutover**, not a gradual transition. All nodes must agree on the same rules at the same height. + +## Bridge Spending: V0 → V1 Transition + +When a V0 note needs to be spent after the cutover, it uses `Spend0`: + +1. The spender references the V0 note by its Name +2. Provides a V0-style signature (directly, not via witness) +3. The outputs (seeds) are V1-format, producing V1 notes + +The Hoon validation logic checks spend-version compatibility: + +```hoon +:: From hoon/common/tx-engine-1.hoon (validate-with-context) +:: v0 note must back a %0 spend +?: ?=(@ -.note) [%.n %v1-spend-version-mismatch] +``` + +- V0 notes (head is a cell) → must use Spend0 +- V1 notes (head is an atom) → must use Spend1 + +This ensures that the more expressive V1 witness/lock system is only used with V1 notes that were created with lock tree commitments. + +## Transaction ID Computation + +In V1, the transaction ID (`TxId`) is computed as a hash of the full spend data including witnesses. This differs from Bitcoin SegWit, where the `txid` excludes witness data (and a separate `wtxid` includes it). + +However, the **sig-hash** (the message that gets signed) excludes the witness: + +```rust +// The sig_hash is computed over (seeds, fee) — not the witness +// This is analogous to Bitcoin's sighash, which signs the transaction +// structure without the signatures themselves +``` + +This means: +- The transaction ID is stable once all witnesses are attached +- The signing message doesn't include signatures (avoiding circular dependency) +- Unlike Bitcoin SegWit, there is no separate "witness txid" — just one ID + +## Fee Implications + +The witness separation enabled the Bythos upgrade (Protocol 012) to introduce **differential fee pricing** for witness vs non-witness data, directly analogous to Bitcoin SegWit's weight system: + +- **Seed words** (outputs): charged at full `base_fee` per word +- **Witness words** (inputs): charged at `base_fee / input_fee_divisor` (1/4 rate) + +See [06-fee-structure.md](06-fee-structure.md) for the full fee analysis. + +## Comparison: Bitcoin SegWit vs Nockchain V1 + +| Aspect | Bitcoin SegWit | Nockchain V1 | +|---|---|---| +| Activation | BIP 9 signaling (soft fork) | Height-gated hard cutover | +| Witness location | Separate witness field in serialization | `Witness` struct in `Spend1` | +| Transaction ID | `txid` excludes witness; `wtxid` includes | Single ID includes all data | +| Signature exclusion | Witness stripped for txid | Witness excluded from sig-hash | +| Fee discount | 4:1 witness weight ratio | 4:1 input fee divisor (Bythos) | +| Legacy compatibility | P2SH-wrapped SegWit | Spend0 bridge spends | +| Script versioning | witness_version byte | Spend enum tag (0 or 1) | +| Extensibility | Led to Taproot (v1) | Led to lock trees + note data | diff --git a/docs/architecture/tx-engine/03-taproot-lock-merkle-proofs.md b/docs/architecture/tx-engine/03-taproot-lock-merkle-proofs.md new file mode 100644 index 000000000..e1e2a9633 --- /dev/null +++ b/docs/architecture/tx-engine/03-taproot-lock-merkle-proofs.md @@ -0,0 +1,187 @@ +# Taproot-Inspired Lock Merkle Proofs (MAST) + +## Bitcoin Taproot Recap + +Bitcoin Taproot (BIP 341/342, activated November 2021) introduced **Merkelized Abstract Syntax Trees (MAST)**: a way to commit to multiple spending scripts in a Merkle tree, where only the executed script branch is revealed on-chain. + +Key Taproot concepts: +- **Key-path spending**: If all parties agree, the output can be spent with a single Schnorr signature (no script revealed) +- **Script-path spending**: Reveal one script from the MAST tree + Merkle proof to root +- **Privacy**: Unexecuted scripts remain hidden behind Merkle hashes +- **Efficiency**: Only the relevant branch is included in the transaction + +## Nockchain's Lock Tree + +Nockchain implements an analogous pattern through its **lock tree** and **lock Merkle proof** system. A lock in V1 is a binary tree of spend conditions, where each leaf is a `SpendCondition` (a list of lock primitives that must all be satisfied — AND logic). Different branches of the tree represent alternative ways to spend (OR logic). + +### The Lock Tree Structure + +``` + Lock Root + / \ + Branch A Branch B + / \ | + Leaf 1 Leaf 2 Leaf 3 + [pkh] [pkh, [tim, + tim] hax] +``` + +Each leaf is a `SpendCondition`: a list of `LockPrimitive` values that are ANDed together. The tree itself provides OR semantics — any one leaf (branch) can authorize spending. + +This maps directly to Taproot's MAST: + +| Taproot | Nockchain | +|---|---| +| Script tree | Lock tree | +| Script leaf | SpendCondition (list of LockPrimitive) | +| Control block | LockMerkleProof | +| Script-path spend | Spend1 with LockMerkleProof | +| Key-path spend | No direct equivalent (always reveals a branch) | + +Notable difference: Nockchain does not have a key-path equivalent. Every spend must reveal at least one branch of the lock tree. This is a deliberate simplification — there is no "cooperative spend" shortcut that bypasses the Merkle proof. + +## LockMerkleProof: The Control Block + +When spending a V1 note, the spender provides a `lock-merkle-proof` that reveals exactly one branch of the lock tree. The authoritative Hoon type (from `tx-engine-1.hoon`): + +```hoon ++$ lock-merkle-proof + $^ :: stub (3-tuple): + [=spend-condition axis=@ =merk-proof:merkle] + :: full (4-tuple with %full tag): + [version=%full =spend-condition axis=@ =merk-proof:merkle] +``` + +Rust serialization mirror: + +```rust +// crates/nockchain-types/src/tx_engine/v1/tx.rs:339-344 +pub enum LockMerkleProof { + Full(LockMerkleProofFull), // Post-Bythos: axis committed in hash + Stub(LockMerkleProofStub), // Pre-Bythos: axis not committed +} +``` + +### Stub Format (Pre-Bythos) + +The stub is a 3-tuple `[spend-condition axis merk-proof]`. The axis is not committed in the hash. + +### Full Format (Post-Bythos) + +The full format is a 4-tuple `[%full spend-condition axis merk-proof]`. The `%full` tag distinguishes it from stub proofs via structural discrimination (a standard Hoon `$^` pattern). + +The difference: Full proofs include the `axis` in the hashable commitment. In Stub proofs, the axis was replaced by a hardcoded placeholder hash, meaning the witness hash did not commit to *which branch* was being executed — a weakness fixed by Bythos. + +## The Axis: Binary Tree Addressing + +The `axis` field is a Nock-native concept for addressing positions in binary trees. In Nock, every value is a binary tree (a "noun"), and tree positions are addressed by axes: + +``` + 1 (root) + / \ + 2 3 + / \ / \ + 4 5 6 7 +``` + +- Axis 1: root +- Axis 2: left child +- Axis 3: right child +- Axis 4: left-left +- Axis 5: left-right +- etc. + +When a lock tree has multiple branches, the axis tells the validator which leaf position the provided `SpendCondition` occupies. The Merkle proof then demonstrates that this leaf, at this axis, hashes up to the lock root committed in the note's Name. + +## Merkle Proof Structure + +```rust +// crates/nockchain-types/src/tx_engine/v1/tx.rs:408-412 +pub struct MerkleProof { + pub root: Hash, // The expected root hash (matches Name.first) + pub path: Vec, // Sibling hashes from leaf to root +} +``` + +The proof contains: +1. **root**: the expected Merkle root (must match the lock root hash stored in the note's Name) +2. **path**: a list of sibling hashes, one per level from the leaf up to the root + +Verification proceeds by: +1. Hash the spend condition (the revealed leaf) +2. At each level, combine with the sibling hash from the path +3. Check that the result equals the declared root +4. Check that the declared root matches the note's lock root (committed in `Name.first`) + +## Hashable Construction + +The hash commitment for a lock Merkle proof is computed differently for stub and full formats. + +### Stub Hashable (Pre-Bythos) + +From `changelog/protocol/012-bythos.md`: + +```hoon +:+ hash+(hash:spend-condition spend-condition.form) + hash+(from-b58:^hash '6mhCSwJQDvbkbiPAUNjetJtVoo1VLtEhmEYoU4hmdGd6ep1F6ayaV4A') + (hashable-merk-proof merk-proof.form) +``` + +The second element is a **hardcoded hash** — a static placeholder that does not depend on the axis. This means two different branches of the same lock tree would produce the same witness hash if their spend conditions happened to hash identically. + +### Full Hashable (Post-Bythos) + +```hoon +:* leaf+version.form + hash+(hash:spend-condition spend-condition.form) + leaf+axis.form + (hashable-merk-proof merk-proof.form) +== +``` + +The full format includes `axis` as a leaf in the hashable tree, ensuring the witness hash commits to the specific branch being executed. This is a stronger security property — the witness is now bound to a particular position in the lock tree. + +## Height-Gated Proof Format Selection + +The proof format is determined by the note's origin page: + +```hoon +=/ parent-lmp=lock-merkle-proof + ?: (gte origin-page.note bythos-phase) + (build-lock-merkle-proof-full:lock parent-lock 1) + (build-lock-merkle-proof-stub:lock parent-lock 1) +``` + +- Notes created **before Bythos** (origin page < 54000): use stub proofs +- Notes created **at/after Bythos** (origin page ≥ 54000): use full proofs + +The validator accepts both formats but gates full proof acceptance on the Bythos activation height. + +## Privacy Properties + +Like Taproot, Nockchain's lock Merkle proofs provide **partial script privacy**: + +- **Revealed**: The spend condition being exercised (one branch of the lock tree) +- **Hidden**: All other branches remain behind Merkle hashes +- **Observable**: The number of tree levels (depth) can hint at the number of branches, but the exact count remains ambiguous + +For a lock tree with N branches: +- The spender reveals 1 spend condition + log2(N) sibling hashes +- The remaining N-1 conditions are hidden + +This is the same privacy model as Bitcoin Taproot script-path spends. + +## Comparison: Bitcoin Taproot vs Nockchain Lock Merkle Proofs + +| Aspect | Bitcoin Taproot | Nockchain Lock Merkle Proof | +|---|---|---| +| Script tree | MAST of TapScript leaves | Binary tree of SpendConditions | +| Key-path spend | Yes (single Schnorr sig, no script revealed) | No (always reveal a branch) | +| Script-path spend | Control block + script + Merkle proof | LockMerkleProof (spend condition + axis + proof) | +| Branch addressing | Control block leaf version + script | Axis (Nock binary tree address) | +| Proof structure | Internal key + parity + Merkle path | Root hash + sibling hash path | +| Hash function | SHA-256 (tagged) | Tip5 | +| Privacy | Unrevealed scripts hidden | Unrevealed conditions hidden | +| Commitment strength | Axis committed via control block | Stub: axis not committed; Full: axis committed | +| Composability | TapScript opcodes | AND within SpendCondition, OR across branches | +| Activation | BIP 9 signaling soft fork | Height-gated hard cutover | diff --git a/docs/architecture/tx-engine/04-eutxo-note-data.md b/docs/architecture/tx-engine/04-eutxo-note-data.md new file mode 100644 index 000000000..006d8339d --- /dev/null +++ b/docs/architecture/tx-engine/04-eutxo-note-data.md @@ -0,0 +1,181 @@ +# Extended UTXO: Note Data as On-Chain Datum + +## Cardano eUTXO Recap + +Cardano's extended UTXO (eUTXO) model extends Bitcoin's UTXO model with three additions: + +1. **Datum**: Arbitrary data attached to each UTXO, available to validator scripts +2. **Redeemer**: Data provided by the spender to the validator when consuming a UTXO +3. **Script context**: The full transaction context available to validator scripts + +The datum allows UTXOs to carry state, enabling stateful smart contracts within the UTXO paradigm — each UTXO becomes a "mini-state-machine" rather than just a value container. + +## Nockchain's Note Data + +V1 notes carry a `note-data` field — a z-map (persistent sorted map) from `@tas` symbol keys to arbitrary nouns. The authoritative Hoon definition: + +```hoon +:: from hoon/common/tx-engine-1.hoon +:: note-data is a z-map of @tas keys to arbitrary noun values +note-data=(z-map @tas *) +``` + +Rust serialization mirror (represents the z-map as a list of key-value pairs with jammed noun blobs): + +```rust +// crates/nockchain-types/src/tx_engine/v1/note.rs:98-99 +pub struct NoteData(pub Vec); + +// crates/nockchain-types/src/tx_engine/v1/note.rs:115-120 +pub struct NoteDataEntry { + pub key: String, // @tas (Hoon "cord" / symbol string) + pub blob: bytes::Bytes, // Jammed noun (arbitrary serialized Nock data) +} +``` + +### How Note Data Attaches to UTXOs + +Note data flows from transaction outputs (seeds) to the notes they create: + +```rust +// crates/nockchain-types/src/tx_engine/v1/tx.rs:158-165 +pub struct Seed { + pub output_source: Option, + pub lock_root: Hash, + pub note_data: NoteData, // Data for the new UTXO + pub gift: Nicks, + pub parent_hash: Hash, +} +``` + +When a transaction is applied to the balance: +1. Each seed creates a new V1 note +2. The seed's `note_data` becomes the note's `note_data` +3. The resulting `NoteV1` carries this data for its entire lifetime + +```rust +// crates/nockchain-types/src/tx_engine/v1/note.rs:58-65 +pub struct NoteV1 { + pub version: Version, + pub origin_page: BlockHeight, + pub name: Name, + pub note_data: NoteData, // Carried from the creating seed + pub assets: Nicks, +} +``` + +### Noun Encoding + +Note data entries use Hoon's `@tas` (text atom symbol) for keys and **jammed nouns** for values. A jammed noun is a compressed binary serialization of an arbitrary Nock value — it can represent any data structure expressible in Nock: + +```rust +// crates/nockchain-types/src/tx_engine/v1/note.rs:128-145 +impl NounEncode for NoteData { + fn to_noun(&self, allocator: &mut A) -> Noun { + self.0.iter().fold(D(0), |map, entry| { + let mut key = make_tas(allocator, &entry.key).as_noun(); + // Cue (decompress) the jammed blob back into a noun + let mut slab: NounSlab = NounSlab::new(); + slab.cue_into(entry.blob.clone()).expect("failed to cue blob"); + let mut value = unsafe { + let &root = slab.root(); + allocator.copy_into(root) + }; + zmap::z_map_put(allocator, &map, &mut key, &mut value, &DefaultTipHasher) + .expect("failed to encode note-data entry") + }) + } +} +``` + +The data is stored on-chain as a z-map (persistent sorted map) of `(@tas, *)` — string keys to arbitrary nouns. This makes it a fully general key-value store per UTXO. + +## Lock-Root Aggregation + +A distinctive feature of Nockchain's note data model: **outputs (seeds) sharing the same lock root have their note-data merged**. + +From `hoon/common/tx-engine-1.hoon`: + +```hoon +++ note-data-by-lock-root + |= sps=form + ^- (z-mip ^hash @tas *) + =/ all-seeds=(list seed) ... + =/ by-lock-root=(z-mip ^hash @tas *) + %+ roll all-seeds + |= [sed=seed acc=(z-mip ^hash @tas *)] + =/ key=hash lock-root.sed + =/ existing=(unit (z-map @tas *)) + (~(get z-by acc) key) + ?~ existing + (~(put z-by acc) key note-data.sed) + =/ merged=(z-map @tas *) + (~(uni z-by u.existing) note-data.sed) + (~(put z-by acc) key merged) + by-lock-root +``` + +This means: +- If a transaction has three outputs to the same lock root, each with note-data `{a: 1}`, `{b: 2}`, `{c: 3}`, they merge into one note with `{a: 1, b: 2, c: 3}` +- Fee calculation charges once for the merged data, not three times +- Size validation (`max-size`) applies to the merged result + +This is an unusual design choice with no direct equivalent in Bitcoin or Cardano. It enables efficient batch data attachment to a single spending authority. + +## Size Limits + +Bythos (Protocol 012) introduced per-output size validation for note data: + +```hoon +++ note-data-exceeds-max + |= [sps=form max=@] + ^- ? + %+ lien ~(tap z-by (note-data-by-lock-root sps)) + |= [key=hash note-data=(z-map @tas *)] + =/ data-size=@ + %- num-of-leaves:shape + %- ~(rep z-by note-data) + |= [[k=@tas v=*] tree=*] + [k v tree] + (gth data-size max) +``` + +The size is measured by counting the number of leaves in the noun tree representation of the note-data map. This is checked per-output (after merging by lock-root), not per-seed. + +The `max-size` limit is configured in `blockchain-constants.data.max-size` and provides protection against bloating the UTXO set with arbitrarily large datum. + +## Comparison: Cardano Datum vs Nockchain Note Data + +| Aspect | Cardano eUTXO Datum | Nockchain NoteData | +|---|---|---| +| Data type | Arbitrary Plutus Data (CBOR) | Arbitrary jammed nouns (key-value map) | +| Structure | Single datum per UTxO | Key-value map (`@tas → *`) per note | +| Attachment | Per output | Per output, merged by lock root | +| Size limits | Protocol parameter (max tx size) | `max-size` per merged output | +| Validator access | Full datum available to script | Note data available to Hoon validation | +| Inline vs reference | Both supported (Vasil) | Always inline | +| Hash commitment | Datum hash in output (pre-Vasil) | Note data included in note structure | +| Fee impact | Contributes to transaction size/fee | Contributes to seed word count for fee | +| Use cases | DeFi state, NFT metadata, oracle data | TBD (protocol-level data attachment) | + +## Key Differences from Full eUTXO + +Nockchain's note data is inspired by eUTXO but is **not a full implementation** of the Cardano model: + +1. **No validator execution per UTXO**: Cardano runs Plutus validators when UTXOs are consumed; Nockchain's lock primitives (`Pkh`, `Tim`, `Hax`, `Burn`) are fixed opcodes, not arbitrary scripts. The note data is not consumed by a user-defined validator. + +2. **No redeemer concept**: In Cardano, the spender provides a "redeemer" argument to the validator. Nockchain's closest equivalent is the `Witness` struct, but its fields are purpose-specific (signatures, preimages), not general-purpose redeemer data. + +3. **No script context**: Cardano validators receive the full transaction context (all inputs, outputs, validity range, etc.). Nockchain's lock primitives check individual conditions without access to the full transaction. + +4. **Key-value structure**: Unlike Cardano's single datum blob, Nockchain uses a structured key-value map, which provides native namespacing and avoids the need for application-level datum parsing. + +The note data is best understood as a **data attachment mechanism** — a way to carry metadata alongside value — rather than a full programmable state machine. It provides the substrate for future extensions where lock primitives or validators could inspect note data during spending. + +## Potential Applications + +The note data mechanism enables: +- **Token metadata**: Attaching token names, symbols, or URIs to value-carrying notes +- **On-chain state**: Carrying application state that transitions with UTXO spending +- **Provenance tracking**: Recording the history or classification of value flows +- **Cross-chain data**: Carrying data needed for bridge or interoperability proofs diff --git a/docs/architecture/tx-engine/05-lock-primitives-script-model.md b/docs/architecture/tx-engine/05-lock-primitives-script-model.md new file mode 100644 index 000000000..70b3d2e4d --- /dev/null +++ b/docs/architecture/tx-engine/05-lock-primitives-script-model.md @@ -0,0 +1,262 @@ +# Lock Primitives: A Composable Script Model + +## Context: Programmable Spending + +Bitcoin uses Script — a stack-based, non-Turing-complete language — to express spending conditions. Cardano uses Plutus validators — Haskell-based programs compiled to Plutus Core. Nockchain takes a third approach: a fixed set of **lock primitives** composed via AND/OR logic, with Merkle tree branch selection providing the OR. + +## LockPrimitive: Authoritative Hoon Definition + +Lock primitives are defined in Hoon (`tx-engine-1.hoon`) as a tagged union using `@tas` (text atom symbol) tags. The Hoon definition is the source of truth: + +```hoon ++$ lock-primitive + $% [%pkh m=@ hashes=(z-set hash)] + [%tim rel=timelock-range-relative abs=timelock-range-absolute] + [%hax hashes=(z-set hash)] + [%brn ~] + == +``` + +A `spend-condition` is a list of lock primitives (AND logic): + +```hoon ++$ spend-condition (list lock-primitive) +``` + +Rust serialization mirror: + +```rust +// crates/nockchain-types/src/tx_engine/v1/tx.rs:478-484 +pub enum LockPrimitive { + Pkh(Pkh), // %pkh - Pay-to-public-key-hash (M-of-N Schnorr) + Tim(LockTim), // %tim - Timelock constraints + Hax(Hax), // %hax - Hash preimage verification + Burn, // %brn - Unspendable (proof-of-burn) +} +``` + +## Pkh: Pay-to-Public-Key-Hash + +Defined in Hoon as `[%pkh m=@ hashes=(z-set hash)]`: a threshold `m` and a set of public key hashes. + +Rust serialization mirror: + +```rust +// crates/nockchain-types/src/tx_engine/v1/tx.rs:534-539 +pub struct Pkh { + pub m: u64, // Required signature count (threshold) + pub hashes: Vec, // z-set of public key hashes +} +``` + +Pkh is the primary authentication primitive. It requires M Schnorr signatures from a set of N public key hashes — a threshold multisig scheme. + +### How Pkh Differs from Bitcoin + +| Feature | Bitcoin P2PKH | Bitcoin P2TR Key-Path | Nockchain Pkh | +|---|---|---|---| +| Key type | ECDSA | Schnorr | Schnorr (Cheetah curve) | +| Hash stores | Single pubkey hash | Tweaked public key | Set of pubkey hashes | +| Multisig | Separate OP_CHECKMULTISIG | MuSig2 aggregation | Native M-of-N threshold | +| Reveals | Full public key on spend | Only aggregated key | Signing pubkeys + signatures (lock stores only hashes) | + +The Pkh primitive uses **public key hashes** rather than public keys directly. The lock commits to `Hash(pubkey)`, and the witness provides `(pubkey, signature)` pairs. The validator: +1. Checks that `Hash(pubkey)` matches one of the committed hashes +2. Verifies the Schnorr signature against the pubkey and sig-hash +3. Counts that at least M valid signatures are provided + +### Witness Satisfaction + +The `PkhSignature` in the witness provides the satisfying data: + +```rust +// crates/nockchain-types/src/tx_engine/v1/tx.rs:296-301 +pub struct PkhSignatureEntry { + pub hash: Hash, // The pubkey hash being satisfied + pub pubkey: SchnorrPubkey, // The actual public key + pub signature: SchnorrSignature, // Schnorr signature +} +``` + +The Hoon validation (`tx-engine-1.hoon`, `check:pkh` at line 1656): + +```hoon +++ check + |= [=form ctx=check-context] + ?& =(m.form ~(wyt z-by pkh.witness.ctx)) :: exactly M signatures provided + =(~ (~(dif z-in ~(key z-by pkh.witness.ctx)) :: all pubkey hashes are in the + h.form)) :: Pkh's permitted set + %- ~(rep z-by pkh.witness.ctx) :: each pubkey hashes to its + |= [[h=^hash pk=schnorr-pubkey sig=schnorr-signature] a=?] :: claimed hash + ?& a =(h (hash:schnorr-pubkey pk)) == + %- batch-verify:affine:belt-schnorr:cheetah :: all signatures are valid + (signatures:pkh-signature pkh.witness.ctx sig-hash.ctx) + == +``` + +This checks four conditions (AND): +1. **Exactly** `m` signatures are provided (not "at least" — the count must match precisely) +2. All provided pubkey hashes are members of the Pkh's permitted hash set +3. Each provided public key hashes to its declared hash (binding pubkey to commitment) +4. All signatures pass batch Schnorr verification against the sig-hash via `batch-verify:affine:belt-schnorr:cheetah` + +## Tim: Timelock Constraints + +Defined in Hoon as `[%tim rel=timelock-range-relative abs=timelock-range-absolute]`. Each range is a pair of optional bounds (min, max). + +Timelocks constrain *when* a note can be spent, with both relative and absolute bounds: + +### Comparison with Bitcoin Timelocks + +| Feature | Bitcoin CLTV (BIP 65) | Bitcoin CSV (BIP 112) | Nockchain Tim | +|---|---|---|---| +| Type | Absolute | Relative | Both combined | +| Granularity | Block height or timestamp | Block count or time | Block height only | +| Min constraint | ✓ (cannot spend before) | ✓ (must wait N blocks) | ✓ (min height/delta) | +| Max constraint | ✗ (no expiry in CLTV) | ✗ (no expiry in CSV) | ✓ (max height/delta) | +| Implementation | OP_CHECKLOCKTIMEVERIFY | OP_CHECKSEQUENCEVERIFY | Lock primitive | + +Nockchain's Tim is more expressive than Bitcoin's individual timelock opcodes because it combines: +- **Absolute minimum**: like CLTV — "cannot spend before block X" +- **Absolute maximum**: "cannot spend after block Y" (not available in Bitcoin) +- **Relative minimum**: like CSV — "must wait N blocks after creation" +- **Relative maximum**: "must spend within N blocks of creation" (not available in Bitcoin) + +The relative constraints use `origin_page` (the block where the note was created) as the reference point, analogous to CSV's reference to the block containing the spending transaction's input. + +### Witness Satisfaction + +The `tim` field in the Witness is currently reserved (always 0). Timelock validation is **context-based** — it checks the current block height against the constraints without requiring witness data. + +The Hoon validation (`tx-engine-1.hoon`, `check:tim` at line 1741): + +```hoon +++ check + |= [=form ctx=check-context] + =/ rmin-ok=? ?~(min.rel.form %.y (gte now.ctx (add since.ctx u.min.rel.form))) + =/ rmax-ok=? ?~(max.rel.form %.y (lte now.ctx (add since.ctx u.max.rel.form))) + =/ amin-ok=? ?~(min.abs.form %.y (gte now.ctx u.min.abs.form)) + =/ amax-ok=? ?~(max.abs.form %.y (lte now.ctx u.max.abs.form)) + &(rmin-ok rmax-ok amin-ok amax-ok) +``` + +Where `now.ctx` is the current block height and `since.ctx` is the note's `origin_page`. Each constraint is optional (`unit`); absent constraints are vacuously satisfied. + +## Hax: Hash Preimage Verification + +Defined in Hoon as `[%hax hashes=(z-set hash)]`: a set of Tip5 hash commitments. + +Hax requires the spender to reveal preimages whose hashes match the committed values. This is the foundation for: +- **Hash Time-Locked Contracts (HTLCs)**: Combined with Tim for atomic swaps +- **Payment channels**: Conditional payments based on secret revelation +- **Commit-reveal schemes**: On-chain commitment to off-chain data + +### Comparison with Bitcoin Hash Locks + +| Feature | Bitcoin OP_SHA256/OP_HASH160 | Nockchain Hax | +|---|---|---| +| Hash function | SHA-256, RIPEMD-160, SHA-1 | Tip5 | +| Multiple preimages | Via script composition | Native set of hash commitments | +| Preimage type | Raw bytes | Jammed nouns (arbitrary Nock data) | + +### Witness Satisfaction + +The `hax` field in the Witness provides preimage revelations: + +```rust +// crates/nockchain-types/src/tx_engine/v1/tx.rs:253-258 +pub struct HaxPreimage { + pub hash: Hash, + pub value: bytes::Bytes, // Jammed noun bytes +} +``` + +The Hoon validation (`tx-engine-1.hoon`, `check:hax` at line 1704): + +```hoon +++ check + |= [=form ctx=check-context] + %- ~(all z-in form) + |= =^hash + =/ preimage (~(get z-by hax.witness.ctx) hash) + ?~ preimage %| + =(hash (hash-noun u.preimage)) +``` + +For each committed hash in the Hax set: +1. Look up the preimage in the witness `hax` map (`z-map hash *`) +2. If no preimage provided, fail +3. Hash the preimage via `hash-noun` (recursive hashable decomposition of the noun tree) and check it matches the committed hash + +The `hash-noun` arm builds a hashable tree from the noun's structure — cells become pairs, atoms become leaves — then hashes via `hash-hashable:tip5`. This is distinct from `hash-noun-varlen` (used in zoon for tree ordering), which uses the Dyck word encoding. + +In Hoon, preimage values are arbitrary nouns (`*`). The Rust serialization represents them as **jammed noun bytes** (`bytes::Bytes`) — Nock's binary serialization format — which are cued (decompressed) back to nouns during validation. This means preimages can be any structured Nock data (lists, trees, records), serialized to bytes for transport. + +## Burn: Unspendable Output + +```rust +LockPrimitive::Burn // Tag: "brn", value: 0 +``` + +Burn creates an **unconditionally unspendable** output. Any SpendCondition containing a Burn primitive can never be satisfied: + +```hoon +%brn %| :: always fails +``` + +This is Nockchain's equivalent of Bitcoin's `OP_RETURN` — used for: +- **Proof-of-burn**: Permanently destroying value +- **Data anchoring**: Committing data to the chain without creating spendable outputs +- **Mining commitments**: Permanently locking value as a proof mechanism + +## Composition Model + +### AND Logic: Within a SpendCondition + +A `SpendCondition` is a **list** of `LockPrimitive` values. All primitives in the list must be satisfied — AND logic: + +```rust +// crates/nockchain-types/src/tx_engine/v1/tx.rs:442-443 +pub struct SpendCondition(pub Vec); +``` + +Example: `[Pkh(1-of-2), Tim(min=100)]` means "provide 1 of 2 valid signatures AND the block height must be ≥ 100." + +### OR Logic: Across Lock Tree Branches + +Different branches of the lock Merkle tree provide OR semantics — any one branch can authorize spending. The spender chooses which branch to reveal via the `LockMerkleProof`. + +Example lock tree: +``` + Root + / \ + Branch A Branch B + [Pkh(2-of-3)] [Tim(height>1000), Hax(secret)] +``` + +This means: "Either provide 2-of-3 signatures, OR wait until block 1000 and reveal the secret." + +### Expressiveness Summary + +The AND/OR composition with four primitive types enables: + +| Pattern | Primitives Used | Use Case | +|---|---|---| +| Simple pay | Pkh(1-of-1) | Standard payment | +| Multisig | Pkh(M-of-N) | Shared custody | +| Timelock + multisig | Pkh + Tim | Vesting schedule | +| HTLC | Pkh + Hax + Tim | Atomic swaps | +| Dead man's switch | Branch A: Pkh(owner) OR Branch B: Pkh(heir) + Tim(delay) | Inheritance | +| Proof-of-burn | Burn | Value destruction | +| Escrow with timeout | Branch A: Pkh(2-of-2) OR Branch B: Pkh(sender) + Tim(expiry) | Conditional escrow | + +### What Cannot Be Expressed + +The fixed primitive set means some Bitcoin Script / Plutus capabilities are not available: +- **Arithmetic conditions**: No equivalent to `OP_ADD`, `OP_LESSTHAN` +- **Introspection**: No access to transaction structure (inputs/outputs) within primitives +- **Loops**: Not possible (but also not possible in Bitcoin Script) +- **State machines**: Cannot inspect note data during validation (unlike Cardano validators) +- **Covenants**: Cannot constrain how outputs must be structured + +This is a deliberate trade-off: simplicity and auditability over expressiveness. The four-primitive model covers the vast majority of practical spending patterns while remaining easy to formally verify. diff --git a/docs/architecture/tx-engine/06-fee-structure.md b/docs/architecture/tx-engine/06-fee-structure.md new file mode 100644 index 000000000..7b7333e88 --- /dev/null +++ b/docs/architecture/tx-engine/06-fee-structure.md @@ -0,0 +1,137 @@ +# Fee Structure: SegWit-Inspired Weight Discounting + +## Bitcoin SegWit Fee Model Recap + +Bitcoin SegWit introduced a **virtual weight** system where witness data is discounted at a 4:1 ratio: + +- Non-witness data: 4 weight units per byte +- Witness data: 1 weight unit per byte +- Block limit: 4,000,000 weight units (effectively ~1MB non-witness + ~3MB witness) + +The rationale: witness data does not contribute to the UTXO set and is only needed for validation, so it should cost less than output data that persists until spent. + +## Nockchain's Fee Model + +Nockchain's fee model evolved through two phases, ultimately arriving at the same principle: **witness (input) data should cost less than output data**. + +All fee logic is defined in Hoon (`hoon/common/tx-engine-1.hoon`), which is the authoritative source. The Rust types in `crates/nockchain-types/` carry the `BlockchainConstants` structure for deserialization but do not implement fee calculation. + +### Pre-Bythos Fee Model (Before Block 54000) + +Before Bythos, fees were calculated uniformly: + +``` +min_fee = max( + (seed_words + witness_words) * (2 * base_fee), + min_fee_constant +) +``` + +Where: +- `base_fee = 2^15 = 32768` (configured, but doubled in practice) +- `min_fee = 256` +- All transaction words charged at the same rate + +### Post-Bythos Fee Model (Block 54000+) + +Bythos (Protocol 012) introduced differential pricing, directly analogous to SegWit's weight discount: + +``` +min_fee = max( + (seed_words * base_fee) + (witness_words * base_fee / input_fee_divisor), + min_fee_constant +) +``` + +Where: +- `base_fee = 2^14 = 16384` (halved from pre-Bythos effective rate) +- `input_fee_divisor = 4` +- `min_fee = 256` + +The Hoon implementation from `tx-engine.hoon` (the versioned facade, line 916): + +```hoon +++ calculate-min-fee + |= [sps=form page-num=page-number] + ^- coins + =/ bythos-active=? (gte page-num bythos-phase) + :: bythos halves base-fee at activation; pre-bythos uses legacy 2x rate + =/ effective-base-fee=coins + ?:(bythos-active base-fee (mul 2 base-fee)) + =/ seed-word-count=@ (count-seed-words [sps page-num]) + =/ witness-word-count=@ (count-witness-words [sps page-num]) + :: inputs pay discounted fee only at/after bythos activation + =/ witness-divisor=@ ?:(bythos-active input-fee-divisor 1) + :: outputs (seeds) pay full effective-base-fee per word + =/ seed-fee=coins (mul seed-word-count effective-base-fee) + :: inputs (witnesses) pay effective-base-fee / input-fee-divisor per word + =/ witness-fee=coins (div (mul witness-word-count effective-base-fee) witness-divisor) + =/ word-fee=coins (add seed-fee witness-fee) + (max word-fee min-fee.data) +``` + +## Blockchain Constants + +Fee parameters are part of the `blockchain-constants` structure defined in Hoon. The Rust side mirrors these for deserialization: + +| Constant | Mainnet Value | Fakenet Value | Purpose | +|---|---|---|---| +| `base-fee` | 16384 (2^14) | 128 | Per-word fee rate for outputs | +| `input-fee-divisor` | 4 | 4 | Witness discount factor | +| `min-fee` | 256 | 256 | Absolute fee floor | +| `max-size` | (configured) | (configured) | Max note-data size per output | +| `bythos-phase` | 54000 | 54000 | Activation height for new fee model | + +## Word Counting + +Fees are denominated in "words" — the number of Nock noun tree nodes in the serialized transaction structure. This is Nock-native sizing: rather than counting bytes (as Bitcoin does), Nockchain counts the structural complexity of the noun representation. + +Two separate word counts are computed: +- **Seed words**: counted from the outputs (seeds), representing data that creates new UTXOs +- **Witness words**: counted from the inputs (witnesses), representing authentication data consumed during validation + +### Note-Data Accounting + +Seed word counting includes note-data, but with an important optimization: **note-data maps for outputs sharing the same lock root are merged first**, and the merged map's leaf count is charged once. + +From `tx-engine-1.hoon`: + +```hoon +++ note-data-by-lock-root + |= sps=form + ^- (z-mip ^hash @tas *) + ... + :: merges note-data from all seeds with the same lock-root +``` + +This prevents double-charging when multiple outputs to the same lock root carry overlapping data. + +## Comparison: Bitcoin SegWit Weight vs Nockchain Fee Formula + +| Aspect | Bitcoin SegWit | Nockchain (Post-Bythos) | +|---|---|---| +| Measurement unit | Bytes → weight units | Noun words | +| Output data rate | 4 WU per byte | `base_fee` per word | +| Witness data rate | 1 WU per byte | `base_fee / 4` per word | +| Discount ratio | 4:1 | 4:1 (configurable via `input-fee-divisor`) | +| Fee floor | Dust relay minimum | `min_fee = 256` | +| Activation | Soft fork (BIP 141) | Height-gated (block 54000) | +| Rationale | Witness data doesn't grow UTXO set | Inputs are consumed; outputs persist | + +The 4:1 discount ratio is identical to Bitcoin SegWit. The rationale is the same: outputs create new UTXOs that the network must store until spent, while input/witness data is validated once and discarded. Discounting witness data incentivizes spending (consuming UTXOs) over creating new ones. + +## Fee Transition Mechanics + +The Bythos upgrade handled the fee transition smoothly: + +1. **Before block 54000**: `effective_base_fee = 2 * base_fee = 2 * 16384 = 32768`, `witness_divisor = 1` + - Net effect: `(all_words) * 32768` +2. **At/after block 54000**: `effective_base_fee = base_fee = 16384`, `witness_divisor = 4` + - Net effect: `(seed_words * 16384) + (witness_words * 4096)` + +For a typical transaction where witness data is ~50% of total words: +- Pre-Bythos: `100 words * 32768 = 3,276,800` +- Post-Bythos: `50 * 16384 + 50 * 4096 = 819,200 + 204,800 = 1,024,000` +- ~69% fee reduction for typical transactions + +This mirrors the practical effect of Bitcoin SegWit, where SegWit transactions paid significantly lower fees than legacy transactions. diff --git a/docs/architecture/tx-engine/07-transaction-validation-pipeline.md b/docs/architecture/tx-engine/07-transaction-validation-pipeline.md new file mode 100644 index 000000000..c86d37257 --- /dev/null +++ b/docs/architecture/tx-engine/07-transaction-validation-pipeline.md @@ -0,0 +1,218 @@ +# Transaction Validation Pipeline + +## Overview + +Transaction validation in Nockchain is a multi-phase process implemented primarily in Hoon (`hoon/common/tx-engine-0.hoon` and `hoon/common/tx-engine-1.hoon`), which is the authoritative source of truth for all validation logic. The Rust layer handles deserialization and networking but defers consensus validation to the Nock VM executing the Hoon kernel. + +Validation proceeds through three phases: +1. **Structural validation** — well-formedness of the transaction itself +2. **Witness verification** — cryptographic proof checking +3. **Context-aware validation** — checking against the current chain state + +## Phase 1: Structural Validation + +Structural validation checks that the transaction is well-formed without reference to the UTXO set or chain state. + +### V0 Transactions + +The V0 `validate` arm in `tx-engine-0.hoon` checks: +- Transaction ID (`tx-id`) matches the hash of the transaction body +- Inputs are non-empty +- Fee amounts are non-negative +- Timelock ranges are internally consistent (min ≤ max) +- Total fees match the sum of per-input fees + +### V1 Transactions + +The V1 `validate` arm in `tx-engine-1.hoon` checks: +- Version tag is `%1` +- Transaction ID matches the hash of the spends +- Spends map is non-empty +- Each spend is internally consistent: + - Seeds (outputs) are non-empty + - Fee is non-negative + - Gift amounts sum correctly + - Spend version tag is valid (`%0` or `%1`) + +## Phase 2: Witness Verification + +### V0: Direct Signature Verification + +V0 spends carry signatures directly. Verification involves: +1. Computing the sig-hash from the spend's seeds and fee +2. For each (pubkey, signature) pair in the signature map: + - Verify the Schnorr signature against the sig-hash +3. Check that at least `keys_required` valid signatures exist (M-of-N threshold) + +### V1: Lock Primitive Satisfaction + +V1 verification is more complex, with each lock primitive type checked independently. + +#### Lock Merkle Proof Validation + +The `check:lock-merkle-proof` arm verifies: +1. The spend condition hashes to the correct leaf +2. The Merkle path from leaf to root is valid (each level combines with the sibling hash) +3. The computed root matches the lock root in the note's Name +4. (Post-Bythos) For full proofs: the axis is committed in the hash and `now ≥ bythos-phase` + +#### PKH Signature Check + +The `check:pkh` arm verifies (all conditions must hold): +1. **Signature count**: The number of witness PKH entries equals `m` (the M-of-N threshold) +2. **Permissible keys**: The set of witness pubkey hashes is a subset of the committed hashes (checked via `z-in dif`) +3. **Hash binding**: Each `Hash(pubkey)` in the witness matches its committed hash (checked via `z-by rep`) +4. **Batch signature verification**: All Schnorr signatures are verified against the sig-hash in a single batch call via `batch-verify:affine:belt-schnorr:cheetah` + +The sig-hash for V1 is computed over the spend's seeds and fee (excluding the witness), ensuring signatures don't circularly depend on themselves. + +#### Timelock Check + +The `check:tim` arm verifies: +1. Current page number (`now.ctx`) satisfies absolute constraints: + - `now ≥ abs.min` (if set) + - `now ≤ abs.max` (if set) +2. Current page number satisfies relative constraints (relative to `since.ctx`, the note's origin page): + - `now ≥ since + rel.min` (if set) + - `now ≤ since + rel.max` (if set) + +No witness data is needed — timelocks are validated against the block context. + +#### Hash Preimage Check + +The `check:hax` arm verifies: +1. For each hash commitment in the Hax primitive +2. Look up the corresponding preimage in the witness `hax` map +3. Hash the preimage and verify it matches the committed hash + +#### Burn Check + +The `%brn` case always returns false — burn primitives can never be satisfied. + +### The check-context Arm + +V1 introduces a unified `check-context` structure that bundles all context needed for witness verification: + +```hoon +++ check-context + =< form + |% + +$ form + $: now=page-number :: current block height + since=page-number :: page height of the note (origin page) + sig-hash=hash :: sig-hash of the spend (for signature verification) + =witness :: the witness data being verified + bythos-phase=page-number :: height at which bythos activates + == + ++ check + |= [=form lock=hash] + ^- ? + ... +``` + +This arm validates a complete spend by: +1. Checking Bythos compatibility (full proofs only after bythos-phase; gates `%full` LMP format to `now >= bythos-phase`) +2. Extracting the spend condition from the lock merkle proof (handles both stub 3-tuple and full 4-tuple formats) +3. Verifying the lock merkle proof against the lock root +4. Checking each lock primitive in the spend condition via `levy` (AND logic): + - `%tim` → check timelocks against current height + - `%hax` → check hash preimages + - `%pkh` → check Schnorr signatures (with batch verification) + - `%brn` → always fail (`%|`) + +All checks must pass (AND logic within a spend condition). + +## Phase 3: Context-Aware Validation + +Added by Bythos (Protocol 012), context-aware validation checks the transaction against the current chain state. + +### validate-with-context + +The `validate-with-context` arm in `tx-engine-1.hoon` takes: +- `balance`: the current UTXO set (z-map of Name → Note) +- `sps`: the transaction's spends +- `page-num`: current block height +- `max-size`: maximum note-data size +- `bythos-phase`: activation height for Bythos rules + +It performs: + +1. **Note-data size check**: Ensure no merged output exceeds `max-size` +2. **For each spend**: + a. **UTXO existence**: The referenced note must exist in the balance + b. **Version matching**: + - V0 note (head is cell) → must use Spend0 + - V1 note (head is atom, version=1) → must use Spend1 + c. **Spend verification**: + - Spend0: verify V0-style signature, check gifts and fees + - Spend1: build check-context, verify lock, check gifts and fees + d. **Gifts and fees**: Total output gifts + fee ≤ input note's assets + +### Error Taxonomy + +The validation pipeline returns named rejection reasons (Hoon `@tas` cords): + +| Error | Meaning | +|---|---| +| `%v1-note-data-exceeds-max-size` | Merged note-data exceeds max-size limit | +| `%v1-input-missing` | Referenced UTXO not in balance | +| `%v1-spend-version-mismatch` | V0 note with Spend1, or V1 note with Spend0 | +| `%v1-note-version-mismatch` | V1 note has unexpected version number | +| `%v1-spend-0-verify-failed` | Spend0 signature verification failed | +| `%v1-spend-0-gifts-failed` | Spend0 output amounts don't balance | +| `%v1-spend-1-lock-failed` | Spend1 lock merkle proof / witness check failed | +| `%v1-spend-1-gifts-failed` | Spend1 output amounts don't balance | + +## Mempool vs Block Validation + +Bythos tightened the gap between mempool admission and block validation: + +**Before Bythos**: Mempool admission used only structural validation (`validate:raw-tx`) plus UTXO presence checks. Context-invalid transactions could circulate in mempools until block processing rejected them. + +**After Bythos**: V1 transactions in the mempool must also pass `validate-with-context` using current chain state: + +```hoon +=/ ctx-valid=(reason:t ~) + ?^ -.raw + [%.y ~] + %- validate-with-context:spends:t + :* get-cur-balance:con + spends.raw + get-cur-height:con + max-size.data.constants.k + bythos-phase.constants.k + == +?. ?=(%.y -.ctx-valid) + ~> %slog.[1 (cat 3 'heard-tx: Transaction context invalid: ' +.ctx-valid)] + `k +``` + +This means transactions that are structurally valid but fail context checks (expired timelocks, invalid lock proofs, oversized note-data, missing UTXOs) are dropped immediately at receipt rather than propagated through the network. + +## Validation Flow Summary + +``` +Transaction arrives + │ + ├─ Phase 1: Structural Validation + │ ├─ Version check + │ ├─ ID integrity + │ ├─ Non-empty inputs + │ └─ Internal consistency + │ + ├─ Phase 2: Witness Verification + │ ├─ Lock Merkle proof validation + │ ├─ PKH signature verification + │ ├─ Timelock satisfaction + │ ├─ Hash preimage verification + │ └─ Burn rejection + │ + └─ Phase 3: Context-Aware Validation (Bythos+) + ├─ Note-data size limits + ├─ UTXO existence + ├─ Version matching + ├─ Full witness+lock verification + └─ Gift/fee balance check +``` + +Each phase can independently reject the transaction with a specific error code. A transaction must pass all three phases to be admitted to the mempool or included in a block. diff --git a/docs/architecture/tx-engine/08-protocol-evolution.md b/docs/architecture/tx-engine/08-protocol-evolution.md new file mode 100644 index 000000000..9454bf270 --- /dev/null +++ b/docs/architecture/tx-engine/08-protocol-evolution.md @@ -0,0 +1,164 @@ +# Protocol Evolution: Upgrade Mechanics and History + +## Upgrade Philosophy + +Nockchain uses **height-gated hard cutovers** for consensus-critical upgrades. Unlike Bitcoin's BIP 9/BIP 8 miner-signaling soft fork mechanism, Nockchain upgrades activate at a predetermined block height with no signaling period. All nodes must upgrade before the activation height or risk forking. + +This approach is documented in `changelog/protocol/SPECIFICATION.md` and reflected in the protocol changelog entries. + +### Comparison: Bitcoin BIP Activation vs Nockchain Height-Gating + +| Aspect | Bitcoin (BIP 9/8) | Nockchain | +|---|---|---| +| Activation trigger | Miner signaling threshold | Fixed block height | +| Signaling period | ~2 weeks per retarget | None | +| Backward compatibility | Soft fork (old nodes accept) | Hard cutover (old nodes may fork) | +| Coordination | Miners signal readiness | All operators must upgrade | +| Rollback | Possible before lock-in | Only safe before activation | +| Upgrade timeline | Months (signaling + lock-in) | Days to weeks (deploy before height) | + +## Protocol Changelog + +All consensus-critical changes are documented in `changelog/protocol/`, with a structured frontmatter format: + +```toml +version = "0.1.11" +status = "final" # draft | final | superseded +consensus_critical = true +activation_height = 54000 +published = "2026-01-19" +activation_target = "2026-03-01" +``` + +## Version History + +### V0 Genesis (Block 0) + +The original transaction engine, defined entirely in `hoon/common/tx-engine-0.hoon` (~2471 lines). + +**Characteristics:** +- Single engine, no version tagging +- Simple M-of-N multisig via `Lock` (keys_required + set of Schnorr pubkeys) +- Signature directly embedded in `Spend` structure +- Coinbase keyed by signer pubkeys +- Notes carry full lock, source, and timelock data +- No note-data (no eUTXO datum) + +### Protocol 009: SegWit Cutover (Block 37350) + +**File**: `changelog/protocol/009-legacy-segwit-cutover-initial.md` + +The most significant upgrade — introduced the V0/V1 split. This is Nockchain's "SegWit moment." + +**Changes:** +1. **Dual engine architecture**: `tx-engine.hoon` became a versioned facade dispatching to `tx-engine-0.hoon` or `tx-engine-1.hoon` +2. **Witness separation**: V1 `Spend1` carries a separate `Witness` struct +3. **Lock trees**: V1 replaces M-of-N locks with Merkle trees of `SpendCondition` branches +4. **Lock primitives**: `Pkh`, `Tim`, `Hax`, `Burn` replace the monolithic lock +5. **Note data**: V1 notes carry `NoteData` (eUTXO-inspired datum) +6. **Bridge spends**: `Spend0` allows V0 notes to transition to V1 outputs +7. **Coinbase format**: Changed from sig-keyed to lock-hash-rooted +8. **Fee floor**: Word-count-based pricing with `base-fee = 2^15` + +**Height-gated rule matrix:** + +| Block Height | V0 Tx | V1 Tx | Coinbase | +|---|---|---|---| +| `< 37350` | Allowed | Rejected | V0 format | +| `≥ 37350` | Rejected | Required | V1 format | + +**State migration**: Kernel state upgraded to v6 to carry versioned consensus objects. + +### Protocol 010: V1-Phase Finalization (Block 39000) + +**File**: `changelog/protocol/010-legacy-v1-phase-39000.md` + +Finalized the V1-phase boundary. The initial plan (Protocol 009) set `v1-phase = 37350`, and Protocol 010 confirmed the transition period ended cleanly at block 39000. + +### Protocol 011: LMP Axis Hotfix + +**File**: `changelog/protocol/011-legacy-lmp-axis-hotfix.md` + +A targeted fix for a lock merkle proof issue related to axis handling — addressed a specific vulnerability or edge case before the Bythos comprehensive fix. + +### Protocol 012: Bythos (Block 54000) + +**File**: `changelog/protocol/012-bythos.md` + +A consensus-critical upgrade addressing two issues discovered after the initial SegWit cutover. + +**Lock Merkle Proof Versioning:** +- Problem: Stub proofs used a hardcoded placeholder instead of committing to the axis field, meaning the witness hash didn't bind to which branch was executed +- Solution: Introduced `lock-merkle-proof-full` with axis in the hashable +- Backward compatibility: Both stub and full formats accepted after activation; stub proofs only before activation +- Format gating by note origin page, not current height + +**Fee Rebalancing:** +- Base fee halved: `2^15 → 2^14` +- Input/witness discount: `input-fee-divisor = 4` (inputs charged at 1/4 output rate) +- Separate seed/witness word counting +- Note-data size validation moved to per-output (after merge by lock-root) + +**Context-Aware Mempool Admission:** +- V1 transactions now validated with `validate-with-context` at mempool receipt +- Transactions failing context checks (expired timelocks, invalid lock proofs, oversized note-data) dropped immediately + +### Protocol 013: Nous (Draft, Non-Consensus) + +**File**: `changelog/protocol/013-nous.md` + +A networking-layer upgrade (not tx-engine). Adds batched transport requests for the libp2p request-response protocol. Included here for completeness — it does not change transaction formats or consensus rules. + +## The Versioned Facade + +The Hoon facade (`hoon/common/tx-engine.hoon`) is the authoritative dispatch layer: + +```hoon +/= v0 /common/tx-engine-0 +/= v1 /common/tx-engine-1 +... +|_ blockchain-constants ++* v0 ~(. ^v0 +63:+<) +``` + +It wraps types as tagged unions for consensus-visible structures: + +```hoon +++ coinbase-split + =< form + |% + +$ form + $% [%0 coinbase-split:v0] + [%1 coinbase-split:v1] + == +``` + +This pattern applies to `page`, `raw-tx`, `local-page`, `coinbase-split`, and other consensus types. The facade checks version tags and routes to the appropriate engine. + +## Kernel State Migration + +Protocol upgrades sometimes require kernel state migration. The kernel state carries the UTXO balance, chain tip, and other consensus data. When the state format changes: + +1. The kernel state version is bumped (e.g., v5 → v6 for Protocol 009) +2. Old state is migrated to the new format on first load +3. Legacy entries are tagged with version markers (e.g., V0 notes in the balance) + +The Hoon kernel manages this migration during node startup. The Rust networking layer is agnostic to the internal state format — it only handles serialized nouns for transport. + +## Upgrade Coordination Model + +Nockchain upgrades follow a predictable pattern: + +1. **Protocol spec published** in `changelog/protocol/` with all technical details +2. **Software updated** to include new rules, gated by activation height +3. **Operators deploy** updated software before activation +4. **Activation height reached** — new rules take effect network-wide +5. **Old software** may fork or reject valid blocks + +There is no soft-fork compatibility layer. This is a deliberate choice: +- Simpler reasoning about consensus rules at any given height +- No ambiguity about which rules apply +- No "anyone can spend" risks from soft fork semantics +- Clear upgrade timeline for operators + +The trade-off is that upgrades require full network coordination. Late upgraders will fork. diff --git a/docs/architecture/tx-engine/09-noun-encoding-data-layer.md b/docs/architecture/tx-engine/09-noun-encoding-data-layer.md new file mode 100644 index 000000000..3d5b4abfa --- /dev/null +++ b/docs/architecture/tx-engine/09-noun-encoding-data-layer.md @@ -0,0 +1,199 @@ +# Noun Encoding and the Nock Data Layer + +## Architecture: Hoon as Source of Truth + +A critical architectural point: **Hoon is the source of truth** for all type definitions and validation logic in the Nockchain transaction engine. The Rust type definitions in `crates/nockchain-types/` are serialization/deserialization mirrors — they exist to encode and decode Hoon nouns for networking and application layers, but they do not define the canonical semantics. + +This means: +- **Consensus validation runs in Nock**: The Hoon kernel executing on the Nock VM is what actually validates transactions, computes fees, checks witnesses, and updates balances +- **Rust types are wire-format adapters**: They serialize Rust structs into nouns (for sending to the Hoon kernel) and deserialize nouns back into Rust structs (for networking, APIs, and wallet logic) +- **The Hoon types are authoritative**: If the Rust types and Hoon types disagree, the Hoon definition is correct and the Rust code must be fixed + +## Nock Nouns: The Universal Data Representation + +Nock (the virtual machine underlying Hoon) has exactly one data type: the **noun**. A noun is either: +- An **atom**: an arbitrary-precision natural number (0, 1, 2, ..., 2^256, ...) +- A **cell**: an ordered pair of nouns `[left right]` + +Every data structure in the system — transactions, notes, balances, blocks, proofs — is ultimately a noun: a binary tree of natural numbers. This is radically different from conventional type systems: + +``` +Transaction (as a noun): +[1 [hash...] [[name spend] [name spend] 0]] + │ │ │ + ver id spends (z-map) +``` + +## NounEncode / NounDecode: The Rust↔Hoon Bridge + +The Rust codebase uses `NounEncode` and `NounDecode` traits to bridge between typed Rust structs and untyped Nock nouns: + +```rust +// Example from crates/nockchain-types/src/tx_engine/v1/tx.rs:25-31 +impl NounEncode for RawTx { + fn to_noun(&self, allocator: &mut A) -> Noun { + let version = self.version.to_noun(allocator); + let id = self.id.to_noun(allocator); + let spends = self.spends.to_noun(allocator); + nockvm::noun::T(allocator, &[version, id, spends]) + } +} +``` + +This creates a noun cell tree: `[version [id spends]]` — matching the Hoon type definition: + +```hoon ++$ form + $: version=%1 + id=tx-id + =spends + == +``` + +### Structural Discrimination + +Because nouns are untyped, version discrimination happens by **examining the shape** of the data. For example, the `Note` enum: + +```rust +// crates/nockchain-types/src/tx_engine/v1/note.rs:77-84 +impl NounDecode for Note { + fn from_noun(noun: &Noun) -> Result { + let hed = noun.as_cell()?.head(); + match hed.is_cell() { + true => Ok(Note::V0(NoteV0::from_noun(noun)?)), // head is cell → V0 + false => Ok(Note::V1(NoteV1::from_noun(noun)?)), // head is atom → V1 + } + } +} +``` + +V0 notes have a `NoteHead` (a cell) as their first element; V1 notes have a `Version` atom (1). The decoder probes the structure to determine the variant. This is idiomatic Hoon — `$^` and `$@` runes discriminate types by whether the head is a cell or atom. + +Similarly, `LockMerkleProof` discriminates stub vs full: + +```rust +// crates/nockchain-types/src/tx_engine/v1/tx.rs:394-406 +impl NounDecode for LockMerkleProof { + fn from_noun(noun: &Noun) -> Result { + if let Ok(full) = LockMerkleProofFull::from_noun(noun) { + if full.version != nockvm_macros::tas!(b"full") { + return Err(NounDecodeError::Custom(...)); + } + return Ok(Self::Full(full)); + } + Ok(Self::Stub(LockMerkleProofStub::from_noun(noun)?)) + } +} +``` + +Try the 4-tuple (Full) first; if it fails or the version tag isn't `%full`, fall back to the 3-tuple (Stub). + +## Z-Maps and Z-Sets: Persistent Sorted Trees + +Nockchain uses **z-maps** and **z-sets** — balanced binary trees ordered by Tip5 hash — for all collection types. These are the Nock-native equivalents of sorted maps and sets. + +### Z-Map (Sorted Map) + +Used for: +- **Spends**: `z-map(Name → Spend)` — associates each UTXO being consumed with its spend proof +- **Balance**: `z-map(Name → Note)` — the full UTXO set +- **PkhSignature**: `z-map(Hash → PkhSignatureEntry)` — signature proofs keyed by pubkey hash +- **NoteData**: `z-map(@tas → *)` — arbitrary key-value data on notes +- **Witness Hax**: `z-map(Hash → *)` — hash preimage reveals keyed by commitment hash + +```rust +// Z-map put operation (from Spends encoding): +zmap::z_map_put(allocator, &acc, &mut key, &mut value, &DefaultTipHasher) +``` + +### Z-Set (Sorted Set) + +Used for: +- **Seeds**: `z-set(Seed)` — the set of outputs in a spend +- **Pkh hashes**: `z-set(Hash)` — the set of authorized pubkey hashes +- **Hax hashes**: `z-set(Hash)` — the set of hash commitments + +```rust +// Z-set put operation (from Seeds encoding): +zset::z_set_put(allocator, &acc, &mut value, &DefaultTipHasher) +``` + +Both structures use `DefaultTipHasher` (Tip5) for ordering, ensuring deterministic tree layout regardless of insertion order. This is essential for consensus — every node must produce the same noun representation for the same logical data. + +## Tip5: The Hash Function + +Nockchain uses **Tip5** as its universal hash function for: +- Merkle trees (lock merkle proofs, block commitments) +- Transaction IDs +- Note Names +- UTXO set ordering (z-map/z-set balancing) +- Public key hashing (Pkh) + +Tip5 produces a 5-element tuple of field elements (`[Belt; 5]`), where each `Belt` is a `u64` reduced modulo a prime: + +```rust +// crates/nockchain-types/src/tx_engine/common/mod.rs:149-150 +pub struct Hash(pub [Belt; 5]); +``` + +### Hashable Construction + +Hoon defines `hashable` arms for each type that specify how to construct the hash input tree: + +```hoon +++ hashable + |= =form + ^- hashable:tip5 + :* leaf+version.form + hash+(hash:spend-condition spend-condition.form) + leaf+axis.form + (hashable-merk-proof merk-proof.form) + == +``` + +The `hashable` type is a tagged tree of `leaf` (raw values) and `hash` (pre-computed hashes) nodes. The Tip5 hasher traverses this tree to produce a deterministic hash. This is defined in Hoon and executed in the Nock VM; the Rust side does not reimplement the hashing logic for consensus purposes. + +## Jamming: Noun Serialization + +**Jam** is Nock's native serialization format — it compresses an arbitrary noun into a byte sequence. **Cue** is the inverse (deserialization). + +Jam is used for: +- **NoteData blobs**: Each note-data entry's value is a jammed noun +- **HaxPreimage values**: Hash preimages are jammed nouns +- **Wire format**: Nouns sent between nodes are jammed for transport + +```rust +// Jamming a noun for storage: +let mut slab: NounSlab = NounSlab::new(); +slab.copy_into(raw_value); +let jam = slab.jam(); // → bytes::Bytes + +// Cueing bytes back into a noun: +slab.cue_into(entry.blob.clone())?; +``` + +Jam achieves compression through structure sharing — if the same sub-noun appears multiple times, it's stored once and referenced by back-pointer. This is particularly efficient for the repetitive structures common in transaction data. + +## Architectural Significance + +### Why Nouns? + +The noun-based architecture has several consequences: + +1. **Language-independent consensus**: The Nock VM specification is ~300 words. Any implementation that correctly evaluates Nock programs will produce identical results. This makes consensus validation implementation-independent. + +2. **Deterministic serialization**: Nouns have a canonical form — the same logical data always produces the same noun, which always produces the same jam bytes, which always produces the same hash. No sorting, normalization, or canonicalization heuristics needed. + +3. **Unified type system**: Everything is a noun. There's no impedance mismatch between "transaction types" and "hash inputs" and "serialized bytes" — they're all the same thing at different levels of interpretation. + +4. **Functional state updates**: Z-maps and z-sets are persistent (immutable) data structures. Updating the balance (UTXO set) produces a new tree that shares structure with the old one, enabling efficient versioning and rollback. + +### Performance Considerations + +The noun-based approach has trade-offs: + +- **Encoding overhead**: Converting between Rust structs and nouns requires allocation and tree traversal. This is more expensive than zero-copy serialization formats like FlatBuffers. +- **Hash computation**: Tip5 hashing traverses the noun tree, which is more expensive than hashing a flat byte array. However, Tip5 is specifically designed for efficient operation within the Nock VM. +- **Memory layout**: Nouns are pointer-heavy binary trees, which are less cache-friendly than contiguous arrays. The Nock VM uses arena allocation to mitigate this. + +These costs are accepted because the noun layer is the consensus boundary — it provides the deterministic execution guarantee that makes decentralized validation possible. Performance-critical paths (like signature verification) are accelerated by **jets** — optimized Rust implementations of commonly-used Nock functions that produce identical results to the pure Nock evaluation. diff --git a/docs/architecture/tx-engine/10-zoon-persistent-data-structures.md b/docs/architecture/tx-engine/10-zoon-persistent-data-structures.md new file mode 100644 index 000000000..1c0762dbb --- /dev/null +++ b/docs/architecture/tx-engine/10-zoon-persistent-data-structures.md @@ -0,0 +1,271 @@ +# Zoon: Hash-Ordered Persistent Trees + +## Overview + +Zoon (`hoon/common/zoon.hoon`, ~700 lines) is Nockchain's vendored balanced tree library, adapted from Hoon's standard `by`/`in` containers. It provides cryptographically-ordered persistent (immutable) collections — z-maps and z-sets — that use Tip5 hashing for deterministic element ordering. These are the data structures that hold the UTXO set, transaction spends, seeds, signatures, and every other collection in the transaction engine. + +Zoon explicitly deprecates the standard Hoon containers: + +```hoon +:: /lib/zoon: vendored types from hoon.hoon ++| %no-by-in +++ by %do-not-use +++ in %do-not-use +++ ju %do-not-use +++ ja %do-not-use +++ bi %do-not-use +``` + +This ensures the entire codebase uses Tip5-ordered trees, which is critical for consensus — all nodes must produce identical noun representations for the same logical data. + +## Import Chain + +``` +zoon.hoon + └─ imports zeke.hoon + └─ re-exports ztd/eight.hoon + └─ ... (full STARK stack, including Tip5) +``` + +Zoon imports `zeke.hoon` (a 1-line re-export of `ztd/eight.hoon`) to get access to the Tip5 hash function. The jet hint `~% %zoon ..stark-engine-jet-hook:z ~` registers the entire zoon library for JIT acceleration by the Nock VM. + +## z-map: Persistent Sorted Map + +```hoon +++ z-map + |$ [key value] + $| (tree (pair key value)) + |=(a=(tree (pair)) ?:(=(~ a) & ~(apt z-by a))) +``` + +A z-map is a balanced binary tree where each node is a `[key value]` pair. The tree maintains two invariants: + +1. **gor-tip ordering**: Keys are ordered by their Tip5 hash (primary key comparison via `gor-tip`) +2. **mor-tip heap property**: Parent nodes have higher `mor-tip` (double-hash) priority than children + +Together, these make the tree a **treap** (tree + heap) — the combination of hash-based ordering and hash-based priority ensures a unique tree shape for any set of keys, regardless of insertion order. + +### z-by: The z-map Engine + +The `z-by` core provides the full API, all jet-hinted with `~/`: + +| Arm | Purpose | Complexity | +|---|---|---| +| `get` | Look up value by key | O(log n) | +| `put` | Insert or update key-value pair | O(log n) | +| `del` | Remove key from map | O(log n) | +| `has` | Check key existence | O(log n) | +| `gas` | Batch insert from list | O(k log n) | +| `uni` | Union (merge two maps, right-biased) | O(n + m) | +| `int` | Intersection of two maps | O(n + m) | +| `dif` | Difference (remove keys in other map) | O(n + m) | +| `bif` | Split map at a key | O(log n) | +| `tap` | Convert to list (in-order traversal) | O(n) | +| `rep` | Fold/reduce over all entries | O(n) | +| `run` | Apply gate to all values | O(n) | +| `urn` | Apply gate to all key-value pairs | O(n) | +| `wyt` | Count entries | O(n) | +| `key` | Extract z-set of keys | O(n) | +| `val` | Extract list of values | O(n) | +| `apt` | Validate tree invariants | O(n) | +| `dig` | Find axis of key in tree | O(log n) | +| `jab` | Update value at key via gate | O(log n) | +| `mar` | Conditional put/delete | O(log n) | +| `uno` | General union with merge function | O(n + m) | + +### The `put` Algorithm + +The `put` arm illustrates how the treap invariants work: + +```hoon +++ put + ~/ %put + |* [b=* c=*] + |- ^+ a + ?~ a + [[b c] ~ ~] + ?: =(b p.n.a) + ?: =(c q.n.a) a + a(n [b c]) + ?: (gor-tip b p.n.a) + =+ d=$(a l.a) + ?> ?=(^ d) + ?: (mor-tip p.n.a p.n.d) + a(l d) + d(r a(l r.d)) + =+ d=$(a r.a) + ?> ?=(^ d) + ?: (mor-tip p.n.a p.n.d) + a(r d) + d(l a(r l.d)) +``` + +1. If tree is empty: create leaf `[[key value] ~ ~]` +2. If key matches: update value +3. Recurse left or right based on `gor-tip` comparison +4. After insertion, check `mor-tip` heap property; if violated, rotate + +## z-set: Persistent Sorted Set + +```hoon +++ z-set + |$ [item] + $| (tree item) + |=(a=(tree) ?:(=(~ a) & ~(apt z-in a))) +``` + +A z-set is a z-map with keys only (no values). The `z-in` engine provides a parallel API: `put`, `has`, `del`, `dif`, `int`, `uni`, `bif`, `tap`, `wyt`, `all`, `any`, `run`, `gas`, `rep`, `dig`. + +## z-mip and z-jug: Nested Collections + +Zoon also provides two higher-order collection types: + +**z-mip** (map of maps): +```hoon +++ z-mip |$ [kex key value] (z-map kex (z-map key value)) +``` + +Used in the tx-engine for `note-data-by-lock-root`: a map from lock-root hash to note-data maps. + +**z-jug** (map of sets): +```hoon +++ z-jug |$ [key value] (z-map key (z-set value)) +``` + +A key-to-set-of-values mapping, with `z-ju` engine providing `put`, `get`, `has`, `del`, `gas`. + +## Ordering Functions: The Cryptographic Heart + +The ordering functions define the tree layout. All comparisons ultimately derive from Tip5 hashing. + +### tip: Primary Hash + +```hoon +++ tip + |= a=* + ^- noun-digest:tip5:z + (hash-noun-varlen:tip5:z a) +``` + +Computes the Tip5 hash of an arbitrary noun. This is a 5-element digest `[Belt; 5]`. + +### double-tip: Secondary Hash + +```hoon +++ double-tip + |= a=* + ^- noun-digest:tip5:z + =/ one (tip a) + (hash-ten-cell:tip5:z one one) +``` + +Hashes the Tip5 digest with itself, producing a secondary hash used for the heap property. + +### dor-tip: Canonical Depth-First Ordering + +```hoon +++ dor-tip + ~/ %dor-tip + |= [a=* b=*] + ^- ? + ?: =(a b) & + ?. ?=(@ a) + ?: ?=(@ b) | + ?: =(-.a -.b) $(a +.a, b +.b) + $(a -.a, b -.b) + ?. ?=(@ b) & + (lth a b) +``` + +A deterministic fallback ordering: atoms before cells, then left-to-right comparison. Used when Tip5 hashes collide (astronomically unlikely but handled for correctness). + +### gor-tip: Primary Key Ordering + +```hoon +++ gor-tip + ~/ %gor-tip + |= [a=* b=*] + ^- ? + =+ [c=(tip a) d=(tip b)] + ?: =(c d) (dor-tip a b) + (lth-tip c d) +``` + +Compare by Tip5 hash; fall back to `dor-tip` on collision. This is the **BST property** — determines left vs right in the tree. + +### mor-tip: Heap Priority Ordering + +```hoon +++ mor-tip + ~/ %mor-tip + |= [a=* b=*] + ^- ? + =+ [c=(double-tip a) d=(double-tip b)] + ?: =(c d) (dor-tip a b) + (lth-tip c d) +``` + +Compare by double-Tip5 hash; fall back to `dor-tip` on collision. This is the **heap property** — determines parent vs child. + +Using two independent hash functions (Tip5 and double-Tip5) for BST ordering and heap priority ensures that the tree shape is deterministic and unique for any set of elements — a cryptographic treap. + +## Why Hash-Based Ordering Matters for Consensus + +In a decentralized system, every node must independently arrive at the same UTXO set representation. If tree layout depended on insertion order, different nodes processing the same transactions in different orders would produce different trees — same logical data, different nouns, different hashes. + +Tip5-based ordering guarantees: +1. **Deterministic layout**: Same elements → same tree → same noun → same hash +2. **No sorting needed**: Elements self-organize via their hashes during insertion +3. **Merkle-like properties**: The tree root hash effectively commits to the entire set's contents +4. **Efficient verification**: Two nodes can compare z-maps by comparing root hashes + +## Rust Jet Implementations + +The ordering functions and z-map/z-set operations are jetted in Rust for performance: + +| Hoon | Rust File | Key Functions | +|---|---|---| +| `tip`, `double-tip`, `gor-tip`, `mor-tip`, `dor-tip`, `lth-tip` | `crates/nockchain-math/src/zoon/common.rs` | `TipHasher` trait, `DefaultTipHasher` | +| `z-map put` | `crates/nockchain-math/src/zoon/zmap.rs` | `z_map_put()`, `z_map_rep()` | +| `z-set put/bif/dif` | `crates/nockchain-math/src/zoon/zset.rs` | `z_set_put()`, `z_set_bif()`, `z_set_dif()` | + +The `TipHasher` trait abstracts the hash function: + +```rust +pub trait TipHasher { + fn hash_noun_varlen( + &self, stack: &mut A, a: Noun, + ) -> Result<[u64; 5], JetErr>; + fn hash_ten_cell(&self, ten: [u64; 10]) -> Result<[u64; 5], JetErr>; +} +``` + +Note that `hash_ten_cell` takes a single `[u64; 10]` array (the two 5-element digests concatenated), not two separate `[u64; 5]` arrays. `DefaultTipHasher` uses Tip5, but the trait allows alternative hashers for testing. + +## Usage in the Transaction Engine + +Every collection type in the tx-engine is a z-map or z-set: + +| Collection | Type | Keys | Values | +|---|---|---|---| +| Spends | z-map | Name (UTXO ID) | Spend (proof) | +| Balance (UTXO set) | z-map | Name | Note | +| Seeds (outputs) | z-set | Seed | — | +| PkhSignature | z-map | Hash (pubkey hash) | PkhSignatureEntry | +| NoteData | z-map | @tas (string key) | * (arbitrary noun) | +| Hax (hash commitments) | z-set | Hash | — | +| Pkh hashes | z-set | Hash | — | +| note-data-by-lock-root | z-mip | Hash (lock root) | @tas → * | + +## Comparison with Other Blockchains + +| Aspect | Bitcoin | Cardano | Nockchain | +|---|---|---|---| +| UTXO set structure | LevelDB flat index | Haskell `Data.Map` | z-map (Tip5-ordered treap) | +| Ordering | None (hash-indexed) | Ord instance | Cryptographic (Tip5 hash) | +| Persistence | Copy-on-write DB | Persistent maps | Structural sharing (immutable trees) | +| Deterministic layout | N/A (DB internal) | Yes (Ord-based) | Yes (hash-based, unique treap) | +| Merkle commitment | Separate Merkle tree | Separate Merkle root | Implicit in tree root hash | +| Collection operations | DB queries | Standard Haskell | z-by/z-in engines (30+ arms each) | + +The key differentiator: Nockchain's UTXO set is itself a cryptographically-ordered tree, not a separate database with a Merkle root bolt-on. The tree structure *is* the commitment. diff --git a/docs/architecture/tx-engine/11-tip5-hash-function.md b/docs/architecture/tx-engine/11-tip5-hash-function.md new file mode 100644 index 000000000..5266c0f23 --- /dev/null +++ b/docs/architecture/tx-engine/11-tip5-hash-function.md @@ -0,0 +1,222 @@ +# Tip5: The Sponge Hash Function + +## Overview + +Tip5 is the cryptographic hash function used throughout Nockchain for commitments, Merkle trees, data structure ordering, and transaction IDs. It belongs to the family of **algebraic sponge hashes** (alongside Poseidon, Neptune, and Rescue) — hash functions designed to be efficient both natively and inside zero-knowledge proof circuits. + +The authoritative implementation is in Hoon (`hoon/common/ztd/three.hoon`), with a Rust jet in `crates/nockchain-math/src/tip5/`. + +## Design Parameters + +| Parameter | Value | Meaning | +|---|---|---| +| Field | Goldilocks, p = 2^64 − 2^32 + 1 | Each state element is a 64-bit field element | +| State size | 16 elements | Total permutation width | +| Capacity | 6 elements | Security margin (never directly exposed) | +| Rate | 10 elements | Input absorbed per permutation | +| Rounds | 7 | Number of permutation rounds | +| Digest length | 5 elements (320 bits) | Output size | +| S-box (first 4) | Lookup table (Fermat map) | x → x^(p−2) via 256-byte table | +| S-box (last 12) | 7th power map | x → x^7 in 4 multiplications | + +## Sponge Construction + +Tip5 uses the standard sponge construction: + +``` +Input: [a₀, a₁, ..., aₙ] + +1. Pad input: append [1, 0, 0, ...] to reach a multiple of RATE (10) +2. Convert to Montgomery form (montify each element) +3. Initialize sponge state (16 elements) +4. For each RATE-sized chunk: + a. XOR chunk into the rate portion of the state + b. Apply permutation (7 rounds) +5. Extract first 5 elements as digest +6. Convert back from Montgomery form +``` + +Two initialization modes exist: +- **Variable-length** (`hash-varlen`): for arbitrary-length inputs +- **Fixed-length** (`hash-10`): optimized for exactly 10 elements (one rate block) + +## Permutation Round Structure + +Each of the 7 rounds applies three layers: + +### 1. S-box Layer + +The S-box provides non-linearity. It applies two different maps depending on position: + +```hoon +++ sbox-layer + ~/ %sbox-layer + |= =state + ?> =((lent state) state-size) + %+ weld + (turn (scag num-split-and-lookup state) split-and-lookup) :: first 4 + %+ turn (slag num-split-and-lookup state) :: last 12 + |= m=melt + =/ sq (bmul m m) :: x² + =/ qu (bmul sq sq) :: x⁴ + :(bmul m sq qu) :: x⁷ = x · x² · x⁴ +``` + +- **First 4 elements**: `split-and-lookup` — decomposes the element into bytes, applies a 256-byte lookup table (implementing the Fermat map x → x^(p−2) ≡ x^(−1)), then recombines +- **Last 12 elements**: 7th power map — computes x^7 using 4 field multiplications (x → x² → x⁴ → x·x²·x⁴) + +The hybrid S-box design is a performance optimization: the lookup-table-based Fermat map is faster for the first few elements, while the power map avoids the table dependency for the remaining elements. + +### 2. MDS Layer + +Linear mixing via a 16×16 circulant MDS (Maximum Distance Separable) matrix: + +```hoon +++ mds-cyclomul-m + ~/ %mds-cyclomul-m + |= v=(list @) + ^- (list @) + %+ turn mds-matrix + |= row=(list @) + (mod (inner-product row v) p) +``` + +The MDS matrix ensures maximum diffusion — every output element depends on every input element. The circulant structure allows optimization via the inner product computation. + +### 3. Round Constant Addition + +Each round adds a unique set of 16 precomputed constants: + +```hoon +++ round + ~/ %round + |= [sponge=tip5-state round-index=@] + =. sponge (mds-cyclomul-m (sbox-layer sponge)) + %^ zip sponge (range state-size) + |= [b=belt i=@] + (badd b (snag (add (mul round-index state-size) i) round-constants)) +``` + +Total round constants: 7 rounds × 16 elements = 112 precomputed Goldilocks field elements. + +## Montgomery Representation + +All permutation arithmetic operates in **Montgomery space** for efficiency: + +- `melt`: a field element in Montgomery form (multiplied by R = 2^64 mod p) +- `montify`: convert belt → melt (multiply by R mod p) +- `mont-reduction`: convert melt → belt (multiply by R^(-1) mod p) +- `montiply`: Montgomery multiplication — `a * b * R^(-1) mod p` in a single operation + +Montgomery multiplication avoids expensive division-based modular reduction by using a shift-based reduction, roughly 4× faster for repeated multiplications. + +## Hash Variants + +### hash-10: Fixed 10-Element Input + +```hoon +++ hash-10 + ~/ %hash-10 + |= input=(list belt) + ^- (list belt) + ?> =((lent input) rate) + ?> (levy input based) + =. input (turn input montify) + =/ sponge (init-tip5-state %fixed) + =. sponge (permutation (weld input (slag rate sponge))) + (turn (scag digest-length sponge) mont-reduction) +``` + +Optimized path for exactly 10 elements. Used by `double-tip` in zoon (hashes two 5-element digests together). + +### hash-varlen: Variable-Length Input + +Pads input, absorbs in rate-sized chunks, returns 5-element digest. Used for hashing arbitrary-length data. + +### hash-noun-varlen: Noun Hashing + +Hashes an arbitrary Nock noun by extracting its **leaf sequence** (atoms at the leaves of the binary tree) and **Dyck word** (the tree structure encoding), then feeding both through the sponge. This ensures that structurally different nouns produce different hashes even if they contain the same atoms. + +### hash-hashable: Commitment Hashing + +Hashes a tagged `hashable` tree structure. The `hashable` type is: + +```hoon ++$ hashable + $% [%leaf p=@] :: raw value + [%hash p=noun-digest] :: pre-computed hash + [%list p=(list hashable)] :: list of hashables + == +``` + +Transaction engine types define `++hashable` arms that build these trees, which `hash-hashable` then traverses to produce deterministic commitments. For example, the lock-merkle-proof-full hashable: + +```hoon +:* leaf+version.form + hash+(hash:spend-condition spend-condition.form) + leaf+axis.form + (hashable-merk-proof merk-proof.form) +== +``` + +### Tog: Deterministic PRNG + +The `tog` interface provides deterministic pseudorandom generation from a sponge state: + +- `belts`: generate N random belt (field element) values +- `felt`: generate a random felt (extension field element) +- `indices`: generate random indices for spot checks + +Used in the STARK proof system for Fiat-Shamir challenges. + +## Rust Jet Implementation + +The Rust jet mirrors the Hoon implementation exactly: + +**Constants** (`crates/nockchain-math/src/tip5/mod.rs`): +- `LOOKUP_TABLE`: 256-byte S-box for the Fermat map +- `ROUND_CONSTANTS`: 112 `u64` values (7 rounds × 16 state elements) +- `MDS_MATRIX_MONT`: 16×16 matrix in Montgomery representation + +**Functions** (`crates/nockchain-math/src/tip5/hash.rs`): +- `permute(sponge: &mut [u64; 16])`: applies 7 rounds in-place +- `hash_varlen(input: &[u64]) -> [u64; 5]`: variable-length hash +- `hash_10(input: &[u64; 10]) -> [u64; 5]`: fixed 10-element hash +- `hash_noun_varlen(noun: &Noun) -> [u64; 5]`: noun hash + +## Usage in the Transaction Engine + +Tip5 is used at every layer of the tx-engine: + +| Usage | Function | Input | +|---|---|---| +| Transaction ID | `hash-hashable` | Hashable tree of spends | +| Lock root | `hash:lock` | Lock tree structure | +| Note Name.first | `hash:lock` | Lock root of the spending conditions | +| Note Name.last | derived | Source hash (parent transaction) | +| Merkle proof siblings | `hash-ten-cell` | Pair of sibling hashes | +| z-map/z-set ordering | `hash-noun-varlen` | Element nouns (via `tip` in zoon) | +| z-map/z-set priority | `hash-ten-cell` + `hash-noun-varlen` | Double-hash (via `double-tip`) | +| PKH commitment | `hash` | Public key → public key hash | +| Hax commitment | `hash` | Preimage → commitment hash | +| Block commitment | `hash-hashable` | Page/block contents | + +## Comparison with Other Blockchain Hash Functions + +| Hash | Blockchain | Field | Design | ZK-Friendly | +|---|---|---|---|---| +| SHA-256d | Bitcoin | Binary (256-bit) | Merkle-Damgård | No | +| Keccak-256 | Ethereum | Binary (256-bit) | Sponge (binary) | No | +| Poseidon | Mina, Filecoin | Various prime fields | Algebraic sponge | Yes | +| Rescue-Prime | Various | Various prime fields | Algebraic sponge | Yes | +| Tip5 | Nockchain | Goldilocks (64-bit) | Algebraic sponge | Yes | + +### Why Tip5 Matters for STARK Compatibility + +Traditional hash functions like SHA-256 and Keccak operate on bits/bytes. Verifying them inside a STARK circuit requires encoding binary operations as algebraic constraints — extremely expensive (thousands of constraints per hash). + +Tip5 operates natively on Goldilocks field elements — the same field used for STARK arithmetic. This means: +- Hashing inside a STARK circuit requires only ~7 × 16 = 112 field operations per permutation +- No binary-to-field conversion overhead +- The S-box, MDS matrix, and round constants all operate in the native field +- Merkle proof verification in-circuit is practical, enabling efficient recursive proofs diff --git a/docs/architecture/tx-engine/12-schnorr-signatures-cheetah-curve.md b/docs/architecture/tx-engine/12-schnorr-signatures-cheetah-curve.md new file mode 100644 index 000000000..19ee3fc83 --- /dev/null +++ b/docs/architecture/tx-engine/12-schnorr-signatures-cheetah-curve.md @@ -0,0 +1,330 @@ +# Schnorr Signatures over the Cheetah Curve + +## Overview + +Nockchain uses **Schnorr signatures** over the **Cheetah elliptic curve** — a STARK-friendly curve defined over a sextic extension of the Goldilocks prime field. The Cheetah curve was designed by Toposware specifically for efficient verification both natively and inside zero-knowledge proof circuits. + +The Rust implementation is in `crates/nockchain-math/src/crypto/cheetah.rs` (~455 lines). The Hoon types are authoritative in `tx-engine-1.hoon`, with Rust serialization mirrors in `crates/nockchain-types/src/tx_engine/`. + +## Why Not secp256k1 or Ed25519? + +Bitcoin uses secp256k1 (ECDSA, later Schnorr via Taproot). Ethereum and many other chains use Ed25519 or secp256k1. These curves operate over 256-bit prime fields that have no special relationship to Nockchain's arithmetic. + +Nockchain's entire proof system operates over the **Goldilocks field** (p = 2^64 − 2^32 + 1). For STARK compatibility, all cryptographic operations — including signature verification — must be efficiently expressible as constraints over this field. A standard 256-bit curve would require multi-limb arithmetic inside the STARK circuit, making signature verification prohibitively expensive. + +The Cheetah curve solves this: its base field *is* Goldilocks, and its extension field arithmetic reduces to `u64` operations over the Goldilocks prime. This means Schnorr signature verification inside a STARK circuit requires only native field operations. + +## Cheetah Curve Specification + +The Cheetah curve comes from Toposware's research on elliptic curves over sextic extensions of small prime fields (ePrint 2022/277). + +### Field Tower + +| Layer | Definition | Meaning | +|---|---|---| +| Base field Fp | p = 2^64 − 2^32 + 1 | Goldilocks prime, fits in one `u64` | +| Extension Fp^6 | Fp[X]/(X^6 − 7) | Degree-6 extension, implemented as `F6lt = [Belt; 6]` | + +The extension field is constructed as Fp^6 = Fp[u] where u^6 = 7. Each element is a 6-tuple of Goldilocks field elements. + +### Curve Parameters + +| Parameter | Value | +|---|---| +| Curve equation | E: y² = x³ + x + b, where b = 395 + u (see `++b` in `ztd/three.hoon:1472`) | +| Prime field | Fp, p = 2^64 − 2^32 + 1 (Goldilocks) | +| Coordinate field | Fp^6 = Fp[u]/(u^6 − 7), each point's (x, y) lives here | +| Group order | `0x7af2599b3b3f22d0563fbf0f990a37b5327aa72330157722d443623eaed4accf` (~255 bits) | +| Scalar field | Z/nZ where n = group order (~255 bits) | +| Security level | ~128 bits (resistant to Pollard-Rho, twist, MOV, cover, decomposition attacks) | +| Generator | `a-gen` / `A_GEN` (predefined constant in `ztd/three.hoon:1535` / `cheetah.rs:22`) | +| Identity | `a-id` / `A_ID` (point at infinity: `[f6-zero f6-one %.y]`) | + +### Why Sextic Extension? + +The degree-6 extension is a carefully chosen balance: + +1. **64-bit base field**: Each base element fits in a single machine word → fast native arithmetic +2. **Large group order**: The ~255-bit subgroup provides ~128 bits of security (comparable to secp256k1) +3. **Attack resistance**: Degree 6 is small enough to resist cover and decomposition attacks specific to extension-field curves, while large enough for security +4. **STARK efficiency**: All field operations reduce to `u64` arithmetic over Goldilocks → efficient as STARK AIR constraints + +## Point Representation + +```rust +// crates/nockchain-math/src/crypto/cheetah.rs:60-65 +pub struct CheetahPoint { + pub x: F6lt, // x-coordinate in Fp^6 + pub y: F6lt, // y-coordinate in Fp^6 + pub inf: bool, // point at infinity flag +} + +pub struct F6lt(pub [Belt; 6]); // 6 Goldilocks field elements +``` + +A point on the Cheetah curve is represented by its (x, y) coordinates in the sextic extension field, plus an infinity flag. Each coordinate is 6 × 64 = 384 bits, so a full point is 768 bits + flag. + +## Extension Field Arithmetic + +The `F6lt` type supports full field arithmetic, implemented via Karatsuba-style multiplication: + +| Operation | Function | Notes | +|---|---|---| +| Addition | `f6_add(a, b)` | Component-wise addition mod p | +| Negation | `f6_neg(a)` | Component-wise negation mod p | +| Subtraction | `f6_sub(a, b)` | `f6_add(a, f6_neg(b))` | +| Multiplication | `f6_mul(a, b)` | Karatsuba via `karat3` — reduces to three degree-3 multiplications | +| Squaring | `f6_square(a)` | Currently `f6_mul(a, a)` (TODO: dedicated Karatsuba-square) | +| Inversion | `f6_inv(a)` | Extended GCD over polynomial ring Fp[X]/(X^6 − 7) | +| Division | `f6_div(a, b)` | `f6_mul(a, f6_inv(b))` | +| Scalar mult | `f6_scal(s, a)` | Multiply each component by base field element | + +The Karatsuba multiplication (`karat3`) is the key optimization: it multiplies two degree-2 polynomials using 3 component multiplications instead of the naive 9, then composes two `karat3` calls to handle the full degree-5 multiplication with reduction by X^6 − 7. + +The reduction by the irreducible polynomial X^6 − 7 appears in `f6_mul` as additions of `Belt(7) * (...)` terms — when a product exceeds degree 5, the X^6 term is replaced by 7 (since X^6 ≡ 7 mod (X^6 − 7)). + +## Point Arithmetic + +| Operation | Function | Notes | +|---|---|---| +| Point addition | `ch_add(p, q)` | Handles identity, negation, and doubling cases | +| Point doubling | `ch_double(p)` | Optimized doubling using tangent slope | +| Point negation | `ch_neg(p)` | Negate y-coordinate | +| Scalar mult (u64) | `ch_scal(n, p)` | Binary method for 64-bit scalars | +| Scalar mult (big) | `ch_scal_big(n, p)` | Binary method for arbitrary-size integers (UBig) | + +The addition formula for non-special cases: + +```rust +// crates/nockchain-math/src/crypto/cheetah.rs:262-271 +pub fn ch_add_unsafe(p: CheetahPoint, q: CheetahPoint) -> Result { + let slope = f6_div(&f6_sub(&p.y, &q.y), &f6_sub(&p.x, &q.x))?; + let x_out = f6_sub(&f6_square(&slope), &f6_add(&p.x, &q.x)); + let y_out = f6_sub(&f6_mul(&slope, &f6_sub(&p.x, &x_out)), &p.y); + Ok(CheetahPoint { x: x_out, y: y_out, inf: false }) +} +``` + +The doubling formula uses the curve equation's derivative (3x² + a where a = 1 for Cheetah): + +```rust +// crates/nockchain-math/src/crypto/cheetah.rs:228-240 +pub fn ch_double_unsafe(x: &F6lt, y: &F6lt) -> Result { + let slope = f6_div( + &f6_add(&f6_scal(Belt(3), &f6_square(x)), &F6_ONE), // 3x² + 1 + &f6_scal(Belt(2), y), // 2y + )?; + let x_out = f6_sub(&f6_square(&slope), &f6_scal(Belt(2), x)); + let y_out = f6_sub(&f6_mul(&slope, &f6_sub(x, &x_out)), y); + Ok(CheetahPoint { x: x_out, y: y_out, inf: false }) +} +``` + +## Curve Validation + +```rust +// crates/nockchain-math/src/crypto/cheetah.rs:114-121 +pub fn in_curve(&self) -> bool { + if *self == A_ID { return true; } + let scaled = ch_scal_big(&G_ORDER, self) + .expect("scalar multiplication should succeed"); + scaled == A_ID // [n]G = ∞ iff G has order dividing n +} +``` + +Validation checks that `[G_ORDER]P = ∞` — the point has order dividing the group order. This is sufficient because the subgroup has prime order, so any non-identity point with this property is a valid group element. + +## Schnorr Signature Scheme (Authoritative Hoon) + +The Schnorr scheme is defined in `hoon/common/ztd/three.hoon` within the `++schnorr` core of the `++cheetah` library. + +### Signing (`++sign`, `three.hoon:1628-1661`) + +```hoon +++ sign + |= [sk-as-32-bit-belts=(list belt) m=noun-digest:tip5] + ^- [c=@ux s=@ux] +``` + +1. Derive public key: `pubkey = [sk]G` +2. Compute nonce deterministically: `nonce = trunc-g-order(hash-varlen(pubkey_x || pubkey_y || m || sk_belts))` +3. Compute commitment point: `R = [nonce]G` +4. Compute challenge: `chal = trunc-g-order(hash-varlen(R_x || R_y || pubkey_x || pubkey_y || m))` +5. Compute response: `sig = (nonce + chal · sk) mod g-order` +6. Return `[chal, sig]` + +The nonce is derived deterministically from the secret key and message (like RFC 6979), avoiding the need for a random number generator. + +### Verification (`++verify`, `three.hoon:1663-1686`) + +```hoon +++ verify + |= [pubkey=a-pt:curve m=noun-digest:tip5 chal=@ux sig=@ux] + ^- ? +``` + +1. Check `0 < chal < g-order` and `0 < sig < g-order` +2. Recover commitment: `R = [sig]G − [chal]pubkey` +3. Recompute challenge: `chal' = trunc-g-order(hash-varlen(R_x || R_y || pubkey_x || pubkey_y || m))` +4. Accept if `chal == chal'` + +This is a standard Schnorr verification: if `sig = nonce + chal·sk`, then `[sig]G − [chal]pubkey = [nonce]G + [chal·sk]G − [chal·sk]G = [nonce]G = R`, recovering the original commitment point. + +### `trunc-g-order`: Hash-to-Scalar (`three.hoon:1695-1706`) + +```hoon +++ trunc-g-order + |= a=(list belt) + (mod (add (snag 0 a) (mul p (snag 1 a)) ...) g-order:curve) +``` + +Converts a Tip5 hash output (list of belts) into a scalar in `[0, g-order)` by interpreting the first 4 elements as a base-p number and reducing modulo the group order. This is the bridge between the hash function's Goldilocks output and the elliptic curve's scalar field. + +### Batch Verification + +```hoon +++ batch-verify + |= batch=(list [pubkey=a-pt:curve m=noun-digest:tip5 chal=@ux sig=@ux]) + (levy batch verify) +``` + +Currently verifies each signature independently. Schnorr's linearity enables future optimization via randomized batch verification. + +## Schnorr Signature Types (Rust Serialization Mirror) + +### SchnorrPubkey + +```rust +// crates/nockchain-types/src/tx_engine/common/mod.rs:15-16 +pub struct SchnorrPubkey(pub CheetahPoint); +``` + +A public key is a point on the Cheetah curve. It derives `NounEncode`/`NounDecode` automatically from `CheetahPoint`. + +### SchnorrSignature + +```rust +// crates/nockchain-types/src/tx_engine/common/mod.rs:30-34 +pub struct SchnorrSignature { + pub chal: [Belt; 8], // Challenge scalar as 8 base field elements + pub sig: [Belt; 8], // Response scalar as 8 base field elements +} +``` + +In Hoon, challenge and response are raw `@ux` atoms. The Rust serialization represents each as 8 Goldilocks field elements (512 bits), which accommodates the ~255-bit group order. The `[Belt; 8]` encoding matches how Hoon nouns serialize large integers as sequences of 64-bit words. + +### PkhSignatureEntry + +```rust +// crates/nockchain-types/src/tx_engine/v1/tx.rs:296-301 +pub struct PkhSignatureEntry { + pub hash: Hash, // The pubkey hash being satisfied + pub pubkey: SchnorrPubkey, // The actual public key + pub signature: SchnorrSignature, // Schnorr signature +} +``` + +When satisfying a `Pkh` lock primitive, the witness provides a `PkhSignatureEntry` for each signing key. The entry binds together: +1. The pubkey hash (committed in the lock) +2. The actual public key (revealed at spend time) +3. The Schnorr signature over the transaction's sig-hash + +### Sig-Hash: What Gets Signed + +The message signed by a Schnorr signature is the **sig-hash** of the transaction — a hash of the seeds (outputs) and fee, but *excluding* the witness data. This is the same SegWit-inspired design described in file 02: the witness does not sign itself, enabling witness malleability prevention. + +## Key Encoding + +Public keys use Base58 serialization: + +```rust +// crates/nockchain-math/src/crypto/cheetah.rs:67-81 +const BYTES: usize = 97; // 1 prefix + 12 Belt elements × 8 bytes each + +pub fn into_base58(&self) -> Result { + let mut bytes = Vec::new(); + bytes.push(0x1); // Prefix byte + for belt in self.y.0.iter().rev().chain(self.x.0.iter().rev()) { + bytes.extend_from_slice(&belt.0.to_be_bytes()); + } + Ok(bs58::encode(bytes).into_string()) +} +``` + +The encoding is: `[0x01 | y₅..y₀ | x₅..x₀]` in big-endian, producing a 97-byte value that encodes to a ~132-character Base58 string. The leading `0x01` byte serves as a version/format prefix. + +Decoding includes `in_curve()` validation to reject points not on the curve. + +## Signature Collection + +The `Signature` type in the witness is a z-map (hash-ordered persistent tree) of `(SchnorrPubkey → SchnorrSignature)` pairs: + +```rust +// crates/nockchain-types/src/tx_engine/common/mod.rs:36-48 +pub struct Signature(pub Vec<(SchnorrPubkey, SchnorrSignature)>); + +impl NounEncode for Signature { + fn to_noun(&self, stack: &mut A) -> Noun { + self.0.iter().fold(D(0), |map, (pubkey, sig)| { + let mut key = pubkey.to_noun(stack); + let mut value = sig.to_noun(stack); + zmap::z_map_put(stack, &map, &mut key, &mut value, &DefaultTipHasher) + .expect("z-map put for signature should not fail") + }) + } +} +``` + +Signatures are stored in a z-map keyed by public key, ensuring deterministic ordering regardless of the order signatures were collected. This is critical for consensus — the noun representation of the witness must be identical across all nodes. + +## The `trunc_g_order` Function + +```rust +// crates/nockchain-math/src/crypto/cheetah.rs:332-339 +pub fn trunc_g_order(a: &[u64]) -> UBig { + let mut result = UBig::from(a[0]); + result += &*P_BIG * UBig::from(a[1]); + result += &*P_BIG_2 * UBig::from(a[2]); + result += &*P_BIG_3 * UBig::from(a[3]); + result % &*G_ORDER +} +``` + +This converts a 4-element array of `u64` values into a scalar modulo the group order, interpreting the array as a base-p number: `a[0] + a[1]·p + a[2]·p² + a[3]·p³ mod G_ORDER`. This is used to convert Tip5 hash outputs (which are in the Goldilocks field) into scalars suitable for elliptic curve operations. + +## Also Available: Ed25519 Jets + +The Nock VM also includes Ed25519 jets for non-consensus cryptographic operations: + +| Jet | Location | Purpose | +|---|---|---| +| `jet_sign` | `crates/nockvm/rust/nockvm/src/jets/lock/ed.rs` | Ed25519 signing | +| `jet_veri` | same | Ed25519 verification | +| `jet_puck` | same | Public key derivation | +| `jet_shar` | same | Shared secret (X25519) | + +These are available in the VM for application-level cryptography (e.g., key derivation, off-chain signatures) but are not used in the consensus-critical transaction engine. The transaction engine exclusively uses Schnorr over Cheetah for on-chain authentication. + +Additionally, `crates/nockchain-math/src/crypto/argon2.rs` provides Argon2 key derivation — used for wallet password hashing, not consensus operations. + +## Comparison with Bitcoin's Signature Evolution + +| Aspect | Bitcoin (pre-Taproot) | Bitcoin (Taproot) | Nockchain | +|---|---|---|---| +| Signature scheme | ECDSA | Schnorr | Schnorr | +| Curve | secp256k1 (256-bit) | secp256k1 (256-bit) | Cheetah (Fp^6 over Goldilocks) | +| Key size | 33 bytes (compressed) | 32 bytes (x-only) | 97 bytes (full point) | +| Signature size | ~72 bytes (DER) | 64 bytes | 128 bytes (8+8 Belt elements) | +| Multisig | OP_CHECKMULTISIG (N pubkeys + N sigs) | MuSig2 (single aggregated key) | Native M-of-N via Pkh primitive | +| ZK-friendly | No | No | Yes (native field arithmetic in STARKs) | +| Batch verification | No | Yes (Schnorr linearity) | Possible (Schnorr linearity) | + +### Key Design Differences + +1. **Larger keys and signatures**: Cheetah operates over Fp^6, so points and scalars are larger than secp256k1. The trade-off is STARK-circuit efficiency. + +2. **No x-only pubkeys**: Bitcoin Taproot uses x-only public keys (32 bytes) where the y-coordinate is implicitly even. Nockchain encodes the full (x, y) point, avoiding the parity ambiguity at the cost of larger keys. + +3. **Hash-based key commitment**: Like Bitcoin P2PKH but unlike Taproot's key-path, Nockchain's Pkh stores `Hash(pubkey)` rather than the pubkey itself. The actual key is revealed only at spend time. + +4. **Schnorr from inception**: Bitcoin migrated from ECDSA to Schnorr over a decade. Nockchain chose Schnorr from V1 inception, benefiting from linearity (enabling potential future MuSig-style aggregation) without legacy compatibility concerns. diff --git a/docs/architecture/tx-engine/13-merkle-trees-and-commitments.md b/docs/architecture/tx-engine/13-merkle-trees-and-commitments.md new file mode 100644 index 000000000..9831bd98e --- /dev/null +++ b/docs/architecture/tx-engine/13-merkle-trees-and-commitments.md @@ -0,0 +1,306 @@ +# Merkle Trees and Commitment Schemes + +## Overview + +Nockchain uses Tip5-based Merkle trees for two distinct purposes: (1) **lock trees** — Taproot-style MAST structures that define spending conditions, and (2) **hashable commitment trees** — deterministic hash commitments over structured data (transactions, proofs, blocks). Both share the same underlying Merkle primitives from `hoon/common/ztd/three.hoon`. + +## Merkle Tree Types + +The authoritative Hoon types from `ztd/three.hoon`: + +### merk: Tagged Merkle Tree + +```hoon +++ merk + |$ [node] + $~ [%leaf *noun-digest:tip5 ~] + $% [%leaf h=noun-digest:tip5 ~] + [%tree h=noun-digest:tip5 t=(pair (merk node) (merk node))] + == +``` + +A `merk` is a tagged union: +- `[%leaf h ~]`: a leaf node containing only a hash digest +- `[%tree h [left right]]`: an internal node containing a hash and two children + +Every node carries its hash `h`, which is either the hash of the leaf data or `hash-ten-cell(h.left, h.right)` for internal nodes. + +### merk-proof: Inclusion Proof + +```hoon ++$ merk-proof [root=noun-digest:tip5 path=(list noun-digest:tip5)] +``` + +A Merkle proof consists of: +- **root**: the expected root hash (must match the committed root) +- **path**: sibling hashes from the leaf up to the root, one per level + +### merk-heap: Flat Array Representation + +```hoon ++$ merk-heap [h=noun-digest:tip5 m=mary] +``` + +A heap-layout Merkle tree stored as a flat `mary` (fixed-stride array). The root hash `h` is stored separately. This representation is used for FRI polynomial commitment Merkle trees where random-access to sibling nodes is needed. + +### mee: Untagged Binary Tree + +```hoon +++ mee + |$ [node] + $~ [%leaf *node] + $% [%leaf n=node] + [%tree l=(mee node) r=(mee node)] + == +``` + +A simpler binary tree without hash annotations, used as an intermediate structure during tree construction (`list-to-balanced-tree`). + +## Tree Construction + +### list-to-balanced-tree + +Converts a flat `mary` (array) into a balanced binary `mee` tree: + +```hoon +++ list-to-balanced-tree + |= lis=mary + ^- [h=@ t=(mee mary)] + :- (xeb len.array.lis) :: height = ceil(log2(len)) + |- + ?> !=(0 len.array.lis) + =/ len len.array.lis + ?: =(1 len) [%leaf ...] + ?: =(2 len) [%tree [%leaf left] [%leaf right]] + :: Split: left gets ceil(len/2), right gets floor(len/2) + [%tree $(lis left-half) $(lis right-half)] +``` + +For odd-length lists, the left subtree gets one extra element, producing a left-heavy balanced tree. + +### build-merk + +Converts a `mary` into a full `merk` with hash annotations: + +```hoon +++ build-merk + |= m=mary + ^- (pair @ (merk mary)) + =/ [h=@ n=(mee mary)] (list-to-balanced-tree m) + :- h + |- + ?: ?=([%leaf *] n) + [%leaf (hash-hashable:tip5 (hashable-mary:tip5 n.n)) ~] + =/ l=(merk mary) $(n l.n) + =/ r=(merk mary) $(n r.n) + [%tree (hash-ten-cell:tip5 h.l h.r) l r] +``` + +1. Build a balanced binary tree from the input array +2. Hash each leaf via `hash-hashable:tip5` +3. Hash each internal node as `hash-ten-cell(left.hash, right.hash)` +4. Return the tree height and annotated Merkle tree + +### build-merk-heap + +Builds a heap-layout Merkle tree (flat array) for efficient random-access proof generation: + +```hoon +++ build-merk-heap + |= m=mary + ^- [depth=@ heap=merk-heap] +``` + +The heap layout stores all nodes in a contiguous array where index 0 is the root, and children of index `i` are at `2i+1` and `2i+2`. This layout is used by the FRI polynomial commitment scheme for efficient Merkle proof construction during STARK proof generation. + +## Proof Generation + +### prove-hashable-by-index + +Generates an inclusion proof for a specific leaf in a hashable tree: + +```hoon +++ prove-hashable-by-index + |= [h=hashable:tip5 idx=@] + ^- [axis=@ proof=merk-proof] +``` + +The algorithm: +1. Count leaves in the left and right subtrees +2. Recurse into the subtree containing the target index +3. At each level, record the sibling's hash in the proof path +4. Convert the leaf position to a Nock axis using `peg` +5. Return both the axis (for the lock Merkle proof) and the proof path + +### build-merk-proof + +Generates a proof from a heap-layout Merkle tree: + +```hoon +++ build-merk-proof + |= [merk=merk-heap axis=@] + ^- merk-proof + :- h.merk :: root hash + |- :: walk from leaf to root collecting siblings + ?: =(0 axis) ~ + =/ parent (div (dec axis) 2) + =/ sibling ?:((mod axis 2) (add axis 1) (sub axis 1)) + [(snag-as-digest:tip5 m.merk sibling) $(axis parent)] +``` + +Starting from the leaf's heap index, it walks up to the root, collecting the sibling hash at each level. The axis-to-heap-index conversion uses `(dec axis)`. + +### verify-merk-proof + +Verifies a Merkle inclusion proof: + +```hoon +++ verify-merk-proof + |= [leaf=noun-digest:tip5 axis=@ merk-proof] + ^- ? + ?: =(1 axis) &(=(root leaf) ?=(~ path)) + ?: =(2 axis) &(=(root (hash-ten-cell:tip5 leaf sib)) ?=(~ t.path)) + ?: =(3 axis) &(=(root (hash-ten-cell:tip5 sib leaf)) ?=(~ t.path)) + :: General case: reconstruct hashes from leaf to root + ?: =((mod axis 2) 0) + $(axis (div axis 2), leaf (hash-ten-cell:tip5 leaf sib), path t.path) + $(axis (div (dec axis) 2), leaf (hash-ten-cell:tip5 sib leaf), path t.path) +``` + +The axis determines sibling ordering at each level: +- **Even axis** (left child): hash as `(leaf, sibling)` +- **Odd axis** (right child): hash as `(sibling, leaf)` + +This uses Nock's binary tree addressing where left children have even axes and right children have odd axes. + +### index-to-axis + +Maps a leaf index (0-based) to a Nock tree axis: + +```hoon +++ index-to-axis + |= [h=@ i=@] + ^- axis + =/ min (bex (dec h)) :: 2^(height-1) + (add min i) +``` + +In a balanced tree of height `h`, the leftmost leaf has axis `2^(h-1)` and leaf `i` has axis `2^(h-1) + i`. + +## Hashable Commitment Construction + +The `hashable` type is the central abstraction for computing deterministic hash commitments over structured data. + +### The Hashable Type + +From `ztd/three.hoon`: + +```hoon ++$ hashable + $% [%leaf p=@] :: raw value + [%hash p=noun-digest] :: pre-computed hash + [%list p=(list hashable)] :: list of hashables + == +``` + +A `hashable` is a tagged tree: +- `leaf+value`: a raw atom to be hashed +- `hash+digest`: an already-computed 5-element Tip5 digest (avoids re-hashing) +- `list+[...]`: a list of sub-hashables, converted to a balanced binary tree for hashing + +### hash-hashable: The Universal Commitment Function + +`hash-hashable:tip5` traverses a hashable tree, producing a single 5-element Tip5 digest: + +- **leaf**: hash the atom via `hash-varlen` +- **hash**: return the pre-computed digest directly +- **list**: convert to a balanced binary tree, hash leaves, then hash pairs up to the root via `hash-ten-cell` + +This provides a **compositional** hashing interface: complex data structures define their `++hashable` arms that construct hashable trees, and `hash-hashable` traverses them uniformly. + +### Transaction Engine Hashable Arms + +Each tx-engine type defines a `++hashable` arm that builds its commitment structure: + +**Lock Merkle Proof (Full format, post-Bythos):** + +```hoon +:* leaf+version.form + hash+(hash:spend-condition spend-condition.form) + leaf+axis.form + (hashable-merk-proof merk-proof.form) +== +``` + +This commits to: the proof format version, the spend condition hash, the axis (which branch), and the Merkle path. + +**Lock Merkle Proof (Stub format, pre-Bythos):** + +```hoon +:+ hash+(hash:spend-condition spend-condition.form) + hash+(from-b58:^hash '6mhCSwJQDvbkbiPAUNjetJtVoo1VLtEhmEYoU4hmdGd6ep1F6ayaV4A') + (hashable-merk-proof merk-proof.form) +``` + +The stub format replaces the axis with a **hardcoded hash** — a static placeholder. This means the witness hash does not commit to which branch is being executed, a weakness fixed by the full format in Bythos. + +**Spend Condition:** + +The spend condition hashable includes all lock primitives in the condition list, each contributing their parameters as leaves and hashes. + +**Transaction (Spends):** + +The transaction ID is computed as `hash-hashable` of the entire spends structure — the map of all inputs being consumed. + +## Usage in the Transaction Engine + +### Lock Tree (MAST) + +The lock tree is the primary use of Merkle trees in the tx-engine: + +1. **Construction**: A lock is built as a Merkle tree of spend conditions. Each leaf is a `SpendCondition` (a list of lock primitives). The root hash becomes `Name.first` — the first component of the UTXO identifier. + +2. **Proof**: When spending, the `LockMerkleProof` reveals one leaf (the spend condition being exercised) plus the sibling hashes from leaf to root. + +3. **Verification**: The validator hashes the revealed spend condition, combines with siblings using the axis to determine ordering, and checks the result equals `Name.first`. + +### Transaction IDs + +The `TxId` (alias for `Hash`) is computed as `hash-hashable` of the transaction's spends structure, providing a unique, deterministic identifier for each transaction. + +### Note Names + +A Note's `Name` is `[first=Hash, last=Hash, null=0]` where: +- `first` = hash of the lock tree root (the spending conditions) +- `last` = hash of the source (parent transaction or coinbase) + +Both components use Tip5 hashing via the hashable commitment machinery. + +### Block Commitments + +Pages (blocks) commit to their contents via Merkle roots over the included transactions, using the same `build-merk` infrastructure. + +### FRI Polynomial Commitments + +The STARK proof system uses `build-merk-heap` to commit to polynomial evaluations. The heap layout enables efficient Merkle proof construction for the spot-check queries in the FRI protocol. + +## Comparison with Other Blockchain Merkle Trees + +| Aspect | Bitcoin | Bitcoin Taproot | Nockchain | +|---|---|---|---| +| Hash function | SHA-256d | SHA-256 (tagged) | Tip5 | +| Transaction Merkle | Binary tree of txids | Same | Hashable tree of spends | +| Script tree | N/A | MAST of TapScript leaves | Lock tree of SpendConditions | +| Proof structure | txid + siblings | Control block + script + path | Axis + spend condition + path | +| UTXO commitment | None (pre-UtreexO) | None | Implicit in z-map root hash | +| ZK-friendly | No (SHA-256 is expensive in circuits) | No | Yes (Tip5 is native field arithmetic) | + +### Nockchain's Merkle Innovation + +Nockchain combines two Merkle patterns: + +1. **Explicit Merkle trees** (lock trees, FRI commitments): traditional hash trees with sibling proofs, matching Taproot's MAST model + +2. **Implicit Merkle structure** (z-map/z-set): the Tip5-ordered treap data structures (zoon) function as hash-committed collections without a separate Merkle tree — the z-map root hash already commits to the entire set's contents and structure + +This means the UTXO set itself is a Merkle-like commitment — comparing two UTXO sets reduces to comparing their z-map root hashes. Bitcoin requires a separate accumulator (like Utreexo) to achieve similar properties. diff --git a/docs/architecture/tx-engine/14-goldilocks-field-arithmetic.md b/docs/architecture/tx-engine/14-goldilocks-field-arithmetic.md new file mode 100644 index 000000000..b633ff02e --- /dev/null +++ b/docs/architecture/tx-engine/14-goldilocks-field-arithmetic.md @@ -0,0 +1,286 @@ +# Goldilocks Field Arithmetic (ztd one and two) + +## Overview + +All cryptographic operations in Nockchain — hashing (Tip5), signatures (Cheetah/Schnorr), Merkle trees, STARK proofs — operate over the **Goldilocks prime field**: p = 2^64 − 2^32 + 1 = 18,446,744,069,414,584,321. + +The foundational arithmetic is defined in two Hoon files: +- `hoon/common/ztd/one.hoon` (~1390 lines): base field types, operations, array utilities +- `hoon/common/ztd/two.hoon` (~1716 lines): extension field, polynomial arithmetic, NTT/FFT + +With Rust mirrors in `crates/nockchain-math/src/belt.rs`, `felt.rs`, `bpoly.rs`, and `poly.rs`. + +## Why Goldilocks? + +The choice of p = 2^64 − 2^32 + 1 is driven by four properties: + +1. **Fits in a single `u64`**: Despite being a 64-bit prime, it's slightly less than 2^64, so every field element is a native machine word. No multi-limb arithmetic needed for base field operations. + +2. **Efficient modular reduction**: The special form p = 2^64 − 2^32 + 1 means modular reduction can use shifts and additions instead of general-purpose division. For a product `a * b mod p`, the 128-bit result can be reduced via the identity `2^64 ≡ 2^32 − 1 (mod p)`. + +3. **Large power-of-two subgroup**: The multiplicative group has order p − 1 = 2^64 − 2^32 = 2^32 · (2^32 − 1), containing a subgroup of order 2^32. This enables efficient NTT (Number Theoretic Transform) / FFT for polynomial operations — critical for STARK proof generation. + +4. **Ecosystem adoption**: The same field is used by Plonky2 (Polygon), Polygon Miden, and other ZK systems, enabling shared tooling and research. + +## ztd/one.hoon: Base Field Types and Operations + +### Core Types + +```hoon ++$ belt @ :: Base field element in [0, p) ++$ felt @ux :: Extension field element (packed atom) ++$ melt @ :: Montgomery-space element ++$ bpoly [len=@ dat=@ux] :: Base field polynomial ++$ fpoly [len=@ dat=@ux] :: Extension field polynomial ++$ poly (list @) :: Coefficient list polynomial ++$ array [len=@ dat=@ux] :: Fixed-stride u64 array ++$ mary [step=@ =array] :: Multi-stride array (step = element width in u64 words) +``` + +**Belt** is the fundamental type — a single Goldilocks field element stored as a `u64` atom. All arithmetic ensures results stay in `[0, p)`. + +**Felt** is a cubic extension field element — three `belt` values packed into a single atom with a high-bit marker. The extension is Fp[x]/(x³ − x − 1), so each felt represents `a₀ + a₁x + a₂x²` where the aᵢ are belts. + +**Melt** is a belt in Montgomery representation (multiplied by R = 2^64 mod p), used for efficient repeated multiplication inside Tip5. + +**Mary** is a strided array — a contiguous block of `u64` words where each element occupies `step` words. Used for polynomials, execution traces, and Merkle tree heaps. + +### Base Field Operations + +```hoon +++ badd :: a + b mod p +++ bneg :: p - a (additive inverse) +++ bsub :: a - b mod p +++ bmul :: a * b mod p (128-bit intermediate, then reduce) +++ bpow :: a^n mod p (repeated squaring) +++ binv :: a^(-1) mod p (Fermat: a^(p-2)) +++ ordered-root :: primitive root of unity of given order +``` + +Rust mirror (`crates/nockchain-math/src/belt.rs`): + +```rust +pub const PRIME: u64 = 18446744069414584321; + +pub struct Belt(pub u64); + +impl Add for Belt { ... } // badd +impl Sub for Belt { ... } // bsub +impl Mul for Belt { ... } // bmul (via u128 intermediate) +impl Neg for Belt { ... } // bneg +impl Belt { + pub fn inv(&self) -> Belt { ... } // binv + pub fn pow(&self, exp: u64) -> Belt { ... } // bpow +} +``` + +The Rust `Belt` type implements standard Rust operator traits, making field arithmetic look like natural integer arithmetic: `a + b`, `a * b`, `-a`. + +### Montgomery Arithmetic + +For Tip5's permutation (7 rounds of repeated field multiplications), Montgomery multiplication avoids per-operation modular division: + +```hoon +++ montify :: belt → melt: multiply by R mod p +++ mont-reduction :: melt → belt: multiply by R^(-1) mod p +++ montiply :: a * b * R^(-1) mod p (single operation) +``` + +Montgomery multiplication `montiply(a, b)` computes `a · b · R^(-1) mod p` using only shifts and additions (no division), roughly 4× faster than standard modular multiplication for sequences of multiplications. + +Rust constants: + +```rust +pub const PRIME: u64 = 18446744069414584321; +const RP: u128 = 340282366841710300967557013911933812736; // R·p +pub const R2: u128 = 18446744065119617025; // R² mod p +pub const H: u64 = 20033703337; // Montgomery reduction constant +pub const ORDER: u64 = 2_u64.pow(32); // Order of multiplicative subgroup +``` + +### Array Utilities (mary) + +The `mary` type and its `ave` core provide array operations central to the STARK proof system: + +| Arm | Purpose | +|---|---| +| `snag` | Index into array | +| `scag` | Take first N elements | +| `slag` | Drop first N elements | +| `weld` | Concatenate arrays | +| `flop` | Reverse array | +| `change-step` | Reinterpret element stride | +| `snag-as-bpoly` | Extract element as bpoly | +| `snag-as-fpoly` | Extract element as fpoly | +| `snag-as-mary` | Extract element as sub-mary | + +These operate on packed binary data with stride-based indexing, providing efficient bulk data manipulation for polynomial evaluation tables and execution traces. + +### Multivariate Polynomials + +ztd/one also defines the `mp-mega` and `mp-graph` types for STARK constraint polynomials: + +```hoon ++$ mp-mega (map bpoly belt) :: Sparse monomial map ++$ mp-graph :: Expression graph + $% [%con a=belt] :: Constant + [%var col=@] :: Variable (column index) + [%rnd t=term] :: Random challenge + [%add a=mp-graph b=mp-graph] :: Addition + [%mul a=mp-graph b=mp-graph] :: Multiplication + ... + == +``` + +The `mp-graph` type represents STARK constraint polynomials as expression trees, preserving semantic structure for efficient evaluation. Variables reference columns in execution trace tables, and random challenges come from the Fiat-Shamir transcript. + +### Base58 Encoding + +```hoon +++ en-base58 :: atom → Base58 string +++ de-base58 :: Base58 string → atom +``` + +Used for human-readable encoding of hashes, public keys, and addresses throughout the system. + +## ztd/two.hoon: Extension Field and Polynomial Math + +ztd/two imports ztd/one and builds the extension field and polynomial infrastructure needed for the STARK proof system. + +### Extension Field Fp³ + +```hoon +|_ deg=_`@`3 :: Extension degree (default 3) +``` + +The extension field is Fp[x]/(x³ − x − 1), configured by the `deg-to-irp` map: + +```hoon +++ deg-to-irp + %- ~(gas by *(map @ bpoly)) + :~ [1 (init-bpoly ~[0 1])] :: x + [2 (init-bpoly ~[2 p-1 1])] :: x² + (p-1)x + 2 + [3 (init-bpoly ~[1 p-1 0 1])] :: x³ − x − 1 → x³ + (p-1)x + 1 + == +``` + +The irreducible polynomial is x³ − x − 1 (stored as `[1, p-1, 0, 1]` using the additive inverse of 1 since we're in Fp). + +### Extension Field Operations + +| Arm | Purpose | +|---|---| +| `fadd` | Addition (component-wise mod p) | +| `fneg` | Negation (component-wise) | +| `fsub` | Subtraction | +| `fmul` | Multiplication with reduction by irreducible poly | +| `finv` | Inversion via extended GCD | +| `fdiv` | Division (`fmul(a, finv(b))`) | +| `fpow` | Exponentiation (repeated squaring) | +| `lift` | Belt → Felt (embed base element) | +| `drop` | Felt → Belt (project to base element) | +| `frip` | Felt → list of Belts (decompose) | +| `frep` | List of Belts → Felt (compose) | + +The Rust mirror in `crates/nockchain-math/src/felt.rs` provides equivalent operations. Note that the Cheetah curve uses a *different* extension — Fp^6 = Fp[X]/(X^6 − 7) — implemented directly in `cheetah.rs` rather than through the general felt machinery. + +### Polynomial Arithmetic + +ztd/two provides full polynomial arithmetic over both base and extension fields: + +| Category | Arms | +|---|---| +| **Basic** | `fpadd`, `fpsub`, `fpmul`, `fpdiv`, `fpmod`, `fpscal` | +| **Evaluation** | `fpeval` (evaluate polynomial at a point) | +| **Interpolation** | `interpolate`, `intercosate` (cosine-domain interpolation) | +| **NTT/FFT** | `fp-ntt`, `bp-ntt`, `fp-fft`, `fp-ifft` | +| **Domain** | `zerofier`, `coseword` (evaluation domains) | +| **GCD** | `fpgcd`, `bpegcd` (extended GCD for inversion) | + +### NTT/FFT + +The Number Theoretic Transform is the finite-field analogue of FFT: + +```hoon +++ fp-ntt :: Forward NTT over extension field polynomials +++ bp-ntt :: Forward NTT over base field polynomials +++ fp-fft :: Forward FFT +++ fp-ifft :: Inverse FFT +``` + +NTT converts between coefficient and evaluation representations of polynomials in O(n log n) field operations. This is critical for STARK proof generation: +- Polynomial multiplication via NTT: O(n log n) instead of O(n²) +- Low-degree extension (LDE): evaluate constraint polynomials on larger domains +- FRI queries: evaluate committed polynomials at specific points + +The Goldilocks field's 2^32-order subgroup enables NTT up to length 2^32, sufficient for practical STARK proof sizes. + +### u32 Arithmetic + +```hoon +++ u32-add :: 32-bit addition with overflow check +++ u32-sub :: 32-bit subtraction +++ u32-mul :: 32-bit multiplication +++ u32-dvr :: 32-bit division with remainder +``` + +Used for the Tip5 S-box lookup table addressing, where bytes are manipulated as 32-bit values. + +### Bignum Arithmetic + +```hoon +++ chunk :: Split atom into multi-word representation +++ merge :: Combine multi-word back to atom +++ valid :: Validate bignum representation +``` + +Used for operations that exceed a single field element, such as scalar multiplication on the Cheetah curve. + +### Shape Utilities + +```hoon +++ shape + ++ grow :: Expand a Dyck word + ++ leaf-sequence :: Extract leaf atoms from a noun tree + ++ num-of-leaves :: Count leaves in a noun tree +``` + +These tree structure utilities are used for noun hashing (extracting the leaf sequence and Dyck word from a noun for Tip5 `hash-noun-varlen`) and for note-data size validation (counting leaves to enforce `max-size` limits). + +## Rust Mirror Architecture + +| Hoon | Rust File | Key Types/Functions | +|---|---|---| +| Belt operations | `crates/nockchain-math/src/belt.rs` | `Belt`, `PRIME`, Add/Sub/Mul/Neg impls, `inv()`, `pow()` | +| Felt operations | `crates/nockchain-math/src/felt.rs` | `Felt`, Karatsuba multiplication | +| Polynomial ops | `crates/nockchain-math/src/poly.rs`, `bpoly.rs` | `bpoly_to_vec`, `init_bpoly`, polynomial arithmetic | +| Mary/array | `crates/nockchain-math/src/structs.rs` | Array manipulation utilities | + +The Rust implementations are jets — they accelerate the Hoon computations while maintaining bit-exact compatibility. The Hoon definitions remain authoritative. + +## Usage in the Transaction Engine + +The field arithmetic from ztd/one and ztd/two underpins every cryptographic operation in the tx-engine: + +| Operation | Types Used | Source | +|---|---|---| +| Tip5 hashing | Belt (field elements), Melt (Montgomery form) | ztd/one | +| Z-map/z-set ordering | Belt (hash digest comparison) | ztd/one | +| Schnorr signatures | Belt (scalars), F6lt (Cheetah point coords) | ztd/one + cheetah.rs | +| Merkle tree hashing | Belt (Tip5 digests) | ztd/one | +| Note-data size limits | Shape utilities (num-of-leaves) | ztd/two | +| STARK proofs | Felt, Bpoly, Fpoly, Mary, NTT/FFT | ztd/one + ztd/two | +| Transaction ID computation | Belt (hash output) | ztd/one | +| Base58 addresses | Belt → Base58 encoding | ztd/one | + +## Comparison with Other Blockchain Fields + +| Blockchain | Field | Size | Special Properties | +|---|---|---|---| +| Bitcoin | Binary (SHA-256) | 256-bit | No algebraic structure | +| Ethereum | Binary (Keccak) | 256-bit | No algebraic structure | +| Mina (Poseidon) | Pasta curves (Fp, Fq) | 255-bit | SNARK-friendly | +| Polygon Miden | Goldilocks (2^64 − 2^32 + 1) | 64-bit | Same field as Nockchain | +| Nockchain | Goldilocks (2^64 − 2^32 + 1) | 64-bit | STARK-friendly, native u64 | + +The Goldilocks field is becoming a standard in the ZK space. Its 64-bit size trades off raw security margin (compensated by the cubic extension for signatures and the sponge capacity for hashing) for dramatic improvements in computational efficiency — every field operation is a single machine instruction rather than multi-limb arithmetic. diff --git a/docs/architecture/tx-engine/15-ztd-stark-proof-stack.md b/docs/architecture/tx-engine/15-ztd-stark-proof-stack.md new file mode 100644 index 000000000..206ab5cff --- /dev/null +++ b/docs/architecture/tx-engine/15-ztd-stark-proof-stack.md @@ -0,0 +1,311 @@ +# The ZTD STARK Proof Stack (ztd three through eight) + +## Overview + +Nockchain's transaction engine sits atop a complete STARK (Scalable Transparent ARgument of Knowledge) proof system, implemented across eight Hoon libraries (`ztd/one.hoon` through `ztd/eight.hoon`). This stack provides: +- The hash function (Tip5) used for all tx-engine commitments +- The Merkle tree primitives used for lock proofs +- The proof-of-work mechanism (proving correct execution of Nock programs) + +The tx-engine accesses the full stack via `zeke.hoon` (a 1-line re-export of `ztd/eight.hoon`), which transitively imports the entire hierarchy. + +## Import Hierarchy + +``` +ztd/one.hoon (~1390 lines) + Belt, Felt, Melt, Bpoly, Fpoly, Mary, Base field arithmetic + │ + ▼ +ztd/two.hoon (~1716 lines) + Extension field, Polynomial arithmetic, NTT/FFT, Interpolation + │ + ▼ +ztd/three.hoon (~2091 lines) + Tip5 hash, Merkle trees, Proof types, u32 arithmetic, Bignum, Cheetah curve + │ + ▼ +ztd/four.hoon (~258 lines) + Proof stream, Fiat-Shamir, Constraint utilities + │ + ▼ +ztd/five.hoon (~1026 lines) + FRI polynomial commitment scheme + │ + ▼ +ztd/six.hoon (~309 lines) + Table/jute interface for execution traces + │ + ▼ +ztd/seven.hoon (~963 lines) + STARK constraint organization, Quotient computation + │ + ▼ +ztd/eight.hoon (~1034 lines) + STARK prover engine, Proof-of-work integration + │ + ▼ +zeke.hoon (1 line) + Re-exports ztd/eight → imported by zoon.hoon and tx-engine +``` + +Total: ~8,787 lines of Hoon implementing a complete STARK proof system from field arithmetic through proof generation. + +## ztd/three.hoon: Cryptographic Primitives + +Covered in detail in files 11 (Tip5) and 13 (Merkle trees). Key components: + +| Component | Purpose | Used by tx-engine? | +|---|---|---| +| Tip5 hash | Sponge hash over Goldilocks | Yes — all commitments | +| Merkle trees (merk) | Binary hash trees with proofs | Yes — lock proofs, block commitments | +| Hashable | Tagged commitment trees | Yes — transaction IDs, lock hashes | +| Tog (PRNG) | Deterministic randomness from sponge | Indirectly — via STARK proofs | +| u32 arithmetic | 32-bit operations for S-box | Yes — Tip5 internals | +| Bignum (chunk/merge) | Multi-word integers | Yes — Cheetah scalar operations | +| Cheetah curve types | Elliptic curve point operations | Yes — Schnorr signatures | + +## ztd/four.hoon: Proof Stream and Fiat-Shamir + +~258 lines providing the interactive-to-non-interactive transformation for STARK proofs. + +### Proof Stream + +The proof stream is the central abstraction for building non-interactive proofs via the Fiat-Shamir heuristic: + +```hoon +++ proof-stream + |_ [objects=(list proof-data) read-index=@ sponge=tip5-state] +``` + +| Arm | Purpose | +|---|---| +| `push` | Append a proof object to the stream | +| `pull` | Read the next proof object from the stream | +| `prover-fiat-shamir` | Generate verifier challenges from prover's transcript | +| `verifier-fiat-shamir` | Regenerate challenges from received proof objects | + +The Fiat-Shamir transform converts an interactive proof protocol (where the verifier sends random challenges) into a non-interactive one (where challenges are derived by hashing the transcript). The `sponge` field is a Tip5 sponge state that absorbs proof objects and squeezes out challenges. + +### Proof Data Types + +```hoon ++$ proof-data + $% [%tip5-digest noun-digest:tip5] + [%merk-proof merk-proof:merkle] + [%merk-data merk-data:merkle] + [%codeword-data codeword-data] + [%fp-codeword-data fp-codeword-data] + [%deep-point deep-point] + [%fp-deep-point fp-deep-point] + == +``` + +Each variant carries a different type of proof artifact — hash digests, Merkle proofs, polynomial evaluation data, and deep composition points. + +### Constraint Utilities + +```hoon ++$ mp-pelt [a=belt b=belt c=belt] :: Triple belt for constraint evaluation +++ mpadd-pelt :: Add two pelts +++ mpmul-pelt :: Multiply two pelts +++ mpsub-pelt :: Subtract two pelts +``` + +The `mp-pelt` type represents constraint polynomial evaluations as triples, used for efficient batch evaluation of STARK AIR constraints. + +## ztd/five.hoon: FRI Commitment Scheme + +~1026 lines implementing the **Fast Reed-Solomon Interactive Oracle Proof of Proximity (FRI)** — the polynomial commitment scheme at the heart of STARKs. + +### FRI Configuration + +```hoon ++$ fri-input + $: offset=belt + omega=belt :: Root of unity for evaluation domain + domain-length=@ :: Size of evaluation domain + expansion-factor=@ :: LDE blow-up factor + num-spot-checks=@ :: Number of random queries + folding-degree=@ :: Degree reduction per FRI round + == +``` + +### FRI Engine + +```hoon +++ fri-door + |_ fri-input + ++ prove :: Generate FRI proof + ++ verify :: Verify FRI proof +``` + +FRI proves that a committed polynomial has degree at most `d` by: +1. Committing to polynomial evaluations via Merkle trees +2. Iteratively folding the polynomial (reducing degree by `folding-degree` per round) +3. Providing spot-check openings with Merkle proofs at random positions +4. The verifier checks that folding was done correctly at the queried positions + +The Merkle trees used for FRI commitments are the same `merk-heap` type from ztd/three, using Tip5 for all hash operations. + +## ztd/six.hoon: Table/Jute Interface + +~309 lines defining the interface between execution traces and STARK constraints. + +### Jute (Junction Table) Types + +```hoon ++$ jute-data :: Data for a single junction table column ++$ jute-funcs :: Functions for a junction table column ++$ row :: Single row of execution trace ++$ matrix :: Multiple rows ++$ table :: Named execution trace table ++$ table-mary :: Table stored as mary (strided array) +``` + +A "jute" (junction table entry) represents a single column in the execution trace — the record of a Nock program's computation step-by-step. Each column captures one aspect of the computation (program counter, stack pointer, memory contents, etc.). + +The STARK proof demonstrates that the execution trace satisfies all transition constraints (each step follows from the previous according to the Nock evaluation rules) and boundary constraints (the input/output match the claimed values). + +## ztd/seven.hoon: STARK Core + +~963 lines implementing the core STARK constraint machinery. + +### Constraint Degrees + +```hoon ++$ constraint-degrees + $: boundary=(list @) + row=(list @) + transition=(list @) + terminal=(list @) + extra=(list @) + == +``` + +Constraint types: +- **Boundary**: Fix values at specific rows (e.g., initial state, final output) +- **Row**: Must hold at every row independently +- **Transition**: Relate consecutive rows (e.g., `state[i+1] = f(state[i])`) +- **Terminal**: Fix values at the last row +- **Extra**: Additional constraints for cross-table lookups + +### STARK Configuration + +```hoon ++$ stark-config ++$ stark-input +``` + +Configuration includes: the number of trace columns, constraint degrees, FRI parameters, and the evaluation domain specification. + +### Quotient Computation + +```hoon +++ quot :: Quotient polynomial computation +``` + +The quotient polynomial is the key object in STARK proofs. Given: +- An execution trace polynomial `T(x)` +- A set of constraint polynomials `C_i(T(x))` +- A zerofier polynomial `Z(x)` that vanishes on the trace domain + +The quotient `Q(x) = C(T(x)) / Z(x)` exists as a polynomial (i.e., the division is exact) if and only if the constraints are satisfied. The STARK proof commits to `Q(x)` and uses FRI to prove its degree is bounded. + +## ztd/eight.hoon: STARK Prover Engine + +~1034 lines implementing the full STARK prover, including proof-of-work integration. + +### STARK Engine + +```hoon +++ stark-engine +++ stark-engine-jet-hook +``` + +The prover follows this pipeline: + +1. **Trace generation**: Execute the Nock program, recording the execution trace +2. **Low-Degree Extension (LDE)**: Extend trace polynomials to a larger evaluation domain +3. **Constraint evaluation**: Evaluate all constraint polynomials over the extended domain +4. **Quotient computation**: Divide constraint evaluations by the zerofier +5. **Composition**: Combine quotient columns using random challenges (Fiat-Shamir) +6. **Deep composition**: Evaluate at a random out-of-domain point +7. **FRI**: Prove the composed polynomial has the expected degree +8. **Serialize**: Package all commitments and openings into a proof stream + +### Proof-of-Work Integration + +```hoon +++ puzzle-nock :: Generate a proof-of-work puzzle +++ powork :: Verify proof-of-work +++ gen-tree :: Generate the execution tree for proof +++ fock :: Proof caching mechanism +``` + +Nockchain's proof-of-work is unique: instead of finding a hash preimage (like Bitcoin), miners must generate a valid STARK proof that a given Nock computation was executed correctly. This means: + +- The "puzzle" is a Nock program to execute +- The "solution" is a STARK proof of correct execution +- Verification is efficient (polylogarithmic in computation size) +- Mining requires actually performing computation, not just hashing + +The `fock` arm provides proof caching — previously generated proofs can be stored and reused, avoiding redundant work for repeated computations. + +## Connection to the Transaction Engine + +The tx-engine does not directly use the STARK prover for transaction validation. Instead, the connection is: + +### Direct Usage + +| ZTD Component | Tx-Engine Usage | +|---|---| +| ztd/one (Belt, Melt) | Field arithmetic for all hash/signature operations | +| ztd/two (Shape, Felt) | Note-data size limits, Cheetah field operations | +| ztd/three (Tip5) | Transaction IDs, lock roots, Merkle proofs, z-map ordering | +| ztd/three (Merkle) | Lock Merkle proofs, block commitments | + +### Indirect Usage + +| ZTD Component | Role | +|---|---| +| ztd/four–eight (STARK prover) | Proof-of-work for block mining | + +The STARK stack generates the proofs that secure the blockchain itself. When a miner produces a block, they must include a STARK proof that the block's Nock computation was executed correctly. The tx-engine's validated transactions are *inputs* to this computation. + +### The Import Chain + +``` +tx-engine-1.hoon + └─ imports zoon.hoon (z-map/z-set for all collections) + └─ imports zeke.hoon + └─ re-exports ztd/eight.hoon + └─ transitively imports one through seven +``` + +This means the tx-engine has access to the *entire* ZTD stack through its dependency chain. While it primarily uses ztd/one through ztd/three for field arithmetic, hashing, and Merkle trees, the full stack is available. + +## Also: zose.hoon — Vendored Crypto Utilities + +`hoon/common/zose.hoon` (~3684 lines) is a vendored copy of Urbit's `zuse` crypto library, providing: + +| Component | Purpose | +|---|---| +| `base58` | Base58 encoding/decoding (Bitcoin-style) | +| `fu` | Modular arithmetic (generic prime field operations) | +| `curt` | Curve25519 scalar multiplication | +| `ga` | GF polynomial arithmetic (Galois fields) | + +These utilities support non-consensus cryptographic operations like key derivation and Ed25519 signatures. + +## Comparison with Other Blockchain Proof Systems + +| Aspect | Bitcoin | Ethereum | Nockchain | +|---|---|---|---| +| Proof of work | SHA-256d hashcash | Ethash (deprecated) → PoS | STARK proof of Nock execution | +| Verification | Check hash < target | N/A (PoS) | Verify STARK proof | +| ZK proofs | None (native) | L2 rollups (external) | Native STARK prover (ztd stack) | +| Hash function | SHA-256 (binary) | Keccak-256 (binary) | Tip5 (algebraic, ZK-friendly) | +| Signature scheme | ECDSA/Schnorr over secp256k1 | ECDSA over secp256k1 | Schnorr over Cheetah (STARK-friendly) | +| Field arithmetic | None (no algebraic structure) | None (no algebraic structure) | Goldilocks field (ztd/one–two) | + +The distinguishing feature of Nockchain's architecture is the **deep integration** between the proof system and the transaction engine. They share the same field (Goldilocks), the same hash function (Tip5), and the same data structures (z-maps, Merkle trees). This co-design means that verifying transactions *inside* a STARK proof is efficient — no impedance mismatch between the consensus layer's cryptographic primitives and the proof system's arithmetic. diff --git a/docs/fixes/HARDEN-JAM-DECODE.md b/docs/fixes/HARDEN-JAM-DECODE.md new file mode 100644 index 000000000..9ab8edf1a --- /dev/null +++ b/docs/fixes/HARDEN-JAM-DECODE.md @@ -0,0 +1,39 @@ +# Harden Jam Decode Regression + +## Summary + +This regression covers two boot-time failure modes in checkpoint and state-jam handling. The first was a corrupt or truncated jam artifact whose encoded byte length could be decoded before any validation had a chance to reject the artifact, allowing the process to panic instead of treating the artifact as a bad checkpoint and falling back to another available checkpoint. The second was a Serf initialization failure where NockStack exhaustion while restoring checkpoint state could be reported to the caller as a generic dropped initialization channel rather than as the allocation problem that actually stopped boot. + +## Background + +Nockchain boot can restore persisted state from checkpoint jams, and operators can also import or export state jams. Checkpoints have multiple valid historical formats, so loading must continue to support V0, V1, and V2 checkpoint artifacts while rejecting corrupt variants deterministically. V2 checkpoints add an envelope around a payload, while older checkpoint formats encode the jammed noun directly. Exported state jams use a separate magic and version but have the same operational risk: they contain a jam byte field whose encoded length must never be trusted enough to allocate blindly. + +The original checkpoint decode path delegated directly to owned bincode decoding for structures that contain `Bytes`. In the corrupt-length case, bincode attempted to allocate a buffer from the declared length before the checkpoint checksum, version, magic, or fallback behavior could run. This meant a single bad checkpoint file could abort the load path even when another valid checkpoint was present and could have been used. + +Serf initialization had a separate error attribution problem. NockStack allocation failures are currently represented as panics with typed payloads inside the VM stack allocator. If such a panic occurred while copying or preserving checkpoint state during Serf thread startup, the Serf thread could exit before sending its initialization result, and the boot caller would observe a generic oneshot channel failure. That obscured the actionable fix for an operator, especially in cases where increasing `--stack-size` could resolve the boot failure. + +## Fix + +Checkpoint and exported state-jam decode now go through a small checked artifact reader before constructing owned values. The reader parses the bincode-standard primitive layout used by these artifacts, including varint lengths, fixed hashes, booleans, and byte slices. It verifies that declared byte lengths fit in memory address space and in the available input before copying bytes into owned `Bytes`, so malformed lengths become normal decode errors instead of allocation panics. + +The checked decoder is used for V0, V1, and V2 checkpoint loading, including the V2 envelope and payload. Valid checkpoint variants still round-trip through the existing encoded format, and checksum validation still remains the integrity check for decoded checkpoint contents. The decoder is also used for exported state jams, with state-jam-specific magic, version, and recovery guidance. + +Operator-facing messages now distinguish corrupt artifacts, unsupported versions, invalid magic, and checksum failures. The messages avoid exposing internal stack traces and instead tell operators whether to restore a checkpoint or state jam from a known-good peer, remove a bad checkpoint so another one can be tried, re-export a state jam, or use a compatible binary for the artifact version. + +Serf initialization now wraps the phases that can trigger NockStack allocation panics during startup. The wrapper is intentionally narrow and only covers initialization phases such as stack allocation, copying checkpoint state, copying cold state, booting the kernel, loading checkpoint state through the kernel, and preserving the loaded state. If the panic payload is a NockStack allocation error, boot receives a direct Serf init allocation error with phase, configured stack size, checkpoint event number, checkpoint kernel hash, current kernel hash, and guidance to retry with a larger `--stack-size`. Unknown Serf init panics are still reported directly as Serf init panics with guidance to preserve the artifact and rerun with backtraces. + +## Regression Coverage + +- `corrupt_checkpoint_length_does_not_panic_and_falls_back_to_previous_checkpoint` covers a bad checkpoint with an absurd jam length alongside a valid older checkpoint. The expected behavior is no panic and successful fallback to the valid checkpoint. +- `serf_thread_init_stack_oom_is_not_collapsed_into_oneshot_error` covers checkpoint restore with insufficient NockStack space. The expected behavior is a direct stack allocation error that includes operator guidance, not a generic channel error. +- Additional focused coverage checks valid V2 checkpoint round-trip, corrupt V2 envelope payload length rejection, valid exported state-jam round-trip, and corrupt exported state-jam length rejection. + +## Operational Expectations + +If an operator sees a malformed checkpoint or state-jam error, the artifact should be treated as corrupt or incomplete. For checkpoints, the operator can remove the bad checkpoint and allow the peer to use another local checkpoint if one exists, or restore checkpoints from a known-good synced peer. For exported state jams, the operator should re-export from a known-good node or bootstrap from a valid checkpoint instead. + +If an operator sees a Serf init allocation error, the first configuration fix is to retry with a larger `--stack-size`, typically `large` or `huge`. If the peer still cannot boot with the largest supported stack size, the operator should preserve the failing artifact for debugging and restore a checkpoint or state jam from a synced peer. The error includes checkpoint and kernel hash context so the operator can distinguish stack sizing from artifact or binary compatibility issues without needing raw panic output. + +## Notes + +This hardening does not add redundant validation of the noun produced by cueing a jam. The fix is deliberately at the artifact boundary: it prevents corrupt container metadata from triggering unsafe allocation behavior before existing checksum, version, magic, and cueing logic can run. PMA and future import/export paths should reuse the same checked artifact reader or an equivalent shared decoder rather than reintroducing direct owned bincode decoding for untrusted jam-bearing artifacts. diff --git a/docs/pma/DESIGN.md b/docs/pma/DESIGN.md new file mode 100644 index 000000000..5ab297a14 --- /dev/null +++ b/docs/pma/DESIGN.md @@ -0,0 +1,181 @@ +# NockVM's Persistent Memory Arena + +Date: 2026-04-29 +Author: @bitemyapp + +This is a near-end-of-implementation-effort summary of the design intent for NockVM's PMA (persistent memory arena). This is intended to help both humans and LLMs understand what the PMA is expected to do and why at a high-level and what considerations and trade-offs went into those decisions. + +## History + +NockVM is a general purpose runtime for Nock, something it shares in common with Urbit's Vere. NockVM started as Ares, a next-generation runtime for Vere designed to overcome some design infelicities that were limiting runtime performance. The principal designer and author was Edward Amsden. I picked up and started working on `nockvm` along with some parts of the Nockchain platform a couple years prior to finishing the PMA. + +NockVM is still slower in a straight line for the runtime execution of Nock than Vere, but the implementation has proved itself in the areas where it's closest to the performance frontier: garbage collection built on copying compaction. NockVM's `NockStack` uses East/West alternating stack frames for runtime data. Not having the overhead of reference-counting and a more efficient garbage collection implementation has enabled NockVM to generally be faster on Linux than on macOS. The opposite is true for Vere, which tends to be faster on Apple Silicon, largely because of the burden reference counting imposes on memory bandwidth and latency. + +The main thing preventing NockVM from exceeding Vere's runtime efficiency at present is that it's still got a relatively naive mutually recursive interpreter. I have taken the current design of NockVM about as far as I can take it in terms of efficiency. I used `hoonc` (compiling Hoon using the Hoon-compiler-in-Hoon running in the Nock runtime) to benchmark and compare NockVM and Vere because it is close to a worst-case benchmark for the current interpreter. My last major nockvm efficiency pass got the runtime performance delta down to 2.5x from 5x. I believe when I started working on NockVM the delta was closer to 10x. + +I believe the next major leap will necessarily come from replacing the recursive interpreter with a bytecode VM. There are some things about how Nock (and Hoon) work that makes this a little trickier than you'd normally anticipate, principally because Nock can generate new formulas to evaluate at runtime. I'm assuming that some blend of AoT and JIT bytecode compilation and caching will be required but I haven't sunk my teeth into this problem yet. + +The more efficient GC design in NockVM has helped tremendously with making Nockchain viable as new blocks get mined and the chain continues to grow. In the pre-PMA design, we handled persistence by checkpointing the subject (all of the state in the VM, "Arvo" in Urbit parlance) to disk at a fixed time interval. This backup-to-disk checkpointing process was a source of some heartache because it required jamming the subject which meant that the time required to construct and persist these checkpoint jams was increasing linearly with the size of the subject. The problem is that the subject itself grows as the chain state grows. For the ZKVM and blockchain state machine to be sound, all of the chain state needs to be addressable as first-class "Nouns" (values in Hoon or Nock) within the VM. + +We didn't notice this until I was testing and comparing PMA performance against the baseline implementation, but we were experiencing parasitic loss from the NockStack having to compact the entire subject in the final flip-top-frame of a `poke` operation. The PMA spares us this and the extra durability work is a few orders of magnitude faster than that was. + +So we needed some kind of persistent memory arena or virtual memory based solution for paging out unused parts of the subject so that the RAM requirements of a Nockchain peer didn't keep increasing linearly with chain history. The PMA in the branch carrying this document appears to have accomplished this. + +## What + +- Works well on macOS and Linux alike. +- Don't require retaining the entire subject in memory simultaneously during ordinary operations +- Durability story should be strictly better from a promptness perspective than baseline's checkpointing intervals +- PMA should recover safely from incomplete writes / slab corruption due to SIGKILL or power loss +- Snapshotting replaces checkpointing, takes ~5 seconds instead of 35-140 seconds as checkpointing requires. +- The PMA should, as much as possible while respecting runtime efficiency and memory constraints, be kind to solid state drives and not burn them out quickly. The NockStack is still being used, it's just ephemeral and the write-back fsyncs to the PMA happen after the event is finished executing and leftovers (persistent updates to the subject) are getting rolled up. +- Runtime execution efficiency is not worse than baseline by more than 10-20%. `hoonc` is close to a worst-case scenario and after some optimization work I got the delta between PMA and baseline down from 4x to 1.13x. +- At 50k+ blocks the PMA peers sit at about 7-8 GiB of steady-state RSS. Baseline is ~18-19 GiB with spikes to ~35-39 GiB while checkpointing, which is much of the time. The default recommendation at time of writing for baseline was to use 64 GiB servers. + +## How + +The PMA is intentionally very conventional and boring, or at least as much as I could make it given the _slightly_ exotic structure of Nock Nouns. In classic PHK/Varnish style, it leans hard on the virtual memory subsystem to handle paging data in and out of RAM as required. + +### Virtual memory + +The existing slab allocation in NockStack was already using anonymous mmaps. It could use `malloc` and I added that as an opt-in to NockStack in the past, but a multi-gigabyte slab allocation requested via `malloc` is presumptively going to turn into an anonymous mmap anyway. Hard to imagine libc deciding to `sbrk` 16 GiB. Instead of an anonymous memory map, the PMA is a shared mmap. "Shared" here refers to the fact that it actually lives as a persistent file on disk and hypothetically other processes could map the same segment into RAM without duplication. Don't do that though, we didn't design the PMA to be multi-process or multi-threading safe. A sufficiently intrepid hacker could probably prototype multi-threaded Nock execution using a shared PMA. Just be forewarned that the PMA effort was pretty gnarly, adding multi-threading to that could be difficult to make 100% correct. + +With a shared mmap we are relying on the kernel's virtual memory subsystem to flush memory mutated by the application to disk, to page out unused (cold) data, and to page-fault in cold data that got read or written to again. + +### Durability + +Much of what's to be said about durability here concerns two main areas of the platform, booting a NockApp and wrapping up `poke` events. Our risk model principally concerned itself with scenarios like power loss, `SIGKILL`, and the like. I explicitly did not attempt to address cosmic bit-flips, nefarious processes with write permissions to the slab, or anything else similarly out-of-pocket. The blockchain is a distributed and replicated computer system, this isn't a computer in a probe going past the asteroid belt. + +Most important parts of the durability story: + +- PMA slab + PMA .meta +- `sqlite3` event log +- Snapshots (lightly augmented copies of the PMA slab) + +The basic order of things persistence-wise is this: + +- NockVM executes `poke` +- If the event is accepted, capture the exact accepted job for the event log. A failed live poke can still synthesize a `%crud` replacement through the ordinary live `poke_swap` path; if that replacement is accepted, the replacement event is what gets logged. +- Commit the accepted event to the `sqlite3` event log in an immediate transaction. Event-log append failure is fatal because the PMA must not be advanced past the SQLite accepted-event boundary. +- Bump allocate the new persistent Arvo frontier into the PMA. +- Rewrite the new root/spine slots being copied so they point at either newly allocated PMA offsets or existing PMA offsets. Existing PMA subtrees are not rewritten during ordinary event commit. +- Write the PMA file trailer. The trailer records PMA file shape, including the allocation offset. It does _not_ record the accepted event number. +- `fdatasync` the PMA file. Failure is fatal on the durability-critical path. +- Write the PMA `.meta` sidecar last. The sidecar records the kernel hash, accepted event number, root pointer for the kernel state, and a checksum. Sidecar write failure is fatal on the durability-critical path. + +The idea here is straightforward. We can't go to the same lengths guarding against corruption in the operative PMA slab that we can for snapshots. But if we're always committing to the `sqlite3` event log before we touch anything in the PMA _and_ the sidecar `.meta` event id (monotonic number for events executed) is the last thing written and fsync'd, then we can detect incomplete write sequences because the PMA's sidecar event id will be behind the latest event id in the event log. In practice, we treat any inequality of the event ids between the PMA and event log on boot as a DR (disaster recovery) trigger. + +This does not make the operative PMA slab a fully checksummed data structure. The fast path validates the sidecar checksum/kernel hash, checks that the PMA opens and that trailer/file metadata are coherent, and requires the PMA sidecar event number to equal SQLite's max event number. It does not hash or traverse the operative slab the way snapshot recovery does. The operative PMA durability model is primarily about detecting torn or incomplete accepted-event persistence, not arbitrary same-boundary PMA data corruption. + +### Event write-back + +After an accepted event, the kernel installs a single new Arvo root. That root is a complete Arvo value, not a separate delta object, but in normal operation it is a mixed graph: newly allocated NockStack cells for the paths rebuilt by the event, novel durable subtrees created by the event, direct atoms, and references to old PMA-resident subtrees. The graph itself is the dirty-path information. + +PMA write-back starts at that new root and walks forward. Stack-allocated cells and indirect atoms are copied into the PMA and rewritten to offset form. Direct atoms are embedded directly. Existing PMA offsets are treated as terminals: the copier writes the offset into the new parent slot and does not descend through the old subtree. Forwarding pointers are used only inside the source copy space to preserve sharing among newly copied objects, so if the new graph references the same novel subtree twice it is copied once and both destination slots reuse the same PMA offset. + +The practical append size for one ordinary event is therefore approximately the new Arvo root plus the rebuilt spines to changed regions plus the novel durable subtrees themselves. Unchanged old subtrees are included by reference, not copied. In units of the implementation, each copied cell is three 64-bit words (`metadata`, `head`, `tail`), each copied indirect atom is two header words plus its data words, and direct atoms do not allocate separate PMA storage. `NOCK_PMA_TIMING_DETAIL` records the appended word count for the Arvo copy as `detail.arvo.alloc_words`; multiplying by eight gives the byte count of PMA data newly appended for that event. + +Ordinary event write-back should not dirty old PMA pages that contain unchanged Arvo subtrees. It dirties the newly appended bump-allocation range, possibly the previously partially filled trailing page, the PMA trailer containing the allocation offset, the `.meta` sidecar, and the SQLite/WAL pages. This is why event commit remains fast when the kernel preserves structural sharing. If kernel code reconstructs a large list or map instead of sharing it, PMA persistence will append that newly reconstructed structure; the PMA cannot infer sharing that the returned Arvo graph did not preserve. + +### Runtime caches and checkpoint bootstrap + +The durable PMA state is only the Arvo/kernel state. PMA boot does not persist or restore derived runtime caches such as `hot`, `warm`, `cold`, jet-test HAMTs, or the memo cache. In particular, the NockVM cold jet state is implemented as HAMTs and linked runtime structures containing native process pointers. Those pointers are appropriate for the current mapped process and stack/PMA arenas, but they are not a stable restart-persistent format. The PMA uses offset-form Nouns for durable state; the cold runtime cache is rebuilt from empty state on boot and repopulated as the kernel runs. + +This is also why first-time PMA bootstrap from legacy checkpoints treats checkpoint jams like state jams. Checkpoint files may contain both kernel state and serialized cold jet state, but PMA migration only imports the kernel state portion and initializes cold state as empty. The checkpoint cold state is an optimization cache, not chain state. Hydrating it during PMA migration increases transient memory pressure substantially and is not required for correctness. After the checkpoint state has been copied into PMA and PMA metadata has been published, later boots use the PMA fast path and rebuild runtime caches in the same way as ordinary PMA restarts. + +### Event log canonicity + +SQLite is the accepted-event authority. The event log uses WAL mode and `synchronous=FULL` unless durability syncing has explicitly been disabled. Boot runs `pragma_quick_check` before trusting the event log for recovery. The `events.event_num` column is unique, and replay reads events in ascending `event_num` order while checking for sequence gaps. + +This matters because PMA-like files are only materializations of state at some accepted event number. The SQLite event log decides what history exists. Operative PMA, snapshots, and checkpoints are boot accelerators or recovery anchors at a particular event boundary. + +The `--disable-fsync` option disables these durability claims. It disables application fsync/fdatasync calls and causes SQLite to use `synchronous=OFF`. That mode can be useful for benchmarking or controlled rebuilds, but it is not the crash-recovery mode described here. + +### Disaster recovery + +On boot, the NockApp checks that the operative PMA sidecar event id matches the latest event id in the `sqlite3` event log database. If they are equal, the operative PMA is eligible for the fast path. If the PMA is ahead of SQLite, behind SQLite, invalid, or missing, boot enters recovery. + +Recovery tries sources in order: + +- Ready snapshots from SQLite. The snapshot is verified, copied back over the operative PMA slab, and SQLite events after the snapshot event number are replayed. +- Legacy checkpoints. A checkpoint is only usable if it is not ahead of SQLite and replay from the checkpoint event number reaches the SQLite max event number. +- Fresh boot plus event-log replay from zero, if the event log is continuous from event 1. + +If no boot base can replay to SQLite's accepted-event boundary, boot fails closed. Replay applies the exact logged job and fails if that job is rejected; replay does not use the live `poke_swap` path to synthesize a different event. + +### Snapshots + +I had a clever idea while working on snapshotting and it pleased me so much that I am compelled to explain it here. I did not use a fixed duration interval (like GC or checkpointing) or count of events processed. Instead, snapshotting uses _event execution time_ as the trigger for initiating snapshot construction. It's a threshold that represents the sum of time spent executing events since the last snapshot. The appeal of this design is that it enables operators to make a very precise trade-off between time spent snapshotting during ordinary operations against the maximum amount of time you want to spend replaying events from the event log in a DR scenario. + +There was some extra effort put into detecting corruption in the snapshots because it's work that doesn't have to get done over-and-over in the course of executing events and because the snapshots are a critical part of our DR (disaster recovery) story for NockVM and NockApps. The `SnapshotManifest` protects itself. `SnapshotManifest` stores magic/version plus PMA shape, event number, root pointers, `used_blake3`, optional `structure_blake3`, and a checksum. The checksum is BLAKE3 over the manifest payload excluding the checksum field, and decode validates it before the manifest is trusted. See `crates/nockapp/src/snapshot.rs:185`. + +The snapshot PMA file is protected by `used_blake3`. When creating a snapshot, the code syncs the source PMA, copies it, then hashes the allocated PMA prefix: `pma.alloc_offset()` * 8 bytes. That hash is stored in the manifest and SQLite ready-snapshot row. See `crates/nockapp/src/snapshot.rs:572`. Verification recomputes that same used-range hash before accepting the snapshot. It also checks manifest `pma_words / alloc_words` against the PMA trailer metadata. If the file is truncated inside the used range, read_exact fails; if any byte in the used prefix changed, `UsedHashMismatch` fails verification. See `crates/nockapp/src/snapshot.rs:313` and `crates/nockapp/src/snapshot.rs:819`. + +Snapshot creation runs a fast verification pass before inserting the snapshot as ready in SQLite. Snapshot restore runs full verification before copying the snapshot back over the operative PMA. Full verification validates the saved root pointer, validates the cold offset, and traverses the noun graph. The optional `structure_blake3` field can pin a structural hash, but current production snapshot creation stores `None` there; the normal cryptographic integrity check is the used-prefix BLAKE3 hash, with full traversal providing structural sanity checks. + +Snapshot kernel-hash semantics intentionally differ from the operative PMA fast path. An operative PMA sidecar with a kernel hash mismatch is rejected. A verified snapshot or checkpoint with an old kernel hash may be loaded into the current kernel with a warning, matching the migration/import semantics already used for checkpoint state. + +### PMA GC + +PMA GC is enabled by default for application boots, with `--gc-interval` controlling the cadence in seconds. The default application interval is one hour (`3600` seconds). Tests and specialized tooling can still disable it with `--gc-interval none`, `--gc-interval 0`, or by constructing `PmaConfig` with no GC interval. + +GC is a discrete phase after the normal event durability path, not a replacement for it. The accepted event is first committed to SQLite, copied into the active PMA, synced, and published in PMA metadata. Only after that durable persistence phase may GC attempt to compact into the inactive slab. If a rotating snapshot is due, it runs after this PMA persistence/GC sequence. + +The GC path treats the SQLite event log plus ready snapshots as the recovery authority. It only runs when a ready snapshot recovery anchor exists. Before touching the inactive slab, it removes and syncs the inactive sidecar metadata. After creating the inactive slab, it removes and syncs the active sidecar metadata before destructively reading/copying from the active slab. That deliberate invalidation means a crash during GC should not leave boot believing the active operative PMA is authoritative. Boot must instead recover from a verified snapshot and replay the SQLite event log to the SQLite boundary. + +GC copies the durable Arvo root into the alternate slab, installs the alternate PMA arena, then publishes the alternate sidecar metadata last. Derived runtime caches (`hot`, `warm`, `cold`, jet-test HAMT, and the memo cache) are rebuilt on the current stack after the slab switch. They are not copied as durable state, and they are not allowed to keep pointers into the old active slab after GC completes. + +GC has a different page-dirtying profile from ordinary event commit. It copies the reachable Arvo DAG from the active slab into the inactive slab. To preserve sharing without an external relocation map, the copier writes forwarding pointers into source objects in the old active slab after copying them. Those in-source forwarding pointers answer "has this source object already been copied, and where did it land?" for later references to the same object. This saves the memory and lookup overhead of a `source offset -> destination offset` relocation table, but it dirties old source pages and makes the source slab invalid as an operative PMA once GC begins. A non-destructive GC would need such an external relocation map or would have to duplicate shared subgraphs. + +## Caveats and risks + +- It is _not_ safe to load or import raw PMA slabs from third parties. Do _not_ do this unless it's between computers you have control over. Use state jams instead! + +- The operative PMA fast path is not snapshot verification. A PMA with sidecar event number equal to SQLite max is accepted after sidecar/trailer/open checks, not after a BLAKE3 scan or noun-graph traversal of the slab. If the threat model includes arbitrary data corruption inside a fully written operative slab, use snapshots/state jams/distributed validation as the recovery boundary. + +- Snapshot restore verifies the manifest/PMA pair, but the recovery path currently uses the SQLite snapshot row to choose the replay boundary before restore. The duplicated row fields and manifest fields should stay identical because they are written together during snapshot creation, but this is not a substitute for SQLite integrity. If SQLite itself is corrupted beyond what `quick_check` and schema constraints catch, recovery decisions can be wrong. + +- PMA GC depends on having a ready snapshot as the fallback recovery anchor. If no ready snapshot exists when the interval fires, GC is skipped rather than invalidating the only operative PMA. This trades space reclamation for crash-recovery clarity. + +- Any operation that touches the whole chain state, such as a new state migration the NockVM kernel `load` poke, could re-hot all of the pages of the entire chain state and cause an unusual RSS spike. At time of writing, we haven't tested this scenario under memory pressure yet. + +- Similarly, a fresh peer syncing the chain state from the genesis block could send requests that lead to a fully-synchronized peer temporarily re-hotting cold pages from early chain history, causing a temporary RSS spike. + +- The PMA should be able to operate in environments with less than the nominal amount of memory required to address the whole subject, but you should expect that memory pressure will cause more page-churn and thus make events execute more slowly in proportion to the degree of memory pressure it is subjected to. + +## Alternative design considered + +You can dig it up in one of the old branches on the GitHub repository but I did implement two different PMAs. The storage design that "won" the contest was the bump allocating PMA. The competing design was a b-tree PMA with mark-and-sweep garbage collection. The relevant decision here is that the PMA's storage model is bump allocation plus periodic copy-style compaction into an alternate slab rather than a mutable b-tree heap. There are a few factors that led to choosing the bump PMA: + +- Copying GC took ~5 seconds to compact the entire 50k+ block subject, mark-and-sweep GC in the b-tree PMA took closer to 35 seconds. + +- b-tree PMA used a little less RAM steady-state by default (~5.8 GiB vs. 7.5 GiB) but it was somewhat more complicated and slightly slower than the bump PMA in terms of executing events. My belief is that the b-tree had lower RSS because it was explicitly attempting to respect page boundaries. The bump PMA doesn't really concern itself with that at all. During copy-style PMA GC, reachable parts of the subject are rewritten to the alternate slab, so surplus page fragmentation/deadweight disappears when the compacted slab becomes operative. + +- Making the mark-and-sweep GC faster is definitely possible but would've added a greater drag coefficient to runtime efficiency. The design attempted to harvest low-utilization pages and compact sub-trees into a smaller number of higher utilization pages. This seemed to work fine but 35 seconds is just intolerable. + +- The Bump PMA's design and simplicity coheres better with the design principles that informed NockVM and NockStack originally, especially NockStack's implementation of garbage collection. + +- I didn't have the wherewithal to deal with large (exceeding the capacity of a single 4K page) atoms in a more intelligent manner in the b-tree PMA. We don't have many examples of this in the NockVM/Nockchain ecosystem but it's my understanding that this is a much more common thing in Urbit applications. + +- Copying compaction is extremely fast even for 10 GiB+ subjects and is a lot closer to the "speed of light" than the mark-and-sweep implementation was. It is harder to get it wrong or make it slow, and keeping it as a discrete post-persistence phase gives it a clearer durability boundary than trying to interleave compaction with event execution. + +So I went with the bump PMA as the storage model. + +## Potential next steps + +- Restore slab high watermark metrics for the slab +- Automatically upgrade slab sizes when we hit a configuration utilization threshold, still doubling +- Add the ability to export a state jam in the background safely without having to shut the Serf down first. +- Rationalize PMA GC triggering so it can safely fire faster than the current hourly default when the workload actually needs it. + - Track PMA append churn since the last GC using the already-measured `pma_arvo_copy` appended word count. This is cheap because event commit already computes the PMA allocation delta. Treat it as an estimate of garbage pressure, not exact truth. + - Track raw active-slab allocation growth since the last completed GC. This is also cheap and gives a hard upper bound on how much compaction could reclaim, but it cannot distinguish live growth from garbage by itself. + - Log exact reclaim efficiency after each GC as `from_alloc_words - to_alloc_words`, then use that history to tune churn/growth thresholds. If GC repeatedly reclaims little, back off even if the timer fires. + - Add a memory-pressure override based on cgroup-aware memory usage. Under Docker or Kubernetes, read cgroup memory current/max rather than host RAM. Use this as a safety signal to page out PMA mappings first and only accelerate GC when PMA churn/growth also indicates real dead space. + - Keep a minimum wall-clock or event-count spacing between GCs. Copy-style GC is fast, but it dirties source pages, writes a full compacted slab, and temporarily maps both slabs, so it should not chase noisy short-term signals. +- Make a tool and in-system hook for tracing leftover write-backs during event commit. The idea is to record appended PMA word counts, approximate dirty-page spans, and where the new Arvo frontier is allocated across the slab. Goal would be to dump a data representation (JSON?) after executing some number of events and a separate binary that generates a static web page visualizing pages dirtied by writes vs. pages only referenced by PMA offsets. + - Stretch goal: ditto but cover reads in the NockStack during event execution as well. + - Stretch goal: ditto but cover GC. +- Add another write-back epicycle that spares write-through to disk without compromising integrity. Basically you write to the PMA without doing any write-back to the PMA .meta's event id for N number of events. If there's a SIGKILL or power cut before the next time you bump it, the event id will get recognized as stale and trigger a DR on boot. You could do this either with maybe `mlock` or by having an in-memory proxy for the event id that doesn't get written to the .meta on each event. +- CDC, building on event log. You'll probably want to figure out delta patch traces for the PMA write routines so that replication doesn't require re-executing the `poke` event though. +- Multi-threaded Nock execution with shared memory (PMA) +- Bytecode VM for NockVM to close the runtime efficiency gap with Vere diff --git a/docs/pma/DURABILITY-OPERATIONS.md b/docs/pma/DURABILITY-OPERATIONS.md new file mode 100644 index 000000000..f9fbc14fc --- /dev/null +++ b/docs/pma/DURABILITY-OPERATIONS.md @@ -0,0 +1,152 @@ +# PMA Durability Operations + +This document is the operator-facing guide for the PMA durability path implemented in `nockapp`. + +## What Exists Now + +The current runtime has: + +1. PMA durability enabled by default for normal NockApp boots. +2. SQLite event durability for accepted events. +3. An immutable `epoch` snapshot. +4. Rotating snapshots with retention of two ready non-epoch snapshots. +5. Verified snapshot restore with replay from SQLite `job_jam`. +6. Fallback across snapshot candidates, then checkpoint/state-jam bootstrap. +7. Orphan snapshot artifact cleanup into `pma/corrupted_pma/`. +8. PMA GC on an interval after normal event durability has completed. + +## Relevant CLI Flags + +These are the main boot flags operators and production testers should know about: + +1. `--ephemeral` + Runs with in-memory NockStack state only. This disables PMA durability, event logs, snapshots, and PMA GC. Production nodes should not use this except for explicit testing. + +2. `--data-dir` + Overrides the root durability directory. + +3. `--event-log-path` + Overrides the SQLite event log path. Default: `data_dir/event-log.sqlite3`. + +4. `--rotating-snapshot-interval-event-time` + Controls how much cumulative accepted event-processing time must elapse before a new rotating snapshot is attempted. Use `none` or `0` to disable rotating snapshots. Default: `900` seconds. + +5. `--gc-interval` + Controls PMA GC cadence in wall-clock seconds. Use `none` or `0` to disable PMA GC. Default: `3600` seconds. + +6. `--bootstrap-from-chkjam` + Copies a jammed checkpoint into the data directory as a bootstrap source. This is a migration/bootstrap path, not the normal steady-state recovery path. + +7. `--disable-fsync` + Disables filesystem sync calls, including SQLite full-sync durability. This is for benchmarks or local testing only, not production durability testing. + +## Relevant Environment Variables + +1. `NOCK_STACK_FREE_GAP_TRIM` + Controls anonymous NockStack free-gap `madvise(MADV_DONTNEED)` after top-frame flips. This is enabled by default with a 512 MiB free-gap threshold. Set to `0`, `false`, `off`, `no`, `disable`, or `disabled` to opt out. + +## On-Disk Layout + +Under `data_dir`: + +1. `event-log.sqlite3` +2. `event-log.sqlite3-wal` and `event-log.sqlite3-shm` when SQLite WAL mode is active +3. `pma/0.pma`, `pma/1.pma`, `pma/0.meta`, and `pma/1.meta` for operative runtime slabs +4. `pma/epoch.pma` and `pma/epoch.manifest` +5. `pma/snap-${TIMESTAMP}.pma` and `pma/snap-${TIMESTAMP}.manifest` +6. `pma/corrupted_pma/` +7. `checkpoints/` for legacy checkpoint jams used during bootstrap or rollback + +Notes: + +1. `epoch` and `snap-*` files are snapshot artifacts. +2. `0.pma` / `1.pma` are the operative runtime slabs. GC can switch which slab is active. +3. `corrupted_pma/` is where orphan snapshot artifacts or crash leftovers are moved for later inspection. +4. The accepted-event log is append-only in the current durability path; plan disk capacity accordingly until pruning/compaction lands. +5. Do not manually edit any of these files while the node is running. + +## Boot Order + +Normal boot first inspects existing operative PMA artifacts and opens the SQLite event log. If PMA artifacts exist and the event log cannot be opened, boot fails closed. + +The recovery decision order is: + +1. Valid operative PMA fast path when the PMA event number equals SQLite max event number. +2. Special first-migration bootstrap: valid PMA with a nonzero event number and an empty event log. +3. Ready snapshots from SQLite, ordered with the active snapshot first and then remaining ready candidates. +4. Checkpoint/state-only bootstrap, if it is not ahead of SQLite and replay can reach SQLite max. +5. Fresh kernel plus event-log replay from event 0, only when SQLite has events and replay continuity is intact. +6. Fresh kernel with no replay only when SQLite has no committed events. + +If PMA is ahead of, behind, missing from, or invalid relative to the event log, it is not treated as authoritative. Boot attempts verified snapshot restore and event-log replay instead. + +If continuity is broken in the event log for the chosen boot base, boot fails rather than silently falling back to stale state. + +## Snapshot Cleanup Behavior + +On boot: + +1. Snapshot artifacts without corresponding ready SQLite rows are moved into `pma/corrupted_pma/`. +2. If a snapshot candidate fails verification, it is marked `failed` in SQLite and boot continues to the next candidate. +3. The active snapshot id is updated when a snapshot is successfully restored. + +## PMA GC Behavior + +PMA GC is a discrete phase after accepted-event durability, not a replacement for it. + +The normal event path first commits the accepted event to SQLite, copies the durable kernel state into PMA, syncs the PMA state, and publishes PMA metadata. Only after that may GC compact into the inactive slab. + +If there is no ready snapshot recovery anchor, GC skips rather than invalidating the only operative PMA recovery source. + +## Metrics + +The following `nockapp` metrics are relevant to durability and recovery: + +1. `nockapp.event_log.append` +2. `nockapp.event_log.commit_failures` +3. `nockapp.snapshot.build` +4. `nockapp.snapshot.build_failures` +5. `nockapp.snapshot.verify` +6. `nockapp.snapshot.verify_failures` +7. `nockapp.snapshot.cleanup` +8. `nockapp.snapshot.cleanup_failures` +9. `nockapp.replay.apply` +10. `nockapp.replay.failures` +11. `nockapp.replay.events` + +## Suggested Dashboards + +Recommended dashboard panels: + +1. Event log append latency: p50 / p95 / p99 of `nockapp.event_log.append`. +2. Snapshot build latency: `nockapp.snapshot.build`. +3. Snapshot verify latency: `nockapp.snapshot.verify`. +4. Snapshot cleanup latency and failure count. +5. Replay duration and replayed event count per boot. +6. Event log commit failure count. +7. Snapshot verify failure count. +8. Replay failure count. + +## Suggested Alerts + +Recommended alerts: + +1. Any non-zero `nockapp.event_log.commit_failures`. +2. Any non-zero `nockapp.snapshot.verify_failures`. +3. Any non-zero `nockapp.replay.failures`. +4. Sustained `nockapp.snapshot.build_failures`. +5. Sustained `nockapp.snapshot.cleanup_failures`. + +## Operator Guidance + +If boot leaves files in `pma/corrupted_pma/`: + +1. Do not delete them immediately. +2. Check the SQLite `snapshots` table and recent logs. +3. Confirm whether the moved files correspond to a crash during snapshot write or a deferred cleanup case. + +If boot fails on event-log continuity: + +1. Treat it as a real durability problem, not a transient startup failure. +2. Inspect the `events` table for missing `event_num` values. +3. Do not assume checkpoint fallback is safe if accepted events may be missing from the log. diff --git a/docs/pma/FAST-HINT-REGISTRATION.md b/docs/pma/FAST-HINT-REGISTRATION.md new file mode 100644 index 000000000..5d38fb2a1 --- /dev/null +++ b/docs/pma/FAST-HINT-REGISTRATION.md @@ -0,0 +1,548 @@ +# `%fast` Hint Registration + +This document explains how `%fast` hint registration works in the current runtime, with special attention to how the cold state is constructed, what data and code participate in that construction, and what constraints the implementation relies on. + +This is a description of the code as it exists today. It is not a design proposal. + +## Scope + +The relevant code lives primarily in: + +- `crates/nockvm/rust/nockvm/src/interpreter.rs` +- `crates/nockvm/rust/nockvm/src/jets/cold.rs` +- `crates/nockvm/rust/nockvm/src/jets/warm.rs` +- `crates/nockvm/rust/nockvm/src/jets/hot.rs` +- `crates/nockvm/rust/nockvm/src/trace/mod.rs` +- `crates/nockvm/rust/nockvm/src/trace/tracing_backend.rs` +- `crates/nockvm/rust/nockvm/src/site.rs` +- `crates/nockvm/rust/nockvm/src/hamt.rs` +- `crates/nockvm/rust/nockvm/src/unifying_equality.rs` + +Persistence and serialization code is also relevant because the cold state is stored and moved around as a runtime object: + +- `crates/nockapp/src/kernel/form.rs` +- `crates/nockapp/src/nockapp/export.rs` + +## Executive Summary + +`%fast` is a dynamic Nock 11 hint that records structural information about cores as they are produced. + +At a high level: + +1. The interpreter evaluates a dynamic hint formula and obtains a clue noun. +2. After the hinted body finishes evaluating, `%fast` extracts a `chum` and a parent formula from that clue. +3. The result core and its parent relationship are registered into `context.cold`. +4. If registration inserted a new entry, the runtime rebuilds `context.warm` from scratch. +5. Later jet lookup uses `warm` to find candidate jets and uses ancestry information stored in `cold` to prove that the candidate really matches the live core. +6. The same cold-state path information is also used for tracing. + +The cold state is therefore a derived structural index over runtime-discovered cores. It is not the kernel state itself. It exists to make jet lookup and tracing possible. + +## Where `%fast` Sits in the Interpreter + +### Nock 11 dispatch + +Opcode 11 is handled in `interpret()` in `interpreter.rs`. + +- A static hint becomes `NockWork::Work11S`. +- A dynamic hint becomes `NockWork::Work11D`. + +The parser distinguishes them by looking at the head of the hint payload: + +- If the head is an atom, it is a static hint. +- If the head is a cell whose head is an atom, it is a dynamic hint. + +For dynamic hints, the interpreter stores: + +- `tag` +- `hint` as a formula to be evaluated +- `body` as the actual formula whose result may be affected or observed + +### Dynamic hint evaluation order + +`Work11D` runs in three stages: + +1. `ComputeHint` +2. `ComputeResult` +3. `Done` + +Those stages call: + +- `hint::match_pre_hint(...)` +- `hint::match_pre_nock(...)` +- `hint::match_post_nock(...)` + +`%fast` only does work in `match_post_nock()`. It does not short-circuit the body. It does not change the result noun. It is a side-effecting registration step that runs after the hinted body has already been evaluated. + +One explicit control-flow constraint is in `hint::is_tail()`: `%fast` is treated as non-tail. That prevents the interpreter from discarding the current frame before post-hint work runs. + +## What `%fast` Reads + +Inside `hint::match_post_nock()`: + +- `res` is the result of evaluating the hinted body. +- `hint` is the already-evaluated clue noun for the dynamic hint. + +The `%fast` logic expects the clue to provide: + +- `chum = clue.slot(2)` +- `parent = clue.slot(6)` + +The code then peels nested hint wrappers off `parent`. While `parent` looks like a Nock 11 formula, it follows `slot(7)` until it reaches the underlying formula. + +After unwrapping, it reads: + +- `parent_formula_op = parent.slot(2)` +- `parent_formula_ax = parent.slot(3)` + +Those values are then used to interpret the parent relationship for the result core: + +- If `parent_formula_op == 1` and `parent_formula_ax == 0`, the result is treated as a root registration. +- Otherwise `parent_formula_ax` is treated as the axis from the result core to its parent core. + +This means `%fast` does not directly consume a parent core from the clue. It consumes a parent formula description and then applies the resulting axis to the actual runtime result `res`. + +## Static Data Versus Dynamic Data + +The runtime uses three related structures: + +### Hot state + +`Hot` is a static table built from `URBIT_HOT_STATE` plus any constant hot entries. + +Each `HotEntry` is: + +```text +(path, axis-in-battery, jet-function) +``` + +`Hot::init()` constructs noun paths from those static path segments and stores them as a linked list of `HotMem` nodes. + +Hot state answers: + +- Which path names are associated with jets? +- At what axis inside the battery does the jetted formula live? + +Hot state does not know which live runtime cores correspond to those paths. + +### Cold state + +`Cold` is the dynamic structural index built by `%fast` registration. + +It answers: + +- Which runtime-discovered cores correspond to which paths? +- For a given path, what ancestry chain of batteries proves that match? +- For a given outer battery or root, which paths might be candidates? + +### Warm state + +`Warm` is derived from `Hot` plus `Cold`. + +It answers: + +- Given a formula, which jet candidates should we try? + +Warm state is the actual formula-indexed jet lookup table used during Nock 9 execution. + +## The Cold State Representation + +`Cold` is a wrapper around `ColdMem`, which holds three HAMTs: + +```text +battery_to_paths : Hamt +root_to_paths : Hamt +path_to_batteries : Hamt +``` + +### `battery_to_paths` + +Key: + +- The outermost battery of a core, meaning `core.slot(2)` + +Value: + +- A linked list of candidate registered paths for cores with that battery + +This exists because identical Nock can appear at multiple places. A battery is not unique enough to identify a path by itself. + +### `root_to_paths` + +Key: + +- A root noun + +Value: + +- A linked list of root paths + +This is the root-only analogue of `battery_to_paths`. + +### `path_to_batteries` + +Key: + +- A registered path noun + +Value: + +- A linked list of possible battery ancestry chains for that path + +This is the proof object used to validate that a live core really corresponds to a registered path. + +## Paths and Chums + +Paths are represented as nested noun lists ending in `0`. + +Examples: + +```text +root path = [chum 0] +child path = [chum parent_path] +``` + +This representation is used consistently across: + +- `Cold::register()` +- `Hot::init()` +- `Warm::init()` +- trace path rendering in `path_to_cord()` + +There are two important implications: + +1. Static hot paths and dynamic `%fast` paths must use the same encoding or warm rebuild cannot join them. +2. `chum` shape matters. The code currently does not validate `chum` beyond a TODO comment in `Cold::register()`. + +In practice, the surrounding code assumes `chum` is path-friendly: + +- an atom, usually printable text +- or a `[name version]` cell used by path formatting logic + +Malformed `chum` values may still register structurally, but they can break or degrade path rendering and logging. + +## Batteries and Batteries Lists + +The most important cold-state type is `Batteries`. + +Each `BatteriesMem` node stores: + +- `battery: Noun` +- `parent_axis: Atom` +- `parent_batteries: Batteries` + +Conceptually, a `Batteries` chain is a proof that says: + +```text +this core's battery is X +its parent is reached by axis A +that parent has battery Y +its parent is reached by axis B +... +eventually we reach a root +``` + +The root is marked by `parent_axis == 0`. + +`BatteriesList` is just a linked list of alternative ancestry chains for one path. Multiple chains may exist because the same path may correspond to multiple structurally distinct but valid candidates. + +## How `Batteries::matches()` Works + +`Batteries::matches(stack, core)` validates a live core against a stored ancestry chain. + +For each stored `(battery, parent_axis)`: + +1. If `parent_axis == 0`, the live `core` must unify with the stored root battery noun. +2. Otherwise: + - the live `core.slot(2)` must unify with the stored battery + - `core = core.slot(parent_axis)` becomes the next parent core + +If the walk reaches a root and all checks succeeded, the chain matches. + +This is the key correctness check in the whole design. Registration and jet lookup both depend on it. + +## How `Cold::register()` Works + +`Cold::register(stack, core, parent_axis, chum) -> Result` + +The return value means: + +- `Ok(true)`: new registration inserted +- `Ok(false)`: registration already existed or was intentionally ignored +- `Err(NoParent)`: a parent path could not be resolved +- `Err(BadNock)`: structural operations like `slot()` failed + +### Root registration + +If `parent_axis == 0`: + +1. Construct `root_path = [chum 0]`. +2. Check `root_to_paths[core]` to see whether that exact root/path pair already exists. +3. If not present: + - create a one-node `Batteries` chain with: + - `battery = core` + - `parent_axis = 0` + - `parent_batteries = NO_BATTERIES` + - prepend that chain to `path_to_batteries[root_path]` + - prepend `root_path` to `root_to_paths[core]` +4. Build a new `ColdMem` containing the updated HAMT roots. + +Notably, the root case does not update `battery_to_paths`. + +### Non-root registration + +If `parent_axis != 0`: + +1. Extract: + - `battery = core.slot(2)` + - `parent = core.slot(parent_axis)` + - `parent_battery = parent.slot(2)` +2. Check for an existing registration: + - look up `battery_to_paths[battery]` + - for each candidate path: + - compare the path head against `chum` + - if that matches, look up `path_to_batteries[path]` + - if any stored ancestry chain matches the full live `core`, the registration already exists +3. Resolve the parent path: + - first through `battery_to_paths[parent_battery]` + - then through `root_to_paths[parent]` +4. For each candidate parent path: + - fetch `path_to_batteries[parent_path]` + - use `BatteriesList::matches(parent)` to validate the actual parent core +5. For each successful parent match: + - construct `my_path = [chum parent_path]` + - create a new `Batteries` node containing: + - `battery` + - `parent_axis` + - `parent_batteries = matched parent chain` + - prepend that chain to `path_to_batteries[my_path]` + - prepend `my_path` to `battery_to_paths[battery]` +6. Build a new `ColdMem` containing the updated HAMT roots. + +If no parent candidate matches, registration returns `Err(NoParent)`. + +## Why the Cold State Uses Three Indices + +The three-way indexing exists because no single key is sufficient: + +- Path alone is useful for warm rebuild and tracing, but registration often starts from a core, not a path. +- Battery alone is useful for narrowing search, but the same battery can appear at multiple paths. +- Full core structural comparison is too expensive to use as the primary hash key. + +The current scheme is: + +1. Use outer battery or root as a coarse filter. +2. Enumerate candidate paths. +3. Use stored ancestry chains to prove the exact match. + +That design is visible directly in `Cold::register()`, `Cold::matches()`, and `Warm::find_jet()`. + +## HAMT Update and Equality Semantics + +The cold-state HAMTs are not used like a conventional immutable map keyed by pure equality. + +Important details: + +- `Hamt::lookup()` takes `&mut Noun` for the key because key comparison uses `unifying_equality()`. +- `unifying_equality()` is not just a read-only comparison. It can rewrite references so equal nouns become unified. +- `Hamt::insert()` returns a new `Hamt` root rather than mutating the old one in place. +- `Cold::register()` follows that style by assembling updated HAMT roots and then allocating a new `ColdMem` to hold them. + +That means the cold-state update pattern is functionally persistent at the top level, but equality checks during lookup and insertion are still allowed to mutate noun references for unification. + +This is a real implementation constraint, not an incidental detail. Code that uses cold-state HAMTs must tolerate mutating equality. + +## How Warm State Is Rebuilt + +If `%fast` registration returns `Ok(true)`, the interpreter immediately runs: + +```rust +context.warm = Warm::init(stack, cold, hot, &context.test_jets) +``` + +This rebuilds the entire warm table. + +`Warm::init()` does: + +1. Iterate every entry in `hot`. +2. For each hot path, call `cold.find(path)` to get the `BatteriesList` for that path. +3. For each batteries chain at that path: + - take the outer battery + - slot into that battery at the hot entry's axis + - use that resulting formula as the warm lookup key +4. Insert a `WarmEntry` containing: + - `batteries` + - `jet` + - `path` + - `test` flag + +Warm is therefore a formula-indexed cache derived from: + +- static jet declarations in `hot` +- runtime structure discovered in `cold` + +## How Warm Lookup Uses Cold Data + +During Nock 9, the interpreter: + +1. Computes the formula from the current core and axis. +2. Looks up candidate entries in `warm` by that formula. +3. For each candidate, uses `batteries.matches(subject_core)` to validate the live core. +4. Runs the jet if one matches. +5. Falls back to raw Nock if no candidate matches. + +This means: + +- `warm` is the fast lookup table +- `cold` provides the structural proof that keeps lookup sound + +Without the ancestry match, formula equality alone would not be sufficient to identify the right core. + +## How Cold Paths Are Used Outside Jet Lookup + +Cold also feeds tracing. + +The interpreter calls `cold.matches(stack, &mut res)` in trace-related paths after some Nock 9 activity. If it finds a path, that path is appended to the trace stack. + +`path_to_cord()` converts the nested noun-list path representation into a slash-separated cord. The tracing backend also extracts the leftmost `chum` for span naming. + +So `%fast` registration affects: + +- jet lookup +- trace labeling +- trace path rendering + +## Additional Consumer Constraint in `Site::new()` + +`Site::new()` does a warm lookup too, but it adds one more constraint: it inspects the first `parent_axis` in each batteries chain and requires that axis 7 be a prefix. + +The comment explains the intent: this avoids considering matches where the sample axis 6 would be part of the jet match. + +This is not part of `Cold::register()` itself, but it is part of the wider `%fast` contract because cached call-site jetting depends on the shape of the stored ancestry data. + +## Structural and Semantic Constraints + +### Explicit constraints in the code + +- `%fast` only does anything in the post-hint path. +- `%fast` is treated as non-tail. +- If the build uses `sham_hints`, `%fast` registration is disabled. +- Dynamic `%fast` requires a clue. A static `%fast` hint has no clue and will only log an error. +- The clue is expected to have meaningful values at slots 2 and 6. +- Nested parent hint wrappers are only unwrapped by following Nock 11 slot 7. +- Root registration is only accepted when the unwrapped parent formula has opcode 1 and axis 0. +- `Cold::register()` expects the result noun to support `slot(2)` and parent-axis navigation as a core. +- `chum` validation is explicitly missing. There is a TODO comment for this. +- `Warm::init()` rebuilds from scratch only when registration reports `Ok(true)`. + +### Implicit constraints in the code + +- Static hot paths and dynamic `%fast` paths must share the same noun encoding. +- `chum` values must be path-compatible in practice, even though the code does not enforce that. +- Parent registrations must happen before child registrations, or child registration fails with `NoParent`. +- The same outer battery can occur at multiple paths, so callers must tolerate candidate fan-out. +- Matching depends on `unifying_equality()`, which can mutate nouns to unify them. The code assumes all nouns passed into these comparisons are safe to unify. +- Matching relies on stable core structure. If the parent axis or core layout convention changes, registration and lookup both break. +- Cold grows monotonically. There is no pruning or delete path in this logic. +- Warm rebuild cost grows with: + - the size of the hot table + - the number of registered paths + - the fan-out of `path_to_batteries` +- The order of linked-list insertion affects lookup order. The implementation assumes that trying candidates sequentially is acceptable. +- Roots are keyed in `root_to_paths`, but ordinary trace lookup through `cold.matches()` starts from outer battery and path-to-batteries. Roots matter mainly for registration of descendants. + +## Representation Constraints + +The cold state is not just nouns. It is a graph of custom runtime structures: + +- `ColdMem` +- `Hamt` +- `NounList` +- `Batteries` +- `BatteriesList` + +Those structures contain raw pointers and embedded nouns. + +That matters because: + +- `Cold` is preserved across stack frame flips using `Preserve`. +- `Cold` is copied into PMA using `PmaCopy`. +- `Cold` can be serialized into a noun with `Cold::into_noun()` and rebuilt with `Cold::from_noun()` plus `Cold::from_vecs()`. + +The representation therefore carries strong locality assumptions. It is cheap to use in-memory, but it is not a naturally relocatable data structure. + +## Persistence and Serialization Constraints + +Cold construction interacts with persistence in several places: + +- `Cold` is copied into PMA as part of the persistent runtime state. +- PMA metadata stores a `cold_offset`. +- Snapshots validate and carry a `cold_offset`. +- Checkpoints convert cold to a noun and then copy that noun into a slab. + +Two constraints are worth calling out explicitly: + +1. The PMA representation of cold relies on pointer-bearing runtime structs being copied into PMA and later reopened. +2. The checkpoint path already contains a warning: + + > Cold state has nouns in it which are *not* copied in `into_noun` + +That warning is in `SaveableCheckpoint::new()` and documents an existing footgun in how cold is exported. + +These persistence details are not part of `%fast` matching logic itself, but they are part of the full cold-state contract because `%fast` registration is what builds the structure being persisted. + +## Failure Modes and Diagnostics + +The current code reports several failures through logging rather than hard failure: + +- no clue for `%fast` +- invalid root parent axis +- bad clue formula +- `NoParent` when parent lookup cannot be resolved +- non-atom or non-UTF-8 `chum` while formatting an error + +Operationally, the main outcomes are: + +- registration succeeds and warm is rebuilt +- registration is a duplicate and nothing changes +- registration fails and the runtime continues without that cold entry + +The steady-state effect of missing registrations is typically reduced jet coverage and reduced path information for tracing, not direct corruption of the kernel state. + +## Performance Characteristics + +The hot path cost of `%fast` registration is not just the insert itself. + +A successful registration can traverse: + +- the clue noun +- nested parent hint wrappers +- the result core +- the parent core +- candidate path lists from `battery_to_paths` +- candidate ancestry chains from `path_to_batteries` +- `Batteries::matches()` walks over the live core hierarchy + +Then, if the registration is new, it triggers a full `Warm::init()` rebuild over the entire hot table. + +So the dominant cost of `%fast` is often: + +- candidate-path fan-out during registration +- full warm rebuild after insertion + +not the single HAMT insert alone. + +## Mental Model + +The cleanest way to think about the system is: + +- `Hot` says which names have jets. +- `%fast` tells the runtime which live cores correspond to those names. +- `Cold` stores that discovered correspondence and the ancestry proof for it. +- `Warm` turns the combination of `Hot` and `Cold` into fast formula-indexed jet lookup. + +The core invariant is: + +> A jet is only sound to run when the runtime can prove that the live core belongs to the registered path associated with that jet. + +In this implementation, that proof is the ancestry chain stored in `Batteries` and validated by `Batteries::matches()`. diff --git a/docs/pma/FOLLOWUP-GC-LOOK-BEFORE-YOU-LEAP.md b/docs/pma/FOLLOWUP-GC-LOOK-BEFORE-YOU-LEAP.md new file mode 100644 index 000000000..c696e5f12 --- /dev/null +++ b/docs/pma/FOLLOWUP-GC-LOOK-BEFORE-YOU-LEAP.md @@ -0,0 +1,42 @@ +# PMA GC Follow-Up: Look Before You Leap + +## Context + +PMA GC currently compacts by copying reachable state from the active from-space PMA slab into the inactive to-space PMA slab, then switching the active slab metadata to point at the compacted PMA. The important detail is that the current copy algorithm uses forwarding pointers in from-space to preserve sharing and avoid duplicate copies. + +Those forwarding pointers intentionally mutate the source noun graph. Because the PMA is file-backed with a shared writable mapping, those mutations can become durable independently of the higher-level GC commit point. + +## Why the old active `.meta` cannot simply stay in place + +The `.meta` file is the boot-time authority that says a PMA slab is a valid root for the current kernel and event number. If the active from-space `.meta` remains present while GC is rewriting from-space with forwarding pointers, a crash or kill during GC can leave a bootable-looking PMA whose contents are no longer valid durable state. + +Forwarding pointers are process-local raw pointers into the to-space mapping, not stable PMA offsets that can be interpreted safely after restart. Keeping `from.meta` until `to.meta` is durable would only be safe if the compaction algorithm did not mutate from-space. + +## Current integrity-first ordering + +The current safe state machine is integrity-first rather than availability-first: + +1. Start with `from.meta` present and `to.meta` absent, so boot selects from-space. +2. Remove any stale inactive `to.meta` and sync the parent directory. +3. Create or truncate to-space. +4. Remove active `from.meta` and sync the parent directory before the first source mutation. +5. Copy reachable state from from-space to to-space, mutating from-space with forwarding pointers. +6. Persist and sync to-space data and trailer. +7. Atomically write `to.meta` and sync the parent directory. +8. Mark the to-space slab as active in memory. + +If the process dies between steps 4 and 7, there may be no valid active PMA and boot must recover from a ready snapshot plus event-log replay. This is expected for the current algorithm. + +## What LAX1 taught us + +The LAX1 incident should not be interpreted as proof that early `from.meta` deletion is wrong. Early deletion is the integrity-preserving choice for a mutating from-space collector. The practical failure was that snapshot fallback recovery was undermined by snapshot cleanup incorrectly treating tracked snapshot artifacts as orphaned and moving them into `corrupted_pma`; that bug has been fixed separately. + +LAX1 also had pathological storage behavior, which increased the probability of being killed inside expensive fsync/fdatasync windows. That made the GC kill window more visible, but it did not change the correctness constraint around forwarding pointers. + +## Recommended smaller follow-ups before changing GC ordering + +1. Strengthen the GC recovery-anchor check before deleting `from.meta`: do not accept only “there is a ready snapshot row”; cheaply verify that the active ready snapshot's PMA and manifest paths exist and that manifest/PMA metadata are readable and internally consistent. +2. Add comments around the `from.meta` deletion explaining that the deletion is deliberately before `copy_from_pma()` because the copy mutates from-space with non-durable forwarding pointers. +3. Add focused tests for the GC-in-progress state: after `from.meta` is removed and before `to.meta` is written, boot should reject both operative PMAs and fall back to a ready snapshot plus event-log replay. +4. Consider operational hardening separately: make the systemd wrapper `exec` the nockchain process and increase `TimeoutStopSec` so normal shutdown has enough time to complete PMA/snapshot sync work. +5. If we want atomic PMA availability across GC later, redesign compaction to avoid mutating from-space, for example with an external relocation table, a private/source scratch mapping, or another non-mutating copy strategy. That is a larger design change and should not be approximated by leaving `from.meta` in place with the current forwarding-pointer collector. diff --git a/docs/pma/NOCK-PMA.md b/docs/pma/NOCK-PMA.md new file mode 100644 index 000000000..dbb558744 --- /dev/null +++ b/docs/pma/NOCK-PMA.md @@ -0,0 +1,619 @@ +# Current status + +> Historical low-level design note. For current public-release guidance, prefer [`DESIGN.md`](./DESIGN.md), [`DURABILITY-OPERATIONS.md`](./DURABILITY-OPERATIONS.md), and [`NOUN-PROVENANCE-AND-BRANDED-HANDLES.md`](./NOUN-PROVENANCE-AND-BRANDED-HANDLES.md). + +The live nockvm still runs on the contiguous arena defined in `open/crates/nockvm/rust/nockvm/src/mem.rs`. That module owns the `Memory` abstraction (wrapping either `memmap2::MmapMut` or `malloc`) and the `NockStack`, a single slab that tracks `frame_offset`, `stack_offset`, and `alloc_offset` as word counts off of a base pointer. Every noun the VM manipulates lives inside that slab until it is explicitly copied into a PMA image. + +- `Memory::allocate` chooses mmap vs. malloc and hands back a base pointer that stays immutable for the life of the VM; all in-VM pointers today are literal `base + offset` derivations performed via `derive_ptr()`, not tagged offsets. + +- `NockStack` models both stack frames and bump-allocation via its `west`/`east` orientation flag, the `AllocationType` enum, and the `pc` (pre-copy) bit that gates when frame flipping or preservation can occur. The reserved slots at the bottom of each frame cache the previous frame/stack/alloc pointers so that copying collectors and frame pops can restore provenance without chasing raw pointers. + +- `open/crates/nockvm/rust/nockvm/src/noun.rs` consumes the stack API through `NounAllocator`, layering +the tag scheme (direct vs. indirect atoms vs. cells) and the forwarding-pointer rules that keep +structural sharing intact while slabs are copied between frames or into the PMA. Helper modules such +as `jets.rs` and `flog.rs` lean on `Preserve`/`preserve_with` from `mem.rs` to ensure nouns stay pinned +during host callbacks. + +- We have not yet switched the runtime over to offset-tagged references: any noun reloaded from a +persisted PMA still has to be patched up by rerunning `derive_ptr()` with the process-local base pointer. + +## A Young System's Programmer's Primer + +1. Read [https://doc.rust-lang.org/nomicon/](https://doc.rust-lang.org/nomicon/) and [https://blog.regehr.org/archives/213](https://blog.regehr.org/archives/213) +2. Meditate on the most vivid possible meaning of the "nasal demons" metaphor for undefined behavior and let it put the fear of God in you +3. Miri is enabled on every test except those it absolutely cannot be made to work for (hi tokio, ffi). If the test executes too slowly in Miri, your test is too slow. Make it faster or more "targeted" to the code coverage you need from Miri. + +Relevant history: + +## Epoch History + +1. 2025-03-28: PR #1167, titled “Offsets, not aliasing” and authored by Chris Allen (`@bitemyapp`). + - This branch (commit e4adb5a8c, 2025‑03‑28) is where the NockStack struct stopped storing live frame/stack/alloc raw pointers and instead began recording `frame_offset`, `stack_offset`, and `alloc_offset` word counts from the slab’s base pointer. The change also introduced `derive_ptr()`/`frame_pointer()` helpers so every access reconstructs a pointer from the base plus offset, and `MemoryState` now snapshots offsets instead of raw pointers (see history at commit e4adb5a8c affecting `open/crates/nockvm/rust/nockvm/src/mem.rs`) +2. 2025‑05‑19 · commit 00d288b1 · PR #1554 “Incremental hierarchy for hoonc” + - Focused on reducing allocator overhead when running hoonc by (a) allowing builds to short-circuit + OOM checks via a `no_check_oom` feature, (b) dropping the expensive assert_no_alloc::permit_alloc + scaffolding around pointer validity checks, and (c) rewriting prev_alloc_offset() to use a single + wrapping_sub instead of branching on the base pointer. Also simplified frame_pop’s null-pointer + panic to avoid heap allocations. Net effect: the stack allocator became leaner and more predictable + under hoonc's incremental compile workload. +3. 2025‑05‑26 · commit a61d3289 · PR #1664 “Least space metric” + - Added the `least_space` field to `NockStack`, threaded it through initialization, resets, and frame + flips, and updated both `west`/`east` allocation paths to maintain a running low-water mark. Exposed + a `least_space()` accessor so the runtime could export a gauge of minimum free words/bytes, enabling + Slam telemetry to flag stacks that are close to exhaustion. +4. 2025‑06‑27 · commit d01347fd · branch “test jets vs hoon” (squash merge) + - Extended the `Preserve` trait with a trivial implementation for `()`, which let the jets test harness + reuse preservation APIs without manufacturing dummy nouns. Small change, but it marked the first + divergence where preservation logic needed to tolerate no-op placeholders. +5. 2025‑07‑01 · commit 0013e50e · branch “Fix rust formatting in open/” + - Pure rustfmt/rust-analyzer cleanup of `mem.rs`: reordered use statements into the standard blocks + (std → crates → local) so the file complied with workspace formatting rules. No behavioral changes, + but it stabilized future diffs for readability. +6. 2025‑07‑24 · commit b6ebdc7a · branch “Tracing backends integration” + - Touched `frame_pop` and the debugging walkers to move from format!-style placeholders to the new + Rust inline formatting (`{ptr:p}`). This kept panic/log strings allocation-free and aligned with the + tracing backend expectations while keeping the underlying mechanics unchanged. +7. 2025‑09‑24 · commit 68b40a80 · branch “gRPC public API / light wallet” + - Updated the `NounAllocator` for `NockStack` impl so that callers using the allocator through the trait could invoke a new `equals()` hook. Under the hood it forwards to `crate::unifying_equality::unifying_equality`, ensuring components like the light-wallet gRPC service can compare nouns without downcasting to `NockStack`. +8. 2025‑10‑06 · commit c809688f · branch “hoonc benchmarking and prewarm best result” + - Largest post-introduction refactor: + * Marked `word_size_of` as #[inline] and pulled in `Vec` to support a heap-based worklist. + * Promoted `frame_push` to pub fn and inlined pop/top helpers for tighter hot-path codegen. + * Replaced the heavyweight NockStack::copy method (which reused the lightweight stack as a + worklist) with a new noun_preserve free function that uses a Vec<(Noun, *mut Noun)>. The new + routine bails early when the root is already direct, already forwarded, or outside the current + frame, dramatically reducing the amount of stack flipping during hoonc prewarm. Preservation + invariants (assert_acyclic, assert_no_forwarding_pointers, assert_no_junior_pointers) still bookend + the operation, but the worklist logic now lives entirely off-stack, improving determinism when the + allocator is hot. + * There was recently a weird interaction between the `axis` vs. `axis.form` issue and hoonc's prewarm bootstrap, at time of writing I'm not totally clear on how the dust settled there but I don't remember Logan saying prewarm in-and-of-itself was implicated, just flagging a risk. + +This foregoing history exemplifies the recent substantive architecture epochs for `mem.rs`: +- initial migration from sword +- partial offset-ification of NockStack indexes into the slab (cf. `e4adb5a8c`) +- perf hardening for hoonc +- observability of stack pressure +- and the more recent noun-preservation rewrite that decouples maintenance work from the lightweight stack. - Subsequent commits are mostly ergonomic or integration tweaks layered on that foundation. + +# Tooling, debugging, profiling + +Make sure your build and test/validation entrypoints you use to iterate on your work are batch-executable (meaning: not daemonized/persistent) Makefile entrypoints that "just work" out of the box with no additional steps required before-hand to make them complete successfully. + +- Memory safety, segfaults, use-after-frees, etc.: + + * ASAN on Linux (ask `@bitemyapp` how but the repo Makefile has breadcrumbs of me doing this) + * guard malloc on macOS ^^ + +- Memory leaks: + + * For Linux, I recommend `bytehound`. Same suggestions as the previous, there are breadcrumbs but ask me how. You'll probably need to flip the `Memory` type to using an ephemeral malloc if the problem you are diagnosing implicates the slab. You will think you have alternatives to `bytehound` for diagnosing leaks on Linux and I will be very surprised if that's true. You'll probably waste your time trying to find something better, I was not successful after many hours. If you find something nicer or better maintained please let me know. + + * macOS: Just use `cargo-instruments` and XCode. Frankly easier than Linux but might be slightly less informative/clear/precise than `bytehound` depending on your circumstances. Seems to work great for Mitchell Hashimoto across the board, idk man. I need to spend more time on it. + +- Performance: + * Cheap and cheerful, works for Linux and macOS, samples native runtime stacks: `samply record make run-my-benchmark-or-whatever`, you'll have to clear detritus threads from Cargo in the Firefox Profiler tab that spawns if the benchmark wasn't already built but the actual benchmark threads should be in there somewhere regardless. + * _There is no legacy tracing JSON profiling for Nock in NockVM_. If there is, I simply forgot to merge the branch deleting it. If it exists, delete it. Please don't use it. Tracy subsumes any need for this and it was wasting bytes and developer time. + * There, however, is _tracing for Nock in NockVM_: use tracy. [watch my youtube video](https://www.youtube.com/watch?v=Z1UA0SzZd6Q) + * Linux only unified nock + native stack profiling: tracy profiler again. make sure you align the locally compiled version of the tracy profiler GUI and the library version in the Cargo project. Look at the repo makefile. + - `macOS` works with Tracy fine, client and server-side but you're going to get the nock traces by default, native (20 khz!) stack samples in Tracy only work on Windows and Linux. No, I don't know why they refuse to support it on macOS. Because they definitely could. They just choose not to. This has a solution: _use Docker_ (do I need to say it again? Look at the Makefile targets and Dockerfile I wrote for this) + - I would strongly encourage you to take advantage of `tracy`'s `ondemand` mode (look at the Cargo features specified for `tracing-tracy`) so that you aren't eating the profiling overhead when the nockapp first boots and loads the slab, but I won't blame you if that's more faffing about than you have patience for. + +# Writing tests gooder + +This is all speaking to Rust norms and structural conventions. I don't care what Uncle Bob thinks an integration test is. Don't tell me, I don't want to know either. + +## Unit tests live in the library modules + +## Integration tests live in separate binaries + +You see all the Rust test cases in `tests/` sub-directories? That's what makes them an integration test. Importantly, _you can have multiple test cases in a single integration test binary_. Too many integration test binaries increase linker surface area, please don't exacerbate that. + +Reasons you'd use an integration test: +# Milestone 1: Offset-addressing + +You need to have a pointer representation that can be used from the Nock code which addresses other objects as offsets from a static (not constant) base pointer (base address). + +Some of this work happened in Chris's earlier offset branch, but it isn't complete and we're still leaning heavily on `derive_ptr()` because Chris didn't want to add a new tag bit or churn the rest of the runtime. That time has passed and we need to rip the bandaid off and finish the other 80% of the work now to set the stage for position-independent addressing for a persistent mmap slab. + +We're going to mmap the PMA, let the system decide where the map the PMA, let the system decide the base address. + +The base address is universal and singular to the PMA slab used for the nockvm instance. + +The NockVM runtime will still be using direct pointers constructed using base address + offset arithmetic to dereference Noun nodes in the PMA. However, the PMA itself will work purely in terms of position-independent addressing which is all offset based so that if you reload the mmap-based PMA slab from disk all the offsets are still valid and simply recalculated in terms of the new base address you got from the virtual memory subsystem of your platform. + +We will need to use pointer tags to distinguish between pointers and offsets. + +On a read you branch on the pointer tag bits and you variously: + +- strip the tag bits, and dereference the resulting pointer +- strip the tag bits, add the offset to a base address, cast that into a pointer, and dereference that pointer + +Discriminant is a single bit in the tag. Signifies whether the reference is in the PMA or in the nockstack noun slab. There's a separate discriminator bit already in the Noun representation for distinguishing Atoms, IndirectAtoms, and Cells. For our purposes, we care about values vs. references. + +After you've established whether the tag bits signify whether a value is a direct pointer to a noun entrypoint or a PMA offset, you now need to to distinguish whether the + +## Milestone 1 discriminant bits hypothetical diff + +### Current status / before milestone 1 + +Before (current master) ― four discriminants only: direct atom, indirect atom, cell, forwarding pointer. + +All allocated variants currently hold raw pointers produced by `NockStack::derive_ptr()` (`open/crates/nockvm/rust/nockvm/src/mem.rs`). + +```rust +/// Mirrors the actual constants in noun.rs. +const DIRECT_MASK: u64 = !(u64::MAX >> 1); // 0x8000_0000_0000_0000 +const DIRECT_TAG: u64 = 0; + +const INDIRECT_MASK: u64 = !(u64::MAX >> 2); // 0xC000_0000_0000_0000 +const INDIRECT_TAG: u64 = u64::MAX & DIRECT_MASK; // pattern 10xx... + +const CELL_MASK: u64 = !(u64::MAX >> 3); // 0xE000_0000_0000_0000 +const CELL_TAG: u64 = u64::MAX & INDIRECT_MASK; // pattern 110x... + +const FORWARDING_MASK: u64 = CELL_MASK; +const FORWARDING_TAG: u64 = u64::MAX & CELL_MASK; // pattern 111x... + +#[repr(transparent)] +#[derive(Clone, Copy)] +pub struct Noun { + raw: u64, +} + +impl Noun { + #[inline] + fn tag_bits(self) -> u64 { + match self.raw & DIRECT_MASK { + DIRECT_TAG => DIRECT_TAG, + _ => self.raw & CELL_MASK, // covers indirect/cell/forwarding + } + } + + #[inline] + fn payload_bits(self) -> u64 { + match self.tag_bits() { + DIRECT_TAG => self.raw, // value <= DIRECT_MAX + INDIRECT_TAG => self.raw & !INDIRECT_MASK, // pointer to IndirectAtom + CELL_TAG => self.raw & !CELL_MASK, // pointer to CellMemory + FORWARDING_TAG => self.raw & !FORWARDING_MASK, // pointer to Allocated + _ => unreachable!(), + } + } +} +``` + +This is exactly what the current noun representation does: a single u64 word whose top three bits +distinguish values, indirect atoms, cells, or transient forwarding pointers; every allocated case stores +a literal pointer to stack memory. + +#### Sidebar about discriminant bits + +Can we please just use a safer library for doing this instead of doing it by hand? There's no performance or clarity downside unless they're doing something dumb. It's an unforced error to keep doing this raw when we're in-progress on changing the design anyway. + +Short‑list after surveying the current ecosystem (no code, just tradeoffs): + +#### Before (just type tags: direct vs indirect vs cell vs forwarding) + +- `bitflags` (1.4+): still the cleanest zero‑cost way to name the masks and expose helper methods. Gives + you a readable bitflags! { struct NounTag: u64 { … } } and keeps the rest of the code close to what we + already have, just without hand‑rolled constants. +- `bitfield-struct`: generates getters/setters for named bit ranges in a `repr(transparent)` wrapper. Useful + if you want a tidy `struct NounBits { #[bits = 1] kind: u8, … }` but don’t want a macro DSL as heavy as + `modular-bitfield`. +- If you want to stick with enums, `strum` + `num_enum::TryFromPrimitive` can encode the three tag states + into an enum without rolling your own match ladder; it’s still zero cost once optimized. (in...theory. in practice I use bit-masking in some places so an enum could give me heartburn later.) + +#### After (type tags + “stack pointer vs PMA offset” location bit) + +- `bitflags` still works here, but pairing it with `bytemuck::TransparentWrapper` lets you define a + `TaggedPtr(u64)` newtype and safely reinterpret between masks/payloads and raw words, which makes the + pointer/offset split less error‑prone. +- `bitfield-struct` or `modular-bitfield` both shine once you have two orthogonal fields (kind + location). + They emit getters that return plain integers, so you branch on location() without remembering which bit + it lives in, and the generated code is just a few shifts/masks. +- For the pointer/offset arm specifically, tagged-pointer (crate) can encode the “pointer with spare + high bits” case; you would still keep your own offset handling, but it gives you a typed wrapper with + compile‑time guarantees that the high bit is reserved for tagging. +- If you’d rather treat the tag word as a mini struct, packed_struct lets you declare + `#[packed_struct(bit_numbering = "msb0")] struct NounBits { #[packed_field(bits="0:0")] location: bool, + … }` and the derive does the rest. Slightly heavier macro, but great when you need to document the + layout inline. + +Bottom line: for the current “before” layout, `bitflags` (optionally with a thin newtype) keeps things +minimal. Once you add the location bit in Milestone 1, stepping up to a `bitfield` derive (`bitfield-struct`, `modular-bitfield`, or `packed_struct`) or a purpose-built tagged-pointer wrapper gives you clearer semantics without runtime cost, and you can choose whichever macro style fits your tolerance for abstraction. + +#### Validating which bitfield/bitflag crate to use + +Assuming they're not messing up the target representation (make note of any applications of `repr` in `noun.rs` in the git history) or outright buggy, it should come down to perf/unforced overhead. + +Get some basic operations (like the case discrimination helpers, chewing through IndirectAtoms of cords, etc.) lifted into a `criterion` benchmark harness, implement all the variations of the same minimal target representation with these verbs attached, and horse-race them with the benchmarks. + +If the benchmarks confuse you grab `@bitemyapp` as he will greatly enjoy being confused with you. I'm not expecting them to be different unless the underlying representations are different. + +Oh yeah, and write tests that verify the exact bit representation of the noun values for each tag-bit discriminant/scenario/type. + +### Discriminant bits / Noun repr after Milestone 1, hypothetical diff + +After (Milestone 1) ― same value/allocated/forwarding taxonomy, but add a location bit so we can +distinguish direct stack pointers from PMA-relative offsets. Reads first branch on the location bit, then +interpret the payload as either a raw pointer (stack slab) or a word offset to be rebased through the PMA +base pointer supplied by `NockStack`. + +The distinction is between the nursery (not persistent, will get thrown away on a stack flip if not permanently allocated) and the persistent non-nursery part of the area (it survived nursery generation on a stack flip/preserve because it was permanently allocated). This distinction exists in the previous system but there was no "persistence" entailed in surviving the nursery and reaching the slab permanently. + +```rust +const LOCATION_BIT: u64 = 1 << 60; // next free bit above CELL_MASK +const VALUE_MASK: u64 = !(DIRECT_MASK | LOCATION_BIT); + +#[derive(Clone, Copy)] +enum PtrKind { + StackPtr(*mut u8), + PmaOffset(u32), // word index inside mmap’d PMA slab +} + +impl Noun { + #[inline] + fn pointer_descriptor(self) -> Option<(u64 /* tag */, PtrKind)> { + let tag = self.tag_bits(); + if tag == DIRECT_TAG { + return None; + } + + let payload = self.raw & VALUE_MASK; + let ptr = if self.raw & LOCATION_BIT == 0 { + PtrKind::StackPtr(payload as *mut u8) + } else { + PtrKind::PmaOffset(payload as u32) + }; + + Some((tag, ptr)) + } + + #[inline] + fn resolve_cell<'a>(&self, base: *const u8) -> Option<&'a CellMemory> { + let (tag, descriptor) = self.pointer_descriptor()?; + if tag != CELL_TAG { + return None; + } + + match descriptor { + PtrKind::StackPtr(ptr) => Some(unsafe { &*(ptr as *const CellMemory) }), + PtrKind::PmaOffset(words) => { + let ptr = unsafe { base.add((words as usize) << 3) } as *const CellMemory; + Some(unsafe { &*ptr }) + } + } + } +} +``` + +This “after” block is the hypothetical partial diff you can paste into `NOCK-PMA.md`: it keeps the +exact bit patterns the runtime already relies on, but demonstrates how Milestone 1 splits the allocated +payload into “stack pointer” vs “offset into PMA,” which is the key new discriminant the doc needs to +communicate. + + +# Milestone 2: Persistence + +Using mmap to persist to disk. We will be assuming only a single reader/writer +for now (Milestone 5 is concurrent reads). + +This consists of two phases: + +## Phase 1 + +Phase 1 is to separate out the NockStack from the arena. + +``` + + ┌──────────────────────┐ ┌──────────────────────┐ + │ NockStack │ │ PMA │ + │(ephemeral, anon mmap)│ │ (persistent, file) │ + │ │ │ │ + │ [frames][stk→ ←alloc]│ │ [bump-allocated │ + │ │ │ nouns in offset │ + │ Cleared after each │ │ form] │ + │ event │ │ │ + │ │ │ Loaded at boot, │ + │ Stack-pointer form │ │ persisted to disk │ + │ only │ │ │ + └──────────────────────┘ └──────────────────────┘ + │ ▲ + │ evacuate_to_pma() │ + └────────────────────────────┘ +``` + +We need to push the persistent arena to a memory slab that is bump-allocated at +the page level. As things stand now, NockStack lives in an anonymous mmap. + +Currently, at the end of every event, NockVM is left with a single stack frame, +the top frame, and a bunch of data to be preserved - the kernel, jet states, and +cache. `preserve()` gets called on all of these, which copies them to the other +side of the memory arena, where then any Nouns that are in stack-pointer form +are retagged into offset form. + +This step is to be replaced with a new copying step, into a file-backed mmap +called the persistent memory arena (PMA). + +Phase 1 will be complete when data is being copied into the PMA at the +conclusion of each event, and NockStack works with references into the PMA for a +single writer/reader. + +### Phase 1 spec + +Here is a more detailed spec for phase 1: + +The central struct for the PMA. `alloc_offset` uses `usize` for now since there +is only one reader/writer, but we will move to `AtomicUsize` when multiplayer +gets enabled. +```rust +/// The Persistent Memory Arena +/// +/// A bump-allocated memory region for storing nouns in offset form. +/// The PMA is backed by a file (in future milestones) and persists across +/// program restarts. +/// +/// "Bump-allocated" means allocation simply increments the `alloc_offset` +/// pointer by the requested size—there is no free list, no compaction, and +/// no mechanism to reclaim memory once allocated. This makes allocation +/// extremely fast (just a pointer bump) but means the PMA grows monotonically +/// until explicitly reset. +/// +/// When a Noun that lives in the PMA needs to be modified, the workflow is: +/// 1. The Noun is read from the PMA (already in offset form) +/// 2. Modifications happen in the NockStack (ephemeral working memory) +/// 3. The modified Noun is copied back to the PMA via `copy_to_pma()` +/// +/// Step 3 only allocates space for the Allocated subtrees that changed. For +/// example, if `[2 3]` becomes `[4 3]`: +/// - The Cell is Allocated, so a NEW cell is allocated in the PMA with head=4, +/// tail=3 with new DirectAtoms for the 4 and 3 since they are not Allocated. +/// - The old `[2 3]` cell remains in the PMA, untouched but now unreachable +/// +/// For Allocated structures, unchanged subtrees that are already in PMA (offset +/// form) are reused without copying. If `[[1 2] 3]` becomes `[[1 2] 4]`: +/// - A NEW outer cell is allocated with tail=4 +/// - The head still points to the existing `[1 2]` in PMA (no copy needed) +/// - Only the old outer cell becomes garbage; `[1 2]` is shared +/// +/// This copy allocates fresh space in the PMA for the new version—the old +/// version is not overwritten or freed, it simply becomes unreachable garbage. +/// Garbage collection (Milestone 4) will eventually reclaim this dead space. +/// +/// Currently Pma is only suitable for a single reader/writer. In the future, +/// `alloc_offset` will be changed to `AtomicUsize` to allow multiple readers. +pub struct Pma { + /// The underlying arena for memory management and pointer resolution + arena: Arc, + /// Current allocation offset in words (bump pointer) + alloc_offset: usize, + /// Path to the backing file (for future file-backed persistence) + path: PathBuf, +} +``` + +As the `Pma` is a place where `Noun`s get allocated, it ought to implement +`NounAllocator`: +```rust +impl NounAllocator for Pma { ... } +``` + +There is a `PmaError` enum for `Result` types coming out of the PMA. +```rust +#[derive(Debug, Error)] +pub enum PmaError { + #[error("PMA is full, cannot allocate {requested} words (available: {available})")] + OutOfMemory { requested: usize, available: usize }, + + #[error("PMA not installed in thread-local storage")] + NotInstalled, + + #[error("Failed to create arena: {0}")] + ArenaError(#[from] NewStackError), +} +``` + +Everything that lives in a NockStack that we'd like to live in the PMA +implements the `PmaCopy` trait: +```rust +pub trait PmaCopy { + /// Copy this value into the PMA. + /// + /// For nouns, this evacuates allocated data (indirect atoms, cells) to the PMA + /// and converts pointers to offset form. Direct atoms are unchanged since they + /// fit in a single word. + /// + /// # Safety + /// The caller must ensure that the stack's arena is installed in thread-local storage. + unsafe fn copy_to_pma(&mut self, stack: &NockStack, pma: &mut Pma); + + /// Assert that this value is fully contained within the PMA. + /// + /// For nouns, this verifies that all allocated data (indirect atoms, cells) + /// resides in the PMA. Direct atoms always pass since they have no allocations. + /// + /// # Panics + /// Panics if any part of this value is not in the PMA. + fn assert_in_pma(&self, pma: &Pma); +} +``` + +`PmaCopy` is implemented for the following types: +```rust +// nouns +impl PmaCopy for Noun { ... } // Calls copy_noun_to_pma below +// The rest of the Noun types probably just call .as_noun().copy_to_pma() +impl PmaCopy for Atom { ... } +impl PmaCopy for IndirectAtom { ... } +impl PmaCopy for DirectAtom { ... } +impl PmaCopy for Allocated { ... } +impl PmaCopy for Cell { ... } +// cache +impl PmaCopy for Hamt { ... } +// jet state +impl PmaCopy for Warm { ... } +impl PmaCopy for WarmEntry { ... } +impl PmaCopy for Hot { ... } +impl PmaCopy for Batteres { ... } +impl PmaCopy for BatteriesList { ... } +impl PmaCopy for NounList { ... } +impl PmaCopy for Cold { ... } +``` +I'm not sure about this one, but `Retag` is implemented for it so I've done it. +```rust +impl PmaCopy for () { ... } // Ctrl-F d01347fd for why this is implemented for Preserve. It also implements + // Retag which makes me think it probably will be. +``` + +The main function to accomplish copying to the PMA for `Nouns`. Something like +this: +```rust +impl PmaCopy for Noun { + /// Copy a noun and all its allocated substructure to the PMA. + /// + /// Uses a worklist algorithm to avoid stack overflow on deep structures. + /// Structural sharing is preserved via forwarding pointers: if the same + /// substructure is referenced multiple times, it's only copied once. + /// + /// # Algorithm + /// 1. Push (noun, destination_ptr) onto worklist + /// 2. Pop and process each item: + /// - Direct atoms: write directly to destination + /// - Already in PMA (offset form): write directly to destination + /// - Has forwarding pointer: write forwarded offset-form to destination + /// - Indirect atom: copy to PMA, set forwarding pointer, write offset-form + /// - Cell: copy metadata to PMA, set forwarding pointer, queue head/tail + /// + /// # Safety + /// - The PMA arena should be installed for reading evacuated nouns afterward + /// - Source nouns will have forwarding pointers set (corrupting the stack data) +... +} +``` + +#### Tests +Summary of tests implemented: + +```rust + // Verifies bump allocation returns sequential offsets and correctly tracks free space. + fn test_pma_allocation() { ... } + // Verifies offset-to-pointer and pointer-to-offset conversions are inverses of each other. + fn test_pma_offset_round_trip() { ... } + // Verifies reset() clears the allocation pointer and reset_to() sets it to a specific offset. + fn test_pma_reset() { ... } + // Verifies thread-local PMA installation, access via with_current(), and cleanup via clear. + fn test_pma_thread_local() { ... } + // Verifies direct atoms are unchanged by evacuation since they fit in a single word. + fn test_evacuate_direct_atom() { ... } + // Verifies indirect atoms (too large for direct representation) are copied to PMA and converted to offset form. + fn test_evacuate_indirect_atom() { ... } + // Verifies a simple cell with direct atom contents is evacuated and readable from PMA. + fn test_evacuate_simple_cell() { ... } + // Verifies nested cell structures are fully evacuated with all sub-cells in offset form. + fn test_evacuate_nested_cells() { ... } + // Verifies cells containing indirect atoms have both the cell and atoms correctly evacuated. + fn test_evacuate_with_indirect_atoms() { ... } + // Verifies structural sharing is preserved: [x x] evacuates x only once, with both refs pointing to same PMA location. + fn test_evacuate_shared_structure() { ... } + // Verifies sharing is preserved across separate evacuate calls via forwarding pointers left in stack memory. + fn test_evacuate_multiple_nouns_preserves_sharing() { ... } + // Verifies evacuating an already-evacuated noun is a no-op that allocates nothing. + fn test_evacuate_already_evacuated() { ... } + // Verifies deeply nested structures are fully evacuated and traversable after evacuation. + fn test_evacuate_deep_tree() { ... } + // Verifies contains_ptr correctly identifies pointers inside vs outside the PMA memory region. + fn test_pma_contains_ptr() { ... } + // Verifies allocation fails gracefully when PMA is full, rolling back the failed allocation. + fn test_pma_out_of_memory() { ... } + // checks that allocating in PMA bumps the alloc ptr + fn test_persistent_arena_allocation_is_monotonic() { ... } + // checks NockStack is empty after moving noun to PMA, + fn test_pma_preserve_moves_noun_and_resets_stack() { ... } + // does a HAMT preserve work? + fn test_preserve_hamt_round_trip() { ... } + // HAMT evacuate with Cells as values and IndirectAtoms as keys + fn test_evacuate_hamt_complex_nouns() { ... } + // jet state round trip tests + fn test_evacuate_warm_round_trip() { ... } + fn test_evacuate_warm_entry_round_trip() { ... } + fn test_evacuate_hot_round_trip() { ... } + fn test_evacuate_batteries_round_trip() { ... } + fn test_evacuate_batteries_list_round_trip() { ... } + fn test_evacuate_noun_list_round_trip() { ... } + fn test_evacuate_cold_round_trip() { ... } +``` +Tests not yet implemented: + +##### Memory alignment and layout: +- `test_evacuate_indirect_atom_alignment` - Verifies indirect atoms of various sizes (1, 2, 3, 7, 8, 9 words) + are properly aligned in PMA and readable without alignment faults. +- `test_evacuate_cell_memory_layout` - Verifies CellMemory fields (metadata, head, tail) are at correct +offsets after evacuation by reading each field independently. + +##### Forwarding pointer edge cases: +- `test_forwarding_pointer_diamond_sharing` - Verifies diamond-shaped DAGs (A→B, A→C, B→D, C→D) preserve all +sharing and D is only copied once. +- `test_forwarding_pointer_wide_sharing` - Verifies a single noun referenced by many (e.g., 100) different +cells is only copied once. +- `test_forwarding_pointer_not_leaked_to_pma` - Verifies no forwarding pointers remain in PMA memory after +evacuation completes (they should only exist transiently in stack memory). + +##### Boundary and edge cases: +- `test_evacuate_maximum_depth_tree` - Verifies evacuation handles very deep trees (e.g., 1000 levels) +without stack overflow in the worklist loop. +- `test_evacuate_large_indirect_atom` - Verifies indirect atoms near the maximum representable size evacuate +correctly. +- `test_evacuate_single_word_indirect_atom` - Verifies the smallest possible indirect atom (just over +DIRECT_MAX) evacuates correctly. +- `test_evacuate_mixed_pma_stack_noun` - Verifies a cell where head is already in PMA and tail is on stack +evacuates correctly (only tail gets copied). + +##### Use-after-evacuation (Miri should catch these): +- `test_stack_memory_not_accessed_after_evacuation` - After evacuation, verify that reading the evacuated +noun uses PMA memory, not stack memory (may need to poison/zero stack to detect). +- `test_evacuate_then_pop_frame_then_read` - Evacuate a noun, pop the stack frame that contained it, then +read from the PMA copy - should work without accessing freed memory. + +##### Concurrent allocation: +For now we are assuming the PMA has only a single writer/reader, so we won't +implement these tests, but they are listed for future reference. +- `test_concurrent_pma_allocation` - Spawns multiple threads that allocate from PMA simultaneously, verifies +no overlapping allocations and total allocated equals sum of individual allocations. +- `test_concurrent_allocation_under_pressure` - Multiple threads racing to allocate when PMA is nearly full, +verifies OOM errors are returned correctly without corruption. + +##### Idempotency and repeated operations: +- `test_evacuate_same_noun_twice_same_call` - Passes the same noun pointer twice in succession; second call +should be pure no-op. +- `test_evacuate_after_pma_reset` - Evacuate, reset PMA, evacuate same structure again - verifies clean +re-evacuation without confusion from old data. + +##### Memory initialization: +- `test_evacuated_metadata_initialized` - Verifies cell metadata is properly copied (not uninitialized) by +checking mug cache bits after evacuation. +- `test_evacuated_indirect_atom_padding_zeroed` - For indirect atoms that don't fill their last word +completely, verify padding bytes are deterministic. + +##### Invalid input detection (debug assertions): +- `test_evacuate_rejects_cyclic_structure` - Verifies the assert_acyclic! macro fires when given a cyclic +noun (if we can construct one). +- `test_evacuate_rejects_existing_forwarding_pointer` - Verifies `assert_no_forwarding_pointers!` fires when +given a noun with pre-existing forwarding pointers. + +##### Arena switching correctness: +- `test_read_pma_noun_with_wrong_arena_installed` - Verifies reading an evacuated noun with the stack arena +(not PMA arena) installed produces incorrect/detectable results or panics. +- `test_arena_switch_mid_traversal` - Verifies that traversing a noun tree requires consistent arena +installation throughout. + +## Phase 2 + +Once we have successfully separated out NockStack from the PMA, we need to +actually implement the ability to load the PMA from disk and make use of it in +ordinary operation of the NockVM. + +# Milestone 3: Mutation and freeing + +# Milestone 4: Garbage collection + +# Milestone 5: Concurrent reads diff --git a/docs/pma/NOCKSTACK-ATTRIBUTION.md b/docs/pma/NOCKSTACK-ATTRIBUTION.md new file mode 100644 index 000000000..817017aae --- /dev/null +++ b/docs/pma/NOCKSTACK-ATTRIBUTION.md @@ -0,0 +1,91 @@ +# PMA / NockStack / Heap Attribution Plan + +Below is a targeted, actionable plan to bucket memory into **NockStack**, **PMA**, and **heap/other anon**. This is designed to work in Docker or on host, and gives you both a point‑in‑time snapshot and a time‑series view. + +**Plan Overview** +- **PMA**: file‑backed mappings under `.../pma/*.mmap` → derive resident % from `smaps` or `mincore`. +- **NockStack**: the single anonymous `mmap` sized to `NOCK_STACK_SIZE * 8` bytes → find by size + anon mapping. +- **Heap/other anon**: `[heap]` + remaining anonymous mappings (excluding NockStack and thread stacks). + +--- + +## 1) Get the exact NockStack size (bytes) +Find the stack size used by your runtime (e.g. `StackSize::Normal`), then compute bytes: + +- Find the size constant: + - `rg -n "StackSize|NOCK_STACK_SIZE" crates/nockapp/src/kernel/boot.rs` +- Compute: + - `stack_bytes = stack_words * 8` + +This value is what you’ll match against `smaps` mappings. + +--- + +## 2) Collect `smaps` inside the container +Inside the container you usually can read `/proc/1/smaps` without host root. + +``` +docker exec -it sh -c 'cat /proc/1/smaps' > /tmp/smaps.txt +``` + +--- + +## 3) Bucket the mappings (PMA / Stack / Heap) +Use `smaps` to sum RSS and Size into buckets. Heuristic rules: + +- **PMA**: pathname contains `/pma/` and ends with `.mmap` +- **NockStack**: anonymous mapping (no pathname), `rw-p`, `Size ~= stack_bytes` + (allow a small tolerance, e.g. ±1–2 pages) +- **Thread stacks**: `[stack]` or `[stack:]` → ignore or report separately +- **Heap/other anon**: `[heap]` + remaining anonymous `rw-p` that aren’t the stack mapping + +If you want a quick oneliner to sanity check: +``` +grep -n "pma/.*\\.mmap" -A20 /tmp/smaps.txt +``` + +--- + +## 4) PMA residency ratio (percent in RAM) +Two options: + +**A) From smaps** +For PMA mappings: `rss_ratio = Rss / Size`. + +**B) From mincore/vmtouch** +If you can install it in the container: +``` +vmtouch -v /data/.data.nockchain/pma/*.mmap +``` +This gives a direct resident‑pages %. + +--- + +## 5) Time‑series: sample every N seconds +Sample buckets every 5–10 seconds during sync and again after sync slows: + +- Phase A: startup / initial sync +- Phase B: mid‑sync +- Phase C: post‑sync idle + +This will show: +- **PMA “hotness”** (resident ratio rising/falling) +- **NockStack high‑water** (stack RSS plateau) +- **Heap/other anon** growth (allocator churn, slabs, jam buffers) + +--- + +## 6) Optional: control checkpoint noise +Even though you said checkpoints are “transparent enough,” for clean baselines: +- Run once with save interval disabled: `--save-interval 0` +- Run once with normal saves enabled +Compare heap/anon spikes to see checkpoint amplification. + +--- + +## What this tells you +- If **PMA** is doing what we want: PMA mapping size grows, but **resident % should drop** when things go cold or under pressure. +- If **NockStack** is the issue: you’ll see a single large anon map with RSS near its size and never dropping. +- If **heap/other anon** is the issue: that bucket grows (NounSlab allocations, jam buffers, libp2p, etc.). + +If you want, I can wire this into the existing `compare_pma_mem.rs` to output all three buckets (plus PMA residency %) on each run. That would make the time‑series trivial. diff --git a/docs/pma/NOUN-PROVENANCE-AND-BRANDED-HANDLES.md b/docs/pma/NOUN-PROVENANCE-AND-BRANDED-HANDLES.md new file mode 100644 index 000000000..88a84ad95 --- /dev/null +++ b/docs/pma/NOUN-PROVENANCE-AND-BRANDED-HANDLES.md @@ -0,0 +1,114 @@ +# Noun Provenance and Branded Handles + +This note consolidates the useful parts of the old `ALIEN-NOUNSPACE-AUDIT.md` and `ALIEN-NOUNSPACE-AUDIT-FOLLOW-UP.md` into a current-state explanation of the problem branded noun handles are meant to solve. + +## Summary + +A raw `Noun` is only a tagged word. For allocated nouns, the payload is meaningful only relative to the arena that owns it: + +- a NockStack arena, +- a PMA arena, +- a NounSlab allocation range, or +- another explicit `NounSpace` that can resolve the pointer or offset. + +PMA makes this invariant unavoidable. Stack pointers, PMA offsets, and slab pointers can all appear in ordinary runtime flows, but they are not interchangeable. A raw noun copied out of its owner and later decoded with the wrong `NounSpace` can become: + +- a dangling pointer, +- an offset resolved against the wrong PMA, +- a mixed tree whose root is local but whose children point elsewhere, +- a value that works in today's call path only because the original owner happens to stay alive nearby. + +The audit found that the main remaining risk was not one isolated bug. It was an API pattern: functions accepted or returned raw `Noun` values after having just used a `NounSpace` to decode them. That erases provenance at exactly the boundary where Rust types should preserve it. + +`NounHandle<'a>` is the first mitigation: it pairs a raw noun with `&'a NounSpace`, so common traversal and decoding helpers can keep the owner nearby. + +Branded handles go a step further. `NounSpace::with_brand(...)` creates a generative brand, and `BrandedNounHandle<'space, 'id>` carries that brand. Code inside the branded scope can only combine handles from the same branded space, and branded handles cannot escape the scope. This is intended to make accidental cross-arena mixing fail at compile time rather than at PMA dereference time. + +## Bug Family + +The audit grouped issues under the term "alien noun" or "alien NounSpace". The concrete patterns were: + +1. Returning a raw `Noun` whose owner has already been dropped. +2. Storing raw `Noun` fields in owned decoded structs without storing the owner. +3. Iterating lists/maps as `Noun` after accepting a `NounSpace` or `NounHandle`. +4. Rebuilding new cells/lists in one allocator from raw children owned by another allocator. +5. Reconstructing a `NounSpace` after the fact from raw pointer ranges. +6. Validating only the root of a tree, while reachable descendants may still live in other arenas. + +These are all the same logical problem: arena provenance is not represented in the type that crosses the API boundary. + +## Current State of the Original Findings + +### Fixed or materially improved + +- `IntoNoun for &str` was a real dangling-pointer constructor: it allocated into a temporary `NounSlab`, returned the raw noun, and dropped the slab. This is now banned with a negative impl (`impl !IntoNoun for &str {}`). Use allocator-taking APIs or slab-producing APIs instead. + +- `noun-serde` used to identity-encode and identity-decode raw `Noun`. That allowed typed decode to erase provenance and typed encode to smuggle foreign nouns into new structures. Raw `Noun` now has negative impls for `NounEncode` and `NounDecode`. + +- Several slab modification paths now re-home returned nouns before installing them under a slab root. This makes `NounSlab::modify(...)`, `modify_noun(...)`, and `modify_with_imports3(...)` much safer for stack-pointer-form nouns. + +- The block explorer has moved much of its parsing toward handle-based helpers. Many helpers now accept `&NounHandle`, and `HoonMapIter` yields `NounHandle` rather than raw nouns. + +- The math-side situation is improved by the presence of handle-shaped traversal APIs such as `NounMathExtHandle::uncell(...)` and handle-yielding map iteration. + +- Cold-state restore is less dangerous than the original audit suggested because the live checkpoint restore path copies checkpoint cold state into the active stack before decoding it. + +### Still worth treating as hazardous API surface + +- `NounSlab::set_root(...)` validates the top-level allocated root, not the full reachable graph. A root cell allocated in the slab can still theoretically contain descendants from another arena if a caller bypasses the re-homing helpers and constructs the cell manually. + +- `NounSlab::rehome_noun(...)` can re-home stack-pointer-form nouns, but it cannot safely import PMA offset-form nouns without a source `NounSpace`. Callers that may receive PMA nouns need an explicit copy/import API that carries the source space. + +- `NounSlab`'s public `NounMap` stores raw noun keys and asks callers to provide a `NounSpace` later. Current uses appear internal to jamming/backrefs where the owner stays live, but the public shape still erases provenance. + +- `nockchain-math::NounMathExt::uncell(...)` still returns raw `[Noun; N]` after taking a `&NounSpace`. The handle variant should be preferred for new code. + +- Some `zkvm-jetpack` helpers still convert Hoon lists into `Vec` and rebuild lists/tuples from raw nouns. Current jet call sites appear intra-stack, but the helpers are structurally unsafe for cross-space use. + +- `jets::cold` has historically treated raw `Noun` as a decoded payload. Some re-homing mitigations have landed, but long-term encode/decode APIs should prefer `NounHandle` or branded forms rather than raw nouns. + +- `bridge::signing::sign_proposal(...)` reconstructs `NounSpace` manually and rejects PMA offset-form nouns. It appears test-only today, but it is the exact pattern branded handles are meant to avoid. + +## Why `NounHandle` Is Not Always Enough + +`NounHandle<'a>` prevents many lifetime mistakes because it carries `&'a NounSpace`. It is the right default for traversal and local parsing. + +However, unbranded handles from different spaces can still have the same Rust type. A function that accepts two `NounHandle<'a>` values can accidentally compare or combine nouns from different spaces if both spaces have compatible lifetimes. The runtime may catch some of this through range checks, but the type system does not know the two handles must share the same owner. + +Branded handles add a generative identity: + +```rust +space.with_brand(|space| { + let noun = space.handle(raw_noun); + // `noun` is tied to this exact branded space. +}); +``` + +Inside the closure, all branded handles carry the same hidden brand. Handles from another `with_brand` call have a different brand and cannot be passed to functions expecting this one. A branded handle also cannot escape the closure because the brand lifetime is generative. + +This directly addresses the audit's main pattern: APIs should not be able to accept "a noun and some space that probably matches" when they actually require "a noun proven to come from this exact space." + +## API Guidance + +For new or touched PMA-sensitive code: + +1. Prefer `NounHandle` inputs over `(Noun, &NounSpace)` pairs. +2. Prefer handle-yielding iterators over `Iterator`. +3. Avoid owned structs with raw `Noun` fields unless the owning slab/space is stored with them and outlives every use. +4. If a noun crosses into a new allocator, copy/import it using an API that takes the source `NounSpace`. +5. If a helper must prove multiple nouns share the same arena, consider a branded API. +6. Do not reconstruct a `NounSpace` from raw pointers after the fact. +7. Do not use raw `Vec` as an intermediate for values that may be re-encoded into another allocator. +8. If a raw-noun escape hatch remains for performance or ergonomics, document the lifetime/provenance precondition at the call site. + +## Practical Migration Priority + +The highest-value long-term cleanups are: + +1. Convert remaining list/map/tuple traversal APIs that return raw nouns to handle-yielding forms. +2. Replace raw-noun intermediate structs in parser paths with borrowed view/parser helpers. +3. Add full-graph or source-space-aware slab import helpers where code may splice PMA nouns into slabs. +4. Remove or fence public APIs that store raw nouns for later comparison. +5. Use branded handles in APIs that require all nouns to come from one exact `NounSpace`. + +The goal is not to eliminate raw `Noun` from the VM internals. The goal is to stop raw nouns from crossing ownership boundaries without a compile-time or explicit runtime proof of provenance. diff --git a/docs/pma/PMA-NOW.md b/docs/pma/PMA-NOW.md new file mode 100644 index 000000000..0f4e93b62 --- /dev/null +++ b/docs/pma/PMA-NOW.md @@ -0,0 +1,142 @@ +# PMA-Ready NockVM Plan (Option 1) + +> Historical implementation-plan/status note. For current public-release guidance, prefer [`DESIGN.md`](./DESIGN.md) and [`DURABILITY-OPERATIONS.md`](./DURABILITY-OPERATIONS.md). + +Goal: convert every allocated noun to carry a position‑independent payload (stack-pointer vs. PMA offset bit), so the Serf slab can be mmap-cloned into read-only CoW replicas without corrupting pointers. We must preserve today’s interpreter hot-path throughput; the only acceptable overhead is a couple of masks/shifts per dereference. We now also want the full nockchain-api stack to run on the memfd-backed PMA so we can measure resident usage under real workloads. + +## Tagged commits + +### retag-perf-hoonc-builds +This tag improves the performance of retag_noun_tree +such that `hoonc` is capable of building the Nockchain kernel in +under 10 minutes. The `hoonc_hotspots` benchmark also builds and runs, with +the following performance regression from `nockchain/master`: + + ``` + | Benchmark | Master | Current Branch | Regression | + |----------------------------|----------|----------------|------------| + | hamt_symbol_table_hot_path | 56.4 µs | 97.1 µs | +72% | + | noun_preserve_deep_core | 77.5 µs | 122.5 µs | +58% | + | unifying_equality_canopy | 32.3 µs | 47.4 µs | +47% | + | interpret_hint_stack | 5.86 µs | 12.5 µs | +113% | + | warm_jet_lookup | 5.57 µs | 12.5 µs | +124% | + | context_cache_churn | 69.3 µs | 98.5 µs | +42% | + | cue_jam_roundtrip | 486.0 µs | 595.6 µs | +23% | + ``` + +Prior to this, it never finished parsing the first file. However, `hoonc` will +crash when it tries to write the file to disk, due to issues with the +`NockStack`/`NounSlab` interface. Basically, there are methods called by the +file and exit driver that want a `NockStack` with `install_arena()` called, +despite the fact that the effects they are processing are all pointer-allocated. +There is a workaround in `98ba6ea` that simply creates emphemeral `NockStack`s +just to satisfy the API requirement, but this is obviously not a real solution. + +## Phase 0 – Preconditions + +- ✅ Branch is frozen around this document; we treat the plan as source of truth. +- ✅ We have a clean baseline after selectively opting out of fmt/clippy per user guidance. +- ✅ Perf baselines captured (`cargo test --release -p nockvm`, `criterion` microbenches); use them when validating further steps. + +## Phase 1 – Arena metadata & pointer math (`open/crates/nockvm/rust/nockvm/src/mem.rs`) + +✅ **Done** + +1. `Arena` now always owns a memfd-backed slab (`rustix::memfd_create` + `ftruncate` + `MmapMut`). It exposes `ptr_from_offset`, `offset_from_ptr`, `map_copy_read_only`. +2. `NockStack` stores an `Arc` and provides `arena()`, `install_arena()`, `ptr_from_offset()`, `offset_from_ptr()`. +3. Allocators (`struct_alloc`, `alloc_cell`, preserve paths, slab copies) tag offsets immediately; forwarding pointers also store offsets. +4. Every stack consumer installs the arena via TLS guards; unit tests, doctests, integration tests now all use small helpers/guards so we no longer see `Arena::with_current` panics. + +## Phase 2 – Tagged pointer representation (`open/crates/nockvm/rust/nockvm/src/noun.rs`) + +✅ **Done** + +1. `TaggedPtr` wraps tag/location/payload and backs every `IndirectAtom`/`Cell` constructor and dereference path. +2. Every stack consumer (`stack.install_arena()` in HAMT, jets, serialization, noun slabs, benches, context builders) ensures the TLS slot holds the active arena before tagged pointers are resolved; `Arena::with_current` now just validates that invariant. +3. Constructors that load from slabs/checkpoints/jam all tag offsets immediately via `stack.offset_from_ptr`. +4. Jets/HAMT/serialization are arena-aware through the install hook, keeping the interpreter hot path unchanged and ready for replicas. + +## Phase 3 – Nursery→permanent retagging (still in `mem.rs` / `noun.rs`) + +✅ **Done** + +- Added `NockStack::retag_noun_tree` plus a lightweight `Retag` trait so `Serf::retag_survivors` rewrites every preserved root (Arvo, scry stack, cache, hot/warm/cold, and the test-jet HAMTs) into offset form immediately after each `preserve_event_update_leftovers`. +- `Hamt`, `WarmEntry`, `Hot`, `Cold`, `Batteries*`, and `NounList` now participate in the retag walker, meaning the caches no longer rely on another preserve pass to shed stack pointers. +- Regression coverage: `hamt_retag_converts_entries_to_offsets` asserts HAMT keys/values are PMA-safe, and the existing debug sweep stays enabled under `debug_assertions` for extra safety when running integration tests. + +## Phase 4 – Interpreter/context plumbing (`open/crates/nockvm/rust/nockvm/src/interpreter.rs` and friends) + +✅/⚠️ + +- ✅ `Context` now stores an `Arc`; `create_context`, HAMT, serialization, jets, benches, noun slabs call `stack.install_arena()` before touching tagged nouns. +- ⚠️ We still rely on TLS for dereferences. To make replicas safe under heavy load we eventually need to pass `&Arena` explicitly to the few hot helpers (slotting, jets) and delete the fallback from `TaggedPtr`. That change is invasive, so it comes **after** we shake out nockchain-api testing with TLS still enabled. + +## Phase 5 – CoW replica scaffolding (nockapp integration + nockchain-api) + +⚠️ **Not started yet** – this is the critical path toward running `nockchain-api` atop the new PMA. + +Tasks needed: +- `Arena::clone_read_only()` exists, but we also need `Arena::map_copy_private_stack()` or an equivalent overlay so replicas get private write space for the nursery/stack without faulting the permanent slab. +- SerfReplica pool with: + - deterministic poke replay stream (leader keeps feeding changes), + - read-only peek scheduling, load-balancing, and back-pressure, + - metrics for replica lag and residency. +- End-to-end integration tests (e.g., `nockapp_grpc` peeking through replicas). +- CLI glue so `nockchain-api` can enable the replica pool behind a feature flag. + +Until this phase is complete, running `nockchain-api` will exercise the memfd PMA but still serialize peeks through the leader arena. + +## Testing & Verification Plan + +1. **Unit / Debug assertions:** + - `noun.rs`: round-trip `as_raw()`/`from_raw()` for each kind/location combination; ensure offsets resolve to the correct addresses via mock arenas. + - `mem.rs`: property tests (`proptest`) verifying `ptr_from_offset(offset_from_ptr(ptr)) == ptr` for various alignments/orientations. + - Preservation tests: craft nouns containing forwarding pointers, run `noun_preserve`, and assert location bits remain valid. + - The serf now runs a debug-only sweep (`debug_assert_offsets`) after each `preserve_event_update_leftovers` that panics if any noun still carries a stack pointer. Running the existing integration tests with `debug_assertions` enabled exercises this automatically. + +2. **Integration / system tests:** + - Existing `nockapp` integration tests (`open/crates/nockapp/tests/integration.rs`) should be run under both `mmap` and `malloc` features to catch regressions. + - Add a new test that imports a checkpoint, clones the arena into a fake replica, and runs a peek to validate that offsets stay stable after remapping. + - For OS-level validation that paging behaves correctly with the memfd-backed slab: + 1. Launch a nockapp instance with a very large snapshot so the slab consumes multiple gigabytes. + 2. Record the PID and inspect `/proc/$PID/smaps` (or `pmap -x`) to capture the `nockstack` VMA’s RSS/Swap. + 3. Trigger a read-only workload (peeks via the upcoming replica pool) so replicas fault pages lazily. + 4. Use `echo 3 | sudo tee /proc/sys/vm/drop_caches`, `mincore`, or `perf mem` to ensure the kernel can drop cold pages and fault them back in for replicas. + 5. Compare before/after RSS to confirm the shared memfd enables CoW instead of duplicating the slab. + +3. **Performance guardrails:** + - Extend the `criterion` benches (e.g., `benches/hoonc_hotspots.rs`) to measure axis traversal and jets with the new tagging. + - Compare against the Phase 0 baseline; permissible regression is ≤1 % on median latency. If larger, profile using `perf` to locate hot spots (tag decoding, arena rebasing, etc.) and optimize. + +4. **Miri & sanitizers:** + - Run `cargo miri test -p nockvm` for pointer-heavy modules (`noun`, `mem`, `interpreter`). We have already ported every unit test/ doctest to install arenas so Miri can run the lightweight subset without TLS failures. + - Address Sanitizer via `RUSTFLAGS="-Zsanitizer=address"` for the nockvm crate to catch misuse of the new pointer math. + +5. **Documentation & rollout:** + - Update `NOCK-PMA.md` with actual code snippets once implemented. + - Document the new invariants in `open/crates/nockvm/DEVELOPERS.md` (how the arena/offset dance works, expectations for jets). + - Provide a migration note in `CHANGELOG.md` describing the new noun representation. + +## Sequencing & Code Review Checklist + +1. Phase 1 + 2 landed; keep the codepath always-on (no flag). +2. Phase 3 retagging is live; next up is Phase 5 (replicas / peek scaling) while maintaining the existing perf guardrails. +3. Every PR must include: + - Updated tests/benchmarks. + - Proof (metrics or benchmark output) that the interpreter hot path did not regress beyond the tolerance window. + +## Driving Toward nockchain-api & PMA Memory Measurements + +To run `nockchain-api` on the memfd-backed PMA and collect resident memory data we need: + +1. ✅ **Phase 3 retagging** is in place, so snapshots/peeks never see stack pointers and the leader slab is PMA-safe. +2. **Replica overlay (Phase 5)** remains pending for multi-replica peek scaling; it is not required for the immediate single-serf RSS study but must land before we farm peeks to read-only stacks. +3. The `nockapp::kernel::boot::Cli` now takes `--data-dir` (full path to the checkpoints directory) and `nockchain-cli` exposes `--identity-path`, so `cargo run -p nockchain-api -- --data-dir test-api/.data.nockchain --identity-path test-api/.nockchain_identity ...` boots directly on the canned 2.5 GiB snapshot. See `open/crates/nockchain-api/README.md` for the exact command and notes on the `nockvm.pma.*` metrics to watch. +3. **Operational checklist** for perf/memory studies: + - Start `nockchain-api` with `NOCKVM_MEMFD_STACK=1` (or a dedicated env) and the replica flag. + - Use `gnort` metrics (`nockvm.pma.*`) to monitor residency, touched pages, and post-peek ratios. + - Record `/proc/$PID/smaps` while issuing peeks via gRPC (simulate pokes concurrently) to ensure the kernel is paging unused chunks. + - Compare RSS vs. the old anonymous mmap builds. +4. **Docs/tests**: update `PMA-NOW.md` (this file) whenever steps shift, and add a “replica smoke test” under `open/crates/nockapp/tests` once Phase 5 lands. + +This plan gets us to position-independent nouns with measured, reviewable steps, unlocking the mmap CoW replicas without jeopardizing interpreter performance. diff --git a/docs/pma/README.md b/docs/pma/README.md new file mode 100644 index 000000000..436249e8a --- /dev/null +++ b/docs/pma/README.md @@ -0,0 +1,13 @@ +# PMA Documentation + +This directory contains PMA documentation that is useful for production testing, public release review, and future PMA-sensitive development. + +## Documents + +- [`DESIGN.md`](./DESIGN.md) — implementation-oriented PMA design, durability model, snapshot model, GC model, and known limits. +- [`DURABILITY-OPERATIONS.md`](./DURABILITY-OPERATIONS.md) — operator and production-testing guide for PMA durability, boot recovery, snapshots, event logs, and GC. +- [`NOUN-PROVENANCE-AND-BRANDED-HANDLES.md`](./NOUN-PROVENANCE-AND-BRANDED-HANDLES.md) — context for alien noun / NounSpace bugs and why `NounHandle` / branded handles exist. +- [`NOCKSTACK-ATTRIBUTION.md`](./NOCKSTACK-ATTRIBUTION.md) — memory attribution guide for PMA, NockStack, and heap/anonymous memory. +- [`FAST-HINT-REGISTRATION.md`](./FAST-HINT-REGISTRATION.md) — current-code guide to `%fast` hint registration, cold/warm/hot state, and persistence implications. +- [`NOCK-PMA.md`](./NOCK-PMA.md) — lower-level historical PMA design notes for NockVM internals. +- [`PMA-NOW.md`](./PMA-NOW.md) — historical implementation-plan/status notes retained for context. diff --git a/hoon/apps/approver/approver.hoon b/hoon/apps/approver/approver.hoon new file mode 100644 index 000000000..2f0e9414d --- /dev/null +++ b/hoon/apps/approver/approver.hoon @@ -0,0 +1,81 @@ +/= * /common/zoon +/= * /common/zeke +/= * /common/wrapper +/= t /common/tx-engine +/= t1 /common/tx-engine-1 +/= s10 /common/slip10 +:: +=< ((moat |) inner) :: wrapped kernel +:: +=> + |% + +$ kernel-state [%state version=%1] + +$ effect + $% [%exit code=@] + [%file %write path=@t contents=@] + == + :: we want schnorr-seckey:t + :: +$ cause [%sign =raw-tx:t1 extended-key=@t] + +$ cause + $% [%sign =raw-tx:t1 extended-key=@t save-file=@t] + == + -- +|% +++ moat (keep kernel-state) :: no state +++ inner + |_ k=kernel-state + :: do-nothing load + ++ load + |= =kernel-state kernel-state + :: crash on peek + ++ peek + |= arg=path + ~| %peeks-not-allowed + !! + :: + ++ poke + |= [wir=wire eny=@ our=@ux now=@da dat=*] + ^- [(list effect) k=kernel-state] + =/ cause ((soft cause) dat) + ?~ cause + ~> %slog.[1 'poke: Bad cause'] + :_ k + [%exit 1]~ + =+ c=u.cause + ?- -.c + :: + %sign + :: process extended key + ~| %bad-key + =/ core (from-extended-key:s10 extended-key.c) + ~| %not-private-key + ?> !=(0 prv:core) + :: get private key + =/ sign-key=schnorr-seckey:t + (from-atom:schnorr-seckey:t private-key:core) + :: alias for raw-tx + =/ rtx=raw-tx:t1 raw-tx.c + :: sign spends + =/ signed-spends=spends:t1 + %- ~(run z-by spends.rtx) + |= spend=spend:t1 + (sign:spend-v1:t spend sign-key) + :: rebuild rtx with signed spends + =/ new-rtx=raw-tx:t1 [version.rtx *tx-id:t1 signed-spends] + :: compute and set the correct id + =. rtx new-rtx(id (compute-id:raw-tx:t1 new-rtx)) + :: validate the spends + ?. (validate:spends:t1 spends.rtx) + ~& 'did not validate' + :_ k [%exit 1]~ + :_ k + :~ + :: write to file + [%file %write save-file.c (jam rtx)] + :: exit the nockapp + [%exit 0] + == + :: + == + -- +-- diff --git a/hoon/apps/dumbnet/inner.hoon b/hoon/apps/dumbnet/inner.hoon index 85852c156..390c3cd9a 100644 --- a/hoon/apps/dumbnet/inner.hoon +++ b/hoon/apps/dumbnet/inner.hoon @@ -7,7 +7,7 @@ /= mine /common/pow /= nv /common/nock-verifier /= zeke /common/zeke -/= * /common/zoon +/= * /common/h-zoon /= * /common/wrapper :: :: Never use c-transact face, always use the lustar `t` @@ -36,14 +36,16 @@ ~& [%nockchain-state-version -.arg] :: cut |^ - =. k ~> %bout (update-constants (check-checkpoints (state-n-to-6 arg))) + =. k ~> %bout (update-constants (check-checkpoints (state-n-to-9 arg))) =. c.k ~> %bout check-and-repair:con + ~| %v1-phase-must-be-lte-asert-phase + ?> (lte v1-phase.constants.k asert-phase.constants.k) k :: this arm should be renamed each state upgrade to state-n-to-[latest] and extended to loop through all upgrades - ++ state-n-to-6 + ++ state-n-to-9 |= arg=load-kernel-state:dk ^- kernel-state:dk - ?. ?=(%6 -.arg) + ?. ?=(%9 -.arg) ~> %slog.[0 'load: State upgrade required'] ?- -.arg :: @@ -53,9 +55,111 @@ %3 $(arg (state-3-to-4 arg)) %4 $(arg (state-4-to-5 arg)) %5 $(arg (state-5-to-6 arg)) + %6 $(arg (state-6-to-7 arg)) + %7 $(arg (state-7-to-8 arg)) + %8 $(arg (state-8-to-9 arg)) == arg :: + :: upgrade kernel state 8 to kernel state 9 + :: h-zoon replaces the remaining consensus z containers with + :: digest-keyed h containers. kernel-state-8 already carries the + :: full post-phase-2 constants shape. this migration preserves constants + :: and only rewrites consensus container representation. + ++ state-8-to-9 + |= arg=kernel-state-8:dk + ^- kernel-state-9:dk + =/ new-c=consensus-state-9:dk + %* . *consensus-state-9:dk + blocks-needed-by (zh-jult blocks-needed-by.c.arg) + excluded-txs (zh-silt excluded-txs.c.arg) + spent-by (zh-jult spent-by.c.arg) + pending-blocks (zh-molt pending-blocks.c.arg) + balance (zh-balmilt blocks.c.arg balance.c.arg) + txs (zh-milt txs.c.arg) + raw-txs (zh-molt raw-txs.c.arg) + blocks (zh-molt blocks.c.arg) + heaviest-block heaviest-block.c.arg + min-timestamps (zh-molt min-timestamps.c.arg) + epoch-start (zh-molt epoch-start.c.arg) + targets (zh-molt targets.c.arg) + btc-data btc-data.c.arg + genesis-seal genesis-seal.c.arg + == + =/ new-m=mining-state-9:dk + %* . *mining-state-9:dk + mining mining.m.arg + shares shares.m.arg + v0-shares v0-shares.m.arg + candidate-block *page:t + candidate-acc *tx-acc:t + next-nonce next-nonce.m.arg + == + :* %9 + c=new-c + a=a.arg + m=new-m + d=d.arg + constants=constants.arg + == + :: + :: upgrade kernel state 7 to kernel state 8 + :: blockchain-constants:v1 gained a sixth asert-* field + :: (asert-anchor-min-timestamp) at phase 2 of 014-aletheia. + :: kernel-state-7 uses the frozen phase-1 shape (five asert + :: fields) so old %7 states still decode; kernel-state-8 uses + :: the full post-phase-2 blockchain-constants:v1. we discard + :: the old constants noun and let update-constants reseed it + :: from *blockchain-constants:t (which now pins the canonical + :: mainnet anchor median-of-11). + ++ state-7-to-8 + |= arg=kernel-state-7:dk + ^- kernel-state-8:dk + :* %8 + c=c.arg + a=a.arg + :: discard stale pre-upgrade candidate; miner will rebuild on next tick + m=m.arg(candidate-block *page:t, candidate-acc *tx-acc:t) + d=d.arg + constants=*blockchain-constants:t + == + :: + :: upgrade kernel state 6 to kernel state 7 + :: blockchain-constants:v1 was extended with five ASERT fields in + :: the original aletheia phase 1. kernel-state-6 uses + :: blockchain-constants-v1-pre-asert (the frozen pre-ASERT shape) so + :: that old %6 states can decode cleanly. kernel-state-7 uses the + :: phase-1 snapshot of blockchain-constants:v1 (five asert-* fields, + :: no asert-anchor-min-timestamp). we discard the old constants and + :: let the chained state-7-to-8 upgrade plus update-constants fill + :: in current defaults below. + ++ state-6-to-7 + |= arg=kernel-state-6:dk + ^- kernel-state-7:dk + :: guard: refuse if mainnet chain has already crossed ASERT activation. + :: replicates is-mainnet logic inline to avoid calling dumb-derived with + :: pre-ASERT constants (incompatible type). + =/ on-mainnet=? + ?~ genesis-seal.c.arg + ?^ genesis-id=(~(get z-by heaviest-chain.d.arg) 0) + =/ genesis (~(get z-by blocks.c.arg) u.genesis-id) + ?~ genesis %.n + =((hash:page-msg:t ~(msg get:local-page:t u.genesis)) realnet-genesis-msg:dk) + %.n + =(realnet-genesis-msg:dk msg-hash.u.genesis-seal.c.arg) + =/ phase asert-phase:*blockchain-constants:t + ?: &(on-mainnet ?=(^ highest-block-height.d.arg) (gte u.highest-block-height.d.arg phase)) + ~> %slog.[0 'FATAL: late-upgrade - mainnet chain crossed ASERT activation under pre-ASERT rules'] + !! + :* %7 + c=c.arg + a=a.arg + :: discard stale pre-upgrade candidate; miner will rebuild on next tick + m=m.arg(candidate-block *page:t, candidate-acc *tx-acc:t) + d=d.arg + constants=*blockchain-constants-v1-phase-1:dk + == + :: :: upgrade kernel state 5 to kernel state 6 ++ state-5-to-6 |= arg=kernel-state-5:dk @@ -95,7 +199,7 @@ next-nonce next-nonce.m.arg == =/ default-constants=blockchain-constants:t *blockchain-constants:t - =/ new-constants=blockchain-constants:t + =/ new-constants=blockchain-constants-v1-pre-asert:dk :* v1-phase.default-constants bythos-phase.default-constants data.default-constants @@ -132,9 +236,9 @@ ~ (~(put z-ju bnb) tx-id block-id) ~> %slog.[0 'load: Indexed blocks by transaction id'] - =/ rtx=(map tx-id:v0:t *) raw-txs.c.arg - =/ bnb=(map tx-id:v0:t *) blocks-needed-by - =/ excluded-map=(map tx-id:v0:t *) (~(dif z-by rtx) bnb) + =/ rtx=(z-map tx-id:v0:t *) raw-txs.c.arg + =/ bnb=(z-map tx-id:v0:t *) blocks-needed-by + =/ excluded-map=(z-map tx-id:v0:t *) (~(dif z-by rtx) bnb) =/ excluded-txs=(z-set tx-id:v0:t) ~(key z-by excluded-map) =+ ?: =(*(z-set tx-id:v0:t) excluded-txs) @@ -283,6 +387,7 @@ ^- (unit (unit *)) ~> %slog.[0 (cat 3 'peek: %' -.arg)] =/ =(pole) arg + |^ ?+ pole ~ :: [%mainnet ~] @@ -293,19 +398,36 @@ :: [%blocks ~] ^- (unit (unit (z-map block-id:t page:t))) - ``(~(run z-by blocks.c.k) to-page:local-page:t) + ``(hz-molt (~(run h-by blocks.c.k) to-page:local-page:t)) + :: + [%h-blocks ~] + ^- (unit (unit (h-map block-id:t page:t))) + ``(~(run h-by blocks.c.k) to-page:local-page:t) :: [%transactions ~] ^- (unit (unit (z-mip block-id:t tx-id:t tx:t))) + ``(hz-milt txs.c.k) + :: + [%h-transactions ~] + ^- (unit (unit (h-mip block-id:t tx-id:t tx:t))) ``txs.c.k :: [%raw-transactions ~] ^- (unit (unit (z-map tx-id:t [=raw-tx:t heard-at=@]))) + ``(hz-molt raw-txs.c.k) + :: + [%h-raw-transactions ~] + ^- (unit (unit (h-map tx-id:t [=raw-tx:t heard-at=@]))) ``raw-txs.c.k :: :: transactions unneeded by any block [%excluded-txs ~] ^- (unit (unit (z-set tx-id:t))) + ``(hz-silt excluded-txs.c.k) + :: + :: transactions unneeded by any block + [%h-excluded-txs ~] + ^- (unit (unit (h-set tx-id:t))) ``excluded-txs.c.k :: :: For %block, %transaction, %raw-transaction, and %balance scries, the ID is @@ -314,7 +436,7 @@ ^- (unit (unit page:t)) :: scry for a validated block (this does not look at pending state) =/ block-id (from-b58:hash:t bid.pole) - `(bind (~(get z-by blocks.c.k) block-id) to-page:local-page:t) + `(bind (~(get h-by blocks.c.k) block-id) to-page:local-page:t) :: [%elders bid=@ ~] :: get ancestor block IDs up to 24 deep for a given block @@ -329,7 +451,16 @@ :: scry for txs included in a validated block ^- (unit (unit (z-map tx-id:t tx:t))) :- ~ - %- ~(get z-by txs.c.k) + %+ bind + %- ~(get h-by txs.c.k) + (from-b58:hash:t bid.pole) + hz-molt + :: + [%h-block-transactions bid=@ ~] + :: scry for txs included in a validated block + ^- (unit (unit (h-map tx-id:t tx:t))) + :- ~ + %- ~(get h-by txs.c.k) (from-b58:hash:t bid.pole) :: [%block-transaction bid=@ tid=@ ~] @@ -337,9 +468,9 @@ ^- (unit (unit tx:t)) =/ tx-id (from-b58:hash:t tid.pole) =/ block-id (from-b58:hash:t bid.pole) - =/ block-txs (~(get z-by txs.c.k) block-id) + =/ block-txs (~(get h-by txs.c.k) block-id) ?~ block-txs ~ - =/ maybe-tx (~(get z-by u.block-txs) tx-id) + =/ maybe-tx (~(get h-by u.block-txs) tx-id) ?~ maybe-tx ~ ``u.maybe-tx :: @@ -363,7 +494,7 @@ (~(get z-by heaviest-chain.d.k) u.num) ?~ id [~ ~] - `(bind (~(get z-by blocks.c.k) u.id) to-page:local-page:t) + `(bind (~(get h-by blocks.c.k) u.id) to-page:local-page:t) :: [%heaviest-chain ~] ^- (unit (unit [page-number:t block-id:t])) @@ -382,38 +513,10 @@ ``heaviest-chain.d.k :: [%heaviest-chain-blocks-range start=@ end=@ ~] - ^- (unit (unit (list [page-number:t block-id:t page:t (z-map tx-id:t tx:t)]))) - =/ start-height ((soft page-number:t) start.pole) - =/ end-height ((soft page-number:t) end.pole) - ?~ start-height ~ - ?~ end-height ~ - :: ensure start <= end - ?: (gth u.start-height u.end-height) - ``~ - :: build list of blocks in range from heaviest chain - =/ result=(list [page-number:t block-id:t page:t (z-map tx-id:t tx:t)]) - =/ height u.start-height - |- ^- (list [page-number:t block-id:t page:t (z-map tx-id:t tx:t)]) - ?: (gth height u.end-height) - ~ - :: get block-id from heaviest chain - =/ block-id=(unit block-id:t) - (~(get z-by heaviest-chain.d.k) height) - ?~ block-id - $(height +(height)) - :: get block data - =/ local-block=(unit local-page:t) - (~(get z-by blocks.c.k) u.block-id) - ?~ local-block - $(height +(height)) - :: get transactions for this block - =/ block-txs=(unit (z-map tx-id:t tx:t)) - (~(get z-by txs.c.k) u.block-id) - =/ txs-map ?~(block-txs ~ u.block-txs) - :: add to result list - :- [height u.block-id (to-page:local-page:t u.local-block) txs-map] - $(height +(height)) - ``result + (heaviest-chain-blocks-range start.pole end.pole %.y) + :: + [%heaviest-chain-blocks-range-no-pow start=@ end=@ ~] + (heaviest-chain-blocks-range start.pole end.pole %.n) :: [%desk-hash ~] ^- (unit (unit (unit @uvI))) @@ -430,14 +533,22 @@ [%balance bid=@ ~] ^- (unit (unit (z-map nname:t nnote:t))) :- ~ - %- ~(get z-by balance.c.k) + %+ bind + %- ~(get h-by balance.c.k) + (from-b58:hash:t bid.pole) + hz-molt + :: + [%h-balance bid=@ ~] + ^- (unit (unit (h-map nname:t nnote:t))) + :- ~ + %- ~(get h-by balance.c.k) (from-b58:hash:t bid.pole) :: [%heaviest-block ~] ^- (unit (unit page:t)) ?~ heaviest-block.c.k [~ ~] - =/ heaviest-block (~(get z-by blocks.c.k) u.heaviest-block.c.k) + =/ heaviest-block (~(get h-by blocks.c.k) u.heaviest-block.c.k) ?~ heaviest-block ~ ``(to-page:local-page:t u.heaviest-block) :: @@ -445,10 +556,22 @@ ^- (unit (unit (z-map nname:t nnote:t))) ?~ heaviest-block.c.k [~ ~] - ?. (~(has z-by blocks.c.k) u.heaviest-block.c.k) + ?. (~(has h-by blocks.c.k) u.heaviest-block.c.k) + [~ ~] + :- ~ + %+ bind + %- ~(get h-by balance.c.k) + u.heaviest-block.c.k + hz-molt + :: + [%h-current-balance ~] + ^- (unit (unit (h-map nname:t nnote:t))) + ?~ heaviest-block.c.k + [~ ~] + ?. (~(has h-by blocks.c.k) u.heaviest-block.c.k) [~ ~] :- ~ - %- ~(get z-by balance.c.k) + %- ~(get h-by balance.c.k) u.heaviest-block.c.k :: [%balance-by-first-name first-name=@t ~] @@ -456,9 +579,9 @@ =/ first-name=hash:t (from-b58:hash:t first-name.pole) ?~ heaviest-block.c.k [~ ~] - ?. (~(has z-by blocks.c.k) u.heaviest-block.c.k) + ?. (~(has h-by blocks.c.k) u.heaviest-block.c.k) [~ ~] - ?~ bal=(~(get z-by balance.c.k) u.heaviest-block.c.k) + ?~ bal=(~(get h-by balance.c.k) u.heaviest-block.c.k) [~ ~] ?~ highest=highest-block-height.d.k [~ ~] @@ -466,7 +589,7 @@ %- some :+ u.highest u.heaviest-block.c.k - %- ~(rep z-by u.bal) + %- ~(rep h-by u.bal) |= [[k=nname:t v=nnote:t] bal=(z-map nname:t nnote:t)] ?. =(~(first-name get:nnote:t v) first-name) bal @@ -477,9 +600,9 @@ =/ pubkey=schnorr-pubkey:t (from-b58:schnorr-pubkey:t key-b58.pole) ?~ heaviest-block.c.k [~ ~] - ?. (~(has z-by blocks.c.k) u.heaviest-block.c.k) + ?. (~(has h-by blocks.c.k) u.heaviest-block.c.k) [~ ~] - ?~ bal=(~(get z-by balance.c.k) u.heaviest-block.c.k) + ?~ bal=(~(get h-by balance.c.k) u.heaviest-block.c.k) [~ ~] ?~ highest=highest-block-height.d.k [~ ~] @@ -487,7 +610,7 @@ %- some :+ u.highest u.heaviest-block.c.k - %- ~(rep z-by u.bal) + %- ~(rep h-by u.bal) |= [[k=nname:t v=nnote:t] pub-bal=(z-map nname:t nnote:t)] :: only include v0 notes; v1 notes use lock-roots ?. ?=(^ -.v) @@ -502,7 +625,7 @@ ^- (unit (unit [(each (z-set sig:t) (z-set hash:t)) (unit page-summary:t)])) ?~ heaviest-block.c.k ``[[%& ~(key z-by v0-shares.m.k)] ~] - =/ heaviest-block (~(get z-by blocks.c.k) u.heaviest-block.c.k) + =/ heaviest-block (~(get h-by blocks.c.k) u.heaviest-block.c.k) ?~ heaviest-block ``[[%& ~(key z-by v0-shares.m.k)] ~] ?~ highest-block-height.d.k @@ -526,7 +649,7 @@ ?~ id [~ ~] =/ pag - (bind (~(get z-by blocks.c.k) u.id) to-page:local-page:t) + (bind (~(get h-by blocks.c.k) u.id) to-page:local-page:t) ?~ pag [~ ~] =/ cb=coinbase-split:t ~(coinbase get:page:t u.pag) @@ -556,7 +679,7 @@ (block-commitment:page:t candidate-block.m.k) :: =/ network-target - (~(got z-by targets.c.k) ~(parent get:page:t candidate-block.m.k)) + (~(got h-by targets.c.k) ~(parent get:page:t candidate-block.m.k)) :: =/ pool-target (chunk:bignum:zeke (div max-tip5-atom:tip5:zeke (bex difficulty.pole))) :: @@ -580,7 +703,8 @@ :- ~ %~ tap z-by ^- (z-map block-id:t page:t) - %- ~(run z-by blocks.c.k) + %- hz-molt + %- ~(run h-by blocks.c.k) |= lp=local-page:t ^- page:t ?^ -.lp lp(pow ~) @@ -589,8 +713,47 @@ [%tx-accepted tid-b58=@t ~] ^- (unit (unit ?)) =+ tid=(from-b58:hash:t tid-b58:pole) - ``(~(has z-by raw-txs.c.k) tid) + ``(~(has h-by raw-txs.c.k) tid) == + ++ heaviest-chain-blocks-range + |= [start=@ end=@ include-pow=?] + ^- (unit (unit (list [page-number:t block-id:t page:t (z-map tx-id:t tx:t)]))) + =/ start-height ((soft page-number:t) start) + =/ end-height ((soft page-number:t) end) + ?~ start-height ~ + ?~ end-height ~ + :: ensure start <= end + ?: (gth u.start-height u.end-height) + ``~ + =/ to-page + ?: include-pow + to-page:local-page:t + to-page-no-pow:local-page:t + :: build list of blocks in range from heaviest chain + =/ result=(list [page-number:t block-id:t page:t (z-map tx-id:t tx:t)]) + =/ height u.start-height + |- ^- (list [page-number:t block-id:t page:t (z-map tx-id:t tx:t)]) + ?: (gth height u.end-height) + ~ + :: get block-id from heaviest chain + =/ block-id=(unit block-id:t) + (~(get z-by heaviest-chain.d.k) height) + ?~ block-id + $(height +(height)) + :: get block data + =/ local-block=(unit local-page:t) + (~(get h-by blocks.c.k) u.block-id) + ?~ local-block + $(height +(height)) + :: get transactions for this block + =/ block-txs=(unit (h-map tx-id:t tx:t)) + (~(get h-by txs.c.k) u.block-id) + =/ txs-map ?~(block-txs ~ (hz-molt u.block-txs)) + :: add to result list + :- [height u.block-id (to-page u.local-block) txs-map] + $(height +(height)) + ``result + -- :: ++ poke |= [wir=wire eny=@ our=@ux now=@da dat=*] @@ -627,7 +790,7 @@ :_ effs =/ version=proof-version:sp (height-to-proof-version:con ~(height get:page:t candidate-block.m.k)) - =/ target (~(got z-by targets.c.k) ~(parent get:page:t candidate-block.m.k)) + =/ target ~(target get:page:t candidate-block.m.k) =/ commit (block-commitment:page:t candidate-block.m.k) ?- version %0 [%mine %0 commit target pow-len:t] @@ -671,9 +834,9 @@ (missing-parent-effects ~(digest get:page:t pag) ~(height get:page:t pag) u.peer-id) :: if we don't have parent and block claims to be heaviest :: request ancestors to catch up or handle reorg - ?. (~(has z-by blocks.c.k) ~(parent get:page:t pag)) + ?. (~(has h-by blocks.c.k) ~(parent get:page:t pag)) ?: %+ compare-heaviness:page:t pag - (~(got z-by blocks.c.k) u.heaviest-block.c.k) + (~(got h-by blocks.c.k) u.heaviest-block.c.k) =/ peer-id=(unit @) (get-peer-id wir) ?~ peer-id ~|("heard-block: Unsupported wire: {}" !!) @@ -759,7 +922,7 @@ [%request %raw-tx %by-id tx-id] :_ k ?: %+ compare-heaviness:page:t pag - (~(got z-by blocks.c.k) (need heaviest-block.c.k)) + (~(got h-by blocks.c.k) (need heaviest-block.c.k)) ~> %slog.[0 'heard-block: Gossiping new heaviest block (transactions pending validation)'] :- [%gossip %0 %heard-block pag] block-effs @@ -807,7 +970,7 @@ |- ?~ ids ~ ?: =(height 0) ~ - ?: (~(has z-by blocks.c.k) i.ids) + ?: (~(has h-by blocks.c.k) i.ids) `[i.ids height] $(ids t.ids, height (dec height)) ?~ latest-known @@ -849,8 +1012,8 @@ :: ++ check-duplicate-block |= digest=block-id:t - ?| (~(has z-by blocks.c.k) digest) - (~(has z-by pending-blocks.c.k) digest) + ?| (~(has h-by blocks.c.k) digest) + (~(has h-by pending-blocks.c.k) digest) == :: ++ check-genesis @@ -997,9 +1160,9 @@ :: won't accidentally throw out a block that contained a valid tx-id :: just because we received a tx that claimed the same id as the valid :: one. - =/ tx-pending-blocks (~(get z-ju blocks-needed-by.c.k) ~(id get:raw-tx:t raw)) + =/ tx-pending-blocks (~(get h-ju blocks-needed-by.c.k) ~(id get:raw-tx:t raw)) =. c.k - %- ~(rep z-in tx-pending-blocks) + %- ~(rep h-in tx-pending-blocks) |= [id=block-id:t c=_c.k] =. c.k c (reject-pending-block:con id) @@ -1071,7 +1234,7 @@ =^ new-effs k %: process-block-with-txs now eny - page:(~(got z-by pending-blocks.c.k) bid) + page:(~(got h-by pending-blocks.c.k) bid) :: if the block is bad, then tell the driver we dont want to see it :: again ~[[%seen %block bid ~]] @@ -1162,7 +1325,10 @@ ~> %slog.[0 'accept-block: New heaviest block!'] =/ span=span-effect:dk :+ %span %new-heaviest-chain - ~['block_height'^n+~(height get:page:t pag) 'heaviest_block_digest'^s+(to-b58:hash:t ~(digest get:page:t pag))] + :~ 'block_height'^n+~(height get:page:t pag) + 'heaviest_block_digest'^s+(to-b58:hash:t ~(digest get:page:t pag)) + 'block_target'^s+(scot %ux (merge:bignum:t ~(target get:page:t pag))) + == :* [%gossip %0 %heard-block pag] span effs @@ -1210,7 +1376,7 @@ :: :: if new block is heaviest, regossip txs that haven't been garbage collected =? effs is-new-heaviest - %- ~(rep z-in excluded-txs.c.k) + %- ~(rep h-in excluded-txs.c.k) |= [=tx-id:t effs=_effs] [[%gossip %0 %heard-tx (got-raw-tx:con tx-id)] effs] :: regossip block transactions if mining @@ -1343,7 +1509,7 @@ :: of height N+1 :: Also emit %seen for the heaviest block so our cache can start to update =/ height=page-number:t - +(~(height get:local-page:t (~(got z-by blocks.c.k) u.heaviest-block.c.k))) + +(~(height get:local-page:t (~(got h-by blocks.c.k) u.heaviest-block.c.k))) =/ born-effects=(list effect:dk) :~ [%request %block %by-height height] [%seen %block u.heaviest-block.c.k `height] @@ -1358,21 +1524,18 @@ ++ do-pow ^- [(list effect:dk) kernel-state:dk] ?> ?=([%pow *] command) - ~& %foo =/ commit=block-commitment:t (block-commitment:page:t candidate-block.m.k) ?. =(bc.command commit) ~> %slog.[1 'do-pow: Mined for wrong (old) block commitment'] [~ k] - ~& %foo1 ?: %+ check-target:mine dig.command - (~(got z-by targets.c.k) ~(parent get:page:t candidate-block.m.k)) + ~(target get:page:t candidate-block.m.k) =. m.k (set-pow:min prf.command) =. m.k set-digest:min =^ heard-block-effs k (heard-block /poke/miner now candidate-block.m.k eny) :_ k heard-block-effs - ~& %foo3 [~ k] :: ++ do-set-mining-key @@ -1484,7 +1647,7 @@ =/ heavy-height=page-number:t ?~ heaviest-block.c.k *page-number:t :: rerequest genesis block - +(~(height get:local-page:t (~(got z-by blocks.c.k) u.heaviest-block.c.k))) + +(~(height get:local-page:t (~(got h-by blocks.c.k) u.heaviest-block.c.k))) =. effects [[%request %block %by-height heavy-height] effects] =. effects @@ -1602,7 +1765,7 @@ %- ~(rep z-in ~(tx-ids get:page:t page)) |= [=tx-id:t effects=(list effect:dk)] ^- (list effect:dk) - =/ tx=raw-tx:t raw-tx:(~(got z-by raw-txs.c.k) tx-id) + =/ tx=raw-tx:t raw-tx:(~(got h-by raw-txs.c.k) tx-id) =/ fec=effect:dk [%gossip %0 %heard-tx tx] [fec effects] :: diff --git a/hoon/apps/dumbnet/lib/asert.hoon b/hoon/apps/dumbnet/lib/asert.hoon new file mode 100644 index 000000000..1faf7ebc9 --- /dev/null +++ b/hoon/apps/dumbnet/lib/asert.hoon @@ -0,0 +1,171 @@ +/= * /common/zeke +:: asert: aserti3-2d difficulty adjustment +:: +:: implements the algorithm formalized by jonathan toomim and shipped on +:: bitcoin cash in 2020. pure math, no consensus-state coupling; operates +:: on atoms with bignum conversion at the +compute-target-bn boundary. +:: +:: the 2^x approximation is a 3rd-order polynomial whose coefficients are +:: tied to rbits=16. changing rbits requires new polynomial coefficients, +:: so rbits is not a parameter. +:: +|% +++ asert-rbits 16 +++ asert-radix ^~((bex asert-rbits)) +:: +:: +poly-factor: 2^(frac/radix) * radix as an integer +:: +:: input frac in [0, radix); output in [radix, 2*radix). +:: 3rd-order minimax polynomial approximation of 2^x on [0, 1), +:: scaled so that evaluating at frac = y*radix yields factor ≈ 2^y * radix: +:: radix + (195_766_423_245_049*frac +:: + 971_821_376 *frac^2 +:: + 5_127 *frac^3 +:: + 2^47) >> 48 +:: the three normalized coefficients (c_k / 2^((k+1)*16)) sum to ~1 so the +:: approximation satisfies p(0) = 1 and p(1) ≈ 2. the `+ 2^47` term is +:: round-to-nearest for the fixed-point divide by 2^48, matching bch's +:: canonical aserti3-2d so published polynomial vectors reproduce exactly. +:: max error versus 2^x is under 0.13% on [0, 1). +++ poly-factor + |= frac=@ + ^- @ + :: precondition: frac < radix (see +decompose-exponent). guard here so + :: any future caller that constructs frac independently fails loudly + :: rather than silently producing a factor outside [radix, 2*radix). + ?> (lth frac asert-radix) + =/ f2 (mul frac frac) + =/ f3 (mul f2 frac) + =/ num :(add (mul 195.766.423.245.049 frac) (mul 971.821.376 f2) (mul 5.127 f3)) + (add asert-radix (rsh [0 48] (add num (bex 47)))) +:: +:: +compute-exponent: (time-diff - ideal*(blocks-since-anchor - 1)) * radix / half-life +:: +:: returns sign-magnitude. sign %.y means non-negative, %.n means negative. +:: time-diff is given as sign-magnitude; blocks-since-anchor is unsigned +:: and must be >= 1 (caller guarantees current-height > anchor-height). +:: the (blocks-since-anchor - 1) factor is PDF Eq. (2) under the anchor- +:: own-ts convention (PDF §1.3 Option 2): time-diff spans parent-time - +:: anchor-time, i.e. (parent.height - anchor.height) = (blocks-since - 1) +:: ideal intervals under perfect schedule. +++ compute-exponent + |= $: time-diff-sign=? + time-diff-mag=@ + blocks-since-anchor=@ + ideal=@ + half-life=@ + == + ^- [sign=? mag=@] + =/ ideal-total (mul ideal (dec blocks-since-anchor)) + ?: time-diff-sign + ?: (gte time-diff-mag ideal-total) + =/ delta (sub time-diff-mag ideal-total) + [%.y (div (mul delta asert-radix) half-life)] + =/ delta (sub ideal-total time-diff-mag) + [%.n (div (mul delta asert-radix) half-life)] + =/ delta (add time-diff-mag ideal-total) + [%.n (div (mul delta asert-radix) half-life)] +:: +:: +decompose-exponent: split signed exponent into (integer shift, fractional) +:: +:: mirrors arithmetic right shift semantics: +:: positive x: shifts = x >> rbits, frac = x mod radix (both in [0, ...)) +:: negative x: shifts = floor(x / radix) (toward -inf), +:: frac = x - shifts*radix, always in [0, radix) +:: so for x = -5 with radix=4: shifts = -2, frac = 3. +++ decompose-exponent + |= [exp-sign=? exp-mag=@] + ^- [shifts-sign=? shifts-mag=@ frac=@] + =/ rem-mag (mod exp-mag asert-radix) + =/ quo-mag (rsh [0 asert-rbits] exp-mag) + ?: exp-sign + [%.y quo-mag rem-mag] + ?: =(0 rem-mag) + [%.n quo-mag 0] + [%.n +(quo-mag) (sub asert-radix rem-mag)] +:: +:: +compute-target: aserti3-2d target for a block given the anchor +:: +:: current-height must be > anchor-height (the caller is computing the +:: target for a strict descendant of the anchor). anchor-min-timestamp +:: is the anchor block's OWN median-of-11 (PDF §1.3 Option 2); +:: current-min-timestamp is the parent block's median-of-11 — so +:: time-diff = parent.min-ts - anchor.min-ts spans (blocks-since - 1) +:: ideal intervals under a perfect schedule. timestamps in seconds. +:: anchor-target is an atom; callers on bignum can use +compute-target-bn. +:: result is clamped to [1, max-target-atom]. +++ compute-target + |= $: anchor-target=@ + anchor-min-timestamp=@ + anchor-height=@ + current-min-timestamp=@ + current-height=@ + ideal-block-time=@ + half-life=@ + max-target-atom=@ + == + ^- @ + ?< (lte current-height anchor-height) + :: anchor-target = 0 would silently produce 0 * factor = 0, which then + :: clamps to 1 below — effectively freezing the chain. reject the + :: misconfiguration at the boundary rather than absorbing it. + ?< =(0 anchor-target) + =/ time-diff-sign=? (gte current-min-timestamp anchor-min-timestamp) + =/ time-diff-mag=@ + ?: time-diff-sign + (sub current-min-timestamp anchor-min-timestamp) + (sub anchor-min-timestamp current-min-timestamp) + =/ blocks-since-anchor (sub current-height anchor-height) + =/ exp + %- compute-exponent + :* time-diff-sign + time-diff-mag + blocks-since-anchor + ideal-block-time + half-life + == + =/ dec (decompose-exponent sign.exp mag.exp) + =/ factor (poly-factor frac.dec) + =/ unshifted (mul anchor-target factor) + =/ max-bits (met 0 max-target-atom) + :: + :: cap shift magnitude so intermediate noun stays bounded. positive + :: shifts beyond max-bits+rbits+2 are guaranteed to saturate at + :: max-target-atom; negative shifts beyond unshifted's bit-length + :: zero the result, which then clamps to 1. + =/ result=@ + ?: shifts-sign.dec + =/ cap (add max-bits (add asert-rbits 2)) + =/ eff ?:((gth shifts-mag.dec cap) cap shifts-mag.dec) + (rsh [0 asert-rbits] (lsh [0 eff] unshifted)) + =/ unshifted-bits (met 0 unshifted) + ?: (gte shifts-mag.dec unshifted-bits) 0 + (rsh [0 asert-rbits] (rsh [0 shifts-mag.dec] unshifted)) + ?: =(0 result) 1 + ?: (gth result max-target-atom) max-target-atom + result +:: +:: +compute-target-bn: thin bignum wrapper over +compute-target +++ compute-target-bn + |= $: anchor-target=bignum:bignum + anchor-min-timestamp=@ + anchor-height=@ + current-min-timestamp=@ + current-height=@ + ideal-block-time=@ + half-life=@ + max-target-atom=@ + == + ^- bignum:bignum + %- chunk:bignum + %- compute-target + :* (merge:bignum anchor-target) + anchor-min-timestamp + anchor-height + current-min-timestamp + current-height + ideal-block-time + half-life + max-target-atom + == +-- diff --git a/hoon/apps/dumbnet/lib/consensus.hoon b/hoon/apps/dumbnet/lib/consensus.hoon index fdae5b59d..93ee4b5b0 100644 --- a/hoon/apps/dumbnet/lib/consensus.hoon +++ b/hoon/apps/dumbnet/lib/consensus.hoon @@ -2,7 +2,8 @@ /= sp /common/stark/prover /= mine /common/pow /= dumb-transact /common/tx-engine -/= * /common/zoon +/= asert /apps/dumbnet/lib/asert +/= * /common/h-zoon :: :: this library is where _every_ update to the consensus state :: occurs, no matter how minor. @@ -12,23 +13,23 @@ :: assert preconditions, provide reason for failure ++ apt ^- (unit @tas) - ?. ~(apt z-by blocks-needed-by.c) `%inapt-blocks-needed-by - ?. ~(apt z-in excluded-txs.c) `%inapt-excluded-txs - ?. ~(apt z-by spent-by.c) `%inapt-spent-by - ?. ~(apt z-by pending-blocks.c) `%inapt-pending-blocks - ?. ~(apt z-by balance.c) `%inapt-balance - ?. ~(apt z-by txs.c) `%inapt-txs + ?. ~(apt h-by blocks-needed-by.c) `%inapt-blocks-needed-by + ?. ~(apt h-in excluded-txs.c) `%inapt-excluded-txs + ?. ~(apt h-by spent-by.c) `%inapt-spent-by + ?. ~(apt h-by pending-blocks.c) `%inapt-pending-blocks + ?. ~(apt h-by balance.c) `%inapt-balance + ?. ~(apt h-by txs.c) `%inapt-txs :: these would take too long but a full semantic verification would include them - ::?. ~(apt z-by raw-txs.c) `%inapt-raw-txs - ::?. ~(apt z-by blocks.c) `%inapt-blocks - ::?. ~(apt z-by min-timestamps.c) `%inapt-min-timestamps - ::?. ~(apt z-by epoch-start.c) `%inapt-epoch-start - ::?. ~(apt z-by targets.c) `%inapt-targets - ?. =(excluded-txs.c (~(int z-in excluded-txs.c) ~(key z-by raw-txs.c))) + ::?. ~(apt h-by raw-txs.c) `%inapt-raw-txs + ::?. ~(apt h-by blocks.c) `%inapt-blocks + ::?. ~(apt h-by min-timestamps.c) `%inapt-min-timestamps + ::?. ~(apt h-by epoch-start.c) `%inapt-epoch-start + ::?. ~(apt h-by targets.c) `%inapt-targets + ?. =(excluded-txs.c (~(int h-in excluded-txs.c) ~(key h-by raw-txs.c))) `%extra-excluded-txs - ?. =(*(z-set tx-id:t) (~(int z-in excluded-txs.c) ~(key z-by blocks-needed-by.c))) + ?. =(*(h-set tx-id:t) (~(int h-in excluded-txs.c) ~(key h-by blocks-needed-by.c))) `%excluded-txs-arent - ?. =(excluded-txs.c (~(dif z-in ~(key z-by raw-txs.c)) ~(key z-by blocks-needed-by.c))) + ?. =(excluded-txs.c (~(dif h-in ~(key h-by raw-txs.c)) ~(key h-by blocks-needed-by.c))) `%txs-fell-through-cracks ~ :: @@ -45,9 +46,9 @@ $(reason %txs-fell-through-cracks) :: %txs-fell-through-cracks - =/ rtx=(z-map tx-id:t *) raw-txs.c - =/ bnb=(z-map tx-id:t *) blocks-needed-by.c - c(excluded-txs ~(key z-by (~(dif z-by rtx) bnb))) + =/ rtx=(h-map tx-id:t *) raw-txs.c + =/ bnb=(h-map tx-id:t *) blocks-needed-by.c + c(excluded-txs ~(key h-by (~(dif h-by rtx) bnb))) == :: :: check for bad state, repair if necessary @@ -60,12 +61,12 @@ ++ has-raw-tx |= tid=tx-id:t ^- ? - (~(has z-by raw-txs.c) tid) + (~(has h-by raw-txs.c) tid) :: ++ get-raw-tx |= tid=tx-id:t ^- (unit raw-tx:t) - =/ tx (~(get z-by raw-txs.c) tid) + =/ tx (~(get h-by raw-txs.c) tid) ?~ tx ~ `raw-tx.u.tx :: ++ got-raw-tx @@ -74,10 +75,18 @@ (need (get-raw-tx tid)) :: :: checkpointed digests for chain stability +:: phase-2 cutover of 014-aletheia pins both the ASERT anchor block +:: (height 65,499) and the first ASERT block (height 65,500) so any +:: competing block at either height is rejected network-wide. the +:: anchor digest is the same digest the phase-1 +find-anchor-min-ts +:: helper would have walked to; pinning it freezes the median-of-11 +:: asert-anchor-min-timestamp now baked into blockchain-constants. ++ checkpointed-digests ^- (z-map page-number:t hash:t) %- ~(gas z-by *(z-map page-number:t hash:t)) - :~ [%16.128 (from-b58:hash:t 'ANjtb2YNFo3cAtLVkjkXXP2DJ2S5ZvByywpxgAa1UhxXM5f8YmiJLWX')] + :~ [%65.500 (from-b58:hash:t '4dr8f3hWcQfgSMUrKRcNb1Z4nwzECbbUuqDYUp8G4WF6G5ocFXzPp2')] + [%65.499 (from-b58:hash:t 'vYekzUpi6o95oA6qHfvcq9kVRzFMZLuUw33YxXQRqNCvBHwU7wys73')] + [%16.128 (from-b58:hash:t 'ANjtb2YNFo3cAtLVkjkXXP2DJ2S5ZvByywpxgAa1UhxXM5f8YmiJLWX')] [%4.032 (from-b58:hash:t 'DhaVTgMz6CMy3ZG3vsci1z9U2Gg7WZL6y3g7bZzfJLUbus1rd8j4BQU')] [%2.448 (from-b58:hash:t '9EChUtcNJumW5DDYgS6UP5UHfHtD6vFH7HoSqjmTuWP2Px6JdpxaR23')] [%720 (from-b58:hash:t 'C4vJRnFNHCLHKHVRJGiYeoiYXS7CyTGrVk2ibEv95HQiZoxRvtr5SRQ')] @@ -122,29 +131,36 @@ (inputs-in-balance raw get-cur-balance-names) :: ++ inputs-in-balance - |= [raw=raw-tx:t balance=(z-set nname:t)] + |= [raw=raw-tx:t balance=(h-set nname:t)] ^- ? :: set of inputs required by tx that are not in balance - =/ in-balance=(z-set nname:t) - (~(dif z-in ~(input-names get:raw-tx:t raw)) balance) + =/ in-balance=(h-set nname:t) + (~(dif h-in (zh-silt ~(input-names get:raw-tx:t raw))) balance) :: %.y: all inputs in .raw are in balance :: %.n: input(s) in .raw not in balance - =(*(z-set nname:t) in-balance) + =(*(h-set nname:t) in-balance) :: ++ get-cur-height ^- page-number:t - ~(height get:local-page:t (~(got z-by blocks.c) (need heaviest-block.c))) + ~(height get:local-page:t (~(got h-by blocks.c) (need heaviest-block.c))) :: ++ get-cur-balance - ^- (z-map nname:t nnote:t) + ^- (h-map nname:t nnote:t) ?~ heaviest-block.c ~> %slog.[1 'get-cur-balance: No known blocks, balance is empty'] - *(z-map nname:t nnote:t) - (~(got z-by balance.c) u.heaviest-block.c) + *(h-map nname:t nnote:t) + =/ heaviest-page=local-page:t + (~(got h-by blocks.c) u.heaviest-block.c) + ?~ balance=(~(get h-by balance.c) u.heaviest-block.c) + ?: =(*page-number:t ~(height get:local-page:t heaviest-page)) + *(h-map nname:t nnote:t) + ~| 'get-cur-balance: Missing balance for non-genesis heaviest block' + !! + u.balance :: ++ get-cur-balance-names - ^- (z-set nname:t) - ~(key z-by get-cur-balance) + ^- (h-set nname:t) + ~(key h-by get-cur-balance) :: :: :: +compute-target: find the new target @@ -193,6 +209,40 @@ ~> %slog.[0 (cat 3 'compute-target: New target: ' (rsh [3 2] (scot %ui next-target-atom)))] next-target-bn :: +:: +compute-target-asert: aserti3-2d target for a post-asert-activation block +:: +:: .child-height is the height the block is (or will be) at; +:: .parent-digest identifies its parent so we can read the parent's +:: median-of-11 from .min-timestamps (written during parent acceptance). +:: callers must guarantee .child-height >= .asert-phase, which implies +:: the min-timestamps lookup succeeds and the height >= anchor invariant +:: holds. used both to validate an accepted page and to compute the +:: target for a candidate block still being constructed. +++ compute-target-asert + |= [child-height=@ parent-digest=block-id:t] + ^- bignum:bignum:t + =/ parent-min-ts=@ + (~(got h-by min-timestamps.c) parent-digest) + :: phase 2 of 014-aletheia: the anchor's median-of-11 is a hardcoded + :: protocol constant captured at the canonical anchor block (height + :: 65,499). paired with the [%65.499 ...] checkpoint in + :: +checkpointed-digests, only one block at the anchor height is + :: admissible network-wide, so reading the constant is consensus- + :: identical to walking ancestry. + =/ anchor-min-ts=@ + asert-anchor-min-timestamp.blockchain-constants + %- chunk:bignum:t + %- compute-target:asert + :* asert-anchor-target-atom.blockchain-constants + anchor-min-ts + asert-anchor-height.blockchain-constants + parent-min-ts + child-height + asert-ideal-block-time.blockchain-constants + asert-half-life.blockchain-constants + max-target-atom:t + == +:: :: +compute-epoch-duration: computes the duration of an epoch in seconds :: :: to mitigate certain types of "time warp" attacks, the timestamp we mark @@ -208,11 +258,11 @@ |= last-block=block-id:t ^- @ =/ prev-last-block=block-id:t - (~(got z-by epoch-start.c) last-block) + (~(got h-by epoch-start.c) last-block) =/ epoch-start=@ - (~(got z-by min-timestamps.c) prev-last-block) + (~(got h-by min-timestamps.c) prev-last-block) =/ epoch-end=@ - (~(got z-by min-timestamps.c) last-block) + (~(got h-by min-timestamps.c) last-block) ~| "compute-epoch-duration: Time warp attack: Negative epoch duration" (sub epoch-end epoch-start) :: @@ -231,10 +281,10 @@ ^- consensus-state:dk :: update balance :: - =? balance.c !=(~ balance.acc) + =? balance.c !=(*(h-map nname:t nnote:t) balance.acc) :: if balance.acc is empty, this would still add the following to balance.c, :: so we do it conditionally. - (~(put z-by balance.c) ~(digest get:page:t pag) balance.acc) + (~(put h-by balance.c) ~(digest get:page:t pag) balance.acc) =/ cb=coinbase-split:t ~(coinbase get:page:t pag) =/ height=page-number:t ~(height get:page:t pag) =/ coinbases=(list coinbase:t) @@ -257,13 +307,13 @@ =. balance.c %+ roll coinbases |= [=coinbase:t bal=_balance.c] - (~(put z-bi bal) ~(digest get:page:t pag) ~(name get:nnote:t coinbase) coinbase) + (~(put h-bi bal) ~(digest get:page:t pag) ~(name get:nnote:t coinbase) coinbase) :: update txs :: =. txs.c - %- ~(rep z-by txs.acc) + %- ~(rep h-by txs.acc) |= [[=tx-id:t =tx:t] txs=_txs.c] - (~(put z-bi txs) ~(digest get:page:t pag) tx-id tx) + (~(put h-bi txs) ~(digest get:page:t pag) tx-id tx) :: :: update epoch map. the first block-id in an epoch maps to its parent, :: and each subsequent block maps to the same block-id as the first. this is helpful @@ -272,30 +322,38 @@ =. epoch-start.c ?: =(*page-number:t ~(height get:page:t pag)) :: genesis block is also considered the last block of the "0th" epoch. - (~(put z-by epoch-start.c) ~(digest get:page:t pag) ~(digest get:page:t pag)) + (~(put h-by epoch-start.c) ~(digest get:page:t pag) ~(digest get:page:t pag)) ?: =(0 ~(epoch-counter get:page:t pag)) - (~(put z-by epoch-start.c) ~(digest get:page:t pag) ~(parent get:page:t pag)) - %- ~(put z-by epoch-start.c) + (~(put h-by epoch-start.c) ~(digest get:page:t pag) ~(parent get:page:t pag)) + %- ~(put h-by epoch-start.c) :- ~(digest get:page:t pag) - (~(got z-by epoch-start.c) ~(parent get:page:t pag)) + (~(got h-by epoch-start.c) ~(parent get:page:t pag)) =. min-timestamps.c (update-min-timestamps now pag) :: =. targets.c + ?: (post-asert-activation:t ~(height get:page:t pag)) + :: post-asert-activation: store pag's own aserti3-2d target. validation and + :: the miner compute ASERT fresh via +compute-target-asert rather + :: than reading this map, so we only populate it for debugging and to + :: keep the map shape consistent across the activation boundary. + %- ~(put h-by targets.c) + :- ~(digest get:page:t pag) + (compute-target-asert ~(height get:page:t pag) ~(parent get:page:t pag)) ?: =(+(~(epoch-counter get:page:t pag)) blocks-per-epoch:t) :: last block of an epoch means update to target - %- ~(put z-by targets.c) + %- ~(put h-by targets.c) :- ~(digest get:page:t pag) (compute-target ~(digest get:page:t pag) ~(target get:page:t pag)) ?: =(~(height get:page:t pag) *page-number:t) :: genesis block - %- ~(put z-by targets.c) + %- ~(put h-by targets.c) [~(digest get:page:t pag) ~(target get:page:t pag)] :: target remains the same throughout an epoch - %- ~(put z-by targets.c) + %- ~(put h-by targets.c) :- ~(digest get:page:t pag) - (~(got z-by targets.c) ~(parent get:page:t pag)) + (~(got h-by targets.c) ~(parent get:page:t pag)) :: note we do not update heaviest-block here, since that is conditional :: and the effects emitted depend on whether we do it. - ?: (~(has z-by pending-blocks.c) ~(digest get:page:t pag)) + ?: (~(has h-by pending-blocks.c) ~(digest get:page:t pag)) (accept-pending-block ~(digest get:page:t pag)) (accept-block pag) :: @@ -322,7 +380,7 @@ ?. version-check ~& [%expected-vs-actual version version:(need ~(pow get:page:t pag))] [%.n %proof-version-invalid] - =/ par=page:t (to-page:local-page:t (~(got z-by blocks.c) ~(parent get:page:t pag))) + =/ par=page:t (to-page:local-page:t (~(got h-by blocks.c) ~(parent get:page:t pag))) :: this is already checked in +heard-block but is done here again :: to avoid a footgun ?. (check-digest:page:t pag) @@ -354,21 +412,25 @@ :: =/ check-timestamp=? ?& %+ gte ~(timestamp get:page:t pag) - (~(got z-by min-timestamps.c) ~(parent get:page:t pag)) + (~(got h-by min-timestamps.c) ~(parent get:page:t pag)) :: (lte ~(timestamp get:page:t pag) (add now-secs max-future-timestamp:t)) == ?. check-timestamp [%.n %page-timestamp-invalid] :: - :: check target - ?. =(~(target get:page:t pag) (~(got z-by targets.c) ~(parent get:page:t pag))) - [%.n %page-target-invalid] - :: :: check height ?. =(~(height get:page:t pag) +(~(height get:page:t par))) [%.n %page-height-invalid] :: + :: check target + =/ expected-target + ?: (post-asert-activation:t ~(height get:page:t pag)) + (compute-target-asert ~(height get:page:t pag) ~(parent get:page:t pag)) + (~(got h-by targets.c) ~(parent get:page:t pag)) + ?. =(~(target get:page:t pag) expected-target) + [%.n %page-target-invalid] + :: :: check if digest matches checkpointed history, skip check if fakenet ?~ genesis-seal.c ~> %slog.[1 'validate-page-without-txs: Fatal error: Genesis seal not set!'] @@ -418,13 +480,14 @@ ?. (check-size pag) ~> %slog.[1 (cat 3 'validate-page-with-txs: Block too large: ' digest-b58)] [%.n %block-too-large] - =/ raw-tx-set=(z-set (unit raw-tx:t)) - (~(run z-in ~(tx-ids get:page:t pag)) |=(=tx-id:t (get-raw-tx tx-id))) - =/ raw-tx-list=(list (unit raw-tx:t)) ~(tap z-in raw-tx-set) + =/ tx-id-list=(list tx-id:t) + ~(tap z-in ~(tx-ids get:page:t pag)) + =/ raw-tx-list=(list (unit raw-tx:t)) + (turn tx-id-list |=(=tx-id:t (get-raw-tx tx-id))) :: initialize balance transfer accumulator with parent block's balance =/ acc=tx-acc:t %+ new:tx-acc:t - (~(get z-by balance.c) ~(parent get:page:t pag)) + (~(get h-by balance.c) ~(parent get:page:t pag)) ~(height get:page:t pag) :: :: test to see that the input notes for all transactions @@ -463,10 +526,35 @@ %1 %+ roll ~(val z-by +.cb) |=([c=coins:t s=coins:t] (add c s)) == - =/ emission-and-fees=coins:t - (add (emission-calc:coinbase:t ~(height get:page:t pag)) fees.u.balance-transfer) + =/ emission=coins:t + (emission-calc:coinbase:t ~(height get:page:t pag)) + =/ emission-and-fees=coins:t (add emission fees.u.balance-transfer) ?. =(emission-and-fees total-split) [%.n %improper-split] + :: + :: Phase-gated v1 coinbase entry count. The +based:coinbase-split:v1 + :: parser allows up to `max-coinbase-split + 1` entries to admit the + :: fund slot post-asert-activation, but pre-activation v1 blocks + :: (v1-phase <= height < asert-phase) carry no fund slot and must + :: continue to cap at `max-coinbase-split` entries — matching the + :: legacy v0 rule. Without this gate, a miner could pre-activation + :: emit a 3-entry v1 coinbase that this branch accepts and stricter + :: implementations reject (consensus split). See + :: docs/2026-05-01-MR2545-EMISSIONS-REVIEW.md P1 #1. + =/ height=page-number:t ~(height get:page:t pag) + ?: ?& ?=([%1 *] cb) + (pre-asert-activation:t height) + (gth ~(wyt z-by +.cb) max-coinbase-split.blockchain-constants) + == + [%.n %coinbase-split-pre-activation-too-many] + :: + :: Post-activation (014-aletheia): coinbase must split 80/20 between + :: the miner and the consensus-known fund address. + ?: (post-asert-activation:t height) + ?. (check-fund-split cb emission) + [%.n %improper-fund-split] + ~> %slog.[0 (cat 3 'validate-page-with-txs: Block validated: ' digest-b58)] + [%.y u.balance-transfer] ~> %slog.[0 (cat 3 'validate-page-with-txs: Block validated: ' digest-b58)] [%.y u.balance-transfer] :: @@ -490,7 +578,7 @@ c(heaviest-block (some ~(digest get:page:t pag))) :: > rather than >= since we take the first heaviest block we've heard ?: %+ compare-heaviness:page:t pag - (~(got z-by blocks.c) (need heaviest-block.c)) + (~(got h-by blocks.c) (need heaviest-block.c)) =/ log-message %+ rap 3 :~ 'update-heaviest: ' @@ -510,17 +598,48 @@ ~> %slog.[0 log-message] c :: +:: +check-fund-split: validate that a post-asert-activation coinbase pays +:: the consensus-known fund address exactly floor(emission/5) atoms. +:: +:: The total-split-equals-(emission+fees) check has already passed +:: by the time this is called (see line ~515 above), and ++based on +:: the v1 coinbase-split caps total entries at max-coinbase-split+1. +:: So we only need to verify that: +:: (a) the split is v1 (post-asert-activation = post-v1-phase), +:: (b) the fund-address slot exists, +:: (c) that slot's coins equal exactly floor(emission/5). +:: The miner side is then `emission - fund-coins + fees`, +:: distributed across however many miner outputs the miner chose +:: (1 or 2; partner mode supported per 014-aletheia). +:: +:: Post-cap special case (height > tail-end): when emission == 0 the +:: expected fund share is 0, but +based:coinbase-split:v1 rejects +:: zero-coin entries — so the only valid representation is fund-slot +:: *absent*, with all fees flowing to miner-side outputs. See +:: docs/2026-05-01-MR2545-EMISSIONS-REVIEW.md P1 #2. +++ check-fund-split + |= [cb=coinbase-split:t emission=coins:t] + ^- ? + ?. ?=([%1 *] cb) %.n + =/ expected-fund-coins=coins:t (div emission 5) + =/ fund-coins=(unit coins:t) + (~(get z-by +.cb) fund-address:t) + ?: =(0 expected-fund-coins) + =(~ fund-coins) + ?~ fund-coins %.n + =(u.fund-coins expected-fund-coins) +:: :: +get-elders: get list of ancestor block IDs up to 24 deep :: (ordered newest->oldest) ++ get-elders |= [d=derived-state:dk bid=block-id:t] ^- (unit [page-number:t (list block-id:t)]) - =/ block (~(get z-by blocks.c) bid) + =/ block (~(get h-by blocks.c) bid) ?~ block ~ =/ unit-height=(unit page-number:t) ?~ heaviest-block.c `0 - =/ heaviest-block (~(get z-by blocks.c) u.heaviest-block.c) + =/ heaviest-block (~(get h-by blocks.c) u.heaviest-block.c) ?~ heaviest-block ~ `(min ~(height get:local-page:t u.heaviest-block) ~(height get:local-page:t u.block)) ?~ unit-height ~ @@ -543,7 +662,7 @@ :: ++ update-min-timestamps |= [now=@da pag=page:t] - ^- (z-map block-id:t @) + ^- (h-map block-id:t @) =/ min-timestamp=@ :: get timestamps of up to N=min-past-blocks prior blocks. =| prev-timestamps=(list @) @@ -560,10 +679,10 @@ (median:t prev-timestamps) %= $ b (dec b) - cur-block (to-page:local-page:t (~(got z-by blocks.c) ~(parent get:page:t cur-block))) + cur-block (to-page:local-page:t (~(got h-by blocks.c) ~(parent get:page:t cur-block))) == :: - (~(put z-by min-timestamps.c) ~(digest get:page:t pag) min-timestamp) + (~(put h-by min-timestamps.c) ~(digest get:page:t pag) min-timestamp) :: :::: pending block and tx functionality :: @@ -572,13 +691,13 @@ ++ accept-block |= pag=page:t ^- consensus-state:dk - ?< (~(has z-by blocks.c) ~(digest get:page:t pag)) - ?< (~(has z-by pending-blocks.c) ~(digest get:page:t pag)) - =. blocks.c (~(put z-by blocks.c) ~(digest get:page:t pag) (to-local-page:page:t pag)) + ?< (~(has h-by blocks.c) ~(digest get:page:t pag)) + ?< (~(has h-by pending-blocks.c) ~(digest get:page:t pag)) + =. blocks.c (~(put h-by blocks.c) ~(digest get:page:t pag) (to-local-page:page:t pag)) %- ~(rep z-in ~(tx-ids get:page:t pag)) |= [=tx-id:t c=_c] - =. blocks-needed-by.c (~(put z-ju blocks-needed-by.c) tx-id ~(digest get:page:t pag)) - =. excluded-txs.c (~(del z-in excluded-txs.c) tx-id) + =. blocks-needed-by.c (~(put h-ju blocks-needed-by.c) tx-id ~(digest get:page:t pag)) + =. excluded-txs.c (~(del h-in excluded-txs.c) tx-id) c :: :: add a block which is waiting on transactions to pending state. @@ -587,57 +706,57 @@ ++ add-pending-block |= pag=page:t ^- [(list tx-id:t) consensus-state:dk] - ?< (~(has z-by blocks.c) ~(digest get:page:t pag)) - ?< (~(has z-by pending-blocks.c) ~(digest get:page:t pag)) - =/ needed=(z-set tx-id:t) + ?< (~(has h-by blocks.c) ~(digest get:page:t pag)) + ?< (~(has h-by pending-blocks.c) ~(digest get:page:t pag)) + =/ needed=(h-set tx-id:t) %- ~(rep z-in ~(tx-ids get:page:t pag)) - |= [=tx-id:t needed=(z-set tx-id:t)] - ?. (~(has z-by raw-txs.c) tx-id) - (~(put z-in needed) tx-id) + |= [=tx-id:t needed=(h-set tx-id:t)] + ?. (~(has h-by raw-txs.c) tx-id) + (~(put h-in needed) tx-id) needed - ?: =(*(z-set tx-id:t) needed) + ?: =(*(h-set tx-id:t) needed) [~ c] :: not missing any transactions - =. pending-blocks.c (~(put z-by pending-blocks.c) ~(digest get:page:t pag) [pag get-cur-height]) + =. pending-blocks.c (~(put h-by pending-blocks.c) ~(digest get:page:t pag) [pag get-cur-height]) =. c %- ~(rep z-in ~(tx-ids get:page:t pag)) |= [=tx-id:t c=_c] - =. blocks-needed-by.c (~(put z-ju blocks-needed-by.c) tx-id ~(digest get:page:t pag)) - =. excluded-txs.c (~(del z-in excluded-txs.c) tx-id) + =. blocks-needed-by.c (~(put h-ju blocks-needed-by.c) tx-id ~(digest get:page:t pag)) + =. excluded-txs.c (~(del h-in excluded-txs.c) tx-id) c - [~(tap z-in needed) c] + [~(tap h-in needed) c] :: :: reject a pending block ++ reject-pending-block |= =block-id:t ^- consensus-state:dk :: block must be pending - ?< (~(has z-by blocks.c) block-id) - =/ pag page:(~(got z-by pending-blocks.c) block-id) + ?< (~(has h-by blocks.c) block-id) + =/ pag page:(~(got h-by pending-blocks.c) block-id) =. c - %- ~(rep z-by ~(tx-ids get:page:t pag)) + %- ~(rep z-in ~(tx-ids get:page:t pag)) |= [=tx-id:t c=_c] - =. blocks-needed-by.c (~(del z-ju blocks-needed-by.c) tx-id ~(digest get:page:t pag)) + =. blocks-needed-by.c (~(del h-ju blocks-needed-by.c) tx-id ~(digest get:page:t pag)) =? excluded-txs.c - ?& ?!((~(has z-by blocks-needed-by.c) tx-id)) :: not in blocks-needed-by - (~(has z-by raw-txs.c) tx-id) :: but in raw-txs + ?& ?!((~(has h-by blocks-needed-by.c) tx-id)) :: not in blocks-needed-by + (~(has h-by raw-txs.c) tx-id) :: but in raw-txs == - (~(put z-in excluded-txs.c) tx-id) + (~(put h-in excluded-txs.c) tx-id) c - =. pending-blocks.c (~(del z-by pending-blocks.c) ~(digest get:page:t pag)) + =. pending-blocks.c (~(del h-by pending-blocks.c) ~(digest get:page:t pag)) c :: :: missing transaction ids from pending blocks ++ missing-tx-ids ^- (list tx-id:t) - %~ tap z-in - ^- (z-set tx-id:t) - %- ~(rep z-by pending-blocks.c) - |= [[block-id:t pag=page:t *] all=(z-set tx-id:t)] - ^- (z-set tx-id:t) + %~ tap h-in + ^- (h-set tx-id:t) + %- ~(rep h-by pending-blocks.c) + |= [[block-id:t pag=page:t *] all=(h-set tx-id:t)] + ^- (h-set tx-id:t) %- ~(rep z-in ~(tx-ids get:page:t pag)) |= [=tx-id:t all=_all] - ?. (~(has z-by raw-txs.c) tx-id) - (~(put z-in all) tx-id) + ?. (~(has h-by raw-txs.c) tx-id) + (~(put h-in all) tx-id) all :: :: move a block from pending-blocks to blocks @@ -645,10 +764,10 @@ |= =block-id:t ^- consensus-state:dk :: block must be pending - ?< (~(has z-by blocks.c) block-id) - =/ pag page:(~(got z-by pending-blocks.c) block-id) - =. pending-blocks.c (~(del z-by pending-blocks.c) ~(digest get:page:t pag)) - =. blocks.c (~(put z-by blocks.c) block-id (to-local-page:page:t pag)) + ?< (~(has h-by blocks.c) block-id) + =/ pag page:(~(got h-by pending-blocks.c) block-id) + =. pending-blocks.c (~(del h-by pending-blocks.c) ~(digest get:page:t pag)) + =. blocks.c (~(put h-by blocks.c) block-id (to-local-page:page:t pag)) c :: :: list of pending blocks which are lower than the minimum retention height @@ -658,12 +777,12 @@ ?~ retain ~ ?~ heaviest-block.c ~ - =/ pag=page:t (to-page:local-page:t (~(got z-by blocks.c) u.heaviest-block.c)) + =/ pag=page:t (to-page:local-page:t (~(got h-by blocks.c) u.heaviest-block.c)) =/ height ~(height get:page:t pag) ?: (lth height u.retain) ~ =/ min-height (sub height u.retain) - %- ~(rep z-by pending-blocks.c) + %- ~(rep h-by pending-blocks.c) |= [[=block-id:t =page:t heard-at=@] dropable=(list block-id:t)] ?: (lte heard-at min-height) [block-id dropable] @@ -681,48 +800,48 @@ ++ inputs-spent |= =raw-tx:t ^- ? - =/ input-names=(z-set nname:t) - ~(input-names get:raw-tx:t raw-tx) - %- ~(any z-in input-names) - ~(has z-by spent-by.c) + =/ input-names=(h-set nname:t) + (zh-silt ~(input-names get:raw-tx:t raw-tx)) + %- ~(any h-in input-names) + ~(has h-by spent-by.c) :: :: Is the transaction needed by a block? ++ needed-by-block |= =tx-id:t ^- ? - (~(has z-by blocks-needed-by.c) tx-id) + (~(has h-by blocks-needed-by.c) tx-id) :: :: add an already-validated raw transaction, producing a list of blocks ready to validate ++ add-raw-tx |= =raw-tx:t ^- [(list block-id:t) consensus-state:dk] =/ =tx-id:t ~(id get:raw-tx:t raw-tx) - ?< (~(has z-by raw-txs.c) tx-id) - =. raw-txs.c (~(put z-by raw-txs.c) tx-id [raw-tx get-cur-height]) + ?< (~(has h-by raw-txs.c) tx-id) + =. raw-txs.c (~(put h-by raw-txs.c) tx-id [raw-tx get-cur-height]) =/ input-names=(z-set nname:t) ~(input-names get:raw-tx:t raw-tx) =. spent-by.c %- ~(rep z-in input-names) |= [=nname:t sb=_spent-by.c] - (~(put z-ju sb) nname tx-id) - =/ bnb (~(get z-ju blocks-needed-by.c) tx-id) - ?: =(*(z-set block-id:t) bnb) - =. excluded-txs.c (~(put z-in excluded-txs.c) tx-id) + (~(put h-ju sb) nname tx-id) + =/ bnb (~(get h-ju blocks-needed-by.c) tx-id) + ?: =(*(h-set block-id:t) bnb) + =. excluded-txs.c (~(put h-in excluded-txs.c) tx-id) [~ c] =/ ready-blocks=(list block-id:t) - %- ~(rep z-in bnb) + %- ~(rep h-in bnb) |= [=block-id:t ready=(list block-id:t)] - =/ pending (~(get z-by pending-blocks.c) block-id) + =/ pending (~(get h-by pending-blocks.c) block-id) ?~ pending ready =/ pag page.u.pending =/ needed %- ~(rep z-in ~(tx-ids get:page:t pag)) - |= [=tx-id:t needed=(z-set tx-id:t)] - ^- (z-set tx-id:t) - ?. (~(has z-by raw-txs.c) tx-id) - (~(put z-in needed) tx-id) + |= [=tx-id:t needed=(h-set tx-id:t)] + ^- (h-set tx-id:t) + ?. (~(has h-by raw-txs.c) tx-id) + (~(put h-in needed) tx-id) needed :: if the block is ready, add it to the ready list - ?: =(*(z-set tx-id:t) needed) + ?: =(*(h-set tx-id:t) needed) [block-id ready] ready [ready-blocks c] @@ -731,46 +850,55 @@ ++ drop-tx |= =tx-id:t ^- consensus-state:dk - ?< (~(has z-by blocks-needed-by.c) tx-id) - ?> (~(has z-in excluded-txs.c) tx-id) - =/ raw-tx raw-tx:(~(got z-by raw-txs.c) tx-id) - =. raw-txs.c (~(del z-by raw-txs.c) tx-id) - =. excluded-txs.c (~(del z-in excluded-txs.c) tx-id) + ?< (~(has h-by blocks-needed-by.c) tx-id) + ?> (~(has h-in excluded-txs.c) tx-id) + =/ raw-tx raw-tx:(~(got h-by raw-txs.c) tx-id) + =. raw-txs.c (~(del h-by raw-txs.c) tx-id) + =. excluded-txs.c (~(del h-in excluded-txs.c) tx-id) =. spent-by.c %- ~(rep z-in ~(input-names get:raw-tx:t raw-tx)) |= [=nname:t sb=_spent-by.c] - (~(del z-ju sb) nname ~(id get:raw-tx:t raw-tx)) + (~(del h-ju sb) nname ~(id get:raw-tx:t raw-tx)) c :: :: transactions which may be dropped (excluded and lower than minimum retention height) ++ dropable-txs |= retain=(unit @) - ^- (z-set tx-id:t) - ?~ heaviest-block.c ~ - =/ height ~(height get:local-page:t (~(got z-by blocks.c) u.heaviest-block.c)) - =/ spent=(z-set tx-id:t) - %- ~(rep z-in excluded-txs.c) - |= [=tx-id:t spent=(z-set tx-id:t)] - ^- (z-set tx-id:t) - =/ raw-tx raw-tx:(~(got z-by raw-txs.c) tx-id) - ?. (inputs-in-heaviest-balance raw-tx) - (~(put z-in spent) tx-id) + ^- (h-set tx-id:t) + ?~ heaviest-block.c *(h-set tx-id:t) + ?: =(*(h-set tx-id:t) excluded-txs.c) + *(h-set tx-id:t) + =/ height ~(height get:local-page:t (~(got h-by blocks.c) u.heaviest-block.c)) + :: Hoist the heaviest-balance name-set out of the per-tx fold: it is + :: loop-invariant (depends only on balance.c / heaviest-block.c, neither + :: mutated here), so computing it once turns the spent-fold from + :: O(|excluded-txs| * |balance|) into O(|balance| + |excluded-txs|). + :: Equivalent to the old per-tx (inputs-in-heaviest-balance raw), which + :: is defined as (inputs-in-balance raw get-cur-balance-names). + =/ cur-balance-names=(h-set nname:t) get-cur-balance-names + =/ spent=(h-set tx-id:t) + %- ~(rep h-in excluded-txs.c) + |= [=tx-id:t spent=(h-set tx-id:t)] + ^- (h-set tx-id:t) + =/ raw-tx raw-tx:(~(got h-by raw-txs.c) tx-id) + ?. (inputs-in-balance raw-tx cur-balance-names) + (~(put h-in spent) tx-id) spent ?~ retain spent ?: (lth height u.retain) spent =/ min-height (sub height u.retain) - %- ~(rep z-in excluded-txs.c) + %- ~(rep h-in excluded-txs.c) |= [=tx-id:t dropable=_spent] - =/ [=raw-tx:t heard-at=@] (~(got z-by raw-txs.c) tx-id) + =/ [=raw-tx:t heard-at=@] (~(got h-by raw-txs.c) tx-id) ?: (lte heard-at min-height) - (~(put z-in dropable) tx-id) + (~(put h-in dropable) tx-id) dropable :: :: drop all dropable transactions ++ drop-dropable-txs |= retain=(unit @) ^- consensus-state:dk - %- ~(rep z-in (dropable-txs retain)) + %- ~(rep h-in (dropable-txs retain)) |= [=tx-id:t con=_c] =. c con (drop-tx tx-id) @@ -779,6 +907,15 @@ ++ garbage-collect |= retain=(unit @) ^- consensus-state:dk + :: Excluded txs are GC'd on a much shorter window than pending blocks + :: (decoupled): keep at most min(retain, 4) blocks of excluded-tx + :: history -- 4 blocks if admin configured never-drop (~) -- so + :: |excluded-txs| stays bounded and the dropable-txs spent-fold does + :: not blow up. Pending blocks keep the full `retain`. + =/ tx-retain=(unit @) + ?~ retain `4 + `(min u.retain 4) + ~> %slog.[0 (cat 3 'garbage-collect: excluded-txs count ' (rsh [3 2] (scot %ui ~(wyt h-in excluded-txs.c))))] =. c (drop-dropable-blocks retain) - (drop-dropable-txs retain) + (drop-dropable-txs tx-retain) -- diff --git a/hoon/apps/dumbnet/lib/derived.hoon b/hoon/apps/dumbnet/lib/derived.hoon index 6581e435c..100b64e33 100644 --- a/hoon/apps/dumbnet/lib/derived.hoon +++ b/hoon/apps/dumbnet/lib/derived.hoon @@ -1,6 +1,6 @@ /= dk /apps/dumbnet/lib/types /= dumb-transact /common/tx-engine -/= * /common/zoon +/= * /common/h-zoon :: |_ [d=derived-state:dk =blockchain-constants:dumb-transact] +* t ~(. dumb-transact blockchain-constants) @@ -14,7 +14,7 @@ =/ heaviest-page=page:t ?: =(~ heaviest-block.c) pag :: genesis block - (to-page:local-page:t (~(got z-by blocks.c) (need heaviest-block.c))) + (to-page:local-page:t (~(got h-by blocks.c) (need heaviest-block.c))) =/ next-parent=block-id:t ~(digest get:page:t heaviest-page) =/ next-height=page-number:t ~(height get:page:t heaviest-page) |- @@ -31,7 +31,7 @@ d %= $ next-height (dec next-height) - next-parent ~(parent get:local-page:t (~(got z-by blocks.c) next-parent)) + next-parent ~(parent get:local-page:t (~(got h-by blocks.c) next-parent)) == ++ update-highest |= height=page-number:t @@ -51,7 +51,7 @@ ^- (unit ?) ?~ genesis-seal.c ?^ genesis-id=(~(get z-by heaviest-chain.d) 0) - =+ genesis=(~(get z-by blocks.c) u.genesis-id) + =+ genesis=(~(get h-by blocks.c) u.genesis-id) ?~ genesis ~ `=((hash:page-msg:t ~(msg get:local-page:t u.genesis)) realnet-genesis-msg:dk) diff --git a/hoon/apps/dumbnet/lib/miner.hoon b/hoon/apps/dumbnet/lib/miner.hoon index 0ce15de39..788521c94 100644 --- a/hoon/apps/dumbnet/lib/miner.hoon +++ b/hoon/apps/dumbnet/lib/miner.hoon @@ -1,7 +1,8 @@ /= dk /apps/dumbnet/lib/types /= sp /common/stark/prover /= dumb-transact /common/tx-engine -/= * /common/zoon +/= asert /apps/dumbnet/lib/asert +/= * /common/h-zoon :: :: everything to do with mining and mining state :: @@ -50,35 +51,35 @@ max-block-size:t :: :: grab all raw-txs that could possibly be included in block. -:: note that this set could include txs that are not spendable +:: note that this map could include txs that are not spendable :: from the current heaviest balance. we rely on the logic inside :: of process:tx-acc to catch these txs and reject them. ++ candidate-txs |= c=consensus-state:dk - ^- (z-set raw-tx:t) + ^- (h-map tx-id:t raw-tx:t) |^ - %- ~(rep z-in candidate-tx-ids) - |= [=tx-id:t txs=(set raw-tx:t)] - =/ raw raw-tx:(~(got z-by raw-txs.c) tx-id) - (~(put z-in txs) raw) + %- ~(rep h-in candidate-tx-ids) + |= [=tx-id:t txs=(h-map tx-id:t raw-tx:t)] + =/ raw raw-tx:(~(got h-by raw-txs.c) tx-id) + (~(put h-by txs) [tx-id raw]) :: :: union of excluded tx-ids and pending block tx ids :: excluding tx-ids already included in candidate block ++ candidate-tx-ids - %- %~ dif z-in - (~(uni z-in excluded-txs.c) pending-block-tx-ids) - ~(tx-ids get:page:t candidate-block.m) + %- %~ dif h-in + (~(uni h-in excluded-txs.c) pending-block-tx-ids) + (zh-silt ~(tx-ids get:page:t candidate-block.m)) :: :: set of available raw-txs from pending blocks ++ pending-block-tx-ids - ^- (z-set tx-id:t) - %- ~(rep z-by pending-blocks.c) - |= [[block-id:t pag=page:t *] all=(z-set tx-id:t)] - ^- (z-set tx-id:t) - %- ~(rep z-in ~(tx-ids get:page:t pag)) + ^- (h-set tx-id:t) + %- ~(rep h-by pending-blocks.c) + |= [[block-id:t pag=page:t *] all=(h-set tx-id:t)] + ^- (h-set tx-id:t) + %- ~(rep h-in (zh-silt ~(tx-ids get:page:t pag))) |= [=tx-id:t all=_all] - ?: (~(has z-by raw-txs.c) tx-id) - (~(put z-in all) tx-id) + ?: (~(has h-by raw-txs.c) tx-id) + (~(put h-in all) tx-id) all -- :: @@ -116,8 +117,8 @@ ^- mining-state:dk :: if the mining pubkey is not set, do nothing ?: no-keys-set m - %- ~(rep z-in (candidate-txs c)) - |= [raw=raw-tx:t min=_m] + %- ~(rep h-by (candidate-txs c)) + |= [[=tx-id:t raw=raw-tx:t] min=_m] =. m min (heard-new-tx raw) :: @@ -212,7 +213,16 @@ =. candidate-block.m ?^ -.candidate-block.m candidate-block.m(coinbase (new:v0:coinbase-split:t new-assets v0-shares.m)) - candidate-block.m(coinbase (new:v1:coinbase-split:t new-assets shares.m)) + :: v1 candidate: dispatch on activation height. Post-activation + :: uses the fee-aware 80/20 fund-aware builder (014-aletheia) which + :: takes emission and fees separately so the fund slot is computed + :: from the subsidy alone; pre-activation retains the existing + :: proportional-allocation arm. + ?: (pre-asert-activation:t height.candidate-block.m) + candidate-block.m(coinbase (new:v1:coinbase-split:t new-assets shares.m)) + =/ emission=coins:t + (emission-calc:coinbase:t height.candidate-block.m) + candidate-block.m(coinbase (new-with-fund-share:v1:coinbase-split:t emission new-fees shares.m)) :: check size of candidate block ?. candidate-block-below-max-size ~> %slog.[3 log-message-exceeds-max-size] @@ -266,21 +276,49 @@ (to-b58:hash:t u.heaviest-block.c) == ~> %slog.[0 log-message] - =/ parent-local=local-page:t (~(got z-by blocks.c) u.heaviest-block.c) + =/ parent-local=local-page:t (~(got h-by blocks.c) u.heaviest-block.c) =/ parent=page:t (to-page:local-page:t parent-local) + :: determine the target the candidate (child of .parent) must have. + :: post-activation: compute aserti3-2d fresh from the anchor and the + :: parent's stored median-of-11. pre-activation: read the next target + :: stored at parent.digest by the epoch rule. + =/ candidate-height=@ +(~(height get:page:t parent)) + =/ candidate-target=bignum:bignum:t + ?: (post-asert-activation:t candidate-height) + =/ parent-min-ts=@ + (~(got h-by min-timestamps.c) u.heaviest-block.c) + :: phase 2 of 014-aletheia: the anchor's median-of-11 is a + :: hardcoded protocol constant. paired with the [%65.499 ...] + :: checkpoint, only the canonical anchor block is admissible + :: at the anchor height, so reading the constant is consensus- + :: identical to the phase-1 parent-walk. + =/ anchor-min-ts=@ + asert-anchor-min-timestamp.blockchain-constants + %- chunk:bignum:t + %- compute-target:asert + :* asert-anchor-target-atom.blockchain-constants + anchor-min-ts + asert-anchor-height.blockchain-constants + parent-min-ts + candidate-height + asert-ideal-block-time.blockchain-constants + asert-half-life.blockchain-constants + max-target-atom:t + == + (~(got h-by targets.c) u.heaviest-block.c) =. candidate-block.m ?^ -.parent :: v0 parent - :: if candidate height is less than cutoff, use v0 new-candidate with v0 shares :: otherwise use v1 new-candidate with v1 shares ?: (lth +(height.parent) v1-phase.blockchain-constants) - (new-candidate:v0:page:t parent now (~(got z-by targets.c) u.heaviest-block.c) v0-shares.m) - (new-candidate:page:t parent now (~(got z-by targets.c) u.heaviest-block.c) shares.m) + (new-candidate:v0:page:t parent now candidate-target v0-shares.m) + (new-candidate:page:t parent now candidate-target shares.m asert-phase.blockchain-constants) :: v1 parent - use v1 new-candidate with v1 shares - (new-candidate:page:t parent now (~(got z-by targets.c) u.heaviest-block.c) shares.m) + (new-candidate:page:t parent now candidate-target shares.m asert-phase.blockchain-constants) =. candidate-acc.m %+ new:tx-acc:t - (~(get z-by balance.c) u.heaviest-block.c) + (~(get h-by balance.c) u.heaviest-block.c) ~(height get:page:t candidate-block.m) :: :: roll over the candidate txs and try to include them in the new candidate block diff --git a/hoon/apps/dumbnet/lib/pending.hoon b/hoon/apps/dumbnet/lib/pending.hoon deleted file mode 100644 index d8f78104e..000000000 --- a/hoon/apps/dumbnet/lib/pending.hoon +++ /dev/null @@ -1,253 +0,0 @@ -/= dcon /apps/dumbnet/lib/consensus -/= dk /apps/dumbnet/lib/types -/= dumb-transact /common/tx-engine -/= * /common/zoon -:: -|_ [p=pending-state:dk c=consensus-state:dk bc=blockchain-constants:dumb-transact] -+* t ~(. dumb-transact bc) -+| %logic -:: +find-ready-blocks: blocks for which .id was the last missing tx -++ has-raw-tx - |= tid=tx-id:t - ^- ? - |((~(has z-by raw-txs.p) tid) (~(has z-by raw-txs.c) tid)) -++ get-raw-tx - |= tid=tx-id:t - ^- (unit raw-tx:t) - =/ p-raw-tx (~(get z-by raw-txs.p) tid) - ?~ p-raw-tx - (~(get z-by raw-txs.c) tid) - p-raw-tx -++ got-raw-tx - |= tid=tx-id:t - ^- raw-tx:t - (need (get-raw-tx tid)) -++ find-ready-blocks - ^- (z-set block-id:t) - :: set of all pending blocks - =/ p-bids=(z-set block-id:t) ~(key z-by pending-blocks.p) - :: set of pending blocks still waiting on txs - =/ w-bids=(z-set block-id:t) ~(key z-by block-tx.p) - (~(dif z-in p-bids) w-bids) -:: -++ inputs-in-spent-by - |= raw=raw-tx:t - ^- ? - =/ inputs-names=(z-set nname:t) (inputs-names:raw-tx:t raw) - =/ spent-by-names=(z-set nname:t) ~(key z-by spent-by.p) - =/ common-names=(z-set nname:t) (~(int z-in spent-by-names) inputs-names) - :: %.y: inputs are present in spent-by - :: %.n: inputs are not present in spent-by - !=(*(z-set nname:t) common-names) -:: -++ refresh-after-new-block - |= retain=(unit @) - ^- pending-state:dk - ?~ retain - :: never drop transactions - p - ?: =(0 u.retain) - :: never retain anything - %_ p - spent-by *(z-map nname:t tx-id:t) - heard-at *(z-map tx-id:t page-number:t) - raw-txs *(z-map tx-id:t raw-tx:t) - == - :: - :: enumerate last N block heights inclusive of current block height - =/ cur-height=page-number:t - ~(get-cur-height dcon c p bc) - =/ min-height=page-number:t - ?: (lth cur-height u.retain) 0 - :: add 1 b/c range is inclusive of cur-height - :: so retain=1 means min-height=cur-height - +((sub cur-height u.retain)) - =/ heard-kvs=(list [tx-id:t page-number:t]) - ~(tap z-by heard-at.p) - :: - =/ keep-drop=[k=(list [tx-id:t page-number:t]) d=(list raw-tx:t)] - %+ roll heard-kvs - |= $: [tid=tx-id:t num=page-number:t] - [keep=(list [tx-id:t page-number:t]) drop=(list raw-tx:t)] - == - =/ raw=raw-tx:t (got-raw-tx tid) - ?: (lth num min-height) - :: tx is old, drop it - [keep [raw drop]] - ?. (~(inputs-in-heaviest-balance dcon c p bc) raw) - :: input(s) in tx not in balance, discard - [keep [raw drop]] - :: tx should stay in heard-at - [[[tid num] keep] drop] - :: - :: new heard-at map from k.keep-drop - =. heard-at.p - (~(gas z-by *(z-map tx-id:t page-number:t)) k.keep-drop) - :: - :: remove d.keep-drop from spent-by map - =. p - %+ roll d.keep-drop - |= [raw=raw-tx:t pen=_p] - =. p pen - (remove-inputs-from-spent-by raw) - :: - :: remove d.keep-drop from raw-txs map. we cannot just - :: make a new map with k.keep-drop since raw-txs also includes - :: txs for pending blocks - =. p - %+ roll d.keep-drop - |= [raw=raw-tx:t pen=_p] - =. p pen - (remove-raw-tx id.raw) - :: - p -+| %getters-and-setters -:: -:: +add-tx-not-in-pending-block: -:: -:: these transactions are treated differently from transactions -:: that are in a pending block. we track their inputs and when -:: we heard them. if we hear another transaction using some -:: of the same inputs, we discard it. if we heard the transaction -:: sufficiently long ago, we drop it from pending state. -++ add-tx-not-in-pending-block - |= [raw=raw-tx:t cur-height=page-number:t] - ^- pending-state:dk - =. p (add-raw-tx raw) - =. p (add-inputs-to-spent-by raw) - =. p (tx-heard-at id.raw cur-height) - p -:: -:: +add-tx-in-pending-block: -:: -:: when a tx is in a pending block, we ignore whether it uses -:: inputs that have been spent in the heaviest balance, or -:: when we heard it. -++ add-tx-in-pending-block - |= raw=raw-tx:t - ^- pending-state:dk - =. p (add-raw-tx raw) - :: this one needs to go before remove-tx-from-tx-block because - :: the keys it needs to inspect are gotten from .tx-block. - =. p (remove-tx-from-block-tx id.raw) - =. p (remove-tx-from-tx-block id.raw) - p -:: -++ tx-heard-at - |= [id=tx-id:t height=page-number:t] - ^- pending-state:dk - p(heard-at (~(put z-by heard-at.p) id height)) -:: -++ add-inputs-to-spent-by - |= raw=raw-tx:t - ^- pending-state:dk - =/ inputs-names=(list nname:t) - ~(tap z-in (inputs-names:raw-tx:t raw)) - =/ new-entries=(list [nname:t tx-id:t]) - (turn inputs-names |=(n=nname:t n^id.raw)) - p(spent-by (~(gas z-by spent-by.p) new-entries)) -:: -++ remove-inputs-from-spent-by - |= raw=raw-tx:t - ^- pending-state:dk - =/ inputs-names=(list nname:t) - ~(tap z-in (inputs-names:raw-tx:t raw)) - =. spent-by.p - %+ roll inputs-names - |= [nom=nname:t spb=_spent-by.p] - (~(del z-by spb) nom) - p -:: -:: +find-pending-tx-ids: pending tx-ids for pending blocks -++ find-pending-tx-ids - ^- (z-set tx-id:t) - %+ roll ~(val z-by block-tx.p) - |= [tx-ids=(z-set tx-id:t) all-tx-ids=(z-set tx-id:t)] - (~(uni z-in all-tx-ids) tx-ids) -:: -++ remove-tx-from-tx-block - |= id=tx-id:t - ^- pending-state:dk - p(tx-block (~(del z-by tx-block.p) id)) -:: -++ remove-tx-from-block-tx - |= tid=tx-id:t - ^- pending-state:dk - =/ block-vals=(unit (z-set block-id:t)) - (~(get z-by tx-block.p) tid) - ?~ block-vals - :: no blocks waiting on this tx, so do nothing - p - =/ block-list=(list block-id:t) - ~(tap z-in u.block-vals) - =. block-tx.p - %+ roll block-list - |= [bid=block-id:t blt=_block-tx.p] - (~(del z-ju blt) bid tid) - p -:: -++ add-pending-block - |= pag=page:t - ^- [(list tx-id:t) pending-state:dk] - :: find missing txs - =/ missing-txs=(list tx-id:t) - :: linear in the number of transactions in a block which is much smaller than - :: the number of transactions in the state, which key:z-by must traverse - %- ~(rep z-in tx-ids.pag) - |= [tid=tx-id:t missing=(list tx-id:t)] - ?. (has-raw-tx tid) - [tid missing] - missing - :: return missing txs and new pending state - :- missing-txs - =; pen=pending-state:dk - :: add to list of pending blocks if there are missing txs - =? pending-blocks.pen - !=(~ missing-txs) - (~(put z-by pending-blocks.pen) digest.pag (to-local-page:page:t pag)) - pen - %+ roll missing-txs - |= [tid=tx-id:t pen=_p] - :: block requires these txs to be complete - =. p pen - =. block-tx.p - (~(put z-ju block-tx.p) digest.pag tid) - :: add block to set of blocks that require this tx - =. tx-block.p - (~(put z-ju tx-block.p) tid digest.pag) - p -:: -++ remove-pending-block - |= bid=block-id:t - ^- pending-state:dk - =. pending-blocks.p (~(del z-by pending-blocks.p) bid) - :: get the txs that were needed for this block. if it was a valid - :: block, this will be empty already, but if it was invalid we - :: want to clean up the state. - =/ tx-vals=(unit (z-set tx-id:t)) - (~(get z-by block-tx.p) bid) - ?~ tx-vals - :: no txs needed by block so just delete from block-tx - =. block-tx.p (~(del z-by block-tx.p) bid) - p - =/ tx-list=(list tx-id:t) - ~(tap z-in u.tx-vals) - =. block-tx.p (~(del z-by block-tx.p) bid) - :: remove the block-id from this map. - =. tx-block.p - %+ roll tx-list - |= [tid=tx-id:t txb=_tx-block.p] - (~(del z-ju txb) tid bid) - p -:: -++ add-raw-tx - |= raw=raw-tx:t - ^- pending-state:dk - p(raw-txs (~(put z-by raw-txs.p) id.raw raw)) -:: -++ remove-raw-tx - |= tid=tx-id:t - ^- pending-state:dk - p(raw-txs (~(del z-by raw-txs.p) tid)) --- - diff --git a/hoon/apps/dumbnet/lib/types.hoon b/hoon/apps/dumbnet/lib/types.hoon index ebd21c9c4..bec11e001 100644 --- a/hoon/apps/dumbnet/lib/types.hoon +++ b/hoon/apps/dumbnet/lib/types.hoon @@ -1,4 +1,4 @@ -/= * /common/zoon +/= * /common/h-zoon /= zeke /common/zeke /= w /common/wrapper /= dt /common/tx-engine @@ -15,6 +15,9 @@ kernel-state-4 kernel-state-5 kernel-state-6 + kernel-state-7 + kernel-state-8 + kernel-state-9 == :: +$ kernel-state-0 @@ -80,6 +83,18 @@ constants=blockchain-constants:v0:dt == :: +:: frozen pre-ASERT snapshot of blockchain-constants:v1 (without the five +:: asert-* fields appended in this PR). used to decode old %6 states that +:: were serialized before the schema change. ++$ blockchain-constants-v1-pre-asert + $: v1-phase=@ + bythos-phase=@ + data=[max-size=@ min-fee=@] + base-fee=@ + input-fee-divisor=@ + blockchain-constants:v0:dt + == +:: +$ kernel-state-6 $: %6 c=consensus-state-6 @@ -87,10 +102,71 @@ m=mining-state-6 :: d=derived-state-6 + constants=blockchain-constants-v1-pre-asert + == +:: +:: frozen phase-1 snapshot of blockchain-constants:v1 (with the five +:: asert-* fields from kernel-state-7 but WITHOUT the phase-2 +:: asert-anchor-min-timestamp field appended in this PR). used to decode +:: old %7 states that were serialized before the schema change. ++$ blockchain-constants-v1-phase-1 + $: v1-phase=@ + bythos-phase=@ + data=[max-size=@ min-fee=@] + base-fee=@ + input-fee-divisor=@ + blockchain-constants:v0:dt + asert-phase=@ + asert-anchor-height=@ + asert-anchor-target-atom=@ + asert-ideal-block-time=@ + asert-half-life=@ + == +:: +:: kernel-state-7 originally had the same shape as kernel-state-6 but +:: tracked the schema change in blockchain-constants:v1: five ASERT fields +:: (asert-phase, anchor-height, anchor-target-atom, ideal-block-time, +:: half-life) were appended to the v1 wrapper in tx-engine-1.hoon. now +:: that a sixth field (asert-anchor-min-timestamp) has been added by +:: phase 2 of 014-aletheia, kernel-state-7 pins the frozen phase-1 +:: snapshot so old %7 states still decode. ++$ kernel-state-7 + $: %7 + c=consensus-state-7 + a=admin-state-7 + m=mining-state-7 + :: + d=derived-state-7 + constants=blockchain-constants-v1-phase-1 + == +:: +:: kernel-state-8 carries the full post-phase-2 blockchain-constants:v1 +:: (six asert-* fields, including the hardcoded anchor median-of-11). +:: the state-7-to-8 upgrade discards the old constants noun and lets +:: +update-constants reseed from *blockchain-constants:t on mainnet. ++$ kernel-state-8 + $: %8 + c=consensus-state-8 + a=admin-state-8 + m=mining-state-8 + :: + d=derived-state-8 + constants=blockchain-constants:v1:dt + == +:: +:: kernel-state-9 moves consensus core maps and sets to h-zoon +:: containers while preserving the post-phase-2 constants shape. ++$ kernel-state-9 + $: %9 + c=consensus-state-9 + a=admin-state-9 + m=mining-state-9 + :: + d=derived-state-9 constants=blockchain-constants:v1:dt == :: -+$ kernel-state kernel-state-6 ++$ kernel-state kernel-state-9 :: +$ consensus-state-0 $+ consensus-state-0 @@ -231,7 +307,55 @@ == == :: -+$ consensus-state consensus-state-6 ++$ consensus-state-7 $+(consensus-state-7 consensus-state-6) +:: ++$ consensus-state-8 $+(consensus-state-8 consensus-state-7) +:: ++$ consensus-state-9 + $+ consensus-state-9 + :: + :: indexes and not-fully-validated state + $: + $: + :: keys in raw-txs must be in EXACTLY ONE OF blocks-needed-by or excluded-txs + blocks-needed-by=(h-jug tx-id:dt block-id:dt) :: dependencies + excluded-txs=(h-set tx-id:dt) :: transactions unneeded by any block + :: + :: every tx-id in spent-by must be in raw-txs and vice-versa + spent-by=(h-jug nname:dt tx-id:dt) + :: + pending-blocks=(h-map block-id:dt [=page:dt heard-at=@]) :: pending blocks + == + :: + :: core consensus state + $: balance=(h-mip block-id:dt nname:dt nnote:dt) + txs=(h-mip block-id:dt tx-id:dt tx:dt) :: fully validated transactions + :: + :: keys in raw-txs must be in EXACTLY ONE OF blocks-needed-by or excluded-txs + raw-txs=(h-map tx-id:dt [=raw-tx:dt heard-at=@]) :: raw transactions + :: + blocks=(h-map block-id:dt local-page:dt) :: fully validated blocks + :: + heaviest-block=(unit block-id:dt) :: most recent heaviest block + :: + :: min timestamp of block that is a child of this block + min-timestamps=(h-map block-id:dt @) + :: this map is used to calculate epoch duration. it is a map of each + :: block-id to the first block-id in that epoch. + epoch-start=(h-map block-id:dt block-id:dt) + :: this map contains the expected target for the child + :: of a given block-id. + targets=(h-map block-id:dt bignum:bignum:dt) + :: + :: Bitcoin block hash for genesis block + ::>) TODO: change face to btc-hash? + btc-data=(unit (unit btc-hash:dt)) + =genesis-seal:dt :: desired seal for genesis block + == + == + +:: ++$ consensus-state consensus-state-9 :: :: you will not have lost any chain state if you lost pending state, you'd just have to :: request data again from peers and reset your mining state @@ -277,7 +401,13 @@ :: +$ admin-state-6 $+(admin-state-6 admin-state-5) :: -+$ admin-state admin-state-6 ++$ admin-state-7 $+(admin-state-7 admin-state-6) +:: ++$ admin-state-8 $+(admin-state-8 admin-state-7) +:: ++$ admin-state-9 $+(admin-state-9 admin-state-8) +:: ++$ admin-state admin-state-9 :: +$ derived-state-0 $+ derived-state-0 @@ -304,7 +434,13 @@ heaviest-chain=(z-map page-number:dt block-id:dt) == :: -+$ derived-state derived-state-6 ++$ derived-state-7 $+(derived-state-7 derived-state-6) +:: ++$ derived-state-8 $+(derived-state-8 derived-state-7) +:: ++$ derived-state-9 $+(derived-state-9 derived-state-8) +:: ++$ derived-state derived-state-9 :: +$ mining-state-0 $+ mining-state-0 @@ -328,6 +464,20 @@ :: +$ mining-state-6 $+ mining-state-6 + $: mining=? :: build candidate blocks? + shares=(z-map hash:dt @) :: shares of coinbase+fees among sighashes (v1) + v0-shares=(z-map sig:v0:dt @) :: shares of coinbase+fees among sigs (v0) + candidate-block=page:dt :: the next block we will attempt to mine. + candidate-acc=* :: old candidates are discarded by upgrades + next-nonce=noun-digest:tip5:zeke :: nonce being mined + == +:: ++$ mining-state-7 $+(mining-state-7 mining-state-6) +:: ++$ mining-state-8 $+(mining-state-8 mining-state-7) +:: ++$ mining-state-9 + $+ mining-state-9 $: mining=? :: build candidate blocks? shares=(z-map hash:dt @) :: shares of coinbase+fees among sighashes (v1) v0-shares=(z-map sig:v0:dt @) :: shares of coinbase+fees among sigs (v0) @@ -336,7 +486,7 @@ next-nonce=noun-digest:tip5:zeke :: nonce being mined == :: -+$ mining-state mining-state-6 ++$ mining-state mining-state-9 :: +$ init-phase $~(%.y ?) :: diff --git a/hoon/apps/wallet/lib/types.hoon b/hoon/apps/wallet/lib/types.hoon index f761616df..19aacc9e7 100644 --- a/hoon/apps/wallet/lib/types.hoon +++ b/hoon/apps/wallet/lib/types.hoon @@ -351,6 +351,58 @@ bc=blockchain-constants:transact == :: + :: frozen pre-ASERT snapshot of blockchain-constants:v1, used to decode + :: old %6 wallet states serialized before the five asert-* fields were added. + +$ blockchain-constants-v1-pre-asert + $: v1-phase=@ + bythos-phase=@ + data=[max-size=@ min-fee=@] + base-fee=@ + input-fee-divisor=@ + blockchain-constants:v0:transact + == + :: + :: frozen phase-1 snapshot of blockchain-constants:v1 (five asert-* + :: fields, no asert-anchor-min-timestamp). used to decode old %7 + :: wallet states serialized before phase 2 of 014-aletheia. + +$ blockchain-constants-v1-phase-1 + $: v1-phase=@ + bythos-phase=@ + data=[max-size=@ min-fee=@] + base-fee=@ + input-fee-divisor=@ + blockchain-constants:v0:transact + asert-phase=@ + asert-anchor-height=@ + asert-anchor-target-atom=@ + asert-ideal-block-time=@ + asert-half-life=@ + == + :: + +$ state-6 + $: %6 + balance=balance-v4 + active-master=active-v4 + keys=keys-v4 + bc=blockchain-constants-v1-pre-asert + == + :: + +$ state-7 + $: %7 + balance=balance-v4 + active-master=active-v4 + keys=keys-v4 + bc=blockchain-constants-v1-phase-1 + == + :: + +$ state-8 + $: %8 + balance=balance-v4 + active-master=active-v4 + keys=keys-v4 + bc=blockchain-constants:transact + == + :: :: $versioned-state: wallet state :: +$ versioned-state @@ -360,9 +412,12 @@ state-3 state-4 state-5 + state-6 + state-7 + state-8 == :: - +$ state $>(%5 versioned-state) + +$ state $>(%8 versioned-state) :: +$ seed-name $~('default-seed' @t) :: @@ -397,6 +452,23 @@ :: ++ key-version ?(%0 %1) :: + +$ create-tx-cause + $: names=(list [first=@t last=@t]) :: base58-encoded name hashes + orders=(list order) + fee=coins:transact :: fee + allow-low-fee=? :: bypass min fee check (unsafe, testing only) + sign-keys=(unit (list [child-index=@ud hardened=?])) :: child key information to sign from + refund-pkh=(unit hash:transact) :: refund pkh for spends over v0 notes + include-data=? :: whether or not we should include note-data. defaults + :: to yes in cli. not including note-data is a power-user option because + :: if the lock is not a standard 1-of-1 pkh or coinbase, the wallet won't + :: be able to guess it, so the funds could be lost forever if the user. + :: doesn't keep track of the lock. + save-raw-tx=? :: if %.y, saves jams of the raw-tx and its hashable into a txs-debug folder + :: in the current working directory + =selection-strategy + == + :: +$ cause $% [%keygen entropy=byts salt=byts] [%derive-child i=@ hardened=? label=(unit @tas)] @@ -417,22 +489,8 @@ [%verify-hash hash-b58=@t sig=@ pk-b58=@t] [%list-notes-by-address address=@t] :: base58-encoded address [%list-notes-by-address-csv address=@t] :: base58-encoded address, CSV format - $: %create-tx - names=(list [first=@t last=@t]) :: base58-encoded name hashes - orders=(list order) - fee=coins:transact :: fee - allow-low-fee=? :: bypass min fee check (unsafe, testing only) - sign-keys=(unit (list [child-index=@ud hardened=?])) :: child key information to sign from - refund-pkh=(unit hash:transact) :: refund pkh for spends over v0 notes - include-data=? :: whether or not we should include note-data. defaults - :: to yes in cli. not including note-data is a power-user option because - :: if the lock is not a standard 1-of-1 pkh or coinbase, the wallet won't - :: be able to guess it, so the funds could be lost forever if the user. - :: doesn't keep track of the lock. - save-raw-tx=? :: if %.y, saves jams of the raw-tx and its hashable into a txs-debug folder - :: in the current working directory - =selection-strategy - == + [%create-tx =create-tx-cause] + [%create-tx-batch requests=(list create-tx-cause)] [%list-active-addresses ~] [%list-notes ~] [%show-key-tree include-values=?] @@ -449,7 +507,7 @@ dat=transaction sign-keys=(unit (list [child-index=@ud hardened=?])) == - [%fakenet ~] + [%fakenet constants=blockchain-constants:transact] == +$ file-cause $% [%write path=@t contents=@t success=?] @@ -562,7 +620,17 @@ $+ versioned-transaction $^(transaction-0 transaction-1) :: - +$ transaction transaction-1 ++$ transaction transaction-1 + :: + :: + +$ active-signer-info + $: child-index=(unit @ud) + hardened=? + absolute-index=(unit @ud) + version=@ + pubkey=schnorr-pubkey:transact + address-b58=@t + == :: :: +$ nockchain-grpc-effect diff --git a/hoon/apps/wallet/lib/utils.hoon b/hoon/apps/wallet/lib/utils.hoon index 1c0838b89..6e0b483c4 100644 --- a/hoon/apps/wallet/lib/utils.hoon +++ b/hoon/apps/wallet/lib/utils.hoon @@ -38,10 +38,10 @@ |% ++ spends |= [raw-spends=spends:v1:transact =input-display:wt page-num=page-number:transact] - =+ bc=*blockchain-constants:transact =/ bythos-active=? (gte page-num bythos-phase.bc) =/ seeds-count=@ - (count-seed-words:spends:transact [raw-spends page-num]) + :: count seeds against the tx-engine instance already bound to this wallet's constants + (count-seed-words:spends:t [raw-spends page-num]) =/ witness-count=@ (count-witness-words [raw-spends input-display page-num]) :: match consensus formula: @@ -64,7 +64,6 @@ :: ++ count-witness-words |= [raw-spends=spends:v1:transact =input-display:wt page-num=page-number:transact] - =+ bc=*blockchain-constants:transact ?: (gte page-num bythos-phase.bc) (count-witness-words-raw [raw-spends input-display]) (count-witness-words-raw [raw-spends input-display]) @@ -325,6 +324,17 @@ 'N/A' (lock:v1:display u.lock.meta) :: + ++ watch-first-name-locks + ^- (list [name=hash:transact lock=(unit lock:transact)]) + =+ subtree=(~(kids of keys.state) watch-path) + %+ murn + ~(tap by kid.subtree) + |= [=trek =meta:wt] + ?. ?=(%first-name -.meta) + ~ + %- some + [name.meta lock.meta] + :: ++ watch-first-names ^- (list @t) =+ subtree=(~(kids of keys.state) watch-path) @@ -337,8 +347,8 @@ ?: (gte (met 3 addr) 132) acc =+ pubkey-hash=(from-b58:hash:transact addr) - =+ simple-name=(simple:v1:first-name:transact pubkey-hash) - =+ coinbase-name=(coinbase:v1:first-name:transact pubkey-hash) + =+ simple-name=(simple:v1:first-name:t pubkey-hash) + =+ coinbase-name=(coinbase:v1:first-name:t pubkey-hash) :+ (to-b58:hash:transact simple-name) (to-b58:hash:transact coinbase-name) acc diff --git a/hoon/apps/wallet/wallet.hoon b/hoon/apps/wallet/wallet.hoon index d67b718e2..20f8ecb77 100644 --- a/hoon/apps/wallet/wallet.hoon +++ b/hoon/apps/wallet/wallet.hoon @@ -5,6 +5,7 @@ /= transact /common/tx-engine /= z /common/zeke /= zo /common/zoon +/= hz /common/h-zoon /= dumb /apps/dumbnet/lib/types /= bridge /apps/bridge/types /= * /common/zose @@ -28,13 +29,14 @@ debug debug:utils warn warn:utils wt ~(. wallet-types bc.state) + t ~(. transact bc.state) :: ++ load |= old=versioned-state:wt ^- state:wt |^ |- - ?: ?=(%5 -.old) + ?: ?=(%8 -.old) old ~> %slog.[0 'load: State upgrade required'] ?- -.old @@ -43,6 +45,9 @@ %2 $(old state-2-3) %3 $(old state-3-4) %4 $(old state-4-5) + %5 $(old state-5-6) + %6 $(old state-6-7) + %7 $(old state-7-8) == :: ++ state-0-1 @@ -120,7 +125,7 @@ == :: ++ state-4-5 - ^- state:wt + ^- state-5:wt ?> ?=(%4 -.old) ~> %slog.[0 'upgrade version 4 to 5'] :* %5 @@ -129,6 +134,39 @@ keys.old *blockchain-constants:transact == + :: + ++ state-5-6 + ^- state-6:wt + ?> ?=(%5 -.old) + ~> %slog.[0 'upgrade version 5 to 6'] + :* %6 + balance.old + active-master.old + keys.old + *blockchain-constants-v1-pre-asert:wt + == + :: + ++ state-6-7 + ^- state-7:wt + ?> ?=(%6 -.old) + ~> %slog.[0 'upgrade version 6 to 7'] + :* %7 + balance.old + active-master.old + keys.old + *blockchain-constants-v1-phase-1:wt + == + :: + ++ state-7-8 + ^- state-8:wt + ?> ?=(%7 -.old) + ~> %slog.[0 'upgrade version 7 to 8'] + :* %8 + balance.old + active-master.old + keys.old + *blockchain-constants:transact + == -- :: ++ peek @@ -142,6 +180,9 @@ :: [%balance ~] ``balance.state + :: + [%blockchain-constants ~] + ``bc.state :: [%state ~] ``state @@ -181,6 +222,83 @@ ?: ?=(%1 -.coil) ~ `~(address to-b58:coil:wt coil) + :: + :: returns tracked first-name lock cache entries in machine-readable form + [%tracked-locks ~] + :+ ~ + ~ + watch-first-name-locks:get:v + :: + :: returns the master signer key hash for the current master key + [%master-signing-key ~] + :+ ~ + ~ + =/ sk=schnorr-seckey:transact + (sign-key:get:v ~) + =/ signer-pkh=hash:transact + %- hash:schnorr-pubkey:transact + %- from-sk:schnorr-pubkey:transact + (to-atom:schnorr-seckey:transact sk) + signer-pkh + :: + :: returns the master signer pubkey for the current master key + [%master-signing-pubkey ~] + :+ ~ + ~ + =/ sk=schnorr-seckey:transact + (sign-key:get:v ~) + %- from-sk:schnorr-pubkey:transact + (to-atom:schnorr-seckey:transact sk) + :: + :: returns local signing entries under the active master key + :: [(unit child-index) hardened (unit absolute-index) version signer-pubkey address-b58] + [%active-signers ~] + :+ ~ + ~ + %+ murn + ~(keys get:v %prv) + |= [pax=trek coil=coil:wt] + ^- (unit active-signer-info:wt) + ?~ pax + ~ + =/ version=@ -.coil + =/ keyc=keyc:s10 ~(keyc get:coil:wt coil) + =/ public-key=@ + public-key:(from-private:s10 keyc) + =/ public-coil=coil:wt + ?+ version ~|('unsupported protocol version' !!) + %0 [%0 [%pub public-key] cc.coil] + %1 [%1 [%pub public-key] cc.coil] + == + =/ pubkey=schnorr-pubkey:transact + (from-ser:schnorr-pubkey:transact public-key) + =/ address-b58=@t + ~(address to-b58:coil:wt public-coil) + =/ relative-path=@t + (crip (en-tape:trek pax)) + ?: =(relative-path '/m') + %- some + :* ~ + %.n + ~ + version + pubkey + address-b58 + == + =/ abs-index=@ud + (slav %ud (crip (slag 1 (trip relative-path)))) + =/ hardened=? + (gte abs-index (bex 31)) + =/ child-index=@ud + ?:(hardened (sub abs-index (bex 31)) abs-index) + %- some + :* (some child-index) + hardened + (some abs-index) + version + pubkey + address-b58 + == == :: ++ poke @@ -215,6 +333,7 @@ %list-notes-by-address (do-list-notes-by-address cause) %list-notes-by-address-csv (do-list-notes-by-address-csv cause) %create-tx (do-create-tx cause) + %create-tx-batch (do-create-tx-batch cause) %sign-multisig-tx (do-sign-multisig-tx cause) %update-balance-grpc (do-update-balance-grpc cause) %sign-message (do-sign-message cause) @@ -239,7 +358,7 @@ %show-master-zprv (do-show-master-zprv cause) %list-master-addresses (do-list-master-addresses cause) %set-active-master-address (do-set-active-master-address cause) - %fakenet [[%exit 0]~ state(bc bc.state(coinbase-timelock-min 1))] + %fakenet [[%exit 0]~ state(bc constants.cause)] :: %file ?- +<.cause @@ -740,7 +859,7 @@ =+ data=data:*blockchain-constants:transact =/ valid=(reason:dumb ~) %- validate-with-context:spends:transact - [notes.balance.state signed-spends height.balance.state max-size.data bythos-phase.bc.state] + [(zh-molt:hz notes.balance.state) signed-spends height.balance.state max-size.data bythos-phase.bc.state] ?- -.valid %.y =/ nock-cause=$>(%fact cause:dumb) @@ -1162,8 +1281,8 @@ ?^ -.note %.n :: look for coinbase notes with target-pkh :: or notes with simple 1-of-1 lock containing - =+ simple-fn=(simple:v1:first-name:transact target-pkh) - =+ coinbase-fn=(coinbase:v1:first-name:transact target-pkh) + =+ simple-fn=(simple:v1:first-name:t target-pkh) + =+ coinbase-fn=(coinbase:v1:first-name:t target-pkh) ?| =(simple-fn -.name.note) =(coinbase-fn -.name.note) == @@ -1214,8 +1333,8 @@ ?^ -.note %.n :: look for coinbase notes with target-pkh :: or notes with simple 1-of-1 lock containing - =+ simple-fn=(simple:v1:first-name:transact target-pkh) - =+ coinbase-fn=(coinbase:v1:first-name:transact target-pkh) + =+ simple-fn=(simple:v1:first-name:t target-pkh) + =+ coinbase-fn=(coinbase:v1:first-name:t target-pkh) ?| =(simple-fn -.name.note) =(coinbase-fn -.name.note) == @@ -1263,10 +1382,8 @@ ++ do-create-tx |= =cause:wt ?> ?=(%create-tx -.cause) - |^ - %- (debug "create-tx: {}") - =/ names=(list nname:transact) (parse-names names.cause) - =/ orders=(list order:wt) orders.cause + =/ tx-cause=create-tx-cause:wt +.cause + %- (debug "create-tx: {}") ?~ active-master.state :_ state :~ :- %markdown @@ -1276,6 +1393,53 @@ """ [%exit 0] == + =/ [=transaction:wt lock-roots-to-watch=(z-set:zo [hash:transact (unit lock:transact)])] + (build-create-tx tx-cause) + =/ effects=(list effect:wt) + (save-transaction transaction save-raw-tx.tx-cause) + ?: ?=(~ lock-roots-to-watch) + [effects state] + :- effects + state(keys (watch-root-locks lock-roots-to-watch)) + :: + ++ do-create-tx-batch + |= =cause:wt + ?> ?=(%create-tx-batch -.cause) + %- (debug "create-tx-batch: {<(lent requests.cause)>} request(s)") + ?~ active-master.state + :_ state + :~ :- %markdown + %- crip + """ + Cannot create a transaction without active master address set. Please import a master key / seed phrase or generate a new one. + """ + [%exit 0] + == + ?~ requests.cause + [[%exit 0]~ state] + =/ built=(list [tx-ser=transaction:wt save-raw-tx=? roots=(z-set:zo [hash:transact (unit lock:transact)])]) + %+ turn requests.cause + |= tx-cause=create-tx-cause:wt + =/ [tx-ser=transaction:wt roots=(z-set:zo [hash:transact (unit lock:transact)])] + (build-create-tx tx-cause) + [tx-ser save-raw-tx.tx-cause roots] + =/ effects=(list effect:wt) + (save-transactions-batch built) + =/ lock-roots-to-watch=(z-set:zo [hash:transact (unit lock:transact)]) + %+ roll built + |= [[tx-ser=transaction:wt save-raw-tx=? roots=(z-set:zo [hash:transact (unit lock:transact)])] acc=(z-set:zo [hash:transact (unit lock:transact)])] + (~(uni z-in:zo acc) roots) + ?: ?=(~ lock-roots-to-watch) + [effects state] + :- effects + state(keys (watch-root-locks lock-roots-to-watch)) + :: + ++ build-create-tx + |= cause=create-tx-cause:wt + ^- [transaction=transaction:wt lock-roots-to-watch=(z-set:zo [hash:transact (unit lock:transact)])] + =/ names=(list nname:transact) + (parse-create-tx-names names.cause) + =/ orders=(list order:wt) orders.cause =/ sign-keys=(list schnorr-seckey:transact) ?~ sign-keys.cause ~[(sign-key:get:v ~)] @@ -1307,101 +1471,113 @@ display display witness-data witness-data == - =/ res=effects=(list effect:wt) - (save-transaction transaction) - ?: ?=(~ lock-roots-to-watch) - [effects.res state] - :- effects.res - state(keys (watch-root-locks lock-roots-to-watch)) - :: - ++ parse-names - |= raw-names=(list [first=@t last=@t]) - ^- (list nname:transact) - %+ turn raw-names - |= [first=@t last=@t] - (from-b58:nname:transact [first last]) - :: - ++ save-transaction - |= tx-ser=transaction:wt - ^- (list effect:wt) - :: we fallback to the hash of the spends as the transaction name - :: when generating filenames to ensure uniqueness. - =/ =raw-tx:v1:transact (new:raw-tx:v1:transact spends.tx-ser) - =/ =tx:v1:transact (new:tx:v1:transact raw-tx height.balance.state) - =/ =witness-data:wt witness-data.tx-ser - =/ fees=@ (roll-fees:spends:v1:transact spends.tx-ser) - =/ markdown-text=@t - %: transaction:v1:display:utils - name.tx-ser - outputs.tx - fees - display.tx-ser - get-note:v - `witness-data - == - :: jam inputs and save as transaction - =/ transaction-jam (jam tx-ser) - =/ tx-path=@t - (crip "./txs/{(trip name.tx-ser)}.tx") - %- (debug "saving transaction to {}") - =/ write-effect=effect:wt - ?. save-raw-tx.cause - [%file %write tx-path transaction-jam] - =/ hashable-path=@t - %- crip - "./txs-debug/{(trip name.tx-ser)}-hashable.jam" - =/ raw-tx-path=@t - %- crip - "./txs-debug/{(trip name.tx-ser)}.jam" - :* %file - %batch-write - :~ [hashable-path (jam [leaf+%1 (hashable:spends:transact spends.tx-ser)])] - [tx-path transaction-jam] - [raw-tx-path (jam raw-tx)] - == - == - =. markdown-text - ;: (cury cat 3) - '\0a## Create Tx' - '\0a - Saved transaction to ' - tx-path - '\0a ' - markdown-text - == - ~[write-effect [%markdown markdown-text]] - :: - ++ gather-watch-roots - |= orders=(list order:wt) - ^- (z-set:zo [hash:transact (unit lock:transact)]) - %- z-silt:zo - %+ murn orders - |= ord=order:wt - ?- -.ord - %pkh ~ - :: - %multisig - =/ allowed=(z-set:zo hash:transact) (z-silt:zo participants.ord) - =/ lock [%pkh [m=threshold.ord allowed]]~ - %- some - :- (hash:lock:transact lock) - `lock - :: - %lock-root - `[root.ord ~] - :: - %bridge-deposit - `[bridge-lock-root-default:bridge ~] + [transaction lock-roots-to-watch] + :: + ++ parse-create-tx-names + |= raw-names=(list [first=@t last=@t]) + ^- (list nname:transact) + %+ turn raw-names + |= [first=@t last=@t] + (from-b58:nname:transact [first last]) + :: + ++ transaction-file-path + |= tx-ser=transaction:wt + ^- @t + (crip "./txs/{(trip name.tx-ser)}.tx") + :: + ++ transaction-write-files + |= [tx-ser=transaction:wt save-raw-tx=?] + ^- (list [path=@t contents=@]) + =/ =raw-tx:v1:transact (new:raw-tx:v1:transact spends.tx-ser) + =/ transaction-jam=@ (jam tx-ser) + =/ tx-path=@t (transaction-file-path tx-ser) + ?. save-raw-tx + ~[[tx-path transaction-jam]] + =/ hashable-path=@t + %- crip + "./txs-debug/{(trip name.tx-ser)}-hashable.jam" + =/ raw-tx-path=@t + %- crip + "./txs-debug/{(trip name.tx-ser)}.jam" + :~ [hashable-path (jam [leaf+%1 (hashable:spends:transact spends.tx-ser)])] + [tx-path transaction-jam] + [raw-tx-path (jam raw-tx)] + == + :: + ++ save-transaction + |= [tx-ser=transaction:wt save-raw-tx=?] + ^- (list effect:wt) + =/ =raw-tx:v1:transact (new:raw-tx:v1:transact spends.tx-ser) + =/ =tx:v1:transact (new:tx:v1:transact raw-tx height.balance.state) + =/ =witness-data:wt witness-data.tx-ser + =/ fees=@ (roll-fees:spends:v1:transact spends.tx-ser) + =/ tx-path=@t (transaction-file-path tx-ser) + =/ markdown-text=@t + %: transaction:v1:display:utils + name.tx-ser + outputs.tx + fees + display.tx-ser + get-note:v + `witness-data == + %- (debug "saving transaction to {}") + =/ files=(list [path=@t contents=@]) + (transaction-write-files tx-ser save-raw-tx) + =/ write-effect=effect:wt + ?. save-raw-tx + [%file %write tx-path (jam tx-ser)] + [%file %batch-write files] + =. markdown-text + ;: (cury cat 3) + '\0a## Create Tx' + '\0a - Saved transaction to ' + tx-path + '\0a ' + markdown-text + == + ~[write-effect [%markdown markdown-text]] + :: + ++ save-transactions-batch + |= txs=(list [tx-ser=transaction:wt save-raw-tx=? roots=(z-set:zo [hash:transact (unit lock:transact)])]) + ^- (list effect:wt) + =/ files=(list [path=@t contents=@]) + %- zing + %+ turn txs + |= [tx-ser=transaction:wt save-raw-tx=? roots=(z-set:zo [hash:transact (unit lock:transact)])] + (transaction-write-files tx-ser save-raw-tx) + ~[[%file %batch-write files]] + :: + ++ gather-watch-roots + |= orders=(list order:wt) + ^- (z-set:zo [hash:transact (unit lock:transact)]) + %- z-silt:zo + %+ murn orders + |= ord=order:wt + ?- -.ord + %pkh ~ :: - ++ watch-root-locks - |= roots=(z-set:zo [hash:transact (unit lock:transact)]) - ^- keys:wt - %- ~(rep z-in:zo roots) - |= [[root=hash:transact lock=(unit lock:transact)] acc=_keys.state] - %- watch-first-name:put:v - [(first:nname:transact root) lock] + %multisig + =/ allowed=(z-set:zo hash:transact) (z-silt:zo participants.ord) + =/ lock [%pkh [m=threshold.ord allowed]]~ + %- some + :- (hash:lock:transact lock) + `lock :: - -- + %lock-root + `[root.ord ~] + :: + %bridge-deposit + `[bridge-lock-root-default:bridge ~] + == + :: + ++ watch-root-locks + |= roots=(z-set:zo [hash:transact (unit lock:transact)]) + ^- keys:wt + %- ~(rep z-in:zo roots) + |= [[root=hash:transact lock=(unit lock:transact)] acc=_keys.state] + %- watch-first-name:put:v + [(first:nname:transact root) lock] :: ++ do-keygen |= =cause:wt diff --git a/hoon/common/h-zoon-test.hoon b/hoon/common/h-zoon-test.hoon new file mode 100644 index 000000000..f59e422f2 --- /dev/null +++ b/hoon/common/h-zoon-test.hoon @@ -0,0 +1,100 @@ +:: common/h-zoon-test.hoon +:: +:: compile-time checks for hashed-key container boundaries. +:: +/= * /common/h-zoon +/= transact /common/tx-engine-0 +|% +++ h-test1 + |= [a=(h-set hashed) b=hashed] + ^- ? + (~(has h-in a) b) +++ h-test2 + |= [a=(h-set noun-digest:tip5:z)] + ^- ? + (~(has h-in a) (atom-to-digest:tip5:z `@ux`5)) +:: this will not compile +:: ++ h-test3 +:: |= [a=(h-set @)] +:: ^- ? +:: %.n +++ h-test4 + |= [a=(h-set nname:transact)] + ^- ? + %.n +++ h-test5 + |= [a=(h-set noun-digest:tip5:z)] + ^- (z-set noun-digest:tip5:z) + (hz-silt a) +++ h-test6 + |= [a=(z-set noun-digest:tip5:z)] + ^- (h-set noun-digest:tip5:z) + (zh-silt a) +:: this won't compile either +:: ++ h-test7 +:: |= [a=(z-set @)] +:: ^- (h-set @) +:: (zh-silt a) +++ h-test8 + |= [a=(h-map noun-digest:tip5:z @tas)] + %- ~(rep h-by a) + |= [[=noun-digest:tip5:z v=@tas] sum=_0] + %+ add sum + `@`v +++ h-test9 + |= [a=(h-map hashed @tas)] + (~(put h-by a) [~ 'abc']) +:: this will not compile either +:: ++ h-test10 +:: |= [a=(h-map noun-digest:tip5:z @tas)] +:: (~(put h-by a) [~ 'abc']) +++ h-test11 + |= [a=(h-map nname:transact @tas)] + (~(put h-by a) [[(atom-to-digest:tip5:z 0x0) (atom-to-digest:tip5:z 0x0) ~] 'abc']) +:: this will not compile either +:: ++ h-test12 +:: |= [a=(h-map nname:transact @tas)] +:: (~(put h-by a) [~ 'abc']) +++ h-test13 + |= [a=(h-set noun-digest:tip5:z)] + (~(put h-in a) (atom-to-digest:tip5:z 0x0)) +++ h-test14 + |= [a=(z-jug noun-digest:tip5:z noun-digest:tip5:z)] + (zh-jult a) +++ h-test15 + |= [a=(z-mip noun-digest:tip5:z noun-digest:tip5:z ~)] + (zh-milt a) +++ h-test16 + |= [a=~] + ^- (h-map ~ @) + =/ b=(h-map ~ @) ~ + (~(put h-by b) [~ 4]) +:: ++ h-test17 +:: |= [a=(h-map @ @)] +:: ^- (h-map @ @) +:: a +++ h-test18 + |= [a=(h-map noun-digest:tip5:z ~)] + (hz-molt a) +++ h-test19 + |= [a=(h-mip noun-digest:tip5:z noun-digest:tip5:z ~)] + (h-test18 (~(got h-by a) (atom-to-digest:tip5:z 0x5))) +++ h-test20 + |= [a=(h-mip noun-digest:tip5:z noun-digest:tip5:z ~)] + (hz-molt (~(got h-by a) (atom-to-digest:tip5:z 0x5))) +:: This won't compile either +:: ++ h-test21 +:: |= [a=~] +:: ^- (h-map noun-digests:z @) +:: =/ b=(h-map noun-digests:z @) ~ +:: (~(put h-by b) [(atom-to-digest:tip5:z 0x1) 0x5]) +:: ++ h-test22 +:: |= [a=(h-set noun-digest:tip5:z)] +:: ^- ? +:: (~(has z-in a) (atom-to-digest:tip5:z `@ux`5)) +:: This won't compile: z-set uses ztree, h-in expects hset-tree +:: ++ h-test23 +:: |= [a=(z-set noun-digest:tip5:z)] +:: ^- ? +:: (~(has h-in a) (atom-to-digest:tip5:z `@ux`5)) +-- diff --git a/hoon/common/h-zoon.hoon b/hoon/common/h-zoon.hoon new file mode 100644 index 000000000..7bbc408f8 --- /dev/null +++ b/hoon/common/h-zoon.hoon @@ -0,0 +1,862 @@ +:: common/h-zoon/hoon +:: +:: deterministic treap containers for digest-shaped keys. +:: h-map and h-set keep zoon tree semantics while comparing +:: existing digest limbs to avoid repeated tip5 hashing. +:: keys are either one tip5 digest or a non-singleton list of tip5 digests. +:: +:: h-map and h-set use %hmap and %hset empty leaves. this gives the hoon +:: guard a real boundary from legacy z-map and z-set nouns. callers cross +:: the boundary with zh-* and hz-* helpers, never by treating one tree kind +:: as the other. +:: +:: the algebraic contract for these containers and the seven h-zoon jets +:: (gor-hip, mor-hip, zh-molt, zh-silt, zh-milt, zh-balmilt, zh-jult) is +:: written down in docs/H-ZOON-TEST-PLAN.md under "Algebraic Contract". +:: when landing a new h-* operation or jet, update the contract there and +:: add the corresponding randomized coverage in tests/zoon/mod/h-zoon and +:: tests/dumb/mod/unit/h-zoon-consensus so reviewers can grep for the law +:: a change must preserve. +:: +/= * /common/zoon +~% %h-zoon ..stark-engine-jet-hook:z ~ +|% +:: ++| %types ++$ hashed $^(noun-digests:z noun-digest:tip5:z) +:: NOTE: empty-checking with ?~ does not work due to tagged empty leaves. Use ?@. +++ hmap-tree + |$ [item] + $~ %hmap + $| $@(%hmap [n=item l=(hmap-tree item) r=(hmap-tree item)]) + |= a=$@(%hmap [n=* l=(hmap-tree) r=(hmap-tree)]) + |- ^- ? + ?@ a =(%hmap a) + ?& $(a l.a) + $(a r.a) + == +:: +++ hset-tree + |$ [item] + $~ %hset + $| $@(%hset [n=item l=(hset-tree item) r=(hset-tree item)]) + |= a=$@(%hset [n=* l=(hset-tree) r=(hset-tree)]) + |- ^- ? + ?@ a =(%hset a) + ?& $(a l.a) + $(a r.a) + == +:: ++| %map +++ h-map + |$ [key value] :: table + $| (hmap-tree (pair key value)) + |=(a=(hmap-tree (pair hashed *)) ?@(a =(%hmap a) ~(apt h-by a))) +:: +++ h-by :: h-map engine + ~/ %h-by + =| a=(hmap-tree (pair hashed *)) :: (h-map) + |@ + ++ all :: logical AND + ~/ %all + |* b=$-(* ?) + |- ^- ? + ?@ a + & + ?&((b q.n.a) $(a l.a) $(a r.a)) + :: + ++ any :: logical OR + ~/ %any + |* b=$-(* ?) + |- ^- ? + ?@ a + | + ?|((b q.n.a) $(a l.a) $(a r.a)) + :: + ++ bif :: splits a h-by b + ~/ %bif + |* b=* + |- ^+ [l=a r=a] + ?@ a + [%hmap %hmap] + ?: =(b p.n.a) + +.a + ?: (gor-hip b p.n.a) + =+ d=$(a l.a) + ?> ?=(^ d) + [l.d a(l r.d)] + =+ d=$(a r.a) + ?> ?=(^ d) + [a(r l.d) r.d] + :: + ++ del :: delete at key b + ~/ %del + |* b=* + |- ^+ a + ?@ a + %hmap + ?. =(b p.n.a) + ?: (gor-hip b p.n.a) + a(l $(a l.a)) + a(r $(a r.a)) + |- ^- [$?(%hmap _a)] + ?@ l.a r.a + ?@ r.a l.a + ?: (mor-hip p.n.l.a p.n.r.a) + l.a(r $(l.a r.l.a)) + r.a(l $(r.a l.r.a)) + :: + ++ dif :: difference + ~/ %dif + |* b=_a + |- ^+ a + ?@ b + a + =+ c=(bif p.n.b) + ?> ?=(^ c) + =+ d=$(a l.c, b l.b) + =+ e=$(a r.c, b r.b) + |- ^- [$?(%hmap _a)] + ?@ d e + ?@ e d + ?: (mor-hip p.n.d p.n.e) + d(r $(d r.d)) + e(l $(e l.e)) + :: + ++ dig :: axis of b key + ~/ %dig + |= b=* + =+ c=1 + |- ^- (unit @) + ?@ a ~ + ?: =(b p.n.a) [~ u=(peg c 2)] + ?: (gor-hip b p.n.a) + $(a l.a, c (peg c 6)) + $(a r.a, c (peg c 7)) + :: + ++ apt :: check correctness + =< $ + =| [l=(unit hashed) r=(unit hashed)] + |. ^- ? + ?@ a =(%hmap a) + =+ (checked-hashed p.n.a) + ?& ?~(l & &((gor-hip p.n.a u.l) !=(p.n.a u.l))) + ?~(r & &((gor-hip u.r p.n.a) !=(u.r p.n.a))) + ?@ l.a & + &((mor-hip p.n.a p.n.l.a) !=(p.n.a p.n.l.a) $(a l.a, l `p.n.a)) + ?@ r.a & + &((mor-hip p.n.a p.n.r.a) !=(p.n.a p.n.r.a) $(a r.a, r `p.n.a)) + == + :: + ++ gas :: concatenate + ~/ %gas + |* b=(list [p=* q=*]) + => .(b `(list _?>(?=(^ a) n.a))`b) + |- ^+ a + ?@ b + a + $(b t.b, a (put p.i.b q.i.b)) + :: + ++ get :: grab value h-by key + ~/ %get + |* b=* + => .(b `_?>(?=(^ a) p.n.a)`b) + |- ^- (unit _?>(?=(^ a) q.n.a)) + ?@ a + ~ + ?: =(b p.n.a) + (some q.n.a) + ?: (gor-hip b p.n.a) + $(a l.a) + $(a r.a) + :: + ++ got :: need value h-by key + ~/ %got + |* b=* + (need (get b)) + :: + ++ gut :: fall value h-by key + ~/ %gut + |* [b=* c=*] + (fall (get b) c) + :: + ++ has :: key existence check + ~/ %has + |* b=* + !=(~ (get b)) + :: + ++ int :: intersection + ~/ %int + |* b=_a + |- ^+ a + ?@ b + %hmap + ?@ a + %hmap + ?: (mor-hip p.n.a p.n.b) + ?: =(p.n.b p.n.a) + b(l $(a l.a, b l.b), r $(a r.a, b r.b)) + ?: (gor-hip p.n.b p.n.a) + %- uni(a $(a l.a, r.b %hmap)) $(b r.b) + %- uni(a $(a r.a, l.b %hmap)) $(b l.b) + ?: =(p.n.a p.n.b) + b(l $(b l.b, a l.a), r $(b r.b, a r.a)) + ?: (gor-hip p.n.a p.n.b) + %- uni(a $(b l.b, r.a %hmap)) $(a r.a) + %- uni(a $(b r.b, l.a %hmap)) $(a l.a) + :: + ++ jab + ~/ %jab + |* [key=_?>(?=(^ a) p.n.a) fun=$-(_?>(?=(^ a) q.n.a) _?>(?=(^ a) q.n.a))] + ^+ a + :: + ?@ a !! + :: + ?: =(key p.n.a) + a(q.n (fun q.n.a)) + :: + ?: (gor-hip key p.n.a) + a(l $(a l.a)) + :: + a(r $(a r.a)) + :: + ++ mar :: add with validation + ~/ %mar + |* [b=* c=(unit *)] + ?~ c + (del b) + (put b u.c) + :: + ++ put :: adds key-value pair + ~/ %put + |* [b=* c=*] + =+ (checked-hashed `hashed`b) + |- ^+ a + ?@ a + [[b c] %hmap %hmap] + ?: =(b p.n.a) + ?: =(c q.n.a) + a + a(n [b c]) + ?: (gor-hip b p.n.a) + =+ d=$(a l.a) + ?> ?=(^ d) + ?: (mor-hip p.n.a p.n.d) + a(l d) + d(r a(l r.d)) + =+ d=$(a r.a) + ?> ?=(^ d) + ?: (mor-hip p.n.a p.n.d) + a(r d) + d(l a(r l.d)) + :: + ++ rep :: reduce to product + ~/ %rep + |* b=_=>(~ |=([* *] +<+)) + |- + ?@ a +<+.b + $(a r.a, +<+.b $(a l.a, +<+.b (b n.a +<+.b))) + :: + ++ rib :: transform + product + ~/ %rib + |* [b=* c=gate] + |- ^+ [b a] + ?@ a [b %hmap] + =+ d=(c n.a b) + =. n.a +.d + =+ e=$(a l.a, b -.d) + =+ f=$(a r.a, b -.e) + [-.f a(l +.e, r +.f)] + :: + ++ run :: apply gate to values + ~/ %run + |* b=gate + |- + ?@ a a + [n=[p=p.n.a q=(b q.n.a)] l=$(a l.a) r=$(a r.a)] + :: + ++ tap :: listify pairs + =< $ + =+ b=`(list _?>(?=(^ a) n.a))`~ + |. ^+ b + ?@ a + b + $(a r.a, b [n.a $(a l.a)]) + :: + ++ uni :: union, merge + ~/ %uni + |* b=_a + |- ^+ a + ?@ b + a + ?@ a + b + ?: =(p.n.b p.n.a) + b(l $(a l.a, b l.b), r $(a r.a, b r.b)) + ?: (mor-hip p.n.a p.n.b) + ?: (gor-hip p.n.b p.n.a) + $(l.a $(a l.a, r.b %hmap), b r.b) + $(r.a $(a r.a, l.b %hmap), b l.b) + ?: (gor-hip p.n.a p.n.b) + $(l.b $(b l.b, r.a %hmap), a r.a) + $(r.b $(b r.b, l.a %hmap), a l.a) + :: + ++ uno :: general union + ~/ %uno + |* b=_a + |* meg=$-([* * *] *) + |- ^+ a + ?@ b + a + ?@ a + b + ?: =(p.n.b p.n.a) + :+ [p.n.a `_?>(?=(^ a) q.n.a)`(meg p.n.a q.n.a q.n.b)] + $(b l.b, a l.a) + $(b r.b, a r.a) + ?: (mor-hip p.n.a p.n.b) + ?: (gor-hip p.n.b p.n.a) + $(l.a $(a l.a, r.b %hmap), b r.b) + $(r.a $(a r.a, l.b %hmap), b l.b) + ?: (gor-hip p.n.a p.n.b) + $(l.b $(b l.b, r.a %hmap), a r.a) + $(r.b $(b r.b, l.a %hmap), a l.a) + :: + ++ urn :: apply gate to nodes + ~/ %urn + |* b=$-([* *] *) + |- + ?@ a %hmap + a(n n.a(q (b p.n.a q.n.a)), l $(a l.a), r $(a r.a)) + :: + ++ wyt :: depth of h-map + =< $ + |. ^- @ + ?@(a 0 +((add $(a l.a) $(a r.a)))) + :: + ++ key :: h-set of keys + =+ b=`(h-set _?>(?=(^ a) p.n.a))`%hset + |- ^+ b + ?@ a b + =. b (~(put h-in b) p.n.a) + =. b $(a l.a, b b) + $(a r.a, b b) + :: + ++ val :: list of vals + =+ b=`(list _?>(?=(^ a) q.n.a))`~ + |- ^+ b + ?@ a b + $(a r.a, b [q.n.a $(a l.a)]) + -- ++| %set +++ h-set + |$ [item] :: h-set + $| (hset-tree item) + |=(a=(hset-tree item) ?@(a =(%hset a) ~(apt h-in a))) +:: +++ h-in :: h-set engine + ~/ %h-in + =| a=(hset-tree hashed) :: (h-set) + |@ + ++ all :: logical AND + ~/ %all + |* b=$-(* ?) + |- ^- ? + ?@ a + & + ?&((b n.a) $(a l.a) $(a r.a)) + :: + ++ any :: logical OR + ~/ %any + |* b=$-(* ?) + |- ^- ? + ?@ a + | + ?|((b n.a) $(a l.a) $(a r.a)) + :: + ++ apt :: check correctness + =< $ + =| [l=(unit hashed) r=(unit hashed)] + |. ^- ? + ?@ a =(%hset a) + =+ (checked-hashed n.a) + ?& ?~(l & &((gor-hip n.a u.l) !=(n.a u.l))) + ?~(r & &((gor-hip u.r n.a) !=(u.r n.a))) + ?@(l.a & ?&((mor-hip n.a n.l.a) !=(n.a n.l.a) $(a l.a, l `n.a))) + ?@(r.a & ?&((mor-hip n.a n.r.a) !=(n.a n.r.a) $(a r.a, r `n.a))) + == + :: + ++ bif :: splits a by b + ~/ %bif + |* b=* + ^+ [l=a r=a] + =< + + |- ^+ a + ?@ a + [b %hset %hset] + ?: =(b n.a) + a + ?: (gor-hip b n.a) + =+ c=$(a l.a) + ?> ?=(^ c) + c(r a(l r.c)) + =+ c=$(a r.a) + ?> ?=(^ c) + c(l a(r l.c)) + :: + ++ del :: b without any a + ~/ %del + |* b=* + |- ^+ a + ?@ a + %hset + ?. =(b n.a) + ?: (gor-hip b n.a) + a(l $(a l.a)) + a(r $(a r.a)) + |- ^- [$?(%hset _a)] + ?@ l.a r.a + ?@ r.a l.a + ?: (mor-hip n.l.a n.r.a) + l.a(r $(l.a r.l.a)) + r.a(l $(r.a l.r.a)) + :: + ++ dif :: difference + ~/ %dif + |* b=_a + |- ^+ a + ?@ b + a + =+ c=(bif n.b) + ?> ?=(^ c) + =+ d=$(a l.c, b l.b) + =+ e=$(a r.c, b r.b) + |- ^- [$?(%hset _a)] + ?@ d e + ?@ e d + ?: (mor-hip n.d n.e) + d(r $(d r.d)) + e(l $(e l.e)) + :: + ++ dig :: axis of a h-in b + ~/ %dig + |= b=* + =+ c=1 + |- ^- (unit @) + ?@ a ~ + ?: =(b n.a) [~ u=(peg c 2)] + ?: (gor-hip b n.a) + $(a l.a, c (peg c 6)) + $(a r.a, c (peg c 7)) + :: + ++ gas :: concatenate + ~/ %gas + |= b=(list _?>(?=(^ a) n.a)) + |- ^+ a + ?@ b + a + $(b t.b, a (put i.b)) + :: +has: does :b exist h-in :a? + :: + ++ has + ~/ %has + |* b=* + ^- ? + :: wrap extracted item type h-in a unit because bunting fails + :: + :: if we used the real item type of _?^(a n.a !!) as the sample type, + :: then hoon would bunt it to create the default sample for the gate. + :: + :: however, bunting that expression fails if :a is ~. if we wrap it + :: h-in a unit, the bunted unit doesn't include the bunted item type. + :: + :: this way we can ensure type safety of :b without needing to perform + :: this failing bunt. it's a hack. + :: + %. [~ b] + |= b=(unit _?>(?=(^ a) n.a)) + => .(b ?>(?=(^ b) u.b)) + |- ^- ? + ?@ a + | + ?: =(b n.a) + & + ?: (gor-hip b n.a) + $(a l.a) + $(a r.a) + :: + ++ int :: intersection + ~/ %int + |* b=_a + |- ^+ a + ?@ b + %hset + ?@ a + %hset + ?: =(n.b n.a) + a(l $(a l.a, b l.b), r $(a r.a, b r.b)) + ?. (mor-hip n.a n.b) + $(a b, b a) + ?: (gor-hip n.b n.a) + %- uni(a $(a l.a, r.b %hset)) $(b r.b) + %- uni(a $(a r.a, l.b %hset)) $(b l.b) + :: + ++ put :: puts b h-in a, sorted + ~/ %put + |* b=hashed + =+ (checked-hashed `hashed`b) + |- ^+ a + ?@ a + [b %hset %hset] + ?: =(b n.a) + a + ?: (gor-hip b n.a) + =+ c=$(a l.a) + ?> ?=(^ c) + ?: (mor-hip n.a n.c) + a(l c) + c(r a(l r.c)) + =+ c=$(a r.a) + ?> ?=(^ c) + ?: (mor-hip n.a n.c) + a(r c) + c(l a(r l.c)) + :: + ++ rep :: reduce to product + ~/ %rep + |* b=_=>(~ |=([* *] +<+)) + |- + ?@ a +<+.b + $(a r.a, +<+.b $(a l.a, +<+.b (b n.a +<+.b))) + :: + ++ run :: apply gate to values + ~/ %run + |* b=gate + =+ c=`(h-set _?>(?=(^ a) (b n.a)))`%hset + |- ?@ a c + =. c (~(put h-in c) (b n.a)) + =. c $(a l.a, c c) + $(a r.a, c c) + :: + ++ tap :: convert to list + =< $ + =+ b=`(list _?>(?=(^ a) n.a))`~ + |. ^+ b + ?@ a + b + $(a r.a, b [n.a $(a l.a)]) + :: + ++ uni :: union + ~/ %uni + |* b=_a + ?: =(a b) a + |- ^+ a + ?@ b + a + ?@ a + b + ?: =(n.b n.a) + b(l $(a l.a, b l.b), r $(a r.a, b r.b)) + ?: (mor-hip n.a n.b) + ?: (gor-hip n.b n.a) + $(l.a $(a l.a, r.b %hset), b r.b) + $(r.a $(a r.a, l.b %hset), b l.b) + ?: (gor-hip n.a n.b) + $(l.b $(b l.b, r.a %hset), a r.a) + $(r.b $(b r.b, l.a %hset), a l.a) + :: + ++ wyt :: size of h-set + =< $ + |. ^- @ + ?@(a 0 +((add $(a l.a) $(a r.a)))) + -- ++| %mip +:: +++ h-mip :: map of maps + |$ [kex key value] + (h-map kex (h-map key value)) +:: +++ h-bi :: mip engine + =| a=(h-map hashed (h-map hashed *)) + |@ + ++ inner + |* b=* + => .(b `_?>(?=(^ a) p.n.a)`b) + ^- _?>(?=(^ a) q.n.a) + (~(gut h-by a) b `_?>(?=(^ a) q.n.a)`%hmap) + :: + ++ del + |* [b=* c=*] + => .(b `_?>(?=(^ a) p.n.a)`b, c `_?>(?=(^ a) ?>(?=(^ q.n.a) p.n.q.n.a))`c) + =+ d=(inner b) + =+ e=(~(del h-by d) c) + ?@ e + (~(del h-by a) b) + (~(put h-by a) b e) + :: + ++ get + |* [b=* c=*] + => .(b `_?>(?=(^ a) p.n.a)`b, c `_?>(?=(^ a) ?>(?=(^ q.n.a) p.n.q.n.a))`c) + ^- (unit _?>(?=(^ a) ?>(?=(^ q.n.a) q.n.q.n.a))) + (~(get h-by (inner b)) c) + :: + ++ got + |* [b=* c=*] + (need (get b c)) + :: + ++ gut + |* [b=* c=* d=*] + (~(gut h-by (inner b)) c d) + :: + ++ has + |* [b=* c=*] + !=(~ (get b c)) + :: + ++ key + |* b=* + ~(key h-by (inner b)) + :: + ++ put + |* [b=* c=* d=*] + => .(b `_?>(?=(^ a) p.n.a)`b, c `_?>(?=(^ a) ?>(?=(^ q.n.a) p.n.q.n.a))`c, d `_?>(?=(^ a) ?>(?=(^ q.n.a) q.n.q.n.a))`d) + %+ ~(put h-by a) b + %. [c d] + %~ put h-by + (inner b) + :: + ++ tap + ::NOTE naive turn-based implementation find-errors ): + =< $ + =+ b=`_?>(?=(^ a) *(list [x=_p.n.a _?>(?=(^ q.n.a) [y=p v=q]:n.q.n.a)]))`~ + |. ^+ b + ?@ a + b + $(a r.a, b (welp (turn ~(tap h-by q.n.a) (lead p.n.a)) $(a l.a))) + -- +:: ++| %jug +:: +++ h-jug + |$ [key value] + (h-map key (h-set value)) +:: +++ h-ju :: h-jug engine + =| a=(hmap-tree (pair hashed (hset-tree hashed))) :: (h-jug) + |@ + ++ del :: del key-set pair + |* [b=* c=*] + => .(b `_?>(?=(^ a) p.n.a)`b, c `_?>(?=(^ a) ?>(?=(^ q.n.a) n.q.n.a))`c) + ^+ a + =+ d=(get b) + =/ e=_?>(?=(^ a) q.n.a) (~(del h-in d) c) + ?@ e + (~(del h-by a) b) + (~(put h-by a) b e) + :: + ++ gas :: concatenate + |* b=(list [p=* q=*]) + => .(b `(list _?>(?=([[* ^] ^] a) [p=p q=n.q]:n.a))`b) + |- ^+ a + ?@ b + a + $(b t.b, a (put p.i.b q.i.b)) + :: + ++ get :: gets h-set by key + |* b=* + => .(b `_?>(?=(^ a) p.n.a)`b) + =+ c=(~(get h-by a) b) + ?~(c `_?>(?=(^ a) q.n.a)`%hset u.c) + :: + ++ has :: existence check + |* [b=* c=*] + ^- ? + (~(has h-in (get b)) c) + :: + ++ put :: add key-h-set pair + |* [b=* c=*] + => .(b `_?>(?=(^ a) p.n.a)`b, c `_?>(?=(^ a) ?>(?=(^ q.n.a) n.q.n.a))`c) + ^+ a + =+ d=(get b) + =/ e=_?>(?=(^ a) q.n.a) (~(put h-in d) c) + |- ^+ a + ?@ a + [[b e] %hmap %hmap] + ?: =(b p.n.a) + ?: =(e q.n.a) + a + a(n [b e]) + ?: (gor-hip b p.n.a) + =+ f=$(a l.a) + ?> ?=(^ f) + ?: (mor-hip p.n.a p.n.f) + a(l f) + f(r a(l r.f)) + =+ f=$(a r.a) + ?> ?=(^ f) + ?: (mor-hip p.n.a p.n.f) + a(r f) + f(l a(r l.f)) + -- +:: ++| %ordering +:: +gor-hip: pre-hashed tip order. +:: +:: this is h-zoon's key order. a key is normalized to a digest list; a +:: single digest acts like a one-item list. each digest compares limbs +:: [4 3 2 1 0], then equal prefixes continue to the next digest. equal +:: lists return %.n and an equal-prefix shorter list loses. +:: +:: there is no dor fallback here. equality means the digest keys collide, +:: and h-zoon treats digest collision resistance as a consensus assumption. +:: +++ gor-hip + ~/ %gor-hip + |= [a=hashed b=hashed] + ^- ? + (gor-digests (hashed-to-digests a) (hashed-to-digests b)) +:: +mor-hip: pre-hashed priority order. +:: +:: this is h-zoon's treap priority order. it uses the same digest-list +:: rules as gor-hip, but compares limbs [0 1 2 3 4], matching double-tip +:: priority without hashing the key noun again. +:: +++ mor-hip + ~/ %mor-hip + |= [a=hashed b=hashed] + ^- ? + (mor-digests (hashed-to-digests a) (hashed-to-digests b)) +:: +++ hashed-to-digests + |= a=hashed + ^- noun-digests:z + ?: ?=([@ @ @ @ @] a) + [a ~] + ?: ?=([[@ @ @ @ @] ~] a) + ~| %hashed-singleton-list-forbidden !! + a +:: +:: validate a `hashed` key without canonicalizing its noun shape. +:: +:: `hashed-to-digests` defines the ordered domain used by `gor-hip` +:: and `mor-hip`: direct digest keys and non-singleton digest-list +:: keys are valid, singleton digest-list keys are ambiguous and crash. +:: +:: comparator validation alone does not cover empty insertion or a +:: single-node `apt`, because neither path needs to compare two keys. +:: `h-map.put`, `h-set.put`, `h-map.apt`, and `h-set.apt` call this +:: arm at their root boundary before the key can enter or validate as +:: an h-container member. +:: +:: return the original noun to preserve each caller's static key type +:: and exact key shape after validation. +++ checked-hashed + |= a=hashed + ^- hashed + =+ (hashed-to-digests a) + a +:: +++ gor-digests + |= [a=noun-digests:z b=noun-digests:z] + ^- ? + ?~ a %.n + ?~ b %.y + =+ c=(digest-to-atom:tip5:z i.a) + =+ d=(digest-to-atom:tip5:z i.b) + ?: (gth c d) %.y + ?: (lth c d) %.n + $(a t.a, b t.b) +:: +++ rev-tip + |= a=noun-digest:tip5:z + ^- noun-digest:tip5:z + =+ [b c d e f]=a + [f e d c b] +:: +++ mor-digests + |= [a=noun-digests:z b=noun-digests:z] + ^- ? + ?~ a %.n + ?~ b %.y + =+ c=(digest-to-atom:tip5:z (rev-tip i.a)) + =+ d=(digest-to-atom:tip5:z (rev-tip i.b)) + ?: (gth c d) %.y + ?: (lth c d) %.n + $(a t.a, b t.b) +:: ++| %h-container-from-container + ++ h-silt :: h-set from list + |* a=(list) + =+ b=`(hset-tree _?>(?=(^ a) i.a))`%hset + (~(gas h-in b) a) + :: + ++ h-molt :: h-map from pair + |* a=(list (pair)) + (~(gas h-by `(hmap-tree [p=_p.i.-.a q=_q.i.-.a])`%hmap) a) + :: + ++ h-malt :: h-map from list + |* a=(list) + (h-molt `(list [p=_-<.a q=_->.a])`a) + :: + ++ zh-molt + ~/ %zh-molt + |* a=(z-map hashed *) + (h-molt ~(tap z-by a)) + :: + ++ zh-jult + ~/ %zh-jult + |* a=(z-jug hashed hashed) + (zh-molt (~(run z-by a) zh-silt)) + :: + ++ zh-milt + ~/ %zh-milt + |* a=(z-mip hashed hashed hashed) + (zh-molt (~(run z-by a) zh-molt)) + :: + :: balance migration is the only z-mip conversion that needs extra context. + :: + :: blocks: block-id -> page(parent-id, ...) + :: balance: block-id -> full note-balance snapshot + :: + :: the balance value at each block is a full z-map snapshot. snapshots near + :: each other usually share most of their source tree with the parent block, + :: because a block changes only a small number of notes. plain zh-milt cannot + :: see that history. it receives only a nested map and must treat every inner + :: balance root as a fresh full map unless the exact root noun repeats. + :: + :: blocks is not part of this arm's hoon result. it exists only to give a jet + :: parent links for conversion order. the jet may read those links, convert + :: parents first, and derive child h-balances from z-map diffs. + :: + :: hashed keys can also be digest lists, but block ids in this fast path must + :: be direct digests. any other valid key shape falls back because the output + :: must preserve exact key nouns, not only compact digest identity. + :: + :: the fallback is deliberately still zh-milt, making this arm's hoon meaning + :: the consensus oracle. the jet must produce exactly the same noun as this + :: line. + ++ zh-balmilt + ~/ %zh-balmilt + |* [blocks=(z-map hashed *) balance=(z-mip hashed hashed hashed)] + (zh-milt balance) + :: + ++ hz-molt + |* a=(h-map hashed *) + (z-molt ~(tap h-by a)) + :: + ++ zh-silt + ~/ %zh-silt + |* a=(z-set hashed) + (h-silt ~(tap z-in a)) + :: + ++ hz-silt + |* a=(h-set hashed) + (z-silt ~(tap h-in a)) + :: + ++ hz-jult + |* a=(h-jug hashed hashed) + (hz-molt (~(run h-by a) hz-silt)) + :: + ++ hz-milt + |* a=(h-mip hashed hashed hashed) + (hz-molt (~(run h-by a) hz-molt)) +-- diff --git a/hoon/common/schedule.hoon b/hoon/common/schedule.hoon index d58b5c546..dfc1b4942 100644 --- a/hoon/common/schedule.hoon +++ b/hoon/common/schedule.hoon @@ -1,50 +1,75 @@ +:: Block emission schedule. +:: +:: Returns NOCK-atom emission for a given block number, spanning genesis +:: through the 2^32-NOCK hard cap. Pre-activation blocks (1..=65,500) +:: preserve the original chain history with three power-of-two halvings +:: at heights 13,150 / 39,448 / 65,500. Eon 2 is truncated from its +:: original end at block 78,895 to the new activation block 65,500; +:: the 13,395 blocks of legacy eon-2 emission that would have followed +:: are absorbed into the new post-activation budget. +:: +:: Post-activation (block > 65,500), the schedule is the unified +:: Aletheia table: a 6-month activation era at 2,048 NOCK/block, then +:: nine 1-year eras with alternating 25% / 33% drops down to a 64-NOCK +:: floor, then a long flat tail until the 2^32 hard cap is met exactly. +:: +:: Cap accounting. Eons 0/1/2 emit 65,536 / 32,768 / 16,384 NOCK per +:: block (powers of two, the actual on-chain history). The Aletheia +:: schedule doc rounds these to multiples of 5 (65,540 / 32,770 / +:: 16,380) for table readability, but the chain has already emitted +:: the power-of-two values, so the implementation preserves them. +:: Cumulative supply through end-of-decay (block 2,060,500) = +:: 3,393,567,232 NOCK. Remaining budget to cap = 901,400,064 NOCK, +:: which is exactly 14,084,376 tail blocks at 64 NOCK each. Therefore +:: emission-tail-end = 2,060,500 + 14,084,376 = 16,144,876, every tail +:: block emits 64 NOCK, and no per-block carve-out for "dust" is +:: needed. |% -++ bmonth 4.383 -++ byear ^~((mul 12 4.383)) ++ atoms-per-nock ^~((bex 16)) :: +:: Era boundaries (block heights, inclusive end of each era). +:: eon-0-end :: end of original eon 0 (3-month halving) +:: eon-1-end :: end of original eon 1 (9-month halving) +:: activation :: end of (truncated) eon 2 + Aletheia activation +:: eon-3-end :: end of 6-month activation era at 2,048 NOCK +:: decay-end :: end of decay phase (eon 12, last 96-NOCK block) +:: tail-end :: last block at which emission is non-zero +++ eon-0-end 13.150 +++ eon-1-end 39.448 +++ activation 65.500 +++ eon-3-end 170.500 +++ decay-end 2.060.500 +++ tail-end 16.144.876 +++ era-blocks 210.000 +:: +:: Per-block reward in NOCK for each post-activation 1-year era beyond +:: eon 3. Index = era - 4 (era 4 = 1,536 NOCK, era 12 = 96 NOCK). +++ decay-rewards + ^- (list @) + ~[1.536 1.024 768 512 384 256 192 128 96] +:: ++ schedule |= block-num=@ ^- @ :: emission is number of atoms - ?: =(0 block-num) 0 :: no coins in genesis block - :: least inconvenient offset to deal with coinless - :: genesis block - =. block-num (dec block-num) - =; emit=@ - ?: (gte block-num (add 2 (mul byear 191))) - :: rate goes to 0 at 191 years and 2 blocks. not strictly - :: necessary since the algorithm would do so anyways, but - :: it makes it clear exactly when emissions stop. - ?> =(0 emit) 0 - ?> !=(0 emit) emit - =/ rate ^~((mul (bex 16) atoms-per-nock)) - =? rate (gth block-num (mul bmonth 3)) - (div rate 2) - =? rate (gth block-num (mul bmonth 9)) - (div rate 2) - =? rate (gth block-num (mul bmonth 18)) - (div rate 2) - =? rate (gth block-num (mul byear 3)) - (div rate 2) - =? rate (gth block-num (mul byear 5)) - (div rate 2) - =? rate (gth block-num (mul byear 8)) - (div rate 2) - =? rate (gth block-num (mul byear 12)) - (div rate 2) - =? rate (gth block-num (mul byear 17)) - (div rate 2) - =? rate (gth block-num (mul byear 23)) - (div rate 2) - ?. (gth block-num (mul byear 30)) - rate - =: rate (div rate 2) - block-num (sub block-num (mul byear 30)) - == - |- - ?: (gth block-num (mul byear 7)) - $(rate (div rate 2), block-num (sub block-num (mul byear 7))) - rate + ?: =(0 block-num) 0 + ?: (gth block-num tail-end) 0 + :: Pre-activation eons preserve the actual on-chain emission + :: (powers of two, not the rounded doc values). + ?: (lte block-num eon-0-end) + ^~((mul (bex 16) atoms-per-nock)) :: 65,536 NOCK + ?: (lte block-num eon-1-end) + ^~((mul (bex 15) atoms-per-nock)) :: 32,768 NOCK + ?: (lte block-num activation) + ^~((mul (bex 14) atoms-per-nock)) :: 16,384 NOCK + :: Eon 3: 6-month activation era at 2,048 NOCK. + ?: (lte block-num eon-3-end) + ^~((mul 2.048 atoms-per-nock)) + :: Decay phase (eons 4..=12): 9 one-year eras of 210k blocks each. + ?: (lte block-num decay-end) + =/ era-idx=@ (div (sub block-num +(eon-3-end)) era-blocks) + (mul (snag era-idx decay-rewards) atoms-per-nock) + :: Tail: 64 NOCK/block until the cap is hit at tail-end. + ^~((mul 64 atoms-per-nock)) :: ++ total-supply |= max-block=@ diff --git a/hoon/common/tx-engine-0.hoon b/hoon/common/tx-engine-0.hoon index 817f32e99..91bd83310 100644 --- a/hoon/common/tx-engine-0.hoon +++ b/hoon/common/tx-engine-0.hoon @@ -266,7 +266,7 @@ ++ hashable |= =form ^- hashable:tip5 - ?~ form leaf+form + ?@ form leaf+form :+ [hash+(hash:schnorr-pubkey p.n.form) (hashable:schnorr-signature q.n.form)] $(form l.form) $(form r.form) @@ -541,7 +541,7 @@ ++ hashable-tx-ids |= tx-ids=(z-set tx-id) ^- hashable:tip5 - ?~ tx-ids leaf+tx-ids + ?@ tx-ids leaf+tx-ids :+ hash+n.tx-ids $(tx-ids l.tx-ids) $(tx-ids r.tx-ids) @@ -719,7 +719,8 @@ |= target-bn=bignum:bn ^- bignum:bn =/ target-atom=@ (merge:bn target-bn) - (chunk:bn (div max-target-atom +(target-atom))) + =/ raw=@ (div max-target-atom +(target-atom)) + (chunk:bn ?:(=(0 raw) 1 raw)) :: ++ to-page-summary |= pag=form @@ -767,6 +768,11 @@ |= lp=form ^- page lp(pow (biff pow.lp |=(j=@ ((soft proof) (cue j))))) + :: + ++ to-page-no-pow + |= lp=form + ^- page + lp(pow ~) -- :: :: +page-msg: (list belt) that enforces that each elt is a belt @@ -938,7 +944,7 @@ ++ hashable |= =form ^- hashable:tip5 - ?~ form leaf+form + ?@ form leaf+form :+ [(hashable:nname p.n.form) (hashable:input q.n.form)] $(form l.form) $(form r.form) @@ -1604,7 +1610,7 @@ ++ hashable-pubkeys |= pubkeys=(z-set schnorr-pubkey) ^- hashable:tip5 - ?~ pubkeys leaf+pubkeys + ?@ pubkeys leaf+pubkeys :+ hash+(hash:schnorr-pubkey n.pubkeys) $(pubkeys l.pubkeys) $(pubkeys r.pubkeys) @@ -1839,7 +1845,7 @@ ++ hashable |= =form ^- hashable:tip5 - ?~ form leaf+form + ?@ form leaf+form :+ [(hashable:sig p.n.form) leaf+q.n.form] $(form l.form) $(form r.form) @@ -2013,7 +2019,7 @@ ++ hashable |= =form ^- hashable:tip5 - ?~ form leaf+form + ?@ form leaf+form :+ (hashable:seed n.form) $(form l.form) $(form r.form) @@ -2021,7 +2027,7 @@ ++ sig-hashable |= =form ^- hashable:tip5 - ?~ form leaf+form + ?@ form leaf+form :+ (sig-hashable:seed n.form) $(form l.form) $(form r.form) diff --git a/hoon/common/tx-engine-1.hoon b/hoon/common/tx-engine-1.hoon index 9b0a021b0..4e0b45234 100644 --- a/hoon/common/tx-engine-1.hoon +++ b/hoon/common/tx-engine-1.hoon @@ -1,6 +1,6 @@ /= v0 /common/tx-engine-0 /= * /common/zeke -/= * /common/zoon +/= * /common/h-zoon |% :: import ++ hash hash:v0 @@ -21,6 +21,42 @@ |$ object (each object term) :: +:: $fund-address: lock-script hash that receives the 20% protocol-fund +:: share of every post-activation coinbase (014-aletheia, asert-phase +:: onward). The lock-root of a 3-of-4 multisig over the four pkhs in +:: /asert-protocol-lock-fund.txt at the repo root; spending the fund +:: therefore requires three of four signatures. +:: +:: Computed once by /scripts/generate-fund-address.hoon and pasted in +:: here as a base58 literal. Re-run that script after any change to +:: the participant set, the threshold, or the lock-script structure +:: to regenerate the value below. The pin is enforced by +:: test-fund-address-is-3-of-4-multisig in +:: /tests/dumb/mod/unit/coinbase-split. +++ fund-address + ^- hash + (from-b58:hash '9EhcJiGhAPcWLYrR9DL4ZPjU2Z9XT6FT2ZFkEEwmSQv7ES2TMC7p6Up') +:: +:: +post-asert-activation: 014-aletheia activation predicate, 2-arg form. +:: Returns %.y when `height` is at or past the asert-phase boundary. +:: The 1-arg wrappers in /common/tx-engine close over the kernel's +:: blockchain-constants; this 2-arg form is for callers (like +:: +new-candidate below) that already have asert-phase as a separate +:: parameter rather than via blockchain-constants. SINGLE source of +:: truth for the boundary semantics — see +:: 014-aletheia-emissions-audit.md finding #3. +++ post-asert-activation + |= [height=@ asert-phase=@] + ^- ? + (gte height asert-phase) +:: +:: +pre-asert-activation: inverse of +post-asert-activation, for readability at +:: call sites that branch on the legacy / activated split. +++ pre-asert-activation + |= [height=@ asert-phase=@] + ^- ? + (lth height asert-phase) +:: :: $page: page with v1 coinbase-split ++ page =< form @@ -41,7 +77,7 @@ == :: ++ new-candidate - |= [par=$^(page:v0 form) now=@da target-bn=bignum:bn =shares] + |= [par=$^(page:v0 form) now=@da target-bn=bignum:bn =shares asert-phase=@] ^- form :: extract common fields from either v0 or v1 parent =/ [par-accumulated-work=bignum:bn par-digest=hash par-epoch-counter=@ par-height=@] @@ -57,6 +93,15 @@ ?: =(+(par-epoch-counter) blocks-per-epoch:v0) 0 +(par-epoch-counter) =/ height=@ +(par-height) + =/ emission=coins (emission-calc:coinbase:v0 height) + :: Pre-activation: 100% of emission to the miner per ++new:coinbase-split. + :: Post-activation: 80% miner / 20% fund (014-aletheia). Fees are zero + :: at candidate construction; the miner module re-runs the builder + :: with the live fee total each time the mempool changes. + =/ cb=coinbase-split + ?: (pre-asert-activation height asert-phase) + (new:coinbase-split emission shares) + (new-with-fund-share:coinbase-split emission 0 shares) %* . *form height height parent par-digest @@ -64,9 +109,7 @@ epoch-counter epoch-counter target target-bn accumulated-work accumulated-work - coinbase %+ new:coinbase-split - (emission-calc:coinbase:v0 height) - shares + coinbase cb == :: ++ to-local-page @@ -91,7 +134,7 @@ ++ hashable-tx-ids |= tx-ids=(z-set tx-id) ^- hashable:tip5 - ?~ tx-ids leaf+tx-ids + ?@ tx-ids leaf+tx-ids :+ hash+n.tx-ids $(tx-ids l.tx-ids) $(tx-ids r.tx-ids) @@ -154,6 +197,11 @@ |= lp=form ^- page lp(pow (biff pow.lp |=(j=@ ((soft proof) (cue j))))) + :: + ++ to-page-no-pow + |= lp=form + ^- page + lp(pow ~) -- ++ timelock-range timelock-range:v0 ++ size size:v0 @@ -172,6 +220,35 @@ :: divisor for input fees (inputs cost 1/divisor of outputs) input-fee-divisor=4 *blockchain-constants:v0 + :: aserti3-2d difficulty adjustment. + :: .asert-phase: activation height. at or after, target is + :: computed per-block via aserti3-2d instead of epoch retarget. + :: .asert-anchor-height / -target-atom: the fixed anchor (height, + :: target-atom) used as the aserti3-2d reference. anchor-target + :: is 2^291: at constant hashrate H, expected blocks/sec is + :: H*target/max, so to cut block time 600s → 150s (4x more + :: blocks per sec) we increase target by ~4x, which reduces + :: per-block difficulty (max/target) by the same factor. + :: 2^291 is the closest power of 2 to 4 * pre-activation + :: mainnet target (~2^291.38); it yields ~3.2x faster blocks + :: (expected ~187s) under the same hashrate, slightly + :: conservative vs the ideal 150s. + :: .asert-anchor-min-timestamp: median-of-11 timestamp at the + :: anchor block (height 65,499). pinned at phase-2 cutover + :: to the value observed when the canonical mainnet block at + :: asert-anchor-height was finalized. replaces phase-1's + :: runtime parent-walk through .blocks / .min-timestamps. + :: .asert-ideal-block-time: post-asert-activation block time in seconds. + :: .asert-half-life: real-time seconds of drift to halve or double. + :: rbits is hardcoded to 16 in lib/asert.hoon (the polynomial + :: coefficients are tied to that precision and cannot be varied + :: per-constants without a hard fork of the polynomial itself). + asert-phase=65.500 + asert-anchor-height=65.499 + asert-anchor-target-atom=^~((bex 291)) + asert-ideal-block-time=150 + asert-half-life=^~((mul 12 ^~((mul 60 60)))) + asert-anchor-min-timestamp=9.223.372.093.639.027.842 == $: v1-phase=@ bythos-phase=@ @@ -179,6 +256,12 @@ base-fee=@ input-fee-divisor=@ blockchain-constants:v0 + asert-phase=@ + asert-anchor-height=@ + asert-anchor-target-atom=@ + asert-ideal-block-time=@ + asert-half-life=@ + asert-anchor-min-timestamp=@ == :: $nname ++ nname @@ -274,7 +357,7 @@ |= =form |^ ^- ? - %- ~(rep by form) + %- ~(rep z-by form) |= [[k=@tas v=*] a=?] ?&(a (^based k) (based-noun v)) ++ based-noun @@ -286,7 +369,7 @@ |= =form ^- hashable:tip5 |^ - ?~ form leaf+~ + ?@ form leaf+~ :+ [leaf+p.n.form (hashable-noun q.n.form)] $(form l.form) $(form r.form) @@ -389,7 +472,7 @@ ++ hashable |= =form ^- hashable:tip5 - ?~ form leaf+~ + ?@ form leaf+~ :+ (hashable:seed n.form) $(form l.form) $(form r.form) @@ -397,7 +480,7 @@ ++ sig-hashable |= =form ^- hashable:tip5 - ?~ form leaf+form + ?@ form leaf+form :+ (sig-hashable:seed n.form) $(form l.form) $(form r.form) @@ -534,9 +617,21 @@ recursion-depth +(recursion-depth) == :: + :: +based: parser-level shape check for v1 coinbase-split. + :: Allows up to `max-coinbase-split:v0 + 1` entries — the +1 is + :: the post-asert-activation fund slot (014-aletheia). Pre-activation v1 + :: builders only emit one entry per `shares` key (capped at + :: `max-coinbase-split` by ++validate:shares), so honest + :: pre-asert-activation blocks never reach the relaxed bound. + :: + :: NOTE: this parser is height-blind by design. The consensus-layer + :: rule that pre-activation v1 blocks cap at `max-coinbase-split` + :: (no fund slot before activation) is enforced in + :: +validate-page-with-txs in apps/dumbnet/lib/consensus.hoon + :: via the %coinbase-split-pre-activation-too-many reason. ++ based |= =form - ?. (lte ~(wyt z-by form) max-coinbase-split:v0) + ?. (lte ~(wyt z-by form) +(max-coinbase-split:v0)) %| %+ levy ~(tap z-by form) |= [h=^hash =coins] @@ -545,10 +640,32 @@ (based:^hash h) == :: + :: +new-with-fund-share: post-asert-activation 80/20 coinbase-split builder. + :: Splits a block's coinbase between the protocol fund and the + :: miner-side recipients: + :: fund = floor(emission / 5) :: 20% of subsidy + :: miner-pool = (emission - fund) + fees :: 80% + all fees + :: The miner-pool is distributed across `shares` via the same + :: proportional-allocation arm as ++new (per-block atom remainders + :: accrue to the first share key in z-map order, preserving the + :: legacy single-miner behaviour and supporting up-to-2 partner + :: splits). The fund is added as one additional output. + :: `shares` must NOT include `fund-address`. + :: Fees are computed from the subsidy alone — folding fees into the + :: fund slot would be rejected by +check-fund-split. + ++ new-with-fund-share + |= [emission=coins fees=coins =shares] + ^- form + ?< (~(has z-by shares) fund-address) + =/ fund-coins=coins (div emission 5) + =/ miner-pool=coins (add fees (sub emission fund-coins)) + =/ miner-split=form (new miner-pool shares) + (~(put z-by miner-split) fund-address fund-coins) + :: ++ hashable |= =form ^- hashable:tip5 - ?~ form leaf+form + ?@ form leaf+form :+ [hash+p.n.form leaf+q.n.form] $(form l.form) $(form r.form) @@ -784,7 +901,7 @@ |= =form ^- hashable:tip5 |- - ?~ form leaf+form + ?@ form leaf+form :+ [(hashable:nname p.n.form) (hashable:spend q.n.form)] $(form l.form) $(form r.form) @@ -832,7 +949,7 @@ (gth data-size max) :: ++ validate-with-context - |= $: balance=(z-map nname nnote) + |= $: balance=(h-map nname nnote) sps=form page-num=page-number max-size=@ @@ -844,7 +961,7 @@ %+ roll ~(tap z-by sps) |= [[nam=nname sp=spend] acc=(reason ~)] ?. ?=(%.y -.acc) acc - =/ mnote=(unit nnote) (~(get z-by balance) nam) + =/ mnote=(unit nnote) (~(get h-by balance) nam) ?~ mnote [%.n %v1-input-missing] =/ note=nnote u.mnote ?- -.sp @@ -988,7 +1105,7 @@ ++ hashable |= =form ^- hashable:tip5 - ?~ form leaf+form + ?@ form leaf+form :+ [(hashable:nname p.n.form) (hashable:input q.n.form)] $(form l.form) $(form r.form) @@ -1222,7 +1339,7 @@ ++ hashable-hax |= m=(z-map ^hash *) ^- hashable:tip5 - ?~ m leaf+m + ?@ m leaf+m :+ [hash+p.n.m (hashable-noun q.n.m)] $(m l.m) $(m r.m) @@ -1521,9 +1638,6 @@ ++ check |= [=form parent-firstname=^hash] ^- ? - ?. =(1 axis.form) - ~> %slog.[0 'axis must be 1 until a protocol upgrade that properly commits to lmp in witness'] - %.n =/ spend-firstname (hash-hashable:tip5 [leaf+& hash+root.merk-proof.form]) ?. =(spend-firstname parent-firstname) @@ -1647,7 +1761,7 @@ ++ hashable-hashes |= hs=(z-set ^hash) ^- hashable:tip5 - ?~ hs leaf+hs + ?@ hs leaf+hs :+ hash+n.hs $(hs l.hs) $(hs r.hs) @@ -1688,7 +1802,7 @@ based:^hash ++ hashable |= =form - ?~ form leaf+~ + ?@ form leaf+~ :* hash+n.form $(form l.form) $(form r.form) @@ -1778,7 +1892,7 @@ |= =form ^- hashable:tip5 |^ - ?~ form leaf+form + ?@ form leaf+form :+ [hash+p.n.form (hashable-val q.n.form)] $(form l.form) $(form r.form) diff --git a/hoon/common/tx-engine.hoon b/hoon/common/tx-engine.hoon index 4b7c0d3bd..f733ac54c 100644 --- a/hoon/common/tx-engine.hoon +++ b/hoon/common/tx-engine.hoon @@ -1,13 +1,13 @@ /= v0 /common/tx-engine-0 /= v1 /common/tx-engine-1 /= * /common/zeke -/= * /common/zoon +/= * /common/h-zoon /= * /common/zose => |% ++ blockchain-constants blockchain-constants:v1 -- |_ blockchain-constants -+* v0 ~(. ^v0 +63:+<) ++* v0 ~(. ^v0 +126:+<) :: constants ++ quarter-ted ^~((div target-epoch-duration 4)) ++ quadruple-ted ^~((mul target-epoch-duration 4)) @@ -15,6 +15,23 @@ ++ max-target ^~((chunk:bignum max-target-atom)) ++ nicks-per-nock ^~((bex 16)) :: +:: +post-asert-activation / +pre-asert-activation: 1-arg activation +:: predicates, bound to the kernel's blockchain-constants. The +:: boundary semantics also live in the 2-arg +:: +post-asert-activation:v1 (used by +new-candidate); the inline +:: `gte` here is the canonical definition for callers that read +:: asert-phase from blockchain-constants. See +:: 014-aletheia-emissions-audit.md finding #3. +++ post-asert-activation + |= height=@ + ^- ? + (gte height asert-phase) +:: +++ pre-asert-activation + |= height=@ + ^- ? + (lth height asert-phase) +:: ++ bignum bignum:v0 ++ block-commitment block-commitment:v0 ++ block-id block-id:v0 @@ -41,6 +58,7 @@ |% +$ form $|(^form |=(* %&)) ++ new ^new + ++ new-with-fund-share ^new-with-fund-share -- ++ based |= =form @@ -64,6 +82,9 @@ ++ genesis-seal genesis-seal:v0 ++ genesis-template genesis-template:v0 ++ hash hash:v0 +:: $fund-address: lock-script hash receiving the 20% protocol-fund +:: share of every post-asert-activation coinbase. See tx-engine-1.hoon. +++ fund-address fund-address:v1 ++ local-page =< form |% @@ -77,6 +98,12 @@ ?^ -.lp (to-page:local-page:v0 lp) (to-page:local-page:v1 lp) :: + ++ to-page-no-pow + |= lp=form + ^- page + ?^ -.lp (to-page-no-pow:local-page:v0 lp) + (to-page-no-pow:local-page:v1 lp) + :: ++ get |_ =form :: @@ -224,69 +251,6 @@ =+ witness:v1 |% +$ form $|(^form |=(* %&)) - :: - :: +make-pkh: build witness with pkh signatures for sig-hash - ++ make-pkh - |= $: root=^hash - sc=spend-condition - sig-hash=^hash - keys=(list [schnorr-seckey schnorr-pubkey]) - == - ^- form - =/ pmap=pkh-signature:v1 - %+ roll keys - |= $: kp=[sk=schnorr-seckey pk=schnorr-pubkey] - acc=pkh-signature:v1 - == - =/ sig=schnorr-signature - %+ sign:affine:belt-schnorr:cheetah - sk.kp - sig-hash - (~(put z-by acc) (hash:schnorr-pubkey pk.kp) [pk.kp sig]) - %* . *^form - lmp (build-lock-merkle-proof:lock sc 1) - pkh pmap - hax *(z-map ^hash *) - tim ~ - == - :: - :: +make-hax: build witness for %hax lock with preimage - ++ make-hax - |= [root=^hash sc=spend-condition h=^hash pre=*] - ^- form - %* . *^form - lmp (build-lock-merkle-proof:lock sc 1) - pkh *(z-map ^hash [pk=schnorr-pubkey sig=schnorr-signature]) - hax (~(put z-by *(z-map ^hash *)) h pre) - tim ~ - == - :: - :: +make-hax-pkh: build witness for combined %hax AND %pkh - ++ make-hax-pkh - |= $: root=^hash - sc=spend-condition - sig-hash=^hash - keys=(list [schnorr-seckey schnorr-pubkey]) - h=^hash - pre=* - == - ^- form - =/ pmap=pkh-signature:v1 - %+ roll keys - |= $: kp=[sk=schnorr-seckey pk=schnorr-pubkey] - acc=pkh-signature:v1 - == - =/ sig=schnorr-signature - %+ sign:affine:belt-schnorr:cheetah - sk.kp - sig-hash - (~(put z-by acc) (hash:schnorr-pubkey pk.kp) [pk.kp sig]) - %* . *^form - lmp (build-lock-merkle-proof:lock sc 1) - pkh pmap - hax (~(put z-by *(z-map ^hash *)) h pre) - tim ~ - == --::+witness :: :: TODO: remove @@ -386,11 +350,13 @@ :: :: +new-candidate: build candidate page for mining with v1 shares :: - :: creates a v1 page with hash-based coinbase-split. + :: creates a v1 page with hash-based coinbase-split. `asert-phase` + :: threads through so post-asert-activation candidates carry the 80/20 + :: miner/fund split (014-aletheia). ++ new-candidate - |= [par=form now=@da target-bn=bignum:bn =shares] + |= [par=form now=@da target-bn=bignum:bn =shares asert-phase=@] ^- form - (new-candidate:page:v1 par now target-bn shares) + (new-candidate:page:v1 par now target-bn shares asert-phase) :: ++ get |_ =form @@ -1116,7 +1082,7 @@ ++ txs =< form |% - +$ form (z-map tx-id tx) + +$ form (h-map tx-id tx) -- :: :: $tx-acc: accumulate transactions against a balance to create a new balance @@ -1124,26 +1090,26 @@ =< form |% +$ form - $: balance=(z-map nname nnote) :: current balance + $: balance=(h-map nname nnote) :: current balance height=page-number :: origin height fees=coins :: total fee =size :: total size =txs :: valid txs == ++ new - |= $: initial-balance=(unit (z-map nname nnote)) + |= $: initial-balance=(unit (h-map nname nnote)) initial-height=page-number == ^- form %* . *form - balance ?~ initial-balance *(z-map nname nnote) + balance ?~ initial-balance *(h-map nname nnote) u.initial-balance height initial-height == :: ++ txs-size-by-set |= form - %- ~(rep z-by txs) + %- ~(rep h-by txs) |= [[=tx-id =tx] sum-sizes=^size] %+ add sum-sizes ~(size get:raw-tx raw-tx.tx) @@ -1215,7 +1181,7 @@ :- %.y %_ form size (add size.form computed-size) - txs (~(put z-by txs.form) id.tx0 agg-tx) + txs (~(put h-by txs.form) id.tx0 agg-tx) == :: ++ add-outputs @@ -1225,9 +1191,9 @@ |: [op=*output:v0 acc=`(reason _form)`[%.y form]] ?. ?=(%.y -.acc) acc =/ f=_form p.acc - ?: (~(has z-by balance.f) name.note.op) + ?: (~(has h-by balance.f) name.note.op) [%.n %v0-output-already-exists] - [%.y f(balance (~(put z-by balance.f) name.note.op note.op))] + [%.y f(balance (~(put h-by balance.f) name.note.op note.op))] :: ++ consume-inputs |= [ips=(z-map nname input:v0) page-num=page-number] @@ -1238,7 +1204,7 @@ == ?. ?=(%.y -.acc) acc =/ [tir=timelock-range f=^form] p.acc - ?. =(`note.ip (~(get z-by balance.f) name.note.ip)) + ?. =(`note.ip (~(get h-by balance.f) name.note.ip)) [%.n %v0-input-missing] =/ new-tir=timelock-range %+ merge:timelock-range tir @@ -1248,7 +1214,7 @@ :- %.y :- new-tir %_ f - balance (~(del z-by balance.f) name.note.ip) + balance (~(del h-by balance.f) name.note.ip) fees (add fees.f fee.spend.ip) == -- @@ -1282,7 +1248,7 @@ :- %.y %_ form size (add size.form ~(size get:raw-tx raw1)) - txs (~(put z-by txs.form) (compute-id:raw-tx raw1) tx1) + txs (~(put h-by txs.form) (compute-id:raw-tx raw1) tx1) == :: ++ add-outputs @@ -1295,9 +1261,9 @@ =/ note=nnote note.op ?. ?=(@ -.note) [%.n %v1-output-wrong-note-version] =/ nam=nname name.note - ?: (~(has z-by balance.f) nam) + ?: (~(has h-by balance.f) nam) [%.n %v1-output-already-exists] - [%.y f(balance (~(put z-by balance.f) nam note))] + [%.y f(balance (~(put h-by balance.f) nam note))] :: ++ consume-inputs |= sps=spends @@ -1308,9 +1274,9 @@ |: [nam=*nname acc=`(reason ^form)`[%.y form]] ?. ?=(%.y -.acc) acc =/ f=^form p.acc - ?. (~(has z-by balance.f) nam) + ?. (~(has h-by balance.f) nam) [%.n %v1-input-missing] - [%.y f(balance (~(del z-by balance.f) nam))] + [%.y f(balance (~(del h-by balance.f) nam))] ?. ?=(%.y -.remove-result) remove-result [%.y p.remove-result(fees (add fees.p.remove-result fees-add))] -- diff --git a/hoon/common/zoon.hoon b/hoon/common/zoon.hoon index 0a12cd521..cca399aa3 100644 --- a/hoon/common/zoon.hoon +++ b/hoon/common/zoon.hoon @@ -3,28 +3,34 @@ ~% %zoon ..stark-engine-jet-hook:z ~ |% :: -+| %no-by-in -++ by %do-not-use -++ in %do-not-use -++ ju %do-not-use -++ ja %do-not-use -++ bi %do-not-use ++| %types +:: NOTE: empty-checking with ?~ does not work due to %ztree. Use ?@. +++ ztree + |$ [item] + $~ ~ + $| $@(?(%~ %ztree) [n=item l=(ztree item) r=(ztree item)]) + |= a=$@(?(%~ %ztree) [n=* l=(ztree) r=(ztree)]) + |- ^- ? + ?@ a =(~ a) + ?& $(a l.a) + $(a r.a) + == :: +| %map ++ z-map |$ [key value] :: table - $| (tree (pair key value)) - |=(a=(tree (pair)) ?:(=(~ a) & ~(apt z-by a))) + $| (ztree (pair key value)) + |=(a=(ztree (pair)) ?@(a =(~ a) ~(apt z-by a))) :: ++ z-by :: z-map engine ~/ %z-by - =| a=(tree (pair)) :: (z-map) + =| a=(ztree (pair)) :: (z-map) |@ ++ all :: logical AND ~/ %all |* b=$-(* ?) |- ^- ? - ?~ a + ?@ a & ?&((b q.n.a) $(a l.a) $(a r.a)) :: @@ -32,7 +38,7 @@ ~/ %any |* b=$-(* ?) |- ^- ? - ?~ a + ?@ a | ?|((b q.n.a) $(a l.a) $(a r.a)) :: @@ -40,7 +46,7 @@ ~/ %bif |* b=* |- ^+ [l=a r=a] - ?~ a + ?@ a [~ ~] ?: =(b p.n.a) +.a @@ -56,15 +62,15 @@ ~/ %del |* b=* |- ^+ a - ?~ a + ?@ a ~ ?. =(b p.n.a) ?: (gor-tip b p.n.a) a(l $(a l.a)) a(r $(a r.a)) - |- ^- [$?(~ _a)] - ?~ l.a r.a - ?~ r.a l.a + |- ^- [$?(?(%~ %ztree) _a)] + ?@ l.a r.a + ?@ r.a l.a ?: (mor-tip p.n.l.a p.n.r.a) l.a(r $(l.a r.l.a)) r.a(l $(r.a l.r.a)) @@ -73,15 +79,15 @@ ~/ %dif |* b=_a |- ^+ a - ?~ b + ?@ b a =+ c=(bif p.n.b) ?> ?=(^ c) =+ d=$(a l.c, b l.b) =+ e=$(a r.c, b r.b) - |- ^- [$?(~ _a)] - ?~ d e - ?~ e d + |- ^- [$?(?(%~ %ztree) _a)] + ?@ d e + ?@ e d ?: (mor-tip p.n.d p.n.e) d(r $(d r.d)) e(l $(e l.e)) @@ -91,7 +97,7 @@ |= b=* =+ c=1 |- ^- (unit @) - ?~ a ~ + ?@ a ~ ?: =(b p.n.a) [~ u=(peg c 2)] ?: (gor-tip b p.n.a) $(a l.a, c (peg c 6)) @@ -101,12 +107,12 @@ =< $ =| [l=(unit) r=(unit)] |. ^- ? - ?~ a & + ?@ a =(~ a) ?& ?~(l & &((gor-tip p.n.a u.l) !=(p.n.a u.l))) ?~(r & &((gor-tip u.r p.n.a) !=(u.r p.n.a))) - ?~ l.a & + ?@ l.a & &((mor-tip p.n.a p.n.l.a) !=(p.n.a p.n.l.a) $(a l.a, l `p.n.a)) - ?~ r.a & + ?@ r.a & &((mor-tip p.n.a p.n.r.a) !=(p.n.a p.n.r.a) $(a r.a, r `p.n.a)) == :: @@ -115,7 +121,7 @@ |* b=(list [p=* q=*]) => .(b `(list _?>(?=(^ a) n.a))`b) |- ^+ a - ?~ b + ?@ b a $(b t.b, a (put p.i.b q.i.b)) :: @@ -124,7 +130,7 @@ |* b=* => .(b `_?>(?=(^ a) p.n.a)`b) |- ^- (unit _?>(?=(^ a) q.n.a)) - ?~ a + ?@ a ~ ?: =(b p.n.a) (some q.n.a) @@ -151,9 +157,9 @@ ~/ %int |* b=_a |- ^+ a - ?~ b + ?@ b ~ - ?~ a + ?@ a ~ ?: (mor-tip p.n.a p.n.b) ?: =(p.n.b p.n.a) @@ -172,7 +178,7 @@ |* [key=_?>(?=(^ a) p.n.a) fun=$-(_?>(?=(^ a) q.n.a) _?>(?=(^ a) q.n.a))] ^+ a :: - ?~ a !! + ?@ a !! :: ?: =(key p.n.a) a(q.n (fun q.n.a)) @@ -193,7 +199,7 @@ ~/ %put |* [b=* c=*] |- ^+ a - ?~ a + ?@ a [[b c] ~ ~] ?: =(b p.n.a) ?: =(c q.n.a) @@ -215,14 +221,14 @@ ~/ %rep |* b=_=>(~ |=([* *] +<+)) |- - ?~ a +<+.b + ?@ a +<+.b $(a r.a, +<+.b $(a l.a, +<+.b (b n.a +<+.b))) :: ++ rib :: transform + product ~/ %rib |* [b=* c=gate] |- ^+ [b a] - ?~ a [b ~] + ?@ a [b ~] =+ d=(c n.a b) =. n.a +.d =+ e=$(a l.a, b -.d) @@ -233,14 +239,14 @@ ~/ %run |* b=gate |- - ?~ a a + ?@ a a [n=[p=p.n.a q=(b q.n.a)] l=$(a l.a) r=$(a r.a)] :: ++ tap :: listify pairs =< $ =+ b=`(list _?>(?=(^ a) n.a))`~ |. ^+ b - ?~ a + ?@ a b $(a r.a, b [n.a $(a l.a)]) :: @@ -248,9 +254,9 @@ ~/ %uni |* b=_a |- ^+ a - ?~ b + ?@ b a - ?~ a + ?@ a b ?: =(p.n.b p.n.a) b(l $(a l.a, b l.b), r $(a r.a, b r.b)) @@ -267,9 +273,9 @@ |* b=_a |* meg=$-([* * *] *) |- ^+ a - ?~ b + ?@ b a - ?~ a + ?@ a b ?: =(p.n.b p.n.a) :+ [p.n.a `_?>(?=(^ a) q.n.a)`(meg p.n.a q.n.a q.n.b)] @@ -287,40 +293,40 @@ ~/ %urn |* b=$-([* *] *) |- - ?~ a ~ + ?@ a ~ a(n n.a(q (b p.n.a q.n.a)), l $(a l.a), r $(a r.a)) :: ++ wyt :: depth of z-map =< $ |. ^- @ - ?~(a 0 +((add $(a l.a) $(a r.a)))) + ?@(a 0 +((add $(a l.a) $(a r.a)))) :: ++ key :: z-set of keys |- ^- (z-set _?>(?=(^ a) p.n.a)) - ?~ a ~ + ?@ a ~ [p.n.a $(a l.a) $(a r.a)] :: ++ val :: list of vals =+ b=`(list _?>(?=(^ a) q.n.a))`~ |- ^+ b - ?~ a b + ?@ a b $(a r.a, b [q.n.a $(a l.a)]) -- +| %set ++ z-set |$ [item] :: z-set - $| (tree item) - |=(a=(tree) ?:(=(~ a) & ~(apt z-in a))) + $| (ztree item) + |=(a=(ztree) ?@(a =(~ a) ~(apt z-in a))) :: ++ z-in :: z-set engine ~/ %z-in - =| a=(tree) :: (z-set) + =| a=(ztree) :: (z-set) |@ ++ all :: logical AND ~/ %all |* b=$-(* ?) |- ^- ? - ?~ a + ?@ a & ?&((b n.a) $(a l.a) $(a r.a)) :: @@ -328,7 +334,7 @@ ~/ %any |* b=$-(* ?) |- ^- ? - ?~ a + ?@ a | ?|((b n.a) $(a l.a) $(a r.a)) :: @@ -336,11 +342,11 @@ =< $ =| [l=(unit) r=(unit)] |. ^- ? - ?~ a & + ?@ a =(~ a) ?& ?~(l & &((gor-tip n.a u.l) !=(n.a u.l))) ?~(r & &((gor-tip u.r n.a) !=(u.r n.a))) - ?~(l.a & ?&((mor-tip n.a n.l.a) !=(n.a n.l.a) $(a l.a, l `n.a))) - ?~(r.a & ?&((mor-tip n.a n.r.a) !=(n.a n.r.a) $(a r.a, r `n.a))) + ?@(l.a & ?&((mor-tip n.a n.l.a) !=(n.a n.l.a) $(a l.a, l `n.a))) + ?@(r.a & ?&((mor-tip n.a n.r.a) !=(n.a n.r.a) $(a r.a, r `n.a))) == :: ++ bif :: splits a by b @@ -349,7 +355,7 @@ ^+ [l=a r=a] =< + |- ^+ a - ?~ a + ?@ a [b ~ ~] ?: =(b n.a) a @@ -365,15 +371,15 @@ ~/ %del |* b=* |- ^+ a - ?~ a + ?@ a ~ ?. =(b n.a) ?: (gor-tip b n.a) a(l $(a l.a)) a(r $(a r.a)) - |- ^- [$?(~ _a)] - ?~ l.a r.a - ?~ r.a l.a + |- ^- [$?(?(%~ %ztree) _a)] + ?@ l.a r.a + ?@ r.a l.a ?: (mor-tip n.l.a n.r.a) l.a(r $(l.a r.l.a)) r.a(l $(r.a l.r.a)) @@ -382,15 +388,15 @@ ~/ %dif |* b=_a |- ^+ a - ?~ b + ?@ b a =+ c=(bif n.b) ?> ?=(^ c) =+ d=$(a l.c, b l.b) =+ e=$(a r.c, b r.b) - |- ^- [$?(~ _a)] - ?~ d e - ?~ e d + |- ^- [$?(?(%~ %ztree) _a)] + ?@ d e + ?@ e d ?: (mor-tip n.d n.e) d(r $(d r.d)) e(l $(e l.e)) @@ -400,7 +406,7 @@ |= b=* =+ c=1 |- ^- (unit @) - ?~ a ~ + ?@ a ~ ?: =(b n.a) [~ u=(peg c 2)] ?: (gor-tip b n.a) $(a l.a, c (peg c 6)) @@ -410,7 +416,7 @@ ~/ %gas |= b=(list _?>(?=(^ a) n.a)) |- ^+ a - ?~ b + ?@ b a $(b t.b, a (put i.b)) :: +has: does :b exist z-in :a? @@ -434,7 +440,7 @@ |= b=(unit _?>(?=(^ a) n.a)) => .(b ?>(?=(^ b) u.b)) |- ^- ? - ?~ a + ?@ a | ?: =(b n.a) & @@ -446,9 +452,9 @@ ~/ %int |* b=_a |- ^+ a - ?~ b + ?@ b ~ - ?~ a + ?@ a ~ ?. (mor-tip n.a n.b) $(a b, b a) @@ -462,7 +468,7 @@ ~/ %put |* b=* |- ^+ a - ?~ a + ?@ a [b ~ ~] ?: =(b n.a) a @@ -482,14 +488,14 @@ ~/ %rep |* b=_=>(~ |=([* *] +<+)) |- - ?~ a +<+.b + ?@ a +<+.b $(a r.a, +<+.b $(a l.a, +<+.b (b n.a +<+.b))) :: ++ run :: apply gate to values ~/ %run |* b=gate =+ c=`(z-set _?>(?=(^ a) (b n.a)))`~ - |- ?~ a c + |- ?@ a c =. c (~(put z-in c) (b n.a)) =. c $(a l.a, c c) $(a r.a, c c) @@ -498,7 +504,7 @@ =< $ =+ b=`(list _?>(?=(^ a) n.a))`~ |. ^+ b - ?~ a + ?@ a b $(a r.a, b [n.a $(a l.a)]) :: @@ -507,9 +513,9 @@ |* b=_a ?: =(a b) a |- ^+ a - ?~ b + ?@ b a - ?~ a + ?@ a b ?: =(n.b n.a) b(l $(a l.a, b l.b), r $(a r.a, b r.b)) @@ -524,7 +530,7 @@ ++ wyt :: size of z-set =< $ |. ^- @ - ?~(a 0 +((add $(a l.a) $(a r.a)))) + ?@(a 0 +((add $(a l.a) $(a r.a)))) -- +| %mip :: @@ -539,7 +545,7 @@ |* [b=* c=*] =+ d=(~(gut z-by a) b ~) =+ e=(~(del z-by d) c) - ?~ e + ?@ e (~(del z-by a) b) (~(put z-by a) b e) :: @@ -577,7 +583,7 @@ =< $ =+ b=`_?>(?=(^ a) *(list [x=_p.n.a _?>(?=(^ q.n.a) [y=p v=q]:n.q.n.a)]))`~ |. ^+ b - ?~ a + ?@ a b $(a r.a, b (welp (turn ~(tap z-by q.n.a) (lead p.n.a)) $(a l.a))) -- @@ -589,14 +595,14 @@ (z-map key (z-set value)) :: ++ z-ju :: z-jug engine - =| a=(tree (pair * (tree))) :: (z-jug) + =| a=(z-map * (z-set)) :: (z-jug) |@ ++ del :: del key-set pair |* [b=* c=*] ^+ a =+ d=(get b) =+ e=(~(del z-in d) c) - ?~ e + ?@ e (~(del z-by a) b) (~(put z-by a) b e) :: @@ -604,14 +610,14 @@ |* b=(list [p=* q=*]) => .(b `(list _?>(?=([[* ^] ^] a) [p=p q=n.q]:n.a))`b) |- ^+ a - ?~ b + ?@ b a $(b t.b, a (put p.i.b q.i.b)) :: ++ get :: gets z-set by key |* b=* =+ c=(~(get z-by a) b) - ?~(c ~ u.c) + ?@(c ~ u.c) :: ++ has :: existence check |* [b=* c=*] @@ -687,12 +693,12 @@ +| %z-container-from-container ++ z-silt :: z-set from list |* a=(list) - =+ b=*(tree _?>(?=(^ a) i.a)) + =+ b=*(ztree _?>(?=(^ a) i.a)) (~(gas z-in b) a) :: ++ z-molt :: z-map from pair |* a=(list (pair)) - (~(gas z-by `(tree [p=_p.i.-.a q=_q.i.-.a])`~) a) + (~(gas z-by `(ztree [p=_p.i.-.a q=_q.i.-.a])`~) a) :: ++ z-malt :: z-map from list |* a=(list) diff --git a/hoon/scripts/generate-fund-address.hoon b/hoon/scripts/generate-fund-address.hoon new file mode 100644 index 000000000..ec2ad8975 --- /dev/null +++ b/hoon/scripts/generate-fund-address.hoon @@ -0,0 +1,33 @@ +/= * /common/tx-engine +/= zo /common/zoon +:: Computes the +fund-address constant for 014-aletheia: the lock-root +:: of a 3-of-4 multisig over the four pkhs in /asert-protocol-lock-fund.txt. +:: The result is a literal hash to be pasted into +fund-address in +:: /common/tx-engine-1.hoon, alongside a link back to this script. +:: Spending the protocol fund post-activation requires three of four +:: signatures from the participants below. +:: +:: Re-run after any change to the participant set or threshold: +:: cargo run --profile release --bin hoon -- \ +:: +:: +:: The trace prints both the 5-tuple atom representation and the +:: base58 encoding of the resulting fund-address. +=/ pkhs=(list hash) + :~ (from-b58:hash '7pGXggKU1AWk3d3wqX2kpKUatTqT68Cv8SQfGzGRQvJvYnQBvagSSjT') + (from-b58:hash '8Mc1U7kdujhPoEwog1BfNsFDtRp8St8UQCHk84iaLdhP4cX9a2CT1MU') + (from-b58:hash 'DAvp9ffoyNTBqAudZN29qc6s8GZfvvvAGvAfEFrqQsCVgSSSkg1SaSm') + (from-b58:hash '9LK7wEcQsmRpEot4qFaV9bwjSE9ZD6tB1kWbgNsFkDa2LEpvBV9WGY3') + == +=/ participant-set=(z-set:zo hash) (z-silt:zo pkhs) +:: 3-of-4 multisig: a single spend-condition with one ++pkh primitive. +:: The +pkh primitive enforces m-of-n by itself, so we don't need to +:: enumerate the four 3-pkh combinations as separate spend-conditions. +=/ multi-lock=lock + [%pkh [m=3 participant-set]]~ +=/ fund-address=hash (hash:lock multi-lock) +=/ fund-address-b58=@t (to-b58:hash fund-address) +~& fund-address+fund-address +~& fund-address-b58+fund-address-b58 +~& fund-address-back-to-hash+(from-b58:hash fund-address-b58) +fund-address diff --git a/scripts/aletheia_asert_vectors.rs b/scripts/aletheia_asert_vectors.rs new file mode 100755 index 000000000..b4cf31a53 --- /dev/null +++ b/scripts/aletheia_asert_vectors.rs @@ -0,0 +1,163 @@ +#!/usr/bin/env -S rust-script +//! Fetch mainnet block timestamps and targets for a post-activation +//! range and emit them as Hoon-syntax test vectors. The output is a +//! `(list [height=@ parent-min-ts=@ target=@])` plus an +//! `++ anchor-min-timestamp` pin and an `++ anchor-target` pin — +//! ready to paste into a test that imports `lib/asert` and pins +//! `compute-target:asert` against observed mainnet outputs. +//! +//! For each post-activation block N, `parent-min-ts` is the +//! median-of-11 of timestamps from blocks (N-11)..=(N-1). This is +//! exactly the value `min-timestamps[parent.digest]` would hold on a +//! node that just accepted block N-1. +//! +//! ```cargo +//! [dependencies] +//! anyhow = "1" +//! serde = { version = "1", features = ["derive"] } +//! serde_json = "1" +//! ``` +use std::process::Command; + +use anyhow::{Context, Result, bail}; +use serde::Deserialize; + +const ANCHOR_HEIGHT: u64 = 65_499; +const ACTIVATION_HEIGHT: u64 = 65_500; +const FIRST_TEST: u64 = ACTIVATION_HEIGHT; +const LAST_TEST: u64 = 65_520; +const LOOKBACK: u64 = 11; +const ENDPOINT: &str = "23.252.122.122:5556"; +const METHOD: &str = "nockchain.public.v2.NockchainBlockService/GetBlockDetails"; + +#[derive(Debug, Deserialize)] +struct WireTarget { + display: String, +} + +#[derive(Debug, Deserialize)] +struct WireDetails { + height: String, + timestamp: String, + target: WireTarget, +} + +#[derive(Debug, Deserialize)] +struct WireEnvelope { + details: Option, + error: Option, +} + +fn fetch_block(height: u64) -> Result { + let payload = format!("{{\"height\":{height}}}"); + let out = Command::new("grpcurl") + .args([ + "-plaintext", + "-max-time", + "30", + "-d", + &payload, + ENDPOINT, + METHOD, + ]) + .output() + .with_context(|| format!("invoking grpcurl for height {height}"))?; + if !out.status.success() { + bail!( + "grpcurl failed for height {height}: {}", + String::from_utf8_lossy(&out.stderr) + ); + } + let env: WireEnvelope = serde_json::from_slice(&out.stdout) + .with_context(|| format!("parsing GetBlockDetails JSON for height {height}"))?; + if let Some(err) = env.error { + bail!("server error at height {height}: {err}"); + } + env.details + .with_context(|| format!("no details field for height {height}")) +} + +fn format_dots(n: u128) -> String { + let s = n.to_string(); + let bytes = s.as_bytes(); + let mut out = String::with_capacity(s.len() + s.len() / 3); + for (i, b) in bytes.iter().enumerate() { + if i > 0 && (bytes.len() - i) % 3 == 0 { + out.push('.'); + } + out.push(*b as char); + } + out +} + +fn format_dots_str(s: &str) -> String { + let bytes = s.as_bytes(); + let mut out = String::with_capacity(s.len() + s.len() / 3); + for (i, b) in bytes.iter().enumerate() { + if i > 0 && (bytes.len() - i) % 3 == 0 { + out.push('.'); + } + out.push(*b as char); + } + out +} + +fn main() -> Result<()> { + // Need timestamps from (FIRST_TEST - LOOKBACK)..=LAST_TEST inclusive, + // plus targets for FIRST_TEST..=LAST_TEST. + let lo_ts = FIRST_TEST - LOOKBACK; + eprintln!("Fetching heights {lo_ts}..={LAST_TEST}"); + let mut blocks: Vec<(u64, u64, String)> = Vec::new(); + for h in lo_ts..=LAST_TEST { + let d = fetch_block(h)?; + let observed_height: u64 = d.height.parse()?; + if observed_height != h { + bail!("height mismatch: requested {h}, got {observed_height}"); + } + let ts: u64 = d.timestamp.parse()?; + blocks.push((h, ts, d.target.display.clone())); + eprintln!(" height={h} ts={ts} target_digits={}", d.target.display.len()); + } + + // Compute parent-min-ts (median-of-11 of timestamps[N-11..=N-1]) + // for each block N in FIRST_TEST..=LAST_TEST. + println!(); + println!(":: Mainnet ASERT cross-check vectors. Each entry is"); + println!(":: [height=@ parent-min-ts=@ observed-target=@], where"); + println!(":: parent-min-ts is the median-of-11 of timestamps for"); + println!(":: blocks (height-11)..=(height-1) — exactly what"); + println!(":: min-timestamps[parent.digest] holds on a node that"); + println!(":: just accepted block (height-1)."); + println!("++ asert-vectors"); + println!(" ^- (list [height=@ parent-min-ts=@ observed-target=@])"); + println!(" :~"); + for &(h, _ts, ref target) in &blocks { + if h < FIRST_TEST { + continue; + } + // window is blocks[(h - lo_ts - LOOKBACK)..(h - lo_ts)] + let end_idx = (h - lo_ts) as usize; + let start_idx = end_idx - LOOKBACK as usize; + let mut window: Vec = blocks[start_idx..end_idx].iter().map(|b| b.1).collect(); + window.sort_unstable(); + let median = window[(LOOKBACK as usize) / 2]; + + println!( + " [height={} parent-min-ts={} observed-target={}]", + format_dots(h as u128), + format_dots(median as u128), + format_dots_str(target), + ); + } + println!(" =="); + println!(); + println!(":: Anchor pins (block 65,499 = asert-anchor-height)."); + println!( + "++ anchor-min-timestamp {}", + format_dots(9_223_372_093_639_027_842u128) + ); + println!("++ anchor-target-atom ^~((bex 291))"); + println!("++ asert-anchor-height {}", format_dots(ANCHOR_HEIGHT as u128)); + + Ok(()) +} diff --git a/scripts/aletheia_phase2_params.rs b/scripts/aletheia_phase2_params.rs new file mode 100755 index 000000000..6be436237 --- /dev/null +++ b/scripts/aletheia_phase2_params.rs @@ -0,0 +1,183 @@ +#!/usr/bin/env -S rust-script +//! Fetch the three Aletheia phase 2 cutover parameters from a Nockchain +//! public gRPC node and emit them in a form ready to paste into Hoon. +//! +//! - asert-anchor-digest = base58 digest of mainnet block 65,499 +//! - asert-anchor-min-timestamp = median-of-11 timestamp across blocks 65,489..=65,499 +//! - asert-activation-digest = base58 digest of mainnet block 65,500 +//! +//! Hash <-> base58 conversion follows `Hash::to_base58` in +//! `open/crates/nockchain-types/src/tx_engine/common/mod.rs:149`: belts are +//! interpreted as a base-p number with p = 2^64 - 2^32 + 1 (the Mersenne +//! prime used as the field modulus), and the resulting bigint is encoded +//! big-endian then base58 (Bitcoin alphabet). +//! +//! ```cargo +//! [dependencies] +//! anyhow = "1" +//! bs58 = "0.5" +//! num-bigint = "0.4" +//! serde = { version = "1", features = ["derive"] } +//! serde_json = "1" +//! ``` + +use std::process::Command; + +use anyhow::{Context, Result, bail}; +use num_bigint::BigUint; +use serde::Deserialize; + +const ANCHOR_HEIGHT: u64 = 65_499; +const ACTIVATION_HEIGHT: u64 = 65_500; +const MIN_PAST_BLOCKS: u64 = 11; +const ENDPOINT: &str = "23.252.122.122:5556"; +const METHOD: &str = "nockchain.public.v2.NockchainBlockService/GetBlockDetails"; + +// Mersenne field prime: 2^64 - 2^32 + 1 +const PRIME: u64 = 0xffffffff00000001; + +#[derive(Debug, Deserialize)] +struct BeltValue { + value: String, +} + +#[derive(Debug, Deserialize)] +struct WireHash { + belt1: BeltValue, + belt2: BeltValue, + belt3: BeltValue, + belt4: BeltValue, + belt5: BeltValue, +} + +#[derive(Debug, Deserialize)] +struct WireDetails { + #[serde(rename = "blockId")] + block_id: WireHash, + height: String, + timestamp: String, +} + +#[derive(Debug, Deserialize)] +struct WireEnvelope { + details: Option, + error: Option, +} + +fn fetch_block(height: u64) -> Result { + let payload = format!("{{\"height\":{height}}}"); + let out = Command::new("grpcurl") + .args([ + "-plaintext", + "-max-time", + "30", + "-d", + &payload, + ENDPOINT, + METHOD, + ]) + .output() + .with_context(|| format!("invoking grpcurl for height {height}"))?; + if !out.status.success() { + bail!( + "grpcurl failed for height {height}: {}", + String::from_utf8_lossy(&out.stderr) + ); + } + let env: WireEnvelope = serde_json::from_slice(&out.stdout) + .with_context(|| format!("parsing GetBlockDetails JSON for height {height}"))?; + if let Some(err) = env.error { + bail!("server error at height {height}: {err}"); + } + let details = env + .details + .with_context(|| format!("no details field for height {height}"))?; + let observed_height: u64 = details.height.parse()?; + if observed_height != height { + bail!("requested height {height} but server returned {observed_height}"); + } + Ok(details) +} + +fn belts_to_b58(h: &WireHash) -> Result { + let belts = [&h.belt1, &h.belt2, &h.belt3, &h.belt4, &h.belt5]; + let prime = BigUint::from(PRIME); + let mut value = BigUint::from(0u8); + let mut power = BigUint::from(1u8); + for b in belts { + let limb: u64 = b + .value + .parse() + .with_context(|| format!("parsing belt value {:?}", b.value))?; + value += BigUint::from(limb) * &power; + power *= ′ + } + Ok(bs58::encode(value.to_bytes_be()).into_string()) +} + +fn main() -> Result<()> { + let lo = ANCHOR_HEIGHT - (MIN_PAST_BLOCKS - 1); + let hi = ANCHOR_HEIGHT; + eprintln!("Fetching blocks {lo}..={hi} for median-of-11"); + + let mut timestamps: Vec = Vec::with_capacity(MIN_PAST_BLOCKS as usize); + let mut anchor_details: Option = None; + for h in lo..=hi { + let d = fetch_block(h)?; + let ts: u64 = d + .timestamp + .parse() + .with_context(|| format!("parsing timestamp at height {h}"))?; + eprintln!(" height={h} timestamp={ts}"); + timestamps.push(ts); + if h == ANCHOR_HEIGHT { + anchor_details = Some(d); + } + } + let anchor = anchor_details.expect("anchor fetched in loop above"); + + eprintln!("Fetching activation block {ACTIVATION_HEIGHT}"); + let activation = fetch_block(ACTIVATION_HEIGHT)?; + + // Median of 11: sort, take the middle (index 5, 0-indexed). + timestamps.sort_unstable(); + let median = timestamps[(MIN_PAST_BLOCKS as usize) / 2]; + + let anchor_b58 = belts_to_b58(&anchor.block_id)?; + let activation_b58 = belts_to_b58(&activation.block_id)?; + + println!(); + println!("=== Aletheia phase 2 parameters ==="); + println!("asert-anchor-digest = '{}'", anchor_b58); + println!("asert-anchor-min-timestamp = {}", median); + println!("asert-activation-digest = '{}'", activation_b58); + println!(); + println!("Hoon snippets:"); + println!( + " [%65.499 (from-b58:hash:t '{}')]", + anchor_b58 + ); + println!( + " [%65.500 (from-b58:hash:t '{}')]", + activation_b58 + ); + println!( + " asert-anchor-min-timestamp={}", + format_with_dots(median) + ); + + Ok(()) +} + +fn format_with_dots(n: u64) -> String { + let s = n.to_string(); + let bytes = s.as_bytes(); + let mut out = String::with_capacity(s.len() + s.len() / 3); + for (i, b) in bytes.iter().enumerate() { + if i > 0 && (bytes.len() - i) % 3 == 0 { + out.push('.'); + } + out.push(*b as char); + } + out +} diff --git a/scripts/block-poke-times.sh b/scripts/block-poke-times.sh new file mode 100755 index 000000000..9a25055b7 --- /dev/null +++ b/scripts/block-poke-times.sh @@ -0,0 +1,53 @@ +#!/usr/bin/env bash +set -euo pipefail + +DB="${1:-.data.nockchain-sync-fsync-on/event-log.sqlite3}" + +if [ ! -f "$DB" ]; then + echo "Event log not found: $DB" + exit 1 +fi + +cat <<'SQL' | sqlite3 -readonly "$DB" +.mode column +.headers on + +SELECT '=== Last 100 heard-block Pokes ===' AS ''; +SELECT + event_num, + wire_tags_json AS tags, + printf('%.1f ms', event_processing_duration_us / 1000.0) AS duration, + datetime(created_at_ms / 1000, 'unixepoch', 'localtime') AS created_at +FROM events +WHERE wire_source = 'libp2p' + AND wire_tags_json LIKE '%heard-block%' +ORDER BY event_num DESC +LIMIT 100; + +SELECT '=== Duration Stats: heard-block (last 100) ===' AS ''; +SELECT + printf('%.1f ms', min(event_processing_duration_us) / 1000.0) AS min_dur, + printf('%.1f ms', avg(event_processing_duration_us) / 1000.0) AS avg_dur, + printf('%.1f ms', max(event_processing_duration_us) / 1000.0) AS max_dur, + printf('%.1f ms', sum(event_processing_duration_us) / 1000.0) AS total_dur, + count(*) AS count +FROM ( + SELECT event_processing_duration_us FROM events + WHERE wire_source = 'libp2p' + AND wire_tags_json LIKE '%heard-block%' + ORDER BY event_num DESC + LIMIT 100 +); + +SELECT '=== Top 20 Slowest heard-block Pokes ===' AS ''; +SELECT + event_num, + wire_tags_json AS tags, + printf('%.1f ms', event_processing_duration_us / 1000.0) AS duration, + datetime(created_at_ms / 1000, 'unixepoch', 'localtime') AS created_at +FROM events +WHERE wire_source = 'libp2p' + AND wire_tags_json LIKE '%heard-block%' +ORDER BY event_processing_duration_us DESC +LIMIT 20; +SQL diff --git a/scripts/compare_pma_mem.rs b/scripts/compare_pma_mem.rs new file mode 100755 index 000000000..24dc03ade --- /dev/null +++ b/scripts/compare_pma_mem.rs @@ -0,0 +1,1460 @@ +#!/usr/bin/env rust-script +//! ```cargo +//! [package] +//! edition = "2021" +//! ``` +use std::collections::HashMap; +use std::convert::TryInto; +use std::env; +use std::ffi::OsString; +use std::fs::{self, File}; +use std::io::{BufRead, BufReader}; +use std::path::{Path, PathBuf}; + +#[derive(Clone, Debug)] +struct ProcInfo { + pid: i32, + cmdline: String, + comm: String, + exe: Option, + cwd: Option, +} + +#[derive(Clone, Debug, Default)] +struct MemTotals { + size_kb: u64, + rss_kb: u64, + pss_kb: u64, + shared_clean_kb: u64, + shared_dirty_kb: u64, + private_clean_kb: u64, + private_dirty_kb: u64, + swap_kb: u64, + swap_pss_kb: u64, +} + +#[derive(Clone, Debug, Default)] +struct StatusMem { + vm_rss_kb: u64, + vm_size_kb: u64, + vm_swap_kb: u64, + rss_anon_kb: u64, + rss_file_kb: u64, + rss_shmem_kb: u64, +} + +#[derive(Clone, Debug, Default)] +struct ProcMemReport { + status: StatusMem, + smaps_rollup: Option, + pma_maps: MemTotals, + pma_map_count: u64, + pma_smaps_available: bool, + pma_alloc_bytes: Option, + checkpoint: Option, +} + +#[derive(Clone, Debug)] +struct CheckpointInfo { + checkpoints_dir: PathBuf, + file_count: u64, + total_bytes: u64, + latest_path: Option, + latest_bytes: u64, +} + +fn main() { + if !Path::new("/proc").is_dir() { + eprintln!("This script requires /proc (Linux)."); + std::process::exit(1); + } + + let (pma_proc, base_proc) = match discover_procs() { + Ok(pair) => pair, + Err(err) => { + eprintln!("{err}"); + std::process::exit(1); + } + }; + + let pma_report = match collect_report( + &pma_proc, + env::var("NOCKCHAIN_PMA_DATA_DIR").ok().map(PathBuf::from), + ) { + Ok(report) => report, + Err(err) => { + eprintln!("Failed to read memory stats for PMA process: {err}"); + std::process::exit(1); + } + }; + let base_report = match collect_report( + &base_proc, + env::var("NOCKCHAIN_BASE_DATA_DIR").ok().map(PathBuf::from), + ) { + Ok(report) => report, + Err(err) => { + eprintln!("Failed to read memory stats for base process: {err}"); + std::process::exit(1); + } + }; + + println!("PMA process:"); + print_proc(&pma_proc, &pma_report); + println!(); + println!("Base process:"); + print_proc(&base_proc, &base_report); + println!(); + print_comparison(&pma_report, &base_report); + println!(); + print_summary(&pma_proc, &pma_report, &base_proc, &base_report); +} + +fn discover_procs() -> Result<(ProcInfo, ProcInfo), String> { + if let (Ok(pma_pid), Ok(base_pid)) = ( + env::var("NOCKCHAIN_PMA_PID"), + env::var("NOCKCHAIN_BASE_PID"), + ) { + let pma_pid = parse_pid(&pma_pid)?; + let base_pid = parse_pid(&base_pid)?; + let pma_info = read_proc_info(pma_pid)?; + let base_info = read_proc_info(base_pid)?; + return Ok((pma_info, base_info)); + } + + let all = list_proc_infos()?; + let candidates: Vec = all + .into_iter() + .filter(|info| is_nockchain_process(info)) + .collect(); + + if candidates.len() < 2 { + return Err(format!( + "Expected at least 2 nockchain processes, found {}.\n\ + You can set NOCKCHAIN_PMA_PID and NOCKCHAIN_BASE_PID to override.", + candidates.len() + )); + } + + let base_dir = env::var("NOCKCHAIN_BASE_DIR").ok().map(PathBuf::from); + let repo_root_opt = env::current_dir().ok().and_then(find_repo_root); + + if let Some((pma, base)) = pick_by_cwd_hint(&candidates) { + return Ok((pma, base)); + } + + if let Some(repo_root) = repo_root_opt.as_ref() { + if let Some((pma, base)) = pick_by_repo_data_dir(&candidates, repo_root) { + return Ok((pma, base)); + } + } + + if let Some((pma, base)) = pick_by_data_dir_flag(&candidates) { + return Ok((pma, base)); + } + + if let Some((pma, base)) = pick_by_pma_presence(&candidates, base_dir.as_ref()) { + return Ok((pma, base)); + } + + if candidates.len() == 2 { + let first_is_docker = is_docker_nockchain(&candidates[0]); + let second_is_docker = is_docker_nockchain(&candidates[1]); + if first_is_docker != second_is_docker { + let (pma, base) = if first_is_docker { + (candidates[0].clone(), candidates[1].clone()) + } else { + (candidates[1].clone(), candidates[0].clone()) + }; + return Ok((pma, base)); + } + } + + let repo_root = match repo_root_opt { + Some(root) => root, + None => { + return Err( + "Could not locate repo root (Cargo.toml + crates/). Run from repo root." + .to_string(), + ); + } + }; + + let mut docker_candidates: Vec = candidates + .iter() + .cloned() + .filter(|info| is_docker_nockchain(info)) + .collect(); + if docker_candidates.len() == 1 { + let pma = docker_candidates.remove(0); + let mut base_candidates: Vec = candidates + .iter() + .cloned() + .filter(|info| info.pid != pma.pid) + .collect(); + base_candidates = apply_base_dir_filter(&base_candidates, base_dir.as_ref()); + let base = choose_single( + base_candidates, + "base", + "Set NOCKCHAIN_BASE_PID or NOCKCHAIN_BASE_DIR.", + )?; + return Ok((pma, base)); + } + + let mut with_pma: Vec = Vec::new(); + let mut without_pma: Vec = Vec::new(); + + for info in candidates.iter().cloned() { + if has_pma_mapping(info.pid).unwrap_or(false) { + with_pma.push(info); + } else { + without_pma.push(info); + } + } + + if with_pma.len() == 1 && !without_pma.is_empty() { + let pma = with_pma.remove(0); + let base_candidates = apply_base_dir_filter(&without_pma, base_dir.as_ref()); + let base = choose_single( + base_candidates, + "base", + "Set NOCKCHAIN_BASE_PID or NOCKCHAIN_BASE_DIR.", + )?; + return Ok((pma, base)); + } + + let mut container_candidates: Vec = Vec::new(); + let mut host_candidates: Vec = Vec::new(); + for info in candidates.iter().cloned() { + if is_container_process(&info) { + container_candidates.push(info); + } else { + host_candidates.push(info); + } + } + if container_candidates.len() == 1 && !host_candidates.is_empty() { + let pma = container_candidates.remove(0); + let base_candidates = apply_base_dir_filter(&host_candidates, base_dir.as_ref()); + let base = choose_single( + base_candidates, + "base", + "Set NOCKCHAIN_BASE_PID or NOCKCHAIN_BASE_DIR.", + )?; + return Ok((pma, base)); + } + + let mut pma_candidates: Vec = Vec::new(); + let mut base_candidates: Vec = Vec::new(); + + for info in candidates { + let is_pma = is_under_root(&info, &repo_root); + let is_base = base_dir + .as_ref() + .map(|base| is_under_root(&info, base)) + .unwrap_or(false); + + if is_pma { + pma_candidates.push(info); + } else if is_base { + base_candidates.push(info); + } else { + base_candidates.push(info); + } + } + + let pma = choose_single( + pma_candidates, + "PMA", + "Set NOCKCHAIN_PMA_PID or run from PMA repo root.", + )?; + + let base = choose_single( + base_candidates, + "base", + "Set NOCKCHAIN_BASE_PID or NOCKCHAIN_BASE_DIR.", + )?; + + Ok((pma, base)) +} + +fn pick_by_cwd_hint(candidates: &[ProcInfo]) -> Option<(ProcInfo, ProcInfo)> { + let mut pma_candidates: Vec = candidates + .iter() + .cloned() + .filter(|info| has_pma_hint_from_paths(info)) + .collect(); + if pma_candidates.len() == 1 && candidates.len() >= 2 { + let pma = pma_candidates.remove(0); + let mut base_candidates: Vec = candidates + .iter() + .cloned() + .filter(|info| info.pid != pma.pid) + .collect(); + if base_candidates.len() == 1 { + return Some((pma, base_candidates.remove(0))); + } + } + None +} + +fn pick_by_repo_data_dir( + candidates: &[ProcInfo], + repo_root: &Path, +) -> Option<(ProcInfo, ProcInfo)> { + let pma_data_dir = repo_root.join(".data.nockchain"); + let mut matches: Vec = candidates + .iter() + .cloned() + .filter(|info| { + resolve_data_dir(info, None) + .map(|dir| dir == pma_data_dir) + .unwrap_or(false) + }) + .collect(); + if matches.len() == 1 && candidates.len() >= 2 { + let pma = matches.remove(0); + let mut base_candidates: Vec = candidates + .iter() + .cloned() + .filter(|info| info.pid != pma.pid) + .collect(); + if base_candidates.len() == 1 { + return Some((pma, base_candidates.remove(0))); + } + } + None +} + +fn pick_by_data_dir_flag(candidates: &[ProcInfo]) -> Option<(ProcInfo, ProcInfo)> { + let mut with_flag: Vec = candidates + .iter() + .cloned() + .filter(|info| parse_data_dir_flag(&info.cmdline).is_some()) + .collect(); + let mut without_flag: Vec = candidates + .iter() + .cloned() + .filter(|info| parse_data_dir_flag(&info.cmdline).is_none()) + .collect(); + if with_flag.len() == 1 && without_flag.len() == 1 { + return Some((with_flag.remove(0), without_flag.remove(0))); + } + None +} + +fn pick_by_pma_presence( + candidates: &[ProcInfo], + base_dir: Option<&PathBuf>, +) -> Option<(ProcInfo, ProcInfo)> { + let mut with_pma = Vec::new(); + let mut without_pma = Vec::new(); + let mut unknown = Vec::new(); + + for info in candidates.iter().cloned() { + match detect_pma_presence(&info) { + Some(true) => with_pma.push(info), + Some(false) => without_pma.push(info), + None => unknown.push(info), + } + } + + if with_pma.len() == 1 { + let pma = with_pma.remove(0); + let base_candidates = if !without_pma.is_empty() { + apply_base_dir_filter(&without_pma, base_dir) + } else { + apply_base_dir_filter(&unknown, base_dir) + }; + if let Ok(base) = choose_single( + base_candidates, + "base", + "Set NOCKCHAIN_BASE_PID or NOCKCHAIN_BASE_DIR.", + ) { + return Some((pma, base)); + } + } + + if with_pma.is_empty() && without_pma.len() == 1 && !unknown.is_empty() { + let base = without_pma.remove(0); + if let Ok(pma) = choose_single( + apply_base_dir_filter(&unknown, base_dir), + "PMA", + "Set NOCKCHAIN_PMA_PID.", + ) { + return Some((pma, base)); + } + } + + if with_pma.len() > 1 && without_pma.len() == 1 { + let base = without_pma.remove(0); + if let Ok(pma) = choose_single( + with_pma, + "PMA", + "Set NOCKCHAIN_PMA_PID.", + ) { + return Some((pma, base)); + } + } + + None +} + +fn apply_base_dir_filter( + candidates: &[ProcInfo], + base_dir: Option<&PathBuf>, +) -> Vec { + let mut candidates: Vec = candidates.to_vec(); + if let Some(base_dir) = base_dir { + let filtered: Vec = candidates + .iter() + .cloned() + .filter(|info| is_under_root(info, base_dir)) + .collect(); + if !filtered.is_empty() { + candidates = filtered; + } + } + candidates +} + +fn choose_single(mut candidates: Vec, label: &str, hint: &str) -> Result { + if candidates.is_empty() { + return Err(format!("No candidate {label} process found. {hint}")); + } + if candidates.len() == 1 { + return Ok(candidates.remove(0)); + } + + if label.eq_ignore_ascii_case("base") { + let non_docker: Vec = candidates + .iter() + .cloned() + .filter(|info| !is_docker_nockchain(info)) + .collect(); + if non_docker.len() == 1 { + return Ok(non_docker[0].clone()); + } + } else if label.eq_ignore_ascii_case("pma") { + let docker: Vec = candidates + .iter() + .cloned() + .filter(|info| is_docker_nockchain(info)) + .collect(); + if docker.len() == 1 { + return Ok(docker[0].clone()); + } + } + + candidates.retain(|info| info.cmdline.contains("--fast-sync")); + if candidates.len() == 1 { + return Ok(candidates.remove(0)); + } + + let mut msg = format!( + "Ambiguous {label} process selection ({} candidates). {hint}\n", + candidates.len() + ); + for info in candidates { + msg.push_str(&format!( + " pid={} comm={} cwd={} cmdline={}\n", + info.pid, + info.comm, + info.cwd + .as_ref() + .map(|p| p.display().to_string()) + .unwrap_or_else(|| "".to_string()), + info.cmdline + )); + } + Err(msg) +} + +fn has_pma_mapping(pid: i32) -> Option { + let smaps_path = PathBuf::from("/proc").join(pid.to_string()).join("smaps"); + if let Some(found) = scan_mapping_file(&smaps_path) { + return Some(found); + } + let maps_path = PathBuf::from("/proc").join(pid.to_string()).join("maps"); + scan_mapping_file(&maps_path) +} + +fn scan_mapping_file(path: &Path) -> Option { + let file = File::open(path).ok()?; + let reader = BufReader::new(file); + for line in reader.lines().flatten() { + if let Some(path_opt) = parse_smaps_header(&line) { + if let Some(path) = path_opt { + if is_pma_mapping(&path) { + return Some(true); + } + } + } + } + Some(false) +} + +fn has_pma_fd(pid: i32) -> Option { + let fd_dir = PathBuf::from("/proc").join(pid.to_string()).join("fd"); + let entries = fs::read_dir(&fd_dir).ok()?; + for entry in entries.flatten() { + if let Ok(target) = fs::read_link(entry.path()) { + let target = target.to_string_lossy(); + if is_pma_mapping(&target) { + return Some(true); + } + } + } + Some(false) +} + +fn detect_pma_presence(info: &ProcInfo) -> Option { + if let Some(found) = has_pma_mapping(info.pid) { + return Some(found); + } + if let Some(found) = has_pma_fd(info.pid) { + return Some(found); + } + if has_pma_hint_from_paths(info) { + return Some(true); + } + if let Some(has_dir) = data_dir_has_pma(info) { + return Some(has_dir); + } + None +} + +fn has_pma_hint_from_paths(info: &ProcInfo) -> bool { + let needles = ["pma-nockchain", "pma_nockchain"]; + if let Some(cwd) = &info.cwd { + let cwd = cwd.to_string_lossy(); + if needles.iter().any(|needle| cwd.contains(needle)) { + return true; + } + } + if let Some(exe) = &info.exe { + let exe = exe.to_string_lossy(); + if needles.iter().any(|needle| exe.contains(needle)) { + return true; + } + } + false +} + +fn data_dir_has_pma(info: &ProcInfo) -> Option { + let data_dir = resolve_data_dir(info, None)?; + let pma_dir = data_dir.join("pma"); + match fs::read_dir(&pma_dir) { + Ok(entries) => { + for entry in entries.flatten() { + let path = entry.path(); + if path.extension().and_then(|s| s.to_str()) == Some("mmap") { + return Some(true); + } + } + Some(false) + } + Err(err) => match err.kind() { + std::io::ErrorKind::NotFound => Some(false), + std::io::ErrorKind::PermissionDenied => None, + _ => None, + }, + } +} + +fn is_container_process(info: &ProcInfo) -> bool { + if is_docker_nockchain(info) { + return true; + } + let path = PathBuf::from("/proc").join(info.pid.to_string()).join("cgroup"); + let contents = read_to_string(&path).unwrap_or_default(); + ["docker", "containerd", "kubepods", "libpod"] + .iter() + .any(|marker| contents.contains(marker)) +} + +fn is_docker_nockchain(info: &ProcInfo) -> bool { + if info.cmdline.contains("/data/.data.nockchain") + || info.cmdline.contains("/usr/local/bin/nockchain") + { + return true; + } + if let Some(exe) = &info.exe { + if exe.ends_with("/usr/local/bin/nockchain") { + return true; + } + } + false +} + +fn is_under_root(info: &ProcInfo, root: &Path) -> bool { + let mut matches = false; + if let Some(cwd) = &info.cwd { + if cwd.starts_with(root) { + matches = true; + } + } + if let Some(exe) = &info.exe { + if exe.starts_with(root) { + matches = true; + } + } + matches +} + +fn parse_pid(pid: &str) -> Result { + pid.parse::() + .map_err(|_| format!("Invalid pid: {pid}")) +} + +fn list_proc_infos() -> Result, String> { + let mut infos = Vec::new(); + let entries = fs::read_dir("/proc").map_err(|e| e.to_string())?; + for entry in entries { + let entry = entry.map_err(|e| e.to_string())?; + let name = entry.file_name(); + let name = match name.to_str() { + Some(s) => s, + None => continue, + }; + let pid = match name.parse::() { + Ok(pid) => pid, + Err(_) => continue, + }; + if let Ok(info) = read_proc_info(pid) { + infos.push(info); + } + } + Ok(infos) +} + +fn read_proc_info(pid: i32) -> Result { + let base = PathBuf::from("/proc").join(pid.to_string()); + let cmdline = read_cmdline(&base.join("cmdline")).unwrap_or_default(); + let comm = read_to_string(&base.join("comm")).unwrap_or_default(); + let exe = fs::read_link(&base.join("exe")).ok(); + let cwd = fs::read_link(&base.join("cwd")).ok(); + + Ok(ProcInfo { + pid, + cmdline, + comm: comm.trim().to_string(), + exe, + cwd, + }) +} + +fn read_cmdline(path: &Path) -> Option { + let data = fs::read(path).ok()?; + if data.is_empty() { + return None; + } + let parts: Vec = data + .split(|b| *b == 0) + .filter(|b| !b.is_empty()) + .map(|b| OsString::from(String::from_utf8_lossy(b).to_string())) + .collect(); + if parts.is_empty() { + return None; + } + let cmdline = parts + .iter() + .map(|s| s.to_string_lossy().to_string()) + .collect::>() + .join(" "); + Some(cmdline) +} + +fn read_to_string(path: &Path) -> Option { + fs::read_to_string(path).ok() +} + +fn is_nockchain_process(info: &ProcInfo) -> bool { + if info.comm == "nockchain" { + return true; + } + if let Some(exe) = &info.exe { + if exe.ends_with("nockchain") { + return true; + } + } + info.cmdline + .split_whitespace() + .next() + .map(|arg0| arg0.ends_with("nockchain") || arg0.ends_with("/nockchain")) + .unwrap_or(false) +} + +fn find_repo_root(start: PathBuf) -> Option { + let mut dir = start; + loop { + if dir.join("Cargo.toml").is_file() && dir.join("crates").is_dir() { + return Some(dir); + } + if !dir.pop() { + return None; + } + } +} + +fn collect_report( + info: &ProcInfo, + data_dir_override: Option, +) -> Result { + let base = PathBuf::from("/proc").join(info.pid.to_string()); + let status = parse_status(&base.join("status")).unwrap_or_default(); + let smaps_rollup = parse_smaps_rollup(&base.join("smaps_rollup")).ok(); + let (pma_maps, pma_map_count, pma_smaps_available) = parse_pma_smaps(&base.join("smaps"))?; + let pma_alloc_bytes = pma_alloc_bytes(info, data_dir_override.as_ref()); + let checkpoint = checkpoint_info(info, data_dir_override); + Ok(ProcMemReport { + status, + smaps_rollup, + pma_maps, + pma_map_count, + pma_smaps_available, + pma_alloc_bytes, + checkpoint, + }) +} + +fn parse_status(path: &Path) -> Option { + let contents = read_to_string(path)?; + let mut map: HashMap<&str, u64> = HashMap::new(); + for line in contents.lines() { + if let Some((key, val)) = parse_kb_line(line) { + map.insert(key, val); + } + } + Some(StatusMem { + vm_rss_kb: *map.get("VmRSS").unwrap_or(&0), + vm_size_kb: *map.get("VmSize").unwrap_or(&0), + vm_swap_kb: *map.get("VmSwap").unwrap_or(&0), + rss_anon_kb: *map.get("RssAnon").unwrap_or(&0), + rss_file_kb: *map.get("RssFile").unwrap_or(&0), + rss_shmem_kb: *map.get("RssShmem").unwrap_or(&0), + }) +} + +fn parse_smaps_rollup(path: &Path) -> Result { + let file = File::open(path).map_err(|e| e.to_string())?; + let reader = BufReader::new(file); + let mut totals = MemTotals::default(); + for line in reader.lines() { + let line = line.map_err(|e| e.to_string())?; + if let Some((key, val)) = parse_kb_line(&line) { + assign_mem_total(&mut totals, key, val); + } + } + Ok(totals) +} + +fn parse_pma_smaps(path: &Path) -> Result<(MemTotals, u64, bool), String> { + let file = match File::open(path) { + Ok(file) => file, + Err(_) => { + return Ok((MemTotals::default(), 0, false)); + } + }; + let reader = BufReader::new(file); + let mut totals = MemTotals::default(); + let mut count = 0u64; + let mut current_is_pma = false; + + for line in reader.lines() { + let line = line.map_err(|e| e.to_string())?; + if let Some(path_opt) = parse_smaps_header(&line) { + current_is_pma = path_opt + .as_ref() + .map(|path| is_pma_mapping(path)) + .unwrap_or(false); + if current_is_pma { + count += 1; + } + continue; + } + if current_is_pma { + if let Some((key, val)) = parse_kb_line(&line) { + assign_mem_total(&mut totals, key, val); + } + } + } + + Ok((totals, count, true)) +} + +fn parse_smaps_header(line: &str) -> Option> { + let mut parts = line.split_whitespace(); + let range = parts.next()?; + if !range.contains('-') { + return None; + } + let perms = parts.next()?; + if perms.len() < 4 { + return None; + } + parts.next()?; + parts.next()?; + parts.next()?; + let path = parts + .next() + .map(|p| p.trim_end_matches("(deleted)").to_string()); + Some(path) +} + +fn is_pma_mapping(path: &str) -> bool { + let path = path.trim(); + (path.contains("/pma/") && path.ends_with(".mmap")) || (path.contains("pma-") && path.ends_with(".mmap")) +} + +fn pma_alloc_bytes(info: &ProcInfo, data_dir_override: Option<&PathBuf>) -> Option { + let data_dir = resolve_data_dir(info, data_dir_override.cloned())?; + let pma_dir = data_dir.join("pma"); + if !pma_dir.is_dir() { + return None; + } + let mut best: Option<(std::time::SystemTime, u64)> = None; + let entries = fs::read_dir(&pma_dir).ok()?; + for entry in entries.flatten() { + let path = entry.path(); + if path.extension().and_then(|s| s.to_str()) != Some("mmap") { + continue; + } + let meta = entry.metadata().ok()?; + let mtime = meta.modified().unwrap_or(std::time::SystemTime::UNIX_EPOCH); + let alloc = read_pma_alloc_bytes(&path)?; + let update = match best { + None => true, + Some((best_mtime, _)) => mtime >= best_mtime, + }; + if update { + best = Some((mtime, alloc)); + } + } + best.map(|(_, alloc)| alloc) +} + +fn read_pma_alloc_bytes(path: &Path) -> Option { + const PMA_MAGIC: u64 = u64::from_le_bytes(*b"NOCKPMA1"); + const PMA_VERSION: u64 = 1; + const PMA_TRAILER_BYTES: usize = 32; + let mut file = File::open(path).ok()?; + let len = file.metadata().ok()?.len(); + if len < PMA_TRAILER_BYTES as u64 { + return None; + } + use std::io::{Read, Seek, SeekFrom}; + file.seek(SeekFrom::End(-(PMA_TRAILER_BYTES as i64))).ok()?; + let mut buf = [0u8; PMA_TRAILER_BYTES]; + file.read_exact(&mut buf).ok()?; + let magic = u64::from_le_bytes(buf[0..8].try_into().ok()?); + let version = u64::from_le_bytes(buf[8..16].try_into().ok()?); + if magic != PMA_MAGIC || version != PMA_VERSION { + return None; + } + let alloc_offset_words = u64::from_le_bytes(buf[24..32].try_into().ok()?); + Some(alloc_offset_words.saturating_mul(8)) +} + +fn checkpoint_info(info: &ProcInfo, data_dir_override: Option) -> Option { + let data_dir = resolve_data_dir(info, data_dir_override)?; + let checkpoints_dir = data_dir.join("checkpoints"); + if !checkpoints_dir.is_dir() { + return None; + } + let mut file_count = 0u64; + let mut total_bytes = 0u64; + let mut latest_path: Option = None; + let mut latest_mtime = std::time::SystemTime::UNIX_EPOCH; + let mut latest_bytes = 0u64; + + let entries = fs::read_dir(&checkpoints_dir).ok()?; + for entry in entries.flatten() { + let path = entry.path(); + let meta = match entry.metadata() { + Ok(meta) => meta, + Err(_) => continue, + }; + if !meta.is_file() { + continue; + } + file_count += 1; + let bytes = meta.len(); + total_bytes = total_bytes.saturating_add(bytes); + let mtime = meta.modified().unwrap_or(std::time::SystemTime::UNIX_EPOCH); + if latest_path.is_none() || mtime >= latest_mtime { + latest_path = Some(path); + latest_mtime = mtime; + latest_bytes = bytes; + } + } + + Some(CheckpointInfo { + checkpoints_dir, + file_count, + total_bytes, + latest_path, + latest_bytes, + }) +} + +fn resolve_data_dir(info: &ProcInfo, data_dir_override: Option) -> Option { + let base = if let Some(override_dir) = data_dir_override { + Some(override_dir) + } else { + parse_data_dir_flag(&info.cmdline).or_else(|| default_data_dir_from_cwd(info)) + }; + + if base.is_none() { + if is_container_process(info) { + return probe_container_data_dir(info); + } + return None; + } + let base = base?; + + if base.is_absolute() { + let rooted = proc_root(info).join(base.strip_prefix("/").unwrap_or(&base)); + if (is_container_process(info) && rooted.exists()) || (!base.exists() && rooted.exists()) { + return Some(rooted); + } + if is_container_process(info) { + if let Some(mapped) = map_container_path_to_host(info, &base) { + if mapped.exists() { + return Some(mapped); + } + } + } + return Some(base); + } + info.cwd.as_ref().map(|cwd| cwd.join(base)) +} + +fn probe_container_data_dir(info: &ProcInfo) -> Option { + let root = proc_root(info); + let candidates = [ + PathBuf::from("/data/.data.nockchain"), + PathBuf::from("/.data.nockchain"), + PathBuf::from("/root/.data.nockchain"), + ]; + for path in candidates { + let rooted = root.join(path.strip_prefix("/").unwrap_or(&path)); + if rooted.is_dir() { + return Some(rooted); + } + if let Some(mapped) = map_container_path_to_host(info, &path) { + if mapped.is_dir() { + return Some(mapped); + } + } + } + None +} + +fn map_container_path_to_host(info: &ProcInfo, container_path: &Path) -> Option { + let mounts_path = PathBuf::from("/proc") + .join(info.pid.to_string()) + .join("mounts"); + let mounts = read_to_string(&mounts_path)?; + let mut best: Option<(usize, PathBuf)> = None; + for line in mounts.lines() { + let mut parts = line.split_whitespace(); + let source = parts.next()?; + let mount_point = parts.next()?; + let mount_point = unescape_mount_field(mount_point); + let source = unescape_mount_field(source); + let mount_point_path = Path::new(&mount_point); + if container_path.starts_with(mount_point_path) { + let rel = container_path.strip_prefix(mount_point_path).unwrap_or(Path::new("")); + let host_path = Path::new(&source).join(rel); + let len = mount_point.len(); + let update = match &best { + None => true, + Some((best_len, _)) => len > *best_len, + }; + if update { + best = Some((len, host_path)); + } + } + } + best.map(|(_, path)| path) +} + +fn unescape_mount_field(input: &str) -> String { + let mut out = String::with_capacity(input.len()); + let mut chars = input.chars().peekable(); + while let Some(ch) = chars.next() { + if ch != '\\' { + out.push(ch); + continue; + } + let mut code = String::new(); + for _ in 0..3 { + match chars.peek() { + Some(c) if c.is_ascii_digit() => { + code.push(*c); + chars.next(); + } + _ => break, + } + } + if code.len() == 3 { + if let Ok(oct) = u8::from_str_radix(&code, 8) { + out.push(oct as char); + continue; + } + } + out.push('\\'); + out.push_str(&code); + } + out +} + +fn proc_root(info: &ProcInfo) -> PathBuf { + PathBuf::from("/proc") + .join(info.pid.to_string()) + .join("root") +} + +fn parse_data_dir_flag(cmdline: &str) -> Option { + let mut iter = cmdline.split_whitespace(); + while let Some(arg) = iter.next() { + if arg == "--data-dir" { + if let Some(value) = iter.next() { + return Some(PathBuf::from(value)); + } + } else if let Some(value) = arg.strip_prefix("--data-dir=") { + return Some(PathBuf::from(value)); + } + } + None +} + +fn default_data_dir_from_cwd(info: &ProcInfo) -> Option { + let cwd = info.cwd.as_ref()?; + Some(cwd.join(".data.nockchain")) +} + +fn parse_kb_line(line: &str) -> Option<(&str, u64)> { + let mut parts = line.split_whitespace(); + let key = parts.next()?; + let val = parts.next()?.parse::().ok()?; + Some((key.trim_end_matches(':'), val)) +} + +fn assign_mem_total(totals: &mut MemTotals, key: &str, val: u64) { + match key { + "Size" => totals.size_kb += val, + "Rss" => totals.rss_kb += val, + "Pss" => totals.pss_kb += val, + "Shared_Clean" => totals.shared_clean_kb += val, + "Shared_Dirty" => totals.shared_dirty_kb += val, + "Private_Clean" => totals.private_clean_kb += val, + "Private_Dirty" => totals.private_dirty_kb += val, + "Swap" => totals.swap_kb += val, + "SwapPss" => totals.swap_pss_kb += val, + _ => {} + } +} + +fn print_proc(info: &ProcInfo, report: &ProcMemReport) { + println!(" pid: {}", info.pid); + println!(" cmdline: {}", info.cmdline); + if let Some(exe) = &info.exe { + println!(" exe: {}", exe.display()); + } + if let Some(cwd) = &info.cwd { + println!(" cwd: {}", cwd.display()); + } + println!( + " status: VmRSS={} VmSize={} VmSwap={} RssAnon={} RssFile={} RssShmem={}", + fmt_mib(report.status.vm_rss_kb), + fmt_mib(report.status.vm_size_kb), + fmt_mib(report.status.vm_swap_kb), + fmt_mib(report.status.rss_anon_kb), + fmt_mib(report.status.rss_file_kb), + fmt_mib(report.status.rss_shmem_kb), + ); + if let Some(rollup) = &report.smaps_rollup { + println!( + " smaps_rollup: Size={} Rss={} Pss={} Private={} Shared={} Swap={}", + fmt_mib(rollup.size_kb), + fmt_mib(rollup.rss_kb), + fmt_mib(rollup.pss_kb), + fmt_mib(rollup.private_clean_kb + rollup.private_dirty_kb), + fmt_mib(rollup.shared_clean_kb + rollup.shared_dirty_kb), + fmt_mib(rollup.swap_kb), + ); + } else { + println!(" smaps_rollup: unavailable"); + } + if !report.pma_smaps_available { + println!(" pma_maps: unavailable (permission denied)"); + } else if report.pma_map_count > 0 { + let ratio = rss_ratio_str(report.pma_maps.rss_kb, report.pma_maps.size_kb); + let alloc = report + .pma_alloc_bytes + .map(fmt_bytes) + .unwrap_or_else(|| "unknown".to_string()); + println!( + " pma_maps: count={} Size={} Rss={} Pss={} Private={} Shared={} Swap={} rss_ratio={} alloc_offset={}", + report.pma_map_count, + fmt_mib(report.pma_maps.size_kb), + fmt_mib(report.pma_maps.rss_kb), + fmt_mib(report.pma_maps.pss_kb), + fmt_mib(report.pma_maps.private_clean_kb + report.pma_maps.private_dirty_kb), + fmt_mib(report.pma_maps.shared_clean_kb + report.pma_maps.shared_dirty_kb), + fmt_mib(report.pma_maps.swap_kb), + ratio, + alloc, + ); + } else { + println!(" pma_maps: none detected"); + } + + if let Some(checkpoint) = &report.checkpoint { + let latest_name = checkpoint + .latest_path + .as_ref() + .and_then(|p| p.file_name()) + .map(|p| p.to_string_lossy().to_string()) + .unwrap_or_else(|| "".to_string()); + println!( + " checkpoints: dir={} files={} total={} latest={} ({})", + checkpoint.checkpoints_dir.display(), + checkpoint.file_count, + fmt_bytes(checkpoint.total_bytes), + fmt_bytes(checkpoint.latest_bytes), + latest_name + ); + } else { + println!(" checkpoints: unavailable"); + } +} + +fn print_comparison(pma: &ProcMemReport, base: &ProcMemReport) { + println!("Comparison:"); + struct Row { + label: String, + pma: String, + base: String, + delta: String, + } + let mut rows = Vec::new(); + rows.push(Row { + label: "VmRSS".to_string(), + pma: fmt_mib(pma.status.vm_rss_kb), + base: fmt_mib(base.status.vm_rss_kb), + delta: fmt_signed_mib(delta_kb_signed( + pma.status.vm_rss_kb, + base.status.vm_rss_kb, + )), + }); + rows.push(Row { + label: "VmSize".to_string(), + pma: fmt_mib(pma.status.vm_size_kb), + base: fmt_mib(base.status.vm_size_kb), + delta: fmt_signed_mib(delta_kb_signed( + pma.status.vm_size_kb, + base.status.vm_size_kb, + )), + }); + rows.push(Row { + label: "RssAnon".to_string(), + pma: fmt_mib(pma.status.rss_anon_kb), + base: fmt_mib(base.status.rss_anon_kb), + delta: fmt_signed_mib(delta_kb_signed( + pma.status.rss_anon_kb, + base.status.rss_anon_kb, + )), + }); + rows.push(Row { + label: "RssFile".to_string(), + pma: fmt_mib(pma.status.rss_file_kb), + base: fmt_mib(base.status.rss_file_kb), + delta: fmt_signed_mib(delta_kb_signed( + pma.status.rss_file_kb, + base.status.rss_file_kb, + )), + }); + rows.push(Row { + label: "VmSwap".to_string(), + pma: fmt_mib(pma.status.vm_swap_kb), + base: fmt_mib(base.status.vm_swap_kb), + delta: fmt_signed_mib(delta_kb_signed( + pma.status.vm_swap_kb, + base.status.vm_swap_kb, + )), + }); + + if pma.pma_smaps_available && pma.pma_map_count > 0 { + rows.push(Row { + label: "PMA map size".to_string(), + pma: fmt_mib(pma.pma_maps.size_kb), + base: if base.pma_map_count > 0 { + fmt_mib(base.pma_maps.size_kb) + } else { + "n/a".to_string() + }, + delta: if base.pma_map_count > 0 { + fmt_signed_mib(delta_kb_signed( + pma.pma_maps.size_kb, + base.pma_maps.size_kb, + )) + } else { + "n/a".to_string() + }, + }); + rows.push(Row { + label: "PMA rss_ratio".to_string(), + pma: rss_ratio_str(pma.pma_maps.rss_kb, pma.pma_maps.size_kb), + base: if base.pma_map_count > 0 { + rss_ratio_str(base.pma_maps.rss_kb, base.pma_maps.size_kb) + } else { + "n/a".to_string() + }, + delta: "n/a".to_string(), + }); + } else if !pma.pma_smaps_available { + rows.push(Row { + label: "PMA map size".to_string(), + pma: "unavailable".to_string(), + base: "n/a".to_string(), + delta: "n/a".to_string(), + }); + rows.push(Row { + label: "PMA rss_ratio".to_string(), + pma: "unavailable".to_string(), + base: "n/a".to_string(), + delta: "n/a".to_string(), + }); + } + + let pma_alloc = pma + .pma_alloc_bytes + .map(fmt_bytes) + .unwrap_or_else(|| "unknown".to_string()); + let base_alloc = base + .pma_alloc_bytes + .map(fmt_bytes) + .unwrap_or_else(|| "n/a".to_string()); + rows.push(Row { + label: "PMA alloc_offset".to_string(), + pma: pma_alloc, + base: base_alloc, + delta: "n/a".to_string(), + }); + + match (&pma.checkpoint, &base.checkpoint) { + (Some(pma_ck), Some(base_ck)) => { + rows.push(Row { + label: "Checkpoint latest".to_string(), + pma: fmt_bytes(pma_ck.latest_bytes), + base: fmt_bytes(base_ck.latest_bytes), + delta: fmt_signed_bytes(delta_bytes_signed( + pma_ck.latest_bytes, + base_ck.latest_bytes, + )), + }); + rows.push(Row { + label: "Checkpoint total".to_string(), + pma: fmt_bytes(pma_ck.total_bytes), + base: fmt_bytes(base_ck.total_bytes), + delta: fmt_signed_bytes(delta_bytes_signed( + pma_ck.total_bytes, + base_ck.total_bytes, + )), + }); + } + _ => { + rows.push(Row { + label: "Checkpoint latest".to_string(), + pma: "n/a".to_string(), + base: "n/a".to_string(), + delta: "n/a".to_string(), + }); + rows.push(Row { + label: "Checkpoint total".to_string(), + pma: "n/a".to_string(), + base: "n/a".to_string(), + delta: "n/a".to_string(), + }); + } + } + + let header_label = "Metric"; + let header_pma = "PMA"; + let header_base = "Base"; + let header_delta = "PMA - base"; + let mut w_label = header_label.len(); + let mut w_pma = header_pma.len(); + let mut w_base = header_base.len(); + let mut w_delta = header_delta.len(); + for row in &rows { + w_label = w_label.max(row.label.len()); + w_pma = w_pma.max(row.pma.len()); + w_base = w_base.max(row.base.len()); + w_delta = w_delta.max(row.delta.len()); + } + + println!( + " {:w2$} {:>w3$} {:>w4$}", + header_label, + header_pma, + header_base, + header_delta, + w1 = w_label, + w2 = w_pma, + w3 = w_base, + w4 = w_delta + ); + println!( + " {:w2$} {:>w3$} {:>w4$}", + "-".repeat(w_label), + "-".repeat(w_pma), + "-".repeat(w_base), + "-".repeat(w_delta), + w1 = w_label, + w2 = w_pma, + w3 = w_base, + w4 = w_delta + ); + for row in rows { + println!( + " {:w2$} {:>w3$} {:>w4$}", + row.label, + row.pma, + row.base, + row.delta, + w1 = w_label, + w2 = w_pma, + w3 = w_base, + w4 = w_delta + ); + } +} + +fn print_summary( + pma_proc: &ProcInfo, + pma: &ProcMemReport, + base_proc: &ProcInfo, + base: &ProcMemReport, +) { + println!("Summary:"); + let pma_maps_ok = pma.pma_smaps_available && pma.pma_map_count > 0; + if !pma.pma_smaps_available { + println!( + " PMA mapping info unavailable for pid {} (permission denied reading smaps).", + pma_proc.pid + ); + println!( + " Notes: run as a user with access to /proc//smaps to see PMA RSS." + ); + } else if pma.pma_map_count == 0 { + println!( + " PMA mapping not detected for pid {}. PMA likely not enabled or mapping path unexpected.", + pma_proc.pid + ); + } else { + let pma_size_mib = kb_to_mib(pma.pma_maps.size_kb); + let pma_rss_mib = kb_to_mib(pma.pma_maps.rss_kb); + println!( + " PMA mapping size {:.1} MiB, RSS {:.1} MiB (ratio {}).", + pma_size_mib, + pma_rss_mib, + rss_ratio_str(pma.pma_maps.rss_kb, pma.pma_maps.size_kb) + ); + } + + let rss_delta = delta_kb_signed(pma.status.vm_rss_kb, base.status.vm_rss_kb); + println!( + " Total RSS delta (PMA - base): {}.", + fmt_signed_mib(rss_delta) + ); + + let verdict = if pma_maps_ok { + let pma_ratio = rss_ratio_value(pma.pma_maps.rss_kb, pma.pma_maps.size_kb); + let pma_size_mib = kb_to_mib(pma.pma_maps.size_kb); + let mut score = 0; + if pma_size_mib > 256.0 { + score += 1; + } + if pma_ratio < 0.9 { + score += 1; + } + if pma.status.rss_file_kb > base.status.rss_file_kb { + score += 1; + } + match score { + 3 => "likely", + 2 => "somewhat likely", + 1 => "inconclusive", + _ => "unlikely", + } + } else { + "inconclusive" + }; + + println!( + " Likelihood PMA paging is working correctly: {}.", + verdict + ); + if pma_maps_ok { + println!( + " Notes: PMA paging is best-effort. If ratio ~= 1.0, the PMA may be small/hot or no memory pressure." + ); + } + println!( + " Processes: PMA pid {} vs base pid {}.", + pma_proc.pid, base_proc.pid + ); +} + +fn fmt_mib(kb: u64) -> String { + format!("{:.1} MiB", kb_to_mib(kb)) +} + +fn fmt_bytes(bytes: u64) -> String { + let mib = (bytes as f64) / (1024.0 * 1024.0); + if mib >= 1024.0 { + format!("{:.2} GiB", mib / 1024.0) + } else { + format!("{:.1} MiB", mib) + } +} + +fn kb_to_mib(kb: u64) -> f64 { + (kb as f64) / 1024.0 +} + +fn delta_kb_signed(a: u64, b: u64) -> i64 { + (a as i64) - (b as i64) +} + +fn delta_bytes_signed(a: u64, b: u64) -> i64 { + (a as i64) - (b as i64) +} + +fn fmt_signed_mib(kb_delta: i64) -> String { + let sign = if kb_delta < 0 { "-" } else { "+" }; + let value = (kb_delta.abs() as f64) / 1024.0; + format!("{}{:.*} MiB", sign, 1, value) +} + +fn fmt_signed_bytes(bytes_delta: i64) -> String { + let sign = if bytes_delta < 0 { "-" } else { "+" }; + let value = bytes_delta.abs() as u64; + format!("{}{}", sign, fmt_bytes(value)) +} + +fn rss_ratio_str(rss_kb: u64, size_kb: u64) -> String { + if size_kb == 0 { + return "n/a".to_string(); + } + format!("{:.3}", (rss_kb as f64) / (size_kb as f64)) +} + +fn rss_ratio_value(rss_kb: u64, size_kb: u64) -> f64 { + if size_kb == 0 { + return 0.0; + } + (rss_kb as f64) / (size_kb as f64) +} diff --git a/scripts/poke-times.sh b/scripts/poke-times.sh new file mode 100755 index 000000000..25380047e --- /dev/null +++ b/scripts/poke-times.sh @@ -0,0 +1,44 @@ +#!/usr/bin/env bash +set -euo pipefail + +DB="${1:-.data.nockchain-sync-fsync-on/event-log.sqlite3}" + +if [ ! -f "$DB" ]; then + echo "Event log not found: $DB" + exit 1 +fi + +cat <<'SQL' | sqlite3 -readonly "$DB" +.mode column +.headers on + +SELECT '=== Last 100 Pokes ===' AS ''; +SELECT + event_num, + wire_source, + wire_tags_json AS tags, + printf('%.1f ms', event_processing_duration_us / 1000.0) AS duration, + datetime(created_at_ms / 1000, 'unixepoch', 'localtime') AS created_at +FROM events +ORDER BY event_num DESC +LIMIT 100; + +SELECT '=== Duration Stats (last 100) ===' AS ''; +SELECT + printf('%.1f ms', min(event_processing_duration_us) / 1000.0) AS min_dur, + printf('%.1f ms', avg(event_processing_duration_us) / 1000.0) AS avg_dur, + printf('%.1f ms', max(event_processing_duration_us) / 1000.0) AS max_dur, + printf('%.1f ms', sum(event_processing_duration_us) / 1000.0) AS total_dur +FROM (SELECT event_processing_duration_us FROM events ORDER BY event_num DESC LIMIT 100); + +SELECT '=== Duration by Source (last 100) ===' AS ''; +SELECT + wire_source, + count(*) AS count, + printf('%.1f ms', min(event_processing_duration_us) / 1000.0) AS min_dur, + printf('%.1f ms', avg(event_processing_duration_us) / 1000.0) AS avg_dur, + printf('%.1f ms', max(event_processing_duration_us) / 1000.0) AS max_dur +FROM (SELECT * FROM events ORDER BY event_num DESC LIMIT 100) +GROUP BY wire_source +ORDER BY count DESC; +SQL diff --git a/scripts/watch-event-log.sh b/scripts/watch-event-log.sh new file mode 100755 index 000000000..0ac658245 --- /dev/null +++ b/scripts/watch-event-log.sh @@ -0,0 +1,46 @@ +#!/usr/bin/env bash +set -euo pipefail + +DB="${1:-.data.nockchain-sync-fsync-on/event-log.sqlite3}" + +if [ ! -f "$DB" ]; then + echo "Event log not found: $DB" + exit 1 +fi + +Q=$(cat <<'SQL' +.mode column +.headers on + +SELECT '=== Event Summary ===' AS ''; +SELECT + count(*) AS total_events, + min(event_num) AS first_event, + max(event_num) AS latest_event +FROM events; + +SELECT '=== Recent Events ===' AS ''; +SELECT + event_num, + wire_source, + printf('%.1f ms', event_processing_duration_us / 1000.0) AS duration, + datetime(created_at_ms / 1000, 'unixepoch', 'localtime') AS created_at +FROM events +ORDER BY event_num DESC +LIMIT 10; + +SELECT '=== Snapshots ===' AS ''; +SELECT + snapshot_id, + kind, + state, + event_num, + datetime(created_at_ms / 1000, 'unixepoch', 'localtime') AS created_at, + timestamp_tag +FROM snapshots +ORDER BY snapshot_id DESC +LIMIT 10; +SQL +) + +echo "$Q" | sqlite3 -readonly "$DB" diff --git a/telegraf.conf b/telegraf.conf new file mode 100644 index 000000000..9685f8e37 --- /dev/null +++ b/telegraf.conf @@ -0,0 +1,20 @@ +[agent] + interval = "10s" + round_interval = true + flush_interval = "10s" + flush_jitter = "1s" + metric_batch_size = 1000 + metric_buffer_limit = 10000 + omit_hostname = true + +[[inputs.statsd]] + protocol = "udp" + service_address = ":8125" + parse_data_dog_tags = true + datadog_extensions = true + +[[outputs.influxdb_v2]] + urls = ["http://influxdb:8086"] + token = "$INFLUX_TOKEN" + organization = "$INFLUX_ORG" + bucket = "$INFLUX_BUCKET"