feat: tiered data storage

.github/workflows/audit.yml

@@ -19,8 +19,8 @@ jobs:
     runs-on: ${{ matrix.platform }}
     steps:
       - name: Checkout source code
-        uses: actions/checkout@v3
+        uses: actions/checkout@v6
       - name: Run security audit
-        uses: rustsec/audit-check@v1.4.1
+        uses: rustsec/audit-check@v2
         with:
           token: ${{ secrets.GITHUB_TOKEN }}

.github/workflows/benchmarks.yml

@@ -13,20 +13,20 @@ jobs:
       TOOLCHAIN: stable
     steps:
       - name: Checkout source code
-        uses: actions/checkout@v3
+        uses: actions/checkout@v6
       - name: Install Rust toolchain
         run: |
           curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y --profile=minimal --default-toolchain stable
           rustup override set stable
       - name: Enable caching for bitcoind
         id: cache-bitcoind
-        uses: actions/cache@v4
+        uses: actions/cache@v5
         with:
           path: bin/bitcoind-${{ runner.os }}-${{ runner.arch }}
           key: bitcoind-29.0-${{ runner.os }}-${{ runner.arch }}
       - name: Enable caching for electrs
         id: cache-electrs
-        uses: actions/cache@v4
+        uses: actions/cache@v5
         with:
           path: bin/electrs-${{ runner.os }}-${{ runner.arch }}
           key: electrs-${{ runner.os }}-${{ runner.arch }}

.github/workflows/cln-integration.yml

@@ -11,20 +11,43 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Checkout repository
-        uses: actions/checkout@v4
+        uses: actions/checkout@v6
 
-      - name: Install dependencies
-        run: |
-          sudo apt-get update -y
-          sudo apt-get install -y socat
+      - name: Create temporary directory for CLN data
+        run: echo "CLN_DATA_DIR=$(mktemp -d)" >> $GITHUB_ENV
 
       - name: Start bitcoind, electrs, and lightningd
         run: docker compose -p ldk-node -f tests/docker/docker-compose-cln.yml up -d
+        env:
+          CLN_DATA_DIR: ${{ env.CLN_DATA_DIR }}
+
+      - name: Wait for CLN to be ready
+        run: |
+          for i in $(seq 1 30); do
+            if docker exec ldk-node-cln-1 lightning-cli --regtest getinfo 2>/dev/null | grep -q '"id"'; then
+              echo "CLN is ready"
+              break
+            fi
+            echo "Waiting for CLN... ($i/30)"
+            sleep 2
+          done
+          docker exec ldk-node-cln-1 lightning-cli --regtest getinfo || {
+            echo "ERROR: CLN not responding"
+            docker compose -p ldk-node -f tests/docker/docker-compose-cln.yml logs cln
+            exit 1
+          }
 
-      - name: Forward lightningd RPC socket
+      - name: Set permissions for CLN data directory
         run: |
-          docker exec ldk-node-cln-1 sh -c "socat -d -d TCP-LISTEN:9937,fork,reuseaddr UNIX-CONNECT:/root/.lightning/regtest/lightning-rpc&"
-          socat -d -d UNIX-LISTEN:/tmp/lightning-rpc,reuseaddr,fork TCP:127.0.0.1:9937&
+          sudo chown -R $(id -u):$(id -g) $CLN_DATA_DIR
+          sudo chmod 755 $CLN_DATA_DIR
+          sudo find $CLN_DATA_DIR -type d -exec chmod 755 {} +
+          sudo find $CLN_DATA_DIR -type f -exec chmod 644 {} +
+        env:
+          CLN_DATA_DIR: ${{ env.CLN_DATA_DIR }}
 
       - name: Run CLN integration tests
-        run: RUSTFLAGS="--cfg cln_test" cargo test --test integration_tests_cln
+        run: CLN_SOCKET_PATH=$CLN_DATA_DIR/regtest/lightning-rpc
+          RUSTFLAGS="--cfg cln_test" cargo test --test integration_tests_cln -- --show-output --test-threads=1
+        env:
+          CLN_DATA_DIR: ${{ env.CLN_DATA_DIR }}

.github/workflows/cron-weekly-rustfmt.yml

@@ -13,7 +13,7 @@ jobs:
     name: Nightly rustfmt
     runs-on: ubuntu-24.04
     steps:
-      - uses: actions/checkout@v5
+      - uses: actions/checkout@v6
       - uses: dtolnay/rust-toolchain@nightly
         with:
           components: rustfmt
@@ -23,7 +23,7 @@ jobs:
       - name: Get the current date
         run: echo "date=$(date +'%Y-%m-%d')" >> $GITHUB_ENV
       - name: Create Pull Request
-        uses: peter-evans/create-pull-request@v7
+        uses: peter-evans/create-pull-request@v8
         with:
           author: Fmt Bot <bot@example.com>
           title: Automated nightly rustfmt (${{ env.date }})

.github/workflows/eclair-integration.yml

@@ -0,0 +1,56 @@
+name: CI Checks - Eclair Integration Tests
+
+on: [push, pull_request]
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  check-eclair:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+
+      - name: Start bitcoind and electrs
+        run: docker compose -p ldk-node -f tests/docker/docker-compose-eclair.yml up -d bitcoin electrs
+
+      - name: Wait for bitcoind to be healthy
+        run: |
+          for i in $(seq 1 30); do
+            if docker compose -p ldk-node -f tests/docker/docker-compose-eclair.yml exec bitcoin bitcoin-cli -regtest -rpcuser=user -rpcpassword=pass getblockchaininfo > /dev/null 2>&1; then
+              echo "bitcoind is ready"
+              exit 0
+            fi
+            echo "Waiting for bitcoind... ($i/30)"
+            sleep 2
+          done
+          echo "ERROR: bitcoind not ready"
+          exit 1
+
+      - name: Create wallets on bitcoind
+        run: |
+          docker compose -p ldk-node -f tests/docker/docker-compose-eclair.yml exec bitcoin bitcoin-cli -regtest -rpcuser=user -rpcpassword=pass createwallet eclair
+          docker compose -p ldk-node -f tests/docker/docker-compose-eclair.yml exec bitcoin bitcoin-cli -regtest -rpcuser=user -rpcpassword=pass -rpcwallet=eclair getnewaddress
+          docker compose -p ldk-node -f tests/docker/docker-compose-eclair.yml exec bitcoin bitcoin-cli -regtest -rpcuser=user -rpcpassword=pass createwallet ldk_node_test
+
+      - name: Start Eclair
+        run: docker compose -p ldk-node -f tests/docker/docker-compose-eclair.yml up -d eclair
+
+      - name: Wait for Eclair to be ready
+        run: |
+          for i in $(seq 1 60); do
+            if curl -sf -u :eclairpassword -X POST http://127.0.0.1:8080/getinfo > /dev/null 2>&1; then
+              echo "Eclair is ready"
+              exit 0
+            fi
+            echo "Waiting for Eclair... ($i/60)"
+            sleep 5
+          done
+          echo "Eclair failed to start"
+          docker compose -p ldk-node -f tests/docker/docker-compose-eclair.yml logs eclair
+          exit 1
+
+      - name: Run Eclair integration tests
+        run: RUSTFLAGS="--cfg eclair_test" cargo test --test integration_tests_eclair -- --show-output --test-threads=1

.github/workflows/hrn-integration.yml

@@ -0,0 +1,45 @@
+name: CI Checks - HRN Integration Tests
+
+on: [push, pull_request]
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  build-and-test:
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Checkout source code
+        uses: actions/checkout@v3
+      - name: Install Rust stable toolchain
+        run: |
+          curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y --profile=minimal --default-toolchain stable
+      - name: Enable caching for bitcoind
+        id: cache-bitcoind
+        uses: actions/cache@v4
+        with:
+          path: bin/bitcoind-${{ runner.os }}-${{ runner.arch }}
+          key: bitcoind-29.0-${{ runner.os }}-${{ runner.arch }}
+      - name: Enable caching for electrs
+        id: cache-electrs
+        uses: actions/cache@v4
+        with:
+          path: bin/electrs-${{ runner.os }}-${{ runner.arch }}
+          key: electrs-${{ runner.os }}-${{ runner.arch }}
+      - name: Download bitcoind/electrs
+        if: "steps.cache-bitcoind.outputs.cache-hit != 'true' || steps.cache-electrs.outputs.cache-hit != 'true'"
+        run: |
+          source ./scripts/download_bitcoind_electrs.sh
+          mkdir -p bin
+          mv "$BITCOIND_EXE" bin/bitcoind-${{ runner.os }}-${{ runner.arch }}
+          mv "$ELECTRS_EXE" bin/electrs-${{ runner.os }}-${{ runner.arch }}
+      - name: Set bitcoind/electrs environment variables
+        run: |
+          echo "BITCOIND_EXE=$( pwd )/bin/bitcoind-${{ runner.os }}-${{ runner.arch }}" >> "$GITHUB_ENV"
+          echo "ELECTRS_EXE=$( pwd )/bin/electrs-${{ runner.os }}-${{ runner.arch }}" >> "$GITHUB_ENV"
+      - name: Run HRN Integration Tests
+        run: |
+          RUSTFLAGS="--cfg no_download --cfg hrn_tests $RUSTFLAGS" cargo test --test integration_tests_hrn
+          RUSTFLAGS="--cfg no_download --cfg hrn_tests $RUSTFLAGS" cargo test --test integration_tests_hrn --features uniffi

.github/workflows/kotlin.yml

@@ -16,10 +16,10 @@ jobs:
 
     steps:
       - name: Checkout repository
-        uses: actions/checkout@v4
+        uses: actions/checkout@v6
 
       - name: Set up JDK
-        uses: actions/setup-java@v3
+        uses: actions/setup-java@v5
         with:
           distribution: temurin
           java-version: 11
@@ -47,7 +47,7 @@ jobs:
         run: ./scripts/uniffi_bindgen_generate_kotlin_android.sh
 
       - name: Start bitcoind and electrs
-        run: docker compose up -d
+        run: docker compose -p ldk-node -f tests/docker/docker-compose.yml up -d
 
       - name: Run ldk-node-jvm tests
         run: |

.github/workflows/lnd-integration.yml

@@ -11,7 +11,7 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Checkout repository
-        uses: actions/checkout@v4
+        uses: actions/checkout@v6
 
       - name: Check and install CMake if needed
         # lnd_grpc_rust (via prost-build v0.10.4) requires CMake >= 3.5 but is incompatible with CMake >= 4.0.
@@ -33,24 +33,31 @@ jobs:
           fi
 
       - name: Create temporary directory for LND data
-        id: create-temp-dir
         run: echo "LND_DATA_DIR=$(mktemp -d)" >> $GITHUB_ENV
 
       - name: Start bitcoind, electrs, and LND
         run: docker compose -p ldk-node -f tests/docker/docker-compose-lnd.yml up -d
         env:
           LND_DATA_DIR: ${{ env.LND_DATA_DIR }}
 
-      - name: Set permissions for LND data directory
-        # In PR 4622 (https://github.com/lightningnetwork/lnd/pull/4622),
-        # LND sets file permissions to 0700, preventing test code from accessing them.
-        # This step ensures the test suite has the necessary permissions.
-        run: sudo chmod -R 755 $LND_DATA_DIR
+      - name: Wait for LND macaroon and set permissions
+        run: |
+          for i in $(seq 1 30); do
+            if docker exec ldk-node-lnd test -f /root/.lnd/data/chain/bitcoin/regtest/admin.macaroon 2>/dev/null; then
+              echo "LND macaroon found"
+              break
+            fi
+            echo "Waiting for LND macaroon... ($i/30)"
+            sleep 2
+          done
+          sudo chmod 755 $LND_DATA_DIR
+          sudo find $LND_DATA_DIR -type d -exec chmod 755 {} +
+          sudo find $LND_DATA_DIR -type f -exec chmod 644 {} +
         env:
           LND_DATA_DIR: ${{ env.LND_DATA_DIR }}
 
       - name: Run LND integration tests
         run: LND_CERT_PATH=$LND_DATA_DIR/tls.cert LND_MACAROON_PATH=$LND_DATA_DIR/data/chain/bitcoin/regtest/admin.macaroon
-          RUSTFLAGS="--cfg lnd_test" cargo test --test integration_tests_lnd -- --exact --show-output
+          RUSTFLAGS="--cfg lnd_test" cargo test --test integration_tests_lnd -- --show-output --test-threads=1
         env:
           LND_DATA_DIR: ${{ env.LND_DATA_DIR }}

.github/workflows/python.yml

@@ -15,7 +15,7 @@ jobs:
 
     steps:
       - name: Checkout repository
-        uses: actions/checkout@v4
+        uses: actions/checkout@v6
 
       - name: Install uv
         uses: astral-sh/setup-uv@v7
@@ -24,7 +24,7 @@ jobs:
         run: ./scripts/uniffi_bindgen_generate_python.sh
 
       - name: Start bitcoind and electrs
-        run: docker compose up -d
+        run: docker compose -p ldk-node -f tests/docker/docker-compose.yml up -d
 
       - name: Run Python unit tests
         env:

.github/workflows/rust.yml

@@ -34,7 +34,7 @@ jobs:
     runs-on: ${{ matrix.platform }}
     steps:
       - name: Checkout source code
-        uses: actions/checkout@v3
+        uses: actions/checkout@v6
       - name: Install Rust ${{ matrix.toolchain }} toolchain
         run: |
           curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y --profile=minimal --default-toolchain ${{ matrix.toolchain }}
@@ -50,13 +50,13 @@ jobs:
         run: echo "RUSTFLAGS=-D warnings" >> "$GITHUB_ENV"
       - name: Enable caching for bitcoind
         id: cache-bitcoind
-        uses: actions/cache@v4
+        uses: actions/cache@v5
         with:
           path: bin/bitcoind-${{ runner.os }}-${{ runner.arch }}
           key: bitcoind-29.0-${{ runner.os }}-${{ runner.arch }}
       - name: Enable caching for electrs
         id: cache-electrs
-        uses: actions/cache@v4
+        uses: actions/cache@v5
         with:
           path: bin/electrs-${{ runner.os }}-${{ runner.arch }}
           key: electrs-${{ runner.os }}-${{ runner.arch }}
@@ -90,6 +90,21 @@ jobs:
         run: |
           RUSTFLAGS="--cfg no_download --cfg cycle_tests" cargo test --features uniffi
 
+  linting:
+    name: Linting
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout source code
+        uses: actions/checkout@v6
+      - name: Install Rust and clippy
+        run: |
+          curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y --profile=minimal --default-toolchain stable
+          rustup component add clippy
+      - name: Ban `unwrap` in library code
+        run: |
+          cargo clippy --lib --verbose --color always -- -A warnings -D clippy::unwrap_used -A clippy::tabs_in_doc_comments
+          cargo clippy --lib --features uniffi --verbose --color always -- -A warnings -D clippy::unwrap_used -A clippy::tabs_in_doc_comments
+
   doc:
     name: Documentation
     runs-on: ubuntu-latest
@@ -99,4 +114,4 @@ jobs:
       - uses: actions/checkout@v6
       - uses: dtolnay/rust-toolchain@nightly
       - uses: dtolnay/install@cargo-docs-rs
-      - run: cargo docs-rs
+      - run: cargo docs-rs

.github/workflows/semver.yml

@@ -6,6 +6,6 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Checkout source code
-        uses: actions/checkout@v4
+        uses: actions/checkout@v6
       - name: Check SemVer
         uses: obi1kenobi/cargo-semver-checks-action@v2

.github/workflows/swift.yml

@@ -12,7 +12,7 @@ jobs:
 
     steps:
       - name: Checkout repository
-        uses: actions/checkout@v4
+        uses: actions/checkout@v6
 
       - name: Set default Rust version to stable
         run: rustup default stable

.github/workflows/vss-integration.yml

@@ -27,11 +27,11 @@ jobs:
 
     steps:
       - name: Checkout code
-        uses: actions/checkout@v3
+        uses: actions/checkout@v6
         with:
           path: ldk-node
       - name: Checkout VSS
-        uses: actions/checkout@v3
+        uses: actions/checkout@v6
         with:
           repository: lightningdevkit/vss-server
           path: vss-server

.github/workflows/vss-no-auth-integration.yml

@@ -27,11 +27,11 @@ jobs:
 
     steps:
       - name: Checkout code
-        uses: actions/checkout@v3
+        uses: actions/checkout@v6
         with:
           path: ldk-node
       - name: Checkout VSS
-        uses: actions/checkout@v3
+        uses: actions/checkout@v6
         with:
           repository: lightningdevkit/vss-server
           path: vss-server