From 5f779882f97cef2a3a16d37350f0d298557f3806 Mon Sep 17 00:00:00 2001
From: Justin Frevert <justinfrevert@gmail.com>
Date: Wed, 1 Jul 2026 14:27:18 -0700
Subject: [PATCH 1/6] Debug runtime upgrade job

Signed-off-by: Justin Frevert <justinfrevert@gmail.com>
---
 .github/workflows/fork-network.yml            | 12 +++--
 .../changed/fork-network-ws-ipv4-connect.md   | 20 ++++++++
 .../src/lib/runtimeUpgradeUtils.ts            | 48 +++++++++++++++++--
 3 files changed, 72 insertions(+), 8 deletions(-)
 create mode 100644 changes/node/changed/fork-network-ws-ipv4-connect.md

diff --git a/.github/workflows/fork-network.yml b/.github/workflows/fork-network.yml
index 7add12815..ffb2a87d2 100644
--- a/.github/workflows/fork-network.yml
+++ b/.github/workflows/fork-network.yml
@@ -215,6 +215,10 @@ jobs:
       #
       # node1's container RPC port 9944 is published on host port 9950 for
       # every well-known network compose file, hence the --rpc-url override.
+      # Use 127.0.0.1, not localhost: under Node 17+ resolver order `localhost`
+      # resolves to ::1 first, and the runner's IPv6 loopback path to the
+      # published port is not routable, so the polkadot-js WsProvider (which has
+      # no connect timeout) hangs forever instead of falling back to IPv4.
       - name: Run runtime upgrade via governance
         if: inputs.upgrade_mode == 'runtime'
         working-directory: local-environment
@@ -229,7 +233,7 @@ jobs:
           npm run "governance-runtime-upgrade:${NETWORK}" -- \
             --from-snapshot "$SNAPSHOT_URL" \
             --wasm "$WASM_REL" \
-            --rpc-url ws://localhost:9950 \
+            --rpc-url ws://127.0.0.1:9950 \
             --council-uris //Dave //Eve //Ferdie \
             --technical-uris //Alice //Bob //Charlie \
             --executor-uri //Alice \
@@ -249,7 +253,7 @@ jobs:
           npm run "full-upgrade:${NETWORK}" -- \
             --from-snapshot "$SNAPSHOT_URL" \
             --wasm "$WASM_REL" \
-            --rpc-url ws://localhost:9950 \
+            --rpc-url ws://127.0.0.1:9950 \
             --council-uris //Dave //Eve //Ferdie \
             --technical-uris //Alice //Bob //Charlie \
             --executor-uri //Alice \
@@ -258,7 +262,7 @@ jobs:
       - name: Wait for block + finality progression
         timeout-minutes: 15
         env:
-          RPC_URL: "http://localhost:9950"
+          RPC_URL: "http://127.0.0.1:9950"
           REQUIRED_BLOCKS: "5"
         run: |
           get_tip() {
@@ -315,7 +319,7 @@ jobs:
       - name: Verify on-chain runtime matches candidate wasm
         if: inputs.upgrade_mode != 'image'
         env:
-          RPC_URL: "http://localhost:9950"
+          RPC_URL: "http://127.0.0.1:9950"
           WASM_REL: ${{ steps.wasm.outputs.wasm_rel }}
         run: |
           wasm_file="local-environment/artifacts/$WASM_REL"
diff --git a/changes/node/changed/fork-network-ws-ipv4-connect.md b/changes/node/changed/fork-network-ws-ipv4-connect.md
new file mode 100644
index 000000000..12d5b244c
--- /dev/null
+++ b/changes/node/changed/fork-network-ws-ipv4-connect.md
@@ -0,0 +1,20 @@
+# Fix fork-network runtime/full upgrade hang on WS connect to localhost
+
+The `fork-network` workflow's `runtime` and `full` upgrade modes could hang
+until the 45-minute job timeout at "Connecting to node at ws://localhost:9950".
+Under the Node 17+ resolver order `localhost` resolves to IPv6 `::1` first; when
+the runner's IPv6 loopback path to the published RPC port is not routable, the
+polkadot-js `WsProvider` (which has no connect timeout of its own) retries that
+address forever instead of falling back to IPv4. The forked chain is healthy the
+whole time — only the client cannot attach — so `image` mode, which never opens
+a `WsProvider`, was unaffected.
+
+The workflow now passes explicit IPv4 hosts (`127.0.0.1`) in the `--rpc-url`
+and curl `RPC_URL` values. In the local-environment tooling, `createApi` now
+fails fast with an actionable error after a bounded connect timeout
+(`API_CONNECT_TIMEOUT_MS`) instead of hanging indefinitely, and its
+`DEFAULT_RPC_URL` is switched to `127.0.0.1` so the default path cannot hit the
+same trap.
+
+PR:
+Issue:
diff --git a/local-environment/src/lib/runtimeUpgradeUtils.ts b/local-environment/src/lib/runtimeUpgradeUtils.ts
index 597865dfa..c4f63a936 100644
--- a/local-environment/src/lib/runtimeUpgradeUtils.ts
+++ b/local-environment/src/lib/runtimeUpgradeUtils.ts
@@ -21,7 +21,15 @@ import type { ISubmittableResult } from "@polkadot/types/types";
 import { u8aToHex } from "@polkadot/util";
 import { blake2AsU8a } from "@polkadot/util-crypto";
 
-export const DEFAULT_RPC_URL = "ws://localhost:9944";
+// Prefer an explicit IPv4 host over `localhost`: under the Node 17+ resolver
+// order `localhost` resolves to ::1 first, and if the IPv6 loopback path to the
+// node's published port is not routable the WsProvider (which has no connect
+// timeout of its own) retries forever instead of falling back to IPv4.
+export const DEFAULT_RPC_URL = "ws://127.0.0.1:9944";
+
+// Fail a stuck connection fast with an actionable error rather than letting the
+// WsProvider auto-reconnect indefinitely (which otherwise hangs the whole run).
+export const API_CONNECT_TIMEOUT_MS = 60_000;
 
 export interface WasmArtifact {
   path: string;
@@ -74,13 +82,45 @@ export function resolveRpcUrl(candidate?: string): string {
   return DEFAULT_RPC_URL;
 }
 
-export async function createApi(rpcUrl: string): Promise<{
+export async function createApi(
+  rpcUrl: string,
+  connectTimeoutMs: number = API_CONNECT_TIMEOUT_MS,
+): Promise<{
   api: ApiPromise;
   provider: WsProvider;
 }> {
   const provider = new WsProvider(rpcUrl);
-  const api = await ApiPromise.create({ provider });
-  return { api, provider };
+
+  let timer: ReturnType<typeof setTimeout> | undefined;
+  const timeout = new Promise<never>((_resolve, reject) => {
+    timer = setTimeout(() => {
+      reject(
+        new Error(
+          `Timed out after ${connectTimeoutMs}ms connecting to ${rpcUrl}. ` +
+            `Is the node RPC reachable there? If the URL uses 'localhost' it may ` +
+            `resolve to IPv6 (::1); try an explicit IPv4 host, e.g. ws://127.0.0.1:<port>.`,
+        ),
+      );
+    }, connectTimeoutMs);
+  });
+
+  try {
+    const api = await Promise.race([ApiPromise.create({ provider }), timeout]);
+    return { api, provider };
+  } catch (err) {
+    // Tear down the socket so a failed connect doesn't keep reconnecting in the
+    // background and hold the process open.
+    try {
+      await provider.disconnect();
+    } catch {
+      // best-effort cleanup
+    }
+    throw err;
+  } finally {
+    if (timer) {
+      clearTimeout(timer);
+    }
+  }
 }
 
 export async function disconnectApi(

From f398dd3cb8cbe031b48bd0d41c74412398c1e978 Mon Sep 17 00:00:00 2001
From: Justin Frevert <justinfrevert@gmail.com>
Date: Wed, 1 Jul 2026 15:29:00 -0700
Subject: [PATCH 2/6] Try fixing connection details

Signed-off-by: Justin Frevert <justinfrevert@gmail.com>
---
 .github/workflows/fork-network.yml            | 54 +++++++++++++++----
 .../src/lib/runtimeUpgradeUtils.ts            | 52 ++++++++++++++++++
 2 files changed, 96 insertions(+), 10 deletions(-)

diff --git a/.github/workflows/fork-network.yml b/.github/workflows/fork-network.yml
index ffb2a87d2..3b7b71a34 100644
--- a/.github/workflows/fork-network.yml
+++ b/.github/workflows/fork-network.yml
@@ -213,12 +213,13 @@ jobs:
       # authority observation source), so these dev-keyring URIs always meet
       # the 2/3 approval threshold on a freshly converted fork.
       #
-      # node1's container RPC port 9944 is published on host port 9950 for
-      # every well-known network compose file, hence the --rpc-url override.
-      # Use 127.0.0.1, not localhost: under Node 17+ resolver order `localhost`
-      # resolves to ::1 first, and the runner's IPv6 loopback path to the
-      # published port is not routable, so the polkadot-js WsProvider (which has
-      # no connect timeout) hangs forever instead of falling back to IPv4.
+      # node1's container RPC port 9944 is published on host port 9950, but on
+      # some self-hosted runners the host loopback -> docker-published-port path
+      # is black-holed (e.g. Docker started with userland-proxy disabled): the
+      # SYN is silently dropped, so the RPC client hangs. So bring the fork up
+      # first, then connect to whichever endpoint actually answers -- preferring
+      # the published port, but falling back to node1's docker bridge IP, which
+      # is routable from the runner regardless of the loopback/DNAT setup.
       - name: Run runtime upgrade via governance
         if: inputs.upgrade_mode == 'runtime'
         working-directory: local-environment
@@ -230,15 +231,44 @@ jobs:
           if [ "$ALLOW_SAME_VERSION" = "true" ]; then
             same_version_flag=(--allow-same-version)
           fi
+
+          # Bring the fork up first so we can probe for a reachable RPC endpoint.
+          npm run "run:${NETWORK}" -- --from-snapshot "$SNAPSHOT_URL"
+
+          node1_ip="$(docker inspect -f '{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' node1 | tr -d '[:space:]')"
+          echo "node1 bridge IP: ${node1_ip:-<none>}"
+          candidates="127.0.0.1:9950 [::1]:9950"
+          [ -n "$node1_ip" ] && candidates="$candidates ${node1_ip}:9944"
+          rpc_ws=""
+          for target in $candidates; do
+            if curl -sS --max-time 5 -o /dev/null \
+                 -H 'Content-Type: application/json' \
+                 -d '{"jsonrpc":"2.0","id":1,"method":"system_health","params":[]}' \
+                 "http://${target}"; then
+              echo "RPC reachable at http://${target}"
+              rpc_ws="ws://${target}"
+              echo "RPC_HTTP=http://${target}" >> "$GITHUB_ENV"
+              break
+            fi
+            echo "RPC NOT reachable at http://${target}"
+          done
+          [ -n "$rpc_ws" ] || { echo "No reachable RPC endpoint found" >&2; exit 1; }
+
           npm run "governance-runtime-upgrade:${NETWORK}" -- \
-            --from-snapshot "$SNAPSHOT_URL" \
+            --skip-run \
             --wasm "$WASM_REL" \
-            --rpc-url ws://127.0.0.1:9950 \
+            --rpc-url "$rpc_ws" \
             --council-uris //Dave //Eve //Ferdie \
             --technical-uris //Alice //Bob //Charlie \
             --executor-uri //Alice \
             "${same_version_flag[@]}"
 
+      # NOTE: full-upgrade brings the fork up itself (image rollout) and has no
+      # --skip-run, so we cannot pre-resolve node1's bridge IP the way the
+      # runtime step does. It therefore still relies on the host published port;
+      # on a runner where the loopback path is black-holed this will time out.
+      # Making full-upgrade robust needs a tooling-side change (resolve the
+      # node's reachable endpoint after its internal bring-up).
       - name: Run full-upgrade
         if: inputs.upgrade_mode == 'full'
         working-directory: local-environment
@@ -262,9 +292,12 @@ jobs:
       - name: Wait for block + finality progression
         timeout-minutes: 15
         env:
-          RPC_URL: "http://127.0.0.1:9950"
           REQUIRED_BLOCKS: "5"
         run: |
+          # Reuse the endpoint the upgrade step found reachable (published port
+          # or node1's bridge IP); fall back to the published port otherwise.
+          RPC_URL="${RPC_HTTP:-http://127.0.0.1:9950}"
+          echo "Using RPC endpoint: $RPC_URL"
           get_tip() {
             curl -sf -H 'Content-Type: application/json' \
               -d '{"jsonrpc":"2.0","id":1,"method":"chain_getHeader","params":[]}' \
@@ -319,9 +352,10 @@ jobs:
       - name: Verify on-chain runtime matches candidate wasm
         if: inputs.upgrade_mode != 'image'
         env:
-          RPC_URL: "http://127.0.0.1:9950"
           WASM_REL: ${{ steps.wasm.outputs.wasm_rel }}
         run: |
+          RPC_URL="${RPC_HTTP:-http://127.0.0.1:9950}"
+          echo "Using RPC endpoint: $RPC_URL"
           wasm_file="local-environment/artifacts/$WASM_REL"
           expected_hex=$(od -An -v -tx1 "$wasm_file" | tr -d ' \n')
           onchain_hex=$(curl -sf -H 'Content-Type: application/json' \
diff --git a/local-environment/src/lib/runtimeUpgradeUtils.ts b/local-environment/src/lib/runtimeUpgradeUtils.ts
index c4f63a936..52820cf5b 100644
--- a/local-environment/src/lib/runtimeUpgradeUtils.ts
+++ b/local-environment/src/lib/runtimeUpgradeUtils.ts
@@ -12,7 +12,9 @@
 // limitations under the License.
 
 import fs from "fs";
+import net from "net";
 import path from "path";
+import { promises as dnsPromises } from "dns";
 import { ApiPromise, WsProvider } from "@polkadot/api";
 import type { SubmittableExtrinsic } from "@polkadot/api/promise/types";
 import { Keyring } from "@polkadot/keyring";
@@ -82,6 +84,41 @@ export function resolveRpcUrl(candidate?: string): string {
   return DEFAULT_RPC_URL;
 }
 
+// Print the definite reason a connect failed: what the host resolved to (and in
+// what order) plus a raw per-address TCP probe. Distinguishes "IPv6 path dead,
+// IPv4 works" from "nothing is listening" without waiting out a job timeout.
+async function logConnectDiagnostics(rpcUrl: string): Promise<void> {
+  const url = new URL(rpcUrl);
+  const port = Number(url.port);
+  const addrs = await dnsPromises
+    .lookup(url.hostname, { all: true, verbatim: true })
+    .catch(() => [] as { address: string; family: number }[]);
+  if (addrs.length) {
+    console.error(
+      `DNS ${url.hostname} (verbatim order): ${addrs
+        .map((a) => `${a.address}(v${a.family})`)
+        .join(", ")}`,
+    );
+  }
+  const targets = addrs.length ? addrs.map((a) => a.address) : [url.hostname];
+  for (const address of targets) {
+    const result = await new Promise<string>((resolve) => {
+      const socket = net.connect({ host: address, port });
+      const done = (msg: string) => {
+        socket.destroy();
+        resolve(msg);
+      };
+      socket.setTimeout(4000);
+      socket.once("connect", () => done("OK"));
+      socket.once("timeout", () => done("TIMEOUT"));
+      socket.once("error", (e: NodeJS.ErrnoException) =>
+        done(`FAIL (${e.code ?? e.message})`),
+      );
+    });
+    console.error(`TCP ${address}:${port} -> ${result}`);
+  }
+}
+
 export async function createApi(
   rpcUrl: string,
   connectTimeoutMs: number = API_CONNECT_TIMEOUT_MS,
@@ -90,6 +127,18 @@ export async function createApi(
   provider: WsProvider;
 }> {
   const provider = new WsProvider(rpcUrl);
+  // Surface socket-level connect activity so a stuck connection is never silent.
+  provider.on("error", (e) =>
+    console.error(
+      `WsProvider error (${rpcUrl}): ${(e as Error)?.message ?? e}`,
+    ),
+  );
+  provider.on("disconnected", () =>
+    console.warn(`WsProvider disconnected (${rpcUrl})`),
+  );
+  provider.on("connected", () =>
+    console.log(`WsProvider connected (${rpcUrl})`),
+  );
 
   let timer: ReturnType<typeof setTimeout> | undefined;
   const timeout = new Promise<never>((_resolve, reject) => {
@@ -108,6 +157,9 @@ export async function createApi(
     const api = await Promise.race([ApiPromise.create({ provider }), timeout]);
     return { api, provider };
   } catch (err) {
+    await logConnectDiagnostics(rpcUrl).catch(() => {
+      // diagnostics are best-effort
+    });
     // Tear down the socket so a failed connect doesn't keep reconnecting in the
     // background and hold the process open.
     try {

From 4b7607cef1a0a1e16b10637659031377da243bda Mon Sep 17 00:00:00 2001
From: Justin Frevert <justinfrevert@gmail.com>
Date: Wed, 1 Jul 2026 15:47:15 -0700
Subject: [PATCH 3/6] runtime upgrade connection probe

Signed-off-by: Justin Frevert <justinfrevert@gmail.com>
---
 .github/workflows/fork-network.yml            | 43 +++++++++++++------
 .../changed/fork-network-ws-ipv4-connect.md   | 34 ++++++++-------
 2 files changed, 50 insertions(+), 27 deletions(-)

diff --git a/.github/workflows/fork-network.yml b/.github/workflows/fork-network.yml
index 3b7b71a34..ab8b0c9a6 100644
--- a/.github/workflows/fork-network.yml
+++ b/.github/workflows/fork-network.yml
@@ -239,20 +239,39 @@ jobs:
           echo "node1 bridge IP: ${node1_ip:-<none>}"
           candidates="127.0.0.1:9950 [::1]:9950"
           [ -n "$node1_ip" ] && candidates="$candidates ${node1_ip}:9944"
+
+          probe() {
+            curl -sS --max-time 5 -o /dev/null \
+              -H 'Content-Type: application/json' \
+              -d '{"jsonrpc":"2.0","id":1,"method":"system_health","params":[]}' \
+              "http://$1"
+          }
+
+          # `compose up -d` returns before the node binds its RPC port, so a
+          # single-shot probe races the node boot and loses. Poll all candidates
+          # until one answers system_health or the deadline hits.
           rpc_ws=""
-          for target in $candidates; do
-            if curl -sS --max-time 5 -o /dev/null \
-                 -H 'Content-Type: application/json' \
-                 -d '{"jsonrpc":"2.0","id":1,"method":"system_health","params":[]}' \
-                 "http://${target}"; then
-              echo "RPC reachable at http://${target}"
-              rpc_ws="ws://${target}"
-              echo "RPC_HTTP=http://${target}" >> "$GITHUB_ENV"
-              break
-            fi
-            echo "RPC NOT reachable at http://${target}"
+          deadline=$((SECONDS + 120))
+          while [ "$SECONDS" -lt "$deadline" ]; do
+            for target in $candidates; do
+              if probe "$target" 2>/dev/null; then
+                echo "RPC reachable at http://${target}"
+                rpc_ws="ws://${target}"
+                echo "RPC_HTTP=http://${target}" >> "$GITHUB_ENV"
+                break 2
+              fi
+            done
+            echo "RPC not ready yet (${SECONDS}s elapsed); retrying..."
+            sleep 3
           done
-          [ -n "$rpc_ws" ] || { echo "No reachable RPC endpoint found" >&2; exit 1; }
+          if [ -z "$rpc_ws" ]; then
+            echo "No reachable RPC endpoint found after ${SECONDS}s; per-target errors:" >&2
+            for target in $candidates; do
+              echo "--- http://${target}" >&2
+              probe "$target" || true
+            done
+            exit 1
+          fi
 
           npm run "governance-runtime-upgrade:${NETWORK}" -- \
             --skip-run \
diff --git a/changes/node/changed/fork-network-ws-ipv4-connect.md b/changes/node/changed/fork-network-ws-ipv4-connect.md
index 12d5b244c..129c19c07 100644
--- a/changes/node/changed/fork-network-ws-ipv4-connect.md
+++ b/changes/node/changed/fork-network-ws-ipv4-connect.md
@@ -1,20 +1,24 @@
-# Fix fork-network runtime/full upgrade hang on WS connect to localhost
+# Fix fork-network runtime upgrade hang connecting to the forked node
 
-The `fork-network` workflow's `runtime` and `full` upgrade modes could hang
-until the 45-minute job timeout at "Connecting to node at ws://localhost:9950".
-Under the Node 17+ resolver order `localhost` resolves to IPv6 `::1` first; when
-the runner's IPv6 loopback path to the published RPC port is not routable, the
-polkadot-js `WsProvider` (which has no connect timeout of its own) retries that
-address forever instead of falling back to IPv4. The forked chain is healthy the
-whole time — only the client cannot attach — so `image` mode, which never opens
-a `WsProvider`, was unaffected.
+The `fork-network` workflow's `runtime` upgrade mode could hang until the
+45-minute job timeout at "Connecting to node at ws://localhost:9950". The forked
+chain is healthy the whole time (all validators up, RPC bound, producing and
+finalizing blocks) — the problem is that on some self-hosted runners the host
+loopback -> docker-published-port path is black-holed (e.g. Docker started with
+`userland-proxy` disabled): the SYN is silently dropped, so the polkadot-js
+`WsProvider` — which has no connect timeout of its own — never establishes and
+never errors. `image` mode is unaffected because it never opens a `WsProvider`.
 
-The workflow now passes explicit IPv4 hosts (`127.0.0.1`) in the `--rpc-url`
-and curl `RPC_URL` values. In the local-environment tooling, `createApi` now
-fails fast with an actionable error after a bounded connect timeout
-(`API_CONNECT_TIMEOUT_MS`) instead of hanging indefinitely, and its
-`DEFAULT_RPC_URL` is switched to `127.0.0.1` so the default path cannot hit the
-same trap.
+The `runtime` step now brings the fork up first, then connects to whichever RPC
+endpoint actually answers — preferring the published port but falling back to
+node1's docker bridge IP, which is routable from the runner regardless of the
+loopback/DNAT setup — and reuses that endpoint for the finality-wait and
+`:code` verification steps. As defense-in-depth, `createApi` in the
+local-environment tooling now fails fast with an actionable error after a bounded
+connect timeout (`API_CONNECT_TIMEOUT_MS`) and logs per-address connectivity
+diagnostics, and its `DEFAULT_RPC_URL` uses an explicit IPv4 host. `full` mode
+brings the fork up internally (no `--skip-run`) so it still relies on the
+published port; making it robust needs a follow-up tooling change.
 
 PR:
 Issue:

From fa3ec5372aac8c279ba744a2b7aa44960cb0691b Mon Sep 17 00:00:00 2001
From: Justin Frevert <justinfrevert@gmail.com>
Date: Wed, 1 Jul 2026 16:28:02 -0700
Subject: [PATCH 4/6] remove logging

Signed-off-by: Justin Frevert <justinfrevert@gmail.com>
---
 .../changed/fork-network-ws-ipv4-connect.md   |  4 +-
 .../src/lib/runtimeUpgradeUtils.ts            | 52 -------------------
 2 files changed, 2 insertions(+), 54 deletions(-)

diff --git a/changes/node/changed/fork-network-ws-ipv4-connect.md b/changes/node/changed/fork-network-ws-ipv4-connect.md
index 129c19c07..7984a21c0 100644
--- a/changes/node/changed/fork-network-ws-ipv4-connect.md
+++ b/changes/node/changed/fork-network-ws-ipv4-connect.md
@@ -15,8 +15,8 @@ node1's docker bridge IP, which is routable from the runner regardless of the
 loopback/DNAT setup — and reuses that endpoint for the finality-wait and
 `:code` verification steps. As defense-in-depth, `createApi` in the
 local-environment tooling now fails fast with an actionable error after a bounded
-connect timeout (`API_CONNECT_TIMEOUT_MS`) and logs per-address connectivity
-diagnostics, and its `DEFAULT_RPC_URL` uses an explicit IPv4 host. `full` mode
+connect timeout (`API_CONNECT_TIMEOUT_MS`), and its `DEFAULT_RPC_URL` uses an
+explicit IPv4 host. `full` mode
 brings the fork up internally (no `--skip-run`) so it still relies on the
 published port; making it robust needs a follow-up tooling change.
 
diff --git a/local-environment/src/lib/runtimeUpgradeUtils.ts b/local-environment/src/lib/runtimeUpgradeUtils.ts
index 52820cf5b..c4f63a936 100644
--- a/local-environment/src/lib/runtimeUpgradeUtils.ts
+++ b/local-environment/src/lib/runtimeUpgradeUtils.ts
@@ -12,9 +12,7 @@
 // limitations under the License.
 
 import fs from "fs";
-import net from "net";
 import path from "path";
-import { promises as dnsPromises } from "dns";
 import { ApiPromise, WsProvider } from "@polkadot/api";
 import type { SubmittableExtrinsic } from "@polkadot/api/promise/types";
 import { Keyring } from "@polkadot/keyring";
@@ -84,41 +82,6 @@ export function resolveRpcUrl(candidate?: string): string {
   return DEFAULT_RPC_URL;
 }
 
-// Print the definite reason a connect failed: what the host resolved to (and in
-// what order) plus a raw per-address TCP probe. Distinguishes "IPv6 path dead,
-// IPv4 works" from "nothing is listening" without waiting out a job timeout.
-async function logConnectDiagnostics(rpcUrl: string): Promise<void> {
-  const url = new URL(rpcUrl);
-  const port = Number(url.port);
-  const addrs = await dnsPromises
-    .lookup(url.hostname, { all: true, verbatim: true })
-    .catch(() => [] as { address: string; family: number }[]);
-  if (addrs.length) {
-    console.error(
-      `DNS ${url.hostname} (verbatim order): ${addrs
-        .map((a) => `${a.address}(v${a.family})`)
-        .join(", ")}`,
-    );
-  }
-  const targets = addrs.length ? addrs.map((a) => a.address) : [url.hostname];
-  for (const address of targets) {
-    const result = await new Promise<string>((resolve) => {
-      const socket = net.connect({ host: address, port });
-      const done = (msg: string) => {
-        socket.destroy();
-        resolve(msg);
-      };
-      socket.setTimeout(4000);
-      socket.once("connect", () => done("OK"));
-      socket.once("timeout", () => done("TIMEOUT"));
-      socket.once("error", (e: NodeJS.ErrnoException) =>
-        done(`FAIL (${e.code ?? e.message})`),
-      );
-    });
-    console.error(`TCP ${address}:${port} -> ${result}`);
-  }
-}
-
 export async function createApi(
   rpcUrl: string,
   connectTimeoutMs: number = API_CONNECT_TIMEOUT_MS,
@@ -127,18 +90,6 @@ export async function createApi(
   provider: WsProvider;
 }> {
   const provider = new WsProvider(rpcUrl);
-  // Surface socket-level connect activity so a stuck connection is never silent.
-  provider.on("error", (e) =>
-    console.error(
-      `WsProvider error (${rpcUrl}): ${(e as Error)?.message ?? e}`,
-    ),
-  );
-  provider.on("disconnected", () =>
-    console.warn(`WsProvider disconnected (${rpcUrl})`),
-  );
-  provider.on("connected", () =>
-    console.log(`WsProvider connected (${rpcUrl})`),
-  );
 
   let timer: ReturnType<typeof setTimeout> | undefined;
   const timeout = new Promise<never>((_resolve, reject) => {
@@ -157,9 +108,6 @@ export async function createApi(
     const api = await Promise.race([ApiPromise.create({ provider }), timeout]);
     return { api, provider };
   } catch (err) {
-    await logConnectDiagnostics(rpcUrl).catch(() => {
-      // diagnostics are best-effort
-    });
     // Tear down the socket so a failed connect doesn't keep reconnecting in the
     // background and hold the process open.
     try {

From bec726a6507e7415c6ce3137c9b21693d962e01e Mon Sep 17 00:00:00 2001
From: Justin Frevert <justinfrevert@gmail.com>
Date: Wed, 1 Jul 2026 16:46:35 -0700
Subject: [PATCH 5/6] local env usability fixes

Signed-off-by: Justin Frevert <justinfrevert@gmail.com>
---
 .github/workflows/fork-network.yml           | 13 +++-
 changes/node/added/local-env-from-genesis.md | 17 +++++
 docs/fork-testing.md                         | 31 ++++++++
 local-environment/README.md                  | 21 ++++++
 local-environment/src/commands/run.ts        | 75 ++++++++++++++++++--
 local-environment/src/index.ts               |  6 +-
 local-environment/src/lib/types.ts           |  7 ++
 7 files changed, 161 insertions(+), 9 deletions(-)
 create mode 100644 changes/node/added/local-env-from-genesis.md

diff --git a/.github/workflows/fork-network.yml b/.github/workflows/fork-network.yml
index ab8b0c9a6..f6aacc2ee 100644
--- a/.github/workflows/fork-network.yml
+++ b/.github/workflows/fork-network.yml
@@ -15,6 +15,15 @@ name: fork-network
 #             federated-authority governance flow
 #   full    - production-shaped rehearsal: image rollout first, then the
 #             runtime upgrade via governance
+#
+# node_image / new_node_image take full Docker image references — anything
+# `docker pull` accepts. Git branches, commit shas, and repo URLs are NOT
+# accepted. Examples:
+#   ghcr.io/midnight-ntwrk/midnight-node:1.0.0                            (release tag)
+#   ghcr.io/midnight-ntwrk/midnight-node:1.0.1-dev-6587676a9bb2-amd64     (CI build of a commit)
+# CI tags a per-commit image <version>-dev-<12-char-tree-hash>-amd64; find it
+# in the commit's continuous-integration run (image_tags step) or compute the
+# hash with `git rev-parse HEAD^{tree} | cut -c1-12`.
 
 on:
   workflow_dispatch:
@@ -32,10 +41,10 @@ on:
         required: true
         default: "368fd98"
       node_image:
-        description: "midnight-node image to fork from (must match the snapshot's runtime host fns)"
+        description: "Full Docker image reference to fork from, e.g. ghcr.io/midnight-ntwrk/midnight-node:1.0.0 or a CI tag like ghcr.io/midnight-ntwrk/midnight-node:1.0.1-dev-<tree-hash>-amd64 — not a git branch/sha/URL. Must match the snapshot's runtime host fns"
         required: true
       new_node_image:
-        description: "midnight-node image to upgrade to (image rollout target and/or runtime wasm source, per upgrade_mode)"
+        description: "Full Docker image reference to upgrade to (image rollout target and/or runtime wasm source, per upgrade_mode). Same format as node_image"
         required: true
       upgrade_mode:
         description: "Upgrade shape: roll the node image, submit a runtime upgrade via governance, or both in sequence"
diff --git a/changes/node/added/local-env-from-genesis.md b/changes/node/added/local-env-from-genesis.md
new file mode 100644
index 000000000..06340ef1a
--- /dev/null
+++ b/changes/node/added/local-env-from-genesis.md
@@ -0,0 +1,17 @@
+# Restore from-genesis bring-up for well-known networks in local-env tooling
+
+`npm run run:<network> -- --from-genesis` brings up a well-known network's base
+compose from block 0 again, instead of requiring `--from-snapshot` (the genesis
+path was removed together with the internal k8s/AWS integration in #1470).
+Nothing is mocked in this mode: validator seed phrases and a main-chain data
+source must be supplied via `--env-file` or the environment, and the CLI warns
+up front about compose variables left unset and about pre-existing chain data
+directories.
+
+Also clarifies the fork-network workflow's `node_image`/`new_node_image` inputs
+(full Docker image references with the expected release and CI tag formats, not
+git branches/shas/URLs) and documents how to find snapshot archives for each
+network from the backup system's public index in `docs/fork-testing.md`.
+
+PR:
+Issue:
diff --git a/docs/fork-testing.md b/docs/fork-testing.md
index e32426956..1e7a3381f 100644
--- a/docs/fork-testing.md
+++ b/docs/fork-testing.md
@@ -23,6 +23,37 @@ Local restore requires:
 - `tar`
 - `zstd` for `.zst` archives
 
+## Finding snapshot archives
+
+The backup system publishes snapshots of the well-known networks behind a
+CloudFront distribution. The manifest at
+`https://dg39snjayoq3t.cloudfront.net/index.json` lists every available
+archive, keyed by network name. To see which networks currently have
+snapshots:
+
+```bash
+curl -fsSL https://dg39snjayoq3t.cloudfront.net/index.json | jq 'keys'
+```
+
+As of this writing the index contains `devnet`, `qanet`, and `govnet`.
+`preview`, `preprod`, and `mainnet` snapshots are not published there yet;
+ask the node team if you need one.
+
+To resolve the latest archive URL for a network (this mirrors what the
+`fork-network` workflow's "Resolve latest snapshot URL" step does):
+
+```bash
+NETWORK=devnet
+curl -fsSL https://dg39snjayoq3t.cloudfront.net/index.json |
+  jq -r --arg net "$NETWORK" '
+    .[$net] | to_entries | map(.value[]) | sort_by(.timestamp) | last
+    | .s3_path | sub("^s3://[^/]+/"; "https://dg39snjayoq3t.cloudfront.net/")'
+```
+
+Pass the resulting URL to `--from-snapshot`. Drop the `last` selection to list
+every archive for the network (older snapshots stay useful for forking a
+specific block height or runtime version).
+
 ## Initial restore
 
 On the first bring-up of a well-known network, pass the snapshot URL to `run`,
diff --git a/local-environment/README.md b/local-environment/README.md
index 4e8796ad6..756b19793 100644
--- a/local-environment/README.md
+++ b/local-environment/README.md
@@ -35,6 +35,10 @@ npm run run:preprod -- --from-snapshot https://example.com/snapshots/preprod-lat
 npm run run:mainnet -- --from-snapshot https://example.com/snapshots/mainnet-latest.tar.gz
 ```
 
+Real snapshot URLs for each network can be resolved from the backup system's
+public index — see
+[Finding snapshot archives](../docs/fork-testing.md#finding-snapshot-archives).
+
 After that initial restore, the same network can be restarted without
 `--from-snapshot` as long as the restored `data/` directories and generated
 mock-authorities output are still present:
@@ -48,6 +52,23 @@ image was built with the same `networkId` as the genesis used to produce the
 snapshot. Recent runtimes validate this at boot and the node will refuse to
 start on a mismatch.
 
+### Starting a well-known network from genesis
+
+`--from-genesis` skips the snapshot/fork flow entirely and brings up the
+network's base compose from block 0:
+
+```bash
+npm run run:devnet -- --from-genesis --env-file ../devnet.env
+```
+
+Unlike fork mode, nothing is mocked: each validator needs its real seed phrase
+(e.g. `MIDNIGHT_NODE_01_0_SEED`) and a reachable main-chain data source (the
+`DB_SYNC_POSTGRES_CONNECTION_STRING_NODE_*` vars) supplied via `--env-file` or
+the process environment. The CLI warns about any compose variables left unset,
+and about existing `data/` directories (nodes resume from existing chain data;
+wipe the network's `data/` directories first for a clean block-0 start).
+Restarting a genesis environment uses the same flag.
+
 ### Upgrade rehearsals
 
 `image-upgrade` rolls service containers from `NODE_IMAGE` to `NEW_NODE_IMAGE`.
diff --git a/local-environment/src/commands/run.ts b/local-environment/src/commands/run.ts
index b9d42ea79..d50deb359 100644
--- a/local-environment/src/commands/run.ts
+++ b/local-environment/src/commands/run.ts
@@ -42,7 +42,8 @@ import {
  * Bring up a network locally:
  * - "local-env" runs the bundled local Cardano/PC stack from compose.
  * - Any well-known network (devnet/qanet/...) is forked from the
- *   provided snapshot via mock-authorities — there is no k8s-backed path.
+ *   provided snapshot via mock-authorities, or brought up from genesis
+ *   with --from-genesis — there is no k8s-backed path.
  */
 export async function run(network: string, runOptions: RunOptions) {
   if (network === "local-env") {
@@ -52,15 +53,20 @@ export async function run(network: string, runOptions: RunOptions) {
   }
 
   assertWellKnownNamespace(network);
-  console.log(
-    `Preparing ${network} local fork (mock-authorities-driven bring-up)`,
-  );
+  if (runOptions.fromGenesis) {
+    console.log(`Bringing up ${network} from genesis (base compose, no fork)`);
+  } else {
+    console.log(
+      `Preparing ${network} local fork (mock-authorities-driven bring-up)`,
+    );
+  }
   await runWellKnownNetwork(network, runOptions);
 }
 
 async function runWellKnownNetwork(namespace: string, runOptions: RunOptions) {
-  const networkConfig = loadNetworkConfig(namespace);
-  const mock = requireMockConfig(namespace, networkConfig);
+  if (runOptions.fromGenesis && runOptions.fromSnapshot) {
+    throw new Error("--from-genesis and --from-snapshot are mutually exclusive.");
+  }
 
   const composeFile = resolveComposeFile(namespace);
   const composeDir = path.dirname(composeFile);
@@ -75,6 +81,14 @@ async function runWellKnownNetwork(namespace: string, runOptions: RunOptions) {
     }
   }
 
+  if (runOptions.fromGenesis) {
+    await runFromGenesis(composeFile, env, runOptions);
+    return;
+  }
+
+  const networkConfig = loadNetworkConfig(namespace);
+  const mock = requireMockConfig(namespace, networkConfig);
+
   let overridePath: string;
   if (runOptions.fromSnapshot) {
     const restoredDirs = await restoreSnapshot({
@@ -134,6 +148,55 @@ async function runWellKnownNetwork(namespace: string, runOptions: RunOptions) {
   });
 }
 
+/**
+ * Bring up the network's base compose from block 0 — no snapshot restore, no
+ * mock-authorities rewrite. The base compose expects the real network inputs
+ * (validator seed phrases, a main-chain data source) via env/--env-file, so
+ * report anything left unset up front rather than letting the chain come up
+ * silently unable to produce blocks.
+ */
+async function runFromGenesis(
+  composeFile: string,
+  env: Record<string, string>,
+  runOptions: RunOptions,
+) {
+  const staleDataDirs = discoverComposeDataMounts(composeFile).filter((dir) =>
+    isNonEmptyDirectory(dir),
+  );
+  if (staleDataDirs.length > 0) {
+    console.warn(
+      `⚠️  Existing chain data found (${staleDataDirs.join(", ")}); nodes will resume from it rather than genesis. Delete these directories first for a clean block-0 start.`,
+    );
+  }
+
+  const unsetVars = collectUnsetComposeVars(composeFile, env);
+  if (unsetVars.length > 0) {
+    console.warn(
+      `⚠️  Env vars referenced by the compose file but not set: ${unsetVars.join(", ")}. From-genesis mode mocks nothing — validator seeds and a main-chain data source must be provided via --env-file or the environment for the chain to produce blocks.`,
+    );
+  }
+
+  await runDockerCompose({
+    composeFile,
+    env,
+    profiles: runOptions.profiles,
+    detach: true,
+  });
+}
+
+/** Env var names referenced by the compose file ($VAR or ${VAR...}) that are unset or blank in env. */
+function collectUnsetComposeVars(
+  composeFile: string,
+  env: Record<string, string>,
+): string[] {
+  const text = fs.readFileSync(composeFile, "utf8");
+  const names = new Set<string>();
+  for (const match of text.matchAll(/\$\{?([A-Z][A-Z0-9_]*)/g)) {
+    names.add(match[1]);
+  }
+  return [...names].filter((name) => !env[name]).sort();
+}
+
 function assertReusableForkState(
   namespace: string,
   composeFile: string,
diff --git a/local-environment/src/index.ts b/local-environment/src/index.ts
index f3b298500..db3ac5163 100644
--- a/local-environment/src/index.ts
+++ b/local-environment/src/index.ts
@@ -84,8 +84,12 @@ program
     "--from-snapshot <uri>",
     "http(s):// snapshot URI to restore before the first well-known-network bring-up. Later runs can omit it to reuse existing local fork state.",
   )
+  .option(
+    "--from-genesis",
+    "Bring up the well-known network's base compose from block 0 instead of forking a snapshot. Requires the network's validator seeds and main-chain data-source env vars (see README).",
+  )
   .description(
-    "Bring up a forked well-known network from a snapshot using mock-authorities, reuse an existing local fork, or run the local-env target.",
+    "Bring up a forked well-known network from a snapshot using mock-authorities, reuse an existing local fork, start a well-known network from genesis, or run the local-env target.",
   )
   .action(async (network: string, options: RunOptions) => {
     await run(network, options);
diff --git a/local-environment/src/lib/types.ts b/local-environment/src/lib/types.ts
index 61dcab592..7fb66d31c 100644
--- a/local-environment/src/lib/types.ts
+++ b/local-environment/src/lib/types.ts
@@ -22,6 +22,13 @@ export interface RunOptions {
    * output.
    */
   fromSnapshot?: string;
+  /**
+   * Bring the well-known network's base compose up from block 0 instead of
+   * forking a snapshot. Nothing is mocked in this mode: validator seed
+   * phrases and a main-chain data source must be supplied via env/--env-file.
+   * Mutually exclusive with fromSnapshot.
+   */
+  fromGenesis?: boolean;
 }
 
 export interface ImageUpgradeOptions extends RunOptions {

From 93eeda64434354ef9a0ce6917f3784ed95f69965 Mon Sep 17 00:00:00 2001
From: Justin Frevert <justinfrevert@gmail.com>
Date: Wed, 15 Jul 2026 09:45:33 -0700
Subject: [PATCH 6/6] pr review comments

Signed-off-by: Justin Frevert <justinfrevert@gmail.com>
---
 changes/node/added/local-env-from-genesis.md  |  14 +-
 local-environment/.gitignore                  |   5 +
 local-environment/README.md                   |  48 +++++-
 local-environment/package.json                |   2 +-
 local-environment/src/commands/run.test.ts    |  60 +++++++
 local-environment/src/commands/run.ts         |  51 +++++-
 local-environment/src/index.ts                |   4 +
 .../src/lib/genesisComposeOverride.test.ts    | 133 +++++++++++++++
 .../src/lib/genesisComposeOverride.ts         | 160 ++++++++++++++++++
 local-environment/src/lib/types.ts            |   6 +
 10 files changed, 470 insertions(+), 13 deletions(-)
 create mode 100644 local-environment/src/commands/run.test.ts
 create mode 100644 local-environment/src/lib/genesisComposeOverride.test.ts
 create mode 100644 local-environment/src/lib/genesisComposeOverride.ts

diff --git a/changes/node/added/local-env-from-genesis.md b/changes/node/added/local-env-from-genesis.md
index 06340ef1a..cf59c9b27 100644
--- a/changes/node/added/local-env-from-genesis.md
+++ b/changes/node/added/local-env-from-genesis.md
@@ -8,10 +8,20 @@ source must be supplied via `--env-file` or the environment, and the CLI warns
 up front about compose variables left unset and about pre-existing chain data
 directories.
 
+Provided seed phrases are wired into node keystores through a generated
+`<network>.genesis.override.yaml` that mounts per-validator seed files and sets
+`AURA_SEED_FILE`/`GRANDPA_SEED_FILE`/`CROSS_CHAIN_SEED_FILE` — the node does
+not consume the `SEED_PHRASE` env var the base compose files pass. Networks
+deployed with a distinct phrase per key type can set per-type vars instead
+(`<VAR>_AURA_SEED`/`<VAR>_GRANDPA_SEED`/`<VAR>_CROSS_CHAIN_SEED` replacing the
+base var's `_SEED` suffix, with the base var as fallback). A
+`--compose-override` option lets fully local runs layer extra compose config on
+top, such as enabling the node's built-in mock main-chain follower.
+
 Also clarifies the fork-network workflow's `node_image`/`new_node_image` inputs
 (full Docker image references with the expected release and CI tag formats, not
 git branches/shas/URLs) and documents how to find snapshot archives for each
 network from the backup system's public index in `docs/fork-testing.md`.
 
-PR:
-Issue:
+PR: https://github.com/midnightntwrk/midnight-node/pull/1807
+Issue: https://github.com/midnightntwrk/midnight-node/issues/1468
diff --git a/local-environment/.gitignore b/local-environment/.gitignore
index e3672eec7..1dad158f5 100644
--- a/local-environment/.gitignore
+++ b/local-environment/.gitignore
@@ -7,3 +7,8 @@ src/networks/well-known/**/*.mock.override.yaml
 src/networks/well-known/**/mocked-config/
 src/networks/well-known/**/res/mock-bridge-data/*-mock.json
 src/networks/well-known/**/data/
+
+# generated genesis-mode state (genesis-config holds raw seed phrases)
+src/networks/well-known/**/*.genesis.override.yaml
+src/networks/well-known/**/genesis-config/
+src/networks/well-known/**/*.mock-follower.override.yaml
diff --git a/local-environment/README.md b/local-environment/README.md
index 9ad0b8b17..69f606072 100644
--- a/local-environment/README.md
+++ b/local-environment/README.md
@@ -62,12 +62,48 @@ npm run run:devnet -- --from-genesis --env-file ../devnet.env
 ```
 
 Unlike fork mode, nothing is mocked: each validator needs its real seed phrase
-(e.g. `MIDNIGHT_NODE_01_0_SEED`) and a reachable main-chain data source (the
-`DB_SYNC_POSTGRES_CONNECTION_STRING_NODE_*` vars) supplied via `--env-file` or
-the process environment. The CLI warns about any compose variables left unset,
-and about existing `data/` directories (nodes resume from existing chain data;
-wipe the network's `data/` directories first for a clean block-0 start).
-Restarting a genesis environment uses the same flag.
+(e.g. `MIDNIGHT_NODE_01_0_SEED`) supplied via `--env-file` or the process
+environment. The node only imports keystore keys from `AURA_SEED_FILE` /
+`GRANDPA_SEED_FILE` / `CROSS_CHAIN_SEED_FILE`, so the CLI writes each provided
+phrase to per-validator seed files under `genesis-config/` (gitignored) and
+generates a `<network>.genesis.override.yaml` that mounts them — the same
+phrase drives all three key types, as in the original genesis deployment. At
+least one seed phrase is required; validators without one start with empty
+keystores and cannot author blocks. Note that only holders of the network's
+genesis authority keys can produce blocks from genesis.
+
+Networks whose validators were deployed with a _distinct_ phrase per key type
+can set per-type vars instead: the base var's `_SEED` suffix is replaced by
+`_AURA_SEED` / `_GRANDPA_SEED` / `_CROSS_CHAIN_SEED` (e.g.
+`MIDNIGHT_NODE_01_0_AURA_SEED`). Each key type falls back to the base var when
+its per-type var is unset; a validator is seeded only when all three key types
+resolve.
+
+Each node also needs a main-chain data source: either the real db-sync
+connection strings (the `DB_SYNC_POSTGRES_CONNECTION_STRING_NODE_*` vars) or
+the node's built-in mock follower, enabled through an extra override file:
+
+```yaml
+# mock-follower.override.yaml
+services:
+  node1: &mock
+    environment:
+      USE_MAIN_CHAIN_FOLLOWER_MOCK: "true"
+      MOCK_REGISTRATIONS_FILE: /res/mock-bridge-data/default-registrations.json
+      DB_SYNC_POSTGRES_CONNECTION_STRING: ""
+  node2: *mock
+  # ... one entry per validator service
+```
+
+```bash
+npm run run:devnet -- --from-genesis --env-file ../devnet.env \
+  --compose-override ../mock-follower.override.yaml
+```
+
+The CLI warns about any compose variables left unset, and about existing
+`data/` directories (nodes resume from existing chain data; wipe the network's
+`data/` directories first for a clean block-0 start). Restarting a genesis
+environment uses the same flags.
 
 ### Upgrade rehearsals
 
diff --git a/local-environment/package.json b/local-environment/package.json
index 85a4e75b8..65eb55be9 100644
--- a/local-environment/package.json
+++ b/local-environment/package.json
@@ -3,7 +3,7 @@
   "version": "1.0.0",
   "main": "index.js",
   "scripts": {
-    "test": "echo \"Error: no test specified\" && exit 1",
+    "test": "node --require ts-node/register --test src/lib/genesisComposeOverride.test.ts src/commands/run.test.ts",
     "run:qanet": "ts-node src/index.ts run qanet",
     "run:devnet": "ts-node src/index.ts run devnet",
     "run:govnet": "ts-node src/index.ts run govnet",
diff --git a/local-environment/src/commands/run.test.ts b/local-environment/src/commands/run.test.ts
new file mode 100644
index 000000000..c3f10ebcf
--- /dev/null
+++ b/local-environment/src/commands/run.test.ts
@@ -0,0 +1,60 @@
+// This file is part of midnight-node.
+// Copyright (C) Midnight Foundation
+// SPDX-License-Identifier: Apache-2.0
+// Licensed under the Apache License, Version 2.0 (the "License");
+// You may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+// http://www.apache.org/licenses/LICENSE-2.0
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+import assert from "node:assert/strict";
+import { test } from "node:test";
+import fs from "fs";
+import os from "os";
+import path from "path";
+import { collectUnsetComposeVars, run } from "./run";
+
+test("collectUnsetComposeVars reports unset or blank vars once, sorted", () => {
+  const dir = fs.mkdtempSync(path.join(os.tmpdir(), "compose-vars-test-"));
+  const composeFile = path.join(dir, "compose.yaml");
+  fs.writeFileSync(
+    composeFile,
+    [
+      "services:",
+      "  node1:",
+      "    environment:",
+      "      SEED_PHRASE: $UNSET_SEED",
+      "      DB_URL: ${BLANK_VAR}",
+      "      IMAGE: $SET_VAR",
+      "      REPEAT: $UNSET_SEED",
+    ].join("\n"),
+  );
+
+  const unset = collectUnsetComposeVars(composeFile, {
+    SET_VAR: "value",
+    BLANK_VAR: "",
+  });
+
+  assert.deepEqual(unset, ["BLANK_VAR", "UNSET_SEED"]);
+});
+
+test("run rejects --from-genesis combined with --from-snapshot", async () => {
+  await assert.rejects(
+    run("devnet", {
+      fromGenesis: true,
+      fromSnapshot: "https://example.com/snapshot.tgz",
+    }),
+    /mutually exclusive/,
+  );
+});
+
+test("run rejects --compose-override without --from-genesis", async () => {
+  await assert.rejects(
+    run("devnet", { composeOverride: ["extra.override.yaml"] }),
+    /--compose-override is only supported together with --from-genesis/,
+  );
+});
diff --git a/local-environment/src/commands/run.ts b/local-environment/src/commands/run.ts
index d50deb359..86716ad77 100644
--- a/local-environment/src/commands/run.ts
+++ b/local-environment/src/commands/run.ts
@@ -37,6 +37,7 @@ import {
   mockOverridePath,
   MOCKED_CONFIG_DIRNAME,
 } from "../lib/mockComposeOverride";
+import { generateGenesisComposeOverride } from "../lib/genesisComposeOverride";
 
 /**
  * Bring up a network locally:
@@ -46,6 +47,12 @@ import {
  *   with --from-genesis — there is no k8s-backed path.
  */
 export async function run(network: string, runOptions: RunOptions) {
+  if (runOptions.composeOverride?.length && !runOptions.fromGenesis) {
+    throw new Error(
+      "--compose-override is only supported together with --from-genesis.",
+    );
+  }
+
   if (network === "local-env") {
     console.log("Running environment with local Cardano/PC resources");
     await runLocalEnvironment(runOptions);
@@ -65,7 +72,9 @@ export async function run(network: string, runOptions: RunOptions) {
 
 async function runWellKnownNetwork(namespace: string, runOptions: RunOptions) {
   if (runOptions.fromGenesis && runOptions.fromSnapshot) {
-    throw new Error("--from-genesis and --from-snapshot are mutually exclusive.");
+    throw new Error(
+      "--from-genesis and --from-snapshot are mutually exclusive.",
+    );
   }
 
   const composeFile = resolveComposeFile(namespace);
@@ -82,7 +91,7 @@ async function runWellKnownNetwork(namespace: string, runOptions: RunOptions) {
   }
 
   if (runOptions.fromGenesis) {
-    await runFromGenesis(composeFile, env, runOptions);
+    await runFromGenesis(namespace, composeFile, env, runOptions);
     return;
   }
 
@@ -154,8 +163,13 @@ async function runWellKnownNetwork(namespace: string, runOptions: RunOptions) {
  * (validator seed phrases, a main-chain data source) via env/--env-file, so
  * report anything left unset up front rather than letting the chain come up
  * silently unable to produce blocks.
+ *
+ * The provided seed phrases are wired into node keystores via a generated
+ * compose override: the node only imports keys from *_SEED_FILE paths, not
+ * from the SEED_PHRASE env var the base compose files pass.
  */
 async function runFromGenesis(
+  namespace: string,
   composeFile: string,
   env: Record<string, string>,
   runOptions: RunOptions,
@@ -169,15 +183,44 @@ async function runFromGenesis(
     );
   }
 
+  const { overridePath, seededServices, missingSeedVars } =
+    generateGenesisComposeOverride({
+      composeFile,
+      network: namespace,
+      env,
+    });
+  if (seededServices.length === 0) {
+    throw new Error(
+      `From-genesis mode needs at least one validator seed phrase, but none of the seed env vars are set (${Object.values(missingSeedVars).join(", ")}). Provide them via --env-file or the environment. Only holders of the network's genesis authority keys can produce blocks from genesis.`,
+    );
+  }
+  console.log(
+    `Generated genesis-mode override: ${overridePath} (seeded: ${seededServices.join(", ")})`,
+  );
+  const missing = Object.entries(missingSeedVars);
+  if (missing.length > 0) {
+    console.warn(
+      `⚠️  No seed phrase for: ${missing.map(([svc, v]) => `${svc} (${v})`).join(", ")}. These validators start with empty keystores and cannot author blocks.`,
+    );
+  }
+
   const unsetVars = collectUnsetComposeVars(composeFile, env);
   if (unsetVars.length > 0) {
     console.warn(
-      `⚠️  Env vars referenced by the compose file but not set: ${unsetVars.join(", ")}. From-genesis mode mocks nothing — validator seeds and a main-chain data source must be provided via --env-file or the environment for the chain to produce blocks.`,
+      `⚠️  Env vars referenced by the compose file but not set: ${unsetVars.join(", ")}. From-genesis mode mocks nothing — a main-chain data source must be provided (db-sync connection strings, or a --compose-override enabling the node's mock follower) for the chain to run.`,
     );
   }
 
+  const extraOverrides = runOptions.composeOverride ?? [];
+  for (const override of extraOverrides) {
+    if (!fs.existsSync(override)) {
+      throw new Error(`--compose-override file not found: ${override}`);
+    }
+  }
+
   await runDockerCompose({
     composeFile,
+    extraComposeFiles: [overridePath, ...extraOverrides],
     env,
     profiles: runOptions.profiles,
     detach: true,
@@ -185,7 +228,7 @@ async function runFromGenesis(
 }
 
 /** Env var names referenced by the compose file ($VAR or ${VAR...}) that are unset or blank in env. */
-function collectUnsetComposeVars(
+export function collectUnsetComposeVars(
   composeFile: string,
   env: Record<string, string>,
 ): string[] {
diff --git a/local-environment/src/index.ts b/local-environment/src/index.ts
index db3ac5163..e10342cce 100644
--- a/local-environment/src/index.ts
+++ b/local-environment/src/index.ts
@@ -88,6 +88,10 @@ program
     "--from-genesis",
     "Bring up the well-known network's base compose from block 0 instead of forking a snapshot. Requires the network's validator seeds and main-chain data-source env vars (see README).",
   )
+  .option(
+    "--compose-override <path...>",
+    "Extra docker-compose override file(s) applied after the generated genesis override (from-genesis mode only), e.g. to point nodes at a mock main-chain follower for fully local runs.",
+  )
   .description(
     "Bring up a forked well-known network from a snapshot using mock-authorities, reuse an existing local fork, start a well-known network from genesis, or run the local-env target.",
   )
diff --git a/local-environment/src/lib/genesisComposeOverride.test.ts b/local-environment/src/lib/genesisComposeOverride.test.ts
new file mode 100644
index 000000000..c592b566f
--- /dev/null
+++ b/local-environment/src/lib/genesisComposeOverride.test.ts
@@ -0,0 +1,133 @@
+// This file is part of midnight-node.
+// Copyright (C) Midnight Foundation
+// SPDX-License-Identifier: Apache-2.0
+// Licensed under the Apache License, Version 2.0 (the "License");
+// You may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+// http://www.apache.org/licenses/LICENSE-2.0
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+import assert from "node:assert/strict";
+import { test } from "node:test";
+import fs from "fs";
+import os from "os";
+import path from "path";
+import YAML from "yaml";
+import {
+  generateGenesisComposeOverride,
+  genesisOverridePath,
+  GENESIS_CONFIG_DIRNAME,
+} from "./genesisComposeOverride";
+
+/** Writes a compose file with the given services into a fresh temp dir. */
+function writeCompose(services: Record<string, unknown>): string {
+  const dir = fs.mkdtempSync(path.join(os.tmpdir(), "genesis-override-test-"));
+  const composeFile = path.join(dir, "testnet.network.yaml");
+  fs.writeFileSync(composeFile, YAML.stringify({ services }));
+  return composeFile;
+}
+
+function readSeed(composeFile: string, service: string, file: string): string {
+  return fs.readFileSync(
+    path.join(
+      path.dirname(composeFile),
+      GENESIS_CONFIG_DIRNAME,
+      "seeds",
+      service,
+      file,
+    ),
+    "utf8",
+  );
+}
+
+test("wires seed files for services whose seed var is set", () => {
+  const composeFile = writeCompose({
+    node1: { environment: { SEED_PHRASE: "$NODE_01_SEED" } },
+    node2: { environment: { SEED_PHRASE: "${NODE_02_SEED}" } },
+    proxy: { environment: { OTHER: "value" } },
+  });
+
+  const result = generateGenesisComposeOverride({
+    composeFile,
+    network: "testnet",
+    env: { NODE_01_SEED: "one two three" },
+  });
+
+  assert.deepEqual(result.seededServices, ["node1"]);
+  assert.deepEqual(result.missingSeedVars, { node2: "NODE_02_SEED" });
+  assert.equal(
+    result.overridePath,
+    genesisOverridePath(path.dirname(composeFile), "testnet"),
+  );
+
+  const override = YAML.parse(fs.readFileSync(result.overridePath, "utf8"));
+  assert.deepEqual(override.services.node1, {
+    environment: {
+      AURA_SEED_FILE: "/seeds/aura.seed",
+      GRANDPA_SEED_FILE: "/seeds/grandpa.seed",
+      CROSS_CHAIN_SEED_FILE: "/seeds/cross_chain.seed",
+      SEED_PHRASE: "",
+    },
+    volumes: [`./${GENESIS_CONFIG_DIRNAME}/seeds/node1:/seeds:ro`],
+  });
+  assert.equal(override.services.node2, undefined);
+  assert.equal(override.services.proxy, undefined);
+
+  for (const file of ["aura.seed", "grandpa.seed", "cross_chain.seed"]) {
+    assert.equal(readSeed(composeFile, "node1", file), "one two three");
+  }
+});
+
+test("per-key-type vars override the base var and fall back to it", () => {
+  const composeFile = writeCompose({
+    node1: { environment: { SEED_PHRASE: "$NODE_01_SEED" } },
+  });
+
+  const result = generateGenesisComposeOverride({
+    composeFile,
+    network: "testnet",
+    env: { NODE_01_SEED: "base phrase", NODE_01_AURA_SEED: "aura phrase" },
+  });
+
+  assert.deepEqual(result.seededServices, ["node1"]);
+  assert.equal(readSeed(composeFile, "node1", "aura.seed"), "aura phrase");
+  assert.equal(readSeed(composeFile, "node1", "grandpa.seed"), "base phrase");
+  assert.equal(
+    readSeed(composeFile, "node1", "cross_chain.seed"),
+    "base phrase",
+  );
+});
+
+test("incomplete per-type seeds without a base-var fallback are not seeded", () => {
+  const composeFile = writeCompose({
+    node1: { environment: { SEED_PHRASE: "$NODE_01_SEED" } },
+  });
+
+  const result = generateGenesisComposeOverride({
+    composeFile,
+    network: "testnet",
+    env: { NODE_01_AURA_SEED: "aura only" },
+  });
+
+  assert.deepEqual(result.seededServices, []);
+  assert.deepEqual(result.missingSeedVars, { node1: "NODE_01_SEED" });
+});
+
+test("a SEED_PHRASE that is not a plain env var reference is skipped", () => {
+  const composeFile = writeCompose({
+    node1: { environment: { SEED_PHRASE: "inline literal phrase" } },
+  });
+
+  const result = generateGenesisComposeOverride({
+    composeFile,
+    network: "testnet",
+    env: {},
+  });
+
+  assert.deepEqual(result.seededServices, []);
+  assert.deepEqual(result.missingSeedVars, {});
+});
diff --git a/local-environment/src/lib/genesisComposeOverride.ts b/local-environment/src/lib/genesisComposeOverride.ts
new file mode 100644
index 000000000..79d4c10df
--- /dev/null
+++ b/local-environment/src/lib/genesisComposeOverride.ts
@@ -0,0 +1,160 @@
+// This file is part of midnight-node.
+// Copyright (C) Midnight Foundation
+// SPDX-License-Identifier: Apache-2.0
+// Licensed under the Apache License, Version 2.0 (the "License");
+// You may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+// http://www.apache.org/licenses/LICENSE-2.0
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+import fs from "fs";
+import path from "path";
+import YAML from "yaml";
+
+export const GENESIS_CONFIG_DIRNAME = "genesis-config";
+
+/** Deterministic path of the override yaml generated by generateGenesisComposeOverride. */
+export function genesisOverridePath(
+  composeDir: string,
+  network: string,
+): string {
+  return path.join(
+    path.resolve(composeDir),
+    `${network}.genesis.override.yaml`,
+  );
+}
+
+export interface GenesisOverrideOptions {
+  /** Base compose file of the well-known network. */
+  composeFile: string;
+  /** Network name (devnet, qanet, ...) — drives the override filename. */
+  network: string;
+  /** Resolved environment (process env + --env-file overrides). */
+  env: Record<string, string>;
+}
+
+export interface GenesisOverrideResult {
+  overridePath: string;
+  /** Services whose seed env var was set and got seed files wired in. */
+  seededServices: string[];
+  /** service -> unset env var name, for services that could not be seeded. */
+  missingSeedVars: Record<string, string>;
+}
+
+/**
+ * Per-key-type seed file names and the env var suffix that overrides each.
+ * For a base var `MIDNIGHT_NODE_01_0_SEED`, the per-type vars are
+ * `MIDNIGHT_NODE_01_0_AURA_SEED`, `..._GRANDPA_SEED`, `..._CROSS_CHAIN_SEED`.
+ */
+const SEED_FILE_TYPES = [
+  { file: "aura.seed", envSuffix: "AURA" },
+  { file: "grandpa.seed", envSuffix: "GRANDPA" },
+  { file: "cross_chain.seed", envSuffix: "CROSS_CHAIN" },
+] as const;
+
+/**
+ * Generates the docker-compose override that wires validator keys into a
+ * from-genesis bring-up.
+ *
+ * The base compose files pass each validator's seed phrase as a `SEED_PHRASE`
+ * env var, but the node only imports keystore keys from AURA_SEED_FILE /
+ * GRANDPA_SEED_FILE / CROSS_CHAIN_SEED_FILE (see node/src/command.rs), so a
+ * bare bring-up would start every validator with an empty keystore. For each
+ * service whose SEED_PHRASE references a set env var, this writes the phrase
+ * to a per-service seed file and points all three *_SEED_FILE vars at it —
+ * the node derives the sr25519/ed25519/ecdsa keypairs from the same phrase,
+ * matching how the pre-fork tooling populated keystores from the single
+ * MIDNIGHT_NODE_*_SEED secret.
+ *
+ * Networks whose validators were deployed with a distinct phrase per key type
+ * can set per-type vars instead (the base var's `_SEED` suffix is replaced by
+ * `_AURA_SEED` / `_GRANDPA_SEED` / `_CROSS_CHAIN_SEED`); each key type falls
+ * back to the base var when its per-type var is unset. A service is seeded
+ * only when all three key types resolve to a phrase.
+ */
+export function generateGenesisComposeOverride(
+  opts: GenesisOverrideOptions,
+): GenesisOverrideResult {
+  const { composeFile, network, env } = opts;
+  const composeDir = path.dirname(path.resolve(composeFile));
+
+  const compose = YAML.parse(fs.readFileSync(composeFile, "utf8"));
+  const services: Record<string, { environment?: Record<string, unknown> }> =
+    compose?.services ?? {};
+
+  const seedsRoot = path.join(composeDir, GENESIS_CONFIG_DIRNAME, "seeds");
+  const overrideServices: Record<string, unknown> = {};
+  const seededServices: string[] = [];
+  const missingSeedVars: Record<string, string> = {};
+
+  for (const [name, service] of Object.entries(services)) {
+    const seedRef = service?.environment?.SEED_PHRASE;
+    if (typeof seedRef !== "string") {
+      continue;
+    }
+    const varMatch = seedRef.match(/^\$\{?([A-Za-z_][A-Za-z0-9_]*)\}?$/);
+    if (!varMatch) {
+      console.warn(
+        `⚠️  ${name}: SEED_PHRASE is not a plain env var reference (${seedRef}); skipping seed wiring for it.`,
+      );
+      continue;
+    }
+    const varName = varMatch[1];
+    const basePhrase = env[varName]?.trim();
+    const varPrefix = varName.replace(/_SEED$/, "");
+    const phrases: Record<string, string> = {};
+    const unresolvedVars: string[] = [];
+    for (const { file, envSuffix } of SEED_FILE_TYPES) {
+      const typeVar = `${varPrefix}_${envSuffix}_SEED`;
+      const phrase = env[typeVar]?.trim() || basePhrase;
+      if (phrase) {
+        phrases[file] = phrase;
+      } else {
+        unresolvedVars.push(typeVar);
+      }
+    }
+    if (unresolvedVars.length === SEED_FILE_TYPES.length) {
+      missingSeedVars[name] = varName;
+      continue;
+    }
+    if (unresolvedVars.length > 0) {
+      console.warn(
+        `⚠️  ${name}: per-key-type seeds are incomplete (${unresolvedVars.join(", ")} unset, no ${varName} fallback); skipping seed wiring for it.`,
+      );
+      missingSeedVars[name] = varName;
+      continue;
+    }
+
+    const serviceSeedDir = path.join(seedsRoot, name);
+    fs.mkdirSync(serviceSeedDir, { recursive: true, mode: 0o700 });
+    for (const { file } of SEED_FILE_TYPES) {
+      fs.writeFileSync(path.join(serviceSeedDir, file), phrases[file], {
+        mode: 0o600,
+      });
+    }
+
+    overrideServices[name] = {
+      environment: {
+        AURA_SEED_FILE: "/seeds/aura.seed",
+        GRANDPA_SEED_FILE: "/seeds/grandpa.seed",
+        CROSS_CHAIN_SEED_FILE: "/seeds/cross_chain.seed",
+        // The node does not consume SEED_PHRASE; blank it so nothing appears
+        // to depend on it and the raw phrase stays out of container env.
+        SEED_PHRASE: "",
+      },
+      volumes: [`./${GENESIS_CONFIG_DIRNAME}/seeds/${name}:/seeds:ro`],
+    };
+    seededServices.push(name);
+  }
+
+  const overridePath = genesisOverridePath(composeDir, network);
+  fs.writeFileSync(
+    overridePath,
+    YAML.stringify({ services: overrideServices }),
+  );
+  return { overridePath, seededServices, missingSeedVars };
+}
diff --git a/local-environment/src/lib/types.ts b/local-environment/src/lib/types.ts
index 7fb66d31c..f1246dd61 100644
--- a/local-environment/src/lib/types.ts
+++ b/local-environment/src/lib/types.ts
@@ -29,6 +29,12 @@ export interface RunOptions {
    * Mutually exclusive with fromSnapshot.
    */
   fromGenesis?: boolean;
+  /**
+   * Extra docker-compose override file(s) applied after the generated genesis
+   * override (from-genesis mode only) — e.g. to enable the node's mock
+   * main-chain follower for fully local runs.
+   */
+  composeOverride?: string[];
 }
 
 export interface ImageUpgradeOptions extends RunOptions {