local env usability fixes

.github/workflows/fork-network.yml

@@ -15,6 +15,15 @@ name: fork-network
 #             federated-authority governance flow
 #   full    - production-shaped rehearsal: image rollout first, then the
 #             runtime upgrade via governance
+#
+# node_image / new_node_image take full Docker image references — anything
+# `docker pull` accepts. Git branches, commit shas, and repo URLs are NOT
+# accepted. Examples:
+#   ghcr.io/midnight-ntwrk/midnight-node:1.0.0                            (release tag)
+#   ghcr.io/midnight-ntwrk/midnight-node:1.0.1-dev-6587676a9bb2-amd64     (CI build of a commit)
+# CI tags a per-commit image <version>-dev-<12-char-tree-hash>-amd64; find it
+# in the commit's continuous-integration run (image_tags step) or compute the
+# hash with `git rev-parse HEAD^{tree} | cut -c1-12`.
 
 on:
   workflow_dispatch:
@@ -32,10 +41,10 @@ on:
         required: true
         default: "368fd98"
       node_image:
-        description: "midnight-node image to fork from (must match the snapshot's runtime host fns)"
+        description: "Full Docker image reference to fork from, e.g. ghcr.io/midnight-ntwrk/midnight-node:1.0.0 or a CI tag like ghcr.io/midnight-ntwrk/midnight-node:1.0.1-dev-<tree-hash>-amd64 — not a git branch/sha/URL. Must match the snapshot's runtime host fns"
         required: true
       new_node_image:
-        description: "midnight-node image to upgrade to (image rollout target and/or runtime wasm source, per upgrade_mode)"
+        description: "Full Docker image reference to upgrade to (image rollout target and/or runtime wasm source, per upgrade_mode). Same format as node_image"
         required: true
       upgrade_mode:
         description: "Upgrade shape: roll the node image, submit a runtime upgrade via governance, or both in sequence"

changes/node/added/local-env-from-genesis.md

@@ -0,0 +1,27 @@
+# Restore from-genesis bring-up for well-known networks in local-env tooling
+
+`npm run run:<network> -- --from-genesis` brings up a well-known network's base
+compose from block 0 again, instead of requiring `--from-snapshot` (the genesis
+path was removed together with the internal k8s/AWS integration in #1470).
+Nothing is mocked in this mode: validator seed phrases and a main-chain data
+source must be supplied via `--env-file` or the environment, and the CLI warns
+up front about compose variables left unset and about pre-existing chain data
+directories.
+
+Provided seed phrases are wired into node keystores through a generated
+`<network>.genesis.override.yaml` that mounts per-validator seed files and sets
+`AURA_SEED_FILE`/`GRANDPA_SEED_FILE`/`CROSS_CHAIN_SEED_FILE` — the node does
+not consume the `SEED_PHRASE` env var the base compose files pass. Networks
+deployed with a distinct phrase per key type can set per-type vars instead
+(`<VAR>_AURA_SEED`/`<VAR>_GRANDPA_SEED`/`<VAR>_CROSS_CHAIN_SEED` replacing the
+base var's `_SEED` suffix, with the base var as fallback). A
+`--compose-override` option lets fully local runs layer extra compose config on
+top, such as enabling the node's built-in mock main-chain follower.
+
+Also clarifies the fork-network workflow's `node_image`/`new_node_image` inputs
+(full Docker image references with the expected release and CI tag formats, not
+git branches/shas/URLs) and documents how to find snapshot archives for each
+network from the backup system's public index in `docs/fork-testing.md`.
+
+PR: https://github.com/midnightntwrk/midnight-node/pull/1807
+Issue: https://github.com/midnightntwrk/midnight-node/issues/1468

docs/fork-testing.md

@@ -23,6 +23,37 @@ Local restore requires:
 - `tar`
 - `zstd` for `.zst` archives
 
+## Finding snapshot archives
+
+The backup system publishes snapshots of the well-known networks behind a
+CloudFront distribution. The manifest at
+`https://dg39snjayoq3t.cloudfront.net/index.json` lists every available
+archive, keyed by network name. To see which networks currently have
+snapshots:
+
+```bash
+curl -fsSL https://dg39snjayoq3t.cloudfront.net/index.json | jq 'keys'
+```
+
+As of this writing the index contains `devnet`, `qanet`, and `govnet`.
+`preview`, `preprod`, and `mainnet` snapshots are not published there yet;
+ask the node team if you need one.
+
+To resolve the latest archive URL for a network (this mirrors what the
+`fork-network` workflow's "Resolve latest snapshot URL" step does):
+
+```bash
+NETWORK=devnet
+curl -fsSL https://dg39snjayoq3t.cloudfront.net/index.json |
+  jq -r --arg net "$NETWORK" '
+    .[$net] | to_entries | map(.value[]) | sort_by(.timestamp) | last
+    | .s3_path | sub("^s3://[^/]+/"; "https://dg39snjayoq3t.cloudfront.net/")'
+```
+
+Pass the resulting URL to `--from-snapshot`. Drop the `last` selection to list
+every archive for the network (older snapshots stay useful for forking a
+specific block height or runtime version).
+
 ## Initial restore
 
 On the first bring-up of a well-known network, pass the snapshot URL to `run`,

local-environment/.gitignore

@@ -7,3 +7,8 @@ src/networks/well-known/**/*.mock.override.yaml
 src/networks/well-known/**/mocked-config/
 src/networks/well-known/**/res/mock-bridge-data/*-mock.json
 src/networks/well-known/**/data/
+
+# generated genesis-mode state (genesis-config holds raw seed phrases)
+src/networks/well-known/**/*.genesis.override.yaml
+src/networks/well-known/**/genesis-config/
+src/networks/well-known/**/*.mock-follower.override.yaml

local-environment/README.md

@@ -35,6 +35,10 @@ npm run run:preprod -- --from-snapshot https://example.com/snapshots/preprod-lat
 npm run run:mainnet -- --from-snapshot https://example.com/snapshots/mainnet-latest.tar.gz
 ```
 
+Real snapshot URLs for each network can be resolved from the backup system's
+public index — see
+[Finding snapshot archives](../docs/fork-testing.md#finding-snapshot-archives).
+
 After that initial restore, the same network can be restarted without
 `--from-snapshot` as long as the restored `data/` directories and generated
 mock-authorities output are still present:
@@ -48,6 +52,59 @@ image was built with the same `networkId` as the genesis used to produce the
 snapshot. Recent runtimes validate this at boot and the node will refuse to
 start on a mismatch.
 
+### Starting a well-known network from genesis
+
+`--from-genesis` skips the snapshot/fork flow entirely and brings up the
+network's base compose from block 0:
+
+```bash
+npm run run:devnet -- --from-genesis --env-file ../devnet.env
+```
+
+Unlike fork mode, nothing is mocked: each validator needs its real seed phrase
+(e.g. `MIDNIGHT_NODE_01_0_SEED`) supplied via `--env-file` or the process
+environment. The node only imports keystore keys from `AURA_SEED_FILE` /
+`GRANDPA_SEED_FILE` / `CROSS_CHAIN_SEED_FILE`, so the CLI writes each provided
+phrase to per-validator seed files under `genesis-config/` (gitignored) and
+generates a `<network>.genesis.override.yaml` that mounts them — the same
+phrase drives all three key types, as in the original genesis deployment. At
+least one seed phrase is required; validators without one start with empty
+keystores and cannot author blocks. Note that only holders of the network's
+genesis authority keys can produce blocks from genesis.
+
+Networks whose validators were deployed with a _distinct_ phrase per key type
+can set per-type vars instead: the base var's `_SEED` suffix is replaced by
+`_AURA_SEED` / `_GRANDPA_SEED` / `_CROSS_CHAIN_SEED` (e.g.
+`MIDNIGHT_NODE_01_0_AURA_SEED`). Each key type falls back to the base var when
+its per-type var is unset; a validator is seeded only when all three key types
+resolve.
+
+Each node also needs a main-chain data source: either the real db-sync
+connection strings (the `DB_SYNC_POSTGRES_CONNECTION_STRING_NODE_*` vars) or
+the node's built-in mock follower, enabled through an extra override file:
+
+```yaml
+# mock-follower.override.yaml
+services:
+  node1: &mock
+    environment:
+      USE_MAIN_CHAIN_FOLLOWER_MOCK: "true"
+      MOCK_REGISTRATIONS_FILE: /res/mock-bridge-data/default-registrations.json
+      DB_SYNC_POSTGRES_CONNECTION_STRING: ""
+  node2: *mock
+  # ... one entry per validator service
+```
+
+```bash
+npm run run:devnet -- --from-genesis --env-file ../devnet.env \
+  --compose-override ../mock-follower.override.yaml
+```
+
+The CLI warns about any compose variables left unset, and about existing
+`data/` directories (nodes resume from existing chain data; wipe the network's
+`data/` directories first for a clean block-0 start). Restarting a genesis
+environment uses the same flags.
+
 ### Upgrade rehearsals
 
 `image-upgrade` rolls service containers from `NODE_IMAGE` to `NEW_NODE_IMAGE`.

local-environment/package.json

@@ -3,7 +3,7 @@
   "version": "1.0.0",
   "main": "index.js",
   "scripts": {
-    "test": "echo \"Error: no test specified\" && exit 1",
+    "test": "node --require ts-node/register --test src/lib/genesisComposeOverride.test.ts src/commands/run.test.ts",
     "run:qanet": "ts-node src/index.ts run qanet",
     "run:devnet": "ts-node src/index.ts run devnet",
     "run:govnet": "ts-node src/index.ts run govnet",

local-environment/src/commands/run.test.ts

@@ -0,0 +1,60 @@
+// This file is part of midnight-node.
+// Copyright (C) Midnight Foundation
+// SPDX-License-Identifier: Apache-2.0
+// Licensed under the Apache License, Version 2.0 (the "License");
+// You may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+// http://www.apache.org/licenses/LICENSE-2.0
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+import assert from "node:assert/strict";
+import { test } from "node:test";
+import fs from "fs";
+import os from "os";
+import path from "path";
+import { collectUnsetComposeVars, run } from "./run";
+
+test("collectUnsetComposeVars reports unset or blank vars once, sorted", () => {
+  const dir = fs.mkdtempSync(path.join(os.tmpdir(), "compose-vars-test-"));
+  const composeFile = path.join(dir, "compose.yaml");
+  fs.writeFileSync(
+    composeFile,
+    [
+      "services:",
+      "  node1:",
+      "    environment:",
+      "      SEED_PHRASE: $UNSET_SEED",
+      "      DB_URL: ${BLANK_VAR}",
+      "      IMAGE: $SET_VAR",
+      "      REPEAT: $UNSET_SEED",
+    ].join("\n"),
+  );
+
+  const unset = collectUnsetComposeVars(composeFile, {
+    SET_VAR: "value",
+    BLANK_VAR: "",
+  });
+
+  assert.deepEqual(unset, ["BLANK_VAR", "UNSET_SEED"]);
+});
+
+test("run rejects --from-genesis combined with --from-snapshot", async () => {
+  await assert.rejects(
+    run("devnet", {
+      fromGenesis: true,
+      fromSnapshot: "https://example.com/snapshot.tgz",
+    }),
+    /mutually exclusive/,
+  );
+});
+
+test("run rejects --compose-override without --from-genesis", async () => {
+  await assert.rejects(
+    run("devnet", { composeOverride: ["extra.override.yaml"] }),
+    /--compose-override is only supported together with --from-genesis/,
+  );
+});