From 7e373a62297aa31bd84fab7f1af858257fa19bbe Mon Sep 17 00:00:00 2001 From: claravanstaden Date: Fri, 24 Apr 2026 09:09:03 +0200 Subject: [PATCH 01/10] governance commands --- control/preimage/src/commands.rs | 53 ++++++++++++++- control/preimage/src/main.rs | 64 ++++++++++++++++--- .../governance-and-operational-processes.md | 30 ++++++--- 3 files changed, 129 insertions(+), 18 deletions(-) diff --git a/control/preimage/src/commands.rs b/control/preimage/src/commands.rs index dd22dded8f..6786759daf 100644 --- a/control/preimage/src/commands.rs +++ b/control/preimage/src/commands.rs @@ -24,10 +24,16 @@ use crate::bridge_hub_runtime::runtime_types::{ }, snowbridge_outbound_queue_primitives::{v1::message::Initializer, OperatingMode}, snowbridge_pallet_ethereum_client, snowbridge_pallet_inbound_queue, - snowbridge_pallet_outbound_queue, snowbridge_pallet_system, + snowbridge_pallet_inbound_queue_v2, snowbridge_pallet_outbound_queue, + snowbridge_pallet_system, snowbridge_pallet_system_v2, }; use crate::bridge_hub_runtime::RuntimeCall as BridgeHubRuntimeCall; +use crate::asset_hub_runtime::runtime_types::{ + snowbridge_core::operating_mode::BasicOperatingMode as AssetHubBasicOperatingMode, + snowbridge_pallet_system_frontend, +}; + #[cfg(feature = "polkadot")] pub mod asset_hub_polkadot_types { pub use crate::asset_hub_runtime::runtime_types::staging_xcm::v5::{ @@ -150,6 +156,51 @@ pub fn outbound_queue_operating_mode(param: &OperatingModeEnum) -> BridgeHubRunt ) } +// V2 variant: halts the inbound-queue-v2 pallet's `submit` extrinsic, blocking +// processing of V2 Ethereum -> Polkadot messages on BridgeHub. +pub fn inbound_queue_v2_operating_mode(param: &OperatingModeEnum) -> BridgeHubRuntimeCall { + let mode = match param { + OperatingModeEnum::Normal => BasicOperatingMode::Normal, + OperatingModeEnum::Halted => BasicOperatingMode::Halted, + }; + BridgeHubRuntimeCall::EthereumInboundQueueV2( + snowbridge_pallet_inbound_queue_v2::pallet::Call::set_operating_mode { mode }, + ) +} + +// V2 variant: sends `Command::SetOperatingMode` to the Gateway via the V2 outbound +// queue. Sets the same Gateway `$.mode` storage as the V1 variant; both are kept +// so governance can halt via whichever outbound path is live. +pub fn gateway_operating_mode_v2( + operating_mode: &GatewayOperatingModeEnum, +) -> BridgeHubRuntimeCall { + let mode = match operating_mode { + GatewayOperatingModeEnum::Normal => OperatingMode::Normal, + GatewayOperatingModeEnum::RejectingOutboundMessages => { + OperatingMode::RejectingOutboundMessages + } + }; + BridgeHubRuntimeCall::EthereumSystemV2( + snowbridge_pallet_system_v2::pallet::Call::set_operating_mode { mode }, + ) +} + +// AssetHub-side: halts the system-frontend pallet. The `PausableExporter` wrapping +// the AssetHub->Ethereum XcmRouter consults `SnowbridgeSystemFrontend::is_paused()` +// and returns `SendError::NotApplicable` when halted, short-circuiting every +// AssetHub->Ethereum export (V1 and V2 share the same wrapper). This is the +// primary outbound halt lever for V2 because `outbound-queue-v2` has no local +// operating-mode storage. +pub fn system_frontend_operating_mode(param: &OperatingModeEnum) -> AssetHubRuntimeCall { + let mode = match param { + OperatingModeEnum::Normal => AssetHubBasicOperatingMode::Normal, + OperatingModeEnum::Halted => AssetHubBasicOperatingMode::Halted, + }; + AssetHubRuntimeCall::SnowbridgeSystemFrontend( + snowbridge_pallet_system_frontend::pallet::Call::set_operating_mode { mode }, + ) +} + pub fn upgrade(params: &UpgradeArgs) -> BridgeHubRuntimeCall { BridgeHubRuntimeCall::EthereumSystem(snowbridge_pallet_system::pallet::Call::upgrade { impl_address: params.logic_address.into_array().into(), diff --git a/control/preimage/src/main.rs b/control/preimage/src/main.rs index a1c0e7bcee..702dab8ff3 100644 --- a/control/preimage/src/main.rs +++ b/control/preimage/src/main.rs @@ -204,24 +204,46 @@ pub struct PricingParametersArgs { #[derive(Debug, Args)] pub struct HaltBridgeArgs { - /// Halt the Ethereum gateway, blocking message from Ethereum to Polkadot in the Ethereum - /// contract. + /// Halt the Ethereum Gateway contract (both V1 and V2 paths). Sends + /// `Command::SetOperatingMode(Halted)` via both V1 and V2 system pallets so the halt + /// is delivered via whichever outbound queue is live. Once processed on Ethereum, + /// this blocks `v2_sendMessage`, `v2_registerToken`, and V1 `sendToken`/`sendMessage` + /// on the Gateway. Delivery is relayer-dependent. #[arg(long, value_name = "HALT_GATEWAY")] gateway: bool, - /// Halt the Ethereum Inbound Queue, blocking messages from BH to AH. + /// Halt both V1 and V2 inbound-queue pallets on BridgeHub, blocking processing of + /// Ethereum -> Polkadot messages. For surgical halts of a single version, use + /// `--inbound-queue-v1` or `--inbound-queue-v2`. #[arg(long, value_name = "HALT_INBOUND_QUEUE")] inbound_queue: bool, - /// Halt the Ethereum Outbound Queue, blocking message from AH to BH. + /// Halt only the V1 inbound-queue pallet on BridgeHub. + #[arg(long, value_name = "HALT_INBOUND_QUEUE_V1")] + inbound_queue_v1: bool, + /// Halt only the V2 inbound-queue pallet on BridgeHub. + #[arg(long, value_name = "HALT_INBOUND_QUEUE_V2")] + inbound_queue_v2: bool, + /// Halt AssetHub -> Ethereum outbound traffic. Halts the V1 outbound-queue pallet + /// on BridgeHub AND the system-frontend pallet on AssetHub; the latter short-circuits + /// the AssetHub->Ethereum `PausableExporter` for both V1 and V2 at the XcmRouter + /// layer (V2's `outbound-queue-v2` has no local halt, so the frontend halt is the + /// primary V2 outbound lever). #[arg(long, value_name = "HALT_OUTBOUND_QUEUE")] outbound_queue: bool, - /// Halt the Ethereum client, blocking consensus updates to the light client. + /// Halt the Ethereum beacon light client, blocking new beacon-header ingestion. + /// Note: this does NOT propagate into the `Verifier::verify` trait impl that + /// downstream consumers (`inbound-queue(-v2)::submit`, + /// `outbound-queue-v2::submit_delivery_receipt`) call — those verify against + /// already-stored finalised state. Halt those consumers individually to block + /// proof-consuming flows during a suspected beacon compromise. #[arg(long, value_name = "HALT_ETHEREUM_CLIENT")] ethereum_client: bool, - /// Set the AH to Ethereum fee to a high amount, effectively blocking messages from AH -> - /// Ethereum. + /// Set the AssetHub -> Ethereum outbound fee to `u128::MAX` for both V1 + /// (`BridgeHubEthereumBaseFee`) and V2 (`BridgeHubEthereumBaseFeeV2`) storage + /// items, effectively deterring user sends via fee pricing. Complementary to the + /// system-frontend halt; does not block at the router layer. #[arg(long, value_name = "ASSETHUB_MAX_FEE")] assethub_max_fee: bool, - /// Halt all parts of the bridge + /// Halt all parts of the bridge (equivalent to passing every other flag). #[arg(long, value_name = "HALT_SNOWBRIDGE")] all: bool, } @@ -419,26 +441,48 @@ async fn run() -> Result<(), Box> { // if no individual option specified, assume halt the whole bridge. if !params.gateway && !params.inbound_queue + && !params.inbound_queue_v1 + && !params.inbound_queue_v2 && !params.outbound_queue && !params.ethereum_client && !params.assethub_max_fee { halt_all = true; } + // Gateway halt commands must be enqueued BEFORE any local outbound-queue + // halt takes effect, otherwise the SetOperatingMode command cannot be + // committed for delivery to Ethereum. Push both V1 and V2 variants so the + // halt is delivered via whichever outbound queue is operational. if params.gateway || halt_all { bh_calls.push(commands::gateway_operating_mode( &GatewayOperatingModeEnum::RejectingOutboundMessages, )); + bh_calls.push(commands::gateway_operating_mode_v2( + &GatewayOperatingModeEnum::RejectingOutboundMessages, + )); } - if params.inbound_queue || halt_all { + if params.inbound_queue || params.inbound_queue_v1 || halt_all { bh_calls.push(commands::inbound_queue_operating_mode( &OperatingModeEnum::Halted, )); } + if params.inbound_queue || params.inbound_queue_v2 || halt_all { + bh_calls.push(commands::inbound_queue_v2_operating_mode( + &OperatingModeEnum::Halted, + )); + } if params.outbound_queue || halt_all { + // V1 local halt on BridgeHub. V2's outbound-queue-v2 has no local halt; + // the system-frontend halt below is the effective V2 outbound lever. bh_calls.push(commands::outbound_queue_operating_mode( &OperatingModeEnum::Halted, )); + // system-frontend halt on AssetHub: short-circuits the PausableExporter + // wrapping the AH->Ethereum router, blocking both V1 and V2 exports at + // the source regardless of user or parachain origin. + ah_calls.push(commands::system_frontend_operating_mode( + &OperatingModeEnum::Halted, + )); } if params.ethereum_client || halt_all { bh_calls.push(commands::ethereum_client_operating_mode( @@ -446,7 +490,9 @@ async fn run() -> Result<(), Box> { )); } if params.assethub_max_fee || halt_all { + // Set both V1 and V2 AssetHub outbound fee storage items to u128::MAX. ah_calls.push(commands::set_assethub_fee(u128::MAX)); + ah_calls.push(commands::set_assethub_fee_v2(u128::MAX)); } if bh_calls.len() > 0 && ah_calls.len() == 0 { send_xcm_bridge_hub(&context, bh_calls).await? diff --git a/docs/resources/governance-and-operational-processes.md b/docs/resources/governance-and-operational-processes.md index f3a74e2292..fa0fcdd8c8 100644 --- a/docs/resources/governance-and-operational-processes.md +++ b/docs/resources/governance-and-operational-processes.md @@ -59,14 +59,28 @@ Preimage Size: 129 The halt-bridge command has the following flags: -* `--all` Halt all the bridge components at once. -* `--ethereum-client` Halt the Ethereum client, blocking consensus updates. -* `--inbound-queue` Halt the Inbound Queue, blocking messages from BridgeHub to AssetHub. -* `--outbound-queue` Halt the Outbound Queue, blocking messages from AssetHub to BridgeHub. -* `--gateway` Halt messages from Ethereum to AssetHub. -* `--assethub-max-fee` Set the AssetHub to Ethereum to the maximum fee, so that users effectively cannot send messages from AssetHub to Ethereum. - -Based on the nature of the emergency, the bridge might need to be halted in its entirety, or partially. +* `--all` Halt all the bridge components at once. This is the default when no other flag is given. +* `--ethereum-client` Halt the Ethereum beacon light client. Blocks new beacon-header ingestion via `EthereumBeaconClient::submit` **and** short-circuits `Verifier::verify` for every downstream consumer on BridgeHub. Concretely this stops: + * inbound V1 `submit` and inbound V2 `submit` (Ethereum → Polkadot messages), and + * `outbound-queue-v2::submit_delivery_receipt` (relayer-reward payouts against `PendingOrders`). + + This is the single lever to pull during a suspected beacon-light-client or sync-committee compromise. `force_checkpoint` remains available (root-only) for recovery. +* `--inbound-queue` Halt both V1 and V2 inbound-queue pallets on BridgeHub, blocking processing of Ethereum → Polkadot messages. For surgical halts of a single version, use the two flags below. +* `--inbound-queue-v1` Halt only the V1 inbound-queue pallet on BridgeHub. +* `--inbound-queue-v2` Halt only the V2 inbound-queue pallet on BridgeHub. +* `--outbound-queue` Halt AssetHub → Ethereum traffic. Halts the V1 outbound-queue pallet on BridgeHub **and** the system-frontend pallet on AssetHub; the latter short-circuits the AssetHub → Ethereum `PausableExporter` for both V1 and V2 at the XcmRouter layer. (V2's `outbound-queue-v2` has no local halt, so the system-frontend halt is the primary V2 outbound lever.) +* `--gateway` Halt the Ethereum Gateway contract (both V1 and V2 paths). Sends `Command::SetOperatingMode(Halted)` via both V1 and V2 system pallets so the halt is delivered via whichever outbound queue is live. Delivery is relayer-dependent, so schedule this **before** any local outbound halt takes effect. +* `--assethub-max-fee` Set the AssetHub → Ethereum outbound fee to `u128::MAX` for both V1 (`BridgeHubEthereumBaseFee`) and V2 (`BridgeHubEthereumBaseFeeV2`), effectively deterring user sends via fee pricing. Complementary to the system-frontend halt; does not block at the router layer. + +Based on the nature of the emergency, the bridge might need to be halted in its entirety, or partially. Pick the narrowest lever that covers the suspected failure mode: + +| Suspected failure mode | Minimum-scope command | +| --- | --- | +| Beacon light client / sync committee compromise | `halt-bridge --ethereum-client` | +| Ethereum Gateway contract compromise | `halt-bridge --gateway --assethub-max-fee` | +| Inbound-queue bug (one version) | `halt-bridge --inbound-queue-v1` or `--inbound-queue-v2` | +| Outbound-queue / system-frontend bug | `halt-bridge --outbound-queue` | +| Uncertain / any component suspect | `halt-bridge --all` | In case of emergency where there is uncertainty of the cause of a problem, it is best to block the bridge in its entirety using `halt-bridge --all`. To block both transfer directions at the earliest point possible, use `halt-bridge --gateway --assethub-max-fee`. From d7dd8830e0e7e7765797ac78acf6735eda1cb4fe Mon Sep 17 00:00:00 2001 From: claravanstaden Date: Mon, 11 May 2026 10:13:57 +0200 Subject: [PATCH 02/10] halt improvements --- control/preimage/src/helpers.rs | 25 ++++++- control/preimage/src/main.rs | 72 +++++++++++++++---- control/preimage/src/treasury_commands.rs | 4 +- .../governance-and-operational-processes.md | 5 ++ 4 files changed, 91 insertions(+), 15 deletions(-) diff --git a/control/preimage/src/helpers.rs b/control/preimage/src/helpers.rs index 361b063eca..2a039f089f 100644 --- a/control/preimage/src/helpers.rs +++ b/control/preimage/src/helpers.rs @@ -134,12 +134,35 @@ pub async fn query_weight_asset_hub( Ok((call_info.weight.ref_time, call_info.weight.proof_size)) } -pub fn utility_force_batch(calls: Vec) -> AssetHubRuntimeCall { +/// Emits `pallet_utility::Call::batch_all`, which rolls back every call if any one +/// fails. The default for governance batches that must commit atomically. +pub fn utility_batch_all(calls: Vec) -> AssetHubRuntimeCall { AssetHubRuntimeCall::Utility( crate::asset_hub_runtime::runtime_types::pallet_utility::pallet::Call::batch_all { calls }, ) } +/// Emits `pallet_utility::Call::batch`, which short-circuits on the first failing call +/// but commits all preceding calls. +#[allow(dead_code)] +pub fn utility_batch(calls: Vec) -> AssetHubRuntimeCall { + AssetHubRuntimeCall::Utility( + crate::asset_hub_runtime::runtime_types::pallet_utility::pallet::Call::batch { calls }, + ) +} + +/// Emits `pallet_utility::Call::force_batch`, which attempts every call regardless of +/// failures and reports per-call results via `ItemFailed` events. Use when each call +/// must fire independently — e.g. the halt-bridge preimage, where one stuck lever +/// (e.g. HRMP transport failure for the BH XCM) should not skip the rest. +pub fn utility_force_batch(calls: Vec) -> AssetHubRuntimeCall { + AssetHubRuntimeCall::Utility( + crate::asset_hub_runtime::runtime_types::pallet_utility::pallet::Call::force_batch { + calls, + }, + ) +} + #[cfg(any(feature = "westend", feature = "paseo"))] pub fn sudo(call: Box) -> AssetHubRuntimeCall { return AssetHubRuntimeCall::Sudo( diff --git a/control/preimage/src/main.rs b/control/preimage/src/main.rs index 702dab8ff3..a71251c693 100644 --- a/control/preimage/src/main.rs +++ b/control/preimage/src/main.rs @@ -11,7 +11,10 @@ use alloy_primitives::{address, utils::parse_units, Address, Bytes, FixedBytes, use clap::{Args, Parser, Subcommand, ValueEnum}; use codec::Encode; use constants::{ASSET_HUB_API, BRIDGE_HUB_API, POLKADOT_DECIMALS, POLKADOT_SYMBOL, RELAY_API}; -use helpers::{force_xcm_version, send_xcm_asset_hub, send_xcm_bridge_hub, utility_force_batch}; +use helpers::{ + force_xcm_version, send_xcm_asset_hub, send_xcm_bridge_hub, utility_batch_all, + utility_force_batch, +}; use snowbridge_preimage_chopsticks::generate_chopsticks_script; use sp_crypto_hashing::blake2_256; use std::{io::Write, path::PathBuf}; @@ -211,6 +214,11 @@ pub struct HaltBridgeArgs { /// on the Gateway. Delivery is relayer-dependent. #[arg(long, value_name = "HALT_GATEWAY")] gateway: bool, + /// V2-only Gateway halt: sends `Command::SetOperatingMode(Halted)` only via the V2 + /// system pallet. Blocks `v2_sendMessage` and `v2_registerToken` on the Gateway once + /// the message is delivered to Ethereum. Leaves V1 `sendToken`/`sendMessage` working. + #[arg(long, value_name = "HALT_GATEWAY_V2")] + gateway_v2: bool, /// Halt both V1 and V2 inbound-queue pallets on BridgeHub, blocking processing of /// Ethereum -> Polkadot messages. For surgical halts of a single version, use /// `--inbound-queue-v1` or `--inbound-queue-v2`. @@ -229,6 +237,13 @@ pub struct HaltBridgeArgs { /// primary V2 outbound lever). #[arg(long, value_name = "HALT_OUTBOUND_QUEUE")] outbound_queue: bool, + /// V2-only router-layer P->E halt: halts only the AssetHub system-frontend pallet, + /// leaving the V1 outbound-queue on BridgeHub running. Short-circuits the + /// `PausableExporter` for V2 (and V1 — frontend halt covers both at the router + /// layer), so V2 P->E messages are rejected with `SendError::NotApplicable` while + /// V1's BH outbound-queue continues to drain in-flight messages. + #[arg(long, value_name = "HALT_SYSTEM_FRONTEND")] + system_frontend: bool, /// Halt the Ethereum beacon light client, blocking new beacon-header ingestion. /// Note: this does NOT propagate into the `Verifier::verify` trait impl that /// downstream consumers (`inbound-queue(-v2)::submit`, @@ -243,6 +258,11 @@ pub struct HaltBridgeArgs { /// system-frontend halt; does not block at the router layer. #[arg(long, value_name = "ASSETHUB_MAX_FEE")] assethub_max_fee: bool, + /// V2-only variant of `--assethub-max-fee`: writes only the + /// `BridgeHubEthereumBaseFeeV2` storage item, leaving V1 fee unchanged. + /// Use this to deter V2 P->E sends while V1 traffic continues unimpeded. + #[arg(long, value_name = "ASSETHUB_MAX_FEE_V2")] + assethub_max_fee_v2: bool, /// Halt all parts of the bridge (equivalent to passing every other flag). #[arg(long, value_name = "HALT_SNOWBRIDGE")] all: bool, @@ -406,7 +426,7 @@ async fn run() -> Result<(), Box> { ], ) .await?; - utility_force_batch(vec![bridge_hub_call, asset_hub_call]) + utility_batch_all(vec![bridge_hub_call, asset_hub_call]) } Command::UpdateAsset(params) => { send_xcm_asset_hub( @@ -432,7 +452,7 @@ async fn run() -> Result<(), Box> { let bridge_hub_call = send_xcm_bridge_hub(&context, vec![set_pricing_parameters]).await?; let asset_hub_call = send_xcm_asset_hub(&context, vec![set_ethereum_fee]).await?; - utility_force_batch(vec![bridge_hub_call, asset_hub_call]) + utility_batch_all(vec![bridge_hub_call, asset_hub_call]) } Command::HaltBridge(params) => { let mut bh_calls = vec![]; @@ -440,12 +460,15 @@ async fn run() -> Result<(), Box> { let mut halt_all = params.all; // if no individual option specified, assume halt the whole bridge. if !params.gateway + && !params.gateway_v2 && !params.inbound_queue && !params.inbound_queue_v1 && !params.inbound_queue_v2 && !params.outbound_queue + && !params.system_frontend && !params.ethereum_client && !params.assethub_max_fee + && !params.assethub_max_fee_v2 { halt_all = true; } @@ -460,6 +483,11 @@ async fn run() -> Result<(), Box> { bh_calls.push(commands::gateway_operating_mode_v2( &GatewayOperatingModeEnum::RejectingOutboundMessages, )); + } else if params.gateway_v2 { + // V2-only: leave V1 SetOperatingMode unsent. + bh_calls.push(commands::gateway_operating_mode_v2( + &GatewayOperatingModeEnum::RejectingOutboundMessages, + )); } if params.inbound_queue || params.inbound_queue_v1 || halt_all { bh_calls.push(commands::inbound_queue_operating_mode( @@ -483,6 +511,11 @@ async fn run() -> Result<(), Box> { ah_calls.push(commands::system_frontend_operating_mode( &OperatingModeEnum::Halted, )); + } else if params.system_frontend { + // V2-only router-layer halt: AH frontend, no V1 BH outbound-queue change. + ah_calls.push(commands::system_frontend_operating_mode( + &OperatingModeEnum::Halted, + )); } if params.ethereum_client || halt_all { bh_calls.push(commands::ethereum_client_operating_mode( @@ -493,15 +526,30 @@ async fn run() -> Result<(), Box> { // Set both V1 and V2 AssetHub outbound fee storage items to u128::MAX. ah_calls.push(commands::set_assethub_fee(u128::MAX)); ah_calls.push(commands::set_assethub_fee_v2(u128::MAX)); + } else if params.assethub_max_fee_v2 { + // V2-only: leave V1 fee untouched. + ah_calls.push(commands::set_assethub_fee_v2(u128::MAX)); } - if bh_calls.len() > 0 && ah_calls.len() == 0 { + // Use `force_batch` (not `batch_all` or `batch`) so every lever fires + // independently — a single failure (e.g. HRMP transport hiccup blocking + // pallet_xcm::send to BridgeHub) must not skip the AH-side halts, and + // vice versa. Per-call failures are reported via `ItemFailed` events. + let ah_call = if ah_calls.len() == 1 { + Some(ah_calls.into_iter().next().unwrap()) + } else if ah_calls.len() > 1 { + Some(utility_force_batch(ah_calls)) + } else { + None + }; + if bh_calls.len() > 0 && ah_call.is_none() { send_xcm_bridge_hub(&context, bh_calls).await? - } else if ah_calls.len() > 0 && bh_calls.len() == 0 { - send_xcm_asset_hub(&context, ah_calls).await? + } else if ah_call.is_some() && bh_calls.len() == 0 { + ah_call.unwrap() } else { - let call1 = send_xcm_bridge_hub(&context, bh_calls).await?; - let call2 = send_xcm_asset_hub(&context, ah_calls).await?; - utility_force_batch(vec![call1, call2]) + let bh_xcm_send = send_xcm_bridge_hub(&context, bh_calls).await?; + // BH XCM-send is the first call so the V2 Gateway halt (the first + // Transact inside the XCM) is processed before any AH-side halt. + utility_force_batch(vec![bh_xcm_send, ah_call.unwrap()]) } } Command::RegisterEther(params) => { @@ -527,7 +575,7 @@ async fn run() -> Result<(), Box> { send_xcm_asset_hub(&context, vec![register_ether_call, set_ether_metadata_call]) .await?; - utility_force_batch(vec![ + utility_batch_all(vec![ bh_set_pricing_call, ah_set_pricing_call, ah_register_ether_call, @@ -552,7 +600,7 @@ async fn run() -> Result<(), Box> { { let metadata_calls = commands::register_erc20_token_metadata(); let reg_call = commands::frequency_token_registrations(); - utility_force_batch(vec![ + utility_batch_all(vec![ send_xcm_asset_hub(&context, metadata_calls).await?, send_xcm_bridge_hub(&context, reg_call).await?, ]) @@ -579,7 +627,7 @@ async fn run() -> Result<(), Box> { let outbound_fee_call = commands::set_assethub_fee_v2(1_000_000_000); let ah_xcm_call = send_xcm_asset_hub(&context, vec![outbound_fee_call]).await?; - utility_force_batch(vec![bh_xcm_call, ah_xcm_call]) + utility_batch_all(vec![bh_xcm_call, ah_xcm_call]) } } Command::ReplaySep2025 => { diff --git a/control/preimage/src/treasury_commands.rs b/control/preimage/src/treasury_commands.rs index 5329fcc225..c649d03cb6 100644 --- a/control/preimage/src/treasury_commands.rs +++ b/control/preimage/src/treasury_commands.rs @@ -9,7 +9,7 @@ use crate::asset_hub_runtime::runtime_types::{ }, }; use crate::asset_hub_runtime::RuntimeCall as AssetHubRuntimeCall; -use crate::helpers::utility_force_batch; +use crate::helpers::utility_batch_all; use polkadot_runtime_constants::currency::UNITS; use polkadot_runtime_constants::time::DAYS; @@ -189,7 +189,7 @@ pub fn treasury_proposal(params: &TreasuryProposal2024Args) -> AssetHubRuntimeCa println!("Spend: {}, {}({})", spend.name, asset_id, asset_amount); } - utility_force_batch(calls) + utility_batch_all(calls) } fn make_treasury_spend( diff --git a/docs/resources/governance-and-operational-processes.md b/docs/resources/governance-and-operational-processes.md index fa0fcdd8c8..d805596f48 100644 --- a/docs/resources/governance-and-operational-processes.md +++ b/docs/resources/governance-and-operational-processes.md @@ -69,8 +69,11 @@ The halt-bridge command has the following flags: * `--inbound-queue-v1` Halt only the V1 inbound-queue pallet on BridgeHub. * `--inbound-queue-v2` Halt only the V2 inbound-queue pallet on BridgeHub. * `--outbound-queue` Halt AssetHub → Ethereum traffic. Halts the V1 outbound-queue pallet on BridgeHub **and** the system-frontend pallet on AssetHub; the latter short-circuits the AssetHub → Ethereum `PausableExporter` for both V1 and V2 at the XcmRouter layer. (V2's `outbound-queue-v2` has no local halt, so the system-frontend halt is the primary V2 outbound lever.) +* `--system-frontend` V2-only router-layer P→E halt: halts only the AssetHub system-frontend pallet, leaving V1's BridgeHub outbound-queue running. V2 P→E messages are rejected with `SendError::NotApplicable` at the `PausableExporter`; V1 traffic continues to drain. Use this when V2 needs to stop but V1 should keep flowing. * `--gateway` Halt the Ethereum Gateway contract (both V1 and V2 paths). Sends `Command::SetOperatingMode(Halted)` via both V1 and V2 system pallets so the halt is delivered via whichever outbound queue is live. Delivery is relayer-dependent, so schedule this **before** any local outbound halt takes effect. +* `--gateway-v2` V2-only Gateway halt: sends `Command::SetOperatingMode(Halted)` only via the V2 system pallet. Once delivered to Ethereum, blocks `v2_sendMessage` and `v2_registerToken`; leaves V1 `sendToken`/`sendMessage` working. Pair with `--inbound-queue-v2`, `--system-frontend`, and `--assethub-max-fee-v2` for a full V2-only pause. * `--assethub-max-fee` Set the AssetHub → Ethereum outbound fee to `u128::MAX` for both V1 (`BridgeHubEthereumBaseFee`) and V2 (`BridgeHubEthereumBaseFeeV2`), effectively deterring user sends via fee pricing. Complementary to the system-frontend halt; does not block at the router layer. +* `--assethub-max-fee-v2` V2-only variant of `--assethub-max-fee`: writes only `BridgeHubEthereumBaseFeeV2`, leaving V1 fee untouched. Pair with `--system-frontend` for a complete V2-only P→E pause without touching V1. Based on the nature of the emergency, the bridge might need to be halted in its entirety, or partially. Pick the narrowest lever that covers the suspected failure mode: @@ -80,6 +83,8 @@ Based on the nature of the emergency, the bridge might need to be halted in its | Ethereum Gateway contract compromise | `halt-bridge --gateway --assethub-max-fee` | | Inbound-queue bug (one version) | `halt-bridge --inbound-queue-v1` or `--inbound-queue-v2` | | Outbound-queue / system-frontend bug | `halt-bridge --outbound-queue` | +| V2 P→E only (V1 keeps flowing) | `halt-bridge --system-frontend --assethub-max-fee-v2` | +| Full V2 pause (both directions, V1 keeps flowing) | `halt-bridge --gateway-v2 --inbound-queue-v2 --system-frontend --assethub-max-fee-v2` | | Uncertain / any component suspect | `halt-bridge --all` | In case of emergency where there is uncertainty of the cause of a problem, it is best to block the bridge in its entirety using `halt-bridge --all`. To block both transfer directions at the earliest point possible, use `halt-bridge --gateway --assethub-max-fee`. From 243112db393cc1f0d3b211ec801cf247f6ace3a2 Mon Sep 17 00:00:00 2001 From: claravanstaden Date: Wed, 13 May 2026 09:34:14 +0200 Subject: [PATCH 03/10] fixes --- control/preimage/src/main.rs | 14 ++-- docs/SUMMARY.md | 1 + .../governance-and-operational-processes.md | 65 +------------------ 3 files changed, 10 insertions(+), 70 deletions(-) diff --git a/control/preimage/src/main.rs b/control/preimage/src/main.rs index a71251c693..9a1b902988 100644 --- a/control/preimage/src/main.rs +++ b/control/preimage/src/main.rs @@ -237,11 +237,11 @@ pub struct HaltBridgeArgs { /// primary V2 outbound lever). #[arg(long, value_name = "HALT_OUTBOUND_QUEUE")] outbound_queue: bool, - /// V2-only router-layer P->E halt: halts only the AssetHub system-frontend pallet, - /// leaving the V1 outbound-queue on BridgeHub running. Short-circuits the - /// `PausableExporter` for V2 (and V1 — frontend halt covers both at the router - /// layer), so V2 P->E messages are rejected with `SendError::NotApplicable` while - /// V1's BH outbound-queue continues to drain in-flight messages. + /// Router-layer P->E halt: halts only the AssetHub system-frontend pallet. Blocks + /// BOTH V1 and V2 P->E at the `PausableExporter`, returning `SendError::NotApplicable` + /// to the XcmRouter. The V1 BridgeHub outbound-queue is left untouched so it keeps + /// draining in-flight V1 messages already enqueued there. There is no V2-only + /// operating-mode halt for P->E; for a V2-only deterrent use `--assethub-max-fee-v2`. #[arg(long, value_name = "HALT_SYSTEM_FRONTEND")] system_frontend: bool, /// Halt the Ethereum beacon light client, blocking new beacon-header ingestion. @@ -512,7 +512,9 @@ async fn run() -> Result<(), Box> { &OperatingModeEnum::Halted, )); } else if params.system_frontend { - // V2-only router-layer halt: AH frontend, no V1 BH outbound-queue change. + // Router-layer halt: AH frontend, blocks both V1 and V2 P->E at the + // PausableExporter. V1 BH outbound-queue left running so in-flight V1 + // messages continue to drain. ah_calls.push(commands::system_frontend_operating_mode( &OperatingModeEnum::Halted, )); diff --git a/docs/SUMMARY.md b/docs/SUMMARY.md index e6fa98ec1c..ef16dbbf37 100644 --- a/docs/SUMMARY.md +++ b/docs/SUMMARY.md @@ -41,6 +41,7 @@ * [Processes for keeping track of dependency changes](resources/processes-for-keeping-track-of-dependency-changes.md) * [Contributing to Snowbridge](resources/updating-snowbridge-pallets-bridgehub-and-assethub-runtimes.md) * [Governance and Operational Processes](resources/governance-and-operational-processes.md) +* [Emergency Procedures](resources/emergency-procedures.md) * [General Governance Updates](resources/governance-updates.md) * [Test Runtime Upgrades](resources/test-runtime-upgrades.md) * [Run Relayers](resources/run-relayers.md) diff --git a/docs/resources/governance-and-operational-processes.md b/docs/resources/governance-and-operational-processes.md index d805596f48..b2e2e05e18 100644 --- a/docs/resources/governance-and-operational-processes.md +++ b/docs/resources/governance-and-operational-processes.md @@ -28,70 +28,7 @@ We expect to need non-emergency governance calls once every few months as we imp ## Emergency Situations -For emergency situations, there are two possible scenarios: - -### 1. Emergency Pause - -In case of an emergency, a call to halt the bridge needs to be executed as soon as possible. Deploying the fix and resuming the bridge can happen afterwards with less time pressure and sensitivity, similar to a normal upgrade. - -The processes to do so is: - -{% code overflow="wrap" %} -``` -git clone https://github.com/Snowfork/snowbridge.git -cd snowbridge/control - -cargo run --bin snowbridge-preimage -- \ - --bridge-hub-api wss://polkadot-bridge-hub-rpc.polkadot.io \ - --asset-hub-api wss://polkadot-asset-hub-rpc.polkadot.io \ - halt-bridge \ - --all -``` -{% endcode %} - -The command will product a preimage hash, to be submitted to the **Whitelisted Caller Track**: - -``` -Preimage Hash: 0xc2569b432fba3b01df7da3c90bb546158480067064d4d5bc88c351fcba4355dd -Preimage Size: 129 -0x1a0408630003000100a90f03242f00000602b28d0b9859460c530101200006028217b42.. -``` - -The halt-bridge command has the following flags: - -* `--all` Halt all the bridge components at once. This is the default when no other flag is given. -* `--ethereum-client` Halt the Ethereum beacon light client. Blocks new beacon-header ingestion via `EthereumBeaconClient::submit` **and** short-circuits `Verifier::verify` for every downstream consumer on BridgeHub. Concretely this stops: - * inbound V1 `submit` and inbound V2 `submit` (Ethereum → Polkadot messages), and - * `outbound-queue-v2::submit_delivery_receipt` (relayer-reward payouts against `PendingOrders`). - - This is the single lever to pull during a suspected beacon-light-client or sync-committee compromise. `force_checkpoint` remains available (root-only) for recovery. -* `--inbound-queue` Halt both V1 and V2 inbound-queue pallets on BridgeHub, blocking processing of Ethereum → Polkadot messages. For surgical halts of a single version, use the two flags below. -* `--inbound-queue-v1` Halt only the V1 inbound-queue pallet on BridgeHub. -* `--inbound-queue-v2` Halt only the V2 inbound-queue pallet on BridgeHub. -* `--outbound-queue` Halt AssetHub → Ethereum traffic. Halts the V1 outbound-queue pallet on BridgeHub **and** the system-frontend pallet on AssetHub; the latter short-circuits the AssetHub → Ethereum `PausableExporter` for both V1 and V2 at the XcmRouter layer. (V2's `outbound-queue-v2` has no local halt, so the system-frontend halt is the primary V2 outbound lever.) -* `--system-frontend` V2-only router-layer P→E halt: halts only the AssetHub system-frontend pallet, leaving V1's BridgeHub outbound-queue running. V2 P→E messages are rejected with `SendError::NotApplicable` at the `PausableExporter`; V1 traffic continues to drain. Use this when V2 needs to stop but V1 should keep flowing. -* `--gateway` Halt the Ethereum Gateway contract (both V1 and V2 paths). Sends `Command::SetOperatingMode(Halted)` via both V1 and V2 system pallets so the halt is delivered via whichever outbound queue is live. Delivery is relayer-dependent, so schedule this **before** any local outbound halt takes effect. -* `--gateway-v2` V2-only Gateway halt: sends `Command::SetOperatingMode(Halted)` only via the V2 system pallet. Once delivered to Ethereum, blocks `v2_sendMessage` and `v2_registerToken`; leaves V1 `sendToken`/`sendMessage` working. Pair with `--inbound-queue-v2`, `--system-frontend`, and `--assethub-max-fee-v2` for a full V2-only pause. -* `--assethub-max-fee` Set the AssetHub → Ethereum outbound fee to `u128::MAX` for both V1 (`BridgeHubEthereumBaseFee`) and V2 (`BridgeHubEthereumBaseFeeV2`), effectively deterring user sends via fee pricing. Complementary to the system-frontend halt; does not block at the router layer. -* `--assethub-max-fee-v2` V2-only variant of `--assethub-max-fee`: writes only `BridgeHubEthereumBaseFeeV2`, leaving V1 fee untouched. Pair with `--system-frontend` for a complete V2-only P→E pause without touching V1. - -Based on the nature of the emergency, the bridge might need to be halted in its entirety, or partially. Pick the narrowest lever that covers the suspected failure mode: - -| Suspected failure mode | Minimum-scope command | -| --- | --- | -| Beacon light client / sync committee compromise | `halt-bridge --ethereum-client` | -| Ethereum Gateway contract compromise | `halt-bridge --gateway --assethub-max-fee` | -| Inbound-queue bug (one version) | `halt-bridge --inbound-queue-v1` or `--inbound-queue-v2` | -| Outbound-queue / system-frontend bug | `halt-bridge --outbound-queue` | -| V2 P→E only (V1 keeps flowing) | `halt-bridge --system-frontend --assethub-max-fee-v2` | -| Full V2 pause (both directions, V1 keeps flowing) | `halt-bridge --gateway-v2 --inbound-queue-v2 --system-frontend --assethub-max-fee-v2` | -| Uncertain / any component suspect | `halt-bridge --all` | - -In case of emergency where there is uncertainty of the cause of a problem, it is best to block the bridge in its entirety using `halt-bridge --all`. To block both transfer directions at the earliest point possible, use `halt-bridge --gateway --assethub-max-fee`. - -### 2. Emergency Upgrade - -Although unlikely, there may be scenarios where the emergency can only be resolved through an upgrade rather than just a pause. In this case, the code for the upgrade may be sensitive, and we would want to avoid overly publicising it until the fix has been executed. +Emergency response (halt-bridge and emergency-upgrade procedures) is documented in [Emergency Procedures](emergency-procedures.md). On-call operators should read that page directly. ## Fallback governance From 56327b2f6a36de7821d6f01b56c960b32ef62ab8 Mon Sep 17 00:00:00 2001 From: claravanstaden Date: Tue, 26 May 2026 11:28:02 +0200 Subject: [PATCH 04/10] Add Emergency Procedures docs page Adds the docs/resources/emergency-procedures.md file referenced by SUMMARY.md and governance-and-operational-processes.md in this PR, extending the on-call runbook with a Producing a halt-bridge preimage section that points to the live governance UI at app.snowbridge.network/governance alongside the snowbridge-preimage CLI. Co-Authored-By: Claude Opus 4.7 (1M context) --- docs/resources/emergency-procedures.md | 78 ++++++++++++++++++++++++++ 1 file changed, 78 insertions(+) create mode 100644 docs/resources/emergency-procedures.md diff --git a/docs/resources/emergency-procedures.md b/docs/resources/emergency-procedures.md new file mode 100644 index 0000000000..72bf61d578 --- /dev/null +++ b/docs/resources/emergency-procedures.md @@ -0,0 +1,78 @@ +# Emergency Procedures + +This page is the on-call runbook for Snowbridge emergency response. It covers the two emergency scenarios: pausing the bridge (a non-destructive operating-mode flip) and emergency upgrades. For routine governance and the broader governance model, see [Governance and Operational Processes](governance-and-operational-processes.md). + +For emergency situations, there are two possible scenarios. + +## Producing a halt-bridge preimage + +There are two interchangeable ways to produce the halt-bridge preimage. Both wrap the same underlying halt SDK and emit identical preimage bytes, so pick whichever is faster to reach during an incident. + +* **Governance page (recommended)**: browser UI at [app.snowbridge.network/governance](https://app.snowbridge.network/governance). Select the halt scope (full halt or per-component), and copy the resulting preimage hash and bytes for submission on the **Whitelisted Caller** track. No local toolchain required, useful when the on-call operator is away from their dev machine. +* **`snowbridge-preimage` CLI**: local Rust binary in the [snowbridge](https://github.com/Snowfork/snowbridge) repo under `control/preimage`. Same halt scopes as the UI, useful for scripted runs or when the governance page is unreachable. Setup and full flag reference are in the [Emergency Pause](#emergency-pause) section below. + +Both paths produce the same on-chain outcome: a preimage to submit on the **Whitelisted Caller** track. The flag semantics described under [Emergency Pause](#emergency-pause) apply to the UI form fields as well. + +## Emergency Pause + +In case of an emergency, a call to halt the bridge needs to be executed as soon as possible. Deploying the fix and resuming the bridge can happen afterwards with less time pressure and sensitivity, similar to a normal upgrade. + +The CLI process is: + +{% code overflow="wrap" %} +``` +git clone https://github.com/Snowfork/snowbridge.git +cd snowbridge/control + +cargo run --bin snowbridge-preimage -- \ + --bridge-hub-api wss://polkadot-bridge-hub-rpc.polkadot.io \ + --asset-hub-api wss://polkadot-asset-hub-rpc.polkadot.io \ + halt-bridge \ + --all +``` +{% endcode %} + +The command will produce a preimage hash, to be submitted to the **Whitelisted Caller Track**: + +``` +Preimage Hash: 0xc2569b432fba3b01df7da3c90bb546158480067064d4d5bc88c351fcba4355dd +Preimage Size: 129 +0x1a0408630003000100a90f03242f00000602b28d0b9859460c530101200006028217b42.. +``` + +The halt-bridge command has the following flags: + +* `--all` Halt all the bridge components at once. This is the default when no other flag is given. +* `--ethereum-client` Halt the Ethereum beacon light client. Blocks new beacon-header ingestion via `EthereumBeaconClient::submit` **and** short-circuits `Verifier::verify` for every downstream consumer on BridgeHub. Concretely this stops: + * inbound V1 `submit` and inbound V2 `submit` (Ethereum → Polkadot messages), and + * `outbound-queue-v2::submit_delivery_receipt` (relayer-reward payouts against `PendingOrders`). + + This is the single lever to pull during a suspected beacon-light-client or sync-committee compromise. `force_checkpoint` remains available (root-only) for recovery. +* `--inbound-queue` Halt both V1 and V2 inbound-queue pallets on BridgeHub, blocking processing of Ethereum → Polkadot messages. For surgical halts of a single version, use the two flags below. +* `--inbound-queue-v1` Halt only the V1 inbound-queue pallet on BridgeHub. +* `--inbound-queue-v2` Halt only the V2 inbound-queue pallet on BridgeHub. +* `--outbound-queue` Halt AssetHub → Ethereum outbound traffic. Halts the V1 outbound-queue pallet on BridgeHub **and** the system-frontend pallet on AssetHub; the latter short-circuits the AssetHub → Ethereum `PausableExporter` for both V1 and V2 at the XcmRouter layer. (V2's `outbound-queue-v2` has no local halt, so the system-frontend halt is the primary V2 outbound lever.) +* `--system-frontend` Router-layer P→E halt: halts only the AssetHub system-frontend pallet. Blocks **both** V1 and V2 P→E at the `PausableExporter` (returns `SendError::NotApplicable` to the XcmRouter). The V1 BridgeHub outbound-queue is left untouched, so in-flight V1 messages already enqueued there keep draining. There is no V2-only operating-mode P→E halt; use `--assethub-max-fee-v2` for a V2-only deterrent. +* `--gateway` Halt the Ethereum Gateway contract (both V1 and V2 paths). Sends `Command::SetOperatingMode(Halted)` via both V1 and V2 system pallets so the halt is delivered via whichever outbound queue is live. Delivery is relayer-dependent, so schedule this **before** any local outbound halt takes effect. +* `--gateway-v2` V2-only Gateway halt: sends `Command::SetOperatingMode(Halted)` only via the V2 system pallet. Once delivered to Ethereum, blocks `v2_sendMessage` and `v2_registerToken`; leaves V1 `sendToken`/`sendMessage` working. Pair with `--inbound-queue-v2` and `--assethub-max-fee-v2` for the closest thing to a V2-only pause (no V2-only operating-mode P→E halt exists). +* `--assethub-max-fee` Set the AssetHub → Ethereum outbound fee to `u128::MAX` for both V1 (`BridgeHubEthereumBaseFee`) and V2 (`BridgeHubEthereumBaseFeeV2`), effectively deterring user sends via fee pricing. Complementary to the system-frontend halt; does not block at the router layer. +* `--assethub-max-fee-v2` V2-only variant of `--assethub-max-fee`: writes only `BridgeHubEthereumBaseFeeV2`, leaving V1 fee untouched. This is the only V2-isolated P→E lever (fee deterrent, not a hard halt). + +Based on the nature of the emergency, the bridge might need to be halted in its entirety, or partially. Pick the narrowest lever that covers the suspected failure mode: + +| Suspected failure mode | Minimum-scope command | +| --- | --- | +| Beacon light client / sync committee compromise | `halt-bridge --ethereum-client` | +| Ethereum Gateway contract compromise | `halt-bridge --gateway --assethub-max-fee` | +| Inbound-queue bug (one version) | `halt-bridge --inbound-queue-v1` or `--inbound-queue-v2` | +| Outbound-queue / system-frontend bug | `halt-bridge --outbound-queue` | +| V2 P→E only (V1 keeps flowing) | `halt-bridge --assethub-max-fee-v2` (fee deterrent, no operating-mode halt) | +| Full V2 pause (V1 keeps flowing, P→E is fee-deterrent only) | `halt-bridge --gateway-v2 --inbound-queue-v2 --assethub-max-fee-v2` | +| Full P→E halt (both V1 and V2) | `halt-bridge --outbound-queue` or `halt-bridge --system-frontend` | +| Uncertain / any component suspect | `halt-bridge --all` | + +In case of emergency where there is uncertainty of the cause of a problem, it is best to block the bridge in its entirety using `halt-bridge --all`. To block both transfer directions at the earliest point possible, use `halt-bridge --gateway --assethub-max-fee`. + +## Emergency Upgrade + +Although unlikely, there may be scenarios where the emergency can only be resolved through an upgrade rather than just a pause. In this case, the code for the upgrade may be sensitive, and we would want to avoid overly publicising it until the fix has been executed. From 5c9c4a8fe64c5a373cc60600722d279a7d63752b Mon Sep 17 00:00:00 2001 From: claravanstaden Date: Tue, 26 May 2026 12:19:14 +0200 Subject: [PATCH 05/10] Expand Emergency Procedures with incident runbook Adds the operational runbook sections around the existing halt-bridge command reference: Detection signals (including bug bounty reports and visibly drained funds), Decision authority (solo halt only for visible exploits, otherwise 2/3 confirm to avoid spooking the community with a halt referendum), Comms during an incident (Slack first, then Element with Parity, then affected integrators, no public comms until fix is deployed), Submitting the preimage (Whitelisted Caller flow with Fellowship-whitelist timing), Verifying the halt (per-flag storage queries with Gateway-side caveat), Resuming the bridge (governance UI is the only practical path), and Post-mortem (Google Doc, 48h SLA). Also reframes the Producing a halt-bridge preimage and Resuming the bridge sections to make app.snowbridge.network/governance the primary path, with the snowbridge-preimage CLI as a fallback only. Co-Authored-By: Claude Opus 4.7 (1M context) --- docs/resources/emergency-procedures.md | 106 +++++++++++++++++++++++-- 1 file changed, 99 insertions(+), 7 deletions(-) diff --git a/docs/resources/emergency-procedures.md b/docs/resources/emergency-procedures.md index 72bf61d578..2065db6112 100644 --- a/docs/resources/emergency-procedures.md +++ b/docs/resources/emergency-procedures.md @@ -1,17 +1,49 @@ # Emergency Procedures -This page is the on-call runbook for Snowbridge emergency response. It covers the two emergency scenarios: pausing the bridge (a non-destructive operating-mode flip) and emergency upgrades. For routine governance and the broader governance model, see [Governance and Operational Processes](governance-and-operational-processes.md). +This page is the on-call runbook for Snowbridge emergency response. It covers detection, decision-making, comms, the two emergency scenarios (pausing the bridge and emergency upgrades), and post-incident steps. For routine governance and the broader governance model, see [Governance and Operational Processes](governance-and-operational-processes.md). -For emergency situations, there are two possible scenarios. +The Snowbridge team is small (three people) and operates all-hands during an incident. Halting the bridge is technically reversible (it's a non-destructive operating-mode flip), but the halt referendum appearing on Polkassembly is itself a public signal that something is wrong, so it isn't free. The thresholds in [Decision authority](#decision-authority) are calibrated accordingly: solo action is reserved for cases where the incident is already publicly visible (funds being drained, active exploit), and confirmation-before-action is the default for everything else. + +## Detection + +Anything on this list is a valid trigger to start the incident flow. When in doubt, post in Slack and treat it as an incident until ruled out. + +* **Funds drained or unexpectedly moved** on either side of the bridge (highest priority, halt first, investigate after). +* **Bug bounty report** (Immunefi or direct) describing an exploitable vulnerability, especially one with a proof-of-concept. +* **Unexpected Gateway events on Ethereum**: messages with unknown nonces, forged commands, or `Command::*` events not produced by a known relayer run. +* **Beacon light-client anomalies**: stalled or non-progressing beacon headers on BridgeHub, unexpected sync-committee rotation, or failures in `EthereumBeaconClient::submit`. +* **Queue depth or message-processing anomalies**: inbound/outbound queues backing up, repeated `Verifier::verify` failures, or messages stuck in pending state. +* **User reports** of failed or missing transfers that don't match a known issue, especially from integration partners (Hydration, others). +* **Anything one of the three of you doesn't immediately understand** and can't rule out within a few minutes. Halt first. + +## Decision authority + +To keep the path to halt short without creating unnecessary alarm: + +* **Solo halt** (any 1 of 3, no confirmation needed): reserved for cases where the incident is already publicly visible and acting fast matters more than coordinating. Concretely: **funds are visibly being drained on-chain, or an active exploit is in progress**. In these cases the halt referendum appearing on-chain isn't new information to the community, the exploit already is. +* **Confirmed halt** (at least 2 of 3 agree before submitting): the default for everything else, including bug bounty reports without active exploitation, anomalies, unexplained queue behaviour, and "I don't understand what I'm seeing." A halt referendum appearing on Polkassembly is itself a public signal that something is wrong, so absent a visible exploit it's worth a few minutes of Slack discussion to confirm before submitting. +* **Escalating to Parity**: at least 2 of 3 team members agree there is a genuine incident. Same threshold as a confirmed halt; in practice these decisions happen together. This is the gate for creating the cross-team Element channel. +* **Public comms**: all three team members agree, and only after the fix is deployed and the bridge is resuming. See [Comms during an incident](#comms-during-an-incident). +* **Emergency upgrade**: coordinated decision with Parity once they are in the loop, since upgrade code is often exploit-sensitive. + +## Comms during an incident + +Order matters. Each step assumes the previous one has happened. + +1. **Team Slack**: post what you've seen in the team channel, include links to the signal (block explorer, alert, bounty report). For non-visible signals (bug reports, anomalies), wait for at least one teammate to confirm before submitting a halt referendum; see [Decision authority](#decision-authority). For a visible exploit or funds being drained, skip ahead and halt now. +2. **Halt the bridge** (if warranted) using the steps in [Producing a halt-bridge preimage](#producing-a-halt-bridge-preimage) and [Submitting the preimage](#submitting-the-preimage). For visible exploits, do this in parallel with steps 3 and 4 below. +3. **Internal confirmation**: once 2 of 3 team members have agreed there is a genuine incident, proceed to step 4. For solo-halt cases this is retroactive: the other two confirm as they come online. +4. **Element channel with Parity**: create a new Element room and invite the Parity Bridges contacts (Adrian, Bastian, Oliver). This is where Fellowship coordination for the Whitelisted Caller submission happens, and where Parity gets the context they need to help. +5. **Affected integrators**: contact integration partners directly (Hydration first, then others) once Parity is in the loop. Channel: whatever direct line you have with them (Element, Slack Connect, Signal). Tell them what's halted and the expected resume timing. +6. **No public comms** (Twitter/X, forum, blog, public Discord) until the fix is deployed and resume is in flight. Premature public disclosure of an unfixed vulnerability is worse than silence. ## Producing a halt-bridge preimage -There are two interchangeable ways to produce the halt-bridge preimage. Both wrap the same underlying halt SDK and emit identical preimage bytes, so pick whichever is faster to reach during an incident. +**Retrieve the halt-bridge preimage from the governance page at [app.snowbridge.network/governance](https://app.snowbridge.network/governance).** Select the halt scope (full halt or per-component), and copy the resulting preimage hash and bytes for submission on the **Whitelisted Caller** track. No local toolchain required, which matters when the on-call operator is away from their dev machine. -* **Governance page (recommended)**: browser UI at [app.snowbridge.network/governance](https://app.snowbridge.network/governance). Select the halt scope (full halt or per-component), and copy the resulting preimage hash and bytes for submission on the **Whitelisted Caller** track. No local toolchain required, useful when the on-call operator is away from their dev machine. -* **`snowbridge-preimage` CLI**: local Rust binary in the [snowbridge](https://github.com/Snowfork/snowbridge) repo under `control/preimage`. Same halt scopes as the UI, useful for scripted runs or when the governance page is unreachable. Setup and full flag reference are in the [Emergency Pause](#emergency-pause) section below. +The flag semantics described under [Emergency Pause](#emergency-pause) apply to the governance page's form fields as well; that section is the canonical reference for what each scope does. -Both paths produce the same on-chain outcome: a preimage to submit on the **Whitelisted Caller** track. The flag semantics described under [Emergency Pause](#emergency-pause) apply to the UI form fields as well. +**Fallback only if the governance page is unreachable**: the `snowbridge-preimage` CLI in the [snowbridge](https://github.com/Snowfork/snowbridge) repo under `control/preimage` produces the same preimage bytes. Setup and full flag reference are in the [Emergency Pause](#emergency-pause) section below. Use it only when the UI is down, since the UI is the single source of truth the team is expected to drive from during an incident. ## Emergency Pause @@ -73,6 +105,66 @@ Based on the nature of the emergency, the bridge might need to be halted in its In case of emergency where there is uncertainty of the cause of a problem, it is best to block the bridge in its entirety using `halt-bridge --all`. To block both transfer directions at the earliest point possible, use `halt-bridge --gateway --assethub-max-fee`. +## Submitting the preimage + +Halt preimages go through Polkadot OpenGov's **Whitelisted Caller** track. This is the fastest emergency track available, but it is not instantaneous: + +1. **Submit the preimage** on Polkadot via Polkassembly/Subsquare or directly via the preimage pallet, using the bytes from the previous step. Note the preimage hash. +2. **Whitelist the call hash**: the Whitelisted Caller track requires the specific call hash to be whitelisted by the Polkadot Fellowship via a Fellowship referendum. This is the bottleneck. In the Element channel with Parity (Adrian, Bastian, Oliver), share the preimage hash and ask them to coordinate a Fellowship referendum to whitelist it. Parity has Fellowship members and direct lines to others. +3. **Submit the Whitelisted Caller referendum** referencing the preimage. Once the Fellowship referendum to whitelist passes, the Whitelisted Caller referendum's confirmation period begins. +4. **Wall-clock expectation**: from preimage produced to call executed on-chain depends on Fellowship response time. Plan for the order of hours, not minutes, even on the emergency track. This is why the order in [Comms during an incident](#comms-during-an-incident) puts Parity escalation in parallel with halt submission. + +## Verifying the halt + +Once the call executes, verify the halt actually took effect on each affected chain. Don't trust the referendum closing as sufficient evidence, query state. + +Per-component checks, all reading the pallet's `OperatingMode` storage item (expected value: `Halted`): + +| Halt flag | Chain | Storage item to query | +| --- | --- | --- | +| `--ethereum-client` | BridgeHub | `ethereumBeaconClient.operatingMode` | +| `--inbound-queue-v1` | BridgeHub | `ethereumInboundQueue.operatingMode` | +| `--inbound-queue-v2` | BridgeHub | `ethereumInboundQueueV2.operatingMode` | +| `--outbound-queue` (BridgeHub side) | BridgeHub | `ethereumOutboundQueue.operatingMode` | +| `--outbound-queue` / `--system-frontend` (AssetHub side) | AssetHub | `systemFrontend.operatingMode` (the snowbridge-system-frontend pallet) | +| `--gateway` (local pallet state) | BridgeHub | `ethereumSystem.operatingMode` and `ethereumSystemV2.operatingMode` | +| `--gateway` (actual contract state on Ethereum) | Ethereum | `Gateway.operatingMode()` returns `Halted`. **This step depends on relayer delivery**, so the on-chain Polkadot referendum executing does not mean the Ethereum contract is halted yet. Watch for the `SetOperatingMode` event on the Gateway and confirm operating mode after expected delivery. | +| `--assethub-max-fee` | AssetHub | `bridgeHubEthereumBaseFee` and `bridgeHubEthereumBaseFeeV2` equal `u128::MAX` (`340282366920938463463374607431768211455`) | +| `--assethub-max-fee-v2` | AssetHub | `bridgeHubEthereumBaseFeeV2` equals `u128::MAX` | + +If the Polkadot side is halted but the Gateway contract is not (because no relayer has delivered the `SetOperatingMode` command yet), users can still call `sendToken` / `sendMessage` on Ethereum, those messages just won't be processed downstream. Treat the Polkadot-side halt as the firm guarantee; the Gateway-side halt as a follow-up. + +## Resuming the bridge + +Once the fix is deployed and the root cause is understood, **retrieve the resume-bridge preimage from the same governance page at [app.snowbridge.network/governance](https://app.snowbridge.network/governance)**. Select the resume action and the scopes to resume (typically matching whatever was halted), and copy the preimage hash and bytes for submission on the **Whitelisted Caller** track. + +The `snowbridge-preimage` CLI does not currently have a top-level `resume-bridge` subcommand, so the governance page is the only practical path. If you need to script it (rare), the underlying SDK function is `buildResumeBridgePreimage` in `@snowbridge/api`. + +Before submitting the resume preimage, confirm: + +* The fix is deployed and verified in production (runtime upgrade live, contracts upgraded if applicable). +* All three team members have signed off on resuming. +* Monitoring is back to normal baselines and there are no fresh anomalies. +* AssetHub fee values are being restored to their pre-incident values (the resume preimage writes `BridgeHubEthereumBaseFee` / `BridgeHubEthereumBaseFeeV2` back to known good defaults, currently `14_929_540_998` for V1 and `1_000_000_000` for V2; double-check these match what was live before the incident). + +The resume preimage submission follows the same Whitelisted Caller flow as [Submitting the preimage](#submitting-the-preimage). Public comms can begin once the resume call has executed and the bridge is observed processing messages again. + ## Emergency Upgrade -Although unlikely, there may be scenarios where the emergency can only be resolved through an upgrade rather than just a pause. In this case, the code for the upgrade may be sensitive, and we would want to avoid overly publicising it until the fix has been executed. +Although unlikely, there may be scenarios where the emergency can only be resolved through an upgrade rather than a pause, for example a critical bug in a pallet's logic that even a halt doesn't fully contain. In this case: + +* **Halt first anyway**. A halt buys time to develop, review, and deploy the upgrade without time pressure. The exceptions are situations where the halt itself would be more harmful than the bug, which is rare. +* **Keep the upgrade code restricted**. Upgrade code for a known unpatched vulnerability is itself exploit material. Develop in a private branch, limit reviewers to the team plus the necessary Parity contacts, and avoid publicising the patch until it has executed on-chain. +* **Coordinate with Parity** on the upgrade pathway (Whitelisted Caller for runtime upgrades, multi-sig for contract upgrades, etc.). The Element channel from [Comms during an incident](#comms-during-an-incident) is the right venue. +* **Resume the bridge** only after the upgrade is verified live, following the [Resuming the bridge](#resuming-the-bridge) checklist. + +## Post-mortem + +Within 48 hours of the bridge being resumed (or sooner if everyone has bandwidth): + +* **Owner**: whichever team member drove the incident (whoever halted first by default). +* **Format**: Google Doc, shared with the team and the Parity contacts who were in the Element channel. +* **Minimum contents**: timeline of events (with timestamps), root cause, what the halt scope was and why, what worked in the response, what didn't, action items with owners and target dates. +* **Action items**: track in the team's normal issue tracker, not the doc itself, so they don't get lost when the doc goes stale. + +Even for false-positive halts (incident turned out to be non-incident), write a short post-mortem. Tuning detection signals to reduce false positives is itself a useful output. From 13f480c046c2ad0044f3fb900cd53085f9b15c66 Mon Sep 17 00:00:00 2001 From: claravanstaden Date: Tue, 26 May 2026 13:59:07 +0200 Subject: [PATCH 06/10] Iterate Emergency Procedures based on team feedback - Detection: trim to drained funds + verified-PoC bug bounty (HackenProof or direct); drop generic anomaly bullets. - Decision authority: distinguish solo halt (visible exploit, funds draining) from confirmed halt (2 team members agree) to avoid spooking the community with a halt referendum for a maybe-incident. Remove team-size denominators so the doc survives team-size changes. - Comms: name the #snowbridge-security Slack channel and Telegram as the integrator channel. - Halt path: governance page is primary; SDK (buildHaltBridgePreimage / buildResumeBridgePreimage) is the fallback. Drop snowbridge-preimage CLI from the runbook. - Submission: document the actual opengov-cli flow: Asset Hub batch (preimage note + public referendum, anyone) and Collectives Chain batch (Fellowship whitelist, rank-3+ Fellow only). Include the enactment-time default caveat. - Rename Emergency Pause to Halt scopes reference; rewrite scope list and failure-mode + verification tables to use UI form-field names instead of CLI flag names. Co-Authored-By: Claude Opus 4.7 (1M context) --- docs/resources/emergency-procedures.md | 162 +++++++++++-------------- 1 file changed, 74 insertions(+), 88 deletions(-) diff --git a/docs/resources/emergency-procedures.md b/docs/resources/emergency-procedures.md index 2065db6112..4afd217d63 100644 --- a/docs/resources/emergency-procedures.md +++ b/docs/resources/emergency-procedures.md @@ -2,117 +2,103 @@ This page is the on-call runbook for Snowbridge emergency response. It covers detection, decision-making, comms, the two emergency scenarios (pausing the bridge and emergency upgrades), and post-incident steps. For routine governance and the broader governance model, see [Governance and Operational Processes](governance-and-operational-processes.md). -The Snowbridge team is small (three people) and operates all-hands during an incident. Halting the bridge is technically reversible (it's a non-destructive operating-mode flip), but the halt referendum appearing on Polkassembly is itself a public signal that something is wrong, so it isn't free. The thresholds in [Decision authority](#decision-authority) are calibrated accordingly: solo action is reserved for cases where the incident is already publicly visible (funds being drained, active exploit), and confirmation-before-action is the default for everything else. +The Snowbridge team is small and operates all-hands during an incident. Halting the bridge is technically reversible (it's a non-destructive operating-mode flip), but the halt referendum appearing on Polkassembly is itself a public signal that something is wrong, so it isn't free. The thresholds in [Decision authority](#decision-authority) are calibrated accordingly: solo action is reserved for cases where the incident is already publicly visible (funds being drained, active exploit), and confirmation-before-action is the default for everything else. ## Detection Anything on this list is a valid trigger to start the incident flow. When in doubt, post in Slack and treat it as an incident until ruled out. * **Funds drained or unexpectedly moved** on either side of the bridge (highest priority, halt first, investigate after). -* **Bug bounty report** (Immunefi or direct) describing an exploitable vulnerability, especially one with a proof-of-concept. -* **Unexpected Gateway events on Ethereum**: messages with unknown nonces, forged commands, or `Command::*` events not produced by a known relayer run. -* **Beacon light-client anomalies**: stalled or non-progressing beacon headers on BridgeHub, unexpected sync-committee rotation, or failures in `EthereumBeaconClient::submit`. -* **Queue depth or message-processing anomalies**: inbound/outbound queues backing up, repeated `Verifier::verify` failures, or messages stuck in pending state. -* **User reports** of failed or missing transfers that don't match a known issue, especially from integration partners (Hydration, others). -* **Anything one of the three of you doesn't immediately understand** and can't rule out within a few minutes. Halt first. +* **Bug bounty report** (HackenProof or direct) describing an exploitable vulnerability, verified by a team member as a valid exploit with a working proof-of-concept. ## Decision authority To keep the path to halt short without creating unnecessary alarm: -* **Solo halt** (any 1 of 3, no confirmation needed): reserved for cases where the incident is already publicly visible and acting fast matters more than coordinating. Concretely: **funds are visibly being drained on-chain, or an active exploit is in progress**. In these cases the halt referendum appearing on-chain isn't new information to the community, the exploit already is. -* **Confirmed halt** (at least 2 of 3 agree before submitting): the default for everything else, including bug bounty reports without active exploitation, anomalies, unexplained queue behaviour, and "I don't understand what I'm seeing." A halt referendum appearing on Polkassembly is itself a public signal that something is wrong, so absent a visible exploit it's worth a few minutes of Slack discussion to confirm before submitting. -* **Escalating to Parity**: at least 2 of 3 team members agree there is a genuine incident. Same threshold as a confirmed halt; in practice these decisions happen together. This is the gate for creating the cross-team Element channel. -* **Public comms**: all three team members agree, and only after the fix is deployed and the bridge is resuming. See [Comms during an incident](#comms-during-an-incident). +* **Solo halt** (1 team member, no confirmation needed): reserved for cases where the incident is already publicly visible and acting fast matters more than coordinating. Concretely: **funds are visibly being drained on-chain, or an active exploit is in progress**. In these cases the halt referendum appearing on-chain isn't new information to the community, the exploit already is. +* **Confirmed halt** (at least 2 team members agree before submitting): the default for everything else, including bug bounty reports without active exploitation, anomalies, unexplained queue behaviour, and "I don't understand what I'm seeing." A halt referendum appearing on Polkassembly is itself a public signal that something is wrong, so absent a visible exploit it's worth a few minutes of Slack discussion to confirm before submitting. +* **Escalating to Parity**: at least 2 team members agree there is a genuine incident. Same threshold as a confirmed halt; in practice these decisions happen together. This is the gate for creating the cross-team Element channel. +* **Public comms**: full team agreement, and only after the fix is deployed and the bridge is resuming. See [Comms during an incident](#comms-during-an-incident). * **Emergency upgrade**: coordinated decision with Parity once they are in the loop, since upgrade code is often exploit-sensitive. ## Comms during an incident -Order matters. Each step assumes the previous one has happened. +Each step assumes the previous one has happened. -1. **Team Slack**: post what you've seen in the team channel, include links to the signal (block explorer, alert, bounty report). For non-visible signals (bug reports, anomalies), wait for at least one teammate to confirm before submitting a halt referendum; see [Decision authority](#decision-authority). For a visible exploit or funds being drained, skip ahead and halt now. +1. **Team Slack**: post what you've seen in `#snowbridge-security`, include links to the signal (block explorer, alert, bounty report). For non-visible signals (bug reports, anomalies), wait for at least one teammate to confirm before submitting a halt referendum; see [Decision authority](#decision-authority). For a visible exploit or funds being drained, skip ahead and halt now. 2. **Halt the bridge** (if warranted) using the steps in [Producing a halt-bridge preimage](#producing-a-halt-bridge-preimage) and [Submitting the preimage](#submitting-the-preimage). For visible exploits, do this in parallel with steps 3 and 4 below. -3. **Internal confirmation**: once 2 of 3 team members have agreed there is a genuine incident, proceed to step 4. For solo-halt cases this is retroactive: the other two confirm as they come online. +3. **Internal confirmation**: once at least 2 team members have agreed there is a genuine incident, proceed to step 4. For solo-halt cases this is retroactive: the rest of the team confirms as they come online. 4. **Element channel with Parity**: create a new Element room and invite the Parity Bridges contacts (Adrian, Bastian, Oliver). This is where Fellowship coordination for the Whitelisted Caller submission happens, and where Parity gets the context they need to help. -5. **Affected integrators**: contact integration partners directly (Hydration first, then others) once Parity is in the loop. Channel: whatever direct line you have with them (Element, Slack Connect, Signal). Tell them what's halted and the expected resume timing. +5. **Affected integrators**: contact integration partners directly (Hydration first, then others) once Parity is in the loop. Channel: Telegram. Tell them what's halted and the expected resume timing. 6. **No public comms** (Twitter/X, forum, blog, public Discord) until the fix is deployed and resume is in flight. Premature public disclosure of an unfixed vulnerability is worse than silence. ## Producing a halt-bridge preimage -**Retrieve the halt-bridge preimage from the governance page at [app.snowbridge.network/governance](https://app.snowbridge.network/governance).** Select the halt scope (full halt or per-component), and copy the resulting preimage hash and bytes for submission on the **Whitelisted Caller** track. No local toolchain required, which matters when the on-call operator is away from their dev machine. +**Retrieve the halt-bridge preimage from the governance page at [app.snowbridge.network/governance](https://app.snowbridge.network/governance).** Select the halt scope (full halt or per-component), and copy the resulting preimage hash and bytes. From here, the preimage bytes are fed into `opengov-cli` to generate the on-chain referendum calls; see [Submitting the preimage](#submitting-the-preimage). -The flag semantics described under [Emergency Pause](#emergency-pause) apply to the governance page's form fields as well; that section is the canonical reference for what each scope does. +See [Halt scopes reference](#halt-scopes-reference) for the canonical description of what each scope on the governance page actually does, and the failure-mode to scope mapping. -**Fallback only if the governance page is unreachable**: the `snowbridge-preimage` CLI in the [snowbridge](https://github.com/Snowfork/snowbridge) repo under `control/preimage` produces the same preimage bytes. Setup and full flag reference are in the [Emergency Pause](#emergency-pause) section below. Use it only when the UI is down, since the UI is the single source of truth the team is expected to drive from during an incident. +**Fallback only if the governance page is unreachable**: call the `buildHaltBridgePreimage` function from the `@snowbridge/api` SDK directly (this is what the governance page wraps). It takes a `HaltBridgeOptions` object whose keys match the scopes listed in [Halt scopes reference](#halt-scopes-reference), and returns the same preimage hash and bytes. Use this only when the UI is down, since the UI is the single source of truth the team is expected to drive from during an incident. -## Emergency Pause +## Halt scopes reference -In case of an emergency, a call to halt the bridge needs to be executed as soon as possible. Deploying the fix and resuming the bridge can happen afterwards with less time pressure and sensitivity, similar to a normal upgrade. +The governance page exposes the following halt scopes as form fields. Pick the narrowest scope that covers the suspected failure mode. -The CLI process is: - -{% code overflow="wrap" %} -``` -git clone https://github.com/Snowfork/snowbridge.git -cd snowbridge/control - -cargo run --bin snowbridge-preimage -- \ - --bridge-hub-api wss://polkadot-bridge-hub-rpc.polkadot.io \ - --asset-hub-api wss://polkadot-asset-hub-rpc.polkadot.io \ - halt-bridge \ - --all -``` -{% endcode %} - -The command will produce a preimage hash, to be submitted to the **Whitelisted Caller Track**: - -``` -Preimage Hash: 0xc2569b432fba3b01df7da3c90bb546158480067064d4d5bc88c351fcba4355dd -Preimage Size: 129 -0x1a0408630003000100a90f03242f00000602b28d0b9859460c530101200006028217b42.. -``` - -The halt-bridge command has the following flags: - -* `--all` Halt all the bridge components at once. This is the default when no other flag is given. -* `--ethereum-client` Halt the Ethereum beacon light client. Blocks new beacon-header ingestion via `EthereumBeaconClient::submit` **and** short-circuits `Verifier::verify` for every downstream consumer on BridgeHub. Concretely this stops: +* **All** Halt all the bridge components at once. This is the default when no other scope is selected. +* **Ethereum client** Halt the Ethereum beacon light client. Blocks new beacon-header ingestion via `EthereumBeaconClient::submit` **and** short-circuits `Verifier::verify` for every downstream consumer on BridgeHub. Concretely this stops: * inbound V1 `submit` and inbound V2 `submit` (Ethereum → Polkadot messages), and * `outbound-queue-v2::submit_delivery_receipt` (relayer-reward payouts against `PendingOrders`). This is the single lever to pull during a suspected beacon-light-client or sync-committee compromise. `force_checkpoint` remains available (root-only) for recovery. -* `--inbound-queue` Halt both V1 and V2 inbound-queue pallets on BridgeHub, blocking processing of Ethereum → Polkadot messages. For surgical halts of a single version, use the two flags below. -* `--inbound-queue-v1` Halt only the V1 inbound-queue pallet on BridgeHub. -* `--inbound-queue-v2` Halt only the V2 inbound-queue pallet on BridgeHub. -* `--outbound-queue` Halt AssetHub → Ethereum outbound traffic. Halts the V1 outbound-queue pallet on BridgeHub **and** the system-frontend pallet on AssetHub; the latter short-circuits the AssetHub → Ethereum `PausableExporter` for both V1 and V2 at the XcmRouter layer. (V2's `outbound-queue-v2` has no local halt, so the system-frontend halt is the primary V2 outbound lever.) -* `--system-frontend` Router-layer P→E halt: halts only the AssetHub system-frontend pallet. Blocks **both** V1 and V2 P→E at the `PausableExporter` (returns `SendError::NotApplicable` to the XcmRouter). The V1 BridgeHub outbound-queue is left untouched, so in-flight V1 messages already enqueued there keep draining. There is no V2-only operating-mode P→E halt; use `--assethub-max-fee-v2` for a V2-only deterrent. -* `--gateway` Halt the Ethereum Gateway contract (both V1 and V2 paths). Sends `Command::SetOperatingMode(Halted)` via both V1 and V2 system pallets so the halt is delivered via whichever outbound queue is live. Delivery is relayer-dependent, so schedule this **before** any local outbound halt takes effect. -* `--gateway-v2` V2-only Gateway halt: sends `Command::SetOperatingMode(Halted)` only via the V2 system pallet. Once delivered to Ethereum, blocks `v2_sendMessage` and `v2_registerToken`; leaves V1 `sendToken`/`sendMessage` working. Pair with `--inbound-queue-v2` and `--assethub-max-fee-v2` for the closest thing to a V2-only pause (no V2-only operating-mode P→E halt exists). -* `--assethub-max-fee` Set the AssetHub → Ethereum outbound fee to `u128::MAX` for both V1 (`BridgeHubEthereumBaseFee`) and V2 (`BridgeHubEthereumBaseFeeV2`), effectively deterring user sends via fee pricing. Complementary to the system-frontend halt; does not block at the router layer. -* `--assethub-max-fee-v2` V2-only variant of `--assethub-max-fee`: writes only `BridgeHubEthereumBaseFeeV2`, leaving V1 fee untouched. This is the only V2-isolated P→E lever (fee deterrent, not a hard halt). - -Based on the nature of the emergency, the bridge might need to be halted in its entirety, or partially. Pick the narrowest lever that covers the suspected failure mode: - -| Suspected failure mode | Minimum-scope command | +* **Inbound queue** Halt both V1 and V2 inbound-queue pallets on BridgeHub, blocking processing of Ethereum → Polkadot messages. For surgical halts of a single version, use the two scopes below. +* **Inbound queue V1** Halt only the V1 inbound-queue pallet on BridgeHub. +* **Inbound queue V2** Halt only the V2 inbound-queue pallet on BridgeHub. +* **Outbound queue** Halt AssetHub → Ethereum outbound traffic. Halts the V1 outbound-queue pallet on BridgeHub **and** the system-frontend pallet on AssetHub; the latter short-circuits the AssetHub → Ethereum `PausableExporter` for both V1 and V2 at the XcmRouter layer. (V2's `outbound-queue-v2` has no local halt, so the system-frontend halt is the primary V2 outbound lever.) +* **System frontend** Router-layer P→E halt: halts only the AssetHub system-frontend pallet. Blocks **both** V1 and V2 P→E at the `PausableExporter` (returns `SendError::NotApplicable` to the XcmRouter). The V1 BridgeHub outbound-queue is left untouched, so in-flight V1 messages already enqueued there keep draining. There is no V2-only operating-mode P→E halt; use **AssetHub max fee V2** for a V2-only deterrent. +* **Gateway** Halt the Ethereum Gateway contract (both V1 and V2 paths). Sends `Command::SetOperatingMode(Halted)` via both V1 and V2 system pallets so the halt is delivered via whichever outbound queue is live. Delivery is relayer-dependent, so schedule this **before** any local outbound halt takes effect. +* **Gateway V2** V2-only Gateway halt: sends `Command::SetOperatingMode(Halted)` only via the V2 system pallet. Once delivered to Ethereum, blocks `v2_sendMessage` and `v2_registerToken`; leaves V1 `sendToken`/`sendMessage` working. Pair with **Inbound queue V2** and **AssetHub max fee V2** for the closest thing to a V2-only pause (no V2-only operating-mode P→E halt exists). +* **AssetHub max fee** Set the AssetHub → Ethereum outbound fee to `u128::MAX` for both V1 (`BridgeHubEthereumBaseFee`) and V2 (`BridgeHubEthereumBaseFeeV2`), effectively deterring user sends via fee pricing. Complementary to the system-frontend halt; does not block at the router layer. +* **AssetHub max fee V2** V2-only variant of **AssetHub max fee**: writes only `BridgeHubEthereumBaseFeeV2`, leaving V1 fee untouched. This is the only V2-isolated P→E lever (fee deterrent, not a hard halt). + +Failure-mode to scope mapping: + +| Suspected failure mode | Minimum scopes to select | | --- | --- | -| Beacon light client / sync committee compromise | `halt-bridge --ethereum-client` | -| Ethereum Gateway contract compromise | `halt-bridge --gateway --assethub-max-fee` | -| Inbound-queue bug (one version) | `halt-bridge --inbound-queue-v1` or `--inbound-queue-v2` | -| Outbound-queue / system-frontend bug | `halt-bridge --outbound-queue` | -| V2 P→E only (V1 keeps flowing) | `halt-bridge --assethub-max-fee-v2` (fee deterrent, no operating-mode halt) | -| Full V2 pause (V1 keeps flowing, P→E is fee-deterrent only) | `halt-bridge --gateway-v2 --inbound-queue-v2 --assethub-max-fee-v2` | -| Full P→E halt (both V1 and V2) | `halt-bridge --outbound-queue` or `halt-bridge --system-frontend` | -| Uncertain / any component suspect | `halt-bridge --all` | +| Beacon light client / sync committee compromise | **Ethereum client** | +| Ethereum Gateway contract compromise | **Gateway** + **AssetHub max fee** | +| Inbound-queue bug (one version) | **Inbound queue V1** or **Inbound queue V2** | +| Outbound-queue / system-frontend bug | **Outbound queue** | +| V2 P→E only (V1 keeps flowing) | **AssetHub max fee V2** (fee deterrent, no operating-mode halt) | +| Full V2 pause (V1 keeps flowing, P→E is fee-deterrent only) | **Gateway V2** + **Inbound queue V2** + **AssetHub max fee V2** | +| Full P→E halt (both V1 and V2) | **Outbound queue** or **System frontend** | +| Uncertain / any component suspect | **All** | -In case of emergency where there is uncertainty of the cause of a problem, it is best to block the bridge in its entirety using `halt-bridge --all`. To block both transfer directions at the earliest point possible, use `halt-bridge --gateway --assethub-max-fee`. +In case of emergency where there is uncertainty of the cause of a problem, it is best to block the bridge in its entirety using **All**. To block both transfer directions at the earliest point possible, use **Gateway** + **AssetHub max fee**. ## Submitting the preimage -Halt preimages go through Polkadot OpenGov's **Whitelisted Caller** track. This is the fastest emergency track available, but it is not instantaneous: +Halt preimages go through Polkadot OpenGov's **Whitelisted Caller** track, which requires the call hash to be whitelisted by the Polkadot Fellowship first. The submission flow is built using the [opengov-cli](https://github.com/joepetrowski/opengov-cli) tool, which takes the preimage bytes and outputs the on-chain calls. This is the fastest emergency track available, but it is not instantaneous. + +Run opengov-cli with the preimage bytes from the previous step: + +{% code overflow="wrap" %} +``` +opengov-cli submit-referendum \ + --proposal 0x \ + --network polkadot \ + --track whitelistedcaller \ + --output AppsUiLink +``` +{% endcode %} + +It prints three individual calls (preimage note, public referendum open, Fellowship whitelist referendum) plus two pre-built batches with direct papi.how links. Use the batches: + +1. **Polkadot Asset Hub batch** (anyone on the team can submit): notes the preimage and opens the public Whitelisted Caller referendum. opengov-cli outputs a direct papi.how link pointing at `asset-hub-polkadot-rpc`. +2. **Polkadot Collectives Chain batch** (requires a Fellow of rank 3 or higher): opens the Fellowship referendum that whitelists the call hash. This is the bottleneck. Coordinate with Parity (Adrian, Bastian, Oliver) in the Element channel to identify a rank-3+ Fellow who can submit it. opengov-cli outputs a direct papi.how link pointing at `polkadot-collectives-rpc`. + +**Enactment time**: opengov-cli defaults enactment to `After(10)` blocks if no `--at ` or `--after ` is passed. For emergencies, leave the default or shorten it; just be aware it's a soft default the tool warns about. -1. **Submit the preimage** on Polkadot via Polkassembly/Subsquare or directly via the preimage pallet, using the bytes from the previous step. Note the preimage hash. -2. **Whitelist the call hash**: the Whitelisted Caller track requires the specific call hash to be whitelisted by the Polkadot Fellowship via a Fellowship referendum. This is the bottleneck. In the Element channel with Parity (Adrian, Bastian, Oliver), share the preimage hash and ask them to coordinate a Fellowship referendum to whitelist it. Parity has Fellowship members and direct lines to others. -3. **Submit the Whitelisted Caller referendum** referencing the preimage. Once the Fellowship referendum to whitelist passes, the Whitelisted Caller referendum's confirmation period begins. -4. **Wall-clock expectation**: from preimage produced to call executed on-chain depends on Fellowship response time. Plan for the order of hours, not minutes, even on the emergency track. This is why the order in [Comms during an incident](#comms-during-an-incident) puts Parity escalation in parallel with halt submission. +**Wall-clock expectation**: from preimage produced to call executed on-chain depends on Fellowship response time and referendum confirmation periods. Plan for the order of hours, not minutes, even on the emergency track. This is why the order in [Comms during an incident](#comms-during-an-incident) puts Parity escalation in parallel with halt submission. ## Verifying the halt @@ -120,34 +106,34 @@ Once the call executes, verify the halt actually took effect on each affected ch Per-component checks, all reading the pallet's `OperatingMode` storage item (expected value: `Halted`): -| Halt flag | Chain | Storage item to query | +| Halt scope | Chain | Storage item to query | | --- | --- | --- | -| `--ethereum-client` | BridgeHub | `ethereumBeaconClient.operatingMode` | -| `--inbound-queue-v1` | BridgeHub | `ethereumInboundQueue.operatingMode` | -| `--inbound-queue-v2` | BridgeHub | `ethereumInboundQueueV2.operatingMode` | -| `--outbound-queue` (BridgeHub side) | BridgeHub | `ethereumOutboundQueue.operatingMode` | -| `--outbound-queue` / `--system-frontend` (AssetHub side) | AssetHub | `systemFrontend.operatingMode` (the snowbridge-system-frontend pallet) | -| `--gateway` (local pallet state) | BridgeHub | `ethereumSystem.operatingMode` and `ethereumSystemV2.operatingMode` | -| `--gateway` (actual contract state on Ethereum) | Ethereum | `Gateway.operatingMode()` returns `Halted`. **This step depends on relayer delivery**, so the on-chain Polkadot referendum executing does not mean the Ethereum contract is halted yet. Watch for the `SetOperatingMode` event on the Gateway and confirm operating mode after expected delivery. | -| `--assethub-max-fee` | AssetHub | `bridgeHubEthereumBaseFee` and `bridgeHubEthereumBaseFeeV2` equal `u128::MAX` (`340282366920938463463374607431768211455`) | -| `--assethub-max-fee-v2` | AssetHub | `bridgeHubEthereumBaseFeeV2` equals `u128::MAX` | +| **Ethereum client** | BridgeHub | `ethereumBeaconClient.operatingMode` | +| **Inbound queue V1** | BridgeHub | `ethereumInboundQueue.operatingMode` | +| **Inbound queue V2** | BridgeHub | `ethereumInboundQueueV2.operatingMode` | +| **Outbound queue** (BridgeHub side) | BridgeHub | `ethereumOutboundQueue.operatingMode` | +| **Outbound queue** / **System frontend** (AssetHub side) | AssetHub | `systemFrontend.operatingMode` (the snowbridge-system-frontend pallet) | +| **Gateway** (local pallet state) | BridgeHub | `ethereumSystem.operatingMode` and `ethereumSystemV2.operatingMode` | +| **Gateway** (actual contract state on Ethereum) | Ethereum | `Gateway.operatingMode()` returns `Halted`. **This step depends on relayer delivery**, so the on-chain Polkadot referendum executing does not mean the Ethereum contract is halted yet. Watch for the `SetOperatingMode` event on the Gateway and confirm operating mode after expected delivery. | +| **AssetHub max fee** | AssetHub | `bridgeHubEthereumBaseFee` and `bridgeHubEthereumBaseFeeV2` equal `u128::MAX` (`340282366920938463463374607431768211455`) | +| **AssetHub max fee V2** | AssetHub | `bridgeHubEthereumBaseFeeV2` equals `u128::MAX` | If the Polkadot side is halted but the Gateway contract is not (because no relayer has delivered the `SetOperatingMode` command yet), users can still call `sendToken` / `sendMessage` on Ethereum, those messages just won't be processed downstream. Treat the Polkadot-side halt as the firm guarantee; the Gateway-side halt as a follow-up. ## Resuming the bridge -Once the fix is deployed and the root cause is understood, **retrieve the resume-bridge preimage from the same governance page at [app.snowbridge.network/governance](https://app.snowbridge.network/governance)**. Select the resume action and the scopes to resume (typically matching whatever was halted), and copy the preimage hash and bytes for submission on the **Whitelisted Caller** track. +Once the fix is deployed and the root cause is understood, **retrieve the resume-bridge preimage from the same governance page at [app.snowbridge.network/governance](https://app.snowbridge.network/governance)**. Select the resume action and the scopes to resume (typically matching whatever was halted), and copy the preimage hash and bytes. Submission follows the same `opengov-cli` + Fellowship Whitelist flow as [Submitting the preimage](#submitting-the-preimage). -The `snowbridge-preimage` CLI does not currently have a top-level `resume-bridge` subcommand, so the governance page is the only practical path. If you need to script it (rare), the underlying SDK function is `buildResumeBridgePreimage` in `@snowbridge/api`. +If the governance page is unreachable, call `buildResumeBridgePreimage` from the `@snowbridge/api` SDK directly (same fallback shape as halting). Before submitting the resume preimage, confirm: * The fix is deployed and verified in production (runtime upgrade live, contracts upgraded if applicable). -* All three team members have signed off on resuming. +* The full team has signed off on resuming. * Monitoring is back to normal baselines and there are no fresh anomalies. * AssetHub fee values are being restored to their pre-incident values (the resume preimage writes `BridgeHubEthereumBaseFee` / `BridgeHubEthereumBaseFeeV2` back to known good defaults, currently `14_929_540_998` for V1 and `1_000_000_000` for V2; double-check these match what was live before the incident). -The resume preimage submission follows the same Whitelisted Caller flow as [Submitting the preimage](#submitting-the-preimage). Public comms can begin once the resume call has executed and the bridge is observed processing messages again. +Public comms can begin once the resume call has executed and the bridge is observed processing messages again. ## Emergency Upgrade From c8c80e42cbcfaf70fcf96fd9004e60d0034b63cc Mon Sep 17 00:00:00 2001 From: claravanstaden Date: Tue, 26 May 2026 14:34:25 +0200 Subject: [PATCH 07/10] Tighten Emergency Procedures and lead with halt mechanics Restructure so halt-action sections (Producing, Halt scopes reference, Submitting, Verifying) sit directly under the intro for fast access under pressure; Detection, Decision authority, and Comms move below. Trim prose throughout: intro 2 paragraphs to 2 lines, scope bullets compressed to one sentence each, Decision authority converted to a table, numbered comms steps shortened. Protocol-relevant details (pallet names, V1/V2 distinctions, relayer-delivery caveats) kept; explanatory framing cut. Co-Authored-By: Claude Opus 4.7 (1M context) --- docs/resources/emergency-procedures.md | 190 ++++++++++++------------- 1 file changed, 91 insertions(+), 99 deletions(-) diff --git a/docs/resources/emergency-procedures.md b/docs/resources/emergency-procedures.md index 4afd217d63..294e757077 100644 --- a/docs/resources/emergency-procedures.md +++ b/docs/resources/emergency-procedures.md @@ -1,85 +1,47 @@ # Emergency Procedures -This page is the on-call runbook for Snowbridge emergency response. It covers detection, decision-making, comms, the two emergency scenarios (pausing the bridge and emergency upgrades), and post-incident steps. For routine governance and the broader governance model, see [Governance and Operational Processes](governance-and-operational-processes.md). +On-call runbook. Halt mechanics are at the top so they're easy to reach under pressure. For routine governance, see [Governance and Operational Processes](governance-and-operational-processes.md). -The Snowbridge team is small and operates all-hands during an incident. Halting the bridge is technically reversible (it's a non-destructive operating-mode flip), but the halt referendum appearing on Polkassembly is itself a public signal that something is wrong, so it isn't free. The thresholds in [Decision authority](#decision-authority) are calibrated accordingly: solo action is reserved for cases where the incident is already publicly visible (funds being drained, active exploit), and confirmation-before-action is the default for everything else. - -## Detection - -Anything on this list is a valid trigger to start the incident flow. When in doubt, post in Slack and treat it as an incident until ruled out. - -* **Funds drained or unexpectedly moved** on either side of the bridge (highest priority, halt first, investigate after). -* **Bug bounty report** (HackenProof or direct) describing an exploitable vulnerability, verified by a team member as a valid exploit with a working proof-of-concept. - -## Decision authority - -To keep the path to halt short without creating unnecessary alarm: - -* **Solo halt** (1 team member, no confirmation needed): reserved for cases where the incident is already publicly visible and acting fast matters more than coordinating. Concretely: **funds are visibly being drained on-chain, or an active exploit is in progress**. In these cases the halt referendum appearing on-chain isn't new information to the community, the exploit already is. -* **Confirmed halt** (at least 2 team members agree before submitting): the default for everything else, including bug bounty reports without active exploitation, anomalies, unexplained queue behaviour, and "I don't understand what I'm seeing." A halt referendum appearing on Polkassembly is itself a public signal that something is wrong, so absent a visible exploit it's worth a few minutes of Slack discussion to confirm before submitting. -* **Escalating to Parity**: at least 2 team members agree there is a genuine incident. Same threshold as a confirmed halt; in practice these decisions happen together. This is the gate for creating the cross-team Element channel. -* **Public comms**: full team agreement, and only after the fix is deployed and the bridge is resuming. See [Comms during an incident](#comms-during-an-incident). -* **Emergency upgrade**: coordinated decision with Parity once they are in the loop, since upgrade code is often exploit-sensitive. - -## Comms during an incident - -Each step assumes the previous one has happened. - -1. **Team Slack**: post what you've seen in `#snowbridge-security`, include links to the signal (block explorer, alert, bounty report). For non-visible signals (bug reports, anomalies), wait for at least one teammate to confirm before submitting a halt referendum; see [Decision authority](#decision-authority). For a visible exploit or funds being drained, skip ahead and halt now. -2. **Halt the bridge** (if warranted) using the steps in [Producing a halt-bridge preimage](#producing-a-halt-bridge-preimage) and [Submitting the preimage](#submitting-the-preimage). For visible exploits, do this in parallel with steps 3 and 4 below. -3. **Internal confirmation**: once at least 2 team members have agreed there is a genuine incident, proceed to step 4. For solo-halt cases this is retroactive: the rest of the team confirms as they come online. -4. **Element channel with Parity**: create a new Element room and invite the Parity Bridges contacts (Adrian, Bastian, Oliver). This is where Fellowship coordination for the Whitelisted Caller submission happens, and where Parity gets the context they need to help. -5. **Affected integrators**: contact integration partners directly (Hydration first, then others) once Parity is in the loop. Channel: Telegram. Tell them what's halted and the expected resume timing. -6. **No public comms** (Twitter/X, forum, blog, public Discord) until the fix is deployed and resume is in flight. Premature public disclosure of an unfixed vulnerability is worse than silence. +Halt is technically reversible but the halt referendum on Polkassembly is public. See [Decision authority](#decision-authority) for when solo action vs team confirmation applies. ## Producing a halt-bridge preimage -**Retrieve the halt-bridge preimage from the governance page at [app.snowbridge.network/governance](https://app.snowbridge.network/governance).** Select the halt scope (full halt or per-component), and copy the resulting preimage hash and bytes. From here, the preimage bytes are fed into `opengov-cli` to generate the on-chain referendum calls; see [Submitting the preimage](#submitting-the-preimage). - -See [Halt scopes reference](#halt-scopes-reference) for the canonical description of what each scope on the governance page actually does, and the failure-mode to scope mapping. +Get the preimage from [app.snowbridge.network/governance](https://app.snowbridge.network/governance). Select the halt scope (see [Halt scopes reference](#halt-scopes-reference)), copy the hash and bytes, and feed bytes into `opengov-cli` (see [Submitting the preimage](#submitting-the-preimage)). -**Fallback only if the governance page is unreachable**: call the `buildHaltBridgePreimage` function from the `@snowbridge/api` SDK directly (this is what the governance page wraps). It takes a `HaltBridgeOptions` object whose keys match the scopes listed in [Halt scopes reference](#halt-scopes-reference), and returns the same preimage hash and bytes. Use this only when the UI is down, since the UI is the single source of truth the team is expected to drive from during an incident. +**Fallback (UI down only)**: call `buildHaltBridgePreimage` from `@snowbridge/api` with a `HaltBridgeOptions` matching the same scopes. ## Halt scopes reference -The governance page exposes the following halt scopes as form fields. Pick the narrowest scope that covers the suspected failure mode. - -* **All** Halt all the bridge components at once. This is the default when no other scope is selected. -* **Ethereum client** Halt the Ethereum beacon light client. Blocks new beacon-header ingestion via `EthereumBeaconClient::submit` **and** short-circuits `Verifier::verify` for every downstream consumer on BridgeHub. Concretely this stops: - * inbound V1 `submit` and inbound V2 `submit` (Ethereum → Polkadot messages), and - * `outbound-queue-v2::submit_delivery_receipt` (relayer-reward payouts against `PendingOrders`). - - This is the single lever to pull during a suspected beacon-light-client or sync-committee compromise. `force_checkpoint` remains available (root-only) for recovery. -* **Inbound queue** Halt both V1 and V2 inbound-queue pallets on BridgeHub, blocking processing of Ethereum → Polkadot messages. For surgical halts of a single version, use the two scopes below. -* **Inbound queue V1** Halt only the V1 inbound-queue pallet on BridgeHub. -* **Inbound queue V2** Halt only the V2 inbound-queue pallet on BridgeHub. -* **Outbound queue** Halt AssetHub → Ethereum outbound traffic. Halts the V1 outbound-queue pallet on BridgeHub **and** the system-frontend pallet on AssetHub; the latter short-circuits the AssetHub → Ethereum `PausableExporter` for both V1 and V2 at the XcmRouter layer. (V2's `outbound-queue-v2` has no local halt, so the system-frontend halt is the primary V2 outbound lever.) -* **System frontend** Router-layer P→E halt: halts only the AssetHub system-frontend pallet. Blocks **both** V1 and V2 P→E at the `PausableExporter` (returns `SendError::NotApplicable` to the XcmRouter). The V1 BridgeHub outbound-queue is left untouched, so in-flight V1 messages already enqueued there keep draining. There is no V2-only operating-mode P→E halt; use **AssetHub max fee V2** for a V2-only deterrent. -* **Gateway** Halt the Ethereum Gateway contract (both V1 and V2 paths). Sends `Command::SetOperatingMode(Halted)` via both V1 and V2 system pallets so the halt is delivered via whichever outbound queue is live. Delivery is relayer-dependent, so schedule this **before** any local outbound halt takes effect. -* **Gateway V2** V2-only Gateway halt: sends `Command::SetOperatingMode(Halted)` only via the V2 system pallet. Once delivered to Ethereum, blocks `v2_sendMessage` and `v2_registerToken`; leaves V1 `sendToken`/`sendMessage` working. Pair with **Inbound queue V2** and **AssetHub max fee V2** for the closest thing to a V2-only pause (no V2-only operating-mode P→E halt exists). -* **AssetHub max fee** Set the AssetHub → Ethereum outbound fee to `u128::MAX` for both V1 (`BridgeHubEthereumBaseFee`) and V2 (`BridgeHubEthereumBaseFeeV2`), effectively deterring user sends via fee pricing. Complementary to the system-frontend halt; does not block at the router layer. -* **AssetHub max fee V2** V2-only variant of **AssetHub max fee**: writes only `BridgeHubEthereumBaseFeeV2`, leaving V1 fee untouched. This is the only V2-isolated P→E lever (fee deterrent, not a hard halt). - -Failure-mode to scope mapping: - -| Suspected failure mode | Minimum scopes to select | +Pick the narrowest scope that covers the failure mode. Governance page form fields: + +* **All** Every component. Default if nothing else selected. +* **Ethereum client** Halts `EthereumBeaconClient::submit` and short-circuits `Verifier::verify` for all BridgeHub consumers. Stops V1 + V2 inbound `submit` and `outbound-queue-v2::submit_delivery_receipt`. Use for beacon-light-client or sync-committee compromise. `force_checkpoint` stays available (root-only) for recovery. +* **Inbound queue** Both V1 + V2 inbound pallets on BridgeHub. +* **Inbound queue V1** V1 inbound only. +* **Inbound queue V2** V2 inbound only. +* **Outbound queue** V1 outbound on BridgeHub **and** AssetHub system-frontend (short-circuits `PausableExporter` for V1 + V2 at XcmRouter). V2 has no local outbound halt, so system-frontend is the primary V2 outbound lever. +* **System frontend** AssetHub system-frontend only. Blocks V1 + V2 P→E at `PausableExporter` (`SendError::NotApplicable`). V1 BridgeHub outbound keeps draining in-flight messages. +* **Gateway** Sends `Command::SetOperatingMode(Halted)` to the Ethereum Gateway via both V1 + V2 system pallets. Delivery is relayer-dependent, so schedule **before** local outbound halts. +* **Gateway V2** V2-only Gateway halt. Blocks `v2_sendMessage` and `v2_registerToken` once delivered. Pair with **Inbound queue V2** + **AssetHub max fee V2** for a V2-only pause. +* **AssetHub max fee** Sets `BridgeHubEthereumBaseFee` + `BridgeHubEthereumBaseFeeV2` to `u128::MAX`. Fee deterrent, not a router halt. +* **AssetHub max fee V2** V2-only variant. Writes only `BridgeHubEthereumBaseFeeV2`. Only V2-isolated P→E lever. + +| Failure mode | Scopes | | --- | --- | | Beacon light client / sync committee compromise | **Ethereum client** | -| Ethereum Gateway contract compromise | **Gateway** + **AssetHub max fee** | +| Ethereum Gateway compromise | **Gateway** + **AssetHub max fee** | | Inbound-queue bug (one version) | **Inbound queue V1** or **Inbound queue V2** | | Outbound-queue / system-frontend bug | **Outbound queue** | -| V2 P→E only (V1 keeps flowing) | **AssetHub max fee V2** (fee deterrent, no operating-mode halt) | -| Full V2 pause (V1 keeps flowing, P→E is fee-deterrent only) | **Gateway V2** + **Inbound queue V2** + **AssetHub max fee V2** | -| Full P→E halt (both V1 and V2) | **Outbound queue** or **System frontend** | -| Uncertain / any component suspect | **All** | +| V2 P→E only (V1 keeps flowing) | **AssetHub max fee V2** (fee deterrent only) | +| Full V2 pause | **Gateway V2** + **Inbound queue V2** + **AssetHub max fee V2** | +| Full P→E halt (V1 + V2) | **Outbound queue** or **System frontend** | +| Uncertain | **All** | -In case of emergency where there is uncertainty of the cause of a problem, it is best to block the bridge in its entirety using **All**. To block both transfer directions at the earliest point possible, use **Gateway** + **AssetHub max fee**. +When uncertain: **All**. To block both directions immediately: **Gateway** + **AssetHub max fee**. ## Submitting the preimage -Halt preimages go through Polkadot OpenGov's **Whitelisted Caller** track, which requires the call hash to be whitelisted by the Polkadot Fellowship first. The submission flow is built using the [opengov-cli](https://github.com/joepetrowski/opengov-cli) tool, which takes the preimage bytes and outputs the on-chain calls. This is the fastest emergency track available, but it is not instantaneous. - -Run opengov-cli with the preimage bytes from the previous step: +Goes through OpenGov **Whitelisted Caller** track. Requires Polkadot Fellowship to whitelist the call first. Use [opengov-cli](https://github.com/joepetrowski/opengov-cli): {% code overflow="wrap" %} ``` @@ -91,66 +53,96 @@ opengov-cli submit-referendum \ ``` {% endcode %} -It prints three individual calls (preimage note, public referendum open, Fellowship whitelist referendum) plus two pre-built batches with direct papi.how links. Use the batches: +Outputs two pre-built batches as papi.how links: -1. **Polkadot Asset Hub batch** (anyone on the team can submit): notes the preimage and opens the public Whitelisted Caller referendum. opengov-cli outputs a direct papi.how link pointing at `asset-hub-polkadot-rpc`. -2. **Polkadot Collectives Chain batch** (requires a Fellow of rank 3 or higher): opens the Fellowship referendum that whitelists the call hash. This is the bottleneck. Coordinate with Parity (Adrian, Bastian, Oliver) in the Element channel to identify a rank-3+ Fellow who can submit it. opengov-cli outputs a direct papi.how link pointing at `polkadot-collectives-rpc`. +1. **Polkadot Asset Hub batch** Preimage note + public Whitelisted Caller referendum. Anyone on the team can submit. +2. **Polkadot Collectives Chain batch** Opens Fellowship whitelist referendum. **Requires rank-3+ Fellow.** Coordinate with Parity (Adrian, Bastian, Oliver) in Element. Bottleneck of the flow. -**Enactment time**: opengov-cli defaults enactment to `After(10)` blocks if no `--at ` or `--after ` is passed. For emergencies, leave the default or shorten it; just be aware it's a soft default the tool warns about. +Enactment defaults to `After(10)` blocks. Override with `--at ` or `--after `. -**Wall-clock expectation**: from preimage produced to call executed on-chain depends on Fellowship response time and referendum confirmation periods. Plan for the order of hours, not minutes, even on the emergency track. This is why the order in [Comms during an incident](#comms-during-an-incident) puts Parity escalation in parallel with halt submission. +**Wall-clock: hours, not minutes.** Run Parity escalation in parallel with submission (see [Comms](#comms-during-an-incident)). ## Verifying the halt -Once the call executes, verify the halt actually took effect on each affected chain. Don't trust the referendum closing as sufficient evidence, query state. - -Per-component checks, all reading the pallet's `OperatingMode` storage item (expected value: `Halted`): +After the call executes, query each affected chain's `OperatingMode` storage (expected: `Halted`). -| Halt scope | Chain | Storage item to query | +| Halt scope | Chain | Storage | | --- | --- | --- | | **Ethereum client** | BridgeHub | `ethereumBeaconClient.operatingMode` | | **Inbound queue V1** | BridgeHub | `ethereumInboundQueue.operatingMode` | | **Inbound queue V2** | BridgeHub | `ethereumInboundQueueV2.operatingMode` | -| **Outbound queue** (BridgeHub side) | BridgeHub | `ethereumOutboundQueue.operatingMode` | -| **Outbound queue** / **System frontend** (AssetHub side) | AssetHub | `systemFrontend.operatingMode` (the snowbridge-system-frontend pallet) | -| **Gateway** (local pallet state) | BridgeHub | `ethereumSystem.operatingMode` and `ethereumSystemV2.operatingMode` | -| **Gateway** (actual contract state on Ethereum) | Ethereum | `Gateway.operatingMode()` returns `Halted`. **This step depends on relayer delivery**, so the on-chain Polkadot referendum executing does not mean the Ethereum contract is halted yet. Watch for the `SetOperatingMode` event on the Gateway and confirm operating mode after expected delivery. | -| **AssetHub max fee** | AssetHub | `bridgeHubEthereumBaseFee` and `bridgeHubEthereumBaseFeeV2` equal `u128::MAX` (`340282366920938463463374607431768211455`) | -| **AssetHub max fee V2** | AssetHub | `bridgeHubEthereumBaseFeeV2` equals `u128::MAX` | +| **Outbound queue** (BridgeHub) | BridgeHub | `ethereumOutboundQueue.operatingMode` | +| **Outbound queue** / **System frontend** (AssetHub) | AssetHub | `systemFrontend.operatingMode` | +| **Gateway** (BridgeHub side) | BridgeHub | `ethereumSystem.operatingMode`, `ethereumSystemV2.operatingMode` | +| **Gateway** (Ethereum contract) | Ethereum | `Gateway.operatingMode() == Halted`. **Relayer-dependent**: watch for `SetOperatingMode` event before confirming. | +| **AssetHub max fee** | AssetHub | `bridgeHubEthereumBaseFee` + `bridgeHubEthereumBaseFeeV2` == `u128::MAX` (`340282366920938463463374607431768211455`) | +| **AssetHub max fee V2** | AssetHub | `bridgeHubEthereumBaseFeeV2` == `u128::MAX` | + +Polkadot-side halt is the firm guarantee. If Gateway isn't halted yet (no relayer delivery), `sendToken`/`sendMessage` on Ethereum still accept calls but nothing downstream processes them. + +## Detection + +Triggers for the incident flow: + +* **Funds drained or unexpectedly moved**. Highest priority. Halt first, investigate after. +* **Bug bounty report** (HackenProof or direct), verified by a team member as a valid exploit with working PoC. + +When in doubt: post in Slack, treat as incident until ruled out. + +## Decision authority + +| Action | Threshold | +| --- | --- | +| Solo halt | 1 member. **Only for visible exploit / funds being drained.** | +| Confirmed halt | 2 members agree. Default for bug bounty, anomalies, "I don't understand what I'm seeing." | +| Escalate to Parity | 2 members agree. Same conversation as confirmed halt in practice. | +| Public comms | Full team. **Only after fix is deployed and bridge is resuming.** | +| Emergency upgrade | Coordinated with Parity. Code is exploit-sensitive. | + +A halt referendum on Polkassembly is public, so solo authority is reserved for cases where the incident is already public (funds moving). Otherwise discuss in Slack first. + +## Comms during an incident + +Each step assumes the previous one has happened. -If the Polkadot side is halted but the Gateway contract is not (because no relayer has delivered the `SetOperatingMode` command yet), users can still call `sendToken` / `sendMessage` on Ethereum, those messages just won't be processed downstream. Treat the Polkadot-side halt as the firm guarantee; the Gateway-side halt as a follow-up. +1. **Slack** `#snowbridge-security` Post the signal (link to explorer, alert, bounty report). Non-visible signals: wait for at least one teammate to confirm before halting. Visible exploit: skip ahead. +2. **Halt** See [Producing](#producing-a-halt-bridge-preimage) + [Submitting](#submitting-the-preimage). For visible exploits, run in parallel with steps 3 and 4. +3. **Internal confirmation** 2+ members agree it's a real incident. Retroactive for solo-halt cases. +4. **Element with Parity** New room, invite Adrian, Bastian, Oliver. Fellowship coordination happens here. +5. **Integrators** Telegram. Hydration first, then others. Tell them what's halted + expected resume timing. +6. **No public comms** (Twitter/X, forum, blog, public Discord) until fix is deployed and resume is in flight. ## Resuming the bridge -Once the fix is deployed and the root cause is understood, **retrieve the resume-bridge preimage from the same governance page at [app.snowbridge.network/governance](https://app.snowbridge.network/governance)**. Select the resume action and the scopes to resume (typically matching whatever was halted), and copy the preimage hash and bytes. Submission follows the same `opengov-cli` + Fellowship Whitelist flow as [Submitting the preimage](#submitting-the-preimage). +Get the resume preimage from [app.snowbridge.network/governance](https://app.snowbridge.network/governance). Select scopes matching what was halted. Submission: same `opengov-cli` + Fellowship Whitelist flow as [Submitting the preimage](#submitting-the-preimage). -If the governance page is unreachable, call `buildResumeBridgePreimage` from the `@snowbridge/api` SDK directly (same fallback shape as halting). +Fallback: `buildResumeBridgePreimage` in `@snowbridge/api`. -Before submitting the resume preimage, confirm: +Before submitting, confirm: -* The fix is deployed and verified in production (runtime upgrade live, contracts upgraded if applicable). -* The full team has signed off on resuming. -* Monitoring is back to normal baselines and there are no fresh anomalies. -* AssetHub fee values are being restored to their pre-incident values (the resume preimage writes `BridgeHubEthereumBaseFee` / `BridgeHubEthereumBaseFeeV2` back to known good defaults, currently `14_929_540_998` for V1 and `1_000_000_000` for V2; double-check these match what was live before the incident). +* Fix is deployed and verified in production. +* Full team has signed off. +* Monitoring is back to baseline, no fresh anomalies. +* AssetHub fee values being restored to pre-incident values. Resume writes `BridgeHubEthereumBaseFee` and `BridgeHubEthereumBaseFeeV2` back to known good defaults (currently `14_929_540_998` for V1, `1_000_000_000` for V2). Double-check these match what was live before. -Public comms can begin once the resume call has executed and the bridge is observed processing messages again. +Public comms can begin once resume executes and the bridge is processing again. ## Emergency Upgrade -Although unlikely, there may be scenarios where the emergency can only be resolved through an upgrade rather than a pause, for example a critical bug in a pallet's logic that even a halt doesn't fully contain. In this case: +For cases a halt alone can't contain (e.g. critical pallet logic bug): -* **Halt first anyway**. A halt buys time to develop, review, and deploy the upgrade without time pressure. The exceptions are situations where the halt itself would be more harmful than the bug, which is rare. -* **Keep the upgrade code restricted**. Upgrade code for a known unpatched vulnerability is itself exploit material. Develop in a private branch, limit reviewers to the team plus the necessary Parity contacts, and avoid publicising the patch until it has executed on-chain. -* **Coordinate with Parity** on the upgrade pathway (Whitelisted Caller for runtime upgrades, multi-sig for contract upgrades, etc.). The Element channel from [Comms during an incident](#comms-during-an-incident) is the right venue. -* **Resume the bridge** only after the upgrade is verified live, following the [Resuming the bridge](#resuming-the-bridge) checklist. +* **Halt first anyway.** Buys time to develop the upgrade without pressure. Skip only if halting is itself harmful. +* **Restrict the code.** Upgrade code for an unpatched vulnerability is itself exploit material. Private branch, limited reviewers (team + necessary Parity contacts). Don't publicise until executed on-chain. +* **Coordinate the pathway with Parity.** Whitelisted Caller for runtime, multi-sig for contracts. Use the Element channel. +* **Resume** only after the upgrade is verified live ([Resuming the bridge](#resuming-the-bridge)). ## Post-mortem -Within 48 hours of the bridge being resumed (or sooner if everyone has bandwidth): +Within 48h of resume: -* **Owner**: whichever team member drove the incident (whoever halted first by default). -* **Format**: Google Doc, shared with the team and the Parity contacts who were in the Element channel. -* **Minimum contents**: timeline of events (with timestamps), root cause, what the halt scope was and why, what worked in the response, what didn't, action items with owners and target dates. -* **Action items**: track in the team's normal issue tracker, not the doc itself, so they don't get lost when the doc goes stale. +* **Owner** Whoever drove the incident (defaults to whoever halted first). +* **Format** Google Doc, shared with team + Parity contacts from the Element channel. +* **Contents** Timeline (timestamps), root cause, halt scope + reason, what worked, what didn't, action items with owners and dates. +* **Action items** Track in the team issue tracker, not the doc itself. -Even for false-positive halts (incident turned out to be non-incident), write a short post-mortem. Tuning detection signals to reduce false positives is itself a useful output. +Also write one for false-positive halts. Tuning detection signals to reduce false positives is itself useful output. From fbe6c56f9928bc81499dc5edbebaeb9e1625c3db Mon Sep 17 00:00:00 2001 From: claravanstaden Date: Tue, 26 May 2026 16:30:49 +0200 Subject: [PATCH 08/10] Add buildHaltBridgeSubmissionUrls to governance SDK Lets the snowbridge-app governance UI emit two papi.how submission links directly from a halt-bridge preimage, replacing the operator's manual opengov-cli + Rust toolchain step: 1. Asset Hub batch URL: utility.forceBatch([preimage.notePreimage( dispatchWhitelistedCallWithPreimage(call)), referenda.submit( Origins(WhitelistedCaller), Lookup{hash, len}, After(n))]). Anyone on the operator team can submit. 2. Collectives Fellowship URL: utility.forceBatch([fellowshipReferenda .submit(FellowshipOrigins(Fellows), Inline(polkadotXcm.send to Asset Hub carrying Transact(whitelist.whitelistCall(preimageHash))), After(10))]). Must be submitted by a rank-3+ Fellow. Mirrors joepetrowski/opengov-cli's polkadot_fellowship_referenda flow (src/submit_referendum.rs ~L628-L811). Includes opengov_submission_check.ts under @snowbridge/operations as a hex-parity check against a recorded opengov-cli fixture, currently passes byte-for-byte on the live Polkadot Asset Hub + Collectives runtimes. resumeBridgeSubmissionUrls is a thin alias because the wire format is identical; both halt and resume preimages go through the same WhitelistedCaller track. UI changes (copy-preimage button, submit-to-AH button, copy-Fellowship- URL button in HaltBridgeForm.tsx) are intentionally out of scope and will follow in snowbridge-app. Co-Authored-By: Claude Opus 4.7 (1M context) --- web/packages/api/src/governance/index.ts | 2 + .../api/src/governance/opengov_submission.ts | 293 ++++++++++++++++++ web/packages/api/src/index.ts | 2 +- .../src/opengov_submission_check.ts | 127 ++++++++ 4 files changed, 423 insertions(+), 1 deletion(-) create mode 100644 web/packages/api/src/governance/index.ts create mode 100644 web/packages/api/src/governance/opengov_submission.ts create mode 100644 web/packages/operations/src/opengov_submission_check.ts diff --git a/web/packages/api/src/governance/index.ts b/web/packages/api/src/governance/index.ts new file mode 100644 index 0000000000..cdd6717bf0 --- /dev/null +++ b/web/packages/api/src/governance/index.ts @@ -0,0 +1,2 @@ +export * from "./halt_bridge" +export * from "./opengov_submission" diff --git a/web/packages/api/src/governance/opengov_submission.ts b/web/packages/api/src/governance/opengov_submission.ts new file mode 100644 index 0000000000..056397aac1 --- /dev/null +++ b/web/packages/api/src/governance/opengov_submission.ts @@ -0,0 +1,293 @@ +import { ApiPromise } from "@polkadot/api" +import { SubmittableExtrinsic } from "@polkadot/api/types" +import { blake2AsHex } from "@polkadot/util-crypto" +import { hexToU8a, u8aToHex } from "@polkadot/util" + +import { HaltBridgePreimage } from "./halt_bridge" + +// Default WS endpoints embedded in the emitted papi.how URLs. These match what +// joepetrowski/opengov-cli prints, so the UI links open against the same +// metadata source opengov-cli would have used. +const DEFAULT_AH_WS = "wss://asset-hub-polkadot-rpc.dwellir.com" +const DEFAULT_COLLECTIVES_WS = "wss://polkadot-collectives-rpc.polkadot.io" + +// papi.how host. opengov-cli emits dev.papi.how URLs, match for parity. +const DEFAULT_PAPI_HOW_BASE = "https://dev.papi.how" + +// Asset Hub para ID, used as the XCM dest from Collectives. AH is where +// pallet_whitelist and pallet_referenda live post-AHM. +const ASSET_HUB_POLKADOT_ID = 1000 + +// opengov-cli hardcodes After(10) for the Fellowship-side enactment. +const FELLOWSHIP_ENACTMENT_AFTER_BLOCKS = 10 + +// papi.how networkId values, matching opengov-cli's output. +const PAPI_HOW_NETWORK_ASSET_HUB = "polkadot_asset_hub" +const PAPI_HOW_NETWORK_COLLECTIVES = "polkadot_collectives" + +export interface SubmissionUrls { + /** + * Asset Hub batch URL: `utility.forceBatch([preimage.notePreimage(wrapped), + * referenda.submit(Origins(WhitelistedCaller), Lookup{hash, len}, After(n))])`. + * Anyone on the operator team can submit this. + */ + assetHubBatchUrl: string + /** + * Hex bytes of the Asset Hub batch call (the `data=` payload of {@link assetHubBatchUrl}), + * exposed for parity testing against opengov-cli. + */ + assetHubBatchCallData: string + /** + * Collectives Chain URL: `utility.forceBatch([fellowshipReferenda.submit( + * Origins(FellowshipOrigins::Fellows), Inline(xcm_send), After(10))])` where + * `xcm_send` is a `polkadotXcm.send` to Asset Hub carrying a Transact of + * `whitelist.whitelistCall(preimageHash)`. **Must be submitted by a Fellow + * of rank 3 or higher.** + */ + fellowshipWhitelistUrl: string + /** + * Hex bytes of the Collectives call (the `data=` payload of {@link fellowshipWhitelistUrl}), + * exposed for parity testing against opengov-cli. + */ + fellowshipWhitelistCallData: string + /** + * Hash of the user-supplied preimage call (the input to this function). + * This is the hash that goes inside `whitelist.whitelistCall` on the + * Fellowship side. **Not** the hash referenced by the AH public referendum + * (that one hashes the wrapped `dispatchWhitelistedCallWithPreimage` call). + */ + preimageHash: string + /** + * Hash of the wrapped preimage (`whitelist.dispatchWhitelistedCallWithPreimage(call)`). + * This is what `preimage.notePreimage` stores on AH and what the AH public + * referendum's `Lookup` references. + */ + wrappedPreimageHash: string + /** + * Length in bytes of the wrapped preimage, the `len` field of the AH + * referendum's `Lookup`. + */ + wrappedPreimageLen: number +} + +export interface SubmissionOptions { + /** + * Enactment delay (in blocks) for the AH public WhitelistedCaller + * referendum. Defaults to 10, matching opengov-cli's default. The + * Fellowship-side enactment is always hardcoded to `After(10)` to match + * opengov-cli. + */ + enactmentAfterBlocks?: number + /** Override the papi.how host. Defaults to dev.papi.how (opengov-cli parity). */ + papiHowBase?: string + /** Override the AH WS URL embedded in the papi.how URL. */ + assetHubWsUrl?: string + /** Override the Collectives WS URL embedded in the papi.how URL. */ + collectivesWsUrl?: string +} + +/** + * Build the two submission URLs a Snowbridge halt preimage needs to land on + * the Polkadot WhitelistedCaller track: + * + * 1. An Asset Hub URL the operator submits to note the (wrapped) preimage + * and open the public Whitelisted Caller referendum. + * 2. A Collectives Chain URL that a rank-3+ Fellow submits to open the + * Fellowship whitelist referendum, which XCM-Transacts into AH to call + * `whitelist.whitelistCall(preimageHash)`. + * + * Mirrors joepetrowski/opengov-cli's `polkadot_fellowship_referenda` flow + * (src/submit_referendum.rs ~L628-L811). The encoded call bytes are byte-for- + * byte identical to what opengov-cli emits for the same preimage (verified by + * the `opengov_submission_check` script in @snowbridge/operations). + * + * @param assetHub Connected ApiPromise for Asset Hub Polkadot (provides + * metadata for `whitelist`, `preimage`, `referenda`, + * `utility`, `Origins`). + * @param collectives Connected ApiPromise for Polkadot Collectives Chain + * (provides metadata for `fellowshipReferenda`, `polkadotXcm`, + * `utility`, `FellowshipOrigins`). + * @param preimage The halt-bridge preimage to wrap, either a {@link HaltBridgePreimage} + * (use its `callData` field) or a 0x-prefixed hex string of + * the raw preimage call bytes. + * @param opts Optional URL/enactment overrides. + */ +export async function buildHaltBridgeSubmissionUrls( + assetHub: ApiPromise, + collectives: ApiPromise, + preimage: HaltBridgePreimage | string, + opts: SubmissionOptions = {}, +): Promise { + const userCallHex = + typeof preimage === "string" ? preimage : preimage.callData + + const assetHubWsUrl = opts.assetHubWsUrl ?? DEFAULT_AH_WS + const collectivesWsUrl = opts.collectivesWsUrl ?? DEFAULT_COLLECTIVES_WS + const papiHowBase = opts.papiHowBase ?? DEFAULT_PAPI_HOW_BASE + const ahEnactmentAfter = + opts.enactmentAfterBlocks ?? FELLOWSHIP_ENACTMENT_AFTER_BLOCKS + + const userCallBytes = hexToU8a(userCallHex) + const preimageHash = blake2AsHex(userCallBytes, 256) + + // Wrap the user call in whitelist.dispatchWhitelistedCallWithPreimage. + // The wrapped bytes are what gets noted on AH and referenced by the + // public referendum; the wrapped hash is distinct from the raw preimage + // hash that the Fellowship whitelist call hashes. + const userCallDecoded = assetHub.createType("Call", userCallBytes) + const wrappedCall = assetHub.tx.whitelist.dispatchWhitelistedCallWithPreimage( + userCallDecoded, + ) + const wrappedCallBytes = wrappedCall.method.toU8a() + const wrappedPreimageHash = blake2AsHex(wrappedCallBytes, 256) + const wrappedPreimageLen = wrappedCallBytes.length + + const assetHubBatch = buildAssetHubBatch( + assetHub, + wrappedCallBytes, + wrappedPreimageHash, + wrappedPreimageLen, + ahEnactmentAfter, + ) + const assetHubBatchCallData = u8aToHex(assetHubBatch.method.toU8a()) + + const collectivesBatch = buildCollectivesBatch( + assetHub, + collectives, + preimageHash, + ) + const fellowshipWhitelistCallData = u8aToHex(collectivesBatch.method.toU8a()) + + return { + assetHubBatchUrl: buildPapiHowUrl( + papiHowBase, + PAPI_HOW_NETWORK_ASSET_HUB, + assetHubWsUrl, + assetHubBatchCallData, + ), + assetHubBatchCallData, + fellowshipWhitelistUrl: buildPapiHowUrl( + papiHowBase, + PAPI_HOW_NETWORK_COLLECTIVES, + collectivesWsUrl, + fellowshipWhitelistCallData, + ), + fellowshipWhitelistCallData, + preimageHash, + wrappedPreimageHash, + wrappedPreimageLen, + } +} + +/** + * Convenience wrapper for the resume case. The wire format is identical + * (resume preimages go through the same WhitelistedCaller track), so this + * just delegates to {@link buildHaltBridgeSubmissionUrls}. + */ +export async function buildResumeBridgeSubmissionUrls( + assetHub: ApiPromise, + collectives: ApiPromise, + preimage: HaltBridgePreimage | string, + opts: SubmissionOptions = {}, +): Promise { + return buildHaltBridgeSubmissionUrls(assetHub, collectives, preimage, opts) +} + +function buildAssetHubBatch( + assetHub: ApiPromise, + wrappedCallBytes: Uint8Array, + wrappedPreimageHash: string, + wrappedPreimageLen: number, + enactmentAfterBlocks: number, +): SubmittableExtrinsic<"promise"> { + const notePreimage = assetHub.tx.preimage.notePreimage( + u8aToHex(wrappedCallBytes), + ) + + // Plain-object args; @polkadot/api resolves the runtime-specific + // OriginCaller and Bounded enums from referenda.submit's metadata. + const submit = assetHub.tx.referenda.submit( + { Origins: "WhitelistedCaller" } as any, + { Lookup: { hash: wrappedPreimageHash, len: wrappedPreimageLen } } as any, + { After: enactmentAfterBlocks } as any, + ) + + return assetHub.tx.utility.forceBatch([notePreimage, submit]) +} + +function buildCollectivesBatch( + assetHub: ApiPromise, + collectives: ApiPromise, + rawPreimageHash: string, +): SubmittableExtrinsic<"promise"> { + // Inner AH call that the Fellowship XCM Transacts into: + // whitelist.whitelistCall(preimageHash) + const whitelistCall = assetHub.tx.whitelist.whitelistCall(rawPreimageHash) + const whitelistCallBytes = whitelistCall.method.toU8a() + + // XCM message: UnpaidExecution + Transact(whitelistCall). No fee asset + // because AH grants free execution to sibling Collectives via the + // UnpaidExecution barrier. + const message = { + V5: [ + { + UnpaidExecution: { + weightLimit: "Unlimited", + checkOrigin: null, + }, + }, + { + Transact: { + originKind: "Xcm", + fallbackMaxWeight: null, + call: { encoded: u8aToHex(whitelistCallBytes) }, + }, + }, + ], + } + + // Dest from Collectives to AH: { parents: 1, interior: X1([Parachain(1000)]) }. + const dest = { + V5: { + parents: 1, + interior: { X1: [{ Parachain: ASSET_HUB_POLKADOT_ID }] }, + }, + } + + const xcmSend = collectives.tx.polkadotXcm.send(dest, message) + const xcmSendBytes = xcmSend.method.toU8a() + + // fellowshipReferenda.submit with FellowshipOrigins::Fellows origin and + // Inline(xcm_send) proposal. opengov-cli only uses Inline when bytes + // fit in the 128-byte BoundedVec; the whitelist-call wrapper is small + // enough that this always holds. + if (xcmSendBytes.length > 128) { + throw new Error( + `buildCollectivesBatch: XCM send bytes (${xcmSendBytes.length}) exceed 128-byte Inline cap; ` + + "would need a separate Preimage::notePreimage on Collectives. Not implemented.", + ) + } + + const submit = collectives.tx.fellowshipReferenda.submit( + { FellowshipOrigins: "Fellows" } as any, + { Inline: u8aToHex(xcmSendBytes) } as any, + { After: FELLOWSHIP_ENACTMENT_AFTER_BLOCKS } as any, + ) + + // Even for a single call, opengov-cli wraps in force_batch for output + // consistency; match it byte-for-byte. + return collectives.tx.utility.forceBatch([submit]) +} + +function buildPapiHowUrl( + base: string, + networkId: string, + wsUrl: string, + callData: string, +): string { + return ( + `${base}/extrinsics#data=${callData}` + + `&networkId=${networkId}` + + `&endpoint=${encodeURIComponent(wsUrl)}` + ) +} diff --git a/web/packages/api/src/index.ts b/web/packages/api/src/index.ts index 17016658ff..6ca999b41b 100644 --- a/web/packages/api/src/index.ts +++ b/web/packages/api/src/index.ts @@ -79,7 +79,7 @@ export * as history from "./history" export * as historyV2 from "./history_v2" export { TransferStatus } from "./history_v2" export * as subsquidV2 from "./subsquid_v2" -export * as governance from "./governance/halt_bridge" +export * as governance from "./governance" export class Context { readonly environment: Environment diff --git a/web/packages/operations/src/opengov_submission_check.ts b/web/packages/operations/src/opengov_submission_check.ts new file mode 100644 index 0000000000..0e8e1a9950 --- /dev/null +++ b/web/packages/operations/src/opengov_submission_check.ts @@ -0,0 +1,127 @@ +/** + * Hex-parity check: verify that @snowbridge/api's `buildHaltBridgeSubmissionUrls` + * produces byte-for-byte the same Asset Hub batch and Collectives extrinsic as + * joepetrowski/opengov-cli, for a known fixture preimage. + * + * The fixture was captured by running opengov-cli locally: + * + * opengov-cli submit-referendum \ + * --proposal 0x \ + * --network polkadot \ + * --track whitelistedcaller \ + * --output AppsUiLink + * + * Outputs the two known papi.how `data=` payloads recorded in + * EXPECTED_ASSET_HUB_HEX and EXPECTED_COLLECTIVES_HEX below. + * + * Connects to live RPCs by default; override with ASSET_HUB_WS / COLLECTIVES_WS. + * Note: runtime upgrades on either chain can shift pallet indices or type + * shapes, in which case the fixture goes stale; capture a fresh one by + * re-running opengov-cli. + * + * Usage: `npx ts-node web/packages/operations/src/opengov_submission_check.ts` + */ + +import { ApiPromise, WsProvider } from "@polkadot/api" +import { governance } from "@snowbridge/api" + +const ASSET_HUB_WS = + process.env.ASSET_HUB_WS ?? "wss://polkadot-asset-hub-rpc.polkadot.io" +const COLLECTIVES_WS = + process.env.COLLECTIVES_WS ?? "wss://polkadot-collectives-rpc.polkadot.io" + +// Fixture preimage from a recorded opengov-cli run, the raw bytes you'd pass +// via `--proposal`. This is a halt-bridge preimage (XCM-send to BridgeHub +// + AH frontend halt + max-fee writes). +const FIXTURE_PREIMAGE = + "0x2804081f0005010100a90f05342f000006020102d96aca89700c5301012000060201826b28be89700c5a0101200006020102ca9a3b000c500101200006020102ca9a3b000c5b0101200006020102ca9a3b000c510001200006020102ca9a3b000c520301200028040c240001000404405fbc5c7ba58845ad1f1a9a7c5bc12fad40ffffffffffffffffffffffffffffffff00040440d0ed50b03e9a49e836dd934b425ba4c340ffffffffffffffffffffffffffffffff" + +// The two `data=` payloads opengov-cli emits for the fixture above. Copied +// verbatim from the recorded output. +const EXPECTED_ASSET_HUB_HEX = + "0x2804080500e10240032804081f0005010100a90f05342f000006020102d96aca89700c5301012000060201826b28be89700c5a0101200006020102ca9a3b000c500101200006020102ca9a3b000c5b0101200006020102ca9a3b000c510001200006020102ca9a3b000c520301200028040c240001000404405fbc5c7ba58845ad1f1a9a7c5bc12fad40ffffffffffffffffffffffffffffffff00040440d0ed50b03e9a49e836dd934b425ba4c340ffffffffffffffffffffffffffffffff3e003f0d023ed76771e81b331f874722bfaef740b3219ddd944fcc33021d53735ede9c0e3ab8000000010a000000" +const EXPECTED_COLLECTIVES_HEX = + "0x2804043d003e0201cc1f0005010100a10f05082f00000603008840009acbcdc7f00d57411325ef59d71f4861d4c9181e07eca54d62f27a7260951710010a000000" + +async function main() { + console.log(`Connecting to AssetHub: ${ASSET_HUB_WS}`) + console.log(`Connecting to Collectives: ${COLLECTIVES_WS}`) + + const [assetHub, collectives] = await Promise.all([ + ApiPromise.create({ provider: new WsProvider(ASSET_HUB_WS) }), + ApiPromise.create({ provider: new WsProvider(COLLECTIVES_WS) }), + ]) + + try { + const urls = await governance.buildHaltBridgeSubmissionUrls( + assetHub, + collectives, + FIXTURE_PREIMAGE, + ) + + console.log("\n--- SDK output ---") + console.log(`preimageHash: ${urls.preimageHash}`) + console.log(`wrappedPreimageHash: ${urls.wrappedPreimageHash}`) + console.log(`wrappedPreimageLen: ${urls.wrappedPreimageLen}`) + console.log(`\nassetHubBatchUrl: ${urls.assetHubBatchUrl}`) + console.log(`fellowshipWhitelist: ${urls.fellowshipWhitelistUrl}`) + + const ahMatch = urls.assetHubBatchCallData === EXPECTED_ASSET_HUB_HEX + const collectivesMatch = + urls.fellowshipWhitelistCallData === EXPECTED_COLLECTIVES_HEX + + console.log("\n--- Parity check vs opengov-cli ---") + reportMatch( + "Asset Hub batch", + ahMatch, + EXPECTED_ASSET_HUB_HEX, + urls.assetHubBatchCallData, + ) + reportMatch( + "Collectives batch", + collectivesMatch, + EXPECTED_COLLECTIVES_HEX, + urls.fellowshipWhitelistCallData, + ) + + if (!ahMatch || !collectivesMatch) { + process.exitCode = 1 + } + } finally { + await Promise.all([assetHub.disconnect(), collectives.disconnect()]) + } +} + +function reportMatch( + label: string, + match: boolean, + expected: string, + actual: string, +) { + if (match) { + console.log(` ${label}: PASS (${actual.length / 2 - 1} bytes)`) + return + } + console.log(` ${label}: FAIL`) + console.log(` expected: ${expected}`) + console.log(` actual: ${actual}`) + const firstDiff = firstDifference(expected, actual) + if (firstDiff !== null) { + console.log( + ` first diff at char ${firstDiff}: expected '${expected[firstDiff] ?? ""}', got '${actual[firstDiff] ?? ""}'`, + ) + } +} + +function firstDifference(a: string, b: string): number | null { + const len = Math.min(a.length, b.length) + for (let i = 0; i < len; i++) { + if (a[i] !== b[i]) return i + } + return a.length === b.length ? null : len +} + +main().catch((e) => { + console.error(e) + process.exit(1) +}) From 25595cc6909be39c33a86c1051893cb128702e24 Mon Sep 17 00:00:00 2001 From: claravanstaden Date: Wed, 27 May 2026 14:55:10 +0200 Subject: [PATCH 09/10] Reframe halt-bridge submission docs around governance UI The governance page at app.snowbridge.network/governance now emits both the preimage AND the two papi.how submission links directly (Asset Hub batch + Fellowship whitelist), via the new buildHaltBridgeSubmissionUrls SDK helper. Reflect that as the primary path: - "Producing a halt-bridge preimage" -> "Producing the preimage and submission links": one step now, the page emits the URLs along with the preimage. - "Submitting the preimage" -> "Submitting": leads with "click Open on AH batch / Copy Fellowship link from the UI", no intermediate tooling. - Resuming section mirrors halt: UI emits both. - opengov-cli demoted to a fallback subsection under Submitting, for the case where the UI is unreachable. - SDK fallbacks updated to include buildHalt/ResumeBridgeSubmissionUrls alongside the preimage builders. Co-Authored-By: Claude Opus 4.7 (1M context) --- docs/resources/emergency-procedures.md | 36 ++++++++++++++------------ 1 file changed, 20 insertions(+), 16 deletions(-) diff --git a/docs/resources/emergency-procedures.md b/docs/resources/emergency-procedures.md index 294e757077..4f6c2cd9e2 100644 --- a/docs/resources/emergency-procedures.md +++ b/docs/resources/emergency-procedures.md @@ -4,11 +4,11 @@ On-call runbook. Halt mechanics are at the top so they're easy to reach under pr Halt is technically reversible but the halt referendum on Polkassembly is public. See [Decision authority](#decision-authority) for when solo action vs team confirmation applies. -## Producing a halt-bridge preimage +## Producing the preimage and submission links -Get the preimage from [app.snowbridge.network/governance](https://app.snowbridge.network/governance). Select the halt scope (see [Halt scopes reference](#halt-scopes-reference)), copy the hash and bytes, and feed bytes into `opengov-cli` (see [Submitting the preimage](#submitting-the-preimage)). +The governance page at [app.snowbridge.network/governance](https://app.snowbridge.network/governance) is the single source of truth during an incident. Select the halt scope (see [Halt scopes reference](#halt-scopes-reference)) and the page emits both the preimage **and** the two ready-to-submit papi.how links. Take the links straight to [Submitting](#submitting). -**Fallback (UI down only)**: call `buildHaltBridgePreimage` from `@snowbridge/api` with a `HaltBridgeOptions` matching the same scopes. +**Fallback (UI down only)**: call `buildHaltBridgePreimage` then `buildHaltBridgeSubmissionUrls` from `@snowbridge/api`. Produces the same preimage + URLs the UI shows. ## Halt scopes reference @@ -39,9 +39,20 @@ Pick the narrowest scope that covers the failure mode. Governance page form fiel When uncertain: **All**. To block both directions immediately: **Gateway** + **AssetHub max fee**. -## Submitting the preimage +## Submitting -Goes through OpenGov **Whitelisted Caller** track. Requires Polkadot Fellowship to whitelist the call first. Use [opengov-cli](https://github.com/joepetrowski/opengov-cli): +Submission goes through OpenGov's **Whitelisted Caller** track, which requires the Polkadot Fellowship to whitelist the call first. From the governance page's result panel, two papi.how links handle this end-to-end: + +1. **Asset Hub batch** Click **Open**. Notes the preimage and opens the public Whitelisted Caller referendum on Asset Hub. Anyone on the team can submit. Sign in papi.how. +2. **Fellowship whitelist** Click **Copy** and share the link in the Parity Element channel (see [Comms](#comms-during-an-incident)). Must be submitted by a Fellow of rank 3 or higher. Bottleneck of the flow. + +Enactment defaults to `After(10)` blocks (matches opengov-cli's default). + +**Wall-clock: hours, not minutes.** Run Parity escalation in parallel with the Asset Hub submission. + +### Fallback: opengov-cli + +If the governance page is unreachable, construct the same submission locally with [opengov-cli](https://github.com/joepetrowski/opengov-cli) and the preimage bytes (which the SDK fallback in [Producing](#producing-the-preimage-and-submission-links) can still generate offline): {% code overflow="wrap" %} ``` @@ -53,14 +64,7 @@ opengov-cli submit-referendum \ ``` {% endcode %} -Outputs two pre-built batches as papi.how links: - -1. **Polkadot Asset Hub batch** Preimage note + public Whitelisted Caller referendum. Anyone on the team can submit. -2. **Polkadot Collectives Chain batch** Opens Fellowship whitelist referendum. **Requires rank-3+ Fellow.** Coordinate with Parity (Adrian, Bastian, Oliver) in Element. Bottleneck of the flow. - -Enactment defaults to `After(10)` blocks. Override with `--at ` or `--after `. - -**Wall-clock: hours, not minutes.** Run Parity escalation in parallel with submission (see [Comms](#comms-during-an-incident)). +Emits the same two papi.how URLs the UI shows. Use only when the UI is down; the UI is the single source of truth the team drives from during an incident. ## Verifying the halt @@ -106,7 +110,7 @@ A halt referendum on Polkassembly is public, so solo authority is reserved for c Each step assumes the previous one has happened. 1. **Slack** `#snowbridge-security` Post the signal (link to explorer, alert, bounty report). Non-visible signals: wait for at least one teammate to confirm before halting. Visible exploit: skip ahead. -2. **Halt** See [Producing](#producing-a-halt-bridge-preimage) + [Submitting](#submitting-the-preimage). For visible exploits, run in parallel with steps 3 and 4. +2. **Halt** See [Producing](#producing-the-preimage-and-submission-links) + [Submitting](#submitting). For visible exploits, run in parallel with steps 3 and 4. 3. **Internal confirmation** 2+ members agree it's a real incident. Retroactive for solo-halt cases. 4. **Element with Parity** New room, invite Adrian, Bastian, Oliver. Fellowship coordination happens here. 5. **Integrators** Telegram. Hydration first, then others. Tell them what's halted + expected resume timing. @@ -114,9 +118,9 @@ Each step assumes the previous one has happened. ## Resuming the bridge -Get the resume preimage from [app.snowbridge.network/governance](https://app.snowbridge.network/governance). Select scopes matching what was halted. Submission: same `opengov-cli` + Fellowship Whitelist flow as [Submitting the preimage](#submitting-the-preimage). +Same flow as halting: the governance page emits the resume preimage and the two submission links. Select scopes matching what was halted, then proceed via [Submitting](#submitting). -Fallback: `buildResumeBridgePreimage` in `@snowbridge/api`. +Fallback: `buildResumeBridgePreimage` + `buildResumeBridgeSubmissionUrls` in `@snowbridge/api`. Before submitting, confirm: From 9d673ee84fc7ed1739d4b18cfd653ea81c8fd6ab Mon Sep 17 00:00:00 2001 From: claravanstaden Date: Tue, 2 Jun 2026 09:51:07 +0200 Subject: [PATCH 10/10] chore: refresh PR head (no-op)