v1.110 — three-way fail-safe campaigns (validate the safety net)

.github/workflows/ci.yml

@@ -117,4 +117,10 @@ jobs:
       - uses: dtolnay/rust-toolchain@stable
       - uses: Swatinem/rust-cache@v2
       - name: Closed-loop safety suite + dispersion campaigns
-        run: cargo test -p falcon-sitl-gz -- --nocapture
+        run: |
+          # falcon-sitl-gz: integration safety tests + the nominal-recovery
+          # Monte-Carlo campaigns (motor-out / attitude / hexa).
+          cargo test -p falcon-sitl-gz -- --nocapture
+          # falcon-core: the THREE-WAY fail-safe campaigns (runaway→terminate,
+          # wind→RTL) that drive the full production FlightSupervisor.
+          cargo test -p falcon-core failsafe -- --nocapture

.github/workflows/verification-gate.yml

@@ -38,7 +38,14 @@ jobs:
   gate:
     name: Verification gate (rivet-driven)
     runs-on: [self-hosted, linux, x64, rust-cpu]
-    timeout-minutes: 30
+    # 45 (was 30): the gate executes every artifact's non-bench-only cargo step
+    # serially. When a PR touches a CENTRAL crate (e.g. falcon-core), the shared
+    # build cache is invalidated and many artifact steps rebuild cold, so the
+    # ~370-artifact sweep can exceed 30 min on a disk-constrained runner — it
+    # timed out at 373/… all-PASS. The extra headroom lets the sweep finish; the
+    # verification itself was never failing. (Root cause to watch: self-hosted
+    # runner disk pressure — see the 94% post-job-hook cleanups.)
+    timeout-minutes: 45
     env:
       CARGO_TERM_COLOR: always
       CARGO_INCREMENTAL: 0

artifacts/verification/FV-FALCON-SIMMC-001.yaml

@@ -56,9 +56,9 @@ artifacts:
       method: simulation
       test-count: 3
       steps:
-        - run: cargo test -p falcon-sitl-gz motor_out_monte_carlo_campaign
-        - run: cargo test -p falcon-sitl-gz att_stab_monte_carlo_campaign
-        - run: cargo test -p falcon-sitl-gz hexa_monte_carlo_campaign
+        - run: "cargo test -p falcon-sitl-gz motor_out_monte_carlo_campaign  # bench-only (2000-trial dispersion; runs in the closed-loop-sim CI job + workspace test)"
+        - run: "cargo test -p falcon-sitl-gz att_stab_monte_carlo_campaign  # bench-only"
+        - run: "cargo test -p falcon-sitl-gz hexa_monte_carlo_campaign  # bench-only"
     links:
       - type: verifies
         target: SWREQ-FALCON-FAULT-P02

artifacts/verification/FV-FALCON-SIMMC-002.yaml

@@ -0,0 +1,56 @@
+artifacts:
+  - id: FV-FALCON-SIMMC-002
+    type: sw-verification
+    title: "Three-way fail-safe campaigns — validating the safety NET across a dispersed regime sweep (v1.110)"
+    status: implemented
+    release: falcon-v1.110.0
+    description: >
+      Extends the dispersion-campaign discipline (FV-FALCON-SIMMC-001) from
+      NOMINAL recovery to the FAIL-SAFES themselves, with the three-way envelope
+      classification: inside the recoverable set no fail-safe fires (no false
+      positive), and OUTSIDE it the CORRECT fail-safe (terminate vs RTL) fires
+      and latches. Where the point-tests
+      (attitude_runaway_terminates_in_flight / high_wind_saturation_commands_rtl)
+      prove ONE condition each, these sweep the regime and validate BOTH the
+      "fires when it must" and "never false-fires" halves — a one-sided pass
+      (a detector wired always-on, or always-off) trips the well-sampled
+      assertion. Each trial drives the FULL production FlightSupervisor (real
+      IEKF + geometric controller + mixer + FSM) via a tilt-injecting backend,
+      the same mechanism as the point-tests.
+
+      crates/falcon-core (mod failsafe_campaign), two campaigns, both pass, both
+      regimes well-sampled (≈60 each), reproducible from (seed, index):
+
+        failsafe_runaway_terminate_campaign — runaway regime: a sustained tilt
+          drawn from (1.15, 1.50) rad, strictly above TILT_RUNAWAY_LIMIT (1.05),
+          MUST drive the FSM to Terminated and LATCH (stays Terminated after the
+          tilt returns to level). No-terminate regime: a tilt drawn from
+          [0, 0.95] rad, strictly below the limit, must NEVER terminate (an RTL
+          via the wind path is not a false terminate). Also asserts no
+          termination during the level take-off.
+        failsafe_wind_rtl_campaign — wind regime: a sustained tilt drawn from
+          [0.38, 0.62] rad (inside the saturation band [0.30, 0.70]) away from
+          home MUST latch the RTL-class failsafe (rtl_latched). No-RTL regime: a
+          tilt drawn from [0, 0.20] rad (below the band) must NEVER latch RTL.
+
+      Honest scope: these are currently DETERMINISTIC tilt sweeps (60/regime
+      densely covers the 1-D range) — the injected tilt is exact, so the
+      "dispersion" is the tilt magnitude + position, not yet sensor noise. Adding
+      gyro/accel noise + bias walk + gust profiles (the full stochastic
+      dispersion vector) is the next slice; that is when the trial count rises to
+      the rule-of-three ~1000/behavior. NAMED GATE: the ci.yml "Closed-loop
+      simulation + Monte-Carlo campaigns" job now also runs
+      `cargo test -p falcon-core failsafe`, so the fail-safe campaigns join the
+      nominal-recovery campaigns as a first-class visible required check.
+    tags: [verification, falcon, simulation, monte-carlo, failsafe, three-way, v1.110]
+    fields:
+      method: simulation
+      test-count: 2
+      steps:
+        - run: "cargo test -p falcon-core failsafe_runaway_terminate_campaign  # bench-only (heavy full-estimator sim; runs in the closed-loop-sim CI job + workspace test)"
+        - run: "cargo test -p falcon-core failsafe_wind_rtl_campaign  # bench-only"
+    links:
+      - type: verifies
+        target: SWREQ-FALCON-RUNAWAY-P01
+      - type: verifies
+        target: SWREQ-FALCON-WINDFS-P01

crates/falcon-core/plain/src/lib.rs

@@ -2756,3 +2756,201 @@ mod tests {
         assert!(-b.pos[2] < 0.25, "must settle on the surface (not float): {} m", -b.pos[2]);
     }
 }
+
+/// v1.110 — Monte-Carlo THREE-WAY fail-safe campaigns. Where the point-tests
+/// prove ONE runaway terminates and ONE wind gust RTLs, these sweep the
+/// dispersion and validate the SAFETY NET itself: inside the recoverable set no
+/// fail-safe fires (no false positive), and outside it the CORRECT fail-safe
+/// (terminate vs RTL) fires and latches. Each trial drives the full production
+/// FlightSupervisor (real IEKF + controller + mixer) via a tilt-injecting
+/// backend — the same mechanism as the point-tests, dispersed and reduced to a
+/// pass-rate + per-regime counts. Reproducible from (seed, index); companion to
+/// the nominal-recovery campaigns in examples/falcon-sitl-gz.
+#[cfg(test)]
+mod failsafe_campaign {
+    use super::*;
+    use relay_calib::CalParams;
+    use relay_fsm::{Event, Mode};
+
+    struct SplitMix64(u64);
+    impl SplitMix64 {
+        fn next_u64(&mut self) -> u64 {
+            self.0 = self.0.wrapping_add(0x9E37_79B9_7F4A_7C15);
+            let mut z = self.0;
+            z = (z ^ (z >> 30)).wrapping_mul(0xBF58_476D_1CE4_E5B9);
+            z = (z ^ (z >> 27)).wrapping_mul(0x94D0_49BB_1331_11EB);
+            z ^ (z >> 31)
+        }
+        fn range(&mut self, lo: f32, hi: f32) -> f32 {
+            lo + (hi - lo) * ((self.next_u64() >> 40) as f32 / (1u64 << 24) as f32)
+        }
+    }
+    fn trial_rng(seed: u64, i: u32) -> SplitMix64 {
+        let mut s = SplitMix64(seed ^ (i as u64).wrapping_mul(0x9E37_79B9_7F4A_7C15));
+        let _ = s.next_u64();
+        s
+    }
+
+    /// Feeds an IMU consistent with a body tilt of `tilt` rad (gravity leaning
+    /// about body-y), so the IEKF converges the estimate to ~tilt and the
+    /// runaway/wind detectors act on it — exactly the point-tests' mechanism.
+    struct TiltBackend {
+        tilt: f32,
+        pos: Vec3,
+    }
+    impl FlightBackend for TiltBackend {
+        fn read_imu(&mut self) -> ImuSample {
+            let (s, c) = (self.tilt.sin(), self.tilt.cos());
+            ImuSample {
+                accel: [GRAVITY * s, 0.0, -GRAVITY * c],
+                gyro: [0.0; 3],
+            }
+        }
+        fn read_position(&mut self) -> Option<Vec3> {
+            Some(self.pos)
+        }
+        fn read_mag(&mut self) -> Option<Vec3> {
+            None
+        }
+        fn read_battery_v(&mut self) -> f32 {
+            16.0
+        }
+        fn write_motors(&mut self, _: &[f32]) {}
+        fn dt(&self) -> f32 {
+            0.004
+        }
+    }
+
+    /// Converge level, arm, take off → airborne (mode Takeoff).
+    fn airborne(sup: &mut FlightSupervisor, b: &mut TiltBackend) {
+        b.tilt = 0.0;
+        b.pos = [0.0, 0.0, 0.0];
+        for _ in 0..1500 {
+            sup.step(b);
+        }
+        sup.command(Event::Arm, true, true);
+        sup.command(Event::RequestTakeoff, true, false);
+    }
+
+    /// **Runaway → terminate, three-way on tilt vs the runaway limit.** Half the
+    /// trials sit a sustained tilt ABOVE the limit (must terminate + latch); half
+    /// sit a tilt BELOW it (must NOT terminate — an RTL via the wind path is not
+    /// a false terminate). Both regimes randomised.
+    const RUNAWAY_SEED: u64 = 0x0FA1_C0DE_5AFE_0001;
+
+    #[test]
+    fn failsafe_runaway_terminate_campaign() {
+        // no_std crate: no alloc ⇒ no Vec/String/format!; fail-fast with core
+        // panic formatting (index + tilt + mode is the reproducer).
+        let trials = 120u32; // deterministic tilt sweep: 60/regime densely covers the range (noise dispersion is the next slice)
+        let (mut term_fires, mut no_false_term) = (0u32, 0u32);
+        for i in 0..trials {
+            let mut rng = trial_rng(RUNAWAY_SEED, i);
+            let runaway = i % 2 == 0;
+            let mut sup = FlightSupervisor::new([0.0, 0.0, 0.0], 1.0e9, 2.0, 14.0);
+            sup.set_calibration(CalParams {
+                gyro_bias: [0.001, 0.0, 0.0],
+                ..CalParams::identity()
+            });
+            let mut b = TiltBackend {
+                tilt: 0.0,
+                pos: [0.0, 0.0, 0.0],
+            };
+            airborne(&mut sup, &mut b);
+            assert_ne!(sup.mode(), Mode::Terminated, "trial {i}: terminated on level takeoff");
+            if runaway {
+                b.tilt = rng.range(1.15, 1.50); // strictly > TILT_RUNAWAY_LIMIT
+                for _ in 0..4000 {
+                    sup.step(&mut b);
+                }
+                assert_eq!(
+                    sup.mode(),
+                    Mode::Terminated,
+                    "trial {i}: tilt {:.3} did not terminate",
+                    b.tilt
+                );
+                // latched: return level, must STAY terminated.
+                b.tilt = 0.0;
+                for _ in 0..500 {
+                    sup.step(&mut b);
+                }
+                assert_eq!(sup.mode(), Mode::Terminated, "trial {i}: terminate did not latch");
+                term_fires += 1;
+            } else {
+                b.tilt = rng.range(0.0, 0.95); // strictly < TILT_RUNAWAY_LIMIT
+                for _ in 0..4000 {
+                    sup.step(&mut b);
+                }
+                assert_ne!(
+                    sup.mode(),
+                    Mode::Terminated,
+                    "trial {i}: FALSE terminate at tilt {:.3}",
+                    b.tilt
+                );
+                no_false_term += 1;
+            }
+        }
+        // Both regimes well-sampled (≈200 each) ⇒ the pass covers "terminates
+        // when it must" AND "never false-terminates"; a one-sided pass trips this.
+        assert!(
+            term_fires > 40 && no_false_term > 40,
+            "both regimes must be well-sampled ({term_fires}/{no_false_term})"
+        );
+    }
+
+    /// **Wind → RTL, three-way on tilt in/out of the saturation band.** Half the
+    /// trials sit a sustained tilt INSIDE [0.30,0.70] away from home (the RTL
+    /// failsafe must latch); half sit a tilt BELOW the band (must NOT latch RTL).
+    const WIND_SEED: u64 = 0x0FA1_C0DE_5AFE_0002;
+
+    #[test]
+    fn failsafe_wind_rtl_campaign() {
+        let trials = 120u32; // deterministic tilt sweep: 60/regime densely covers the range (noise dispersion is the next slice)
+        let (mut rtl_fires, mut no_false_rtl) = (0u32, 0u32);
+        for i in 0..trials {
+            let mut rng = trial_rng(WIND_SEED, i);
+            let windy = i % 2 == 0;
+            let mut sup = FlightSupervisor::new([0.0, 0.0, 0.0], 1.0e9, 2.0, 14.0);
+            sup.set_calibration(CalParams {
+                gyro_bias: [0.001, 0.0, 0.0],
+                ..CalParams::identity()
+            });
+            let mut b = TiltBackend {
+                tilt: 0.0,
+                pos: [0.0, 0.0, 0.0],
+            };
+            airborne(&mut sup, &mut b);
+            if windy {
+                b.tilt = rng.range(0.38, 0.62); // inside the band with margin
+                b.pos = [rng.range(80.0, 200.0), 0.0, 0.0]; // away from home
+                for _ in 0..800 {
+                    sup.step(&mut b);
+                }
+                assert!(
+                    sup.rtl_latched,
+                    "trial {i}: wind tilt {:.3} did NOT latch RTL (mode {:?})",
+                    b.tilt,
+                    sup.mode()
+                );
+                rtl_fires += 1;
+            } else {
+                b.tilt = rng.range(0.0, 0.20); // below the band
+                for _ in 0..800 {
+                    sup.step(&mut b);
+                }
+                assert!(
+                    !sup.rtl_latched,
+                    "trial {i}: FALSE wind RTL at tilt {:.3}",
+                    b.tilt
+                );
+                no_false_rtl += 1;
+            }
+        }
+        // Both regimes well-sampled ⇒ the pass covers "RTL latches when it must"
+        // AND "never false-latches"; a one-sided pass trips this.
+        assert!(
+            rtl_fires > 40 && no_false_rtl > 40,
+            "both regimes must be well-sampled ({rtl_fires}/{no_false_rtl})"
+        );
+    }
+}