Sim: exercise unreliable delivery + reliability recovery over multi-hop

src/dapps/dapps.core.tests/MeshFabricScenarioTests.cs

@@ -1,4 +1,5 @@
 using AwesomeAssertions;
+using dapps.meshcore;
 using dapps.meshcore.sim;
 using Xunit;
 
@@ -57,6 +58,50 @@ public async Task FullStack_FourRelayHops_BidirectionalDelivery_Discovery_NoDupl
         a.DiscoveredPeers.Should().Contain("GB7C-1");
     }
 
+    [Theory]
+    [InlineData(0.3)]
+    [InlineData(0.4)]
+    public async Task LossyMultiHop_ReliabilityRecoversEveryMessage_ExactlyOnce(double lossPerEdge)
+    {
+        // A - R1 - R2 - R3 - B over FOUR lossy hops. End-to-end reliability (ACK + resend)
+        // must recover every message despite heavy per-edge loss, and idempotent inbound
+        // must still deliver each exactly once - a lost ACK makes the sender resend, so B
+        // sees duplicates it must dedup. This is the whole reason the reliability layer
+        // exists, and multi-hop compounds the loss; the earlier tests ran at 0% loss.
+        var f = new MeshFabric(seed: 20260701);
+        f.AddRelay("R1"); f.AddRelay("R2"); f.AddRelay("R3");
+
+        // Accelerate the resend timings so this runs in CI-time rather than on the 20 s
+        // production backoff (WaitUntil returns as soon as everything arrives, so the
+        // common case finishes in a few seconds; the deadline is only a safety cap).
+        var fast = new MeshCoreReliability.Options(
+            BaseBackoff: TimeSpan.FromMilliseconds(120), Multiplier: 1.4,
+            MaxBackoff: TimeSpan.FromMilliseconds(600), MaxLifetime: TimeSpan.FromSeconds(60));
+        var poll = TimeSpan.FromMilliseconds(100);
+
+        var a = new MeshDappsNode(f, "GB7A-1", reliabilityOptions: fast, resendPoll: poll);
+        var b = new MeshDappsNode(f, "GB7B-1", reliabilityOptions: fast, resendPoll: poll);
+        f.Connect("GB7A-1", "R1", lossPerEdge);
+        f.Connect("R1", "R2", lossPerEdge);
+        f.Connect("R2", "R3", lossPerEdge);
+        f.Connect("R3", "GB7B-1", lossPerEdge);
+
+        using var cts = new CancellationTokenSource();
+        var loops = Task.WhenAll(a.RunAsync(cts.Token), b.RunAsync(cts.Token));
+
+        const int N = 5;
+        for (var i = 0; i < N; i++)
+            await a.SendAsync("GB7B-1", i, $"A->B #{i}", cts.Token);
+
+        await WaitUntil(() => b.DistinctSeqs == N, TimeSpan.FromSeconds(50));
+        cts.Cancel();
+        try { await loops; } catch { /* cancelled */ }
+
+        b.DistinctSeqs.Should().Be(N, "reliability resends recovered every message despite {0:P0} per-hop loss", lossPerEdge);
+        b.Delivered.Should().Be(N, "idempotent: each delivered exactly once despite resends after lost ACKs");
+        f.Dropped.Should().BeGreaterThan(0, "loss was genuinely exercised (not a no-op path)");
+    }
+
     [Fact]
     public async Task FullStack_FanIn_ManySendersOneCollector_AllDeliveredOnce()
     {

src/dapps/dapps.meshcore.sim/MeshDappsNode.cs

@@ -21,6 +21,7 @@ public sealed class MeshDappsNode
     private readonly MeshCoreReliability? _reliability;
     private readonly RecordingInbox _inbox;
     private readonly string _channel;
+    private readonly TimeSpan _resendPoll;
     private readonly ConcurrentDictionary<string, int> _discovered = new(StringComparer.OrdinalIgnoreCase);
 
     public string Callsign { get; }
@@ -33,19 +34,30 @@ public sealed class MeshDappsNode
     /// <summary>Peers this node learned purely by hearing their traffic (passive discovery).</summary>
     public IReadOnlyCollection<string> DiscoveredPeers => _discovered.Keys.ToList();
 
-    public MeshDappsNode(MeshFabric fabric, string callsign, string channel = "dapps-sim", bool reliable = true, string scope = "")
+    /// <param name="reliabilityOptions">Override the ACK/resend timings. Null uses the
+    /// production defaults (20 s base backoff); an accelerated profile lets a loss-recovery
+    /// scenario run in CI-time instead of minutes.</param>
+    /// <param name="resendPoll">How often the resend loop checks for due retransmits.
+    /// Should be no slower than the backoff, or it becomes the bottleneck.</param>
+    public MeshDappsNode(MeshFabric fabric, string callsign, string channel = "dapps-sim", bool reliable = true, string scope = "",
+        MeshCoreReliability.Options? reliabilityOptions = null, TimeSpan? resendPoll = null)
     {
         Callsign = callsign;
         _channel = channel;
+        _resendPoll = resendPoll ?? TimeSpan.FromSeconds(2);
         _link = fabric.AddLeaf(callsign, scope);
         var opts = new MeshCoreBearerOptions
         {
             Region = "uk-test", ChannelName = channel, ChannelIndex = 1,
             LocalCallsign = callsign, NodeName = callsign, ReliableDelivery = reliable,
             LbtGuardMs = 0,          // no radio timing to wait on in the sim
             AirtimeBudgetSecPerHour = 3600,   // don't let the governor gate the scenario
+            // Propagation is instantaneous here, so the occupancy estimate (from overheard
+            // packet airtime over wall-clock) is an artifact - disable congestion backoff
+            // so it can't confound reliability-recovery scenarios by refusing resends.
+            CongestionBackoffFraction = 0,
         };
-        _reliability = reliable ? new MeshCoreReliability() : null;
+        _reliability = reliable ? new MeshCoreReliability(reliabilityOptions) : null;
         _inbox = new RecordingInbox(callsign);
         _backhaul = new MeshCoreCompanionBackhaul(_link, opts, new TxBudget(opts.AirtimeBudgetSecPerHour),
             NullLogger.Instance, reliability: _reliability);
@@ -78,7 +90,7 @@ private async Task ResendLoopAsync(CancellationToken ct)
     {
         while (!ct.IsCancellationRequested)
         {
-            try { await Task.Delay(TimeSpan.FromSeconds(2), ct); } catch { break; }
+            try { await Task.Delay(_resendPoll, ct); } catch { break; }
             var now = DateTime.UtcNow;
             _reliability!.DropExpired(now);
             foreach (var (m, local) in _reliability.DueResends(now))

src/dapps/dapps.meshcore.sim/MeshFabric.cs

@@ -59,6 +59,9 @@ private sealed class Node
     /// new packet on), and total re-floods - coarse observability for scenarios.</summary>
     public long Deliveries { get; private set; }
     public long Refloods { get; private set; }
+    /// <summary>Transmissions dropped by per-edge loss - lets a test assert loss was
+    /// genuinely exercised (not a no-op) rather than rely on a probabilistic control.</summary>
+    public long Dropped { get; private set; }
 
     /// <param name="seed">Seed for the loss RNG so scenarios are reproducible.</param>
     /// <param name="snrQuarterDb">Reported SNR in quarter-dB (default 40 = 10 dB).</param>
@@ -141,7 +144,7 @@ private void Transmit(Node tx, int receiversPathLen, string packetId, string sco
     {
         foreach (var (toId, loss) in tx.Neighbours)
         {
-            if (loss > 0 && _rng.NextDouble() < loss) continue;   // lost on this edge
+            if (loss > 0 && _rng.NextDouble() < loss) { Dropped++; continue; }   // lost on this edge
             var nbr = _nodes[toId];
 
             // Overhearing is a PHY event: it happens whenever RF arrives, even for a

src/dapps/dapps.meshcore.sim/README.md

@@ -49,15 +49,23 @@ cts.Cancel();
 ```
 
 See `MeshFabricTests` (fast, deterministic flood/dedup/scope + transport-over-multi-hop)
-and `MeshFabricScenarioTests` (the full bearer stack over a relay backbone) in
-`dapps.core.tests`.
+and `MeshFabricScenarioTests` (the full bearer stack over a relay backbone, including a
+**lossy multi-hop reliability-recovery** case) in `dapps.core.tests`.
+
+### Loss + reliability
+
+Edges take a per-transmission drop probability, so a scenario can run at 30–40 %/hop
+over a multi-hop backbone and assert the reliability layer (ACK + resend) recovers **every**
+message while idempotent inbound still delivers each **exactly once** (a lost ACK makes the
+sender resend, so the receiver must dedup the duplicate). `MeshDappsNode` takes an
+accelerated `MeshCoreReliability.Options` so this runs in CI-time rather than on the 20 s
+production backoff, and it disables congestion-backoff (the occupancy estimate is an
+artifact under instant propagation — otherwise resend traffic would throttle itself).
 
 ## Scope / limits
 
 - Models the **channel**, not the serial link's liveness — the simulated link is always
   healthy (watchdog/recovery is out of scope).
-- No-loss runs are fully deterministic and fast; **loss-recovery** exercises reliability
-  resends on a multi-second cadence, so drive those from a longer harness run rather than
-  a CI unit test.
 - Propagation is instantaneous (no per-hop latency model yet) — correctness (delivery,
-  dedup, containment) is faithful; fine-grained timing/contention is not.
+  dedup, containment, loss-recovery) is faithful; fine-grained timing/contention is not,
+  which is also why occupancy-driven congestion backoff is disabled in the sim node.