From 53a5efd4aee8a1f0e725dfe696cd8f76f8af58b5 Mon Sep 17 00:00:00 2001 From: Tom M0LTE <37816024+M0LTE@users.noreply.github.com> Date: Wed, 1 Jul 2026 13:25:42 +0000 Subject: [PATCH] Sim: exercise unreliable delivery + reliability recovery over multi-hop The fabric had a per-edge loss knob but every committed test ran at 0% loss, so the reliability layer (the whole reason it exists) was never exercised over the sim - a real gap. Close it. - MeshDappsNode takes an accelerated MeshCoreReliability.Options + resend poll, so a loss-recovery scenario runs in CI-time instead of on the 20 s production backoff. It also disables congestion-backoff: propagation is instantaneous in the sim, so the occupancy estimate is an artifact, and under heavy loss the extra resend traffic would otherwise inflate it and throttle recovery (a genuine production interaction, but a confound here). - MeshFabric gains a Dropped counter so a test can assert loss was genuinely exercised rather than lean on a flaky probabilistic control. - New test LossyMultiHop_ReliabilityRecoversEveryMessage_ExactlyOnce (0.3 and 0.4 per-hop loss over four hops): asserts every message is recovered, each delivered exactly once (idempotent under resend-after-lost-ACK), and Dropped > 0. Ran 5x with no flakiness; recovers in ~3-6 s. Co-Authored-By: Claude Opus 4.8 (1M context) Claude-Session: https://claude.ai/code/session_01KLbwvhE2cKCe8WPZNg8k17 --- .../MeshFabricScenarioTests.cs | 45 +++++++++++++++++++ src/dapps/dapps.meshcore.sim/MeshDappsNode.cs | 18 ++++++-- src/dapps/dapps.meshcore.sim/MeshFabric.cs | 5 ++- src/dapps/dapps.meshcore.sim/README.md | 20 ++++++--- 4 files changed, 78 insertions(+), 10 deletions(-) diff --git a/src/dapps/dapps.core.tests/MeshFabricScenarioTests.cs b/src/dapps/dapps.core.tests/MeshFabricScenarioTests.cs index a60dc40..baa1b21 100644 --- a/src/dapps/dapps.core.tests/MeshFabricScenarioTests.cs +++ b/src/dapps/dapps.core.tests/MeshFabricScenarioTests.cs @@ -1,4 +1,5 @@ using AwesomeAssertions; +using dapps.meshcore; using dapps.meshcore.sim; using Xunit; @@ -57,6 +58,50 @@ public async Task FullStack_FourRelayHops_BidirectionalDelivery_Discovery_NoDupl a.DiscoveredPeers.Should().Contain("GB7C-1"); } + [Theory] + [InlineData(0.3)] + [InlineData(0.4)] + public async Task LossyMultiHop_ReliabilityRecoversEveryMessage_ExactlyOnce(double lossPerEdge) + { + // A - R1 - R2 - R3 - B over FOUR lossy hops. End-to-end reliability (ACK + resend) + // must recover every message despite heavy per-edge loss, and idempotent inbound + // must still deliver each exactly once - a lost ACK makes the sender resend, so B + // sees duplicates it must dedup. This is the whole reason the reliability layer + // exists, and multi-hop compounds the loss; the earlier tests ran at 0% loss. + var f = new MeshFabric(seed: 20260701); + f.AddRelay("R1"); f.AddRelay("R2"); f.AddRelay("R3"); + + // Accelerate the resend timings so this runs in CI-time rather than on the 20 s + // production backoff (WaitUntil returns as soon as everything arrives, so the + // common case finishes in a few seconds; the deadline is only a safety cap). + var fast = new MeshCoreReliability.Options( + BaseBackoff: TimeSpan.FromMilliseconds(120), Multiplier: 1.4, + MaxBackoff: TimeSpan.FromMilliseconds(600), MaxLifetime: TimeSpan.FromSeconds(60)); + var poll = TimeSpan.FromMilliseconds(100); + + var a = new MeshDappsNode(f, "GB7A-1", reliabilityOptions: fast, resendPoll: poll); + var b = new MeshDappsNode(f, "GB7B-1", reliabilityOptions: fast, resendPoll: poll); + f.Connect("GB7A-1", "R1", lossPerEdge); + f.Connect("R1", "R2", lossPerEdge); + f.Connect("R2", "R3", lossPerEdge); + f.Connect("R3", "GB7B-1", lossPerEdge); + + using var cts = new CancellationTokenSource(); + var loops = Task.WhenAll(a.RunAsync(cts.Token), b.RunAsync(cts.Token)); + + const int N = 5; + for (var i = 0; i < N; i++) + await a.SendAsync("GB7B-1", i, $"A->B #{i}", cts.Token); + + await WaitUntil(() => b.DistinctSeqs == N, TimeSpan.FromSeconds(50)); + cts.Cancel(); + try { await loops; } catch { /* cancelled */ } + + b.DistinctSeqs.Should().Be(N, "reliability resends recovered every message despite {0:P0} per-hop loss", lossPerEdge); + b.Delivered.Should().Be(N, "idempotent: each delivered exactly once despite resends after lost ACKs"); + f.Dropped.Should().BeGreaterThan(0, "loss was genuinely exercised (not a no-op path)"); + } + [Fact] public async Task FullStack_FanIn_ManySendersOneCollector_AllDeliveredOnce() { diff --git a/src/dapps/dapps.meshcore.sim/MeshDappsNode.cs b/src/dapps/dapps.meshcore.sim/MeshDappsNode.cs index 8bbd800..41cd27a 100644 --- a/src/dapps/dapps.meshcore.sim/MeshDappsNode.cs +++ b/src/dapps/dapps.meshcore.sim/MeshDappsNode.cs @@ -21,6 +21,7 @@ public sealed class MeshDappsNode private readonly MeshCoreReliability? _reliability; private readonly RecordingInbox _inbox; private readonly string _channel; + private readonly TimeSpan _resendPoll; private readonly ConcurrentDictionary _discovered = new(StringComparer.OrdinalIgnoreCase); public string Callsign { get; } @@ -33,10 +34,17 @@ public sealed class MeshDappsNode /// Peers this node learned purely by hearing their traffic (passive discovery). public IReadOnlyCollection DiscoveredPeers => _discovered.Keys.ToList(); - public MeshDappsNode(MeshFabric fabric, string callsign, string channel = "dapps-sim", bool reliable = true, string scope = "") + /// Override the ACK/resend timings. Null uses the + /// production defaults (20 s base backoff); an accelerated profile lets a loss-recovery + /// scenario run in CI-time instead of minutes. + /// How often the resend loop checks for due retransmits. + /// Should be no slower than the backoff, or it becomes the bottleneck. + public MeshDappsNode(MeshFabric fabric, string callsign, string channel = "dapps-sim", bool reliable = true, string scope = "", + MeshCoreReliability.Options? reliabilityOptions = null, TimeSpan? resendPoll = null) { Callsign = callsign; _channel = channel; + _resendPoll = resendPoll ?? TimeSpan.FromSeconds(2); _link = fabric.AddLeaf(callsign, scope); var opts = new MeshCoreBearerOptions { @@ -44,8 +52,12 @@ public MeshDappsNode(MeshFabric fabric, string callsign, string channel = "dapps LocalCallsign = callsign, NodeName = callsign, ReliableDelivery = reliable, LbtGuardMs = 0, // no radio timing to wait on in the sim AirtimeBudgetSecPerHour = 3600, // don't let the governor gate the scenario + // Propagation is instantaneous here, so the occupancy estimate (from overheard + // packet airtime over wall-clock) is an artifact - disable congestion backoff + // so it can't confound reliability-recovery scenarios by refusing resends. + CongestionBackoffFraction = 0, }; - _reliability = reliable ? new MeshCoreReliability() : null; + _reliability = reliable ? new MeshCoreReliability(reliabilityOptions) : null; _inbox = new RecordingInbox(callsign); _backhaul = new MeshCoreCompanionBackhaul(_link, opts, new TxBudget(opts.AirtimeBudgetSecPerHour), NullLogger.Instance, reliability: _reliability); @@ -78,7 +90,7 @@ private async Task ResendLoopAsync(CancellationToken ct) { while (!ct.IsCancellationRequested) { - try { await Task.Delay(TimeSpan.FromSeconds(2), ct); } catch { break; } + try { await Task.Delay(_resendPoll, ct); } catch { break; } var now = DateTime.UtcNow; _reliability!.DropExpired(now); foreach (var (m, local) in _reliability.DueResends(now)) diff --git a/src/dapps/dapps.meshcore.sim/MeshFabric.cs b/src/dapps/dapps.meshcore.sim/MeshFabric.cs index 221d05e..741bcaf 100644 --- a/src/dapps/dapps.meshcore.sim/MeshFabric.cs +++ b/src/dapps/dapps.meshcore.sim/MeshFabric.cs @@ -59,6 +59,9 @@ private sealed class Node /// new packet on), and total re-floods - coarse observability for scenarios. public long Deliveries { get; private set; } public long Refloods { get; private set; } + /// Transmissions dropped by per-edge loss - lets a test assert loss was + /// genuinely exercised (not a no-op) rather than rely on a probabilistic control. + public long Dropped { get; private set; } /// Seed for the loss RNG so scenarios are reproducible. /// Reported SNR in quarter-dB (default 40 = 10 dB). @@ -141,7 +144,7 @@ private void Transmit(Node tx, int receiversPathLen, string packetId, string sco { foreach (var (toId, loss) in tx.Neighbours) { - if (loss > 0 && _rng.NextDouble() < loss) continue; // lost on this edge + if (loss > 0 && _rng.NextDouble() < loss) { Dropped++; continue; } // lost on this edge var nbr = _nodes[toId]; // Overhearing is a PHY event: it happens whenever RF arrives, even for a diff --git a/src/dapps/dapps.meshcore.sim/README.md b/src/dapps/dapps.meshcore.sim/README.md index 68afd56..4c8c15d 100644 --- a/src/dapps/dapps.meshcore.sim/README.md +++ b/src/dapps/dapps.meshcore.sim/README.md @@ -49,15 +49,23 @@ cts.Cancel(); ``` See `MeshFabricTests` (fast, deterministic flood/dedup/scope + transport-over-multi-hop) -and `MeshFabricScenarioTests` (the full bearer stack over a relay backbone) in -`dapps.core.tests`. +and `MeshFabricScenarioTests` (the full bearer stack over a relay backbone, including a +**lossy multi-hop reliability-recovery** case) in `dapps.core.tests`. + +### Loss + reliability + +Edges take a per-transmission drop probability, so a scenario can run at 30–40 %/hop +over a multi-hop backbone and assert the reliability layer (ACK + resend) recovers **every** +message while idempotent inbound still delivers each **exactly once** (a lost ACK makes the +sender resend, so the receiver must dedup the duplicate). `MeshDappsNode` takes an +accelerated `MeshCoreReliability.Options` so this runs in CI-time rather than on the 20 s +production backoff, and it disables congestion-backoff (the occupancy estimate is an +artifact under instant propagation — otherwise resend traffic would throttle itself). ## Scope / limits - Models the **channel**, not the serial link's liveness — the simulated link is always healthy (watchdog/recovery is out of scope). -- No-loss runs are fully deterministic and fast; **loss-recovery** exercises reliability - resends on a multi-second cadence, so drive those from a longer harness run rather than - a CI unit test. - Propagation is instantaneous (no per-hop latency model yet) — correctness (delivery, - dedup, containment) is faithful; fine-grained timing/contention is not. + dedup, containment, loss-recovery) is faithful; fine-grained timing/contention is not, + which is also why occupancy-driven congestion backoff is disabled in the sim node.