diff --git a/src/dapps/dapps.core.tests/MeshFabricScenarioTests.cs b/src/dapps/dapps.core.tests/MeshFabricScenarioTests.cs index a60dc40..baa1b21 100644 --- a/src/dapps/dapps.core.tests/MeshFabricScenarioTests.cs +++ b/src/dapps/dapps.core.tests/MeshFabricScenarioTests.cs @@ -1,4 +1,5 @@ using AwesomeAssertions; +using dapps.meshcore; using dapps.meshcore.sim; using Xunit; @@ -57,6 +58,50 @@ public async Task FullStack_FourRelayHops_BidirectionalDelivery_Discovery_NoDupl a.DiscoveredPeers.Should().Contain("GB7C-1"); } + [Theory] + [InlineData(0.3)] + [InlineData(0.4)] + public async Task LossyMultiHop_ReliabilityRecoversEveryMessage_ExactlyOnce(double lossPerEdge) + { + // A - R1 - R2 - R3 - B over FOUR lossy hops. End-to-end reliability (ACK + resend) + // must recover every message despite heavy per-edge loss, and idempotent inbound + // must still deliver each exactly once - a lost ACK makes the sender resend, so B + // sees duplicates it must dedup. This is the whole reason the reliability layer + // exists, and multi-hop compounds the loss; the earlier tests ran at 0% loss. + var f = new MeshFabric(seed: 20260701); + f.AddRelay("R1"); f.AddRelay("R2"); f.AddRelay("R3"); + + // Accelerate the resend timings so this runs in CI-time rather than on the 20 s + // production backoff (WaitUntil returns as soon as everything arrives, so the + // common case finishes in a few seconds; the deadline is only a safety cap). + var fast = new MeshCoreReliability.Options( + BaseBackoff: TimeSpan.FromMilliseconds(120), Multiplier: 1.4, + MaxBackoff: TimeSpan.FromMilliseconds(600), MaxLifetime: TimeSpan.FromSeconds(60)); + var poll = TimeSpan.FromMilliseconds(100); + + var a = new MeshDappsNode(f, "GB7A-1", reliabilityOptions: fast, resendPoll: poll); + var b = new MeshDappsNode(f, "GB7B-1", reliabilityOptions: fast, resendPoll: poll); + f.Connect("GB7A-1", "R1", lossPerEdge); + f.Connect("R1", "R2", lossPerEdge); + f.Connect("R2", "R3", lossPerEdge); + f.Connect("R3", "GB7B-1", lossPerEdge); + + using var cts = new CancellationTokenSource(); + var loops = Task.WhenAll(a.RunAsync(cts.Token), b.RunAsync(cts.Token)); + + const int N = 5; + for (var i = 0; i < N; i++) + await a.SendAsync("GB7B-1", i, $"A->B #{i}", cts.Token); + + await WaitUntil(() => b.DistinctSeqs == N, TimeSpan.FromSeconds(50)); + cts.Cancel(); + try { await loops; } catch { /* cancelled */ } + + b.DistinctSeqs.Should().Be(N, "reliability resends recovered every message despite {0:P0} per-hop loss", lossPerEdge); + b.Delivered.Should().Be(N, "idempotent: each delivered exactly once despite resends after lost ACKs"); + f.Dropped.Should().BeGreaterThan(0, "loss was genuinely exercised (not a no-op path)"); + } + [Fact] public async Task FullStack_FanIn_ManySendersOneCollector_AllDeliveredOnce() { diff --git a/src/dapps/dapps.meshcore.sim/MeshDappsNode.cs b/src/dapps/dapps.meshcore.sim/MeshDappsNode.cs index 8bbd800..41cd27a 100644 --- a/src/dapps/dapps.meshcore.sim/MeshDappsNode.cs +++ b/src/dapps/dapps.meshcore.sim/MeshDappsNode.cs @@ -21,6 +21,7 @@ public sealed class MeshDappsNode private readonly MeshCoreReliability? _reliability; private readonly RecordingInbox _inbox; private readonly string _channel; + private readonly TimeSpan _resendPoll; private readonly ConcurrentDictionary _discovered = new(StringComparer.OrdinalIgnoreCase); public string Callsign { get; } @@ -33,10 +34,17 @@ public sealed class MeshDappsNode /// Peers this node learned purely by hearing their traffic (passive discovery). public IReadOnlyCollection DiscoveredPeers => _discovered.Keys.ToList(); - public MeshDappsNode(MeshFabric fabric, string callsign, string channel = "dapps-sim", bool reliable = true, string scope = "") + /// Override the ACK/resend timings. Null uses the + /// production defaults (20 s base backoff); an accelerated profile lets a loss-recovery + /// scenario run in CI-time instead of minutes. + /// How often the resend loop checks for due retransmits. + /// Should be no slower than the backoff, or it becomes the bottleneck. + public MeshDappsNode(MeshFabric fabric, string callsign, string channel = "dapps-sim", bool reliable = true, string scope = "", + MeshCoreReliability.Options? reliabilityOptions = null, TimeSpan? resendPoll = null) { Callsign = callsign; _channel = channel; + _resendPoll = resendPoll ?? TimeSpan.FromSeconds(2); _link = fabric.AddLeaf(callsign, scope); var opts = new MeshCoreBearerOptions { @@ -44,8 +52,12 @@ public MeshDappsNode(MeshFabric fabric, string callsign, string channel = "dapps LocalCallsign = callsign, NodeName = callsign, ReliableDelivery = reliable, LbtGuardMs = 0, // no radio timing to wait on in the sim AirtimeBudgetSecPerHour = 3600, // don't let the governor gate the scenario + // Propagation is instantaneous here, so the occupancy estimate (from overheard + // packet airtime over wall-clock) is an artifact - disable congestion backoff + // so it can't confound reliability-recovery scenarios by refusing resends. + CongestionBackoffFraction = 0, }; - _reliability = reliable ? new MeshCoreReliability() : null; + _reliability = reliable ? new MeshCoreReliability(reliabilityOptions) : null; _inbox = new RecordingInbox(callsign); _backhaul = new MeshCoreCompanionBackhaul(_link, opts, new TxBudget(opts.AirtimeBudgetSecPerHour), NullLogger.Instance, reliability: _reliability); @@ -78,7 +90,7 @@ private async Task ResendLoopAsync(CancellationToken ct) { while (!ct.IsCancellationRequested) { - try { await Task.Delay(TimeSpan.FromSeconds(2), ct); } catch { break; } + try { await Task.Delay(_resendPoll, ct); } catch { break; } var now = DateTime.UtcNow; _reliability!.DropExpired(now); foreach (var (m, local) in _reliability.DueResends(now)) diff --git a/src/dapps/dapps.meshcore.sim/MeshFabric.cs b/src/dapps/dapps.meshcore.sim/MeshFabric.cs index 221d05e..741bcaf 100644 --- a/src/dapps/dapps.meshcore.sim/MeshFabric.cs +++ b/src/dapps/dapps.meshcore.sim/MeshFabric.cs @@ -59,6 +59,9 @@ private sealed class Node /// new packet on), and total re-floods - coarse observability for scenarios. public long Deliveries { get; private set; } public long Refloods { get; private set; } + /// Transmissions dropped by per-edge loss - lets a test assert loss was + /// genuinely exercised (not a no-op) rather than rely on a probabilistic control. + public long Dropped { get; private set; } /// Seed for the loss RNG so scenarios are reproducible. /// Reported SNR in quarter-dB (default 40 = 10 dB). @@ -141,7 +144,7 @@ private void Transmit(Node tx, int receiversPathLen, string packetId, string sco { foreach (var (toId, loss) in tx.Neighbours) { - if (loss > 0 && _rng.NextDouble() < loss) continue; // lost on this edge + if (loss > 0 && _rng.NextDouble() < loss) { Dropped++; continue; } // lost on this edge var nbr = _nodes[toId]; // Overhearing is a PHY event: it happens whenever RF arrives, even for a diff --git a/src/dapps/dapps.meshcore.sim/README.md b/src/dapps/dapps.meshcore.sim/README.md index 68afd56..4c8c15d 100644 --- a/src/dapps/dapps.meshcore.sim/README.md +++ b/src/dapps/dapps.meshcore.sim/README.md @@ -49,15 +49,23 @@ cts.Cancel(); ``` See `MeshFabricTests` (fast, deterministic flood/dedup/scope + transport-over-multi-hop) -and `MeshFabricScenarioTests` (the full bearer stack over a relay backbone) in -`dapps.core.tests`. +and `MeshFabricScenarioTests` (the full bearer stack over a relay backbone, including a +**lossy multi-hop reliability-recovery** case) in `dapps.core.tests`. + +### Loss + reliability + +Edges take a per-transmission drop probability, so a scenario can run at 30–40 %/hop +over a multi-hop backbone and assert the reliability layer (ACK + resend) recovers **every** +message while idempotent inbound still delivers each **exactly once** (a lost ACK makes the +sender resend, so the receiver must dedup the duplicate). `MeshDappsNode` takes an +accelerated `MeshCoreReliability.Options` so this runs in CI-time rather than on the 20 s +production backoff, and it disables congestion-backoff (the occupancy estimate is an +artifact under instant propagation — otherwise resend traffic would throttle itself). ## Scope / limits - Models the **channel**, not the serial link's liveness — the simulated link is always healthy (watchdog/recovery is out of scope). -- No-loss runs are fully deterministic and fast; **loss-recovery** exercises reliability - resends on a multi-second cadence, so drive those from a longer harness run rather than - a CI unit test. - Propagation is instantaneous (no per-hop latency model yet) — correctness (delivery, - dedup, containment) is faithful; fine-grained timing/contention is not. + dedup, containment, loss-recovery) is faithful; fine-grained timing/contention is not, + which is also why occupancy-driven congestion backoff is disabled in the sim node.