Kademlia: intra-lookup RPC retransmit for loss-tolerant lookups

projects/quorum-distsys-k7r2/JOURNAL.md

@@ -465,8 +465,11 @@ contrast with Chord: a *tree* not a ring, an **XOR metric** not clockwise distan
         lookup latency / hop-count vs. network size to make the O(log N) scaling visible.
   - [ ] **Value expiry & re-publication** — TTLs on stored pairs and the caching of a value at the closest
         node that missed on the lookup path (Kademlia's read-through cache), shown decaying over time.
-  - [ ] **Intra-lookup RPC retransmit** — retry a timed-out probe once before dropping it, so single async
-        lookups stay exact under loss too (not just the eventual tables); measure the exactness gain.
+  - [x] **Intra-lookup RPC retransmit** — a timed-out probe is retransmitted up to `lookupRetries` (=1)
+        times before it is given up (Kademlia retransmits RPCs), *without* evicting the peer from the
+        routing table. Measured live: lookups from one node under packet loss went from 21/45 to 31/35
+        exact at 10% drop, with drop=0 staying a perfect 35/35 — so a single async lookup now tolerates
+        loss too, not just the eventually-refreshed tables.
   - [ ] **Routing-table refresh scheduling** — per-bucket last-touched timers (refresh only stale buckets)
         instead of a blanket periodic sweep, matching the paper; show refresh traffic drop.
 
@@ -1144,7 +1147,11 @@ dead ends, and Herlihy & Wing's locality theorem. Self-contained in `src/linz/*`
   retrievable from every node**; absent-key not-found (no false positive); heal after two crashes;
   restart-rejoin; **tables converging under 15% message loss** (a dropped reply no longer evicts a live
   contact — only a failed liveness ping does, which is what makes the α-parallel lookup loss-tolerant);
-  determinism. Suite **165 → 178/178**. Drove the built `#/kademlia` route in headless Chromium: the
+  determinism. Suite **165 → 178/178**. Also landed a
+  backlog item the same session — **intra-lookup RPC retransmit** (a timed-out probe is retried up to
+  `lookupRetries`=1 times before being dropped, without evicting the peer): live lookups under packet
+  loss went from 21/45 to 31/35 exact at 10% drop, drop=0 staying a perfect 35/35. Drove the built
+  `#/kademlia` route in headless Chromium: the
   8-node network converges, all three invariants green (26/26 required buckets populated, 64/64 (node,
   target) lookups exact), and the trie renders node D's buckets-as-subtrees correctly (bucket 7 holds 4
   of the 6 far nodes, bucket 5 holds F) — no page errors. Full gate green (scope + conformance + lint +

projects/quorum-distsys-k7r2/src/protocols/kademlia/kademlia.ts

@@ -124,6 +124,7 @@ export function createKademlia(config: KademliaConfig = DEFAULT_KADEMLIA_CONFIG)
       kind,
       shortlist: seed,
       status: status as Lookup['status'],
+      retries: {},
       inflight: 0,
       rounds: 0,
       value: null,
@@ -344,11 +345,23 @@ export function createKademlia(config: KademliaConfig = DEFAULT_KADEMLIA_CONFIG)
         const lk = s.lookups[lid];
         if (!lk) return;
         if (lk.status[pid] === 'pending') {
-          // The probe timed out *for this lookup* — try the next-closest contact.
-          // We do NOT evict the peer from the routing table on a single missed
-          // reply (a dropped packet ≠ a dead node); k-bucket eviction only ever
-          // happens after a direct liveness ping fails during a full-bucket
-          // replacement. This is what makes the α-parallel lookup loss-tolerant.
+          // Retransmit the probe up to `lookupRetries` times before giving up —
+          // Kademlia retransmits RPCs, so a single dropped packet doesn't cost
+          // the lookup its answer. Only after the retries are exhausted do we
+          // treat the peer as dead *for this lookup* (still WITHOUT evicting it
+          // from the routing table — a dropped packet ≠ a dead node; k-bucket
+          // eviction happens only when a direct liveness ping fails). This is
+          // what keeps even a single async lookup exact under packet loss.
+          const used = lk.retries[pid] ?? 0;
+          if (used < config.lookupRetries) {
+            lk.retries[pid] = used + 1;
+            const nm = nameOf(s, pid);
+            if (nm !== null) {
+              ctx.send(nm, 'KFind', { lid: lk.id, target: lk.target, kind: lk.kind, src: s.id } as FindReq);
+              ctx.setTimer(`find:${lk.id}:${pid}`, config.rpcTimeout);
+              return; // stay pending, one more probe in flight
+            }
+          }
           lk.status[pid] = 'dead';
           lk.inflight = Math.max(0, lk.inflight - 1);
           pump(ctx, s, lk);

projects/quorum-distsys-k7r2/src/protocols/kademlia/types.ts

@@ -17,6 +17,10 @@ export interface KademliaConfig {
   alpha: number;
   /** How long to wait for an RPC reply before declaring a contact dead. */
   rpcTimeout: number;
+  /** How many times a single lookup probe is retransmitted before it is given
+   *  up (Kademlia retransmits RPCs) — this keeps a lookup exact under packet
+   *  loss, not just the eventually-refreshed routing tables. */
+  lookupRetries: number;
   /** Period of the background bucket-refresh lookup that keeps tables fresh. */
   refreshInterval: number;
   /** Period at which a node re-publishes (STOREs) the keys it originated. */
@@ -28,6 +32,7 @@ export const DEFAULT_KADEMLIA_CONFIG: KademliaConfig = {
   k: 4,
   alpha: 3,
   rpcTimeout: 260,
+  lookupRetries: 1,
   refreshInterval: 900,
   republishInterval: 1600,
 };
@@ -56,6 +61,8 @@ export interface Lookup {
   /** Every candidate id we have heard of for this lookup (incl. self). */
   shortlist: number[];
   status: Record<number, CandStatus>;
+  /** per-peer retransmit count for the in-flight probe (loss tolerance). */
+  retries: Record<number, number>;
   inflight: number;
   rounds: number;
   /** For value lookups: the value once some node returns it. */