Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 5 additions & 0 deletions noq-proto/src/connection/packet_builder.rs
Original file line number Diff line number Diff line change
Expand Up @@ -281,6 +281,11 @@ impl<'a, 'b> PacketBuilder<'a, 'b> {
let space_id = self.space;
let (size, padded, sent) = self.finish(conn, now);

// Count only tracked sends. Off-path PATH_CHALLENGE/PATH_RESPONSE packets call
// `finish` directly and only share the path's logical identifier, not the
// network path itself, so they stay out of per-path stats.
conn.path_stats.for_path(path_id).sent_packets += 1;

let size = match padded || ack_eliciting {
true => size as u16,
false => 0,
Expand Down
19 changes: 19 additions & 0 deletions noq-proto/src/connection/stats.rs
Original file line number Diff line number Diff line change
Expand Up @@ -229,6 +229,13 @@ pub struct PathStats {
pub congestion_events: u64,
/// Spurious congestion events on the connection.
pub spurious_congestion_events: u64,
/// The number of packets sent on this path.
///
/// This counts individual QUIC packets, which can differ from
/// [`UdpStats::datagrams`] when multiple packets are coalesced into a single UDP
/// datagram (e.g. during the handshake). Together with [`Self::lost_packets`] this
/// allows computing an accurate packet loss rate.
pub sent_packets: u64,
/// The number of packets lost on this path.
pub lost_packets: u64,
/// The number of bytes lost on this path.
Expand Down Expand Up @@ -262,6 +269,13 @@ pub struct ConnectionStats {
pub frame_tx: FrameStats,
/// Statistics about frames received on the connection.
pub frame_rx: FrameStats,
/// The number of packets sent on the connection.
///
/// This counts individual QUIC packets, which can differ from
/// [`UdpStats::datagrams`] when multiple packets are coalesced into a single UDP
/// datagram (e.g. during the handshake). Together with [`Self::lost_packets`] this
/// allows computing an accurate packet loss rate.
pub sent_packets: u64,
/// The number of packets lost on the connection.
pub lost_packets: u64,
/// The number of bytes lost on the connection.
Expand All @@ -283,6 +297,7 @@ impl std::ops::Add<PathStats> for ConnectionStats {
cwnd: _,
congestion_events: _,
spurious_congestion_events: _,
sent_packets,
lost_packets,
lost_bytes,
sent_plpmtud_probes: _,
Expand All @@ -295,6 +310,7 @@ impl std::ops::Add<PathStats> for ConnectionStats {
udp_rx: self.udp_rx + udp_rx,
frame_tx: self.frame_tx + frame_tx,
frame_rx: self.frame_rx + frame_rx,
sent_packets: self.sent_packets + sent_packets,

Copy link
Copy Markdown
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I wonder if this is good enough. These are connection wide so I'm on the fence on whether off-path packets should also taken into account. I'm tempted to say yes, but also it's probably convoluted to implement. What do you all think about this?

Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I think the important part is that lost_packets and sent_packets make sense in relation to each other. We should never have lost_packets > sent_packets, for example.
If off-path packets are taken into account for lost_packets (which I haven't checked!), then they should be taken into account for sent_packets.

Copy link
Copy Markdown
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

@mvanhorn this is solid advice. Would you mind checking that this is consistent so that we can move forward with this PR?

lost_packets: self.lost_packets + lost_packets,
lost_bytes: self.lost_bytes + lost_bytes,
}
Expand All @@ -314,6 +330,7 @@ impl std::ops::AddAssign<PathStats> for ConnectionStats {
cwnd: _,
congestion_events: _,
spurious_congestion_events: _,
sent_packets: path_sent_packets,
lost_packets: path_lost_packets,
lost_bytes: path_lost_bytes,
sent_plpmtud_probes: _,
Expand All @@ -326,13 +343,15 @@ impl std::ops::AddAssign<PathStats> for ConnectionStats {
udp_rx,
frame_tx,
frame_rx,
sent_packets,
lost_packets,
lost_bytes,
} = self;
*udp_tx += path_udp_tx;
*udp_rx += path_udp_rx;
*frame_tx += path_frame_tx;
*frame_rx += path_frame_rx;
*sent_packets += path_sent_packets;
*lost_packets += path_lost_packets;
*lost_bytes += path_lost_bytes;
}
Expand Down
93 changes: 93 additions & 0 deletions noq-proto/src/tests/multipath.rs
Original file line number Diff line number Diff line change
Expand Up @@ -1161,6 +1161,99 @@ fn path_scheduling_path_status() -> TestResult {
Ok(())
}

/// Tracks `sent_packets` per path and on the connection, and checks it against
/// `udp_tx.datagrams`.
///
/// `sent_packets` counts individual QUIC packets, while `udp_tx.datagrams` counts UDP
/// datagrams. Since packets can be coalesced into a single datagram (e.g. during the
/// handshake), `sent_packets` must always be at least `udp_tx.datagrams`, and the
/// connection total must equal the sum across paths. See
/// <https://github.com/n0-computer/noq/issues/565>.
#[test]
fn sent_packets_tracked_per_path() -> TestResult {
let _guard = subscribe();
let mut pair = ConnPair::builder().enable_multipath().connect();

// After the handshake, path 0 has sent at least one packet, and the per-packet
// counter must be at least the per-datagram counter (they diverge when packets are
// coalesced into a single datagram).
let stats_path0 = pair.conn_mut(Client).path_stats(PathId::ZERO).unwrap();
assert!(
stats_path0.sent_packets > 0,
"path 0 should have sent packets after the handshake"
);
assert!(
stats_path0.sent_packets >= stats_path0.udp_tx.datagrams,
"sent_packets ({}) must be >= udp_tx.datagrams ({})",
stats_path0.sent_packets,
stats_path0.udp_tx.datagrams,
);

// Open a 2nd path and send some data on it.
let server_addr = pair.routes.public_server_addr();
let path_1 = pair.open_path(
Client,
FourTuple::from_remote(server_addr),
PathStatus::Available,
)?;
pair.drive();
assert_matches!(
pair.poll(Client),
Some(Event::Path(crate::PathEvent::Established { id })) if id == path_1
);

// Put path 0 into Backup so the scheduler prefers the new path 1 for application
// data (mirrors `path_scheduling_path_status`).
pair.set_path_status(Client, PathId::ZERO, PathStatus::Backup)?;
pair.drive();

let path0_before = pair
.conn_mut(Client)
.path_stats(PathId::ZERO)
.unwrap()
.sent_packets;
let path1_before = pair
.conn_mut(Client)
.path_stats(path_1)
.unwrap()
.sent_packets;

// Sending a STREAM frame goes out on path 1, so its counter should grow. Path 0 may
// still emit a few control/ACK packets, but the bulk of the new packets land on
// path 1.
let s = pair.streams(Client).open(Dir::Uni).unwrap();
pair.send_stream(Client, s).write(b"hello").unwrap();
pair.drive();

let path0_stats = pair.conn_mut(Client).path_stats(PathId::ZERO).unwrap();
let path1_stats = pair.conn_mut(Client).path_stats(path_1).unwrap();
let path0_delta = path0_stats.sent_packets - path0_before;
let path1_delta = path1_stats.sent_packets - path1_before;
assert!(
path1_delta > 0,
"path 1 sent_packets should grow after sending a STREAM frame"
);
assert!(
path1_delta > path0_delta,
"the STREAM frame should drive more packets on path 1 ({path1_delta}) \
than on path 0 ({path0_delta})"
);

// Per-packet counter stays consistent with per-datagram counter on each path.
assert!(path0_stats.sent_packets >= path0_stats.udp_tx.datagrams);
assert!(path1_stats.sent_packets >= path1_stats.udp_tx.datagrams);

// The connection total must equal the sum across the live paths.
let conn_stats = pair.conn_mut(Client).stats();
assert_eq!(
conn_stats.sent_packets,
path0_stats.sent_packets + path1_stats.sent_packets,
"connection sent_packets must equal the sum across paths"
);

Ok(())
}

#[test]
fn server_abandon_last_verified_path() -> TestResult {
// The client abandons the last verified path the server has. The server is expected to
Expand Down
Loading