From bb82d1eaa5c86c1254814baefe984f957fa3b17d Mon Sep 17 00:00:00 2001 From: gd-0 Date: Tue, 8 Sep 2026 20:02:00 +0100 Subject: [PATCH 1/6] feat(network): add opt-in io_uring UDP backend --- Cargo.lock | 1 + crates/flux-network/Cargo.toml | 3 + crates/flux-network/UDP_URING.md | 157 ++++ .../udp-uring/baseline-benchmark.patch | 88 +++ .../benches/results/udp-uring/branch-1.txt | 27 + .../benches/results/udp-uring/branch-2.txt | 27 + .../benches/results/udp-uring/branch-3.txt | 27 + .../benches/results/udp-uring/branch-4.txt | 27 + .../benches/results/udp-uring/branch-5.txt | 27 + .../benches/results/udp-uring/main-1.txt | 17 + .../benches/results/udp-uring/main-2.txt | 17 + .../benches/results/udp-uring/main-3.txt | 17 + .../benches/results/udp-uring/main-4.txt | 17 + .../benches/results/udp-uring/main-5.txt | 17 + crates/flux-network/benches/udp_pipeline.rs | 77 +- crates/flux-network/src/network_driver.rs | 7 +- crates/flux-network/src/udp/connector.rs | 96 ++- crates/flux-network/src/udp/mod.rs | 41 ++ crates/flux-network/src/udp/peer.rs | 49 +- crates/flux-network/src/udp/sys.rs | 150 +++- crates/flux-network/src/udp/sys/uring.rs | 682 ++++++++++++++++++ .../tests/support/udp_connector.rs | 612 ++++++++++++++++ crates/flux-network/tests/tcp_dcache.rs | 9 + crates/flux-network/tests/udp_connector.rs | 612 +--------------- crates/flux-network/tests/udp_uring.rs | 107 +++ 25 files changed, 2221 insertions(+), 690 deletions(-) create mode 100644 crates/flux-network/UDP_URING.md create mode 100644 crates/flux-network/benches/results/udp-uring/baseline-benchmark.patch create mode 100644 crates/flux-network/benches/results/udp-uring/branch-1.txt create mode 100644 crates/flux-network/benches/results/udp-uring/branch-2.txt create mode 100644 crates/flux-network/benches/results/udp-uring/branch-3.txt create mode 100644 crates/flux-network/benches/results/udp-uring/branch-4.txt create mode 100644 crates/flux-network/benches/results/udp-uring/branch-5.txt create mode 100644 crates/flux-network/benches/results/udp-uring/main-1.txt create mode 100644 crates/flux-network/benches/results/udp-uring/main-2.txt create mode 100644 crates/flux-network/benches/results/udp-uring/main-3.txt create mode 100644 crates/flux-network/benches/results/udp-uring/main-4.txt create mode 100644 crates/flux-network/benches/results/udp-uring/main-5.txt create mode 100644 crates/flux-network/src/udp/sys/uring.rs create mode 100644 crates/flux-network/tests/support/udp_connector.rs create mode 100644 crates/flux-network/tests/udp_uring.rs diff --git a/Cargo.lock b/Cargo.lock index 5f967e9..82d789a 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -664,6 +664,7 @@ dependencies = [ "flux-timing", "flux-utils", "httparse", + "io-uring", "libc", "mio", "serde", diff --git a/crates/flux-network/Cargo.toml b/crates/flux-network/Cargo.toml index d05f118..8eda13f 100644 --- a/crates/flux-network/Cargo.toml +++ b/crates/flux-network/Cargo.toml @@ -20,6 +20,9 @@ serde.workspace = true tracing.workspace = true wincode = { workspace = true, optional = true } +[target.'cfg(target_os = "linux")'.dependencies] +io-uring.workspace = true + [dev-dependencies] core_affinity.workspace = true criterion.workspace = true diff --git a/crates/flux-network/UDP_URING.md b/crates/flux-network/UDP_URING.md new file mode 100644 index 0000000..93249ec --- /dev/null +++ b/crates/flux-network/UDP_URING.md @@ -0,0 +1,157 @@ +# UDP io_uring + +An opt-in Linux backend for `NetworkDriver`. TCP and the default UDP syscall +backend retain their existing behavior. + +```rust +use flux_network::{NetworkDriver, Transport, UdpConfig}; +use flux_network::udp::{UdpIo, UringConfig}; + +let mut network = NetworkDriver::default() + .with_transport(Transport::Udp(UdpConfig { + io: UdpIo::Uring(UringConfig::default()), + ..UdpConfig::lan() + })) + .with_socket_buf_size(16 * 1024 * 1024); +// Use connect/listen_at, write_or_enqueue_with and poll_with as usual. +``` + +Both peers may choose their backend independently. There is no wire-format +change. Selecting io_uring is explicit: socket creation returns `None` and logs +the setup error if the kernel denies or lacks the required facilities. Linux +6.0+ is required for multishot recvmsg and synchronous cancellation; kernel +security policy may still disable them. + +## Implementation and costs + +- One ring per UDP socket, with one multishot receive and a provided-buffer ring. + GRO entries use the existing datagram validation and reassembly path. +- Ordinary io_uring `SendMsg` operations carry both data and control packets. + Compatible runs within a batch retain GSO, including two-fragment messages + and batches crossing peer boundaries. GSO failure retries the original + datagrams without segmentation. +- Each pending send owns a pooled copy of its bytes. Admission to this bounded + queue counts as acceptance by the local socket backend. Completion errors + become packet loss, handled by the existing reliability protocol. Neither + ACK processing nor reconnect can invalidate kernel-owned bytes. +- Completion passes are bounded. ACKs are emitted between receive passes so + callback processing cannot defer them behind a whole send window. Buffer + exhaustion ends/rearms the multishot request after buffers are recycled. +- Normal polling uses no waiting for completions and skips idle kernel entries + using the task-work flag. There is no async executor, SQPOLL thread, or + zero-copy send machinery. +- Moving an active driver to another thread synchronously retires outstanding + requests before rearming them on the new issuer. Drop also synchronously + cancels requests before releasing buffers and the socket. Bind on the worker + that will drive I/O to avoid handoff work. If kernel cancellation fails, + borrowed allocations are retained rather than freed underneath the kernel. + +Default limits are 64 send slots and 32 receive buffers: approximately 6 MiB +per socket, plus ring metadata and the existing protocol state. A GSO group +uses one send slot. Counts are configurable; receive count must be a power of +two, both counts must be nonzero, and their sum must not exceed 4096. Ring +capacity is separate from the protocol's send/receive windows and kernel +socket-buffer limits. UDP still has no receiver flow control. + +The workspace already depended on `io-uring`; `flux-network` now uses the same +version as a Linux-only dependency. The new public `UdpConfig::io` field means +exhaustive struct literals need an update; literals using `..Default::default()` +or `..UdpConfig::lan()` keep compiling. Account for this source compatibility +change when preparing the next release. This branch does not bump versions. + +## Measurements + +Measured on 2026-09-08 against main +`e59fe83be3d31f29b5a149d736bc0c5cb399cbea`, which already includes UDP GSO/GRO. + +AMD Ryzen 9 9950X, Linux `7.1.5-arch1-2`, Rust `1.91.0`, release optimization, +`target-cpu=native`. IPv4 loopback; sender on CPU 30 and receiver on CPU 31 +(different physical cores). Eight broadcast receivers share the receiver +thread. Socket buffers request 16 MiB; this host's send/receive sysctl maxima +are 4 MiB (Linux reports doubled effective socket accounting limits). + +Five measured rounds followed one discarded run of each binary. Main and +branch alternate order; UDP and io_uring also alternate order within the +branch binary. No builds or tests ran concurrently with the measured rounds. +The isolated main build changes only the benchmark harness, using the saved +[benchmark patch](benches/results/udp-uring/baseline-benchmark.patch); its +networking implementation is unchanged. + +These are medians of each run's reported statistic, not pooled percentiles. +`FLUX_BENCH_SCALE=16` gives 65,536 / 16,384 / 512 deliveries for 2 KiB / 64 KiB / +2 MiB burst and broadcast cases. Paced cases have 2,000 messages and a 100 µs +minimum send interval; the large case cannot sustain that interval. The same +outstanding-message bounds apply to both backends. There is no per-scenario +warmup: initial message costs remain in the measurements. Socket setup and +handshake are outside the timed interval. + +Latency is from the transport's wire timestamp to the receive callback; it +excludes serialization before that timestamp. Throughput includes the whole +send/receive interval. CPU ns/B sums sender and receiver thread CPU time, +including their busy polling; it is not an idle-efficiency measurement. + +| Workload | Main MiB/s | io_uring MiB/s | Change | Main p50 / p99 µs | io_uring p50 / p99 µs | Main / io_uring CPU ns/B | +| --- | ---: | ---: | ---: | ---: | ---: | ---: | +| paced/2k | 20 | 20 | +0.0% | 4.7 / 5.7 | 2.4 / 3.4 | 95.436 / 96.388 | +| paced/64k | 625 | 625 | +0.0% | 8.9 / 356.5 | 9.5 / 36.8 | 2.959 / 2.955 | +| paced/2m | 7,446 | 7,397 | -0.7% | 229.2 / 283.5 | 231.8 / 240.6 | 0.193 / 0.201 | +| burst/2k | 442 | 772 | +74.7% | 4.8 / 5.8 | 2.4 / 3.3 | 2.420 / 1.393 | +| burst/64k | 6,891 | 7,108 | +3.1% | 9.0 / 12.2 | 9.5 / 12.6 | 0.201 / 0.191 | +| burst/2m | 7,589 | 7,329 | -3.4% | 219.2 / 259.1 | 230.7 / 242.1 | 0.187 / 0.201 | +| bcast/2k | 562 | 950 | +69.0% | 17.0 / 34.4 | 18.7 / 35.2 | 1.778 / 1.075 | +| bcast/64k | 1,402 | 7,499 | +434.9% | 207.7 / 345.3 | 44.1 / 69.4 | 0.765 / 0.181 | +| bcast/2m | 6,820 | 6,700 | -1.8% | 1,344.5 / 4,861.2 | 3,431.1 / 6,699.6 | 0.211 / 0.246 | + +The strongest gains are 2 KiB bursts (+75% throughput, roughly half median +latency) and 64 KiB broadcasts (5.35x throughput, p99 345 → 69 µs). The latter +also benefits from retaining GSO across mixed-peer batches: these results +measure the complete backend implementation, not an isolated io_uring syscall +substitution. The syscall backend could independently adopt that grouping. + +Large messages are a tradeoff: 2 MiB burst throughput is 3.4% lower, and 2 MiB +broadcast median latency increases from 1.34 ms to 3.43 ms while throughput +falls 1.8%. The extra send copy and different batching/scheduling costs remain. +This is why io_uring stays opt-in. Paced p99 varies substantially between runs; +consult the raw samples rather than treating a median as a guarantee. + +The single-loss relay case completed in a median 6.21 ms on main and 5.26 ms +with io_uring. Every run observed 1,793 data datagrams, including 2 repeated +sequences. Each run sends only one message, so this is a recovery check, not a +useful p99 measurement. Relay CPU is excluded from the CPU column. + +All raw runs are in [benches/results/udp-uring](benches/results/udp-uring). +`main-N.txt` is the isolated main build; `branch-N.txt` includes both backends +on this branch. The branch's syscall control preserves similar throughput +(e.g. 441 vs 442 MiB/s for 2 KiB bursts, 1,406 vs 1,402 for 64 KiB broadcasts). +These are local loopback measurements, not physical-NIC, WAN, or AWS results. +Non-Linux builds were not cross-compiled. + +## Reproduce and validate + +```sh +FLUX_BENCH_TRANSPORT=udp,uring FLUX_BENCH_SCALE=16 \ + cargo bench -p flux-network --bench udp_pipeline +FLUX_BENCH_TRANSPORT=udp,uring FLUX_BENCH_SCALE=16 FLUX_BENCH_REVERSE=1 \ + cargo bench -p flux-network --bench udp_pipeline +``` + +To reconstruct the main baseline, extract the commit above into a separate +directory with `git archive`, apply `baseline-benchmark.patch` there with +`patch -p1`, and run the same benchmark with `FLUX_BENCH_TRANSPORT=udp`. +The patch only adds filtering, sample scaling, and CPU-time instrumentation. +The branch also accepts `FLUX_BENCH_SIZE=2m` to restrict size-dependent cases. + +Validation passed: + +- `just fmt` +- `just clippy` +- `cargo test --workspace --all-features --locked`: 362 passed, 4 existing + ignored documentation tests, no failures. + +Both backends run the same UDP integration suite. Additional coverage checks +mixed-backend peers, DCache delivery, sustained large messages with slow debug +callbacks, owned send buffers, queue saturation, IPv4/IPv6, mixed GSO groups, +fallback, GRO validation, receive-pool exhaustion, 16-bit descriptor-tail +wraparound, and driver movement across threads. Tests require local socket +and io_uring permissions. Local runs used `RUSTC_WRAPPER=` and a temporary +`CARGO_TARGET_DIR` because the shared build cache was sandbox-restricted. diff --git a/crates/flux-network/benches/results/udp-uring/baseline-benchmark.patch b/crates/flux-network/benches/results/udp-uring/baseline-benchmark.patch new file mode 100644 index 0000000..0f5fbe2 --- /dev/null +++ b/crates/flux-network/benches/results/udp-uring/baseline-benchmark.patch @@ -0,0 +1,88 @@ +--- a/crates/flux-network/benches/udp_pipeline.rs ++++ b/crates/flux-network/benches/udp_pipeline.rs +@@ -13,0 +14,5 @@ ++//! `FLUX_BENCH_TRANSPORT=udp,uring` selects backends; `FLUX_BENCH_REVERSE=1` ++//! reverses their order. `FLUX_BENCH_SCALE=16` increases burst/broadcast sample ++//! counts. `FLUX_BENCH_SIZE=2m` filters paced/burst/broadcast sizes. CPU ns/B ++//! sums sender and receiver thread CPU time; it excludes the loss relay. ++//! +@@ -56,2 +61,10 @@ +-fn transports() -> [(&'static str, Transport); 2] { +- [("tcp", Transport::default()), ("udp", Transport::Udp(udp_config()))] ++fn transports() -> Vec<(&'static str, Transport)> { ++ let mut transports = vec![("tcp", Transport::default()), ("udp", Transport::Udp(udp_config()))]; ++ if let Ok(filter) = std::env::var("FLUX_BENCH_TRANSPORT") { ++ transports.retain(|(name, _)| filter.split(',').any(|selected| selected == *name)); ++ assert!(!transports.is_empty(), "unknown FLUX_BENCH_TRANSPORT"); ++ } ++ if std::env::var_os("FLUX_BENCH_REVERSE").is_some() { ++ transports.reverse(); ++ } ++ transports +@@ -68 +81,4 @@ +- (count, window) ++ let scale: usize = std::env::var("FLUX_BENCH_SCALE") ++ .map_or(1, |s| s.parse().expect("invalid FLUX_BENCH_SCALE")); ++ assert!((1..=64).contains(&scale)); ++ (count * scale, window) +@@ -74,0 +91 @@ ++ cpu: Duration, +@@ -84 +101 @@ +- "{name:<28} n={:<6} p50={:>9.1}µs p99={:>9.1}µs max={:>9.1}µs {mibps:>9.0} MiB/s {extra}", ++ "{name:<28} n={:<6} p50={:>9.1}µs p99={:>9.1}µs max={:>9.1}µs {mibps:>9.0} MiB/s cpu={:.3}ns/B {extra}", +@@ -88,0 +106 @@ ++ self.cpu.as_nanos() as f64 / self.bytes as f64, +@@ -136,0 +155,11 @@ ++} ++ ++fn thread_cpu_time() -> Duration { ++ let mut time: libc::timespec = unsafe { std::mem::zeroed() }; ++ assert_eq!( ++ unsafe { ++ libc::clock_gettime(libc::CLOCK_THREAD_CPUTIME_ID, std::ptr::from_mut(&mut time)) ++ }, ++ 0 ++ ); ++ Duration::new(time.tv_sec as u64, time.tv_nsec as u32) +@@ -155,0 +185 @@ ++ let cpu_start = thread_cpu_time(); +@@ -167 +197 @@ +- lat ++ (lat, thread_cpu_time() - cpu_start) +@@ -172,0 +203 @@ ++ let cpu_start = thread_cpu_time(); +@@ -191,3 +222,4 @@ +- let latencies_ns = rx_thread.join().unwrap(); +- assert_eq!(latencies_ns.len(), expected, "receiver timed out"); +- Stats { latencies_ns, elapsed, bytes: expected * size } ++ let sender_cpu = thread_cpu_time() - cpu_start; ++ let (latencies_ns, receiver_cpu) = rx_thread.join().unwrap(); ++ assert_eq!(latencies_ns.len(), expected, "receiver timed out after {sent} sends"); ++ Stats { latencies_ns, elapsed, bytes: expected * size, cpu: sender_cpu + receiver_cpu } +@@ -279 +311,4 @@ +- for (size_name, size) in SIZES { ++ for (size_name, size) in SIZES ++ .into_iter() ++ .filter(|(name, _)| std::env::var("FLUX_BENCH_SIZE").ok().is_none_or(|s| s == *name)) ++ { +@@ -297 +332,4 @@ +- for (size_name, size) in SIZES { ++ for (size_name, size) in SIZES ++ .into_iter() ++ .filter(|(name, _)| std::env::var("FLUX_BENCH_SIZE").ok().is_none_or(|s| s == *name)) ++ { +@@ -315,0 +354,2 @@ ++ for (name, transport) in ++ transports().into_iter().filter(|(_, t)| matches!(t, Transport::Udp(_))) +@@ -321 +361 @@ +- transport: Transport::Udp(udp_config()), ++ transport, +@@ -333 +373 @@ +- s.row("loss1/udp/2m", &format!("datagrams={data} retransmitted={retx}")); ++ s.row(&format!("loss1/{name}/2m"), &format!("datagrams={data} retransmitted={retx}")); +@@ -337 +377,4 @@ +- for (size_name, size) in SIZES { ++ for (size_name, size) in SIZES ++ .into_iter() ++ .filter(|(name, _)| std::env::var("FLUX_BENCH_SIZE").ok().is_none_or(|s| s == *name)) ++ { diff --git a/crates/flux-network/benches/results/udp-uring/branch-1.txt b/crates/flux-network/benches/results/udp-uring/branch-1.txt new file mode 100644 index 0000000..1f17c03 --- /dev/null +++ b/crates/flux-network/benches/results/udp-uring/branch-1.txt @@ -0,0 +1,27 @@ +== paced: one message per 100µs, one receiver == +paced/uring/2k n=2000 p50= 2.4µs p99= 3.4µs max= 67.9µs 20 MiB/s cpu=96.393ns/B +paced/udp/2k n=2000 p50= 4.8µs p99= 328.1µs max= 2324.2µs 20 MiB/s cpu=94.873ns/B +paced/uring/64k n=2000 p50= 9.5µs p99= 13.4µs max= 1762.7µs 625 MiB/s cpu=2.959ns/B +paced/udp/64k n=2000 p50= 9.1µs p99= 706.3µs max= 2653.7µs 625 MiB/s cpu=2.956ns/B +paced/uring/2m n=2000 p50= 233.3µs p99= 249.3µs max= 2046.2µs 7362 MiB/s cpu=0.201ns/B +paced/udp/2m n=2000 p50= 208.0µs p99= 238.8µs max= 2905.7µs 8001 MiB/s cpu=0.178ns/B + +== burst: bounded outstanding, one receiver == +burst/uring/2k n=65536 p50= 2.4µs p99= 3.3µs max= 2563.5µs 767 MiB/s cpu=1.409ns/B window=256 +burst/udp/2k n=65536 p50= 4.8µs p99= 5.9µs max= 3029.5µs 441 MiB/s cpu=2.422ns/B window=256 +burst/uring/64k n=16384 p50= 9.4µs p99= 12.4µs max= 2874.5µs 7291 MiB/s cpu=0.184ns/B window=146 +burst/udp/64k n=16384 p50= 8.9µs p99= 12.0µs max= 2799.0µs 7044 MiB/s cpu=0.195ns/B window=146 +burst/uring/2m n=512 p50= 230.7µs p99= 569.2µs max= 2604.1µs 7329 MiB/s cpu=0.201ns/B window=4 +burst/udp/2m n=512 p50= 225.3µs p99= 284.2µs max= 3285.2µs 7344 MiB/s cpu=0.195ns/B window=4 + +== loss1: one 2 MiB message, exactly one datagram dropped by a relay == +loss1/uring/2m n=1 p50= 5365.6µs p99= 5365.6µs max= 5365.6µs 357 MiB/s cpu=3.212ns/B datagrams=1793 retransmitted=2 +loss1/udp/2m n=1 p50= 6156.9µs p99= 6156.9µs max= 6156.9µs 303 MiB/s cpu=2.845ns/B datagrams=1793 retransmitted=2 + +== bcast: one sender, 8 receivers on one listener == +bcast/uring/2k n=65536 p50= 18.7µs p99= 35.4µs max= 356.2µs 942 MiB/s cpu=1.083ns/B window=256 +bcast/udp/2k n=65536 p50= 16.9µs p99= 31.6µs max= 1995.7µs 562 MiB/s cpu=1.777ns/B window=256 +bcast/uring/64k n=16384 p50= 46.1µs p99= 71.8µs max= 138.3µs 7123 MiB/s cpu=0.190ns/B window=146 +bcast/udp/64k n=16384 p50= 212.4µs p99= 343.5µs max= 2958.4µs 1406 MiB/s cpu=0.758ns/B window=146 +bcast/uring/2m n=512 p50= 3584.3µs p99= 7297.2µs max= 8506.7µs 6522 MiB/s cpu=0.254ns/B window=4 +bcast/udp/2m n=512 p50= 1435.3µs p99= 4666.5µs max= 6445.6µs 6485 MiB/s cpu=0.219ns/B window=4 diff --git a/crates/flux-network/benches/results/udp-uring/branch-2.txt b/crates/flux-network/benches/results/udp-uring/branch-2.txt new file mode 100644 index 0000000..78c7833 --- /dev/null +++ b/crates/flux-network/benches/results/udp-uring/branch-2.txt @@ -0,0 +1,27 @@ +== paced: one message per 100µs, one receiver == +paced/udp/2k n=2000 p50= 4.8µs p99= 5.6µs max= 56.2µs 20 MiB/s cpu=95.407ns/B +paced/uring/2k n=2000 p50= 2.4µs p99= 4.3µs max= 906.7µs 20 MiB/s cpu=96.159ns/B +paced/udp/64k n=2000 p50= 9.0µs p99= 509.5µs max= 2451.0µs 625 MiB/s cpu=2.957ns/B +paced/uring/64k n=2000 p50= 9.8µs p99= 207.8µs max= 2131.1µs 625 MiB/s cpu=2.954ns/B +paced/udp/2m n=2000 p50= 231.7µs p99= 299.8µs max= 4451.2µs 7228 MiB/s cpu=0.200ns/B +paced/uring/2m n=2000 p50= 231.8µs p99= 238.2µs max= 2775.7µs 7397 MiB/s cpu=0.201ns/B + +== burst: bounded outstanding, one receiver == +burst/udp/2k n=65536 p50= 4.8µs p99= 5.8µs max= 2822.9µs 441 MiB/s cpu=2.420ns/B window=256 +burst/uring/2k n=65536 p50= 2.4µs p99= 3.3µs max= 2956.5µs 773 MiB/s cpu=1.392ns/B window=256 +burst/udp/64k n=16384 p50= 9.3µs p99= 12.9µs max= 3028.8µs 6681 MiB/s cpu=0.210ns/B window=146 +burst/uring/64k n=16384 p50= 9.5µs p99= 18.0µs max= 3148.4µs 7091 MiB/s cpu=0.191ns/B window=146 +burst/udp/2m n=512 p50= 220.4µs p99= 240.9µs max= 3213.8µs 7498 MiB/s cpu=0.190ns/B window=4 +burst/uring/2m n=512 p50= 218.5µs p99= 241.8µs max= 2487.1µs 7672 MiB/s cpu=0.194ns/B window=4 + +== loss1: one 2 MiB message, exactly one datagram dropped by a relay == +loss1/udp/2m n=1 p50= 6246.1µs p99= 6246.1µs max= 6246.1µs 296 MiB/s cpu=3.014ns/B datagrams=1793 retransmitted=2 +loss1/uring/2m n=1 p50= 4717.3µs p99= 4717.3µs max= 4717.3µs 394 MiB/s cpu=2.741ns/B datagrams=1793 retransmitted=2 + +== bcast: one sender, 8 receivers on one listener == +bcast/udp/2k n=65536 p50= 17.2µs p99= 31.5µs max= 1643.0µs 563 MiB/s cpu=1.784ns/B window=256 +bcast/uring/2k n=65536 p50= 18.7µs p99= 35.0µs max= 85.9µs 954 MiB/s cpu=1.074ns/B window=256 +bcast/udp/64k n=16384 p50= 211.5µs p99= 344.0µs max= 1742.5µs 1406 MiB/s cpu=0.757ns/B window=146 +bcast/uring/64k n=16384 p50= 44.4µs p99= 70.3µs max= 124.5µs 7463 MiB/s cpu=0.182ns/B window=146 +bcast/udp/2m n=512 p50= 1655.0µs p99= 6342.0µs max= 8185.5µs 6653 MiB/s cpu=0.223ns/B window=4 +bcast/uring/2m n=512 p50= 3212.1µs p99= 6699.6µs max= 7581.0µs 6776 MiB/s cpu=0.243ns/B window=4 diff --git a/crates/flux-network/benches/results/udp-uring/branch-3.txt b/crates/flux-network/benches/results/udp-uring/branch-3.txt new file mode 100644 index 0000000..d2ff48c --- /dev/null +++ b/crates/flux-network/benches/results/udp-uring/branch-3.txt @@ -0,0 +1,27 @@ +== paced: one message per 100µs, one receiver == +paced/uring/2k n=2000 p50= 2.4µs p99= 3.4µs max= 150.9µs 20 MiB/s cpu=96.390ns/B +paced/udp/2k n=2000 p50= 4.8µs p99= 168.2µs max= 2164.6µs 20 MiB/s cpu=94.907ns/B +paced/uring/64k n=2000 p50= 9.4µs p99= 36.8µs max= 1597.4µs 625 MiB/s cpu=2.955ns/B +paced/udp/64k n=2000 p50= 9.1µs p99= 645.4µs max= 2593.0µs 625 MiB/s cpu=2.956ns/B +paced/uring/2m n=2000 p50= 219.1µs p99= 226.3µs max= 1995.9µs 7782 MiB/s cpu=0.192ns/B +paced/udp/2m n=2000 p50= 215.1µs p99= 228.2µs max= 3220.1µs 7771 MiB/s cpu=0.183ns/B + +== burst: bounded outstanding, one receiver == +burst/uring/2k n=65536 p50= 2.4µs p99= 4.4µs max= 2456.5µs 772 MiB/s cpu=1.393ns/B window=256 +burst/udp/2k n=65536 p50= 4.8µs p99= 5.8µs max= 3212.3µs 443 MiB/s cpu=2.410ns/B window=256 +burst/uring/64k n=16384 p50= 9.4µs p99= 12.4µs max= 2521.1µs 7190 MiB/s cpu=0.188ns/B window=146 +burst/udp/64k n=16384 p50= 9.1µs p99= 12.1µs max= 3126.3µs 6837 MiB/s cpu=0.203ns/B window=146 +burst/uring/2m n=512 p50= 231.8µs p99= 242.1µs max= 1894.4µs 7295 MiB/s cpu=0.203ns/B window=4 +burst/udp/2m n=512 p50= 225.0µs p99= 245.1µs max= 3138.8µs 7347 MiB/s cpu=0.194ns/B window=4 + +== loss1: one 2 MiB message, exactly one datagram dropped by a relay == +loss1/uring/2m n=1 p50= 5261.6µs p99= 5261.6µs max= 5261.6µs 362 MiB/s cpu=3.183ns/B datagrams=1793 retransmitted=2 +loss1/udp/2m n=1 p50= 6341.0µs p99= 6341.0µs max= 6341.0µs 293 MiB/s cpu=2.999ns/B datagrams=1793 retransmitted=2 + +== bcast: one sender, 8 receivers on one listener == +bcast/uring/2k n=65536 p50= 18.9µs p99= 35.2µs max= 84.3µs 949 MiB/s cpu=1.076ns/B window=256 +bcast/udp/2k n=65536 p50= 17.2µs p99= 30.5µs max= 1583.6µs 566 MiB/s cpu=1.765ns/B window=256 +bcast/uring/64k n=16384 p50= 43.8µs p99= 68.7µs max= 116.5µs 7499 MiB/s cpu=0.181ns/B window=146 +bcast/udp/64k n=16384 p50= 211.0µs p99= 347.3µs max= 2499.7µs 1400 MiB/s cpu=0.761ns/B window=146 +bcast/uring/2m n=512 p50= 3688.4µs p99= 7156.3µs max= 7792.3µs 6443 MiB/s cpu=0.257ns/B window=4 +bcast/udp/2m n=512 p50= 1321.7µs p99= 2843.1µs max= 3790.1µs 7092 MiB/s cpu=0.198ns/B window=4 diff --git a/crates/flux-network/benches/results/udp-uring/branch-4.txt b/crates/flux-network/benches/results/udp-uring/branch-4.txt new file mode 100644 index 0000000..3fb89b1 --- /dev/null +++ b/crates/flux-network/benches/results/udp-uring/branch-4.txt @@ -0,0 +1,27 @@ +== paced: one message per 100µs, one receiver == +paced/udp/2k n=2000 p50= 4.8µs p99= 5.5µs max= 54.5µs 20 MiB/s cpu=95.421ns/B +paced/uring/2k n=2000 p50= 2.4µs p99= 3.9µs max= 949.7µs 20 MiB/s cpu=96.165ns/B +paced/udp/64k n=2000 p50= 9.2µs p99= 481.3µs max= 2432.1µs 625 MiB/s cpu=2.957ns/B +paced/uring/64k n=2000 p50= 9.5µs p99= 135.3µs max= 2039.7µs 625 MiB/s cpu=2.954ns/B +paced/udp/2m n=2000 p50= 227.7µs p99= 250.5µs max= 3518.5µs 7560 MiB/s cpu=0.189ns/B +paced/uring/2m n=2000 p50= 232.2µs p99= 240.6µs max= 2488.2µs 7364 MiB/s cpu=0.201ns/B + +== burst: bounded outstanding, one receiver == +burst/udp/2k n=65536 p50= 4.8µs p99= 6.5µs max= 2834.1µs 439 MiB/s cpu=2.437ns/B window=256 +burst/uring/2k n=65536 p50= 2.4µs p99= 3.2µs max= 2790.4µs 776 MiB/s cpu=1.385ns/B window=256 +burst/udp/64k n=16384 p50= 9.2µs p99= 11.8µs max= 3593.8µs 6738 MiB/s cpu=0.207ns/B window=146 +burst/uring/64k n=16384 p50= 9.5µs p99= 15.2µs max= 2969.4µs 7086 MiB/s cpu=0.191ns/B window=146 +burst/udp/2m n=512 p50= 213.8µs p99= 237.6µs max= 3026.2µs 7771 MiB/s cpu=0.182ns/B window=4 +burst/uring/2m n=512 p50= 222.1µs p99= 237.5µs max= 2690.7µs 7503 MiB/s cpu=0.197ns/B window=4 + +== loss1: one 2 MiB message, exactly one datagram dropped by a relay == +loss1/udp/2m n=1 p50= 6181.2µs p99= 6181.2µs max= 6181.2µs 300 MiB/s cpu=2.937ns/B datagrams=1793 retransmitted=2 +loss1/uring/2m n=1 p50= 4563.1µs p99= 4563.1µs max= 4563.1µs 407 MiB/s cpu=2.600ns/B datagrams=1793 retransmitted=2 + +== bcast: one sender, 8 receivers on one listener == +bcast/udp/2k n=65536 p50= 17.5µs p99= 31.0µs max= 1655.4µs 565 MiB/s cpu=1.758ns/B window=256 +bcast/uring/2k n=65536 p50= 18.7µs p99= 35.0µs max= 74.4µs 954 MiB/s cpu=1.070ns/B window=256 +bcast/udp/64k n=16384 p50= 209.9µs p99= 342.7µs max= 1455.8µs 1411 MiB/s cpu=0.754ns/B window=146 +bcast/uring/64k n=16384 p50= 44.0µs p99= 67.9µs max= 124.7µs 7538 MiB/s cpu=0.179ns/B window=146 +bcast/udp/2m n=512 p50= 1378.0µs p99= 5145.2µs max= 6673.5µs 6877 MiB/s cpu=0.209ns/B window=4 +bcast/uring/2m n=512 p50= 1373.0µs p99= 2804.0µs max= 4350.4µs 6889 MiB/s cpu=0.239ns/B window=4 diff --git a/crates/flux-network/benches/results/udp-uring/branch-5.txt b/crates/flux-network/benches/results/udp-uring/branch-5.txt new file mode 100644 index 0000000..8772997 --- /dev/null +++ b/crates/flux-network/benches/results/udp-uring/branch-5.txt @@ -0,0 +1,27 @@ +== paced: one message per 100µs, one receiver == +paced/uring/2k n=2000 p50= 2.4µs p99= 3.4µs max= 80.6µs 20 MiB/s cpu=96.388ns/B +paced/udp/2k n=2000 p50= 4.8µs p99= 357.1µs max= 2352.8µs 20 MiB/s cpu=94.886ns/B +paced/uring/64k n=2000 p50= 9.7µs p99= 19.3µs max= 1957.4µs 625 MiB/s cpu=2.956ns/B +paced/udp/64k n=2000 p50= 9.0µs p99= 627.2µs max= 2569.8µs 625 MiB/s cpu=2.956ns/B +paced/uring/2m n=2000 p50= 229.2µs p99= 251.2µs max= 2853.4µs 7518 MiB/s cpu=0.197ns/B +paced/udp/2m n=2000 p50= 210.1µs p99= 234.8µs max= 3168.3µs 7913 MiB/s cpu=0.180ns/B + +== burst: bounded outstanding, one receiver == +burst/uring/2k n=65536 p50= 2.4µs p99= 3.3µs max= 2511.4µs 770 MiB/s cpu=1.398ns/B window=256 +burst/udp/2k n=65536 p50= 4.8µs p99= 5.8µs max= 3883.5µs 444 MiB/s cpu=2.388ns/B window=256 +burst/uring/64k n=16384 p50= 9.5µs p99= 12.6µs max= 3123.2µs 7108 MiB/s cpu=0.191ns/B window=146 +burst/udp/64k n=16384 p50= 9.0µs p99= 13.2µs max= 3224.0µs 6884 MiB/s cpu=0.201ns/B window=146 +burst/uring/2m n=512 p50= 231.6µs p99= 242.8µs max= 2772.9µs 7272 MiB/s cpu=0.202ns/B window=4 +burst/udp/2m n=512 p50= 226.9µs p99= 233.6µs max= 3008.8µs 7349 MiB/s cpu=0.194ns/B window=4 + +== loss1: one 2 MiB message, exactly one datagram dropped by a relay == +loss1/uring/2m n=1 p50= 5259.1µs p99= 5259.1µs max= 5259.1µs 358 MiB/s cpu=3.181ns/B datagrams=1793 retransmitted=2 +loss1/udp/2m n=1 p50= 8808.8µs p99= 8808.8µs max= 8808.8µs 225 MiB/s cpu=5.182ns/B datagrams=1793 retransmitted=2 + +== bcast: one sender, 8 receivers on one listener == +bcast/uring/2k n=65536 p50= 18.8µs p99= 35.2µs max= 72.3µs 950 MiB/s cpu=1.075ns/B window=256 +bcast/udp/2k n=65536 p50= 17.2µs p99= 30.5µs max= 1532.2µs 562 MiB/s cpu=1.774ns/B window=256 +bcast/uring/64k n=16384 p50= 44.1µs p99= 69.4µs max= 133.0µs 7541 MiB/s cpu=0.179ns/B window=146 +bcast/udp/64k n=16384 p50= 209.6µs p99= 341.2µs max= 1612.9µs 1416 MiB/s cpu=0.749ns/B window=146 +bcast/uring/2m n=512 p50= 3431.1µs p99= 6646.8µs max= 7164.7µs 6700 MiB/s cpu=0.246ns/B window=4 +bcast/udp/2m n=512 p50= 1359.1µs p99= 2644.3µs max= 3605.9µs 6944 MiB/s cpu=0.199ns/B window=4 diff --git a/crates/flux-network/benches/results/udp-uring/main-1.txt b/crates/flux-network/benches/results/udp-uring/main-1.txt new file mode 100644 index 0000000..99e3710 --- /dev/null +++ b/crates/flux-network/benches/results/udp-uring/main-1.txt @@ -0,0 +1,17 @@ +== paced: one message per 100µs, one receiver == +paced/udp/2k n=2000 p50= 4.8µs p99= 6.2µs max= 37.6µs 20 MiB/s cpu=95.455ns/B +paced/udp/64k n=2000 p50= 9.1µs p99= 458.5µs max= 2403.1µs 625 MiB/s cpu=2.957ns/B +paced/udp/2m n=2000 p50= 236.2µs p99= 282.1µs max= 2643.4µs 7308 MiB/s cpu=0.198ns/B + +== burst: bounded outstanding, one receiver == +burst/udp/2k n=65536 p50= 4.8µs p99= 5.9µs max= 2430.4µs 442 MiB/s cpu=2.420ns/B window=256 +burst/udp/64k n=16384 p50= 8.9µs p99= 11.9µs max= 2623.9µs 7094 MiB/s cpu=0.194ns/B window=146 +burst/udp/2m n=512 p50= 219.2µs p99= 270.5µs max= 3493.1µs 7589 MiB/s cpu=0.187ns/B window=4 + +== loss1: one 2 MiB message, exactly one datagram dropped by a relay == +loss1/udp/2m n=1 p50= 6211.3µs p99= 6211.3µs max= 6211.3µs 287 MiB/s cpu=3.150ns/B datagrams=1793 retransmitted=2 + +== bcast: one sender, 8 receivers on one listener == +bcast/udp/2k n=65536 p50= 17.3µs p99= 31.9µs max= 2139.8µs 564 MiB/s cpu=1.778ns/B window=256 +bcast/udp/64k n=16384 p50= 211.5µs p99= 345.3µs max= 419.5µs 1402 MiB/s cpu=0.765ns/B window=146 +bcast/udp/2m n=512 p50= 1342.2µs p99= 5379.5µs max= 6375.9µs 6846 MiB/s cpu=0.211ns/B window=4 diff --git a/crates/flux-network/benches/results/udp-uring/main-2.txt b/crates/flux-network/benches/results/udp-uring/main-2.txt new file mode 100644 index 0000000..b2d4017 --- /dev/null +++ b/crates/flux-network/benches/results/udp-uring/main-2.txt @@ -0,0 +1,17 @@ +== paced: one message per 100µs, one receiver == +paced/udp/2k n=2000 p50= 4.7µs p99= 5.5µs max= 31.4µs 20 MiB/s cpu=95.420ns/B +paced/udp/64k n=2000 p50= 8.9µs p99= 337.3µs max= 2285.4µs 625 MiB/s cpu=2.959ns/B +paced/udp/2m n=2000 p50= 228.9µs p99= 297.2µs max= 2699.0µs 7446 MiB/s cpu=0.193ns/B + +== burst: bounded outstanding, one receiver == +burst/udp/2k n=65536 p50= 4.8µs p99= 5.8µs max= 2658.0µs 439 MiB/s cpu=2.435ns/B window=256 +burst/udp/64k n=16384 p50= 9.0µs p99= 15.0µs max= 3153.0µs 6972 MiB/s cpu=0.198ns/B window=146 +burst/udp/2m n=512 p50= 211.4µs p99= 239.8µs max= 2388.1µs 7840 MiB/s cpu=0.181ns/B window=4 + +== loss1: one 2 MiB message, exactly one datagram dropped by a relay == +loss1/udp/2m n=1 p50= 6310.8µs p99= 6310.8µs max= 6310.8µs 282 MiB/s cpu=3.205ns/B datagrams=1793 retransmitted=2 + +== bcast: one sender, 8 receivers on one listener == +bcast/udp/2k n=65536 p50= 17.2µs p99= 316.4µs max= 2276.3µs 560 MiB/s cpu=1.777ns/B window=256 +bcast/udp/64k n=16384 p50= 207.7µs p99= 344.0µs max= 356.5µs 1414 MiB/s cpu=0.753ns/B window=146 +bcast/udp/2m n=512 p50= 1315.6µs p99= 2859.3µs max= 4793.0µs 6817 MiB/s cpu=0.212ns/B window=4 diff --git a/crates/flux-network/benches/results/udp-uring/main-3.txt b/crates/flux-network/benches/results/udp-uring/main-3.txt new file mode 100644 index 0000000..b3b0b96 --- /dev/null +++ b/crates/flux-network/benches/results/udp-uring/main-3.txt @@ -0,0 +1,17 @@ +== paced: one message per 100µs, one receiver == +paced/udp/2k n=2000 p50= 4.7µs p99= 5.4µs max= 48.7µs 20 MiB/s cpu=95.436ns/B +paced/udp/64k n=2000 p50= 8.9µs p99= 435.7µs max= 2374.7µs 625 MiB/s cpu=2.957ns/B +paced/udp/2m n=2000 p50= 216.5µs p99= 283.5µs max= 2549.4µs 7608 MiB/s cpu=0.189ns/B + +== burst: bounded outstanding, one receiver == +burst/udp/2k n=65536 p50= 4.7µs p99= 6.4µs max= 2735.1µs 442 MiB/s cpu=2.421ns/B window=256 +burst/udp/64k n=16384 p50= 9.0µs p99= 142.2µs max= 2771.2µs 6891 MiB/s cpu=0.201ns/B window=146 +burst/udp/2m n=512 p50= 211.9µs p99= 224.4µs max= 2815.5µs 7809 MiB/s cpu=0.182ns/B window=4 + +== loss1: one 2 MiB message, exactly one datagram dropped by a relay == +loss1/udp/2m n=1 p50= 6172.2µs p99= 6172.2µs max= 6172.2µs 288 MiB/s cpu=3.067ns/B datagrams=1793 retransmitted=2 + +== bcast: one sender, 8 receivers on one listener == +bcast/udp/2k n=65536 p50= 17.0µs p99= 37.7µs max= 2368.8µs 563 MiB/s cpu=1.773ns/B window=256 +bcast/udp/64k n=16384 p50= 211.4µs p99= 342.6µs max= 990.3µs 1410 MiB/s cpu=0.758ns/B window=146 +bcast/udp/2m n=512 p50= 1372.5µs p99= 5468.6µs max= 7813.7µs 6820 MiB/s cpu=0.210ns/B window=4 diff --git a/crates/flux-network/benches/results/udp-uring/main-4.txt b/crates/flux-network/benches/results/udp-uring/main-4.txt new file mode 100644 index 0000000..6311833 --- /dev/null +++ b/crates/flux-network/benches/results/udp-uring/main-4.txt @@ -0,0 +1,17 @@ +== paced: one message per 100µs, one receiver == +paced/udp/2k n=2000 p50= 4.7µs p99= 6.2µs max= 56.6µs 20 MiB/s cpu=95.439ns/B +paced/udp/64k n=2000 p50= 8.9µs p99= 340.1µs max= 2285.9µs 625 MiB/s cpu=2.960ns/B +paced/udp/2m n=2000 p50= 233.8µs p99= 288.9µs max= 3134.6µs 7361 MiB/s cpu=0.196ns/B + +== burst: bounded outstanding, one receiver == +burst/udp/2k n=65536 p50= 4.8µs p99= 5.8µs max= 2663.3µs 445 MiB/s cpu=2.403ns/B window=256 +burst/udp/64k n=16384 p50= 9.1µs p99= 12.2µs max= 3505.6µs 6857 MiB/s cpu=0.203ns/B window=146 +burst/udp/2m n=512 p50= 231.1µs p99= 259.1µs max= 3046.1µs 7335 MiB/s cpu=0.195ns/B window=4 + +== loss1: one 2 MiB message, exactly one datagram dropped by a relay == +loss1/udp/2m n=1 p50= 6141.2µs p99= 6141.2µs max= 6141.2µs 293 MiB/s cpu=3.060ns/B datagrams=1793 retransmitted=2 + +== bcast: one sender, 8 receivers on one listener == +bcast/udp/2k n=65536 p50= 16.6µs p99= 34.4µs max= 2306.9µs 561 MiB/s cpu=1.784ns/B window=256 +bcast/udp/64k n=16384 p50= 182.9µs p99= 345.6µs max= 361.7µs 1398 MiB/s cpu=0.768ns/B window=146 +bcast/udp/2m n=512 p50= 1344.5µs p99= 4861.2µs max= 7029.2µs 6895 MiB/s cpu=0.206ns/B window=4 diff --git a/crates/flux-network/benches/results/udp-uring/main-5.txt b/crates/flux-network/benches/results/udp-uring/main-5.txt new file mode 100644 index 0000000..afcd501 --- /dev/null +++ b/crates/flux-network/benches/results/udp-uring/main-5.txt @@ -0,0 +1,17 @@ +== paced: one message per 100µs, one receiver == +paced/udp/2k n=2000 p50= 4.7µs p99= 5.7µs max= 75.5µs 20 MiB/s cpu=95.411ns/B +paced/udp/64k n=2000 p50= 9.1µs p99= 356.5µs max= 2306.2µs 625 MiB/s cpu=2.959ns/B +paced/udp/2m n=2000 p50= 229.2µs p99= 260.9µs max= 2679.1µs 7489 MiB/s cpu=0.192ns/B + +== burst: bounded outstanding, one receiver == +burst/udp/2k n=65536 p50= 4.8µs p99= 5.8µs max= 2491.4µs 442 MiB/s cpu=2.413ns/B window=256 +burst/udp/64k n=16384 p50= 9.2µs p99= 11.7µs max= 3150.8µs 6642 MiB/s cpu=0.212ns/B window=146 +burst/udp/2m n=512 p50= 239.6µs p99= 301.6µs max= 2976.3µs 7107 MiB/s cpu=0.203ns/B window=4 + +== loss1: one 2 MiB message, exactly one datagram dropped by a relay == +loss1/udp/2m n=1 p50= 6353.6µs p99= 6353.6µs max= 6353.6µs 278 MiB/s cpu=3.233ns/B datagrams=1793 retransmitted=2 + +== bcast: one sender, 8 receivers on one listener == +bcast/udp/2k n=65536 p50= 17.0µs p99= 30.3µs max= 65.5µs 562 MiB/s cpu=1.784ns/B window=256 +bcast/udp/64k n=16384 p50= 197.1µs p99= 345.9µs max= 382.8µs 1398 MiB/s cpu=0.769ns/B window=146 +bcast/udp/2m n=512 p50= 1379.3µs p99= 3527.3µs max= 5245.0µs 6794 MiB/s cpu=0.211ns/B window=4 diff --git a/crates/flux-network/benches/udp_pipeline.rs b/crates/flux-network/benches/udp_pipeline.rs index 0638c9b..469b2e4 100644 --- a/crates/flux-network/benches/udp_pipeline.rs +++ b/crates/flux-network/benches/udp_pipeline.rs @@ -11,6 +11,11 @@ //! MiB message. Recovery should resend one datagram, not the message. //! - `bcast`: one sender broadcasting to 8 receivers on one listener socket. //! +//! `FLUX_BENCH_TRANSPORT=udp,uring` selects backends; `FLUX_BENCH_REVERSE=1` +//! reverses their order. `FLUX_BENCH_SCALE=16` increases burst/broadcast sample +//! counts. `FLUX_BENCH_SIZE=2m` filters paced/burst/broadcast sizes. CPU ns/B +//! sums sender and receiver thread CPU time; it excludes the loss relay. +//! //! Run with `cargo bench -p flux-network --bench udp_pipeline`. use std::{ @@ -53,8 +58,24 @@ fn udp_config() -> UdpConfig { UdpConfig { max_message_size: 4 * 1024 * 1024, ..UdpConfig::lan() } } -fn transports() -> [(&'static str, Transport); 2] { - [("tcp", Transport::default()), ("udp", Transport::Udp(udp_config()))] +fn transports() -> Vec<(&'static str, Transport)> { + let mut transports = vec![("tcp", Transport::default()), ("udp", Transport::Udp(udp_config()))]; + #[cfg(target_os = "linux")] + transports.push(( + "uring", + Transport::Udp(UdpConfig { + io: flux_network::udp::UdpIo::Uring(flux_network::udp::UringConfig::default()), + ..udp_config() + }), + )); + if let Ok(filter) = std::env::var("FLUX_BENCH_TRANSPORT") { + transports.retain(|(name, _)| filter.split(',').any(|selected| selected == *name)); + assert!(!transports.is_empty(), "unknown FLUX_BENCH_TRANSPORT"); + } + if std::env::var_os("FLUX_BENCH_REVERSE").is_some() { + transports.reverse(); + } + transports } fn connector(transport: Transport) -> NetworkDriver { @@ -65,13 +86,17 @@ fn connector(transport: Transport) -> NetworkDriver { fn burst_plan(size: usize) -> (usize, usize) { let count = (64 * 1024 * 1024 / size).clamp(16, 4096); let window = (UdpConfig::default().send_window / 2 / size.div_ceil(1171)).clamp(1, 256); - (count, window) + let scale: usize = std::env::var("FLUX_BENCH_SCALE") + .map_or(1, |s| s.parse().expect("invalid FLUX_BENCH_SCALE")); + assert!((1..=64).contains(&scale)); + (count * scale, window) } struct Stats { latencies_ns: Vec, elapsed: Duration, bytes: usize, + cpu: Duration, } impl Stats { @@ -81,11 +106,12 @@ impl Stats { let pct = |p: f64| l[((l.len() - 1) as f64 * p) as usize] as f64 / 1000.0; let mibps = self.bytes as f64 / self.elapsed.as_secs_f64() / (1024.0 * 1024.0); println!( - "{name:<28} n={:<6} p50={:>9.1}µs p99={:>9.1}µs max={:>9.1}µs {mibps:>9.0} MiB/s {extra}", + "{name:<28} n={:<6} p50={:>9.1}µs p99={:>9.1}µs max={:>9.1}µs {mibps:>9.0} MiB/s cpu={:.3}ns/B {extra}", l.len(), pct(0.5), pct(0.99), pct(1.0), + self.cpu.as_nanos() as f64 / self.bytes as f64, ); } } @@ -136,6 +162,17 @@ struct Scenario { window: usize, } +fn thread_cpu_time() -> Duration { + let mut time: libc::timespec = unsafe { std::mem::zeroed() }; + assert_eq!( + unsafe { + libc::clock_gettime(libc::CLOCK_THREAD_CPUTIME_ID, std::ptr::from_mut(&mut time)) + }, + 0 + ); + Duration::new(time.tv_sec as u64, time.tv_nsec as u32) +} + /// Sends `count` messages from the server while the receiver thread counts /// them and records one-way latency. fn run(sc: Scenario) -> Stats { @@ -153,6 +190,7 @@ fn run(sc: Scenario) -> Stats { thread::spawn(move || { pin(0); let mut lat = Vec::with_capacity(expected); + let cpu_start = thread_cpu_time(); while lat.len() < expected && !stop.load(Ordering::Relaxed) { for r in &mut receivers { r.poll_with(|e| { @@ -164,12 +202,13 @@ fn run(sc: Scenario) -> Stats { got.store(lat.len(), Ordering::Relaxed); } let _ = done_tx.send(()); - lat + (lat, thread_cpu_time() - cpu_start) }) }; pin(1); let start = Instant::now(); + let cpu_start = thread_cpu_time(); let mut sent = 0; let mut next_send = start; while done_rx.try_recv().is_err() { @@ -188,9 +227,10 @@ fn run(sc: Scenario) -> Stats { } } let elapsed = start.elapsed(); - let latencies_ns = rx_thread.join().unwrap(); - assert_eq!(latencies_ns.len(), expected, "receiver timed out"); - Stats { latencies_ns, elapsed, bytes: expected * size } + let sender_cpu = thread_cpu_time() - cpu_start; + let (latencies_ns, receiver_cpu) = rx_thread.join().unwrap(); + assert_eq!(latencies_ns.len(), expected, "receiver timed out after {sent} sends"); + Stats { latencies_ns, elapsed, bytes: expected * size, cpu: sender_cpu + receiver_cpu } } /// Raises the relay's socket buffers so it never drops a burst itself. @@ -276,7 +316,10 @@ impl Drop for Relay { fn main() { pin(usize::MAX); println!("== paced: one message per 100µs, one receiver =="); - for (size_name, size) in SIZES { + for (size_name, size) in SIZES + .into_iter() + .filter(|(name, _)| std::env::var("FLUX_BENCH_SIZE").ok().is_none_or(|s| s == *name)) + { for (name, transport) in transports() { let addr = free_addr(); let s = run(Scenario { @@ -294,7 +337,10 @@ fn main() { } println!("\n== burst: bounded outstanding, one receiver =="); - for (size_name, size) in SIZES { + for (size_name, size) in SIZES + .into_iter() + .filter(|(name, _)| std::env::var("FLUX_BENCH_SIZE").ok().is_none_or(|s| s == *name)) + { let (count, window) = burst_plan(size); for (name, transport) in transports() { let addr = free_addr(); @@ -313,12 +359,14 @@ fn main() { } println!("\n== loss1: one 2 MiB message, exactly one datagram dropped by a relay =="); + for (name, transport) in + transports().into_iter().filter(|(_, t)| matches!(t, Transport::Udp(_))) { let server_addr = free_addr(); // Drop the 900th of 1789 data datagrams. let relay = Relay::start(server_addr, 900); let s = run(Scenario { - transport: Transport::Udp(udp_config()), + transport, listen: server_addr, dial: relay.addr, clients: 1, @@ -330,11 +378,14 @@ fn main() { let data = relay.data.load(Ordering::Relaxed); let retx = relay.retransmits.load(Ordering::Relaxed); drop(relay); - s.row("loss1/udp/2m", &format!("datagrams={data} retransmitted={retx}")); + s.row(&format!("loss1/{name}/2m"), &format!("datagrams={data} retransmitted={retx}")); } println!("\n== bcast: one sender, {BCAST_PEERS} receivers on one listener =="); - for (size_name, size) in SIZES { + for (size_name, size) in SIZES + .into_iter() + .filter(|(name, _)| std::env::var("FLUX_BENCH_SIZE").ok().is_none_or(|s| s == *name)) + { let (count, window) = burst_plan(size); let count = count / BCAST_PEERS; for (name, transport) in transports() { diff --git a/crates/flux-network/src/network_driver.rs b/crates/flux-network/src/network_driver.rs index 153d5bf..49f7a59 100644 --- a/crates/flux-network/src/network_driver.rs +++ b/crates/flux-network/src/network_driver.rs @@ -96,9 +96,12 @@ impl Inner { } } -/// Poll-driven message transport built on `mio`, over TCP or reliable UDP +/// Poll-driven message transport over TCP or reliable UDP /// (see [`Transport`]). The API and events are identical for both. /// +/// TCP uses `mio`; UDP selects syscall or Linux `io_uring` I/O through +/// [`UdpConfig`]. +/// /// Manages: /// - **Outbound (client) connections** created via [`connect`]. These are /// **auto-retried** on failure/disconnect: TCP on its configured reconnect @@ -245,7 +248,7 @@ impl NetworkDriver { /// /// This call: /// 1) attempts outbound reconnects if due - /// 2) polls `mio` with a zero timeout + /// 2) polls readiness or drains `io_uring` completions without waiting /// 3) for each event calls `handler` with the appropriate [`PollEvent`] /// 4) returns whether any IO events were processed /// diff --git a/crates/flux-network/src/udp/connector.rs b/crates/flux-network/src/udp/connector.rs index 9054f0f..2f595dd 100644 --- a/crates/flux-network/src/udp/connector.rs +++ b/crates/flux-network/src/udp/connector.rs @@ -17,13 +17,13 @@ use flux::spine::{SpineProducerWithDCache, SpineProducers}; use flux_communication::Timer; use flux_timing::{Duration, Instant, Nanos}; use flux_utils::{DCache, DCachePtr, safe_panic}; -use mio::{Events, Interest, Poll, Registry, Token, event::Event, net::UdpSocket}; +use mio::{Events, Interest, Poll, Registry, Token}; use tracing::{debug, info, warn}; use super::{ UdpConfig, peer::{MsgStore, PushOutcome, RxPayload, SendOutcome, Staged, UdpPeer, send_batch}, - sys::{BATCH, RecvBatch, SendBatch}, + sys::{BATCH, RecvBatch, SendBatch, UdpSocket}, wire::{HEADER_SIZE, Header, Kind}, }; use crate::{ @@ -134,7 +134,11 @@ impl UdpManager { store: MsgStore::new(), batch: SendBatch::new(), staged: [Staged { peer: 0, seq: 0 }; BATCH], - recv: Some(RecvBatch::new(udp.max_datagram_size)), + recv: match udp.io { + super::UdpIo::Syscall => Some(RecvBatch::new(udp.max_datagram_size)), + #[cfg(target_os = "linux")] + super::UdpIo::Uring(_) => None, + }, pending_disconnects: Vec::new(), next_token: 0, tick_interval: udp.min_rto / 2_u32, @@ -165,6 +169,13 @@ impl UdpManager { if let Some(size) = self.config.socket_buf_size { set_socket_buf_size(&socket, size); } + #[cfg(target_os = "linux")] + if let super::UdpIo::Uring(config) = self.udp.io { + socket.ring = Some(std::cell::RefCell::new(super::sys::uring::Ring::new( + socket.as_raw_fd(), + config, + )?)); + } let token = self.next_token(); self.registry.register(&mut socket, token, Interest::READABLE)?; self.sockets.push(Endpoint { token, socket, listener, writable_armed: false }); @@ -333,7 +344,7 @@ impl UdpManager { /// peers. Arms WRITABLE if the kernel stopped accepting. fn flush_socket(&mut self, k: usize, now: Instant) { let entry = &mut self.sockets[k]; - let fd = entry.socket.as_raw_fd(); + let socket = &entry.socket; let mut n = 0; for i in 0..self.peers.len() { if self.peers[i].socket_token != entry.token { @@ -344,7 +355,14 @@ impl UdpManager { self.staged[n] = Staged { peer: i, seq }; n += 1; if n == BATCH { - if !Self::dispatch(&mut self.batch, &self.staged, &mut self.peers, fd, n, now) { + if !Self::dispatch( + &mut self.batch, + &self.staged, + &mut self.peers, + socket, + n, + now, + ) { arm_writable(&self.registry, entry); return; } @@ -352,7 +370,8 @@ impl UdpManager { } } } - if n != 0 && !Self::dispatch(&mut self.batch, &self.staged, &mut self.peers, fd, n, now) { + if n != 0 && !Self::dispatch(&mut self.batch, &self.staged, &mut self.peers, socket, n, now) + { arm_writable(&self.registry, entry); } } @@ -363,11 +382,11 @@ impl UdpManager { batch: &mut SendBatch, staged: &[Staged; BATCH], peers: &mut [UdpPeer], - fd: i32, + socket: &UdpSocket, n: usize, now: Instant, ) -> bool { - let accepted = send_batch(batch, fd, n); + let accepted = send_batch(batch, socket, n); for s in &staged[..accepted] { peers[s.peer].mark_sent(s.seq, now); } @@ -516,12 +535,18 @@ impl UdpManager { } /// Readable/writable event on the socket at `k`. - fn handle_event(&mut self, k: usize, event: &Event, dcache: Option<&DCache>, deliver: &mut F) - where + fn handle_event( + &mut self, + k: usize, + readable: bool, + writable: bool, + dcache: Option<&DCache>, + deliver: &mut F, + ) where F: for<'a> FnMut(PollEvent>), { let now = Instant::now(); - if event.is_readable() { + if readable { let fd = self.sockets[k].socket.as_raw_fd(); let mut recv = self.recv.take().expect("recv batch in use"); loop { @@ -546,7 +571,7 @@ impl UdpManager { self.recv = Some(recv); } - if event.is_writable() { + if writable { self.sockets[k].writable_armed = false; self.flush_socket(k, now); } @@ -557,7 +582,7 @@ impl UdpManager { arm_writable(&self.registry, entry); } } - if event.is_writable() && !entry.writable_armed { + if writable && !entry.writable_armed { if let Err(err) = self.registry.reregister(&mut entry.socket, entry.token, Interest::READABLE) { @@ -589,6 +614,49 @@ impl UdpManager { self.next_tick = now + self.tick_interval; self.tick(now); } + #[cfg(target_os = "linux")] + if matches!(self.udp.io, super::UdpIo::Uring(_)) { + for k in 0..self.sockets.len() { + // Drain in bounded passes, emitting ACKs between passes so + // slow callbacks cannot hold back the sender's whole window. + for _ in 0..BATCH { + let work = self.sockets[k].socket.ring.as_ref().unwrap().borrow_mut().poll(); + o |= work; + let mut received_any = false; + loop { + let received = + self.sockets[k].socket.ring.as_ref().unwrap().borrow_mut().receive(); + let Some(received) = received else { break }; + received_any = true; + if let Some((datagrams, from)) = + received.datagrams(self.udp.max_datagram_size) + { + for bytes in datagrams { + let Some(header) = Header::decode(bytes) else { continue }; + let dgram = + Datagram { header, payload: &bytes[HEADER_SIZE..], from, now }; + self.on_datagram(k, &dgram, dcache, deliver); + } + } + self.sockets[k] + .socket + .ring + .as_ref() + .unwrap() + .borrow_mut() + .recycle(received); + } + self.handle_event(k, false, false, dcache, deliver); + if !work && !received_any { + break; + } + } + let writable = self.sockets[k].writable_armed; + self.handle_event(k, false, writable, dcache, deliver); + self.sockets[k].socket.ring.as_ref().unwrap().borrow_mut().submit(); + } + return o | self.drain_pending_disconnects(deliver); + } // Taken out so `handle_event` can borrow `self`; put back below. let mut events = std::mem::replace(&mut self.events, Events::with_capacity(0)); if let Err(e) = self.poll.poll(&mut events, Some(std::time::Duration::ZERO)) { @@ -602,7 +670,7 @@ impl UdpManager { debug!(token = ?event.token(), "ignoring stale udp readiness event"); continue; }; - self.handle_event(k, event, dcache, deliver); + self.handle_event(k, event.is_readable(), event.is_writable(), dcache, deliver); } self.events = events; o |= self.drain_pending_disconnects(deliver); diff --git a/crates/flux-network/src/udp/mod.rs b/crates/flux-network/src/udp/mod.rs index 7a675ac..3c28640 100644 --- a/crates/flux-network/src/udp/mod.rs +++ b/crates/flux-network/src/udp/mod.rs @@ -17,6 +17,38 @@ mod wire; pub(crate) use connector::UdpManager; +/// Socket I/O implementation. The wire protocol is identical for both backends. +#[derive(Clone, Copy, Debug, Default)] +pub enum UdpIo { + #[default] + Syscall, + /// Linux `io_uring` with bounded per-socket buffers. Requires synchronous + /// cancellation support (Linux 6.0+); socket creation fails if unavailable. + #[cfg(target_os = "linux")] + Uring(UringConfig), +} + +/// Per-socket operation limits. Buffers are allocated when the socket opens. +/// +/// Each entry reserves approximately 64 KiB; defaults use about 6 MiB/socket. +/// `recv_entries` must be a power of two, both counts must be nonzero, and +/// their sum must not exceed 4096. +#[cfg(target_os = "linux")] +#[derive(Clone, Copy, Debug)] +pub struct UringConfig { + /// Outstanding sends, including GSO groups and control packets. + pub send_entries: u16, + /// Buffers shared by the socket's multishot receive. + pub recv_entries: u16, +} + +#[cfg(target_os = "linux")] +impl Default for UringConfig { + fn default() -> Self { + Self { send_entries: 64, recv_entries: 32 } + } +} + /// Tuning for [`crate::Transport::Udp`]. /// /// The retransmit timeout is measured from acks (RFC 6298) and clamped to @@ -26,6 +58,9 @@ pub(crate) use connector::UdpManager; /// `max_datagram_size`. #[derive(Clone, Copy, Debug)] pub struct UdpConfig { + /// Local I/O backend; peers may use different backends. Queued `io_uring` + /// sends progress through `NetworkDriver::poll_with`, like other backlogs. + pub io: UdpIo, /// Datagram size including the 29-byte header. 1200 stays under the /// 1280-byte IPv6 minimum MTU. pub max_datagram_size: usize, @@ -52,6 +87,7 @@ pub struct UdpConfig { impl Default for UdpConfig { fn default() -> Self { Self { + io: UdpIo::Syscall, max_datagram_size: 1200, send_window: 16 * 1024, recv_window: 16 * 1024, @@ -93,6 +129,11 @@ impl UdpConfig { } pub(crate) fn validate(&self) { + #[cfg(target_os = "linux")] + if let UdpIo::Uring(config) = self.io { + assert!(config.send_entries > 0 && config.recv_entries.is_power_of_two()); + assert!(u32::from(config.send_entries) + u32::from(config.recv_entries) <= 4096); + } assert!( self.max_datagram_size > wire::HEADER_SIZE && self.max_datagram_size <= wire::MAX_DATAGRAM_SIZE, diff --git a/crates/flux-network/src/udp/peer.rs b/crates/flux-network/src/udp/peer.rs index f6fa452..7b69c7c 100644 --- a/crates/flux-network/src/udp/peer.rs +++ b/crates/flux-network/src/udp/peer.rs @@ -1,17 +1,17 @@ //! Per-peer reliability state. One [`UdpPeer`] per remote address; the //! connector owns the sockets and passes them in. -use std::{collections::VecDeque, io, net::SocketAddr, os::fd::AsRawFd}; +use std::{collections::VecDeque, io, net::SocketAddr}; use flux_communication::Timer; use flux_timing::{Duration, Instant, Nanos}; use flux_utils::{DCache, DCacheRef}; -use mio::{Token, net::UdpSocket}; +use mio::Token; use tracing::{debug, warn}; use super::{ UdpConfig, - sys::{BATCH, SendBatch, SockAddr}, + sys::{BATCH, SendBatch, SockAddr, UdpSocket}, wire::{HEADER_SIZE, Header, Kind, fragment_count, write_session}, }; @@ -43,11 +43,12 @@ pub(crate) enum PushOutcome { TooLarge, } -/// Sends a filled batch of `n` datagrams. Returns how many the kernel took; -/// errors other than `WouldBlock` count as sent so the RTO path retries them. +/// Sends a filled batch of `n` datagrams. Returns how many the socket backend +/// accepted; errors other than `WouldBlock` count as sent so the RTO path +/// retries them. #[inline] -pub(crate) fn send_batch(batch: &mut SendBatch, fd: i32, n: usize) -> usize { - match batch.send(fd) { +pub(crate) fn send_batch(batch: &mut SendBatch, socket: &UdpSocket, n: usize) -> usize { + match socket.send_batch(batch) { Ok(k) => k, Err(e) if e.kind() == io::ErrorKind::WouldBlock => 0, Err(e) => { @@ -357,7 +358,7 @@ impl TxWindow { range: std::ops::Range, mut pred: impl FnMut(&Self, u64) -> bool, store: &MsgStore, - fd: i32, + socket: &UdpSocket, to: &SockAddr, batch: &mut SendBatch, now: Instant, @@ -373,7 +374,7 @@ impl TxWindow { pushed[n] = seq; n += 1; if n == BATCH { - let accepted = send_batch(batch, fd, n); + let accepted = send_batch(batch, socket, n); for &seq in &pushed[..accepted] { self.mark_sent(seq, true, now); } @@ -385,7 +386,7 @@ impl TxWindow { } } if n != 0 { - let accepted = send_batch(batch, fd, n); + let accepted = send_batch(batch, socket, n); for &seq in &pushed[..accepted] { self.mark_sent(seq, true, now); } @@ -422,7 +423,7 @@ impl TxWindow { rto: Duration, reorder: Duration, store: &mut MsgStore, - fd: i32, + socket: &UdpSocket, to: &SockAddr, batch: &mut SendBatch, now: Instant, @@ -487,7 +488,7 @@ impl TxWindow { if let Some(end) = self.hole_scan_end() { let pred = |tx: &Self, seq: u64| tx.is_hole(seq, rto, reorder, now); - let (_, blocked) = self.send_where(self.base..end, pred, store, fd, to, batch, now); + let (_, blocked) = self.send_where(self.base..end, pred, store, socket, to, batch, now); acked.blocked = blocked; } acked @@ -529,7 +530,7 @@ impl TxWindow { max_rto: Duration, reorder: Duration, store: &MsgStore, - fd: i32, + socket: &UdpSocket, to: &SockAddr, batch: &mut SendBatch, now: Instant, @@ -556,7 +557,7 @@ impl TxWindow { } false }; - self.send_where(self.base..self.next_send, pred, store, fd, to, batch, now) + self.send_where(self.base..self.next_send, pred, store, socket, to, batch, now) } /// Restarts every retained message from its first fragment under @@ -1098,7 +1099,7 @@ impl UdpPeer { self.rto.current(), self.rto.reorder_window(), store, - socket.as_raw_fd(), + socket, &self.native_addr, batch, now, @@ -1133,7 +1134,7 @@ impl UdpPeer { self.config.max_rto, self.rto.reorder_window(), store, - socket.as_raw_fd(), + socket, &self.native_addr, batch, now, @@ -1203,13 +1204,13 @@ mod tests { let (s, to) = sock(); let mut batch = SendBatch::new(); let now = Instant::now(); - let fd = s.as_raw_fd(); + let socket = &s; send_all(&mut tx, 0..8); assert_eq!(tx.next_send, 8); // Ack 0..3 cumulatively, 5 and 7 selectively: 3, 4, 6 are holes. let bits = 0b0000_1010_u64.to_le_bytes(); let zero = Duration::ZERO; - let acked = tx.on_ack(3, 8, &bits, zero, zero, &mut store, fd, &to, &mut batch, now); + let acked = tx.on_ack(3, 8, &bits, zero, zero, &mut store, socket, &to, &mut batch, now); assert!(!acked.blocked); assert!(acked.rtt.is_some()); assert_eq!(tx.base, 3); @@ -1217,7 +1218,7 @@ mod tests { assert_eq!(store.free.len(), 2); assert_eq!(tx.free(), 59, "slots of the unfinished message stay reserved"); assert!(tx.is_acked(5) && tx.is_acked(7) && !tx.is_acked(4)); - let acked = tx.on_ack(8, 0, &[], zero, zero, &mut store, fd, &to, &mut batch, now); + let acked = tx.on_ack(8, 0, &[], zero, zero, &mut store, socket, &to, &mut batch, now); assert!(acked.rtt.is_none(), "holes were retransmitted, no clean sample"); assert_eq!(tx.free(), 64); assert!(tx.messages.is_empty()); @@ -1231,7 +1232,7 @@ mod tests { let mut store = MsgStore::new(); let mut tx = TxWindow::new(config.send_window); let (s, to) = sock(); - let fd = s.as_raw_fd(); + let socket = &s; let mut batch = SendBatch::new(); let now = Instant::now(); let zero = Duration::ZERO; @@ -1239,7 +1240,7 @@ mod tests { assert!(push(&mut tx, &mut store, stride, &[1])); } send_all(&mut tx, 0..64); - tx.on_ack(64, 0, &[], zero, zero, &mut store, fd, &to, &mut batch, now); + tx.on_ack(64, 0, &[], zero, zero, &mut store, socket, &to, &mut batch, now); assert_eq!(tx.base, 64); for _ in 0..8 { assert!(push(&mut tx, &mut store, stride, &[2])); @@ -1247,7 +1248,7 @@ mod tests { send_all(&mut tx, 64..72); // Old ack: cumulative 1, selective bit for seq 2, which aliases seq 66. let bits = 0b1_u64.to_le_bytes(); - tx.on_ack(1, 8, &bits, zero, zero, &mut store, fd, &to, &mut batch, now); + tx.on_ack(1, 8, &bits, zero, zero, &mut store, socket, &to, &mut batch, now); assert_eq!(tx.base, 64); assert!(!tx.is_acked(66)); } @@ -1259,13 +1260,13 @@ mod tests { let mut store = MsgStore::new(); let mut tx = TxWindow::new(config.send_window); let (s, to) = sock(); - let fd = s.as_raw_fd(); + let socket = &s; let mut batch = SendBatch::new(); let now = Instant::now(); let zero = Duration::ZERO; assert!(push(&mut tx, &mut store, stride, &vec![0; stride * 4])); send_all(&mut tx, 0..4); - tx.on_ack(2, 0, &[], zero, zero, &mut store, fd, &to, &mut batch, now); + tx.on_ack(2, 0, &[], zero, zero, &mut store, socket, &to, &mut batch, now); assert_eq!(tx.base, 2); tx.rewind(0xabcd); assert_eq!((tx.base, tx.next_send), (0, 0), "restarts from the first fragment"); diff --git a/crates/flux-network/src/udp/sys.rs b/crates/flux-network/src/udp/sys.rs index 758db34..d59a591 100644 --- a/crates/flux-network/src/udp/sys.rs +++ b/crates/flux-network/src/udp/sys.rs @@ -9,6 +9,88 @@ use std::{ ptr, slice, }; +#[cfg(target_os = "linux")] +pub(crate) mod uring; + +/// Datagram socket and its optional completion backend. +pub(crate) struct UdpSocket { + #[cfg(target_os = "linux")] + // Retire kernel requests before closing the socket below. + pub(crate) ring: Option>, + socket: mio::net::UdpSocket, +} + +impl UdpSocket { + pub(crate) fn bind(addr: SocketAddr) -> io::Result { + Ok(Self { + socket: mio::net::UdpSocket::bind(addr)?, + #[cfg(target_os = "linux")] + ring: None, + }) + } + + #[cfg(test)] + pub(crate) fn local_addr(&self) -> io::Result { + self.socket.local_addr() + } + + pub(crate) fn send_to(&self, bytes: &[u8], addr: SocketAddr) -> io::Result { + #[cfg(target_os = "linux")] + if let Some(ring) = &self.ring { + return ring.borrow_mut().send(bytes, SockAddr::new(addr)); + } + self.socket.send_to(bytes, addr) + } + + pub(crate) fn send_batch(&self, batch: &mut SendBatch) -> io::Result { + #[cfg(target_os = "linux")] + if let Some(ring) = &self.ring { + return ring.borrow_mut().send_batch(batch); + } + batch.send(std::os::fd::AsRawFd::as_raw_fd(self)) + } +} + +impl std::os::fd::AsRawFd for UdpSocket { + fn as_raw_fd(&self) -> RawFd { + self.socket.as_raw_fd() + } +} + +impl mio::event::Source for UdpSocket { + fn register( + &mut self, + registry: &mio::Registry, + token: mio::Token, + interests: mio::Interest, + ) -> io::Result<()> { + #[cfg(target_os = "linux")] + if self.ring.is_some() { + return Ok(()); + } + self.socket.register(registry, token, interests) + } + fn reregister( + &mut self, + registry: &mio::Registry, + token: mio::Token, + interests: mio::Interest, + ) -> io::Result<()> { + #[cfg(target_os = "linux")] + if self.ring.is_some() { + return Ok(()); + } + self.socket.reregister(registry, token, interests) + } + fn deregister(&mut self, registry: &mio::Registry) -> io::Result<()> { + #[cfg(target_os = "linux")] + if self.ring.is_some() { + return Ok(()); + } + self.socket.deregister(registry) + } +} + /// Datagrams per syscall. pub(crate) const BATCH: usize = 32; @@ -368,36 +450,50 @@ impl RecvBatch { /// entries. pub(crate) fn datagrams(&self, i: usize) -> Option<(std::slice::Chunks<'_, u8>, SocketAddr)> { debug_assert!(i < self.len); - let hdr = &self.hdrs[i]; - if hdr.msg_hdr.msg_flags & (libc::MSG_TRUNC | libc::MSG_CTRUNC) != 0 { - return None; - } - let len = hdr.msg_len as usize; - #[cfg(not(target_os = "linux"))] - let segment_size = len; - #[cfg(target_os = "linux")] - let segment_size = if hdr.msg_hdr.msg_controllen != 0 { - let control = &self.controls[i]; - let control_len = - unsafe { libc::CMSG_LEN(mem::size_of::() as _) as usize }; - if hdr.msg_hdr.msg_controllen < control_len || - control.header.cmsg_len != control_len || - control.header.cmsg_level != libc::SOL_UDP || - control.header.cmsg_type != libc::UDP_GRO - { - return None; - } - usize::try_from(control.size).ok()? - } else { - len - }; - if segment_size == 0 || segment_size > self.datagram_size { + let start = i * self.stride; + decode_datagrams( + &self.hdrs[i], + &self.addrs[i], + #[cfg(target_os = "linux")] + &self.controls[i], + &self.bufs[start..start + self.stride], + self.datagram_size, + ) + } +} + +fn decode_datagrams<'a>( + hdr: &MMsgHdr, + addr: &libc::sockaddr_storage, + #[cfg(target_os = "linux")] control: &GroControl, + bytes: &'a [u8], + datagram_size: usize, +) -> Option<(std::slice::Chunks<'a, u8>, SocketAddr)> { + if hdr.msg_hdr.msg_flags & (libc::MSG_TRUNC | libc::MSG_CTRUNC) != 0 { + return None; + } + let len = hdr.msg_len as usize; + #[cfg(not(target_os = "linux"))] + let segment_size = len; + #[cfg(target_os = "linux")] + let segment_size = if hdr.msg_hdr.msg_controllen != 0 { + let control_len = unsafe { libc::CMSG_LEN(mem::size_of::() as _) as usize }; + if hdr.msg_hdr.msg_controllen < control_len || + control.header.cmsg_len != control_len || + control.header.cmsg_level != libc::SOL_UDP || + control.header.cmsg_type != libc::UDP_GRO + { return None; } - let from = SockAddr::decode(&self.addrs[i], hdr.msg_hdr.msg_namelen)?; - let start = i * self.stride; - Some((self.bufs[start..start + len].chunks(segment_size), from)) + usize::try_from(control.size).ok()? + } else { + len + }; + if segment_size == 0 || segment_size > datagram_size { + return None; } + let from = SockAddr::decode(addr, hdr.msg_hdr.msg_namelen)?; + Some((bytes[..len].chunks(segment_size), from)) } #[cfg(test)] diff --git a/crates/flux-network/src/udp/sys/uring.rs b/crates/flux-network/src/udp/sys/uring.rs new file mode 100644 index 0000000..04e90e7 --- /dev/null +++ b/crates/flux-network/src/udp/sys/uring.rs @@ -0,0 +1,682 @@ +//! Bounded completion I/O. Kernel requests own pooled buffers, never peer +//! state. + +use std::{collections::VecDeque, io, mem, os::fd::RawFd, ptr}; + +use io_uring::{IoUring, opcode, types}; +use tracing::{debug, warn}; + +use super::{ + GRO_CONTROL_SPACE, GroControl, MMsgHdr, SEGMENT_CONTROL_SPACE, SegmentControl, SendBatch, + SockAddr, decode_datagrams, iovec_mut, +}; +use crate::udp::UringConfig; + +const RX_BIT: u64 = 1 << 63; +const BUFFER_SIZE: usize = 65_535; + +const NAME_SIZE: usize = mem::size_of::(); +const RX_SIZE: usize = BUFFER_SIZE + 16 + NAME_SIZE + GRO_CONTROL_SPACE; + +struct Rx { + bytes: Box<[u8]>, + len: usize, +} + +impl Rx { + fn new() -> Self { + Self { bytes: vec![0; RX_SIZE].into_boxed_slice(), len: 0 } + } +} + +struct Provided { + base: std::ptr::NonNull, + layout: std::alloc::Layout, + tail: u16, + mask: u16, +} + +impl Provided { + fn new(entries: u16) -> Self { + let page = unsafe { libc::sysconf(libc::_SC_PAGESIZE) }; + assert!(page > 0); + let layout = std::alloc::Layout::from_size_align( + usize::from(entries) * mem::size_of::(), + page as usize, + ) + .unwrap(); + let base = std::ptr::NonNull::new(unsafe { std::alloc::alloc_zeroed(layout) }.cast()) + .unwrap_or_else(|| std::alloc::handle_alloc_error(layout)); + Self { base, layout, tail: 0, mask: entries - 1 } + } + + fn provide(&mut self, index: usize, bytes: &mut [u8]) { + // Only consumed entries are reused. Publish the buffer after its + // descriptor, and do not touch its bytes until the receive CQE. + unsafe { + let entry = &mut *self.base.as_ptr().add(usize::from(self.tail & self.mask)); + entry.set_addr(bytes.as_mut_ptr() as u64); + entry.set_len(bytes.len() as u32); + entry.set_bid(index as u16); + self.tail = self.tail.wrapping_add(1); + let tail = types::BufRingEntry::tail(self.base.as_ptr()) + .cast::(); + (*tail).store(self.tail, std::sync::atomic::Ordering::Release); + } + } +} + +impl Drop for Provided { + fn drop(&mut self) { + unsafe { + std::alloc::dealloc(self.base.as_ptr().cast(), self.layout); + } + } +} + +fn receive_header() -> libc::msghdr { + let mut header: libc::msghdr = unsafe { mem::zeroed() }; + header.msg_namelen = NAME_SIZE as _; + header.msg_controllen = GRO_CONTROL_SPACE; + header +} + +struct Tx { + header: libc::msghdr, + addr: SockAddr, + control: SegmentControl, + iov: libc::iovec, + bytes: Box<[u8]>, + len: usize, + segment: usize, + offset: usize, + fallback: bool, +} + +impl Tx { + fn new() -> Box { + Box::new(Self { + header: unsafe { mem::zeroed() }, + addr: SockAddr::new("0.0.0.0:0".parse().unwrap()), + control: unsafe { mem::zeroed() }, + iov: unsafe { mem::zeroed() }, + bytes: vec![0; BUFFER_SIZE].into_boxed_slice(), + len: 0, + segment: 0, + offset: 0, + fallback: false, + }) + } + + fn prepare(&mut self, fd: RawFd, index: usize) -> io_uring::squeue::Entry { + let end = if self.fallback { (self.offset + self.segment).min(self.len) } else { self.len }; + self.iov = iovec_mut(&mut self.bytes[self.offset..end]); + self.header.msg_name = ptr::from_mut(&mut self.addr.storage).cast(); + self.header.msg_namelen = self.addr.len; + self.header.msg_iov = ptr::from_mut(&mut self.iov); + self.header.msg_iovlen = 1; + self.header.msg_control = ptr::null_mut(); + self.header.msg_controllen = 0; + if self.segment != 0 && !self.fallback { + self.control.header = libc::cmsghdr { + cmsg_len: unsafe { libc::CMSG_LEN(mem::size_of::() as _) as usize }, + cmsg_level: libc::SOL_UDP, + cmsg_type: libc::UDP_SEGMENT, + }; + self.control.size = self.segment as u16; + self.header.msg_control = ptr::from_mut(&mut self.control).cast(); + self.header.msg_controllen = SEGMENT_CONTROL_SPACE; + } + opcode::SendMsg::new(types::Fd(fd), ptr::from_ref(&self.header)) + .flags(libc::MSG_DONTWAIT as _) + .build() + .user_data(index as u64) + } +} + +pub(crate) struct Received { + index: usize, + slot: Rx, +} + +impl Received { + pub(crate) fn datagrams( + &self, + max_size: usize, + ) -> Option<(std::slice::Chunks<'_, u8>, std::net::SocketAddr)> { + let rx = &self.slot; + let out = types::RecvMsgOut::parse(&rx.bytes[..rx.len], &receive_header()).ok()?; + if out.is_name_data_truncated() || + out.is_control_data_truncated() || + out.is_payload_truncated() + { + return None; + } + let mut addr: libc::sockaddr_storage = unsafe { mem::zeroed() }; + let mut control: GroControl = unsafe { mem::zeroed() }; + // The parser bounds these slices by the configured metadata capacities. + unsafe { + ptr::copy_nonoverlapping( + out.name_data().as_ptr(), + ptr::from_mut(&mut addr).cast(), + out.name_data().len(), + ); + ptr::copy_nonoverlapping( + out.control_data().as_ptr(), + ptr::from_mut(&mut control).cast(), + out.control_data().len(), + ); + } + let mut hdr: MMsgHdr = unsafe { mem::zeroed() }; + hdr.msg_len = out.payload_data().len() as u32; + hdr.msg_hdr.msg_namelen = out.incoming_name_len(); + hdr.msg_hdr.msg_controllen = out.control_data().len(); + // Return a slice of the owned buffer, independent of the parser's borrow. + let offset = out.payload_data().as_ptr() as usize - rx.bytes.as_ptr() as usize; + decode_datagrams( + &hdr, + &addr, + &control, + &rx.bytes[offset..offset + hdr.msg_len as usize], + max_size, + ) + } +} + +pub(crate) struct Ring { + io: IoUring, + fd: RawFd, + owner: std::thread::ThreadId, + // Boxes keep msghdr, iovec and ancillary pointers stable across moves. + rx: Vec>, + provided: Option, + receive_header: Box, + rx_active: bool, + available: usize, + #[allow(clippy::vec_box)] + tx: Vec>, + free_tx: Vec, + ready: VecDeque, + completions: Vec<(u64, i32, u32)>, +} + +// SAFETY: requests only access their boxed buffers. Moving the owner does not +// move the buffers; RefCell on the socket excludes concurrent access. +#[allow(clippy::non_send_fields_in_send_ty)] +unsafe impl Send for Ring {} + +impl Ring { + pub(crate) fn new(fd: RawFd, config: UringConfig) -> io::Result { + let count = u32::from(config.send_entries) + u32::from(config.recv_entries); + let ring = IoUring::builder() + .setup_coop_taskrun() + .setup_taskrun_flag() + .build(count.next_power_of_two())?; + // Require the teardown primitive before posting any borrowed pointers. + cancel(&ring)?; + let provided = Provided::new(config.recv_entries); + // SAFETY: the aligned descriptor ring outlives its registration. + unsafe { + ring.submitter().register_buf_ring_with_flags( + provided.base.as_ptr() as u64, + config.recv_entries, + 0, + 0, + )?; + } + let mut this = Self { + io: ring, + fd, + owner: std::thread::current().id(), + rx: (0..config.recv_entries).map(|_| Some(Rx::new())).collect(), + provided: Some(provided), + receive_header: Box::new(receive_header()), + rx_active: false, + available: config.recv_entries.into(), + tx: (0..config.send_entries).map(|_| Tx::new()).collect(), + free_tx: (0..usize::from(config.send_entries)).rev().collect(), + ready: VecDeque::with_capacity(config.recv_entries.into()), + completions: Vec::with_capacity(count as usize + 1), + }; + for i in 0..this.rx.len() { + this.provided.as_mut().unwrap().provide(i, &mut this.rx[i].as_mut().unwrap().bytes); + } + this.arm_receive(); + this.io.submit()?; + Ok(this) + } + + fn push(&mut self, entry: &io_uring::squeue::Entry) { + // At most one SQE per slot. Ring capacity covers every TX and RX slot. + // SAFETY: slots remain allocated and unchanged until their CQE. + unsafe { + self.io.submission().push(entry).expect("UDP operation slots exceed SQ capacity"); + }; + } + + fn arm_receive(&mut self) { + if self.rx_active || self.available == 0 { + return; + } + let entry = + opcode::RecvMsgMulti::new(types::Fd(self.fd), ptr::from_ref(&*self.receive_header), 0) + .build() + .user_data(RX_BIT); + self.push(&entry); + self.rx_active = true; + } + + fn arm_send(&mut self, index: usize) { + let entry = self.tx[index].prepare(self.fd, index); + self.push(&entry); + } + + /// A successful enqueue owns the bytes, like acceptance into a socket + /// send buffer. Later errors are packet loss, recovered by the protocol. + pub(crate) fn send(&mut self, bytes: &[u8], to: SockAddr) -> io::Result { + if bytes.len() > BUFFER_SIZE { + return Err(io::Error::from_raw_os_error(libc::EMSGSIZE)); + } + let index = self.free_tx.pop().ok_or(io::ErrorKind::WouldBlock)?; + let tx = &mut self.tx[index]; + tx.bytes[..bytes.len()].copy_from_slice(bytes); + tx.len = bytes.len(); + tx.addr = to; + tx.segment = 0; + tx.offset = 0; + tx.fallback = false; + self.arm_send(index); + Ok(bytes.len()) + } + + pub(crate) fn send_batch(&mut self, batch: &mut SendBatch) -> io::Result { + if self.free_tx.is_empty() { + self.poll(); + } + let n = mem::take(&mut batch.len); + let mut accepted = 0; + while accepted < n { + let Some(index) = self.free_tx.pop() else { break }; + // A batch may straddle peers or message tails. Segment each + // compatible run instead of losing GSO for the entire batch. + let size = batch.iovs[accepted].iter().map(|iov| iov.iov_len).sum::(); + let mut end = accepted + 1; + if batch.gso && size != 0 { + while end < n && batch.addrs[end] == batch.addrs[accepted] { + let len = batch.iovs[end].iter().map(|iov| iov.iov_len).sum::(); + if len == 0 || len > size || (end - accepted) * size + len > BUFFER_SIZE { + break; + } + end += 1; + if len < size { + break; + } + } + } + let segment = if end > accepted + 1 { size } else { 0 }; + let tx = &mut self.tx[index]; + tx.len = 0; + tx.addr = batch.addrs[accepted]; + tx.segment = segment; + tx.offset = 0; + tx.fallback = false; + for i in accepted..end { + for iov in &batch.iovs[i] { + // SAFETY: SendBatch's slices remain live throughout this call. + let bytes = unsafe { + std::slice::from_raw_parts(iov.iov_base.cast::(), iov.iov_len) + }; + tx.bytes[tx.len..tx.len + bytes.len()].copy_from_slice(bytes); + tx.len += bytes.len(); + } + } + self.arm_send(index); + accepted = end; + } + if accepted == 0 && n != 0 { Err(io::ErrorKind::WouldBlock.into()) } else { Ok(accepted) } + } + + pub(crate) fn submit(&mut self) { + let owner = std::thread::current().id(); + if self.owner != owner { + // Receive task work belongs to the submitting thread. Retire those + // requests before a moved driver starts polling on another thread. + cancel(&self.io).expect("couldn't transfer UDP io_uring to this thread"); + self.owner = owner; + } + let needs_enter = { + let sq = self.io.submission(); + !sq.is_empty() || sq.taskrun() + }; + if needs_enter { + if let Err(err) = self.io.submit() { + debug!(?err, "UDP io_uring submit failed"); + } + } + } + + pub(crate) fn poll(&mut self) -> bool { + self.submit(); + self.completions.clear(); + self.completions + .extend(self.io.completion().map(|c| (c.user_data(), c.result(), c.flags()))); + for i in 0..self.completions.len() { + let (id, result, flags) = self.completions[i]; + if id & RX_BIT != 0 { + if !io_uring::cqueue::more(flags) { + self.rx_active = false; + } + if let Some(index) = io_uring::cqueue::buffer_select(flags) { + let index = usize::from(index); + self.available -= 1; + if result >= 0 { + self.rx[index].as_mut().unwrap().len = result as usize; + self.ready.push_back(index); + } else { + self.provided + .as_mut() + .unwrap() + .provide(index, &mut self.rx[index].as_mut().unwrap().bytes); + self.available += 1; + } + } + if result < 0 && result != -libc::ENOBUFS && result != -libc::ECANCELED { + debug!(result, "UDP io_uring receive failed"); + } + } else { + let index = id as usize; + let tx = &mut self.tx[index]; + if result < 0 && tx.segment != 0 && !tx.fallback { + // GSO errors reject the whole message. Retry its original + // datagrams without offload, retaining the same buffer. + tx.fallback = true; + self.arm_send(index); + } else if tx.fallback && result >= 0 && tx.offset + tx.iov.iov_len < tx.len { + tx.offset += tx.iov.iov_len; + self.arm_send(index); + } else { + if result < 0 { + debug!(result, "UDP io_uring send failed"); + } + self.free_tx.push(index); + } + } + } + self.arm_receive(); + !self.completions.is_empty() + } + + pub(crate) fn receive(&mut self) -> Option { + let index = self.ready.pop_front()?; + Some(Received { index, slot: self.rx[index].take().unwrap() }) + } + + pub(crate) fn recycle(&mut self, received: Received) { + let index = received.index; + self.rx[index] = Some(received.slot); + self.provided.as_mut().unwrap().provide(index, &mut self.rx[index].as_mut().unwrap().bytes); + self.available += 1; + self.arm_receive(); + } +} + +fn cancel(ring: &IoUring) -> io::Result<()> { + loop { + match ring.submitter().register_sync_cancel(None, types::CancelBuilder::any()) { + Ok(()) => return Ok(()), + Err(err) if err.raw_os_error() == Some(libc::ENOENT) => return Ok(()), + Err(err) if err.kind() == io::ErrorKind::Interrupted => {} + Err(err) => return Err(err), + } + } +} + +impl Drop for Ring { + fn drop(&mut self) { + // Unsubmitted SQEs are never submitted again. Cancel all kernel-owned + // operations synchronously before releasing their memory or socket. + match cancel(&self.io) { + Ok(()) => { + if let Err(err) = self.io.submitter().unregister_buf_ring(0) { + warn!(?err, "UDP buffer ring unregister failed; retaining receive buffers"); + mem::forget(mem::take(&mut self.rx)); + mem::forget(self.provided.take()); + } + } + Err(err) => { + // A failed cancellation cannot justify freeing kernel buffers. + warn!(?err, "UDP io_uring cancellation failed; retaining operation buffers"); + mem::forget(mem::take(&mut self.rx)); + mem::forget(mem::take(&mut self.tx)); + mem::forget(self.provided.take()); + // The multishot request also borrows this header. + mem::forget(mem::replace(&mut self.receive_header, Box::new(receive_header()))); + } + } + } +} + +#[cfg(test)] +mod tests { + use std::{ + net::UdpSocket, + os::fd::AsRawFd, + time::{Duration, Instant}, + }; + + use super::*; + + fn config() -> UringConfig { + UringConfig { send_entries: 2, recv_entries: 2 } + } + + fn receive(ring: &mut Ring) -> Received { + let deadline = Instant::now() + Duration::from_secs(2); + loop { + ring.poll(); + if let Some(rx) = ring.receive() { + return rx; + } + assert!(Instant::now() < deadline, "receive completion timed out"); + } + } + + #[test] + fn owned_sends_backpressure_and_receive_recycling() { + for bind in ["127.0.0.1:0", "[::1]:0"] { + let socket = UdpSocket::bind(bind).unwrap(); + let remote = UdpSocket::bind(bind).unwrap(); + remote.set_read_timeout(Some(Duration::from_secs(2))).unwrap(); + let mut ring = Ring::new(socket.as_raw_fd(), config()).unwrap(); + let to = SockAddr::new(remote.local_addr().unwrap()); + let mut payload = vec![1; 100]; + assert_eq!(ring.send(&payload, to).unwrap(), 100); + payload.fill(2); + assert_eq!(ring.send(&payload, to).unwrap(), 100); + payload.fill(3); + assert_eq!(ring.send(&payload, to).unwrap_err().kind(), io::ErrorKind::WouldBlock); + ring.poll(); + let mut buf = [0; 1200]; + let mut seen = Vec::new(); + for _ in 0..2 { + let n = remote.recv(&mut buf).unwrap(); + assert_eq!(n, 100); + assert!(buf[..n].iter().all(|b| *b == buf[0])); + seen.push(buf[0]); + } + seen.sort_unstable(); + assert_eq!(seen, [1, 2]); + for i in 0..16 { + remote.send_to(&[i; 10], socket.local_addr().unwrap()).unwrap(); + let rx = receive(&mut ring); + let (mut datagrams, from) = rx.datagrams(1200).unwrap(); + assert_eq!(from, remote.local_addr().unwrap()); + assert_eq!(datagrams.next().unwrap(), &[i; 10]); + assert!(datagrams.next().is_none()); + ring.recycle(rx); + } + // Drop with receives both submitted and queued for rearming. + drop(ring); + drop(socket); + } + } + + #[test] + fn gso_and_fallback_preserve_boundaries() { + for fallback in [false, true] { + let socket = UdpSocket::bind("127.0.0.1:0").unwrap(); + let remote = UdpSocket::bind("127.0.0.1:0").unwrap(); + remote.set_nonblocking(true).unwrap(); + if fallback { + let enabled: libc::c_int = 1; + assert_eq!( + unsafe { + libc::setsockopt( + socket.as_raw_fd(), + libc::SOL_SOCKET, + libc::SO_NO_CHECK, + ptr::from_ref(&enabled).cast(), + mem::size_of_val(&enabled) as _, + ) + }, + 0 + ); + } + let mut ring = Ring::new(socket.as_raw_fd(), config()).unwrap(); + let mut batch = SendBatch::new(); + batch.enable_gso(socket.as_raw_fd()).unwrap(); + let to = SockAddr::new(remote.local_addr().unwrap()); + let headers = [[1; 29], [2; 29], [3; 29], [4; 29]]; + let payload = [0x5a; 1171]; + for (i, header) in headers.iter().enumerate() { + batch.push(header, &payload[..if i == 3 { 17 } else { 1171 }], &to); + } + assert_eq!(ring.send_batch(&mut batch).unwrap(), 4); + let deadline = Instant::now() + Duration::from_secs(2); + let mut seen = [false; 4]; + let mut count = 0; + let mut buf = [0; 2048]; + while count < 4 { + ring.poll(); + if let Ok(n) = remote.recv(&mut buf) { + let i = usize::from(buf[0] - 1); + assert!(!seen[i]); + seen[i] = true; + assert_eq!(&buf[..29], &headers[i]); + assert_eq!(&buf[29..n], &payload[..if i == 3 { 17 } else { 1171 }]); + count += 1; + } + assert!(Instant::now() < deadline, "GSO/fallback timed out"); + } + } + } + + #[test] + fn exhausted_receive_pool_rearms_and_tail_wraps() { + let socket = UdpSocket::bind("127.0.0.1:0").unwrap(); + let remote = UdpSocket::bind("127.0.0.1:0").unwrap(); + let mut ring = Ring::new(socket.as_raw_fd(), config()).unwrap(); + for i in 0..8 { + remote.send_to(&[i], socket.local_addr().unwrap()).unwrap(); + } + let deadline = Instant::now() + Duration::from_secs(2); + while ring.rx_active || ring.available != 0 { + ring.poll(); + assert!(Instant::now() < deadline, "buffer exhaustion CQE missing"); + } + let mut seen = [false; 8]; + for _ in 0..8 { + let rx = receive(&mut ring); + let (mut packets, _) = rx.datagrams(1200).unwrap(); + let i = usize::from(packets.next().unwrap()[0]); + assert!(!seen[i]); + seen[i] = true; + ring.recycle(rx); + } + assert!(seen.into_iter().all(|v| v)); + for _ in 0..65_536 { + remote.send_to(b"wrap", socket.local_addr().unwrap()).unwrap(); + let rx = receive(&mut ring); + assert_eq!(rx.datagrams(1200).unwrap().0.next().unwrap(), b"wrap"); + ring.recycle(rx); + } + } + + #[test] + fn driver_moves_after_submitting_receives() { + let socket = UdpSocket::bind("127.0.0.1:0").unwrap(); + let remote = UdpSocket::bind("127.0.0.1:0").unwrap(); + let mut ring = Ring::new(socket.as_raw_fd(), config()).unwrap(); + // The original issuer exits while the socket and ring move onward. + let (mut ring, socket) = std::thread::spawn(move || { + ring.poll(); + (ring, socket) + }) + .join() + .unwrap(); + remote.send_to(b"moved", socket.local_addr().unwrap()).unwrap(); + let rx = receive(&mut ring); + assert_eq!(rx.datagrams(1200).unwrap().0.next().unwrap(), b"moved"); + ring.recycle(rx); + } + + #[test] + fn mixed_gso_runs_keep_their_destinations_and_lengths() { + let socket = UdpSocket::bind("127.0.0.1:0").unwrap(); + let peers = + [UdpSocket::bind("127.0.0.1:0").unwrap(), UdpSocket::bind("127.0.0.1:0").unwrap()]; + let to = peers.each_ref().map(|p| SockAddr::new(p.local_addr().unwrap())); + for p in &peers { + p.set_read_timeout(Some(Duration::from_secs(2))).unwrap(); + } + let mut ring = + Ring::new(socket.as_raw_fd(), UringConfig { send_entries: 8, recv_entries: 2 }) + .unwrap(); + let mut batch = SendBatch::new(); + batch.enable_gso(socket.as_raw_fd()).unwrap(); + let headers: [[u8; 29]; 8] = std::array::from_fn(|i| [i as u8; 29]); + let payload = [0x5a; 1171]; + let lengths = [1171, 17, 1171, 1171, 1171, 3, 1171, 1171]; + for i in 0..8 { + batch.push(&headers[i], &payload[..lengths[i]], &to[i / 4]); + } + assert_eq!(ring.send_batch(&mut batch).unwrap(), 8); + ring.poll(); + let mut seen = [false; 8]; + for (peer_index, peer) in peers.iter().enumerate() { + for _ in 0..4 { + let mut buf = [0; 2048]; + let len = peer.recv(&mut buf).unwrap(); + let i = usize::from(buf[0]); + assert_eq!(i / 4, peer_index); + assert!(!seen[i]); + seen[i] = true; + assert_eq!(&buf[..29], &headers[i]); + assert_eq!(&buf[29..len], &payload[..lengths[i]]); + } + } + } + + #[test] + fn gro_and_oversized_receives() { + let socket = UdpSocket::bind("127.0.0.1:0").unwrap(); + let remote = UdpSocket::bind("127.0.0.1:0").unwrap(); + super::super::RecvBatch::enable_gro(socket.as_raw_fd()).unwrap(); + let mut ring = Ring::new(socket.as_raw_fd(), config()).unwrap(); + let mut batch = SendBatch::new(); + batch.enable_gso(remote.as_raw_fd()).unwrap(); + let to = SockAddr::new(socket.local_addr().unwrap()); + for _ in 0..4 { + batch.push(&[1; 29], &[2; 1171], &to); + } + assert_eq!(batch.send(remote.as_raw_fd()).unwrap(), 4); + let rx = receive(&mut ring); + let (datagrams, _) = rx.datagrams(1200).unwrap(); + assert_eq!(datagrams.count(), 4); + ring.recycle(rx); + remote.send_to(&[0; 1201], socket.local_addr().unwrap()).unwrap(); + let rx = receive(&mut ring); + assert!(rx.datagrams(1200).is_none()); + ring.recycle(rx); + } +} diff --git a/crates/flux-network/tests/support/udp_connector.rs b/crates/flux-network/tests/support/udp_connector.rs new file mode 100644 index 0000000..f3a920f --- /dev/null +++ b/crates/flux-network/tests/support/udp_connector.rs @@ -0,0 +1,612 @@ +use std::{ + net::{Ipv4Addr, SocketAddr, UdpSocket}, + sync::{ + Arc, + atomic::{AtomicBool, AtomicUsize, Ordering}, + }, + thread, + time::{Duration, Instant}, +}; + +use flux_network::{NetworkDriver, PollEvent, SendBehavior, Transport, UdpConfig}; +use mio::Token; + +fn free_addr() -> SocketAddr { + UdpSocket::bind((Ipv4Addr::LOCALHOST, 0)).unwrap().local_addr().unwrap() +} + +fn udp(config: UdpConfig) -> NetworkDriver { + let config = UdpConfig { io: IO, ..config }; + NetworkDriver::default().with_transport(Transport::Udp(config)) +} + +/// Handshake between a fresh listener and client, returning the accepted and +/// the outbound token. `dial` differs from `addr` when a relay sits between. +fn connect_via( + server: &mut NetworkDriver, + client: &mut NetworkDriver, + addr: SocketAddr, + dial: SocketAddr, +) -> (Token, Token) { + server.listen_at(addr).unwrap(); + let client_token = client.connect(dial).unwrap(); + let mut accepted = None; + let deadline = Instant::now() + Duration::from_secs(5); + while accepted.is_none() { + assert!(Instant::now() < deadline, "no accept"); + server.poll_with(|e| { + if let PollEvent::Accept { stream, .. } = e { + accepted = Some(stream); + } + }); + client.poll_with(|_| {}); + thread::sleep(Duration::from_micros(50)); + } + while client.currently_disconnected().count() != 0 { + assert!(Instant::now() < deadline, "handshake"); + server.poll_with(|_| {}); + client.poll_with(|_| {}); + thread::sleep(Duration::from_micros(50)); + } + // A hello retry may still be in flight if the ack took longer than one + // RTO. Let it land now, while the peer it belongs to still exists. + for _ in 0..5 { + server.poll_with(|_| {}); + client.poll_with(|_| {}); + } + (accepted.unwrap(), client_token) +} + +fn connect_pair( + server: &mut NetworkDriver, + client: &mut NetworkDriver, + addr: SocketAddr, +) -> (Token, Token) { + connect_via(server, client, addr, addr) +} + +fn checksum(bytes: &[u8]) -> u64 { + bytes + .iter() + .fold(0xcbf2_9ce4_8422_2325_u64, |h, b| (h ^ u64::from(*b)).wrapping_mul(0x100_0000_01b3)) +} + +/// Test message: 4-byte id, then `len` bytes derived from the id. +fn make_msg(id: u32, len: usize) -> Vec { + let mut v = Vec::with_capacity(4 + len); + v.extend_from_slice(&id.to_le_bytes()); + v.extend((0..len).map(|i| (id as usize).wrapping_mul(31).wrapping_add(i * 7) as u8)); + v +} + +fn msg_id(payload: &[u8]) -> u32 { + u32::from_le_bytes(payload[..4].try_into().unwrap()) +} + +#[test] +fn udp_roundtrip_before_handshake_completes() { + let addr = free_addr(); + let mut server = udp(UdpConfig::lan()); + server.listen_at(addr).unwrap(); + let mut client = udp(UdpConfig::lan()); + let tok = client.connect(addr).unwrap(); + // Queued before the hello ack arrives; must go out once it does. + client.write_or_enqueue_with(SendBehavior::Single(tok), |b| b.extend_from_slice(b"ping")); + + let mut accepted = None; + let mut request_seen = false; + let mut reply_seen = false; + let deadline = Instant::now() + Duration::from_secs(5); + while !reply_seen { + assert!(Instant::now() < deadline, "roundtrip timed out"); + server.poll_with(|e| match e { + PollEvent::Accept { stream, .. } => accepted = Some(stream), + PollEvent::Message { token, payload, .. } => { + assert_eq!(Some(token), accepted); + assert_eq!(payload, b"ping"); + request_seen = true; + } + _ => {} + }); + if request_seen && !reply_seen { + server.write_or_enqueue_with(SendBehavior::Single(accepted.unwrap()), |b| { + b.extend_from_slice(b"pong"); + }); + request_seen = false; + } + client.poll_with(|e| { + if let PollEvent::Message { token, payload, .. } = e { + assert_eq!(token, tok); + assert_eq!(payload, b"pong"); + reply_seen = true; + } + }); + thread::sleep(Duration::from_micros(50)); + } +} + +#[test] +fn udp_broadcast_mixed_sizes_to_two_subscribers() { + let addr = free_addr(); + let mut server = udp(UdpConfig::lan()); + server.listen_at(addr).unwrap(); + let mut a = udp(UdpConfig::lan()); + let mut b = udp(UdpConfig::lan()); + a.connect(addr).unwrap(); + b.connect(addr).unwrap(); + let mut accepted = 0; + let deadline = Instant::now() + Duration::from_secs(5); + while accepted < 2 { + assert!(Instant::now() < deadline, "accepts"); + server.poll_with(|e| { + if let PollEvent::Accept { .. } = e { + accepted += 1; + } + }); + a.poll_with(|_| {}); + b.poll_with(|_| {}); + } + + // 1 byte, one datagram, one stride exactly, and a 2 MiB message. + let sizes = [1usize, 100, 1171, 1172, 5000, 2 * 1024 * 1024]; + let msgs: Vec> = + sizes.iter().enumerate().map(|(i, s)| make_msg(i as u32, *s)).collect(); + for m in &msgs { + server.write_or_enqueue_with(SendBehavior::Broadcast, |buf| buf.extend_from_slice(m)); + } + + let expected: Vec = msgs.iter().map(|m| checksum(m)).collect(); + let mut got_a = Vec::new(); + let mut got_b = Vec::new(); + let deadline = Instant::now() + Duration::from_secs(10); + while got_a.len() < msgs.len() || got_b.len() < msgs.len() { + assert!(Instant::now() < deadline, "broadcast delivery"); + server.poll_with(|_| {}); + a.poll_with(|e| { + if let PollEvent::Message { payload, .. } = e { + got_a.push((msg_id(payload), checksum(payload))); + } + }); + b.poll_with(|e| { + if let PollEvent::Message { payload, .. } = e { + got_b.push((msg_id(payload), checksum(payload))); + } + }); + } + for got in [got_a, got_b] { + assert_eq!(got.len(), msgs.len()); + for (id, sum) in got { + assert_eq!(sum, expected[id as usize], "message {id} corrupted"); + } + } +} + +/// Forwards datagrams between one client and the server, dropping every +/// `drop_every`-th datagram in each direction. +struct LossyRelay { + addr: SocketAddr, + stop: Arc, + dropped: Arc, + handle: Option>, +} + +impl LossyRelay { + fn start(server: SocketAddr, drop_every: usize) -> Self { + let socket = UdpSocket::bind((Ipv4Addr::LOCALHOST, 0)).unwrap(); + socket.set_read_timeout(Some(Duration::from_millis(5))).unwrap(); + let addr = socket.local_addr().unwrap(); + let stop = Arc::new(AtomicBool::new(false)); + let dropped = Arc::new(AtomicUsize::new(0)); + let (stop_c, dropped_c) = (stop.clone(), dropped.clone()); + let handle = thread::spawn(move || { + let mut buf = vec![0u8; 65_536]; + let mut client: Option = None; + let mut count = 0usize; + while !stop_c.load(Ordering::Relaxed) { + let Ok((n, from)) = socket.recv_from(&mut buf) else { continue }; + let to = if from == server { + let Some(c) = client else { continue }; + c + } else { + client = Some(from); + server + }; + count += 1; + if count.is_multiple_of(drop_every) { + dropped_c.fetch_add(1, Ordering::Relaxed); + continue; + } + let _ = socket.send_to(&buf[..n], to); + } + }); + Self { addr, stop, dropped, handle: Some(handle) } + } +} + +impl Drop for LossyRelay { + fn drop(&mut self) { + self.stop.store(true, Ordering::Relaxed); + self.handle.take().unwrap().join().unwrap(); + } +} + +#[test] +fn udp_delivers_everything_exactly_once_under_loss() { + const N: u32 = 400; + let server_addr = free_addr(); + let relay = LossyRelay::start(server_addr, 7); + + let mut server = udp(UdpConfig::lan()); + let mut client = udp(UdpConfig::lan()); + let (accepted, _) = connect_via(&mut server, &mut client, server_addr, relay.addr); + + // Both directions at once: server pushes to the client, client replies. + let msgs: Vec> = (0..N).map(|i| make_msg(i, 1 + (i as usize * 613) % 4000)).collect(); + for m in &msgs { + server.write_or_enqueue_with(SendBehavior::Single(accepted), |b| b.extend_from_slice(m)); + } + + let mut seen = vec![false; N as usize]; + let mut received = 0; + let mut echoed_back = 0; + let deadline = Instant::now() + Duration::from_secs(20); + while received < N || echoed_back < N { + assert!(Instant::now() < deadline, "loss recovery: {received} rx, {echoed_back} echoed"); + let mut echo = Vec::new(); + client.poll_with(|e| { + if let PollEvent::Message { payload, .. } = e { + let id = msg_id(payload) as usize; + assert_eq!(checksum(payload), checksum(&msgs[id]), "message {id} corrupted"); + assert!(!seen[id], "message {id} delivered twice"); + seen[id] = true; + received += 1; + echo.push(id as u32); + } + }); + for id in echo { + client.write_or_enqueue_with(SendBehavior::Broadcast, |b| { + b.extend_from_slice(&id.to_le_bytes()); + }); + } + server.poll_with(|e| { + if let PollEvent::Message { payload, .. } = e { + assert_eq!(payload.len(), 4); + echoed_back += 1; + } + }); + thread::sleep(Duration::from_micros(20)); + } + assert!(relay.dropped.load(Ordering::Relaxed) > 0, "relay dropped nothing"); +} + +#[test] +fn udp_client_disconnect_is_a_new_session() { + let addr = free_addr(); + let mut server = udp(UdpConfig::lan()); + let mut client = udp(UdpConfig::lan()); + let (first, tok) = connect_pair(&mut server, &mut client, addr); + + client.disconnect(tok); + assert_eq!(client.currently_disconnected().count(), 1); + client.write_or_enqueue_with(SendBehavior::Single(tok), |b| b.extend_from_slice(b"after")); + + let mut events = Vec::new(); + let mut payload_on = None; + let mut reconnected = false; + let deadline = Instant::now() + Duration::from_secs(5); + while payload_on.is_none() || !reconnected { + assert!(Instant::now() < deadline, "reconnect"); + server.poll_with(|e| match e { + PollEvent::Disconnect { token } => events.push(("disconnect", token)), + PollEvent::Accept { stream, .. } => events.push(("accept", stream)), + PollEvent::Message { token, payload, .. } => { + assert_eq!(payload, b"after"); + payload_on = Some(token); + } + PollEvent::Reconnect { .. } => unreachable!(), + }); + client.poll_with(|e| { + if let PollEvent::Reconnect { token } = e { + assert_eq!(token, tok); + reconnected = true; + } + }); + thread::sleep(Duration::from_micros(50)); + } + assert_eq!(events[0], ("disconnect", first)); + assert_eq!(events[1].0, "accept"); + assert_ne!(events[1].1, first); + assert_eq!(payload_on, Some(events[1].1), "backlog replayed on the new session"); + assert_eq!(client.currently_disconnected().count(), 0); +} + +#[test] +fn udp_drop_backlog_on_disconnect_discards_queued() { + let addr = free_addr(); + let mut server = udp(UdpConfig::lan()); + let mut client = udp(UdpConfig::lan()).with_drop_outbound_backlog_on_disconnect(true); + let (_, tok) = connect_pair(&mut server, &mut client, addr); + + client.disconnect(tok); + client.write_or_enqueue_with(SendBehavior::Single(tok), |b| b.extend_from_slice(b"lost")); + let mut reconnected = false; + let mut got = 0; + let deadline = Instant::now() + Duration::from_millis(500); + while Instant::now() < deadline { + server.poll_with(|e| { + if let PollEvent::Message { .. } = e { + got += 1; + } + }); + client.poll_with(|e| { + if let PollEvent::Reconnect { .. } = e { + reconnected = true; + } + }); + } + assert!(reconnected); + assert_eq!(got, 0); +} + +#[test] +fn udp_server_restart_reconnects_client() { + let addr = free_addr(); + let mut client = udp(UdpConfig::lan()).with_user_timeout(2_000); + let tok; + { + let mut server = udp(UdpConfig::lan()); + let (_, t) = connect_pair(&mut server, &mut client, addr); + tok = t; + } + // Old server gone. A new one on the same port must be rejoined without + // waiting for the 2s peer timeout: its reset acks trigger renegotiation. + let mut server = udp(UdpConfig::lan()); + server.listen_at(addr).unwrap(); + let start = Instant::now(); + let mut disconnected = false; + let mut reconnected = false; + let mut accepted = false; + while !(disconnected && reconnected && accepted) { + assert!(start.elapsed() < Duration::from_secs(5), "server restart recovery"); + client.poll_with(|e| match e { + PollEvent::Disconnect { token } => { + assert_eq!(token, tok); + disconnected = true; + } + PollEvent::Reconnect { token } => { + assert_eq!(token, tok); + reconnected = true; + } + _ => {} + }); + server.poll_with(|e| { + if let PollEvent::Accept { .. } = e { + accepted = true; + } + }); + thread::sleep(Duration::from_micros(50)); + } + assert!(start.elapsed() < Duration::from_millis(1500), "took the slow timeout path"); +} + +#[test] +fn udp_peer_timeout_disconnects_silent_client() { + let addr = free_addr(); + let mut server = udp(UdpConfig::lan()).with_user_timeout(300); + let mut client = udp(UdpConfig::lan()); + let (accepted, _) = connect_pair(&mut server, &mut client, addr); + drop(client); + + let mut disconnected = None; + let start = Instant::now(); + while disconnected.is_none() { + assert!(start.elapsed() < Duration::from_secs(5), "peer timeout"); + server.poll_with(|e| { + if let PollEvent::Disconnect { token } = e { + disconnected = Some(token); + } + }); + thread::sleep(Duration::from_millis(1)); + } + assert_eq!(disconnected, Some(accepted)); + assert!(start.elapsed() >= Duration::from_millis(250)); +} + +#[test] +fn udp_backlog_limit_disconnects_non_consuming_peer() { + let addr = free_addr(); + let mut server = + udp(UdpConfig::lan()).with_max_backlog(8, flux_timing::Duration::from_millis(50)); + let mut client = udp(UdpConfig::lan()); + let (accepted, _) = connect_pair(&mut server, &mut client, addr); + // Client stops polling: nothing gets acked. + + let mut disconnected = false; + let deadline = Instant::now() + Duration::from_secs(5); + while !disconnected { + assert!(Instant::now() < deadline, "backlog disconnect"); + server.write_or_enqueue_with(SendBehavior::Single(accepted), |b| { + b.extend_from_slice(&[0; 1000]); + }); + server.poll_with(|e| { + if let PollEvent::Disconnect { token } = e { + assert_eq!(token, accepted); + disconnected = true; + } + }); + thread::sleep(Duration::from_millis(1)); + } + let _ = &client; +} + +#[test] +fn udp_ignores_junk_datagrams() { + let addr = free_addr(); + let mut server = udp(UdpConfig::lan()); + let mut client = udp(UdpConfig::lan()); + let (accepted, _) = connect_pair(&mut server, &mut client, addr); + let junk = UdpSocket::bind((Ipv4Addr::LOCALHOST, 0)).unwrap(); + junk.send_to(b"not flux", addr).unwrap(); + junk.send_to(&[0xFF; 1200], addr).unwrap(); + junk.send_to(&[0; 2000], addr).unwrap(); + server.write_or_enqueue_with(SendBehavior::Single(accepted), |b| b.extend_from_slice(b"ok")); + let mut got = false; + let deadline = Instant::now() + Duration::from_secs(5); + while !got { + assert!(Instant::now() < deadline, "junk tolerance"); + server.poll_with(|e| assert!(!matches!(e, PollEvent::Accept { .. }))); + client.poll_with(|e| { + if let PollEvent::Message { payload, .. } = e { + assert_eq!(payload, b"ok"); + got = true; + } + }); + } +} + +/// A peer dropped mid-broadcast must not take the shared payload with it. +#[test] +fn udp_broadcast_survives_dropping_a_peer_mid_way() { + let addr = free_addr(); + let mut server = udp(UdpConfig::lan()).with_max_backlog(4, flux_timing::Duration::ZERO); + server.listen_at(addr).unwrap(); + let mut stalled = udp(UdpConfig::lan()); + let mut live = udp(UdpConfig::lan()); + stalled.connect(addr).unwrap(); + live.connect(addr).unwrap(); + let mut accepted = 0; + let deadline = Instant::now() + Duration::from_secs(5); + while accepted < 2 || live.currently_disconnected().count() != 0 { + assert!(Instant::now() < deadline, "accepts"); + server.poll_with(|e| accepted += usize::from(matches!(e, PollEvent::Accept { .. }))); + stalled.poll_with(|_| {}); + live.poll_with(|_| {}); + } + // `stalled` stops polling: its unacked count grows past the backlog limit + // while `live` keeps consuming. + let msgs: Vec> = (0..12).map(|i| make_msg(i, 3000)).collect(); + let mut got = Vec::new(); + let mut dropped = 0; + let deadline = Instant::now() + Duration::from_secs(5); + for m in &msgs { + server.write_or_enqueue_with(SendBehavior::Broadcast, |b| b.extend_from_slice(m)); + let until = Instant::now() + Duration::from_millis(20); + while Instant::now() < until { + server.poll_with(|e| dropped += usize::from(matches!(e, PollEvent::Disconnect { .. }))); + live.poll_with(|e| { + if let PollEvent::Message { payload, .. } = e { + got.push((msg_id(payload), checksum(payload))); + } + }); + } + } + while got.len() < msgs.len() { + assert!(Instant::now() < deadline, "live peer delivery: got {}", got.len()); + server.poll_with(|_| {}); + live.poll_with(|e| { + if let PollEvent::Message { payload, .. } = e { + got.push((msg_id(payload), checksum(payload))); + } + }); + } + assert_eq!(dropped, 1, "the stalled peer is dropped exactly once"); + for (id, sum) in got { + assert_eq!(sum, checksum(&msgs[id as usize]), "message {id} corrupted"); + } +} + +/// Messages in flight when the session is cut arrive whole under the new +/// session, however much of them the receiver had already acked. +#[test] +fn udp_reconnect_replays_messages_queued_before_disconnect() { + let addr = free_addr(); + let mut server = udp(UdpConfig::lan()); + let mut client = udp(UdpConfig::lan()); + let (_, tok) = connect_pair(&mut server, &mut client, addr); + let big = make_msg(7, 500_000); + client.write_or_enqueue_with(SendBehavior::Single(tok), |b| b.extend_from_slice(&big)); + client.write_or_enqueue_with(SendBehavior::Single(tok), |b| b.extend_from_slice(b"small")); + // Let some fragments through and get acked, then cut the session. How + // much lands first depends on the receive buffer; either message may. + let mut got = Vec::new(); + for _ in 0..3 { + client.poll_with(|_| {}); + server.poll_with(|e| { + if let PollEvent::Message { payload, .. } = e { + got.push(payload.to_vec()); + } + }); + } + client.disconnect(tok); + + // Everything not cumulatively acked is replayed, so the small message may + // arrive twice: delivery across a reconnect is at-least-once. + let deadline = Instant::now() + Duration::from_secs(5); + while !got.iter().any(|m| m.len() == big.len()) { + assert!(Instant::now() < deadline, "replay"); + server.poll_with(|e| { + if let PollEvent::Message { payload, .. } = e { + got.push(payload.to_vec()); + } + }); + client.poll_with(|_| {}); + } + assert!(got.iter().any(|m| m == b"small")); + assert!(got.iter().any(|m| checksum(m) == checksum(&big)), "big message replayed intact"); +} + +/// A server that drops an accepted peer makes the client renegotiate. +#[test] +fn udp_server_disconnect_reconnects_client() { + let addr = free_addr(); + let mut server = udp(UdpConfig::lan()).with_user_timeout(5_000); + let mut client = udp(UdpConfig::lan()).with_user_timeout(5_000); + let (accepted, tok) = connect_pair(&mut server, &mut client, addr); + server.disconnect(accepted); + // Client traffic hits a listener with no session for it and gets reset. + client.write_or_enqueue_with(SendBehavior::Single(tok), |b| b.extend_from_slice(b"x")); + + let start = Instant::now(); + let (mut disconnected, mut reconnected, mut reaccepted) = (false, false, false); + while !(disconnected && reconnected && reaccepted) { + assert!(start.elapsed() < Duration::from_secs(5), "server-side disconnect recovery"); + client.poll_with(|e| match e { + PollEvent::Disconnect { token } => { + assert_eq!(token, tok); + disconnected = true; + } + PollEvent::Reconnect { token } => { + assert_eq!(token, tok); + reconnected = true; + } + _ => {} + }); + server.poll_with(|e| reaccepted |= matches!(e, PollEvent::Accept { .. })); + thread::sleep(Duration::from_micros(50)); + } + assert!(start.elapsed() < Duration::from_secs(1), "did not wait for the peer timeout"); +} + +/// A message that cannot fit the send window drops the peer instead of +/// vanishing silently. +#[test] +fn udp_window_exhaustion_disconnects_instead_of_dropping() { + let addr = free_addr(); + let config = UdpConfig { send_window: 64, max_message_size: 64 * 1171, ..UdpConfig::lan() }; + let mut server = udp(config); + let mut client = udp(config); + let (accepted, _) = connect_pair(&mut server, &mut client, addr); + // Client never polls: nothing is acked, the window fills, the 65th + // single-fragment message cannot be queued. + let mut disconnected = None; + for _ in 0..70 { + server.write_or_enqueue_with(SendBehavior::Single(accepted), |b| b.extend_from_slice(b"m")); + server.poll_with(|e| { + if let PollEvent::Disconnect { token } = e { + disconnected = Some(token); + } + }); + } + assert_eq!(disconnected, Some(accepted)); + let _ = &client; +} diff --git a/crates/flux-network/tests/tcp_dcache.rs b/crates/flux-network/tests/tcp_dcache.rs index a843e1c..d4f6758 100644 --- a/crates/flux-network/tests/tcp_dcache.rs +++ b/crates/flux-network/tests/tcp_dcache.rs @@ -98,6 +98,15 @@ fn dcache_multi_stream_udp() { dcache_multi_stream(Transport::Udp(UdpConfig::lan())); } +#[cfg(target_os = "linux")] +#[test] +fn dcache_multi_stream_udp_uring() { + dcache_multi_stream(Transport::Udp(UdpConfig { + io: flux_network::udp::UdpIo::Uring(flux_network::udp::UringConfig::default()), + ..UdpConfig::lan() + })); +} + /// Two streams into the same dcache-backed spine queue. /// Verifies dcache bytes match the queue message (same shmem region). #[allow(clippy::significant_drop_tightening)] diff --git a/crates/flux-network/tests/udp_connector.rs b/crates/flux-network/tests/udp_connector.rs index 59d0ff1..c4b30e7 100644 --- a/crates/flux-network/tests/udp_connector.rs +++ b/crates/flux-network/tests/udp_connector.rs @@ -1,611 +1,3 @@ -use std::{ - net::{Ipv4Addr, SocketAddr, UdpSocket}, - sync::{ - Arc, - atomic::{AtomicBool, AtomicUsize, Ordering}, - }, - thread, - time::{Duration, Instant}, -}; +const IO: flux_network::udp::UdpIo = flux_network::udp::UdpIo::Syscall; -use flux_network::{NetworkDriver, PollEvent, SendBehavior, Transport, UdpConfig}; -use mio::Token; - -fn free_addr() -> SocketAddr { - UdpSocket::bind((Ipv4Addr::LOCALHOST, 0)).unwrap().local_addr().unwrap() -} - -fn udp(config: UdpConfig) -> NetworkDriver { - NetworkDriver::default().with_transport(Transport::Udp(config)) -} - -/// Handshake between a fresh listener and client, returning the accepted and -/// the outbound token. `dial` differs from `addr` when a relay sits between. -fn connect_via( - server: &mut NetworkDriver, - client: &mut NetworkDriver, - addr: SocketAddr, - dial: SocketAddr, -) -> (Token, Token) { - server.listen_at(addr).unwrap(); - let client_token = client.connect(dial).unwrap(); - let mut accepted = None; - let deadline = Instant::now() + Duration::from_secs(5); - while accepted.is_none() { - assert!(Instant::now() < deadline, "no accept"); - server.poll_with(|e| { - if let PollEvent::Accept { stream, .. } = e { - accepted = Some(stream); - } - }); - client.poll_with(|_| {}); - thread::sleep(Duration::from_micros(50)); - } - while client.currently_disconnected().count() != 0 { - assert!(Instant::now() < deadline, "handshake"); - server.poll_with(|_| {}); - client.poll_with(|_| {}); - thread::sleep(Duration::from_micros(50)); - } - // A hello retry may still be in flight if the ack took longer than one - // RTO. Let it land now, while the peer it belongs to still exists. - for _ in 0..5 { - server.poll_with(|_| {}); - client.poll_with(|_| {}); - } - (accepted.unwrap(), client_token) -} - -fn connect_pair( - server: &mut NetworkDriver, - client: &mut NetworkDriver, - addr: SocketAddr, -) -> (Token, Token) { - connect_via(server, client, addr, addr) -} - -fn checksum(bytes: &[u8]) -> u64 { - bytes - .iter() - .fold(0xcbf2_9ce4_8422_2325_u64, |h, b| (h ^ u64::from(*b)).wrapping_mul(0x100_0000_01b3)) -} - -/// Test message: 4-byte id, then `len` bytes derived from the id. -fn make_msg(id: u32, len: usize) -> Vec { - let mut v = Vec::with_capacity(4 + len); - v.extend_from_slice(&id.to_le_bytes()); - v.extend((0..len).map(|i| (id as usize).wrapping_mul(31).wrapping_add(i * 7) as u8)); - v -} - -fn msg_id(payload: &[u8]) -> u32 { - u32::from_le_bytes(payload[..4].try_into().unwrap()) -} - -#[test] -fn udp_roundtrip_before_handshake_completes() { - let addr = free_addr(); - let mut server = udp(UdpConfig::lan()); - server.listen_at(addr).unwrap(); - let mut client = udp(UdpConfig::lan()); - let tok = client.connect(addr).unwrap(); - // Queued before the hello ack arrives; must go out once it does. - client.write_or_enqueue_with(SendBehavior::Single(tok), |b| b.extend_from_slice(b"ping")); - - let mut accepted = None; - let mut request_seen = false; - let mut reply_seen = false; - let deadline = Instant::now() + Duration::from_secs(5); - while !reply_seen { - assert!(Instant::now() < deadline, "roundtrip timed out"); - server.poll_with(|e| match e { - PollEvent::Accept { stream, .. } => accepted = Some(stream), - PollEvent::Message { token, payload, .. } => { - assert_eq!(Some(token), accepted); - assert_eq!(payload, b"ping"); - request_seen = true; - } - _ => {} - }); - if request_seen && !reply_seen { - server.write_or_enqueue_with(SendBehavior::Single(accepted.unwrap()), |b| { - b.extend_from_slice(b"pong"); - }); - request_seen = false; - } - client.poll_with(|e| { - if let PollEvent::Message { token, payload, .. } = e { - assert_eq!(token, tok); - assert_eq!(payload, b"pong"); - reply_seen = true; - } - }); - thread::sleep(Duration::from_micros(50)); - } -} - -#[test] -fn udp_broadcast_mixed_sizes_to_two_subscribers() { - let addr = free_addr(); - let mut server = udp(UdpConfig::lan()); - server.listen_at(addr).unwrap(); - let mut a = udp(UdpConfig::lan()); - let mut b = udp(UdpConfig::lan()); - a.connect(addr).unwrap(); - b.connect(addr).unwrap(); - let mut accepted = 0; - let deadline = Instant::now() + Duration::from_secs(5); - while accepted < 2 { - assert!(Instant::now() < deadline, "accepts"); - server.poll_with(|e| { - if let PollEvent::Accept { .. } = e { - accepted += 1; - } - }); - a.poll_with(|_| {}); - b.poll_with(|_| {}); - } - - // 1 byte, one datagram, one stride exactly, and a 2 MiB message. - let sizes = [1usize, 100, 1171, 1172, 5000, 2 * 1024 * 1024]; - let msgs: Vec> = - sizes.iter().enumerate().map(|(i, s)| make_msg(i as u32, *s)).collect(); - for m in &msgs { - server.write_or_enqueue_with(SendBehavior::Broadcast, |buf| buf.extend_from_slice(m)); - } - - let expected: Vec = msgs.iter().map(|m| checksum(m)).collect(); - let mut got_a = Vec::new(); - let mut got_b = Vec::new(); - let deadline = Instant::now() + Duration::from_secs(10); - while got_a.len() < msgs.len() || got_b.len() < msgs.len() { - assert!(Instant::now() < deadline, "broadcast delivery"); - server.poll_with(|_| {}); - a.poll_with(|e| { - if let PollEvent::Message { payload, .. } = e { - got_a.push((msg_id(payload), checksum(payload))); - } - }); - b.poll_with(|e| { - if let PollEvent::Message { payload, .. } = e { - got_b.push((msg_id(payload), checksum(payload))); - } - }); - } - for got in [got_a, got_b] { - assert_eq!(got.len(), msgs.len()); - for (id, sum) in got { - assert_eq!(sum, expected[id as usize], "message {id} corrupted"); - } - } -} - -/// Forwards datagrams between one client and the server, dropping every -/// `drop_every`-th datagram in each direction. -struct LossyRelay { - addr: SocketAddr, - stop: Arc, - dropped: Arc, - handle: Option>, -} - -impl LossyRelay { - fn start(server: SocketAddr, drop_every: usize) -> Self { - let socket = UdpSocket::bind((Ipv4Addr::LOCALHOST, 0)).unwrap(); - socket.set_read_timeout(Some(Duration::from_millis(5))).unwrap(); - let addr = socket.local_addr().unwrap(); - let stop = Arc::new(AtomicBool::new(false)); - let dropped = Arc::new(AtomicUsize::new(0)); - let (stop_c, dropped_c) = (stop.clone(), dropped.clone()); - let handle = thread::spawn(move || { - let mut buf = vec![0u8; 65_536]; - let mut client: Option = None; - let mut count = 0usize; - while !stop_c.load(Ordering::Relaxed) { - let Ok((n, from)) = socket.recv_from(&mut buf) else { continue }; - let to = if from == server { - let Some(c) = client else { continue }; - c - } else { - client = Some(from); - server - }; - count += 1; - if count.is_multiple_of(drop_every) { - dropped_c.fetch_add(1, Ordering::Relaxed); - continue; - } - let _ = socket.send_to(&buf[..n], to); - } - }); - Self { addr, stop, dropped, handle: Some(handle) } - } -} - -impl Drop for LossyRelay { - fn drop(&mut self) { - self.stop.store(true, Ordering::Relaxed); - self.handle.take().unwrap().join().unwrap(); - } -} - -#[test] -fn udp_delivers_everything_exactly_once_under_loss() { - const N: u32 = 400; - let server_addr = free_addr(); - let relay = LossyRelay::start(server_addr, 7); - - let mut server = udp(UdpConfig::lan()); - let mut client = udp(UdpConfig::lan()); - let (accepted, _) = connect_via(&mut server, &mut client, server_addr, relay.addr); - - // Both directions at once: server pushes to the client, client replies. - let msgs: Vec> = (0..N).map(|i| make_msg(i, 1 + (i as usize * 613) % 4000)).collect(); - for m in &msgs { - server.write_or_enqueue_with(SendBehavior::Single(accepted), |b| b.extend_from_slice(m)); - } - - let mut seen = vec![false; N as usize]; - let mut received = 0; - let mut echoed_back = 0; - let deadline = Instant::now() + Duration::from_secs(20); - while received < N || echoed_back < N { - assert!(Instant::now() < deadline, "loss recovery: {received} rx, {echoed_back} echoed"); - let mut echo = Vec::new(); - client.poll_with(|e| { - if let PollEvent::Message { payload, .. } = e { - let id = msg_id(payload) as usize; - assert_eq!(checksum(payload), checksum(&msgs[id]), "message {id} corrupted"); - assert!(!seen[id], "message {id} delivered twice"); - seen[id] = true; - received += 1; - echo.push(id as u32); - } - }); - for id in echo { - client.write_or_enqueue_with(SendBehavior::Broadcast, |b| { - b.extend_from_slice(&id.to_le_bytes()); - }); - } - server.poll_with(|e| { - if let PollEvent::Message { payload, .. } = e { - assert_eq!(payload.len(), 4); - echoed_back += 1; - } - }); - thread::sleep(Duration::from_micros(20)); - } - assert!(relay.dropped.load(Ordering::Relaxed) > 0, "relay dropped nothing"); -} - -#[test] -fn udp_client_disconnect_is_a_new_session() { - let addr = free_addr(); - let mut server = udp(UdpConfig::lan()); - let mut client = udp(UdpConfig::lan()); - let (first, tok) = connect_pair(&mut server, &mut client, addr); - - client.disconnect(tok); - assert_eq!(client.currently_disconnected().count(), 1); - client.write_or_enqueue_with(SendBehavior::Single(tok), |b| b.extend_from_slice(b"after")); - - let mut events = Vec::new(); - let mut payload_on = None; - let mut reconnected = false; - let deadline = Instant::now() + Duration::from_secs(5); - while payload_on.is_none() || !reconnected { - assert!(Instant::now() < deadline, "reconnect"); - server.poll_with(|e| match e { - PollEvent::Disconnect { token } => events.push(("disconnect", token)), - PollEvent::Accept { stream, .. } => events.push(("accept", stream)), - PollEvent::Message { token, payload, .. } => { - assert_eq!(payload, b"after"); - payload_on = Some(token); - } - PollEvent::Reconnect { .. } => unreachable!(), - }); - client.poll_with(|e| { - if let PollEvent::Reconnect { token } = e { - assert_eq!(token, tok); - reconnected = true; - } - }); - thread::sleep(Duration::from_micros(50)); - } - assert_eq!(events[0], ("disconnect", first)); - assert_eq!(events[1].0, "accept"); - assert_ne!(events[1].1, first); - assert_eq!(payload_on, Some(events[1].1), "backlog replayed on the new session"); - assert_eq!(client.currently_disconnected().count(), 0); -} - -#[test] -fn udp_drop_backlog_on_disconnect_discards_queued() { - let addr = free_addr(); - let mut server = udp(UdpConfig::lan()); - let mut client = udp(UdpConfig::lan()).with_drop_outbound_backlog_on_disconnect(true); - let (_, tok) = connect_pair(&mut server, &mut client, addr); - - client.disconnect(tok); - client.write_or_enqueue_with(SendBehavior::Single(tok), |b| b.extend_from_slice(b"lost")); - let mut reconnected = false; - let mut got = 0; - let deadline = Instant::now() + Duration::from_millis(500); - while Instant::now() < deadline { - server.poll_with(|e| { - if let PollEvent::Message { .. } = e { - got += 1; - } - }); - client.poll_with(|e| { - if let PollEvent::Reconnect { .. } = e { - reconnected = true; - } - }); - } - assert!(reconnected); - assert_eq!(got, 0); -} - -#[test] -fn udp_server_restart_reconnects_client() { - let addr = free_addr(); - let mut client = udp(UdpConfig::lan()).with_user_timeout(2_000); - let tok; - { - let mut server = udp(UdpConfig::lan()); - let (_, t) = connect_pair(&mut server, &mut client, addr); - tok = t; - } - // Old server gone. A new one on the same port must be rejoined without - // waiting for the 2s peer timeout: its reset acks trigger renegotiation. - let mut server = udp(UdpConfig::lan()); - server.listen_at(addr).unwrap(); - let start = Instant::now(); - let mut disconnected = false; - let mut reconnected = false; - let mut accepted = false; - while !(disconnected && reconnected && accepted) { - assert!(start.elapsed() < Duration::from_secs(5), "server restart recovery"); - client.poll_with(|e| match e { - PollEvent::Disconnect { token } => { - assert_eq!(token, tok); - disconnected = true; - } - PollEvent::Reconnect { token } => { - assert_eq!(token, tok); - reconnected = true; - } - _ => {} - }); - server.poll_with(|e| { - if let PollEvent::Accept { .. } = e { - accepted = true; - } - }); - thread::sleep(Duration::from_micros(50)); - } - assert!(start.elapsed() < Duration::from_millis(1500), "took the slow timeout path"); -} - -#[test] -fn udp_peer_timeout_disconnects_silent_client() { - let addr = free_addr(); - let mut server = udp(UdpConfig::lan()).with_user_timeout(300); - let mut client = udp(UdpConfig::lan()); - let (accepted, _) = connect_pair(&mut server, &mut client, addr); - drop(client); - - let mut disconnected = None; - let start = Instant::now(); - while disconnected.is_none() { - assert!(start.elapsed() < Duration::from_secs(5), "peer timeout"); - server.poll_with(|e| { - if let PollEvent::Disconnect { token } = e { - disconnected = Some(token); - } - }); - thread::sleep(Duration::from_millis(1)); - } - assert_eq!(disconnected, Some(accepted)); - assert!(start.elapsed() >= Duration::from_millis(250)); -} - -#[test] -fn udp_backlog_limit_disconnects_non_consuming_peer() { - let addr = free_addr(); - let mut server = - udp(UdpConfig::lan()).with_max_backlog(8, flux_timing::Duration::from_millis(50)); - let mut client = udp(UdpConfig::lan()); - let (accepted, _) = connect_pair(&mut server, &mut client, addr); - // Client stops polling: nothing gets acked. - - let mut disconnected = false; - let deadline = Instant::now() + Duration::from_secs(5); - while !disconnected { - assert!(Instant::now() < deadline, "backlog disconnect"); - server.write_or_enqueue_with(SendBehavior::Single(accepted), |b| { - b.extend_from_slice(&[0; 1000]); - }); - server.poll_with(|e| { - if let PollEvent::Disconnect { token } = e { - assert_eq!(token, accepted); - disconnected = true; - } - }); - thread::sleep(Duration::from_millis(1)); - } - let _ = &client; -} - -#[test] -fn udp_ignores_junk_datagrams() { - let addr = free_addr(); - let mut server = udp(UdpConfig::lan()); - let mut client = udp(UdpConfig::lan()); - let (accepted, _) = connect_pair(&mut server, &mut client, addr); - let junk = UdpSocket::bind((Ipv4Addr::LOCALHOST, 0)).unwrap(); - junk.send_to(b"not flux", addr).unwrap(); - junk.send_to(&[0xFF; 1200], addr).unwrap(); - junk.send_to(&[0; 2000], addr).unwrap(); - server.write_or_enqueue_with(SendBehavior::Single(accepted), |b| b.extend_from_slice(b"ok")); - let mut got = false; - let deadline = Instant::now() + Duration::from_secs(5); - while !got { - assert!(Instant::now() < deadline, "junk tolerance"); - server.poll_with(|e| assert!(!matches!(e, PollEvent::Accept { .. }))); - client.poll_with(|e| { - if let PollEvent::Message { payload, .. } = e { - assert_eq!(payload, b"ok"); - got = true; - } - }); - } -} - -/// A peer dropped mid-broadcast must not take the shared payload with it. -#[test] -fn udp_broadcast_survives_dropping_a_peer_mid_way() { - let addr = free_addr(); - let mut server = udp(UdpConfig::lan()).with_max_backlog(4, flux_timing::Duration::ZERO); - server.listen_at(addr).unwrap(); - let mut stalled = udp(UdpConfig::lan()); - let mut live = udp(UdpConfig::lan()); - stalled.connect(addr).unwrap(); - live.connect(addr).unwrap(); - let mut accepted = 0; - let deadline = Instant::now() + Duration::from_secs(5); - while accepted < 2 || live.currently_disconnected().count() != 0 { - assert!(Instant::now() < deadline, "accepts"); - server.poll_with(|e| accepted += usize::from(matches!(e, PollEvent::Accept { .. }))); - stalled.poll_with(|_| {}); - live.poll_with(|_| {}); - } - // `stalled` stops polling: its unacked count grows past the backlog limit - // while `live` keeps consuming. - let msgs: Vec> = (0..12).map(|i| make_msg(i, 3000)).collect(); - let mut got = Vec::new(); - let mut dropped = 0; - let deadline = Instant::now() + Duration::from_secs(5); - for m in &msgs { - server.write_or_enqueue_with(SendBehavior::Broadcast, |b| b.extend_from_slice(m)); - let until = Instant::now() + Duration::from_millis(20); - while Instant::now() < until { - server.poll_with(|e| dropped += usize::from(matches!(e, PollEvent::Disconnect { .. }))); - live.poll_with(|e| { - if let PollEvent::Message { payload, .. } = e { - got.push((msg_id(payload), checksum(payload))); - } - }); - } - } - while got.len() < msgs.len() { - assert!(Instant::now() < deadline, "live peer delivery: got {}", got.len()); - server.poll_with(|_| {}); - live.poll_with(|e| { - if let PollEvent::Message { payload, .. } = e { - got.push((msg_id(payload), checksum(payload))); - } - }); - } - assert_eq!(dropped, 1, "the stalled peer is dropped exactly once"); - for (id, sum) in got { - assert_eq!(sum, checksum(&msgs[id as usize]), "message {id} corrupted"); - } -} - -/// Messages in flight when the session is cut arrive whole under the new -/// session, however much of them the receiver had already acked. -#[test] -fn udp_reconnect_replays_messages_queued_before_disconnect() { - let addr = free_addr(); - let mut server = udp(UdpConfig::lan()); - let mut client = udp(UdpConfig::lan()); - let (_, tok) = connect_pair(&mut server, &mut client, addr); - let big = make_msg(7, 500_000); - client.write_or_enqueue_with(SendBehavior::Single(tok), |b| b.extend_from_slice(&big)); - client.write_or_enqueue_with(SendBehavior::Single(tok), |b| b.extend_from_slice(b"small")); - // Let some fragments through and get acked, then cut the session. How - // much lands first depends on the receive buffer; either message may. - let mut got = Vec::new(); - for _ in 0..3 { - client.poll_with(|_| {}); - server.poll_with(|e| { - if let PollEvent::Message { payload, .. } = e { - got.push(payload.to_vec()); - } - }); - } - client.disconnect(tok); - - // Everything not cumulatively acked is replayed, so the small message may - // arrive twice: delivery across a reconnect is at-least-once. - let deadline = Instant::now() + Duration::from_secs(5); - while !got.iter().any(|m| m.len() == big.len()) { - assert!(Instant::now() < deadline, "replay"); - server.poll_with(|e| { - if let PollEvent::Message { payload, .. } = e { - got.push(payload.to_vec()); - } - }); - client.poll_with(|_| {}); - } - assert!(got.iter().any(|m| m == b"small")); - assert!(got.iter().any(|m| checksum(m) == checksum(&big)), "big message replayed intact"); -} - -/// A server that drops an accepted peer makes the client renegotiate. -#[test] -fn udp_server_disconnect_reconnects_client() { - let addr = free_addr(); - let mut server = udp(UdpConfig::lan()).with_user_timeout(5_000); - let mut client = udp(UdpConfig::lan()).with_user_timeout(5_000); - let (accepted, tok) = connect_pair(&mut server, &mut client, addr); - server.disconnect(accepted); - // Client traffic hits a listener with no session for it and gets reset. - client.write_or_enqueue_with(SendBehavior::Single(tok), |b| b.extend_from_slice(b"x")); - - let start = Instant::now(); - let (mut disconnected, mut reconnected, mut reaccepted) = (false, false, false); - while !(disconnected && reconnected && reaccepted) { - assert!(start.elapsed() < Duration::from_secs(5), "server-side disconnect recovery"); - client.poll_with(|e| match e { - PollEvent::Disconnect { token } => { - assert_eq!(token, tok); - disconnected = true; - } - PollEvent::Reconnect { token } => { - assert_eq!(token, tok); - reconnected = true; - } - _ => {} - }); - server.poll_with(|e| reaccepted |= matches!(e, PollEvent::Accept { .. })); - thread::sleep(Duration::from_micros(50)); - } - assert!(start.elapsed() < Duration::from_secs(1), "did not wait for the peer timeout"); -} - -/// A message that cannot fit the send window drops the peer instead of -/// vanishing silently. -#[test] -fn udp_window_exhaustion_disconnects_instead_of_dropping() { - let addr = free_addr(); - let config = UdpConfig { send_window: 64, max_message_size: 64 * 1171, ..UdpConfig::lan() }; - let mut server = udp(config); - let mut client = udp(config); - let (accepted, _) = connect_pair(&mut server, &mut client, addr); - // Client never polls: nothing is acked, the window fills, the 65th - // single-fragment message cannot be queued. - let mut disconnected = None; - for _ in 0..70 { - server.write_or_enqueue_with(SendBehavior::Single(accepted), |b| b.extend_from_slice(b"m")); - server.poll_with(|e| { - if let PollEvent::Disconnect { token } = e { - disconnected = Some(token); - } - }); - } - assert_eq!(disconnected, Some(accepted)); - let _ = &client; -} +include!("support/udp_connector.rs"); diff --git a/crates/flux-network/tests/udp_uring.rs b/crates/flux-network/tests/udp_uring.rs new file mode 100644 index 0000000..a991a4a --- /dev/null +++ b/crates/flux-network/tests/udp_uring.rs @@ -0,0 +1,107 @@ +#![cfg(target_os = "linux")] + +const IO: flux_network::udp::UdpIo = + flux_network::udp::UdpIo::Uring(flux_network::udp::UringConfig { + send_entries: 64, + recv_entries: 32, + }); + +include!("support/udp_connector.rs"); + +#[test] +fn mixed_backends_interoperate() { + for server_uring in [false, true] { + let config = |uring| UdpConfig { + io: if uring { IO } else { flux_network::udp::UdpIo::Syscall }, + ..UdpConfig::lan() + }; + let mut server = + NetworkDriver::default().with_transport(Transport::Udp(config(server_uring))); + let mut client = + NetworkDriver::default().with_transport(Transport::Udp(config(!server_uring))); + let addr = free_addr(); + let (accepted, outbound) = connect_pair(&mut server, &mut client, addr); + let payload = make_msg(17, 256 * 1024); + server.write_or_enqueue_with(SendBehavior::Single(accepted), |buf| { + buf.extend_from_slice(&payload); + }); + client.write_or_enqueue_with(SendBehavior::Single(outbound), |buf| { + buf.extend_from_slice(&payload); + }); + let deadline = Instant::now() + Duration::from_secs(5); + let (mut server_got, mut client_got) = (false, false); + while !server_got || !client_got { + server.poll_with(|e| { + if let PollEvent::Message { payload: got, .. } = e { + assert_eq!(got, payload); + assert!(!server_got); + server_got = true; + } + }); + client.poll_with(|e| { + if let PollEvent::Message { payload: got, .. } = e { + assert_eq!(got, payload); + assert!(!client_got); + client_got = true; + } + }); + assert!(Instant::now() < deadline); + } + } +} + +#[test] +fn sustained_large_messages_keep_acknowledgements_current() { + const COUNT: usize = 64; + let addr = free_addr(); + let mut server = udp(UdpConfig::lan()).with_socket_buf_size(16 * 1024 * 1024); + let mut client = udp(UdpConfig::lan()).with_socket_buf_size(16 * 1024 * 1024); + let (accepted, _) = connect_pair(&mut server, &mut client, addr); + let got = Arc::new(AtomicUsize::new(0)); + let progress = got.clone(); + let deadline = Instant::now() + Duration::from_secs(10); + let receiver = thread::spawn(move || { + let mut seen = [false; COUNT]; + while progress.load(Ordering::Relaxed) < COUNT { + client.poll_with(|event| { + if let PollEvent::Message { payload, .. } = event { + assert_eq!(payload.len(), 2 * 1024 * 1024); + let id = msg_id(payload) as usize; + assert!(!seen[id]); + assert!(payload[4..].iter().all(|b| *b == 0x5a)); + seen[id] = true; + progress.fetch_add(1, Ordering::Relaxed); + } + }); + assert!( + Instant::now() < deadline, + "large-message receiver stalled at {}", + progress.load(Ordering::Relaxed) + ); + } + }); + let mut sent = 0; + let mut payload = vec![0x5a; 2 * 1024 * 1024]; + while got.load(Ordering::Relaxed) < COUNT { + if sent < COUNT && sent - got.load(Ordering::Relaxed) < 4 { + payload[..4].copy_from_slice(&(sent as u32).to_le_bytes()); + server.write_or_enqueue_with(SendBehavior::Single(accepted), |buf| { + buf.extend_from_slice(&payload); + }); + sent += 1; + } + server.poll_with(|event| { + assert!( + !matches!(event, PollEvent::Disconnect { .. }), + "sender disconnected at {sent} sent, {} received", + got.load(Ordering::Relaxed) + ); + }); + assert!( + Instant::now() < deadline, + "large-message sender stalled at {sent} sent, {} received", + got.load(Ordering::Relaxed) + ); + } + receiver.join().unwrap(); +} From afcad76ec0386eb31ab09f76a4b3e965ce6fa672 Mon Sep 17 00:00:00 2001 From: gd-0 Date: Tue, 8 Sep 2026 20:13:33 +0100 Subject: [PATCH 2/6] fix(network): retain blocked ACKs and bound io_uring receive passes --- crates/flux-network/UDP_URING.md | 47 +++++++++++-------- .../benches/results/udp-uring/branch-1.txt | 40 ++++++++-------- .../benches/results/udp-uring/branch-2.txt | 40 ++++++++-------- .../benches/results/udp-uring/branch-3.txt | 40 ++++++++-------- .../benches/results/udp-uring/branch-4.txt | 40 ++++++++-------- .../benches/results/udp-uring/branch-5.txt | 40 ++++++++-------- .../benches/results/udp-uring/main-1.txt | 20 ++++---- .../benches/results/udp-uring/main-2.txt | 20 ++++---- .../benches/results/udp-uring/main-3.txt | 20 ++++---- .../benches/results/udp-uring/main-4.txt | 20 ++++---- .../benches/results/udp-uring/main-5.txt | 20 ++++---- crates/flux-network/src/udp/connector.rs | 5 +- crates/flux-network/src/udp/peer.rs | 46 ++++++++++++++++-- crates/flux-network/src/udp/sys.rs | 3 +- crates/flux-network/tests/udp_uring.rs | 39 +++++++++++++++ 15 files changed, 264 insertions(+), 176 deletions(-) diff --git a/crates/flux-network/UDP_URING.md b/crates/flux-network/UDP_URING.md index 93249ec..662ccdb 100644 --- a/crates/flux-network/UDP_URING.md +++ b/crates/flux-network/UDP_URING.md @@ -34,9 +34,12 @@ security policy may still disable them. queue counts as acceptance by the local socket backend. Completion errors become packet loss, handled by the existing reliability protocol. Neither ACK processing nor reconnect can invalidate kernel-owned bytes. -- Completion passes are bounded. ACKs are emitted between receive passes so +- Each poll processes at most 32 receive passes per socket, each capped at + `recv_entries` buffers (a GRO buffer can contain several datagrams). + ACKs are emitted between receive passes so callback processing cannot defer them behind a whole send window. Buffer exhaustion ends/rearms the multishot request after buffers are recycled. + An ACK blocked by send-queue capacity remains pending for retry. - Normal polling uses no waiting for completions and skips idle kernel entries using the task-work flag. There is no async executor, SQPOLL thread, or zero-copy send machinery. @@ -70,6 +73,7 @@ AMD Ryzen 9 9950X, Linux `7.1.5-arch1-2`, Rust `1.91.0`, release optimization, thread. Socket buffers request 16 MiB; this host's send/receive sysctl maxima are 4 MiB (Linux reports doubled effective socket accounting limits). +Samples were refreshed after the ACK-backpressure and bounded-drain audit fixes. Five measured rounds followed one discarded run of each binary. Main and branch alternate order; UDP and io_uring also alternate order within the branch binary. No builds or tests ran concurrently with the measured rounds. @@ -92,29 +96,31 @@ including their busy polling; it is not an idle-efficiency measurement. | Workload | Main MiB/s | io_uring MiB/s | Change | Main p50 / p99 µs | io_uring p50 / p99 µs | Main / io_uring CPU ns/B | | --- | ---: | ---: | ---: | ---: | ---: | ---: | -| paced/2k | 20 | 20 | +0.0% | 4.7 / 5.7 | 2.4 / 3.4 | 95.436 / 96.388 | -| paced/64k | 625 | 625 | +0.0% | 8.9 / 356.5 | 9.5 / 36.8 | 2.959 / 2.955 | -| paced/2m | 7,446 | 7,397 | -0.7% | 229.2 / 283.5 | 231.8 / 240.6 | 0.193 / 0.201 | -| burst/2k | 442 | 772 | +74.7% | 4.8 / 5.8 | 2.4 / 3.3 | 2.420 / 1.393 | -| burst/64k | 6,891 | 7,108 | +3.1% | 9.0 / 12.2 | 9.5 / 12.6 | 0.201 / 0.191 | -| burst/2m | 7,589 | 7,329 | -3.4% | 219.2 / 259.1 | 230.7 / 242.1 | 0.187 / 0.201 | -| bcast/2k | 562 | 950 | +69.0% | 17.0 / 34.4 | 18.7 / 35.2 | 1.778 / 1.075 | -| bcast/64k | 1,402 | 7,499 | +434.9% | 207.7 / 345.3 | 44.1 / 69.4 | 0.765 / 0.181 | -| bcast/2m | 6,820 | 6,700 | -1.8% | 1,344.5 / 4,861.2 | 3,431.1 / 6,699.6 | 0.211 / 0.246 | +| paced/2k | 20 | 20 | +0.0% | 4.7 / 5.6 | 2.4 / 3.4 | 95.432 / 96.388 | +| paced/64k | 625 | 625 | +0.0% | 9.0 / 452.2 | 9.5 / 172.7 | 2.958 / 2.955 | +| paced/2m | 7,430 | 7,408 | -0.3% | 232.5 / 249.5 | 231.6 / 239.1 | 0.193 / 0.200 | +| burst/2k | 442 | 773 | +74.9% | 4.8 / 5.8 | 2.4 / 3.3 | 2.420 / 1.396 | +| burst/64k | 6,849 | 7,178 | +4.8% | 9.1 / 11.7 | 9.5 / 12.7 | 0.203 / 0.188 | +| burst/2m | 7,285 | 7,315 | +0.4% | 234.5 / 254.6 | 229.9 / 236.9 | 0.197 / 0.202 | +| bcast/2k | 561 | 946 | +68.6% | 16.6 / 40.5 | 18.8 / 35.1 | 1.784 / 1.084 | +| bcast/64k | 1,398 | 7,461 | +433.7% | 187.4 / 345.8 | 44.5 / 69.6 | 0.767 / 0.181 | +| bcast/2m | 6,855 | 7,043 | +2.7% | 1,333.8 / 4,693.6 | 3,232.8 / 6,186.5 | 0.207 / 0.233 | The strongest gains are 2 KiB bursts (+75% throughput, roughly half median -latency) and 64 KiB broadcasts (5.35x throughput, p99 345 → 69 µs). The latter +latency) and 64 KiB broadcasts (5.34x throughput, p99 346 → 70 µs). The latter also benefits from retaining GSO across mixed-peer batches: these results measure the complete backend implementation, not an isolated io_uring syscall substitution. The syscall backend could independently adopt that grouping. -Large messages are a tradeoff: 2 MiB burst throughput is 3.4% lower, and 2 MiB -broadcast median latency increases from 1.34 ms to 3.43 ms while throughput -falls 1.8%. The extra send copy and different batching/scheduling costs remain. -This is why io_uring stays opt-in. Paced p99 varies substantially between runs; -consult the raw samples rather than treating a median as a guarantee. +Large messages remain a tradeoff: 2 MiB burst throughput is within 1% of +main, but 3.0% below the branch's syscall control (7,315 vs 7,541 MiB/s). +Broadcast throughput is 2.7% above main, while median latency increases from +1.33 ms to 3.23 ms and CPU cost rises from 0.207 to 0.233 ns/B. The extra send +copy and different batching/scheduling costs remain. This is why io_uring +stays opt-in. Paced p99 varies substantially between runs; consult the raw +samples rather than treating a median as a guarantee. -The single-loss relay case completed in a median 6.21 ms on main and 5.26 ms +The single-loss relay case completed in a median 6.09 ms on main and 5.39 ms with io_uring. Every run observed 1,793 data datagrams, including 2 repeated sequences. Each run sends only one message, so this is a recovery check, not a useful p99 measurement. Relay CPU is excluded from the CPU column. @@ -122,7 +128,7 @@ useful p99 measurement. Relay CPU is excluded from the CPU column. All raw runs are in [benches/results/udp-uring](benches/results/udp-uring). `main-N.txt` is the isolated main build; `branch-N.txt` includes both backends on this branch. The branch's syscall control preserves similar throughput -(e.g. 441 vs 442 MiB/s for 2 KiB bursts, 1,406 vs 1,402 for 64 KiB broadcasts). +(e.g. 441 vs 442 MiB/s for 2 KiB bursts, 1,408 vs 1,398 for 64 KiB broadcasts). These are local loopback measurements, not physical-NIC, WAN, or AWS results. Non-Linux builds were not cross-compiled. @@ -145,13 +151,14 @@ Validation passed: - `just fmt` - `just clippy` -- `cargo test --workspace --all-features --locked`: 362 passed, 4 existing +- `cargo test --workspace --all-features --locked`: 364 passed, 4 existing ignored documentation tests, no failures. Both backends run the same UDP integration suite. Additional coverage checks mixed-backend peers, DCache delivery, sustained large messages with slow debug callbacks, owned send buffers, queue saturation, IPv4/IPv6, mixed GSO groups, -fallback, GRO validation, receive-pool exhaustion, 16-bit descriptor-tail +fallback, GRO validation, receive-pool exhaustion, ACK retry under send +backpressure, duplex delivery with one send/receive slot, 16-bit descriptor-tail wraparound, and driver movement across threads. Tests require local socket and io_uring permissions. Local runs used `RUSTC_WRAPPER=` and a temporary `CARGO_TARGET_DIR` because the shared build cache was sandbox-restricted. diff --git a/crates/flux-network/benches/results/udp-uring/branch-1.txt b/crates/flux-network/benches/results/udp-uring/branch-1.txt index 1f17c03..554908b 100644 --- a/crates/flux-network/benches/results/udp-uring/branch-1.txt +++ b/crates/flux-network/benches/results/udp-uring/branch-1.txt @@ -1,27 +1,27 @@ == paced: one message per 100µs, one receiver == -paced/uring/2k n=2000 p50= 2.4µs p99= 3.4µs max= 67.9µs 20 MiB/s cpu=96.393ns/B -paced/udp/2k n=2000 p50= 4.8µs p99= 328.1µs max= 2324.2µs 20 MiB/s cpu=94.873ns/B -paced/uring/64k n=2000 p50= 9.5µs p99= 13.4µs max= 1762.7µs 625 MiB/s cpu=2.959ns/B -paced/udp/64k n=2000 p50= 9.1µs p99= 706.3µs max= 2653.7µs 625 MiB/s cpu=2.956ns/B -paced/uring/2m n=2000 p50= 233.3µs p99= 249.3µs max= 2046.2µs 7362 MiB/s cpu=0.201ns/B -paced/udp/2m n=2000 p50= 208.0µs p99= 238.8µs max= 2905.7µs 8001 MiB/s cpu=0.178ns/B +paced/uring/2k n=2000 p50= 2.4µs p99= 3.4µs max= 80.7µs 20 MiB/s cpu=96.401ns/B +paced/udp/2k n=2000 p50= 4.8µs p99= 384.1µs max= 2380.2µs 20 MiB/s cpu=94.905ns/B +paced/uring/64k n=2000 p50= 9.5µs p99= 64.3µs max= 2030.5µs 625 MiB/s cpu=2.955ns/B +paced/udp/64k n=2000 p50= 9.2µs p99= 760.4µs max= 2701.4µs 625 MiB/s cpu=2.956ns/B +paced/uring/2m n=2000 p50= 232.9µs p99= 242.3µs max= 2234.0µs 7353 MiB/s cpu=0.201ns/B +paced/udp/2m n=2000 p50= 210.0µs p99= 233.7µs max= 3326.9µs 7960 MiB/s cpu=0.179ns/B == burst: bounded outstanding, one receiver == -burst/uring/2k n=65536 p50= 2.4µs p99= 3.3µs max= 2563.5µs 767 MiB/s cpu=1.409ns/B window=256 -burst/udp/2k n=65536 p50= 4.8µs p99= 5.9µs max= 3029.5µs 441 MiB/s cpu=2.422ns/B window=256 -burst/uring/64k n=16384 p50= 9.4µs p99= 12.4µs max= 2874.5µs 7291 MiB/s cpu=0.184ns/B window=146 -burst/udp/64k n=16384 p50= 8.9µs p99= 12.0µs max= 2799.0µs 7044 MiB/s cpu=0.195ns/B window=146 -burst/uring/2m n=512 p50= 230.7µs p99= 569.2µs max= 2604.1µs 7329 MiB/s cpu=0.201ns/B window=4 -burst/udp/2m n=512 p50= 225.3µs p99= 284.2µs max= 3285.2µs 7344 MiB/s cpu=0.195ns/B window=4 +burst/uring/2k n=65536 p50= 2.4µs p99= 3.3µs max= 2519.5µs 772 MiB/s cpu=1.396ns/B window=256 +burst/udp/2k n=65536 p50= 4.8µs p99= 5.8µs max= 3429.4µs 445 MiB/s cpu=2.395ns/B window=256 +burst/uring/64k n=16384 p50= 9.5µs p99= 12.1µs max= 3058.8µs 7106 MiB/s cpu=0.191ns/B window=146 +burst/udp/64k n=16384 p50= 9.1µs p99= 11.8µs max= 3436.4µs 6846 MiB/s cpu=0.204ns/B window=146 +burst/uring/2m n=512 p50= 229.9µs p99= 236.9µs max= 2884.9µs 7315 MiB/s cpu=0.202ns/B window=4 +burst/udp/2m n=512 p50= 227.7µs p99= 244.0µs max= 2952.3µs 7318 MiB/s cpu=0.195ns/B window=4 == loss1: one 2 MiB message, exactly one datagram dropped by a relay == -loss1/uring/2m n=1 p50= 5365.6µs p99= 5365.6µs max= 5365.6µs 357 MiB/s cpu=3.212ns/B datagrams=1793 retransmitted=2 -loss1/udp/2m n=1 p50= 6156.9µs p99= 6156.9µs max= 6156.9µs 303 MiB/s cpu=2.845ns/B datagrams=1793 retransmitted=2 +loss1/uring/2m n=1 p50= 5413.6µs p99= 5413.6µs max= 5413.6µs 351 MiB/s cpu=3.289ns/B datagrams=1793 retransmitted=2 +loss1/udp/2m n=1 p50= 8873.2µs p99= 8873.2µs max= 8873.2µs 221 MiB/s cpu=5.179ns/B datagrams=1793 retransmitted=2 == bcast: one sender, 8 receivers on one listener == -bcast/uring/2k n=65536 p50= 18.7µs p99= 35.4µs max= 356.2µs 942 MiB/s cpu=1.083ns/B window=256 -bcast/udp/2k n=65536 p50= 16.9µs p99= 31.6µs max= 1995.7µs 562 MiB/s cpu=1.777ns/B window=256 -bcast/uring/64k n=16384 p50= 46.1µs p99= 71.8µs max= 138.3µs 7123 MiB/s cpu=0.190ns/B window=146 -bcast/udp/64k n=16384 p50= 212.4µs p99= 343.5µs max= 2958.4µs 1406 MiB/s cpu=0.758ns/B window=146 -bcast/uring/2m n=512 p50= 3584.3µs p99= 7297.2µs max= 8506.7µs 6522 MiB/s cpu=0.254ns/B window=4 -bcast/udp/2m n=512 p50= 1435.3µs p99= 4666.5µs max= 6445.6µs 6485 MiB/s cpu=0.219ns/B window=4 +bcast/uring/2k n=65536 p50= 18.7µs p99= 34.9µs max= 78.3µs 938 MiB/s cpu=1.093ns/B window=256 +bcast/udp/2k n=65536 p50= 17.3µs p99= 30.9µs max= 1789.7µs 565 MiB/s cpu=1.765ns/B window=256 +bcast/uring/64k n=16384 p50= 44.5µs p99= 69.6µs max= 128.7µs 7443 MiB/s cpu=0.181ns/B window=146 +bcast/udp/64k n=16384 p50= 210.3µs p99= 342.0µs max= 2528.8µs 1413 MiB/s cpu=0.749ns/B window=146 +bcast/uring/2m n=512 p50= 3318.4µs p99= 6161.2µs max= 6783.3µs 7043 MiB/s cpu=0.233ns/B window=4 +bcast/udp/2m n=512 p50= 1360.5µs p99= 3951.8µs max= 4640.5µs 6980 MiB/s cpu=0.201ns/B window=4 diff --git a/crates/flux-network/benches/results/udp-uring/branch-2.txt b/crates/flux-network/benches/results/udp-uring/branch-2.txt index 78c7833..3c823a8 100644 --- a/crates/flux-network/benches/results/udp-uring/branch-2.txt +++ b/crates/flux-network/benches/results/udp-uring/branch-2.txt @@ -1,27 +1,27 @@ == paced: one message per 100µs, one receiver == -paced/udp/2k n=2000 p50= 4.8µs p99= 5.6µs max= 56.2µs 20 MiB/s cpu=95.407ns/B -paced/uring/2k n=2000 p50= 2.4µs p99= 4.3µs max= 906.7µs 20 MiB/s cpu=96.159ns/B -paced/udp/64k n=2000 p50= 9.0µs p99= 509.5µs max= 2451.0µs 625 MiB/s cpu=2.957ns/B -paced/uring/64k n=2000 p50= 9.8µs p99= 207.8µs max= 2131.1µs 625 MiB/s cpu=2.954ns/B -paced/udp/2m n=2000 p50= 231.7µs p99= 299.8µs max= 4451.2µs 7228 MiB/s cpu=0.200ns/B -paced/uring/2m n=2000 p50= 231.8µs p99= 238.2µs max= 2775.7µs 7397 MiB/s cpu=0.201ns/B +paced/udp/2k n=2000 p50= 4.7µs p99= 6.0µs max= 54.2µs 20 MiB/s cpu=95.431ns/B +paced/uring/2k n=2000 p50= 2.4µs p99= 4.1µs max= 1080.5µs 20 MiB/s cpu=96.127ns/B +paced/udp/64k n=2000 p50= 8.9µs p99= 498.6µs max= 2451.9µs 625 MiB/s cpu=2.957ns/B +paced/uring/64k n=2000 p50= 9.5µs p99= 434.7µs max= 2378.5µs 625 MiB/s cpu=2.953ns/B +paced/udp/2m n=2000 p50= 240.2µs p99= 256.3µs max= 3402.6µs 7198 MiB/s cpu=0.201ns/B +paced/uring/2m n=2000 p50= 219.9µs p99= 253.7µs max= 2659.0µs 7689 MiB/s cpu=0.194ns/B == burst: bounded outstanding, one receiver == -burst/udp/2k n=65536 p50= 4.8µs p99= 5.8µs max= 2822.9µs 441 MiB/s cpu=2.420ns/B window=256 -burst/uring/2k n=65536 p50= 2.4µs p99= 3.3µs max= 2956.5µs 773 MiB/s cpu=1.392ns/B window=256 -burst/udp/64k n=16384 p50= 9.3µs p99= 12.9µs max= 3028.8µs 6681 MiB/s cpu=0.210ns/B window=146 -burst/uring/64k n=16384 p50= 9.5µs p99= 18.0µs max= 3148.4µs 7091 MiB/s cpu=0.191ns/B window=146 -burst/udp/2m n=512 p50= 220.4µs p99= 240.9µs max= 3213.8µs 7498 MiB/s cpu=0.190ns/B window=4 -burst/uring/2m n=512 p50= 218.5µs p99= 241.8µs max= 2487.1µs 7672 MiB/s cpu=0.194ns/B window=4 +burst/udp/2k n=65536 p50= 4.8µs p99= 5.8µs max= 2664.1µs 441 MiB/s cpu=2.437ns/B window=256 +burst/uring/2k n=65536 p50= 2.4µs p99= 3.2µs max= 2444.8µs 773 MiB/s cpu=1.400ns/B window=256 +burst/udp/64k n=16384 p50= 9.2µs p99= 11.8µs max= 3550.8µs 6735 MiB/s cpu=0.208ns/B window=146 +burst/uring/64k n=16384 p50= 9.4µs p99= 12.2µs max= 3002.8µs 7365 MiB/s cpu=0.181ns/B window=146 +burst/udp/2m n=512 p50= 222.0µs p99= 239.7µs max= 3490.5µs 7484 MiB/s cpu=0.189ns/B window=4 +burst/uring/2m n=512 p50= 218.8µs p99= 229.5µs max= 2668.9µs 7667 MiB/s cpu=0.193ns/B window=4 == loss1: one 2 MiB message, exactly one datagram dropped by a relay == -loss1/udp/2m n=1 p50= 6246.1µs p99= 6246.1µs max= 6246.1µs 296 MiB/s cpu=3.014ns/B datagrams=1793 retransmitted=2 -loss1/uring/2m n=1 p50= 4717.3µs p99= 4717.3µs max= 4717.3µs 394 MiB/s cpu=2.741ns/B datagrams=1793 retransmitted=2 +loss1/udp/2m n=1 p50= 6207.4µs p99= 6207.4µs max= 6207.4µs 298 MiB/s cpu=2.932ns/B datagrams=1793 retransmitted=2 +loss1/uring/2m n=1 p50= 4580.3µs p99= 4580.3µs max= 4580.3µs 408 MiB/s cpu=2.586ns/B datagrams=1793 retransmitted=2 == bcast: one sender, 8 receivers on one listener == -bcast/udp/2k n=65536 p50= 17.2µs p99= 31.5µs max= 1643.0µs 563 MiB/s cpu=1.784ns/B window=256 -bcast/uring/2k n=65536 p50= 18.7µs p99= 35.0µs max= 85.9µs 954 MiB/s cpu=1.074ns/B window=256 -bcast/udp/64k n=16384 p50= 211.5µs p99= 344.0µs max= 1742.5µs 1406 MiB/s cpu=0.757ns/B window=146 -bcast/uring/64k n=16384 p50= 44.4µs p99= 70.3µs max= 124.5µs 7463 MiB/s cpu=0.182ns/B window=146 -bcast/udp/2m n=512 p50= 1655.0µs p99= 6342.0µs max= 8185.5µs 6653 MiB/s cpu=0.223ns/B window=4 -bcast/uring/2m n=512 p50= 3212.1µs p99= 6699.6µs max= 7581.0µs 6776 MiB/s cpu=0.243ns/B window=4 +bcast/udp/2k n=65536 p50= 17.2µs p99= 31.4µs max= 1759.9µs 568 MiB/s cpu=1.754ns/B window=256 +bcast/uring/2k n=65536 p50= 18.8µs p99= 35.3µs max= 80.1µs 946 MiB/s cpu=1.085ns/B window=256 +bcast/udp/64k n=16384 p50= 211.5µs p99= 343.5µs max= 1985.4µs 1406 MiB/s cpu=0.756ns/B window=146 +bcast/uring/64k n=16384 p50= 44.6µs p99= 70.1µs max= 140.4µs 7461 MiB/s cpu=0.181ns/B window=146 +bcast/udp/2m n=512 p50= 1388.2µs p99= 5729.2µs max= 7346.6µs 6848 MiB/s cpu=0.208ns/B window=4 +bcast/uring/2m n=512 p50= 1499.2µs p99= 4005.6µs max= 4768.5µs 6911 MiB/s cpu=0.238ns/B window=4 diff --git a/crates/flux-network/benches/results/udp-uring/branch-3.txt b/crates/flux-network/benches/results/udp-uring/branch-3.txt index d2ff48c..011633e 100644 --- a/crates/flux-network/benches/results/udp-uring/branch-3.txt +++ b/crates/flux-network/benches/results/udp-uring/branch-3.txt @@ -1,27 +1,27 @@ == paced: one message per 100µs, one receiver == -paced/uring/2k n=2000 p50= 2.4µs p99= 3.4µs max= 150.9µs 20 MiB/s cpu=96.390ns/B -paced/udp/2k n=2000 p50= 4.8µs p99= 168.2µs max= 2164.6µs 20 MiB/s cpu=94.907ns/B -paced/uring/64k n=2000 p50= 9.4µs p99= 36.8µs max= 1597.4µs 625 MiB/s cpu=2.955ns/B -paced/udp/64k n=2000 p50= 9.1µs p99= 645.4µs max= 2593.0µs 625 MiB/s cpu=2.956ns/B -paced/uring/2m n=2000 p50= 219.1µs p99= 226.3µs max= 1995.9µs 7782 MiB/s cpu=0.192ns/B -paced/udp/2m n=2000 p50= 215.1µs p99= 228.2µs max= 3220.1µs 7771 MiB/s cpu=0.183ns/B +paced/uring/2k n=2000 p50= 2.4µs p99= 3.4µs max= 40.0µs 20 MiB/s cpu=96.437ns/B +paced/udp/2k n=2000 p50= 4.8µs p99= 448.5µs max= 2444.6µs 20 MiB/s cpu=94.849ns/B +paced/uring/64k n=2000 p50= 9.4µs p99= 172.7µs max= 2119.9µs 625 MiB/s cpu=2.956ns/B +paced/udp/64k n=2000 p50= 9.0µs p99= 779.9µs max= 2727.9µs 625 MiB/s cpu=2.956ns/B +paced/uring/2m n=2000 p50= 231.9µs p99= 239.1µs max= 3346.7µs 7403 MiB/s cpu=0.200ns/B +paced/udp/2m n=2000 p50= 234.1µs p99= 239.4µs max= 3088.5µs 7260 MiB/s cpu=0.199ns/B == burst: bounded outstanding, one receiver == -burst/uring/2k n=65536 p50= 2.4µs p99= 4.4µs max= 2456.5µs 772 MiB/s cpu=1.393ns/B window=256 -burst/udp/2k n=65536 p50= 4.8µs p99= 5.8µs max= 3212.3µs 443 MiB/s cpu=2.410ns/B window=256 -burst/uring/64k n=16384 p50= 9.4µs p99= 12.4µs max= 2521.1µs 7190 MiB/s cpu=0.188ns/B window=146 -burst/udp/64k n=16384 p50= 9.1µs p99= 12.1µs max= 3126.3µs 6837 MiB/s cpu=0.203ns/B window=146 -burst/uring/2m n=512 p50= 231.8µs p99= 242.1µs max= 1894.4µs 7295 MiB/s cpu=0.203ns/B window=4 -burst/udp/2m n=512 p50= 225.0µs p99= 245.1µs max= 3138.8µs 7347 MiB/s cpu=0.194ns/B window=4 +burst/uring/2k n=65536 p50= 2.4µs p99= 3.3µs max= 2650.1µs 766 MiB/s cpu=1.407ns/B window=256 +burst/udp/2k n=65536 p50= 4.8µs p99= 5.9µs max= 3366.4µs 438 MiB/s cpu=2.435ns/B window=256 +burst/uring/64k n=16384 p50= 9.5µs p99= 12.7µs max= 2447.3µs 7178 MiB/s cpu=0.188ns/B window=146 +burst/udp/64k n=16384 p50= 9.1µs p99= 11.8µs max= 3300.3µs 6787 MiB/s cpu=0.205ns/B window=146 +burst/uring/2m n=512 p50= 232.5µs p99= 242.3µs max= 2468.1µs 7270 MiB/s cpu=0.203ns/B window=4 +burst/udp/2m n=512 p50= 213.6µs p99= 230.8µs max= 2898.7µs 7709 MiB/s cpu=0.183ns/B window=4 == loss1: one 2 MiB message, exactly one datagram dropped by a relay == -loss1/uring/2m n=1 p50= 5261.6µs p99= 5261.6µs max= 5261.6µs 362 MiB/s cpu=3.183ns/B datagrams=1793 retransmitted=2 -loss1/udp/2m n=1 p50= 6341.0µs p99= 6341.0µs max= 6341.0µs 293 MiB/s cpu=2.999ns/B datagrams=1793 retransmitted=2 +loss1/uring/2m n=1 p50= 5389.2µs p99= 5389.2µs max= 5389.2µs 351 MiB/s cpu=3.282ns/B datagrams=1793 retransmitted=2 +loss1/udp/2m n=1 p50= 8768.1µs p99= 8768.1µs max= 8768.1µs 226 MiB/s cpu=5.171ns/B datagrams=1793 retransmitted=2 == bcast: one sender, 8 receivers on one listener == -bcast/uring/2k n=65536 p50= 18.9µs p99= 35.2µs max= 84.3µs 949 MiB/s cpu=1.076ns/B window=256 -bcast/udp/2k n=65536 p50= 17.2µs p99= 30.5µs max= 1583.6µs 566 MiB/s cpu=1.765ns/B window=256 -bcast/uring/64k n=16384 p50= 43.8µs p99= 68.7µs max= 116.5µs 7499 MiB/s cpu=0.181ns/B window=146 -bcast/udp/64k n=16384 p50= 211.0µs p99= 347.3µs max= 2499.7µs 1400 MiB/s cpu=0.761ns/B window=146 -bcast/uring/2m n=512 p50= 3688.4µs p99= 7156.3µs max= 7792.3µs 6443 MiB/s cpu=0.257ns/B window=4 -bcast/udp/2m n=512 p50= 1321.7µs p99= 2843.1µs max= 3790.1µs 7092 MiB/s cpu=0.198ns/B window=4 +bcast/uring/2k n=65536 p50= 18.8µs p99= 35.1µs max= 73.2µs 952 MiB/s cpu=1.080ns/B window=256 +bcast/udp/2k n=65536 p50= 17.3µs p99= 30.6µs max= 1354.2µs 567 MiB/s cpu=1.764ns/B window=256 +bcast/uring/64k n=16384 p50= 45.0µs p99= 70.0µs max= 116.5µs 7412 MiB/s cpu=0.183ns/B window=146 +bcast/udp/64k n=16384 p50= 211.9µs p99= 343.1µs max= 2545.4µs 1408 MiB/s cpu=0.759ns/B window=146 +bcast/uring/2m n=512 p50= 3410.7µs p99= 6854.9µs max= 7521.4µs 6714 MiB/s cpu=0.246ns/B window=4 +bcast/udp/2m n=512 p50= 1425.0µs p99= 2761.3µs max= 3752.1µs 6681 MiB/s cpu=0.210ns/B window=4 diff --git a/crates/flux-network/benches/results/udp-uring/branch-4.txt b/crates/flux-network/benches/results/udp-uring/branch-4.txt index 3fb89b1..16a9bc5 100644 --- a/crates/flux-network/benches/results/udp-uring/branch-4.txt +++ b/crates/flux-network/benches/results/udp-uring/branch-4.txt @@ -1,27 +1,27 @@ == paced: one message per 100µs, one receiver == -paced/udp/2k n=2000 p50= 4.8µs p99= 5.5µs max= 54.5µs 20 MiB/s cpu=95.421ns/B -paced/uring/2k n=2000 p50= 2.4µs p99= 3.9µs max= 949.7µs 20 MiB/s cpu=96.165ns/B -paced/udp/64k n=2000 p50= 9.2µs p99= 481.3µs max= 2432.1µs 625 MiB/s cpu=2.957ns/B -paced/uring/64k n=2000 p50= 9.5µs p99= 135.3µs max= 2039.7µs 625 MiB/s cpu=2.954ns/B -paced/udp/2m n=2000 p50= 227.7µs p99= 250.5µs max= 3518.5µs 7560 MiB/s cpu=0.189ns/B -paced/uring/2m n=2000 p50= 232.2µs p99= 240.6µs max= 2488.2µs 7364 MiB/s cpu=0.201ns/B +paced/udp/2k n=2000 p50= 4.8µs p99= 5.5µs max= 48.6µs 20 MiB/s cpu=95.447ns/B +paced/uring/2k n=2000 p50= 2.4µs p99= 4.2µs max= 798.4µs 20 MiB/s cpu=96.215ns/B +paced/udp/64k n=2000 p50= 9.1µs p99= 515.2µs max= 2456.4µs 625 MiB/s cpu=2.958ns/B +paced/uring/64k n=2000 p50= 9.6µs p99= 321.3µs max= 2260.8µs 625 MiB/s cpu=2.955ns/B +paced/udp/2m n=2000 p50= 238.8µs p99= 254.4µs max= 4154.9µs 7257 MiB/s cpu=0.200ns/B +paced/uring/2m n=2000 p50= 219.5µs p99= 226.6µs max= 2734.6µs 7755 MiB/s cpu=0.192ns/B == burst: bounded outstanding, one receiver == -burst/udp/2k n=65536 p50= 4.8µs p99= 6.5µs max= 2834.1µs 439 MiB/s cpu=2.437ns/B window=256 -burst/uring/2k n=65536 p50= 2.4µs p99= 3.2µs max= 2790.4µs 776 MiB/s cpu=1.385ns/B window=256 -burst/udp/64k n=16384 p50= 9.2µs p99= 11.8µs max= 3593.8µs 6738 MiB/s cpu=0.207ns/B window=146 -burst/uring/64k n=16384 p50= 9.5µs p99= 15.2µs max= 2969.4µs 7086 MiB/s cpu=0.191ns/B window=146 -burst/udp/2m n=512 p50= 213.8µs p99= 237.6µs max= 3026.2µs 7771 MiB/s cpu=0.182ns/B window=4 -burst/uring/2m n=512 p50= 222.1µs p99= 237.5µs max= 2690.7µs 7503 MiB/s cpu=0.197ns/B window=4 +burst/udp/2k n=65536 p50= 4.8µs p99= 5.8µs max= 3052.9µs 439 MiB/s cpu=2.438ns/B window=256 +burst/uring/2k n=65536 p50= 2.4µs p99= 3.4µs max= 2978.2µs 776 MiB/s cpu=1.390ns/B window=256 +burst/udp/64k n=16384 p50= 9.2µs p99= 13.5µs max= 3695.2µs 6667 MiB/s cpu=0.211ns/B window=146 +burst/uring/64k n=16384 p50= 9.4µs p99= 13.3µs max= 3345.8µs 7394 MiB/s cpu=0.181ns/B window=146 +burst/udp/2m n=512 p50= 216.2µs p99= 282.9µs max= 3181.4µs 7560 MiB/s cpu=0.188ns/B window=4 +burst/uring/2m n=512 p50= 218.5µs p99= 228.5µs max= 2571.7µs 7680 MiB/s cpu=0.193ns/B window=4 == loss1: one 2 MiB message, exactly one datagram dropped by a relay == -loss1/udp/2m n=1 p50= 6181.2µs p99= 6181.2µs max= 6181.2µs 300 MiB/s cpu=2.937ns/B datagrams=1793 retransmitted=2 -loss1/uring/2m n=1 p50= 4563.1µs p99= 4563.1µs max= 4563.1µs 407 MiB/s cpu=2.600ns/B datagrams=1793 retransmitted=2 +loss1/udp/2m n=1 p50= 6135.7µs p99= 6135.7µs max= 6135.7µs 302 MiB/s cpu=2.932ns/B datagrams=1793 retransmitted=2 +loss1/uring/2m n=1 p50= 4526.4µs p99= 4526.4µs max= 4526.4µs 410 MiB/s cpu=2.572ns/B datagrams=1793 retransmitted=2 == bcast: one sender, 8 receivers on one listener == -bcast/udp/2k n=65536 p50= 17.5µs p99= 31.0µs max= 1655.4µs 565 MiB/s cpu=1.758ns/B window=256 -bcast/uring/2k n=65536 p50= 18.7µs p99= 35.0µs max= 74.4µs 954 MiB/s cpu=1.070ns/B window=256 -bcast/udp/64k n=16384 p50= 209.9µs p99= 342.7µs max= 1455.8µs 1411 MiB/s cpu=0.754ns/B window=146 -bcast/uring/64k n=16384 p50= 44.0µs p99= 67.9µs max= 124.7µs 7538 MiB/s cpu=0.179ns/B window=146 -bcast/udp/2m n=512 p50= 1378.0µs p99= 5145.2µs max= 6673.5µs 6877 MiB/s cpu=0.209ns/B window=4 -bcast/uring/2m n=512 p50= 1373.0µs p99= 2804.0µs max= 4350.4µs 6889 MiB/s cpu=0.239ns/B window=4 +bcast/udp/2k n=65536 p50= 17.4µs p99= 31.4µs max= 1743.5µs 566 MiB/s cpu=1.759ns/B window=256 +bcast/uring/2k n=65536 p50= 18.7µs p99= 35.0µs max= 77.8µs 954 MiB/s cpu=1.070ns/B window=256 +bcast/udp/64k n=16384 p50= 208.6µs p99= 342.6µs max= 1654.6µs 1411 MiB/s cpu=0.754ns/B window=146 +bcast/uring/64k n=16384 p50= 44.4µs p99= 68.5µs max= 135.7µs 7471 MiB/s cpu=0.180ns/B window=146 +bcast/udp/2m n=512 p50= 1364.5µs p99= 5455.6µs max= 6912.9µs 7009 MiB/s cpu=0.204ns/B window=4 +bcast/uring/2m n=512 p50= 3199.3µs p99= 6353.6µs max= 6675.1µs 7064 MiB/s cpu=0.232ns/B window=4 diff --git a/crates/flux-network/benches/results/udp-uring/branch-5.txt b/crates/flux-network/benches/results/udp-uring/branch-5.txt index 8772997..c61ee24 100644 --- a/crates/flux-network/benches/results/udp-uring/branch-5.txt +++ b/crates/flux-network/benches/results/udp-uring/branch-5.txt @@ -1,27 +1,27 @@ == paced: one message per 100µs, one receiver == -paced/uring/2k n=2000 p50= 2.4µs p99= 3.4µs max= 80.6µs 20 MiB/s cpu=96.388ns/B -paced/udp/2k n=2000 p50= 4.8µs p99= 357.1µs max= 2352.8µs 20 MiB/s cpu=94.886ns/B -paced/uring/64k n=2000 p50= 9.7µs p99= 19.3µs max= 1957.4µs 625 MiB/s cpu=2.956ns/B -paced/udp/64k n=2000 p50= 9.0µs p99= 627.2µs max= 2569.8µs 625 MiB/s cpu=2.956ns/B -paced/uring/2m n=2000 p50= 229.2µs p99= 251.2µs max= 2853.4µs 7518 MiB/s cpu=0.197ns/B -paced/udp/2m n=2000 p50= 210.1µs p99= 234.8µs max= 3168.3µs 7913 MiB/s cpu=0.180ns/B +paced/uring/2k n=2000 p50= 2.4µs p99= 3.3µs max= 44.8µs 20 MiB/s cpu=96.388ns/B +paced/udp/2k n=2000 p50= 4.7µs p99= 458.1µs max= 2454.1µs 20 MiB/s cpu=94.881ns/B +paced/uring/64k n=2000 p50= 9.4µs p99= 104.1µs max= 2040.3µs 625 MiB/s cpu=2.956ns/B +paced/udp/64k n=2000 p50= 9.0µs p99= 10.1µs max= 1249.2µs 625 MiB/s cpu=2.966ns/B +paced/uring/2m n=2000 p50= 231.6µs p99= 237.7µs max= 2123.8µs 7408 MiB/s cpu=0.200ns/B +paced/udp/2m n=2000 p50= 217.1µs p99= 229.8µs max= 3756.7µs 7721 MiB/s cpu=0.185ns/B == burst: bounded outstanding, one receiver == -burst/uring/2k n=65536 p50= 2.4µs p99= 3.3µs max= 2511.4µs 770 MiB/s cpu=1.398ns/B window=256 -burst/udp/2k n=65536 p50= 4.8µs p99= 5.8µs max= 3883.5µs 444 MiB/s cpu=2.388ns/B window=256 -burst/uring/64k n=16384 p50= 9.5µs p99= 12.6µs max= 3123.2µs 7108 MiB/s cpu=0.191ns/B window=146 -burst/udp/64k n=16384 p50= 9.0µs p99= 13.2µs max= 3224.0µs 6884 MiB/s cpu=0.201ns/B window=146 -burst/uring/2m n=512 p50= 231.6µs p99= 242.8µs max= 2772.9µs 7272 MiB/s cpu=0.202ns/B window=4 -burst/udp/2m n=512 p50= 226.9µs p99= 233.6µs max= 3008.8µs 7349 MiB/s cpu=0.194ns/B window=4 +burst/uring/2k n=65536 p50= 2.4µs p99= 3.3µs max= 2558.2µs 777 MiB/s cpu=1.388ns/B window=256 +burst/udp/2k n=65536 p50= 4.8µs p99= 6.8µs max= 3460.7µs 442 MiB/s cpu=2.405ns/B window=256 +burst/uring/64k n=16384 p50= 9.5µs p99= 14.5µs max= 2710.8µs 7102 MiB/s cpu=0.191ns/B window=146 +burst/udp/64k n=16384 p50= 9.0µs p99= 14.1µs max= 2953.6µs 6843 MiB/s cpu=0.203ns/B window=146 +burst/uring/2m n=512 p50= 233.1µs p99= 306.5µs max= 2166.3µs 7237 MiB/s cpu=0.204ns/B window=4 +burst/udp/2m n=512 p50= 219.0µs p99= 290.7µs max= 3091.0µs 7541 MiB/s cpu=0.188ns/B window=4 == loss1: one 2 MiB message, exactly one datagram dropped by a relay == -loss1/uring/2m n=1 p50= 5259.1µs p99= 5259.1µs max= 5259.1µs 358 MiB/s cpu=3.181ns/B datagrams=1793 retransmitted=2 -loss1/udp/2m n=1 p50= 8808.8µs p99= 8808.8µs max= 8808.8µs 225 MiB/s cpu=5.182ns/B datagrams=1793 retransmitted=2 +loss1/uring/2m n=1 p50= 5441.4µs p99= 5441.4µs max= 5441.4µs 349 MiB/s cpu=3.334ns/B datagrams=1793 retransmitted=2 +loss1/udp/2m n=1 p50= 6228.9µs p99= 6228.9µs max= 6228.9µs 299 MiB/s cpu=2.910ns/B datagrams=1793 retransmitted=2 == bcast: one sender, 8 receivers on one listener == -bcast/uring/2k n=65536 p50= 18.8µs p99= 35.2µs max= 72.3µs 950 MiB/s cpu=1.075ns/B window=256 -bcast/udp/2k n=65536 p50= 17.2µs p99= 30.5µs max= 1532.2µs 562 MiB/s cpu=1.774ns/B window=256 -bcast/uring/64k n=16384 p50= 44.1µs p99= 69.4µs max= 133.0µs 7541 MiB/s cpu=0.179ns/B window=146 -bcast/udp/64k n=16384 p50= 209.6µs p99= 341.2µs max= 1612.9µs 1416 MiB/s cpu=0.749ns/B window=146 -bcast/uring/2m n=512 p50= 3431.1µs p99= 6646.8µs max= 7164.7µs 6700 MiB/s cpu=0.246ns/B window=4 -bcast/udp/2m n=512 p50= 1359.1µs p99= 2644.3µs max= 3605.9µs 6944 MiB/s cpu=0.199ns/B window=4 +bcast/uring/2k n=65536 p50= 18.9µs p99= 35.3µs max= 87.1µs 945 MiB/s cpu=1.084ns/B window=256 +bcast/udp/2k n=65536 p50= 17.4µs p99= 30.5µs max= 1319.4µs 567 MiB/s cpu=1.753ns/B window=256 +bcast/uring/64k n=16384 p50= 43.8µs p99= 69.0µs max= 138.8µs 7514 MiB/s cpu=0.180ns/B window=146 +bcast/udp/64k n=16384 p50= 211.3µs p99= 343.0µs max= 2534.9µs 1408 MiB/s cpu=0.757ns/B window=146 +bcast/uring/2m n=512 p50= 3232.8µs p99= 6186.5µs max= 7257.1µs 7126 MiB/s cpu=0.231ns/B window=4 +bcast/udp/2m n=512 p50= 1407.6µs p99= 4641.6µs max= 6066.8µs 6822 MiB/s cpu=0.206ns/B window=4 diff --git a/crates/flux-network/benches/results/udp-uring/main-1.txt b/crates/flux-network/benches/results/udp-uring/main-1.txt index 99e3710..82adf62 100644 --- a/crates/flux-network/benches/results/udp-uring/main-1.txt +++ b/crates/flux-network/benches/results/udp-uring/main-1.txt @@ -1,17 +1,17 @@ == paced: one message per 100µs, one receiver == -paced/udp/2k n=2000 p50= 4.8µs p99= 6.2µs max= 37.6µs 20 MiB/s cpu=95.455ns/B -paced/udp/64k n=2000 p50= 9.1µs p99= 458.5µs max= 2403.1µs 625 MiB/s cpu=2.957ns/B -paced/udp/2m n=2000 p50= 236.2µs p99= 282.1µs max= 2643.4µs 7308 MiB/s cpu=0.198ns/B +paced/udp/2k n=2000 p50= 4.7µs p99= 5.5µs max= 48.8µs 20 MiB/s cpu=95.423ns/B +paced/udp/64k n=2000 p50= 9.0µs p99= 405.1µs max= 2346.1µs 625 MiB/s cpu=2.960ns/B +paced/udp/2m n=2000 p50= 232.5µs p99= 250.3µs max= 2403.3µs 7430 MiB/s cpu=0.193ns/B == burst: bounded outstanding, one receiver == -burst/udp/2k n=65536 p50= 4.8µs p99= 5.9µs max= 2430.4µs 442 MiB/s cpu=2.420ns/B window=256 -burst/udp/64k n=16384 p50= 8.9µs p99= 11.9µs max= 2623.9µs 7094 MiB/s cpu=0.194ns/B window=146 -burst/udp/2m n=512 p50= 219.2µs p99= 270.5µs max= 3493.1µs 7589 MiB/s cpu=0.187ns/B window=4 +burst/udp/2k n=65536 p50= 4.7µs p99= 5.8µs max= 2984.1µs 443 MiB/s cpu=2.415ns/B window=256 +burst/udp/64k n=16384 p50= 9.1µs p99= 11.7µs max= 3330.8µs 6849 MiB/s cpu=0.203ns/B window=146 +burst/udp/2m n=512 p50= 236.0µs p99= 254.6µs max= 3188.7µs 7192 MiB/s cpu=0.200ns/B window=4 == loss1: one 2 MiB message, exactly one datagram dropped by a relay == -loss1/udp/2m n=1 p50= 6211.3µs p99= 6211.3µs max= 6211.3µs 287 MiB/s cpu=3.150ns/B datagrams=1793 retransmitted=2 +loss1/udp/2m n=1 p50= 6107.3µs p99= 6107.3µs max= 6107.3µs 295 MiB/s cpu=2.964ns/B datagrams=1793 retransmitted=2 == bcast: one sender, 8 receivers on one listener == -bcast/udp/2k n=65536 p50= 17.3µs p99= 31.9µs max= 2139.8µs 564 MiB/s cpu=1.778ns/B window=256 -bcast/udp/64k n=16384 p50= 211.5µs p99= 345.3µs max= 419.5µs 1402 MiB/s cpu=0.765ns/B window=146 -bcast/udp/2m n=512 p50= 1342.2µs p99= 5379.5µs max= 6375.9µs 6846 MiB/s cpu=0.211ns/B window=4 +bcast/udp/2k n=65536 p50= 16.5µs p99= 40.5µs max= 2439.2µs 561 MiB/s cpu=1.787ns/B window=256 +bcast/udp/64k n=16384 p50= 183.9µs p99= 345.6µs max= 362.8µs 1398 MiB/s cpu=0.767ns/B window=146 +bcast/udp/2m n=512 p50= 1310.8µs p99= 5420.6µs max= 7348.5µs 7087 MiB/s cpu=0.199ns/B window=4 diff --git a/crates/flux-network/benches/results/udp-uring/main-2.txt b/crates/flux-network/benches/results/udp-uring/main-2.txt index b2d4017..a7d94e2 100644 --- a/crates/flux-network/benches/results/udp-uring/main-2.txt +++ b/crates/flux-network/benches/results/udp-uring/main-2.txt @@ -1,17 +1,17 @@ == paced: one message per 100µs, one receiver == -paced/udp/2k n=2000 p50= 4.7µs p99= 5.5µs max= 31.4µs 20 MiB/s cpu=95.420ns/B -paced/udp/64k n=2000 p50= 8.9µs p99= 337.3µs max= 2285.4µs 625 MiB/s cpu=2.959ns/B -paced/udp/2m n=2000 p50= 228.9µs p99= 297.2µs max= 2699.0µs 7446 MiB/s cpu=0.193ns/B +paced/udp/2k n=2000 p50= 4.7µs p99= 5.5µs max= 54.8µs 20 MiB/s cpu=95.450ns/B +paced/udp/64k n=2000 p50= 8.8µs p99= 445.4µs max= 2394.2µs 625 MiB/s cpu=2.959ns/B +paced/udp/2m n=2000 p50= 228.0µs p99= 243.4µs max= 2605.8µs 7548 MiB/s cpu=0.190ns/B == burst: bounded outstanding, one receiver == -burst/udp/2k n=65536 p50= 4.8µs p99= 5.8µs max= 2658.0µs 439 MiB/s cpu=2.435ns/B window=256 -burst/udp/64k n=16384 p50= 9.0µs p99= 15.0µs max= 3153.0µs 6972 MiB/s cpu=0.198ns/B window=146 -burst/udp/2m n=512 p50= 211.4µs p99= 239.8µs max= 2388.1µs 7840 MiB/s cpu=0.181ns/B window=4 +burst/udp/2k n=65536 p50= 4.8µs p99= 5.8µs max= 2649.6µs 441 MiB/s cpu=2.427ns/B window=256 +burst/udp/64k n=16384 p50= 9.0µs p99= 11.0µs max= 1015.8µs 7082 MiB/s cpu=0.197ns/B window=146 +burst/udp/2m n=512 p50= 229.8µs p99= 270.7µs max= 3486.4µs 7392 MiB/s cpu=0.193ns/B window=4 == loss1: one 2 MiB message, exactly one datagram dropped by a relay == -loss1/udp/2m n=1 p50= 6310.8µs p99= 6310.8µs max= 6310.8µs 282 MiB/s cpu=3.205ns/B datagrams=1793 retransmitted=2 +loss1/udp/2m n=1 p50= 6088.1µs p99= 6088.1µs max= 6088.1µs 305 MiB/s cpu=2.876ns/B datagrams=1793 retransmitted=2 == bcast: one sender, 8 receivers on one listener == -bcast/udp/2k n=65536 p50= 17.2µs p99= 316.4µs max= 2276.3µs 560 MiB/s cpu=1.777ns/B window=256 -bcast/udp/64k n=16384 p50= 207.7µs p99= 344.0µs max= 356.5µs 1414 MiB/s cpu=0.753ns/B window=146 -bcast/udp/2m n=512 p50= 1315.6µs p99= 2859.3µs max= 4793.0µs 6817 MiB/s cpu=0.212ns/B window=4 +bcast/udp/2k n=65536 p50= 16.5µs p99= 30.6µs max= 1698.3µs 557 MiB/s cpu=1.803ns/B window=256 +bcast/udp/64k n=16384 p50= 180.8µs p99= 345.8µs max= 364.5µs 1397 MiB/s cpu=0.764ns/B window=146 +bcast/udp/2m n=512 p50= 1324.7µs p99= 4800.1µs max= 6321.8µs 7002 MiB/s cpu=0.200ns/B window=4 diff --git a/crates/flux-network/benches/results/udp-uring/main-3.txt b/crates/flux-network/benches/results/udp-uring/main-3.txt index b3b0b96..10e7643 100644 --- a/crates/flux-network/benches/results/udp-uring/main-3.txt +++ b/crates/flux-network/benches/results/udp-uring/main-3.txt @@ -1,17 +1,17 @@ == paced: one message per 100µs, one receiver == -paced/udp/2k n=2000 p50= 4.7µs p99= 5.4µs max= 48.7µs 20 MiB/s cpu=95.436ns/B -paced/udp/64k n=2000 p50= 8.9µs p99= 435.7µs max= 2374.7µs 625 MiB/s cpu=2.957ns/B -paced/udp/2m n=2000 p50= 216.5µs p99= 283.5µs max= 2549.4µs 7608 MiB/s cpu=0.189ns/B +paced/udp/2k n=2000 p50= 4.7µs p99= 5.6µs max= 67.3µs 20 MiB/s cpu=95.459ns/B +paced/udp/64k n=2000 p50= 9.0µs p99= 462.5µs max= 2416.8µs 625 MiB/s cpu=2.958ns/B +paced/udp/2m n=2000 p50= 237.4µs p99= 252.5µs max= 2741.8µs 7299 MiB/s cpu=0.199ns/B == burst: bounded outstanding, one receiver == -burst/udp/2k n=65536 p50= 4.7µs p99= 6.4µs max= 2735.1µs 442 MiB/s cpu=2.421ns/B window=256 -burst/udp/64k n=16384 p50= 9.0µs p99= 142.2µs max= 2771.2µs 6891 MiB/s cpu=0.201ns/B window=146 -burst/udp/2m n=512 p50= 211.9µs p99= 224.4µs max= 2815.5µs 7809 MiB/s cpu=0.182ns/B window=4 +burst/udp/2k n=65536 p50= 4.8µs p99= 5.8µs max= 2555.0µs 442 MiB/s cpu=2.420ns/B window=256 +burst/udp/64k n=16384 p50= 9.1µs p99= 12.4µs max= 3446.5µs 6758 MiB/s cpu=0.207ns/B window=146 +burst/udp/2m n=512 p50= 237.9µs p99= 268.4µs max= 3076.9µs 7231 MiB/s cpu=0.199ns/B window=4 == loss1: one 2 MiB message, exactly one datagram dropped by a relay == -loss1/udp/2m n=1 p50= 6172.2µs p99= 6172.2µs max= 6172.2µs 288 MiB/s cpu=3.067ns/B datagrams=1793 retransmitted=2 +loss1/udp/2m n=1 p50= 6116.4µs p99= 6116.4µs max= 6116.4µs 294 MiB/s cpu=2.987ns/B datagrams=1793 retransmitted=2 == bcast: one sender, 8 receivers on one listener == -bcast/udp/2k n=65536 p50= 17.0µs p99= 37.7µs max= 2368.8µs 563 MiB/s cpu=1.773ns/B window=256 -bcast/udp/64k n=16384 p50= 211.4µs p99= 342.6µs max= 990.3µs 1410 MiB/s cpu=0.758ns/B window=146 -bcast/udp/2m n=512 p50= 1372.5µs p99= 5468.6µs max= 7813.7µs 6820 MiB/s cpu=0.210ns/B window=4 +bcast/udp/2k n=65536 p50= 16.8µs p99= 182.7µs max= 2640.5µs 567 MiB/s cpu=1.759ns/B window=256 +bcast/udp/64k n=16384 p50= 187.4µs p99= 347.2µs max= 358.7µs 1391 MiB/s cpu=0.775ns/B window=146 +bcast/udp/2m n=512 p50= 1333.8µs p99= 2664.3µs max= 4598.8µs 6800 MiB/s cpu=0.209ns/B window=4 diff --git a/crates/flux-network/benches/results/udp-uring/main-4.txt b/crates/flux-network/benches/results/udp-uring/main-4.txt index 6311833..a66aa13 100644 --- a/crates/flux-network/benches/results/udp-uring/main-4.txt +++ b/crates/flux-network/benches/results/udp-uring/main-4.txt @@ -1,17 +1,17 @@ == paced: one message per 100µs, one receiver == -paced/udp/2k n=2000 p50= 4.7µs p99= 6.2µs max= 56.6µs 20 MiB/s cpu=95.439ns/B -paced/udp/64k n=2000 p50= 8.9µs p99= 340.1µs max= 2285.9µs 625 MiB/s cpu=2.960ns/B -paced/udp/2m n=2000 p50= 233.8µs p99= 288.9µs max= 3134.6µs 7361 MiB/s cpu=0.196ns/B +paced/udp/2k n=2000 p50= 4.8µs p99= 6.1µs max= 128.9µs 20 MiB/s cpu=95.432ns/B +paced/udp/64k n=2000 p50= 9.0µs p99= 452.2µs max= 2405.3µs 625 MiB/s cpu=2.958ns/B +paced/udp/2m n=2000 p50= 237.8µs p99= 249.5µs max= 2569.9µs 7293 MiB/s cpu=0.198ns/B == burst: bounded outstanding, one receiver == -burst/udp/2k n=65536 p50= 4.8µs p99= 5.8µs max= 2663.3µs 445 MiB/s cpu=2.403ns/B window=256 -burst/udp/64k n=16384 p50= 9.1µs p99= 12.2µs max= 3505.6µs 6857 MiB/s cpu=0.203ns/B window=146 -burst/udp/2m n=512 p50= 231.1µs p99= 259.1µs max= 3046.1µs 7335 MiB/s cpu=0.195ns/B window=4 +burst/udp/2k n=65536 p50= 4.8µs p99= 5.8µs max= 2963.3µs 442 MiB/s cpu=2.396ns/B window=256 +burst/udp/64k n=16384 p50= 9.0µs p99= 11.6µs max= 2762.5µs 6887 MiB/s cpu=0.202ns/B window=146 +burst/udp/2m n=512 p50= 220.9µs p99= 237.2µs max= 3897.4µs 7670 MiB/s cpu=0.185ns/B window=4 == loss1: one 2 MiB message, exactly one datagram dropped by a relay == -loss1/udp/2m n=1 p50= 6141.2µs p99= 6141.2µs max= 6141.2µs 293 MiB/s cpu=3.060ns/B datagrams=1793 retransmitted=2 +loss1/udp/2m n=1 p50= 6037.8µs p99= 6037.8µs max= 6037.8µs 308 MiB/s cpu=2.840ns/B datagrams=1793 retransmitted=2 == bcast: one sender, 8 receivers on one listener == -bcast/udp/2k n=65536 p50= 16.6µs p99= 34.4µs max= 2306.9µs 561 MiB/s cpu=1.784ns/B window=256 -bcast/udp/64k n=16384 p50= 182.9µs p99= 345.6µs max= 361.7µs 1398 MiB/s cpu=0.768ns/B window=146 -bcast/udp/2m n=512 p50= 1344.5µs p99= 4861.2µs max= 7029.2µs 6895 MiB/s cpu=0.206ns/B window=4 +bcast/udp/2k n=65536 p50= 16.6µs p99= 30.6µs max= 1736.9µs 562 MiB/s cpu=1.778ns/B window=256 +bcast/udp/64k n=16384 p50= 211.7µs p99= 342.1µs max= 974.9µs 1410 MiB/s cpu=0.754ns/B window=146 +bcast/udp/2m n=512 p50= 1389.6µs p99= 2275.6µs max= 3802.8µs 6855 MiB/s cpu=0.207ns/B window=4 diff --git a/crates/flux-network/benches/results/udp-uring/main-5.txt b/crates/flux-network/benches/results/udp-uring/main-5.txt index afcd501..be69ba8 100644 --- a/crates/flux-network/benches/results/udp-uring/main-5.txt +++ b/crates/flux-network/benches/results/udp-uring/main-5.txt @@ -1,17 +1,17 @@ == paced: one message per 100µs, one receiver == -paced/udp/2k n=2000 p50= 4.7µs p99= 5.7µs max= 75.5µs 20 MiB/s cpu=95.411ns/B -paced/udp/64k n=2000 p50= 9.1µs p99= 356.5µs max= 2306.2µs 625 MiB/s cpu=2.959ns/B -paced/udp/2m n=2000 p50= 229.2µs p99= 260.9µs max= 2679.1µs 7489 MiB/s cpu=0.192ns/B +paced/udp/2k n=2000 p50= 4.7µs p99= 6.0µs max= 62.9µs 20 MiB/s cpu=95.431ns/B +paced/udp/64k n=2000 p50= 9.1µs p99= 465.4µs max= 2417.8µs 625 MiB/s cpu=2.958ns/B +paced/udp/2m n=2000 p50= 228.9µs p99= 244.0µs max= 2751.7µs 7530 MiB/s cpu=0.190ns/B == burst: bounded outstanding, one receiver == -burst/udp/2k n=65536 p50= 4.8µs p99= 5.8µs max= 2491.4µs 442 MiB/s cpu=2.413ns/B window=256 -burst/udp/64k n=16384 p50= 9.2µs p99= 11.7µs max= 3150.8µs 6642 MiB/s cpu=0.212ns/B window=146 -burst/udp/2m n=512 p50= 239.6µs p99= 301.6µs max= 2976.3µs 7107 MiB/s cpu=0.203ns/B window=4 +burst/udp/2k n=65536 p50= 4.8µs p99= 5.8µs max= 2648.4µs 440 MiB/s cpu=2.420ns/B window=256 +burst/udp/64k n=16384 p50= 9.2µs p99= 11.9µs max= 3298.6µs 6659 MiB/s cpu=0.211ns/B window=146 +burst/udp/2m n=512 p50= 234.5µs p99= 241.0µs max= 2772.8µs 7285 MiB/s cpu=0.197ns/B window=4 == loss1: one 2 MiB message, exactly one datagram dropped by a relay == -loss1/udp/2m n=1 p50= 6353.6µs p99= 6353.6µs max= 6353.6µs 278 MiB/s cpu=3.233ns/B datagrams=1793 retransmitted=2 +loss1/udp/2m n=1 p50= 6074.5µs p99= 6074.5µs max= 6074.5µs 298 MiB/s cpu=2.937ns/B datagrams=1793 retransmitted=2 == bcast: one sender, 8 receivers on one listener == -bcast/udp/2k n=65536 p50= 17.0µs p99= 30.3µs max= 65.5µs 562 MiB/s cpu=1.784ns/B window=256 -bcast/udp/64k n=16384 p50= 197.1µs p99= 345.9µs max= 382.8µs 1398 MiB/s cpu=0.769ns/B window=146 -bcast/udp/2m n=512 p50= 1379.3µs p99= 3527.3µs max= 5245.0µs 6794 MiB/s cpu=0.211ns/B window=4 +bcast/udp/2k n=65536 p50= 17.2µs p99= 152.1µs max= 2640.8µs 560 MiB/s cpu=1.784ns/B window=256 +bcast/udp/64k n=16384 p50= 202.0µs p99= 345.8µs max= 372.5µs 1398 MiB/s cpu=0.770ns/B window=146 +bcast/udp/2m n=512 p50= 1372.5µs p99= 4693.6µs max= 6457.7µs 6845 MiB/s cpu=0.207ns/B window=4 diff --git a/crates/flux-network/src/udp/connector.rs b/crates/flux-network/src/udp/connector.rs index 2f595dd..1aecd4a 100644 --- a/crates/flux-network/src/udp/connector.rs +++ b/crates/flux-network/src/udp/connector.rs @@ -615,7 +615,7 @@ impl UdpManager { self.tick(now); } #[cfg(target_os = "linux")] - if matches!(self.udp.io, super::UdpIo::Uring(_)) { + if let super::UdpIo::Uring(config) = self.udp.io { for k in 0..self.sockets.len() { // Drain in bounded passes, emitting ACKs between passes so // slow callbacks cannot hold back the sender's whole window. @@ -623,7 +623,8 @@ impl UdpManager { let work = self.sockets[k].socket.ring.as_ref().unwrap().borrow_mut().poll(); o |= work; let mut received_any = false; - loop { + // ACK processing can reap more receives while sending. + for _ in 0..config.recv_entries { let received = self.sockets[k].socket.ring.as_ref().unwrap().borrow_mut().receive(); let Some(received) = received else { break }; diff --git a/crates/flux-network/src/udp/peer.rs b/crates/flux-network/src/udp/peer.rs index 7b69c7c..abf7146 100644 --- a/crates/flux-network/src/udp/peer.rs +++ b/crates/flux-network/src/udp/peer.rs @@ -876,9 +876,12 @@ impl UdpPeer { } .encode(&mut self.ctrl); let n = self.rx.write_bitmap(n_bits, &mut self.ctrl[HEADER_SIZE..]); - self.ack_due = false; - self.last_send = now; - send_datagram(socket, self.addr, &self.ctrl[..HEADER_SIZE + n]) + let outcome = send_datagram(socket, self.addr, &self.ctrl[..HEADER_SIZE + n]); + self.ack_due = outcome == SendOutcome::WouldBlock; + if outcome == SendOutcome::Done { + self.last_send = now; + } + outcome } /// Stages the message in store `slot`. @@ -1156,6 +1159,43 @@ impl UdpPeer { mod tests { use super::*; + #[cfg(target_os = "linux")] + #[test] + fn blocked_ack_is_retried_after_send_capacity_returns() { + use std::os::fd::AsRawFd; + + use crate::udp::{UringConfig, sys::uring::Ring}; + + let remote = std::net::UdpSocket::bind("127.0.0.1:0").unwrap(); + remote.set_read_timeout(Some(std::time::Duration::from_secs(2))).unwrap(); + let addr = remote.local_addr().unwrap(); + let (mut socket, _) = sock(); + socket.ring = Some(std::cell::RefCell::new( + Ring::new(socket.as_raw_fd(), UringConfig { send_entries: 1, recv_entries: 1 }) + .unwrap(), + )); + let mut peer = UdpPeer::new(addr, Token(1), Token(0), 7, cfg(), None); + socket.send_to(b"occupy", addr).unwrap(); + assert_eq!(peer.send_ack(&socket, Instant::now()), SendOutcome::WouldBlock); + assert!(peer.take_ack_due(), "backpressure must retain the pending ACK"); + let deadline = std::time::Instant::now() + std::time::Duration::from_secs(2); + loop { + socket.ring.as_ref().unwrap().borrow_mut().poll(); + if peer.send_ack(&socket, Instant::now()) == SendOutcome::Done { + break; + } + assert!(std::time::Instant::now() < deadline); + } + assert!(!peer.take_ack_due()); + socket.ring.as_ref().unwrap().borrow_mut().submit(); + let mut bytes = [0; 1200]; + assert_eq!(remote.recv(&mut bytes).unwrap(), 6); + let n = remote.recv(&mut bytes).unwrap(); + let header = Header::decode(&bytes[..n]).unwrap(); + assert_eq!(header.kind, Kind::Ack); + assert_eq!(header.session, 7); + } + fn cfg() -> UdpConfig { UdpConfig { send_window: 64, diff --git a/crates/flux-network/src/udp/sys.rs b/crates/flux-network/src/udp/sys.rs index d59a591..78f677a 100644 --- a/crates/flux-network/src/udp/sys.rs +++ b/crates/flux-network/src/udp/sys.rs @@ -1,4 +1,5 @@ -//! Batched datagram syscalls. Linux uses `sendmmsg`/`recvmmsg`; elsewhere the +//! Datagram I/O with an optional Linux completion backend. +//! Batched syscalls use `sendmmsg`/`recvmmsg` on Linux; elsewhere the //! same API loops over `sendmsg`/`recvmsg`. Both are non-blocking and send or //! receive whatever is available right now: there is no accumulation delay. diff --git a/crates/flux-network/tests/udp_uring.rs b/crates/flux-network/tests/udp_uring.rs index a991a4a..e81e66a 100644 --- a/crates/flux-network/tests/udp_uring.rs +++ b/crates/flux-network/tests/udp_uring.rs @@ -8,6 +8,45 @@ const IO: flux_network::udp::UdpIo = include!("support/udp_connector.rs"); +#[test] +fn duplex_progress_with_minimum_ring_capacity() { + let config = UdpConfig { + io: flux_network::udp::UdpIo::Uring(flux_network::udp::UringConfig { + send_entries: 1, + recv_entries: 1, + }), + ..UdpConfig::lan() + }; + let mut server = NetworkDriver::default().with_transport(Transport::Udp(config)); + let mut client = NetworkDriver::default().with_transport(Transport::Udp(config)); + let (accepted, outbound) = connect_pair(&mut server, &mut client, free_addr()); + for id in 0..32 { + let payload = make_msg(id, 64 * 1024); + server.write_or_enqueue_with(SendBehavior::Single(accepted), |buf| { + buf.extend_from_slice(&payload); + }); + client.write_or_enqueue_with(SendBehavior::Single(outbound), |buf| { + buf.extend_from_slice(&payload); + }); + let (mut server_got, mut client_got) = (false, false); + let deadline = Instant::now() + Duration::from_secs(5); + while !server_got || !client_got { + for (driver, got) in [(&mut server, &mut server_got), (&mut client, &mut client_got)] { + driver.poll_with(|event| match event { + PollEvent::Message { payload: bytes, .. } => { + assert_eq!(bytes, payload); + assert!(!*got); + *got = true; + } + PollEvent::Disconnect { .. } => panic!("duplex peer disconnected"), + _ => {} + }); + } + assert!(Instant::now() < deadline, "duplex message {id} stalled"); + } + } +} + #[test] fn mixed_backends_interoperate() { for server_uring in [false, true] { From 7c7b3eab721b81c6635d40efaabbfa975e9f071a Mon Sep 17 00:00:00 2001 From: gd-0 Date: Tue, 8 Sep 2026 22:09:16 +0100 Subject: [PATCH 3/6] chore(network): remove checked-in io_uring benchmark reports --- crates/flux-network/UDP_URING.md | 164 ------------------ .../udp-uring/baseline-benchmark.patch | 88 ---------- .../benches/results/udp-uring/branch-1.txt | 27 --- .../benches/results/udp-uring/branch-2.txt | 27 --- .../benches/results/udp-uring/branch-3.txt | 27 --- .../benches/results/udp-uring/branch-4.txt | 27 --- .../benches/results/udp-uring/branch-5.txt | 27 --- .../benches/results/udp-uring/main-1.txt | 17 -- .../benches/results/udp-uring/main-2.txt | 17 -- .../benches/results/udp-uring/main-3.txt | 17 -- .../benches/results/udp-uring/main-4.txt | 17 -- .../benches/results/udp-uring/main-5.txt | 17 -- 12 files changed, 472 deletions(-) delete mode 100644 crates/flux-network/UDP_URING.md delete mode 100644 crates/flux-network/benches/results/udp-uring/baseline-benchmark.patch delete mode 100644 crates/flux-network/benches/results/udp-uring/branch-1.txt delete mode 100644 crates/flux-network/benches/results/udp-uring/branch-2.txt delete mode 100644 crates/flux-network/benches/results/udp-uring/branch-3.txt delete mode 100644 crates/flux-network/benches/results/udp-uring/branch-4.txt delete mode 100644 crates/flux-network/benches/results/udp-uring/branch-5.txt delete mode 100644 crates/flux-network/benches/results/udp-uring/main-1.txt delete mode 100644 crates/flux-network/benches/results/udp-uring/main-2.txt delete mode 100644 crates/flux-network/benches/results/udp-uring/main-3.txt delete mode 100644 crates/flux-network/benches/results/udp-uring/main-4.txt delete mode 100644 crates/flux-network/benches/results/udp-uring/main-5.txt diff --git a/crates/flux-network/UDP_URING.md b/crates/flux-network/UDP_URING.md deleted file mode 100644 index 662ccdb..0000000 --- a/crates/flux-network/UDP_URING.md +++ /dev/null @@ -1,164 +0,0 @@ -# UDP io_uring - -An opt-in Linux backend for `NetworkDriver`. TCP and the default UDP syscall -backend retain their existing behavior. - -```rust -use flux_network::{NetworkDriver, Transport, UdpConfig}; -use flux_network::udp::{UdpIo, UringConfig}; - -let mut network = NetworkDriver::default() - .with_transport(Transport::Udp(UdpConfig { - io: UdpIo::Uring(UringConfig::default()), - ..UdpConfig::lan() - })) - .with_socket_buf_size(16 * 1024 * 1024); -// Use connect/listen_at, write_or_enqueue_with and poll_with as usual. -``` - -Both peers may choose their backend independently. There is no wire-format -change. Selecting io_uring is explicit: socket creation returns `None` and logs -the setup error if the kernel denies or lacks the required facilities. Linux -6.0+ is required for multishot recvmsg and synchronous cancellation; kernel -security policy may still disable them. - -## Implementation and costs - -- One ring per UDP socket, with one multishot receive and a provided-buffer ring. - GRO entries use the existing datagram validation and reassembly path. -- Ordinary io_uring `SendMsg` operations carry both data and control packets. - Compatible runs within a batch retain GSO, including two-fragment messages - and batches crossing peer boundaries. GSO failure retries the original - datagrams without segmentation. -- Each pending send owns a pooled copy of its bytes. Admission to this bounded - queue counts as acceptance by the local socket backend. Completion errors - become packet loss, handled by the existing reliability protocol. Neither - ACK processing nor reconnect can invalidate kernel-owned bytes. -- Each poll processes at most 32 receive passes per socket, each capped at - `recv_entries` buffers (a GRO buffer can contain several datagrams). - ACKs are emitted between receive passes so - callback processing cannot defer them behind a whole send window. Buffer - exhaustion ends/rearms the multishot request after buffers are recycled. - An ACK blocked by send-queue capacity remains pending for retry. -- Normal polling uses no waiting for completions and skips idle kernel entries - using the task-work flag. There is no async executor, SQPOLL thread, or - zero-copy send machinery. -- Moving an active driver to another thread synchronously retires outstanding - requests before rearming them on the new issuer. Drop also synchronously - cancels requests before releasing buffers and the socket. Bind on the worker - that will drive I/O to avoid handoff work. If kernel cancellation fails, - borrowed allocations are retained rather than freed underneath the kernel. - -Default limits are 64 send slots and 32 receive buffers: approximately 6 MiB -per socket, plus ring metadata and the existing protocol state. A GSO group -uses one send slot. Counts are configurable; receive count must be a power of -two, both counts must be nonzero, and their sum must not exceed 4096. Ring -capacity is separate from the protocol's send/receive windows and kernel -socket-buffer limits. UDP still has no receiver flow control. - -The workspace already depended on `io-uring`; `flux-network` now uses the same -version as a Linux-only dependency. The new public `UdpConfig::io` field means -exhaustive struct literals need an update; literals using `..Default::default()` -or `..UdpConfig::lan()` keep compiling. Account for this source compatibility -change when preparing the next release. This branch does not bump versions. - -## Measurements - -Measured on 2026-09-08 against main -`e59fe83be3d31f29b5a149d736bc0c5cb399cbea`, which already includes UDP GSO/GRO. - -AMD Ryzen 9 9950X, Linux `7.1.5-arch1-2`, Rust `1.91.0`, release optimization, -`target-cpu=native`. IPv4 loopback; sender on CPU 30 and receiver on CPU 31 -(different physical cores). Eight broadcast receivers share the receiver -thread. Socket buffers request 16 MiB; this host's send/receive sysctl maxima -are 4 MiB (Linux reports doubled effective socket accounting limits). - -Samples were refreshed after the ACK-backpressure and bounded-drain audit fixes. -Five measured rounds followed one discarded run of each binary. Main and -branch alternate order; UDP and io_uring also alternate order within the -branch binary. No builds or tests ran concurrently with the measured rounds. -The isolated main build changes only the benchmark harness, using the saved -[benchmark patch](benches/results/udp-uring/baseline-benchmark.patch); its -networking implementation is unchanged. - -These are medians of each run's reported statistic, not pooled percentiles. -`FLUX_BENCH_SCALE=16` gives 65,536 / 16,384 / 512 deliveries for 2 KiB / 64 KiB / -2 MiB burst and broadcast cases. Paced cases have 2,000 messages and a 100 µs -minimum send interval; the large case cannot sustain that interval. The same -outstanding-message bounds apply to both backends. There is no per-scenario -warmup: initial message costs remain in the measurements. Socket setup and -handshake are outside the timed interval. - -Latency is from the transport's wire timestamp to the receive callback; it -excludes serialization before that timestamp. Throughput includes the whole -send/receive interval. CPU ns/B sums sender and receiver thread CPU time, -including their busy polling; it is not an idle-efficiency measurement. - -| Workload | Main MiB/s | io_uring MiB/s | Change | Main p50 / p99 µs | io_uring p50 / p99 µs | Main / io_uring CPU ns/B | -| --- | ---: | ---: | ---: | ---: | ---: | ---: | -| paced/2k | 20 | 20 | +0.0% | 4.7 / 5.6 | 2.4 / 3.4 | 95.432 / 96.388 | -| paced/64k | 625 | 625 | +0.0% | 9.0 / 452.2 | 9.5 / 172.7 | 2.958 / 2.955 | -| paced/2m | 7,430 | 7,408 | -0.3% | 232.5 / 249.5 | 231.6 / 239.1 | 0.193 / 0.200 | -| burst/2k | 442 | 773 | +74.9% | 4.8 / 5.8 | 2.4 / 3.3 | 2.420 / 1.396 | -| burst/64k | 6,849 | 7,178 | +4.8% | 9.1 / 11.7 | 9.5 / 12.7 | 0.203 / 0.188 | -| burst/2m | 7,285 | 7,315 | +0.4% | 234.5 / 254.6 | 229.9 / 236.9 | 0.197 / 0.202 | -| bcast/2k | 561 | 946 | +68.6% | 16.6 / 40.5 | 18.8 / 35.1 | 1.784 / 1.084 | -| bcast/64k | 1,398 | 7,461 | +433.7% | 187.4 / 345.8 | 44.5 / 69.6 | 0.767 / 0.181 | -| bcast/2m | 6,855 | 7,043 | +2.7% | 1,333.8 / 4,693.6 | 3,232.8 / 6,186.5 | 0.207 / 0.233 | - -The strongest gains are 2 KiB bursts (+75% throughput, roughly half median -latency) and 64 KiB broadcasts (5.34x throughput, p99 346 → 70 µs). The latter -also benefits from retaining GSO across mixed-peer batches: these results -measure the complete backend implementation, not an isolated io_uring syscall -substitution. The syscall backend could independently adopt that grouping. - -Large messages remain a tradeoff: 2 MiB burst throughput is within 1% of -main, but 3.0% below the branch's syscall control (7,315 vs 7,541 MiB/s). -Broadcast throughput is 2.7% above main, while median latency increases from -1.33 ms to 3.23 ms and CPU cost rises from 0.207 to 0.233 ns/B. The extra send -copy and different batching/scheduling costs remain. This is why io_uring -stays opt-in. Paced p99 varies substantially between runs; consult the raw -samples rather than treating a median as a guarantee. - -The single-loss relay case completed in a median 6.09 ms on main and 5.39 ms -with io_uring. Every run observed 1,793 data datagrams, including 2 repeated -sequences. Each run sends only one message, so this is a recovery check, not a -useful p99 measurement. Relay CPU is excluded from the CPU column. - -All raw runs are in [benches/results/udp-uring](benches/results/udp-uring). -`main-N.txt` is the isolated main build; `branch-N.txt` includes both backends -on this branch. The branch's syscall control preserves similar throughput -(e.g. 441 vs 442 MiB/s for 2 KiB bursts, 1,408 vs 1,398 for 64 KiB broadcasts). -These are local loopback measurements, not physical-NIC, WAN, or AWS results. -Non-Linux builds were not cross-compiled. - -## Reproduce and validate - -```sh -FLUX_BENCH_TRANSPORT=udp,uring FLUX_BENCH_SCALE=16 \ - cargo bench -p flux-network --bench udp_pipeline -FLUX_BENCH_TRANSPORT=udp,uring FLUX_BENCH_SCALE=16 FLUX_BENCH_REVERSE=1 \ - cargo bench -p flux-network --bench udp_pipeline -``` - -To reconstruct the main baseline, extract the commit above into a separate -directory with `git archive`, apply `baseline-benchmark.patch` there with -`patch -p1`, and run the same benchmark with `FLUX_BENCH_TRANSPORT=udp`. -The patch only adds filtering, sample scaling, and CPU-time instrumentation. -The branch also accepts `FLUX_BENCH_SIZE=2m` to restrict size-dependent cases. - -Validation passed: - -- `just fmt` -- `just clippy` -- `cargo test --workspace --all-features --locked`: 364 passed, 4 existing - ignored documentation tests, no failures. - -Both backends run the same UDP integration suite. Additional coverage checks -mixed-backend peers, DCache delivery, sustained large messages with slow debug -callbacks, owned send buffers, queue saturation, IPv4/IPv6, mixed GSO groups, -fallback, GRO validation, receive-pool exhaustion, ACK retry under send -backpressure, duplex delivery with one send/receive slot, 16-bit descriptor-tail -wraparound, and driver movement across threads. Tests require local socket -and io_uring permissions. Local runs used `RUSTC_WRAPPER=` and a temporary -`CARGO_TARGET_DIR` because the shared build cache was sandbox-restricted. diff --git a/crates/flux-network/benches/results/udp-uring/baseline-benchmark.patch b/crates/flux-network/benches/results/udp-uring/baseline-benchmark.patch deleted file mode 100644 index 0f5fbe2..0000000 --- a/crates/flux-network/benches/results/udp-uring/baseline-benchmark.patch +++ /dev/null @@ -1,88 +0,0 @@ ---- a/crates/flux-network/benches/udp_pipeline.rs -+++ b/crates/flux-network/benches/udp_pipeline.rs -@@ -13,0 +14,5 @@ -+//! `FLUX_BENCH_TRANSPORT=udp,uring` selects backends; `FLUX_BENCH_REVERSE=1` -+//! reverses their order. `FLUX_BENCH_SCALE=16` increases burst/broadcast sample -+//! counts. `FLUX_BENCH_SIZE=2m` filters paced/burst/broadcast sizes. CPU ns/B -+//! sums sender and receiver thread CPU time; it excludes the loss relay. -+//! -@@ -56,2 +61,10 @@ --fn transports() -> [(&'static str, Transport); 2] { -- [("tcp", Transport::default()), ("udp", Transport::Udp(udp_config()))] -+fn transports() -> Vec<(&'static str, Transport)> { -+ let mut transports = vec![("tcp", Transport::default()), ("udp", Transport::Udp(udp_config()))]; -+ if let Ok(filter) = std::env::var("FLUX_BENCH_TRANSPORT") { -+ transports.retain(|(name, _)| filter.split(',').any(|selected| selected == *name)); -+ assert!(!transports.is_empty(), "unknown FLUX_BENCH_TRANSPORT"); -+ } -+ if std::env::var_os("FLUX_BENCH_REVERSE").is_some() { -+ transports.reverse(); -+ } -+ transports -@@ -68 +81,4 @@ -- (count, window) -+ let scale: usize = std::env::var("FLUX_BENCH_SCALE") -+ .map_or(1, |s| s.parse().expect("invalid FLUX_BENCH_SCALE")); -+ assert!((1..=64).contains(&scale)); -+ (count * scale, window) -@@ -74,0 +91 @@ -+ cpu: Duration, -@@ -84 +101 @@ -- "{name:<28} n={:<6} p50={:>9.1}µs p99={:>9.1}µs max={:>9.1}µs {mibps:>9.0} MiB/s {extra}", -+ "{name:<28} n={:<6} p50={:>9.1}µs p99={:>9.1}µs max={:>9.1}µs {mibps:>9.0} MiB/s cpu={:.3}ns/B {extra}", -@@ -88,0 +106 @@ -+ self.cpu.as_nanos() as f64 / self.bytes as f64, -@@ -136,0 +155,11 @@ -+} -+ -+fn thread_cpu_time() -> Duration { -+ let mut time: libc::timespec = unsafe { std::mem::zeroed() }; -+ assert_eq!( -+ unsafe { -+ libc::clock_gettime(libc::CLOCK_THREAD_CPUTIME_ID, std::ptr::from_mut(&mut time)) -+ }, -+ 0 -+ ); -+ Duration::new(time.tv_sec as u64, time.tv_nsec as u32) -@@ -155,0 +185 @@ -+ let cpu_start = thread_cpu_time(); -@@ -167 +197 @@ -- lat -+ (lat, thread_cpu_time() - cpu_start) -@@ -172,0 +203 @@ -+ let cpu_start = thread_cpu_time(); -@@ -191,3 +222,4 @@ -- let latencies_ns = rx_thread.join().unwrap(); -- assert_eq!(latencies_ns.len(), expected, "receiver timed out"); -- Stats { latencies_ns, elapsed, bytes: expected * size } -+ let sender_cpu = thread_cpu_time() - cpu_start; -+ let (latencies_ns, receiver_cpu) = rx_thread.join().unwrap(); -+ assert_eq!(latencies_ns.len(), expected, "receiver timed out after {sent} sends"); -+ Stats { latencies_ns, elapsed, bytes: expected * size, cpu: sender_cpu + receiver_cpu } -@@ -279 +311,4 @@ -- for (size_name, size) in SIZES { -+ for (size_name, size) in SIZES -+ .into_iter() -+ .filter(|(name, _)| std::env::var("FLUX_BENCH_SIZE").ok().is_none_or(|s| s == *name)) -+ { -@@ -297 +332,4 @@ -- for (size_name, size) in SIZES { -+ for (size_name, size) in SIZES -+ .into_iter() -+ .filter(|(name, _)| std::env::var("FLUX_BENCH_SIZE").ok().is_none_or(|s| s == *name)) -+ { -@@ -315,0 +354,2 @@ -+ for (name, transport) in -+ transports().into_iter().filter(|(_, t)| matches!(t, Transport::Udp(_))) -@@ -321 +361 @@ -- transport: Transport::Udp(udp_config()), -+ transport, -@@ -333 +373 @@ -- s.row("loss1/udp/2m", &format!("datagrams={data} retransmitted={retx}")); -+ s.row(&format!("loss1/{name}/2m"), &format!("datagrams={data} retransmitted={retx}")); -@@ -337 +377,4 @@ -- for (size_name, size) in SIZES { -+ for (size_name, size) in SIZES -+ .into_iter() -+ .filter(|(name, _)| std::env::var("FLUX_BENCH_SIZE").ok().is_none_or(|s| s == *name)) -+ { diff --git a/crates/flux-network/benches/results/udp-uring/branch-1.txt b/crates/flux-network/benches/results/udp-uring/branch-1.txt deleted file mode 100644 index 554908b..0000000 --- a/crates/flux-network/benches/results/udp-uring/branch-1.txt +++ /dev/null @@ -1,27 +0,0 @@ -== paced: one message per 100µs, one receiver == -paced/uring/2k n=2000 p50= 2.4µs p99= 3.4µs max= 80.7µs 20 MiB/s cpu=96.401ns/B -paced/udp/2k n=2000 p50= 4.8µs p99= 384.1µs max= 2380.2µs 20 MiB/s cpu=94.905ns/B -paced/uring/64k n=2000 p50= 9.5µs p99= 64.3µs max= 2030.5µs 625 MiB/s cpu=2.955ns/B -paced/udp/64k n=2000 p50= 9.2µs p99= 760.4µs max= 2701.4µs 625 MiB/s cpu=2.956ns/B -paced/uring/2m n=2000 p50= 232.9µs p99= 242.3µs max= 2234.0µs 7353 MiB/s cpu=0.201ns/B -paced/udp/2m n=2000 p50= 210.0µs p99= 233.7µs max= 3326.9µs 7960 MiB/s cpu=0.179ns/B - -== burst: bounded outstanding, one receiver == -burst/uring/2k n=65536 p50= 2.4µs p99= 3.3µs max= 2519.5µs 772 MiB/s cpu=1.396ns/B window=256 -burst/udp/2k n=65536 p50= 4.8µs p99= 5.8µs max= 3429.4µs 445 MiB/s cpu=2.395ns/B window=256 -burst/uring/64k n=16384 p50= 9.5µs p99= 12.1µs max= 3058.8µs 7106 MiB/s cpu=0.191ns/B window=146 -burst/udp/64k n=16384 p50= 9.1µs p99= 11.8µs max= 3436.4µs 6846 MiB/s cpu=0.204ns/B window=146 -burst/uring/2m n=512 p50= 229.9µs p99= 236.9µs max= 2884.9µs 7315 MiB/s cpu=0.202ns/B window=4 -burst/udp/2m n=512 p50= 227.7µs p99= 244.0µs max= 2952.3µs 7318 MiB/s cpu=0.195ns/B window=4 - -== loss1: one 2 MiB message, exactly one datagram dropped by a relay == -loss1/uring/2m n=1 p50= 5413.6µs p99= 5413.6µs max= 5413.6µs 351 MiB/s cpu=3.289ns/B datagrams=1793 retransmitted=2 -loss1/udp/2m n=1 p50= 8873.2µs p99= 8873.2µs max= 8873.2µs 221 MiB/s cpu=5.179ns/B datagrams=1793 retransmitted=2 - -== bcast: one sender, 8 receivers on one listener == -bcast/uring/2k n=65536 p50= 18.7µs p99= 34.9µs max= 78.3µs 938 MiB/s cpu=1.093ns/B window=256 -bcast/udp/2k n=65536 p50= 17.3µs p99= 30.9µs max= 1789.7µs 565 MiB/s cpu=1.765ns/B window=256 -bcast/uring/64k n=16384 p50= 44.5µs p99= 69.6µs max= 128.7µs 7443 MiB/s cpu=0.181ns/B window=146 -bcast/udp/64k n=16384 p50= 210.3µs p99= 342.0µs max= 2528.8µs 1413 MiB/s cpu=0.749ns/B window=146 -bcast/uring/2m n=512 p50= 3318.4µs p99= 6161.2µs max= 6783.3µs 7043 MiB/s cpu=0.233ns/B window=4 -bcast/udp/2m n=512 p50= 1360.5µs p99= 3951.8µs max= 4640.5µs 6980 MiB/s cpu=0.201ns/B window=4 diff --git a/crates/flux-network/benches/results/udp-uring/branch-2.txt b/crates/flux-network/benches/results/udp-uring/branch-2.txt deleted file mode 100644 index 3c823a8..0000000 --- a/crates/flux-network/benches/results/udp-uring/branch-2.txt +++ /dev/null @@ -1,27 +0,0 @@ -== paced: one message per 100µs, one receiver == -paced/udp/2k n=2000 p50= 4.7µs p99= 6.0µs max= 54.2µs 20 MiB/s cpu=95.431ns/B -paced/uring/2k n=2000 p50= 2.4µs p99= 4.1µs max= 1080.5µs 20 MiB/s cpu=96.127ns/B -paced/udp/64k n=2000 p50= 8.9µs p99= 498.6µs max= 2451.9µs 625 MiB/s cpu=2.957ns/B -paced/uring/64k n=2000 p50= 9.5µs p99= 434.7µs max= 2378.5µs 625 MiB/s cpu=2.953ns/B -paced/udp/2m n=2000 p50= 240.2µs p99= 256.3µs max= 3402.6µs 7198 MiB/s cpu=0.201ns/B -paced/uring/2m n=2000 p50= 219.9µs p99= 253.7µs max= 2659.0µs 7689 MiB/s cpu=0.194ns/B - -== burst: bounded outstanding, one receiver == -burst/udp/2k n=65536 p50= 4.8µs p99= 5.8µs max= 2664.1µs 441 MiB/s cpu=2.437ns/B window=256 -burst/uring/2k n=65536 p50= 2.4µs p99= 3.2µs max= 2444.8µs 773 MiB/s cpu=1.400ns/B window=256 -burst/udp/64k n=16384 p50= 9.2µs p99= 11.8µs max= 3550.8µs 6735 MiB/s cpu=0.208ns/B window=146 -burst/uring/64k n=16384 p50= 9.4µs p99= 12.2µs max= 3002.8µs 7365 MiB/s cpu=0.181ns/B window=146 -burst/udp/2m n=512 p50= 222.0µs p99= 239.7µs max= 3490.5µs 7484 MiB/s cpu=0.189ns/B window=4 -burst/uring/2m n=512 p50= 218.8µs p99= 229.5µs max= 2668.9µs 7667 MiB/s cpu=0.193ns/B window=4 - -== loss1: one 2 MiB message, exactly one datagram dropped by a relay == -loss1/udp/2m n=1 p50= 6207.4µs p99= 6207.4µs max= 6207.4µs 298 MiB/s cpu=2.932ns/B datagrams=1793 retransmitted=2 -loss1/uring/2m n=1 p50= 4580.3µs p99= 4580.3µs max= 4580.3µs 408 MiB/s cpu=2.586ns/B datagrams=1793 retransmitted=2 - -== bcast: one sender, 8 receivers on one listener == -bcast/udp/2k n=65536 p50= 17.2µs p99= 31.4µs max= 1759.9µs 568 MiB/s cpu=1.754ns/B window=256 -bcast/uring/2k n=65536 p50= 18.8µs p99= 35.3µs max= 80.1µs 946 MiB/s cpu=1.085ns/B window=256 -bcast/udp/64k n=16384 p50= 211.5µs p99= 343.5µs max= 1985.4µs 1406 MiB/s cpu=0.756ns/B window=146 -bcast/uring/64k n=16384 p50= 44.6µs p99= 70.1µs max= 140.4µs 7461 MiB/s cpu=0.181ns/B window=146 -bcast/udp/2m n=512 p50= 1388.2µs p99= 5729.2µs max= 7346.6µs 6848 MiB/s cpu=0.208ns/B window=4 -bcast/uring/2m n=512 p50= 1499.2µs p99= 4005.6µs max= 4768.5µs 6911 MiB/s cpu=0.238ns/B window=4 diff --git a/crates/flux-network/benches/results/udp-uring/branch-3.txt b/crates/flux-network/benches/results/udp-uring/branch-3.txt deleted file mode 100644 index 011633e..0000000 --- a/crates/flux-network/benches/results/udp-uring/branch-3.txt +++ /dev/null @@ -1,27 +0,0 @@ -== paced: one message per 100µs, one receiver == -paced/uring/2k n=2000 p50= 2.4µs p99= 3.4µs max= 40.0µs 20 MiB/s cpu=96.437ns/B -paced/udp/2k n=2000 p50= 4.8µs p99= 448.5µs max= 2444.6µs 20 MiB/s cpu=94.849ns/B -paced/uring/64k n=2000 p50= 9.4µs p99= 172.7µs max= 2119.9µs 625 MiB/s cpu=2.956ns/B -paced/udp/64k n=2000 p50= 9.0µs p99= 779.9µs max= 2727.9µs 625 MiB/s cpu=2.956ns/B -paced/uring/2m n=2000 p50= 231.9µs p99= 239.1µs max= 3346.7µs 7403 MiB/s cpu=0.200ns/B -paced/udp/2m n=2000 p50= 234.1µs p99= 239.4µs max= 3088.5µs 7260 MiB/s cpu=0.199ns/B - -== burst: bounded outstanding, one receiver == -burst/uring/2k n=65536 p50= 2.4µs p99= 3.3µs max= 2650.1µs 766 MiB/s cpu=1.407ns/B window=256 -burst/udp/2k n=65536 p50= 4.8µs p99= 5.9µs max= 3366.4µs 438 MiB/s cpu=2.435ns/B window=256 -burst/uring/64k n=16384 p50= 9.5µs p99= 12.7µs max= 2447.3µs 7178 MiB/s cpu=0.188ns/B window=146 -burst/udp/64k n=16384 p50= 9.1µs p99= 11.8µs max= 3300.3µs 6787 MiB/s cpu=0.205ns/B window=146 -burst/uring/2m n=512 p50= 232.5µs p99= 242.3µs max= 2468.1µs 7270 MiB/s cpu=0.203ns/B window=4 -burst/udp/2m n=512 p50= 213.6µs p99= 230.8µs max= 2898.7µs 7709 MiB/s cpu=0.183ns/B window=4 - -== loss1: one 2 MiB message, exactly one datagram dropped by a relay == -loss1/uring/2m n=1 p50= 5389.2µs p99= 5389.2µs max= 5389.2µs 351 MiB/s cpu=3.282ns/B datagrams=1793 retransmitted=2 -loss1/udp/2m n=1 p50= 8768.1µs p99= 8768.1µs max= 8768.1µs 226 MiB/s cpu=5.171ns/B datagrams=1793 retransmitted=2 - -== bcast: one sender, 8 receivers on one listener == -bcast/uring/2k n=65536 p50= 18.8µs p99= 35.1µs max= 73.2µs 952 MiB/s cpu=1.080ns/B window=256 -bcast/udp/2k n=65536 p50= 17.3µs p99= 30.6µs max= 1354.2µs 567 MiB/s cpu=1.764ns/B window=256 -bcast/uring/64k n=16384 p50= 45.0µs p99= 70.0µs max= 116.5µs 7412 MiB/s cpu=0.183ns/B window=146 -bcast/udp/64k n=16384 p50= 211.9µs p99= 343.1µs max= 2545.4µs 1408 MiB/s cpu=0.759ns/B window=146 -bcast/uring/2m n=512 p50= 3410.7µs p99= 6854.9µs max= 7521.4µs 6714 MiB/s cpu=0.246ns/B window=4 -bcast/udp/2m n=512 p50= 1425.0µs p99= 2761.3µs max= 3752.1µs 6681 MiB/s cpu=0.210ns/B window=4 diff --git a/crates/flux-network/benches/results/udp-uring/branch-4.txt b/crates/flux-network/benches/results/udp-uring/branch-4.txt deleted file mode 100644 index 16a9bc5..0000000 --- a/crates/flux-network/benches/results/udp-uring/branch-4.txt +++ /dev/null @@ -1,27 +0,0 @@ -== paced: one message per 100µs, one receiver == -paced/udp/2k n=2000 p50= 4.8µs p99= 5.5µs max= 48.6µs 20 MiB/s cpu=95.447ns/B -paced/uring/2k n=2000 p50= 2.4µs p99= 4.2µs max= 798.4µs 20 MiB/s cpu=96.215ns/B -paced/udp/64k n=2000 p50= 9.1µs p99= 515.2µs max= 2456.4µs 625 MiB/s cpu=2.958ns/B -paced/uring/64k n=2000 p50= 9.6µs p99= 321.3µs max= 2260.8µs 625 MiB/s cpu=2.955ns/B -paced/udp/2m n=2000 p50= 238.8µs p99= 254.4µs max= 4154.9µs 7257 MiB/s cpu=0.200ns/B -paced/uring/2m n=2000 p50= 219.5µs p99= 226.6µs max= 2734.6µs 7755 MiB/s cpu=0.192ns/B - -== burst: bounded outstanding, one receiver == -burst/udp/2k n=65536 p50= 4.8µs p99= 5.8µs max= 3052.9µs 439 MiB/s cpu=2.438ns/B window=256 -burst/uring/2k n=65536 p50= 2.4µs p99= 3.4µs max= 2978.2µs 776 MiB/s cpu=1.390ns/B window=256 -burst/udp/64k n=16384 p50= 9.2µs p99= 13.5µs max= 3695.2µs 6667 MiB/s cpu=0.211ns/B window=146 -burst/uring/64k n=16384 p50= 9.4µs p99= 13.3µs max= 3345.8µs 7394 MiB/s cpu=0.181ns/B window=146 -burst/udp/2m n=512 p50= 216.2µs p99= 282.9µs max= 3181.4µs 7560 MiB/s cpu=0.188ns/B window=4 -burst/uring/2m n=512 p50= 218.5µs p99= 228.5µs max= 2571.7µs 7680 MiB/s cpu=0.193ns/B window=4 - -== loss1: one 2 MiB message, exactly one datagram dropped by a relay == -loss1/udp/2m n=1 p50= 6135.7µs p99= 6135.7µs max= 6135.7µs 302 MiB/s cpu=2.932ns/B datagrams=1793 retransmitted=2 -loss1/uring/2m n=1 p50= 4526.4µs p99= 4526.4µs max= 4526.4µs 410 MiB/s cpu=2.572ns/B datagrams=1793 retransmitted=2 - -== bcast: one sender, 8 receivers on one listener == -bcast/udp/2k n=65536 p50= 17.4µs p99= 31.4µs max= 1743.5µs 566 MiB/s cpu=1.759ns/B window=256 -bcast/uring/2k n=65536 p50= 18.7µs p99= 35.0µs max= 77.8µs 954 MiB/s cpu=1.070ns/B window=256 -bcast/udp/64k n=16384 p50= 208.6µs p99= 342.6µs max= 1654.6µs 1411 MiB/s cpu=0.754ns/B window=146 -bcast/uring/64k n=16384 p50= 44.4µs p99= 68.5µs max= 135.7µs 7471 MiB/s cpu=0.180ns/B window=146 -bcast/udp/2m n=512 p50= 1364.5µs p99= 5455.6µs max= 6912.9µs 7009 MiB/s cpu=0.204ns/B window=4 -bcast/uring/2m n=512 p50= 3199.3µs p99= 6353.6µs max= 6675.1µs 7064 MiB/s cpu=0.232ns/B window=4 diff --git a/crates/flux-network/benches/results/udp-uring/branch-5.txt b/crates/flux-network/benches/results/udp-uring/branch-5.txt deleted file mode 100644 index c61ee24..0000000 --- a/crates/flux-network/benches/results/udp-uring/branch-5.txt +++ /dev/null @@ -1,27 +0,0 @@ -== paced: one message per 100µs, one receiver == -paced/uring/2k n=2000 p50= 2.4µs p99= 3.3µs max= 44.8µs 20 MiB/s cpu=96.388ns/B -paced/udp/2k n=2000 p50= 4.7µs p99= 458.1µs max= 2454.1µs 20 MiB/s cpu=94.881ns/B -paced/uring/64k n=2000 p50= 9.4µs p99= 104.1µs max= 2040.3µs 625 MiB/s cpu=2.956ns/B -paced/udp/64k n=2000 p50= 9.0µs p99= 10.1µs max= 1249.2µs 625 MiB/s cpu=2.966ns/B -paced/uring/2m n=2000 p50= 231.6µs p99= 237.7µs max= 2123.8µs 7408 MiB/s cpu=0.200ns/B -paced/udp/2m n=2000 p50= 217.1µs p99= 229.8µs max= 3756.7µs 7721 MiB/s cpu=0.185ns/B - -== burst: bounded outstanding, one receiver == -burst/uring/2k n=65536 p50= 2.4µs p99= 3.3µs max= 2558.2µs 777 MiB/s cpu=1.388ns/B window=256 -burst/udp/2k n=65536 p50= 4.8µs p99= 6.8µs max= 3460.7µs 442 MiB/s cpu=2.405ns/B window=256 -burst/uring/64k n=16384 p50= 9.5µs p99= 14.5µs max= 2710.8µs 7102 MiB/s cpu=0.191ns/B window=146 -burst/udp/64k n=16384 p50= 9.0µs p99= 14.1µs max= 2953.6µs 6843 MiB/s cpu=0.203ns/B window=146 -burst/uring/2m n=512 p50= 233.1µs p99= 306.5µs max= 2166.3µs 7237 MiB/s cpu=0.204ns/B window=4 -burst/udp/2m n=512 p50= 219.0µs p99= 290.7µs max= 3091.0µs 7541 MiB/s cpu=0.188ns/B window=4 - -== loss1: one 2 MiB message, exactly one datagram dropped by a relay == -loss1/uring/2m n=1 p50= 5441.4µs p99= 5441.4µs max= 5441.4µs 349 MiB/s cpu=3.334ns/B datagrams=1793 retransmitted=2 -loss1/udp/2m n=1 p50= 6228.9µs p99= 6228.9µs max= 6228.9µs 299 MiB/s cpu=2.910ns/B datagrams=1793 retransmitted=2 - -== bcast: one sender, 8 receivers on one listener == -bcast/uring/2k n=65536 p50= 18.9µs p99= 35.3µs max= 87.1µs 945 MiB/s cpu=1.084ns/B window=256 -bcast/udp/2k n=65536 p50= 17.4µs p99= 30.5µs max= 1319.4µs 567 MiB/s cpu=1.753ns/B window=256 -bcast/uring/64k n=16384 p50= 43.8µs p99= 69.0µs max= 138.8µs 7514 MiB/s cpu=0.180ns/B window=146 -bcast/udp/64k n=16384 p50= 211.3µs p99= 343.0µs max= 2534.9µs 1408 MiB/s cpu=0.757ns/B window=146 -bcast/uring/2m n=512 p50= 3232.8µs p99= 6186.5µs max= 7257.1µs 7126 MiB/s cpu=0.231ns/B window=4 -bcast/udp/2m n=512 p50= 1407.6µs p99= 4641.6µs max= 6066.8µs 6822 MiB/s cpu=0.206ns/B window=4 diff --git a/crates/flux-network/benches/results/udp-uring/main-1.txt b/crates/flux-network/benches/results/udp-uring/main-1.txt deleted file mode 100644 index 82adf62..0000000 --- a/crates/flux-network/benches/results/udp-uring/main-1.txt +++ /dev/null @@ -1,17 +0,0 @@ -== paced: one message per 100µs, one receiver == -paced/udp/2k n=2000 p50= 4.7µs p99= 5.5µs max= 48.8µs 20 MiB/s cpu=95.423ns/B -paced/udp/64k n=2000 p50= 9.0µs p99= 405.1µs max= 2346.1µs 625 MiB/s cpu=2.960ns/B -paced/udp/2m n=2000 p50= 232.5µs p99= 250.3µs max= 2403.3µs 7430 MiB/s cpu=0.193ns/B - -== burst: bounded outstanding, one receiver == -burst/udp/2k n=65536 p50= 4.7µs p99= 5.8µs max= 2984.1µs 443 MiB/s cpu=2.415ns/B window=256 -burst/udp/64k n=16384 p50= 9.1µs p99= 11.7µs max= 3330.8µs 6849 MiB/s cpu=0.203ns/B window=146 -burst/udp/2m n=512 p50= 236.0µs p99= 254.6µs max= 3188.7µs 7192 MiB/s cpu=0.200ns/B window=4 - -== loss1: one 2 MiB message, exactly one datagram dropped by a relay == -loss1/udp/2m n=1 p50= 6107.3µs p99= 6107.3µs max= 6107.3µs 295 MiB/s cpu=2.964ns/B datagrams=1793 retransmitted=2 - -== bcast: one sender, 8 receivers on one listener == -bcast/udp/2k n=65536 p50= 16.5µs p99= 40.5µs max= 2439.2µs 561 MiB/s cpu=1.787ns/B window=256 -bcast/udp/64k n=16384 p50= 183.9µs p99= 345.6µs max= 362.8µs 1398 MiB/s cpu=0.767ns/B window=146 -bcast/udp/2m n=512 p50= 1310.8µs p99= 5420.6µs max= 7348.5µs 7087 MiB/s cpu=0.199ns/B window=4 diff --git a/crates/flux-network/benches/results/udp-uring/main-2.txt b/crates/flux-network/benches/results/udp-uring/main-2.txt deleted file mode 100644 index a7d94e2..0000000 --- a/crates/flux-network/benches/results/udp-uring/main-2.txt +++ /dev/null @@ -1,17 +0,0 @@ -== paced: one message per 100µs, one receiver == -paced/udp/2k n=2000 p50= 4.7µs p99= 5.5µs max= 54.8µs 20 MiB/s cpu=95.450ns/B -paced/udp/64k n=2000 p50= 8.8µs p99= 445.4µs max= 2394.2µs 625 MiB/s cpu=2.959ns/B -paced/udp/2m n=2000 p50= 228.0µs p99= 243.4µs max= 2605.8µs 7548 MiB/s cpu=0.190ns/B - -== burst: bounded outstanding, one receiver == -burst/udp/2k n=65536 p50= 4.8µs p99= 5.8µs max= 2649.6µs 441 MiB/s cpu=2.427ns/B window=256 -burst/udp/64k n=16384 p50= 9.0µs p99= 11.0µs max= 1015.8µs 7082 MiB/s cpu=0.197ns/B window=146 -burst/udp/2m n=512 p50= 229.8µs p99= 270.7µs max= 3486.4µs 7392 MiB/s cpu=0.193ns/B window=4 - -== loss1: one 2 MiB message, exactly one datagram dropped by a relay == -loss1/udp/2m n=1 p50= 6088.1µs p99= 6088.1µs max= 6088.1µs 305 MiB/s cpu=2.876ns/B datagrams=1793 retransmitted=2 - -== bcast: one sender, 8 receivers on one listener == -bcast/udp/2k n=65536 p50= 16.5µs p99= 30.6µs max= 1698.3µs 557 MiB/s cpu=1.803ns/B window=256 -bcast/udp/64k n=16384 p50= 180.8µs p99= 345.8µs max= 364.5µs 1397 MiB/s cpu=0.764ns/B window=146 -bcast/udp/2m n=512 p50= 1324.7µs p99= 4800.1µs max= 6321.8µs 7002 MiB/s cpu=0.200ns/B window=4 diff --git a/crates/flux-network/benches/results/udp-uring/main-3.txt b/crates/flux-network/benches/results/udp-uring/main-3.txt deleted file mode 100644 index 10e7643..0000000 --- a/crates/flux-network/benches/results/udp-uring/main-3.txt +++ /dev/null @@ -1,17 +0,0 @@ -== paced: one message per 100µs, one receiver == -paced/udp/2k n=2000 p50= 4.7µs p99= 5.6µs max= 67.3µs 20 MiB/s cpu=95.459ns/B -paced/udp/64k n=2000 p50= 9.0µs p99= 462.5µs max= 2416.8µs 625 MiB/s cpu=2.958ns/B -paced/udp/2m n=2000 p50= 237.4µs p99= 252.5µs max= 2741.8µs 7299 MiB/s cpu=0.199ns/B - -== burst: bounded outstanding, one receiver == -burst/udp/2k n=65536 p50= 4.8µs p99= 5.8µs max= 2555.0µs 442 MiB/s cpu=2.420ns/B window=256 -burst/udp/64k n=16384 p50= 9.1µs p99= 12.4µs max= 3446.5µs 6758 MiB/s cpu=0.207ns/B window=146 -burst/udp/2m n=512 p50= 237.9µs p99= 268.4µs max= 3076.9µs 7231 MiB/s cpu=0.199ns/B window=4 - -== loss1: one 2 MiB message, exactly one datagram dropped by a relay == -loss1/udp/2m n=1 p50= 6116.4µs p99= 6116.4µs max= 6116.4µs 294 MiB/s cpu=2.987ns/B datagrams=1793 retransmitted=2 - -== bcast: one sender, 8 receivers on one listener == -bcast/udp/2k n=65536 p50= 16.8µs p99= 182.7µs max= 2640.5µs 567 MiB/s cpu=1.759ns/B window=256 -bcast/udp/64k n=16384 p50= 187.4µs p99= 347.2µs max= 358.7µs 1391 MiB/s cpu=0.775ns/B window=146 -bcast/udp/2m n=512 p50= 1333.8µs p99= 2664.3µs max= 4598.8µs 6800 MiB/s cpu=0.209ns/B window=4 diff --git a/crates/flux-network/benches/results/udp-uring/main-4.txt b/crates/flux-network/benches/results/udp-uring/main-4.txt deleted file mode 100644 index a66aa13..0000000 --- a/crates/flux-network/benches/results/udp-uring/main-4.txt +++ /dev/null @@ -1,17 +0,0 @@ -== paced: one message per 100µs, one receiver == -paced/udp/2k n=2000 p50= 4.8µs p99= 6.1µs max= 128.9µs 20 MiB/s cpu=95.432ns/B -paced/udp/64k n=2000 p50= 9.0µs p99= 452.2µs max= 2405.3µs 625 MiB/s cpu=2.958ns/B -paced/udp/2m n=2000 p50= 237.8µs p99= 249.5µs max= 2569.9µs 7293 MiB/s cpu=0.198ns/B - -== burst: bounded outstanding, one receiver == -burst/udp/2k n=65536 p50= 4.8µs p99= 5.8µs max= 2963.3µs 442 MiB/s cpu=2.396ns/B window=256 -burst/udp/64k n=16384 p50= 9.0µs p99= 11.6µs max= 2762.5µs 6887 MiB/s cpu=0.202ns/B window=146 -burst/udp/2m n=512 p50= 220.9µs p99= 237.2µs max= 3897.4µs 7670 MiB/s cpu=0.185ns/B window=4 - -== loss1: one 2 MiB message, exactly one datagram dropped by a relay == -loss1/udp/2m n=1 p50= 6037.8µs p99= 6037.8µs max= 6037.8µs 308 MiB/s cpu=2.840ns/B datagrams=1793 retransmitted=2 - -== bcast: one sender, 8 receivers on one listener == -bcast/udp/2k n=65536 p50= 16.6µs p99= 30.6µs max= 1736.9µs 562 MiB/s cpu=1.778ns/B window=256 -bcast/udp/64k n=16384 p50= 211.7µs p99= 342.1µs max= 974.9µs 1410 MiB/s cpu=0.754ns/B window=146 -bcast/udp/2m n=512 p50= 1389.6µs p99= 2275.6µs max= 3802.8µs 6855 MiB/s cpu=0.207ns/B window=4 diff --git a/crates/flux-network/benches/results/udp-uring/main-5.txt b/crates/flux-network/benches/results/udp-uring/main-5.txt deleted file mode 100644 index be69ba8..0000000 --- a/crates/flux-network/benches/results/udp-uring/main-5.txt +++ /dev/null @@ -1,17 +0,0 @@ -== paced: one message per 100µs, one receiver == -paced/udp/2k n=2000 p50= 4.7µs p99= 6.0µs max= 62.9µs 20 MiB/s cpu=95.431ns/B -paced/udp/64k n=2000 p50= 9.1µs p99= 465.4µs max= 2417.8µs 625 MiB/s cpu=2.958ns/B -paced/udp/2m n=2000 p50= 228.9µs p99= 244.0µs max= 2751.7µs 7530 MiB/s cpu=0.190ns/B - -== burst: bounded outstanding, one receiver == -burst/udp/2k n=65536 p50= 4.8µs p99= 5.8µs max= 2648.4µs 440 MiB/s cpu=2.420ns/B window=256 -burst/udp/64k n=16384 p50= 9.2µs p99= 11.9µs max= 3298.6µs 6659 MiB/s cpu=0.211ns/B window=146 -burst/udp/2m n=512 p50= 234.5µs p99= 241.0µs max= 2772.8µs 7285 MiB/s cpu=0.197ns/B window=4 - -== loss1: one 2 MiB message, exactly one datagram dropped by a relay == -loss1/udp/2m n=1 p50= 6074.5µs p99= 6074.5µs max= 6074.5µs 298 MiB/s cpu=2.937ns/B datagrams=1793 retransmitted=2 - -== bcast: one sender, 8 receivers on one listener == -bcast/udp/2k n=65536 p50= 17.2µs p99= 152.1µs max= 2640.8µs 560 MiB/s cpu=1.784ns/B window=256 -bcast/udp/64k n=16384 p50= 202.0µs p99= 345.8µs max= 372.5µs 1398 MiB/s cpu=0.770ns/B window=146 -bcast/udp/2m n=512 p50= 1372.5µs p99= 4693.6µs max= 6457.7µs 6845 MiB/s cpu=0.207ns/B window=4 From 89b648c21811481028324e9422bf6111ba5419ea Mon Sep 17 00:00:00 2001 From: gd-0 Date: Tue, 8 Sep 2026 22:21:37 +0100 Subject: [PATCH 4/6] refactor(network): trim io_uring plumbing Drop the redundant Box around pooled send slots (the Vec is never resized), add a UdpSocket::ring accessor, skip the extra per-socket ACK scan on idle io_uring polls, and dedupe the bench size filter. --- crates/flux-network/benches/udp_pipeline.rs | 20 ++++++++------------ crates/flux-network/src/udp/connector.rs | 18 +++++------------- crates/flux-network/src/udp/sys.rs | 6 ++++++ crates/flux-network/src/udp/sys/uring.rs | 12 ++++++------ 4 files changed, 25 insertions(+), 31 deletions(-) diff --git a/crates/flux-network/benches/udp_pipeline.rs b/crates/flux-network/benches/udp_pipeline.rs index 469b2e4..964f5ae 100644 --- a/crates/flux-network/benches/udp_pipeline.rs +++ b/crates/flux-network/benches/udp_pipeline.rs @@ -78,6 +78,11 @@ fn transports() -> Vec<(&'static str, Transport)> { transports } +fn sizes() -> impl Iterator { + let filter = std::env::var("FLUX_BENCH_SIZE").ok(); + SIZES.into_iter().filter(move |(name, _)| filter.as_deref().is_none_or(|s| s == *name)) +} + fn connector(transport: Transport) -> NetworkDriver { NetworkDriver::default().with_transport(transport).with_socket_buf_size(BIG_SOCKET_BUF) } @@ -316,10 +321,7 @@ impl Drop for Relay { fn main() { pin(usize::MAX); println!("== paced: one message per 100µs, one receiver =="); - for (size_name, size) in SIZES - .into_iter() - .filter(|(name, _)| std::env::var("FLUX_BENCH_SIZE").ok().is_none_or(|s| s == *name)) - { + for (size_name, size) in sizes() { for (name, transport) in transports() { let addr = free_addr(); let s = run(Scenario { @@ -337,10 +339,7 @@ fn main() { } println!("\n== burst: bounded outstanding, one receiver =="); - for (size_name, size) in SIZES - .into_iter() - .filter(|(name, _)| std::env::var("FLUX_BENCH_SIZE").ok().is_none_or(|s| s == *name)) - { + for (size_name, size) in sizes() { let (count, window) = burst_plan(size); for (name, transport) in transports() { let addr = free_addr(); @@ -382,10 +381,7 @@ fn main() { } println!("\n== bcast: one sender, {BCAST_PEERS} receivers on one listener =="); - for (size_name, size) in SIZES - .into_iter() - .filter(|(name, _)| std::env::var("FLUX_BENCH_SIZE").ok().is_none_or(|s| s == *name)) - { + for (size_name, size) in sizes() { let (count, window) = burst_plan(size); let count = count / BCAST_PEERS; for (name, transport) in transports() { diff --git a/crates/flux-network/src/udp/connector.rs b/crates/flux-network/src/udp/connector.rs index 1aecd4a..ca54cb0 100644 --- a/crates/flux-network/src/udp/connector.rs +++ b/crates/flux-network/src/udp/connector.rs @@ -620,14 +620,12 @@ impl UdpManager { // Drain in bounded passes, emitting ACKs between passes so // slow callbacks cannot hold back the sender's whole window. for _ in 0..BATCH { - let work = self.sockets[k].socket.ring.as_ref().unwrap().borrow_mut().poll(); + let work = self.sockets[k].socket.ring().poll(); o |= work; let mut received_any = false; // ACK processing can reap more receives while sending. for _ in 0..config.recv_entries { - let received = - self.sockets[k].socket.ring.as_ref().unwrap().borrow_mut().receive(); - let Some(received) = received else { break }; + let Some(received) = self.sockets[k].socket.ring().receive() else { break }; received_any = true; if let Some((datagrams, from)) = received.datagrams(self.udp.max_datagram_size) @@ -639,22 +637,16 @@ impl UdpManager { self.on_datagram(k, &dgram, dcache, deliver); } } - self.sockets[k] - .socket - .ring - .as_ref() - .unwrap() - .borrow_mut() - .recycle(received); + self.sockets[k].socket.ring().recycle(received); } - self.handle_event(k, false, false, dcache, deliver); if !work && !received_any { break; } + self.handle_event(k, false, false, dcache, deliver); } let writable = self.sockets[k].writable_armed; self.handle_event(k, false, writable, dcache, deliver); - self.sockets[k].socket.ring.as_ref().unwrap().borrow_mut().submit(); + self.sockets[k].socket.ring().submit(); } return o | self.drain_pending_disconnects(deliver); } diff --git a/crates/flux-network/src/udp/sys.rs b/crates/flux-network/src/udp/sys.rs index 78f677a..258ae17 100644 --- a/crates/flux-network/src/udp/sys.rs +++ b/crates/flux-network/src/udp/sys.rs @@ -35,6 +35,12 @@ impl UdpSocket { self.socket.local_addr() } + /// Panics unless the socket was opened with [`crate::udp::UdpIo::Uring`]. + #[cfg(target_os = "linux")] + pub(crate) fn ring(&self) -> std::cell::RefMut<'_, uring::Ring> { + self.ring.as_ref().expect("io_uring socket").borrow_mut() + } + pub(crate) fn send_to(&self, bytes: &[u8], addr: SocketAddr) -> io::Result { #[cfg(target_os = "linux")] if let Some(ring) = &self.ring { diff --git a/crates/flux-network/src/udp/sys/uring.rs b/crates/flux-network/src/udp/sys/uring.rs index 04e90e7..3ffb206 100644 --- a/crates/flux-network/src/udp/sys/uring.rs +++ b/crates/flux-network/src/udp/sys/uring.rs @@ -94,8 +94,8 @@ struct Tx { } impl Tx { - fn new() -> Box { - Box::new(Self { + fn new() -> Self { + Self { header: unsafe { mem::zeroed() }, addr: SockAddr::new("0.0.0.0:0".parse().unwrap()), control: unsafe { mem::zeroed() }, @@ -105,7 +105,7 @@ impl Tx { segment: 0, offset: 0, fallback: false, - }) + } } fn prepare(&mut self, fd: RawFd, index: usize) -> io_uring::squeue::Entry { @@ -187,14 +187,14 @@ pub(crate) struct Ring { io: IoUring, fd: RawFd, owner: std::thread::ThreadId, - // Boxes keep msghdr, iovec and ancillary pointers stable across moves. rx: Vec>, provided: Option, + // Boxed: the multishot SQE points at it and `Ring` itself moves. receive_header: Box, rx_active: bool, available: usize, - #[allow(clippy::vec_box)] - tx: Vec>, + // Never resized, so SQE pointers into entries stay stable. + tx: Vec, free_tx: Vec, ready: VecDeque, completions: Vec<(u64, i32, u32)>, From ab2d2959ef72305f49bfdf1725c603a7d724f536 Mon Sep 17 00:00:00 2001 From: gd-0 Date: Wed, 9 Sep 2026 09:02:57 +0100 Subject: [PATCH 5/6] fix(network): isolate send slots and flush syscall ACKs between batches --- crates/flux-network/src/udp/connector.rs | 18 ++++-- crates/flux-network/src/udp/sys/uring.rs | 15 ++--- .../tests/support/udp_connector.rs | 56 +++++++++++++++++++ crates/flux-network/tests/udp_uring.rs | 56 ------------------- 4 files changed, 76 insertions(+), 69 deletions(-) diff --git a/crates/flux-network/src/udp/connector.rs b/crates/flux-network/src/udp/connector.rs index ca54cb0..f9a8e68 100644 --- a/crates/flux-network/src/udp/connector.rs +++ b/crates/flux-network/src/udp/connector.rs @@ -567,6 +567,7 @@ impl UdpManager { self.on_datagram(k, &dgram, dcache, deliver); } } + self.flush_acks(k, Instant::now()); } self.recv = Some(recv); } @@ -575,13 +576,8 @@ impl UdpManager { self.sockets[k].writable_armed = false; self.flush_socket(k, now); } + self.flush_acks(k, now); let entry = &mut self.sockets[k]; - let token = entry.token; - for peer in self.peers.iter_mut().filter(|p| p.socket_token == token) { - if peer.take_ack_due() && peer.send_ack(&entry.socket, now) == SendOutcome::WouldBlock { - arm_writable(&self.registry, entry); - } - } if writable && !entry.writable_armed { if let Err(err) = self.registry.reregister(&mut entry.socket, entry.token, Interest::READABLE) @@ -591,6 +587,16 @@ impl UdpManager { } } + fn flush_acks(&mut self, k: usize, now: Instant) { + let entry = &mut self.sockets[k]; + let token = entry.token; + for peer in self.peers.iter_mut().filter(|p| p.socket_token == token) { + if peer.take_ack_due() && peer.send_ack(&entry.socket, now) == SendOutcome::WouldBlock { + arm_writable(&self.registry, entry); + } + } + } + #[inline] fn drain_pending_disconnects(&mut self, deliver: &mut F) -> bool where diff --git a/crates/flux-network/src/udp/sys/uring.rs b/crates/flux-network/src/udp/sys/uring.rs index 3ffb206..c0c8134 100644 --- a/crates/flux-network/src/udp/sys/uring.rs +++ b/crates/flux-network/src/udp/sys/uring.rs @@ -94,8 +94,8 @@ struct Tx { } impl Tx { - fn new() -> Self { - Self { + fn new() -> Box { + Box::new(Self { header: unsafe { mem::zeroed() }, addr: SockAddr::new("0.0.0.0:0".parse().unwrap()), control: unsafe { mem::zeroed() }, @@ -105,7 +105,7 @@ impl Tx { segment: 0, offset: 0, fallback: false, - } + }) } fn prepare(&mut self, fd: RawFd, index: usize) -> io_uring::squeue::Entry { @@ -193,15 +193,16 @@ pub(crate) struct Ring { receive_header: Box, rx_active: bool, available: usize, - // Never resized, so SQE pointers into entries stay stable. - tx: Vec, + // Vec indexing must not reborrow headers retained by other in-flight SQEs. + #[allow(clippy::vec_box)] + tx: Vec>, free_tx: Vec, ready: VecDeque, completions: Vec<(u64, i32, u32)>, } -// SAFETY: requests only access their boxed buffers. Moving the owner does not -// move the buffers; RefCell on the socket excludes concurrent access. +// SAFETY: requests point into stable heap allocations. Moving the owner does +// not move those allocations; RefCell on the socket excludes concurrent access. #[allow(clippy::non_send_fields_in_send_ty)] unsafe impl Send for Ring {} diff --git a/crates/flux-network/tests/support/udp_connector.rs b/crates/flux-network/tests/support/udp_connector.rs index f3a920f..b0fa649 100644 --- a/crates/flux-network/tests/support/udp_connector.rs +++ b/crates/flux-network/tests/support/udp_connector.rs @@ -610,3 +610,59 @@ fn udp_window_exhaustion_disconnects_instead_of_dropping() { assert_eq!(disconnected, Some(accepted)); let _ = &client; } + +#[test] +fn sustained_large_messages_keep_acknowledgements_current() { + const COUNT: usize = 64; + let addr = free_addr(); + let mut server = udp(UdpConfig::lan()).with_socket_buf_size(16 * 1024 * 1024); + let mut client = udp(UdpConfig::lan()).with_socket_buf_size(16 * 1024 * 1024); + let (accepted, _) = connect_pair(&mut server, &mut client, addr); + let got = Arc::new(AtomicUsize::new(0)); + let progress = got.clone(); + let deadline = Instant::now() + Duration::from_secs(10); + let receiver = thread::spawn(move || { + let mut seen = [false; COUNT]; + while progress.load(Ordering::Relaxed) < COUNT { + client.poll_with(|event| { + if let PollEvent::Message { payload, .. } = event { + assert_eq!(payload.len(), 2 * 1024 * 1024); + let id = msg_id(payload) as usize; + assert!(!seen[id]); + assert!(payload[4..].iter().all(|b| *b == 0x5a)); + seen[id] = true; + progress.fetch_add(1, Ordering::Relaxed); + } + }); + assert!( + Instant::now() < deadline, + "large-message receiver stalled at {}", + progress.load(Ordering::Relaxed) + ); + } + }); + let mut sent = 0; + let mut payload = vec![0x5a; 2 * 1024 * 1024]; + while got.load(Ordering::Relaxed) < COUNT { + if sent < COUNT && sent - got.load(Ordering::Relaxed) < 4 { + payload[..4].copy_from_slice(&(sent as u32).to_le_bytes()); + server.write_or_enqueue_with(SendBehavior::Single(accepted), |buf| { + buf.extend_from_slice(&payload); + }); + sent += 1; + } + server.poll_with(|event| { + assert!( + !matches!(event, PollEvent::Disconnect { .. }), + "sender disconnected at {sent} sent, {} received", + got.load(Ordering::Relaxed) + ); + }); + assert!( + Instant::now() < deadline, + "large-message sender stalled at {sent} sent, {} received", + got.load(Ordering::Relaxed) + ); + } + receiver.join().unwrap(); +} diff --git a/crates/flux-network/tests/udp_uring.rs b/crates/flux-network/tests/udp_uring.rs index e81e66a..994fc29 100644 --- a/crates/flux-network/tests/udp_uring.rs +++ b/crates/flux-network/tests/udp_uring.rs @@ -88,59 +88,3 @@ fn mixed_backends_interoperate() { } } } - -#[test] -fn sustained_large_messages_keep_acknowledgements_current() { - const COUNT: usize = 64; - let addr = free_addr(); - let mut server = udp(UdpConfig::lan()).with_socket_buf_size(16 * 1024 * 1024); - let mut client = udp(UdpConfig::lan()).with_socket_buf_size(16 * 1024 * 1024); - let (accepted, _) = connect_pair(&mut server, &mut client, addr); - let got = Arc::new(AtomicUsize::new(0)); - let progress = got.clone(); - let deadline = Instant::now() + Duration::from_secs(10); - let receiver = thread::spawn(move || { - let mut seen = [false; COUNT]; - while progress.load(Ordering::Relaxed) < COUNT { - client.poll_with(|event| { - if let PollEvent::Message { payload, .. } = event { - assert_eq!(payload.len(), 2 * 1024 * 1024); - let id = msg_id(payload) as usize; - assert!(!seen[id]); - assert!(payload[4..].iter().all(|b| *b == 0x5a)); - seen[id] = true; - progress.fetch_add(1, Ordering::Relaxed); - } - }); - assert!( - Instant::now() < deadline, - "large-message receiver stalled at {}", - progress.load(Ordering::Relaxed) - ); - } - }); - let mut sent = 0; - let mut payload = vec![0x5a; 2 * 1024 * 1024]; - while got.load(Ordering::Relaxed) < COUNT { - if sent < COUNT && sent - got.load(Ordering::Relaxed) < 4 { - payload[..4].copy_from_slice(&(sent as u32).to_le_bytes()); - server.write_or_enqueue_with(SendBehavior::Single(accepted), |buf| { - buf.extend_from_slice(&payload); - }); - sent += 1; - } - server.poll_with(|event| { - assert!( - !matches!(event, PollEvent::Disconnect { .. }), - "sender disconnected at {sent} sent, {} received", - got.load(Ordering::Relaxed) - ); - }); - assert!( - Instant::now() < deadline, - "large-message sender stalled at {sent} sent, {} received", - got.load(Ordering::Relaxed) - ); - } - receiver.join().unwrap(); -} From fd57087ca8d79a9127a4580176e4275f72efa386 Mon Sep 17 00:00:00 2001 From: gd-0 Date: Wed, 9 Sep 2026 09:34:38 +0100 Subject: [PATCH 6/6] ci: honor socket buffer sizes requested by UDP tests --- .github/workflows/lint.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index 70fb9cc..0c933c9 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -62,5 +62,10 @@ jobs: with: rustflags: "" + # Large-window UDP tests request 16 MiB; avoid silently capping their + # socket buffers below the in-flight window. UDP has no receiver flow control. + - name: Configure socket buffer limits + run: sudo sysctl -w net.core.rmem_max=16777216 net.core.wmem_max=16777216 + - name: Run tests run: cargo test --workspace --all-features --locked