feat: Faster UDP/IO on Apple platforms

This uses Apple's private sendmsg_x and recvmsg_x system calls for
multi-packet UDP I/O.

quinn-udp/Cargo.toml

@@ -18,6 +18,8 @@ default = ["tracing", "log"]
 # Configure `tracing` to log events via `log` if no `tracing` subscriber exists.
 log = ["tracing/log"]
 direct-log = ["dep:log"]
+# Use private Apple APIs to send multiple packets in a single syscall.
+fast-apple-datapath = []
 
 [dependencies]
 libc = "0.2.158"
@@ -30,8 +32,15 @@ once_cell = { workspace = true }
 windows-sys = { workspace = true }
 
 [dev-dependencies]
-criterion = "0.5"
+criterion = { version = "0.5", default-features = false }
 
-[target.'cfg(any(target_os = "linux", target_os = "windows"))'.bench]
+[build-dependencies]
+cfg_aliases = "0.2"
+
+[lib]
+# See https://github.com/bheisler/criterion.rs/blob/master/book/src/faq.md#cargo-bench-gives-unrecognized-option-errors-for-valid-command-line-options
+bench = false
+
+[[bench]]
 name = "throughput"
 harness = false

quinn-udp/benches/throughput.rs

@@ -1,8 +1,8 @@
 use criterion::{criterion_group, criterion_main, Criterion};
-use quinn_udp::{RecvMeta, Transmit, UdpSocketState};
+use quinn_udp::{RecvMeta, Transmit, UdpSocketState, BATCH_SIZE};
 use std::cmp::min;
 use std::net::{Ipv4Addr, Ipv6Addr};
-use std::{io::IoSliceMut, net::UdpSocket, slice};
+use std::{io::IoSliceMut, net::UdpSocket};
 
 pub fn criterion_benchmark(c: &mut Criterion) {
     const TOTAL_BYTES: usize = 10 * 1024 * 1024;
@@ -28,8 +28,12 @@ pub fn criterion_benchmark(c: &mut Criterion) {
     // Reverse non-blocking flag set by `UdpSocketState` to make the test non-racy
     recv.set_nonblocking(false).unwrap();
 
-    let mut receive_buffer = vec![0; MAX_BUFFER_SIZE];
-    let mut meta = RecvMeta::default();
+    let mut receive_buffers = vec![vec![0; SEGMENT_SIZE * recv_state.gro_segments()]; BATCH_SIZE];
+    let mut receive_slices = receive_buffers
+        .iter_mut()
+        .map(|buf| IoSliceMut::new(buf))
+        .collect::<Vec<_>>();
+    let mut meta = vec![RecvMeta::default(); BATCH_SIZE];
 
     for gso_enabled in [false, true] {
         let mut group = c.benchmark_group(format!("gso_{}", gso_enabled));
@@ -56,14 +60,11 @@ pub fn criterion_benchmark(c: &mut Criterion) {
                     let mut received_segments = 0;
                     while received_segments < segments {
                         let n = recv_state
-                            .recv(
-                                (&recv).into(),
-                                &mut [IoSliceMut::new(&mut receive_buffer)],
-                                slice::from_mut(&mut meta),
-                            )
+                            .recv((&recv).into(), &mut receive_slices, &mut meta)
                             .unwrap();
-                        assert_eq!(n, 1);
-                        received_segments += meta.len / meta.stride;
+                        for i in meta.iter().take(n) {
+                            received_segments += i.len / i.stride;
+                        }
                     }
                     assert_eq!(received_segments, segments);
                 }

quinn-udp/build.rs

@@ -0,0 +1,26 @@
+use cfg_aliases::cfg_aliases;
+
+fn main() {
+    // Setup cfg aliases
+    cfg_aliases! {
+        // Platforms
+        apple: {
+            any(
+                target_os = "macos",
+                target_os = "ios",
+                target_os = "tvos",
+                target_os = "visionos"
+            )
+        },
+        bsd: {
+            any(
+                target_os = "freebsd",
+                target_os = "openbsd",
+                target_os = "netbsd"
+            )
+        },
+        // Convenience aliases
+        apple_fast: { all(apple, feature = "fast-apple-datapath") },
+        apple_slow: { all(apple, not(feature = "fast-apple-datapath")) },
+    }
+}

quinn-udp/src/cmsg/unix.rs

@@ -32,6 +32,29 @@ impl MsgHdr for libc::msghdr {
     }
 }
 
+#[cfg(apple_fast)]
+impl MsgHdr for crate::imp::msghdr_x {
+    type ControlMessage = libc::cmsghdr;
+
+    fn cmsg_first_hdr(&self) -> *mut Self::ControlMessage {
+        let selfp = self as *const _ as *mut libc::msghdr;
+        unsafe { libc::CMSG_FIRSTHDR(selfp) }
+    }
+
+    fn cmsg_nxt_hdr(&self, cmsg: &Self::ControlMessage) -> *mut Self::ControlMessage {
+        let selfp = self as *const _ as *mut libc::msghdr;
+        unsafe { libc::CMSG_NXTHDR(selfp, cmsg) }
+    }
+
+    fn set_control_len(&mut self, len: usize) {
+        self.msg_controllen = len as _;
+    }
+
+    fn control_len(&self) -> usize {
+        self.msg_controllen as _
+    }
+}
+
 /// Helpers for [`libc::cmsghdr`]
 impl CMsgHdr for libc::cmsghdr {
     fn cmsg_len(length: usize) -> usize {