diff --git a/README.md b/README.md index 836462e1..38a7750d 100644 --- a/README.md +++ b/README.md @@ -340,7 +340,7 @@ the same. Building -------- -quiceh requires Rust 1.66 or later to build. The latest stable Rust release can +quiceh requires Rust 1.79 or later to build. The latest stable Rust release can be installed using [rustup](https://rustup.rs/). Once the Rust build environment is setup, the quiceh source code can be fetched diff --git a/apps/src/args.rs b/apps/src/args.rs index ac3c3411..bef872a6 100644 --- a/apps/src/args.rs +++ b/apps/src/args.rs @@ -137,7 +137,7 @@ impl Args for CommonArgs { let early_data = args.get_bool("--early-data"); - let dump_packet_path = if args.get_str("--dump-packets") != "" { + let dump_packet_path = if !args.get_str("--dump-packets").is_empty() { Some(args.get_str("--dump-packets").to_string()) } else { None @@ -155,7 +155,7 @@ impl Args for CommonArgs { let enable_active_migration = args.get_bool("--enable-active-migration"); let max_field_section_size = - if args.get_str("--max-field-section-size") != "" { + if !args.get_str("--max-field-section-size").is_empty() { Some( args.get_str("--max-field-section-size") .parse::() @@ -166,7 +166,7 @@ impl Args for CommonArgs { }; let qpack_max_table_capacity = - if args.get_str("--qpack-max-table-capacity") != "" { + if !args.get_str("--qpack-max-table-capacity").is_empty() { Some( args.get_str("--qpack-max-table-capacity") .parse::() @@ -177,7 +177,7 @@ impl Args for CommonArgs { }; let qpack_blocked_streams = - if args.get_str("--qpack-blocked-streams") != "" { + if !args.get_str("--qpack-blocked-streams").is_empty() { Some( args.get_str("--qpack-blocked-streams") .parse::() @@ -319,7 +319,7 @@ impl Args for ClientArgs { let version = args.get_str("--wire-version"); let version = u32::from_str_radix(version, 16).unwrap(); - let dump_response_path = if args.get_str("--dump-responses") != "" { + let dump_response_path = if !args.get_str("--dump-responses").is_empty() { Some(args.get_str("--dump-responses").to_string()) } else { None @@ -464,6 +464,7 @@ Options: --qpack-blocked-streams STREAMS Limit of streams that can be blocked while decoding. Any value other that 0 is currently unsupported. --disable-pacing Disable pacing (linux only). --initial-cwnd-packets PACKETS The initial congestion window size in terms of packet count [default: 10]. + --enable-hidden-copy Assemble a stream frame and control through encryption. -h --help Show this screen. "; @@ -477,6 +478,7 @@ pub struct ServerArgs { pub key: String, pub disable_pacing: bool, pub enable_pmtud: bool, + pub enable_hidden_copy: bool, } impl Args for ServerArgs { @@ -491,6 +493,7 @@ impl Args for ServerArgs { let key = args.get_str("--key").to_string(); let disable_pacing = args.get_bool("--disable-pacing"); let enable_pmtud = args.get_bool("--enable-pmtud"); + let enable_hidden_copy = args.get_bool("--enable-hidden-copy"); ServerArgs { listen, @@ -501,6 +504,7 @@ impl Args for ServerArgs { key, disable_pacing, enable_pmtud, + enable_hidden_copy, } } } diff --git a/apps/src/bin/quiceh-client.rs b/apps/src/bin/quiceh-client.rs index e0a6c294..b5db3ced 100644 --- a/apps/src/bin/quiceh-client.rs +++ b/apps/src/bin/quiceh-client.rs @@ -38,7 +38,7 @@ fn main() { let conn_args = CommonArgs::with_docopt(&docopt); let args = ClientArgs::with_docopt(&docopt); - match connect(args, conn_args, stdout_sink) { + match connect::(args, conn_args, stdout_sink) { Err(ClientError::HandshakeFail) => std::process::exit(-1), Err(ClientError::HttpFail) => std::process::exit(-2), diff --git a/apps/src/bin/quiceh-server.rs b/apps/src/bin/quiceh-server.rs index a7bcccc9..df37df76 100644 --- a/apps/src/bin/quiceh-server.rs +++ b/apps/src/bin/quiceh-server.rs @@ -125,6 +125,7 @@ fn main() { config.set_max_stream_window(conn_args.max_stream_window); config.enable_pacing(pacing); + config.enable_hidden_copy_for_zc_sender(args.enable_hidden_copy); let mut keylog = None; @@ -355,7 +356,7 @@ fn main() { debug!("New connection: dcid={:?} scid={:?}", hdr.dcid, scid); #[allow(unused_mut)] - let mut conn = quiceh::accept( + let mut conn = quiceh::accept_with_buf_factory( &scid, odcid.as_ref(), local_addr, diff --git a/apps/src/client.rs b/apps/src/client.rs index 7481b5ad..de78aa6d 100644 --- a/apps/src/client.rs +++ b/apps/src/client.rs @@ -27,6 +27,8 @@ use crate::args::*; use crate::common::*; use quiceh::AppRecvBufMap; +use quiceh::BufFactory; +use quiceh::BufSplit; use std::cell::RefCell; use std::io::prelude::*; @@ -52,10 +54,13 @@ pub enum ClientError { Other(String), } -pub fn connect( +pub fn connect>( args: ClientArgs, conn_args: CommonArgs, output_sink: impl FnMut(String) + 'static, -) -> Result<(), ClientError> { +) -> Result<(), ClientError> +where + ::Buf: BufSplit, +{ let mut buf = [0; 65536 * BATCH_SIZE]; let mut out = [0; MAX_DATAGRAM_SIZE]; @@ -176,7 +181,7 @@ pub fn connect( config.enable_dgram(true, 1000, 1000); } - let mut http_conn: Option> = None; + let mut http_conn: Option>> = None; let mut app_proto_selected = false; @@ -208,7 +213,7 @@ pub fn connect( .unwrap(); // Create a QUIC connection and initiate handshake. - let mut conn = quiceh::connect( + let mut conn = quiceh::connect_with_buffer_factory( connect_url.domain(), &scid, local_addr, diff --git a/apps/src/common.rs b/apps/src/common.rs index 6b0fa26b..bf06bc93 100644 --- a/apps/src/common.rs +++ b/apps/src/common.rs @@ -50,6 +50,9 @@ use quiceh::ConnectionId; use quiceh::h3::NameValue; use quiceh::h3::Priority; +use quiceh::BufFactory; +use quiceh::BufSplit; +use std::sync::Arc; pub fn stdout_sink(out: String) { print!("{out}"); @@ -69,21 +72,90 @@ pub struct PartialRequest { pub req: Vec, } -pub struct PartialResponse { +#[derive(Debug, Clone, Default)] +pub struct BufResponseFactory; + +#[derive(Debug, Clone, Default)] +pub struct BufResponse { + inner: Arc>, + start: usize, + end: usize, +} + +impl BufResponse { + fn new(inner: Arc>, start: usize, end: usize) -> Self { + Self { inner, start, end } + } + + fn len(&self) -> usize { + self.end - self.start + } +} + +impl From> for BufResponse { + fn from(value: Vec) -> Self { + BufResponse { + start: 0, + end: value.len(), + inner: Arc::new(value.into_boxed_slice()), + } + } +} + +impl BufFactory for BufResponseFactory { + type Buf = BufResponse; + + fn buf_from_slice(buf: &[u8]) -> Self::Buf { + BufResponse { + start: 0, + end: buf.len(), + inner: Arc::new(buf.into()), + } + } +} + +impl BufSplit for BufResponse { + // Split the buffer at a given point, after the split the old buffer + // must only contain the first `at` bytes, while the newly produced + // buffer must containt the remaining bytes. + fn split_at(&mut self, at: usize) -> Self { + assert!(at <= self.len(), "split_at index out of bounds"); + + let newend = self.start + at; + let buf = BufResponse::new(self.inner.clone(), newend, self.end); + + self.end = newend; + + buf + } +} + +impl AsRef<[u8]> for BufResponse { + fn as_ref(&self) -> &[u8] { + &self.inner[self.start..self.end] + } +} + +pub struct PartialResponse +where + F: quiceh::BufFactory, +{ pub headers: Option>, pub priority: Option, - pub body: Vec, - - pub written: usize, + pub body: ::Buf, + pub remaining_len: usize, } pub type ClientId = u64; -pub struct Client { - pub conn: quiceh::Connection, +pub struct Client +where + F: BufFactory, +{ + pub conn: quiceh::Connection, - pub http_conn: Option>, + pub http_conn: Option>>, pub client_id: ClientId, @@ -315,8 +387,8 @@ pub fn priority_from_query_string(url: &url::Url) -> Option { } } -fn send_h3_dgram( - conn: &mut quiceh::Connection, flow_id: u64, dgram_content: &[u8], +fn send_h3_dgram( + conn: &mut quiceh::Connection, flow_id: u64, dgram_content: &[u8], ) -> quiceh::Result<()> { info!( "sending HTTP/3 DATAGRAM on flow_id={} with data {:?}", @@ -335,25 +407,25 @@ fn send_h3_dgram( conn.dgram_send(&d) } -pub trait HttpConn { +pub trait HttpConn { fn send_requests( - &mut self, conn: &mut quiceh::Connection, target_path: &Option, + &mut self, conn: &mut quiceh::Connection, target_path: &Option, ); fn handle_responses( - &mut self, conn: &mut quiceh::Connection, buf: &mut [u8], + &mut self, conn: &mut quiceh::Connection, buf: &mut [u8], req_start: &std::time::Instant, ); fn handle_responses_on_quic_v3( - &mut self, conn: &mut quiceh::Connection, + &mut self, conn: &mut quiceh::Connection, app_buffers: &mut quiceh::AppRecvBufMap, req_start: &std::time::Instant, ); fn report_incomplete(&self, start: &std::time::Instant) -> bool; fn handle_requests( - &mut self, conn: &mut quiceh::Connection, + &mut self, conn: &mut quiceh::Connection, partial_requests: &mut HashMap, partial_responses: &mut HashMap, root: &str, index: &str, buf: &mut [u8], @@ -361,13 +433,13 @@ pub trait HttpConn { ) -> quiceh::h3::Result<()>; fn handle_requests_on_quic_v3( - &mut self, conn: &mut quiceh::Connection, + &mut self, conn: &mut quiceh::Connection, partial_responses: &mut HashMap, root: &str, index: &str, app_buffers: &mut quiceh::AppRecvBufMap, ) -> quiceh::h3::Result<()>; fn handle_writable( - &mut self, conn: &mut quiceh::Connection, + &mut self, conn: &mut quiceh::Connection, partial_responses: &mut HashMap, stream_id: u64, ); } @@ -420,10 +492,13 @@ impl Default for Http09Conn { } impl Http09Conn { - pub fn with_urls( + pub fn with_urls>( urls: &[url::Url], reqs_cardinal: u64, output_sink: Rc>, - ) -> Box { + ) -> Box> + where + ::Buf: BufSplit, + { let mut reqs = Vec::new(); for url in urls { for i in 1..=reqs_cardinal { @@ -450,9 +525,12 @@ impl Http09Conn { } } -impl HttpConn for Http09Conn { +impl> HttpConn for Http09Conn +where + ::Buf: BufSplit, +{ fn send_requests( - &mut self, conn: &mut quiceh::Connection, target_path: &Option, + &mut self, conn: &mut quiceh::Connection, target_path: &Option, ) { let mut reqs_done = 0; @@ -494,7 +572,7 @@ impl HttpConn for Http09Conn { } fn handle_responses_on_quic_v3( - &mut self, conn: &mut quiceh::Connection, + &mut self, conn: &mut quiceh::Connection, app_buffers: &mut quiceh::AppRecvBufMap, req_start: &std::time::Instant, ) { for s in conn.readable() { @@ -562,7 +640,7 @@ impl HttpConn for Http09Conn { } fn handle_responses( - &mut self, conn: &mut quiceh::Connection, buf: &mut [u8], + &mut self, conn: &mut quiceh::Connection, buf: &mut [u8], req_start: &std::time::Instant, ) { // Process all readable streams. @@ -649,7 +727,7 @@ impl HttpConn for Http09Conn { } fn handle_requests_on_quic_v3( - &mut self, conn: &mut quiceh::Connection, + &mut self, conn: &mut quiceh::Connection, partial_responses: &mut HashMap, root: &str, index: &str, app_buffers: &mut quiceh::AppRecvBufMap, ) -> quiceh::h3::Result<()> { @@ -696,11 +774,17 @@ impl HttpConn for Http09Conn { body.len(), s ); + let bodylen = body.len(); - let written = match conn.stream_send(s, &body, true) { + let (written, remaining) = match conn.stream_send_zc( + s, + body.into(), + Some(bodylen), + true, + ) { Ok(v) => v, - Err(quiceh::Error::Done) => 0, + Err(quiceh::Error::Done) => (0, None), Err(e) => { error!( @@ -712,12 +796,12 @@ impl HttpConn for Http09Conn { return Err(From::from(e)); }, }; - if written < body.len() { + if let Some(body) = remaining { let response = PartialResponse { headers: None, priority: None, body, - written, + remaining_len: bodylen.saturating_sub(written), }; partial_responses.insert(s, response); @@ -733,7 +817,7 @@ impl HttpConn for Http09Conn { } fn handle_requests( - &mut self, conn: &mut quiceh::Connection, + &mut self, conn: &mut quiceh::Connection, partial_requests: &mut HashMap, partial_responses: &mut HashMap, root: &str, index: &str, buf: &mut [u8], @@ -810,10 +894,17 @@ impl HttpConn for Http09Conn { s ); - let written = match conn.stream_send(s, &body, true) { + let bodylen = body.len(); + + let (written, remaining) = match conn.stream_send_zc( + s, + body.into(), + Some(bodylen), + true, + ) { Ok(v) => v, - Err(quiceh::Error::Done) => 0, + Err(quiceh::Error::Done) => (0, None), Err(e) => { error!( @@ -825,12 +916,12 @@ impl HttpConn for Http09Conn { }, }; - if written < body.len() { + if let Some(body) = remaining { let response = PartialResponse { headers: None, priority: None, body, - written, + remaining_len: bodylen.saturating_sub(written), }; partial_responses.insert(s, response); @@ -843,7 +934,7 @@ impl HttpConn for Http09Conn { } fn handle_writable( - &mut self, conn: &mut quiceh::Connection, + &mut self, conn: &mut quiceh::Connection, partial_responses: &mut HashMap, stream_id: u64, ) { trace!("{} stream {} is writable", conn.trace_id(), stream_id); @@ -853,12 +944,17 @@ impl HttpConn for Http09Conn { } let resp = partial_responses.get_mut(&stream_id).unwrap(); - let body = &resp.body[resp.written..]; - - let written = match conn.stream_send(stream_id, body, true) { + let body = resp.body.clone(); + + let (written, remaining) = match conn.stream_send_zc( + stream_id, + body, + Some(resp.remaining_len), + true, + ) { Ok(v) => v, - Err(quiceh::Error::Done) => 0, + Err(quiceh::Error::Done) => (0, None), Err(e) => { partial_responses.remove(&stream_id); @@ -868,9 +964,13 @@ impl HttpConn for Http09Conn { }, }; - resp.written += written; - - if resp.written == resp.body.len() { + if let Some(body) = remaining { + resp.body = body; + resp.remaining_len = resp.remaining_len.saturating_sub(written); + if resp.remaining_len == 0 { + partial_responses.remove(&stream_id); + } + } else if written == resp.remaining_len { partial_responses.remove(&stream_id); } } @@ -932,15 +1032,15 @@ pub struct Http3Conn { impl Http3Conn { #[allow(clippy::too_many_arguments)] - pub fn with_urls( - conn: &mut quiceh::Connection, urls: &[url::Url], reqs_cardinal: u64, + pub fn with_urls>( + conn: &mut quiceh::Connection, urls: &[url::Url], reqs_cardinal: u64, req_headers: &[String], body: &Option>, method: &str, send_priority_update: bool, max_field_section_size: Option, qpack_max_table_capacity: Option, qpack_blocked_streams: Option, dump_json: Option, dgram_sender: Option, output_sink: Rc>, - ) -> Box { + ) -> Box> { let mut reqs = Vec::new(); for url in urls { for i in 1..=reqs_cardinal { @@ -1026,13 +1126,13 @@ impl Http3Conn { Box::new(h_conn) } - pub fn with_conn( - conn: &mut quiceh::Connection, max_field_section_size: Option, + pub fn with_conn>( + conn: &mut quiceh::Connection, max_field_section_size: Option, qpack_max_table_capacity: Option, qpack_blocked_streams: Option, dgram_sender: Option, output_sink: Rc>, - ) -> std::result::Result, String> { + ) -> std::result::Result>, String> { let h3_conn = quiceh::h3::Connection::with_transport( conn, &make_h3_config( @@ -1060,8 +1160,8 @@ impl Http3Conn { /// poll the h3_conn with either poll() or poll_v3() depending on the /// connection context. - fn poll_internal( - &mut self, conn: &mut quiceh::Connection, + fn poll_internal( + &mut self, conn: &mut quiceh::Connection, app_buffers: &mut Option<&mut quiceh::AppRecvBufMap>, ) -> quiceh::h3::Result<(u64, quiceh::h3::Event)> { if let Some(ref mut app_buffers) = app_buffers { @@ -1296,9 +1396,12 @@ impl Http3Conn { } } -impl HttpConn for Http3Conn { +impl> HttpConn for Http3Conn +where + F::Buf: BufSplit, +{ fn send_requests( - &mut self, conn: &mut quiceh::Connection, target_path: &Option, + &mut self, conn: &mut quiceh::Connection, target_path: &Option, ) { let mut reqs_done = 0; @@ -1399,7 +1502,7 @@ impl HttpConn for Http3Conn { } fn handle_responses_on_quic_v3( - &mut self, conn: &mut quiceh::Connection, + &mut self, conn: &mut quiceh::Connection, app_buffers: &mut quiceh::AppRecvBufMap, req_start: &std::time::Instant, ) { loop { @@ -1434,8 +1537,9 @@ impl HttpConn for Http3Conn { b }, - - Err(quiceh::h3::Error::Done) => panic!("Error::Done"), + // This may happen if we get QUIC packet with the HTTP Frame hdr only + // It trigers an Event::Data, but there is nothing to read (yet). + Err(quiceh::h3::Error::Done) => break, Err(e) => panic!("Error reading conn: {:?}", e), }; @@ -1556,7 +1660,7 @@ impl HttpConn for Http3Conn { } fn handle_responses( - &mut self, conn: &mut quiceh::Connection, buf: &mut [u8], + &mut self, conn: &mut quiceh::Connection, buf: &mut [u8], req_start: &std::time::Instant, ) { loop { @@ -1728,7 +1832,7 @@ impl HttpConn for Http3Conn { } fn handle_requests_on_quic_v3( - &mut self, conn: &mut quiceh::Connection, + &mut self, conn: &mut quiceh::Connection, partial_responses: &mut HashMap, root: &str, index: &str, app_buffers: &mut quiceh::AppRecvBufMap, ) -> quiceh::h3::Result<()> { @@ -1744,7 +1848,7 @@ impl HttpConn for Http3Conn { } fn handle_requests( - &mut self, conn: &mut quiceh::Connection, + &mut self, conn: &mut quiceh::Connection, _partial_requests: &mut HashMap, partial_responses: &mut HashMap, root: &str, index: &str, buf: &mut [u8], @@ -1809,11 +1913,7 @@ impl HttpConn for Http3Conn { #[cfg(feature = "sfv")] let priority = - match quiceh::h3::Priority::try_from(priority.as_slice()) - { - Ok(v) => v, - Err(_) => quiceh::h3::Priority::default(), - }; + quiceh::h3::Priority::try_from(priority.as_slice()).unwrap_or_default(); #[cfg(not(feature = "sfv"))] let priority = quiceh::h3::Priority::default(); @@ -1831,11 +1931,12 @@ impl HttpConn for Http3Conn { Ok(v) => v, Err(quiceh::h3::Error::StreamBlocked) => { + let len = body.len(); let response = PartialResponse { headers: Some(headers), priority: Some(priority), - body, - written: 0, + body: body.into(), + remaining_len: len, }; partial_responses.insert(stream_id, response); @@ -1852,12 +1953,12 @@ impl HttpConn for Http3Conn { break; }, } - + let len = body.len(); let response = PartialResponse { headers: None, priority: None, - body, - written: 0, + body: body.into(), + remaining_len: len, }; partial_responses.insert(stream_id, response); @@ -1945,7 +2046,7 @@ impl HttpConn for Http3Conn { } fn handle_writable( - &mut self, conn: &mut quiceh::Connection, + &mut self, conn: &mut quiceh::Connection, partial_responses: &mut HashMap, stream_id: u64, ) { debug!("{} stream {} is writable", conn.trace_id(), stream_id); @@ -1975,26 +2076,30 @@ impl HttpConn for Http3Conn { resp.headers = None; resp.priority = None; + let body = resp.body.clone(); + + match self.h3_conn.send_body_zc(conn, stream_id, body, true) { + Ok((written, remaining)) => + if let Some(body) = remaining { + resp.remaining_len = + resp.remaining_len.saturating_sub(written); + resp.body = body; + if resp.remaining_len == 0 { + partial_responses.remove(&stream_id); + } + } else if written == resp.remaining_len { + partial_responses.remove(&stream_id); + }, - let body = &resp.body[resp.written..]; - - let written = match self.h3_conn.send_body(conn, stream_id, body, true) { - Ok(v) => v, - - Err(quiceh::h3::Error::Done) => 0, + Err(quiceh::h3::Error::Done) => { + info!("{} sending on stream_id {} returned Error::Done; likely we're lacking stream + capacity", conn.trace_id(), stream_id); + }, Err(e) => { partial_responses.remove(&stream_id); - error!("{} stream send failed {:?}", conn.trace_id(), e); - return; }, }; - - resp.written += written; - - if resp.written == resp.body.len() { - partial_responses.remove(&stream_id); - } } } diff --git a/bench_i71165.sh b/bench_i71165.sh index 139bcff1..2e4a9477 100755 --- a/bench_i71165.sh +++ b/bench_i71165.sh @@ -6,7 +6,7 @@ BENCH=$1 PROFILE=$2 sudo cpupower -c 0 frequency-set -d $SET_MIN -u $SET_MAX -g performance -taskset -c 0 cargo bench --bench $BENCH --profile $PROFILE -sudo cpupower -c 0 frequency-set -d $MIN -u $MAX -g powersave +taskset -c 0 cargo bench --bench $BENCH --profile $PROFILE -- --profile-time=15 +sudo cpupower -c 0 frequency-set -d $MIN -u $MAX -g performance diff --git a/octets_rev/src/lib.rs b/octets_rev/src/lib.rs index ae979a85..490250bf 100644 --- a/octets_rev/src/lib.rs +++ b/octets_rev/src/lib.rs @@ -28,6 +28,7 @@ use likely_stable::if_unlikely; /// Zero-copy abstraction for parsing and constructing network packets. use std::mem; use std::ptr; +use std::ops::Deref; /// A specialized [`Result`] type for [`OctetsMut`] operations. /// @@ -488,6 +489,14 @@ impl<'a> AsRef<[u8]> for Octets<'a> { } } +impl<'a> Deref for Octets<'a> { + type Target = [u8]; + + fn deref(&self) -> &[u8] { + &self.buf[self.off..] + } +} + /// A zero-copy mutable byte buffer. /// /// Like `Octets` but mutable. diff --git a/quiceh/Cargo.toml b/quiceh/Cargo.toml index 10d10117..b14a4496 100644 --- a/quiceh/Cargo.toml +++ b/quiceh/Cargo.toml @@ -77,6 +77,7 @@ smallvec = { version = "1.10", features = ["serde", "union"] } likely_stable = "0.1.2" cpu-time = "1.0.0" itertools = "0.12.1" +criterion-cycles-per-byte = "0.6.1" [target."cfg(windows)".dependencies] winapi = { version = "0.3", features = ["wincrypt", "ws2def", "ws2ipdef", "ws2tcpip"] } @@ -84,13 +85,12 @@ winapi = { version = "0.3", features = ["wincrypt", "ws2def", "ws2ipdef", "ws2tc [dev-dependencies] mio = { version = "0.8", features = ["net", "os-poll"] } url = "2.5" -# criterion = { version = "0.5.1", features = ["html_reports"] } -criterion = { version = "0.5.1"} +criterion = { version = "0.5.1" } +pprof = { version = "0.13", features = ["flamegraph", "criterion"] } [lib] crate-type = ["lib", "staticlib", "cdylib"] - [[bench]] name = "quic_benchmarks" harness = false @@ -102,3 +102,15 @@ harness = false [[bench]] name = "h3_benchmarks" harness = false + +[[bench]] +name = "send_cwin_bench" +harness = false + +[[bench]] +name = "send_cwin_profile_bench" +harness = false + +[[bench]] +name = "profile_bench_send" +harness = false diff --git a/quiceh/benches/bench_util.rs b/quiceh/benches/bench_util.rs index 3316900f..b9744c01 100644 --- a/quiceh/benches/bench_util.rs +++ b/quiceh/benches/bench_util.rs @@ -2,12 +2,54 @@ use cpu_time::ProcessTime; use criterion::measurement::Measurement; use criterion::measurement::ValueFormatter; use criterion::Throughput; +use quiceh::BufFactory; +use quiceh::BufSplit; use std::time::Duration; +use std::sync::Arc; const NANOS_PER_SEC: u64 = 1_000_000_000; -/// Keeps track of QUIC streams and enforces stream limits. +#[derive(Debug, Clone, Default)] +pub struct BenchBufFactory; + +#[derive(Debug, Clone, Default)] +pub struct BenchBuf(Arc>); + +impl BufFactory for BenchBufFactory { + type Buf = BenchBuf; + + fn buf_from_slice(buf: &[u8]) -> Self::Buf { + BenchBuf(Arc::new(buf.into())) + } +} + +impl AsRef<[u8]> for BenchBuf { + fn as_ref(&self) -> &[u8] { + &self.0[..] + } +} + +impl From> for BenchBuf { + fn from(value: Vec) -> Self { + BenchBuf(Arc::new(value.into_boxed_slice())) + } +} +impl BufSplit for BenchBuf { + fn split_at(&mut self, _at: usize) -> Self { + // There is enough capacity in the stream send buffer for the simple + // bench of 10 QUIC packets we run. This function is then never called + // internally in quiceh. + // + // Should we implement a bench test case that 1) send more than what the + // stream's capacity can do, 2) process ack packets from the peer, + // 3) send more. This would however capture some of the cost of + // the receive path as well. + unimplemented!(); + } +} + +/// Keeps track of QUIC streams and enforces stream limits. pub struct CPUTime; impl Measurement for CPUTime { type Intermediate = ProcessTime; diff --git a/quiceh/benches/send_cwin_bench.rs b/quiceh/benches/send_cwin_bench.rs new file mode 100644 index 00000000..b6feed6e --- /dev/null +++ b/quiceh/benches/send_cwin_bench.rs @@ -0,0 +1,191 @@ +mod bench_util; +use bench_util::*; +use criterion::black_box; +use criterion::criterion_group; +use criterion::criterion_main; +use criterion::BatchSize; +use criterion::BenchmarkId; +use criterion::Criterion; +use criterion::Throughput; +use quiceh::testing::Pipe; +use quiceh::BufFactory; +use criterion_cycles_per_byte::CyclesPerByte; + +use quiceh::BufSplit; + +const MAX_DATAGRAM_SIZE: usize = 1350; + +fn bench_stream_send(pipe: &mut Pipe, outbuf: &mut [u8], sendbuf: &[u8]) { + pipe.client.stream_send(4, sendbuf, true).unwrap(); + + loop { + let (write, send_info) = match pipe.client.send_on_path(outbuf, None, None) { + Ok(v) => v, + + Err(quiceh::Error::Done) => { + // we sent everything + break; + }, + + Err(e) => { + panic!("An error occured {:?}", e); + } + }; + black_box(write); + black_box(send_info); + } + black_box(outbuf); +} + +fn bench_stream_send_zc>( + pipe: &mut Pipe, outbuf: &mut [u8], benchbuf: &F::Buf, +) where + ::Buf: BufSplit, +{ + pipe.client + .stream_send_zc( + 4, + benchbuf.clone(), + Some(10000), + true, + ) + .unwrap(); + loop { + let (write, send_info) = match pipe.client.send_on_path(outbuf, None, None) { + Ok(v) => v, + + Err(quiceh::Error::Done) => { + break; + }, + + Err(e) => { + panic!("An error occured {:?}", e); + } + }; + black_box(write); + black_box(send_info); + } + black_box(outbuf); +} + +fn bench_sender(c: &mut Criterion, config: &mut quiceh::Config, name: &str) { + let mut group = c.benchmark_group(name); + group.throughput(Throughput::Bytes(10000)); + + let sendbuf = vec![0; 10000]; + let benchbuf = BenchBufFactory::buf_from_slice(&sendbuf); + + group.bench_with_input( + BenchmarkId::new("zerocopy_send_path", 10000), + &benchbuf, + |b, benchbuf| { + b.iter_batched_ref( + || { + let mut pipe = + Pipe::::with_config(config).unwrap(); + pipe.handshake().unwrap(); + let outbuf = vec![0; 65535]; + (pipe, outbuf) + }, + |(ref mut pipe, ref mut outbuf)| bench_stream_send_zc(pipe, outbuf, benchbuf), + BatchSize::SmallInput, + ) + }); + + group.bench_with_input( + BenchmarkId::new("send_path", 10000), + &sendbuf, + |b, sendbuf| { + b.iter_batched_ref( + || { + let mut pipe = Pipe::with_config(config).unwrap(); + pipe.handshake().unwrap(); + let outbuf = vec![0; 65535]; + (pipe, outbuf) + }, + |(ref mut pipe, ref mut outbuf)| { + bench_stream_send(pipe, outbuf, sendbuf) + }, + BatchSize::SmallInput, + ); + }, + ); + + + group.finish(); +} + +fn send_bench_hidden_copy(c: &mut Criterion) { + let mut config = + quiceh::Config::new(quiceh::PROTOCOL_VERSION_VREVERSO).unwrap(); + config + .set_application_protos(&[b"proto1", b"proto2"]) + .unwrap(); + config + .load_cert_chain_from_pem_file("examples/cert.crt") + .unwrap(); + config + .load_priv_key_from_pem_file("examples/cert.key") + .unwrap(); + config + .set_application_protos(&[b"proto1", b"proto2"]) + .unwrap(); + config.set_max_recv_udp_payload_size(MAX_DATAGRAM_SIZE); + config.set_max_send_udp_payload_size(MAX_DATAGRAM_SIZE); + config.set_initial_max_data(10_000_000_000); + config.set_max_stream_window(25_165_824); + config.set_initial_max_stream_data_uni(10_000_000_000); + config.set_initial_max_streams_bidi(10_000_000_000); + config.set_initial_max_stream_data_bidi_local(10_000_000_000); + config.set_initial_max_stream_data_bidi_remote(10_000_000_000); + config.set_initial_congestion_window_packets(20_000_000); + config.verify_peer(false); + config.enable_pacing(false); + config.enable_hidden_copy_for_zc_sender(true); + + bench_sender(c, &mut config, "send_path_with_hidden_copy"); + +} + +fn send_bench_no_hidden_copy(c: &mut Criterion) { + let mut config = + quiceh::Config::new(quiceh::PROTOCOL_VERSION_VREVERSO).unwrap(); + config + .set_application_protos(&[b"proto1", b"proto2"]) + .unwrap(); + config + .load_cert_chain_from_pem_file("examples/cert.crt") + .unwrap(); + config + .load_priv_key_from_pem_file("examples/cert.key") + .unwrap(); + config + .set_application_protos(&[b"proto1", b"proto2"]) + .unwrap(); + config.set_max_recv_udp_payload_size(MAX_DATAGRAM_SIZE); + config.set_max_send_udp_payload_size(MAX_DATAGRAM_SIZE); + config.set_initial_max_data(10_000_000_000); + config.set_max_stream_window(25_165_824); + config.set_initial_max_stream_data_uni(10_000_000_000); + config.set_initial_max_streams_bidi(10_000_000_000); + config.set_initial_max_stream_data_bidi_local(10_000_000_000); + config.set_initial_max_stream_data_bidi_remote(10_000_000_000); + config.set_initial_congestion_window_packets(20_000_000); + config.verify_peer(false); + config.enable_pacing(false); + config.enable_hidden_copy_for_zc_sender(false); + + bench_sender(c, &mut config, "send_path_no_hidden_copy"); + +} + +criterion_group! { + name = send_cwin_bench; + config = Criterion::default() + .measurement_time(std::time::Duration::from_millis(10000)) + .with_measurement(CyclesPerByte) + .sample_size(5000); + targets = send_bench_no_hidden_copy, send_bench_hidden_copy +} + +criterion_main!(send_cwin_bench); diff --git a/quiceh/benches/send_cwin_profile_bench.rs b/quiceh/benches/send_cwin_profile_bench.rs new file mode 100644 index 00000000..910b0643 --- /dev/null +++ b/quiceh/benches/send_cwin_profile_bench.rs @@ -0,0 +1,214 @@ +mod bench_util; +use bench_util::*; +use criterion::black_box; +use criterion::criterion_group; +use criterion::criterion_main; +use criterion::BatchSize; +use criterion::BenchmarkId; +use criterion::Criterion; +use criterion::Throughput; +use quiceh::testing::Pipe; +use quiceh::BufFactory; +use quiceh::BufSplit; +use pprof::criterion::{PProfProfiler, Output}; + +const MAX_DATAGRAM_SIZE: usize = 1350; + + +// /!\ this code is designed for flamegraph inspection of the send path logic; not to get reliable +// efficiency measurements. +fn bench_stream_send(pipe: &mut Pipe, config: &mut quiceh::Config, outbuf: &mut [u8], sendbuf: &[u8], stream_id: &mut u64) { + // Every 1000 iteration we destroy the pipe. Otherwise we have memory issues since it send may + // grow the pipe's mem. We don't re-do an handshake at each sample bench outside of the timed + // function, or the flamegraph would be unreadable. + if *stream_id > 1_000 { + std::mem::swap(pipe, &mut Pipe::with_config(config).unwrap()); + pipe.handshake().unwrap(); + *stream_id = 0; + } + *stream_id += 4; + pipe.client.stream_send(*stream_id, sendbuf, true).unwrap(); + + loop { + let (write, send_info) = match pipe.client.send_on_path(outbuf, None, None) { + Ok(v) => v, + + Err(quiceh::Error::Done) => { + // we sent everything + break; + }, + + Err(e) => { + panic!("An error occured {:?}", e); + } + }; + black_box(write); + black_box(send_info); + } + black_box(outbuf); +} + +fn bench_stream_send_zc>( + pipe: &mut Pipe, config: &mut quiceh::Config, outbuf: &mut [u8], benchbuf: &F::Buf, stream_id: &mut u64, +) where + ::Buf: BufSplit, +{ + if *stream_id > 1_000 { + std::mem::swap(pipe, &mut Pipe::with_config(config).unwrap()); + pipe.handshake().unwrap(); + *stream_id = 0; + } + *stream_id += 4; + pipe.client + .stream_send_zc( + *stream_id, + benchbuf.clone(), + Some(10000), + true, + ) + .unwrap(); + loop { + let (write, send_info) = match pipe.client.send_on_path(outbuf, None, None) { + Ok(v) => v, + + Err(quiceh::Error::Done) => { + break; + }, + + Err(e) => { + panic!("An error occured {:?}", e); + } + }; + black_box(write); + black_box(send_info); + } + black_box(outbuf); +} + +fn bench_sender(c: &mut Criterion, config: &mut quiceh::Config, name: &str) { + let mut group = c.benchmark_group(name); + group.throughput(Throughput::Bytes(10000)); + + let mut pipe = Pipe::with_config(config).unwrap(); + pipe.handshake().unwrap(); + let sendbuf = vec![0; 10000]; + let mut stream_id = 0; + group.bench_with_input( + BenchmarkId::new("send_path", 10000), + &sendbuf, + |b, sendbuf| { + b.iter_batched_ref( + || { + let outbuf = vec![0; 65535]; + outbuf + }, + |ref mut outbuf| { + bench_stream_send(&mut pipe, config, outbuf, sendbuf, &mut stream_id) + }, + BatchSize::SmallInput, + ); + }, + ); + + let benchbuf = BenchBufFactory::buf_from_slice(&sendbuf); + let mut pipe = + Pipe::::with_config(config).unwrap(); + pipe.handshake().unwrap(); + stream_id = 0; + + group.bench_with_input( + BenchmarkId::new("zerocopy_send_path", 10000), + &benchbuf, + |b, benchbuf| { + b.iter_batched_ref( + || { + let outbuf = vec![0; 65535]; + outbuf + }, + |ref mut outbuf| bench_stream_send_zc(&mut pipe, config, outbuf, benchbuf, &mut stream_id), + BatchSize::SmallInput, + ) + }); + + group.finish(); +} + +fn send_bench_hidden_copy(c: &mut Criterion) { + let mut config = + quiceh::Config::new(quiceh::PROTOCOL_VERSION_VREVERSO).unwrap(); + config + .set_application_protos(&[b"proto1", b"proto2"]) + .unwrap(); + config + .load_cert_chain_from_pem_file("examples/cert.crt") + .unwrap(); + config + .load_priv_key_from_pem_file("examples/cert.key") + .unwrap(); + config + .set_application_protos(&[b"proto1", b"proto2"]) + .unwrap(); + config.set_max_recv_udp_payload_size(MAX_DATAGRAM_SIZE); + config.set_max_send_udp_payload_size(MAX_DATAGRAM_SIZE); + config.set_initial_max_data(10_000_000_000); + config.set_max_stream_window(25_165_824); + config.set_initial_max_stream_data_uni(10_000_000_000); + config.set_initial_max_streams_bidi(10_000_000_000); + config.set_initial_max_stream_data_bidi_local(10_000_000_000); + config.set_initial_max_stream_data_bidi_remote(10_000_000_000); + config.set_initial_congestion_window_packets(20_000_000); + config.verify_peer(false); + config.enable_pacing(false); + config.enable_hidden_copy_for_zc_sender(true); + + bench_sender(c, &mut config, "send_path_with_hidden_copy"); + +} + +fn send_bench_no_hidden_copy(c: &mut Criterion) { + let mut config = + quiceh::Config::new(quiceh::PROTOCOL_VERSION_VREVERSO).unwrap(); + config + .set_application_protos(&[b"proto1", b"proto2"]) + .unwrap(); + config + .load_cert_chain_from_pem_file("examples/cert.crt") + .unwrap(); + config + .load_priv_key_from_pem_file("examples/cert.key") + .unwrap(); + config + .set_application_protos(&[b"proto1", b"proto2"]) + .unwrap(); + config.set_max_recv_udp_payload_size(MAX_DATAGRAM_SIZE); + config.set_max_send_udp_payload_size(MAX_DATAGRAM_SIZE); + config.set_initial_max_data(10_000_000_000); + config.set_max_stream_window(25_165_824); + config.set_initial_max_stream_data_uni(10_000_000_000); + config.set_initial_max_streams_bidi(10_000_000_000); + config.set_initial_max_stream_data_bidi_local(10_000_000_000); + config.set_initial_max_stream_data_bidi_remote(10_000_000_000); + config.set_initial_congestion_window_packets(20_000_000); + config.verify_peer(false); + config.enable_pacing(false); + config.enable_hidden_copy_for_zc_sender(false); + + bench_sender(c, &mut config, "send_path_no_hidden_copy"); + +} + +criterion_group! { + name = send_cwin_profile_bench; + config = Criterion::default() + .measurement_time(std::time::Duration::from_millis(1000)) + .with_profiler({ + let mut options = pprof::flamegraph::Options::default(); + options.image_width = Some(10000); + PProfProfiler::new(999, Output::Flamegraph(Some(options))) + }) + .with_measurement(CPUTime) + .sample_size(5000); + targets = send_bench_no_hidden_copy, send_bench_hidden_copy +} + +criterion_main!(send_cwin_profile_bench); diff --git a/quiceh/src/crypto/boringssl.rs b/quiceh/src/crypto/boringssl.rs index 13c92d1f..076e3ccd 100644 --- a/quiceh/src/crypto/boringssl.rs +++ b/quiceh/src/crypto/boringssl.rs @@ -109,7 +109,7 @@ impl Open { impl Seal { pub fn seal_with_u64_counter( &self, counter: u64, ad: &[u8], buf: &mut [u8], in_len: usize, - extra_in: Option<&[u8]>, + in_buf: Option<&[u8]>, extra_in: Option<&[u8]>, scatter_crypt: bool, ) -> Result { if cfg!(feature = "fuzzing") { if let Some(extra) = extra_in { @@ -122,31 +122,54 @@ impl Seal { let tag_len = self.alg().tag_len(); - let mut out_tag_len = tag_len; - + // So: if we use hidden_copy + // + // buf is the desitination buffer whose length + // ends at the payload offset. + // + // If REVERSO: + // in_buf is the stream_frame of in_buf.len() bytes + // extra_in is pointing to buf at payload_offset+stream_len. + // remaining length should hold extra_in_len + tag_len + // + // if V1: + // in_buf is the ctrl data which is contained inside buf at position payload_offset + // this is expected to be written in buf at the same position. + // the length of in_buf should be able to hold ctrl_len, stream_len and tag_len + // extra_in contains the stream_frame of stream_len bytes. + // let (extra_in_ptr, extra_in_len) = match extra_in { Some(v) => (v.as_ptr(), v.len()), - - None => (std::ptr::null(), 0), + None => (std::ptr::null(), 0), }; - // Make sure all the outputs combined fit in the buffer. - if in_len + tag_len + extra_in_len > buf.len() { + let mut out_tag_len = tag_len + extra_in_len; + + // Make sure it fits in the buffer. + if !scatter_crypt && in_len + tag_len > buf.len() { return Err(Error::CryptoFail); } let nonce = make_nonce(&self.packet.nonce, counter); + let in_ptr = if let Some(in_buf) = in_buf { + in_buf.as_ptr() + } else { + buf.as_ptr() + }; + + let rc = unsafe { + let out_tag_mut_ptr = buf.as_mut_ptr().add(in_len); EVP_AEAD_CTX_seal_scatter( &self.packet.ctx, // ctx buf.as_mut_ptr(), // out - buf[in_len..].as_mut_ptr(), // out_tag + out_tag_mut_ptr, // out_tag &mut out_tag_len, // out_tag_len tag_len + extra_in_len, // max_out_tag_len nonce[..].as_ptr(), // nonce nonce.len(), // nonce_len - buf.as_ptr(), // inp + in_ptr, // inp in_len, // in_len extra_in_ptr, // extra_in extra_in_len, // extra_in_len diff --git a/quiceh/src/frame.rs b/quiceh/src/frame.rs index 681213f7..5282b270 100644 --- a/quiceh/src/frame.rs +++ b/quiceh/src/frame.rs @@ -30,6 +30,7 @@ use crate::Error; use crate::Result; use crate::packet; +use crate::range_buf::RangeBuf; use crate::ranges; use crate::stream; use likely_stable::if_likely; @@ -88,7 +89,13 @@ pub enum Frame { }, Crypto { - data: stream::RangeBuf, + data: RangeBuf, + }, + + CryptoVec { + offset: u64, + length: usize, + rbvec: Vec, }, CryptoHeader { @@ -102,7 +109,7 @@ pub enum Frame { Stream { stream_id: u64, - data: stream::RangeBuf, + data: RangeBuf, }, StreamV3 { @@ -266,7 +273,7 @@ impl Frame { b.get_bytes_with_varint_length()?) }}; // TODO protocol reverso could get rid of RangeBuf. - let data = stream::RangeBuf::from(data.as_ref(), offset, false); + let data = ::from(data.as_ref(), offset, false); Frame::Crypto { data } }, @@ -679,6 +686,7 @@ impl Frame { }, Frame::CryptoHeader { .. } => (), + Frame::CryptoVec { .. } => (), Frame::NewToken { token } => { if_likely! {version == crate::PROTOCOL_VERSION_VREVERSO => { @@ -1019,6 +1027,13 @@ impl Frame { data.len() // data }, + Frame::CryptoVec { offset, length, .. } => { + 1 + // frame type + octets_rev::varint_len(*offset) + + 2 + + length // data + }, + Frame::CryptoHeader { offset, length, .. } => { 1 + // frame type octets_rev::varint_len(*offset) + // offset @@ -1264,7 +1279,8 @@ impl Frame { length: data.len() as u64, }, - Frame::CryptoHeader { offset, length } => QuicFrame::Crypto { + Frame::CryptoHeader { offset, length } | + Frame::CryptoVec { offset, length, .. } => QuicFrame::Crypto { offset: *offset, length: *length as u64, }, @@ -1458,6 +1474,10 @@ impl std::fmt::Debug for Frame { write!(f, "CRYPTO off={offset} len={length}")?; }, + Frame::CryptoVec { offset, length, .. } => { + write!(f, "CRYPTO off={offset} len={length}")?; + }, + Frame::NewToken { token } => { write!(f, "NEW_TOKEN len={}", token.len())?; }, @@ -1824,7 +1844,7 @@ fn parse_stream_frame( let fin = first & 0x01 != 0; let data = b.get_bytes(len)?; - let data = stream::RangeBuf::from(data.as_ref(), offset, fin); + let data = ::from(data.as_ref(), offset, fin); Ok(Frame::Stream { stream_id, data }) }} @@ -2267,7 +2287,7 @@ mod tests { let data = [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12]; let frame = Frame::Crypto { - data: stream::RangeBuf::from(&data, 1230976, false), + data: ::from(&data, 1230976, false), }; let wire_len = { @@ -2398,7 +2418,7 @@ mod tests { }; let frame = Frame::Stream { stream_id: 32, - data: stream::RangeBuf::from(&data, 1230976, true), + data: ::from(&data, 1230976, true), }; let wire_len = { @@ -2472,7 +2492,7 @@ mod tests { let frame = Frame::Stream { stream_id: 32, - data: stream::RangeBuf::from(&data, MAX_STREAM_SIZE - 11, true), + data: ::from(&data, MAX_STREAM_SIZE - 11, true), }; let wire_len = { diff --git a/quiceh/src/h3/mod.rs b/quiceh/src/h3/mod.rs index f6c4b50b..42757077 100644 --- a/quiceh/src/h3/mod.rs +++ b/quiceh/src/h3/mod.rs @@ -313,6 +313,9 @@ use qlog::events::EventImportance; #[cfg(feature = "qlog")] use qlog::events::EventType; +use crate::range_buf::BufFactory; +use crate::BufSplit; + /// List of ALPN tokens of supported HTTP/3 versions. /// /// This can be passed directly to the [`Config::set_application_protos()`] @@ -1046,8 +1049,8 @@ impl Connection { /// /// [`StreamLimit`]: ../enum.Error.html#variant.StreamLimit /// [`InternalError`]: ../enum.Error.html#variant.InternalError - pub fn with_transport( - conn: &mut super::Connection, config: &Config, + pub fn with_transport( + conn: &mut super::Connection, config: &Config, ) -> Result { let is_client = !conn.is_server; if is_client && !(conn.is_established() || conn.is_in_early_data()) { @@ -1097,8 +1100,8 @@ impl Connection { /// /// [`send_body()`]: struct.Connection.html#method.send_body /// [`StreamBlocked`]: enum.Error.html#variant.StreamBlocked - pub fn send_request( - &mut self, conn: &mut super::Connection, headers: &[T], fin: bool, + pub fn send_request( + &mut self, conn: &mut super::Connection, headers: &[T], fin: bool, ) -> Result { // If we received a GOAWAY from the peer, MUST NOT initiate new // requests. @@ -1110,7 +1113,7 @@ impl Connection { self.streams.insert( stream_id, - stream::Stream::new(stream_id, true, conn.version), + ::new(stream_id, true, conn.version), ); // The underlying QUIC stream does not exist yet, so calls to e.g. @@ -1152,9 +1155,9 @@ impl Connection { /// /// [`send_body()`]: struct.Connection.html#method.send_body /// [`StreamBlocked`]: enum.Error.html#variant.StreamBlocked - pub fn send_response( - &mut self, conn: &mut super::Connection, stream_id: u64, headers: &[T], - fin: bool, + pub fn send_response( + &mut self, conn: &mut super::Connection, stream_id: u64, + headers: &[T], fin: bool, ) -> Result<()> { let priority = Default::default(); @@ -1179,9 +1182,9 @@ impl Connection { /// /// [`StreamBlocked`]: enum.Error.html#variant.StreamBlocked /// [Extensible Priority]: https://www.rfc-editor.org/rfc/rfc9218.html#section-4. - pub fn send_response_with_priority( - &mut self, conn: &mut super::Connection, stream_id: u64, headers: &[T], - priority: &Priority, fin: bool, + pub fn send_response_with_priority( + &mut self, conn: &mut super::Connection, stream_id: u64, + headers: &[T], priority: &Priority, fin: bool, ) -> Result<()> { if !self.streams.contains_key(&stream_id) { return Err(Error::FrameUnexpected); @@ -1218,9 +1221,9 @@ impl Connection { Ok(header_block) } - fn send_headers( - &mut self, conn: &mut super::Connection, stream_id: u64, headers: &[T], - fin: bool, + fn send_headers( + &mut self, conn: &mut super::Connection, stream_id: u64, + headers: &[T], fin: bool, ) -> Result<()> { let mut d = [42; 10]; let mut b = octets_rev::OctetsMut::with_slice(&mut d); @@ -1314,13 +1317,90 @@ impl Connection { /// writable again. /// /// [`Done`]: enum.Error.html#variant.Done - pub fn send_body( - &mut self, conn: &mut super::Connection, stream_id: u64, body: &[u8], + pub fn send_body( + &mut self, conn: &mut super::Connection, stream_id: u64, body: &[u8], fin: bool, ) -> Result { + self.do_send_body( + conn, + stream_id, + body, + fin, + |conn: &mut super::Connection, + stream_id: u64, + body: &[u8], + body_len: usize, + fin: bool| { + Ok(conn + .stream_send(stream_id, &body[..body_len], fin) + .map(|v| (v, v))?) + }, + ) + } + + /// Sends an HTTP/3 body chunk provided as a raw buffer on the given stream. + /// + /// If the capacity allows it the buffer will be appended to the stream's + /// send queue with zero copying. + /// + /// On success the number of bytes written is returned, or [`Done`] if no + /// bytes could be written (e.g. because the stream is blocked). + /// + /// Note that the number of written bytes returned can be lower than the + /// length of the input buffer when the underlying QUIC stream doesn't have + /// enough capacity for the operation to complete. + /// + /// When a partial write happens (including when [`Done`] is returned) the + /// remaining (unwrittent) buffer will also be returned. The application + /// should retry the operation once the stream is reported as writable + /// again. + /// + /// [`Done`]: enum.Error.html#variant.Done + pub fn send_body_zc( + &mut self, conn: &mut super::Connection, stream_id: u64, body: F::Buf, + fin: bool, + ) -> Result<(usize, Option)> + where + F: BufFactory, + F::Buf: BufSplit, + { + self.do_send_body( + conn, + stream_id, + body, + fin, + |conn: &mut super::Connection, + stream_id: u64, + body: F::Buf, + body_len: usize, + fin: bool| { + Ok(conn + .stream_send_zc(stream_id, body, Some(body_len), fin) + .map(|v| (v.0, v))?) + }, + ) + } + + fn do_send_body( + &mut self, conn: &mut super::Connection, stream_id: u64, body: B, + fin: bool, write_fn: SND, + ) -> Result + where + F: BufFactory, + B: AsRef<[u8]>, + SND: FnOnce( + &mut super::Connection, + u64, + B, + usize, + bool, + ) -> Result<(usize, R)>, + { let mut d = [42; 10]; let mut b = octets_rev::OctetsMut::with_slice(&mut d); + let len = body.as_ref().len(); + // Validate that it is sane to send data on the stream. if stream_id % 4 != 0 || (stream_id == 0 && @@ -1341,12 +1421,12 @@ impl Connection { }; // Avoid sending 0-length DATA frames when the fin flag is false. - if body.is_empty() && !fin { + if len == 0 && !fin { return Err(Error::Done); } let overhead = octets_rev::varint_len(frame::DATA_FRAME_TYPE_ID) + - octets_rev::varint_len(body.len() as u64); + octets_rev::varint_len(len as u64); let stream_cap = match conn.stream_capacity(stream_id) { Ok(v) => v, @@ -1367,11 +1447,11 @@ impl Connection { } // Cap the frame payload length to the stream's capacity. - let body_len = std::cmp::min(body.len(), stream_cap - overhead); + let body_len = std::cmp::min(len, stream_cap - overhead); // If we can't send the entire body, set the fin flag to false so the // application can try again later. - let fin = if body_len != body.len() { false } else { fin }; + let fin = if body_len != len { false } else { fin }; // Again, avoid sending 0-length DATA frames when the fin flag is false. if body_len == 0 && !fin { @@ -1386,7 +1466,7 @@ impl Connection { // Return how many bytes were written, excluding the frame header. // Sending body separately avoids unnecessary copy. - let written = conn.stream_send(stream_id, &body[..body_len], fin)?; + let (written, ret) = write_fn(conn, stream_id, body, body_len, fin)?; trace!( "{} tx frm DATA stream={} len={} fin={}", @@ -1408,7 +1488,7 @@ impl Connection { q.add_event_data_now(ev_data).ok(); }); - if written < body.len() { + if written < len { // Ensure the peer is notified that the connection or stream is // blocked when the stream's capacity is limited by flow control. // @@ -1418,11 +1498,11 @@ impl Connection { let _ = conn.stream_writable(stream_id, overhead + 1); } - if fin && written == body.len() && conn.stream_finished(stream_id) { + if fin && written == len && conn.stream_finished(stream_id) { self.streams.remove(&stream_id); } - Ok(written) + Ok(ret) } /// Returns whether the peer enabled HTTP/3 DATAGRAM frame support. @@ -1432,7 +1512,9 @@ impl Connection { /// method. /// /// [`poll()`]: struct.Connection.html#method.poll - pub fn dgram_enabled_by_peer(&self, conn: &super::Connection) -> bool { + pub fn dgram_enabled_by_peer( + &self, conn: &super::Connection, + ) -> bool { self.peer_settings.h3_datagram == Some(1) && conn.dgram_max_writable_len().is_some() } @@ -1471,8 +1553,8 @@ impl Connection { /// [`body_consumed()`]: struct.Connection.html#method.body_consumed /// [`Data`]: enum.Event.html#variant.Data /// [`Done`]: enum.Error.html#variant.Done - pub fn recv_body_v3<'a>( - &mut self, conn: &mut super::Connection, stream_id: u64, + pub fn recv_body_v3<'a, F: BufFactory>( + &mut self, conn: &mut super::Connection, stream_id: u64, app_buf: &'a mut crate::AppRecvBufMap, ) -> Result<(&'a [u8], usize)> { if conn.version != crate::PROTOCOL_VERSION_VREVERSO { @@ -1512,9 +1594,9 @@ impl Connection { /// processing state. /// /// [`recv_body_v3()`]: struct.Connection.html#method.recv_body_v3 - pub fn body_consumed( - &mut self, conn: &mut super::Connection, stream_id: u64, consumed: usize, - app_buf: &mut crate::AppRecvBufMap, + pub fn body_consumed( + &mut self, conn: &mut super::Connection, stream_id: u64, + consumed: usize, app_buf: &mut crate::AppRecvBufMap, ) -> Result<()> { if conn.version != crate::PROTOCOL_VERSION_VREVERSO { return Err(Error::InvalidAPICall( @@ -1560,8 +1642,9 @@ impl Connection { /// [`poll()`]: struct.Connection.html#method.poll /// [`Data`]: enum.Event.html#variant.Data /// [`Done`]: enum.Error.html#variant.Done - pub fn recv_body( - &mut self, conn: &mut super::Connection, stream_id: u64, out: &mut [u8], + pub fn recv_body( + &mut self, conn: &mut super::Connection, stream_id: u64, + out: &mut [u8], ) -> Result { if conn.version != crate::PROTOCOL_VERSION_V1 { return Err(Error::InvalidAPICall( @@ -1641,8 +1724,8 @@ impl Connection { /// /// [`StreamBlocked`]: enum.Error.html#variant.StreamBlocked /// [Extensible Priority]: https://www.rfc-editor.org/rfc/rfc9218.html#section-4. - pub fn send_priority_update_for_request( - &mut self, conn: &mut super::Connection, stream_id: u64, + pub fn send_priority_update_for_request( + &mut self, conn: &mut super::Connection, stream_id: u64, priority: &Priority, ) -> Result<()> { let mut d = [42; 20]; @@ -1794,7 +1877,9 @@ impl Connection { /// [`recv_dgram()`]: struct.Connection.html#method.recv_dgram /// [`take_last_priority_update()`]: struct.Connection.html#method.take_last_priority_update /// [`close()`]: ../struct.Connection.html#method.close - pub fn poll(&mut self, conn: &mut super::Connection) -> Result<(u64, Event)> { + pub fn poll( + &mut self, conn: &mut super::Connection, + ) -> Result<(u64, Event)> { if conn.version != crate::PROTOCOL_VERSION_V1 { return Err(Error::InvalidAPICall( "This function should be called on a \ @@ -1842,16 +1927,16 @@ impl Connection { /// [`recv_dgram()`]: struct.Connection.html#method.recv_dgram /// [`take_last_priority_update()`]: struct.Connection.html#method.take_last_priority_update /// [`close()`]: ../struct.Connection.html#method.close - pub fn poll_v3( - &mut self, conn: &mut super::Connection, + pub fn poll_v3( + &mut self, conn: &mut super::Connection, app_buf: &mut crate::AppRecvBufMap, ) -> Result<(u64, Event)> { self.poll_internal(conn, Some(app_buf)) } #[inline(always)] - fn poll_internal( - &mut self, conn: &mut super::Connection, + fn poll_internal( + &mut self, conn: &mut super::Connection, mut app_buf: Option<&mut crate::AppRecvBufMap>, ) -> Result<(u64, Event)> { // When connection close is initiated by the local application (e.g. due @@ -1968,8 +2053,8 @@ impl Connection { /// required to call [`close()`] themselves. /// /// [`close()`]: ../struct.Connection.html#method.close - pub fn send_goaway( - &mut self, conn: &mut super::Connection, id: u64, + pub fn send_goaway( + &mut self, conn: &mut super::Connection, id: u64, ) -> Result<()> { let mut id = id; @@ -2036,8 +2121,8 @@ impl Connection { self.peer_settings.raw.as_deref() } - fn open_uni_stream( - &mut self, conn: &mut super::Connection, ty: u64, + fn open_uni_stream( + &mut self, conn: &mut super::Connection, ty: u64, ) -> Result { let stream_id = self.next_uni_stream_id; @@ -2073,8 +2158,8 @@ impl Connection { Ok(stream_id) } - fn open_qpack_encoder_stream( - &mut self, conn: &mut super::Connection, + fn open_qpack_encoder_stream( + &mut self, conn: &mut super::Connection, ) -> Result<()> { let stream_id = self.open_uni_stream(conn, stream::QPACK_ENCODER_STREAM_TYPE_ID)?; @@ -2096,8 +2181,8 @@ impl Connection { Ok(()) } - fn open_qpack_decoder_stream( - &mut self, conn: &mut super::Connection, + fn open_qpack_decoder_stream( + &mut self, conn: &mut super::Connection, ) -> Result<()> { let stream_id = self.open_uni_stream(conn, stream::QPACK_DECODER_STREAM_TYPE_ID)?; @@ -2120,8 +2205,8 @@ impl Connection { } /// Send GREASE frames on the provided stream ID. - fn send_grease_frames( - &mut self, conn: &mut super::Connection, stream_id: u64, + fn send_grease_frames( + &mut self, conn: &mut super::Connection, stream_id: u64, ) -> Result<()> { let mut d = [0; 8]; @@ -2213,7 +2298,9 @@ impl Connection { /// Opens a new unidirectional stream with a GREASE type and sends some /// unframed payload. - fn open_grease_stream(&mut self, conn: &mut super::Connection) -> Result<()> { + fn open_grease_stream( + &mut self, conn: &mut super::Connection, + ) -> Result<()> { let ty = grease_value(); match self.open_uni_stream(conn, ty) { Ok(stream_id) => { @@ -2247,7 +2334,9 @@ impl Connection { } /// Sends SETTINGS frame based on HTTP/3 configuration. - fn send_settings(&mut self, conn: &mut super::Connection) -> Result<()> { + fn send_settings( + &mut self, conn: &mut super::Connection, + ) -> Result<()> { let stream_id = match self .open_uni_stream(conn, stream::HTTP3_CONTROL_STREAM_TYPE_ID) { @@ -2332,8 +2421,8 @@ impl Connection { Ok(()) } - fn process_control_stream( - &mut self, conn: &mut super::Connection, stream_id: u64, + fn process_control_stream( + &mut self, conn: &mut super::Connection, stream_id: u64, app_buf: &mut Option<&mut crate::AppRecvBufMap>, ) -> Result<(u64, Event)> { let is_finished = if conn.version == crate::PROTOCOL_VERSION_VREVERSO { @@ -2380,14 +2469,13 @@ impl Connection { Err(Error::Done) } - fn process_readable_stream( - &mut self, conn: &mut super::Connection, stream_id: u64, polling: bool, - app_buf: &mut Option<&mut crate::AppRecvBufMap>, + fn process_readable_stream( + &mut self, conn: &mut super::Connection, stream_id: u64, + polling: bool, app_buf: &mut Option<&mut crate::AppRecvBufMap>, ) -> Result<(u64, Event)> { self.streams.entry(stream_id).or_insert_with(|| { - stream::Stream::new(stream_id, false, conn.version) + ::new(stream_id, false, conn.version) }); - // We need to get a fresh reference to the stream for each // iteration, to avoid borrowing `self` for the entire duration // of the loop, because we'll need to borrow it again in the @@ -2881,8 +2969,8 @@ impl Connection { }; } - fn process_frame( - &mut self, conn: &mut super::Connection, stream_id: u64, + fn process_frame( + &mut self, conn: &mut super::Connection, stream_id: u64, frame: frame::Frame, payload_len: u64, app_buf: &mut Option<&mut crate::AppRecvBufMap>, ) -> Result<(u64, Event)> { @@ -3214,7 +3302,7 @@ impl Connection { .streams .entry(prioritized_element_id) .or_insert_with(|| { - stream::Stream::new( + ::new( prioritized_element_id, false, conn.version, @@ -3288,6 +3376,7 @@ pub mod testing { use super::*; use crate::testing; + use crate::range_buf::DefaultBufFactory; /// Session is an HTTP/3 test helper structure. It holds a client, server /// and pipe that allows them to communicate. @@ -3303,14 +3392,17 @@ pub mod testing { /// request, responses and individual headers. The full quiceh API remains /// available for any test that need to do unconventional things (such as /// bad behaviour that triggers errors). - pub struct Session { - pub pipe: testing::Pipe, + pub struct Session + where + F: BufFactory, + { + pub pipe: testing::Pipe, pub client: Connection, pub server: Connection, } - impl Session { - pub fn new() -> Result { + impl Session { + pub fn new() -> Result> { let mut config = crate::Config::new(crate::PROTOCOL_VERSION)?; config.load_cert_chain_from_pem_file("examples/cert.crt")?; config.load_priv_key_from_pem_file("examples/cert.key")?; @@ -3326,12 +3418,12 @@ pub mod testing { config.set_ack_delay_exponent(8); let h3_config = Config::new()?; - Session::with_configs(&mut config, &h3_config) + Session::::with_configs(&mut config, &h3_config) } pub fn with_configs( config: &mut crate::Config, h3_config: &Config, - ) -> Result { + ) -> Result> { let pipe = testing::Pipe::with_config(config)?; let client_dgram = pipe.client.dgram_enabled(); let server_dgram = pipe.server.dgram_enabled(); @@ -3555,6 +3647,20 @@ pub mod testing { Ok(bytes) } + pub fn send_body_server_zc( + &mut self, stream: u64, body: F::Buf, fin: bool + ) -> Result<(usize, Option)> + where + F: BufFactory, + F::Buf: BufSplit, + { + let ret = self.server + .send_body_zc(&mut self.pipe.server, stream, body, fin)?; + + self.advance().ok(); + Ok(ret) + } + /// Fetches DATA payload from the client /// /// On success, it returns a slice of the DATA payload @@ -3720,6 +3826,9 @@ mod tests { use super::*; use super::testing::*; + use crate::testing::Pipe; + use crate::range_buf::BufFactory; + use crate::testing::BufTestFactory; #[test] /// Make sure that random GREASE values is within the specified limit. @@ -3754,14 +3863,14 @@ mod tests { let h3_config = Config::new().unwrap(); // Perform initial handshake. - let mut pipe = crate::testing::Pipe::with_config(&mut config).unwrap(); + let mut pipe = ::with_config(&mut config).unwrap(); assert_eq!(pipe.handshake(), Ok(())); // Extract session, let session = pipe.client.session().unwrap(); // Configure session on new connection. - let mut pipe = crate::testing::Pipe::with_config(&mut config).unwrap(); + let mut pipe = ::with_config(&mut config).unwrap(); assert_eq!(pipe.client.set_session(session), Ok(())); // Can't create an H3 connection until the QUIC connection is determined @@ -3783,7 +3892,7 @@ mod tests { let frames = [crate::frame::Frame::Stream { stream_id: 6, - data: crate::stream::RangeBuf::from(b"aaaaa", 0, true), + data: ::from(b"aaaaa", 0, true), }]; assert_eq!( @@ -3819,7 +3928,7 @@ mod tests { #[test] /// Send a request with no body, get a response with no body. fn request_no_body_response_no_body() { - let mut s = Session::new().unwrap(); + let mut s = ::new().unwrap(); s.handshake().unwrap(); let off_by = if crate::PROTOCOL_VERSION == crate::PROTOCOL_VERSION_VREVERSO { @@ -3855,7 +3964,7 @@ mod tests { #[test] /// Send a request with no body, get a response with one DATA frame. fn request_no_body_response_one_chunk() { - let mut s = Session::new().unwrap(); + let mut s = ::new().unwrap(); s.handshake().unwrap(); let off_by = if crate::PROTOCOL_VERSION == crate::PROTOCOL_VERSION_VREVERSO { @@ -3904,10 +4013,87 @@ mod tests { assert_eq!(s.poll_client(), Err(Error::Done)); } + + #[test] + /// Send a request with no body, and get a response with multiple DATA frames + /// on server using hidden copy encryption + fn request_no_body_response_many_chunks_sent_without_copy() { + let mut config = crate::Config::new(crate::PROTOCOL_VERSION).unwrap(); + config + .load_cert_chain_from_pem_file("examples/cert.crt") + .unwrap(); + config + .load_priv_key_from_pem_file("examples/cert.key") + .unwrap(); + config.set_application_protos(&[b"h3"]).unwrap(); + config.set_initial_max_data(10_000_000); + config.set_initial_max_stream_data_uni(10 * 32 * 1024); + config.set_initial_max_stream_data_bidi_local(10 * 32 * 1024); + config.set_initial_max_stream_data_bidi_remote(10 * 32 * 1024); + config.set_initial_max_streams_bidi(150); + config.set_initial_max_streams_uni(150); + config.enable_hidden_copy_for_zc_sender(true); + + let h3_config = Config::new().unwrap(); + let mut s = Session::::with_configs(&mut config, &h3_config).unwrap(); + + s.handshake().unwrap(); + + let (stream, req) = s.send_request(true).unwrap(); + + let ev_headers = Event::Headers { + list: req, + has_body: false, + }; + + assert_eq!(s.poll_server(), Ok((stream, ev_headers))); + assert_eq!(s.poll_server(), Ok((stream, Event::Finished))); + + let resp = s.send_response(stream, false).unwrap(); + + let bodies = [vec![42; 100], vec![42; 1000], vec![42; 10000]]; + + for i in 0..bodies.len() - 1 { + s.send_body_server_zc( + stream, BufTestFactory::buf_from_slice(&bodies[i]), false, + ).unwrap(); + } + + s.send_body_server_zc( + stream, BufTestFactory::buf_from_slice(&bodies[bodies.len() - 1]), true, + ).unwrap(); + + + let ev_headers = Event::Headers { + list: resp, + has_body: true, + }; + + assert_eq!(s.poll_client(), Ok((stream, ev_headers))); + assert_eq!(s.poll_client(), Ok((stream, Event::Data))); + assert_eq!(s.poll_client(), Err(Error::Done)); + + for i in 0..bodies.len() { + if crate::PROTOCOL_VERSION == crate::PROTOCOL_VERSION_VREVERSO { + let (b, tot_exp_len) = s.recv_body_v3_client(stream).unwrap(); + assert_eq!(b.len(), bodies[i].len()); + assert_eq!(b, &bodies[i]); + assert_eq!(tot_exp_len, bodies[i].len()); + assert!(s.body_consumed_client(stream, tot_exp_len).is_ok()); + } else { + let mut recv_buf = vec![0; bodies[i].len()]; + assert_eq!( + s.recv_body_client(stream, &mut recv_buf), + Ok(bodies[i].len()) + ); + assert_eq!(&recv_buf, &bodies[i]); + } + } + } #[test] /// Send a request with no body, get a response with multiple DATA frames. fn request_no_body_response_many_chunks() { - let mut s = Session::new().unwrap(); + let mut s = ::new().unwrap(); s.handshake().unwrap(); let (stream, req) = s.send_request(true).unwrap(); @@ -3964,7 +4150,7 @@ mod tests { /// which we partially consume. fn request_no_body_response_many_chunks_partially_consumed_v3_only() { if crate::PROTOCOL_VERSION == crate::PROTOCOL_VERSION_VREVERSO { - let mut s = Session::new().unwrap(); + let mut s = ::new().unwrap(); s.handshake().unwrap(); let (stream, req) = s.send_request(true).unwrap(); @@ -4016,7 +4202,7 @@ mod tests { #[test] /// Send a request with one DATA frame, get a response with no body. fn request_one_chunk_response_no_body() { - let mut s = Session::new().unwrap(); + let mut s = ::new().unwrap(); s.handshake().unwrap(); let (stream, req) = s.send_request(false).unwrap(); @@ -4058,7 +4244,7 @@ mod tests { #[test] /// Send a request with multiple DATA frames, get a response with no body. fn request_many_chunks_response_no_body() { - let mut s = Session::new().unwrap(); + let mut s = ::new().unwrap(); s.handshake().unwrap(); let (stream, req) = s.send_request(false).unwrap(); @@ -4113,7 +4299,7 @@ mod tests { /// Send a request with multiple DATA frames, get a response with one DATA /// frame. fn many_requests_many_chunks_response_one_chunk() { - let mut s = Session::new().unwrap(); + let mut s = ::new().unwrap(); s.handshake().unwrap(); let mut reqs = Vec::new(); @@ -4247,7 +4433,7 @@ mod tests { /// Send a request with no body, get a response with one DATA frame and an /// empty FIN after reception from the client. fn request_no_body_response_one_chunk_empty_fin() { - let mut s = Session::new().unwrap(); + let mut s = ::new().unwrap(); s.handshake().unwrap(); let (stream, req) = s.send_request(true).unwrap(); @@ -4295,7 +4481,7 @@ mod tests { /// Send a request with no body, get a response with no body followed by /// GREASE that is STREAM frame with a FIN. fn request_no_body_response_no_body_with_grease() { - let mut s = Session::new().unwrap(); + let mut s = ::new().unwrap(); s.handshake().unwrap(); let off_by = @@ -4355,7 +4541,7 @@ mod tests { #[test] /// Try to send DATA frames before HEADERS. fn body_response_before_headers() { - let mut s = Session::new().unwrap(); + let mut s = ::new().unwrap(); s.handshake().unwrap(); let off_by = if crate::PROTOCOL_VERSION == crate::PROTOCOL_VERSION_VREVERSO { @@ -4388,7 +4574,7 @@ mod tests { /// Try to send DATA frames on wrong streams, ensure the API returns an /// error before anything hits the transport layer. fn send_body_invalid_client_stream() { - let mut s = Session::new().unwrap(); + let mut s = ::new().unwrap(); s.handshake().unwrap(); let off_by = @@ -4450,7 +4636,7 @@ mod tests { /// Try to send DATA frames on wrong streams, ensure the API returns an /// error before anything hits the transport layer. fn send_body_invalid_server_stream() { - let mut s = Session::new().unwrap(); + let mut s = ::new().unwrap(); s.handshake().unwrap(); let off_by = if crate::PROTOCOL_VERSION == crate::PROTOCOL_VERSION_VREVERSO { @@ -4510,7 +4696,7 @@ mod tests { #[test] /// Send a MAX_PUSH_ID frame from the client on a valid stream. fn max_push_id_from_client_good() { - let mut s = Session::new().unwrap(); + let mut s = ::new().unwrap(); s.handshake().unwrap(); s.send_frame_client( @@ -4526,7 +4712,7 @@ mod tests { #[test] /// Send a MAX_PUSH_ID frame from the client on an invalid stream. fn max_push_id_from_client_bad_stream() { - let mut s = Session::new().unwrap(); + let mut s = ::new().unwrap(); s.handshake().unwrap(); let (stream, req) = s.send_request(false).unwrap(); @@ -4551,7 +4737,7 @@ mod tests { /// Send a sequence of MAX_PUSH_ID frames from the client that attempt to /// reduce the limit. fn max_push_id_from_client_limit_reduction() { - let mut s = Session::new().unwrap(); + let mut s = ::new().unwrap(); s.handshake().unwrap(); s.send_frame_client( @@ -4574,7 +4760,7 @@ mod tests { #[test] /// Send a MAX_PUSH_ID frame from the server, which is forbidden. fn max_push_id_from_server() { - let mut s = Session::new().unwrap(); + let mut s = ::new().unwrap(); s.handshake().unwrap(); s.send_frame_server( @@ -4590,7 +4776,7 @@ mod tests { #[test] /// Send a PUSH_PROMISE frame from the client, which is forbidden. fn push_promise_from_client() { - let mut s = Session::new().unwrap(); + let mut s = ::new().unwrap(); s.handshake().unwrap(); let (stream, req) = s.send_request(false).unwrap(); @@ -4619,7 +4805,7 @@ mod tests { #[test] /// Send a CANCEL_PUSH frame from the client. fn cancel_push_from_client() { - let mut s = Session::new().unwrap(); + let mut s = ::new().unwrap(); s.handshake().unwrap(); s.send_frame_client( @@ -4635,7 +4821,7 @@ mod tests { #[test] /// Send a CANCEL_PUSH frame from the client on an invalid stream. fn cancel_push_from_client_bad_stream() { - let mut s = Session::new().unwrap(); + let mut s = ::new().unwrap(); s.handshake().unwrap(); let (stream, req) = s.send_request(false).unwrap(); @@ -4659,7 +4845,7 @@ mod tests { #[test] /// Send a CANCEL_PUSH frame from the client. fn cancel_push_from_server() { - let mut s = Session::new().unwrap(); + let mut s = ::new().unwrap(); s.handshake().unwrap(); s.send_frame_server( @@ -4675,7 +4861,7 @@ mod tests { #[test] /// Send a GOAWAY frame from the client. fn goaway_from_client_good() { - let mut s = Session::new().unwrap(); + let mut s = ::new().unwrap(); s.handshake().unwrap(); let off_by = if crate::PROTOCOL_VERSION == crate::PROTOCOL_VERSION_VREVERSO { @@ -4695,7 +4881,7 @@ mod tests { #[test] /// Send a GOAWAY frame from the server. fn goaway_from_server_good() { - let mut s = Session::new().unwrap(); + let mut s = ::new().unwrap(); s.handshake().unwrap(); s.server.send_goaway(&mut s.pipe.server, 4000).unwrap(); @@ -4708,7 +4894,7 @@ mod tests { #[test] /// A client MUST NOT send a request after it receives GOAWAY. fn client_request_after_goaway() { - let mut s = Session::new().unwrap(); + let mut s = ::new().unwrap(); s.handshake().unwrap(); s.server.send_goaway(&mut s.pipe.server, 4000).unwrap(); @@ -4723,7 +4909,7 @@ mod tests { #[test] /// Send a GOAWAY frame from the server, using an invalid goaway ID. fn goaway_from_server_invalid_id() { - let mut s = Session::new().unwrap(); + let mut s = ::new().unwrap(); s.handshake().unwrap(); s.send_frame_server( @@ -4740,7 +4926,7 @@ mod tests { /// Send multiple GOAWAY frames from the server, that increase the goaway /// ID. fn goaway_from_server_increase_id() { - let mut s = Session::new().unwrap(); + let mut s = ::new().unwrap(); s.handshake().unwrap(); let off_by = if crate::PROTOCOL_VERSION == crate::PROTOCOL_VERSION_VREVERSO { @@ -4857,7 +5043,7 @@ mod tests { #[test] /// Send a PRIORITY_UPDATE for request stream from the client. fn priority_update_request() { - let mut s = Session::new().unwrap(); + let mut s = ::new().unwrap(); s.handshake().unwrap(); let off_by = if crate::PROTOCOL_VERSION == crate::PROTOCOL_VERSION_VREVERSO { @@ -4885,7 +5071,7 @@ mod tests { #[test] /// Send a PRIORITY_UPDATE for request stream from the client. fn priority_update_single_stream_rearm() { - let mut s = Session::new().unwrap(); + let mut s = ::new().unwrap(); s.handshake().unwrap(); let off_by = if crate::PROTOCOL_VERSION == crate::PROTOCOL_VERSION_VREVERSO { @@ -4963,7 +5149,7 @@ mod tests { /// Send multiple PRIORITY_UPDATE frames for different streams from the /// client across multiple flights of exchange. fn priority_update_request_multiple_stream_arm_multiple_flights() { - let mut s = Session::new().unwrap(); + let mut s = ::new().unwrap(); s.handshake().unwrap(); let off_by = if crate::PROTOCOL_VERSION == crate::PROTOCOL_VERSION_VREVERSO { @@ -5039,7 +5225,7 @@ mod tests { /// Send multiple PRIORITY_UPDATE frames for different streams from the /// client across a single flight. fn priority_update_request_multiple_stream_arm_single_flight() { - let mut s = Session::new().unwrap(); + let mut s = ::new().unwrap(); s.handshake().unwrap(); let off_by = if crate::PROTOCOL_VERSION == crate::PROTOCOL_VERSION_VREVERSO { @@ -5107,7 +5293,7 @@ mod tests { /// Send a PRIORITY_UPDATE for a request stream, before and after the stream /// has been completed. fn priority_update_request_collected_completed() { - let mut s = Session::new().unwrap(); + let mut s = ::new().unwrap(); s.handshake().unwrap(); let off_by = if crate::PROTOCOL_VERSION == crate::PROTOCOL_VERSION_VREVERSO { @@ -5181,7 +5367,7 @@ mod tests { /// Send a PRIORITY_UPDATE for a request stream, before and after the stream /// has been stopped. fn priority_update_request_collected_stopped() { - let mut s = Session::new().unwrap(); + let mut s = ::new().unwrap(); s.handshake().unwrap(); let off_by = if crate::PROTOCOL_VERSION == crate::PROTOCOL_VERSION_VREVERSO { @@ -5256,7 +5442,7 @@ mod tests { #[test] /// Send a PRIORITY_UPDATE for push stream from the client. fn priority_update_push() { - let mut s = Session::new().unwrap(); + let mut s = ::new().unwrap(); s.handshake().unwrap(); s.send_frame_client( @@ -5276,7 +5462,7 @@ mod tests { /// Send a PRIORITY_UPDATE for request stream from the client but for an /// incorrect stream type. fn priority_update_request_bad_stream() { - let mut s = Session::new().unwrap(); + let mut s = ::new().unwrap(); s.handshake().unwrap(); s.send_frame_client( @@ -5303,7 +5489,7 @@ mod tests { // behavior only existing if we have crate::PROTOCOL_VERSION_VREVERSO. // XXX Todo if a better idea comes. if crate::PROTOCOL_VERSION == crate::PROTOCOL_VERSION_VREVERSO { - let mut s = Session::new().unwrap(); + let mut s = ::new().unwrap(); s.handshake().unwrap(); s.send_frame_client( @@ -5324,7 +5510,7 @@ mod tests { /// Send a PRIORITY_UPDATE for push stream from the client but for an /// incorrect stream type. fn priority_update_push_bad_stream() { - let mut s = Session::new().unwrap(); + let mut s = ::new().unwrap(); s.handshake().unwrap(); s.send_frame_client( @@ -5343,7 +5529,7 @@ mod tests { #[test] /// Send a PRIORITY_UPDATE for request stream from the server. fn priority_update_request_from_server() { - let mut s = Session::new().unwrap(); + let mut s = ::new().unwrap(); s.handshake().unwrap(); s.send_frame_server( @@ -5362,7 +5548,7 @@ mod tests { #[test] /// Send a PRIORITY_UPDATE for request stream from the server. fn priority_update_push_from_server() { - let mut s = Session::new().unwrap(); + let mut s = ::new().unwrap(); s.handshake().unwrap(); let off_by = if crate::PROTOCOL_VERSION == crate::PROTOCOL_VERSION_VREVERSO { @@ -5399,7 +5585,7 @@ mod tests { #[test] /// Client opens multiple control streams, which is forbidden. fn open_multiple_control_streams() { - let mut s = Session::new().unwrap(); + let mut s = ::new().unwrap(); s.handshake().unwrap(); let stream_id = s.client.next_uni_stream_id; @@ -5424,7 +5610,7 @@ mod tests { #[test] /// Client closes the control stream, which is forbidden. fn close_control_stream() { - let mut s = Session::new().unwrap(); + let mut s = ::new().unwrap(); s.handshake().unwrap(); let mut control_stream_closed = false; @@ -5459,7 +5645,7 @@ mod tests { #[test] /// Client closes QPACK stream, which is forbidden. fn close_qpack_stream() { - let mut s = Session::new().unwrap(); + let mut s = ::new().unwrap(); s.handshake().unwrap(); let mut qpack_stream_closed = false; @@ -5497,7 +5683,7 @@ mod tests { fn qpack_data() { // TODO: QPACK instructions are ignored until dynamic table support is // added so we just test that the data is safely ignored. - let mut s = Session::new().unwrap(); + let mut s = ::new().unwrap(); s.handshake().unwrap(); let e_stream_id = s.client.local_qpack_streams.encoder_stream_id.unwrap(); @@ -5528,7 +5714,7 @@ mod tests { #[test] /// Tests limits for the stream state buffer maximum size. fn max_state_buf_size() { - let mut s = Session::new().unwrap(); + let mut s = ::new().unwrap(); s.handshake().unwrap(); let off_by = if crate::PROTOCOL_VERSION == crate::PROTOCOL_VERSION_VREVERSO { @@ -5582,7 +5768,7 @@ mod tests { assert_eq!(s.poll_server(), Ok((0 + off_by, Event::Data))); // GREASE frames consume the state buffer, so need to be limited. - let mut s = Session::new().unwrap(); + let mut s = ::new().unwrap(); s.handshake().unwrap(); let mut d = [42; 128]; @@ -5613,7 +5799,7 @@ mod tests { fn stream_backpressure() { let bytes = vec![1, 2, 3, 4, 5, 6, 7, 8, 9, 10]; - let mut s = Session::new().unwrap(); + let mut s = ::new().unwrap(); s.handshake().unwrap(); let (stream, req) = s.send_request(false).unwrap(); @@ -5703,7 +5889,7 @@ mod tests { let mut h3_config = Config::new().unwrap(); h3_config.set_max_field_section_size(65); - let mut s = Session::with_configs(&mut config, &h3_config).unwrap(); + let mut s = ::with_configs(&mut config, &h3_config).unwrap(); s.handshake().unwrap(); let off_by = @@ -5741,7 +5927,7 @@ mod tests { #[test] /// Tests that Error::TransportError contains a transport error. fn transport_error() { - let mut s = Session::new().unwrap(); + let mut s = ::new().unwrap(); s.handshake().unwrap(); let off_by = if crate::PROTOCOL_VERSION == crate::PROTOCOL_VERSION_VREVERSO { @@ -5792,7 +5978,7 @@ mod tests { #[test] /// Tests that sending DATA before HEADERS causes an error. fn data_before_headers() { - let mut s = Session::new().unwrap(); + let mut s = ::new().unwrap(); s.handshake().unwrap(); let off_by = if crate::PROTOCOL_VERSION == crate::PROTOCOL_VERSION_VREVERSO { @@ -5829,7 +6015,7 @@ mod tests { #[test] /// Tests that calling poll() after an error occurred does nothing. fn poll_after_error() { - let mut s = Session::new().unwrap(); + let mut s = ::new().unwrap(); s.handshake().unwrap(); let off_by = if crate::PROTOCOL_VERSION == crate::PROTOCOL_VERSION_VREVERSO { @@ -5884,7 +6070,7 @@ mod tests { let h3_config = Config::new().unwrap(); - let mut s = Session::with_configs(&mut config, &h3_config).unwrap(); + let mut s = ::with_configs(&mut config, &h3_config).unwrap(); s.handshake().unwrap(); let off_by = @@ -5949,7 +6135,7 @@ mod tests { let h3_config = Config::new().unwrap(); - let mut s = Session::with_configs(&mut config, &h3_config).unwrap(); + let mut s = ::with_configs(&mut config, &h3_config).unwrap(); s.handshake().unwrap(); let off_by = @@ -6019,7 +6205,7 @@ mod tests { let h3_config = Config::new().unwrap(); - let mut s = Session::with_configs(&mut config, &h3_config).unwrap(); + let mut s = ::with_configs(&mut config, &h3_config).unwrap(); s.handshake().unwrap(); let off_by = @@ -6176,7 +6362,7 @@ mod tests { let h3_config = Config::new().unwrap(); - let mut s = Session::with_configs(&mut config, &h3_config).unwrap(); + let mut s = ::with_configs(&mut config, &h3_config).unwrap(); s.handshake().unwrap(); let off_by = @@ -6259,7 +6445,7 @@ mod tests { let h3_config = Config::new().unwrap(); - let mut s = Session::with_configs(&mut config, &h3_config).unwrap(); + let mut s = ::with_configs(&mut config, &h3_config).unwrap(); s.handshake().unwrap(); let off_by = @@ -6330,7 +6516,7 @@ mod tests { #[test] /// Test handling of 0-length DATA writes with and without fin. fn zero_length_data() { - let mut s = Session::new().unwrap(); + let mut s = ::new().unwrap(); s.handshake().unwrap(); let off_by = if crate::PROTOCOL_VERSION == crate::PROTOCOL_VERSION_VREVERSO { @@ -6433,7 +6619,7 @@ mod tests { let h3_config = Config::new().unwrap(); - let mut s = Session::with_configs(&mut config, &h3_config).unwrap(); + let mut s = ::with_configs(&mut config, &h3_config).unwrap(); s.handshake().unwrap(); let off_by = @@ -6500,7 +6686,7 @@ mod tests { config.grease(false); let h3_config = Config::new().unwrap(); - let mut s = Session::with_configs(&mut config, &h3_config).unwrap(); + let mut s = ::with_configs(&mut config, &h3_config).unwrap(); s.handshake().unwrap(); @@ -6530,7 +6716,7 @@ mod tests { let h3_config = Config::new().unwrap(); - let mut s = Session::with_configs(&mut config, &h3_config).unwrap(); + let mut s = ::with_configs(&mut config, &h3_config).unwrap(); assert_eq!(s.pipe.handshake(), Ok(())); s.client.send_settings(&mut s.pipe.client).unwrap(); @@ -6574,7 +6760,7 @@ mod tests { let h3_config = Config::new().unwrap(); - let mut s = Session::with_configs(&mut config, &h3_config).unwrap(); + let mut s = ::with_configs(&mut config, &h3_config).unwrap(); assert_eq!(s.pipe.handshake(), Ok(())); s.client.control_stream_id = Some( @@ -6626,7 +6812,7 @@ mod tests { let h3_config = Config::new().unwrap(); - let mut s = Session::with_configs(&mut config, &h3_config).unwrap(); + let mut s = ::with_configs(&mut config, &h3_config).unwrap(); assert_eq!(s.pipe.handshake(), Ok(())); s.client.control_stream_id = Some( @@ -6740,7 +6926,7 @@ mod tests { .set_additional_settings(vec![(42, 43), (44, 45)]) .unwrap(); - let mut s = Session::with_configs(&mut config, &h3_config).unwrap(); + let mut s = ::with_configs(&mut config, &h3_config).unwrap(); assert_eq!(s.pipe.handshake(), Ok(())); assert_eq!(s.pipe.advance(), Ok(())); @@ -6767,7 +6953,7 @@ mod tests { /// Send a single DATAGRAM. fn single_dgram() { let mut buf = [0; 65535]; - let mut s = Session::new().unwrap(); + let mut s = ::new().unwrap(); s.handshake().unwrap(); let off_by = if crate::PROTOCOL_VERSION == crate::PROTOCOL_VERSION_VREVERSO { @@ -6793,7 +6979,7 @@ mod tests { /// Send multiple DATAGRAMs. fn multiple_dgram() { let mut buf = [0; 65535]; - let mut s = Session::new().unwrap(); + let mut s = ::new().unwrap(); s.handshake().unwrap(); let off_by = if crate::PROTOCOL_VERSION == crate::PROTOCOL_VERSION_VREVERSO { @@ -6830,7 +7016,7 @@ mod tests { /// Send more DATAGRAMs than the send queue allows. fn multiple_dgram_overflow() { let mut buf = [0; 65535]; - let mut s = Session::new().unwrap(); + let mut s = ::new().unwrap(); s.handshake().unwrap(); let off_by = if crate::PROTOCOL_VERSION == crate::PROTOCOL_VERSION_VREVERSO { @@ -6878,7 +7064,7 @@ mod tests { config.enable_dgram(true, 100, 100); let h3_config = Config::new().unwrap(); - let mut s = Session::with_configs(&mut config, &h3_config).unwrap(); + let mut s = ::with_configs(&mut config, &h3_config).unwrap(); s.handshake().unwrap(); let off_by = if crate::PROTOCOL_VERSION == crate::PROTOCOL_VERSION_VREVERSO { @@ -6928,7 +7114,7 @@ mod tests { config.enable_dgram(true, 100, 100); let h3_config = Config::new().unwrap(); - let mut s = Session::with_configs(&mut config, &h3_config).unwrap(); + let mut s = ::with_configs(&mut config, &h3_config).unwrap(); s.handshake().unwrap(); let off_by = if crate::PROTOCOL_VERSION == crate::PROTOCOL_VERSION_VREVERSO { @@ -7035,7 +7221,7 @@ mod tests { config.enable_dgram(true, 100, 100); let h3_config = Config::new().unwrap(); - let mut s = Session::with_configs(&mut config, &h3_config).unwrap(); + let mut s = ::with_configs(&mut config, &h3_config).unwrap(); s.handshake().unwrap(); let off_by = if crate::PROTOCOL_VERSION == crate::PROTOCOL_VERSION_VREVERSO { @@ -7183,7 +7369,7 @@ mod tests { /// Tests that the Finished event is not issued for streams of unknown type /// (e.g. GREASE). fn finished_is_for_requests() { - let mut s = Session::new().unwrap(); + let mut s = ::new().unwrap(); s.handshake().unwrap(); assert_eq!(s.poll_client(), Err(Error::Done)); @@ -7199,7 +7385,7 @@ mod tests { #[test] /// Tests that streams are marked as finished only once. fn finished_once() { - let mut s = Session::new().unwrap(); + let mut s = ::new().unwrap(); s.handshake().unwrap(); let (stream, req) = s.send_request(false).unwrap(); @@ -7242,7 +7428,7 @@ mod tests { fn data_event_rearm() { let bytes = [1, 2, 3, 4, 5, 6, 7, 8, 9, 10]; - let mut s = Session::new().unwrap(); + let mut s = ::new().unwrap(); s.handshake().unwrap(); let (stream, req) = s.send_request(false).unwrap(); @@ -7489,7 +7675,7 @@ mod tests { config.enable_dgram(true, 100, 100); let h3_config = Config::new().unwrap(); - let mut s = Session::with_configs(&mut config, &h3_config).unwrap(); + let mut s = ::with_configs(&mut config, &h3_config).unwrap(); s.handshake().unwrap(); let off_by = if crate::PROTOCOL_VERSION == crate::PROTOCOL_VERSION_VREVERSO { @@ -7563,7 +7749,7 @@ mod tests { fn reset_stream() { let mut buf = [0; 65535]; - let mut s = Session::new().unwrap(); + let mut s = ::new().unwrap(); s.handshake().unwrap(); // Client sends request. @@ -7631,7 +7817,7 @@ mod tests { #[test] fn reset_finished_at_server() { - let mut s = Session::new().unwrap(); + let mut s = ::new().unwrap(); s.handshake().unwrap(); let off_by = if crate::PROTOCOL_VERSION == crate::PROTOCOL_VERSION_VREVERSO { @@ -7681,7 +7867,7 @@ mod tests { #[test] fn reset_finished_at_server_with_data_pending() { - let mut s = Session::new().unwrap(); + let mut s = ::new().unwrap(); s.handshake().unwrap(); // Client sends HEADERS and doesn't fin. @@ -7719,7 +7905,7 @@ mod tests { #[test] fn reset_finished_at_client() { let mut buf = [0; 65535]; - let mut s = Session::new().unwrap(); + let mut s = ::new().unwrap(); s.handshake().unwrap(); // Client sends HEADERS and doesn't fin diff --git a/quiceh/src/h3/qpack/mod.rs b/quiceh/src/h3/qpack/mod.rs index 0e81c00d..6f1d587f 100644 --- a/quiceh/src/h3/qpack/mod.rs +++ b/quiceh/src/h3/qpack/mod.rs @@ -161,7 +161,8 @@ mod tests { let mut enc = Encoder::new(); // Indexed name with literal value - let headers1 = vec![crate::h3::Header::new(b"location", b" ")]; + let headers1 = + vec![crate::h3::Header::new(b"location", b" ")]; assert_eq!(enc.encode(&headers1, &mut encoded), Ok(19)); // Literal name and value diff --git a/quiceh/src/h3/stream.rs b/quiceh/src/h3/stream.rs index 3244ed8e..6e06ad28 100644 --- a/quiceh/src/h3/stream.rs +++ b/quiceh/src/h3/stream.rs @@ -24,6 +24,8 @@ // NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS // SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +use crate::range_buf::BufFactory; + use super::Error; use super::Result; @@ -397,17 +399,24 @@ impl Stream { /// Read the connection and acquire a reference to the data containing the /// state - pub fn try_acquire_state_buffer<'a>( - &mut self, conn: &mut crate::Connection, app_buf: &'a mut AppRecvBufMap, + pub fn try_acquire_state_buffer<'a, F: BufFactory>( + &mut self, conn: &mut crate::Connection, + app_buf: &'a mut AppRecvBufMap, ) -> Result<&'a [u8]> { // In v3, the state is kept mixed with data. We eventually // give a slice to the upper layer containing the DATA from // the data frame. // // This gives everything readable until it is explicitely - // marrked as consumed. + // marked as consumed. let b = match conn.stream_recv_v3(self.id, app_buf) { Ok((b, len, _)) => { + trace!( + "{} Acquiring {} bytes of the HTTP/3 frame from Stream {} ", + conn.trace_id(), + len, + self.id + ); // We read nothing form the QUIC stream if len == 0 { self.reset_data_event(); @@ -419,6 +428,12 @@ impl Stream { }, Err(e) => { + trace!( + "{} failed to recv bytes from Stream {}, error {:?}", + conn.trace_id(), + self.id, + e + ); // The stream is not readable anymore, so re-arm the Data event. if e == crate::Error::Done { self.reset_data_event(); @@ -432,8 +447,8 @@ impl Stream { } /// Mark the data acquired from the state buffer as consumed. - pub fn mark_state_buffer_consumed( - &mut self, conn: &mut crate::Connection, consumed: usize, + pub fn mark_state_buffer_consumed( + &mut self, conn: &mut crate::Connection, consumed: usize, app_buf: &mut AppRecvBufMap, ) -> Result<()> { self.state_off += consumed; @@ -455,8 +470,8 @@ impl Stream { /// /// When not enough data can be read to complete the state, this returns /// `Error::Done`. - pub fn try_fill_buffer( - &mut self, conn: &mut crate::Connection, + pub fn try_fill_buffer( + &mut self, conn: &mut crate::Connection, ) -> Result<()> { // If no bytes are required to be read, return early. if self.state_buffer_complete() { @@ -665,8 +680,9 @@ impl Stream { /// Tries to get a reference to the DATA payload for the application to /// eventually consume. - pub fn try_acquire_data<'a>( - &mut self, conn: &mut crate::Connection, app_buf: &'a mut AppRecvBufMap, + pub fn try_acquire_data<'a, F: BufFactory>( + &mut self, conn: &mut crate::Connection, + app_buf: &'a mut AppRecvBufMap, ) -> Result<(&'a [u8], usize, bool)> { let (b, len, fin) = match conn.stream_recv_v3(self.id, app_buf) { Ok(v) => v, @@ -691,8 +707,8 @@ impl Stream { } /// Tries to read DATA payload from the transport stream. - pub fn try_consume_data( - &mut self, conn: &mut crate::Connection, out: &mut [u8], + pub fn try_consume_data( + &mut self, conn: &mut crate::Connection, out: &mut [u8], ) -> Result<(usize, bool)> { let left = std::cmp::min(out.len(), self.state_len - self.state_off); @@ -724,8 +740,8 @@ impl Stream { } /// Marks DATA payload read and consumed (up to `consumed`). - pub fn mark_data_consumed( - &mut self, conn: &mut crate::Connection, app_buf: &mut AppRecvBufMap, + pub fn mark_data_consumed( + &mut self, conn: &mut crate::Connection, app_buf: &mut AppRecvBufMap, consumed: usize, ) -> Result<()> { // Account for DATA consumed by the app @@ -839,6 +855,11 @@ impl Stream { } } + trace!("H3 connection moves to state {:?} with expected len {}", + new_state, + expected_len + ); + self.state = new_state; self.state_off = 0; self.state_len = expected_len; @@ -854,7 +875,7 @@ mod tests { use super::*; fn open_uni(b: &mut octets_rev::OctetsMut, ty: u64) -> Result { - let stream = Stream::new(2, false, crate::PROTOCOL_VERSION); + let stream = ::new(2, false, crate::PROTOCOL_VERSION); assert_eq!(stream.state, State::StreamType); b.put_varint(ty)?; @@ -1445,7 +1466,7 @@ mod tests { #[test] fn request_no_data() { - let mut stream = Stream::new(0, false, crate::PROTOCOL_VERSION); + let mut stream = ::new(0, false, crate::PROTOCOL_VERSION); assert_eq!(stream.ty, Some(Type::Request)); assert_eq!(stream.state, State::FrameType); @@ -1455,7 +1476,7 @@ mod tests { #[test] fn request_good() { - let mut stream = Stream::new(0, false, crate::PROTOCOL_VERSION); + let mut stream = ::new(0, false, crate::PROTOCOL_VERSION); let mut d = vec![42; 128]; let mut b = octets_rev::OctetsMut::with_slice(&mut d); @@ -1842,7 +1863,7 @@ mod tests { #[test] fn data_before_headers() { - let mut stream = Stream::new(0, false, crate::PROTOCOL_VERSION); + let mut stream = ::new(0, false, crate::PROTOCOL_VERSION); let mut d = vec![42; 128]; let mut b = octets_rev::OctetsMut::with_slice(&mut d); @@ -1968,7 +1989,7 @@ mod tests { let mut d = vec![42; 128]; let mut b = octets_rev::OctetsMut::with_slice(&mut d); - let mut stream = Stream::new(0, false, crate::PROTOCOL_VERSION); + let mut stream = ::new(0, false, crate::PROTOCOL_VERSION); assert_eq!(stream.ty, Some(Type::Request)); assert_eq!(stream.state, State::FrameType); diff --git a/quiceh/src/lib.rs b/quiceh/src/lib.rs index 3c03d25c..af517aaa 100644 --- a/quiceh/src/lib.rs +++ b/quiceh/src/lib.rs @@ -462,6 +462,8 @@ use std::collections::VecDeque; use likely_stable::if_likely; use smallvec::SmallVec; +use range_buf::DefaultBufFactory; + /// The current QUIC wire version. pub const PROTOCOL_VERSION: u32 = PROTOCOL_VERSION_VREVERSO; @@ -855,6 +857,7 @@ pub enum QlogLevel { } /// Stores configuration shared between multiple connections. +#[derive(Clone)] pub struct Config { local_transport_params: TransportParams, @@ -889,6 +892,8 @@ pub struct Config { max_amplification_factor: usize, disable_dcid_reuse: bool, + + use_hidden_crypt_copy_for_zc: bool, } // See https://quicwg.org/base-drafts/rfc9000.html#section-15 @@ -957,6 +962,7 @@ impl Config { max_amplification_factor: MAX_AMPLIFICATION_FACTOR, disable_dcid_reuse: false, + use_hidden_crypt_copy_for_zc: false, }) } @@ -1354,6 +1360,14 @@ impl Config { self.pacing = v; } + /// Configures whether to enable usage of encryption's + /// hidden copy to assemble a QUIC packet + /// + /// The default value is `false` + pub fn enable_hidden_copy_for_zc_sender(&mut self, v: bool) { + self.use_hidden_crypt_copy_for_zc = v; + } + /// Sets the max value for pacing rate. /// /// By default pacing rate is not limited. @@ -1429,7 +1443,10 @@ impl Config { } /// A QUIC connection. -pub struct Connection { +pub struct Connection +where + F: BufFactory, +{ /// QUIC wire version used for the connection. version: u32, @@ -1525,7 +1542,7 @@ pub struct Connection { lost_bytes: u64, /// Streams map, indexed by stream ID. - streams: stream::StreamMap, + streams: stream::StreamMap, /// Peer's original destination connection ID. Used by the client to /// validate the server's transport parameter. @@ -1650,6 +1667,12 @@ pub struct Connection { /// The anti-amplification limit factor. max_amplification_factor: usize, + + /// Do we use zerocopy while emitting stream frames? This may + /// result to many short packets if the application makes many stream_send + /// calls on the same stream id with small datasizes. This would be unadvised to + /// use if the Application aims to send many short sized data. + use_hidden_crypt_copy_for_zc: bool, } /// Creates a new server-side connection. @@ -1681,6 +1704,21 @@ pub fn accept( Ok(conn) } +/// Creates a new server-side connection, with a custom buffer generation +/// method. +/// +/// The buffers generated can be anything that can be drereferenced as a byte +/// slice. See [`accept`] and [`BufFactory`] for more info. +#[inline] +pub fn accept_with_buf_factory( + scid: &ConnectionId, odcid: Option<&ConnectionId>, local: SocketAddr, + peer: SocketAddr, config: &mut Config, +) -> Result> { + let conn = Connection::new(scid, odcid, local, peer, config, true)?; + + Ok(conn) +} + /// Creates a new client-side connection. /// /// The `scid` parameter is used as the connection's source connection ID, @@ -1713,6 +1751,25 @@ pub fn connect( Ok(conn) } +/// Creates a new client-side connection, with a custom buffer generation +/// method. +/// +/// The buffers generated can be anything that can be drereferenced as a byte +/// slice. See [`connect`] and [`BufFactory`] for more info. +#[inline] +pub fn connect_with_buffer_factory( + server_name: Option<&str>, scid: &ConnectionId, local: SocketAddr, + peer: SocketAddr, config: &mut Config, +) -> Result> { + let mut conn = Connection::new(scid, None, local, peer, config, false)?; + + if let Some(server_name) = server_name { + conn.handshake.set_host_name(server_name)?; + } + + Ok(conn) +} + /// Writes a version negotiation packet. /// /// The `scid` and `dcid` parameters are the source connection ID and the @@ -1822,21 +1879,68 @@ pub fn version_is_supported(version: u32) -> bool { /// Frames such as Stream, Crypto, etc. have already been written into the /// buffer at their intended position. macro_rules! push_frames_to_pkt { - ($out:expr, $frames:expr, $ver:expr) => {{ + ($out:expr, $frames:expr, $usehiddencopy:expr, $ver:expr) => {{ for frame in $frames.iter() { match frame { /* * Some frames have been already been encoded to avoid a copy from * frame.to_bytes(). We just need to $out.skip them. **/ - frame::Frame::StreamHeader { stream_id, offset, length, ..} => { - let hdr_len = 1 + // frame type - octets_rev::varint_len(*stream_id) + // stream_id - octets_rev::varint_len(*offset) + // offset - 2; // length, always encode as 2-byte varint - // this has already been added to the buffer. - $out.skip(length + hdr_len)?; + frame::Frame::StreamHeader { stream_id, offset, length, fin } => { + if $usehiddencopy { + if $ver == crate::PROTOCOL_VERSION_VREVERSO { + // not in place VReverso + frame::encode_stream_footer( + *stream_id, + *offset, + *length as u64, + *fin, + &mut $out, + )?; + } else { + // not in place QUIC v1 + frame::encode_stream_header( + *stream_id, + *offset, + *length as u64, + *fin, + &mut $out, + )?; + } + } else { + let hdr_len = 1 + // frame type + octets_rev::varint_len(*stream_id) + // stream_id + octets_rev::varint_len(*offset) + // offset + 2; // length, always encode as 2-byte varint + + //this has already been added to the buffer. + $out.skip(length + hdr_len)?; + } + }, + frame::Frame::CryptoVec { offset, length, rbvec } => { + if rbvec.len() > 0 { + if $ver == crate::PROTOCOL_VERSION_VREVERSO { + for rb in rbvec { + $out.put_bytes(&rb[..])?; + } + frame::encode_crypto_footer( + *offset, + *length as u64, + &mut $out + )?; + } else { + frame::encode_crypto_header( + *offset, + *length as u64, + &mut $out + )?; + for rb in rbvec { + $out.put_bytes(&rb[..])?; + } + } + } + } frame::Frame::DatagramHeader { length } => { let hdr_len = 1 + // frame type 2; // length, always encode as 2-byte varint @@ -1874,27 +1978,6 @@ macro_rules! push_frame_to_vec { }}; } -/// Pushes a frame to the output packet if there is enough space. -/// -/// Returns `true` on success, `false` otherwise. In case of failure it means -/// there is no room to add the frame in the packet. You may retry to add the -/// frame later. -macro_rules! push_frame_to_pkt { - ($out:expr, $frames:expr, $frame:expr, $left:expr, $ver:expr) => {{ - if $frame.wire_len() <= $left { - $left -= $frame.wire_len(); - - $frame.to_bytes(&mut $out, $ver)?; - - $frames.push($frame); - - true - } else { - false - } - }}; -} - /// Executes the provided body if the qlog feature is enabled, quiceh has been /// configured with a log writer, the event's importance is within the /// configured level. @@ -1915,7 +1998,7 @@ macro_rules! qlog_with_type { const QLOG_PARAMS_SET: EventType = EventType::TransportEventType(TransportEventType::ParametersSet); -#[cfg(feature = "qlog")] +#[cfg(feature= "qlog")] const QLOG_PACKET_RX: EventType = EventType::TransportEventType(TransportEventType::PacketReceived); @@ -1953,11 +2036,11 @@ impl Default for QlogInfo { } } -impl Connection { +impl Connection { fn new( scid: &ConnectionId, odcid: Option<&ConnectionId>, local: SocketAddr, peer: SocketAddr, config: &mut Config, is_server: bool, - ) -> Result { + ) -> Result> { let tls = config.tls_ctx.new_handshake()?; Connection::with_tls(scid, odcid, local, peer, config, tls, is_server) } @@ -1965,7 +2048,7 @@ impl Connection { fn with_tls( scid: &ConnectionId, odcid: Option<&ConnectionId>, local: SocketAddr, peer: SocketAddr, config: &Config, tls: tls::Handshake, is_server: bool, - ) -> Result { + ) -> Result> { let max_rx_data = config.local_transport_params.initial_max_data; let scid_as_hex: Vec = @@ -2155,6 +2238,8 @@ impl Connection { stopped_stream_remote_count: 0, max_amplification_factor: config.max_amplification_factor, + + use_hidden_crypt_copy_for_zc: config.use_hidden_crypt_copy_for_zc, }; if let Some(odcid) = odcid { @@ -3150,7 +3235,6 @@ impl Connection { if self.is_server && !self.got_peer_conn_id { self.set_initial_dcid(hdr.scid.clone(), None, recv_pid)?; - if !self.did_retry { self.local_transport_params .original_destination_connection_id = @@ -3722,7 +3806,13 @@ impl Connection { // Limit output packet size to respect the sender and receiver's // maximum UDP payload size limit. - let mut left = cmp::min(out.len(), self.max_send_udp_payload_size()); + let mut left = if self.use_hidden_crypt_copy_for_zc { + // We reserve a bit more memory for scatter encryption alignment on blocksize -- only + // works for AES for this impl. + cmp::min(out.len(), self.max_send_udp_payload_size() + 16) - 16 + } else { + cmp::min(out.len(), self.max_send_udp_payload_size()) + }; let send_pid = match (from, to) { (Some(f), Some(t)) => self @@ -4341,7 +4431,6 @@ impl Connection { max: self.streams.max_streams_uni_next(), }; - // if push_frame_to_pkt!(b, frames, frame, left, self.version) { if push_frame_to_vec!(frames, frame, left, cumul) { self.streams.update_max_streams_uni(); @@ -4572,114 +4661,76 @@ impl Connection { do_dgram { if let Some(max_dgram_payload) = max_dgram_len { + // Datagrames frames are pop'ed from the queue to be pushed into + // the control's buffer. They need to keep the same order. This should + // not involve any copy of the underlying data contained in a Vec. + let mut tmp_frames: Vec = Vec::new(); + // We use tmp_left to keep track of left while poping datagrams. + // left will be update while pushing into the control buffer. + if self.version == PROTOCOL_VERSION_VREVERSO && !has_fixed_overhead && + self.dgram_send_queue.peek_front_len().is_some() { + left += 6; + has_fixed_overhead = true; + } + + let mut tmp_left = left; while let Some(len) = self.dgram_send_queue.peek_front_len() { - // Ok, so we're trying to send a Datagram. V3's header - // overhead for Datagram is 2 bytes. We - // assumed it to be 8, we fix it now. - let hdr_off = if_likely! {self.version == PROTOCOL_VERSION_VREVERSO => { - if !has_fixed_overhead { - left += 6; - has_fixed_overhead = true; - } - b.off() - } else { - b.off() + cumul - }}; + let hdr_len = 1 + // frame type 2; // length, always encode as 2-byte varint - if (hdr_len + len) <= left { + if (hdr_len + len) <= tmp_left { // Front of the queue fits this packet, send it. match self.dgram_send_queue.pop() { Some(data) => { - // Encode the frame. - // - // Instead of creating a `frame::Frame` object, - // encode the frame directly into the packet - // buffer. - // - // First we reserve some space in the output - // buffer for writing the frame header (we - // assume the length field is always a 2-byte - // varint as we don't know the value yet). - // - // Then we emit the data from the DATAGRAM's - // buffer. - // - // Finally we go back and encode the frame - // header with the now available information. - if_likely! { self.version == PROTOCOL_VERSION_VREVERSO => { - - // Write stream data into the packet buffer; normally right - // after the encrypted header. - b.as_mut()[..len].copy_from_slice(&data); - - //Advance the buffer. - b.skip(len)?; - - // Encode the header reversed. - frame::encode_dgram_footer( - len as u64, - &mut b, - )?; - - // Back to the initial index. - b.rewind(len+hdr_len)?; - - } else { - let (mut dgram_hdr, mut dgram_payload) = - b.split_at(hdr_off + hdr_len)?; - - dgram_payload.as_mut()[..len] - .copy_from_slice(&data); - - // Encode the frame's header. - // - // Due to how `OctetsMut::split_at()` works, - // `dgram_hdr` starts from the initial offset - // of `b` (rather than the current offset), so - // it needs to be advanced to the initial frame - // offset. - dgram_hdr.skip(hdr_off)?; - - frame::encode_dgram_header( - len as u64, - &mut dgram_hdr, - )?; - }}; - - let frame = - frame::Frame::DatagramHeader { length: len }; - - // We always write data a the beginning the - // packet. - // If we use PROTOCOL_VERSION_VREVERSO, this can - // be leveraged to - // enable a - // zero-copy contiguous application buffer. If we - // don't use V3, - // it shouldn't matter where we put it in the - // packet. - if_likely! {self.version == PROTOCOL_VERSION_VREVERSO => { - // we already know that left > frame.wire_len() - let wire_len = frame.wire_len(); - left -= wire_len; - // adjust left that was computed based on a 8 bytes overhead - cumul += wire_len; - frames.insert(0, frame); - ack_eliciting = true; - in_flight = true; - dgram_emitted = true; - // just one dgram per Quic packet to enable the zero-copy - // contiguous receiver - break; - } else { - if push_frame_to_vec!(frames, frame, left, cumul) { - ack_eliciting = true; - in_flight = true; - dgram_emitted = true; - } - }}; + + // Will be written as extra_in + let frame = frame::Frame::Datagram { + data, + }; + + tmp_left -= frame.wire_len(); + tmp_frames.push(frame); + //} else { + //// Encode the frame. + //// + //// Instead of creating a `frame::Frame` object, + //// encode the frame directly into the packet + //// buffer. + //// + //// First we reserve some space in the output + //// buffer for writing the frame header (we + //// assume the length field is always a 2-byte + //// varint as we don't know the value yedgram_send_max_sizet). + //// + //// Then we emit the data from the DATAGRAM's + //// buffer. + //// + //// Finally we go back and encode the frame + //// header with the now available information. + //let (mut dgram_hdr, mut dgram_payload) = + //b.split_at(hdr_off + hdr_len)?; + + //dgram_payload.as_mut()[..len] + //.copy_from_slice(&data); + + //// Encode the frame's header. + //// + //// Due to how `OctetsMut::split_at()` works, + //// `dgram_hdr` starts from the initial offset + //// of `b` (rather than the current offset), so + //// it needs to be advanced to the initial frame + //// offset. + //dgram_hdr.skip(hdr_off)?; + + //frame::encode_dgram_header( + //len as u64, + //&mut dgram_hdr, + //)?; + + //frame::Frame::DatagramHeader { length: len } + //}}; + }, None => continue, @@ -4690,23 +4741,42 @@ impl Connection { } else { // Revert back the fixed overhead. We might try to send a // Stream Frame. - if has_fixed_overhead { + if has_fixed_overhead && tmp_frames.is_empty() { left -= 6; has_fixed_overhead = false; } break; } } + + if self.version == crate::PROTOCOL_VERSION_VREVERSO { + for frame in tmp_frames.into_iter().rev() { + if push_frame_to_vec!(frames, frame, left, cumul) { + ack_eliciting = true; + in_flight = true; + dgram_emitted = true; + } + } + } else { + for frame in tmp_frames { + if push_frame_to_vec!(frames, frame, left, cumul) { + ack_eliciting = true; + in_flight = true; + dgram_emitted = true; + } + } + } } } // Create a single STREAM frame for the first stream that is flushable. - if (pkt_type == packet::Type::Short || pkt_type == packet::Type::ZeroRTT) && + let maybe_stream_header = if (pkt_type == packet::Type::Short || pkt_type == packet::Type::ZeroRTT) && left > frame::MAX_STREAM_OVERHEAD && !is_closing && path.active() && !dgram_emitted { + let mut maybe_frame = frame::Frame::StreamHeader { stream_id: 0, offset: 0, length: 0, fin: false }; let hdr_off = if_likely! {self.version == PROTOCOL_VERSION_VREVERSO => { b.off() } else { b.off() + cumul }}; let max_stream_window = self.streams.max_stream_window; @@ -4795,54 +4865,76 @@ impl Connection { }, }; - let (len, fin) = if_likely! {self.version == PROTOCOL_VERSION_VREVERSO => { - // Write stream data into the packet buffer; normally right after - // the encrypted header. - let (len, fin) = - stream.send.emit(&mut b.as_mut()[..max_len])?; - // Advance the buffer - b.skip(len)?; - - // Encode the header reversed - frame::encode_stream_footer( - stream_id, - stream_off, - len as u64, - fin, - &mut b, + let (len, fin) = if self.use_hidden_crypt_copy_for_zc { + // We'll copy the stream data while encrypting. If the stream + // data is fragmented in smaller sizes than the max payload len + // this method would yield more packets. + stream.send.rangebuf_len(max_len)? + } else { + // We copy the data directly into the buffer. Encryption will be inplace. + let (len, fin) = if_likely! {self.version == PROTOCOL_VERSION_VREVERSO => { + // Write stream data into the packet buffer; normally right after + // the encrypted header. + let (len, fin) = + stream.send.emit(&mut b.as_mut()[..max_len])?; + // Advance the buffer + b.skip(len)?; + + // Encode the header reversed + frame::encode_stream_footer( + stream_id, + stream_off, + len as u64, + fin, + &mut b, )?; - // back to the initial index. - b.rewind(len+hdr_len)?; - - (len, fin) - } else { + // back to the initial index. + b.rewind(len+hdr_len)?; - let (mut stream_hdr, mut stream_payload) = - b.split_at(hdr_off + hdr_len)?; + (len, fin) + } else { - // Write stream data into the packet buffer. - let (len, fin) = - stream.send.emit(&mut stream_payload.as_mut()[..max_len])?; + let (mut stream_hdr, mut stream_payload) = + b.split_at(hdr_off + hdr_len)?; - // Encode the frame's header. - // - // Due to how `OctetsMut::split_at()` works, `stream_hdr` starts - // from the initial offset of `b` (rather than the current - // offset), so it needs to be advanced to the initial frame - // offset. - stream_hdr.skip(hdr_off)?; + // Write stream data into the packet buffer. + let (len, fin) = + stream.send.emit(&mut stream_payload.as_mut()[..max_len])?; - frame::encode_stream_header( - stream_id, - stream_off, - len as u64, - fin, - &mut stream_hdr, + // Encode the frame's header. + // + // Due to how `OctetsMut::split_at()` works, `stream_hdr` starts + // from the initial offset of `b` (rather than the current + // offset), so it needs to be advanced to the initial frame + // offset. + stream_hdr.skip(hdr_off)?; + + frame::encode_stream_header( + stream_id, + stream_off, + len as u64, + fin, + &mut stream_hdr, )?; + (len, fin) + }}; + + let priority_key = Arc::clone(&stream.priority_key); + // If the stream is no longer flushable, remove it from the queue + if !stream.is_flushable() { + self.streams.remove_flushable(&priority_key); + } else if stream.incremental { + // Shuffle the incremental stream to the back of the + // queue. + self.streams.remove_flushable(&priority_key); + self.streams.insert_flushable(&priority_key); + } + (len, fin) - }}; + + }; let frame = frame::Frame::StreamHeader { stream_id, @@ -4855,39 +4947,53 @@ impl Connection { // we already know that left > frame.wire_len() let wire_len = frame.wire_len(); left -= wire_len; + // XXX we should refactor to avoid this. frames.insert(0, frame); ack_eliciting = true; in_flight = true; has_data = true; cumul += wire_len; } else { - if push_frame_to_vec!(frames, frame, left, cumul) { + let wire_len = frame.wire_len(); + if self.use_hidden_crypt_copy_for_zc { + maybe_frame = frame; + left -= wire_len; + cumul += wire_len; ack_eliciting = true; in_flight = true; has_data = true; + } else { + if push_frame_to_vec!(frames, frame, left, cumul) { + ack_eliciting = true; + in_flight = true; + has_data = true; + } } }}; - let priority_key = Arc::clone(&stream.priority_key); - // If the stream is no longer flushable, remove it from the queue - if !stream.is_flushable() { - self.streams.remove_flushable(&priority_key); - } else if stream.incremental { - // Shuffle the incremental stream to the back of the - // queue. - self.streams.remove_flushable(&priority_key); - self.streams.insert_flushable(&priority_key); - } - #[cfg(feature = "fuzzing")] - // Coalesce STREAM frames when fuzzing. - if left > frame::MAX_STREAM_OVERHEAD { + // Coalesce STREAM frames when fuzzing + if left > frame::MAX_STREAM_OVERHEAD && + self.version == crate::PROTOCOL_VERSION_V1 { continue; + // XXX support this with VReverso } break; } - } + + let res = match maybe_frame { + frame::Frame::StreamHeader { length, .. } => { + if length > 0 { + Some(maybe_frame) + } else { + None + } + }, + _ => None, + }; + res + } else {None}; // Alternate trying to send DATAGRAMs next time. self.emit_dgram = !dgram_emitted; @@ -4927,57 +5033,72 @@ impl Connection { } }}; if let Some(max_len) = left.checked_sub(hdr_len) { - let len = if_likely! {self.version == PROTOCOL_VERSION_VREVERSO => { - // located potentally after other control frames - b.skip(cumul)?; - let (len, _) = pkt_space + let frame = if self.use_hidden_crypt_copy_for_zc { + + let (rbvec, length) = pkt_space .crypto_stream .send - .emit(&mut b.as_mut()[..max_len])?; - // this advances b to the Crypto Hdr expected location. - b.skip(len)?; - // Encode the header reversed - frame::encode_crypto_footer( - crypto_off, - len as u64, - &mut b, - )?; - // back to the initial index. - b.rewind(len+hdr_len+cumul)?; - len + .emit_rangebuf_vec(max_len); + + frame::Frame::CryptoVec { + offset: crypto_off, + length, + rbvec, + } + } else { - let (mut crypto_hdr, mut crypto_payload) = - b.split_at(hdr_off + hdr_len)?; - // Write stream data into the packet buffer. - let (len, _) = pkt_space - .crypto_stream - .send - .emit(&mut crypto_payload.as_mut()[..max_len])?; + let len = if_likely! {self.version == PROTOCOL_VERSION_VREVERSO => { + //located potentally after other control frames + b.skip(cumul)?; + let (len, _) = pkt_space + .crypto_stream + .send + .emit(&mut b.as_mut()[..max_len])?; + // this advances b to the Crypto Hdr expected location. + b.skip(len)?; + // Encode the header reversed + frame::encode_crypto_footer( + crypto_off, + len as u64, + &mut b, + )?; + // back to the initial index. + b.rewind(len+hdr_len+cumul)?; + len + } else { - // Encode the frame's header. - // - // Due to how `OctetsMut::split_at()` works, `crypto_hdr` starts - // from the initial offset of `b` (rather than the current - // offset), so it needs to be advanced to the - // initial frame offset. - crypto_hdr.skip(hdr_off)?; - - frame::encode_crypto_header( - crypto_off, - len as u64, - &mut crypto_hdr, - )?; - - len - }}; + let (mut crypto_hdr, mut crypto_payload) = + b.split_at(hdr_off + hdr_len)?; + // Write stream data into the packet buffer. + let (len, _) = pkt_space + .crypto_stream + .send + .emit(&mut crypto_payload.as_mut()[..max_len])?; - let frame = frame::Frame::CryptoHeader { - offset: crypto_off, - length: len, + // Encode the frame's header. + // + // Due to how `OctetsMut::split_at()` works, `crypto_hdr` starts + // from the initial offset of `b` (rather than the current + // offset), so it needs to be advanced to the + // initial frame offset. + crypto_hdr.skip(hdr_off)?; + + frame::encode_crypto_header( + crypto_off, + len as u64, + &mut crypto_hdr, + )?; + + len + }}; + + frame::Frame::CryptoHeader { + offset: crypto_off, + length: len, + } }; - #[allow(unused_assignments)] if push_frame_to_vec!(frames, frame, left, cumul) { ack_eliciting = true; in_flight = true; @@ -4995,8 +5116,6 @@ impl Connection { } }}; - // Add the frames to the buffer. - push_frames_to_pkt!(b, frames, self.version); // If no other ack-eliciting frame is sent, include a PING frame // - if PTO probe needed; OR @@ -5010,7 +5129,7 @@ impl Connection { { let frame = frame::Frame::Ping { mtu_probe: None }; - if push_frame_to_pkt!(b, frames, frame, left, self.version) { + if push_frame_to_vec!(frames, frame, left, cumul) { ack_eliciting = true; in_flight = true; } @@ -5021,7 +5140,7 @@ impl Connection { path.recovery.ping_sent(epoch); } - if frames.is_empty() { + if frames.is_empty() && maybe_stream_header.is_none() { // When we reach this point we are not able to write more, so set // app_limited to false. path.recovery.update_app_limited(false); @@ -5042,31 +5161,85 @@ impl Connection { { let frame = frame::Frame::Padding { len: left }; - if push_frame_to_pkt!(b, frames, frame, left, self.version) { + if push_frame_to_vec!(frames, frame, left, cumul) { in_flight = true; } } // Pad payload so that it's always at least 4 bytes if QUIC V1, or 12 // bytes if QUIC V3. - if b.off() - payload_offset < payload_min_len { - let payload_len = b.off() - payload_offset; - let len = payload_min_len - payload_len; + if cumul < payload_min_len { + //let payload_len = b.off() - payload_offset; + let len = payload_min_len - cumul; let frame = frame::Frame::Padding { len }; - if payload_len + frame.wire_len() > left { + if cumul + frame.wire_len() > left { // We need to bypass the cwnd restriction; or // we can't send this packed normally containing // an ack. left = frame.wire_len(); } #[allow(unused_assignments)] - if push_frame_to_pkt!(b, frames, frame, left, self.version) { + if push_frame_to_vec!(frames, frame, left, cumul) { in_flight = true; } } - let payload_len = b.off() - payload_offset; + // Add the frames to the buffer. + + // With sender copy avoidance, we need a buffer to pack all control + // The data will be send without copy; ctrl information are written + // inside this buffer with the goal to encrypt them next to the zero-copy + // encryted data, in out. + let (mut b_start, ctrl, b_len, b_ctrl_len) = if self.use_hidden_crypt_copy_for_zc { + + // If we're in VReverso, the frame should be first. + let (b_start, mut b_ctrl, stream_len) = if self.version == crate::PROTOCOL_VERSION_VREVERSO { + let stream_len = if let Some(frame::Frame::StreamHeader { length, ..}) = frames.get(0) { + *length + } else { + 0_usize + }; + // ctrl cleartext is written inside the destination buffer, aligned + // on a multiple of the AES blocksize. + let align = stream_len % 16; + let (b, b_ctrl) = b.split_at(payload_offset + stream_len + align)?; + (b, b_ctrl, stream_len) + } else { + // In V1 we would start with the ctrl. + let (b, b_ctrl) = b.split_at(payload_offset)?; + // The StreamHeader frame should be last. + if let Some(frame) = maybe_stream_header { + let frame_len = frame.wire_len(); + cumul -= frame_len; + left += frame_len; + let len = if let frame::Frame::StreamHeader { length, ..} = frame { + length + } else { + 0_usize + }; + + #[allow(unused_assignments)] + if push_frame_to_vec!(frames, frame, left, cumul) { + (b, b_ctrl, len) + } else { + (b, b_ctrl, 0_usize) + } + } else { (b, b_ctrl, 0_usize) } + }; + //let mut ctrl = vec![0; cumul-stream_len]; + //let mut b_ctrl = octets_rev::OctetsMut::with_slice(&mut ctrl); + push_frames_to_pkt!(b_ctrl, frames, true, self.version); + let b_ctrl_len = b_ctrl.off(); + b_ctrl.rewind(b_ctrl_len)?; + (b_start, Some(b_ctrl), stream_len, b_ctrl_len) + } else { + push_frames_to_pkt!(b, frames, false, self.version); + let b_len = b.off() - payload_offset; + (b, None, b_len, 0_usize) + }; + + let payload_len = b_len + b_ctrl_len; // Fill in payload length. if pkt_type != packet::Type::Short { @@ -5077,7 +5250,7 @@ impl Connection { }}; let (_, mut payload_with_len) = - b.split_at(header_offset_for_length)?; + b_start.split_at(header_offset_for_length)?; payload_with_len .put_varint_with_len(len as u64, PAYLOAD_LENGTH_LEN)?; } @@ -5144,21 +5317,149 @@ impl Connection { None => return Err(Error::InvalidState), }; - let enc_hdr_len = if_likely! {self.version == crate::PROTOCOL_VERSION_VREVERSO => { + + let written = if self.use_hidden_crypt_copy_for_zc { + + let sentry = if_likely! {self.version == PROTOCOL_VERSION_VREVERSO => { + if let Some(frame::Frame::StreamHeader { stream_id, ..}) = frames.get(0) { + Some(self.streams.entry(*stream_id)) + } else { + None + } + } else { + if let Some(frame::Frame::StreamHeader { stream_id, ..}) = frames.last() { + Some(self.streams.entry(*stream_id)) + } else { + None + } + }}; + + let written = if_likely! {self.version == PROTOCOL_VERSION_VREVERSO => { + // We encrypt with the data in inbuf and the ctrl data in extra_in, starting + // with the reversed stream frame + let rangebuf = sentry + .as_ref() + .map(|v| { + match v { + std::collections::hash_map::Entry::Occupied(v) => + v.get() + .send + .rangebuf_get() + .and_then(|rb| Some(&rb[..b_len])), + _ => None, + } + }) + .flatten(); + + if let Some(ctrl) = ctrl { + packet::encrypt_pkt( + &mut b_start, + rangebuf, + pn, + b_len, + payload_offset, + Some(&ctrl.as_ref()[..b_ctrl_len]), + aead, + self.use_hidden_crypt_copy_for_zc, + )? + } else { + packet::encrypt_pkt( + &mut b_start, + rangebuf, + pn, + b_len, + payload_offset, + None, + aead, + self.use_hidden_crypt_copy_for_zc, + )? + } + } else { + let rangebuf = sentry + .as_ref() + .map(|v| { + match v { + std::collections::hash_map::Entry::Occupied(v) => + v.get() + .send + .rangebuf_get() + .and_then(|rb| Some(&rb[..b_len])), + _ => None, + } + }) + .flatten(); + // We encrypt with the data in extra_in and the ctrl in inbuf, with + // the stream header at the end of the ctrl. + if let Some(ctrl) = ctrl { + packet::encrypt_pkt( + &mut b_start, + Some(&ctrl.as_ref()), + pn, + b_ctrl_len, + payload_offset, + rangebuf, + aead, + self.use_hidden_crypt_copy_for_zc, + )? + } else { + packet::encrypt_pkt( + &mut b_start, + None, + pn, + b_len, + payload_offset, + rangebuf, + aead, + self.use_hidden_crypt_copy_for_zc, + )? + } + }}; + + if let Some(sentry) = sentry { + let mut is_flushable = false; + let mut is_incremental = false; + let mut priority_key = Default::default(); + sentry.and_modify(|s| { + s.send.rangebuf_consume(b_len); + is_flushable = s.is_flushable(); + is_incremental = s.incremental; + priority_key = Arc::clone(&s.priority_key); + }); + + if !is_flushable { + self.streams.remove_flushable(&priority_key); + } else if is_incremental { + self.streams.remove_flushable(&priority_key); + self.streams.insert_flushable(&priority_key); + } + } + + written + } else { + packet::encrypt_pkt( + &mut b_start, + None, + pn, + payload_len, + payload_offset, + None, + aead, + self.use_hidden_crypt_copy_for_zc, + )? + }; + + + let enc_hdr_len = if_likely! {self.version == PROTOCOL_VERSION_VREVERSO => { pn_len + self.expected_stream_id_len + self.truncated_offset_len } else { pn_len }}; - let written = packet::encrypt_pkt( - &mut b, - pn, - enc_hdr_len, - payload_len, - payload_offset, - None, - aead, - self.version, - )?; + + // safe since payload_offset is guaranteed to be < out.len() + unsafe { + let (left, right) = out.split_at_mut_unchecked(payload_offset); + packet::encrypt_hdr_unchecked(left, enc_hdr_len, &right[..], aead, self.version)?; + } self.expected_stream_id_len = 1; self.truncated_offset_len = 1; @@ -5647,6 +5948,55 @@ impl Connection { pub fn stream_send( &mut self, stream_id: u64, buf: &[u8], fin: bool, ) -> Result { + self.stream_do_send( + stream_id, + buf, + fin, + |stream: &mut stream::Stream, + buf: &[u8], + cap: usize, + fin: bool| { + stream.send.write(&buf[..cap], fin).map(|v| (v, v)) + }, + ) + } + + /// Writes data to a stream with zero copying, instead, it appends the + /// provided buffer directly to the send queue if the capacity allows + /// it. + /// + /// When a partial write happens (including when [`Done`] is returned) the + /// remaining (unwrittent) buffer will also be returned. The application + /// should retry the operation once the stream is reported as writable + /// again. + pub fn stream_send_zc( + &mut self, stream_id: u64, buf: F::Buf, len: Option, fin: bool, + ) -> Result<(usize, Option)> + where + F::Buf: BufSplit, + { + self.stream_do_send( + stream_id, + buf, + fin, + |stream: &mut stream::Stream, + buf: F::Buf, + cap: usize, + fin: bool| { + let len = len.unwrap_or(usize::MAX).min(cap); + let (sent, remaining) = stream.send.append_buf(buf, len, fin)?; + Ok((sent, (sent, remaining))) + }, + ) + } + + fn stream_do_send( + &mut self, stream_id: u64, buf: B, fin: bool, write_fn: SND, + ) -> Result + where + B: AsRef<[u8]>, + SND: FnOnce(&mut stream::Stream, B, usize, bool) -> Result<(usize, R)>, + { // We can't write on the peer's unidirectional streams. if !stream::is_bidi(stream_id) && !stream::is_local(stream_id, self.is_server) @@ -5654,12 +6004,14 @@ impl Connection { return Err(Error::InvalidStreamState(stream_id)); } + let len = buf.as_ref().len(); + // Mark the connection as blocked if the connection-level flow control // limit doesn't let us buffer all the data. // // Note that this is separate from "send capacity" as that also takes // congestion control into consideration. - if self.max_tx_data - self.tx_data < buf.len() as u64 { + if self.max_tx_data - self.tx_data < len as u64 { self.blocked_limit = Some(self.max_tx_data); } @@ -5682,7 +6034,7 @@ impl Connection { // // When the cap is zero, the method returns Ok(0) *only* when the passed // buffer is empty. We return Error::Done otherwise. - if cap == 0 && !buf.is_empty() { + if cap == 0 && len > 0 { if was_writable { // When `stream_writable_next()` returns a stream, the writable // mark is removed, but because the stream is blocked by the @@ -5696,13 +6048,13 @@ impl Connection { return Err(Error::Done); } - let (buf, fin, blocked_by_cap) = if cap < buf.len() { - (&buf[..cap], false, true) + let (cap, fin, blocked_by_cap) = if cap < len { + (cap, false, true) } else { - (buf, fin, false) + (len, fin, false) }; - let sent = match stream.send.write(buf, fin) { + let (sent, ret) = match write_fn(stream, buf, cap, fin) { Ok(v) => v, Err(e) => { @@ -5718,9 +6070,9 @@ impl Connection { let writable = stream.is_writable(); - let empty_fin = buf.is_empty() && fin; + let empty_fin = len == 0 && fin; - if sent < buf.len() { + if sent < cap { let max_off = stream.send.max_off(); if stream.send.blocked_at() != Some(max_off) { @@ -5773,7 +6125,7 @@ impl Connection { q.add_event_data_with_instant(ev_data, now).ok(); }); - if sent == 0 && !buf.is_empty() { + if sent == 0 && cap > 0 { return Err(Error::Done); } @@ -5783,7 +6135,7 @@ impl Connection { self.streams.insert_writable(&priority_key); } - Ok(sent) + Ok(ret) } /// Sets the priority for a stream. @@ -6531,7 +6883,7 @@ impl Connection { /// # Ok::<(), quiceh::Error>(()) /// ``` #[inline] - pub fn dgram_purge_outgoing bool>(&mut self, f: F) { + pub fn dgram_purge_outgoing bool>(&mut self, f: FN) { self.dgram_send_queue.purge(f); } @@ -7681,6 +8033,13 @@ impl Connection { let send_path = self.paths.get(send_pid)?; if (self.is_established() || self.is_in_early_data()) && (self.should_send_handshake_done() || + self.streams.has_flushable() || + self.streams.has_almost_full() || + self.streams.has_blocked() || + self.streams.has_reset() || + self.streams.has_stopped() || + self.ids.has_new_scids() || + self.ids.has_retire_dcids() || self.almost_full || self.blocked_limit.is_some() || self.dgram_send_queue.has_pending() || @@ -7689,13 +8048,6 @@ impl Connection { .map_or(false, |conn_err| conn_err.is_app) || self.streams.should_update_max_streams_bidi() || self.streams.should_update_max_streams_uni() || - self.streams.has_flushable() || - self.streams.has_almost_full() || - self.streams.has_blocked() || - self.streams.has_reset() || - self.streams.has_stopped() || - self.ids.has_new_scids() || - self.ids.has_retire_dcids() || send_path.pmtud.get_probe_status() || send_path.needs_ack_eliciting || send_path.probing_required()) @@ -7715,7 +8067,7 @@ impl Connection { /// a new one otherwise. fn get_or_create_stream( &mut self, id: u64, local: bool, - ) -> Result<&mut stream::Stream> { + ) -> Result<&mut stream::Stream> { self.streams.get_or_create( id, &self.local_transport_params, @@ -7919,7 +8271,7 @@ impl Connection { }, frame::Frame::CryptoHeader { .. } => unreachable!(), - + frame::Frame::CryptoVec { .. } => unreachable!(), // TODO: implement stateless retry frame::Frame::NewToken { .. } => if self.is_server { @@ -9346,15 +9698,72 @@ impl TransportParams { pub mod testing { use super::*; - pub struct Pipe { - pub client: Connection, - pub server: Connection, + #[derive(Debug, Clone, Default)] + pub struct BufTestFactory; + #[derive(Debug, Clone, Default, PartialEq)] + pub struct BufTest { + inner: Arc>, + start: usize, + end: usize, + } + + impl BufTest { + fn new(inner: Arc>, start: usize, end: usize) -> Self { + Self { inner, start, end } + } + + fn len(&self) -> usize { + self.end - self.start + } + + } + + impl BufFactory for BufTestFactory { + type Buf = BufTest; + + fn buf_from_slice(buf: &[u8]) -> Self::Buf { + BufTest { + start: 0, + end: buf.len(), + inner: Arc::new(buf.into()), + } + } + } + + impl BufSplit for BufTest { + // Split the buffer at a given point, after the split the old buffer + // must only contain the first `at` bytes, while the newly produced + // buffer must containt the remaining bytes. + fn split_at(&mut self, at: usize) -> Self { + assert!(at <= self.len(), "split_at index out of bounds"); + + let newend = self.start + at; + let buf = BufTest::new(self.inner.clone(), newend, self.end); + + self.end = newend; + + buf + } + } + + impl AsRef<[u8]> for BufTest { + fn as_ref(&self) -> &[u8] { + &self.inner[self.start..self.end] + } + } + + pub struct Pipe + where + F: BufFactory, + { + pub client: Connection, + pub server: Connection, pub client_app_buffers: AppRecvBufMap, pub server_app_buffers: AppRecvBufMap, } - impl Pipe { - pub fn new() -> Result { + impl Pipe { + pub fn new() -> Result> { let mut config = Config::new(crate::PROTOCOL_VERSION)?; config.load_cert_chain_from_pem_file("examples/cert.crt")?; config.load_priv_key_from_pem_file("examples/cert.key")?; @@ -9380,16 +9789,16 @@ pub mod testing { "127.0.0.1:4321".parse().unwrap() } - pub fn with_config(config: &mut Config) -> Result { + pub fn with_config(config: &mut Config) -> Result> { let mut client_scid = [0; 16]; rand::rand_bytes(&mut client_scid[..]); let client_scid = ConnectionId::from_ref(&client_scid); - let client_addr = Pipe::client_addr(); + let client_addr = ::client_addr(); let mut server_scid = [0; 16]; rand::rand_bytes(&mut server_scid[..]); let server_scid = ConnectionId::from_ref(&server_scid); - let server_addr = Pipe::server_addr(); + let server_addr = ::server_addr(); let mut client_app_buffers = AppRecvBufMap::new(3, stream::MAX_STREAM_WINDOW, 1000, 1000); @@ -9403,14 +9812,14 @@ pub mod testing { .unwrap(); Ok(Pipe { - client: connect( + client: connect_with_buffer_factory( Some("quic.tech"), &client_scid, client_addr, server_addr, config, )?, - server: accept( + server: accept_with_buf_factory( &server_scid, None, server_addr, @@ -9424,26 +9833,26 @@ pub mod testing { pub fn with_config_and_scid_lengths( config: &mut Config, client_scid_len: usize, server_scid_len: usize, - ) -> Result { + ) -> Result> { let mut client_scid = vec![0; client_scid_len]; rand::rand_bytes(&mut client_scid[..]); let client_scid = ConnectionId::from_ref(&client_scid); - let client_addr = Pipe::client_addr(); + let client_addr = ::client_addr(); let mut server_scid = vec![0; server_scid_len]; rand::rand_bytes(&mut server_scid[..]); let server_scid = ConnectionId::from_ref(&server_scid); - let server_addr = Pipe::server_addr(); + let server_addr = ::server_addr(); Ok(Pipe { - client: connect( + client: connect_with_buffer_factory( Some("quic.tech"), &client_scid, client_addr, server_addr, config, )?, - server: accept( + server: accept_with_buf_factory( &server_scid, None, server_addr, @@ -9465,16 +9874,16 @@ pub mod testing { }) } - pub fn with_client_config(client_config: &mut Config) -> Result { + pub fn with_client_config(client_config: &mut Config) -> Result> { let mut client_scid = [0; 16]; rand::rand_bytes(&mut client_scid[..]); let client_scid = ConnectionId::from_ref(&client_scid); - let client_addr = Pipe::client_addr(); + let client_addr = ::client_addr(); let mut server_scid = [0; 16]; rand::rand_bytes(&mut server_scid[..]); let server_scid = ConnectionId::from_ref(&server_scid); - let server_addr = Pipe::server_addr(); + let server_addr = ::server_addr(); let mut config = Config::new(crate::PROTOCOL_VERSION)?; config.load_cert_chain_from_pem_file("examples/cert.crt")?; @@ -9488,14 +9897,14 @@ pub mod testing { config.set_ack_delay_exponent(8); Ok(Pipe { - client: connect( + client: connect_with_buffer_factory( Some("quic.tech"), &client_scid, client_addr, server_addr, client_config, )?, - server: accept( + server: accept_with_buf_factory( &server_scid, None, server_addr, @@ -9517,16 +9926,16 @@ pub mod testing { }) } - pub fn with_server_config(server_config: &mut Config) -> Result { + pub fn with_server_config(server_config: &mut Config) -> Result> { let mut client_scid = [0; 16]; rand::rand_bytes(&mut client_scid[..]); let client_scid = ConnectionId::from_ref(&client_scid); - let client_addr = Pipe::client_addr(); + let client_addr = ::client_addr(); let mut server_scid = [0; 16]; rand::rand_bytes(&mut server_scid[..]); let server_scid = ConnectionId::from_ref(&server_scid); - let server_addr = Pipe::server_addr(); + let server_addr = Pipe::::server_addr(); let mut config = Config::new(crate::PROTOCOL_VERSION)?; config.set_application_protos(&[b"proto1", b"proto2"])?; @@ -9538,14 +9947,14 @@ pub mod testing { config.set_ack_delay_exponent(8); Ok(Pipe { - client: connect( + client: connect_with_buffer_factory( Some("quic.tech"), &client_scid, client_addr, server_addr, &mut config, )?, - server: accept( + server: accept_with_buf_factory( &server_scid, None, server_addr, @@ -9569,26 +9978,26 @@ pub mod testing { pub fn with_client_and_server_config( client_config: &mut Config, server_config: &mut Config, - ) -> Result { + ) -> Result> { let mut client_scid = [0; 16]; rand::rand_bytes(&mut client_scid[..]); let client_scid = ConnectionId::from_ref(&client_scid); - let client_addr = Pipe::client_addr(); + let client_addr = Pipe::::client_addr(); let mut server_scid = [0; 16]; rand::rand_bytes(&mut server_scid[..]); let server_scid = ConnectionId::from_ref(&server_scid); - let server_addr = Pipe::server_addr(); + let server_addr = Pipe::::server_addr(); Ok(Pipe { - client: connect( + client: connect_with_buffer_factory( Some("quic.tech"), &client_scid, client_addr, server_addr, client_config, )?, - server: accept( + server: accept_with_buf_factory( &server_scid, None, server_addr, @@ -9730,9 +10139,9 @@ pub mod testing { } } - pub fn recv_send( - conn: &mut Connection, app_buffers: &mut AppRecvBufMap, buf: &mut [u8], - len: usize, stream_id: Option, + pub fn recv_send( + conn: &mut Connection, app_buffers: &mut AppRecvBufMap, + buf: &mut [u8], len: usize, stream_id: Option, ) -> Result { let active_path = conn.paths.get_active()?; let info = RecvInfo { @@ -9770,8 +10179,8 @@ pub mod testing { Ok(off) } - pub fn process_flight( - conn: &mut Connection, app_buffers: &mut AppRecvBufMap, + pub fn process_flight( + conn: &mut Connection, app_buffers: &mut AppRecvBufMap, flight: Vec<(Vec, SendInfo)>, ) -> Result<()> { for (mut pkt, si) in flight { @@ -9785,8 +10194,8 @@ pub mod testing { Ok(()) } - pub fn emit_flight_with_max_buffer( - conn: &mut Connection, out_size: usize, from: Option, + pub fn emit_flight_with_max_buffer( + conn: &mut Connection, out_size: usize, from: Option, to: Option, ) -> Result, SendInfo)>> { let mut flight = Vec::new(); @@ -9815,21 +10224,22 @@ pub mod testing { Ok(flight) } - pub fn emit_flight_on_path( - conn: &mut Connection, from: Option, to: Option, + pub fn emit_flight_on_path( + conn: &mut Connection, from: Option, + to: Option, ) -> Result, SendInfo)>> { emit_flight_with_max_buffer(conn, 65535, from, to) } - pub fn emit_flight( - conn: &mut Connection, + pub fn emit_flight( + conn: &mut Connection, ) -> Result, SendInfo)>> { emit_flight_on_path(conn, None, None) } - pub fn encode_pkt( - conn: &mut Connection, pkt_type: packet::Type, frames: &[frame::Frame], - buf: &mut [u8], + pub fn encode_pkt( + conn: &mut Connection, pkt_type: packet::Type, + frames: &[frame::Frame], buf: &mut [u8], ) -> Result { let mut b = octets_rev::OctetsMut::with_slice(buf); @@ -9925,22 +10335,25 @@ pub mod testing { let written = packet::encrypt_pkt( &mut b, + None, pn, - hdr_enc_len, payload_len, payload_offset, None, aead, - conn.version, + conn.use_hidden_crypt_copy_for_zc, )?; + let (mut header, payload) = b.split_at(payload_offset)?; + packet::encrypt_hdr(&mut header, hdr_enc_len, payload.as_ref(), aead, crate::PROTOCOL_VERSION)?; + space.next_pkt_num += 1; Ok(written) } - pub fn decode_pkt( - conn: &mut Connection, buf: &mut [u8], app_buffers: &mut AppRecvBufMap, + pub fn decode_pkt( + conn: &mut Connection, buf: &mut [u8], app_buffers: &mut AppRecvBufMap, ) -> Result> { let mut b = octets_rev::OctetsMut::with_slice(buf); @@ -10090,7 +10503,11 @@ pub mod testing { #[cfg(test)] mod tests { + use crate::range_buf::RangeBuf; + use super::*; + use testing::Pipe; + use testing::BufTestFactory; #[test] fn transport_params() { @@ -10198,7 +10615,7 @@ mod tests { .unwrap(); config.verify_peer(false); - let mut pipe = testing::Pipe::with_client_config(&mut config).unwrap(); + let mut pipe = ::with_client_config(&mut config).unwrap(); assert_eq!(pipe.handshake(), Err(Error::UnknownVersion)); } @@ -10226,7 +10643,7 @@ mod tests { .unwrap(); config.verify_peer(false); - let mut pipe = testing::Pipe::with_client_config(&mut config).unwrap(); + let mut pipe = ::with_client_config(&mut config).unwrap(); let (mut len, _) = pipe.client.send(&mut buf).unwrap(); @@ -10252,7 +10669,7 @@ mod tests { .set_application_protos(&[b"proto1", b"proto2"]) .unwrap(); - let mut pipe = testing::Pipe::with_client_config(&mut config).unwrap(); + let mut pipe = ::with_client_config(&mut config).unwrap(); assert_eq!(pipe.handshake(), Ok(())); } @@ -10302,7 +10719,7 @@ mod tests { .unwrap(); client_config.verify_peer(true); - let mut pipe = testing::Pipe::with_client_and_server_config( + let mut pipe = ::with_client_and_server_config( &mut client_config, &mut server_config, ) @@ -10333,7 +10750,7 @@ mod tests { // Try to validate client certificate. config.verify_peer(true); - let mut pipe = testing::Pipe::with_server_config(&mut config).unwrap(); + let mut pipe = ::with_server_config(&mut config).unwrap(); assert_eq!(pipe.handshake(), Ok(())); // Client didn't send a certificate. @@ -10344,7 +10761,7 @@ mod tests { fn missing_initial_source_connection_id() { let mut buf = [0; 65535]; - let mut pipe = testing::Pipe::new().unwrap(); + let mut pipe = ::new().unwrap(); // Reset initial_source_connection_id. pipe.client @@ -10366,7 +10783,7 @@ mod tests { fn invalid_initial_source_connection_id() { let mut buf = [0; 65535]; - let mut pipe = testing::Pipe::new().unwrap(); + let mut pipe = ::new().unwrap(); // Scramble initial_source_connection_id. pipe.client @@ -10386,7 +10803,7 @@ mod tests { #[test] fn handshake() { - let mut pipe = testing::Pipe::new().unwrap(); + let mut pipe = ::new().unwrap(); assert_eq!(pipe.handshake(), Ok(())); assert_eq!( @@ -10399,7 +10816,7 @@ mod tests { #[test] fn handshake_done() { - let mut pipe = testing::Pipe::new().unwrap(); + let mut pipe = ::new().unwrap(); // Disable session tickets on the server (SSL_OP_NO_TICKET) to avoid // triggering 1-RTT packet send with a CRYPTO frame. @@ -10412,7 +10829,7 @@ mod tests { #[test] fn handshake_confirmation() { - let mut pipe = testing::Pipe::new().unwrap(); + let mut pipe = ::new().unwrap(); // Client sends initial flight. let flight = testing::emit_flight(&mut pipe.client).unwrap(); @@ -10523,7 +10940,7 @@ mod tests { config.set_ticket_key(&SESSION_TICKET_KEY).unwrap(); // Perform initial handshake. - let mut pipe = testing::Pipe::with_server_config(&mut config).unwrap(); + let mut pipe = ::with_server_config(&mut config).unwrap(); assert_eq!(pipe.handshake(), Ok(())); assert!(pipe.client.is_established()); @@ -10552,7 +10969,7 @@ mod tests { config.set_initial_max_streams_bidi(3); config.set_ticket_key(&SESSION_TICKET_KEY).unwrap(); - let mut pipe = testing::Pipe::with_server_config(&mut config).unwrap(); + let mut pipe = ::with_server_config(&mut config).unwrap(); assert_eq!(pipe.client.set_session(session), Ok(())); assert_eq!(pipe.handshake(), Ok(())); @@ -10574,7 +10991,7 @@ mod tests { .unwrap(); config.verify_peer(false); - let mut pipe = testing::Pipe::with_client_config(&mut config).unwrap(); + let mut pipe = ::with_client_config(&mut config).unwrap(); assert_eq!(pipe.handshake(), Err(Error::TlsFail)); assert_eq!(pipe.client.application_proto(), b""); @@ -10611,14 +11028,14 @@ mod tests { config.verify_peer(false); // Perform initial handshake. - let mut pipe = testing::Pipe::with_config(&mut config).unwrap(); + let mut pipe = ::with_config(&mut config).unwrap(); assert_eq!(pipe.handshake(), Ok(())); // Extract session, let session = pipe.client.session().unwrap(); // Configure session on new connection. - let mut pipe = testing::Pipe::with_config(&mut config).unwrap(); + let mut pipe = ::with_config(&mut config).unwrap(); assert_eq!(pipe.client.set_session(session), Ok(())); // Client sends initial flight. @@ -10630,7 +11047,7 @@ mod tests { let frames = [frame::Frame::Stream { stream_id: 4, - data: stream::RangeBuf::from(b"aaaaa", 0, true), + data: ::from(b"aaaaa", 0, true), }]; assert_eq!( @@ -10686,14 +11103,14 @@ mod tests { config.verify_peer(false); // Perform initial handshake. - let mut pipe = testing::Pipe::with_config(&mut config).unwrap(); + let mut pipe = ::with_config(&mut config).unwrap(); assert_eq!(pipe.handshake(), Ok(())); // Extract session, let session = pipe.client.session().unwrap(); // Configure session on new connection. - let mut pipe = testing::Pipe::with_config(&mut config).unwrap(); + let mut pipe = ::with_config(&mut config).unwrap(); assert_eq!(pipe.client.set_session(session), Ok(())); // Client sends initial flight. @@ -10705,7 +11122,7 @@ mod tests { let frames = [frame::Frame::Stream { stream_id: 4, - data: stream::RangeBuf::from(b"aaaaa", 0, true), + data: ::from(b"aaaaa", 0, true), }]; let len = @@ -10767,14 +11184,14 @@ mod tests { config.verify_peer(false); // Perform initial handshake. - let mut pipe = testing::Pipe::with_config(&mut config).unwrap(); + let mut pipe = ::with_config(&mut config).unwrap(); assert_eq!(pipe.handshake(), Ok(())); // Extract session, let session = pipe.client.session().unwrap(); // Configure session on new connection. - let mut pipe = testing::Pipe::with_config(&mut config).unwrap(); + let mut pipe = ::with_config(&mut config).unwrap(); assert_eq!(pipe.client.set_session(session), Ok(())); // Client sends initial flight. @@ -10785,7 +11202,7 @@ mod tests { let frames = [frame::Frame::Stream { stream_id: 4, - data: stream::RangeBuf::from(b"aaaaa", 0, true), + data: ::from(b"aaaaa", 0, true), }]; let len = @@ -10825,13 +11242,13 @@ mod tests { config.verify_peer(false); // Perform initial handshake. - let mut pipe = testing::Pipe::with_config(&mut config).unwrap(); + let mut pipe = ::with_config(&mut config).unwrap(); assert_eq!(pipe.handshake(), Ok(())); // Client send a 1-byte frame that starts from the crypto stream offset // limit. let frames = [frame::Frame::Crypto { - data: stream::RangeBuf::from(b"a", MAX_CRYPTO_STREAM_OFFSET, false), + data: ::from(b"a", MAX_CRYPTO_STREAM_OFFSET, false), }]; let pkt_type = packet::Type::Short; @@ -10896,7 +11313,7 @@ mod tests { .set_application_protos(&[b"proto1", b"proto2"]) .unwrap(); - let mut pipe = testing::Pipe::with_server_config(&mut config).unwrap(); + let mut pipe = ::with_server_config(&mut config).unwrap(); let flight = testing::emit_flight(&mut pipe.client).unwrap(); let client_sent = flight.iter().fold(0, |out, p| out + p.0.len()); @@ -10929,7 +11346,7 @@ mod tests { .unwrap(); config.set_max_amplification_factor(CUSTOM_AMPLIFICATION_FACTOR); - let mut pipe = testing::Pipe::with_server_config(&mut config).unwrap(); + let mut pipe = ::with_server_config(&mut config).unwrap(); let flight = testing::emit_flight(&mut pipe.client).unwrap(); let client_sent = flight.iter().fold(0, |out, p| out + p.0.len()); @@ -10948,7 +11365,7 @@ mod tests { #[test] fn stream() { - let mut pipe = testing::Pipe::new().unwrap(); + let mut pipe = ::new().unwrap(); assert_eq!(pipe.handshake(), Ok(())); assert_eq!(pipe.client.stream_send(4, b"hello, world", true), Ok(12)); @@ -10997,7 +11414,7 @@ mod tests { config.set_initial_max_stream_data_bidi_local(30); config.set_initial_max_stream_data_bidi_remote(30); config.set_initial_max_data(30); - let mut pipe = testing::Pipe::with_config(&mut config).unwrap(); + let mut pipe = ::with_config(&mut config).unwrap(); assert_eq!(pipe.handshake(), Ok(())); assert_eq!( @@ -11076,20 +11493,20 @@ mod tests { config.set_initial_max_stream_data_bidi_remote(30); config.set_initial_max_data(30); - let mut pipe = testing::Pipe::with_config(&mut config).unwrap(); + let mut pipe = ::with_config(&mut config).unwrap(); assert_eq!(pipe.handshake(), Ok(())); let frames = [ frame::Frame::Stream { stream_id: 4, - data: stream::RangeBuf::from(b"aaaaa", 0, false), + data: ::from(b"aaaaa", 0, false), }, frame::Frame::Stream { stream_id: 4, - data: stream::RangeBuf::from(b"bbbbb", 5, false), + data: ::from(b"bbbbb", 5, false), }, frame::Frame::Stream { stream_id: 4, - data: stream::RangeBuf::from(b"ccccc", 10, false), + data: ::from(b"ccccc", 10, false), }, ]; let pkt_type = packet::Type::Short; @@ -11162,11 +11579,12 @@ mod tests { config.set_initial_max_streams_uni(10 * 32 * 1024); config.set_initial_max_stream_data_uni(10 * 32 * 1024); config.set_initial_max_data(10 * 32 * 1024); + config.enable_hidden_copy_for_zc_sender(true); let datasize = 12000; let sendbuf = [0; 12000]; - let mut pipe = testing::Pipe::with_config(&mut config).unwrap(); + let mut pipe = ::with_config(&mut config).unwrap(); assert_eq!(pipe.handshake(), Ok(())); for _ in 1..10 { @@ -11220,14 +11638,14 @@ mod tests { config.verify_peer(false); // Perform initial handshake. - let mut pipe = testing::Pipe::with_config(&mut config).unwrap(); + let mut pipe = ::with_config(&mut config).unwrap(); assert_eq!(pipe.handshake(), Ok(())); // Extract session, let session = pipe.client.session().unwrap(); // Configure session on new connection. - let mut pipe = testing::Pipe::with_config(&mut config).unwrap(); + let mut pipe = ::with_config(&mut config).unwrap(); assert_eq!(pipe.client.set_session(session), Ok(())); // Client sends initial flight. @@ -11290,7 +11708,7 @@ mod tests { config.set_initial_max_streams_uni(0); config.verify_peer(false); - let mut pipe = testing::Pipe::with_config(&mut config).unwrap(); + let mut pipe = ::with_config(&mut config).unwrap(); assert_eq!(pipe.handshake(), Ok(())); // In 32bit arch, send_capacity() should be min(2^32+5, cwnd), @@ -11306,12 +11724,12 @@ mod tests { fn empty_stream_frame() { let mut buf = [0; 65535]; - let mut pipe = testing::Pipe::new().unwrap(); + let mut pipe = ::new().unwrap(); assert_eq!(pipe.handshake(), Ok(())); let frames = [frame::Frame::Stream { stream_id: 4, - data: stream::RangeBuf::from(b"aaaaa", 0, false), + data: ::from(b"aaaaa", 0, false), }]; let pkt_type = packet::Type::Short; @@ -11347,7 +11765,7 @@ mod tests { let frames = [frame::Frame::Stream { stream_id: 4, - data: stream::RangeBuf::from(b"", 5, true), + data: ::from(b"", 5, true), }]; let pkt_type = packet::Type::Short; if pipe.client.version == PROTOCOL_VERSION_VREVERSO { @@ -11382,7 +11800,7 @@ mod tests { let frames = [frame::Frame::Stream { stream_id: 4, - data: stream::RangeBuf::from(b"", 15, true), + data: ::from(b"", 15, true), }]; let pkt_type = packet::Type::Short; @@ -11396,7 +11814,7 @@ mod tests { fn update_key_request() { let mut b = [0; 15]; - let mut pipe = testing::Pipe::new().unwrap(); + let mut pipe = ::new().unwrap(); assert_eq!(pipe.handshake(), Ok(())); assert_eq!(pipe.advance(), Ok(())); @@ -11465,7 +11883,7 @@ mod tests { fn update_key_request_twice_error() { let mut buf = [0; 65535]; - let mut pipe = testing::Pipe::new().unwrap(); + let mut pipe = ::new().unwrap(); assert_eq!(pipe.handshake(), Ok(())); assert_eq!(pipe.advance(), Ok(())); @@ -11475,11 +11893,11 @@ mod tests { let frames = [ frame::Frame::Stream { stream_id: 4, - data: stream::RangeBuf::from(b"hello", 0, false), + data: ::from(b"hello", 0, false), }, frame::Frame::Stream { stream_id: 4, - data: stream::RangeBuf::from(b"hello", 5, false), + data: ::from(b"hello", 5, false), }, ]; @@ -11518,7 +11936,7 @@ mod tests { fn max_stream_data_receive_uni() { let mut buf = [0; 65535]; - let mut pipe = testing::Pipe::new().unwrap(); + let mut pipe = ::new().unwrap(); assert_eq!(pipe.handshake(), Ok(())); // Client opens unidirectional stream. @@ -11542,7 +11960,7 @@ mod tests { fn empty_payload() { let mut buf = [0; 65535]; - let mut pipe = testing::Pipe::new().unwrap(); + let mut pipe = ::new().unwrap(); assert_eq!(pipe.handshake(), Ok(())); // Send a packet with no frames. @@ -11557,7 +11975,7 @@ mod tests { fn min_payload() { let mut buf = [0; 65535]; - let mut pipe = testing::Pipe::new().unwrap(); + let mut pipe = ::new().unwrap(); // Send a non-ack-eliciting packet. let frames = [frame::Frame::Padding { len: 4 }]; @@ -11605,21 +12023,21 @@ mod tests { // TODO in V3, should we test this with 1 Stream Frame per packet instead? let mut buf = [0; 65535]; - let mut pipe = testing::Pipe::new().unwrap(); + let mut pipe = ::new().unwrap(); assert_eq!(pipe.handshake(), Ok(())); let frames = [ frame::Frame::Stream { stream_id: 0, - data: stream::RangeBuf::from(b"aaaaaaaaaaaaaaa", 0, false), + data: ::from(b"aaaaaaaaaaaaaaa", 0, false), }, frame::Frame::Stream { stream_id: 4, - data: stream::RangeBuf::from(b"aaaaaaaaaaaaaaa", 0, false), + data: ::from(b"aaaaaaaaaaaaaaa", 0, false), }, frame::Frame::Stream { stream_id: 8, - data: stream::RangeBuf::from(b"a", 0, false), + data: ::from(b"a", 0, false), }, ]; @@ -11634,23 +12052,23 @@ mod tests { fn flow_control_limit_dup() { let mut buf = [0; 65535]; - let mut pipe = testing::Pipe::new().unwrap(); + let mut pipe = ::new().unwrap(); assert_eq!(pipe.handshake(), Ok(())); let frames = [ // One byte less than stream limit. frame::Frame::Stream { stream_id: 4, - data: stream::RangeBuf::from(b"aaaaaaaaaaaaaa", 0, false), + data: ::from(b"aaaaaaaaaaaaaa", 0, false), }, // Same stream, but one byte more. frame::Frame::Stream { stream_id: 4, - data: stream::RangeBuf::from(b"aaaaaaaaaaaaaaa", 0, false), + data: ::from(b"aaaaaaaaaaaaaaa", 0, false), }, frame::Frame::Stream { stream_id: 8, - data: stream::RangeBuf::from(b"aaaaaaaaaaaaaaa", 0, false), + data: ::from(b"aaaaaaaaaaaaaaa", 0, false), }, ]; @@ -11691,17 +12109,17 @@ mod tests { fn flow_control_update() { let mut buf = [0; 65535]; - let mut pipe = testing::Pipe::new().unwrap(); + let mut pipe = ::new().unwrap(); assert_eq!(pipe.handshake(), Ok(())); let frames = [ frame::Frame::Stream { stream_id: 4, - data: stream::RangeBuf::from(b"aaaaaaaaaaaaaaa", 0, false), + data: ::from(b"aaaaaaaaaaaaaaa", 0, false), }, frame::Frame::Stream { stream_id: 8, - data: stream::RangeBuf::from(b"a", 0, false), + data: ::from(b"a", 0, false), }, ]; @@ -11749,7 +12167,7 @@ mod tests { let frames = [frame::Frame::Stream { stream_id: 4, - data: stream::RangeBuf::from(b"a", 1, false), + data: ::from(b"a", 1, false), }]; let len = pipe @@ -11797,7 +12215,7 @@ mod tests { /// Tests that flow control is properly updated even when a stream is shut /// down. fn flow_control_drain() { - let mut pipe = testing::Pipe::new().unwrap(); + let mut pipe = ::new().unwrap(); assert_eq!(pipe.handshake(), Ok(())); // Client opens a stream and sends some data. @@ -11831,12 +12249,12 @@ mod tests { fn stream_flow_control_limit_bidi() { let mut buf = [0; 65535]; - let mut pipe = testing::Pipe::new().unwrap(); + let mut pipe = ::new().unwrap(); assert_eq!(pipe.handshake(), Ok(())); let frames = [frame::Frame::Stream { stream_id: 4, - data: stream::RangeBuf::from(b"aaaaaaaaaaaaaaaa", 0, true), + data: ::from(b"aaaaaaaaaaaaaaaa", 0, true), }]; let pkt_type = packet::Type::Short; @@ -11850,12 +12268,12 @@ mod tests { fn stream_flow_control_limit_uni() { let mut buf = [0; 65535]; - let mut pipe = testing::Pipe::new().unwrap(); + let mut pipe = ::new().unwrap(); assert_eq!(pipe.handshake(), Ok(())); let frames = [frame::Frame::Stream { stream_id: 2, - data: stream::RangeBuf::from(b"aaaaaaaaaaa", 0, true), + data: ::from(b"aaaaaaaaaaa", 0, true), }]; let pkt_type = packet::Type::Short; @@ -11869,12 +12287,12 @@ mod tests { fn stream_flow_control_update() { let mut buf = [0; 65535]; - let mut pipe = testing::Pipe::new().unwrap(); + let mut pipe = ::new().unwrap(); assert_eq!(pipe.handshake(), Ok(())); let frames = [frame::Frame::Stream { stream_id: 4, - data: stream::RangeBuf::from(b"aaaaaaaaa", 0, false), + data: ::from(b"aaaaaaaaa", 0, false), }]; let pkt_type = packet::Type::Short; @@ -11897,7 +12315,7 @@ mod tests { let frames = [frame::Frame::Stream { stream_id: 4, - data: stream::RangeBuf::from(b"a", 9, false), + data: ::from(b"a", 9, false), }]; let len = pipe @@ -11934,7 +12352,7 @@ mod tests { fn stream_left_bidi() { let mut buf = [0; 65535]; - let mut pipe = testing::Pipe::new().unwrap(); + let mut pipe = ::new().unwrap(); assert_eq!(pipe.handshake(), Ok(())); assert_eq!(3, pipe.client.peer_streams_left_bidi()); @@ -11962,7 +12380,7 @@ mod tests { fn stream_left_uni() { let mut buf = [0; 65535]; - let mut pipe = testing::Pipe::new().unwrap(); + let mut pipe = ::new().unwrap(); assert_eq!(pipe.handshake(), Ok(())); assert_eq!(3, pipe.client.peer_streams_left_uni()); @@ -11990,37 +12408,37 @@ mod tests { fn stream_limit_bidi() { let mut buf = [0; 65535]; - let mut pipe = testing::Pipe::new().unwrap(); + let mut pipe = ::new().unwrap(); assert_eq!(pipe.handshake(), Ok(())); let frames = [ frame::Frame::Stream { stream_id: 4, - data: stream::RangeBuf::from(b"a", 0, false), + data: ::from(b"a", 0, false), }, frame::Frame::Stream { stream_id: 8, - data: stream::RangeBuf::from(b"a", 0, false), + data: ::from(b"a", 0, false), }, frame::Frame::Stream { stream_id: 12, - data: stream::RangeBuf::from(b"a", 0, false), + data: ::from(b"a", 0, false), }, frame::Frame::Stream { stream_id: 16, - data: stream::RangeBuf::from(b"a", 0, false), + data: ::from(b"a", 0, false), }, frame::Frame::Stream { stream_id: 20, - data: stream::RangeBuf::from(b"a", 0, false), + data: ::from(b"a", 0, false), }, frame::Frame::Stream { stream_id: 24, - data: stream::RangeBuf::from(b"a", 0, false), + data: ::from(b"a", 0, false), }, frame::Frame::Stream { stream_id: 28, - data: stream::RangeBuf::from(b"a", 0, false), + data: ::from(b"a", 0, false), }, ]; @@ -12035,7 +12453,7 @@ mod tests { fn stream_limit_max_bidi() { let mut buf = [0; 65535]; - let mut pipe = testing::Pipe::new().unwrap(); + let mut pipe = ::new().unwrap(); assert_eq!(pipe.handshake(), Ok(())); let frames = [frame::Frame::MaxStreamsBidi { max: MAX_STREAM_ID }]; @@ -12060,37 +12478,37 @@ mod tests { fn stream_limit_uni() { let mut buf = [0; 65535]; - let mut pipe = testing::Pipe::new().unwrap(); + let mut pipe = ::new().unwrap(); assert_eq!(pipe.handshake(), Ok(())); let frames = [ frame::Frame::Stream { stream_id: 2, - data: stream::RangeBuf::from(b"a", 0, false), + data: ::from(b"a", 0, false), }, frame::Frame::Stream { stream_id: 6, - data: stream::RangeBuf::from(b"a", 0, false), + data: ::from(b"a", 0, false), }, frame::Frame::Stream { stream_id: 10, - data: stream::RangeBuf::from(b"a", 0, false), + data: ::from(b"a", 0, false), }, frame::Frame::Stream { stream_id: 14, - data: stream::RangeBuf::from(b"a", 0, false), + data: ::from(b"a", 0, false), }, frame::Frame::Stream { stream_id: 18, - data: stream::RangeBuf::from(b"a", 0, false), + data: ::from(b"a", 0, false), }, frame::Frame::Stream { stream_id: 22, - data: stream::RangeBuf::from(b"a", 0, false), + data: ::from(b"a", 0, false), }, frame::Frame::Stream { stream_id: 26, - data: stream::RangeBuf::from(b"a", 0, false), + data: ::from(b"a", 0, false), }, ]; @@ -12105,7 +12523,7 @@ mod tests { fn stream_limit_max_uni() { let mut buf = [0; 65535]; - let mut pipe = testing::Pipe::new().unwrap(); + let mut pipe = ::new().unwrap(); assert_eq!(pipe.handshake(), Ok(())); let frames = [frame::Frame::MaxStreamsUni { max: MAX_STREAM_ID }]; @@ -12130,7 +12548,7 @@ mod tests { fn stream_left_reset_bidi() { let mut buf = [0; 65535]; - let mut pipe = testing::Pipe::new().unwrap(); + let mut pipe = ::new().unwrap(); assert_eq!(pipe.handshake(), Ok(())); assert_eq!(3, pipe.client.peer_streams_left_bidi()); @@ -12239,7 +12657,7 @@ mod tests { #[test] fn stream_reset_counts() { - let mut pipe = testing::Pipe::new().unwrap(); + let mut pipe = ::new().unwrap(); assert_eq!(pipe.handshake(), Ok(())); let off_by = if crate::PROTOCOL_VERSION == crate::PROTOCOL_VERSION_VREVERSO { @@ -12313,7 +12731,7 @@ mod tests { #[test] fn stream_stop_counts() { - let mut pipe = testing::Pipe::new().unwrap(); + let mut pipe = ::new().unwrap(); assert_eq!(pipe.handshake(), Ok(())); let off_by = if crate::PROTOCOL_VERSION == crate::PROTOCOL_VERSION_VREVERSO { @@ -12378,7 +12796,7 @@ mod tests { fn streams_blocked_max_bidi() { let mut buf = [0; 65535]; - let mut pipe = testing::Pipe::new().unwrap(); + let mut pipe = ::new().unwrap(); assert_eq!(pipe.handshake(), Ok(())); let frames = [frame::Frame::StreamsBlockedBidi { @@ -12405,7 +12823,7 @@ mod tests { fn streams_blocked_max_uni() { let mut buf = [0; 65535]; - let mut pipe = testing::Pipe::new().unwrap(); + let mut pipe = ::new().unwrap(); assert_eq!(pipe.handshake(), Ok(())); let frames = [frame::Frame::StreamsBlockedUni { @@ -12428,25 +12846,195 @@ mod tests { ); } + #[test] + fn stream_data_with_hidden_copy_using_stream_send() { + let mut config = Config::new(crate::PROTOCOL_VERSION).unwrap(); + + config + .load_cert_chain_from_pem_file("examples/cert.crt") + .unwrap(); + config + .load_priv_key_from_pem_file("examples/cert.key") + .unwrap(); + config + .set_application_protos(&[b"proto1", b"proto2"]) + .unwrap(); + config.set_initial_max_data(30); + config.set_initial_max_stream_data_bidi_local(30); + config.set_initial_max_stream_data_bidi_remote(30); + config.set_initial_max_streams_bidi(3); + config.enable_early_data(); + config.enable_hidden_copy_for_zc_sender(true); + config.verify_peer(false); + + // Perform initial handshake. + let mut pipe = ::with_config(&mut config).unwrap(); + assert_eq!(pipe.handshake(), Ok(())); + + assert_eq!(pipe.client.stream_send(4, b"boup", true), Ok(4)); + assert_eq!(pipe.advance(), Ok(())); + + assert_eq!(pipe.client.stream_send(8, b"hello, world", false), Ok(12)); + assert_eq!(pipe.client.stream_send(8, b"ciao, world", true), Ok(11)); + + let mut buf = [0; 128]; + + // Sender would send chunks of 5 bytes, with no copy accross the two buffers. + // So we expect 6 packets. + for _ in 0..6 { + let (len, _) = pipe.client.send(&mut buf).unwrap(); + assert_eq!(pipe.server_recv(&mut buf[..len]), Ok(len)); + } + + if crate::PROTOCOL_VERSION == crate::PROTOCOL_VERSION_VREVERSO { + let (b, len, fin) = pipe + .server + .stream_recv_v3(8, &mut pipe.server_app_buffers) + .unwrap(); + assert_eq!(b, b"hello, worldciao, world"); + assert_eq!((len, fin), (23, true)); + assert!(pipe + .server + .stream_consumed(8, len, &mut pipe.server_app_buffers) + .is_ok()); + } else { + assert_eq!(pipe.server.stream_recv(8, &mut buf), Ok((23, true))); + } + + } + + + #[test] + fn stream_data_with_hidden_copy_using_mixed_stream_send_and_zc() { + let mut config = Config::new(crate::PROTOCOL_VERSION).unwrap(); + + config + .load_cert_chain_from_pem_file("examples/cert.crt") + .unwrap(); + config + .load_priv_key_from_pem_file("examples/cert.key") + .unwrap(); + config + .set_application_protos(&[b"proto1", b"proto2"]) + .unwrap(); + config.set_initial_max_data(30); + config.set_initial_max_stream_data_bidi_local(30); + config.set_initial_max_stream_data_bidi_remote(30); + config.set_initial_max_streams_bidi(3); + config.enable_early_data(); + config.enable_hidden_copy_for_zc_sender(true); + config.verify_peer(false); + + let mut pipe = Pipe::::with_config(&mut config).unwrap(); + assert_eq!(pipe.handshake(), Ok(())); + + // In testing, stream_send cuts every 5 chars. + assert_eq!(pipe.client.stream_send(8, b"hello, world", false), Ok(12)); + assert_eq!(pipe.client.stream_send_zc(8, BufTestFactory::buf_from_slice(b"ciao, world"), Some(11), true), Ok((11, None))); + + let mut buf = [0; 256]; + for _ in 0..3 { + let (len, _) = pipe.client.send(&mut buf).unwrap(); + assert_eq!(pipe.server_recv(&mut buf[..len]), Ok(len)); + } + // Getting the information from stream_send_zc + let (len, _) = pipe.client.send(&mut buf).unwrap(); + assert_eq!(pipe.server_recv(&mut buf[..len]), Ok(len)); + + if crate::PROTOCOL_VERSION == crate::PROTOCOL_VERSION_VREVERSO { + let (b, len, fin) = pipe + .server + .stream_recv_v3(8, &mut pipe.server_app_buffers) + .unwrap(); + assert_eq!(b, b"hello, worldciao, world"); + assert_eq!((len, fin), (23, true)); + assert!(pipe + .server + .stream_consumed(8, len, &mut pipe.server_app_buffers) + .is_ok()); + } else { + assert_eq!(pipe.server.stream_recv(8, &mut buf), Ok((23, true))); + } + } + + #[test] + fn stream_data_with_hidden_copy_using_stream_send_zc() { + + let mut config = Config::new(crate::PROTOCOL_VERSION).unwrap(); + + config + .load_cert_chain_from_pem_file("examples/cert.crt") + .unwrap(); + config + .load_priv_key_from_pem_file("examples/cert.key") + .unwrap(); + config + .set_application_protos(&[b"proto1", b"proto2"]) + .unwrap(); + config.set_initial_max_data(30); + config.set_initial_max_stream_data_bidi_local(30); + config.set_initial_max_stream_data_bidi_remote(30); + config.set_initial_max_streams_bidi(3); + config.enable_early_data(); + config.enable_hidden_copy_for_zc_sender(true); + config.verify_peer(false); + + // Perform initial handshake. + let mut pipe = Pipe::::with_config(&mut config).unwrap(); + assert_eq!(pipe.handshake(), Ok(())); + + assert_eq!(pipe.client.stream_send_zc(4, BufTestFactory::buf_from_slice(b"boup"), Some(4), true), Ok((4, None))); + assert_eq!(pipe.advance(), Ok(())); + + assert_eq!(pipe.client.stream_send_zc(8, BufTestFactory::buf_from_slice(b"hello, world"), Some(12), false), Ok((12, None))); + assert_eq!(pipe.client.stream_send_zc(8, BufTestFactory::buf_from_slice(b"ciao, world"), Some(11), true), Ok((11, None))); + + let mut buf = [0; 256]; + // Sender would send 1 packet for each buffer since we enabled usage of the + // encryption's hidden copy for packet assembly. + let (len1, _) = pipe.client.send(&mut buf).unwrap(); + let (len2, _) = pipe.client.send(&mut buf[len1..]).unwrap(); + + assert_eq!(pipe.server_recv(&mut buf[..len1]), Ok(len1)); + assert_eq!(pipe.server_recv(&mut buf[len1..len1+len2]), Ok(len2)); + if crate::PROTOCOL_VERSION == crate::PROTOCOL_VERSION_VREVERSO { + assert_eq!((len1, len2), (53, 52)); + let (b, len, fin) = pipe + .server + .stream_recv_v3(8, &mut pipe.server_app_buffers) + .unwrap(); + assert_eq!(b, b"hello, worldciao, world"); + assert_eq!((len, fin), (23, true)); + assert!(pipe + .server + .stream_consumed(8, len, &mut pipe.server_app_buffers) + .is_ok()); + } else { + assert_eq!((len1, len2), (51, 50)); + assert_eq!(pipe.server.stream_recv(8, &mut buf), Ok((23, true))); + } + + } + #[test] fn stream_data_overlap() { let mut buf = [0; 65535]; - let mut pipe = testing::Pipe::new().unwrap(); + let mut pipe = ::new().unwrap(); assert_eq!(pipe.handshake(), Ok(())); let frames = [ frame::Frame::Stream { stream_id: 4, - data: stream::RangeBuf::from(b"aaaaa", 0, false), + data: ::from(b"aaaaa", 0, false), }, frame::Frame::Stream { stream_id: 4, - data: stream::RangeBuf::from(b"bbbbb", 3, false), + data: ::from(b"bbbbb", 3, false), }, frame::Frame::Stream { stream_id: 4, - data: stream::RangeBuf::from(b"ccccc", 6, false), + data: ::from(b"ccccc", 6, false), }, ]; @@ -12506,7 +13094,7 @@ mod tests { .is_ok()); let frames = [frame::Frame::Stream { stream_id: 4, - data: stream::RangeBuf::from(b"b", 5, false), + data: ::from(b"b", 5, false), }]; // we re-send the wrong frame but with the correct offset :) assert!(pipe @@ -12532,21 +13120,21 @@ mod tests { fn stream_data_overlap_with_reordering() { let mut buf = [0; 65535]; - let mut pipe = testing::Pipe::new().unwrap(); + let mut pipe = ::new().unwrap(); assert_eq!(pipe.handshake(), Ok(())); let frames = [ frame::Frame::Stream { stream_id: 4, - data: stream::RangeBuf::from(b"aaaaa", 0, false), + data: ::from(b"aaaaa", 0, false), }, frame::Frame::Stream { stream_id: 4, - data: stream::RangeBuf::from(b"ccccc", 6, false), + data: ::from(b"ccccc", 6, false), }, frame::Frame::Stream { stream_id: 4, - data: stream::RangeBuf::from(b"bbbbb", 3, false), + data: ::from(b"bbbbb", 3, false), }, ]; @@ -12610,7 +13198,7 @@ mod tests { let mut b = [0; 15]; let mut buf = [0; 65535]; - let mut pipe = testing::Pipe::new().unwrap(); + let mut pipe = ::new().unwrap(); assert_eq!(pipe.handshake(), Ok(())); // Client sends some data. @@ -12716,7 +13304,7 @@ mod tests { let mut b = [0; 15]; let mut buf = [0; 65535]; - let mut pipe = testing::Pipe::new().unwrap(); + let mut pipe = ::new().unwrap(); assert_eq!(pipe.handshake(), Ok(())); // Client sends some data. @@ -12831,17 +13419,17 @@ mod tests { fn reset_stream_flow_control() { let mut buf = [0; 65535]; - let mut pipe = testing::Pipe::new().unwrap(); + let mut pipe = ::new().unwrap(); assert_eq!(pipe.handshake(), Ok(())); let frames = [ frame::Frame::Stream { stream_id: 0, - data: stream::RangeBuf::from(b"aaaaaaaaaaaaaaa", 0, false), + data: ::from(b"aaaaaaaaaaaaaaa", 0, false), }, frame::Frame::Stream { stream_id: 4, - data: stream::RangeBuf::from(b"a", 0, false), + data: ::from(b"a", 0, false), }, frame::Frame::ResetStream { stream_id: 4, @@ -12850,7 +13438,7 @@ mod tests { }, frame::Frame::Stream { stream_id: 8, - data: stream::RangeBuf::from(b"a", 0, false), + data: ::from(b"a", 0, false), }, ]; @@ -12867,13 +13455,13 @@ mod tests { fn reset_stream_flow_control_stream() { let mut buf = [0; 65535]; - let mut pipe = testing::Pipe::new().unwrap(); + let mut pipe = ::new().unwrap(); assert_eq!(pipe.handshake(), Ok(())); let frames = [ frame::Frame::Stream { stream_id: 4, - data: stream::RangeBuf::from(b"a", 0, false), + data: ::from(b"a", 0, false), }, frame::Frame::ResetStream { stream_id: 4, @@ -12893,7 +13481,7 @@ mod tests { fn path_challenge() { let mut buf = [0; 65535]; - let mut pipe = testing::Pipe::new().unwrap(); + let mut pipe = ::new().unwrap(); assert_eq!(pipe.handshake(), Ok(())); let frames = [frame::Frame::PathChallenge { data: [0xba; 8] }]; @@ -12936,7 +13524,7 @@ mod tests { fn early_1rtt_packet() { let mut buf = [0; 65535]; - let mut pipe = testing::Pipe::new().unwrap(); + let mut pipe = ::new().unwrap(); // Client sends initial flight let flight = testing::emit_flight(&mut pipe.client).unwrap(); @@ -12970,7 +13558,7 @@ mod tests { // Send 1-RTT packet #0. let frames = [frame::Frame::Stream { stream_id: 0, - data: stream::RangeBuf::from(b"hello, world", 0, true), + data: ::from(b"hello, world", 0, true), }]; let pkt_type = packet::Type::Short; @@ -12983,7 +13571,7 @@ mod tests { // Send 1-RTT packet #1. let frames = [frame::Frame::Stream { stream_id: 4, - data: stream::RangeBuf::from(b"hello, world", 0, true), + data: ::from(b"hello, world", 0, true), }]; let written = @@ -13027,7 +13615,7 @@ mod tests { let mut buf = [0; 65535]; - let mut pipe = testing::Pipe::new().unwrap(); + let mut pipe = ::new().unwrap(); assert_eq!(pipe.handshake(), Ok(())); // Client sends some data, and closes stream. @@ -13182,7 +13770,7 @@ mod tests { let mut buf = [0; 65535]; - let mut pipe = testing::Pipe::new().unwrap(); + let mut pipe = ::new().unwrap(); assert_eq!(pipe.handshake(), Ok(())); // Client sends some data, and closes stream. @@ -13286,7 +13874,7 @@ mod tests { config.set_initial_max_streams_uni(0); config.verify_peer(false); - let mut pipe = testing::Pipe::with_config(&mut config).unwrap(); + let mut pipe = ::with_config(&mut config).unwrap(); assert_eq!(pipe.handshake(), Ok(())); // Client sends some data. -- Note; changed to helloworld for avoiding @@ -13356,7 +13944,7 @@ mod tests { fn stream_shutdown_read() { let mut buf = [0; 65535]; - let mut pipe = testing::Pipe::new().unwrap(); + let mut pipe = ::new().unwrap(); assert_eq!(pipe.handshake(), Ok(())); // Client sends some data. @@ -13451,7 +14039,7 @@ mod tests { fn stream_shutdown_read_after_fin() { let mut buf = [0; 65535]; - let mut pipe = testing::Pipe::new().unwrap(); + let mut pipe = ::new().unwrap(); assert_eq!(pipe.handshake(), Ok(())); // Client sends some data. @@ -13529,7 +14117,7 @@ mod tests { config.set_initial_max_streams_bidi(10); config.verify_peer(false); - let mut pipe = testing::Pipe::with_config(&mut config).unwrap(); + let mut pipe = ::with_config(&mut config).unwrap(); assert_eq!(pipe.handshake(), Ok(())); assert_eq!(pipe.client.stream_send(4, b"a", false), Ok(1)); @@ -13571,7 +14159,7 @@ mod tests { #[test] fn stream_shutdown_uni() { - let mut pipe = testing::Pipe::new().unwrap(); + let mut pipe = ::new().unwrap(); assert_eq!(pipe.handshake(), Ok(())); // Exchange some data on uni streams. @@ -13597,7 +14185,7 @@ mod tests { fn stream_shutdown_write() { let mut buf = [0; 65535]; - let mut pipe = testing::Pipe::new().unwrap(); + let mut pipe = ::new().unwrap(); assert_eq!(pipe.handshake(), Ok(())); // Client sends some data. @@ -13732,7 +14320,7 @@ mod tests { config.set_initial_max_streams_uni(0); config.verify_peer(false); - let mut pipe = testing::Pipe::with_config(&mut config).unwrap(); + let mut pipe = ::with_config(&mut config).unwrap(); assert_eq!(pipe.handshake(), Ok(())); // Client sends some data. @@ -13795,7 +14383,7 @@ mod tests { fn stream_round_robin() { let mut buf = [0; 65535]; - let mut pipe = testing::Pipe::new().unwrap(); + let mut pipe = ::new().unwrap(); assert_eq!(pipe.handshake(), Ok(())); let off_by = @@ -13846,7 +14434,7 @@ mod tests { iter.next(), Some(&frame::Frame::Stream { stream_id: 8, - data: stream::RangeBuf::from(b"aaaaaaa", 0, false), + data: ::from(b"aaaaaaa", 0, false), }) ); } @@ -13873,7 +14461,7 @@ mod tests { frames.first(), Some(&frame::Frame::Stream { stream_id: 0, - data: stream::RangeBuf::from(b"aaaaaaa", 0, false), + data: ::from(b"aaaaaaa", 0, false), }) ); } @@ -13900,7 +14488,7 @@ mod tests { frames.first(), Some(&frame::Frame::Stream { stream_id: 4, - data: stream::RangeBuf::from(b"aaaaaaa", 0, false), + data: ::from(b"aaaaaaa", 0, false), }) ); } @@ -13909,7 +14497,7 @@ mod tests { #[test] /// Tests the readable iterator. fn stream_readable() { - let mut pipe = testing::Pipe::new().unwrap(); + let mut pipe = ::new().unwrap(); assert_eq!(pipe.handshake(), Ok(())); // No readable streams. @@ -13999,7 +14587,7 @@ mod tests { #[test] /// Tests the writable iterator. fn stream_writable() { - let mut pipe = testing::Pipe::new().unwrap(); + let mut pipe = ::new().unwrap(); assert_eq!(pipe.handshake(), Ok(())); // No writable streams. @@ -14110,7 +14698,7 @@ mod tests { config.set_initial_max_streams_uni(5); config.verify_peer(false); - let mut pipe = testing::Pipe::with_config(&mut config).unwrap(); + let mut pipe = ::with_config(&mut config).unwrap(); assert_eq!(pipe.handshake(), Ok(())); // Client creates stream and sends some data. @@ -14159,7 +14747,7 @@ mod tests { /// Tests that we don't exceed the per-connection flow control limit set by /// the peer. fn flow_control_limit_send() { - let mut pipe = testing::Pipe::new().unwrap(); + let mut pipe = ::new().unwrap(); assert_eq!(pipe.handshake(), Ok(())); assert_eq!( @@ -14186,7 +14774,7 @@ mod tests { /// the server to close the connection immediately. fn invalid_initial_server() { let mut buf = [0; 65535]; - let mut pipe = testing::Pipe::new().unwrap(); + let mut pipe = ::new().unwrap(); let frames = [frame::Frame::Padding { len: 10 }]; @@ -14218,7 +14806,7 @@ mod tests { /// the client to close the connection immediately. fn invalid_initial_client() { let mut buf = [0; 65535]; - let mut pipe = testing::Pipe::new().unwrap(); + let mut pipe = ::new().unwrap(); // Client sends initial flight. let (len, _) = pipe.client.send(&mut buf).unwrap(); @@ -14262,7 +14850,7 @@ mod tests { /// valid packet cause the server to close the connection immediately. fn invalid_initial_payload() { let mut buf = [0; 65535]; - let mut pipe = testing::Pipe::new().unwrap(); + let mut pipe = ::new().unwrap(); let mut b = octets_rev::OctetsMut::with_slice(&mut buf); @@ -14323,16 +14911,20 @@ mod tests { let written = packet::encrypt_pkt( &mut b, + None, pn, - pn_len, payload_len, payload_offset, None, aead, - pipe.client.version, + false, ) .unwrap(); + let (mut header, payload) = b.split_at(payload_offset).unwrap(); + packet::encrypt_hdr(&mut header, pn_len, payload.as_ref(), &aead, crate::PROTOCOL_VERSION) + .expect("header encrypt"); + assert_eq!(pipe.server.timeout(), None); assert_eq!( @@ -14348,7 +14940,7 @@ mod tests { fn invalid_packet() { let mut buf = [0; 65535]; - let mut pipe = testing::Pipe::new().unwrap(); + let mut pipe = ::new().unwrap(); assert_eq!(pipe.handshake(), Ok(())); let frames = [frame::Frame::Padding { len: 10 }]; @@ -14378,7 +14970,7 @@ mod tests { fn recv_empty_buffer() { let mut buf = [0; 65535]; - let mut pipe = testing::Pipe::new().unwrap(); + let mut pipe = ::new().unwrap(); assert_eq!(pipe.handshake(), Ok(())); assert_eq!(pipe.server_recv(&mut buf[..0]), Err(Error::BufferTooShort)); @@ -14390,7 +14982,7 @@ mod tests { let mut buf = [0; 65535]; - let mut pipe = testing::Pipe::new().unwrap(); + let mut pipe = ::new().unwrap(); assert_eq!(pipe.handshake(), Ok(())); let off_by = @@ -14523,7 +15115,7 @@ mod tests { config.set_initial_max_streams_bidi(3); config.verify_peer(false); - let mut pipe = testing::Pipe::with_config(&mut config).unwrap(); + let mut pipe = ::with_config(&mut config).unwrap(); assert_eq!(pipe.handshake(), Ok(())); // Client sends some data, and closes stream. @@ -14615,7 +15207,7 @@ mod tests { 0 }; - let mut pipe = testing::Pipe::with_config(&mut config).unwrap(); + let mut pipe = ::with_config(&mut config).unwrap(); assert_eq!(pipe.handshake(), Ok(())); // Client sends stream data. @@ -14709,7 +15301,7 @@ mod tests { config.set_initial_max_streams_uni(3); config.verify_peer(false); - let mut pipe = testing::Pipe::with_config(&mut config).unwrap(); + let mut pipe = ::with_config(&mut config).unwrap(); assert_eq!(pipe.handshake(), Ok(())); // Client sends stream data. @@ -14773,7 +15365,7 @@ mod tests { /// data in the buffer, and that the buffer becomes readable on the other /// side. fn stream_zero_length_fin() { - let mut pipe = testing::Pipe::new().unwrap(); + let mut pipe = ::new().unwrap(); assert_eq!(pipe.handshake(), Ok(())); assert_eq!( @@ -14838,7 +15430,7 @@ mod tests { /// data in the buffer, that the buffer becomes readable on the other /// side and stays readable even if the stream is fin'd locally. fn stream_zero_length_fin_deferred_collection() { - let mut pipe = testing::Pipe::new().unwrap(); + let mut pipe = ::new().unwrap(); assert_eq!(pipe.handshake(), Ok(())); assert_eq!( @@ -14926,7 +15518,7 @@ mod tests { /// Tests that the stream gets created with stream_send() even if there's /// no data in the buffer and the fin flag is not set. fn stream_zero_length_non_fin() { - let mut pipe = testing::Pipe::new().unwrap(); + let mut pipe = ::new().unwrap(); assert_eq!(pipe.handshake(), Ok(())); assert_eq!(pipe.client.stream_send(4, b"", false), Ok(0)); @@ -14946,7 +15538,7 @@ mod tests { fn collect_streams() { let mut buf = [0; 65535]; - let mut pipe = testing::Pipe::new().unwrap(); + let mut pipe = ::new().unwrap(); assert_eq!(pipe.handshake(), Ok(())); let mut stream_id = 0; if crate::PROTOCOL_VERSION == crate::PROTOCOL_VERSION_VREVERSO { @@ -15016,7 +15608,7 @@ mod tests { let frames = [frame::Frame::Stream { stream_id, - data: stream::RangeBuf::from(b"aa", 0, false), + data: ::from(b"aa", 0, false), }]; let pkt_type = packet::Type::Short; @@ -15055,7 +15647,7 @@ mod tests { #[test] fn peer_cert() { - let mut pipe = testing::Pipe::new().unwrap(); + let mut pipe = ::new().unwrap(); assert_eq!(pipe.handshake(), Ok(())); match pipe.client.peer_cert() { @@ -15078,7 +15670,7 @@ mod tests { .set_application_protos(&[b"proto1", b"proto2"]) .unwrap(); - let mut pipe = testing::Pipe::with_server_config(&mut config).unwrap(); + let mut pipe = ::with_server_config(&mut config).unwrap(); assert_eq!(pipe.handshake(), Ok(())); match pipe.client.peer_cert_chain() { @@ -15103,7 +15695,7 @@ mod tests { .set_application_protos(&[b"proto1", b"proto2"]) .unwrap(); - let mut pipe = testing::Pipe::with_server_config(&mut config).unwrap(); + let mut pipe = ::with_server_config(&mut config).unwrap(); // Client sends initial flight. let (mut len, _) = pipe.client.send(&mut buf).unwrap(); @@ -15142,7 +15734,7 @@ mod tests { pipe.server = accept( &scid, Some(&odcid), - testing::Pipe::server_addr(), + ::server_addr(), from, &mut config, ) @@ -15170,7 +15762,7 @@ mod tests { .set_application_protos(&[b"proto1", b"proto2"]) .unwrap(); - let mut pipe = testing::Pipe::with_server_config(&mut config).unwrap(); + let mut pipe = ::with_server_config(&mut config).unwrap(); // Client sends initial flight. let (mut len, _) = pipe.client.send(&mut buf).unwrap(); @@ -15202,9 +15794,14 @@ mod tests { // Server accepts connection and send first flight. But original // destination connection ID is ignored. let from = "127.0.0.1:1234".parse().unwrap(); - pipe.server = - accept(&scid, None, testing::Pipe::server_addr(), from, &mut config) - .unwrap(); + pipe.server = accept( + &scid, + None, + ::server_addr(), + from, + &mut config, + ) + .unwrap(); assert_eq!(pipe.server_recv(&mut buf[..len]), Ok(len)); let flight = testing::emit_flight(&mut pipe.server).unwrap(); @@ -15234,7 +15831,7 @@ mod tests { .set_application_protos(&[b"proto1", b"proto2"]) .unwrap(); - let mut pipe = testing::Pipe::with_server_config(&mut config).unwrap(); + let mut pipe = ::with_server_config(&mut config).unwrap(); // Client sends initial flight. let (mut len, _) = pipe.client.send(&mut buf).unwrap(); @@ -15270,7 +15867,7 @@ mod tests { pipe.server = accept( &scid, Some(&odcid), - testing::Pipe::server_addr(), + ::server_addr(), from, &mut config, ) @@ -15294,7 +15891,7 @@ mod tests { fn zero_length_new_token() { let mut buf = [0; 65535]; - let mut pipe = testing::Pipe::new().unwrap(); + let mut pipe = ::new().unwrap(); assert_eq!(pipe.handshake(), Ok(())); let frames = vec![frame::Frame::NewToken { token: vec![] }]; @@ -15316,7 +15913,7 @@ mod tests { fn client_sent_new_token() { let mut buf = [0; 65535]; - let mut pipe = testing::Pipe::new().unwrap(); + let mut pipe = ::new().unwrap(); assert_eq!(pipe.handshake(), Ok(())); let frames = vec![frame::Frame::NewToken { @@ -15345,7 +15942,7 @@ mod tests { #[test] fn connection_must_be_send() { - let mut pipe = testing::Pipe::new().unwrap(); + let mut pipe = ::new().unwrap(); check_send(&mut pipe.client); } @@ -15359,7 +15956,7 @@ mod tests { #[test] fn connection_must_be_sync() { - let mut pipe = testing::Pipe::new().unwrap(); + let mut pipe = ::new().unwrap(); check_sync(&mut pipe.client); } @@ -15367,7 +15964,7 @@ mod tests { fn data_blocked() { let mut buf = [0; 65535]; - let mut pipe = testing::Pipe::new().unwrap(); + let mut pipe = ::new().unwrap(); assert_eq!(pipe.handshake(), Ok(())); assert_eq!(pipe.client.stream_send(0, b"aaaaaaaaaa", false), Ok(10)); @@ -15412,11 +16009,10 @@ mod tests { iter.next(), Some(&frame::Frame::Stream { stream_id: 8, - data: stream::RangeBuf::from(b"aaaaaaaaaa", 0, false), + data: ::from(b"aaaaaaaaaa", 0, false), }) ); } - assert_eq!(iter.next(), None); } @@ -15424,7 +16020,7 @@ mod tests { fn stream_data_blocked() { let mut buf = [0; 65535]; - let mut pipe = testing::Pipe::new().unwrap(); + let mut pipe = ::new().unwrap(); assert_eq!(pipe.handshake(), Ok(())); assert_eq!(pipe.client.stream_send(4, b"aaaaa", false), Ok(5)); @@ -15488,7 +16084,7 @@ mod tests { iter.next(), Some(&frame::Frame::Stream { stream_id: 4, - data: stream::RangeBuf::from(b"aaaaaaaaaaaaaaa", 0, false), + data: ::from(b"aaaaaaaaaaaaaaa", 0, false), }) ); } @@ -15529,7 +16125,7 @@ mod tests { iter.next(), Some(&frame::Frame::Stream { stream_id: 8, - data: stream::RangeBuf::from(b"a", 0, false), + data: ::from(b"a", 0, false), }) ); } @@ -15549,7 +16145,7 @@ mod tests { #[test] fn stream_data_blocked_unblocked_flow_control() { let mut buf = [0; 65535]; - let mut pipe = testing::Pipe::new().unwrap(); + let mut pipe = ::new().unwrap(); assert_eq!(pipe.handshake(), Ok(())); assert_eq!( @@ -15667,7 +16263,7 @@ mod tests { stream_id = 4; } - let mut pipe = testing::Pipe::with_client_config(&mut config).unwrap(); + let mut pipe = ::with_client_config(&mut config).unwrap(); assert_eq!(pipe.handshake(), Ok(())); // Client sends stream data. @@ -15719,7 +16315,7 @@ mod tests { config.set_max_recv_udp_payload_size(1200); config.verify_peer(false); - let mut pipe = testing::Pipe::with_client_config(&mut config).unwrap(); + let mut pipe = ::with_client_config(&mut config).unwrap(); assert_eq!(pipe.handshake(), Ok(())); // Client sends stream data. @@ -15774,7 +16370,7 @@ mod tests { config.set_max_recv_udp_payload_size(1200); config.verify_peer(false); - let mut pipe = testing::Pipe::with_config(&mut config).unwrap(); + let mut pipe = ::with_config(&mut config).unwrap(); assert_eq!(pipe.handshake(), Ok(())); // Client sends stream data bigger than cwnd (it will never arrive to the @@ -15853,7 +16449,7 @@ mod tests { config.set_max_recv_udp_payload_size(1200); config.verify_peer(false); - let mut pipe = testing::Pipe::with_config(&mut config).unwrap(); + let mut pipe = ::with_config(&mut config).unwrap(); assert_eq!(pipe.handshake(), Ok(())); // Client sends stream data bigger than cwnd (it will never arrive to the @@ -15936,7 +16532,7 @@ mod tests { stream_id = 4; } - let mut pipe = testing::Pipe::with_client_config(&mut config).unwrap(); + let mut pipe = ::with_client_config(&mut config).unwrap(); assert_eq!(pipe.handshake(), Ok(())); // Client sends stream data. @@ -15995,7 +16591,7 @@ mod tests { stream_id = 4; } - let mut pipe = testing::Pipe::with_client_config(&mut config).unwrap(); + let mut pipe = ::with_client_config(&mut config).unwrap(); assert_eq!(pipe.handshake(), Ok(())); // Client sends stream data. @@ -16053,7 +16649,7 @@ mod tests { stream_id = 4; } - let mut pipe = testing::Pipe::with_client_config(&mut config).unwrap(); + let mut pipe = ::with_client_config(&mut config).unwrap(); assert_eq!(pipe.handshake(), Ok(())); // Client sends stream data. @@ -16098,7 +16694,7 @@ mod tests { fn limit_ack_ranges() { let mut buf = [0; 65535]; - let mut pipe = testing::Pipe::new().unwrap(); + let mut pipe = ::new().unwrap(); assert_eq!(pipe.handshake(), Ok(())); let epoch = packet::Epoch::Application; @@ -16177,7 +16773,7 @@ mod tests { config.set_initial_max_streams_uni(0); config.verify_peer(false); - let mut pipe = testing::Pipe::with_config(&mut config).unwrap(); + let mut pipe = ::with_config(&mut config).unwrap(); assert_eq!(pipe.handshake(), Ok(())); let off_by = @@ -16334,7 +16930,7 @@ mod tests { } else { assert_eq!(stream, &frame::Frame::Stream { stream_id: 8, - data: stream::RangeBuf::from(&out, off, false), + data: ::from(&out, off, false), }); } @@ -16369,7 +16965,7 @@ mod tests { } else { assert_eq!(stream, &frame::Frame::Stream { stream_id: 16, - data: stream::RangeBuf::from(&out, off, false), + data: ::from(&out, off, false), }); } @@ -16404,7 +17000,7 @@ mod tests { } else { assert_eq!(stream, &frame::Frame::Stream { stream_id: 20, - data: stream::RangeBuf::from(&out, off, false), + data: ::from(&out, off, false), }); } @@ -16447,7 +17043,7 @@ mod tests { frames.first(), Some(&frame::Frame::Stream { stream_id: 12, - data: stream::RangeBuf::from(&out, off, false), + data: ::from(&out, off, false), }) ); } @@ -16472,7 +17068,7 @@ mod tests { } else { assert_eq!(stream, &frame::Frame::Stream { stream_id: 4, - data: stream::RangeBuf::from(&out, off, false), + data: ::from(&out, off, false), }); } @@ -16507,7 +17103,7 @@ mod tests { } else { assert_eq!(stream, &frame::Frame::Stream { stream_id: 0, - data: stream::RangeBuf::from(&out, off, false), + data: ::from(&out, off, false), }); } @@ -16545,7 +17141,7 @@ mod tests { config.set_initial_max_streams_uni(0); config.verify_peer(false); - let mut pipe = testing::Pipe::with_config(&mut config).unwrap(); + let mut pipe = ::with_config(&mut config).unwrap(); assert_eq!(pipe.handshake(), Ok(())); assert_eq!(pipe.client.stream_send(4, b"a", false), Ok(1)); @@ -16651,7 +17247,7 @@ mod tests { iter.next(), Some(&frame::Frame::Stream { stream_id: 12, - data: stream::RangeBuf::from(b"b", 0, false), + data: ::from(b"b", 0, false), }) ); } @@ -16680,7 +17276,7 @@ mod tests { iter.next(), Some(&frame::Frame::Stream { stream_id: 4, - data: stream::RangeBuf::from(b"b", 0, false), + data: ::from(b"b", 0, false), }) ); } @@ -16710,7 +17306,7 @@ mod tests { iter.next(), Some(&frame::Frame::Stream { stream_id: 16, - data: stream::RangeBuf::from(b"b", 0, false), + data: ::from(b"b", 0, false), }) ); } @@ -16739,7 +17335,7 @@ mod tests { iter.next(), Some(&frame::Frame::Stream { stream_id: 8, - data: stream::RangeBuf::from(b"b", 0, false), + data: ::from(b"b", 0, false), }) ); } @@ -16774,7 +17370,7 @@ mod tests { config.enable_dgram(true, 10, 10); config.verify_peer(false); - let mut pipe = testing::Pipe::with_config(&mut config).unwrap(); + let mut pipe = ::with_config(&mut config).unwrap(); assert_eq!(pipe.handshake(), Ok(())); assert_eq!(pipe.client.stream_send(4, b"a", false), Ok(1)); @@ -16860,7 +17456,7 @@ mod tests { } else { assert_eq!(stream, &frame::Frame::Stream { stream_id: 4, - data: stream::RangeBuf::from(&out, off_0, false), + data: ::from(&out, off_0, false), }); } @@ -16910,7 +17506,7 @@ mod tests { } else { assert_eq!(stream, &frame::Frame::Stream { stream_id: 8, - data: stream::RangeBuf::from(&out, off_4, false), + data: ::from(&out, off_4, false), }); } @@ -16929,7 +17525,7 @@ mod tests { fn early_retransmit() { let mut buf = [0; 65535]; - let mut pipe = testing::Pipe::new().unwrap(); + let mut pipe = ::new().unwrap(); assert_eq!(pipe.handshake(), Ok(())); // Client sends stream data. @@ -16994,10 +17590,11 @@ mod tests { iter.next(), Some(&frame::Frame::Stream { stream_id: 4, - data: stream::RangeBuf::from(b"b", 0, false), + data: ::from(b"b", 0, false), }) ); } + assert_eq!(pipe.client.stats().retrans, 1); } @@ -17006,7 +17603,7 @@ mod tests { fn dont_coalesce_probes() { let mut buf = [0; 65535]; - let mut pipe = testing::Pipe::new().unwrap(); + let mut pipe = ::new().unwrap(); // Client sends Initial packet. let (len, _) = pipe.client.send(&mut buf).unwrap(); @@ -17089,7 +17686,7 @@ mod tests { fn coalesce_padding_short() { let mut buf = [0; 65535]; - let mut pipe = testing::Pipe::new().unwrap(); + let mut pipe = ::new().unwrap(); // Client sends first flight. let (len, _) = pipe.client.send(&mut buf).unwrap(); @@ -17134,7 +17731,7 @@ mod tests { .set_application_protos(&[b"proto1", b"proto2"]) .unwrap(); - let mut pipe = testing::Pipe::with_server_config(&mut config).unwrap(); + let mut pipe = ::with_server_config(&mut config).unwrap(); assert!(!pipe.client.handshake_status().has_handshake_keys); assert!(!pipe.client.handshake_status().peer_verified_address); @@ -17180,7 +17777,7 @@ mod tests { fn handshake_packet_type_corruption() { let mut buf = [0; 65535]; - let mut pipe = testing::Pipe::new().unwrap(); + let mut pipe = ::new().unwrap(); // Client sends padded Initial. let (len, _) = pipe.client.send(&mut buf).unwrap(); @@ -17227,7 +17824,7 @@ mod tests { #[test] fn dgram_send_fails_invalidstate() { - let mut pipe = testing::Pipe::new().unwrap(); + let mut pipe = ::new().unwrap(); assert_eq!(pipe.handshake(), Ok(())); assert_eq!( @@ -17261,7 +17858,7 @@ mod tests { config.set_max_recv_udp_payload_size(1200); config.verify_peer(false); - let mut pipe = testing::Pipe::with_config(&mut config).unwrap(); + let mut pipe = ::with_config(&mut config).unwrap(); assert_eq!(pipe.handshake(), Ok(())); for _ in 0..1000 { @@ -17342,7 +17939,7 @@ mod tests { config.enable_dgram(true, 10, 10); config.verify_peer(false); - let mut pipe = testing::Pipe::with_config(&mut config).unwrap(); + let mut pipe = ::with_config(&mut config).unwrap(); assert_eq!(pipe.handshake(), Ok(())); assert_eq!(pipe.client.dgram_send(b"hello, world"), Ok(())); @@ -17379,7 +17976,7 @@ mod tests { config.enable_dgram(true, 2, 3); config.verify_peer(false); - let mut pipe = testing::Pipe::with_config(&mut config).unwrap(); + let mut pipe = ::with_config(&mut config).unwrap(); assert_eq!(pipe.handshake(), Ok(())); assert_eq!(pipe.client.dgram_send_queue_len(), 0); @@ -17453,7 +18050,7 @@ mod tests { config.enable_dgram(true, 10, 2); config.verify_peer(false); - let mut pipe = testing::Pipe::with_config(&mut config).unwrap(); + let mut pipe = ::with_config(&mut config).unwrap(); assert_eq!(pipe.handshake(), Ok(())); assert_eq!(pipe.client.dgram_send(b"hello, world"), Ok(())); @@ -17500,7 +18097,7 @@ mod tests { config.set_max_recv_udp_payload_size(1200); config.verify_peer(false); - let mut pipe = testing::Pipe::with_config(&mut config).unwrap(); + let mut pipe = ::with_config(&mut config).unwrap(); assert_eq!(pipe.handshake(), Ok(())); assert_eq!(pipe.client.dgram_send(b"hello, world"), Ok(())); @@ -17547,7 +18144,7 @@ mod tests { config.set_max_recv_udp_payload_size(1452); config.verify_peer(false); - let mut pipe = testing::Pipe::with_config(&mut config).unwrap(); + let mut pipe = ::with_config(&mut config).unwrap(); // Before handshake (before peer settings) we don't know max dgram size assert_eq!(pipe.client.dgram_max_writable_len(), None); @@ -17603,7 +18200,7 @@ mod tests { config.set_max_recv_udp_payload_size(1452); config.verify_peer(false); - let mut pipe = testing::Pipe::with_config(&mut config).unwrap(); + let mut pipe = ::with_config(&mut config).unwrap(); assert_eq!(pipe.handshake(), Ok(())); // No readable data. @@ -17677,7 +18274,7 @@ mod tests { fn close() { let mut buf = [0; 65535]; - let mut pipe = testing::Pipe::new().unwrap(); + let mut pipe = ::new().unwrap(); assert_eq!(pipe.handshake(), Ok(())); assert_eq!(pipe.client.close(false, 0x1234, b"hello?"), Ok(())); @@ -17720,7 +18317,7 @@ mod tests { fn app_close_by_client() { let mut buf = [0; 65535]; - let mut pipe = testing::Pipe::new().unwrap(); + let mut pipe = ::new().unwrap(); assert_eq!(pipe.handshake(), Ok(())); assert_eq!(pipe.client.close(true, 0x1234, b"hello!"), Ok(())); @@ -17760,7 +18357,7 @@ mod tests { #[cfg(not(feature = "openssl"))] #[test] fn app_close_by_server_during_handshake_private_key_failure() { - let mut pipe = testing::Pipe::new().unwrap(); + let mut pipe = ::new().unwrap(); pipe.server.handshake.set_failing_private_key_method(); // Client sends initial flight. @@ -17826,7 +18423,7 @@ mod tests { #[test] fn app_close_by_server_during_handshake_not_established() { - let mut pipe = testing::Pipe::new().unwrap(); + let mut pipe = ::new().unwrap(); // Client sends initial flight. let flight = testing::emit_flight(&mut pipe.client).unwrap(); @@ -17891,7 +18488,7 @@ mod tests { #[test] fn app_close_by_server_during_handshake_established() { - let mut pipe = testing::Pipe::new().unwrap(); + let mut pipe = ::new().unwrap(); // Client sends initial flight. let flight = testing::emit_flight(&mut pipe.client).unwrap(); @@ -17964,7 +18561,7 @@ mod tests { #[test] fn peer_error() { - let mut pipe = testing::Pipe::new().unwrap(); + let mut pipe = ::new().unwrap(); assert_eq!(pipe.handshake(), Ok(())); assert_eq!(pipe.server.close(false, 0x1234, b"hello?"), Ok(())); @@ -17982,7 +18579,7 @@ mod tests { #[test] fn app_peer_error() { - let mut pipe = testing::Pipe::new().unwrap(); + let mut pipe = ::new().unwrap(); assert_eq!(pipe.handshake(), Ok(())); assert_eq!(pipe.server.close(true, 0x1234, b"hello!"), Ok(())); @@ -18000,7 +18597,7 @@ mod tests { #[test] fn local_error() { - let mut pipe = testing::Pipe::new().unwrap(); + let mut pipe = ::new().unwrap(); assert_eq!(pipe.handshake(), Ok(())); assert_eq!(pipe.server.local_error(), None); @@ -18052,7 +18649,7 @@ mod tests { // Larger than the client server_config.set_max_send_udp_payload_size(1500); - let mut pipe = testing::Pipe { + let mut pipe = Pipe:: { client: connect( Some("quic.tech"), &client_scid, @@ -18140,7 +18737,7 @@ mod tests { config.set_initial_max_streams_bidi(10); config.verify_peer(false); - let mut pipe = testing::Pipe::with_config(&mut config).unwrap(); + let mut pipe = ::with_config(&mut config).unwrap(); assert_eq!(pipe.handshake(), Ok(())); assert_eq!(pipe.client.stream_send(4, b"hello!", true), Ok(6)); @@ -18245,7 +18842,7 @@ mod tests { let server_scid = ConnectionId::from_ref(&server_scid); let server_addr = "127.0.0.1:4321".parse().unwrap(); - let mut pipe = testing::Pipe { + let mut pipe = Pipe:: { client: connect( Some("quic.tech"), &client_scid, @@ -18280,7 +18877,7 @@ mod tests { config.set_max_recv_udp_payload_size(1200); config.verify_peer(false); - let mut pipe = testing::Pipe::with_client_config(&mut config).unwrap(); + let mut pipe = ::with_client_config(&mut config).unwrap(); assert_eq!(pipe.handshake(), Ok(())); // Client opens stream 4 and 8. @@ -18324,6 +18921,7 @@ mod tests { // Server sends PTO probe (not limited to cwnd), // to update last_tx_data. let (len, _) = pipe.server.send(&mut buf).unwrap(); + assert_eq!(len, 1200); // Client sends STOP_SENDING to decrease tx_data @@ -18356,7 +18954,7 @@ mod tests { config.verify_peer(false); config.set_active_connection_id_limit(3); - let mut pipe = testing::Pipe::with_config(&mut config).unwrap(); + let mut pipe = ::with_config(&mut config).unwrap(); assert_eq!(pipe.handshake(), Ok(())); // So far, there should not have any QUIC event. @@ -18416,7 +19014,7 @@ mod tests { config.verify_peer(false); config.set_active_connection_id_limit(2); - let mut pipe = testing::Pipe::with_config(&mut config).unwrap(); + let mut pipe = ::with_config(&mut config).unwrap(); assert_eq!(pipe.handshake(), Ok(())); let mut frames = Vec::new(); @@ -18498,7 +19096,7 @@ mod tests { config.verify_peer(false); config.set_active_connection_id_limit(2); - let mut pipe = testing::Pipe::with_config(&mut config).unwrap(); + let mut pipe = ::with_config(&mut config).unwrap(); assert_eq!(pipe.handshake(), Ok(())); let mut frames = Vec::new(); @@ -18580,7 +19178,7 @@ mod tests { config.verify_peer(false); config.set_active_connection_id_limit(2); - let mut pipe = testing::Pipe::with_config(&mut config).unwrap(); + let mut pipe = ::with_config(&mut config).unwrap(); assert_eq!(pipe.handshake(), Ok(())); // So far, there should not have any QUIC event. @@ -18665,7 +19263,7 @@ mod tests { config.verify_peer(false); config.set_active_connection_id_limit(2); - let mut pipe = testing::Pipe::with_config(&mut config).unwrap(); + let mut pipe = ::with_config(&mut config).unwrap(); assert_eq!(pipe.handshake(), Ok(())); let scid = pipe.client.source_id().into_owned(); @@ -18722,7 +19320,7 @@ mod tests { config.verify_peer(false); config.set_active_connection_id_limit(3); - let mut pipe = testing::Pipe::with_config(&mut config).unwrap(); + let mut pipe = ::with_config(&mut config).unwrap(); assert_eq!(pipe.handshake(), Ok(())); let (scid_1, reset_token_1) = testing::create_cid_and_reset_token(16); @@ -18769,7 +19367,7 @@ mod tests { config.verify_peer(false); config.set_active_connection_id_limit(2); - let mut pipe = testing::Pipe::with_config(&mut config).unwrap(); + let mut pipe = ::with_config(&mut config).unwrap(); assert_eq!(pipe.handshake(), Ok(())); // So far, there should not have any QUIC event. @@ -18856,8 +19454,8 @@ mod tests { fn pipe_with_exchanged_cids( config: &mut Config, client_scid_len: usize, server_scid_len: usize, additional_cids: usize, - ) -> testing::Pipe { - let mut pipe = testing::Pipe::with_config_and_scid_lengths( + ) -> Pipe { + let mut pipe = ::with_config_and_scid_lengths( config, client_scid_len, server_scid_len, @@ -18927,10 +19525,10 @@ mod tests { config.verify_peer(false); config.set_active_connection_id_limit(2); - let mut pipe = testing::Pipe::with_config(&mut config).unwrap(); + let mut pipe = ::with_config(&mut config).unwrap(); assert_eq!(pipe.handshake(), Ok(())); - let server_addr = testing::Pipe::server_addr(); + let server_addr = ::server_addr(); let client_addr_2 = "127.0.0.1:5678".parse().unwrap(); // We cannot probe a new path if there are not enough identifiers. @@ -19014,7 +19612,7 @@ mod tests { let mut pipe = pipe_with_exchanged_cids(&mut config, 16, 16, 1); - let server_addr = testing::Pipe::server_addr(); + let server_addr = ::server_addr(); let client_addr_2 = "127.0.0.1:5678".parse().unwrap(); assert_eq!(pipe.client.probe_path(client_addr_2, server_addr), Ok(1)); @@ -19079,7 +19677,7 @@ mod tests { let mut pipe = pipe_with_exchanged_cids(&mut config, 16, 16, 1); - let server_addr = testing::Pipe::server_addr(); + let server_addr = ::server_addr(); let client_addr_2 = "127.0.0.1:5678".parse().unwrap(); assert_eq!(pipe.client.probe_path(client_addr_2, server_addr), Ok(1)); @@ -19131,7 +19729,7 @@ mod tests { config.set_initial_max_stream_data_uni(10); config.set_initial_max_streams_uni(3); - let mut pipe = testing::Pipe::with_config(&mut config).unwrap(); + let mut pipe = ::with_config(&mut config).unwrap(); assert_eq!(pipe.handshake(), Ok(())); // Server sends stream data. @@ -19171,7 +19769,7 @@ mod tests { let mut pipe = pipe_with_exchanged_cids(&mut config, 16, 16, 1); - let server_addr = testing::Pipe::server_addr(); + let server_addr = ::server_addr(); let client_addr_2 = "127.0.0.1:5678".parse().unwrap(); assert_eq!(pipe.client.probe_path(client_addr_2, server_addr), Ok(1)); // Limited MTU of 1199 bytes for some reason. @@ -19226,7 +19824,7 @@ mod tests { let mut pipe = pipe_with_exchanged_cids(&mut config, 16, 16, 1); - let server_addr = testing::Pipe::server_addr(); + let server_addr = ::server_addr(); let client_addr_2 = "127.0.0.1:5678".parse().unwrap(); assert_eq!(pipe.client.probe_path(client_addr_2, server_addr), Ok(1)); @@ -19295,7 +19893,7 @@ mod tests { config.set_active_connection_id_limit(2); let mut pipe = pipe_with_exchanged_cids(&mut config, 16, 16, 1); - let server_addr = testing::Pipe::server_addr(); + let server_addr = ::server_addr(); let client_addr_2 = "127.0.0.1:5678".parse().unwrap(); assert_eq!(pipe.client.probe_path(client_addr_2, server_addr), Ok(1)); @@ -19323,8 +19921,8 @@ mod tests { let mut pipe = pipe_with_exchanged_cids(&mut config, 16, 16, 3); - let server_addr = testing::Pipe::server_addr(); - let client_addr = testing::Pipe::client_addr(); + let server_addr = ::server_addr(); + let client_addr = ::client_addr(); let client_addr_2 = "127.0.0.1:5678".parse().unwrap(); assert_eq!(pipe.client.probe_path(client_addr_2, server_addr), Ok(1)); @@ -19532,7 +20130,7 @@ mod tests { let mut pipe = pipe_with_exchanged_cids(&mut config, 16, 16, 2); - let server_addr = testing::Pipe::server_addr(); + let server_addr = ::server_addr(); let client_addr_2 = "127.0.0.1:5678".parse().unwrap(); let client_addr_3 = "127.0.0.1:9012".parse().unwrap(); let client_addr_4 = "127.0.0.1:8908".parse().unwrap(); @@ -19745,7 +20343,7 @@ mod tests { let mut pipe = pipe_with_exchanged_cids(&mut config, 0, 16, 1); - let server_addr = testing::Pipe::server_addr(); + let server_addr = ::server_addr(); let client_addr_2 = "127.0.0.1:5678".parse().unwrap(); // The client migrates on a path that was not previously @@ -19822,8 +20420,8 @@ mod tests { let mut pipe = pipe_with_exchanged_cids(&mut config, 16, 16, 1); - let client_addr = testing::Pipe::client_addr(); - let server_addr = testing::Pipe::server_addr(); + let client_addr = ::client_addr(); + let server_addr = ::server_addr(); let client_addr_2 = "127.0.0.1:5678".parse().unwrap(); assert_eq!(pipe.client.probe_path(client_addr_2, server_addr), Ok(1)); @@ -19902,8 +20500,8 @@ mod tests { let mut pipe = pipe_with_exchanged_cids(&mut config, 16, 16, 1); - let client_addr = testing::Pipe::client_addr(); - let server_addr = testing::Pipe::server_addr(); + let client_addr = ::client_addr(); + let server_addr = ::server_addr(); let spoofed_client_addr = "127.0.0.1:6666".parse().unwrap(); const DATA_BYTES: usize = 24000; @@ -20038,7 +20636,7 @@ mod tests { fn consecutive_non_ack_eliciting() { let mut buf = [0; 65535]; - let mut pipe = testing::Pipe::new().unwrap(); + let mut pipe = ::new().unwrap(); assert_eq!(pipe.handshake(), Ok(())); // Client sends a bunch of PING frames, causing server to ACK (ACKs aren't @@ -20100,7 +20698,7 @@ mod tests { #[test] fn send_ack_eliciting_causes_ping() { // First establish a connection - let mut pipe = testing::Pipe::new().unwrap(); + let mut pipe = ::new().unwrap(); assert_eq!(pipe.handshake(), Ok(())); // Queue a PING frame @@ -20128,7 +20726,7 @@ mod tests { #[test] fn send_ack_eliciting_no_ping() { // First establish a connection - let mut pipe = testing::Pipe::new().unwrap(); + let mut pipe = ::new().unwrap(); assert_eq!(pipe.handshake(), Ok(())); // Queue a PING frame @@ -20202,7 +20800,7 @@ mod tests { config.set_initial_max_streams_uni(0); config.verify_peer(false); - let mut pipe = testing::Pipe::with_config(&mut config).unwrap(); + let mut pipe = ::with_config(&mut config).unwrap(); assert_eq!(pipe.handshake(), Ok(())); assert_eq!(pipe.server.streams.len(), 0); @@ -20415,7 +21013,7 @@ mod tests { config.set_initial_max_streams_bidi(3); let mut pipe = - testing::Pipe::with_config_and_scid_lengths(&mut config, 16, 16) + ::with_config_and_scid_lengths(&mut config, 16, 16) .unwrap(); assert_eq!(pipe.handshake(), Ok(())); @@ -20430,7 +21028,7 @@ mod tests { } assert_eq!(pipe.advance(), Ok(())); - let server_addr = testing::Pipe::server_addr(); + let server_addr = ::server_addr(); let client_addr_2 = "127.0.0.1:5678".parse().unwrap(); // Client probes path before sending CIDs (simulating race condition) @@ -20472,15 +21070,18 @@ mod tests { let written = packet::encrypt_pkt( &mut b, + None, pn, - pn_len, payload_len, payload_offset, None, aead, - pipe.client.version, + false, ) .expect("packet encrypt"); + let (mut header, payload) = b.split_at(payload_offset).unwrap(); + packet::encrypt_hdr(&mut header, pn_len, payload.as_ref(), &aead, crate::PROTOCOL_VERSION) + .expect("header encrypt"); space.next_pkt_num += 1; pipe.server @@ -20524,11 +21125,11 @@ mod tests { config.discover_pmtu(true); // Perform initial handshake. - let mut pipe = testing::Pipe::with_config(&mut config).unwrap(); + let mut pipe = ::with_config(&mut config).unwrap(); assert_eq!(pipe.handshake(), Ok(())); - let server_addr = testing::Pipe::server_addr(); - let client_addr = testing::Pipe::client_addr(); + let server_addr = ::server_addr(); + let client_addr = ::client_addr(); let pid_1 = pipe .server .paths @@ -20570,11 +21171,11 @@ mod tests { config.discover_pmtu(true); // Perform initial handshake. - let mut pipe = testing::Pipe::with_config(&mut config).unwrap(); + let mut pipe = ::with_config(&mut config).unwrap(); assert_eq!(pipe.handshake(), Ok(())); - let server_addr = testing::Pipe::server_addr(); - let client_addr = testing::Pipe::client_addr(); + let server_addr = ::server_addr(); + let client_addr = ::client_addr(); let pid_1 = pipe .server .paths @@ -20613,6 +21214,8 @@ pub use crate::recovery::CongestionControlAlgorithm; pub use crate::stream::StreamIter; +pub use crate::range_buf::BufFactory; +pub use crate::range_buf::BufSplit; pub use crate::stream::app_recv_buf::AppRecvBufMap; mod cid; @@ -20628,6 +21231,7 @@ mod packet; mod path; mod pmtud; mod rand; +mod range_buf; mod ranges; mod recovery; mod stream; diff --git a/quiceh/src/packet.rs b/quiceh/src/packet.rs index 8b9bd085..1f526506 100644 --- a/quiceh/src/packet.rs +++ b/quiceh/src/packet.rs @@ -884,18 +884,18 @@ pub fn decrypt_pkt<'a>( Ok((b.get_bytes(payload_len)?, payload_len)) } -pub fn encrypt_hdr( - b: &mut octets_rev::OctetsMut, enc_len: usize, payload: &[u8], +#[inline] +fn encrypt_hdr_inner( + first: &mut [u8], rest: &mut [u8], enc_len: usize, payload: &[u8], aead: &crypto::Seal, version: u32, ) -> Result<()> { - let sample; - let (mut first, mut rest) = b.split_at(1)?; - let first = first.as_mut(); - + if enc_len > rest.len() { + return Err(Error::BufferTooShort); + } if_likely! {version == crate::PROTOCOL_VERSION_VREVERSO => { // considering max 4 bytes for the streamid and 4 bytes for the buffer offset. // for which the encoding/decoding would work in a similar fashion than for the packet number. - sample = &payload + let sample = &payload [MAX_PKT_NUM_STREAMID_OFFSET_LEN - enc_len..SAMPLE_LEN + (MAX_PKT_NUM_STREAMID_OFFSET_LEN - enc_len)]; let mask = aead.new_mask_13(sample)?; if Header::is_long(first[0]) { @@ -903,13 +903,14 @@ pub fn encrypt_hdr( } else { first[0] ^= mask[0] & 0x1f; } - let buf = rest.slice_last(enc_len)?; + let len = rest.len(); + let buf = &mut rest[len - enc_len..]; for i in 0..enc_len { buf[i] ^= mask[i + 1]; } } else { - sample = &payload + let sample = &payload [MAX_PKT_NUM_LEN - enc_len..SAMPLE_LEN + (MAX_PKT_NUM_LEN - enc_len)]; let mask = aead.new_mask(sample)?; if Header::is_long(first[0]) { @@ -917,7 +918,8 @@ pub fn encrypt_hdr( } else { first[0] ^= mask[0] & 0x1f; } - let buf = rest.slice_last(enc_len)?; + let len = rest.len(); + let buf = &mut rest[len - enc_len..]; for i in 0..enc_len { buf[i] ^= mask[i + 1]; @@ -927,22 +929,44 @@ pub fn encrypt_hdr( Ok(()) } -pub fn encrypt_pkt( - b: &mut octets_rev::OctetsMut, pn: u64, hdr_enc_len: usize, - payload_len: usize, payload_offset: usize, extra_in: Option<&[u8]>, + +pub unsafe fn encrypt_hdr_unchecked( + header: &mut [u8], enc_len: usize, payload: &[u8], + aead: &crypto::Seal, version: u32, +) -> Result<()> { + + let (first, rest) = header.split_at_mut_unchecked(1); + encrypt_hdr_inner(first, rest, enc_len, payload, aead, version) +} + +pub fn encrypt_hdr( + b: &mut octets_rev::OctetsMut, enc_len: usize, payload: &[u8], aead: &crypto::Seal, version: u32, +) -> Result<()> { + let (mut first, mut rest) = b.split_at(1)?; + let first = first.as_mut(); + let rest = rest.as_mut(); + encrypt_hdr_inner(first, rest, enc_len, payload, aead, version) +} + +pub fn encrypt_pkt( + b: &mut octets_rev::OctetsMut, inbuf: Option<&[u8]>, + pn: u64, payload_len: usize, payload_offset: usize, extra_in: Option<&[u8]>, + aead: &crypto::Seal, scatter_crypt: bool, ) -> Result { - let (mut header, mut payload) = b.split_at(payload_offset)?; + let (header, mut payload) = b.split_at(payload_offset)?; let ciphertext_len = aead.seal_with_u64_counter( pn, header.as_ref(), payload.as_mut(), payload_len, + inbuf, extra_in, + scatter_crypt, )?; - encrypt_hdr(&mut header, hdr_enc_len, payload.as_ref(), aead, version)?; + //encrypt_hdr(&mut header, hdr_enc_len, payload.as_ref(), aead, version)?; Ok(payload_offset + ciphertext_len) } @@ -1197,7 +1221,7 @@ impl PktNumSpace { crypto_0rtt_open: None, - crypto_stream: stream::Stream::new( + crypto_stream: ::new( 0, // dummy u64::MAX, u64::MAX, @@ -1211,7 +1235,7 @@ impl PktNumSpace { } pub fn clear(&mut self) { - self.crypto_stream = stream::Stream::new( + self.crypto_stream = ::new( 0, // dummy u64::MAX, u64::MAX, @@ -1920,15 +1944,18 @@ mod tests { let written = encrypt_pkt( &mut b, + None, pn, - pn_len, payload_len, payload_offset, None, &aead, - crate::PROTOCOL_VERSION_V1, + false, ) .unwrap(); + let (mut header, payload) = b.split_at(payload_offset).unwrap(); + encrypt_hdr(&mut header, pn_len, payload.as_ref(), &aead, crate::PROTOCOL_VERSION_V1) + .expect("header encrypt"); assert_eq!(written, expected_pkt.len()); assert_eq!(&out[..written], expected_pkt); @@ -2260,16 +2287,20 @@ mod tests { let written = encrypt_pkt( &mut b, + None, pn, - enc_len, payload_len, payload_offset, None, &aead, - crate::PROTOCOL_VERSION_V1, + false, ) .unwrap(); + let (mut header, payload) = b.split_at(payload_offset).unwrap(); + encrypt_hdr(&mut header, enc_len, payload.as_ref(), &aead, crate::PROTOCOL_VERSION_V1) + .expect("header encrypt"); + assert_eq!(written, expected_pkt.len()); } diff --git a/quiceh/src/range_buf.rs b/quiceh/src/range_buf.rs new file mode 100644 index 00000000..f0a0da16 --- /dev/null +++ b/quiceh/src/range_buf.rs @@ -0,0 +1,204 @@ +use std::cmp; +use std::fmt::Debug; +use std::marker::PhantomData; +use std::ops::Deref; +use std::sync::Arc; + +/// Buffer holding data at a specific offset. +/// +/// The data is stored in a `Vec` in such a way that it can be shared +/// between multiple `RangeBuf` objects. +/// +/// Each `RangeBuf` will have its own view of that buffer, where the `start` +/// value indicates the initial offset within the `Vec`, and `len` indicates the +/// number of bytes, starting from `start` that are included. +/// +/// In addition, `pos` indicates the current offset within the `Vec`, starting +/// from the very beginning of the `Vec`. +/// +/// Finally, `off` is the starting offset for the specific `RangeBuf` within the +/// stream the buffer belongs to. +#[derive(Clone, Debug, Default)] +pub struct RangeBuf +where + F: BufFactory, +{ + /// The internal buffer holding the data. + /// + /// To avoid needless allocations when a RangeBuf is split, this field + /// should be reference-counted so it can be shared between multiple + /// RangeBuf objects, and sliced using the `start` and `len` values. + pub(crate) data: F::Buf, + + /// The initial offset within the internal buffer. + pub(crate) start: usize, + + /// The current offset within the internal buffer. + pub(crate) pos: usize, + + /// The number of bytes in the buffer, from the initial offset. + pub(crate) len: usize, + + /// The offset of the buffer within a stream. + pub(crate) off: u64, + + /// Whether this contains the final byte in the stream. + pub(crate) fin: bool, + + _bf: PhantomData, +} + +/// A trait for providing internal storage buffers for [`RangeBuf`]. +/// The associated type `Buf` can be any type that dereferences to +/// a slice, but should be fast to clone, eg. by wrapping it with an +/// [`Arc`]. +pub trait BufFactory: Clone + Default + Debug { + /// The type of the generated buffer. + type Buf: Clone + Debug + AsRef<[u8]>; + + /// Generate a new buffer from a given slice, the buffer must contain the + /// same data as the original slice. + fn buf_from_slice(buf: &[u8]) -> Self::Buf; +} + +/// A trait that enables zero-copy sends to quiche. When buffers produced +/// by the `BufFactory` implement this trait, quiche and h3 can supply the +/// raw buffers to be sent, instead of slices that must be copied first. +pub trait BufSplit { + /// Split the buffer at a given point, after the split the old buffer + /// must only contain the first `at` bytes, while the newly produced + /// buffer must containt the remaining bytes. + fn split_at(&mut self, at: usize) -> Self; +} + +/// The default [`BufFactory`] allocates buffers on the heap on demand. +#[derive(Debug, Clone, Default)] +pub struct DefaultBufFactory; + +/// The default [`BufFactory::Buf`] is a boxed slice wrapped in an [`Arc`]. +#[derive(Debug, Clone, Default)] +pub struct DefaultBuf(Arc>); + +impl BufFactory for DefaultBufFactory { + type Buf = DefaultBuf; + + fn buf_from_slice(buf: &[u8]) -> Self::Buf { + DefaultBuf(Arc::new(buf.into())) + } +} + +impl AsRef<[u8]> for DefaultBuf { + fn as_ref(&self) -> &[u8] { + &self.0[..] + } +} + +impl RangeBuf +where + F::Buf: Clone, +{ + /// Creates a new `RangeBuf` from the given slice. + pub fn from(buf: &[u8], off: u64, fin: bool) -> RangeBuf { + Self::from_raw(F::buf_from_slice(buf), off, fin) + } + + pub fn from_raw(data: F::Buf, off: u64, fin: bool) -> RangeBuf { + RangeBuf { + len: data.as_ref().len(), + data, + start: 0, + pos: 0, + off, + fin, + _bf: Default::default(), + } + } + + /// Returns whether `self` holds the final offset in the stream. + pub fn fin(&self) -> bool { + self.fin + } + + /// Returns the starting offset of `self`. + pub fn off(&self) -> u64 { + (self.off - self.start as u64) + self.pos as u64 + } + + /// Returns the final offset of `self`. + pub fn max_off(&self) -> u64 { + self.off() + self.len() as u64 + } + + /// Returns the length of `self`. + pub fn len(&self) -> usize { + self.len - (self.pos - self.start) + } + + /// Returns true if `self` has a length of zero bytes. + pub fn is_empty(&self) -> bool { + self.len() == 0 + } + + /// Consumes the starting `count` bytes of `self`. + pub fn consume(&mut self, count: usize) { + self.pos += count; + } + + /// Splits the buffer into two at the given index. + pub fn split_off(&mut self, at: usize) -> RangeBuf + where + F::Buf: Clone + AsRef<[u8]>, + { + assert!( + at <= self.len, + "`at` split index (is {}) should be <= len (is {})", + at, + self.len + ); + + let buf = RangeBuf { + data: self.data.clone(), + start: self.start + at, + pos: cmp::max(self.pos, self.start + at), + len: self.len - at, + off: self.off + at as u64, + _bf: Default::default(), + fin: self.fin, + }; + + self.pos = cmp::min(self.pos, self.start + at); + self.len = at; + self.fin = false; + + buf + } +} + +impl Deref for RangeBuf { + type Target = [u8]; + + fn deref(&self) -> &[u8] { + &self.data.as_ref()[self.pos..self.start + self.len] + } +} + +impl Ord for RangeBuf { + fn cmp(&self, other: &RangeBuf) -> cmp::Ordering { + // Invert ordering to implement min-heap. + self.off.cmp(&other.off).reverse() + } +} + +impl PartialOrd for RangeBuf { + fn partial_cmp(&self, other: &RangeBuf) -> Option { + Some(self.cmp(other)) + } +} + +impl Eq for RangeBuf {} + +impl PartialEq for RangeBuf { + fn eq(&self, other: &RangeBuf) -> bool { + self.off == other.off + } +} diff --git a/quiceh/src/stream/app_recv_buf.rs b/quiceh/src/stream/app_recv_buf.rs index c0a4644a..332cd90f 100644 --- a/quiceh/src/stream/app_recv_buf.rs +++ b/quiceh/src/stream/app_recv_buf.rs @@ -1,6 +1,7 @@ use super::recv_buf::RecvBuf; use super::Stream; use super::DEFAULT_STREAM_WINDOW; +use crate::BufFactory; use std::collections::hash_map; use std::collections::VecDeque; @@ -151,8 +152,8 @@ impl AppRecvBufMap { } } - pub(crate) fn read_mut( - &mut self, stream_id: u64, stream: &mut Stream, + pub(crate) fn read_mut( + &mut self, stream_id: u64, stream: &mut Stream, ) -> Result<&mut [u8]> { let buf = match self.buffers.entry(stream_id) { hash_map::Entry::Vacant(_v) => { @@ -164,8 +165,8 @@ impl AppRecvBufMap { Ok(buf) } - pub(crate) fn advance_if_possible( - &mut self, stream_id: u64, stream: &mut Stream, + pub(crate) fn advance_if_possible( + &mut self, stream_id: u64, stream: &mut Stream, ) -> Result<()> { match self.buffers.entry(stream_id) { hash_map::Entry::Vacant(_v) => Err(Error::AppRecvBufNotFound), @@ -174,8 +175,8 @@ impl AppRecvBufMap { } } - pub(crate) fn has_consumed( - &mut self, stream_id: u64, stream: Option<&Stream>, consumed: usize, + pub(crate) fn has_consumed( + &mut self, stream_id: u64, stream: Option<&Stream>, consumed: usize, ) -> Result { match self.buffers.entry(stream_id) { hash_map::Entry::Occupied(v) => { @@ -382,8 +383,8 @@ impl AppRecvBuf { /// has been consumed by the application. It returns whether the buffer /// can be collected, and how many bytes are available for read. #[inline] - pub fn has_consumed( - &mut self, stream: Option<&Stream>, consumed: usize, + pub fn has_consumed( + &mut self, stream: Option<&Stream>, consumed: usize, ) -> Result<(bool, usize)> { self.consumed = self.consumed.saturating_add(consumed); if let Some(stream) = stream { diff --git a/quiceh/src/stream/mod.rs b/quiceh/src/stream/mod.rs index 921d66ec..ad4fab65 100644 --- a/quiceh/src/stream/mod.rs +++ b/quiceh/src/stream/mod.rs @@ -39,6 +39,8 @@ use intrusive_collections::RBTreeAtomicLink; use smallvec::SmallVec; +use crate::range_buf::DefaultBufFactory; +use crate::BufFactory; use crate::Error; use crate::Result; @@ -85,9 +87,9 @@ pub type StreamIdHashSet = HashSet; /// Keeps track of QUIC streams and enforces stream limits. #[derive(Default)] -pub struct StreamMap { +pub struct StreamMap { /// Map of streams indexed by stream ID. - streams: StreamIdHashMap, + streams: StreamIdHashMap>, /// Set of streams that were completed and garbage collected. /// @@ -163,10 +165,10 @@ pub struct StreamMap { pub max_stream_window: u64, } -impl StreamMap { +impl StreamMap { pub fn new( max_streams_bidi: u64, max_streams_uni: u64, max_stream_window: u64, - ) -> StreamMap { + ) -> Self { StreamMap { local_max_streams_bidi: max_streams_bidi, local_max_streams_bidi_next: max_streams_bidi, @@ -181,15 +183,20 @@ impl StreamMap { } /// Returns the stream with the given ID if it exists. - pub fn get(&self, id: u64) -> Option<&Stream> { + pub fn get(&self, id: u64) -> Option<&Stream> { self.streams.get(&id) } /// Returns the mutable stream with the given ID if it exists. - pub fn get_mut(&mut self, id: u64) -> Option<&mut Stream> { + pub fn get_mut(&mut self, id: u64) -> Option<&mut Stream> { self.streams.get_mut(&id) } + /// Returns an Entry to the Stream ID id. + pub fn entry(&mut self, id: u64) -> hash_map::Entry> { + self.streams.entry(id) + } + /// Returns the mutable stream with the given ID if it exists, or creates /// a new one otherwise. /// @@ -206,7 +213,7 @@ impl StreamMap { &mut self, id: u64, local_params: &crate::TransportParams, peer_params: &crate::TransportParams, local: bool, is_server: bool, version: u32, - ) -> Result<&mut Stream> { + ) -> Result<&mut Stream> { let (stream, is_new_and_writable) = match self.streams.entry(id) { hash_map::Entry::Vacant(v) => { // Stream has already been closed and garbage collected. @@ -677,12 +684,12 @@ impl StreamMap { } /// A QUIC stream. -pub struct Stream { +pub struct Stream { /// Receive-side stream buffer. pub recv: recv_buf::RecvBuf, /// Send-side stream buffer. - pub send: send_buf::SendBuf, + pub send: send_buf::SendBuf, pub send_lowat: usize, @@ -701,12 +708,12 @@ pub struct Stream { pub priority_key: Arc, } -impl Stream { +impl Stream { /// Creates a new stream with the given flow control limits. pub fn new( id: u64, max_rx_data: u64, max_tx_data: u64, bidi: bool, local: bool, max_window: u64, version: u32, - ) -> Stream { + ) -> Self { let priority_key = Arc::new(StreamPriorityKey { id, ..Default::default() @@ -999,150 +1006,16 @@ impl PartialEq for RecvBufInfo { self.start_off == other.start_off } } - -/// Buffer holding data at a specific offset. -/// -/// The data is stored in a `Vec` in such a way that it can be shared -/// between multiple `RangeBuf` objects. -/// -/// Each `RangeBuf` will have its own view of that buffer, where the `start` -/// value indicates the initial offset within the `Vec`, and `len` indicates the -/// number of bytes, starting from `start` that are included. -/// -/// In addition, `pos` indicates the current offset within the `Vec`, starting -/// from the very beginning of the `Vec`. -/// -/// Finally, `off` is the starting offset for the specific `RangeBuf` within the -/// stream the buffer belongs to. -#[derive(Clone, Debug, Default, Eq)] -pub struct RangeBuf { - /// The internal buffer holding the data. - /// - /// To avoid needless allocations when a RangeBuf is split, this field is - /// reference-counted and can be shared between multiple RangeBuf objects, - /// and sliced using the `start` and `len` values. - data: Arc>, - - /// The initial offset within the internal buffer. - start: usize, - - /// The current offset within the internal buffer. - pos: usize, - - /// The number of bytes in the buffer, from the initial offset. - len: usize, - - /// The offset of the buffer within a stream. - off: u64, - - /// Whether this contains the final byte in the stream. - fin: bool, -} - -impl RangeBuf { - /// Creates a new `RangeBuf` from the given slice. - pub fn from(buf: &[u8], off: u64, fin: bool) -> RangeBuf { - RangeBuf { - data: Arc::new(Vec::from(buf)), - start: 0, - pos: 0, - len: buf.len(), - off, - fin, - } - } - - /// Returns whether `self` holds the final offset in the stream. - pub fn fin(&self) -> bool { - self.fin - } - - /// Returns the starting offset of `self`. - pub fn off(&self) -> u64 { - (self.off - self.start as u64) + self.pos as u64 - } - - /// Returns the final offset of `self`. - pub fn max_off(&self) -> u64 { - self.off() + self.len() as u64 - } - - /// Returns the length of `self`. - pub fn len(&self) -> usize { - self.len - (self.pos - self.start) - } - - /// Returns true if `self` has a length of zero bytes. - pub fn is_empty(&self) -> bool { - self.len() == 0 - } - - /// Consumes the starting `count` bytes of `self`. - pub fn consume(&mut self, count: usize) { - self.pos += count; - } - - /// Splits the buffer into two at the given index. - pub fn split_off(&mut self, at: usize) -> RangeBuf { - assert!( - at <= self.len, - "`at` split index (is {}) should be <= len (is {})", - at, - self.len - ); - - let buf = RangeBuf { - data: self.data.clone(), - start: self.start + at, - pos: cmp::max(self.pos, self.start + at), - len: self.len - at, - off: self.off + at as u64, - fin: self.fin, - }; - - self.pos = cmp::min(self.pos, self.start + at); - self.len = at; - self.fin = false; - - buf - } -} - -impl std::ops::Deref for RangeBuf { - type Target = [u8]; - - fn deref(&self) -> &[u8] { - &self.data[self.pos..self.start + self.len] - } -} - -impl Ord for RangeBuf { - fn cmp(&self, other: &RangeBuf) -> cmp::Ordering { - // Invert ordering to implement min-heap. - self.off.cmp(&other.off).reverse() - } -} - -impl PartialOrd for RangeBuf { - fn partial_cmp(&self, other: &RangeBuf) -> Option { - Some(self.cmp(other)) - } -} - -impl PartialEq for RangeBuf { - fn eq(&self, other: &RangeBuf) -> bool { - self.off == other.off - } -} - #[cfg(test)] mod tests { + use crate::range_buf::RangeBuf; + use super::*; use app_recv_buf::*; #[test] fn recv_flow_control() { - let mut stream = Stream::new( + let mut stream = ::new( 0, 15, 0, @@ -1151,7 +1024,7 @@ mod tests { DEFAULT_STREAM_WINDOW, crate::PROTOCOL_VERSION, ); - let mut app_buf = AppRecvBuf::new(1, Some(42), 100, 1000); + let mut app_buf = ::new(1, Some(42), 100, 1000); assert!(!stream.recv.almost_full()); let mut buf = [0; 32]; @@ -1181,7 +1054,7 @@ mod tests { assert_eq!(stream.recv.write_v3(thirdinfo), Err(Error::FlowControl)); assert!(app_buf.advance_if_possible(&mut stream.recv).is_ok()); assert_eq!(app_buf.read_mut(&mut stream.recv).unwrap().len(), 10); - assert!(app_buf.has_consumed(None, 10).is_ok()); + assert!(app_buf.has_consumed::(None, 10).is_ok()); assert!(!stream.recv.is_fin()); } @@ -1202,7 +1075,7 @@ mod tests { #[test] fn recv_past_fin() { - let mut stream = Stream::new( + let mut stream = ::new( 0, 15, 0, @@ -1229,7 +1102,7 @@ mod tests { #[test] fn recv_fin_dup() { - let mut stream = Stream::new( + let mut stream = ::new( 0, 15, 0, @@ -1264,7 +1137,7 @@ mod tests { #[test] fn recv_fin_change() { - let mut stream = Stream::new( + let mut stream = ::new( 0, 15, 0, @@ -1291,7 +1164,7 @@ mod tests { #[test] fn recv_fin_lower_than_received() { - let mut stream = Stream::new( + let mut stream = ::new( 0, 15, 0, @@ -1311,7 +1184,7 @@ mod tests { #[test] fn recv_fin_flow_control() { - let mut stream = Stream::new( + let mut stream = ::new( 0, 15, 0, @@ -1348,7 +1221,7 @@ mod tests { #[test] fn recv_fin_reset_mismatch() { - let mut stream = Stream::new( + let mut stream = ::new( 0, 15, 0, @@ -1372,7 +1245,7 @@ mod tests { #[test] fn recv_reset_dup() { - let mut stream = Stream::new( + let mut stream = ::new( 0, 15, 0, @@ -1397,7 +1270,7 @@ mod tests { #[test] fn recv_reset_change() { - let mut stream = Stream::new( + let mut stream = ::new( 0, 15, 0, @@ -1422,7 +1295,7 @@ mod tests { #[test] fn recv_reset_lower_than_received() { - let mut stream = Stream::new( + let mut stream = ::new( 0, 15, 0, @@ -1449,7 +1322,7 @@ mod tests { fn send_flow_control() { let mut buf = [0; 25]; - let mut stream = Stream::new( + let mut stream = ::new( 0, 0, 15, @@ -1500,7 +1373,7 @@ mod tests { #[test] fn send_past_fin() { - let mut stream = Stream::new( + let mut stream = ::new( 0, 0, 15, @@ -1524,7 +1397,7 @@ mod tests { #[test] fn send_fin_dup() { - let mut stream = Stream::new( + let mut stream = ::new( 0, 0, 15, @@ -1543,7 +1416,7 @@ mod tests { #[test] fn send_undo_fin() { - let mut stream = Stream::new( + let mut stream = ::new( 0, 0, 15, @@ -1566,7 +1439,7 @@ mod tests { fn send_fin_max_data_match() { let mut buf = [0; 15]; - let mut stream = Stream::new( + let mut stream = ::new( 0, 0, 15, @@ -1590,7 +1463,7 @@ mod tests { fn send_fin_zero_length() { let mut buf = [0; 5]; - let mut stream = Stream::new( + let mut stream = ::new( 0, 0, 15, @@ -1614,7 +1487,7 @@ mod tests { fn send_ack() { let mut buf = [0; 5]; - let mut stream = Stream::new( + let mut stream = ::new( 0, 0, 15, @@ -1652,7 +1525,7 @@ mod tests { fn send_ack_reordering() { let mut buf = [0; 5]; - let mut stream = Stream::new( + let mut stream = ::new( 0, 0, 15, @@ -1704,7 +1577,7 @@ mod tests { // below_off() is true (the overlap MUST be the same data). This is too // expensive to be an thing, so we work around this in V3. if crate::PROTOCOL_VERSION == crate::PROTOCOL_VERSION_V1 { - let mut stream = Stream::new( + let mut stream = ::new( 0, 15, 0, @@ -1735,7 +1608,7 @@ mod tests { #[test] fn stream_complete() { - let mut stream = Stream::new( + let mut stream = ::new( 0, 30, 30, @@ -1804,7 +1677,7 @@ mod tests { fn send_fin_zero_length_output() { let mut buf = [0; 5]; - let mut stream = Stream::new( + let mut stream = ::new( 0, 0, 15, @@ -1842,7 +1715,7 @@ mod tests { fn send_emit() { let mut buf = [0; 5]; - let mut stream = Stream::new( + let mut stream = ::new( 0, 0, 20, @@ -1902,7 +1775,7 @@ mod tests { fn send_emit_ack() { let mut buf = [0; 5]; - let mut stream = Stream::new( + let mut stream = ::new( 0, 0, 20, @@ -1977,7 +1850,7 @@ mod tests { fn send_emit_retransmit() { let mut buf = [0; 5]; - let mut stream = Stream::new( + let mut stream = ::new( 0, 0, 20, @@ -2095,7 +1968,7 @@ mod tests { #[test] fn rangebuf_split_off() { - let mut buf = RangeBuf::from(b"helloworld", 5, true); + let mut buf = ::from(b"helloworld", 5, true); assert_eq!(buf.start, 0); assert_eq!(buf.pos, 0); assert_eq!(buf.len, 10); @@ -2215,7 +2088,7 @@ mod tests { let local_tp = crate::TransportParams::default(); let peer_tp = crate::TransportParams::default(); - let mut streams = StreamMap::new(5, 5, 5); + let mut streams = ::new(5, 5, 5); let stream_id = 500; assert!(!is_local(stream_id, true), "stream id is peer initiated"); @@ -2243,7 +2116,7 @@ mod tests { let local_tp = crate::TransportParams::default(); let peer_tp = crate::TransportParams::default(); - let mut streams = StreamMap::new(5, 5, 5); + let mut streams = ::new(5, 5, 5); for stream_id in [8, 12, 4] { assert!(is_local(stream_id, false), "stream id is client initiated"); @@ -2267,7 +2140,7 @@ mod tests { let local_tp = crate::TransportParams::default(); let peer_tp = crate::TransportParams::default(); - let mut streams = StreamMap::new(3, 3, 3); + let mut streams = ::new(3, 3, 3); // Highest permitted let stream_id = if crate::PROTOCOL_VERSION == crate::PROTOCOL_VERSION_V1 { @@ -2406,7 +2279,7 @@ mod tests { ..Default::default() }; - let mut streams = StreamMap::new(100, 100, 100); + let mut streams = ::new(100, 100, 100); // Streams where the urgency descends (becomes more important). No stream // shares an urgency. diff --git a/quiceh/src/stream/recv_buf.rs b/quiceh/src/stream/recv_buf.rs index 927cfb59..60277166 100644 --- a/quiceh/src/stream/recv_buf.rs +++ b/quiceh/src/stream/recv_buf.rs @@ -35,9 +35,9 @@ use crate::Result; use crate::flowcontrol; -use super::RangeBuf; use super::RecvBufInfo; use super::DEFAULT_STREAM_WINDOW; +use crate::range_buf::RangeBuf; use std::collections::btree_map; use likely_stable::if_likely; @@ -302,6 +302,7 @@ impl RecvBuf { Ok(()) } + /// Writes data from the receive buffer into the given output buffer. /// /// Only contiguous data is written to the output buffer, starting from @@ -547,6 +548,7 @@ impl RecvBuf { #[cfg(test)] mod tests { use super::*; + use crate::range_buf::DefaultBufFactory; use crate::stream::app_recv_buf::AppRecvBuf; #[test] @@ -591,7 +593,7 @@ mod tests { (app_buf.read_mut(&mut recv).unwrap().len(), recv.is_fin()), (5, false) ); - assert!(app_buf.has_consumed(None, 5).is_ok()); + assert!(app_buf.has_consumed::(None, 5).is_ok()); } // Don't store non-fin empty buffer. @@ -921,7 +923,7 @@ mod tests { assert_eq!(recv.off, 0); assert_eq!(recv.heap.len(), 0); assert_eq!(app_buf.read_mut(&mut recv).unwrap().len(), 9); - assert!(app_buf.has_consumed(None, 9).is_ok()); + assert!(app_buf.has_consumed::(None, 9).is_ok()); assert_eq!(recv.is_fin(), false); } @@ -1002,7 +1004,7 @@ mod tests { assert_eq!(recv.off, 0); assert_eq!(recv.heap.len(), 0); assert_eq!(app_buf.read_mut(&mut recv).unwrap().len(), 9); - assert!(app_buf.has_consumed(None, 9).is_ok()); + assert!(app_buf.has_consumed::(None, 9).is_ok()); assert_eq!(recv.heap.len(), 0); } assert_eq!(recv.len, 9); @@ -1058,7 +1060,7 @@ mod tests { assert_eq!(recv.off, 0); assert!(app_buf.advance_if_possible(&mut recv).is_ok()); assert_eq!(app_buf.read_mut(&mut recv).unwrap().len(), 9); - assert!(app_buf.has_consumed(None, 9).is_ok()); + assert!(app_buf.has_consumed::(None, 9).is_ok()); assert!(!recv.is_fin()); assert_eq!(recv.heap.len(), 0); } @@ -1117,7 +1119,7 @@ mod tests { assert_eq!(recv.heap.len(), 1); assert!(app_buf.advance_if_possible(&mut recv).is_ok()); assert_eq!(app_buf.read_mut(&mut recv).unwrap().len(), 9); - assert!(app_buf.has_consumed(None, 9).is_ok()); + assert!(app_buf.has_consumed::(None, 9).is_ok()); assert_eq!(recv.heap.len(), 0); } @@ -1188,7 +1190,7 @@ mod tests { assert_eq!(recv.heap.len(), 2); assert!(app_buf.advance_if_possible(&mut recv).is_ok()); assert_eq!(app_buf.read_mut(&mut recv).unwrap().len(), 18); - assert!(app_buf.has_consumed(None, 18).is_ok()); + assert!(app_buf.has_consumed::(None, 18).is_ok()); assert_eq!(recv.heap.len(), 0); } assert_eq!(recv.len, 18); @@ -1247,7 +1249,7 @@ mod tests { assert_eq!(recv.off, 0); assert_eq!(recv.heap.len(), 0); assert_eq!(app_buf.read_mut(&mut recv).unwrap().len(), 9); - assert!(app_buf.has_consumed(None, 9).is_ok()); + assert!(app_buf.has_consumed::(None, 9).is_ok()); assert!(!recv.is_fin()); assert_eq!(recv.len, 9); assert_eq!(recv.off, 9); @@ -1302,7 +1304,7 @@ mod tests { assert_eq!(recv.heap.len(), 1); assert!(app_buf.advance_if_possible(&mut recv).is_ok()); assert_eq!(app_buf.read_mut(&mut recv).unwrap().len(), 12); - assert!(app_buf.has_consumed(None, 12).is_ok()); + assert!(app_buf.has_consumed::(None, 12).is_ok()); assert!(recv.is_fin()); } assert_eq!(recv.len, 12); @@ -1383,7 +1385,7 @@ mod tests { assert_eq!(recv.off, 0); assert_eq!(recv.heap.len(), 2); assert_eq!(app_buf.read_mut(&mut recv).unwrap().len(), 2); - assert!(app_buf.has_consumed(None, 2).is_ok()); + assert!(app_buf.has_consumed::(None, 2).is_ok()); assert!(!recv.is_fin()); } @@ -1462,7 +1464,7 @@ mod tests { assert_eq!(recv.off, 0); assert_eq!(recv.heap.len(), 2); assert_eq!(app_buf.read_mut(&mut recv).unwrap().len(), 5); - assert!(app_buf.has_consumed(None, 5).is_ok()); + assert!(app_buf.has_consumed::(None, 5).is_ok()); assert!(!recv.is_fin()); assert_eq!(recv.len, 16); assert_eq!(recv.off, 5); @@ -1531,7 +1533,7 @@ mod tests { assert_eq!(recv.heap.len(), 2); assert!(app_buf.advance_if_possible(&mut recv).is_ok()); assert_eq!(app_buf.read_mut(&mut recv).unwrap().len(), 15); - assert!(app_buf.has_consumed(None, 15).is_ok()); + assert!(app_buf.has_consumed::(None, 15).is_ok()); assert!(recv.is_fin()); } assert_eq!(recv.len, 15); @@ -1636,7 +1638,7 @@ mod tests { assert_eq!(recv.heap.len(), 5); assert!(app_buf.advance_if_possible(&mut recv).is_ok()); assert_eq!(app_buf.read_mut(&mut recv).unwrap().len(), 14); - assert!(app_buf.has_consumed(None, 14).is_ok()); + assert!(app_buf.has_consumed::(None, 14).is_ok()); assert!(!recv.is_fin()); assert_eq!(recv.heap.len(), 0); } diff --git a/quiceh/src/stream/send_buf.rs b/quiceh/src/stream/send_buf.rs index 10f4e54a..37d5c145 100644 --- a/quiceh/src/stream/send_buf.rs +++ b/quiceh/src/stream/send_buf.rs @@ -28,18 +28,56 @@ use std::cmp; use std::collections::VecDeque; +use crate::range_buf::BufSplit; +use crate::range_buf::RangeBuf; +use crate::BufFactory; use crate::Error; use crate::Result; +use crate::range_buf::DefaultBufFactory; use crate::ranges; -use super::RangeBuf; - #[cfg(test)] const SEND_BUFFER_SIZE: usize = 5; #[cfg(not(test))] -const SEND_BUFFER_SIZE: usize = 4096; +const SEND_BUFFER_SIZE: usize = 4096*4; + +struct SendReserve<'a, F: BufFactory> { + inner: &'a mut SendBuf, + reserved: usize, + fin: bool, +} + +impl<'a, F: BufFactory> SendReserve<'a, F> { + fn append_buf(&mut self, buf: F::Buf) -> Result<()> { + let len = buf.as_ref().len(); + let inner = &mut self.inner; + + if len > self.reserved { + return Err(Error::BufferTooShort); + } + + let fin: bool = self.reserved == len && self.fin; + + let buf = RangeBuf::from_raw(buf, inner.off, fin); + + // The new data can simply be appended at the end of the send buffer. + inner.data.push_back(buf); + + inner.off += len as u64; + inner.len += len as u64; + self.reserved -= len; + + Ok(()) + } +} + +impl<'a, F: BufFactory> Drop for SendReserve<'a, F> { + fn drop(&mut self) { + assert_eq!(self.reserved, 0) + } +} /// Send-side stream buffer. /// @@ -51,9 +89,12 @@ const SEND_BUFFER_SIZE: usize = 4096; /// inserted at the start of the buffer (this is to allow data that needs to be /// retransmitted to be re-buffered). #[derive(Debug, Default)] -pub struct SendBuf { +pub struct SendBuf +where + F: BufFactory, +{ /// Chunks of data to be sent, ordered by offset. - data: VecDeque, + data: VecDeque>, /// The index of the buffer that needs to be sent next. pos: usize, @@ -87,33 +128,25 @@ pub struct SendBuf { error: Option, } -impl SendBuf { +impl SendBuf { /// Creates a new send buffer. - pub fn new(max_data: u64) -> SendBuf { + pub fn new(max_data: u64) -> SendBuf { SendBuf { max_data, ..SendBuf::default() } } - /// Inserts the given slice of data at the end of the buffer. - /// - /// The number of bytes that were actually stored in the buffer is returned - /// (this may be lower than the size of the input buffer, in case of partial - /// writes). - pub fn write(&mut self, mut data: &[u8], mut fin: bool) -> Result { - let max_off = self.off + data.len() as u64; + /// Try to reserve the required number of bytes to be sent + fn reserve_for_write( + &mut self, mut len: usize, mut fin: bool, + ) -> Result> { + let max_off = self.off + len as u64; // Get the stream send capacity. This will return an error if the stream // was stopped. - let capacity = self.cap()?; - - if data.len() > capacity { - // Truncate the input buffer according to the stream's capacity. - let len = capacity; - data = &data[..len]; - - // We are not buffering the full input, so clear the fin flag. + if len > self.cap()? { + len = self.cap()?; fin = false; } @@ -135,34 +168,175 @@ impl SendBuf { // Don't queue data that was already fully acked. if self.ack_off() >= max_off { - return Ok(data.len()); + return Ok(SendReserve { + inner: self, + reserved: 0, + fin, + }); } - // We already recorded the final offset, so we can just discard the - // empty buffer now. - if data.is_empty() { - return Ok(data.len()); + Ok(SendReserve { + inner: self, + reserved: len, + fin, + }) + } + + /// Inserts the given slice of data at the end of the buffer. + /// + /// The number of bytes that were actually stored in the buffer is returned + /// (this may be lower than the size of the input buffer, in case of partial + /// writes). + pub fn write(&mut self, data: &[u8], fin: bool) -> Result { + let mut reserve = self.reserve_for_write(data.len(), fin)?; + + if reserve.reserved == 0 { + return Ok(0); } - let mut len = 0; + let ret = reserve.reserved; // Split the remaining input data into consistently-sized buffers to // avoid fragmentation. - for chunk in data.chunks(SEND_BUFFER_SIZE) { - len += chunk.len(); + for chunk in data[..reserve.reserved].chunks(SEND_BUFFER_SIZE) { + reserve.append_buf(F::buf_from_slice(chunk))?; + } + + Ok(ret) + } + + /// Inserts the given buffer of data at the end of the buffer. + /// + /// The number of bytes that were actually stored in the buffer is returned + /// (this may be lower than the size of the input buffer, in case of partial + /// writes, in which case the unwritten buffer is also returned). + pub fn append_buf( + &mut self, mut data: F::Buf, cap: usize, fin: bool, + ) -> Result<(usize, Option)> + where + F::Buf: BufSplit, + { + let len = data.as_ref().len(); + let mut reserve = self.reserve_for_write(cap.min(len), fin)?; + + if reserve.reserved == 0 { + return Ok((0, Some(data))); + } + + let remainder = + (reserve.reserved < len).then(|| data.split_at(reserve.reserved)); + + let ret = reserve.reserved; + + reserve.append_buf(data)?; + + Ok((ret, remainder)) + } + + pub fn rangebuf_len(&mut self, cap: usize) -> Result<(usize, bool)> { - let fin = len == data.len() && fin; + let next_off = self.off_front(); + let mut fin = self.fin_off == Some(next_off); + let mut buf_len = 0; - let buf = RangeBuf::from(chunk, self.off, fin); + while let Some(buf) = self.data.get(self.pos) { + let off_front = self.off_front(); + if self.is_empty() || + off_front >= self.off || + off_front != next_off || + off_front >= self.max_data + { + break; + } - // The new data can simply be appended at the end of the send buffer. - self.data.push_back(buf); + if buf.is_empty() { + self.pos += 1; + continue; + } - self.off += chunk.len() as u64; - self.len += chunk.len() as u64; + buf_len = cmp::min(buf.len(), cap); + + fin = self.fin_off == Some(buf.off() + buf_len as u64); + break; + } + + Ok((buf_len, fin)) + } + + pub fn rangebuf_get(&self) -> Option<&RangeBuf> { + self.data.get(self.pos) + } + + pub fn rangebuf_consume(&mut self, consumed: usize) { + + let buf = match self.data.get_mut(self.pos) { + Some(v) => v, + None => return, + }; + + if buf.is_empty() { + return; + } + + self.len -= consumed as u64; + + let next_off = buf.off() + consumed as u64; + + if consumed == buf.len() { + self.pos += 1; + } + + buf.consume(consumed); + + self.emit_off = cmp::max(self.emit_off, next_off); + } + + pub fn emit_rangebuf_vec(&mut self, max_len: usize) -> (Vec>, usize) { + + let mut out_len = max_len; + let out_off = self.off_front(); + + let mut next_off = out_off; + let mut cryptovec: Vec> = Vec::new(); + + while out_len > 0 { + let off_front = self.off_front(); + + if self.is_empty() || + off_front >= self.off || + off_front != next_off || + off_front >= self.max_data + { + break; + } + + let buf = match self.data.get_mut(self.pos) { + Some(v) => v, + None => break, + }; + + let buf_len = cmp::min(buf.len(), out_len); + let partial = buf_len < buf.len(); + + self.len -= buf_len as u64; + + out_len -= buf_len; + + next_off = buf.off() + buf_len as u64; + + let mut rbuf = buf.clone(); + + buf.consume(buf_len); + + if partial { + rbuf.split_off(buf_len); + } else { + self.pos += 1; + } + cryptovec.push(rbuf); } - Ok(len) + (cryptovec, max_len - out_len) } /// Writes data from the send buffer into the given output buffer. @@ -495,7 +669,7 @@ mod tests { fn empty_write() { let mut buf = [0; 5]; - let mut send = SendBuf::new(u64::MAX); + let mut send = ::new(u64::MAX); assert_eq!(send.len, 0); let (written, fin) = send.emit(&mut buf).unwrap(); @@ -507,7 +681,7 @@ mod tests { fn multi_write() { let mut buf = [0; 128]; - let mut send = SendBuf::new(u64::MAX); + let mut send = ::new(u64::MAX); assert_eq!(send.len, 0); let first = b"something"; @@ -530,7 +704,7 @@ mod tests { fn split_write() { let mut buf = [0; 10]; - let mut send = SendBuf::new(u64::MAX); + let mut send = ::new(u64::MAX); assert_eq!(send.len, 0); let first = b"something"; @@ -573,7 +747,7 @@ mod tests { fn resend() { let mut buf = [0; 15]; - let mut send = SendBuf::new(u64::MAX); + let mut send = ::new(u64::MAX); assert_eq!(send.len, 0); assert_eq!(send.off_front(), 0); @@ -636,7 +810,7 @@ mod tests { fn write_blocked_by_off() { let mut buf = [0; 10]; - let mut send = SendBuf::default(); + let mut send = ::default(); assert_eq!(send.len, 0); let first = b"something"; @@ -706,7 +880,7 @@ mod tests { fn zero_len_write() { let mut buf = [0; 10]; - let mut send = SendBuf::new(u64::MAX); + let mut send = ::new(u64::MAX); assert_eq!(send.len, 0); let first = b"something"; @@ -731,7 +905,7 @@ mod tests { fn send_buf_len_on_retransmit() { let mut buf = [0; 15]; - let mut send = SendBuf::new(u64::MAX); + let mut send = ::new(u64::MAX); assert_eq!(send.len, 0); assert_eq!(send.off_front(), 0); @@ -757,7 +931,7 @@ mod tests { #[test] fn send_buf_final_size_retransmit() { let mut buf = [0; 50]; - let mut send = SendBuf::new(u64::MAX); + let mut send = ::new(u64::MAX); send.write(&buf, false).unwrap(); assert_eq!(send.off_front(), 0); diff --git a/quiceh/src/tls/mod.rs b/quiceh/src/tls/mod.rs index 0ba4daa8..14dd015e 100644 --- a/quiceh/src/tls/mod.rs +++ b/quiceh/src/tls/mod.rs @@ -129,6 +129,7 @@ pub static QUICHE_EX_DATA_INDEX: Lazy = Lazy::new(|| unsafe { SSL_get_ex_new_index(0, ptr::null(), ptr::null(), ptr::null(), ptr::null()) }); +#[derive(Clone)] pub struct Context(*mut SSL_CTX); impl Context { diff --git a/rustfmt.toml b/rustfmt.toml index b63fbf70..be3feb03 100644 --- a/rustfmt.toml +++ b/rustfmt.toml @@ -52,7 +52,7 @@ color = "Auto" unstable_features = true disable_all_formatting = false skip_children = false -hide_parse_errors = false +show_parse_errors = false error_on_line_overflow = false error_on_unformatted = false ignore = []