From e89b1339a2fd6d795def384760fd0451bfe3caec Mon Sep 17 00:00:00 2001 From: amrbashir Date: Thu, 2 Mar 2023 21:05:10 +0200 Subject: [PATCH 01/15] fix(core): rewrite `asset` protocol streaming, closes #6375 --- .../core-asset-protocol-streaming-crash.md | 5 + core/tauri/src/asset_protocol.rs | 194 +++++++++++++++++ core/tauri/src/lib.rs | 3 +- core/tauri/src/manager.rs | 200 +----------------- 4 files changed, 206 insertions(+), 196 deletions(-) create mode 100644 .changes/core-asset-protocol-streaming-crash.md create mode 100644 core/tauri/src/asset_protocol.rs diff --git a/.changes/core-asset-protocol-streaming-crash.md b/.changes/core-asset-protocol-streaming-crash.md new file mode 100644 index 000000000000..4b9a71934d08 --- /dev/null +++ b/.changes/core-asset-protocol-streaming-crash.md @@ -0,0 +1,5 @@ +--- +'tauri': 'patch' +--- + +Fix crash when streaming large files through `asset` protocol. diff --git a/core/tauri/src/asset_protocol.rs b/core/tauri/src/asset_protocol.rs new file mode 100644 index 000000000000..a02c81c90afc --- /dev/null +++ b/core/tauri/src/asset_protocol.rs @@ -0,0 +1,194 @@ +#![cfg(protocol_asset)] + +use crate::api::file::SafePathBuf; +use crate::scope::FsScope; +use rand::RngCore; +use std::io::SeekFrom; +use std::pin::Pin; +use tauri_runtime::http::HttpRange; +use tauri_runtime::http::{ + header::*, status::StatusCode, MimeType, Request, Response, ResponseBuilder, +}; +use tauri_utils::debug_eprintln; +use tokio::fs::File; +use tokio::io::{AsyncRead, AsyncReadExt, AsyncSeekExt, AsyncWriteExt}; +use url::Position; +use url::Url; + +pub fn asset_protocol_handler( + request: &Request, + scope: FsScope, + window_origin: String, +) -> Result> { + let parsed_path = Url::parse(request.uri())?; + let filtered_path = &parsed_path[..Position::AfterPath]; + let path = filtered_path + .strip_prefix("asset://localhost/") + // the `strip_prefix` only returns None when a request is made to `https://tauri.$P` on Windows + // where `$P` is not `localhost/*` + .unwrap_or(""); + let path = percent_encoding::percent_decode(path.as_bytes()) + .decode_utf8_lossy() + .to_string(); + + if let Err(e) = SafePathBuf::new(path.clone().into()) { + debug_eprintln!("asset protocol path \"{}\" is not valid: {}", path, e); + return ResponseBuilder::new().status(403).body(Vec::new()); + } + + if !scope.is_allowed(&path) { + debug_eprintln!("asset protocol not configured to allow the path: {}", path); + return ResponseBuilder::new().status(403).body(Vec::new()); + } + + let mut resp = ResponseBuilder::new().header("Access-Control-Allow-Origin", &window_origin); + + crate::async_runtime::block_on(async move { + let mut file = File::open(&path).await?; + // get file length + let len = { + let old_pos = file.seek(SeekFrom::Current(0)).await?; + let len = file.seek(SeekFrom::End(0)).await?; + file.seek(SeekFrom::Start(old_pos)).await?; + len + }; + // get file mime type + let mime_type = { + let mut magic_bytes = [0; 8192]; + let old_pos = file.seek(SeekFrom::Current(0)).await?; + file.read(&mut magic_bytes).await?; + file.seek(SeekFrom::Start(old_pos)).await?; + MimeType::parse(&magic_bytes, &path) + }; + + resp = resp.header(CONTENT_TYPE, &mime_type); + + // handle 206 (partial range) http requests + let response = if let Some(range_header) = request + .headers() + .get("range") + .and_then(|r| r.to_str().map(|r| r.to_string()).ok()) + { + resp = resp.header(ACCEPT_RANGES, "bytes"); + + let not_satisfiable = || { + ResponseBuilder::new() + .status(StatusCode::RANGE_NOT_SATISFIABLE) + .header(CONTENT_RANGE, format!("bytes */{len}")) + .body(vec![]) + }; + + // parse range header + let ranges = if let Ok(ranges) = HttpRange::parse(&range_header, len) { + ranges + .iter() + // map the output back to spec range , example: 0-499 + .map(|r| (r.start, r.start + r.length - 1)) + .collect::>() + } else { + return not_satisfiable(); + }; + + /// only send 1MB or less at a time + const MAX_LEN: u64 = 1000 * 1024; + + if ranges.len() == 1 { + let &(start, mut end) = ranges.first().unwrap(); + + // check if a range is not satisfiable + // + // this should be already taken care of by the range parsing library + // but checking here again for extra assurance + if start >= len || end >= len || end < start { + return not_satisfiable(); + } + + // adjust for MAX_LEN + end = start + (end - start).min(len - start).min(MAX_LEN - 1); + + file.seek(SeekFrom::Start(start)).await?; + + let mut stream: Pin> = Box::pin(file); + if end + 1 < len { + stream = Box::pin(stream.take(end + 1 - start)); + } + + let mut buf = Vec::new(); + stream.read_to_end(&mut buf).await?; + + resp = resp.header(CONTENT_RANGE, format!("bytes {start}-{end}/{len}")); + resp = resp.header(CONTENT_LENGTH, end + 1 - start); + resp = resp.status(StatusCode::PARTIAL_CONTENT); + resp.body(buf) + } else { + let mut buf = Vec::new(); + let ranges = ranges + .iter() + .filter_map(|&(start, mut end)| { + // filter out unsatisfiable ranges + // + // this should be already taken care of by the range parsing library + // but checking here again for extra assurance + if start >= len || end >= len || end < start { + None + } else { + end = start + (end - start).min(len - start).min(MAX_LEN - 1); + Some((start, end)) + } + }) + .collect::>(); + + let boundary = random_boundary(); + let boundary_sep = format!("\r\n--{boundary}\r\n"); + let boundary_closer = format!("\r\n--{boundary}\r\n"); + + resp = resp.header( + CONTENT_TYPE, + format!("multipart/byteranges; boundary={boundary}"), + ); + + drop(file); + + for (end, start) in ranges { + buf.write_all(boundary_sep.as_bytes()).await?; + buf + .write_all(format!("{CONTENT_TYPE}: {mime_type}\r\n").as_bytes()) + .await?; + buf + .write_all(format!("{CONTENT_RANGE}: bytes {start}-{end}/{len}\r\n").as_bytes()) + .await?; + buf.write_all("\r\n".as_bytes()).await?; + + let mut file = File::open(&path).await?; + file.seek(SeekFrom::Start(start)).await?; + file + .take(if end + 1 < len { end + 1 - start } else { len }) + .read_to_end(&mut buf) + .await?; + } + buf.write_all(boundary_closer.as_bytes()).await?; + + resp.body(buf) + } + } else { + resp = resp.header(CONTENT_LENGTH, len); + let mut buf = vec![0; len as usize]; + file.read_to_end(&mut buf).await?; + resp.body(buf) + }; + + response + }) +} + +fn random_boundary() -> String { + let mut x = [0 as u8; 30]; + rand::thread_rng().fill_bytes(&mut x); + (&x[..]) + .iter() + .map(|&x| format!("{:x}", x)) + .fold(String::new(), |mut a, x| { + a.push_str(x.as_str()); + a + }) +} diff --git a/core/tauri/src/lib.rs b/core/tauri/src/lib.rs index 9488753ec07d..4fff66304ff4 100644 --- a/core/tauri/src/lib.rs +++ b/core/tauri/src/lib.rs @@ -184,13 +184,14 @@ mod pattern; pub mod plugin; pub mod window; use tauri_runtime as runtime; +#[cfg(protocol_asset)] +mod asset_protocol; /// The allowlist scopes. pub mod scope; mod state; #[cfg(updater)] #[cfg_attr(doc_cfg, doc(cfg(feature = "updater")))] pub mod updater; - pub use tauri_utils as utils; /// A Tauri [`Runtime`] wrapper around wry. diff --git a/core/tauri/src/manager.rs b/core/tauri/src/manager.rs index e0545c09230c..a76d6a455374 100644 --- a/core/tauri/src/manager.rs +++ b/core/tauri/src/manager.rs @@ -507,203 +507,13 @@ impl WindowManager { #[cfg(protocol_asset)] if !registered_scheme_protocols.contains(&"asset".into()) { - use crate::api::file::SafePathBuf; - use tokio::io::{AsyncReadExt, AsyncSeekExt}; - use url::Position; let asset_scope = self.state().get::().asset_protocol.clone(); pending.register_uri_scheme_protocol("asset", move |request| { - let parsed_path = Url::parse(request.uri())?; - let filtered_path = &parsed_path[..Position::AfterPath]; - let path = filtered_path - .strip_prefix("asset://localhost/") - // the `strip_prefix` only returns None when a request is made to `https://tauri.$P` on Windows - // where `$P` is not `localhost/*` - .unwrap_or(""); - let path = percent_encoding::percent_decode(path.as_bytes()) - .decode_utf8_lossy() - .to_string(); - - if let Err(e) = SafePathBuf::new(path.clone().into()) { - debug_eprintln!("asset protocol path \"{}\" is not valid: {}", path, e); - return HttpResponseBuilder::new().status(403).body(Vec::new()); - } - - if !asset_scope.is_allowed(&path) { - debug_eprintln!("asset protocol not configured to allow the path: {}", path); - return HttpResponseBuilder::new().status(403).body(Vec::new()); - } - - let path_ = path.clone(); - - let mut response = - HttpResponseBuilder::new().header("Access-Control-Allow-Origin", &window_origin); - - // handle 206 (partial range) http request - if let Some(range) = request - .headers() - .get("range") - .and_then(|r| r.to_str().map(|r| r.to_string()).ok()) - { - #[derive(Default)] - struct RangeMetadata { - file: Option, - range: Option, - metadata: Option, - headers: HashMap<&'static str, String>, - status_code: u16, - body: Vec, - } - - let mut range_metadata = crate::async_runtime::safe_block_on(async move { - let mut data = RangeMetadata::default(); - // open the file - let mut file = match tokio::fs::File::open(path_.clone()).await { - Ok(file) => file, - Err(e) => { - debug_eprintln!("Failed to open asset: {}", e); - data.status_code = 404; - return data; - } - }; - // Get the file size - let file_size = match file.metadata().await { - Ok(metadata) => { - let len = metadata.len(); - data.metadata.replace(metadata); - len - } - Err(e) => { - debug_eprintln!("Failed to read asset metadata: {}", e); - data.file.replace(file); - data.status_code = 404; - return data; - } - }; - // parse the range - let range = match crate::runtime::http::HttpRange::parse( - &if range.ends_with("-*") { - range.chars().take(range.len() - 1).collect::() - } else { - range.clone() - }, - file_size, - ) { - Ok(r) => r, - Err(e) => { - debug_eprintln!("Failed to parse range {}: {:?}", range, e); - data.file.replace(file); - data.status_code = 400; - return data; - } - }; - - // FIXME: Support multiple ranges - // let support only 1 range for now - if let Some(range) = range.first() { - data.range.replace(*range); - let mut real_length = range.length; - // prevent max_length; - // specially on webview2 - if range.length > file_size / 3 { - // max size sent (400ko / request) - // as it's local file system we can afford to read more often - real_length = std::cmp::min(file_size - range.start, 1024 * 400); - } - - // last byte we are reading, the length of the range include the last byte - // who should be skipped on the header - let last_byte = range.start + real_length - 1; - - data.headers.insert("Connection", "Keep-Alive".into()); - data.headers.insert("Accept-Ranges", "bytes".into()); - data - .headers - .insert("Content-Length", real_length.to_string()); - data.headers.insert( - "Content-Range", - format!("bytes {}-{last_byte}/{file_size}", range.start), - ); - - if let Err(e) = file.seek(std::io::SeekFrom::Start(range.start)).await { - debug_eprintln!("Failed to seek file to {}: {}", range.start, e); - data.file.replace(file); - data.status_code = 422; - return data; - } - - let mut f = file.take(real_length); - let r = f.read_to_end(&mut data.body).await; - file = f.into_inner(); - data.file.replace(file); - - if let Err(e) = r { - debug_eprintln!("Failed read file: {}", e); - data.status_code = 422; - return data; - } - // partial content - data.status_code = 206; - } else { - data.status_code = 200; - } - - data - }); - - for (k, v) in range_metadata.headers { - response = response.header(k, v); - } - - let mime_type = if let (Some(mut file), Some(metadata), Some(range)) = ( - range_metadata.file, - range_metadata.metadata, - range_metadata.range, - ) { - // if we're already reading the beginning of the file, we do not need to re-read it - if range.start == 0 { - MimeType::parse(&range_metadata.body, &path) - } else { - let (status, bytes) = crate::async_runtime::safe_block_on(async move { - let mut status = None; - if let Err(e) = file.rewind().await { - debug_eprintln!("Failed to rewind file: {}", e); - status.replace(422); - (status, Vec::with_capacity(0)) - } else { - // taken from https://docs.rs/infer/0.9.0/src/infer/lib.rs.html#240-251 - let limit = std::cmp::min(metadata.len(), 8192) as usize + 1; - let mut bytes = Vec::with_capacity(limit); - if let Err(e) = file.take(8192).read_to_end(&mut bytes).await { - debug_eprintln!("Failed read file: {}", e); - status.replace(422); - } - (status, bytes) - } - }); - if let Some(s) = status { - range_metadata.status_code = s; - } - MimeType::parse(&bytes, &path) - } - } else { - MimeType::parse(&range_metadata.body, &path) - }; - response - .mimetype(&mime_type) - .status(range_metadata.status_code) - .body(range_metadata.body) - } else { - match crate::async_runtime::safe_block_on(async move { tokio::fs::read(path_).await }) { - Ok(data) => { - let mime_type = MimeType::parse(&data, &path); - response.mimetype(&mime_type).body(data) - } - Err(e) => { - debug_eprintln!("Failed to read file: {}", e); - response.status(404).body(Vec::new()) - } - } - } + crate::asset_protocol::asset_protocol_handler( + request, + asset_scope.clone(), + window_origin.clone(), + ) }); } From 30a4847af4125792cf1f675d7be3378c34428562 Mon Sep 17 00:00:00 2001 From: amrbashir Date: Thu, 2 Mar 2023 21:17:11 +0200 Subject: [PATCH 02/15] clippy --- core/tauri/src/asset_protocol.rs | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/core/tauri/src/asset_protocol.rs b/core/tauri/src/asset_protocol.rs index a02c81c90afc..ae349f0b90d4 100644 --- a/core/tauri/src/asset_protocol.rs +++ b/core/tauri/src/asset_protocol.rs @@ -1,3 +1,7 @@ +// Copyright 2019-2023 Tauri Programme within The Commons Conservancy +// SPDX-License-Identifier: Apache-2.0 +// SPDX-License-Identifier: MIT + #![cfg(protocol_asset)] use crate::api::file::SafePathBuf; @@ -56,7 +60,7 @@ pub fn asset_protocol_handler( let mime_type = { let mut magic_bytes = [0; 8192]; let old_pos = file.seek(SeekFrom::Current(0)).await?; - file.read(&mut magic_bytes).await?; + file.read_exact(&mut magic_bytes).await?; file.seek(SeekFrom::Start(old_pos)).await?; MimeType::parse(&magic_bytes, &path) }; @@ -172,7 +176,7 @@ pub fn asset_protocol_handler( } } else { resp = resp.header(CONTENT_LENGTH, len); - let mut buf = vec![0; len as usize]; + let mut buf = vec![0_u8; len as usize]; file.read_to_end(&mut buf).await?; resp.body(buf) }; @@ -182,11 +186,11 @@ pub fn asset_protocol_handler( } fn random_boundary() -> String { - let mut x = [0 as u8; 30]; + let mut x = [0_u8; 30]; rand::thread_rng().fill_bytes(&mut x); - (&x[..]) + (x[..]) .iter() - .map(|&x| format!("{:x}", x)) + .map(|&x| format!("{x:x}")) .fold(String::new(), |mut a, x| { a.push_str(x.as_str()); a From 48d1b7d065ee6e35f2a569529a63302ab15f53a4 Mon Sep 17 00:00:00 2001 From: amrbashir Date: Thu, 2 Mar 2023 21:57:47 +0200 Subject: [PATCH 03/15] update streaming example --- examples/streaming/main.rs | 180 ++++++++++++++++++++++++++----------- 1 file changed, 126 insertions(+), 54 deletions(-) diff --git a/examples/streaming/main.rs b/examples/streaming/main.rs index cee84c6b1640..1637a6258aaf 100644 --- a/examples/streaming/main.rs +++ b/examples/streaming/main.rs @@ -6,12 +6,11 @@ fn main() { use std::{ - cmp::min, - io::{Read, Seek, SeekFrom}, + io::{Read, Seek, SeekFrom, Write}, path::PathBuf, process::{Command, Stdio}, }; - use tauri::http::{HttpRange, ResponseBuilder}; + use tauri::http::{header::*, status::StatusCode, HttpRange, ResponseBuilder}; let video_file = PathBuf::from("test_video.mp4"); let video_url = @@ -38,8 +37,6 @@ fn main() { tauri::Builder::default() .invoke_handler(tauri::generate_handler![video_uri]) .register_uri_scheme_protocol("stream", move |_app, request| { - // prepare our response - let mut response = ResponseBuilder::new(); // get the file path let path = request.uri().strip_prefix("stream://localhost/").unwrap(); let path = percent_encoding::percent_decode(path.as_bytes()) @@ -47,65 +44,126 @@ fn main() { .to_string(); if path != "example/test_video.mp4" { - // return error 404 if it's not out video - return response.mimetype("text/plain").status(404).body(Vec::new()); + // return error 404 if it's not our video + return ResponseBuilder::new().status(404).body(Vec::new()); } - // read our file - let mut content = std::fs::File::open(&video_file)?; - let mut buf = Vec::new(); + let mut file = std::fs::File::open(&path)?; - // default status code - let mut status_code = 200; + let len = { + let old_pos = file.seek(SeekFrom::Current(0))?; + let len = file.seek(SeekFrom::End(0))?; + file.seek(SeekFrom::Start(old_pos))?; + len + }; + + let mut resp = ResponseBuilder::new().header(CONTENT_TYPE, "video/mp4"); // if the webview sent a range header, we need to send a 206 in return // Actually only macOS and Windows are supported. Linux will ALWAYS return empty headers. - if let Some(range) = request.headers().get("range") { - // Get the file size - let file_size = content.metadata().unwrap().len(); - - // we parse the range header with tauri helper - let range = HttpRange::parse(range.to_str().unwrap(), file_size).unwrap(); - // let support only 1 range for now - let first_range = range.first(); - if let Some(range) = first_range { - let mut real_length = range.length; - - // prevent max_length; - // specially on webview2 - if range.length > file_size / 3 { - // max size sent (400ko / request) - // as it's local file system we can afford to read more often - real_length = min(file_size - range.start, 1024 * 400); + let response = if let Some(range_header) = request.headers().get("range") { + let not_satisfiable = || { + ResponseBuilder::new() + .status(StatusCode::RANGE_NOT_SATISFIABLE) + .header(CONTENT_RANGE, format!("bytes */{len}")) + .body(vec![]) + }; + + // parse range header + let ranges = if let Ok(ranges) = HttpRange::parse(range_header.to_str()?, len) { + ranges + .iter() + // map the output back to spec range , example: 0-499 + .map(|r| (r.start, r.start + r.length - 1)) + .collect::>() + } else { + return not_satisfiable(); + }; + + /// only send 1MB or less at a time + const MAX_LEN: u64 = 1000 * 1024; + + if ranges.len() == 1 { + let &(start, mut end) = ranges.first().unwrap(); + + // check if a range is not satisfiable + // + // this should be already taken care of by HttpRange::parse + // but checking here again for extra assurance + if start >= len || end >= len || end < start { + return not_satisfiable(); } - // last byte we are reading, the length of the range include the last byte - // who should be skipped on the header - let last_byte = range.start + real_length - 1; - // partial content - status_code = 206; - - // Only macOS and Windows are supported, if you set headers in linux they are ignored - response = response - .header("Connection", "Keep-Alive") - .header("Accept-Ranges", "bytes") - .header("Content-Length", real_length) - .header( - "Content-Range", - format!("bytes {}-{}/{}", range.start, last_byte, file_size), - ); - - // FIXME: Add ETag support (caching on the webview) - - // seek our file bytes - content.seek(SeekFrom::Start(range.start))?; - content.take(real_length).read_to_end(&mut buf)?; + // adjust for MAX_LEN + end = start + (end - start).min(len - start).min(MAX_LEN - 1); + + file.seek(SeekFrom::Start(start))?; + + let mut stream: Box = Box::new(file); + if end + 1 < len { + stream = Box::new(stream.take(end + 1 - start)); + } + + let mut buf = Vec::new(); + stream.read_to_end(&mut buf)?; + + resp = resp.header(CONTENT_RANGE, format!("bytes {start}-{end}/{len}")); + resp = resp.header(CONTENT_LENGTH, end + 1 - start); + resp = resp.status(StatusCode::PARTIAL_CONTENT); + resp.body(buf) } else { - content.read_to_end(&mut buf)?; - } - } + let mut buf = Vec::new(); + let ranges = ranges + .iter() + .filter_map(|&(start, mut end)| { + // filter out unsatisfiable ranges + // + // this should be already taken care of by HttpRange::parse + // but checking here again for extra assurance + if start >= len || end >= len || end < start { + None + } else { + end = start + (end - start).min(len - start).min(MAX_LEN - 1); + Some((start, end)) + } + }) + .collect::>(); + + let boundary = "sadasdq2e"; + let boundary_sep = format!("\r\n--{boundary}\r\n"); + let boundary_closer = format!("\r\n--{boundary}\r\n"); + + resp = resp.header( + CONTENT_TYPE, + format!("multipart/byteranges; boundary={boundary}"), + ); + + drop(file); + + for (end, start) in ranges { + buf.write_all(boundary_sep.as_bytes())?; + buf.write_all(format!("{CONTENT_TYPE}: video/mp4\r\n").as_bytes())?; + buf.write_all(format!("{CONTENT_RANGE}: bytes {start}-{end}/{len}\r\n").as_bytes())?; + buf.write_all("\r\n".as_bytes())?; + + let mut file = std::fs::File::open(&path)?; + file.seek(SeekFrom::Start(start))?; + file + .take(if end + 1 < len { end + 1 - start } else { len }) + .read_to_end(&mut buf)?; + } + buf.write_all(boundary_closer.as_bytes())?; - response.mimetype("video/mp4").status(status_code).body(buf) + resp.body(buf) + } + } else { + resp = resp.header(CONTENT_LENGTH, len); + let mut buf = vec![0; len as usize]; + file.read_to_end(&mut buf)?; + resp.body(buf) + }; + + response }) .run(tauri::generate_context!( "../../examples/streaming/tauri.conf.json" @@ -127,3 +185,17 @@ fn video_uri() -> (&'static str, std::path::PathBuf) { #[cfg(not(feature = "protocol-asset"))] ("stream", "example/test_video.mp4".into()) } + +// fn random_boundary() -> String { +// use rand::RngCore; + +// let mut x = [0 as u8; 30]; +// rand::thread_rng().fill_bytes(&mut x); +// (&x[..]) +// .iter() +// .map(|&x| format!("{:x}", x)) +// .fold(String::new(), |mut a, x| { +// a.push_str(x.as_str()); +// a +// }) +// } From 6d3c5090d5557b4b94565a6021ca0e4b5a182700 Mon Sep 17 00:00:00 2001 From: amrbashir Date: Fri, 3 Mar 2023 02:44:43 +0200 Subject: [PATCH 04/15] clippy --- core/tauri/src/asset_protocol.rs | 4 ++-- examples/streaming/main.rs | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/core/tauri/src/asset_protocol.rs b/core/tauri/src/asset_protocol.rs index ae349f0b90d4..6a152a25a43e 100644 --- a/core/tauri/src/asset_protocol.rs +++ b/core/tauri/src/asset_protocol.rs @@ -51,7 +51,7 @@ pub fn asset_protocol_handler( let mut file = File::open(&path).await?; // get file length let len = { - let old_pos = file.seek(SeekFrom::Current(0)).await?; + let old_pos = file.stream_position().await?; let len = file.seek(SeekFrom::End(0)).await?; file.seek(SeekFrom::Start(old_pos)).await?; len @@ -59,7 +59,7 @@ pub fn asset_protocol_handler( // get file mime type let mime_type = { let mut magic_bytes = [0; 8192]; - let old_pos = file.seek(SeekFrom::Current(0)).await?; + let old_pos = file.stream_position().await?; file.read_exact(&mut magic_bytes).await?; file.seek(SeekFrom::Start(old_pos)).await?; MimeType::parse(&magic_bytes, &path) diff --git a/examples/streaming/main.rs b/examples/streaming/main.rs index 1637a6258aaf..043f1978a92e 100644 --- a/examples/streaming/main.rs +++ b/examples/streaming/main.rs @@ -51,7 +51,7 @@ fn main() { let mut file = std::fs::File::open(&path)?; let len = { - let old_pos = file.seek(SeekFrom::Current(0))?; + let old_pos = file.stream_position()?; let len = file.seek(SeekFrom::End(0))?; file.seek(SeekFrom::Start(old_pos))?; len From 732032f4a8bdaa40ecac72a4cb966bf33ea404ec Mon Sep 17 00:00:00 2001 From: amrbashir Date: Sat, 4 Mar 2023 19:24:10 +0200 Subject: [PATCH 05/15] use `Vec::with_capacity` instead --- core/tauri/src/asset_protocol.rs | 2 +- examples/streaming/main.rs | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/core/tauri/src/asset_protocol.rs b/core/tauri/src/asset_protocol.rs index 6a152a25a43e..6cf218528c22 100644 --- a/core/tauri/src/asset_protocol.rs +++ b/core/tauri/src/asset_protocol.rs @@ -176,7 +176,7 @@ pub fn asset_protocol_handler( } } else { resp = resp.header(CONTENT_LENGTH, len); - let mut buf = vec![0_u8; len as usize]; + let mut buf = Vec::with_capacity(len as usize); file.read_to_end(&mut buf).await?; resp.body(buf) }; diff --git a/examples/streaming/main.rs b/examples/streaming/main.rs index 043f1978a92e..ddada1036dd1 100644 --- a/examples/streaming/main.rs +++ b/examples/streaming/main.rs @@ -158,7 +158,7 @@ fn main() { } } else { resp = resp.header(CONTENT_LENGTH, len); - let mut buf = vec![0; len as usize]; + let mut buf = Vec::with_capacity(len as usize); file.read_to_end(&mut buf)?; resp.body(buf) }; From 51ad263f8ba52ffe6c8f7bf8dd37d43034d0250f Mon Sep 17 00:00:00 2001 From: amrbashir Date: Sun, 5 Mar 2023 00:48:55 +0200 Subject: [PATCH 06/15] cleanup --- core/tauri/src/asset_protocol.rs | 39 ++++++++++++++++---------------- 1 file changed, 20 insertions(+), 19 deletions(-) diff --git a/core/tauri/src/asset_protocol.rs b/core/tauri/src/asset_protocol.rs index 6cf218528c22..63f399a3a12d 100644 --- a/core/tauri/src/asset_protocol.rs +++ b/core/tauri/src/asset_protocol.rs @@ -8,14 +8,13 @@ use crate::api::file::SafePathBuf; use crate::scope::FsScope; use rand::RngCore; use std::io::SeekFrom; -use std::pin::Pin; use tauri_runtime::http::HttpRange; use tauri_runtime::http::{ header::*, status::StatusCode, MimeType, Request, Response, ResponseBuilder, }; use tauri_utils::debug_eprintln; use tokio::fs::File; -use tokio::io::{AsyncRead, AsyncReadExt, AsyncSeekExt, AsyncWriteExt}; +use tokio::io::{AsyncReadExt, AsyncSeekExt, AsyncWriteExt}; use url::Position; use url::Url; @@ -93,7 +92,7 @@ pub fn asset_protocol_handler( return not_satisfiable(); }; - /// only send 1MB or less at a time + /// The Maximum bytes we send in one range const MAX_LEN: u64 = 1000 * 1024; if ranges.len() == 1 { @@ -107,18 +106,15 @@ pub fn asset_protocol_handler( return not_satisfiable(); } - // adjust for MAX_LEN + // adjust end byte for MAX_LEN end = start + (end - start).min(len - start).min(MAX_LEN - 1); - file.seek(SeekFrom::Start(start)).await?; - - let mut stream: Pin> = Box::pin(file); - if end + 1 < len { - stream = Box::pin(stream.take(end + 1 - start)); - } + // calculate number of bytes needed to be read + let bytes_to_read = end + 1 - start; - let mut buf = Vec::new(); - stream.read_to_end(&mut buf).await?; + let mut buf = Vec::with_capacity(bytes_to_read as usize); + file.seek(SeekFrom::Start(start)).await?; + file.read_buf(&mut buf).await?; resp = resp.header(CONTENT_RANGE, format!("bytes {start}-{end}/{len}")); resp = resp.header(CONTENT_LENGTH, end + 1 - start); @@ -136,6 +132,7 @@ pub fn asset_protocol_handler( if start >= len || end >= len || end < start { None } else { + // adjust end byte for MAX_LEN end = start + (end - start).min(len - start).min(MAX_LEN - 1); Some((start, end)) } @@ -151,25 +148,29 @@ pub fn asset_protocol_handler( format!("multipart/byteranges; boundary={boundary}"), ); - drop(file); - for (end, start) in ranges { + // a new range is being written, write the range boundary buf.write_all(boundary_sep.as_bytes()).await?; + + // write the needed headers `Content-Type` and `Content-Range` buf .write_all(format!("{CONTENT_TYPE}: {mime_type}\r\n").as_bytes()) .await?; buf .write_all(format!("{CONTENT_RANGE}: bytes {start}-{end}/{len}\r\n").as_bytes()) .await?; + + // separator to indicate start of the range body buf.write_all("\r\n".as_bytes()).await?; - let mut file = File::open(&path).await?; + // calculate number of bytes needed to be read + let bytes_to_read = end + 1 - start; + + buf.reserve_exact(bytes_to_read as usize); file.seek(SeekFrom::Start(start)).await?; - file - .take(if end + 1 < len { end + 1 - start } else { len }) - .read_to_end(&mut buf) - .await?; + file.read_buf(&mut buf).await?; } + // all ranges have been written, write the closing boundary buf.write_all(boundary_closer.as_bytes()).await?; resp.body(buf) From 125fe61ff049f4e2ba94ec98625ed2dbb527182a Mon Sep 17 00:00:00 2001 From: amrbashir Date: Sun, 5 Mar 2023 01:01:14 +0200 Subject: [PATCH 07/15] more optimizations and cleanups --- core/tauri/src/asset_protocol.rs | 7 ++++-- examples/streaming/main.rs | 38 ++++++++++++++++++-------------- 2 files changed, 26 insertions(+), 19 deletions(-) diff --git a/core/tauri/src/asset_protocol.rs b/core/tauri/src/asset_protocol.rs index 63f399a3a12d..ea2f5e1802a0 100644 --- a/core/tauri/src/asset_protocol.rs +++ b/core/tauri/src/asset_protocol.rs @@ -48,6 +48,7 @@ pub fn asset_protocol_handler( crate::async_runtime::block_on(async move { let mut file = File::open(&path).await?; + // get file length let len = { let old_pos = file.stream_position().await?; @@ -55,6 +56,7 @@ pub fn asset_protocol_handler( file.seek(SeekFrom::Start(old_pos)).await?; len }; + // get file mime type let mime_type = { let mut magic_bytes = [0; 8192]; @@ -166,9 +168,10 @@ pub fn asset_protocol_handler( // calculate number of bytes needed to be read let bytes_to_read = end + 1 - start; - buf.reserve_exact(bytes_to_read as usize); + let mut local_buf = Vec::with_capacity(bytes_to_read as usize); file.seek(SeekFrom::Start(start)).await?; - file.read_buf(&mut buf).await?; + file.read_buf(&mut local_buf).await?; + buf.extend_from_slice(&local_buf); } // all ranges have been written, write the closing boundary buf.write_all(boundary_closer.as_bytes()).await?; diff --git a/examples/streaming/main.rs b/examples/streaming/main.rs index ddada1036dd1..1528e96c8eb6 100644 --- a/examples/streaming/main.rs +++ b/examples/streaming/main.rs @@ -50,6 +50,7 @@ fn main() { let mut file = std::fs::File::open(&path)?; + // get file length let len = { let old_pos = file.stream_position()?; let len = file.seek(SeekFrom::End(0))?; @@ -80,7 +81,7 @@ fn main() { return not_satisfiable(); }; - /// only send 1MB or less at a time + /// The Maximum bytes we send in one range const MAX_LEN: u64 = 1000 * 1024; if ranges.len() == 1 { @@ -94,18 +95,18 @@ fn main() { return not_satisfiable(); } - // adjust for MAX_LEN + // adjust end byte for MAX_LEN end = start + (end - start).min(len - start).min(MAX_LEN - 1); - file.seek(SeekFrom::Start(start))?; - - let mut stream: Box = Box::new(file); - if end + 1 < len { - stream = Box::new(stream.take(end + 1 - start)); - } + // calculate number of bytes needed to be read + let bytes_to_read = end + 1 - start; - let mut buf = Vec::new(); - stream.read_to_end(&mut buf)?; + // allocate a buf with a suitable capacity + let mut buf = Vec::with_capacity(bytes_to_read as usize); + // seek the file to the starting byte + file.seek(SeekFrom::Start(start))?; + // read the needed bytes + file.take(bytes_to_read).read_to_end(&mut buf)?; resp = resp.header(CONTENT_RANGE, format!("bytes {start}-{end}/{len}")); resp = resp.header(CONTENT_LENGTH, end + 1 - start); @@ -123,6 +124,7 @@ fn main() { if start >= len || end >= len || end < start { None } else { + // adjust end byte for MAX_LEN end = start + (end - start).min(len - start).min(MAX_LEN - 1); Some((start, end)) } @@ -138,19 +140,21 @@ fn main() { format!("multipart/byteranges; boundary={boundary}"), ); - drop(file); - for (end, start) in ranges { + // a new range is being written, write the range boundary buf.write_all(boundary_sep.as_bytes())?; + + // write the needed headers `Content-Type` and `Content-Range` buf.write_all(format!("{CONTENT_TYPE}: video/mp4\r\n").as_bytes())?; buf.write_all(format!("{CONTENT_RANGE}: bytes {start}-{end}/{len}\r\n").as_bytes())?; + + // separator to indicate start of the range body buf.write_all("\r\n".as_bytes())?; - let mut file = std::fs::File::open(&path)?; - file.seek(SeekFrom::Start(start))?; - file - .take(if end + 1 < len { end + 1 - start } else { len }) - .read_to_end(&mut buf)?; + let mut local_buf = vec![0_u8, bytes_to_read as usize]; + file.seek(SeekFrom::Start(start)); + file.read_exact(&mut local_buf); + buf.extend_from_slice(&local_buf); } buf.write_all(boundary_closer.as_bytes())?; From 7ff65c103d8ccf3368db4d55538ea98c959d00ea Mon Sep 17 00:00:00 2001 From: amrbashir Date: Sun, 5 Mar 2023 01:10:30 +0200 Subject: [PATCH 08/15] fix streaming example --- examples/streaming/main.rs | 3 +++ 1 file changed, 3 insertions(+) diff --git a/examples/streaming/main.rs b/examples/streaming/main.rs index 1528e96c8eb6..567eb63b4541 100644 --- a/examples/streaming/main.rs +++ b/examples/streaming/main.rs @@ -151,6 +151,9 @@ fn main() { // separator to indicate start of the range body buf.write_all("\r\n".as_bytes())?; + // calculate number of bytes needed to be read + let bytes_to_read = end + 1 - start; + let mut local_buf = vec![0_u8, bytes_to_read as usize]; file.seek(SeekFrom::Start(start)); file.read_exact(&mut local_buf); From d325810e3e6ab3ed0d904e19ced4c8df840bc510 Mon Sep 17 00:00:00 2001 From: amrbashir Date: Sun, 5 Mar 2023 01:24:28 +0200 Subject: [PATCH 09/15] adjust comments --- core/tauri/src/asset_protocol.rs | 4 ++-- examples/streaming/main.rs | 5 +++-- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/core/tauri/src/asset_protocol.rs b/core/tauri/src/asset_protocol.rs index ea2f5e1802a0..63cc057bd849 100644 --- a/core/tauri/src/asset_protocol.rs +++ b/core/tauri/src/asset_protocol.rs @@ -116,7 +116,7 @@ pub fn asset_protocol_handler( let mut buf = Vec::with_capacity(bytes_to_read as usize); file.seek(SeekFrom::Start(start)).await?; - file.read_buf(&mut buf).await?; + file.take(bytes_to_read).read_to_end(&mut buf).await?; resp = resp.header(CONTENT_RANGE, format!("bytes {start}-{end}/{len}")); resp = resp.header(CONTENT_LENGTH, end + 1 - start); @@ -162,7 +162,7 @@ pub fn asset_protocol_handler( .write_all(format!("{CONTENT_RANGE}: bytes {start}-{end}/{len}\r\n").as_bytes()) .await?; - // separator to indicate start of the range body + // write the separator to indicate the start of the range body buf.write_all("\r\n".as_bytes()).await?; // calculate number of bytes needed to be read diff --git a/examples/streaming/main.rs b/examples/streaming/main.rs index 567eb63b4541..740a343e78d9 100644 --- a/examples/streaming/main.rs +++ b/examples/streaming/main.rs @@ -148,17 +148,18 @@ fn main() { buf.write_all(format!("{CONTENT_TYPE}: video/mp4\r\n").as_bytes())?; buf.write_all(format!("{CONTENT_RANGE}: bytes {start}-{end}/{len}\r\n").as_bytes())?; - // separator to indicate start of the range body + // write the separator to indicate the start of the range body buf.write_all("\r\n".as_bytes())?; // calculate number of bytes needed to be read let bytes_to_read = end + 1 - start; - let mut local_buf = vec![0_u8, bytes_to_read as usize]; + let mut local_buf = vec![0_u8; bytes_to_read as usize]; file.seek(SeekFrom::Start(start)); file.read_exact(&mut local_buf); buf.extend_from_slice(&local_buf); } + // all ranges have been written, write the closing boundary buf.write_all(boundary_closer.as_bytes())?; resp.body(buf) From ca4e23890ffff211d5bb63a04697c1158f548dd4 Mon Sep 17 00:00:00 2001 From: amrbashir Date: Sun, 5 Mar 2023 01:46:01 +0200 Subject: [PATCH 10/15] clippy --- examples/streaming/main.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/examples/streaming/main.rs b/examples/streaming/main.rs index 740a343e78d9..4fb372b62e3a 100644 --- a/examples/streaming/main.rs +++ b/examples/streaming/main.rs @@ -155,8 +155,8 @@ fn main() { let bytes_to_read = end + 1 - start; let mut local_buf = vec![0_u8; bytes_to_read as usize]; - file.seek(SeekFrom::Start(start)); - file.read_exact(&mut local_buf); + file.seek(SeekFrom::Start(start))?; + file.read_exact(&mut local_buf)?; buf.extend_from_slice(&local_buf); } // all ranges have been written, write the closing boundary From 1293400e84806e4705c88ccb764a1064be42047e Mon Sep 17 00:00:00 2001 From: amrbashir Date: Thu, 13 Apr 2023 19:23:29 +0200 Subject: [PATCH 11/15] fix reading small files --- core/tauri/src/asset_protocol.rs | 41 ++++++++++++++++++++++---------- 1 file changed, 28 insertions(+), 13 deletions(-) diff --git a/core/tauri/src/asset_protocol.rs b/core/tauri/src/asset_protocol.rs index 63cc057bd849..0953b9d74330 100644 --- a/core/tauri/src/asset_protocol.rs +++ b/core/tauri/src/asset_protocol.rs @@ -58,12 +58,18 @@ pub fn asset_protocol_handler( }; // get file mime type - let mime_type = { - let mut magic_bytes = [0; 8192]; + let (mime_type, read_bytes) = { + let nbytes = len.min(8192); + let mut magic_buf = Vec::with_capacity(nbytes as usize); let old_pos = file.stream_position().await?; - file.read_exact(&mut magic_bytes).await?; + (&mut file).take(nbytes).read_to_end(&mut magic_buf).await?; file.seek(SeekFrom::Start(old_pos)).await?; - MimeType::parse(&magic_bytes, &path) + ( + MimeType::parse(&magic_buf, &path), + // return the `magic_bytes` if we read the whole file + // to avoid reading it again later if this is not a range request + if len < 8192 { Some(magic_buf) } else { None }, + ) }; resp = resp.header(CONTENT_TYPE, &mime_type); @@ -87,7 +93,7 @@ pub fn asset_protocol_handler( let ranges = if let Ok(ranges) = HttpRange::parse(&range_header, len) { ranges .iter() - // map the output back to spec range , example: 0-499 + // map the output to spec range , example: 0-499 .map(|r| (r.start, r.start + r.length - 1)) .collect::>() } else { @@ -97,6 +103,7 @@ pub fn asset_protocol_handler( /// The Maximum bytes we send in one range const MAX_LEN: u64 = 1000 * 1024; + // single-part range header if ranges.len() == 1 { let &(start, mut end) = ranges.first().unwrap(); @@ -112,17 +119,18 @@ pub fn asset_protocol_handler( end = start + (end - start).min(len - start).min(MAX_LEN - 1); // calculate number of bytes needed to be read - let bytes_to_read = end + 1 - start; + let nbytes = end + 1 - start; - let mut buf = Vec::with_capacity(bytes_to_read as usize); + let mut buf = Vec::with_capacity(nbytes as usize); file.seek(SeekFrom::Start(start)).await?; - file.take(bytes_to_read).read_to_end(&mut buf).await?; + file.take(nbytes).read_to_end(&mut buf).await?; resp = resp.header(CONTENT_RANGE, format!("bytes {start}-{end}/{len}")); resp = resp.header(CONTENT_LENGTH, end + 1 - start); resp = resp.status(StatusCode::PARTIAL_CONTENT); resp.body(buf) } else { + // multi-part range header let mut buf = Vec::new(); let ranges = ranges .iter() @@ -166,11 +174,11 @@ pub fn asset_protocol_handler( buf.write_all("\r\n".as_bytes()).await?; // calculate number of bytes needed to be read - let bytes_to_read = end + 1 - start; + let nbytes = end + 1 - start; - let mut local_buf = Vec::with_capacity(bytes_to_read as usize); + let mut local_buf = Vec::with_capacity(nbytes as usize); file.seek(SeekFrom::Start(start)).await?; - file.read_buf(&mut local_buf).await?; + (&mut file).take(nbytes).read_to_end(&mut local_buf).await?; buf.extend_from_slice(&local_buf); } // all ranges have been written, write the closing boundary @@ -179,9 +187,16 @@ pub fn asset_protocol_handler( resp.body(buf) } } else { + // avoid reading the file if we already read it + // as part of mime type detection + let buf = if let Some(b) = read_bytes { + b + } else { + let mut local_buf = Vec::with_capacity(len as usize); + file.read_to_end(&mut local_buf).await?; + local_buf + }; resp = resp.header(CONTENT_LENGTH, len); - let mut buf = Vec::with_capacity(len as usize); - file.read_to_end(&mut buf).await?; resp.body(buf) }; From 9ba71279827ae1e5b01763de7500118681005342 Mon Sep 17 00:00:00 2001 From: amrbashir Date: Tue, 23 May 2023 17:35:53 +0300 Subject: [PATCH 12/15] remove unused comment --- examples/streaming/main.rs | 23 ++++++++--------------- 1 file changed, 8 insertions(+), 15 deletions(-) diff --git a/examples/streaming/main.rs b/examples/streaming/main.rs index 4fb372b62e3a..181329ccefa0 100644 --- a/examples/streaming/main.rs +++ b/examples/streaming/main.rs @@ -4,6 +4,8 @@ #![cfg_attr(not(debug_assertions), windows_subsystem = "windows")] +use std::sync::{Arc, Mutex}; + fn main() { use std::{ io::{Read, Seek, SeekFrom, Write}, @@ -34,6 +36,9 @@ fn main() { assert!(video_file.exists()); } + // NOTE: for production use `rand` crate to generate a random boundary + let boundary_id = Arc::new(Mutex::new(0)); + tauri::Builder::default() .invoke_handler(tauri::generate_handler![video_uri]) .register_uri_scheme_protocol("stream", move |_app, request| { @@ -131,7 +136,9 @@ fn main() { }) .collect::>(); - let boundary = "sadasdq2e"; + let mut id = boundary_id.lock().unwrap(); + *id = *id + 1; + let boundary = format!("sadasq2e{id}"); let boundary_sep = format!("\r\n--{boundary}\r\n"); let boundary_closer = format!("\r\n--{boundary}\r\n"); @@ -193,17 +200,3 @@ fn video_uri() -> (&'static str, std::path::PathBuf) { #[cfg(not(feature = "protocol-asset"))] ("stream", "example/test_video.mp4".into()) } - -// fn random_boundary() -> String { -// use rand::RngCore; - -// let mut x = [0 as u8; 30]; -// rand::thread_rng().fill_bytes(&mut x); -// (&x[..]) -// .iter() -// .map(|&x| format!("{:x}", x)) -// .fold(String::new(), |mut a, x| { -// a.push_str(x.as_str()); -// a -// }) -// } From 837a95d12a688e4b6d57e14a6fff3e4e57f9c06e Mon Sep 17 00:00:00 2001 From: amrbashir Date: Tue, 23 May 2023 17:43:38 +0300 Subject: [PATCH 13/15] clippy --- examples/streaming/main.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/streaming/main.rs b/examples/streaming/main.rs index 181329ccefa0..6716a697f08e 100644 --- a/examples/streaming/main.rs +++ b/examples/streaming/main.rs @@ -137,7 +137,7 @@ fn main() { .collect::>(); let mut id = boundary_id.lock().unwrap(); - *id = *id + 1; + *id += 1; let boundary = format!("sadasq2e{id}"); let boundary_sep = format!("\r\n--{boundary}\r\n"); let boundary_closer = format!("\r\n--{boundary}\r\n"); From d18bfa2f3c1055775f9714787d734c44f23bcf7f Mon Sep 17 00:00:00 2001 From: Lucas Nogueira Date: Tue, 23 May 2023 15:21:45 -0300 Subject: [PATCH 14/15] fix example [skip ci] --- examples/streaming/main.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/examples/streaming/main.rs b/examples/streaming/main.rs index 6716a697f08e..087aa4566bbc 100644 --- a/examples/streaming/main.rs +++ b/examples/streaming/main.rs @@ -48,7 +48,7 @@ fn main() { .decode_utf8_lossy() .to_string(); - if path != "example/test_video.mp4" { + if path != "test_video.mp4" { // return error 404 if it's not our video return ResponseBuilder::new().status(404).body(Vec::new()); } @@ -198,5 +198,5 @@ fn video_uri() -> (&'static str, std::path::PathBuf) { } #[cfg(not(feature = "protocol-asset"))] - ("stream", "example/test_video.mp4".into()) + ("stream", "test_video.mp4".into()) } From 67546e8a510d7deebcb63d546197154a8fde0d1a Mon Sep 17 00:00:00 2001 From: Lucas Nogueira Date: Tue, 23 May 2023 15:24:12 -0300 Subject: [PATCH 15/15] update change file [skip ci] --- .changes/core-asset-protocol-streaming-crash.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.changes/core-asset-protocol-streaming-crash.md b/.changes/core-asset-protocol-streaming-crash.md index 4b9a71934d08..44bcd09a31af 100644 --- a/.changes/core-asset-protocol-streaming-crash.md +++ b/.changes/core-asset-protocol-streaming-crash.md @@ -1,5 +1,5 @@ --- -'tauri': 'patch' +'tauri': 'patch:enhance' --- -Fix crash when streaming large files through `asset` protocol. +Enhance the `asset` protocol to support streaming of large files.