diff --git a/crates/librqbit/src/api.rs b/crates/librqbit/src/api.rs index c3808289..aea4aba7 100644 --- a/crates/librqbit/src/api.rs +++ b/crates/librqbit/src/api.rs @@ -533,23 +533,23 @@ fn make_torrent_details( output_folder: String, ) -> Result { let files = info - .iter_filenames_and_lengths() + .iter_file_details() .context("error iterating filenames and lengths")? .enumerate() - .map(|(idx, (filename_it, length))| { - let name = match filename_it.to_string() { + .map(|(idx, d)| { + let name = match d.filename.to_string() { Ok(s) => s, Err(err) => { warn!("error reading filename: {:?}", err); "".to_string() } }; - let components = filename_it.to_vec().unwrap_or_default(); + let components = d.filename.to_vec().unwrap_or_default(); let included = only_files.map(|o| o.contains(&idx)).unwrap_or(true); TorrentDetailsResponseFile { name, components, - length, + length: d.len, included, } }) @@ -568,10 +568,11 @@ fn torrent_file_mime_type( info: &TorrentMetaV1Info, file_idx: usize, ) -> Result<&'static str> { - info.iter_filenames_and_lengths()? + info.iter_file_details()? .nth(file_idx) - .and_then(|(f, _)| { - f.iter_components() + .and_then(|d| { + d.filename + .iter_components() .last() .and_then(|r| r.ok()) .and_then(|s| mime_guess::from_path(s).first_raw()) diff --git a/crates/librqbit/src/create_torrent_file.rs b/crates/librqbit/src/create_torrent_file.rs index caecaef5..18ea5dc7 100644 --- a/crates/librqbit/src/create_torrent_file.rs +++ b/crates/librqbit/src/create_torrent_file.rs @@ -124,7 +124,13 @@ async fn create_torrent_raw<'a>( .components() .map(|c| osstr_to_bytes(c.as_os_str()).into()) .collect(); - output_files.push(TorrentMetaV1File { length, path }); + output_files.push(TorrentMetaV1File { + length, + path, + attr: None, + sha1: None, + symlink_path: None, + }); continue 'outer; } @@ -154,6 +160,9 @@ async fn create_torrent_raw<'a>( } else { Some(output_files) }, + attr: None, + sha1: None, + symlink_path: None, }) } diff --git a/crates/librqbit/src/file_info.rs b/crates/librqbit/src/file_info.rs index 0d122bc8..fe13b8ce 100644 --- a/crates/librqbit/src/file_info.rs +++ b/crates/librqbit/src/file_info.rs @@ -1,10 +1,13 @@ use std::path::PathBuf; +use librqbit_core::torrent_metainfo::FileDetailsAttrs; + #[derive(Debug, Clone)] pub struct FileInfo { pub relative_filename: PathBuf, pub offset_in_torrent: u64, pub piece_range: std::ops::Range, + pub attrs: FileDetailsAttrs, pub len: u64, } diff --git a/crates/librqbit/src/file_ops.rs b/crates/librqbit/src/file_ops.rs index 0442991d..1c84237b 100644 --- a/crates/librqbit/src/file_ops.rs +++ b/crates/librqbit/src/file_ops.rs @@ -21,6 +21,7 @@ use crate::{ pub fn update_hash_from_file( file_id: usize, + file_info: &FileInfo, mut pos: u64, files: &dyn TorrentStorage, hash: &mut Sha1, @@ -30,9 +31,15 @@ pub fn update_hash_from_file( let mut read = 0; while bytes_to_read > 0 { let chunk = std::cmp::min(buf.len(), bytes_to_read); - files - .pread_exact(file_id, pos, &mut buf[..chunk]) - .with_context(|| format!("failed reading chunk of size {chunk}, read so far {read}"))?; + if file_info.attrs.padding { + buf[..chunk].fill(0); + } else { + files + .pread_exact(file_id, pos, &mut buf[..chunk]) + .with_context(|| { + format!("failed reading chunk of size {chunk}, read so far {read}") + })?; + } bytes_to_read -= chunk; read += chunk; pos += chunk as u64; @@ -138,6 +145,7 @@ impl<'a> FileOps<'a> { if let Err(err) = update_hash_from_file( current_file.index, + current_file.fi, pos, self.files, &mut computed_hash, @@ -181,7 +189,8 @@ impl<'a> FileOps<'a> { let mut piece_remaining_bytes = piece_length as usize; - for (file_idx, (name, file_len)) in self.torrent.iter_filenames_and_lengths()?.enumerate() { + for (file_idx, fi) in self.file_infos.iter().enumerate() { + let file_len = fi.len; if absolute_offset > file_len { absolute_offset -= file_len; continue; @@ -198,6 +207,7 @@ impl<'a> FileOps<'a> { ); update_hash_from_file( file_idx, + fi, absolute_offset, self.files, &mut h, @@ -205,7 +215,10 @@ impl<'a> FileOps<'a> { to_read_in_file, ) .with_context(|| { - format!("error reading {to_read_in_file} bytes, file_id: {file_idx} (\"{name:?}\")") + format!( + "error reading {to_read_in_file} bytes, file_id: {file_idx} (\"{:?}\")", + fi.relative_filename + ) })?; piece_remaining_bytes -= to_read_in_file; @@ -246,7 +259,8 @@ impl<'a> FileOps<'a> { let mut absolute_offset = self.lengths.chunk_absolute_offset(chunk_info); let mut buf = result_buf; - for (file_idx, file_len) in self.torrent.iter_file_lengths()?.enumerate() { + for (file_idx, file_info) in self.file_infos.iter().enumerate() { + let file_len = file_info.len; if absolute_offset > file_len { absolute_offset -= file_len; continue; @@ -262,11 +276,15 @@ impl<'a> FileOps<'a> { absolute_offset, &chunk_info ); - self.files - .pread_exact(file_idx, absolute_offset, &mut buf[..to_read_in_file]) - .with_context(|| { - format!("error reading {file_idx} bytes, file_id: {to_read_in_file}") - })?; + if file_info.attrs.padding { + buf[..to_read_in_file].fill(0); + } else { + self.files + .pread_exact(file_idx, absolute_offset, &mut buf[..to_read_in_file]) + .with_context(|| { + format!("error reading {file_idx} bytes, file_id: {to_read_in_file}") + })?; + } buf = &mut buf[to_read_in_file..]; @@ -292,7 +310,8 @@ impl<'a> FileOps<'a> { let mut buf = data.block.as_ref(); let mut absolute_offset = self.lengths.chunk_absolute_offset(chunk_info); - for (file_idx, (name, file_len)) in self.torrent.iter_filenames_and_lengths()?.enumerate() { + for (file_idx, file_info) in self.file_infos.iter().enumerate() { + let file_len = file_info.len; if absolute_offset > file_len { absolute_offset -= file_len; continue; @@ -311,9 +330,16 @@ impl<'a> FileOps<'a> { to_write, absolute_offset ); - self.files - .pwrite_all(file_idx, absolute_offset, &buf[..to_write]) - .with_context(|| format!("error writing to file {file_idx} (\"{name:?}\")"))?; + if !file_info.attrs.padding { + self.files + .pwrite_all(file_idx, absolute_offset, &buf[..to_write]) + .with_context(|| { + format!( + "error writing to file {file_idx} (\"{:?}\")", + file_info.relative_filename + ) + })?; + } buf = &buf[to_write..]; if buf.is_empty() { break; diff --git a/crates/librqbit/src/http_api.rs b/crates/librqbit/src/http_api.rs index e70de8bd..69744ad8 100644 --- a/crates/librqbit/src/http_api.rs +++ b/crates/librqbit/src/http_api.rs @@ -153,10 +153,10 @@ impl HttpApi { let mut playlist_items = handle .shared() .info - .iter_filenames_and_lengths()? + .iter_file_details()? .enumerate() - .filter_map(|(file_idx, (filename, _))| { - let filename = filename.to_vec().ok()?.join("/"); + .filter_map(|(file_idx, file_details)| { + let filename = file_details.filename.to_vec().ok()?.join("/"); let is_playable = mime_guess::from_path(&filename) .first() .map(|mime| { diff --git a/crates/librqbit/src/session.rs b/crates/librqbit/src/session.rs index a8305da6..27238392 100644 --- a/crates/librqbit/src/session.rs +++ b/crates/librqbit/src/session.rs @@ -156,8 +156,9 @@ fn compute_only_files_regex>( ) -> anyhow::Result> { let filename_re = regex::Regex::new(filename_re).context("filename regex is incorrect")?; let mut only_files = Vec::new(); - for (idx, (filename, _)) in torrent.iter_filenames_and_lengths()?.enumerate() { - let full_path = filename + for (idx, fd) in torrent.iter_file_details()?.enumerate() { + let full_path = fd + .filename .to_pathbuf() .with_context(|| format!("filename of file {idx} is not valid utf8"))?; if filename_re.is_match(full_path.to_str().unwrap()) { @@ -191,12 +192,12 @@ fn compute_only_files( } (None, Some(filename_re)) => { let only_files = compute_only_files_regex(info, &filename_re)?; - for (idx, (filename, _)) in info.iter_filenames_and_lengths()?.enumerate() { + for (idx, fd) in info.iter_file_details()?.enumerate() { if !only_files.contains(&idx) { continue; } if !list_only { - info!(?filename, "will download"); + info!(filename=?fd.filename, "will download"); } } Ok(Some(only_files)) @@ -1043,8 +1044,8 @@ impl Session { info: &TorrentMetaV1Info, ) -> anyhow::Result> { let files = info - .iter_filenames_and_lengths()? - .map(|(f, l)| Ok((f.to_pathbuf()?, l))) + .iter_file_details()? + .map(|fd| Ok((fd.filename.to_pathbuf()?, fd.len))) .collect::>>()?; if files.len() < 2 { return Ok(None); @@ -1141,13 +1142,14 @@ impl Session { let lengths = Lengths::from_torrent(&info)?; let file_infos = info - .iter_file_details(&lengths)? + .iter_file_details_ext(&lengths)? .map(|fd| { Ok::<_, anyhow::Error>(FileInfo { - relative_filename: fd.filename.to_pathbuf()?, + relative_filename: fd.details.filename.to_pathbuf()?, offset_in_torrent: fd.offset, piece_range: fd.pieces, - len: fd.len, + len: fd.details.len, + attrs: fd.details.attrs(), }) }) .collect::>>()?; diff --git a/crates/librqbit/src/storage/filesystem/fs.rs b/crates/librqbit/src/storage/filesystem/fs.rs index 74ac5961..791e01e1 100644 --- a/crates/librqbit/src/storage/filesystem/fs.rs +++ b/crates/librqbit/src/storage/filesystem/fs.rs @@ -151,23 +151,26 @@ impl TorrentStorage for FilesystemStorage { fn init(&mut self, meta: &ManagedTorrentShared) -> anyhow::Result<()> { let mut files = Vec::::new(); - for file_details in meta.info.iter_file_details(&meta.lengths)? { + for file_details in meta.file_infos.iter() { let mut full_path = self.output_folder.clone(); - let relative_path = file_details - .filename - .to_pathbuf() - .context("error converting file to path")?; + let relative_path = &file_details.relative_filename; full_path.push(relative_path); std::fs::create_dir_all(full_path.parent().context("bug: no parent")?)?; - let file = if meta.options.allow_overwrite { - OpenOptions::new() - .create(true) - .truncate(false) - .read(true) - .write(true) - .open(&full_path) - .with_context(|| format!("error opening {full_path:?} in read/write mode"))? + let file = if file_details.attrs.padding { + OpenedFile::new_dummy() + } else if meta.options.allow_overwrite { + OpenedFile::new( + OpenOptions::new() + .create(true) + .truncate(false) + .read(true) + .write(true) + .open(&full_path) + .with_context(|| { + format!("error opening {full_path:?} in read/write mode") + })?, + ) } else { // create_new does not seem to work with read(true), so calling this twice. OpenOptions::new() @@ -180,9 +183,9 @@ impl TorrentStorage for FilesystemStorage { &full_path ) })?; - OpenOptions::new().read(true).write(true).open(&full_path)? + OpenedFile::new(OpenOptions::new().read(true).write(true).open(&full_path)?) }; - files.push(OpenedFile::new(file)); + files.push(file); } self.opened_files = files; diff --git a/crates/librqbit/src/storage/filesystem/opened_file.rs b/crates/librqbit/src/storage/filesystem/opened_file.rs index f1f4e067..051c652e 100644 --- a/crates/librqbit/src/storage/filesystem/opened_file.rs +++ b/crates/librqbit/src/storage/filesystem/opened_file.rs @@ -14,6 +14,12 @@ impl OpenedFile { } } + pub fn new_dummy() -> Self { + Self { + file: RwLock::new(None), + } + } + pub fn take(&self) -> anyhow::Result> { let mut f = self.file.write(); Ok(f.take()) diff --git a/crates/librqbit/src/torrent_state/initializing.rs b/crates/librqbit/src/torrent_state/initializing.rs index ed4e2288..b81ccb79 100644 --- a/crates/librqbit/src/torrent_state/initializing.rs +++ b/crates/librqbit/src/torrent_state/initializing.rs @@ -245,6 +245,9 @@ impl TorrentStateInitializing { .unwrap_or(true) { let now = Instant::now(); + if fi.attrs.padding { + continue; + } if let Err(err) = self.files.ensure_file_length(idx, fi.len) { warn!( "Error setting length for file {:?} to {}: {:#?}", diff --git a/crates/librqbit/src/upnp_server_adapter.rs b/crates/librqbit/src/upnp_server_adapter.rs index 7a2e2784..ab60f37f 100644 --- a/crates/librqbit/src/upnp_server_adapter.rs +++ b/crates/librqbit/src/upnp_server_adapter.rs @@ -68,10 +68,10 @@ impl TorrentFileTreeNode { let last_url_bit = torrent .shared() .info - .iter_filenames_and_lengths() + .iter_file_details() .ok() .and_then(|mut it| it.nth(fid)) - .and_then(|(fi, _)| fi.to_vec().ok()) + .and_then(|fd| fd.filename.to_vec().ok()) .map(|components| { components .into_iter() @@ -111,10 +111,10 @@ struct TorrentFileTree { } fn is_single_file_at_root(info: &TorrentMetaV1Info) -> bool { - info.iter_filenames_and_lengths() + info.iter_file_details() .into_iter() .flatten() - .flat_map(|(f, _)| f.iter_components()) + .flat_map(|fd| fd.filename.iter_components()) .nth(1) .is_none() } @@ -123,10 +123,10 @@ impl TorrentFileTree { fn build(torent_id: TorrentId, info: &TorrentMetaV1Info) -> anyhow::Result { if is_single_file_at_root(info) { let filename = info - .iter_filenames_and_lengths()? + .iter_file_details()? .next() .context("bug")? - .0 + .filename .iter_components() .last() .context("bug")??; @@ -159,8 +159,8 @@ impl TorrentFileTree { let mut name_cache = HashMap::new(); - for (fid, (fi, _)) in info.iter_filenames_and_lengths()?.enumerate() { - let components = match fi.to_vec() { + for (fid, fd) in info.iter_file_details()?.enumerate() { + let components = match fd.filename.to_vec() { Ok(v) => v, Err(_) => continue, }; @@ -402,9 +402,15 @@ mod tests { .map(|f| TorrentMetaV1File { length: 1, path: f.split("/").map(|f| f.as_bytes().into()).collect(), + attr: None, + sha1: None, + symlink_path: None, }) .collect(), ), + attr: None, + sha1: None, + symlink_path: None, }, comment: None, created_by: None, diff --git a/crates/librqbit_core/src/torrent_metainfo.rs b/crates/librqbit_core/src/torrent_metainfo.rs index ee61bd5b..89aa1850 100644 --- a/crates/librqbit_core/src/torrent_metainfo.rs +++ b/crates/librqbit_core/src/torrent_metainfo.rs @@ -6,6 +6,7 @@ use clone_to_owned::CloneToOwned; use itertools::Either; use serde::{Deserialize, Serialize}; use std::{iter::once, path::PathBuf}; +use tracing::debug; use crate::{hash_id::Id20, lengths::Lengths}; @@ -99,6 +100,16 @@ pub struct TorrentMetaV1Info { // Single-file mode #[serde(skip_serializing_if = "Option::is_none")] pub length: Option, + #[serde(default = "none", skip_serializing_if = "Option::is_none")] + pub attr: Option, + #[serde(default = "none", skip_serializing_if = "Option::is_none")] + pub sha1: Option, + #[serde( + default = "none", + rename = "symlink path", + skip_serializing_if = "Option::is_none" + )] + pub symlink_path: Option>, #[serde(skip_serializing_if = "Option::is_none")] pub md5sum: Option, @@ -174,14 +185,57 @@ where } } +#[derive(Default, Debug, Clone, Copy)] +pub struct FileDetailsAttrs { + pub symlink: bool, + pub hidden: bool, + pub padding: bool, + pub executable: bool, +} + pub struct FileDetails<'a, BufType> { pub filename: FileIteratorName<'a, BufType>, - pub offset: u64, pub len: u64, + + // bep-47 + attr: Option<&'a BufType>, + pub sha1: Option<&'a BufType>, + pub symlink_path: Option<&'a [BufType]>, +} + +impl<'a, BufType> FileDetails<'a, BufType> +where + BufType: AsRef<[u8]>, +{ + pub fn attrs(&self) -> FileDetailsAttrs { + let attrs = match self.attr { + Some(attrs) => attrs, + None => return FileDetailsAttrs::default(), + }; + let mut result = FileDetailsAttrs::default(); + for byte in attrs.as_ref().iter().copied() { + match byte { + b'l' => result.symlink = true, + b'h' => result.hidden = true, + b'p' => result.padding = true, + b'x' => result.executable = true, + other => debug!(attr = other, "unknown file attribute"), + } + } + result + } +} + +pub struct FileDetailsExt<'a, BufType> { + pub details: FileDetails<'a, BufType>, + // absolute offset in torrent if it was a flat blob of bytes + pub offset: u64, + + // the pieces that contain this file pub pieces: std::ops::Range, } -impl<'a, BufType> FileDetails<'a, BufType> { +impl<'a, BufType> FileDetailsExt<'a, BufType> { pub fn pieces_usize(&self) -> std::ops::Range { self.pieces.start as usize..self.pieces.end as usize } @@ -203,60 +257,77 @@ impl> TorrentMetaV1Info { } #[inline(never)] - pub fn iter_filenames_and_lengths( + pub fn iter_file_details( &self, - ) -> anyhow::Result, u64)>> { + ) -> anyhow::Result>> { match (self.length, self.files.as_ref()) { // Single-file - (Some(length), None) => Ok(Either::Left(once(( - FileIteratorName::Single(self.name.as_ref()), - length, - )))), + (Some(length), None) => Ok(Either::Left(once(FileDetails { + filename: FileIteratorName::Single(self.name.as_ref()), + len: length, + attr: self.attr.as_ref(), + sha1: self.sha1.as_ref(), + symlink_path: self.symlink_path.as_deref(), + }))), // Multi-file (None, Some(files)) => { if files.is_empty() { anyhow::bail!("expected multi-file torrent to have at least one file") } - Ok(Either::Right( - files - .iter() - .map(|f| (FileIteratorName::Tree(&f.path), f.length)), - )) + Ok(Either::Right(files.iter().map(|f| FileDetails { + filename: FileIteratorName::Tree(&f.path), + len: f.length, + attr: f.attr.as_ref(), + sha1: f.sha1.as_ref(), + symlink_path: f.symlink_path.as_deref(), + }))) } _ => anyhow::bail!("torrent can't be both in single and multi-file mode"), } } pub fn iter_file_lengths(&self) -> anyhow::Result + '_> { - Ok(self.iter_filenames_and_lengths()?.map(|(_, l)| l)) + Ok(self.iter_file_details()?.map(|d| d.len)) } // NOTE: lenghts MUST be construced with Lenghts::from_torrent, otherwise // the yielded results will be garbage. - pub fn iter_file_details<'a>( + pub fn iter_file_details_ext<'a>( &'a self, lengths: &'a Lengths, - ) -> anyhow::Result> + 'a> { - Ok(self - .iter_filenames_and_lengths()? - .scan(0u64, |acc_offset, (filename, len)| { - let offset = *acc_offset; - *acc_offset += len; - Some(FileDetails { - filename, - pieces: lengths.iter_pieces_within_offset(offset, len), - offset, - len, - }) - })) + ) -> anyhow::Result> + 'a> { + Ok(self.iter_file_details()?.scan(0u64, |acc_offset, details| { + let offset = *acc_offset; + *acc_offset += details.len; + Some(FileDetailsExt { + pieces: lengths.iter_pieces_within_offset(offset, details.len), + details, + offset, + }) + })) } } +const fn none() -> Option { + None +} + #[derive(Deserialize, Serialize, Debug, Clone, PartialEq, Eq)] pub struct TorrentMetaV1File { pub length: u64, pub path: Vec, + + #[serde(default = "none", skip_serializing_if = "Option::is_none")] + pub attr: Option, + #[serde(default = "none", skip_serializing_if = "Option::is_none")] + pub sha1: Option, + #[serde( + default = "none", + rename = "symlink path", + skip_serializing_if = "Option::is_none" + )] + pub symlink_path: Option>, } impl TorrentMetaV1File @@ -282,6 +353,9 @@ where TorrentMetaV1File { length: self.length, path: self.path.clone_to_owned(within_buffer), + attr: self.attr.clone_to_owned(within_buffer), + sha1: self.sha1.clone_to_owned(within_buffer), + symlink_path: self.symlink_path.clone_to_owned(within_buffer), } } } @@ -300,6 +374,9 @@ where length: self.length, md5sum: self.md5sum.clone_to_owned(within_buffer), files: self.files.clone_to_owned(within_buffer), + attr: self.attr.clone_to_owned(within_buffer), + sha1: self.sha1.clone_to_owned(within_buffer), + symlink_path: self.symlink_path.clone_to_owned(within_buffer), } } } diff --git a/crates/rqbit/src/main.rs b/crates/rqbit/src/main.rs index 68061251..a03c87ca 100644 --- a/crates/rqbit/src/main.rs +++ b/crates/rqbit/src/main.rs @@ -762,17 +762,15 @@ async fn async_main(opts: Opts, cancel: CancellationToken) -> anyhow::Result<()> only_files, .. }) => { - for (idx, (filename, len)) in - info.iter_filenames_and_lengths()?.enumerate() - { + for (idx, fd) in info.iter_file_details()?.enumerate() { let included = match &only_files { Some(files) => files.contains(&idx), None => true, }; info!( - "File {}, size {}{}", - filename.to_string()?, - SF::new(len), + "File {:?}, size {}{}", + fd.filename, + SF::new(fd.len), if included { "" } else { ", will skip" } ) }