Merge pull request #269 from ikatson/bep-47

BEP-47 padding files + refactor related code

crates/librqbit/src/api.rs

@@ -533,23 +533,23 @@ fn make_torrent_details(
     output_folder: String,
 ) -> Result<TorrentDetailsResponse> {
     let files = info
-        .iter_filenames_and_lengths()
+        .iter_file_details()
         .context("error iterating filenames and lengths")?
         .enumerate()
-        .map(|(idx, (filename_it, length))| {
-            let name = match filename_it.to_string() {
+        .map(|(idx, d)| {
+            let name = match d.filename.to_string() {
                 Ok(s) => s,
                 Err(err) => {
                     warn!("error reading filename: {:?}", err);
                     "<INVALID NAME>".to_string()
                 }
             };
-            let components = filename_it.to_vec().unwrap_or_default();
+            let components = d.filename.to_vec().unwrap_or_default();
             let included = only_files.map(|o| o.contains(&idx)).unwrap_or(true);
             TorrentDetailsResponseFile {
                 name,
                 components,
-                length,
+                length: d.len,
                 included,
             }
         })
@@ -568,10 +568,11 @@ fn torrent_file_mime_type(
     info: &TorrentMetaV1Info<ByteBufOwned>,
     file_idx: usize,
 ) -> Result<&'static str> {
-    info.iter_filenames_and_lengths()?
+    info.iter_file_details()?
         .nth(file_idx)
-        .and_then(|(f, _)| {
-            f.iter_components()
+        .and_then(|d| {
+            d.filename
+                .iter_components()
                 .last()
                 .and_then(|r| r.ok())
                 .and_then(|s| mime_guess::from_path(s).first_raw())

crates/librqbit/src/create_torrent_file.rs

@@ -124,7 +124,13 @@ async fn create_torrent_raw<'a>(
                     .components()
                     .map(|c| osstr_to_bytes(c.as_os_str()).into())
                     .collect();
-                output_files.push(TorrentMetaV1File { length, path });
+                output_files.push(TorrentMetaV1File {
+                    length,
+                    path,
+                    attr: None,
+                    sha1: None,
+                    symlink_path: None,
+                });
                 continue 'outer;
             }
 
@@ -154,6 +160,9 @@ async fn create_torrent_raw<'a>(
         } else {
             Some(output_files)
         },
+        attr: None,
+        sha1: None,
+        symlink_path: None,
     })
 }
 

crates/librqbit/src/file_info.rs

@@ -1,10 +1,13 @@
 use std::path::PathBuf;
 
+use librqbit_core::torrent_metainfo::FileDetailsAttrs;
+
 #[derive(Debug, Clone)]
 pub struct FileInfo {
     pub relative_filename: PathBuf,
     pub offset_in_torrent: u64,
     pub piece_range: std::ops::Range<u32>,
+    pub attrs: FileDetailsAttrs,
     pub len: u64,
 }
 

crates/librqbit/src/file_ops.rs

@@ -21,6 +21,7 @@ use crate::{
 
 pub fn update_hash_from_file<Sha1: ISha1>(
     file_id: usize,
+    file_info: &FileInfo,
     mut pos: u64,
     files: &dyn TorrentStorage,
     hash: &mut Sha1,
@@ -30,9 +31,15 @@ pub fn update_hash_from_file<Sha1: ISha1>(
     let mut read = 0;
     while bytes_to_read > 0 {
         let chunk = std::cmp::min(buf.len(), bytes_to_read);
-        files
-            .pread_exact(file_id, pos, &mut buf[..chunk])
-            .with_context(|| format!("failed reading chunk of size {chunk}, read so far {read}"))?;
+        if file_info.attrs.padding {
+            buf[..chunk].fill(0);
+        } else {
+            files
+                .pread_exact(file_id, pos, &mut buf[..chunk])
+                .with_context(|| {
+                    format!("failed reading chunk of size {chunk}, read so far {read}")
+                })?;
+        }
         bytes_to_read -= chunk;
         read += chunk;
         pos += chunk as u64;
@@ -138,6 +145,7 @@ impl<'a> FileOps<'a> {
 
                 if let Err(err) = update_hash_from_file(
                     current_file.index,
+                    current_file.fi,
                     pos,
                     self.files,
                     &mut computed_hash,
@@ -181,7 +189,8 @@ impl<'a> FileOps<'a> {
 
         let mut piece_remaining_bytes = piece_length as usize;
 
-        for (file_idx, (name, file_len)) in self.torrent.iter_filenames_and_lengths()?.enumerate() {
+        for (file_idx, fi) in self.file_infos.iter().enumerate() {
+            let file_len = fi.len;
             if absolute_offset > file_len {
                 absolute_offset -= file_len;
                 continue;
@@ -198,14 +207,18 @@ impl<'a> FileOps<'a> {
             );
             update_hash_from_file(
                 file_idx,
+                fi,
                 absolute_offset,
                 self.files,
                 &mut h,
                 &mut buf,
                 to_read_in_file,
             )
             .with_context(|| {
-                format!("error reading {to_read_in_file} bytes, file_id: {file_idx} (\"{name:?}\")")
+                format!(
+                    "error reading {to_read_in_file} bytes, file_id: {file_idx} (\"{:?}\")",
+                    fi.relative_filename
+                )
             })?;
 
             piece_remaining_bytes -= to_read_in_file;
@@ -246,7 +259,8 @@ impl<'a> FileOps<'a> {
         let mut absolute_offset = self.lengths.chunk_absolute_offset(chunk_info);
         let mut buf = result_buf;
 
-        for (file_idx, file_len) in self.torrent.iter_file_lengths()?.enumerate() {
+        for (file_idx, file_info) in self.file_infos.iter().enumerate() {
+            let file_len = file_info.len;
             if absolute_offset > file_len {
                 absolute_offset -= file_len;
                 continue;
@@ -262,11 +276,15 @@ impl<'a> FileOps<'a> {
                 absolute_offset,
                 &chunk_info
             );
-            self.files
-                .pread_exact(file_idx, absolute_offset, &mut buf[..to_read_in_file])
-                .with_context(|| {
-                    format!("error reading {file_idx} bytes, file_id: {to_read_in_file}")
-                })?;
+            if file_info.attrs.padding {
+                buf[..to_read_in_file].fill(0);
+            } else {
+                self.files
+                    .pread_exact(file_idx, absolute_offset, &mut buf[..to_read_in_file])
+                    .with_context(|| {
+                        format!("error reading {file_idx} bytes, file_id: {to_read_in_file}")
+                    })?;
+            }
 
             buf = &mut buf[to_read_in_file..];
 
@@ -292,7 +310,8 @@ impl<'a> FileOps<'a> {
         let mut buf = data.block.as_ref();
         let mut absolute_offset = self.lengths.chunk_absolute_offset(chunk_info);
 
-        for (file_idx, (name, file_len)) in self.torrent.iter_filenames_and_lengths()?.enumerate() {
+        for (file_idx, file_info) in self.file_infos.iter().enumerate() {
+            let file_len = file_info.len;
             if absolute_offset > file_len {
                 absolute_offset -= file_len;
                 continue;
@@ -311,9 +330,16 @@ impl<'a> FileOps<'a> {
                 to_write,
                 absolute_offset
             );
-            self.files
-                .pwrite_all(file_idx, absolute_offset, &buf[..to_write])
-                .with_context(|| format!("error writing to file {file_idx} (\"{name:?}\")"))?;
+            if !file_info.attrs.padding {
+                self.files
+                    .pwrite_all(file_idx, absolute_offset, &buf[..to_write])
+                    .with_context(|| {
+                        format!(
+                            "error writing to file {file_idx} (\"{:?}\")",
+                            file_info.relative_filename
+                        )
+                    })?;
+            }
             buf = &buf[to_write..];
             if buf.is_empty() {
                 break;

crates/librqbit/src/http_api.rs

@@ -153,10 +153,10 @@ impl HttpApi {
             let mut playlist_items = handle
                 .shared()
                 .info
-                .iter_filenames_and_lengths()?
+                .iter_file_details()?
                 .enumerate()
-                .filter_map(|(file_idx, (filename, _))| {
-                    let filename = filename.to_vec().ok()?.join("/");
+                .filter_map(|(file_idx, file_details)| {
+                    let filename = file_details.filename.to_vec().ok()?.join("/");
                     let is_playable = mime_guess::from_path(&filename)
                         .first()
                         .map(|mime| {

crates/librqbit/src/session.rs

@@ -156,8 +156,9 @@ fn compute_only_files_regex<ByteBuf: AsRef<[u8]>>(
 ) -> anyhow::Result<Vec<usize>> {
     let filename_re = regex::Regex::new(filename_re).context("filename regex is incorrect")?;
     let mut only_files = Vec::new();
-    for (idx, (filename, _)) in torrent.iter_filenames_and_lengths()?.enumerate() {
-        let full_path = filename
+    for (idx, fd) in torrent.iter_file_details()?.enumerate() {
+        let full_path = fd
+            .filename
             .to_pathbuf()
             .with_context(|| format!("filename of file {idx} is not valid utf8"))?;
         if filename_re.is_match(full_path.to_str().unwrap()) {
@@ -191,12 +192,12 @@ fn compute_only_files(
         }
         (None, Some(filename_re)) => {
             let only_files = compute_only_files_regex(info, &filename_re)?;
-            for (idx, (filename, _)) in info.iter_filenames_and_lengths()?.enumerate() {
+            for (idx, fd) in info.iter_file_details()?.enumerate() {
                 if !only_files.contains(&idx) {
                     continue;
                 }
                 if !list_only {
-                    info!(?filename, "will download");
+                    info!(filename=?fd.filename, "will download");
                 }
             }
             Ok(Some(only_files))
@@ -1043,8 +1044,8 @@ impl Session {
         info: &TorrentMetaV1Info<ByteBufOwned>,
     ) -> anyhow::Result<Option<PathBuf>> {
         let files = info
-            .iter_filenames_and_lengths()?
-            .map(|(f, l)| Ok((f.to_pathbuf()?, l)))
+            .iter_file_details()?
+            .map(|fd| Ok((fd.filename.to_pathbuf()?, fd.len)))
             .collect::<anyhow::Result<Vec<(PathBuf, u64)>>>()?;
         if files.len() < 2 {
             return Ok(None);
@@ -1141,13 +1142,14 @@ impl Session {
 
             let lengths = Lengths::from_torrent(&info)?;
             let file_infos = info
-                .iter_file_details(&lengths)?
+                .iter_file_details_ext(&lengths)?
                 .map(|fd| {
                     Ok::<_, anyhow::Error>(FileInfo {
-                        relative_filename: fd.filename.to_pathbuf()?,
+                        relative_filename: fd.details.filename.to_pathbuf()?,
                         offset_in_torrent: fd.offset,
                         piece_range: fd.pieces,
-                        len: fd.len,
+                        len: fd.details.len,
+                        attrs: fd.details.attrs(),
                     })
                 })
                 .collect::<anyhow::Result<Vec<FileInfo>>>()?;

crates/librqbit/src/storage/filesystem/fs.rs

@@ -151,23 +151,26 @@ impl TorrentStorage for FilesystemStorage {
 
     fn init(&mut self, meta: &ManagedTorrentShared) -> anyhow::Result<()> {
         let mut files = Vec::<OpenedFile>::new();
-        for file_details in meta.info.iter_file_details(&meta.lengths)? {
+        for file_details in meta.file_infos.iter() {
             let mut full_path = self.output_folder.clone();
-            let relative_path = file_details
-                .filename
-                .to_pathbuf()
-                .context("error converting file to path")?;
+            let relative_path = &file_details.relative_filename;
             full_path.push(relative_path);
 
             std::fs::create_dir_all(full_path.parent().context("bug: no parent")?)?;
-            let file = if meta.options.allow_overwrite {
-                OpenOptions::new()
-                    .create(true)
-                    .truncate(false)
-                    .read(true)
-                    .write(true)
-                    .open(&full_path)
-                    .with_context(|| format!("error opening {full_path:?} in read/write mode"))?
+            let file = if file_details.attrs.padding {
+                OpenedFile::new_dummy()
+            } else if meta.options.allow_overwrite {
+                OpenedFile::new(
+                    OpenOptions::new()
+                        .create(true)
+                        .truncate(false)
+                        .read(true)
+                        .write(true)
+                        .open(&full_path)
+                        .with_context(|| {
+                            format!("error opening {full_path:?} in read/write mode")
+                        })?,
+                )
             } else {
                 // create_new does not seem to work with read(true), so calling this twice.
                 OpenOptions::new()
@@ -180,9 +183,9 @@ impl TorrentStorage for FilesystemStorage {
                             &full_path
                         )
                     })?;
-                OpenOptions::new().read(true).write(true).open(&full_path)?
+                OpenedFile::new(OpenOptions::new().read(true).write(true).open(&full_path)?)
             };
-            files.push(OpenedFile::new(file));
+            files.push(file);
         }
 
         self.opened_files = files;

crates/librqbit/src/storage/filesystem/opened_file.rs

@@ -14,6 +14,12 @@ impl OpenedFile {
         }
     }
 
+    pub fn new_dummy() -> Self {
+        Self {
+            file: RwLock::new(None),
+        }
+    }
+
     pub fn take(&self) -> anyhow::Result<Option<File>> {
         let mut f = self.file.write();
         Ok(f.take())

crates/librqbit/src/torrent_state/initializing.rs

@@ -245,6 +245,9 @@ impl TorrentStateInitializing {
                     .unwrap_or(true)
                 {
                     let now = Instant::now();
+                    if fi.attrs.padding {
+                        continue;
+                    }
                     if let Err(err) = self.files.ensure_file_length(idx, fi.len) {
                         warn!(
                             "Error setting length for file {:?} to {}: {:#?}",