diff --git a/Cargo.lock b/Cargo.lock index e00746f594a2..dc76105ab303 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4237,6 +4237,7 @@ dependencies = [ "tar", "tempfile", "tokio", + "tokio-stream", "tokio-util", "tracing", "tracing-subscriber", @@ -7105,9 +7106,9 @@ dependencies = [ [[package]] name = "tokio-stream" -version = "0.1.11" +version = "0.1.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d660770404473ccd7bc9f8b28494a811bc18542b915c0855c51e8f419d5223ce" +checksum = "8fb52b74f05dbf495a8fba459fdc331812b96aa086d9eb78101fa0d4569c3313" dependencies = [ "futures-core", "pin-project-lite", diff --git a/Cargo.toml b/Cargo.toml index 95d7563700a2..80b1d0a9c9a7 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -91,6 +91,7 @@ regex = { version = "1.6.0" } serde_yaml = { version = "0.9.16" } serde-wasm-bindgen = { version = "0.4.5" } tokio = { version = "1.23.0", features = ["full", "tracing"] } +tokio-stream = { version = "0.1.12", features = ["fs"] } tokio-util = { version = "0.7.4", features = ["full"] } wasm-bindgen = { version = "0.2.83", features = ["serde-serialize"] } wasm-bindgen-test = { version = "0.3.33" } diff --git a/build/base/src/fs/wrappers.rs b/build/base/src/fs/wrappers.rs index 71ed4647b4a0..5f9794c2ed41 100644 --- a/build/base/src/fs/wrappers.rs +++ b/build/base/src/fs/wrappers.rs @@ -5,6 +5,7 @@ use crate::prelude::*; +use std::fs::DirEntry; use std::fs::File; use std::fs::Metadata; @@ -20,6 +21,12 @@ pub fn metadata>(path: P) -> Result { std::fs::metadata(&path).anyhow_err() } +/// See [std::fs::symlink_metadata]. +#[context("Failed to obtain symlink metadata for file: {}", path.as_ref().display())] +pub fn symlink_metadata>(path: P) -> Result { + std::fs::symlink_metadata(&path).anyhow_err() +} + /// See [std::fs::copy]. #[context("Failed to copy file from {} to {}", from.as_ref().display(), to.as_ref().display())] pub fn copy(from: impl AsRef, to: impl AsRef) -> Result { @@ -39,9 +46,15 @@ pub fn read(path: impl AsRef) -> Result> { } /// See [std::fs::read_dir]. -#[context("Failed to read the directory: {}", path.as_ref().display())] -pub fn read_dir(path: impl AsRef) -> Result { - std::fs::read_dir(&path).anyhow_err() +pub fn read_dir(path: impl AsRef) -> Result>> { + let path = path.as_ref().to_path_buf(); + let read_dir = std::fs::read_dir(&path) + .map_err(|e| anyhow!("Failed to read the directory: '{}'. Error: {}", path.display(), e))?; + Ok(read_dir.into_iter().map(move |elem_result| { + elem_result.map_err(|e| { + anyhow!("Failed to read sub-item from the directory '{}'. Error: {}", path.display(), e) + }) + })) } /// See [std::fs::read_to_string]. diff --git a/build/build/src/project/backend.rs b/build/build/src/project/backend.rs index b71af472220d..633dae0c1e89 100644 --- a/build/build/src/project/backend.rs +++ b/build/build/src/project/backend.rs @@ -88,8 +88,8 @@ pub async fn bundled_engine_versions( let mut ret = vec![]; let mut dir_reader = ide_ci::fs::tokio::read_dir(&project_manager_bundle.dist).await?; - while let Some(entry) = dir_reader.next_entry().await? { - if entry.metadata().await?.is_dir() { + while let Some(entry) = dir_reader.try_next().await? { + if ide_ci::fs::tokio::metadata(&entry.path()).await?.is_dir() { ret.push(Version::from_str(entry.file_name().as_str())?); } } diff --git a/build/ci_utils/Cargo.toml b/build/ci_utils/Cargo.toml index 0e1467022ab8..4ec09d4079c7 100644 --- a/build/ci_utils/Cargo.toml +++ b/build/ci_utils/Cargo.toml @@ -72,6 +72,7 @@ sysinfo = "0.26.2" tar = "0.4.37" tempfile = "3.2.0" tokio = { workspace = true } +tokio-stream = { workspace = true } tokio-util = { workspace = true } tracing = { version = "0.1.37" } tracing-subscriber = { version = "0.3.11", features = ["env-filter"] } diff --git a/build/ci_utils/src/fs/wrappers/tokio.rs b/build/ci_utils/src/fs/wrappers/tokio.rs index 287bc9253fc3..77a048d89271 100644 --- a/build/ci_utils/src/fs/wrappers/tokio.rs +++ b/build/ci_utils/src/fs/wrappers/tokio.rs @@ -11,6 +11,16 @@ pub fn metadata>(path: P) -> BoxFuture<'static, Result>(path: P) -> BoxFuture<'static, Result> { + let path = path.as_ref().to_owned(); + tokio::fs::symlink_metadata(path.clone()) + .with_context(move || { + format!("Failed to obtain symlink metadata for file: {}", path.display()) + }) + .boxed() +} + #[context("Failed to open path for reading: {}", path.as_ref().display())] pub async fn open(path: impl AsRef) -> Result { File::open(&path).await.anyhow_err() @@ -38,9 +48,20 @@ pub async fn create_dir_all(path: impl AsRef) -> Result { tokio::fs::create_dir_all(&path).await.anyhow_err() } -#[context("Failed to read the directory: {}", path.as_ref().display())] -pub async fn read_dir(path: impl AsRef) -> Result { - tokio::fs::read_dir(&path).await.anyhow_err() +pub async fn read_dir( + path: impl AsRef, +) -> Result>> { + let path = path.as_ref().to_path_buf(); + let read_dir = tokio::fs::read_dir(&path) + .await + .with_context(|| format!("Failed to read the directory: {}", path.display()))?; + + let stream = tokio_stream::wrappers::ReadDirStream::new(read_dir); + Ok(stream.map(move |entry| { + entry.with_context(|| { + format!("Failed to get entry when reading the directory: {}", path.display()) + }) + })) } #[context("Failed to remove directory with the subtree: {}", path.as_ref().display())] diff --git a/build/enso-formatter/src/lib.rs b/build/enso-formatter/src/lib.rs index 64ca45f8fd79..634b09418edb 100644 --- a/build/enso-formatter/src/lib.rs +++ b/build/enso-formatter/src/lib.rs @@ -367,8 +367,9 @@ pub struct RustSourcePath { /// uses non-documented API and is slow as well (8 seconds for the whole codebase). It should be /// possible to improve the latter solution to get good performance, but it seems way harder than it /// should be. -pub async fn process_path(path: impl AsRef, action: Action) -> Result { - let paths = discover_paths(&path)?; +#[context("Enso Formatter: failed to process root path '{}'.", path.as_ref().display())] +pub async fn process_path(path: impl AsRef + Copy, action: Action) -> Result { + let paths = discover_paths(path)?; let total = paths.len(); let mut hash_map = HashMap::::new(); for (i, sub_path) in paths.iter().enumerate() { @@ -396,12 +397,14 @@ pub async fn process_path(path: impl AsRef, action: Action) -> Result { } /// Discover all paths containing Rust sources, recursively. +#[context("Discovering Rust paths failed for '{}' failed.", path.as_ref().display())] pub fn discover_paths(path: impl AsRef) -> Result> { let mut vec = Vec::default(); - discover_paths_internal(&mut vec, path, false)?; + discover_paths_internal(&mut vec, &path, false)?; Ok(vec) } +#[context("Discovering paths failed for '{}' failed.", path.as_ref().display())] pub fn discover_paths_internal( vec: &mut Vec, path: impl AsRef, @@ -409,7 +412,11 @@ pub fn discover_paths_internal( ) -> Result { use ide_ci::fs; let path = path.as_ref(); - let md = fs::metadata(path)?; + // Below we use `symlink_metadata` instead of `metadata` because the latter follows symlinks. + // We don't want the formatter to fail if it encounters a symlink to a non-existing file. + // All files to be formatted should be reachable from the repository root without following + // any symlinks. + let md = fs::symlink_metadata(path)?; if md.is_dir() && !path.file_name().contains(&"target") { let dir_name = path.file_name(); // FIXME: This should cover 'tests' folder also, but only the files that contain actual