[#6357] feat (gvfs-fuse): Passing file type argument in the Filesystem::stat() to improve the performance of open_dal_filesystem

clients/filesystem-fuse/src/default_raw_filesystem.rs

@@ -26,6 +26,7 @@ use crate::opened_file::{FileHandle, OpenFileFlags, OpenedFile};
 use crate::opened_file_manager::OpenedFileManager;
 use async_trait::async_trait;
 use bytes::Bytes;
+use fuse3::FileType::{Directory, RegularFile};
 use fuse3::{Errno, FileType};
 use std::collections::HashMap;
 use std::ffi::OsStr;
@@ -86,7 +87,12 @@ impl<T: PathFileSystem> DefaultRawFileSystem<T> {
             None => {
                 // allocate new file id
                 file_stat.set_file_id(parent_file_id, self.next_file_id());
-                file_manager.insert(file_stat.parent_file_id, file_stat.file_id, &file_stat.path);
+                file_manager.insert(
+                    file_stat.parent_file_id,
+                    file_stat.file_id,
+                    &file_stat.path,
+                    file_stat.kind,
+                );
             }
             Some(file) => {
                 // use the exist file id
@@ -130,9 +136,15 @@ impl<T: PathFileSystem> DefaultRawFileSystem<T> {
         file_manager.remove(path);
     }
 
-    async fn insert_file_entry_locked(&self, parent_file_id: u64, file_id: u64, path: &Path) {
+    async fn insert_file_entry_locked(
+        &self,
+        parent_file_id: u64,
+        file_id: u64,
+        path: &Path,
+        kind: FileType,
+    ) {
         let mut file_manager = self.file_entry_manager.write().await;
-        file_manager.insert(parent_file_id, file_id, path);
+        file_manager.insert(parent_file_id, file_id, path, kind);
     }
 
     fn get_meta_file_stat(&self) -> FileStat {
@@ -159,13 +171,15 @@ impl<T: PathFileSystem> RawFileSystem for DefaultRawFileSystem<T> {
             ROOT_DIR_PARENT_FILE_ID,
             ROOT_DIR_FILE_ID,
             Path::new(ROOT_DIR_PATH),
+            Directory,
         )
         .await;
 
         self.insert_file_entry_locked(
             ROOT_DIR_FILE_ID,
             FS_META_FILE_ID,
             Path::new(FS_META_FILE_PATH),
+            RegularFile,
         )
         .await;
         self.fs.init().await
@@ -197,7 +211,7 @@ impl<T: PathFileSystem> RawFileSystem for DefaultRawFileSystem<T> {
         }
 
         let file_entry = self.get_file_entry(file_id).await?;
-        let mut file_stat = self.fs.stat(&file_entry.path).await?;
+        let mut file_stat = self.fs.stat(&file_entry.path, file_entry.kind).await?;
         file_stat.set_file_id(file_entry.parent_file_id, file_entry.file_id);
         Ok(file_stat)
     }
@@ -209,7 +223,7 @@ impl<T: PathFileSystem> RawFileSystem for DefaultRawFileSystem<T> {
 
         let parent_file_entry = self.get_file_entry(parent_file_id).await?;
         let path = parent_file_entry.path.join(name);
-        let mut file_stat = self.fs.stat(&path).await?;
+        let mut file_stat = self.fs.lookup(&path).await?;
         // fill the file id to file stat
         self.resolve_file_id_to_filestat(&mut file_stat, parent_file_id)
             .await;
@@ -270,6 +284,7 @@ impl<T: PathFileSystem> RawFileSystem for DefaultRawFileSystem<T> {
             parent_file_id,
             file_without_id.file_stat.file_id,
             &file_without_id.file_stat.path,
+            RegularFile,
         )
         .await;
 
@@ -287,7 +302,7 @@ impl<T: PathFileSystem> RawFileSystem for DefaultRawFileSystem<T> {
         filestat.set_file_id(parent_file_id, self.next_file_id());
 
         // insert the new file to file entry manager
-        self.insert_file_entry_locked(parent_file_id, filestat.file_id, &filestat.path)
+        self.insert_file_entry_locked(parent_file_id, filestat.file_id, &filestat.path, Directory)
             .await;
         Ok(filestat.file_id)
     }
@@ -401,6 +416,7 @@ struct FileEntry {
     file_id: u64,
     parent_file_id: u64,
     path: PathBuf,
+    kind: FileType,
 }
 
 /// FileEntryManager is manage all the file entries in memory. it is used manger the file relationship and name mapping.
@@ -428,11 +444,12 @@ impl FileEntryManager {
         self.file_path_map.get(path).cloned()
     }
 
-    fn insert(&mut self, parent_file_id: u64, file_id: u64, path: &Path) {
+    fn insert(&mut self, parent_file_id: u64, file_id: u64, path: &Path, kind: FileType) {
         let file_entry = FileEntry {
             file_id,
             parent_file_id,
             path: path.into(),
+            kind: kind,
         };
         self.file_id_map.insert(file_id, file_entry.clone());
         self.file_path_map.insert(path.into(), file_entry);
@@ -452,7 +469,7 @@ mod tests {
     #[test]
     fn test_file_entry_manager() {
         let mut manager = FileEntryManager::new();
-        manager.insert(1, 2, Path::new("a/b"));
+        manager.insert(1, 2, Path::new("a/b"), Directory);
         let file = manager.get_file_entry_by_id(2).unwrap();
         assert_eq!(file.file_id, 2);
         assert_eq!(file.parent_file_id, 1);

clients/filesystem-fuse/src/filesystem.rs

@@ -115,7 +115,10 @@ pub(crate) trait PathFileSystem: Send + Sync {
     async fn init(&self) -> Result<()>;
 
     /// Get the file stat by file path, if the file exists, return the file stat
-    async fn stat(&self, path: &Path) -> Result<FileStat>;
+    async fn stat(&self, path: &Path, kind: FileType) -> Result<FileStat>;
+
+    /// Lookup the file stat by file path, if the file exists, return the file stat
+    async fn lookup(&self, path: &Path) -> Result<FileStat>;
 
     /// Read the directory by file path, if the directory exists, return the file stat list
     async fn read_dir(&self, path: &Path) -> Result<Vec<FileStat>>;
@@ -318,7 +321,7 @@ pub(crate) mod tests {
 
         pub(crate) async fn test_path_file_system(&mut self) {
             // test root dir
-            let resutl = self.fs.stat(Path::new("/")).await;
+            let resutl = self.fs.stat(Path::new("/"), Directory).await;
             assert!(resutl.is_ok());
             let root_file_stat = resutl.unwrap();
             self.assert_file_stat(&root_file_stat, Path::new("/"), Directory, 0);
@@ -347,7 +350,7 @@ pub(crate) mod tests {
         }
 
         async fn test_stat_file(&mut self, path: &Path, expect_kind: FileType, expect_size: u64) {
-            let file_stat = self.fs.stat(path).await;
+            let file_stat = self.fs.stat(path, expect_kind).await;
             assert!(file_stat.is_ok());
             let file_stat = file_stat.unwrap();
             self.assert_file_stat(&file_stat, path, expect_kind, expect_size);
@@ -403,7 +406,7 @@ pub(crate) mod tests {
         }
 
         async fn test_file_not_found(&self, path: &Path) {
-            let not_found_file = self.fs.stat(path).await;
+            let not_found_file = self.fs.stat(path, RegularFile).await;
             assert!(not_found_file.is_err());
         }
 

clients/filesystem-fuse/src/gravitino_fileset_filesystem.rs

@@ -21,7 +21,7 @@ use crate::filesystem::{FileStat, FileSystemCapacity, FileSystemContext, PathFil
 use crate::gravitino_client::GravitinoClient;
 use crate::opened_file::{OpenFileFlags, OpenedFile};
 use async_trait::async_trait;
-use fuse3::Errno;
+use fuse3::{Errno, FileType};
 use std::path::{Path, PathBuf};
 
 /// GravitinoFileSystem is a filesystem that is associated with a fileset in Gravitino.
@@ -74,9 +74,16 @@ impl PathFileSystem for GravitinoFilesetFileSystem {
         self.physical_fs.init().await
     }
 
-    async fn stat(&self, path: &Path) -> Result<FileStat> {
+    async fn stat(&self, path: &Path, kind: FileType) -> Result<FileStat> {
         let raw_path = self.gvfs_path_to_raw_path(path);
-        let mut file_stat = self.physical_fs.stat(&raw_path).await?;
+        let mut file_stat = self.physical_fs.stat(&raw_path, kind).await?;
+        file_stat.path = self.raw_path_to_gvfs_path(&file_stat.path)?;
+        Ok(file_stat)
+    }
+
+    async fn lookup(&self, path: &Path) -> Result<FileStat> {
+        let raw_path = self.gvfs_path_to_raw_path(path);
+        let mut file_stat = self.physical_fs.lookup(&raw_path).await?;
         file_stat.path = self.raw_path_to_gvfs_path(&file_stat.path)?;
         Ok(file_stat)
     }

clients/filesystem-fuse/src/memory_filesystem.rs

@@ -70,7 +70,7 @@ impl PathFileSystem for MemoryFileSystem {
         Ok(())
     }
 
-    async fn stat(&self, path: &Path) -> Result<FileStat> {
+    async fn stat(&self, path: &Path, _kind: FileType) -> Result<FileStat> {
         self.file_map
             .read()
             .unwrap()
@@ -79,6 +79,10 @@ impl PathFileSystem for MemoryFileSystem {
             .ok_or(Errno::from(libc::ENOENT))
     }
 
+    async fn lookup(&self, path: &Path) -> Result<FileStat> {
+        self.stat(path, RegularFile).await
+    }
+
     async fn read_dir(&self, path: &Path) -> Result<Vec<FileStat>> {
         let file_map = self.file_map.read().unwrap();
 
@@ -92,7 +96,7 @@ impl PathFileSystem for MemoryFileSystem {
     }
 
     async fn open_file(&self, path: &Path, flags: OpenFileFlags) -> Result<OpenedFile> {
-        let file_stat = self.stat(path).await?;
+        let file_stat = self.stat(path, RegularFile).await?;
         let mut opened_file = OpenedFile::new(file_stat);
         match opened_file.file_stat.kind {
             Directory => Ok(opened_file),

clients/filesystem-fuse/src/open_dal_filesystem.rs

@@ -66,7 +66,24 @@ impl PathFileSystem for OpenDalFileSystem {
         Ok(())
     }
 
-    async fn stat(&self, path: &Path) -> Result<FileStat> {
+    async fn stat(&self, path: &Path, kind: FileType) -> Result<FileStat> {
+        let file_name = match kind {
+            Directory => build_dir_path(path),
+            _ => path.to_string_lossy().to_string(),
+        };
+        let meta = self
+            .op
+            .stat(&file_name)
+            .await
+            .map_err(opendal_error_to_errno)?;
+
+        let mut file_stat = FileStat::new_file_filestat_with_path(path, 0);
+        self.opendal_meta_to_file_stat(&meta, &mut file_stat);
+
+        Ok(file_stat)
+    }
+
+    async fn lookup(&self, path: &Path) -> Result<FileStat> {
         let file_name = path.to_string_lossy().to_string();
         let meta_result = self.op.stat(&file_name).await;
 
@@ -114,7 +131,7 @@ impl PathFileSystem for OpenDalFileSystem {
     }
 
     async fn open_file(&self, path: &Path, flags: OpenFileFlags) -> Result<OpenedFile> {
-        let file_stat = self.stat(path).await?;
+        let file_stat = self.stat(path, RegularFile).await?;
         debug_assert!(file_stat.kind == RegularFile);
 
         let mut file = OpenedFile::new(file_stat);
@@ -155,7 +172,7 @@ impl PathFileSystem for OpenDalFileSystem {
     }
 
     async fn open_dir(&self, path: &Path, _flags: OpenFileFlags) -> Result<OpenedFile> {
-        let file_stat = self.stat(path).await?;
+        let file_stat = self.stat(path, Directory).await?;
         debug_assert!(file_stat.kind == Directory);
 
         let opened_file = OpenedFile::new(file_stat);
@@ -185,7 +202,7 @@ impl PathFileSystem for OpenDalFileSystem {
             .create_dir(&dir_name)
             .await
             .map_err(opendal_error_to_errno)?;
-        let file_stat = self.stat(path).await?;
+        let file_stat = self.stat(path, Directory).await?;
         Ok(file_stat)
     }
 

clients/filesystem-fuse/src/s3_filesystem.rs

@@ -24,6 +24,7 @@ use crate::open_dal_filesystem::OpenDalFileSystem;
 use crate::opened_file::{OpenFileFlags, OpenedFile};
 use crate::utils::{parse_location, GvfsResult};
 use async_trait::async_trait;
+use fuse3::FileType;
 use log::error;
 use opendal::layers::LoggingLayer;
 use opendal::services::S3;
@@ -94,8 +95,12 @@ impl PathFileSystem for S3FileSystem {
         Ok(())
     }
 
-    async fn stat(&self, path: &Path) -> Result<FileStat> {
-        self.open_dal_fs.stat(path).await
+    async fn stat(&self, path: &Path, kind: FileType) -> Result<FileStat> {
+        self.open_dal_fs.stat(path, kind).await
+    }
+
+    async fn lookup(&self, path: &Path) -> Result<FileStat> {
+        self.open_dal_fs.lookup(path).await
     }
 
     async fn read_dir(&self, path: &Path) -> Result<Vec<FileStat>> {

clients/filesystem-fuse/tests/bin/gvfs_fuse.sh

@@ -37,8 +37,6 @@ check_gvfs_fuse_ready() {
 }
 
 start_gvfs_fuse() {
-  MOUNT_DIR=$CLIENT_FUSE_DIR/target/gvfs
-
   umount $MOUNT_DIR > /dev/null 2>&1 || true
   if [ ! -d "$MOUNT_DIR" ]; then
     echo "Create the mount point"

clients/filesystem-fuse/tests/bin/run_fuse_testers.sh

@@ -27,6 +27,8 @@ source ./gvfs_fuse.sh
 source ./localstatck.sh
 
 TEST_CONFIG_FILE=$CLIENT_FUSE_DIR/target/debug/gvfs-fuse.toml
+MOUNT_DIR=$CLIENT_FUSE_DIR/target/gvfs
+
 
 start_servers() {
   start_localstack

clients/filesystem-fuse/tests/bin/run_s3fs_testers.sh

@@ -70,4 +70,3 @@ else
   exit 1
 fi
 
-