From 909fe5cd6cb8281627c17ca81a55218d532cd4f5 Mon Sep 17 00:00:00 2001 From: yuhui Date: Thu, 16 Jan 2025 11:29:35 +0800 Subject: [PATCH 1/9] gvfs fuse Support daemonize --- clients/filesystem-fuse/Cargo.toml | 2 + clients/filesystem-fuse/src/command_args.rs | 57 ++++++ clients/filesystem-fuse/src/config.rs | 12 +- clients/filesystem-fuse/src/fuse_server.rs | 3 +- clients/filesystem-fuse/src/main.rs | 183 +++++++++++++++--- clients/filesystem-fuse/src/s3_filesystem.rs | 2 +- .../filesystem-fuse/tests/bin/gvfs_fuse.sh | 6 +- clients/filesystem-fuse/tests/fuse_test.rs | 2 +- 8 files changed, 224 insertions(+), 43 deletions(-) create mode 100644 clients/filesystem-fuse/src/command_args.rs diff --git a/clients/filesystem-fuse/Cargo.toml b/clients/filesystem-fuse/Cargo.toml index 3760bd5285f..e6006836894 100644 --- a/clients/filesystem-fuse/Cargo.toml +++ b/clients/filesystem-fuse/Cargo.toml @@ -35,7 +35,9 @@ name = "gvfs_fuse" [dependencies] async-trait = "0.1" bytes = "1.6.0" +clap = { version = "4.5.24", features = ["derive"] } config = "0.13" +daemonize = "0.5.0" dashmap = "6.1.0" fuse3 = { version = "0.8.1", "features" = ["tokio-runtime", "unprivileged"] } futures-util = "0.3.30" diff --git a/clients/filesystem-fuse/src/command_args.rs b/clients/filesystem-fuse/src/command_args.rs new file mode 100644 index 00000000000..4e64eefa59b --- /dev/null +++ b/clients/filesystem-fuse/src/command_args.rs @@ -0,0 +1,57 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +use clap::{Parser, Subcommand}; + +#[derive(Parser, Debug)] +#[command( + name = "gvfs-fuse", + version = "1.0", + about = "A FUSE-based file system client" +)] +pub(crate) struct Arguments { + #[command(subcommand)] + pub(crate) command: Commands, +} + +#[derive(Subcommand, Debug)] +pub(crate) enum Commands { + Mount { + #[arg(help = "Mount point for the filesystem")] + mount_point: String, + + #[arg(help = "Location of GVFS fileset URI")] + location: String, + + #[arg(short, long)] + config: Option, + + #[arg(short, long, help = "Debug level", default_value_t = 0)] + debug: u8, + + #[arg(short, long, default_value_t = false, help = "Run in foreground")] + foreground: bool, + }, + Umount { + #[arg(help = "Mount point to umount")] + mount_point: String, + + #[arg(short, long, help = "Force umount")] + force: bool, + }, +} diff --git a/clients/filesystem-fuse/src/config.rs b/clients/filesystem-fuse/src/config.rs index 17908fd08fc..96b720b4ed4 100644 --- a/clients/filesystem-fuse/src/config.rs +++ b/clients/filesystem-fuse/src/config.rs @@ -205,19 +205,19 @@ impl AppConfig { .unwrap_or_else(|e| panic!("Failed to set default for {}: {}", entity.name, e)) } - pub fn from_file(config_file_path: Option<&str>) -> GvfsResult { + pub fn from_file(config_file_path: Option) -> GvfsResult { let builder = Self::crete_default_config_builder(); let config_path = { if config_file_path.is_some() { let path = config_file_path.unwrap(); //check config file exists - if fs::metadata(path).is_err() { + if fs::metadata(&path).is_err() { return Err( ConfigNotFound.to_error("The configuration file not found".to_string()) ); } - info!("Use configuration file: {}", path); + info!("Use configuration file: {}", &path); path } else { //use default config @@ -232,11 +232,11 @@ impl AppConfig { CONF_FUSE_CONFIG_PATH.default ); } - CONF_FUSE_CONFIG_PATH.default + CONF_FUSE_CONFIG_PATH.default.to_string() } }; let config = builder - .add_source(config::File::with_name(config_path).required(true)) + .add_source(config::File::with_name(&config_path).required(true)) .build(); if let Err(e) = config { let msg = format!("Failed to build configuration: {}", e); @@ -302,7 +302,7 @@ mod test { #[test] fn test_config_from_file() { - let config = AppConfig::from_file(Some("tests/conf/config_test.toml")).unwrap(); + let config = AppConfig::from_file(Some("tests/conf/config_test.toml".to_string())).unwrap(); assert_eq!(config.fuse.file_mask, 0o644); assert_eq!(config.fuse.dir_mask, 0o755); assert_eq!(config.filesystem.block_size, 8192); diff --git a/clients/filesystem-fuse/src/fuse_server.rs b/clients/filesystem-fuse/src/fuse_server.rs index a059686e16c..0e8bd518334 100644 --- a/clients/filesystem-fuse/src/fuse_server.rs +++ b/clients/filesystem-fuse/src/fuse_server.rs @@ -20,6 +20,7 @@ use crate::utils::GvfsResult; use fuse3::raw::{Filesystem, Session}; use fuse3::MountOptions; use log::{error, info}; +use std::path::Path; use std::process::exit; use std::sync::Arc; use tokio::select; @@ -46,7 +47,7 @@ impl FuseServer { /// Starts the FUSE filesystem and blocks until it is stopped. pub async fn start(&self, fuse_fs: impl Filesystem + Sync + 'static) -> GvfsResult<()> { //check if the mount point exists - if !std::path::Path::new(&self.mount_point).exists() { + if !Path::new(&self.mount_point).exists() { error!("Mount point {} does not exist", self.mount_point); exit(libc::ENOENT); } diff --git a/clients/filesystem-fuse/src/main.rs b/clients/filesystem-fuse/src/main.rs index 3534e033465..e86e943cf16 100644 --- a/clients/filesystem-fuse/src/main.rs +++ b/clients/filesystem-fuse/src/main.rs @@ -16,49 +16,170 @@ * specific language governing permissions and limitations * under the License. */ -use fuse3::Errno; +mod command_args; + +use crate::command_args::Commands; +use clap::Parser; +use daemonize::Daemonize; use gvfs_fuse::config::AppConfig; use gvfs_fuse::{gvfs_mount, gvfs_unmount}; use log::{error, info}; +use std::fs::OpenOptions; +use std::path::Path; +use std::process::{exit, Command}; +use std::{env, io}; +use tokio::runtime::Runtime; use tokio::signal; +use tokio::signal::unix::{signal, SignalKind}; -#[tokio::main] -async fn main() -> fuse3::Result<()> { - tracing_subscriber::fmt().init(); +fn make_daemon() { + let log_file_name = "/tmp/gvfs-fuse.log"; + let log_file = OpenOptions::new() + .create(true) + .append(true) + .open(log_file_name) + .unwrap(); + let log_err_file = OpenOptions::new().write(true).open(log_file_name).unwrap(); - // todo need inmprove the args parsing - let args: Vec = std::env::args().collect(); - let (mount_point, mount_from, config_path) = match args.len() { - 4 => (args[1].clone(), args[2].clone(), args[3].clone()), - _ => { - error!("Usage: {} ", args[0]); - return Err(Errno::from(libc::EINVAL)); - } - }; + let cwd = env::current_dir(); + if let Err(e) = cwd { + error!("Error getting current directory: {}", e); + return; + } + let cwd = cwd.unwrap(); + + let daemonize = Daemonize::new() + .pid_file("/tmp/gvfs-fuse.pid") + .chown_pid_file(true) + .working_directory(&cwd) + .stdout(log_file) + .stderr(log_err_file); - //todo(read config file from args) - let config = AppConfig::from_file(Some(&config_path)); - if let Err(e) = &config { - error!("Failed to load config: {:?}", e); - return Err(Errno::from(libc::EINVAL)); + match daemonize.start() { + Ok(_) => info!("Gvfs-fuse Daemon started successfully"), + Err(e) => { + error!("Gvfs-fuse Daemon failed to start: {:?}", e); + exit(-1) + } } - let config = config.unwrap(); - let handle = tokio::spawn(async move { - let result = gvfs_mount(&mount_point, &mount_from, &config).await; - if let Err(e) = result { - error!("Failed to mount gvfs: {:?}", e); - return Err(Errno::from(libc::EINVAL)); +} + +fn mount_fuse(config: AppConfig, mount_point: String, target: String) -> io::Result<()> { + let rt = Runtime::new().unwrap(); + rt.block_on(async { + let handle = tokio::spawn(async move { + let result = gvfs_mount(&mount_point, &target, &config).await; + if let Err(e) = result { + error!("Failed to mount gvfs: {:?}", e); + return Err(io::Error::from(io::ErrorKind::InvalidInput)); + } + Ok(()) + }); + + let mut term_signal = signal(SignalKind::terminate())?; + tokio::select! { + _ = handle => {} + _ = signal::ctrl_c() => { + info!("Received Ctrl+C, unmounting gvfs...") + } + _ = term_signal.recv()=> { + info!("Received SIGTERM, unmounting gvfs...") + } } + + let _ = gvfs_unmount().await; Ok(()) - }); + }) +} + +#[cfg(target_os = "macos")] +fn do_umount(mp: &str, force: bool) -> std::io::Result<()> { + let cmd_result = if force { + Command::new("umount").arg("-f").arg(mp).output() + } else { + Command::new("umount").arg(mp).output() + }; - tokio::select! { - _ = handle => {} - _ = signal::ctrl_c() => { - info!("Received Ctrl+C, unmounting gvfs..."); + handle_command_result(cmd_result) +} + +#[cfg(target_os = "linux")] +fn do_umount(mp: &str, force: bool) -> std::io::Result<()> { + let cmd_result = + if Path::new("/bin/fusermount").exists() || Path::new("/usr/bin/fusermount").exists() { + if force { + Command::new("fusermount").arg("-uz").arg(mp).output() + } else { + Command::new("fusermount").arg("-u").arg(mp).output() + } + } else if force { + Command::new("umount").arg("-l").arg(mp).output() + } else { + Command::new("umount").arg(mp).output() + }; + + handle_command_result(cmd_result) +} + +fn handle_command_result(cmd_result: io::Result) -> io::Result<()> { + match cmd_result { + Ok(output) => { + if !output.status.success() { + let stderr = String::from_utf8_lossy(&output.stderr); + Err(io::Error::new(io::ErrorKind::Other, stderr.to_string())) + } else { + Ok(()) + } } + Err(e) => Err(e), } +} - let _ = gvfs_unmount().await; - Ok(()) +#[cfg(not(any(target_os = "macos", target_os = "linux")))] +fn do_umount(_mp: &str, _force: bool) -> std::io::Result<()> { + Err(std::io::Error::new( + std::io::ErrorKind::Other, + format!("OS {} is not supported", env::consts::OS), + )) +} + +fn main() -> Result<(), i32> { + tracing_subscriber::fmt().init(); + let args = command_args::Arguments::parse(); + match args.command { + Commands::Mount { + mount_point, + location, + config, + debug: _, + foreground, + } => { + let app_config = AppConfig::from_file(config); + if let Err(e) = &app_config { + error!("Failed to load config: {:?}", e); + return Err(-1); + }; + let app_config = app_config.unwrap(); + let result = if foreground { + mount_fuse(app_config, mount_point, location) + } else { + make_daemon(); + mount_fuse(app_config, mount_point, location) + }; + + if let Err(e) = result { + error!("Failed to mount gvfs: {:?}", e.to_string()); + return Err(-1); + }; + Ok(()) + } + Commands::Umount { mount_point, force } => { + let result = do_umount(&mount_point, force); + if let Err(e) = result { + error!("Failed to unmount gvfs: {:?}", e.to_string()); + return Err(-1); + }; + Ok(()) + } + } } diff --git a/clients/filesystem-fuse/src/s3_filesystem.rs b/clients/filesystem-fuse/src/s3_filesystem.rs index 35a091b3fe1..e397d31ccf3 100644 --- a/clients/filesystem-fuse/src/s3_filesystem.rs +++ b/clients/filesystem-fuse/src/s3_filesystem.rs @@ -255,7 +255,7 @@ pub(crate) mod tests { config_file_name = source_file_name; } - AppConfig::from_file(Some(config_file_name)).unwrap() + AppConfig::from_file(Some(config_file_name.to_string())).unwrap() } #[tokio::test] diff --git a/clients/filesystem-fuse/tests/bin/gvfs_fuse.sh b/clients/filesystem-fuse/tests/bin/gvfs_fuse.sh index e706d8e2c0d..b6a04f3b2a0 100644 --- a/clients/filesystem-fuse/tests/bin/gvfs_fuse.sh +++ b/clients/filesystem-fuse/tests/bin/gvfs_fuse.sh @@ -52,14 +52,14 @@ start_gvfs_fuse() { make build echo "Starting gvfs-fuse-daemon" - $CLIENT_FUSE_DIR/target/debug/gvfs-fuse $MOUNT_DIR $MOUNT_FROM_LOCATION $TEST_CONFIG_FILE > \ + $CLIENT_FUSE_DIR/target/debug/gvfs-fuse mount $MOUNT_DIR $MOUNT_FROM_LOCATION -c $TEST_CONFIG_FILE -f > \ $CLIENT_FUSE_DIR/target/debug/fuse.log 2>&1 & check_gvfs_fuse_ready cd - } stop_gvfs_fuse() { - # Stop the gvfs-fuse process if it's running - pkill -INT gvfs-fuse || true + # Unmount the gvfs-fuse + $CLIENT_FUSE_DIR/target/debug/gvfs-fuse umount $MOUNT_DIR echo "Stopping gvfs-fuse-daemon" } \ No newline at end of file diff --git a/clients/filesystem-fuse/tests/fuse_test.rs b/clients/filesystem-fuse/tests/fuse_test.rs index 41e385c49f1..29f47a3dae9 100644 --- a/clients/filesystem-fuse/tests/fuse_test.rs +++ b/clients/filesystem-fuse/tests/fuse_test.rs @@ -42,7 +42,7 @@ impl FuseTest { info!("Start gvfs fuse server"); let mount_point = self.mount_point.clone(); - let config = AppConfig::from_file(Some("tests/conf/gvfs_fuse_memory.toml")) + let config = AppConfig::from_file(Some("tests/conf/gvfs_fuse_memory.toml".to_string())) .expect("Failed to load config"); self.runtime.spawn(async move { let result = gvfs_mount(&mount_point, "", &config).await; From 8a7c9da4c4ac95f0bd5cb68ec39db703975b6d10 Mon Sep 17 00:00:00 2001 From: yuhui Date: Mon, 20 Jan 2025 10:46:26 +0800 Subject: [PATCH 2/9] Fix --- clients/filesystem-fuse/src/main.rs | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/clients/filesystem-fuse/src/main.rs b/clients/filesystem-fuse/src/main.rs index e86e943cf16..12e7f43a0a1 100644 --- a/clients/filesystem-fuse/src/main.rs +++ b/clients/filesystem-fuse/src/main.rs @@ -25,7 +25,6 @@ use gvfs_fuse::config::AppConfig; use gvfs_fuse::{gvfs_mount, gvfs_unmount}; use log::{error, info}; use std::fs::OpenOptions; -use std::path::Path; use std::process::{exit, Command}; use std::{env, io}; use tokio::runtime::Runtime; @@ -65,7 +64,7 @@ fn make_daemon() { } fn mount_fuse(config: AppConfig, mount_point: String, target: String) -> io::Result<()> { - let rt = Runtime::new().unwrap(); + let rt = Runtime::new()?; rt.block_on(async { let handle = tokio::spawn(async move { let result = gvfs_mount(&mount_point, &target, &config).await; From 0a7a3af36ece0fd2cc3fda3e426491aac2189fb6 Mon Sep 17 00:00:00 2001 From: yuhui Date: Mon, 20 Jan 2025 15:19:13 +0800 Subject: [PATCH 3/9] Fix log filename and pid filename --- clients/filesystem-fuse/conf/gvfs_fuse.toml | 1 + clients/filesystem-fuse/src/config.rs | 62 +++++++++++++++---- clients/filesystem-fuse/src/lib.rs | 3 + clients/filesystem-fuse/src/main.rs | 45 +++++++++----- .../tests/conf/config_test.toml | 2 + .../tests/conf/gvfs_fuse_s3.toml | 1 + 6 files changed, 85 insertions(+), 29 deletions(-) diff --git a/clients/filesystem-fuse/conf/gvfs_fuse.toml b/clients/filesystem-fuse/conf/gvfs_fuse.toml index 4bde0e9e1bd..27e52fc7d57 100644 --- a/clients/filesystem-fuse/conf/gvfs_fuse.toml +++ b/clients/filesystem-fuse/conf/gvfs_fuse.toml @@ -20,6 +20,7 @@ file_mask = 0o600 dir_mask = 0o700 fs_type = "memory" +data_path = "target/gvfs-fuse" [fuse.properties] diff --git a/clients/filesystem-fuse/src/config.rs b/clients/filesystem-fuse/src/config.rs index 96b720b4ed4..890cb7d31dc 100644 --- a/clients/filesystem-fuse/src/config.rs +++ b/clients/filesystem-fuse/src/config.rs @@ -24,6 +24,7 @@ use serde::Deserialize; use std::collections::HashMap; use std::fs; +// FuseConfig pub(crate) const CONF_FUSE_FILE_MASK: ConfigEntity = ConfigEntity::new( FuseConfig::MODULE_NAME, "file_mask", @@ -45,13 +46,28 @@ pub(crate) const CONF_FUSE_FS_TYPE: ConfigEntity<&'static str> = ConfigEntity::n "memory", ); -pub(crate) const CONF_FUSE_CONFIG_PATH: ConfigEntity<&'static str> = ConfigEntity::new( +pub(crate) const CONF_FUSE_CONFIG_FILE_PATH: ConfigEntity<&'static str> = ConfigEntity::new( FuseConfig::MODULE_NAME, "config_path", "The path of the FUSE configuration file", - "/etc/gvfs/gvfs.toml", + "/etc/gvfs-fuse/config.toml", ); +pub(crate) const CONF_FUSE_DATA_DIR: ConfigEntity<&'static str> = ConfigEntity::new( + FuseConfig::MODULE_NAME, + "data_dir", + "The data path of GVFS FUSE", + "/var/data/gvfs-fuse", +); + +pub(crate) const CONF_FUSE_LOG_DIR: ConfigEntity<&'static str> = ConfigEntity::new( + FuseConfig::MODULE_NAME, + "log_dir", + "The log path of GVFS FUSE", + "logs", //relative to the data path +); + +// FilesystemConfig pub(crate) const CONF_FILESYSTEM_BLOCK_SIZE: ConfigEntity = ConfigEntity::new( FilesystemConfig::MODULE_NAME, "block_size", @@ -59,6 +75,7 @@ pub(crate) const CONF_FILESYSTEM_BLOCK_SIZE: ConfigEntity = ConfigEntity::n 4096, ); +// GravitinoConfig pub(crate) const CONF_GRAVITINO_URI: ConfigEntity<&'static str> = ConfigEntity::new( GravitinoConfig::MODULE_NAME, "uri", @@ -125,22 +142,32 @@ impl Default for DefaultConfig { ConfigValue::String(CONF_FUSE_FS_TYPE), ); configs.insert( - Self::compose_key(CONF_FUSE_CONFIG_PATH), - ConfigValue::String(CONF_FUSE_CONFIG_PATH), + Self::compose_key(CONF_FUSE_CONFIG_FILE_PATH), + ConfigValue::String(CONF_FUSE_CONFIG_FILE_PATH), ); configs.insert( - Self::compose_key(CONF_GRAVITINO_URI), - ConfigValue::String(CONF_GRAVITINO_URI), + Self::compose_key(CONF_FUSE_DATA_DIR), + ConfigValue::String(CONF_FUSE_DATA_DIR), ); configs.insert( - Self::compose_key(CONF_GRAVITINO_METALAKE), - ConfigValue::String(CONF_GRAVITINO_METALAKE), + Self::compose_key(CONF_FUSE_LOG_DIR), + ConfigValue::String(CONF_FUSE_LOG_DIR), ); + configs.insert( Self::compose_key(CONF_FILESYSTEM_BLOCK_SIZE), ConfigValue::U32(CONF_FILESYSTEM_BLOCK_SIZE), ); + configs.insert( + Self::compose_key(CONF_GRAVITINO_URI), + ConfigValue::String(CONF_GRAVITINO_URI), + ); + configs.insert( + Self::compose_key(CONF_GRAVITINO_METALAKE), + ConfigValue::String(CONF_GRAVITINO_METALAKE), + ); + DefaultConfig { configs } } } @@ -220,19 +247,20 @@ impl AppConfig { info!("Use configuration file: {}", &path); path } else { - //use default config - if fs::metadata(CONF_FUSE_CONFIG_PATH.default).is_err() { + if fs::metadata(CONF_FUSE_CONFIG_FILE_PATH.default).is_err() { + //use default config warn!( "The default configuration file is not found, using the default configuration" ); return Ok(AppConfig::default()); } else { + //use the default configuration file warn!( "Using the default config file {}", - CONF_FUSE_CONFIG_PATH.default + CONF_FUSE_CONFIG_FILE_PATH.default ); } - CONF_FUSE_CONFIG_PATH.default.to_string() + CONF_FUSE_CONFIG_FILE_PATH.default.to_string() } }; let config = builder @@ -265,7 +293,11 @@ pub struct FuseConfig { #[serde(default)] pub fs_type: String, #[serde(default)] - pub config_path: String, + pub config_file_path: String, + #[serde(default)] + pub data_dir: String, + #[serde(default)] + pub log_dir: String, #[serde(default)] pub properties: HashMap, } @@ -305,6 +337,8 @@ mod test { let config = AppConfig::from_file(Some("tests/conf/config_test.toml".to_string())).unwrap(); assert_eq!(config.fuse.file_mask, 0o644); assert_eq!(config.fuse.dir_mask, 0o755); + assert_eq!(config.fuse.data_dir, "/target/gvfs-fuse"); + assert_eq!(config.fuse.log_dir, "/target/gvfs-fuse/logs"); assert_eq!(config.filesystem.block_size, 8192); assert_eq!(config.gravitino.uri, "http://localhost:8090"); assert_eq!(config.gravitino.metalake, "test"); @@ -323,6 +357,8 @@ mod test { let config = AppConfig::default(); assert_eq!(config.fuse.file_mask, 0o600); assert_eq!(config.fuse.dir_mask, 0o700); + assert_eq!(config.fuse.data_dir, "/var/data/gvfs-fuse"); + assert_eq!(config.fuse.log_dir, "logs"); assert_eq!(config.filesystem.block_size, 4096); assert_eq!(config.gravitino.uri, "http://localhost:8090"); assert_eq!(config.gravitino.metalake, ""); diff --git a/clients/filesystem-fuse/src/lib.rs b/clients/filesystem-fuse/src/lib.rs index 41a9a5335d5..65cd878f34d 100644 --- a/clients/filesystem-fuse/src/lib.rs +++ b/clients/filesystem-fuse/src/lib.rs @@ -49,6 +49,9 @@ macro_rules! test_enable_with { pub const RUN_TEST_WITH_S3: &str = "RUN_TEST_WITH_S3"; pub const RUN_TEST_WITH_FUSE: &str = "RUN_TEST_WITH_FUSE"; +pub const LOG_FILE_NAME: &str = "gvfs-fuse.log"; +pub const PID_FILE_NAME: &str = "gvfs-fuse.pid"; + pub async fn gvfs_mount(mount_to: &str, mount_from: &str, config: &AppConfig) -> GvfsResult<()> { gvfs_fuse::mount(mount_to, mount_from, config).await } diff --git a/clients/filesystem-fuse/src/main.rs b/clients/filesystem-fuse/src/main.rs index 12e7f43a0a1..536c8365228 100644 --- a/clients/filesystem-fuse/src/main.rs +++ b/clients/filesystem-fuse/src/main.rs @@ -22,35 +22,43 @@ use crate::command_args::Commands; use clap::Parser; use daemonize::Daemonize; use gvfs_fuse::config::AppConfig; -use gvfs_fuse::{gvfs_mount, gvfs_unmount}; +use gvfs_fuse::{gvfs_mount, gvfs_unmount, LOG_FILE_NAME, PID_FILE_NAME}; use log::{error, info}; -use std::fs::OpenOptions; +use std::fs::{create_dir_all, OpenOptions}; +use std::io; +use std::path::Path; use std::process::{exit, Command}; -use std::{env, io}; use tokio::runtime::Runtime; use tokio::signal; use tokio::signal::unix::{signal, SignalKind}; -fn make_daemon() { - let log_file_name = "/tmp/gvfs-fuse.log"; +fn make_daemon(config: &AppConfig) -> io::Result<()> { + let data_dir_name = Path::new(&config.fuse.data_dir).to_path_buf(); + if !data_dir_name.exists() { + Err(io::Error::new( + io::ErrorKind::NotFound, + format!("Data directory {} not found", &config.fuse.data_dir), + ))? + }; + + let log_dir_name = data_dir_name.join(&config.fuse.log_dir); + if !log_dir_name.exists() { + create_dir_all(&log_dir_name)? + }; + let log_file_name = Path::new(&config.fuse.log_dir).join(LOG_FILE_NAME); let log_file = OpenOptions::new() .create(true) .append(true) - .open(log_file_name) + .open(&log_file_name) .unwrap(); - let log_err_file = OpenOptions::new().write(true).open(log_file_name).unwrap(); + let log_err_file = OpenOptions::new().append(true).open(log_file_name).unwrap(); - let cwd = env::current_dir(); - if let Err(e) = cwd { - error!("Error getting current directory: {}", e); - return; - } - let cwd = cwd.unwrap(); + let pid_file_name = data_dir_name.join(PID_FILE_NAME); let daemonize = Daemonize::new() - .pid_file("/tmp/gvfs-fuse.pid") + .pid_file(pid_file_name) .chown_pid_file(true) - .working_directory(&cwd) + .working_directory(&data_dir_name) .stdout(log_file) .stderr(log_err_file); @@ -61,6 +69,7 @@ fn make_daemon() { exit(-1) } } + Ok(()) } fn mount_fuse(config: AppConfig, mount_point: String, target: String) -> io::Result<()> { @@ -162,7 +171,11 @@ fn main() -> Result<(), i32> { let result = if foreground { mount_fuse(app_config, mount_point, location) } else { - make_daemon(); + let result = make_daemon(&app_config); + if let Err(e) = result { + error!("Failed to daemonize: {:?}", e); + return Err(-1); + }; mount_fuse(app_config, mount_point, location) }; diff --git a/clients/filesystem-fuse/tests/conf/config_test.toml b/clients/filesystem-fuse/tests/conf/config_test.toml index 524e0aa94fb..e7cbf02c7f3 100644 --- a/clients/filesystem-fuse/tests/conf/config_test.toml +++ b/clients/filesystem-fuse/tests/conf/config_test.toml @@ -20,6 +20,8 @@ file_mask= 0o644 dir_mask= 0o755 fs_type = "memory" +data_dir = "/target/gvfs-fuse" +log_dir = "/target/gvfs-fuse/logs" [fuse.properties] key1 = "value1" diff --git a/clients/filesystem-fuse/tests/conf/gvfs_fuse_s3.toml b/clients/filesystem-fuse/tests/conf/gvfs_fuse_s3.toml index d0ff8e5ddec..3735b2a20da 100644 --- a/clients/filesystem-fuse/tests/conf/gvfs_fuse_s3.toml +++ b/clients/filesystem-fuse/tests/conf/gvfs_fuse_s3.toml @@ -20,6 +20,7 @@ file_mask= 0o600 dir_mask= 0o700 fs_type = "gvfs" +data_path = "target/debug/gvfs-fuse" [fuse.properties] key1 = "value1" From 387fa9c8448dc28fed45802a241acb940ea49e8c Mon Sep 17 00:00:00 2001 From: yuhui Date: Mon, 20 Jan 2025 15:32:24 +0800 Subject: [PATCH 4/9] Fix --- clients/filesystem-fuse/src/main.rs | 37 ++++++++++++++++++++++------- 1 file changed, 29 insertions(+), 8 deletions(-) diff --git a/clients/filesystem-fuse/src/main.rs b/clients/filesystem-fuse/src/main.rs index 536c8365228..9b9aea394c8 100644 --- a/clients/filesystem-fuse/src/main.rs +++ b/clients/filesystem-fuse/src/main.rs @@ -24,15 +24,15 @@ use daemonize::Daemonize; use gvfs_fuse::config::AppConfig; use gvfs_fuse::{gvfs_mount, gvfs_unmount, LOG_FILE_NAME, PID_FILE_NAME}; use log::{error, info}; -use std::fs::{create_dir_all, OpenOptions}; +use std::fs::{create_dir, OpenOptions}; use std::io; use std::path::Path; -use std::process::{exit, Command}; +use std::process::Command; use tokio::runtime::Runtime; use tokio::signal; use tokio::signal::unix::{signal, SignalKind}; -fn make_daemon(config: &AppConfig) -> io::Result<()> { +fn init_work_dirs(config: &AppConfig, mount_point: &str) -> io::Result<()> { let data_dir_name = Path::new(&config.fuse.data_dir).to_path_buf(); if !data_dir_name.exists() { Err(io::Error::new( @@ -41,11 +41,23 @@ fn make_daemon(config: &AppConfig) -> io::Result<()> { ))? }; + let mount_point_name = data_dir_name.join(mount_point); + if !mount_point_name.exists() { + create_dir(&mount_point_name)? + }; + let log_dir_name = data_dir_name.join(&config.fuse.log_dir); if !log_dir_name.exists() { - create_dir_all(&log_dir_name)? + create_dir(&log_dir_name)? }; - let log_file_name = Path::new(&config.fuse.log_dir).join(LOG_FILE_NAME); + + Ok(()) +} + +fn make_daemon(config: &AppConfig) -> io::Result<()> { + let data_dir_name = Path::new(&config.fuse.data_dir); + let log_dir_name = data_dir_name.join(&config.fuse.log_dir); + let log_file_name = log_dir_name.join(LOG_FILE_NAME); let log_file = OpenOptions::new() .create(true) .append(true) @@ -58,15 +70,17 @@ fn make_daemon(config: &AppConfig) -> io::Result<()> { let daemonize = Daemonize::new() .pid_file(pid_file_name) .chown_pid_file(true) - .working_directory(&data_dir_name) + .working_directory(data_dir_name) .stdout(log_file) .stderr(log_err_file); match daemonize.start() { Ok(_) => info!("Gvfs-fuse Daemon started successfully"), Err(e) => { - error!("Gvfs-fuse Daemon failed to start: {:?}", e); - exit(-1) + return Err(io::Error::new( + io::ErrorKind::Other, + format!("Gvfs-fuse Daemon failed to start: {:?}", e), + )) } } Ok(()) @@ -168,6 +182,13 @@ fn main() -> Result<(), i32> { return Err(-1); }; let app_config = app_config.unwrap(); + + let result = init_work_dirs(&app_config, &mount_point); + if let Err(e) = result { + error!("Failed to initialize working directories: {:?}", e); + return Err(-1); + } + let result = if foreground { mount_fuse(app_config, mount_point, location) } else { From 9976988bbbd66b6c4ab3ef719ad8d15dbadc6219 Mon Sep 17 00:00:00 2001 From: Yuhui Date: Mon, 20 Jan 2025 15:34:28 +0800 Subject: [PATCH 5/9] Update clients/filesystem-fuse/src/command_args.rs Co-authored-by: Qiming Teng --- clients/filesystem-fuse/src/command_args.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/clients/filesystem-fuse/src/command_args.rs b/clients/filesystem-fuse/src/command_args.rs index 4e64eefa59b..8db9bcef832 100644 --- a/clients/filesystem-fuse/src/command_args.rs +++ b/clients/filesystem-fuse/src/command_args.rs @@ -35,7 +35,7 @@ pub(crate) enum Commands { #[arg(help = "Mount point for the filesystem")] mount_point: String, - #[arg(help = "Location of GVFS fileset URI")] + #[arg(help = "The URI of the GVFS fileset")] location: String, #[arg(short, long)] From 9c26f361b235fed9ab192409bbdc78bd1767feae Mon Sep 17 00:00:00 2001 From: yuhui Date: Mon, 20 Jan 2025 17:40:09 +0800 Subject: [PATCH 6/9] Update --- clients/filesystem-fuse/tests/bin/gvfs_fuse.sh | 3 ++- clients/filesystem-fuse/tests/conf/gvfs_fuse_s3.toml | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/clients/filesystem-fuse/tests/bin/gvfs_fuse.sh b/clients/filesystem-fuse/tests/bin/gvfs_fuse.sh index b6a04f3b2a0..3089a9a1a1f 100644 --- a/clients/filesystem-fuse/tests/bin/gvfs_fuse.sh +++ b/clients/filesystem-fuse/tests/bin/gvfs_fuse.sh @@ -52,6 +52,7 @@ start_gvfs_fuse() { make build echo "Starting gvfs-fuse-daemon" + mkdir -p target/debug/gvfs-dir $CLIENT_FUSE_DIR/target/debug/gvfs-fuse mount $MOUNT_DIR $MOUNT_FROM_LOCATION -c $TEST_CONFIG_FILE -f > \ $CLIENT_FUSE_DIR/target/debug/fuse.log 2>&1 & check_gvfs_fuse_ready @@ -62,4 +63,4 @@ stop_gvfs_fuse() { # Unmount the gvfs-fuse $CLIENT_FUSE_DIR/target/debug/gvfs-fuse umount $MOUNT_DIR echo "Stopping gvfs-fuse-daemon" -} \ No newline at end of file +} diff --git a/clients/filesystem-fuse/tests/conf/gvfs_fuse_s3.toml b/clients/filesystem-fuse/tests/conf/gvfs_fuse_s3.toml index 3735b2a20da..69a3fcaf36d 100644 --- a/clients/filesystem-fuse/tests/conf/gvfs_fuse_s3.toml +++ b/clients/filesystem-fuse/tests/conf/gvfs_fuse_s3.toml @@ -20,7 +20,7 @@ file_mask= 0o600 dir_mask= 0o700 fs_type = "gvfs" -data_path = "target/debug/gvfs-fuse" +data_dir = "target/debug/gvfs-dir" [fuse.properties] key1 = "value1" From d097c1df50a588bdf85d1ba591bc514ee2717a8b Mon Sep 17 00:00:00 2001 From: yuhui Date: Mon, 20 Jan 2025 18:00:48 +0800 Subject: [PATCH 7/9] pass file type to PathFileSystem::stat --- .../src/default_raw_filesystem.rs | 22 ++++++++++++------- clients/filesystem-fuse/src/filesystem.rs | 11 ++++++---- .../src/gravitino_fileset_filesystem.rs | 13 ++++++++--- .../filesystem-fuse/src/memory_filesystem.rs | 8 +++++-- .../src/open_dal_filesystem.rs | 21 ++++++++++++++---- clients/filesystem-fuse/src/s3_filesystem.rs | 9 ++++++-- 6 files changed, 61 insertions(+), 23 deletions(-) diff --git a/clients/filesystem-fuse/src/default_raw_filesystem.rs b/clients/filesystem-fuse/src/default_raw_filesystem.rs index d1d8e7605df..84944a9b89d 100644 --- a/clients/filesystem-fuse/src/default_raw_filesystem.rs +++ b/clients/filesystem-fuse/src/default_raw_filesystem.rs @@ -31,6 +31,7 @@ use std::collections::HashMap; use std::ffi::OsStr; use std::path::{Path, PathBuf}; use std::sync::atomic::AtomicU64; +use fuse3::FileType::{Directory, RegularFile}; use tokio::sync::RwLock; /// DefaultRawFileSystem is a simple implementation for the file system. @@ -86,7 +87,7 @@ impl DefaultRawFileSystem { None => { // allocate new file id file_stat.set_file_id(parent_file_id, self.next_file_id()); - file_manager.insert(file_stat.parent_file_id, file_stat.file_id, &file_stat.path); + file_manager.insert(file_stat.parent_file_id, file_stat.file_id, &file_stat.path, file_stat.kind); } Some(file) => { // use the exist file id @@ -130,9 +131,9 @@ impl DefaultRawFileSystem { file_manager.remove(path); } - async fn insert_file_entry_locked(&self, parent_file_id: u64, file_id: u64, path: &Path) { + async fn insert_file_entry_locked(&self, parent_file_id: u64, file_id: u64, path: &Path, kind: FileType) { let mut file_manager = self.file_entry_manager.write().await; - file_manager.insert(parent_file_id, file_id, path); + file_manager.insert(parent_file_id, file_id, path, kind); } fn get_meta_file_stat(&self) -> FileStat { @@ -159,6 +160,7 @@ impl RawFileSystem for DefaultRawFileSystem { ROOT_DIR_PARENT_FILE_ID, ROOT_DIR_FILE_ID, Path::new(ROOT_DIR_PATH), + Directory ) .await; @@ -166,6 +168,7 @@ impl RawFileSystem for DefaultRawFileSystem { ROOT_DIR_FILE_ID, FS_META_FILE_ID, Path::new(FS_META_FILE_PATH), + RegularFile ) .await; self.fs.init().await @@ -197,7 +200,7 @@ impl RawFileSystem for DefaultRawFileSystem { } let file_entry = self.get_file_entry(file_id).await?; - let mut file_stat = self.fs.stat(&file_entry.path).await?; + let mut file_stat = self.fs.stat(&file_entry.path, file_entry.kind).await?; file_stat.set_file_id(file_entry.parent_file_id, file_entry.file_id); Ok(file_stat) } @@ -209,7 +212,7 @@ impl RawFileSystem for DefaultRawFileSystem { let parent_file_entry = self.get_file_entry(parent_file_id).await?; let path = parent_file_entry.path.join(name); - let mut file_stat = self.fs.stat(&path).await?; + let mut file_stat = self.fs.lookup(&path).await?; // fill the file id to file stat self.resolve_file_id_to_filestat(&mut file_stat, parent_file_id) .await; @@ -270,6 +273,7 @@ impl RawFileSystem for DefaultRawFileSystem { parent_file_id, file_without_id.file_stat.file_id, &file_without_id.file_stat.path, + RegularFile ) .await; @@ -287,7 +291,7 @@ impl RawFileSystem for DefaultRawFileSystem { filestat.set_file_id(parent_file_id, self.next_file_id()); // insert the new file to file entry manager - self.insert_file_entry_locked(parent_file_id, filestat.file_id, &filestat.path) + self.insert_file_entry_locked(parent_file_id, filestat.file_id, &filestat.path, Directory) .await; Ok(filestat.file_id) } @@ -401,6 +405,7 @@ struct FileEntry { file_id: u64, parent_file_id: u64, path: PathBuf, + kind: FileType, } /// FileEntryManager is manage all the file entries in memory. it is used manger the file relationship and name mapping. @@ -428,11 +433,12 @@ impl FileEntryManager { self.file_path_map.get(path).cloned() } - fn insert(&mut self, parent_file_id: u64, file_id: u64, path: &Path) { + fn insert(&mut self, parent_file_id: u64, file_id: u64, path: &Path, kind: FileType) { let file_entry = FileEntry { file_id, parent_file_id, path: path.into(), + kind: kind }; self.file_id_map.insert(file_id, file_entry.clone()); self.file_path_map.insert(path.into(), file_entry); @@ -452,7 +458,7 @@ mod tests { #[test] fn test_file_entry_manager() { let mut manager = FileEntryManager::new(); - manager.insert(1, 2, Path::new("a/b")); + manager.insert(1, 2, Path::new("a/b"), Directory); let file = manager.get_file_entry_by_id(2).unwrap(); assert_eq!(file.file_id, 2); assert_eq!(file.parent_file_id, 1); diff --git a/clients/filesystem-fuse/src/filesystem.rs b/clients/filesystem-fuse/src/filesystem.rs index dcf35f8ebca..964f9ccb219 100644 --- a/clients/filesystem-fuse/src/filesystem.rs +++ b/clients/filesystem-fuse/src/filesystem.rs @@ -115,7 +115,10 @@ pub(crate) trait PathFileSystem: Send + Sync { async fn init(&self) -> Result<()>; /// Get the file stat by file path, if the file exists, return the file stat - async fn stat(&self, path: &Path) -> Result; + async fn stat(&self, path: &Path, kind: FileType) -> Result; + + /// Lookup the file stat by file path, if the file exists, return the file stat + async fn lookup(&self, path: &Path) -> Result; /// Read the directory by file path, if the directory exists, return the file stat list async fn read_dir(&self, path: &Path) -> Result>; @@ -318,7 +321,7 @@ pub(crate) mod tests { pub(crate) async fn test_path_file_system(&mut self) { // test root dir - let resutl = self.fs.stat(Path::new("/")).await; + let resutl = self.fs.stat(Path::new("/"), Directory).await; assert!(resutl.is_ok()); let root_file_stat = resutl.unwrap(); self.assert_file_stat(&root_file_stat, Path::new("/"), Directory, 0); @@ -347,7 +350,7 @@ pub(crate) mod tests { } async fn test_stat_file(&mut self, path: &Path, expect_kind: FileType, expect_size: u64) { - let file_stat = self.fs.stat(path).await; + let file_stat = self.fs.stat(path, RegularFile).await; assert!(file_stat.is_ok()); let file_stat = file_stat.unwrap(); self.assert_file_stat(&file_stat, path, expect_kind, expect_size); @@ -403,7 +406,7 @@ pub(crate) mod tests { } async fn test_file_not_found(&self, path: &Path) { - let not_found_file = self.fs.stat(path).await; + let not_found_file = self.fs.stat(path, RegularFile).await; assert!(not_found_file.is_err()); } diff --git a/clients/filesystem-fuse/src/gravitino_fileset_filesystem.rs b/clients/filesystem-fuse/src/gravitino_fileset_filesystem.rs index 04236dfe841..ba717673b90 100644 --- a/clients/filesystem-fuse/src/gravitino_fileset_filesystem.rs +++ b/clients/filesystem-fuse/src/gravitino_fileset_filesystem.rs @@ -21,7 +21,7 @@ use crate::filesystem::{FileStat, FileSystemCapacity, FileSystemContext, PathFil use crate::gravitino_client::GravitinoClient; use crate::opened_file::{OpenFileFlags, OpenedFile}; use async_trait::async_trait; -use fuse3::Errno; +use fuse3::{Errno, FileType}; use std::path::{Path, PathBuf}; /// GravitinoFileSystem is a filesystem that is associated with a fileset in Gravitino. @@ -74,9 +74,16 @@ impl PathFileSystem for GravitinoFilesetFileSystem { self.physical_fs.init().await } - async fn stat(&self, path: &Path) -> Result { + async fn stat(&self, path: &Path, kind: FileType) -> Result { let raw_path = self.gvfs_path_to_raw_path(path); - let mut file_stat = self.physical_fs.stat(&raw_path).await?; + let mut file_stat = self.physical_fs.stat(&raw_path, kind).await?; + file_stat.path = self.raw_path_to_gvfs_path(&file_stat.path)?; + Ok(file_stat) + } + + async fn lookup(&self, path: &Path) -> Result { + let raw_path = self.gvfs_path_to_raw_path(path); + let mut file_stat = self.physical_fs.lookup(&raw_path).await?; file_stat.path = self.raw_path_to_gvfs_path(&file_stat.path)?; Ok(file_stat) } diff --git a/clients/filesystem-fuse/src/memory_filesystem.rs b/clients/filesystem-fuse/src/memory_filesystem.rs index f56e65ea33a..f50c541bc67 100644 --- a/clients/filesystem-fuse/src/memory_filesystem.rs +++ b/clients/filesystem-fuse/src/memory_filesystem.rs @@ -70,7 +70,7 @@ impl PathFileSystem for MemoryFileSystem { Ok(()) } - async fn stat(&self, path: &Path) -> Result { + async fn stat(&self, path: &Path, _kind: FileType) -> Result { self.file_map .read() .unwrap() @@ -79,6 +79,10 @@ impl PathFileSystem for MemoryFileSystem { .ok_or(Errno::from(libc::ENOENT)) } + async fn lookup(&self, path: &Path) -> Result { + self.stat(path, RegularFile).await + } + async fn read_dir(&self, path: &Path) -> Result> { let file_map = self.file_map.read().unwrap(); @@ -92,7 +96,7 @@ impl PathFileSystem for MemoryFileSystem { } async fn open_file(&self, path: &Path, _flags: OpenFileFlags) -> Result { - let file_stat = self.stat(path).await?; + let file_stat = self.stat(path, RegularFile).await?; let mut opened_file = OpenedFile::new(file_stat); match opened_file.file_stat.kind { Directory => Ok(opened_file), diff --git a/clients/filesystem-fuse/src/open_dal_filesystem.rs b/clients/filesystem-fuse/src/open_dal_filesystem.rs index d32b014d1f0..56907fcb262 100644 --- a/clients/filesystem-fuse/src/open_dal_filesystem.rs +++ b/clients/filesystem-fuse/src/open_dal_filesystem.rs @@ -59,7 +59,20 @@ impl PathFileSystem for OpenDalFileSystem { Ok(()) } - async fn stat(&self, path: &Path) -> Result { + async fn stat(&self, path: &Path, kind: FileType) -> Result { + let file_name = match kind { + Directory => build_dir_path(path), + _ => path.to_string_lossy().to_string() + }; + let meta= self.op.stat(&file_name).await.map_err(opendal_error_to_errno)?; + + let mut file_stat = FileStat::new_file_filestat_with_path(path, 0); + self.opendal_meta_to_file_stat(&meta, &mut file_stat); + + Ok(file_stat) + } + + async fn lookup(&self, path: &Path) -> Result { let file_name = path.to_string_lossy().to_string(); let meta_result = self.op.stat(&file_name).await; @@ -107,7 +120,7 @@ impl PathFileSystem for OpenDalFileSystem { } async fn open_file(&self, path: &Path, flags: OpenFileFlags) -> Result { - let file_stat = self.stat(path).await?; + let file_stat = self.stat(path, RegularFile).await?; debug_assert!(file_stat.kind == RegularFile); let mut file = OpenedFile::new(file_stat); @@ -132,7 +145,7 @@ impl PathFileSystem for OpenDalFileSystem { } async fn open_dir(&self, path: &Path, _flags: OpenFileFlags) -> Result { - let file_stat = self.stat(path).await?; + let file_stat = self.stat(path, Directory).await?; debug_assert!(file_stat.kind == Directory); let opened_file = OpenedFile::new(file_stat); @@ -160,7 +173,7 @@ impl PathFileSystem for OpenDalFileSystem { .create_dir(&dir_name) .await .map_err(opendal_error_to_errno)?; - let file_stat = self.stat(path).await?; + let file_stat = self.stat(path, Directory).await?; Ok(file_stat) } diff --git a/clients/filesystem-fuse/src/s3_filesystem.rs b/clients/filesystem-fuse/src/s3_filesystem.rs index e397d31ccf3..5b4430a94fd 100644 --- a/clients/filesystem-fuse/src/s3_filesystem.rs +++ b/clients/filesystem-fuse/src/s3_filesystem.rs @@ -30,6 +30,7 @@ use opendal::services::S3; use opendal::{Builder, Operator}; use std::collections::HashMap; use std::path::Path; +use fuse3::FileType; pub(crate) struct S3FileSystem { open_dal_fs: OpenDalFileSystem, @@ -94,8 +95,12 @@ impl PathFileSystem for S3FileSystem { Ok(()) } - async fn stat(&self, path: &Path) -> Result { - self.open_dal_fs.stat(path).await + async fn stat(&self, path: &Path, kind: FileType) -> Result { + self.open_dal_fs.stat(path, kind).await + } + + async fn lookup(&self, path: &Path) -> Result { + self.open_dal_fs.lookup(path).await } async fn read_dir(&self, path: &Path) -> Result> { From c7f17013c35ef370e9fada7308725f1702c6b3a9 Mon Sep 17 00:00:00 2001 From: yuhui Date: Wed, 22 Jan 2025 11:25:11 +0800 Subject: [PATCH 8/9] Fix ci error --- .../src/default_raw_filesystem.rs | 25 +++++++++++++------ clients/filesystem-fuse/src/filesystem.rs | 2 +- .../src/open_dal_filesystem.rs | 8 ++++-- clients/filesystem-fuse/src/s3_filesystem.rs | 2 +- .../filesystem-fuse/tests/bin/gvfs_fuse.sh | 4 +-- .../tests/bin/run_fuse_testers.sh | 2 ++ .../tests/bin/run_s3fs_testers.sh | 1 - 7 files changed, 29 insertions(+), 15 deletions(-) mode change 100644 => 100755 clients/filesystem-fuse/tests/bin/run_s3fs_testers.sh diff --git a/clients/filesystem-fuse/src/default_raw_filesystem.rs b/clients/filesystem-fuse/src/default_raw_filesystem.rs index 84944a9b89d..08b480e43ef 100644 --- a/clients/filesystem-fuse/src/default_raw_filesystem.rs +++ b/clients/filesystem-fuse/src/default_raw_filesystem.rs @@ -26,12 +26,12 @@ use crate::opened_file::{FileHandle, OpenFileFlags, OpenedFile}; use crate::opened_file_manager::OpenedFileManager; use async_trait::async_trait; use bytes::Bytes; +use fuse3::FileType::{Directory, RegularFile}; use fuse3::{Errno, FileType}; use std::collections::HashMap; use std::ffi::OsStr; use std::path::{Path, PathBuf}; use std::sync::atomic::AtomicU64; -use fuse3::FileType::{Directory, RegularFile}; use tokio::sync::RwLock; /// DefaultRawFileSystem is a simple implementation for the file system. @@ -87,7 +87,12 @@ impl DefaultRawFileSystem { None => { // allocate new file id file_stat.set_file_id(parent_file_id, self.next_file_id()); - file_manager.insert(file_stat.parent_file_id, file_stat.file_id, &file_stat.path, file_stat.kind); + file_manager.insert( + file_stat.parent_file_id, + file_stat.file_id, + &file_stat.path, + file_stat.kind, + ); } Some(file) => { // use the exist file id @@ -131,7 +136,13 @@ impl DefaultRawFileSystem { file_manager.remove(path); } - async fn insert_file_entry_locked(&self, parent_file_id: u64, file_id: u64, path: &Path, kind: FileType) { + async fn insert_file_entry_locked( + &self, + parent_file_id: u64, + file_id: u64, + path: &Path, + kind: FileType, + ) { let mut file_manager = self.file_entry_manager.write().await; file_manager.insert(parent_file_id, file_id, path, kind); } @@ -160,7 +171,7 @@ impl RawFileSystem for DefaultRawFileSystem { ROOT_DIR_PARENT_FILE_ID, ROOT_DIR_FILE_ID, Path::new(ROOT_DIR_PATH), - Directory + Directory, ) .await; @@ -168,7 +179,7 @@ impl RawFileSystem for DefaultRawFileSystem { ROOT_DIR_FILE_ID, FS_META_FILE_ID, Path::new(FS_META_FILE_PATH), - RegularFile + RegularFile, ) .await; self.fs.init().await @@ -273,7 +284,7 @@ impl RawFileSystem for DefaultRawFileSystem { parent_file_id, file_without_id.file_stat.file_id, &file_without_id.file_stat.path, - RegularFile + RegularFile, ) .await; @@ -438,7 +449,7 @@ impl FileEntryManager { file_id, parent_file_id, path: path.into(), - kind: kind + kind: kind, }; self.file_id_map.insert(file_id, file_entry.clone()); self.file_path_map.insert(path.into(), file_entry); diff --git a/clients/filesystem-fuse/src/filesystem.rs b/clients/filesystem-fuse/src/filesystem.rs index 964f9ccb219..b8437d39ad3 100644 --- a/clients/filesystem-fuse/src/filesystem.rs +++ b/clients/filesystem-fuse/src/filesystem.rs @@ -350,7 +350,7 @@ pub(crate) mod tests { } async fn test_stat_file(&mut self, path: &Path, expect_kind: FileType, expect_size: u64) { - let file_stat = self.fs.stat(path, RegularFile).await; + let file_stat = self.fs.stat(path, expect_kind).await; assert!(file_stat.is_ok()); let file_stat = file_stat.unwrap(); self.assert_file_stat(&file_stat, path, expect_kind, expect_size); diff --git a/clients/filesystem-fuse/src/open_dal_filesystem.rs b/clients/filesystem-fuse/src/open_dal_filesystem.rs index 56907fcb262..46f2f7ce7b2 100644 --- a/clients/filesystem-fuse/src/open_dal_filesystem.rs +++ b/clients/filesystem-fuse/src/open_dal_filesystem.rs @@ -62,9 +62,13 @@ impl PathFileSystem for OpenDalFileSystem { async fn stat(&self, path: &Path, kind: FileType) -> Result { let file_name = match kind { Directory => build_dir_path(path), - _ => path.to_string_lossy().to_string() + _ => path.to_string_lossy().to_string(), }; - let meta= self.op.stat(&file_name).await.map_err(opendal_error_to_errno)?; + let meta = self + .op + .stat(&file_name) + .await + .map_err(opendal_error_to_errno)?; let mut file_stat = FileStat::new_file_filestat_with_path(path, 0); self.opendal_meta_to_file_stat(&meta, &mut file_stat); diff --git a/clients/filesystem-fuse/src/s3_filesystem.rs b/clients/filesystem-fuse/src/s3_filesystem.rs index 5b4430a94fd..f4e9fd440d7 100644 --- a/clients/filesystem-fuse/src/s3_filesystem.rs +++ b/clients/filesystem-fuse/src/s3_filesystem.rs @@ -24,13 +24,13 @@ use crate::open_dal_filesystem::OpenDalFileSystem; use crate::opened_file::{OpenFileFlags, OpenedFile}; use crate::utils::{parse_location, GvfsResult}; use async_trait::async_trait; +use fuse3::FileType; use log::error; use opendal::layers::LoggingLayer; use opendal::services::S3; use opendal::{Builder, Operator}; use std::collections::HashMap; use std::path::Path; -use fuse3::FileType; pub(crate) struct S3FileSystem { open_dal_fs: OpenDalFileSystem, diff --git a/clients/filesystem-fuse/tests/bin/gvfs_fuse.sh b/clients/filesystem-fuse/tests/bin/gvfs_fuse.sh index b6a04f3b2a0..47fab551131 100644 --- a/clients/filesystem-fuse/tests/bin/gvfs_fuse.sh +++ b/clients/filesystem-fuse/tests/bin/gvfs_fuse.sh @@ -37,8 +37,6 @@ check_gvfs_fuse_ready() { } start_gvfs_fuse() { - MOUNT_DIR=$CLIENT_FUSE_DIR/target/gvfs - umount $MOUNT_DIR > /dev/null 2>&1 || true if [ ! -d "$MOUNT_DIR" ]; then echo "Create the mount point" @@ -62,4 +60,4 @@ stop_gvfs_fuse() { # Unmount the gvfs-fuse $CLIENT_FUSE_DIR/target/debug/gvfs-fuse umount $MOUNT_DIR echo "Stopping gvfs-fuse-daemon" -} \ No newline at end of file +} diff --git a/clients/filesystem-fuse/tests/bin/run_fuse_testers.sh b/clients/filesystem-fuse/tests/bin/run_fuse_testers.sh index 6dc38c48f07..d1dd127427c 100755 --- a/clients/filesystem-fuse/tests/bin/run_fuse_testers.sh +++ b/clients/filesystem-fuse/tests/bin/run_fuse_testers.sh @@ -27,6 +27,8 @@ source ./gvfs_fuse.sh source ./localstatck.sh TEST_CONFIG_FILE=$CLIENT_FUSE_DIR/target/debug/gvfs-fuse.toml +MOUNT_DIR=$CLIENT_FUSE_DIR/target/gvfs + start_servers() { start_localstack diff --git a/clients/filesystem-fuse/tests/bin/run_s3fs_testers.sh b/clients/filesystem-fuse/tests/bin/run_s3fs_testers.sh old mode 100644 new mode 100755 index ac5f9812c93..04f560b00de --- a/clients/filesystem-fuse/tests/bin/run_s3fs_testers.sh +++ b/clients/filesystem-fuse/tests/bin/run_s3fs_testers.sh @@ -61,4 +61,3 @@ else exit 1 fi - From 544ec93909f4d5338e466d7138fa8d0728b0cd8d Mon Sep 17 00:00:00 2001 From: yuhui Date: Fri, 24 Jan 2025 17:30:27 +0800 Subject: [PATCH 9/9] Fix relation path problem --- clients/filesystem-fuse/src/main.rs | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/clients/filesystem-fuse/src/main.rs b/clients/filesystem-fuse/src/main.rs index 9b9aea394c8..90a26803304 100644 --- a/clients/filesystem-fuse/src/main.rs +++ b/clients/filesystem-fuse/src/main.rs @@ -34,6 +34,7 @@ use tokio::signal::unix::{signal, SignalKind}; fn init_work_dirs(config: &AppConfig, mount_point: &str) -> io::Result<()> { let data_dir_name = Path::new(&config.fuse.data_dir).to_path_buf(); + let data_dir_name = data_dir_name.canonicalize()?; if !data_dir_name.exists() { Err(io::Error::new( io::ErrorKind::NotFound, @@ -183,6 +184,16 @@ fn main() -> Result<(), i32> { }; let app_config = app_config.unwrap(); + let mount_point = { + let path = Path::new(&mount_point).canonicalize(); + if let Err(e) = path { + error!("Failed to resolve mount point: {:?}", e); + return Err(-1); + }; + let path = path.unwrap(); + path.to_string_lossy().to_string() + }; + let result = init_work_dirs(&app_config, &mount_point); if let Err(e) = result { error!("Failed to initialize working directories: {:?}", e);