From 2c7d81e11720edaca2345235e8181e4dc8ceb864 Mon Sep 17 00:00:00 2001 From: Abhinandan Purkait Date: Tue, 25 Jul 2023 14:39:49 +0000 Subject: [PATCH] feat(clone): enable option fs id change to clone uuid Signed-off-by: Abhinandan Purkait --- .../csi-driver/src/bin/controller/client.rs | 1 + .../src/bin/controller/controller.rs | 23 +- .../csi-driver/src/bin/node/filesystem_ops.rs | 311 ++++++++++++++++++ .../csi-driver/src/bin/node/filesystem_vol.rs | 99 ++++-- .../csi-driver/src/bin/node/findmnt.rs | 11 +- .../csi-driver/src/bin/node/format.rs | 60 ++-- control-plane/csi-driver/src/bin/node/main.rs | 2 + .../csi-driver/src/bin/node/mount.rs | 26 +- control-plane/csi-driver/src/bin/node/node.rs | 6 +- .../csi-driver/src/bin/node/nodeplugin_svc.rs | 4 +- control-plane/csi-driver/src/context.rs | 79 +++-- control-plane/csi-driver/src/filesystem.rs | 31 ++ control-plane/csi-driver/src/lib.rs | 2 + 13 files changed, 547 insertions(+), 108 deletions(-) create mode 100644 control-plane/csi-driver/src/bin/node/filesystem_ops.rs create mode 100644 control-plane/csi-driver/src/filesystem.rs diff --git a/control-plane/csi-driver/src/bin/controller/client.rs b/control-plane/csi-driver/src/bin/controller/client.rs index 96441a7c1..15f266b73 100644 --- a/control-plane/csi-driver/src/bin/controller/client.rs +++ b/control-plane/csi-driver/src/bin/controller/client.rs @@ -78,6 +78,7 @@ impl From> for ApiClientError { StatusCode::INSUFFICIENT_STORAGE => Self::ResourceExhausted(detailed), StatusCode::SERVICE_UNAVAILABLE => Self::Unavailable(detailed), StatusCode::PRECONDITION_FAILED => Self::PreconditionFailed(detailed), + StatusCode::BAD_REQUEST => Self::InvalidArgument(detailed), status => Self::GenericOperation(status, detailed), } } diff --git a/control-plane/csi-driver/src/bin/controller/controller.rs b/control-plane/csi-driver/src/bin/controller/controller.rs index 77ba5ba7f..04a741971 100644 --- a/control-plane/csi-driver/src/bin/controller/controller.rs +++ b/control-plane/csi-driver/src/bin/controller/controller.rs @@ -91,6 +91,7 @@ impl From for Status { ApiClientError::Conflict(reason) => Status::aborted(reason), ApiClientError::Aborted(reason) => Status::aborted(reason), ApiClientError::Unavailable(reason) => Status::unavailable(reason), + ApiClientError::InvalidArgument(reason) => Status::invalid_argument(reason), // TODO: Revisit the error mapping. Currently handled specifically for snapshot create. // ApiClientError::PreconditionFailed(reason) => Status::resource_exhausted(reason), // ApiClientError::ResourceExhausted(reason) => Status::resource_exhausted(reason), @@ -257,6 +258,8 @@ impl rpc::csi::controller_server::Controller for CsiControllerSvc { None => false, }; + let mut volume_context = args.parameters.clone(); + // First check if the volume already exists. match IoEngineApiClient::get_client() .get_volume_for_create(&parsed_vol_uuid) @@ -281,7 +284,7 @@ impl rpc::csi::controller_server::Controller for CsiControllerSvc { let sts_affinity_group_name = context.sts_affinity_group(); - match volume_content_source { + let volume = match volume_content_source { Some(snapshot_uuid) => { IoEngineApiClient::get_client() .create_snapshot_volume( @@ -293,7 +296,7 @@ impl rpc::csi::controller_server::Controller for CsiControllerSvc { thin, sts_affinity_group_name.clone().map(AffinityGroup::new), ) - .await?; + .await? } None => { IoEngineApiClient::get_client() @@ -305,8 +308,16 @@ impl rpc::csi::controller_server::Controller for CsiControllerSvc { thin, sts_affinity_group_name.clone().map(AffinityGroup::new), ) - .await?; + .await? } + }; + + // Append the 'fsId' : 'volume id' to the context if change was requested for the + // clone. + if volume.spec.content_source.is_some() + && context.clone_fs_id_as_volume_id().unwrap_or(false) + { + volume_context.insert("fsId".to_string(), volume_uuid.clone()); } if let Some(ag_name) = sts_affinity_group_name { @@ -325,7 +336,7 @@ impl rpc::csi::controller_server::Controller for CsiControllerSvc { let volume = rpc::csi::Volume { capacity_bytes: size as i64, volume_id: volume_uuid, - volume_context: args.parameters.clone(), + volume_context, content_source: args.volume_content_source, accessible_topology: vt_mapper.volume_accessible_topology(), }; @@ -387,9 +398,7 @@ impl rpc::csi::controller_server::Controller for CsiControllerSvc { let _guard = csi_driver::limiter::VolumeOpGuard::new(volume_id)?; match args.volume_capability { - Some(c) => { - check_volume_capabilities(&[c])?; - } + Some(c) => check_volume_capabilities(&[c])?, None => { return Err(Status::invalid_argument("Missing volume capability")); } diff --git a/control-plane/csi-driver/src/bin/node/filesystem_ops.rs b/control-plane/csi-driver/src/bin/node/filesystem_ops.rs new file mode 100644 index 000000000..e79dac693 --- /dev/null +++ b/control-plane/csi-driver/src/bin/node/filesystem_ops.rs @@ -0,0 +1,311 @@ +//! This module consists of the various filesystem specific operations utility. Including creation +//! of filesystem, changing the parameter of filesystem like uuid, default mount options for +//! specific filesystem, repairing of the filesystem, retrieving specific properties of the +//! filesystem. + +use crate::mount; +use csi_driver::filesystem::FileSystem as Fs; +use devinfo::{blkid::probe::Probe, DevInfoError}; +use std::process::Output; +use tokio::process::Command; +use tonic::async_trait; +use tracing::{debug, trace}; +use uuid::Uuid; + +/// `nouuid` mount flag, to allow duplicate fs uuid. +const XFS_NO_UUID_FLAG: &str = "nouuid"; + +/// Error type filesystem operations. +type Error = String; + +/// Ext4 filesystem type. +pub(crate) struct Ext4Fs; +/// XFS filesystem type. +pub(crate) struct XFs; + +/// Filesystem type for csi node ops, wrapper over the parent Filesystem enum. +#[derive(Clone, Debug, Eq, PartialEq)] +pub(crate) struct FileSystem(Fs); + +impl From for FileSystem { + fn from(value: Fs) -> Self { + Self(value) + } +} + +impl AsRef for FileSystem { + fn as_ref(&self) -> &str { + self.0.as_ref() + } +} + +impl std::fmt::Display for FileSystem { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "{}", self.0.as_ref()) + } +} + +impl FileSystem { + /// Convert the top level enumeration to specific filesystem types to access the supported + /// operations by them. + pub(crate) fn fs_ops(&self) -> Result<&dyn FileSystemOps, Error> { + static EXT4FS: Ext4Fs = Ext4Fs {}; + static XFS: XFs = XFs {}; + match self.0 { + Fs::Ext4 => Ok(&EXT4FS), + Fs::Xfs => Ok(&XFS), + _ => Err(format!("Unsupported filesystem {self}")), + } + } + /// Get a specific filesystem property by specifying the device path. + pub(crate) fn property(device: &str, property: &str) -> Result { + let probe = Probe::new_from_filename(device)?; + probe.do_probe()?; + probe.lookup_value(property) + } +} + +#[async_trait] +pub(crate) trait FileSystemOps: Send + Sync { + /// Create the filesystem using its fs util. + async fn create(&self, device: &str) -> Result<(), Error>; + /// Get the default mount options along with the user passed options for specific filesystems. + fn mount_flags(&self, mount_flags: Vec) -> Vec; + /// Unmount the filesystem if the filesystem uuid and the provided uuid differ. + fn unmount_on_fs_id_diff( + &self, + device_path: &str, + fs_staging_path: &str, + volume_uuid: &Uuid, + ) -> Result<(), Error>; + /// Repair the filesystem with specific filesystem utility. + async fn repair( + &self, + device: &str, + staging_path: &str, + options: &[String], + volume_uuid: &Uuid, + ) -> Result<(), Error>; + /// Set the filesystem uuid. + async fn set_uuid(&self, device: &str, volume_uuid: &Uuid) -> Result<(), Error>; + /// Set the filesystem uuid with repair if needed. + async fn set_uuid_with_repair( + &self, + device: &str, + staging_path: &str, + options: &[String], + volume_uuid: &Uuid, + ) -> Result<(), Error> { + if self.set_uuid(device, volume_uuid).await.is_err() { + self.repair(device, staging_path, options, volume_uuid) + .await?; + self.set_uuid(device, volume_uuid).await?; + } + Ok(()) + } +} + +#[async_trait] +impl FileSystemOps for Ext4Fs { + async fn create(&self, device: &str) -> Result<(), Error> { + let binary = format!("mkfs.{}", "ext4"); + let output = Command::new(&binary) + .arg(device) + .output() + .await + .map_err(|error| format!("failed to execute {binary}: {error}"))?; + ack_command_output(output, binary) + } + + fn mount_flags(&self, mount_flags: Vec) -> Vec { + mount_flags + } + + fn unmount_on_fs_id_diff( + &self, + _device_path: &str, + _fs_staging_path: &str, + _volume_uuid: &Uuid, + ) -> Result<(), Error> { + Ok(()) + } + + async fn repair( + &self, + device: &str, + _staging_path: &str, + _options: &[String], + _volume_uuid: &Uuid, + ) -> Result<(), Error> { + let binary = "e2fsck".to_string(); + let output = Command::new(&binary) + .arg("-y") + .arg("-f") + .arg(device) + .output() + .await + .map_err(|error| format!("failed to execute {binary}: {error}"))?; + + trace!( + "Output from {} command: {}, status code: {:?}", + binary, + String::from_utf8(output.stdout.clone()).unwrap(), + output.status.code() + ); + + // For ext4 fs repair using e2fsck the success condition is when status code is 0 or 1. + if !output.status.success() && output.status.code() != Some(1) { + return Err(format!( + "{} command failed: {}", + binary, + String::from_utf8(output.stderr).unwrap() + )); + } + Ok(()) + } + + async fn set_uuid(&self, device: &str, volume_uuid: &Uuid) -> Result<(), Error> { + if let Ok(probed_uuid) = FileSystem::property(device, "UUID") { + if probed_uuid == volume_uuid.to_string() { + return Ok(()); + } + } + + let binary = "tune2fs".to_string(); + let output = Command::new(&binary) + .arg("-U") + .arg(volume_uuid.to_string()) + .arg(device) + .output() + .await + .map_err(|error| format!("failed to execute {binary}: {error}"))?; + + if !output.status.success() { + return Err(format!( + "{} command failed: {}", + binary, + String::from_utf8(output.stderr).unwrap() + )); + } + + let probe_uuid = FileSystem::property(device, "UUID") + .map_err(|error| format!("Failed to get UUID of device {device}: {error}"))?; + + if volume_uuid.to_string() != probe_uuid { + return Err(format!( + "failed to set filesystem uuid using : {binary}, {}", + String::from_utf8(output.stdout).unwrap_or_default() + )); + } + debug!("Changed filesystem uuid to {volume_uuid} for {device}"); + Ok(()) + } +} + +#[async_trait] +impl FileSystemOps for XFs { + async fn create(&self, device: &str) -> Result<(), Error> { + let binary = format!("mkfs.{}", "xfs"); + let output = Command::new(&binary) + .arg(device) + .output() + .await + .map_err(|error| format!("failed to execute {binary}: {error}"))?; + ack_command_output(output, binary) + } + + fn mount_flags(&self, mount_flags: Vec) -> Vec { + let mut mount_flags = mount_flags; + if !mount_flags.contains(&XFS_NO_UUID_FLAG.to_string()) { + mount_flags.push(XFS_NO_UUID_FLAG.to_string()) + } + mount_flags + } + + fn unmount_on_fs_id_diff( + &self, + device_path: &str, + fs_staging_path: &str, + volume_uuid: &Uuid, + ) -> Result<(), Error> { + mount::unmount_on_fs_id_diff(device_path, fs_staging_path, volume_uuid) + } + + /// Xfs filesystem needs an unmount to clear the log, so that the parameters can be changed. + /// Mount the filesystem to a defined path and then unmount it. + async fn repair( + &self, + device: &str, + staging_path: &str, + options: &[String], + volume_uuid: &Uuid, + ) -> Result<(), Error> { + mount::filesystem_mount(device, staging_path, &FileSystem(Fs::Xfs), options).map_err(|error| { + format!( + "(xfs repairing) Failed to mount device {device} onto {staging_path} for {volume_uuid} : {error}", + ) + })?; + mount::filesystem_unmount(staging_path).map_err(|error| { + format!( + "(xfs repairing) Failed to unmount device {device} from {staging_path} for {volume_uuid} : {error}", + ) + }) + } + + async fn set_uuid(&self, device: &str, volume_uuid: &Uuid) -> Result<(), Error> { + if let Ok(probed_uuid) = FileSystem::property(device, "UUID") { + if probed_uuid == volume_uuid.to_string() { + return Ok(()); + } + } + + let binary = "xfs_admin".to_string(); + let output = Command::new(&binary) + .arg("-U") + .arg(volume_uuid.to_string()) + .arg(device) + .output() + .await + .map_err(|error| format!("failed to execute {binary}: {error}"))?; + + if !output.status.success() { + return Err(format!( + "{} command failed: {}", + binary, + String::from_utf8(output.stderr).unwrap() + )); + } + + let probe_uuid = FileSystem::property(device, "UUID") + .map_err(|error| format!("Failed to get UUID of device {device}: {error}"))?; + + if volume_uuid.to_string() != probe_uuid { + return Err(format!( + "failed to set filesystem uuid using : {binary}, {}", + String::from_utf8(output.stdout).unwrap_or_default() + )); + } + debug!("Changed filesystem uuid to {volume_uuid} for {device}"); + Ok(()) + } +} + +// Acknowledge the output from Command. +fn ack_command_output(output: Output, binary: String) -> Result<(), Error> { + trace!( + "Output from {} command: {}, status code: {:?}", + binary, + String::from_utf8(output.stdout.clone()).unwrap(), + output.status.code() + ); + + if output.status.success() { + return Ok(()); + } + + Err(format!( + "{} command failed: {}", + binary, + String::from_utf8(output.stderr).unwrap() + )) +} diff --git a/control-plane/csi-driver/src/bin/node/filesystem_vol.rs b/control-plane/csi-driver/src/bin/node/filesystem_vol.rs index f33221921..dfe4d7649 100644 --- a/control-plane/csi-driver/src/bin/node/filesystem_vol.rs +++ b/control-plane/csi-driver/src/bin/node/filesystem_vol.rs @@ -1,19 +1,22 @@ //! Functions for CSI stage, unstage, publish and unpublish filesystem volumes. use crate::{ + filesystem_ops::FileSystem, format::prepare_device, mount::{self, subset, ReadOnly}, }; use csi_driver::{ - context::FileSystem, csi::{ volume_capability::MountVolume, NodePublishVolumeRequest, NodeStageVolumeRequest, NodeUnpublishVolumeRequest, NodeUnstageVolumeRequest, }, + filesystem::FileSystem as Fs, + PublishParams, }; use std::{fs, io::ErrorKind, path::PathBuf}; use tonic::{Code, Status}; use tracing::{debug, error, info}; +use uuid::Uuid; macro_rules! failure { (Code::$code:ident, $msg:literal) => {{ error!($msg); Status::new(Code::$code, $msg) }}; @@ -26,7 +29,19 @@ pub(crate) async fn stage_fs_volume( mnt: &MountVolume, filesystems: &[FileSystem], ) -> Result<(), Status> { - let volume_id = &msg.volume_id; + let volume_uuid = Uuid::parse_str(&msg.volume_id).map_err(|error| { + failure!( + Code::InvalidArgument, + "Failed to stage volume {}: not a valid UUID: {}", + &msg.volume_id, + error + ) + })?; + + // Extract the fs_id from the context, will only be set if requested and its a clone/restore. + let params = PublishParams::try_from(&msg.publish_context)?; + let fs_id = params.fs_id(); + let fs_staging_path = &msg.staging_target_path; // One final check for fs volumes, ignore for block volumes. @@ -36,13 +51,13 @@ pub(crate) async fn stage_fs_volume( Code::Internal, format!( "Failed to create mountpoint {} for volume {}: {}", - &fs_staging_path, volume_id, err + &fs_staging_path, volume_uuid, err ), )); } } - debug!("Staging volume {} to {}", volume_id, fs_staging_path); + debug!("Staging volume {} to {}", volume_uuid, fs_staging_path); let fstype = if mnt.fs_type.is_empty() { &filesystems[0] @@ -56,7 +71,7 @@ pub(crate) async fn stage_fs_volume( return Err(failure!( Code::InvalidArgument, "Failed to stage volume {}: unsupported filesystem type: {}", - volume_id, + volume_uuid, mnt.fs_type )); } @@ -70,15 +85,33 @@ pub(crate) async fn stage_fs_volume( ); info!( %existing, - "Volume {} is already staged to {}", volume_id, fs_staging_path + "Volume {} is already staged to {}", volume_uuid, fs_staging_path ); - // todo: validate other flags? - if mnt.mount_flags.readonly() != existing.options.readonly() { - mount::remount(fs_staging_path, mnt.mount_flags.readonly())?; - } + // If clone's fs id change was requested and we were not able to change it in first attempt + // unmount and continue the stage again. + let continue_stage = if fs_id.is_some() { + continue_after_unmount_on_fs_id_diff(fstype ,device_path, fs_staging_path, &volume_uuid) + .map_err(|error| { + failure!( + Code::FailedPrecondition, + "Failed to stage volume {}: staging path unmount on fs id difference failed: {}", + volume_uuid, + error + ) + })? + } else { + false + }; + + if !continue_stage { + // todo: validate other flags? + if mnt.mount_flags.readonly() != existing.options.readonly() { + mount::remount(fs_staging_path, mnt.mount_flags.readonly())?; + } - return Ok(()); + return Ok(()); + } } // abort if device is mounted somewhere else @@ -86,7 +119,7 @@ pub(crate) async fn stage_fs_volume( return Err(failure!( Code::AlreadyExists, "Failed to stage volume {}: device {} is already mounted elsewhere", - volume_id, + volume_uuid, device_path )); } @@ -96,16 +129,31 @@ pub(crate) async fn stage_fs_volume( return Err(failure!( Code::AlreadyExists, "Failed to stage volume {}: another device is already mounted onto {}", - volume_id, + volume_uuid, fs_staging_path )); } - if let Err(error) = prepare_device(device_path, fstype).await { + let mount_flags = fstype + .fs_ops() + .map_err(|error| { + failure!( + Code::Internal, + "Failed to stage volume {}: could not get mount flags for {}, {}", + volume_uuid, + fstype, + error + ) + })? + .mount_flags(mnt.mount_flags.clone()); + + if let Err(error) = + prepare_device(fstype, device_path, fs_staging_path, &mount_flags, fs_id).await + { return Err(failure!( Code::Internal, "Failed to stage volume {}: error preparing device {}: {}", - volume_id, + volume_uuid, device_path, error )); @@ -113,20 +161,19 @@ pub(crate) async fn stage_fs_volume( debug!("Mounting device {} onto {}", device_path, fs_staging_path); - if let Err(error) = - mount::filesystem_mount(device_path, fs_staging_path, fstype, &mnt.mount_flags) + if let Err(error) = mount::filesystem_mount(device_path, fs_staging_path, fstype, &mount_flags) { return Err(failure!( Code::Internal, "Failed to stage volume {}: failed to mount device {} onto {}: {}", - volume_id, + volume_uuid, device_path, fs_staging_path, error )); } - info!("Volume {} staged to {}", volume_id, fs_staging_path); + info!("Volume {} staged to {}", volume_uuid, fs_staging_path); Ok(()) } @@ -377,3 +424,17 @@ pub(crate) fn unpublish_fs_volume(msg: &NodeUnpublishVolumeRequest) -> Result<() info!("Volume {} unpublished from {}", volume_id, target_path); Ok(()) } + +/// Check if we can continue the staging incase the change fs id failed mid way and we want to retry +/// the flow. +fn continue_after_unmount_on_fs_id_diff( + fstype: &FileSystem, + device_path: &str, + fs_staging_path: &str, + volume_uuid: &Uuid, +) -> Result { + fstype + .fs_ops()? + .unmount_on_fs_id_diff(device_path, fs_staging_path, volume_uuid)?; + Ok(fstype == &Fs::Xfs.into()) +} diff --git a/control-plane/csi-driver/src/bin/node/findmnt.rs b/control-plane/csi-driver/src/bin/node/findmnt.rs index d1de5c590..70837ac1e 100644 --- a/control-plane/csi-driver/src/bin/node/findmnt.rs +++ b/control-plane/csi-driver/src/bin/node/findmnt.rs @@ -1,5 +1,5 @@ -use crate::error::DeviceError; -use csi_driver::context::FileSystem; +use crate::{error::DeviceError, filesystem_ops::FileSystem}; +use csi_driver::filesystem::FileSystem as Fs; use serde_json::Value; use std::{collections::HashMap, process::Command, str::FromStr, string::String, vec::Vec}; @@ -193,14 +193,15 @@ pub(crate) fn get_mountpaths(device_path: &str) -> Result, Devi if let Some(fstype) = entry.get(FSTYPE_KEY) { mountpaths.push(DeviceMount::new( mountpath.to_string(), - FileSystem::from_str(fstype) - .unwrap_or(FileSystem::Unsupported(fstype.to_string())), + Fs::from_str(fstype) + .unwrap_or(Fs::Unsupported(fstype.to_string())) + .into(), )) } else { error!("Missing fstype for {}", mountpath); mountpaths.push(DeviceMount::new( mountpath.to_string(), - FileSystem::Unsupported("".to_string()), + Fs::Unsupported("".to_string()).into(), )) } } else { diff --git a/control-plane/csi-driver/src/bin/node/format.rs b/control-plane/csi-driver/src/bin/node/format.rs index e2c2ed479..bfc6a14c3 100644 --- a/control-plane/csi-driver/src/bin/node/format.rs +++ b/control-plane/csi-driver/src/bin/node/format.rs @@ -1,46 +1,32 @@ //! Utility function for formatting a device with filesystem -use csi_driver::context::FileSystem; - -use devinfo::blkid::probe::Probe; -use std::process::Command; -use tracing::{debug, trace}; - -pub(crate) async fn prepare_device(device: &str, fstype: &FileSystem) -> Result<(), String> { +use crate::filesystem_ops::FileSystem; + +use tracing::debug; +use uuid::Uuid; + +/// Prepare the filesystem before mount, change parameters if requested. +pub(crate) async fn prepare_device( + fstype: &FileSystem, + device: &str, + staging_path: &str, + options: &[String], + fs_id: &Option, +) -> Result<(), String> { debug!("Probing device {}", device); + let fs = FileSystem::property(device, "TYPE"); - let probe = - Probe::new_from_filename(device).map_err(|error| format!("probe setup failed: {error}"))?; - - if let Err(error) = probe.do_probe() { - return Err(format!("probe failed: {error}")); - } + let fs_ops = fstype.fs_ops()?; - if let Ok(fs) = probe.lookup_value("TYPE") { + if let Ok(fs) = fs { debug!("Found existing filesystem ({}) on device {}", fs, device); + if let Some(fs_id) = fs_id { + debug!("Attempting to set uuid for filesystem {fs_id}, device: {device}"); + fs_ops + .set_uuid_with_repair(device, staging_path, options, fs_id) + .await?; + } return Ok(()); } - debug!("Creating new filesystem ({}) on device {}", fstype, device); - - let binary = format!("mkfs.{fstype}"); - let output = Command::new(&binary) - .arg(device) - .output() - .map_err(|error| format!("failed to execute {binary}: {error}"))?; - - trace!( - "Output from {} command: {}", - binary, - String::from_utf8(output.stdout.clone()).unwrap() - ); - - if output.status.success() { - return Ok(()); - } - - Err(format!( - "{} command failed: {}", - binary, - String::from_utf8(output.stderr).unwrap() - )) + fs_ops.create(device).await } diff --git a/control-plane/csi-driver/src/bin/node/main.rs b/control-plane/csi-driver/src/bin/node/main.rs index 65db686b4..e4d988293 100644 --- a/control-plane/csi-driver/src/bin/node/main.rs +++ b/control-plane/csi-driver/src/bin/node/main.rs @@ -8,6 +8,8 @@ pub(crate) mod config; mod dev; #[cfg(target_os = "linux")] mod error; +/// Filesystem specific operations. +pub(crate) mod filesystem_ops; #[cfg(target_os = "linux")] mod filesystem_vol; #[cfg(target_os = "linux")] diff --git a/control-plane/csi-driver/src/bin/node/mount.rs b/control-plane/csi-driver/src/bin/node/mount.rs index b78ae91e2..5752b7303 100644 --- a/control-plane/csi-driver/src/bin/node/mount.rs +++ b/control-plane/csi-driver/src/bin/node/mount.rs @@ -1,10 +1,12 @@ //! Utility functions for mounting and unmounting filesystems. -use csi_driver::context::FileSystem; - +use crate::filesystem_ops::FileSystem; +use csi_driver::filesystem::FileSystem as Fs; use devinfo::mountinfo::{MountInfo, MountIter}; + use std::{collections::HashSet, io::Error}; use sys_mount::{unmount, FilesystemType, Mount, MountFlags, UnmountFlags}; use tracing::{debug, info}; +use uuid::Uuid; // Simple trait for checking if the readonly (ro) option // is present in a "list" of options, while allowing for @@ -88,7 +90,7 @@ pub(super) fn subset(first: &[String], second: &[String]) -> bool { /// Return supported filesystems. pub(crate) fn probe_filesystems() -> Vec { - vec![FileSystem::Xfs, FileSystem::Ext4] + vec![Fs::Xfs.into(), Fs::Ext4.into()] } // Utility function to transform a vector of options @@ -351,3 +353,21 @@ async fn wait_file_removal( )), } } + +/// If the filesystem uuid doesn't match with the provided uuid, unmount the device. +pub(crate) fn unmount_on_fs_id_diff( + device_path: &str, + fs_staging_path: &str, + volume_uuid: &Uuid, +) -> Result<(), String> { + if let Ok(probed_uuid) = FileSystem::property(device_path, "UUID") { + if probed_uuid == volume_uuid.to_string() { + return Ok(()); + } + } + filesystem_unmount(fs_staging_path).map_err(|error| { + format!( + "Failed to unmount on fs id difference, device {device_path} from {fs_staging_path} for {volume_uuid}, {error}", + ) + }) +} diff --git a/control-plane/csi-driver/src/bin/node/node.rs b/control-plane/csi-driver/src/bin/node/node.rs index 619e6c73d..b53001b62 100644 --- a/control-plane/csi-driver/src/bin/node/node.rs +++ b/control-plane/csi-driver/src/bin/node/node.rs @@ -1,12 +1,10 @@ use crate::{ block_vol::{publish_block_volume, unpublish_block_volume}, dev::Device, + filesystem_ops::FileSystem, filesystem_vol::{publish_fs_volume, stage_fs_volume, unpublish_fs_volume, unstage_fs_volume}, }; -use csi_driver::{ - context::FileSystem, - csi::volume_capability::{access_mode::Mode, AccessType}, -}; +use csi_driver::csi::volume_capability::{access_mode::Mode, AccessType}; use rpc::{ csi, csi::{ diff --git a/control-plane/csi-driver/src/bin/node/nodeplugin_svc.rs b/control-plane/csi-driver/src/bin/node/nodeplugin_svc.rs index 8a1db797b..e4206f43a 100644 --- a/control-plane/csi-driver/src/bin/node/nodeplugin_svc.rs +++ b/control-plane/csi-driver/src/bin/node/nodeplugin_svc.rs @@ -5,7 +5,7 @@ use crate::{ dev::{Detach, Device, DeviceError}, findmnt, mount, }; -use csi_driver::context::FileSystem; +use csi_driver::filesystem::FileSystem as Fs; use snafu::{OptionExt, ResultExt, Snafu}; use tokio::process::Command; @@ -168,7 +168,7 @@ pub(crate) async fn find_mount( } } debug!(volume.uuid = volume_id, ?fstype, "Found fstype for volume"); - if fstype == FileSystem::DevTmpFs { + if fstype == Fs::DevTmpFs.into() { Ok(Some(TypeOfMount::RawBlock)) } else { Ok(Some(TypeOfMount::FileSystem)) diff --git a/control-plane/csi-driver/src/context.rs b/control-plane/csi-driver/src/context.rs index 6be60f59e..f7d2a0c39 100644 --- a/control-plane/csi-driver/src/context.rs +++ b/control-plane/csi-driver/src/context.rs @@ -1,3 +1,7 @@ +use crate::filesystem::FileSystem; +use stor_port::types::v0::openapi::models::VolumeShareProtocol; +use utils::K8S_STS_PVC_NAMING_REGEX; + use regex::Regex; use std::{ collections::HashMap, @@ -5,39 +9,9 @@ use std::{ num::ParseIntError, str::{FromStr, ParseBoolError}, }; -use stor_port::types::v0::openapi::models::VolumeShareProtocol; use strum_macros::{AsRefStr, Display, EnumString}; use tracing::log::warn; -use utils::K8S_STS_PVC_NAMING_REGEX; - -/// A type to enumerate used filesystems. -#[derive(EnumString, Clone, Debug, Eq, PartialEq)] -#[strum(serialize_all = "lowercase")] -pub enum FileSystem { - Ext4, - Xfs, - DevTmpFs, - Unsupported(String), -} - -// Implement as ref for the FileSystem. -impl AsRef for FileSystem { - fn as_ref(&self) -> &str { - match self { - FileSystem::Ext4 => "ext4", - FileSystem::Xfs => "xfs", - FileSystem::DevTmpFs => "devtmpfs", - FileSystem::Unsupported(inner) => inner, - } - } -} - -// Implement Display for the filesystem -impl std::fmt::Display for FileSystem { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - write!(f, "{}", self.as_ref()) - } -} +use uuid::{Error as UuidError, Uuid}; /// Parse string protocol into REST API protocol enum. pub fn parse_protocol(proto: Option<&String>) -> Result { @@ -70,6 +44,10 @@ pub enum Parameters { PvcNamespace, #[strum(serialize = "stsAffinityGroup")] StsAffinityGroup, + #[strum(serialize = "cloneFsIdAsVolumeId")] + CloneFsIdAsVolumeId, + #[strum(serialize = "fsId")] + FsId, } impl Parameters { fn parse_u32(value: Option<&String>) -> Result, ParseIntError> { @@ -84,6 +62,12 @@ impl Parameters { None => None, }) } + fn parse_uuid(value: Option<&String>) -> Result, UuidError> { + Ok(match value { + Some(value) => value.parse::().map(Some)?, + None => None, + }) + } /// Parse the value for `Self::NvmeCtrlLossTmo`. pub fn ctrl_loss_tmo(value: Option<&String>) -> Result, ParseIntError> { Self::parse_u32(value) @@ -104,15 +88,27 @@ impl Parameters { pub fn sts_affinity_group(value: Option<&String>) -> Result, ParseBoolError> { Self::parse_bool(value) } + /// Parse the value for `Self::CloneFsAsIdVolumeId` + pub fn clone_fs_id_as_volume_id( + value: Option<&String>, + ) -> Result, ParseBoolError> { + Self::parse_bool(value) + } + /// Parse the value for `Self::FsId` + pub fn fs_id(value: Option<&String>) -> Result, UuidError> { + Self::parse_uuid(value) + } } /// Volume publish parameters. #[allow(dead_code)] +#[derive(Debug)] pub struct PublishParams { io_timeout: Option, ctrl_loss_tmo: Option, keep_alive_tmo: Option, fs_type: Option, + fs_id: Option, } impl PublishParams { /// Get the `Parameters::IoTimeout` value. @@ -127,6 +123,10 @@ impl PublishParams { pub fn keep_alive_tmo(&self) -> &Option { &self.keep_alive_tmo } + /// Get the `Parameters::FsId` value. + pub fn fs_id(&self) -> &Option { + &self.fs_id + } /// Convert `Self` into a publish context. pub fn into_context(self) -> HashMap { let mut publish_context = HashMap::new(); @@ -146,6 +146,9 @@ impl PublishParams { keep_alive_tmo.to_string(), ); } + if let Some(fs_id) = self.fs_id() { + publish_context.insert(Parameters::FsId.to_string(), fs_id.to_string()); + } publish_context } @@ -169,12 +172,15 @@ impl TryFrom<&HashMap> for PublishParams { let keep_alive_tmo = Parameters::keep_alive_tmo(args.get(Parameters::NvmeKeepAliveTmo.as_ref())) .map_err(|_| tonic::Status::invalid_argument("Invalid keep_alive_tmo"))?; + let fs_id = Parameters::fs_id(args.get(Parameters::FsId.as_ref())) + .map_err(|_| tonic::Status::invalid_argument("Invalid fs_id"))?; Ok(Self { io_timeout, ctrl_loss_tmo, keep_alive_tmo, fs_type, + fs_id, }) } } @@ -186,6 +192,7 @@ pub struct CreateParams { share_protocol: VolumeShareProtocol, replica_count: u8, sts_affinity_group: Option, + clone_fs_id_as_volume_id: Option, } impl CreateParams { /// Get the `Parameters::ShareProtocol` value. @@ -201,6 +208,10 @@ impl CreateParams { pub fn sts_affinity_group(&self) -> &Option { &self.sts_affinity_group } + /// Get the `Parameters::CloneFsIdAsVolumeId` value. + pub fn clone_fs_id_as_volume_id(&self) -> &Option { + &self.clone_fs_id_as_volume_id + } } impl TryFrom<&HashMap> for CreateParams { type Error = tonic::Status; @@ -243,11 +254,17 @@ impl TryFrom<&HashMap> for CreateParams { None }; + let clone_fs_id_as_volume_id = Parameters::clone_fs_id_as_volume_id( + args.get(Parameters::CloneFsIdAsVolumeId.as_ref()), + ) + .map_err(|_| tonic::Status::invalid_argument("Invalid clone_fs_id_as_volume_id"))?; + Ok(Self { publish_params, share_protocol, replica_count, sts_affinity_group: sts_affinity_group_name, + clone_fs_id_as_volume_id, }) } } diff --git a/control-plane/csi-driver/src/filesystem.rs b/control-plane/csi-driver/src/filesystem.rs new file mode 100644 index 000000000..8982e360a --- /dev/null +++ b/control-plane/csi-driver/src/filesystem.rs @@ -0,0 +1,31 @@ +//! This module consists of the filesystem type definition shared between controller and node. +use strum_macros::EnumString; + +/// A type to enumerate used filesystems. +#[derive(EnumString, Clone, Debug, Eq, PartialEq)] +#[strum(serialize_all = "lowercase")] +pub enum FileSystem { + Ext4, + Xfs, + DevTmpFs, + Unsupported(String), +} + +// Implement as ref for the FileSystem. +impl AsRef for FileSystem { + fn as_ref(&self) -> &str { + match self { + FileSystem::Ext4 => "ext4", + FileSystem::Xfs => "xfs", + FileSystem::DevTmpFs => "devtmpfs", + FileSystem::Unsupported(inner) => inner, + } + } +} + +// Implement Display for the filesystem +impl std::fmt::Display for FileSystem { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "{}", self.as_ref()) + } +} diff --git a/control-plane/csi-driver/src/lib.rs b/control-plane/csi-driver/src/lib.rs index 4abae62a1..cc53efa0e 100644 --- a/control-plane/csi-driver/src/lib.rs +++ b/control-plane/csi-driver/src/lib.rs @@ -56,5 +56,7 @@ pub use rpc::csi; /// The volume contexts. pub mod context; +/// Filesystem Operations. +pub mod filesystem; /// Volume concurrency limiter. pub mod limiter;