diff --git a/.gitignore b/.gitignore index a29982038..bd9b84f55 100644 --- a/.gitignore +++ b/.gitignore @@ -26,3 +26,4 @@ __pycache__ /rpc/mayastor-api /local-fio-0-verify.state /report.xml +/tmp diff --git a/control-plane/agents/src/bin/core/tests/pool/mod.rs b/control-plane/agents/src/bin/core/tests/pool/mod.rs index f8f657b88..67f3ddd2f 100644 --- a/control-plane/agents/src/bin/core/tests/pool/mod.rs +++ b/control-plane/agents/src/bin/core/tests/pool/mod.rs @@ -1041,10 +1041,10 @@ async fn slow_create() { { let cluster = ClusterBuilder::builder() .with_io_engines(1) - .with_reconcile_period(Duration::from_secs(2), Duration::from_secs(2)) - .with_cache_period("1s") + .with_reconcile_period(Duration::from_millis(250), Duration::from_millis(250)) + .with_cache_period("200ms") .with_options(|o| o.with_io_engine_devices(vec![lvol.path()])) - .with_req_timeouts(Duration::from_secs(2), Duration::from_secs(2)) + .with_req_timeouts(Duration::from_millis(500), Duration::from_millis(500)) .compose_build(|b| b.with_clean(true)) .await .unwrap(); diff --git a/control-plane/agents/src/bin/core/tests/snapshot/fs_cons_snapshot.rs b/control-plane/agents/src/bin/core/tests/snapshot/fs_cons_snapshot.rs index f3a56552c..c48e957bb 100644 --- a/control-plane/agents/src/bin/core/tests/snapshot/fs_cons_snapshot.rs +++ b/control-plane/agents/src/bin/core/tests/snapshot/fs_cons_snapshot.rs @@ -8,7 +8,7 @@ struct DeviceDisconnect(nvmeadm::NvmeTarget); impl Drop for DeviceDisconnect { fn drop(&mut self) { if self.0.disconnect().is_err() { - std::process::Command::new("sudo") + std::process::Command::new(env!("SUDO")) .args(["nvme", "disconnect-all"]) .status() .unwrap(); diff --git a/control-plane/agents/src/bin/core/tests/volume/capacity.rs b/control-plane/agents/src/bin/core/tests/volume/capacity.rs index fe0dead3c..375bb22a1 100644 --- a/control-plane/agents/src/bin/core/tests/volume/capacity.rs +++ b/control-plane/agents/src/bin/core/tests/volume/capacity.rs @@ -89,7 +89,7 @@ struct DeviceDisconnect(nvmeadm::NvmeTarget); impl Drop for DeviceDisconnect { fn drop(&mut self) { if self.0.disconnect().is_err() { - std::process::Command::new("sudo") + std::process::Command::new(env!("SUDO")) .args(["nvme", "disconnect-all"]) .status() .unwrap(); diff --git a/deployer/src/infra/io_engine.rs b/deployer/src/infra/io_engine.rs index 4e6d145c5..544bed9bf 100644 --- a/deployer/src/infra/io_engine.rs +++ b/deployer/src/infra/io_engine.rs @@ -11,6 +11,7 @@ use utils::DEFAULT_GRPC_CLIENT_ADDR; impl ComponentAction for IoEngine { fn configure(&self, options: &StartOptions, cfg: Builder) -> Result { let mut cfg = cfg; + let host_tmp = crate::host_tmp()?; for i in 0 .. options.io_engines + options.idle_io_engines { let io_engine_socket = format!("{}:10124", cfg.next_ip_for_name(&Self::name(i, options))?); @@ -53,7 +54,7 @@ impl ComponentAction for IoEngine { .with_env("NEXUS_NVMF_ANA_ENABLE", "1") .with_env("NVMF_TGT_CRDT", "0") .with_env("ENABLE_SNAPSHOT_REBUILD", "true") - .with_bind("/tmp", "/host/tmp") + .with_bind(&host_tmp, "/host/tmp") .with_bind("/var/run/dpdk", "/var/run/dpdk"); let core_list = match options.io_engine_isolate { diff --git a/deployer/src/lib.rs b/deployer/src/lib.rs index 5b7740375..62e56b4a9 100644 --- a/deployer/src/lib.rs +++ b/deployer/src/lib.rs @@ -786,3 +786,12 @@ impl std::fmt::Debug for ClusterLabel { write!(f, "{self}") } } + +/// Get the host tmp folder for this workspace. +pub fn host_tmp() -> Result { + let root_tmp = format!("{root}/tmp", root = env!("WORKSPACE_ROOT")); + if !std::path::Path::new(&root_tmp).exists() { + std::fs::create_dir(&root_tmp)?; + } + Ok(root_tmp) +} diff --git a/scripts/rust/deployer-cleanup.sh b/scripts/rust/deployer-cleanup.sh index 4367ad6e8..58d72a65d 100755 --- a/scripts/rust/deployer-cleanup.sh +++ b/scripts/rust/deployer-cleanup.sh @@ -2,8 +2,26 @@ SCRIPT_DIR="$(dirname "$0")" export ROOT_DIR="$SCRIPT_DIR/../.." +SUDO=$(which sudo) -sudo nvme disconnect-all +cleanup_ws_tmp() { + # This contains tmp for container artifacts, example: pool disk images + tmp_dir="$(realpath "$ROOT_DIR/tmp")" + + devices=$(losetup -l -J | jq -r --arg tmp_dir=$tmp_dir '.loopdevices[]|select(."back-file" | startswith($tmp_dir))') + for device in $(echo $devices | jq -r '.loopdevices[].name'); do + echo "Found stale loop device: $device" + + $SUDO $(which vgremove) -y --select="pv_name=$device" || : + $SUDO losetup -D "$device" + done + + rm -rf "$tmp_dir" + + return 0 +} + +$SUDO nvme disconnect-all "$ROOT_DIR"/target/debug/deployer stop for c in $(docker ps -a --filter "label=io.composer.test.name" --format '{{.ID}}') ; do @@ -12,7 +30,7 @@ for c in $(docker ps -a --filter "label=io.composer.test.name" --format '{{.ID}} done for n in $(docker network ls --filter "label=io.composer.test.name" --format '{{.ID}}') ; do - docker network rm "$n" || ( sudo systemctl restart docker && docker network rm "$n" ) + docker network rm "$n" || ( $SUDO systemctl restart docker && docker network rm "$n" ) done -exit 0 \ No newline at end of file +cleanup_ws_tmp diff --git a/shell.nix b/shell.nix index a88e0987f..021273ef7 100644 --- a/shell.nix +++ b/shell.nix @@ -91,6 +91,7 @@ mkShell { [ ! -z "${io-engine}" ] && cowsay "${io-engine-moth}" [ ! -z "${io-engine}" ] && export IO_ENGINE_BIN="${io-engine-moth}" export PATH="$PATH:$(pwd)/target/debug" + export SUDO=$(which sudo || echo /run/wrappers/bin/sudo) DOCKER_CONFIG=~/.docker/config.json if [ -f "$DOCKER_CONFIG" ]; then diff --git a/tests/bdd/common/deployer.py b/tests/bdd/common/deployer.py index 8a891d13a..385d7b770 100644 --- a/tests/bdd/common/deployer.py +++ b/tests/bdd/common/deployer.py @@ -196,18 +196,21 @@ def node_name(id: int): @staticmethod def create_disks(len=1, size=100 * 1024 * 1024): - disks = list(map(lambda x: f"/tmp/disk_{x}.img", range(1, len + 1))) + host_tmp = workspace_tmp() + disks = list(map(lambda x: f"disk_{x}.img", range(1, len + 1))) for disk in disks: + disk = f"{host_tmp}/{disk}" if os.path.exists(disk): os.remove(disk) with open(disk, "w") as file: file.truncate(size) # /tmp is mapped into /host/tmp within the io-engine containers - return list(map(lambda file: f"/host{file}", disks)) + return list(map(lambda file: f"/host/tmp/{file}", disks)) @staticmethod def delete_disks(len=1): - disks = list(map(lambda x: f"/tmp/disk_{x}.img", range(1, len + 1))) + host_tmp = workspace_tmp() + disks = list(map(lambda x: f"{host_tmp}/disk_{x}.img", range(1, len + 1))) for disk in disks: if os.path.exists(disk): os.remove(disk) @@ -224,3 +227,10 @@ def cache_period(): @staticmethod def restart_node(node_name): Docker.restart_container(node_name) + + +def workspace_tmp(): + root = os.getenv("WORKSPACE_ROOT") + path = f"{root}/tmp" + os.makedirs(path, exist_ok=True) + return path diff --git a/utils/deployer-cluster/src/lib.rs b/utils/deployer-cluster/src/lib.rs index 45c397c7a..ec9f6fcba 100644 --- a/utils/deployer-cluster/src/lib.rs +++ b/utils/deployer-cluster/src/lib.rs @@ -648,19 +648,19 @@ impl TmpDiskFileInner { disk } fn make_new(name: &str) -> Self { - let path = Self::make_path(name); + let (path, container) = Self::make_path(name); + let pool_id = PoolId::new(); Self { - // the io-engine is setup with a bind mount from /tmp to /host/tmp - uri: format!("aio:///host{}?blk_size=512&uuid={}", path, PoolId::new()), + // the io-engine is setup with a bind mount from /workspace/tmp to /host/tmp + uri: format!("aio://{container}{path}?blk_size=512&uuid={pool_id}"), path, cleanup: true, } } - fn make_path(name: &str) -> String { - // todo: use known path to facilitate cleanup. - // let root = std::env::var("WORKSPACE_ROOT").as_deref().unwrap_or("/tmp"); - let root = "/tmp"; - format!("{root}/io-engine-disk-{name}") + fn make_path(name: &str) -> (String, String) { + let file = format!("io-engine-disk-{name}"); + let host_tmp = deployer_lib::host_tmp().expect("workspace error"); + (format!("{host_tmp}/{file}"), format!("/host/tmp/{file}")) } fn uri(&self) -> &str { &self.uri diff --git a/utils/deployer-cluster/src/lvm.rs b/utils/deployer-cluster/src/lvm.rs index fb75c2bd9..f7b0fbd9b 100644 --- a/utils/deployer-cluster/src/lvm.rs +++ b/utils/deployer-cluster/src/lvm.rs @@ -69,7 +69,7 @@ impl VolGroup { /// The output string is returned. fn command(args: &[&str]) -> Result { let cmd = args.first().unwrap(); - let output = std::process::Command::new("sudo") + let output = std::process::Command::new(env!("SUDO")) .arg("-E") .args(args) .output()?;