test: use tmp in project workspace

Use a tmp folder from the workspace allowing us to cleanup up things like
LVM volumes a lot easier as we can just purge it.

Signed-off-by: Tiago Castro <[email protected]>

.gitignore

@@ -26,3 +26,4 @@ __pycache__
 /rpc/mayastor-api
 /local-fio-0-verify.state
 /report.xml
+/tmp

control-plane/agents/src/bin/core/tests/pool/mod.rs

@@ -1041,10 +1041,10 @@ async fn slow_create() {
     {
         let cluster = ClusterBuilder::builder()
             .with_io_engines(1)
-            .with_reconcile_period(Duration::from_secs(2), Duration::from_secs(2))
-            .with_cache_period("1s")
+            .with_reconcile_period(Duration::from_millis(250), Duration::from_millis(250))
+            .with_cache_period("200ms")
             .with_options(|o| o.with_io_engine_devices(vec![lvol.path()]))
-            .with_req_timeouts(Duration::from_secs(2), Duration::from_secs(2))
+            .with_req_timeouts(Duration::from_millis(500), Duration::from_millis(500))
             .compose_build(|b| b.with_clean(true))
             .await
             .unwrap();

control-plane/agents/src/bin/core/tests/snapshot/fs_cons_snapshot.rs

@@ -8,7 +8,7 @@ struct DeviceDisconnect(nvmeadm::NvmeTarget);
 impl Drop for DeviceDisconnect {
     fn drop(&mut self) {
         if self.0.disconnect().is_err() {
-            std::process::Command::new("sudo")
+            std::process::Command::new(env!("SUDO"))
                 .args(["nvme", "disconnect-all"])
                 .status()
                 .unwrap();

control-plane/agents/src/bin/core/tests/volume/capacity.rs

@@ -89,7 +89,7 @@ struct DeviceDisconnect(nvmeadm::NvmeTarget);
 impl Drop for DeviceDisconnect {
     fn drop(&mut self) {
         if self.0.disconnect().is_err() {
-            std::process::Command::new("sudo")
+            std::process::Command::new(env!("SUDO"))
                 .args(["nvme", "disconnect-all"])
                 .status()
                 .unwrap();

deployer/src/infra/io_engine.rs

@@ -11,6 +11,7 @@ use utils::DEFAULT_GRPC_CLIENT_ADDR;
 impl ComponentAction for IoEngine {
     fn configure(&self, options: &StartOptions, cfg: Builder) -> Result<Builder, Error> {
         let mut cfg = cfg;
+        let host_tmp = crate::host_tmp()?;
         for i in 0 .. options.io_engines + options.idle_io_engines {
             let io_engine_socket =
                 format!("{}:10124", cfg.next_ip_for_name(&Self::name(i, options))?);
@@ -53,7 +54,7 @@ impl ComponentAction for IoEngine {
             .with_env("NEXUS_NVMF_ANA_ENABLE", "1")
             .with_env("NVMF_TGT_CRDT", "0")
             .with_env("ENABLE_SNAPSHOT_REBUILD", "true")
-            .with_bind("/tmp", "/host/tmp")
+            .with_bind(&host_tmp, "/host/tmp")
             .with_bind("/var/run/dpdk", "/var/run/dpdk");
 
             let core_list = match options.io_engine_isolate {

deployer/src/lib.rs

@@ -786,3 +786,12 @@ impl std::fmt::Debug for ClusterLabel {
         write!(f, "{self}")
     }
 }
+
+/// Get the host tmp folder for this workspace.
+pub fn host_tmp() -> Result<String, std::io::Error> {
+    let root_tmp = format!("{root}/tmp", root = env!("WORKSPACE_ROOT"));
+    if !std::path::Path::new(&root_tmp).exists() {
+        std::fs::create_dir(&root_tmp)?;
+    }
+    Ok(root_tmp)
+}

scripts/rust/deployer-cleanup.sh

@@ -2,8 +2,26 @@
 
 SCRIPT_DIR="$(dirname "$0")"
 export ROOT_DIR="$SCRIPT_DIR/../.."
+SUDO=$(which sudo)
 
-sudo nvme disconnect-all
+cleanup_ws_tmp() {
+  # This contains tmp for container artifacts, example: pool disk images
+  tmp_dir="$(realpath "$ROOT_DIR/tmp")"
+
+  devices=$(losetup -l -J | jq -r --arg tmp_dir=$tmp_dir '.loopdevices[]|select(."back-file" | startswith($tmp_dir))')
+  for device in $(echo $devices | jq -r '.loopdevices[].name'); do
+    echo "Found stale loop device: $device"
+
+    $SUDO $(which vgremove) -y --select="pv_name=$device" || :
+    $SUDO losetup -D "$device"
+  done
+
+  rm -rf "$tmp_dir"
+
+  return 0
+}
+
+$SUDO nvme disconnect-all
 "$ROOT_DIR"/target/debug/deployer stop
 
 for c in $(docker ps -a --filter "label=io.composer.test.name" --format '{{.ID}}') ; do
@@ -12,7 +30,7 @@ for c in $(docker ps -a --filter "label=io.composer.test.name" --format '{{.ID}}
 done
 
 for n in $(docker network ls --filter "label=io.composer.test.name" --format '{{.ID}}') ; do
-  docker network rm "$n" || ( sudo systemctl restart docker && docker network rm "$n" )
+  docker network rm "$n" || ( $SUDO systemctl restart docker && docker network rm "$n" )
 done
 
-exit 0
+cleanup_ws_tmp

shell.nix

@@ -91,6 +91,7 @@ mkShell {
     [ ! -z "${io-engine}" ] && cowsay "${io-engine-moth}"
     [ ! -z "${io-engine}" ] && export IO_ENGINE_BIN="${io-engine-moth}"
     export PATH="$PATH:$(pwd)/target/debug"
+    export SUDO=$(which sudo || echo /run/wrappers/bin/sudo)
 
     DOCKER_CONFIG=~/.docker/config.json
     if [ -f "$DOCKER_CONFIG" ]; then

tests/bdd/common/deployer.py

@@ -196,18 +196,21 @@ def node_name(id: int):
 
     @staticmethod
     def create_disks(len=1, size=100 * 1024 * 1024):
-        disks = list(map(lambda x: f"/tmp/disk_{x}.img", range(1, len + 1)))
+        host_tmp = workspace_tmp()
+        disks = list(map(lambda x: f"disk_{x}.img", range(1, len + 1)))
         for disk in disks:
+            disk = f"{host_tmp}/{disk}"
             if os.path.exists(disk):
                 os.remove(disk)
             with open(disk, "w") as file:
                 file.truncate(size)
         # /tmp is mapped into /host/tmp within the io-engine containers
-        return list(map(lambda file: f"/host{file}", disks))
+        return list(map(lambda file: f"/host/tmp/{file}", disks))
 
     @staticmethod
     def delete_disks(len=1):
-        disks = list(map(lambda x: f"/tmp/disk_{x}.img", range(1, len + 1)))
+        host_tmp = workspace_tmp()
+        disks = list(map(lambda x: f"{host_tmp}/disk_{x}.img", range(1, len + 1)))
         for disk in disks:
             if os.path.exists(disk):
                 os.remove(disk)
@@ -224,3 +227,10 @@ def cache_period():
     @staticmethod
     def restart_node(node_name):
         Docker.restart_container(node_name)
+
+
+def workspace_tmp():
+    root = os.getenv("WORKSPACE_ROOT")
+    path = f"{root}/tmp"
+    os.makedirs(path, exist_ok=True)
+    return path

utils/deployer-cluster/src/lib.rs

@@ -648,19 +648,19 @@ impl TmpDiskFileInner {
         disk
     }
     fn make_new(name: &str) -> Self {
-        let path = Self::make_path(name);
+        let (path, container) = Self::make_path(name);
+        let pool_id = PoolId::new();
         Self {
-            // the io-engine is setup with a bind mount from /tmp to /host/tmp
-            uri: format!("aio:///host{}?blk_size=512&uuid={}", path, PoolId::new()),
+            // the io-engine is setup with a bind mount from /workspace/tmp to /host/tmp
+            uri: format!("aio://{container}{path}?blk_size=512&uuid={pool_id}"),
             path,
             cleanup: true,
         }
     }
-    fn make_path(name: &str) -> String {
-        // todo: use known path to facilitate cleanup.
-        // let root = std::env::var("WORKSPACE_ROOT").as_deref().unwrap_or("/tmp");
-        let root = "/tmp";
-        format!("{root}/io-engine-disk-{name}")
+    fn make_path(name: &str) -> (String, String) {
+        let file = format!("io-engine-disk-{name}");
+        let host_tmp = deployer_lib::host_tmp().expect("workspace error");
+        (format!("{host_tmp}/{file}"), format!("/host/tmp/{file}"))
     }
     fn uri(&self) -> &str {
         &self.uri

utils/deployer-cluster/src/lvm.rs

@@ -69,7 +69,7 @@ impl VolGroup {
     /// The output string is returned.
     fn command(args: &[&str]) -> Result<String, anyhow::Error> {
         let cmd = args.first().unwrap();
-        let output = std::process::Command::new("sudo")
+        let output = std::process::Command::new(env!("SUDO"))
             .arg("-E")
             .args(args)
             .output()?;