test: use tmp in project workspace

Use a tmp folder from the workspace allowing us to cleanup up things like
LVM volumes a lot easier as we can just purge it.

Signed-off-by: Tiago Castro <[email protected]>

.gitignore

@@ -26,3 +26,4 @@ __pycache__
 /rpc/mayastor-api
 /local-fio-0-verify.state
 /report.xml
+/.tmp

control-plane/agents/src/bin/core/tests/pool/mod.rs

@@ -1035,16 +1035,16 @@ async fn destroy_after_restart() {
 async fn slow_create() {
     const POOL_SIZE_BYTES: u64 = 128 * 1024 * 1024;
 
-    let vg = deployer_cluster::lvm::VolGroup::new("slow-pool-1", POOL_SIZE_BYTES).unwrap();
+    let vg = deployer_cluster::lvm::VolGroup::new("slow-pool", POOL_SIZE_BYTES).unwrap();
     let lvol = vg.create_lvol("lvol0", POOL_SIZE_BYTES / 2).unwrap();
     lvol.suspend().unwrap();
     {
         let cluster = ClusterBuilder::builder()
             .with_io_engines(1)
-            .with_reconcile_period(Duration::from_secs(2), Duration::from_secs(2))
-            .with_cache_period("1s")
+            .with_reconcile_period(Duration::from_millis(250), Duration::from_millis(250))
+            .with_cache_period("200ms")
             .with_options(|o| o.with_io_engine_devices(vec![lvol.path()]))
-            .with_req_timeouts(Duration::from_secs(2), Duration::from_secs(2))
+            .with_req_timeouts(Duration::from_millis(500), Duration::from_millis(500))
             .compose_build(|b| b.with_clean(true))
             .await
             .unwrap();

control-plane/agents/src/bin/core/tests/snapshot/fs_cons_snapshot.rs

@@ -8,7 +8,7 @@ struct DeviceDisconnect(nvmeadm::NvmeTarget);
 impl Drop for DeviceDisconnect {
     fn drop(&mut self) {
         if self.0.disconnect().is_err() {
-            std::process::Command::new("sudo")
+            std::process::Command::new(env!("SUDO"))
                 .args(["nvme", "disconnect-all"])
                 .status()
                 .unwrap();

control-plane/agents/src/bin/core/tests/volume/capacity.rs

@@ -89,7 +89,7 @@ struct DeviceDisconnect(nvmeadm::NvmeTarget);
 impl Drop for DeviceDisconnect {
     fn drop(&mut self) {
         if self.0.disconnect().is_err() {
-            std::process::Command::new("sudo")
+            std::process::Command::new(env!("SUDO"))
                 .args(["nvme", "disconnect-all"])
                 .status()
                 .unwrap();

deployer/src/infra/io_engine.rs

@@ -11,6 +11,7 @@ use utils::DEFAULT_GRPC_CLIENT_ADDR;
 impl ComponentAction for IoEngine {
     fn configure(&self, options: &StartOptions, cfg: Builder) -> Result<Builder, Error> {
         let mut cfg = cfg;
+        let host_tmp = crate::host_tmp()?;
         for i in 0 .. options.io_engines + options.idle_io_engines {
             let io_engine_socket =
                 format!("{}:10124", cfg.next_ip_for_name(&Self::name(i, options))?);
@@ -53,7 +54,7 @@ impl ComponentAction for IoEngine {
             .with_env("NEXUS_NVMF_ANA_ENABLE", "1")
             .with_env("NVMF_TGT_CRDT", "0")
             .with_env("ENABLE_SNAPSHOT_REBUILD", "true")
-            .with_bind("/tmp", "/host/tmp")
+            .with_bind(&host_tmp, "/host/tmp")
             .with_bind("/var/run/dpdk", "/var/run/dpdk");
 
             let core_list = match options.io_engine_isolate {

deployer/src/lib.rs

@@ -786,3 +786,12 @@ impl std::fmt::Debug for ClusterLabel {
         write!(f, "{self}")
     }
 }
+
+/// Get the host tmp folder for this workspace.
+pub fn host_tmp() -> Result<String, std::io::Error> {
+    let root_tmp = format!("{root}/.tmp", root = env!("WORKSPACE_ROOT"));
+    if !std::path::Path::new(&root_tmp).exists() {
+        std::fs::create_dir(&root_tmp)?;
+    }
+    Ok(root_tmp)
+}

scripts/rust/deployer-cleanup.sh

@@ -2,8 +2,26 @@
 
 SCRIPT_DIR="$(dirname "$0")"
 export ROOT_DIR="$SCRIPT_DIR/../.."
+SUDO=$(which sudo)
 
-sudo nvme disconnect-all
+cleanup_ws_tmp() {
+  # This contains tmp for container artifacts, example: pool disk images
+  tmp_dir="$(realpath "$ROOT_DIR/.tmp")"
+
+  devices=$(losetup -l -J | jq -r --arg tmp_dir=$tmp_dir '.loopdevices[]|select(."back-file" | startswith($tmp_dir))')
+  for device in $(echo $devices | jq -r '.loopdevices[].name'); do
+    echo "Found stale loop device: $device"
+
+    $SUDO $(which vgremove) -y --select="pv_name=$device" || :
+    $SUDO losetup -d "$device"
+  done
+
+  $SUDO rm -rf "$tmp_dir"
+
+  return 0
+}
+
+$SUDO nvme disconnect-all
 "$ROOT_DIR"/target/debug/deployer stop
 
 for c in $(docker ps -a --filter "label=io.composer.test.name" --format '{{.ID}}') ; do
@@ -12,7 +30,7 @@ for c in $(docker ps -a --filter "label=io.composer.test.name" --format '{{.ID}}
 done
 
 for n in $(docker network ls --filter "label=io.composer.test.name" --format '{{.ID}}') ; do
-  docker network rm "$n" || ( sudo systemctl restart docker && docker network rm "$n" )
+  docker network rm "$n" || ( $SUDO systemctl restart docker && docker network rm "$n" )
 done
 
-exit 0
+cleanup_ws_tmp

shell.nix

@@ -91,6 +91,7 @@ mkShell {
     [ ! -z "${io-engine}" ] && cowsay "${io-engine-moth}"
     [ ! -z "${io-engine}" ] && export IO_ENGINE_BIN="${io-engine-moth}"
     export PATH="$PATH:$(pwd)/target/debug"
+    export SUDO=$(which sudo || echo /run/wrappers/bin/sudo)
 
     DOCKER_CONFIG=~/.docker/config.json
     if [ -f "$DOCKER_CONFIG" ]; then

tests/bdd/common/deployer.py

@@ -196,18 +196,21 @@ def node_name(id: int):
 
     @staticmethod
     def create_disks(len=1, size=100 * 1024 * 1024):
-        disks = list(map(lambda x: f"/tmp/disk_{x}.img", range(1, len + 1)))
+        host_tmp = workspace_tmp()
+        disks = list(map(lambda x: f"disk_{x}.img", range(1, len + 1)))
         for disk in disks:
+            disk = f"{host_tmp}/{disk}"
             if os.path.exists(disk):
                 os.remove(disk)
             with open(disk, "w") as file:
                 file.truncate(size)
         # /tmp is mapped into /host/tmp within the io-engine containers
-        return list(map(lambda file: f"/host{file}", disks))
+        return list(map(lambda file: f"/host/tmp/{file}", disks))
 
     @staticmethod
     def delete_disks(len=1):
-        disks = list(map(lambda x: f"/tmp/disk_{x}.img", range(1, len + 1)))
+        host_tmp = workspace_tmp()
+        disks = list(map(lambda x: f"{host_tmp}/disk_{x}.img", range(1, len + 1)))
         for disk in disks:
             if os.path.exists(disk):
                 os.remove(disk)
@@ -224,3 +227,10 @@ def cache_period():
     @staticmethod
     def restart_node(node_name):
         Docker.restart_container(node_name)
+
+
+def workspace_tmp():
+    root = os.getenv("WORKSPACE_ROOT")
+    path = f"{root}/.tmp"
+    os.makedirs(path, exist_ok=True)
+    return path

tests/bdd/features/capacity/thin/volume/test_create.py

@@ -35,6 +35,7 @@
 @pytest.fixture(autouse=True)
 def init(disks):
     Deployer.start(1)
+    Deployer.create_disks(1, POOL_SIZE)
     ApiClient.pools_api().put_node_pool(
         NODE_NAME, POOL_UUID, CreatePoolBody([f"{disks[0]}"])
     )
@@ -48,26 +49,9 @@ def init(disks):
 
 
 @pytest.fixture
-def tmp_files():
-    files = []
-    for index in range(0, 1):
-        files.append(f"/tmp/disk_{index}")
-    yield files
-
-
-@pytest.fixture
-def disks(tmp_files):
-    for disk in tmp_files:
-        if os.path.exists(disk):
-            os.remove(disk)
-        with open(disk, "w") as file:
-            file.truncate(POOL_SIZE)
-    # /tmp is mapped into /host/tmp within the io-engine containers
-    yield list(map(lambda file: f"/host{file}", tmp_files))
-
-    for disk in tmp_files:
-        if os.path.exists(disk):
-            os.remove(disk)
+def disks():
+    yield Deployer.create_disks(1, size=POOL_SIZE)
+    Deployer.cleanup_disks(1)
 
 
 @pytest.fixture(scope="function")

tests/bdd/features/cordon/node/test_cordon_node.py

@@ -58,14 +58,6 @@ def init_scenario(init, disks):
     remove_all_cordons(NODE_NAME_2)
 
 
-@pytest.fixture
-def tmp_files():
-    files = []
-    for index in range(0, 1):
-        files.append(f"/tmp/disk_{index}")
-    yield files
-
-
 @pytest.fixture(scope="module")
 def disks():
     yield Deployer.create_disks(2, size=100 * 1024 * 1024)

tests/bdd/features/garbage-collection/replicas/test_feature.py

@@ -29,26 +29,15 @@
 IO_ENGINE_1 = "io-engine-1"
 IO_ENGINE_2 = "io-engine-2"
 
-POOL_DISK1 = "pdisk1.img"
 POOL1_UUID = "4cc6ee64-7232-497d-a26f-38284a444980"
-POOL_DISK2 = "pdisk2.img"
 POOL2_UUID = "4cc6ee64-7232-497d-a26f-38284a444990"
 
 
 @pytest.fixture(scope="function")
 def create_pool_disk_images():
-    # When starting Io-Engine instances with the deployer a bind mount is created from /tmp to
-    # /host/tmp, so create disk images in /tmp
-    for disk in [POOL_DISK1, POOL_DISK2]:
-        path = "/tmp/{}".format(disk)
-        with open(path, "w") as file:
-            file.truncate(20 * 1024 * 1024)
-
-    yield
-    for disk in [POOL_DISK1, POOL_DISK2]:
-        path = "/tmp/{}".format(disk)
-        if os.path.exists(path):
-            os.remove(path)
+    pools = Deployer.create_disks(2, size=20 * 1024 * 1024)
+    yield pools
+    Deployer.cleanup_disks(len(pools))
 
 
 # This fixture will be automatically used by all tests.
@@ -63,12 +52,12 @@ def init(create_pool_disk_images):
     ApiClient.pools_api().put_node_pool(
         IO_ENGINE_1,
         POOL1_UUID,
-        CreatePoolBody(["aio:///host/tmp/{}".format(POOL_DISK1)]),
+        CreatePoolBody(["aio://{}".format(create_pool_disk_images[0])]),
     )
     ApiClient.pools_api().put_node_pool(
         IO_ENGINE_2,
         POOL2_UUID,
-        CreatePoolBody(["aio:///host/tmp/{}".format(POOL_DISK2)]),
+        CreatePoolBody(["aio://{}".format(create_pool_disk_images[1])]),
     )
 
     # Create and publish a volume on node 1

tests/bdd/features/ha/core-agent/test_target_switchover.py

@@ -41,25 +41,9 @@
 
 
 @pytest.fixture
-def tmp_files():
-    files = []
-    for index in range(0, 2):
-        files.append(f"/tmp/disk_{index}")
-    yield files
-
-
-@pytest.fixture
-def disks(tmp_files):
-    for disk in tmp_files:
-        if os.path.exists(disk):
-            os.remove(disk)
-        with open(disk, "w") as file:
-            file.truncate(100 * 1024 * 1024)
-    # /tmp is mapped into /host/tmp within the io-engine containers
-    yield list(map(lambda file: f"/host{file}", tmp_files))
-    for disk in tmp_files:
-        if os.path.exists(disk):
-            os.remove(disk)
+def disks():
+    yield Deployer.create_disks(2, size=100 * 1024 * 1024)
+    Deployer.cleanup_disks(2)
 
 
 @pytest.fixture(scope="module")

tests/bdd/features/ha/test_robustness.py

@@ -240,25 +240,10 @@ def the_volume_target_node_has_iopath_fixed():
 
 
 @pytest.fixture
-def tmp_files():
-    files = []
-    for index in range(0, 2):
-        files.append(f"/tmp/disk_{index}")
-    yield files
-
-
-@pytest.fixture
-def disks(tmp_files):
-    for disk in tmp_files:
-        if os.path.exists(disk):
-            os.remove(disk)
-        with open(disk, "w") as file:
-            file.truncate(POOL_SIZE)
-    # /tmp is mapped into /host/tmp within the io-engine containers
-    yield list(map(lambda file: f"/host{file}", tmp_files))
-    for disk in tmp_files:
-        if os.path.exists(disk):
-            os.remove(disk)
+def disks():
+    pools = Deployer.create_disks(2, size=POOL_SIZE)
+    yield pools
+    Deployer.cleanup_disks(len(pools))
 
 
 @pytest.fixture

tests/bdd/features/pool/create/test_disks.py

@@ -117,25 +117,10 @@ def pool_attempt(context):
 
 
 @pytest.fixture
-def tmp_files():
-    files = []
-    for index in range(0, POOL_DISK_COUNT + 1):
-        files.append(f"/tmp/disk_{index}")
-    yield files
-
-
-@pytest.fixture
-def disks(tmp_files):
-    for disk in tmp_files:
-        if os.path.exists(disk):
-            os.remove(disk)
-        with open(disk, "w") as file:
-            file.truncate(100 * 1024 * 1024)
-    # /tmp is mapped into /host/tmp within the io-engine containers
-    yield list(map(lambda file: f"/host{file}", tmp_files))
-    for disk in tmp_files:
-        if os.path.exists(disk):
-            os.remove(disk)
+def disks():
+    pools = Deployer.create_disks(POOL_DISK_COUNT, size=100 * 1024 * 1024)
+    yield pools
+    Deployer.cleanup_disks(len(pools))
 
 
 @pytest.fixture

tests/bdd/features/pool/delete/test_feature.py

@@ -143,7 +143,6 @@ def the_pool_should_be_deleted(pool, attempt_delete_the_pool):
 
 """" Implementations """
 
-POOLS_PER_NODE = 2
 VOLUME_UUID = "5cd5378e-3f05-47f1-a830-a0f5873a1449"
 VOLUME_SIZE = 10485761
 
@@ -161,32 +160,10 @@ def nodes():
 
 
 @pytest.fixture
-def tmp_files(nodes):
-    files = []
-    for node in range(len(nodes)):
-        node_files = []
-        for disk in range(0, POOLS_PER_NODE + 1):
-            node_files.append(f"/tmp/node-{node}-disk_{disk}")
-        files.append(node_files)
-    yield files
-
-
-@pytest.fixture
-def node_disks(tmp_files):
-    for node_disks in tmp_files:
-        for disk in node_disks:
-            if os.path.exists(disk):
-                os.remove(disk)
-            with open(disk, "w") as file:
-                file.truncate(100 * 1024 * 1024)
-
-    # /tmp is mapped into /host/tmp within the io-engine containers
-    yield list(map(lambda arr: list(map(lambda file: f"/host{file}", arr)), tmp_files))
-
-    for node_disks in tmp_files:
-        for disk in node_disks:
-            if os.path.exists(disk):
-                os.remove(disk)
+def node_disks(nodes):
+    pools = Deployer.create_disks(1, size=100 * 1024 * 1024)
+    yield pools
+    Deployer.cleanup_disks(len(pools))
 
 
 @pytest.fixture
@@ -202,9 +179,8 @@ def pool(node_disks, nodes):
     assert len(nodes) > 0
     assert len(node_disks) > 0
     node = nodes[0]
-    disks = node_disks[0]
     pool = ApiClient.pools_api().put_node_pool(
-        node.id, f"{node.id}-pool", CreatePoolBody([disks[0]])
+        node.id, f"{node.id}-pool", CreatePoolBody([node_disks[0]])
     )
     yield pool