[ISSUE-1202]: Automate test

Signed-off-by: Malgorzata Dutka <[email protected]>

tests/e2e-test-framework/framework/const.py

@@ -31,6 +31,7 @@
 # statuses
 STATUS_ONLINE = "ONLINE"
 STATUS_OFFLINE = "OFFLINE"
+STATUS_OPERATIVE = "OPERATIVE"
 
 # health
 HEALTH_GOOD = "GOOD"
@@ -39,10 +40,17 @@
 # fake attach
 FAKE_ATTACH_INVOLVED = "FakeAttachInvolved"
 FAKE_ATTACH_CLEARED = "FakeAttachCleared"
+DRIVE_HEALTH_FAILURE = "DriveHealthFailure"
+DRIVE_READY_FOR_REMOVAL = "DriveReadyForRemoval"
+VOLUME_BAD_HEALTH = "VolumeBadHealth"
+DRIVE_READY_FOR_PHYSICAL_REMOVAL = "DriveReadyForPhysicalRemoval"
 
 # plurals
 DRIVES_PLURAL = "drives"
 AC_PLURAL = "availablecapacities"
 ACR_PLURAL = "availablecapacityreservations"
 LVG_PLURAL = "logicalvolumegroups"
 VOLUMES_PLURAL = "volumes"
+
+# led
+LED_STATE_1 = "1"

tests/e2e-test-framework/framework/utils.py

@@ -691,3 +691,8 @@ def recreate_pod(self, name: str, namespace: str) -> V1Pod:
         logging.info(f"pod {name} is ready")
 
         return pod
+
+    def delete_pvc(self, namespace: str):
+        self.core_v1_api.delete_collection_namespaced_persistent_volume_claim(
+                namespace=namespace
+            )

tests/e2e-test-framework/tests/test_drive_replacement_multi_volumes.py

@@ -0,0 +1,190 @@
+import pytest
+import logging
+
+import framework.const as const
+
+from framework.sts import STS
+from framework.utils import Utils
+from framework.drive import DriveUtils
+
+
+
+
+class TestAutoDriveReplacementWithMultipleVolumesPerPod:
+    @classmethod
+    @pytest.fixture(autouse=True)
+    def setup_class(
+        cls,
+        namespace: str,
+        drive_utils_executors: dict[str, DriveUtils],
+        utils: Utils,
+    ):
+        cls.namespace = namespace
+        cls.name = "test-auto-drive-replacement-multiple-volumes"
+        cls.timeout = 120
+        cls.replicas = 1
+
+        cls.utils = utils
+
+        cls.drive_utils = drive_utils_executors
+        cls.sts = STS(cls.namespace, cls.name, cls.replicas)
+        cls.sts.delete()
+        cls.sts.create(storage_classes=[const.SSD_SC, const.HDD_SC])
+
+        yield
+
+        cls.sts.delete()
+
+    @pytest.mark.hal
+    def test_5921_auto_drive_replacement_with_multiple_volumes_per_pod(self):
+    # 1. get volume and volume groups for deployed pod
+        assert (
+            self.sts.verify(self.timeout) is True
+        ), f"STS: {self.name} failed to reach desired number of replicas: {self.replicas}"
+        pod = self.utils.list_pods(name_prefix=self.name)[0]
+        logging.info(f"Pod {type(pod)}: {pod}")
+        node_ip = self.utils.get_pod_node_ip(
+            pod_name=pod.metadata.name, namespace=self.namespace
+        )
+        volumes = self.utils.list_volumes(pod_name=pod.metadata.name)
+        logging.info(f"Pod's volumes: {volumes}")
+        # get all drives
+        drives = []
+        for volume in volumes:
+            drive = self.utils.get_drive_cr(
+                volume_name=volume["metadata"]["name"], 
+                namespace=volume["metadata"]["namespace"])
+            drives.append(drive)
+        logging.info(f"drives: {drives}")
+    # 2. simulate drive failure. Annotate drive used by pod with health=BAD
+        for drive in drives:
+            drive_name = drive["metadata"]["name"]
+            self.utils.annotate_custom_resource(
+                resource_name=drive_name, 
+                resource_type="drives", 
+                annotation_key="health", 
+                annotation_value="BAD"
+            )
+            logging.info(f"drive: {drive_name} was annotated with health=BAD")
+    # 3. wait until drive health is BAD, status=ONLINE, usage=RELEASING.
+        for drive in drives:
+            drive_name = drive["metadata"]["name"]
+            logging.info(f"Waiting for drive: {drive_name}")
+            self.utils.wait_drive(
+                name=drive_name, 
+                expected_status=const.STATUS_ONLINE, 
+                expected_health=const.HEALTH_BAD, 
+                expected_usage=const.USAGE_RELEASING
+            ) 
+            logging.info(f"drive {drive_name} went in Status: {const.STATUS_ONLINE}, Health: {const.HEALTH_BAD}, Usage: {const.USAGE_RELEASING}")
+    # 4. wait until volume health is BAD, status=OPERATIVE, usage=RELEASING.
+        for volume in volumes:
+            volume_name = drive["metadata"]["name"]
+            logging.info(f"Waiting for volume: {volume_name}")
+            self.utils.wait_volume(
+                name=volume_name, 
+                expected_health=const.HEALTH_BAD, 
+                expected_usage=const.USAGE_RELEASING, 
+                expected_operational_status=const.STATUS_OPERATIVE
+            )
+            logging.info(f"volume {drive_name} went in OperationalStatus: {const.STATUS_OPERATIVE}, Health: {const.HEALTH_BAD}, Usage: {const.USAGE_RELEASING}") 
+    # 5. check events and locate event related to DriveHealthFailure
+        for drive in drives:
+            drive_name = drive["metadata"]["name"]
+            assert self.utils.event_in(
+            resource_name=drive_name,
+            reason=const.DRIVE_HEALTH_FAILURE,
+            ), f"event {const.DRIVE_HEALTH_FAILURE} for drive {drive_name} not found"
+    # 6. annotate volume with release=done
+        for volume in volumes:
+            volume_name = drive["metadata"]["name"]
+            self.utils.annotate_custom_resource(
+                resource_name=volume_name, 
+                resource_type="volumes", 
+                annotation_key="release", 
+                annotation_value="done", 
+                namespace=volume['metadata']['namespace']
+            )
+            logging.info(f"volume: {volume_name} was annotated with release=done")
+    # 7. check drive usages are RELEASED and event DriveReadyForRemoval is generated
+        for drive in drives:
+            self.utils.wait_drive(
+                name=drive['metadata']['name'], 
+                expected_usage=const.USAGE_RELEASED
+            )
+    # 8. check events and locate event related to VolumeBadHealth
+        for drive in drives:
+            drive_name = drive["metadata"]["name"]
+            assert self.utils.event_in(
+            resource_name=drive_name,
+            reason=const.DRIVE_READY_FOR_REMOVAL,
+            ), f"event {const.DRIVE_READY_FOR_REMOVAL} for drive {drive_name} not found"
+    # 9. check volumes are RELEASED
+        for volume in volumes:
+            self.utils.wait_volume(
+                name=volume['metadata']['name'],
+                expected_usage=const.USAGE_RELEASED
+            )
+            logging.info(f"volume {drive_name} went in Usage: {const.USAGE_RELEASING}") 
+    # 10. check events and locate event related to VolumeBadHealth
+        for volume in volumes:
+            volume_name = volume["metadata"]["name"]
+            assert self.utils.event_in(
+            resource_name=volume_name,
+            reason=const.VOLUME_BAD_HEALTH,
+            ), f"event {const.VOLUME_BAD_HEALTH} for volume {volume_name} not found"
+    # 11. delete pod and pvc
+        self.utils.delete_pvcs_in_namespace(namespace=pod.metadata.name)
+        self.utils.recreate_pod(name=pod.metadata.name, namespace=self.namespace)
+    # 12. check Drive status to be REMOVING or REMOVED and LED state to be 1 (if drive supports LED ) or 2 (if drive does not support LED) Status to be ONLINE #TODO: status LED 2 => another test case
+        for drive in drives:
+            self.utils.wait_drive(
+                name=drive['metadata']['name'], 
+                expected_status=const.STATUS_ONLINE, 
+                expected_usage=const.USAGE_REMOVED, 
+                expected_health=const.HEALTH_BAD, 
+                expected_led_state=const.LED_STATE_1
+            )
+            logging.info(f"drive {drive_name} went in Status: {const.STATUS_ONLINE}, Health: {const.HEALTH_BAD}, Usage: {const.USAGE_REMOVED}, LedState: {const.LED_STATE_1}") 
+    # 13. check for events: DriveReadyForPhysicalRemoval   
+        for drive in drives:
+            drive_name = drive["metadata"]["name"]
+            assert self.utils.event_in(
+            resource_name=drive_name,
+            reason=const.DRIVE_READY_FOR_PHYSICAL_REMOVAL,
+            ), f"event {const.DRIVE_READY_FOR_PHYSICAL_REMOVAL} for drive {drive_name} not found"
+    # 14. Get Node ID on which drives reside, Obtain path for affected drives, Identify node name for corresponding node id and remove drives
+        for drive in drives:
+            drive_name = drive["metadata"]["name"]
+            drive_path = drive["spec"]["Path"]
+            assert drive_path, f"Drive path for drive {drive_name} not found"
+            logging.info(f"drive_path: {drive_path}")
+
+            host_num = self.drive_utils[node_ip].get_host_num(drive_path)
+            scsi_id = self.drive_utils[node_ip].get_scsi_id(drive_path)
+            assert scsi_id, f"scsi_id for drive {drive_name} not found"
+            logging.info(f"scsi_id: {scsi_id}")
+
+            self.drive_utils[node_ip].remove(scsi_id)
+            logging.info(f"drive {drive_path}, {scsi_id} removed")
+    # 16. Check for events DriveSuccessfullyRemoved in kubernetes events
+        for drive in drives:
+            drive_name = drive["metadata"]["name"]
+            assert self.utils.event_in(
+            resource_name=drive_name,
+            reason=const.DRIVE_SUCCESSFULLY_REMOVED,
+            ), f"event {const.DRIVE_SUCCESSFULLY_REMOVED} for drive {drive_name} not found" 
+    # 17. Check that DriveCR was removed. 
+    # 18. If there is another drive available with enough capacity then POD create in step 0 should be in Running state. If not the POD should be in Pending state.
+    # 19. Rescan bus for activating removed drives
+        for drive in drives:
+            drive_name = drive["metadata"]["name"]
+            self.drive_utils[node_ip].restore(host_num=host_num)
+            logging.info(
+                f"waiting for a drive {drive_name} to be {const.STATUS_ONLINE}"
+            )
+            self.utils.wait_drive(
+                name=drive_name, expected_status=const.STATUS_ONLINE
+            )
+
+