Merge #1516

1516: feat(injection): added support for bdev I/O fault injection r=dsavitskiy a=dsavitskiy Added support for injecting faults are SPDK Bdev I/O level. This allows to inject errors e.g. for replicas (lvols). Currently, only injecting NVMe status fault at submission stage is supported. InjectionBuilder is added to support constructing both injection struct instances and injection URIs to be used with io-engine API. Co-authored-by: Dmitry Savitskiy <[email protected]>
openebs · Oct 16, 2023 · caf7976 · caf7976
2 parents 821bbb1 + 5124b20
commit caf7976
Show file tree

Hide file tree

Showing 20 changed files with 1,086 additions and 463 deletions.
diff --git a/Cargo.lock b/Cargo.lock
diff --git a/io-engine-tests/src/pool.rs b/io-engine-tests/src/pool.rs
@@ -52,6 +52,18 @@ impl PoolBuilder {
         self.with_bdev(&bdev)
     }
 
+    pub fn with_malloc_blk_size(
+        self,
+        bdev_name: &str,
+        size_mb: u64,
+        blk_size: u64,
+    ) -> Self {
+        let bdev = format!(
+            "malloc:///{bdev_name}?size_mb={size_mb}&blk_size={blk_size}"
+        );
+        self.with_bdev(&bdev)
+    }
+
     pub fn rpc(&self) -> SharedRpcHandle {
         self.rpc.clone()
     }

diff --git a/io-engine/Cargo.toml b/io-engine/Cargo.toml
@@ -56,6 +56,7 @@ colored_json = "3.3.0"
 crc = "3.0.1"
 crossbeam = "0.8.2"
 crossbeam-sync = "0.0.0"
+derive_builder = "0.12.0"
 dns-lookup = "2.0.3"
 env_logger = "0.10.0"
 etcd-client = "0.12.1"

diff --git a/io-engine/src/bdev/device.rs b/io-engine/src/bdev/device.rs
@@ -297,6 +297,7 @@ impl BlockDeviceHandle for SpdkBlockDeviceHandle {
                 cb_arg,
                 #[cfg(feature = "fault-injection")]
                 inj_op: InjectIoCtx::with_iovs(
+                    FaultDomain::BlockDevice,
                     self.get_device(),
                     IoType::Read,
                     offset_blocks,
@@ -309,9 +310,7 @@ impl BlockDeviceHandle for SpdkBlockDeviceHandle {
         )?;
 
         #[cfg(feature = "fault-injection")]
-        inject_submission_error(FaultDomain::BlockDevice, unsafe {
-            &(*ctx).inj_op
-        })?;
+        inject_submission_error(unsafe { &(*ctx).inj_op })?;
 
         let (desc, chan) = self.handle.io_tuple();
         let rc = unsafe {
@@ -355,6 +354,7 @@ impl BlockDeviceHandle for SpdkBlockDeviceHandle {
                 cb_arg,
                 #[cfg(feature = "fault-injection")]
                 inj_op: InjectIoCtx::with_iovs(
+                    FaultDomain::BlockDevice,
                     self.get_device(),
                     IoType::Write,
                     offset_blocks,
@@ -367,9 +367,7 @@ impl BlockDeviceHandle for SpdkBlockDeviceHandle {
         )?;
 
         #[cfg(feature = "fault-injection")]
-        inject_submission_error(FaultDomain::BlockDevice, unsafe {
-            &(*ctx).inj_op
-        })?;
+        inject_submission_error(unsafe { &(*ctx).inj_op })?;
 
         let (desc, chan) = self.handle.io_tuple();
         let rc = unsafe {
@@ -411,7 +409,7 @@ impl BlockDeviceHandle for SpdkBlockDeviceHandle {
                 cb,
                 cb_arg,
                 #[cfg(feature = "fault-injection")]
-                inj_op: Default::default(),
+                inj_op: InjectIoCtx::new(FaultDomain::BlockDevice),
             },
             offset_blocks,
             num_blocks,
@@ -454,7 +452,7 @@ impl BlockDeviceHandle for SpdkBlockDeviceHandle {
                 cb,
                 cb_arg,
                 #[cfg(feature = "fault-injection")]
-                inj_op: InjectIoCtx::default(),
+                inj_op: InjectIoCtx::new(FaultDomain::BlockDevice),
             },
             0,
             0,
@@ -493,7 +491,7 @@ impl BlockDeviceHandle for SpdkBlockDeviceHandle {
                 cb,
                 cb_arg,
                 #[cfg(feature = "fault-injection")]
-                inj_op: InjectIoCtx::default(),
+                inj_op: InjectIoCtx::new(FaultDomain::BlockDevice),
             },
             offset_blocks,
             num_blocks,
@@ -536,7 +534,7 @@ impl BlockDeviceHandle for SpdkBlockDeviceHandle {
                 cb,
                 cb_arg,
                 #[cfg(feature = "fault-injection")]
-                inj_op: InjectIoCtx::default(),
+                inj_op: InjectIoCtx::new(FaultDomain::BlockDevice),
             },
             offset_blocks,
             num_blocks,
@@ -634,7 +632,7 @@ impl BlockDeviceHandle for SpdkBlockDeviceHandle {
                 cb,
                 cb_arg,
                 #[cfg(feature = "fault-injection")]
-                inj_op: InjectIoCtx::default(),
+                inj_op: InjectIoCtx::new(FaultDomain::BlockDevice),
             },
             0,
             0,
@@ -757,8 +755,7 @@ extern "C" fn bdev_io_completion(
     };
 
     #[cfg(feature = "fault-injection")]
-    let status =
-        inject_completion_error(FaultDomain::BlockDevice, &bio.inj_op, status);
+    let status = inject_completion_error(&bio.inj_op, status);
 
     (bio.cb)(&bio.device, status, bio.cb_arg);
 

diff --git a/io-engine/src/bdev/nexus/nexus_io.rs b/io-engine/src/bdev/nexus/nexus_io.rs
@@ -751,20 +751,18 @@ impl<'n> NexusBio<'n> {
     ) -> Result<(), CoreError> {
         use crate::core::fault_injection::{
             inject_submission_error,
-            FaultDomain::Nexus,
+            FaultDomain::NexusChild,
             InjectIoCtx,
         };
 
-        inject_submission_error(
-            Nexus,
-            &InjectIoCtx::with_iovs(
-                hdl.get_device(),
-                self.io_type(),
-                self.offset(),
-                self.num_blocks(),
-                self.iovs(),
-            ),
-        )
+        inject_submission_error(&InjectIoCtx::with_iovs(
+            NexusChild,
+            hdl.get_device(),
+            self.io_type(),
+            self.offset(),
+            self.num_blocks(),
+            self.iovs(),
+        ))
     }
 
     /// Checks if an error is to be injected upon completion.
@@ -777,13 +775,13 @@ impl<'n> NexusBio<'n> {
     ) -> IoCompletionStatus {
         use crate::core::fault_injection::{
             inject_completion_error,
-            FaultDomain::Nexus,
+            FaultDomain::NexusChild,
             InjectIoCtx,
         };
 
         inject_completion_error(
-            Nexus,
             &InjectIoCtx::with_iovs(
+                NexusChild,
                 child,
                 self.io_type(),
                 self.offset(),

diff --git a/io-engine/src/bdev/nvmx/handle.rs b/io-engine/src/bdev/nvmx/handle.rs
@@ -357,11 +357,7 @@ fn complete_nvme_command(ctx: *mut NvmeIoCtx, cpl: *const spdk_nvme_cpl) {
     };
 
     #[cfg(feature = "fault-injection")]
-    let status = inject_completion_error(
-        FaultDomain::BlockDevice,
-        &io_ctx.inj_op,
-        status,
-    );
+    let status = inject_completion_error(&io_ctx.inj_op, status);
 
     (io_ctx.cb)(&*inner.device, status, io_ctx.cb_arg);
 
@@ -771,6 +767,7 @@ impl BlockDeviceHandle for NvmeDeviceHandle {
                 num_blocks,
                 #[cfg(feature = "fault-injection")]
                 inj_op: InjectIoCtx::with_iovs(
+                    FaultDomain::BlockDevice,
                     self.get_device(),
                     IoType::Read,
                     offset_blocks,
@@ -783,9 +780,7 @@ impl BlockDeviceHandle for NvmeDeviceHandle {
         )?;
 
         #[cfg(feature = "fault-injection")]
-        inject_submission_error(FaultDomain::BlockDevice, unsafe {
-            &(*bio).inj_op
-        })?;
+        inject_submission_error(unsafe { &(*bio).inj_op })?;
 
         let rc = if iovs.len() == 1 {
             unsafe {
@@ -858,6 +853,7 @@ impl BlockDeviceHandle for NvmeDeviceHandle {
                 num_blocks,
                 #[cfg(feature = "fault-injection")]
                 inj_op: InjectIoCtx::with_iovs(
+                    FaultDomain::BlockDevice,
                     self.get_device(),
                     IoType::Write,
                     offset_blocks,
@@ -870,9 +866,7 @@ impl BlockDeviceHandle for NvmeDeviceHandle {
         )?;
 
         #[cfg(feature = "fault-injection")]
-        inject_submission_error(FaultDomain::BlockDevice, unsafe {
-            &(*bio).inj_op
-        })?;
+        inject_submission_error(unsafe { &(*bio).inj_op })?;
 
         let rc = if iovs.len() == 1 {
             unsafe {
@@ -949,7 +943,7 @@ impl BlockDeviceHandle for NvmeDeviceHandle {
                 op: IoType::Compare,
                 num_blocks,
                 #[cfg(feature = "fault-injection")]
-                inj_op: Default::default(),
+                inj_op: InjectIoCtx::new(FaultDomain::BlockDevice),
             },
             offset_blocks,
             num_blocks,
@@ -1047,7 +1041,7 @@ impl BlockDeviceHandle for NvmeDeviceHandle {
                 op: IoType::Flush,
                 num_blocks,
                 #[cfg(feature = "fault-injection")]
-                inj_op: Default::default(),
+                inj_op: InjectIoCtx::new(FaultDomain::BlockDevice),
             },
             0,
             num_blocks, // Flush all device blocks.
@@ -1110,7 +1104,7 @@ impl BlockDeviceHandle for NvmeDeviceHandle {
                 op: IoType::Unmap,
                 num_blocks,
                 #[cfg(feature = "fault-injection")]
-                inj_op: Default::default(),
+                inj_op: InjectIoCtx::new(FaultDomain::BlockDevice),
             },
             offset_blocks,
             num_blocks,
@@ -1209,7 +1203,7 @@ impl BlockDeviceHandle for NvmeDeviceHandle {
                 op: IoType::WriteZeros,
                 num_blocks,
                 #[cfg(feature = "fault-injection")]
-                inj_op: Default::default(),
+                inj_op: InjectIoCtx::new(FaultDomain::BlockDevice),
             },
             offset_blocks,
             num_blocks,