fix(nexus): fix

Signed-off-by: Dmitry Savitskiy <[email protected]>

io-engine/src/bdev/nexus/nexus_bdev_children.rs

@@ -329,7 +329,7 @@ impl<'n> Nexus<'n> {
                 // Detach the child from the I/O path, and close its handles.
                 if let Some(device) = child.get_device_name() {
                     self.detach_device(&device).await;
-                    self.disconnect_detached_devices().await;
+                    self.disconnect_all_detached_devices().await;
                 }
 
                 // Close child's device.
@@ -985,6 +985,13 @@ impl<'n> Nexus<'n> {
         // in order to prevent an I/O race when retiring a device.
         self.detach_device(&dev).await;
 
+        info!("d'conn ctrlr-failed...");
+        self.traverse_io_channels_async((), |channel, _| {
+            channel.disconnect_detached_devices(|h| h.is_ctrlr_failed());
+        })
+        .await;
+        info!("d'conn ctrlr-failed ok");
+
         // Disconnect, destroy and close the device. The subsystem must be
         // paused to do this properly.
         {
@@ -994,7 +1001,9 @@ impl<'n> Nexus<'n> {
 
             // Disconnect the previously detached device handles. This has to be
             // done after the pause to prevent an I/O race.
-            self.disconnect_detached_devices().await;
+            info!("d'conn all...");
+            self.disconnect_all_detached_devices().await;
+            info!("d'conn all ok");
 
             res?;
 
@@ -1090,11 +1099,11 @@ impl<'n> Nexus<'n> {
 
     /// Disconnects all the detached devices on all I/O channels by dropping
     /// their handles.
-    pub(crate) async fn disconnect_detached_devices(&self) {
+    pub(crate) async fn disconnect_all_detached_devices(&self) {
         debug!("{self:?}: disconnecting all detached devices ...");
 
         self.traverse_io_channels_async((), |channel, _| {
-            channel.disconnect_detached_devices();
+            channel.disconnect_detached_devices(|_| true);
         })
         .await;
 
@@ -1174,7 +1183,7 @@ impl<'n> Nexus<'n> {
                 // Step 1: Close I/O channels for all children.
                 for dev in nexus.child_devices() {
                     nexus.detach_device(&dev).await;
-                    nexus.disconnect_detached_devices().await;
+                    nexus.disconnect_all_detached_devices().await;
 
                     device_cmd_queue().enqueue(DeviceCommand::RetireDevice {
                         nexus_name: nexus.name.clone(),

io-engine/src/bdev/nexus/nexus_channel.rs

@@ -240,14 +240,37 @@ impl<'n> NexusChannel<'n> {
         debug!("{self:?}: device '{device_name}' detached");
     }
 
-    /// Disconnects all previously detached device handles by dropping them.
-    pub(super) fn disconnect_detached_devices(&mut self) {
-        debug!(
-            "{self:?}: disconnecting {n} detached device handles...",
-            n = self.detached.len()
+    /// Disconnects previously detached device handles by dropping them.
+    /// Devices to drop are filtered by the given predicate: true to drop
+    /// a device, false to keep it.
+    pub(super) fn disconnect_detached_devices<F>(&mut self, mut drop_pred: F)
+    where
+        F: FnMut(&dyn BlockDeviceHandle) -> bool,
+    {
+        let cnt = self
+            .detached
+            .iter()
+            .filter(|h| drop_pred(h.as_ref()))
+            .count();
+
+        if cnt == 0 {
+            debug!("{self:?}: no devices to disconnect");
+            return;
+        }
+
+        let n = self.detached.len();
+
+        info!(
+            "{self:?}: disconnecting {cnt} of {n} detached device handles...",
+        );
+
+        self.detached.retain(|h| !drop_pred(h.as_ref()));
+
+        info!(
+            "{self:?}: {cnt} of {n} detached device handles disconnected, \
+            {m} remain(s)",
+            m = self.detached.len()
         );
-        self.detached.clear();
-        debug!("{self:?}: all detached device handles disconnected");
     }
 
     /// Refreshing our channels simply means that we either have a child going

io-engine/src/bdev/nexus/nexus_io.rs

@@ -643,7 +643,7 @@ impl<'n> NexusBio<'n> {
             self.ctx_mut().failed += 1;
 
             self.channel_mut().detach_device(&device);
-            self.channel_mut().disconnect_detached_devices();
+            self.channel_mut().disconnect_detached_devices(|_| true);
 
             if let Some(log) = self.fault_device(
                 &device,

io-engine/src/bdev/nvmx/controller_inner.rs

@@ -300,6 +300,8 @@ impl SpdkNvmeController {
         }
     }
 
+    /// Returns a pointer to the underlying SPDK struct.
+    #[inline(always)]
     pub fn as_ptr(&self) -> *mut spdk_nvme_ctrlr {
         self.0.as_ptr()
     }

io-engine/src/bdev/nvmx/handle.rs

@@ -1523,6 +1523,11 @@ impl BlockDeviceHandle for NvmeDeviceHandle {
         let id = inner.ext_host_id();
         Ok(*id)
     }
+
+    /// TODO
+    fn is_ctrlr_failed(&self) -> bool {
+        self.ctrlr.is_failed
+    }
 }
 
 impl Drop for NvmeDeviceHandle {

io-engine/src/core/block_device.rs

@@ -476,6 +476,11 @@ pub trait BlockDeviceHandle {
         cb: IoCompletionCallback,
         cb_arg: IoCompletionCallbackArg,
     ) -> Result<(), CoreError>;
+
+    /// TODO
+    fn is_ctrlr_failed(&self) -> bool {
+        false
+    }
 }
 
 fn block_device_io_completion(