diff --git a/ci.nix b/ci.nix index d60fc0a637..295c03c973 100644 --- a/ci.nix +++ b/ci.nix @@ -1,4 +1,4 @@ -{ nospdk ? false, norust ? false, spdk_rel ? false }: +{ nospdk ? false, norust ? false, spdk_rel ? false, asan ? false }: let sources = import ./nix/sources.nix; pkgs = import sources.nixpkgs { @@ -61,7 +61,8 @@ mkShell { xfsprogs yasm ] ++ (if (nospdk) then [ spdk.buildInputs ] else [ spdk ]) - ++ pkgs.lib.optional (!norust) channel.stable + ++ pkgs.lib.optional (!norust && asan) channel.asan + ++ pkgs.lib.optional (!norust && !asan) channel.stable ++ pkgs.lib.optional (!norust) channel.nightly; RUST_NIGHTLY_PATH = channel.nightly; @@ -72,12 +73,38 @@ mkShell { FIO_SPDK = if nospdk then null else "${spdk}/fio/spdk_nvme"; ETCD_BIN = "${etcd}/bin/etcd"; + IO_ENGINE_DIR = if asan then "target/x86_64-unknown-linux-gnu/debug" else "target/debug"; + + # ASAN-related Cargo settings. + ASAN_ENABLE = if asan then "1" else null; + ASAN_OPTIONS = if asan then "detect_leaks=0" else null; + ASAN_BUILD_ENV = if asan then "shell" else null; + RUSTFLAGS = if asan then "-Zsanitizer=address" else null; + CARGO_BUILD_RUSTFLAGS = if asan then "-Zbuild-std" else null; + CARGO_BUILD_TARGET = if asan then "x86_64-unknown-linux-gnu" else null; + CARGO_PROFILE_DEV_PANIC = if asan then "unwind" else null; + RUST_BACKTRACE = if asan then "full" else null; + shellHook = '' + ${pkgs.lib.optionalString (asan) "export LLVM_SYMBOLIZER_DIR=$(dirname $(realpath $(which llvm-symbolizer)))"} + + ${pkgs.lib.optionalString (asan) "echo 'AddressSanitizer is enabled, forcing nightly rustc.'"} + ${pkgs.lib.optionalString (asan) "echo ' ASAN_ENABLE :' $\{ASAN_ENABLE\}"} + ${pkgs.lib.optionalString (asan) "echo ' ASAN_OPTIONS :' $\{ASAN_OPTIONS\}"} + ${pkgs.lib.optionalString (asan) "echo ' RUSTFLAGS :' $\{RUSTFLAGS\}"} + ${pkgs.lib.optionalString (asan) "echo ' CARGO_BUILD_RUSTFLAGS :' $\{CARGO_BUILD_RUSTFLAGS\}"} + ${pkgs.lib.optionalString (asan) "echo ' CARGO_BUILD_TARGET :' $\{CARGO_BUILD_TARGET\}"} + ${pkgs.lib.optionalString (asan) "echo ' CARGO_PROFILE_DEV_PANIC :' $\{CARGO_PROFILE_DEV_PANIC\}"} + ${pkgs.lib.optionalString (asan) "echo ' RUST_BACKTRACE :' $\{RUST_BACKTRACE\}"} + ${pkgs.lib.optionalString (asan) "echo ' LLVM_SYMBOLIZER_DIR :' $\{LLVM_SYMBOLIZER_DIR\}"} + ${pkgs.lib.optionalString (asan) "echo"} + ${pkgs.lib.optionalString (!nospdk) "echo 'SPDK version :' $(echo $SPDK_PATH | sed 's/.*libspdk-//g')"} ${pkgs.lib.optionalString (!nospdk) "echo 'SPDK path :' $SPDK_PATH"} ${pkgs.lib.optionalString (!nospdk) "echo 'SPDK FIO plugin :' $FIO_SPDK"} ${pkgs.lib.optionalString (!norust) "echo 'Rust version :' $(rustc --version 2> /dev/null || echo '${norustc_msg}')"} ${pkgs.lib.optionalString (!norust) "echo 'Rust path :' $(which rustc 2> /dev/null || echo '${norustc_msg}')"} + echo 'I/O engine dir :' $IO_ENGINE_DIR ${pkgs.lib.optionalString (nospdk) "cowsay ${nospdk_moth}"} ${pkgs.lib.optionalString (nospdk) "export CFLAGS=-msse4"} ${pkgs.lib.optionalString (nospdk) "echo"} diff --git a/io-engine/src/bdev/nexus/nexus_bdev.rs b/io-engine/src/bdev/nexus/nexus_bdev.rs index 1b295f4b2f..8e49f7a5b8 100644 --- a/io-engine/src/bdev/nexus/nexus_bdev.rs +++ b/io-engine/src/bdev/nexus/nexus_bdev.rs @@ -851,10 +851,11 @@ impl<'n> Nexus<'n> { let name = self.name.clone(); // After calling unregister_bdev_async(), Nexus is gone. + let evt = self.event(EventAction::Delete); match self.as_mut().bdev_mut().unregister_bdev_async().await { Ok(_) => { info!("Nexus '{name}': nexus destroyed ok"); - self.event(EventAction::Delete).generate(); + evt.generate(); Ok(()) } Err(err) => { @@ -1142,6 +1143,20 @@ impl<'n> Nexus<'n> { unsafe { Pin::new_unchecked(self.bdev_mut()) } } + /// Gets a nexus reference from an untyped bdev. + /// # Warning: + /// No checks are performed (e.g. bdev module name check), as it is assumed + /// that the provided bdev is a nexus bdev. + #[inline(always)] + pub(crate) unsafe fn unsafe_from_untyped_bdev( + bdev: spdk_rs::UntypedBdev, + ) -> &'n Nexus<'n> { + spdk_rs::Bdev::>::unsafe_from_inner_ptr( + bdev.unsafe_inner_ptr() as *mut _, + ) + .data() + } + /// Sets the required alignment of the Nexus. pub(crate) unsafe fn set_required_alignment( self: Pin<&mut Self>, diff --git a/io-engine/src/bdev/nexus/nexus_bdev_children.rs b/io-engine/src/bdev/nexus/nexus_bdev_children.rs index 889cada33c..2e5eca3334 100644 --- a/io-engine/src/bdev/nexus/nexus_bdev_children.rs +++ b/io-engine/src/bdev/nexus/nexus_bdev_children.rs @@ -326,9 +326,10 @@ impl<'n> Nexus<'n> { // Close and remove the child. let res = match self.lookup_child(uri) { Some(child) => { - // Remove child from the I/O path. + // Detach the child from the I/O path, and close its handles. if let Some(device) = child.get_device_name() { - self.disconnect_device(&device).await; + self.detach_device(&device).await; + self.disconnect_all_detached_devices().await; } // Close child's device. @@ -974,18 +975,39 @@ impl<'n> Nexus<'n> { return Ok(()); } - // Disconnect the device from all the channels. + // Detach the device from all the channels. + // // After disconnecting, the device will no longer be used by the // channels, and all I/Os failing due to this device will eventually // resubmit and succeeded (if any healthy children are left). - self.disconnect_device(&dev).await; + // + // Device disconnection is done in two steps (detach, than disconnect) + // in order to prevent an I/O race when retiring a device. + self.detach_device(&dev).await; + + // Disconnect the devices with failed controllers _before_ pause, + // otherwise pause would stuck. Keep all controoled that are _not_ + // failed (e.g., in the case I/O failed due to ENOSPC). + self.traverse_io_channels_async((), |channel, _| { + channel.disconnect_detached_devices(|h| h.is_ctrlr_failed()); + }) + .await; - // Destroy (close) the device. The subsystem must be paused to do this - // properly. + // Disconnect, destroy and close the device. The subsystem must be + // paused to do this properly. { debug!("{self:?}: retire: pausing..."); - self.as_mut().pause().await?; - debug!("{self:?}: retire: pausing ok"); + let res = self.as_mut().pause().await; + match &res { + Ok(_) => debug!("{self:?}: retire: pausing ok"), + Err(e) => warn!("{self:?}: retire: pausing failed: {e}"), + }; + + // Disconnect the all previously detached device handles. This has + // to be done after the pause to prevent an I/O race. + self.disconnect_all_detached_devices().await; + + res?; self.child_retire_destroy_device(&dev).await; @@ -1055,20 +1077,39 @@ impl<'n> Nexus<'n> { Ok(()) } - /// Disconnects a device from all I/O channels. - pub(crate) async fn disconnect_device(&self, dev: &str) { + /// Detaches the device's handles from all I/O channels. + /// + /// The detached handles must be disconnected and dropped by a + /// `disconnect_detached_devices()` call. + /// + /// Device disconnection is done in two steps (detach, than disconnect) in + /// order to prevent an I/O race when retiring a device. + pub(crate) async fn detach_device(&self, dev: &str) { if !self.has_io_device { return; } - debug!("{self:?}: disconnecting '{dev}' from all channels ..."); + debug!("{self:?}: detaching '{dev}' from all channels..."); self.traverse_io_channels_async(dev, |channel, dev| { - channel.disconnect_device(dev); + channel.detach_device(dev); + }) + .await; + + debug!("{self:?}: '{dev}' detached from all I/O channels"); + } + + /// Disconnects all the detached devices on all I/O channels by dropping + /// their handles. + pub(crate) async fn disconnect_all_detached_devices(&self) { + debug!("{self:?}: disconnecting all detached devices ..."); + + self.traverse_io_channels_async((), |channel, _| { + channel.disconnect_detached_devices(|_| true); }) .await; - debug!("{self:?}: '{dev}' disconnected from all I/O channels"); + debug!("{self:?}: disconnected all detached devices"); } /// Destroys the device being retired. @@ -1143,7 +1184,8 @@ impl<'n> Nexus<'n> { // Step 1: Close I/O channels for all children. for dev in nexus.child_devices() { - nexus.disconnect_device(&dev).await; + nexus.detach_device(&dev).await; + nexus.disconnect_all_detached_devices().await; device_cmd_queue().enqueue(DeviceCommand::RetireDevice { nexus_name: nexus.name.clone(), diff --git a/io-engine/src/bdev/nexus/nexus_channel.rs b/io-engine/src/bdev/nexus/nexus_channel.rs index 92fb3fef4c..24f694c8ea 100644 --- a/io-engine/src/bdev/nexus/nexus_channel.rs +++ b/io-engine/src/bdev/nexus/nexus_channel.rs @@ -17,6 +17,7 @@ use spdk_rs::Thread; pub struct NexusChannel<'n> { writers: Vec>, readers: Vec>, + detached: Vec>, io_logs: Vec, previous_reader: UnsafeCell, fail_fast: u32, @@ -123,6 +124,7 @@ impl<'n> NexusChannel<'n> { Self { writers, readers, + detached: Vec::new(), io_logs: nexus.io_log_channels(), previous_reader: UnsafeCell::new(0), nexus: unsafe { nexus.pinned_mut() }, @@ -209,16 +211,66 @@ impl<'n> NexusChannel<'n> { } } - /// Disconnects a child device from the I/O path. - pub fn disconnect_device(&mut self, device_name: &str) { + /// Detaches a child device from this I/O channel, moving the device's + /// handles to the list of detached devices to disconnect later. + /// + /// The detached handles must be disconnected and dropped by a + /// `disconnect_detached_devices()` call. + pub(super) fn detach_device(&mut self, device_name: &str) { self.previous_reader = UnsafeCell::new(0); - self.readers - .retain(|c| c.get_device().device_name() != device_name); - self.writers - .retain(|c| c.get_device().device_name() != device_name); + if let Some(d) = self + .readers + .iter() + .position(|c| c.get_device().device_name() == device_name) + { + let t = self.readers.remove(d); + self.detached.push(t); + } + + if let Some(d) = self + .writers + .iter() + .position(|c| c.get_device().device_name() == device_name) + { + let t = self.writers.remove(d); + self.detached.push(t); + } + + debug!("{self:?}: device '{device_name}' detached"); + } + + /// Disconnects previously detached device handles by dropping them. + /// Devices to drop are filtered by the given predicate: true to drop + /// a device, false to keep it. + pub(super) fn disconnect_detached_devices(&mut self, mut drop_pred: F) + where + F: FnMut(&dyn BlockDeviceHandle) -> bool, + { + let cnt = self + .detached + .iter() + .filter(|h| drop_pred(h.as_ref())) + .count(); + + if cnt == 0 { + debug!("{self:?}: no devices to disconnect"); + return; + } - debug!("{self:?}: device '{device_name}' disconnected"); + let n = self.detached.len(); + + info!( + "{self:?}: disconnecting {cnt} of {n} detached device handles...", + ); + + self.detached.retain(|h| !drop_pred(h.as_ref())); + + info!( + "{self:?}: {cnt} of {n} detached device handles disconnected, \ + {m} remain(s)", + m = self.detached.len() + ); } /// Refreshing our channels simply means that we either have a child going diff --git a/io-engine/src/bdev/nexus/nexus_io.rs b/io-engine/src/bdev/nexus/nexus_io.rs index 8caaa9c9a7..317f18200c 100644 --- a/io-engine/src/bdev/nexus/nexus_io.rs +++ b/io-engine/src/bdev/nexus/nexus_io.rs @@ -642,7 +642,8 @@ impl<'n> NexusBio<'n> { // set the IO as failed in the submission stage. self.ctx_mut().failed += 1; - self.channel_mut().disconnect_device(&device); + self.channel_mut().detach_device(&device); + self.channel_mut().disconnect_detached_devices(|_| true); if let Some(log) = self.fault_device( &device, diff --git a/io-engine/src/bdev/nvmx/controller_inner.rs b/io-engine/src/bdev/nvmx/controller_inner.rs index 1b40c6b58a..35d3b8ec92 100644 --- a/io-engine/src/bdev/nvmx/controller_inner.rs +++ b/io-engine/src/bdev/nvmx/controller_inner.rs @@ -300,6 +300,8 @@ impl SpdkNvmeController { } } + /// Returns a pointer to the underlying SPDK struct. + #[inline(always)] pub fn as_ptr(&self) -> *mut spdk_nvme_ctrlr { self.0.as_ptr() } diff --git a/io-engine/src/bdev/nvmx/handle.rs b/io-engine/src/bdev/nvmx/handle.rs index 073196a7ee..879247b5ed 100644 --- a/io-engine/src/bdev/nvmx/handle.rs +++ b/io-engine/src/bdev/nvmx/handle.rs @@ -1523,6 +1523,11 @@ impl BlockDeviceHandle for NvmeDeviceHandle { let id = inner.ext_host_id(); Ok(*id) } + + /// Determines if the underlying controller is failed. + fn is_ctrlr_failed(&self) -> bool { + self.ctrlr.is_failed + } } impl Drop for NvmeDeviceHandle { diff --git a/io-engine/src/core/block_device.rs b/io-engine/src/core/block_device.rs index d0a4e24d09..5a0a0703fc 100644 --- a/io-engine/src/core/block_device.rs +++ b/io-engine/src/core/block_device.rs @@ -476,6 +476,11 @@ pub trait BlockDeviceHandle { cb: IoCompletionCallback, cb_arg: IoCompletionCallbackArg, ) -> Result<(), CoreError>; + + /// Determines if the underlying controller is failed. + fn is_ctrlr_failed(&self) -> bool { + false + } } fn block_device_io_completion( diff --git a/io-engine/src/core/env.rs b/io-engine/src/core/env.rs index 0dc1207f00..704d7d7382 100644 --- a/io-engine/src/core/env.rs +++ b/io-engine/src/core/env.rs @@ -906,6 +906,10 @@ impl MayastorEnvironment { // setup the logger as soon as possible self.init_logger(); + if option_env!("ASAN_ENABLE").unwrap_or_default() == "1" { + print_asan_env(); + } + self.load_yaml_config(); if let Some(ptpl) = &self.ptpl_dir { @@ -1064,3 +1068,25 @@ fn make_hostnqn(node_name: Option<&String>) -> Option { node_name.map(|n| format!("{NVME_NQN_PREFIX}:node-name:{n}")) }) } + +fn print_asan_env() { + fn print_var(s: &str, v: Option<&str>) { + let v = v.unwrap_or_default(); + info!(" {s:25} = {v}"); + } + + warn!("Compiled with Address Sanitizer enabled"); + print_var("ASAN_OPTIONS", option_env!("ASAN_OPTIONS")); + print_var("ASAN_BUILD_ENV", option_env!("ASAN_BUILD_ENV")); + print_var("RUSTFLAGS", option_env!("RUSTFLAGS")); + print_var( + "CARGO_BUILD_RUSTFLAGS", + option_env!("CARGO_BUILD_RUSTFLAGS"), + ); + print_var("CARGO_BUILD_TARGET", option_env!("CARGO_BUILD_TARGET")); + print_var( + "CARGO_PROFILE_DEV_PANIC", + option_env!("CARGO_PROFILE_DEV_PANIC"), + ); + print_var("RUST_BACKTRACE", option_env!("RUST_BACKTRACE")); +} diff --git a/io-engine/src/core/mod.rs b/io-engine/src/core/mod.rs index 92994d3882..962623e3ed 100644 --- a/io-engine/src/core/mod.rs +++ b/io-engine/src/core/mod.rs @@ -458,6 +458,7 @@ impl ToErrno for CoreError { /// Logical volume layer failure. #[derive(Debug, Copy, Clone, Eq, PartialOrd, PartialEq)] +#[repr(C)] pub enum LvolFailure { NoSpace, } @@ -472,6 +473,7 @@ pub enum IoSubmissionFailure { // Generic I/O completion status for block devices, which supports per-protocol // error domains. #[derive(Copy, Clone, Eq, PartialOrd, PartialEq)] +#[repr(C)] pub enum IoCompletionStatus { Success, NvmeError(NvmeStatus), @@ -501,10 +503,9 @@ impl From for IoCompletionStatus { match s { NvmeStatus::NO_SPACE | NvmeStatus::Generic(SPDK_NVME_SC_CAPACITY_EXCEEDED) => { - IoCompletionStatus::LvolError(LvolFailure::NoSpace) + Self::LvolError(LvolFailure::NoSpace) } - - _ => IoCompletionStatus::NvmeError(s), + _ => Self::NvmeError(s), } } } diff --git a/io-engine/src/lvs/lvs_store.rs b/io-engine/src/lvs/lvs_store.rs index 9cc8660b71..fa90037674 100644 --- a/io-engine/src/lvs/lvs_store.rs +++ b/io-engine/src/lvs/lvs_store.rs @@ -683,6 +683,8 @@ impl Lvs { let base_bdev = self.base_bdev(); + let evt = self.event(EventAction::Delete); + unsafe { vbdev_lvs_destruct( self.as_inner_ptr(), @@ -700,7 +702,7 @@ impl Lvs { info!("{}: lvs destroyed successfully", self_str); - self.event(EventAction::Delete).generate(); + evt.generate(); bdev_destroy(&base_bdev.bdev_uri_original_str().unwrap()) .await diff --git a/io-engine/src/subsys/nvmf/subsystem.rs b/io-engine/src/subsys/nvmf/subsystem.rs index 80d68f2c36..e716dfd4c9 100644 --- a/io-engine/src/subsys/nvmf/subsystem.rs +++ b/io-engine/src/subsys/nvmf/subsystem.rs @@ -1,4 +1,5 @@ use std::{ + convert::TryFrom, ffi::{c_void, CString}, fmt::{self, Debug, Display, Formatter}, mem::size_of, @@ -60,11 +61,12 @@ use spdk_rs::{ }; use crate::{ - bdev::{nexus::nexus_lookup_nqn, nvmx::NVME_CONTROLLERS}, + bdev::{nexus::NEXUS_MODULE_NAME, nvmx::NVME_CONTROLLERS, Nexus}, constants::{NVME_CONTROLLER_MODEL_ID, NVME_NQN_PREFIX}, core::{Bdev, Reactors, UntypedBdev}, eventing::{EventMetaGen, EventWithMeta}, ffihelper::{cb_arg, done_cb, AsStr, FfiResult, IntoCString}, + lvs::Lvol, subsys::{ make_subsystem_serial, nvmf::{transport::TransportId, Error, NVMF_TGT}, @@ -236,70 +238,54 @@ impl NvmfSubsystem { /// Subsystem event handlers. extern "C" fn nvmf_subsystem_event_handler( - subsystem: *mut spdk_nvmf_subsystem, + subsys: *mut spdk_nvmf_subsystem, event: spdk_nvmf_subsystem_event, ctx: *mut c_void, _cb_arg: *mut c_void, ) { - let subsystem = NvmfSubsystem::from(subsystem); - let nqn = subsystem.get_nqn(); - let nexus = nexus_lookup_nqn(&nqn); + let s = NvmfSubsystem::from(subsys); let event = NvmfSubsystemEvent::from_cb_args(event, ctx); - debug!("NVMF subsystem event {subsystem:?}: {event:?}"); + debug!("NVMF subsystem event {s:?}: {event:?}"); - match event { - NvmfSubsystemEvent::HostConnect(ctrlr) => { - info!( - "Subsystem '{nqn}': host connected: '{host}'", - host = ctrlr.hostnqn() - ); + let nqn_tgt = NqnTarget::lookup(&s.get_nqn()); + if matches!(nqn_tgt, NqnTarget::None) { + warn!( + "NVMF subsystem event {s:?}: {event:?}: \ + target for event NQN not found" + ); + } - ctrlr - .event(EventAction::NvmeConnect, subsystem.meta()) - .generate(); + match event { + NvmfSubsystemEvent::HostConnect(c) => { + c.event(EventAction::NvmeConnect, s.meta()).generate(); - if let Some(nex) = nexus { - nex.add_initiator(&ctrlr.hostnqn()); - subsystem.host_connect_nexus(ctrlr); - } else { - subsystem.host_connect_replica(ctrlr); + match nqn_tgt { + NqnTarget::Nexus(n) => s.host_connect_nexus(c, n), + NqnTarget::Replica(r) => s.host_connect_replica(c, r), + NqnTarget::None => {} } } - NvmfSubsystemEvent::HostDisconnect(ctrlr) => { - info!( - "Subsystem '{nqn}': host disconnected: '{host}'", - host = ctrlr.hostnqn() - ); + NvmfSubsystemEvent::HostDisconnect(c) => { + c.event(EventAction::NvmeDisconnect, s.meta()).generate(); - ctrlr - .event(EventAction::NvmeDisconnect, subsystem.meta()) - .generate(); - - if let Some(nex) = nexus { - nex.rm_initiator(&ctrlr.hostnqn()); - subsystem.host_disconnect_nexus(ctrlr); - } else { - subsystem.host_disconnect_replica(ctrlr); + match nqn_tgt { + NqnTarget::Nexus(n) => s.host_disconnect_nexus(c, n), + NqnTarget::Replica(r) => s.host_disconnect_replica(c, r), + NqnTarget::None => {} } } - NvmfSubsystemEvent::HostKeepAliveTimeout(ctrlr) => { - warn!( - "Subsystem '{nqn}': host keep alive timeout: '{host}'", - host = ctrlr.hostnqn() - ); - - ctrlr - .event(EventAction::NvmeKeepAliveTimeout, subsystem.meta()) + NvmfSubsystemEvent::HostKeepAliveTimeout(c) => { + c.event(EventAction::NvmeKeepAliveTimeout, s.meta()) .generate(); - if let Some(nex) = nexus { - nex.initiator_keep_alive_timeout(&ctrlr.hostnqn()); + match nqn_tgt { + NqnTarget::Nexus(n) => s.host_kato_nexus(c, n), + NqnTarget::Replica(r) => s.host_kato_replica(c, r), + NqnTarget::None => {} } } - NvmfSubsystemEvent::Unknown => { - // ignore unknown events - } + NvmfSubsystemEvent::Unknown => {} // ignore unknown events } } @@ -329,7 +315,16 @@ impl NvmfSubsystem { } /// Called upon a host connection to a nexus. - fn host_connect_nexus(&self, ctrlr: NvmfController) { + fn host_connect_nexus(&self, ctrlr: NvmfController, nex: &Nexus) { + info!( + "Host '{host}' connected to subsystem '{subsys}' on \ + nexus '{nex:?}'", + host = ctrlr.hostnqn(), + subsys = self.get_nqn(), + ); + + nex.add_initiator(&ctrlr.hostnqn()); + unsafe { spdk_nvmf_ctrlr_set_cpl_error_cb( ctrlr.0.as_ptr(), @@ -340,7 +335,16 @@ impl NvmfSubsystem { } /// Called upon a host disconnection from a nexus. - fn host_disconnect_nexus(&self, ctrlr: NvmfController) { + fn host_disconnect_nexus(&self, ctrlr: NvmfController, nex: &Nexus) { + info!( + "Host '{host}' disconnected from subsystem '{subsys}' on \ + nexus '{nex:?}'", + host = ctrlr.hostnqn(), + subsys = self.get_nqn(), + ); + + nex.rm_initiator(&ctrlr.hostnqn()); + unsafe { spdk_nvmf_ctrlr_set_cpl_error_cb( ctrlr.0.as_ptr(), @@ -350,6 +354,18 @@ impl NvmfSubsystem { } } + /// Called upon a host keep alive timeout (KATO) on a nexus. + fn host_kato_nexus(&self, ctrlr: NvmfController, nex: &Nexus) { + warn!( + "Host '{host}': keep alive timeout on subsystem '{subsys}' on \ + nexus '{nex:?}'", + host = ctrlr.hostnqn(), + subsys = self.get_nqn(), + ); + + nex.initiator_keep_alive_timeout(&ctrlr.hostnqn()); + } + /// Completion error callback for replicas. unsafe extern "C" fn replica_cpl_error_cb( req: *mut spdk_nvmf_request, @@ -375,7 +391,14 @@ impl NvmfSubsystem { } /// Called upon a host connection to a replica. - fn host_connect_replica(&self, ctrlr: NvmfController) { + fn host_connect_replica(&self, ctrlr: NvmfController, lvol: Lvol) { + info!( + "Host '{host}' connected to subsystem '{subsys}' on \ + replica '{lvol:?}'", + host = ctrlr.hostnqn(), + subsys = self.get_nqn(), + ); + unsafe { spdk_nvmf_ctrlr_set_cpl_error_cb( ctrlr.0.as_ptr(), @@ -386,7 +409,14 @@ impl NvmfSubsystem { } /// Called upon a host disconnection from a replica. - fn host_disconnect_replica(&self, ctrlr: NvmfController) { + fn host_disconnect_replica(&self, ctrlr: NvmfController, lvol: Lvol) { + info!( + "Host '{host}' disconnected from subsystem '{subsys}' on \ + replica '{lvol:?}'", + host = ctrlr.hostnqn(), + subsys = self.get_nqn(), + ); + unsafe { spdk_nvmf_ctrlr_set_cpl_error_cb( ctrlr.0.as_ptr(), @@ -396,6 +426,16 @@ impl NvmfSubsystem { } } + /// Called upon a host keep alive timeout (KATO) on a replica. + fn host_kato_replica(&self, ctrlr: NvmfController, lvol: Lvol) { + warn!( + "Host '{host}': keep alive timeout on subsystem '{subsys}' on \ + replica '{lvol:?}'", + host = ctrlr.hostnqn(), + subsys = self.get_nqn(), + ); + } + /// create a new subsystem where the NQN is based on the UUID pub fn new(uuid: &str) -> Result { let nqn = make_nqn(uuid).into_cstring(); @@ -1076,3 +1116,41 @@ impl NvmfSubsystem { fn make_nqn(id: &str) -> String { format!("{NVME_NQN_PREFIX}:{id}") } + +/// NQN target. +pub enum NqnTarget<'a> { + Nexus(&'a Nexus<'a>), + Replica(Lvol), + None, +} + +impl<'a> NqnTarget<'a> { + pub fn lookup(nqn: &str) -> Self { + let Some(bdev) = UntypedBdev::bdev_first() else { + return Self::None; + }; + + let parts: Vec<&str> = nqn.split(':').collect(); + if parts.len() != 2 || parts[0] != NVME_NQN_PREFIX { + return Self::None; + } + + let name = parts[1]; + + for b in bdev.into_iter() { + match b.driver() { + NEXUS_MODULE_NAME if b.name() == name => { + return Self::Nexus(unsafe { + Nexus::unsafe_from_untyped_bdev(*b) + }); + } + "lvol" if b.name() == name => { + return Lvol::try_from(b).map_or(Self::None, Self::Replica) + } + _ => {} + } + } + + Self::None + } +} diff --git a/io-engine/src/subsys/nvmf/target.rs b/io-engine/src/subsys/nvmf/target.rs index e97b607e81..56c23f281a 100644 --- a/io-engine/src/subsys/nvmf/target.rs +++ b/io-engine/src/subsys/nvmf/target.rs @@ -373,17 +373,33 @@ impl Target { } } - let cfg = Config::get(); - let trid_nexus = TransportId::new(cfg.nexus_opts.nvmf_nexus_port); - let trid_replica = TransportId::new(cfg.nexus_opts.nvmf_replica_port); + // TODO: properly fix use-after-free on spdk_nvmf_tgt_stop_listen call. + if option_env!("ASAN_ENABLE").unwrap_or_default() == "1" { + warn!( + "Compiled with Address Sanitizer enabled: \ + won't stop listening targets as it causes \ + use-after-free error" + ); + } else { + let cfg = Config::get(); + let trid_nexus = TransportId::new(cfg.nexus_opts.nvmf_nexus_port); + let trid_replica = + TransportId::new(cfg.nexus_opts.nvmf_replica_port); - unsafe { - spdk_nvmf_tgt_stop_listen(self.tgt.as_ptr(), trid_replica.as_ptr()) - }; + unsafe { + spdk_nvmf_tgt_stop_listen( + self.tgt.as_ptr(), + trid_replica.as_ptr(), + ) + }; - unsafe { - spdk_nvmf_tgt_stop_listen(self.tgt.as_ptr(), trid_nexus.as_ptr()) - }; + unsafe { + spdk_nvmf_tgt_stop_listen( + self.tgt.as_ptr(), + trid_nexus.as_ptr(), + ) + }; + } unsafe { spdk_nvmf_tgt_destroy( diff --git a/nix/lib/rust.nix b/nix/lib/rust.nix index 969144c3ee..4ad077933e 100644 --- a/nix/lib/rust.nix +++ b/nix/lib/rust.nix @@ -2,8 +2,11 @@ let pkgs = import sources.nixpkgs { overlays = [ (import sources.rust-overlay) ]; }; + nightly_version = "2023-08-25"; + stable_version = "1.72.0"; in with pkgs; rec { - nightly = rust-bin.nightly."2023-08-25".default; - stable = rust-bin.stable."1.72.0".default; + nightly = rust-bin.nightly.${nightly_version}.default; + stable = rust-bin.stable.${stable_version}.default; + asan = rust-bin.nightly.${nightly_version}.default; } diff --git a/shell.nix b/shell.nix index 5328a22d19..8ed0b4a2b6 100644 --- a/shell.nix +++ b/shell.nix @@ -62,12 +62,15 @@ mkShell { ETCD_BIN = "${etcd}/bin/etcd"; ETCDCTL_API = "3"; + IO_ENGINE_DIR = "target/debug"; + shellHook = '' ${pkgs.lib.optionalString (!nospdk) "echo 'SPDK version :' $(echo $SPDK_PATH | sed 's/.*libspdk-//g')"} ${pkgs.lib.optionalString (!nospdk) "echo 'SPDK path :' $SPDK_PATH"} ${pkgs.lib.optionalString (!nospdk) "echo 'SPDK FIO plugin :' $FIO_SPDK"} echo 'Rust version :' $(rustc --version 2> /dev/null || echo '${norustc_msg}') echo 'Rust path :' $(which rustc 2> /dev/null || echo '${norustc_msg}') + echo 'I/O engine dir :' $IO_ENGINE_DIR ${pkgs.lib.optionalString (nospdk) "cowsay ${nospdk_moth}"} ${pkgs.lib.optionalString (nospdk) "export CFLAGS=-msse4"} ${pkgs.lib.optionalString (nospdk) "echo"} diff --git a/spdk-rs b/spdk-rs index d029b9f18d..a71dd7f12c 160000 --- a/spdk-rs +++ b/spdk-rs @@ -1 +1 @@ -Subproject commit d029b9f18d28f7345d84bf8c694c75bbab44d0ca +Subproject commit a71dd7f12c7cc80c4e064688e68b86bfaf5477b1 diff --git a/test/python/common/util.py b/test/python/common/util.py index 5e32bf1ecd..4a8adc4edf 100644 --- a/test/python/common/util.py +++ b/test/python/common/util.py @@ -10,4 +10,4 @@ def mayastor_target_dir(): # For now assume only Debug builds, but we might want to consider using # a variable to access Release binaries too. - return "%s/target/debug" % os.environ["SRCDIR"] + return "%s/%s" % (os.environ["SRCDIR"], os.environ["IO_ENGINE_DIR"]) diff --git a/test/python/cross-grpc-version/nexus/docker-compose.yml b/test/python/cross-grpc-version/nexus/docker-compose.yml index f051a3eca4..908a0652b9 100644 --- a/test/python/cross-grpc-version/nexus/docker-compose.yml +++ b/test/python/cross-grpc-version/nexus/docker-compose.yml @@ -11,7 +11,9 @@ services: - MY_POD_IP=10.0.0.2 - NEXUS_NVMF_ANA_ENABLE=1 - NEXUS_NVMF_RESV_ENABLE=1 - command: ${SRCDIR}/target/debug/io-engine -g 0.0.0.0 -l 1,2 -r /tmp/ms0.sock + - PATH=${LLVM_SYMBOLIZER_DIR:-} + - ASAN_OPTIONS=detect_leaks=0 + command: ${SRCDIR}/${IO_ENGINE_DIR}/io-engine -g 0.0.0.0 -l 1,2 -r /tmp/ms0.sock networks: mayastor_net: ipv4_address: 10.0.0.2 @@ -37,7 +39,9 @@ services: - MY_POD_IP=10.0.0.3 - NEXUS_NVMF_ANA_ENABLE=1 - NEXUS_NVMF_RESV_ENABLE=1 - command: ${SRCDIR}/target/debug/io-engine -g 0.0.0.0 -l 3,4 -r /tmp/ms1.sock + - PATH=${LLVM_SYMBOLIZER_DIR:-} + - ASAN_OPTIONS=detect_leaks=0 + command: ${SRCDIR}/${IO_ENGINE_DIR}/io-engine -g 0.0.0.0 -l 3,4 -r /tmp/ms1.sock networks: mayastor_net: ipv4_address: 10.0.0.3 diff --git a/test/python/cross-grpc-version/pool/docker-compose.yml b/test/python/cross-grpc-version/pool/docker-compose.yml index c62b126465..515a6aed35 100644 --- a/test/python/cross-grpc-version/pool/docker-compose.yml +++ b/test/python/cross-grpc-version/pool/docker-compose.yml @@ -11,7 +11,9 @@ services: - MY_POD_IP=10.0.0.2 - NEXUS_NVMF_ANA_ENABLE=1 - NEXUS_NVMF_RESV_ENABLE=1 - command: ${SRCDIR}/target/debug/io-engine -g 0.0.0.0 -l 1,2 -r /tmp/ms0.sock --env-context=--iova-mode=pa + - PATH=${LLVM_SYMBOLIZER_DIR:-} + - ASAN_OPTIONS=detect_leaks=0 + command: ${SRCDIR}/${IO_ENGINE_DIR}/io-engine -g 0.0.0.0 -l 1,2 -r /tmp/ms0.sock --env-context=--iova-mode=pa networks: mayastor_net: ipv4_address: 10.0.0.2 diff --git a/test/python/cross-grpc-version/rebuild/docker-compose.yml b/test/python/cross-grpc-version/rebuild/docker-compose.yml index 9eda4ea91d..a9dfb650c0 100644 --- a/test/python/cross-grpc-version/rebuild/docker-compose.yml +++ b/test/python/cross-grpc-version/rebuild/docker-compose.yml @@ -11,7 +11,9 @@ services: - MY_POD_IP=10.0.0.2 - NEXUS_NVMF_ANA_ENABLE=1 - NEXUS_NVMF_RESV_ENABLE=1 - command: ${SRCDIR}/target/debug/io-engine -g 0.0.0.0 -l 1,2 -r /tmp/ms0.sock + - PATH=${LLVM_SYMBOLIZER_DIR:-} + - ASAN_OPTIONS=detect_leaks=0 + command: ${SRCDIR}/${IO_ENGINE_DIR}/io-engine -g 0.0.0.0 -l 1,2 -r /tmp/ms0.sock networks: mayastor_net: ipv4_address: 10.0.0.2 diff --git a/test/python/cross-grpc-version/replica/docker-compose.yml b/test/python/cross-grpc-version/replica/docker-compose.yml index 22c2929120..dd1fae339a 100644 --- a/test/python/cross-grpc-version/replica/docker-compose.yml +++ b/test/python/cross-grpc-version/replica/docker-compose.yml @@ -11,7 +11,9 @@ services: - MY_POD_IP=10.0.0.2 - NEXUS_NVMF_ANA_ENABLE=1 - NEXUS_NVMF_RESV_ENABLE=1 - command: ${SRCDIR}/target/debug/io-engine -g 0.0.0.0 -l 0,1 -r /tmp/ms0.sock + - PATH=${LLVM_SYMBOLIZER_DIR:-} + - ASAN_OPTIONS=detect_leaks=0 + command: ${SRCDIR}/${IO_ENGINE_DIR}/io-engine -g 0.0.0.0 -l 0,1 -r /tmp/ms0.sock networks: mayastor_net: ipv4_address: 10.0.0.2 diff --git a/test/python/pool/docker-compose.yml b/test/python/pool/docker-compose.yml index 695baf42bd..98b2b9ff37 100644 --- a/test/python/pool/docker-compose.yml +++ b/test/python/pool/docker-compose.yml @@ -7,7 +7,9 @@ services: - MY_POD_IP=10.0.0.2 - NEXUS_NVMF_RESV_ENABLE=1 - RUST_LOG=mayastor=trace - command: ${SRCDIR}/target/debug/io-engine -g 0.0.0.0 -l 1,2 -r /tmp/ms0.sock + - PATH=${LLVM_SYMBOLIZER_DIR:-} + - ASAN_OPTIONS=detect_leaks=0 + command: ${SRCDIR}/${IO_ENGINE_DIR}/io-engine -g 0.0.0.0 -l 1,2 -r /tmp/ms0.sock networks: mayastor_net: ipv4_address: 10.0.0.2 diff --git a/test/python/tests/ana_client/docker-compose.yml b/test/python/tests/ana_client/docker-compose.yml index 39d41662a6..a55a0f5447 100644 --- a/test/python/tests/ana_client/docker-compose.yml +++ b/test/python/tests/ana_client/docker-compose.yml @@ -11,7 +11,9 @@ services: - MY_POD_IP=10.0.0.2 - NEXUS_NVMF_ANA_ENABLE=1 - NEXUS_NVMF_RESV_ENABLE=1 - command: ${SRCDIR}/target/debug/io-engine -g 0.0.0.0 -l 1,2 -r /tmp/ms0.sock + - PATH=${LLVM_SYMBOLIZER_DIR:-} + - ASAN_OPTIONS=detect_leaks=0 + command: ${SRCDIR}/${IO_ENGINE_DIR}/io-engine -g 0.0.0.0 -l 1,2 -r /tmp/ms0.sock networks: mayastor_net: ipv4_address: 10.0.0.2 @@ -37,7 +39,9 @@ services: - MY_POD_IP=10.0.0.3 - NEXUS_NVMF_ANA_ENABLE=1 - NEXUS_NVMF_RESV_ENABLE=1 - command: ${SRCDIR}/target/debug/io-engine -g 0.0.0.0 -l 3,4 -r /tmp/ms1.sock + - PATH=${LLVM_SYMBOLIZER_DIR:-} + - ASAN_OPTIONS=detect_leaks=0 + command: ${SRCDIR}/${IO_ENGINE_DIR}/io-engine -g 0.0.0.0 -l 3,4 -r /tmp/ms1.sock networks: mayastor_net: ipv4_address: 10.0.0.3 @@ -63,7 +67,9 @@ services: - MY_POD_IP=10.0.0.4 - NEXUS_NVMF_ANA_ENABLE=1 - NEXUS_NVMF_RESV_ENABLE=1 - command: ${SRCDIR}/target/debug/io-engine -g 0.0.0.0 -l 5,6 -r /tmp/ms2.sock + - PATH=${LLVM_SYMBOLIZER_DIR:-} + - ASAN_OPTIONS=detect_leaks=0 + command: ${SRCDIR}/${IO_ENGINE_DIR}/io-engine -g 0.0.0.0 -l 5,6 -r /tmp/ms2.sock networks: mayastor_net: ipv4_address: 10.0.0.4 @@ -90,7 +96,9 @@ services: - NVME_KATO_MS=1000 - NEXUS_NVMF_ANA_ENABLE=1 - NEXUS_NVMF_RESV_ENABLE=1 - command: ${SRCDIR}/target/debug/io-engine -g 0.0.0.0 -l 0,7 -r /tmp/ms3.sock + - PATH=${LLVM_SYMBOLIZER_DIR:-} + - ASAN_OPTIONS=detect_leaks=0 + command: ${SRCDIR}/${IO_ENGINE_DIR}/io-engine -g 0.0.0.0 -l 0,7 -r /tmp/ms3.sock networks: mayastor_net: ipv4_address: 10.0.0.5 diff --git a/test/python/tests/cli_controller/docker-compose.yml b/test/python/tests/cli_controller/docker-compose.yml index 4d9d4fa251..797ae4e38c 100644 --- a/test/python/tests/cli_controller/docker-compose.yml +++ b/test/python/tests/cli_controller/docker-compose.yml @@ -11,7 +11,9 @@ services: - MY_POD_IP=10.0.0.3 - NEXUS_NVMF_ANA_ENABLE=1 - NEXUS_NVMF_RESV_ENABLE=1 - command: ${SRCDIR}/target/debug/io-engine -g 0.0.0.0 -l 3,4 -r /tmp/ms1.sock + - PATH=${LLVM_SYMBOLIZER_DIR:-} + - ASAN_OPTIONS=detect_leaks=0 + command: ${SRCDIR}/${IO_ENGINE_DIR}/io-engine -g 0.0.0.0 -l 3,4 -r /tmp/ms1.sock networks: mayastor_net: ipv4_address: 10.0.0.3 @@ -37,7 +39,9 @@ services: - MY_POD_IP=10.0.0.4 - NEXUS_NVMF_ANA_ENABLE=1 - NEXUS_NVMF_RESV_ENABLE=1 - command: ${SRCDIR}/target/debug/io-engine -g 0.0.0.0 -l 5,6 -r /tmp/ms2.sock + - PATH=${LLVM_SYMBOLIZER_DIR:-} + - ASAN_OPTIONS=detect_leaks=0 + command: ${SRCDIR}/${IO_ENGINE_DIR}/io-engine -g 0.0.0.0 -l 5,6 -r /tmp/ms2.sock networks: mayastor_net: ipv4_address: 10.0.0.4 @@ -64,7 +68,9 @@ services: - NVME_KATO_MS=1000 - NEXUS_NVMF_ANA_ENABLE=1 - NEXUS_NVMF_RESV_ENABLE=1 - command: ${SRCDIR}/target/debug/io-engine -g 0.0.0.0 -l 0,7 -r /tmp/ms3.sock + - PATH=${LLVM_SYMBOLIZER_DIR:-} + - ASAN_OPTIONS=detect_leaks=0 + command: ${SRCDIR}/${IO_ENGINE_DIR}/io-engine -g 0.0.0.0 -l 0,7 -r /tmp/ms3.sock networks: mayastor_net: ipv4_address: 10.0.0.5 diff --git a/test/python/tests/nexus/docker-compose.yml b/test/python/tests/nexus/docker-compose.yml index f98c7ce06a..2f86c4a650 100644 --- a/test/python/tests/nexus/docker-compose.yml +++ b/test/python/tests/nexus/docker-compose.yml @@ -11,7 +11,9 @@ services: - MY_POD_IP=10.0.0.2 - NEXUS_NVMF_ANA_ENABLE=1 - NEXUS_NVMF_RESV_ENABLE=1 - command: ${SRCDIR}/target/debug/io-engine -g 0.0.0.0 -l 1 -r /tmp/ms0.sock + - PATH=${LLVM_SYMBOLIZER_DIR:-} + - ASAN_OPTIONS=detect_leaks=0 + command: ${SRCDIR}/${IO_ENGINE_DIR}/io-engine -g 0.0.0.0 -l 1 -r /tmp/ms0.sock networks: mayastor_net: ipv4_address: 10.0.0.2 @@ -37,7 +39,9 @@ services: - MY_POD_IP=10.0.0.3 - NEXUS_NVMF_ANA_ENABLE=1 - NEXUS_NVMF_RESV_ENABLE=1 - command: ${SRCDIR}/target/debug/io-engine -g 0.0.0.0 -l 2 -r /tmp/ms1.sock + - PATH=${LLVM_SYMBOLIZER_DIR:-} + - ASAN_OPTIONS=detect_leaks=0 + command: ${SRCDIR}/${IO_ENGINE_DIR}/io-engine -g 0.0.0.0 -l 2 -r /tmp/ms1.sock networks: mayastor_net: ipv4_address: 10.0.0.3 @@ -63,7 +67,9 @@ services: - MY_POD_IP=10.0.0.4 - NEXUS_NVMF_ANA_ENABLE=1 - NEXUS_NVMF_RESV_ENABLE=1 - command: ${SRCDIR}/target/debug/io-engine -g 0.0.0.0 -l 3 -r /tmp/ms2.sock + - PATH=${LLVM_SYMBOLIZER_DIR:-} + - ASAN_OPTIONS=detect_leaks=0 + command: ${SRCDIR}/${IO_ENGINE_DIR}/io-engine -g 0.0.0.0 -l 3 -r /tmp/ms2.sock networks: mayastor_net: ipv4_address: 10.0.0.4 @@ -92,7 +98,9 @@ services: - NEXUS_NVMF_RESV_ENABLE=1 # required when using "null" devices (which can be written to but cannot be read from) - NEXUS_DONT_READ_LABELS=true - command: ${SRCDIR}/target/debug/io-engine -g 0.0.0.0 -l 4 -r /tmp/ms3.sock + - PATH=${LLVM_SYMBOLIZER_DIR:-} + - ASAN_OPTIONS=detect_leaks=0 + command: ${SRCDIR}/${IO_ENGINE_DIR}/io-engine -g 0.0.0.0 -l 4 -r /tmp/ms3.sock networks: mayastor_net: ipv4_address: 10.0.0.5 diff --git a/test/python/tests/nexus_fault/docker-compose.yml b/test/python/tests/nexus_fault/docker-compose.yml index 3a8e677284..f207aa65dd 100644 --- a/test/python/tests/nexus_fault/docker-compose.yml +++ b/test/python/tests/nexus_fault/docker-compose.yml @@ -11,7 +11,9 @@ services: - MY_POD_IP=10.0.0.2 - NEXUS_NVMF_ANA_ENABLE=1 - NEXUS_NVMF_RESV_ENABLE=1 - command: ${SRCDIR}/target/debug/io-engine -g 0.0.0.0 -l 1 -r /tmp/ms0.sock + - PATH=${LLVM_SYMBOLIZER_DIR:-} + - ASAN_OPTIONS=detect_leaks=0 + command: ${SRCDIR}/${IO_ENGINE_DIR}/io-engine -g 0.0.0.0 -l 1 -r /tmp/ms0.sock networks: mayastor_net: ipv4_address: 10.0.0.2 @@ -37,7 +39,9 @@ services: - MY_POD_IP=10.0.0.3 - NEXUS_NVMF_ANA_ENABLE=1 - NEXUS_NVMF_RESV_ENABLE=1 - command: ${SRCDIR}/target/debug/io-engine -g 0.0.0.0 -l 2 -r /tmp/ms1.sock + - PATH=${LLVM_SYMBOLIZER_DIR:-} + - ASAN_OPTIONS=detect_leaks=0 + command: ${SRCDIR}/${IO_ENGINE_DIR}/io-engine -g 0.0.0.0 -l 2 -r /tmp/ms1.sock networks: mayastor_net: ipv4_address: 10.0.0.3 diff --git a/test/python/tests/nexus_multipath/docker-compose.yml b/test/python/tests/nexus_multipath/docker-compose.yml index 39d41662a6..a55a0f5447 100644 --- a/test/python/tests/nexus_multipath/docker-compose.yml +++ b/test/python/tests/nexus_multipath/docker-compose.yml @@ -11,7 +11,9 @@ services: - MY_POD_IP=10.0.0.2 - NEXUS_NVMF_ANA_ENABLE=1 - NEXUS_NVMF_RESV_ENABLE=1 - command: ${SRCDIR}/target/debug/io-engine -g 0.0.0.0 -l 1,2 -r /tmp/ms0.sock + - PATH=${LLVM_SYMBOLIZER_DIR:-} + - ASAN_OPTIONS=detect_leaks=0 + command: ${SRCDIR}/${IO_ENGINE_DIR}/io-engine -g 0.0.0.0 -l 1,2 -r /tmp/ms0.sock networks: mayastor_net: ipv4_address: 10.0.0.2 @@ -37,7 +39,9 @@ services: - MY_POD_IP=10.0.0.3 - NEXUS_NVMF_ANA_ENABLE=1 - NEXUS_NVMF_RESV_ENABLE=1 - command: ${SRCDIR}/target/debug/io-engine -g 0.0.0.0 -l 3,4 -r /tmp/ms1.sock + - PATH=${LLVM_SYMBOLIZER_DIR:-} + - ASAN_OPTIONS=detect_leaks=0 + command: ${SRCDIR}/${IO_ENGINE_DIR}/io-engine -g 0.0.0.0 -l 3,4 -r /tmp/ms1.sock networks: mayastor_net: ipv4_address: 10.0.0.3 @@ -63,7 +67,9 @@ services: - MY_POD_IP=10.0.0.4 - NEXUS_NVMF_ANA_ENABLE=1 - NEXUS_NVMF_RESV_ENABLE=1 - command: ${SRCDIR}/target/debug/io-engine -g 0.0.0.0 -l 5,6 -r /tmp/ms2.sock + - PATH=${LLVM_SYMBOLIZER_DIR:-} + - ASAN_OPTIONS=detect_leaks=0 + command: ${SRCDIR}/${IO_ENGINE_DIR}/io-engine -g 0.0.0.0 -l 5,6 -r /tmp/ms2.sock networks: mayastor_net: ipv4_address: 10.0.0.4 @@ -90,7 +96,9 @@ services: - NVME_KATO_MS=1000 - NEXUS_NVMF_ANA_ENABLE=1 - NEXUS_NVMF_RESV_ENABLE=1 - command: ${SRCDIR}/target/debug/io-engine -g 0.0.0.0 -l 0,7 -r /tmp/ms3.sock + - PATH=${LLVM_SYMBOLIZER_DIR:-} + - ASAN_OPTIONS=detect_leaks=0 + command: ${SRCDIR}/${IO_ENGINE_DIR}/io-engine -g 0.0.0.0 -l 0,7 -r /tmp/ms3.sock networks: mayastor_net: ipv4_address: 10.0.0.5 diff --git a/test/python/tests/publish/docker-compose.yml b/test/python/tests/publish/docker-compose.yml index 78e2834536..f494889ea6 100644 --- a/test/python/tests/publish/docker-compose.yml +++ b/test/python/tests/publish/docker-compose.yml @@ -11,7 +11,9 @@ services: - MY_POD_IP=10.0.0.2 - NEXUS_NVMF_ANA_ENABLE=1 - NEXUS_NVMF_RESV_ENABLE=1 - command: ${SRCDIR}/target/debug/io-engine -g 0.0.0.0 -l 1,2 -r /tmp/ms0.sock + - PATH=${LLVM_SYMBOLIZER_DIR:-} + - ASAN_OPTIONS=detect_leaks=0 + command: ${SRCDIR}/${IO_ENGINE_DIR}/io-engine -g 0.0.0.0 -l 1,2 -r /tmp/ms0.sock networks: mayastor_net: ipv4_address: 10.0.0.2 @@ -37,7 +39,9 @@ services: - MY_POD_IP=10.0.0.3 - NEXUS_NVMF_ANA_ENABLE=1 - NEXUS_NVMF_RESV_ENABLE=1 - command: ${SRCDIR}/target/debug/io-engine -g 0.0.0.0 -l 3,4 -r /tmp/ms1.sock + - PATH=${LLVM_SYMBOLIZER_DIR:-} + - ASAN_OPTIONS=detect_leaks=0 + command: ${SRCDIR}/${IO_ENGINE_DIR}/io-engine -g 0.0.0.0 -l 3,4 -r /tmp/ms1.sock networks: mayastor_net: ipv4_address: 10.0.0.3 diff --git a/test/python/tests/rebuild/docker-compose.yml b/test/python/tests/rebuild/docker-compose.yml index cd1457bad9..e03cc0cc52 100644 --- a/test/python/tests/rebuild/docker-compose.yml +++ b/test/python/tests/rebuild/docker-compose.yml @@ -11,7 +11,9 @@ services: - MY_POD_IP=10.0.0.2 - NEXUS_NVMF_ANA_ENABLE=1 - NEXUS_NVMF_RESV_ENABLE=1 - command: ${SRCDIR}/target/debug/io-engine -g 0.0.0.0 -l 1,2 -r /tmp/ms0.sock + - PATH=${LLVM_SYMBOLIZER_DIR:-} + - ASAN_OPTIONS=detect_leaks=0 + command: ${SRCDIR}/${IO_ENGINE_DIR}/io-engine -g 0.0.0.0 -l 1,2 -r /tmp/ms0.sock networks: mayastor_net: ipv4_address: 10.0.0.2 diff --git a/test/python/tests/replica/docker-compose.yml b/test/python/tests/replica/docker-compose.yml index cd1457bad9..e03cc0cc52 100644 --- a/test/python/tests/replica/docker-compose.yml +++ b/test/python/tests/replica/docker-compose.yml @@ -11,7 +11,9 @@ services: - MY_POD_IP=10.0.0.2 - NEXUS_NVMF_ANA_ENABLE=1 - NEXUS_NVMF_RESV_ENABLE=1 - command: ${SRCDIR}/target/debug/io-engine -g 0.0.0.0 -l 1,2 -r /tmp/ms0.sock + - PATH=${LLVM_SYMBOLIZER_DIR:-} + - ASAN_OPTIONS=detect_leaks=0 + command: ${SRCDIR}/${IO_ENGINE_DIR}/io-engine -g 0.0.0.0 -l 1,2 -r /tmp/ms0.sock networks: mayastor_net: ipv4_address: 10.0.0.2 diff --git a/test/python/tests/replica_uuid/docker-compose.yml b/test/python/tests/replica_uuid/docker-compose.yml index cd1457bad9..e03cc0cc52 100644 --- a/test/python/tests/replica_uuid/docker-compose.yml +++ b/test/python/tests/replica_uuid/docker-compose.yml @@ -11,7 +11,9 @@ services: - MY_POD_IP=10.0.0.2 - NEXUS_NVMF_ANA_ENABLE=1 - NEXUS_NVMF_RESV_ENABLE=1 - command: ${SRCDIR}/target/debug/io-engine -g 0.0.0.0 -l 1,2 -r /tmp/ms0.sock + - PATH=${LLVM_SYMBOLIZER_DIR:-} + - ASAN_OPTIONS=detect_leaks=0 + command: ${SRCDIR}/${IO_ENGINE_DIR}/io-engine -g 0.0.0.0 -l 1,2 -r /tmp/ms0.sock networks: mayastor_net: ipv4_address: 10.0.0.2 diff --git a/test/python/tests/rpc/docker-compose.yml b/test/python/tests/rpc/docker-compose.yml index f3638f9dc7..3175c21c9d 100644 --- a/test/python/tests/rpc/docker-compose.yml +++ b/test/python/tests/rpc/docker-compose.yml @@ -11,7 +11,9 @@ services: - MY_POD_IP=10.0.0.3 - NEXUS_NVMF_ANA_ENABLE=1 - NEXUS_NVMF_RESV_ENABLE=1 - command: ${SRCDIR}/target/debug/io-engine -g 0.0.0.0 -l 3,4 -r /tmp/ms1.sock + - PATH=${LLVM_SYMBOLIZER_DIR:-} + - ASAN_OPTIONS=detect_leaks=0 + command: ${SRCDIR}/${IO_ENGINE_DIR}/io-engine -g 0.0.0.0 -l 3,4 -r /tmp/ms1.sock networks: mayastor_net: ipv4_address: 10.0.0.3 diff --git a/test/python/v1/nexus/docker-compose.yml b/test/python/v1/nexus/docker-compose.yml index 9f4c6844e2..4b11c9e432 100644 --- a/test/python/v1/nexus/docker-compose.yml +++ b/test/python/v1/nexus/docker-compose.yml @@ -11,7 +11,9 @@ services: - MY_POD_IP=10.0.0.2 - NEXUS_NVMF_ANA_ENABLE=1 - NEXUS_NVMF_RESV_ENABLE=1 - command: ${SRCDIR}/target/debug/io-engine -g 0.0.0.0 -l 1,2 -r /tmp/ms0.sock + - PATH=${LLVM_SYMBOLIZER_DIR:-} + - ASAN_OPTIONS=detect_leaks=0 + command: ${SRCDIR}/${IO_ENGINE_DIR}/io-engine -g 0.0.0.0 -l 1,2 -r /tmp/ms0.sock networks: mayastor_net: ipv4_address: 10.0.0.2 @@ -37,7 +39,9 @@ services: - MY_POD_IP=10.0.0.3 - NEXUS_NVMF_ANA_ENABLE=1 - NEXUS_NVMF_RESV_ENABLE=1 - command: ${SRCDIR}/target/debug/io-engine -g 0.0.0.0 -l 2 -r /tmp/ms1.sock + - PATH=${LLVM_SYMBOLIZER_DIR:-} + - ASAN_OPTIONS=detect_leaks=0 + command: ${SRCDIR}/${IO_ENGINE_DIR}/io-engine -g 0.0.0.0 -l 2 -r /tmp/ms1.sock networks: mayastor_net: ipv4_address: 10.0.0.3 @@ -63,7 +67,9 @@ services: - MY_POD_IP=10.0.0.4 - NEXUS_NVMF_ANA_ENABLE=1 - NEXUS_NVMF_RESV_ENABLE=1 - command: ${SRCDIR}/target/debug/io-engine -g 0.0.0.0 -l 3 -r /tmp/ms2.sock + - PATH=${LLVM_SYMBOLIZER_DIR:-} + - ASAN_OPTIONS=detect_leaks=0 + command: ${SRCDIR}/${IO_ENGINE_DIR}/io-engine -g 0.0.0.0 -l 3 -r /tmp/ms2.sock networks: mayastor_net: ipv4_address: 10.0.0.4 @@ -92,7 +98,9 @@ services: - NEXUS_NVMF_RESV_ENABLE=1 # required when using "null" devices (which can be written to but cannot be read from) - NEXUS_DONT_READ_LABELS=true - command: ${SRCDIR}/target/debug/io-engine -g 0.0.0.0 -l 4 -r /tmp/ms3.sock + - PATH=${LLVM_SYMBOLIZER_DIR:-} + - ASAN_OPTIONS=detect_leaks=0 + command: ${SRCDIR}/${IO_ENGINE_DIR}/io-engine -g 0.0.0.0 -l 4 -r /tmp/ms3.sock networks: mayastor_net: ipv4_address: 10.0.0.5 diff --git a/test/python/v1/pool/docker-compose.yml b/test/python/v1/pool/docker-compose.yml index cd1457bad9..e03cc0cc52 100644 --- a/test/python/v1/pool/docker-compose.yml +++ b/test/python/v1/pool/docker-compose.yml @@ -11,7 +11,9 @@ services: - MY_POD_IP=10.0.0.2 - NEXUS_NVMF_ANA_ENABLE=1 - NEXUS_NVMF_RESV_ENABLE=1 - command: ${SRCDIR}/target/debug/io-engine -g 0.0.0.0 -l 1,2 -r /tmp/ms0.sock + - PATH=${LLVM_SYMBOLIZER_DIR:-} + - ASAN_OPTIONS=detect_leaks=0 + command: ${SRCDIR}/${IO_ENGINE_DIR}/io-engine -g 0.0.0.0 -l 1,2 -r /tmp/ms0.sock networks: mayastor_net: ipv4_address: 10.0.0.2 diff --git a/test/python/v1/rebuild/docker-compose.yml b/test/python/v1/rebuild/docker-compose.yml index cd1457bad9..e03cc0cc52 100644 --- a/test/python/v1/rebuild/docker-compose.yml +++ b/test/python/v1/rebuild/docker-compose.yml @@ -11,7 +11,9 @@ services: - MY_POD_IP=10.0.0.2 - NEXUS_NVMF_ANA_ENABLE=1 - NEXUS_NVMF_RESV_ENABLE=1 - command: ${SRCDIR}/target/debug/io-engine -g 0.0.0.0 -l 1,2 -r /tmp/ms0.sock + - PATH=${LLVM_SYMBOLIZER_DIR:-} + - ASAN_OPTIONS=detect_leaks=0 + command: ${SRCDIR}/${IO_ENGINE_DIR}/io-engine -g 0.0.0.0 -l 1,2 -r /tmp/ms0.sock networks: mayastor_net: ipv4_address: 10.0.0.2 diff --git a/test/python/v1/replica/docker-compose.yml b/test/python/v1/replica/docker-compose.yml index 99982a5288..7fe863759d 100644 --- a/test/python/v1/replica/docker-compose.yml +++ b/test/python/v1/replica/docker-compose.yml @@ -11,7 +11,9 @@ services: - MY_POD_IP=10.0.0.2 - NEXUS_NVMF_ANA_ENABLE=1 - NEXUS_NVMF_RESV_ENABLE=1 - command: ${SRCDIR}/target/debug/io-engine -g 0.0.0.0 -l 1 -r /tmp/ms0.sock + - PATH=${LLVM_SYMBOLIZER_DIR:-} + - ASAN_OPTIONS=detect_leaks=0 + command: ${SRCDIR}/${IO_ENGINE_DIR}/io-engine -g 0.0.0.0 -l 1 -r /tmp/ms0.sock networks: mayastor_net: ipv4_address: 10.0.0.2 diff --git a/utils/dependencies b/utils/dependencies index fc24d16b83..f65be5f88a 160000 --- a/utils/dependencies +++ b/utils/dependencies @@ -1 +1 @@ -Subproject commit fc24d16b837e991e3687a8c933ffcbc2b59d4cf5 +Subproject commit f65be5f88ae74a50ff3fa3bbcd4ee3ca4a2b0399