diff --git a/doc/run.md b/doc/run.md index 6f2dd9094..2528ad85f 100644 --- a/doc/run.md +++ b/doc/run.md @@ -18,7 +18,7 @@ It will also need at least 512 2 MB Hugepages configured. > Learn more about hugepages: [parts 1][hugepages-lwn-one], [2][hugepages-lwn-two], > [3][hugepages-lwn-three], [4][hugepages-lwn-four], [5][hugepages-lwn-five]. - + In NixOS: ```nix @@ -78,7 +78,7 @@ In order to use the full feature set of Mayastor, some or all of the following c + `nvme_loop`: NVMe Loop Device support To load these on NixOS: - + ```nix # /etc/nixos/configuration.nix boot.kernelModules = [ @@ -91,7 +91,40 @@ In order to use the full feature set of Mayastor, some or all of the following c ```bash modprobe nbd nvmet nvmet_rdma nvme_fabrics nvme_tcp nvme_rdma nvme_loop ``` -* An NVMe device. (Typically via PCI-E through an standard slot or [M.2][m-dot-2] port) +* For Asymmetric Namespace Access (ANA) support (early preview), the following kernel build configuration enabled: + + `CONFIG_NVME_MULTIPATH`: enables support for multipath access to NVMe subsystems + + This is usually already enabled in distributions kernels, at least for RHEL/CentOS 8.2, Ubuntu 20.04 LTS, and SUSE Linux Enterprise 15.2. + + On some distributions such as RHEL 8, the feature must be enabled manually: + + ```sh + # /etc/modprobe.d/nvme-multipath + options nvme_core multipath=1 + ``` + + followed by reloading the `nvme-core` module or rebooting. + + To build this on NixOS: + + ```nix + # /etc/nixos/configuration.nix + boot.kernelPackages = pkgs.linuxPackages; + boot.kernelPatches = [ { + name = "nvme-multipath"; + patch = null; + extraConfig = '' + NVME_MULTIPATH y + ''; + } ]; + ``` + + followed by: + + ```sh + sudo nixos-rebuild boot + ``` +* An NVMe device. (Typically via PCI-E through a standard slot or [M.2][m-dot-2] port) * A version of [`nix`][nix-install] configured as in the [build guide.][doc-build] ## Running binaries directly @@ -158,7 +191,7 @@ Why these parameters? - `--privileged` to allow controlling memory policies. > **TODO:** We can use [control groups][control-groups] for this! -- `-v /dev:/dev:rw` is needed to get access to any raw device you might want to consume as local +- `-v /dev:/dev:rw` is needed to get access to any raw device you might want to consume as local storage and huge pages - `-v /dev/shm:/dev/shm:rw` is needed as for a circular buffer that can trace any IO operations as they happen @@ -184,7 +217,7 @@ nixpkgs.overlays = [ ]; systemd.services.mayastor = { - wantedBy = [ "multi-user.target" ]; + wantedBy = [ "multi-user.target" ]; after = [ "network.target" ]; description = "A cloud native declarative data plane."; serviceConfig = { @@ -273,4 +306,3 @@ production Mayastor deployment and operation instructions. [lxd]: https://linuxcontainers.org/ [libvirtd]: https://libvirt.org/index.html [terraform-readme]: ./terraform/readme.adoc -[aarch64-branch]: \ No newline at end of file diff --git a/mayastor/src/bdev/nexus/nexus_bdev.rs b/mayastor/src/bdev/nexus/nexus_bdev.rs index 53193fc02..5bae40405 100644 --- a/mayastor/src/bdev/nexus/nexus_bdev.rs +++ b/mayastor/src/bdev/nexus/nexus_bdev.rs @@ -13,6 +13,7 @@ use std::{ use futures::{channel::oneshot, future::join_all}; use nix::errno::Errno; +use rpc::mayastor::NvmeAnaState; use serde::Serialize; use snafu::{ResultExt, Snafu}; use tonic::{Code, Status}; @@ -106,6 +107,8 @@ pub enum Error { AlreadyShared { name: String }, #[snafu(display("The nexus {} has not been shared", name))] NotShared { name: String }, + #[snafu(display("The nexus {} has not been shared over NVMf", name))] + NotSharedNvmf { name: String }, #[snafu(display("Failed to share nexus over NBD {}", name))] ShareNbdNexus { source: NbdError, name: String }, #[snafu(display("Failed to share iscsi nexus {}", name))] @@ -233,6 +236,8 @@ pub enum Error { }, #[snafu(display("Invalid ShareProtocol value {}", sp_value))] InvalidShareProtocol { sp_value: i32 }, + #[snafu(display("Invalid NvmeAnaState value {}", ana_value))] + InvalidNvmeAnaState { ana_value: i32 }, #[snafu(display("Failed to create nexus {}", name))] NexusCreate { name: String }, #[snafu(display("Failed to destroy nexus {}", name))] @@ -252,6 +257,16 @@ pub enum Error { FailedGetHandle, #[snafu(display("Failed to create snapshot on nexus {}", name))] FailedCreateSnapshot { name: String, source: CoreError }, + #[snafu(display("NVMf subsystem error: {}", e))] + SubsysNvmfError { e: String }, +} + +impl From for Error { + fn from(error: subsys::NvmfError) -> Self { + Error::SubsysNvmfError { + e: error.to_string(), + } + } } impl From for tonic::Status { @@ -272,6 +287,9 @@ impl From for tonic::Status { Error::NotShared { .. } => Status::invalid_argument(e.to_string()), + Error::NotSharedNvmf { + .. + } => Status::invalid_argument(e.to_string()), Error::CreateChild { .. } => Status::invalid_argument(e.to_string()), @@ -630,6 +648,43 @@ impl Nexus { Ok(()) } + /// get ANA state of the NVMe subsystem + pub async fn get_ana_state(&self) -> Result { + if let Some(Protocol::Nvmf) = self.shared() { + if let Some(subsystem) = NvmfSubsystem::nqn_lookup(&self.name) { + let ana_state = subsystem.get_ana_state().await? as i32; + return NvmeAnaState::from_i32(ana_state).ok_or({ + Error::InvalidNvmeAnaState { + ana_value: ana_state, + } + }); + } + } + + Err(Error::NotSharedNvmf { + name: self.name.clone(), + }) + } + + /// set ANA state of the NVMe subsystem + pub async fn set_ana_state( + &self, + ana_state: NvmeAnaState, + ) -> Result<(), Error> { + if let Some(Protocol::Nvmf) = self.shared() { + if let Some(subsystem) = NvmfSubsystem::nqn_lookup(&self.name) { + subsystem.pause().await?; + let res = subsystem.set_ana_state(ana_state as u32).await; + subsystem.resume().await?; + return Ok(res?); + } + } + + Err(Error::NotSharedNvmf { + name: self.name.clone(), + }) + } + /// register the bdev with SPDK and set the callbacks for io channel /// creation. Once this function is called, the device is visible and can /// be used for IO. diff --git a/mayastor/src/bin/mayastor-client/nexus_cli.rs b/mayastor/src/bin/mayastor-client/nexus_cli.rs index 57a167733..26b21882f 100644 --- a/mayastor/src/bin/mayastor-client/nexus_cli.rs +++ b/mayastor/src/bin/mayastor-client/nexus_cli.rs @@ -54,6 +54,24 @@ pub fn subcommands<'a, 'b>() -> App<'a, 'b> { .help("uuid for the nexus"), ); + let ana_state = SubCommand::with_name("ana_state") + .about("get or set the NVMe ANA state of the nexus") + .arg( + Arg::with_name("uuid") + .required(true) + .index(1) + .help("uuid for the nexus"), + ) + .arg( + Arg::with_name("state") + .required(false) + .index(2) + .possible_value("optimized") + .possible_value("non_optimized") + .possible_value("inaccessible") + .help("NVMe ANA state of the nexus"), + ); + let add = SubCommand::with_name("add") .about("add a child") .arg( @@ -122,6 +140,7 @@ pub fn subcommands<'a, 'b>() -> App<'a, 'b> { .subcommand(add) .subcommand(remove) .subcommand(unpublish) + .subcommand(ana_state) .subcommand(list) .subcommand(children) .subcommand(nexus_child_cli::subcommands()) @@ -138,6 +157,7 @@ pub async fn handler( ("children", Some(args)) => nexus_children(ctx, &args).await, ("publish", Some(args)) => nexus_publish(ctx, &args).await, ("unpublish", Some(args)) => nexus_unpublish(ctx, &args).await, + ("ana_state", Some(args)) => nexus_nvme_ana_state(ctx, &args).await, ("add", Some(args)) => nexus_add(ctx, &args).await, ("remove", Some(args)) => nexus_remove(ctx, &args).await, ("child", Some(args)) => nexus_child_cli::handler(ctx, args).await, @@ -321,6 +341,66 @@ async fn nexus_unpublish( Ok(()) } +async fn nexus_nvme_ana_state( + ctx: Context, + matches: &ArgMatches<'_>, +) -> Result<(), Status> { + let uuid = matches.value_of("uuid").unwrap().to_string(); + let ana_state = matches.value_of("state").unwrap_or("").to_string(); + if ana_state.is_empty() { + nexus_get_nvme_ana_state(ctx, uuid).await + } else { + nexus_set_nvme_ana_state(ctx, uuid, ana_state).await + } +} + +async fn nexus_get_nvme_ana_state( + mut ctx: Context, + uuid: String, +) -> Result<(), Status> { + ctx.v2(&format!("Getting NVMe ANA state for nexus {}", uuid)); + let resp = ctx + .client + .get_nvme_ana_state(rpc::GetNvmeAnaStateRequest { + uuid: uuid.clone(), + }) + .await?; + ctx.v1(ana_state_idx_to_str(resp.get_ref().ana_state)); + Ok(()) +} + +async fn nexus_set_nvme_ana_state( + mut ctx: Context, + uuid: String, + ana_state_str: String, +) -> Result<(), Status> { + let ana_state: rpc::NvmeAnaState = match ana_state_str.parse() { + Ok(a) => a, + _ => { + return Err(Status::new( + Code::Internal, + "Invalid value of NVMe ANA state".to_owned(), + )); + } + }; + + ctx.v2(&format!( + "Setting NVMe ANA state for nexus {} to {:?}", + uuid, ana_state + )); + ctx.client + .set_nvme_ana_state(rpc::SetNvmeAnaStateRequest { + uuid: uuid.clone(), + ana_state: ana_state.into(), + }) + .await?; + ctx.v1(&format!( + "Set NVMe ANA state for nexus {} to {:?}", + uuid, ana_state + )); + Ok(()) +} + async fn nexus_add( mut ctx: Context, matches: &ArgMatches<'_>, @@ -363,6 +443,17 @@ async fn nexus_remove( Ok(()) } +fn ana_state_idx_to_str(idx: i32) -> &'static str { + match rpc::NvmeAnaState::from_i32(idx).unwrap() { + rpc::NvmeAnaState::NvmeAnaInvalidState => "invalid", + rpc::NvmeAnaState::NvmeAnaOptimizedState => "optimized", + rpc::NvmeAnaState::NvmeAnaNonOptimizedState => "non_optimized", + rpc::NvmeAnaState::NvmeAnaInaccessibleState => "inaccessible", + rpc::NvmeAnaState::NvmeAnaPersistentLossState => "persistent_loss", + rpc::NvmeAnaState::NvmeAnaChangeState => "change", + } +} + fn nexus_state_to_str(idx: i32) -> &'static str { match rpc::NexusState::from_i32(idx).unwrap() { rpc::NexusState::NexusUnknown => "unknown", diff --git a/mayastor/src/bin/mayastor.rs b/mayastor/src/bin/mayastor.rs index 45a0fe34f..dfde01073 100644 --- a/mayastor/src/bin/mayastor.rs +++ b/mayastor/src/bin/mayastor.rs @@ -42,12 +42,33 @@ fn main() -> Result<(), Box> { let free_pages: u32 = sysfs::parse_value(&hugepage_path, "free_hugepages")?; let nr_pages: u32 = sysfs::parse_value(&hugepage_path, "nr_hugepages")?; let uring_supported = uring::kernel_support(); + let nvme_core_path = Path::new("/sys/module/nvme_core/parameters"); + let nvme_mp: String = + match sysfs::parse_value::(&nvme_core_path, "multipath") { + Ok(s) => match s.as_str() { + "Y" => "yes".to_string(), + "N" => "disabled".to_string(), + u => format!("unknown value {}", u), + }, + Err(e) => { + if e.kind() == std::io::ErrorKind::NotFound { + if nvme_core_path.exists() { + "not built".to_string() + } else { + "nvme not loaded".to_string() + } + } else { + format!("unknown error: {}", e) + } + } + }; info!("Starting Mayastor .."); info!( "kernel io_uring support: {}", if uring_supported { "yes" } else { "no" } ); + info!("kernel nvme initiator multipath support: {}", nvme_mp); info!("free_pages: {} nr_pages: {}", free_pages, nr_pages); let grpc_endpoint = grpc::endpoint(args.grpc_endpoint.clone()); diff --git a/mayastor/src/grpc/mayastor_grpc.rs b/mayastor/src/grpc/mayastor_grpc.rs index 41a400c5e..979a1c61b 100644 --- a/mayastor/src/grpc/mayastor_grpc.rs +++ b/mayastor/src/grpc/mayastor_grpc.rs @@ -289,6 +289,52 @@ impl mayastor_server::Mayastor for MayastorSvc { .await } + #[instrument(level = "debug", err)] + async fn get_nvme_ana_state( + &self, + request: Request, + ) -> GrpcResult { + let args = request.into_inner(); + let uuid = args.uuid.clone(); + debug!("Getting NVMe ANA state for nexus {} ...", uuid); + + let ana_state = locally! { async move { + nexus_lookup(&args.uuid)?.get_ana_state().await + }}; + + info!("Got nexus {} NVMe ANA state {:?}", uuid, ana_state); + Ok(Response::new(GetNvmeAnaStateReply { + ana_state: ana_state as i32, + })) + } + + #[instrument(level = "debug", err)] + async fn set_nvme_ana_state( + &self, + request: Request, + ) -> GrpcResult { + let args = request.into_inner(); + let uuid = args.uuid.clone(); + debug!("Setting NVMe ANA state for nexus {} ...", uuid); + + let ana_state = match NvmeAnaState::from_i32(args.ana_state) { + Some(ana_state) => ana_state, + None => { + return Err(nexus_bdev::Error::InvalidNvmeAnaState { + ana_value: args.ana_state as i32, + } + .into()); + } + }; + + locally! { async move { + nexus_lookup(&args.uuid)?.set_ana_state(ana_state).await + }}; + + info!("Set nexus {} NVMe ANA state {:?}", uuid, ana_state); + Ok(Response::new(Null {})) + } + #[instrument(level = "debug", err)] async fn child_operation( &self, diff --git a/mayastor/src/subsys/nvmf/mod.rs b/mayastor/src/subsys/nvmf/mod.rs index 3a3e51122..a4c346374 100644 --- a/mayastor/src/subsys/nvmf/mod.rs +++ b/mayastor/src/subsys/nvmf/mod.rs @@ -63,16 +63,18 @@ pub enum Error { PgError { msg: String }, #[snafu(display("Failed to create transport {}", msg))] Transport { source: Errno, msg: String }, - #[snafu(display("Failed to create subsystem for {} {} error: {}", source.desc(), nqn, msg))] + #[snafu(display("Failed nvmf subsystem operation for {} {} error: {}", source.desc(), nqn, msg))] Subsystem { source: Errno, nqn: String, msg: String, }, - #[snafu(display("Failed to create share for {} {}", bdev, msg))] + #[snafu(display("Failed to create share for {} {}", bdev, msg))] Share { bdev: String, msg: String }, - #[snafu(display("Failed to add namespace for {} {}", bdev, msg))] + #[snafu(display("Failed to add namespace for {} {}", bdev, msg))] Namespace { bdev: String, msg: String }, + #[snafu(display("Failed to find listener for {} {}", nqn, trid))] + Listener { nqn: String, trid: String }, } thread_local! { diff --git a/mayastor/src/subsys/nvmf/subsystem.rs b/mayastor/src/subsys/nvmf/subsystem.rs index 3774e5f34..af417f388 100644 --- a/mayastor/src/subsys/nvmf/subsystem.rs +++ b/mayastor/src/subsys/nvmf/subsystem.rs @@ -12,6 +12,8 @@ use futures::channel::oneshot; use nix::errno::Errno; use spdk_sys::{ + nvmf_subsystem_find_listener, + nvmf_subsystem_set_ana_state, spdk_bdev_nvme_opts, spdk_nvmf_ns_get_bdev, spdk_nvmf_ns_opts, @@ -471,6 +473,54 @@ impl NvmfSubsystem { } } + /// get ANA state + pub async fn get_ana_state(&self) -> Result { + let cfg = Config::get(); + let trid_replica = TransportId::new(cfg.nexus_opts.nvmf_replica_port); + let listener = unsafe { + nvmf_subsystem_find_listener(self.0.as_ptr(), trid_replica.as_ptr()) + }; + if listener.is_null() { + Err(Error::Listener { + nqn: self.get_nqn(), + trid: trid_replica.to_string(), + }) + } else { + Ok(unsafe { (*listener).ana_state }) + } + } + + /// set ANA state: optimized, non_optimized, inaccessible + /// subsystem must be in paused or inactive state + pub async fn set_ana_state(&self, ana_state: u32) -> Result<(), Error> { + extern "C" fn set_ana_state_cb(arg: *mut c_void, status: i32) { + let s = unsafe { Box::from_raw(arg as *mut oneshot::Sender) }; + s.send(status).unwrap(); + } + let cfg = Config::get(); + let trid_replica = TransportId::new(cfg.nexus_opts.nvmf_replica_port); + + let (s, r) = oneshot::channel::(); + + unsafe { + nvmf_subsystem_set_ana_state( + self.0.as_ptr(), + trid_replica.as_ptr(), + ana_state, + Some(set_ana_state_cb), + cb_arg(s), + ); + } + + r.await + .expect("Cancellation is not supported") + .to_result(|e| Error::Subsystem { + source: Errno::from_i32(-e), + nqn: self.get_nqn(), + msg: "failed to set_ana_state of the subsystem".to_string(), + }) + } + /// destroy all subsystems associated with our target, subsystems must be in /// stopped state pub fn destroy_all() { diff --git a/mayastor/tests/nexus_multipath.rs b/mayastor/tests/nexus_multipath.rs index c61c965f4..27def0afe 100644 --- a/mayastor/tests/nexus_multipath.rs +++ b/mayastor/tests/nexus_multipath.rs @@ -8,6 +8,7 @@ use rpc::mayastor::{ CreateNexusRequest, CreatePoolRequest, CreateReplicaRequest, + NvmeAnaState, PublishNexusRequest, ShareProtocolNexus, }; @@ -71,11 +72,12 @@ async fn nexus_multipath() { let mayastor = MayastorTest::new(MayastorCliArgs::default()); let ip0 = hdls[0].endpoint.ip(); let nexus_name = format!("nexus-{}", UUID); + let name = nexus_name.clone(); mayastor .spawn(async move { // create nexus on local node with remote replica as child nexus_create( - &nexus_name, + &name, 32 * 1024 * 1024, Some(UUID), &[format!("nvmf://{}:8420/{}:{}", ip0, HOSTNQN, UUID)], @@ -83,7 +85,7 @@ async fn nexus_multipath() { .await .unwrap(); // publish nexus on local node over nvmf - nexus_lookup(&nexus_name) + nexus_lookup(&name) .unwrap() .share(ShareProtocolNexus::NexusNvmf, None) .await @@ -138,6 +140,51 @@ async fn nexus_multipath() { ); } + let output_list = Command::new("nvme").args(&["list"]).output().unwrap(); + assert!( + output_list.status.success(), + "failed to list nvme devices, {}", + output_list.status + ); + let sl = String::from_utf8(output_list.stdout).unwrap(); + let nvmems: Vec<&str> = sl + .lines() + .filter(|line| line.contains("Mayastor NVMe controller")) + .collect(); + assert_eq!(nvmems.len(), 1); + let ns = nvmems[0].split(' ').collect::>()[0]; + + mayastor + .spawn(async move { + // set nexus on local node ANA state to non-optimized + nexus_lookup(&nexus_name) + .unwrap() + .set_ana_state(NvmeAnaState::NvmeAnaNonOptimizedState) + .await + .unwrap(); + }) + .await; + + // +- nvme0 tcp traddr=127.0.0.1 trsvcid=8420 live + let output_subsys = Command::new("nvme") + .args(&["list-subsys"]) + .args(&[ns]) + .output() + .unwrap(); + assert!( + output_subsys.status.success(), + "failed to list nvme subsystem, {}", + output_subsys.status + ); + let subsys = String::from_utf8(output_subsys.stdout).unwrap(); + let nvmec: Vec<&str> = subsys + .lines() + .filter(|line| line.contains("traddr=127.0.0.1")) + .collect(); + assert_eq!(nvmec.len(), 1); + let nv: Vec<&str> = nvmec[0].split(' ').collect(); + assert_eq!(nv[7], "non-optimized", "incorrect ANA state"); + // NQN: disconnected 2 controller(s) let output_dis = Command::new("nvme") .args(&["disconnect"]) @@ -152,11 +199,8 @@ async fn nexus_multipath() { let s = String::from_utf8(output_dis.stdout).unwrap(); let v: Vec<&str> = s.split(' ').collect(); tracing::info!("nvme disconnected: {:?}", v); - assert!(v.len() == 4); - assert!(v[1] == "disconnected"); - assert!( - v[0] == format!("NQN:{}", &nqn), - "mismatched NQN disconnected" - ); - assert!(v[2] == "2", "mismatched number of controllers disconnected"); + assert_eq!(v.len(), 4); + assert_eq!(v[1], "disconnected"); + assert_eq!(v[0], format!("NQN:{}", &nqn), "mismatched NQN disconnected"); + assert_eq!(v[2], "2", "mismatched number of controllers disconnected"); } diff --git a/rpc/proto/mayastor.proto b/rpc/proto/mayastor.proto index e887af13c..612dd15cd 100644 --- a/rpc/proto/mayastor.proto +++ b/rpc/proto/mayastor.proto @@ -51,6 +51,10 @@ service Mayastor { rpc PublishNexus (PublishNexusRequest) returns (PublishNexusReply) {} rpc UnpublishNexus (UnpublishNexusRequest) returns (Null) {} + // NVMe ANA state + rpc GetNvmeAnaState (GetNvmeAnaStateRequest) returns (GetNvmeAnaStateReply) {} + rpc SetNvmeAnaState (SetNvmeAnaStateRequest) returns (Null) {} + // Nexus child operations rpc ChildOperation(ChildNexusRequest) returns (Null) {} @@ -273,6 +277,28 @@ message UnpublishNexusRequest { string uuid = 1; // uuid of the nexus which to destroy } +enum NvmeAnaState { + NVME_ANA_INVALID_STATE = 0; // invalid, do not use + NVME_ANA_OPTIMIZED_STATE = 0x1; + NVME_ANA_NON_OPTIMIZED_STATE = 0x2; + NVME_ANA_INACCESSIBLE_STATE = 0x3; + NVME_ANA_PERSISTENT_LOSS_STATE = 0x4; // not yet supported + NVME_ANA_CHANGE_STATE = 0xF; // not yet supported +} + +message GetNvmeAnaStateRequest { + string uuid = 1; // uuid of the nexus +} + +message GetNvmeAnaStateReply { + NvmeAnaState ana_state = 1; +} + +message SetNvmeAnaStateRequest { + string uuid = 1; // uuid of the nexus + NvmeAnaState ana_state = 2; +} + enum ChildAction { offline = 0; online = 1; diff --git a/rpc/src/lib.rs b/rpc/src/lib.rs index c12fc0375..b49d8fd62 100644 --- a/rpc/src/lib.rs +++ b/rpc/src/lib.rs @@ -11,6 +11,12 @@ extern crate tonic; #[allow(clippy::redundant_closure)] #[allow(clippy::upper_case_acronyms)] pub mod mayastor { + use std::str::FromStr; + + #[derive(Debug)] + pub enum Error { + ParseError, + } impl From<()> for Null { fn from(_: ()) -> Self { @@ -18,5 +24,17 @@ pub mod mayastor { } } + impl FromStr for NvmeAnaState { + type Err = Error; + fn from_str(state: &str) -> Result { + match state { + "optimized" => Ok(Self::NvmeAnaOptimizedState), + "non_optimized" => Ok(Self::NvmeAnaNonOptimizedState), + "inaccessible" => Ok(Self::NvmeAnaInaccessibleState), + _ => Err(Error::ParseError), + } + } + } + include!(concat!(env!("OUT_DIR"), "/mayastor.rs")); } diff --git a/spdk-sys/build.rs b/spdk-sys/build.rs index a921d1cc3..d33099de1 100644 --- a/spdk-sys/build.rs +++ b/spdk-sys/build.rs @@ -84,6 +84,8 @@ fn main() { .whitelist_function("^vbdev_.*") .whitelist_function("^nvme_cmd_.*") .whitelist_function("^nvme_status_.*") + .whitelist_function("^nvmf_subsystem_find_listener") + .whitelist_function("^nvmf_subsystem_set_ana_state") .whitelist_function("^nvmf_tgt_accept") .blacklist_type("^longfunc") .whitelist_var("^NVMF.*") diff --git a/test/grpc/test_nexus.js b/test/grpc/test_nexus.js index 2b2e09fc0..9faafdc2a 100644 --- a/test/grpc/test_nexus.js +++ b/test/grpc/test_nexus.js @@ -685,6 +685,42 @@ describe('nexus', function () { }); }); + it('should change ANA state to inaccessible', (done) => { + client.setNvmeAnaState( + { + uuid: UUID, + ana_state: enums.NVME_ANA_INACCESSIBLE_STATE + }, + done + ); + }); + + it('should get ANA state as inaccessible', (done) => { + client.getNvmeAnaState({ uuid: UUID }, (err, res) => { + if (err) done(err); + assert.equal(res.ana_state, 'NVME_ANA_INACCESSIBLE_STATE'); + done(); + }); + }); + + it('should change ANA state back to optimized', (done) => { + client.setNvmeAnaState( + { + uuid: UUID, + ana_state: enums.NVME_ANA_OPTIMIZED_STATE + }, + done + ); + }); + + it('should get ANA state as optimized', (done) => { + client.getNvmeAnaState({ uuid: UUID }, (err, res) => { + if (err) done(err); + assert.equal(res.ana_state, 'NVME_ANA_OPTIMIZED_STATE'); + done(); + }); + }); + it('should write to nvmf replica', (done) => { common.execAsRoot( common.getCmdPath('initiator'), @@ -694,10 +730,7 @@ describe('nexus', function () { }); it('should un-publish the nvmf nexus device', (done) => { - client.unpublishNexus({ uuid: UUID }, (err, res) => { - if (err) done(err); - done(); - }); + client.unpublishNexus({ uuid: UUID }, done); }); }); // End of describe('nvmf datapath')