feat(nexus): implement set ANA state for an NVMf-published Nexus

Implement set_ana_state in nexus_bdev::Nexus and expose that over
gRPC and the client CLI. The states supported by SPDK are
optimized, non_optimized and inaccessible. Add a cargo test to ensure
the state change is picked up by the NVMe initiator.

Fix the wording of nvmf::Error::Subsystem as it has become more widely
used, not just for creates.

mayastor/src/bdev/nexus/nexus_bdev.rs

@@ -13,6 +13,7 @@ use std::{
 
 use futures::{channel::oneshot, future::join_all};
 use nix::errno::Errno;
+use rpc::mayastor::NvmeAnaState;
 use serde::Serialize;
 use snafu::{ResultExt, Snafu};
 use tonic::{Code, Status};
@@ -106,6 +107,8 @@ pub enum Error {
     AlreadyShared { name: String },
     #[snafu(display("The nexus {} has not been shared", name))]
     NotShared { name: String },
+    #[snafu(display("The nexus {} has not been shared over NVMf", name))]
+    NotSharedNvmf { name: String },
     #[snafu(display("Failed to share nexus over NBD {}", name))]
     ShareNbdNexus { source: NbdError, name: String },
     #[snafu(display("Failed to share iscsi nexus {}", name))]
@@ -233,6 +236,8 @@ pub enum Error {
     },
     #[snafu(display("Invalid ShareProtocol value {}", sp_value))]
     InvalidShareProtocol { sp_value: i32 },
+    #[snafu(display("Invalid NvmeAnaState value {}", ana_value))]
+    InvalidNvmeAnaState { ana_value: i32 },
     #[snafu(display("Failed to create nexus {}", name))]
     NexusCreate { name: String },
     #[snafu(display("Failed to destroy nexus {}", name))]
@@ -252,6 +257,16 @@ pub enum Error {
     FailedGetHandle,
     #[snafu(display("Failed to create snapshot on nexus {}", name))]
     FailedCreateSnapshot { name: String, source: CoreError },
+    #[snafu(display("NVMf subsystem error: {}", e))]
+    SubsysNvmfError { e: String },
+}
+
+impl From<subsys::NvmfError> for Error {
+    fn from(error: subsys::NvmfError) -> Self {
+        Error::SubsysNvmfError {
+            e: error.to_string(),
+        }
+    }
 }
 
 impl From<Error> for tonic::Status {
@@ -272,6 +287,9 @@ impl From<Error> for tonic::Status {
             Error::NotShared {
                 ..
             } => Status::invalid_argument(e.to_string()),
+            Error::NotSharedNvmf {
+                ..
+            } => Status::invalid_argument(e.to_string()),
             Error::CreateChild {
                 ..
             } => Status::invalid_argument(e.to_string()),
@@ -630,6 +648,25 @@ impl Nexus {
         Ok(())
     }
 
+    /// set ANA state of the NVMe subsystem
+    pub async fn set_ana_state(
+        &self,
+        ana_state: NvmeAnaState,
+    ) -> Result<(), Error> {
+        if let Some(Protocol::Nvmf) = self.shared() {
+            if let Some(subsystem) = NvmfSubsystem::nqn_lookup(&self.name) {
+                subsystem.pause().await.unwrap();
+                let res = subsystem.set_ana_state(ana_state as u32).await;
+                subsystem.resume().await.unwrap();
+                return Ok(res?);
+            }
+        }
+
+        Err(Error::NotSharedNvmf {
+            name: self.name.clone(),
+        })
+    }
+
     /// register the bdev with SPDK and set the callbacks for io channel
     /// creation. Once this function is called, the device is visible and can
     /// be used for IO.

mayastor/src/bin/mayastor-client/nexus_cli.rs

@@ -54,6 +54,13 @@ pub fn subcommands<'a, 'b>() -> App<'a, 'b> {
                 .help("uuid for the nexus"),
         );
 
+    let set_ana_state = SubCommand::with_name("set_ana_state")
+        .about("set the NVMe ANA state of the nexus")
+        .arg(Arg::with_name("uuid").required(true).index(1)
+            .help("uuid for the nexus"))
+        .arg(Arg::with_name("ana_state").required(true).index(2)
+            .help("NVMe ANA state (optimized,non_optimized,inaccessible) of the nexus"));
+
     let add = SubCommand::with_name("add")
         .about("add a child")
         .arg(
@@ -122,6 +129,7 @@ pub fn subcommands<'a, 'b>() -> App<'a, 'b> {
         .subcommand(add)
         .subcommand(remove)
         .subcommand(unpublish)
+        .subcommand(set_ana_state)
         .subcommand(list)
         .subcommand(children)
         .subcommand(nexus_child_cli::subcommands())
@@ -138,6 +146,9 @@ pub async fn handler(
         ("children", Some(args)) => nexus_children(ctx, &args).await,
         ("publish", Some(args)) => nexus_publish(ctx, &args).await,
         ("unpublish", Some(args)) => nexus_unpublish(ctx, &args).await,
+        ("set_ana_state", Some(args)) => {
+            nexus_set_nvme_ana_state(ctx, &args).await
+        }
         ("add", Some(args)) => nexus_add(ctx, &args).await,
         ("remove", Some(args)) => nexus_remove(ctx, &args).await,
         ("child", Some(args)) => nexus_child_cli::handler(ctx, args).await,
@@ -321,6 +332,40 @@ async fn nexus_unpublish(
     Ok(())
 }
 
+async fn nexus_set_nvme_ana_state(
+    mut ctx: Context,
+    matches: &ArgMatches<'_>,
+) -> Result<(), Status> {
+    let uuid = matches.value_of("uuid").unwrap().to_string();
+    let ana_state = match matches.value_of("ana_state").unwrap() {
+        "optimized" => rpc::NvmeAnaState::NvmeAnaOptimizedState,
+        "non_optimized" => rpc::NvmeAnaState::NvmeAnaNonOptimizedState,
+        "inaccessible" => rpc::NvmeAnaState::NvmeAnaInaccessibleState,
+        _ => {
+            return Err(Status::new(
+                Code::Internal,
+                "Invalid value of NVMe ANA state".to_owned(),
+            ));
+        }
+    };
+
+    ctx.v2(&format!(
+        "Setting NVMe ANA state for nexus {} to {:?}",
+        uuid, ana_state
+    ));
+    ctx.client
+        .set_nvme_ana_state(rpc::SetNvmeAnaStateRequest {
+            uuid: uuid.clone(),
+            ana_state: ana_state.into(),
+        })
+        .await?;
+    ctx.v1(&format!(
+        "Set NVMe ANA state for nexus {} to {:?}",
+        uuid, ana_state
+    ));
+    Ok(())
+}
+
 async fn nexus_add(
     mut ctx: Context,
     matches: &ArgMatches<'_>,

mayastor/src/grpc/mayastor_grpc.rs

@@ -289,6 +289,33 @@ impl mayastor_server::Mayastor for MayastorSvc {
         .await
     }
 
+    #[instrument(level = "debug", err)]
+    async fn set_nvme_ana_state(
+        &self,
+        request: Request<SetNvmeAnaStateRequest>,
+    ) -> GrpcResult<Null> {
+        let args = request.into_inner();
+        let uuid = args.uuid.clone();
+        debug!("Setting NVMe ANA state for nexus {} ...", uuid);
+
+        let ana_state = match NvmeAnaState::from_i32(args.ana_state) {
+            Some(ana_state) => ana_state,
+            None => {
+                return Err(nexus_bdev::Error::InvalidNvmeAnaState {
+                    ana_value: args.ana_state as i32,
+                }
+                .into());
+            }
+        };
+
+        locally! { async move {
+            nexus_lookup(&args.uuid)?.set_ana_state(ana_state).await
+        }};
+
+        info!("Set nexus {} NVMe ANA state {:?}", uuid, ana_state);
+        Ok(Response::new(Null {}))
+    }
+
     #[instrument(level = "debug", err)]
     async fn child_operation(
         &self,

mayastor/src/subsys/nvmf/mod.rs

@@ -63,15 +63,15 @@ pub enum Error {
     PgError { msg: String },
     #[snafu(display("Failed to create transport {}", msg))]
     Transport { source: Errno, msg: String },
-    #[snafu(display("Failed to create subsystem for {} {} error: {}", source.desc(), nqn, msg))]
+    #[snafu(display("Failed nvmf subsystem operation for {} {} error: {}", source.desc(), nqn, msg))]
     Subsystem {
         source: Errno,
         nqn: String,
         msg: String,
     },
-    #[snafu(display("Failed to create share for  {} {}", bdev, msg))]
+    #[snafu(display("Failed to create share for {} {}", bdev, msg))]
     Share { bdev: String, msg: String },
-    #[snafu(display("Failed to add namespace for  {} {}", bdev, msg))]
+    #[snafu(display("Failed to add namespace for {} {}", bdev, msg))]
     Namespace { bdev: String, msg: String },
 }
 

mayastor/src/subsys/nvmf/subsystem.rs

@@ -12,6 +12,7 @@ use futures::channel::oneshot;
 use nix::errno::Errno;
 
 use spdk_sys::{
+    nvmf_subsystem_set_ana_state,
     spdk_bdev_nvme_opts,
     spdk_nvmf_ns_get_bdev,
     spdk_nvmf_ns_opts,
@@ -471,6 +472,35 @@ impl NvmfSubsystem {
         }
     }
 
+    /// set ANA state: optimized, non_optimized, inaccessible
+    /// subsystem must be in paused or inactive state
+    pub async fn set_ana_state(&self, ana_state: u32) -> Result<(), Error> {
+        extern "C" fn set_ana_state_cb(arg: *mut c_void, status: i32) {
+            let s = unsafe { Box::from_raw(arg as *mut oneshot::Sender<i32>) };
+            s.send(status).unwrap();
+        }
+        let cfg = Config::get();
+        let trid_replica = TransportId::new(cfg.nexus_opts.nvmf_replica_port);
+
+        let (s, r) = oneshot::channel::<i32>();
+
+        unsafe {
+            nvmf_subsystem_set_ana_state(
+                self.0.as_ptr(),
+                trid_replica.as_ptr(),
+                ana_state,
+                Some(set_ana_state_cb),
+                cb_arg(s),
+            );
+        }
+
+        r.await.unwrap().to_result(|e| Error::Subsystem {
+            source: Errno::from_i32(e),
+            nqn: self.get_nqn(),
+            msg: "failed to set_ana_state of the subsystem".to_string(),
+        })
+    }
+
     /// destroy all subsystems associated with our target, subsystems must be in
     /// stopped state
     pub fn destroy_all() {

mayastor/tests/nexus_multipath.rs

@@ -8,6 +8,7 @@ use rpc::mayastor::{
     CreateNexusRequest,
     CreatePoolRequest,
     CreateReplicaRequest,
+    NvmeAnaState,
     PublishNexusRequest,
     ShareProtocolNexus,
 };
@@ -71,19 +72,20 @@ async fn nexus_multipath() {
     let mayastor = MayastorTest::new(MayastorCliArgs::default());
     let ip0 = hdls[0].endpoint.ip();
     let nexus_name = format!("nexus-{}", UUID);
+    let name = nexus_name.clone();
     mayastor
         .spawn(async move {
             // create nexus on local node with remote replica as child
             nexus_create(
-                &nexus_name,
+                &name,
                 32 * 1024 * 1024,
                 Some(UUID),
                 &[format!("nvmf://{}:8420/{}:{}", ip0, HOSTNQN, UUID)],
             )
             .await
             .unwrap();
             // publish nexus on local node over nvmf
-            nexus_lookup(&nexus_name)
+            nexus_lookup(&name)
                 .unwrap()
                 .share(ShareProtocolNexus::NexusNvmf, None)
                 .await
@@ -138,6 +140,51 @@ async fn nexus_multipath() {
         );
     }
 
+    let output_list = Command::new("nvme").args(&["list"]).output().unwrap();
+    assert!(
+        output_list.status.success(),
+        "failed to list nvme devices, {}",
+        output_list.status
+    );
+    let sl = String::from_utf8(output_list.stdout).unwrap();
+    let nvmems: Vec<&str> = sl
+        .lines()
+        .filter(|line| line.contains("Mayastor NVMe controller"))
+        .collect();
+    assert_eq!(nvmems.len(), 1);
+    let ns = nvmems[0].split(' ').collect::<Vec<_>>()[0];
+
+    mayastor
+        .spawn(async move {
+            // set nexus on local node ANA state to non-optimized
+            nexus_lookup(&nexus_name)
+                .unwrap()
+                .set_ana_state(NvmeAnaState::NvmeAnaNonOptimizedState)
+                .await
+                .unwrap();
+        })
+        .await;
+
+    //  +- nvme0 tcp traddr=127.0.0.1 trsvcid=8420 live <ana_state>
+    let output_subsys = Command::new("nvme")
+        .args(&["list-subsys"])
+        .args(&[ns])
+        .output()
+        .unwrap();
+    assert!(
+        output_subsys.status.success(),
+        "failed to list nvme subsystem, {}",
+        output_subsys.status
+    );
+    let subsys = String::from_utf8(output_subsys.stdout).unwrap();
+    let nvmec: Vec<&str> = subsys
+        .lines()
+        .filter(|line| line.contains("traddr=127.0.0.1"))
+        .collect();
+    assert_eq!(nvmec.len(), 1);
+    let nv: Vec<&str> = nvmec[0].split(' ').collect();
+    assert_eq!(nv[7], "non-optimized", "incorrect ANA state");
+
     // NQN:<nqn> disconnected 2 controller(s)
     let output_dis = Command::new("nvme")
         .args(&["disconnect"])
@@ -152,11 +199,8 @@ async fn nexus_multipath() {
     let s = String::from_utf8(output_dis.stdout).unwrap();
     let v: Vec<&str> = s.split(' ').collect();
     tracing::info!("nvme disconnected: {:?}", v);
-    assert!(v.len() == 4);
-    assert!(v[1] == "disconnected");
-    assert!(
-        v[0] == format!("NQN:{}", &nqn),
-        "mismatched NQN disconnected"
-    );
-    assert!(v[2] == "2", "mismatched number of controllers disconnected");
+    assert_eq!(v.len(), 4);
+    assert_eq!(v[1], "disconnected");
+    assert_eq!(v[0], format!("NQN:{}", &nqn), "mismatched NQN disconnected");
+    assert_eq!(v[2], "2", "mismatched number of controllers disconnected");
 }

rpc/proto/mayastor.proto

@@ -51,6 +51,9 @@ service Mayastor {
   rpc PublishNexus (PublishNexusRequest) returns (PublishNexusReply) {}
   rpc UnpublishNexus (UnpublishNexusRequest) returns (Null) {}
 
+  // NVMe ANA state
+  rpc SetNvmeAnaState (SetNvmeAnaStateRequest) returns (Null) {}
+
   // Nexus child operations
   rpc ChildOperation(ChildNexusRequest) returns (Null) {}
 
@@ -273,6 +276,20 @@ message UnpublishNexusRequest {
   string uuid = 1;   // uuid of the nexus which to destroy
 }
 
+enum NvmeAnaState {
+  NVME_ANA_INVALID_STATE = 0;           // invalid, do not use
+  NVME_ANA_OPTIMIZED_STATE = 0x1;
+  NVME_ANA_NON_OPTIMIZED_STATE = 0x2;
+  NVME_ANA_INACCESSIBLE_STATE = 0x3;
+  NVME_ANA_PERSISTENT_LOSS_STATE = 0x4; // not yet supported
+  NVME_ANA_CHANGE_STATE = 0xF;          // not yet supported
+}
+
+message SetNvmeAnaStateRequest {
+  string uuid = 1;   // uuid of the nexus
+  NvmeAnaState ana_state = 2;
+}
+
 enum ChildAction {
   offline = 0;
   online = 1;

spdk-sys/build.rs

@@ -84,6 +84,7 @@ fn main() {
         .whitelist_function("^vbdev_.*")
         .whitelist_function("^nvme_cmd_.*")
         .whitelist_function("^nvme_status_.*")
+        .whitelist_function("^nvmf_subsystem_set_ana_state")
         .whitelist_function("^nvmf_tgt_accept")
         .blacklist_type("^longfunc")
         .whitelist_var("^NVMF.*")