feat(nexus): implement get ANA state for an NVMf-published Nexus

Implement the counterpart to the set, also exposed over gRPC and CLI.
Verify the state had been set in the gRPC test.

Address review comments, implementing FromStr for NvmeRpcState in the
rpc crate and fix error paths.

mayastor/src/bdev/nexus/nexus_bdev.rs

@@ -648,16 +648,34 @@ impl Nexus {
         Ok(())
     }
 
+    /// get ANA state of the NVMe subsystem
+    pub async fn get_ana_state(&self) -> Result<NvmeAnaState, Error> {
+        if let Some(Protocol::Nvmf) = self.shared() {
+            if let Some(subsystem) = NvmfSubsystem::nqn_lookup(&self.name) {
+                let ana_state = subsystem.get_ana_state().await? as i32;
+                return NvmeAnaState::from_i32(ana_state).ok_or({
+                    Error::InvalidNvmeAnaState {
+                        ana_value: ana_state,
+                    }
+                });
+            }
+        }
+
+        Err(Error::NotSharedNvmf {
+            name: self.name.clone(),
+        })
+    }
+
     /// set ANA state of the NVMe subsystem
     pub async fn set_ana_state(
         &self,
         ana_state: NvmeAnaState,
     ) -> Result<(), Error> {
         if let Some(Protocol::Nvmf) = self.shared() {
             if let Some(subsystem) = NvmfSubsystem::nqn_lookup(&self.name) {
-                subsystem.pause().await.unwrap();
+                subsystem.pause().await?;
                 let res = subsystem.set_ana_state(ana_state as u32).await;
-                subsystem.resume().await.unwrap();
+                subsystem.resume().await?;
                 return Ok(res?);
             }
         }

mayastor/src/bin/mayastor-client/nexus_cli.rs

@@ -54,12 +54,23 @@ pub fn subcommands<'a, 'b>() -> App<'a, 'b> {
                 .help("uuid for the nexus"),
         );
 
-    let set_ana_state = SubCommand::with_name("set_ana_state")
-        .about("set the NVMe ANA state of the nexus")
-        .arg(Arg::with_name("uuid").required(true).index(1)
-            .help("uuid for the nexus"))
-        .arg(Arg::with_name("ana_state").required(true).index(2)
-            .help("NVMe ANA state (optimized,non_optimized,inaccessible) of the nexus"));
+    let ana_state = SubCommand::with_name("ana_state")
+        .about("get or set the NVMe ANA state of the nexus")
+        .arg(
+            Arg::with_name("uuid")
+                .required(true)
+                .index(1)
+                .help("uuid for the nexus"),
+        )
+        .arg(
+            Arg::with_name("state")
+                .required(false)
+                .index(2)
+                .possible_value("optimized")
+                .possible_value("non_optimized")
+                .possible_value("inaccessible")
+                .help("NVMe ANA state of the nexus"),
+        );
 
     let add = SubCommand::with_name("add")
         .about("add a child")
@@ -129,7 +140,7 @@ pub fn subcommands<'a, 'b>() -> App<'a, 'b> {
         .subcommand(add)
         .subcommand(remove)
         .subcommand(unpublish)
-        .subcommand(set_ana_state)
+        .subcommand(ana_state)
         .subcommand(list)
         .subcommand(children)
         .subcommand(nexus_child_cli::subcommands())
@@ -146,9 +157,7 @@ pub async fn handler(
         ("children", Some(args)) => nexus_children(ctx, &args).await,
         ("publish", Some(args)) => nexus_publish(ctx, &args).await,
         ("unpublish", Some(args)) => nexus_unpublish(ctx, &args).await,
-        ("set_ana_state", Some(args)) => {
-            nexus_set_nvme_ana_state(ctx, &args).await
-        }
+        ("ana_state", Some(args)) => nexus_nvme_ana_state(ctx, &args).await,
         ("add", Some(args)) => nexus_add(ctx, &args).await,
         ("remove", Some(args)) => nexus_remove(ctx, &args).await,
         ("child", Some(args)) => nexus_child_cli::handler(ctx, args).await,
@@ -332,15 +341,41 @@ async fn nexus_unpublish(
     Ok(())
 }
 
-async fn nexus_set_nvme_ana_state(
-    mut ctx: Context,
+async fn nexus_nvme_ana_state(
+    ctx: Context,
     matches: &ArgMatches<'_>,
 ) -> Result<(), Status> {
     let uuid = matches.value_of("uuid").unwrap().to_string();
-    let ana_state = match matches.value_of("ana_state").unwrap() {
-        "optimized" => rpc::NvmeAnaState::NvmeAnaOptimizedState,
-        "non_optimized" => rpc::NvmeAnaState::NvmeAnaNonOptimizedState,
-        "inaccessible" => rpc::NvmeAnaState::NvmeAnaInaccessibleState,
+    let ana_state = matches.value_of("state").unwrap_or("").to_string();
+    if ana_state.is_empty() {
+        nexus_get_nvme_ana_state(ctx, uuid).await
+    } else {
+        nexus_set_nvme_ana_state(ctx, uuid, ana_state).await
+    }
+}
+
+async fn nexus_get_nvme_ana_state(
+    mut ctx: Context,
+    uuid: String,
+) -> Result<(), Status> {
+    ctx.v2(&format!("Getting NVMe ANA state for nexus {}", uuid));
+    let resp = ctx
+        .client
+        .get_nvme_ana_state(rpc::GetNvmeAnaStateRequest {
+            uuid: uuid.clone(),
+        })
+        .await?;
+    ctx.v1(ana_state_idx_to_str(resp.get_ref().ana_state));
+    Ok(())
+}
+
+async fn nexus_set_nvme_ana_state(
+    mut ctx: Context,
+    uuid: String,
+    ana_state_str: String,
+) -> Result<(), Status> {
+    let ana_state: rpc::NvmeAnaState = match ana_state_str.parse() {
+        Ok(a) => a,
         _ => {
             return Err(Status::new(
                 Code::Internal,
@@ -408,6 +443,17 @@ async fn nexus_remove(
     Ok(())
 }
 
+fn ana_state_idx_to_str(idx: i32) -> &'static str {
+    match rpc::NvmeAnaState::from_i32(idx).unwrap() {
+        rpc::NvmeAnaState::NvmeAnaInvalidState => "invalid",
+        rpc::NvmeAnaState::NvmeAnaOptimizedState => "optimized",
+        rpc::NvmeAnaState::NvmeAnaNonOptimizedState => "non_optimized",
+        rpc::NvmeAnaState::NvmeAnaInaccessibleState => "inaccessible",
+        rpc::NvmeAnaState::NvmeAnaPersistentLossState => "persistent_loss",
+        rpc::NvmeAnaState::NvmeAnaChangeState => "change",
+    }
+}
+
 fn nexus_state_to_str(idx: i32) -> &'static str {
     match rpc::NexusState::from_i32(idx).unwrap() {
         rpc::NexusState::NexusUnknown => "unknown",

mayastor/src/grpc/mayastor_grpc.rs

@@ -289,6 +289,25 @@ impl mayastor_server::Mayastor for MayastorSvc {
         .await
     }
 
+    #[instrument(level = "debug", err)]
+    async fn get_nvme_ana_state(
+        &self,
+        request: Request<GetNvmeAnaStateRequest>,
+    ) -> GrpcResult<GetNvmeAnaStateReply> {
+        let args = request.into_inner();
+        let uuid = args.uuid.clone();
+        debug!("Getting NVMe ANA state for nexus {} ...", uuid);
+
+        let ana_state = locally! { async move {
+            nexus_lookup(&args.uuid)?.get_ana_state().await
+        }};
+
+        info!("Got nexus {} NVMe ANA state {:?}", uuid, ana_state);
+        Ok(Response::new(GetNvmeAnaStateReply {
+            ana_state: ana_state as i32,
+        }))
+    }
+
     #[instrument(level = "debug", err)]
     async fn set_nvme_ana_state(
         &self,

mayastor/src/subsys/nvmf/mod.rs

@@ -73,6 +73,8 @@ pub enum Error {
     Share { bdev: String, msg: String },
     #[snafu(display("Failed to add namespace for {} {}", bdev, msg))]
     Namespace { bdev: String, msg: String },
+    #[snafu(display("Failed to find listener for {} {}", nqn, trid))]
+    Listener { nqn: String, trid: String },
 }
 
 thread_local! {

mayastor/src/subsys/nvmf/subsystem.rs

@@ -12,6 +12,7 @@ use futures::channel::oneshot;
 use nix::errno::Errno;
 
 use spdk_sys::{
+    nvmf_subsystem_find_listener,
     nvmf_subsystem_set_ana_state,
     spdk_bdev_nvme_opts,
     spdk_nvmf_ns_get_bdev,
@@ -472,6 +473,23 @@ impl NvmfSubsystem {
         }
     }
 
+    /// get ANA state
+    pub async fn get_ana_state(&self) -> Result<u32, Error> {
+        let cfg = Config::get();
+        let trid_replica = TransportId::new(cfg.nexus_opts.nvmf_replica_port);
+        let listener = unsafe {
+            nvmf_subsystem_find_listener(self.0.as_ptr(), trid_replica.as_ptr())
+        };
+        if listener.is_null() {
+            Err(Error::Listener {
+                nqn: self.get_nqn(),
+                trid: trid_replica.to_string(),
+            })
+        } else {
+            Ok(unsafe { (*listener).ana_state })
+        }
+    }
+
     /// set ANA state: optimized, non_optimized, inaccessible
     /// subsystem must be in paused or inactive state
     pub async fn set_ana_state(&self, ana_state: u32) -> Result<(), Error> {
@@ -494,11 +512,13 @@ impl NvmfSubsystem {
             );
         }
 
-        r.await.unwrap().to_result(|e| Error::Subsystem {
-            source: Errno::from_i32(e),
-            nqn: self.get_nqn(),
-            msg: "failed to set_ana_state of the subsystem".to_string(),
-        })
+        r.await
+            .expect("Cancellation is not supported")
+            .to_result(|e| Error::Subsystem {
+                source: Errno::from_i32(-e),
+                nqn: self.get_nqn(),
+                msg: "failed to set_ana_state of the subsystem".to_string(),
+            })
     }
 
     /// destroy all subsystems associated with our target, subsystems must be in

rpc/proto/mayastor.proto

@@ -52,6 +52,7 @@ service Mayastor {
   rpc UnpublishNexus (UnpublishNexusRequest) returns (Null) {}
 
   // NVMe ANA state
+  rpc GetNvmeAnaState (GetNvmeAnaStateRequest) returns (GetNvmeAnaStateReply) {}
   rpc SetNvmeAnaState (SetNvmeAnaStateRequest) returns (Null) {}
 
   // Nexus child operations
@@ -285,6 +286,14 @@ enum NvmeAnaState {
   NVME_ANA_CHANGE_STATE = 0xF;          // not yet supported
 }
 
+message GetNvmeAnaStateRequest {
+  string uuid = 1;   // uuid of the nexus
+}
+
+message GetNvmeAnaStateReply {
+  NvmeAnaState ana_state = 1;
+}
+
 message SetNvmeAnaStateRequest {
   string uuid = 1;   // uuid of the nexus
   NvmeAnaState ana_state = 2;

rpc/src/lib.rs

@@ -11,12 +11,30 @@ extern crate tonic;
 #[allow(clippy::redundant_closure)]
 #[allow(clippy::upper_case_acronyms)]
 pub mod mayastor {
+    use std::str::FromStr;
+
+    #[derive(Debug)]
+    pub enum Error {
+        ParseError,
+    }
 
     impl From<()> for Null {
         fn from(_: ()) -> Self {
             Self {}
         }
     }
 
+    impl FromStr for NvmeAnaState {
+        type Err = Error;
+        fn from_str(state: &str) -> Result<Self, Self::Err> {
+            match state {
+                "optimized" => Ok(Self::NvmeAnaOptimizedState),
+                "non_optimized" => Ok(Self::NvmeAnaNonOptimizedState),
+                "inaccessible" => Ok(Self::NvmeAnaInaccessibleState),
+                _ => Err(Error::ParseError),
+            }
+        }
+    }
+
     include!(concat!(env!("OUT_DIR"), "/mayastor.rs"));
 }

spdk-sys/build.rs

@@ -84,6 +84,7 @@ fn main() {
         .whitelist_function("^vbdev_.*")
         .whitelist_function("^nvme_cmd_.*")
         .whitelist_function("^nvme_status_.*")
+        .whitelist_function("^nvmf_subsystem_find_listener")
         .whitelist_function("^nvmf_subsystem_set_ana_state")
         .whitelist_function("^nvmf_tgt_accept")
         .blacklist_type("^longfunc")

test/grpc/test_nexus.js

@@ -691,26 +691,36 @@ describe('nexus', function () {
           uuid: UUID,
           ana_state: enums.NVME_ANA_INACCESSIBLE_STATE
         },
-        (err, res) => {
-          if (err) done(err);
-          done();
-        }
+        done
       );
     });
 
+    it('should get ANA state as inaccessible', (done) => {
+      client.getNvmeAnaState({ uuid: UUID }, (err, res) => {
+        if (err) done(err);
+        assert.equal(res.ana_state, 'NVME_ANA_INACCESSIBLE_STATE');
+        done();
+      });
+    });
+
     it('should change ANA state back to optimized', (done) => {
       client.setNvmeAnaState(
         {
           uuid: UUID,
           ana_state: enums.NVME_ANA_OPTIMIZED_STATE
         },
-        (err, res) => {
-          if (err) done(err);
-          done();
-        }
+        done
       );
     });
 
+    it('should get ANA state as optimized', (done) => {
+      client.getNvmeAnaState({ uuid: UUID }, (err, res) => {
+        if (err) done(err);
+        assert.equal(res.ana_state, 'NVME_ANA_OPTIMIZED_STATE');
+        done();
+      });
+    });
+
     it('should write to nvmf replica', (done) => {
       common.execAsRoot(
         common.getCmdPath('initiator'),
@@ -720,10 +730,7 @@ describe('nexus', function () {
     });
 
     it('should un-publish the nvmf nexus device', (done) => {
-      client.unpublishNexus({ uuid: UUID }, (err, res) => {
-        if (err) done(err);
-        done();
-      });
+      client.unpublishNexus({ uuid: UUID }, done);
     });
   }); // End of describe('nvmf datapath')