Merge #1759

1759: test: add a test to enable and test rdma connect r=dsharma-dc a=dsharma-dc

- Added a simple test to create and publish a nexus. Connect it over rdma.
- The connect API is modified to take transport parameter.
- The test automatically detects a suitable interface to create a soft-RDMA device, creates it and deletes it at the end of the test.

Co-authored-by: Diwakar Sharma <[email protected]>

io-engine-tests/src/lib.rs

@@ -8,6 +8,7 @@ use std::{io, io::Write, process::Command, time::Duration};
 
 use crossbeam::channel::{after, select, unbounded};
 use once_cell::sync::OnceCell;
+use regex::Regex;
 use run_script::{self, ScriptOptions};
 use url::{ParseError, Url};
 
@@ -36,6 +37,8 @@ pub mod snapshot;
 pub mod test;
 pub mod test_task;
 
+use cli_tools::run_command_args;
+
 pub use compose::MayastorTest;
 
 /// call F cnt times, and sleep for a duration between each invocation
@@ -550,6 +553,86 @@ pub fn reactor_run_millis(milliseconds: u64) {
     reactor_poll!(r);
 }
 
+// We try to figure out the net interface that is being used to connect
+// to internet(public IP). Then use that interface name to setup rdma rxe
+// device.
+//
+// # host we want to "reach"
+// host=google.com
+//
+// # get the ip of that host (works with dns and /etc/hosts. In case we get
+// # multiple IP addresses, we just want one of them
+// host_ip=$(getent ahosts "$host" | awk '{print $1; exit}')
+//
+// # only list the interface used to reach a specific host/IP. We only want the
+// part # between dev and the remainder (use grep for that)
+// ip route get "$host_ip" | grep -Po '(?<=(dev ))(\S+)'
+pub fn setup_rdma_rxe_device() -> String {
+    let test_host = "google.com";
+    let ns_entries = run_command_args(
+        "getent",
+        vec!["ahosts", test_host],
+        Some("get ip of test host"),
+    )
+    .unwrap();
+
+    let test_host_ip = ns_entries
+        .1
+        .first()
+        .unwrap()
+        .to_string_lossy()
+        .split_whitespace()
+        .next()
+        .unwrap()
+        .to_owned();
+
+    let ent = &run_command_args(
+        "ip",
+        vec!["route", "get", test_host_ip.as_str()],
+        Some("get routing entry"),
+    )
+    .unwrap()
+    .1[0];
+
+    // match/grep the interface name
+    let pattern = r"dev (\S+)";
+    let re = Regex::new(pattern).unwrap();
+
+    let iface = re
+        .captures(ent.to_str().unwrap())
+        .unwrap()
+        .get(1)
+        .expect("interface not found")
+        .as_str();
+    println!("Using interface {} to create rdma rxe device", iface);
+
+    // now create the software rdma device.
+    let _ = run_command_args(
+        "rdma",
+        vec![
+            "link",
+            "add",
+            "io-engine-rxe0",
+            "type",
+            "rxe",
+            "netdev",
+            iface,
+        ],
+        Some("Create rxe device"),
+    );
+
+    iface.to_string()
+}
+
+pub fn delete_rdma_rxe_device() {
+    let _ = run_command_args(
+        "rdma",
+        vec!["link", "delete", "io-engine-rxe0"],
+        Some("Delete rxe device"),
+    )
+    .expect("rxe device delete");
+}
+
 pub fn composer_init() {
     std::fs::create_dir_all("/var/run/dpdk").ok();
     let path = std::path::PathBuf::from(std::env!("CARGO_MANIFEST_DIR"));

io-engine-tests/src/nvme.rs

@@ -15,7 +15,7 @@ pub struct NmveConnectGuard {
 
 impl NmveConnectGuard {
     pub fn connect(target_addr: &str, nqn: &str) -> Self {
-        nvme_connect(target_addr, nqn, true);
+        nvme_connect(target_addr, nqn, "tcp", true);
 
         Self {
             nqn: nqn.to_string(),
@@ -85,11 +85,12 @@ pub fn nvme_discover(target_addr: &str) -> Vec<BTreeMap<String, String>> {
 pub fn nvme_connect(
     target_addr: &str,
     nqn: &str,
+    transport: &str,
     must_succeed: bool,
 ) -> ExitStatus {
     let status = Command::new("nvme")
         .args(["connect"])
-        .args(["-t", "tcp"])
+        .args(["-t", transport])
         .args(["-a", target_addr])
         .args(["-s", "8420"])
         .args(["-c", "1"])

io-engine/tests/nexus_io.rs

@@ -200,12 +200,12 @@ async fn nexus_io_multipath() {
         .unwrap();
 
     let nqn = format!("{HOSTNQN}:nexus-{NEXUS_UUID}");
-    nvme_connect("127.0.0.1", &nqn, true);
+    nvme_connect("127.0.0.1", &nqn, "tcp", true);
 
     // The first attempt will fail with "Duplicate cntlid x with y" error from
     // kernel
     for i in 0 .. 2 {
-        let status_c0 = nvme_connect(&ip0.to_string(), &nqn, false);
+        let status_c0 = nvme_connect(&ip0.to_string(), &nqn, "tcp", false);
         if i == 0 && status_c0.success() {
             break;
         }
@@ -270,7 +270,7 @@ async fn nexus_io_multipath() {
 
     // Connect to remote replica to check key registered
     let rep_nqn = format!("{HOSTNQN}:{REPL_UUID}");
-    nvme_connect(&ip0.to_string(), &rep_nqn, true);
+    nvme_connect(&ip0.to_string(), &rep_nqn, "tcp", true);
 
     let rep_dev = get_mayastor_nvme_device();
 
@@ -404,7 +404,7 @@ async fn nexus_io_resv_acquire() {
 
     // Connect to remote replica to check key registered
     let rep_nqn = format!("{HOSTNQN}:{REPL_UUID}");
-    nvme_connect(&ip0.to_string(), &rep_nqn, true);
+    nvme_connect(&ip0.to_string(), &rep_nqn, "tcp", true);
 
     let rep_dev = get_mayastor_nvme_device();
 
@@ -601,7 +601,7 @@ async fn nexus_io_resv_preempt() {
     // Connect to remote replica to check key registered
     let rep_nqn = format!("{HOSTNQN}:{REPL_UUID}");
 
-    nvme_connect(&ip0.to_string(), &rep_nqn, true);
+    nvme_connect(&ip0.to_string(), &rep_nqn, "tcp", true);
 
     let rep_dev = get_mayastor_nvme_device();
 
@@ -748,7 +748,7 @@ async fn nexus_io_resv_preempt() {
         .await
         .unwrap();
 
-    nvme_connect(&ip0.to_string(), &rep_nqn, true);
+    nvme_connect(&ip0.to_string(), &rep_nqn, "tcp", true);
     let rep_dev = get_mayastor_nvme_device();
 
     // After restart the reservations should still be in place!
@@ -899,7 +899,7 @@ async fn nexus_io_resv_preempt_tabled() {
         // Connect to remote replica to check key registered
         let rep_nqn = format!("{HOSTNQN}:{REPL_UUID}");
 
-        nvme_connect(&ip0.to_string(), &rep_nqn, true);
+        nvme_connect(&ip0.to_string(), &rep_nqn, "tcp", true);
 
         let rep_dev = get_mayastor_nvme_device();
 

io-engine/tests/nvmf.rs

@@ -1,5 +1,6 @@
 use io_engine::{
     bdev_api::bdev_create,
+    constants::NVME_NQN_PREFIX,
     core::{
         mayastor_env_stop,
         MayastorCliArgs,
@@ -10,21 +11,79 @@ use io_engine::{
     subsys::{NvmfSubsystem, SubType},
 };
 
+use io_engine_tests::{delete_rdma_rxe_device, setup_rdma_rxe_device};
+
 pub mod common;
-use common::compose::{
-    rpc::v0::{
-        mayastor::{BdevShareRequest, BdevUri, CreateReply},
-        GrpcConnect,
+use common::{
+    compose::{
+        rpc::{
+            v0::{
+                mayastor::{
+                    BdevShareRequest,
+                    BdevUri,
+                    CreateReply,
+                    ShareProtocolNexus,
+                },
+                GrpcConnect,
+            },
+            v1::{
+                nexus::{CreateNexusRequest, PublishNexusRequest},
+                pool::CreatePoolRequest,
+                replica::CreateReplicaRequest,
+                GrpcConnect as v1GrpcConnect,
+                RpcHandle,
+            },
+        },
+        Binary,
+        Builder,
+        ComposeTest,
+        NetworkMode,
     },
-    Binary,
-    Builder,
-    ComposeTest,
+    nvme::{nvme_connect, nvme_disconnect_nqn},
 };
 use regex::Regex;
 
 static DISKNAME1: &str = "/tmp/disk1.img";
 static BDEVNAME1: &str = "aio:///tmp/disk1.img?blk_size=512";
 
+fn nexus_uuid() -> String {
+    "cdc2a7db-3ac3-403a-af80-7fadc1581c47".to_string()
+}
+fn nexus_name() -> String {
+    "nexus0".to_string()
+}
+fn repl_uuid() -> String {
+    "65acdaac-14c4-41d8-a55e-d03bfd7185a4".to_string()
+}
+fn repl_name() -> String {
+    "repl0".to_string()
+}
+fn pool_uuid() -> String {
+    "6e3c062c-293b-46e6-8ab3-ff13c1643437".to_string()
+}
+fn pool_name() -> String {
+    "tpool".to_string()
+}
+
+pub async fn create_nexus(h: &mut RpcHandle, children: Vec<String>) {
+    h.nexus
+        .create_nexus(CreateNexusRequest {
+            name: nexus_name(),
+            uuid: nexus_uuid(),
+            size: 60 * 1024 * 1024,
+            min_cntl_id: 1,
+            max_cntl_id: 1,
+            resv_key: 1,
+            preempt_key: 0,
+            children,
+            nexus_info_key: nexus_name(),
+            resv_type: None,
+            preempt_policy: 0,
+        })
+        .await
+        .unwrap();
+}
+
 #[common::spdk_test]
 fn nvmf_target() {
     common::mayastor_test_init();
@@ -195,3 +254,90 @@ async fn nvmf_set_target_interface() {
     // test_fail("10.15.0.0/16", vec!["-T", "mac:123"]).await;
     // test_fail("10.15.0.0/16", vec!["-T", "ip:hello"]).await;
 }
+
+#[tokio::test]
+#[ignore]
+async fn test_rdma_target() {
+    common::composer_init();
+
+    let iface = setup_rdma_rxe_device();
+    let test = Builder::new()
+        .name("cargo-test")
+        .network_mode(NetworkMode::Host)
+        .add_container_bin(
+            "ms_0",
+            Binary::from_dbg("io-engine")
+                .with_args(vec![
+                    "-l",
+                    "1,2",
+                    "--enable-rdma",
+                    "-T",
+                    iface.as_str(),
+                ])
+                .with_privileged(Some(true)),
+        )
+        .with_clean(true)
+        .build()
+        .await
+        .unwrap();
+
+    let conn = v1GrpcConnect::new(&test);
+    let mut hdl = conn.grpc_handle("ms_0").await.unwrap();
+    println!("ms_0 grpc endpoint {:?}", hdl.endpoint);
+
+    hdl.pool
+        .create_pool(CreatePoolRequest {
+            name: pool_name(),
+            uuid: Some(pool_uuid()),
+            pooltype: 0,
+            disks: vec!["malloc:///disk0?size_mb=100".into()],
+            cluster_size: None,
+            md_args: None,
+        })
+        .await
+        .unwrap();
+
+    hdl.replica
+        .create_replica(CreateReplicaRequest {
+            name: repl_name(),
+            uuid: repl_uuid(),
+            pooluuid: pool_uuid(),
+            size: 80 * 1024 * 1024,
+            thin: false,
+            share: 1,
+            ..Default::default()
+        })
+        .await
+        .unwrap();
+
+    let child0 = format!("bdev:///{}", repl_name());
+    create_nexus(&mut hdl, vec![child0.clone()]).await;
+    let device_uri = hdl
+        .nexus
+        .publish_nexus(PublishNexusRequest {
+            uuid: nexus_uuid(),
+            key: "".to_string(),
+            share: ShareProtocolNexus::NexusNvmf as i32,
+            ..Default::default()
+        })
+        .await
+        .expect("Failed to publish nexus")
+        .into_inner()
+        .nexus
+        .unwrap()
+        .device_uri;
+
+    let url = url::Url::parse(device_uri.as_str()).unwrap();
+    assert!(url.scheme() == "nvmf+rdma+tcp");
+
+    let host = url.host_str().unwrap();
+    let nqn = format!("{NVME_NQN_PREFIX}:{}", nexus_name());
+    let conn_status = nvme_connect(host, &nqn, "rdma", true);
+    assert!(conn_status.success());
+
+    nvme_disconnect_nqn(&nqn);
+    // Explicitly destroy this test's containers so that rxe device can be
+    // deleted.
+    test.down().await;
+    delete_rdma_rxe_device();
+}

io-engine/tests/snapshot_nexus.rs

@@ -680,7 +680,7 @@ async fn test_snapshot_ancestor_usage() {
      * should now own all new data.
      */
     let nqn = format!("{NVME_NQN_PREFIX}:{}", nexus_name());
-    nvme_connect("127.0.0.1", &nqn, true);
+    nvme_connect("127.0.0.1", &nqn, "tcp", true);
 
     let (s, r) = oneshot::channel::<()>();
     tokio::spawn(async move {

scripts/cargo-test.sh

@@ -19,6 +19,17 @@ trap cleanup_handler INT QUIT TERM HUP EXIT
 export PATH=$PATH:${HOME}/.cargo/bin
 set -euxo pipefail
 
+# Warn if rdma-rxe and nvme-rdme kernel modules are not
+# available. Absence of rdma-rxe can be ignored on hardware
+# RDMA setups.
+if ! lsmod | grep -q rdma_rxe; then
+  echo "Warning: rdma_rxe kernel module is not loaded. Please load it for rdma tests to work."
+fi
+
+if ! lsmod | grep -q nvme_rdma; then
+  echo "Warning: nvme_rdma kernel module is not loaded. Please load it for rdma tests to work."
+fi
+
 ( cd jsonrpc && cargo test )
 # test dependencies
 cargo build --bins --features=io-engine-testing