TEST/MD/IB: Add alignment to test_ib_md + add memh flags check to tes…

…t_ib_xfer

test/gtest/uct/ib/test_ib_md.cc

@@ -229,10 +229,14 @@ void
 test_ib_md::test_mkey_pack_mt_internal(unsigned access_mask, bool invalidate)
 {
     constexpr size_t size = 1 * UCS_MBYTE;
-    std::array<char, size> buffer;
     unsigned pack_flags, dereg_flags;
+    void* buffer;
+    int ret;
     uct_mem_h memh;
 
+    ret = ucs_posix_memalign(&buffer, UCS_KBYTE, size, "mkey_pack_mt");
+    ASSERT_EQ(0, ret) << "Allocation failed";
+
     if ((access_mask & UCT_MD_MEM_ACCESS_REMOTE_ATOMIC) && is_bf_arm()) {
         UCS_TEST_SKIP_R("FIXME: AMO reg key bug on BF device, skipping");
         return;
@@ -249,13 +253,9 @@ test_ib_md::test_mkey_pack_mt_internal(unsigned access_mask, bool invalidate)
         pack_flags = dereg_flags = 0;
     }
 
-    ucs_status_t status = reg_mem(access_mask, buffer.data(), size, &memh);
+    ucs_status_t status = reg_mem(access_mask, buffer, size, &memh);
     ASSERT_UCS_OK(status);
 
-    /* memh isn't always registered as multithreaded due to the following error:
-       mlx5dv_devx_obj_create(CREATE_MKEY, mode=KSM) failed, syndrome 0x103e77: Remote I/O error
-    */
-
     uct_ib_mem_t *ib_memh = (uct_ib_mem_t *)memh;
     EXPECT_TRUE(ib_memh->flags & UCT_IB_MEM_MULTITHREADED);
 
@@ -279,6 +279,8 @@ test_ib_md::test_mkey_pack_mt_internal(unsigned access_mask, bool invalidate)
     params.comp        = &comp().comp;
     status = uct_md_mem_dereg_v2(md(), &params);
     EXPECT_UCS_OK(status);
+
+    ucs_free(buffer);
 }
 
 void test_ib_md::test_mkey_pack_mt(bool invalidate) {

test/gtest/uct/ib/test_ib_xfer.cc

@@ -7,6 +7,7 @@
 #include <uct/test_p2p_rma.h>
 #include <uct/test_p2p_mix.h>
 
+#include <uct/ib/base/ib_md.h>
 
 class uct_p2p_rma_test_xfer : public uct_p2p_rma_test {};
 
@@ -129,13 +130,33 @@ UCS_TEST_P(uct_p2p_mix_test_alloc_methods, mix1000)
     run(1000);
 }
 
-UCS_TEST_P(uct_p2p_mix_test_alloc_methods, mix1000_multithreaded,
+UCT_INSTANTIATE_IB_TEST_CASE(uct_p2p_mix_test_alloc_methods)
+
+
+class uct_p2p_mix_test_alloc_methods_mt : public uct_p2p_mix_test {
+protected:
+    virtual void random_op(const mapped_buffer &sendbuf,
+                           const mapped_buffer &recvbuf) override
+    {
+        uint32_t mt_flag = UCT_IB_MEM_MULTITHREADED;
+
+        /* The potential reason of failure is alignment of the memory.
+         * Should be aligned to max atomic size
+         * or even to uct_ib_md_ext_config_t::mt_reg_chunk size on old FW.*/
+        EXPECT_TRUE(((uct_ib_mem_t *)sendbuf.memh())->flags & mt_flag);
+        EXPECT_TRUE(((uct_ib_mem_t *)recvbuf.memh())->flags & mt_flag);
+
+        uct_p2p_mix_test::random_op(sendbuf, recvbuf);
+    }
+};
+
+UCS_TEST_P(uct_p2p_mix_test_alloc_methods_mt, mix1000,
            "REG_MT_THRESH=1", "REG_MT_CHUNK=1K", "REG_MT_BIND=y")
 {
     run(1000);
 }
 
-UCT_INSTANTIATE_IB_TEST_CASE(uct_p2p_mix_test_alloc_methods)
+UCT_INSTANTIATE_IB_TEST_CASE(uct_p2p_mix_test_alloc_methods_mt)
 
 
 class uct_p2p_mix_test_indirect_atomic : public uct_p2p_mix_test {};

test/gtest/uct/test_p2p_mix.h

@@ -56,15 +56,14 @@ class uct_p2p_mix_test : public uct_p2p_test {
                           const mapped_buffer &recvbuf,
                           uct_completion_t *comp);
 
-    void random_op(const mapped_buffer &sendbuf, const mapped_buffer &recvbuf);
+    virtual void random_op(const mapped_buffer &sendbuf, const mapped_buffer &recvbuf);
 
     void run(unsigned count);
 
     virtual void init();
 
     virtual void cleanup();
 
-private:
     std::vector<send_func_t> m_avail_send_funcs;
     size_t                   m_send_size;
     static uint32_t          am_pending;