UCT/IB/MLX5: Print KSM details to log

src/uct/ib/mlx5/dv/ib_mlx5dv_md.c

@@ -89,11 +89,11 @@ uct_ib_mlx5_devx_reg_ksm(uct_ib_mlx5_md_t *md, int atomic, uint64_t address,
                                        out, sizeof(out));
     if (mr == NULL) {
         if (reason != NULL) {
-            ucs_debug("mlx5dv_devx_obj_create(CREATE_MKEY, mode=KSM) failed, "
-                      "syndrome 0x%x: %m",
+            ucs_debug("%s: mlx5dv_devx_obj_create(CREATE_MKEY, mode=KSM, "
+                      "start_addr=0x%lx length=%zu) failed, syndrome 0x%x: %m",
+                      uct_ib_device_name(&md->super.dev), address, length,
                       UCT_IB_MLX5DV_GET(create_mkey_out, out, syndrome));
         }
-
         return UCS_ERR_UNSUPPORTED;
     }
 
@@ -121,23 +121,35 @@ uct_ib_mlx5_devx_reg_ksm_data(uct_ib_mlx5_md_t *md, int atomic, void *address,
 {
     void *mr_address = address;
     ucs_status_t status;
+    struct ibv_mr **mr;
     char *in;
     void *klm;
-    int i;
+
+    if (reason != NULL) {
+        ucs_trace("%s: initialize ksm_data list for %s address %p length %zu "
+                  "iova 0x%" PRIx64,
+                  uct_ib_device_name(&md->super.dev), reason, address, length,
+                  iova);
+    }
 
     status = uct_ib_mlx5_alloc_mkey_inbox(ksm_data->mr_num, &in);
     if (status != UCS_OK) {
         return UCS_ERR_NO_MEMORY;
     }
 
+    ucs_log_indent(+1);
     klm = UCT_IB_MLX5DV_ADDR_OF(create_mkey_in, in, klm_pas_mtt);
-    for (i = 0; i < ksm_data->mr_num; i++) {
+    ucs_carray_for_each(mr, ksm_data->mrs, ksm_data->mr_num) {
+        ucs_trace("klm[%ld] va %p mr [addr %p len %zu lkey 0x%x]",
+                  mr - ksm_data->mrs, mr_address, (*mr)->addr, (*mr)->length,
+                  (*mr)->lkey);
         UCT_IB_MLX5DV_SET64(klm, klm, address, (uintptr_t)mr_address);
-        UCT_IB_MLX5DV_SET(klm, klm, byte_count, ksm_data->mrs[i]->length);
-        UCT_IB_MLX5DV_SET(klm, klm, mkey, ksm_data->mrs[i]->lkey);
+        UCT_IB_MLX5DV_SET(klm, klm, byte_count, (*mr)->length);
+        UCT_IB_MLX5DV_SET(klm, klm, mkey, (*mr)->lkey);
         klm        = UCS_PTR_BYTE_OFFSET(klm, UCT_IB_MLX5DV_ST_SZ_BYTES(klm));
-        mr_address = UCS_PTR_BYTE_OFFSET(mr_address, ksm_data->mrs[i]->length);
+        mr_address = UCS_PTR_BYTE_OFFSET(mr_address, (*mr)->length);
     }
+    ucs_log_indent(-1);
 
     status = uct_ib_mlx5_devx_reg_ksm(md, atomic, iova, length,
                                       ksm_data->mr_num,
@@ -149,29 +161,41 @@ uct_ib_mlx5_devx_reg_ksm_data(uct_ib_mlx5_md_t *md, int atomic, void *address,
 
 static ucs_status_t
 uct_ib_mlx5_devx_reg_ksm_data_addr(uct_ib_mlx5_md_t *md, struct ibv_mr *mr,
-                                   uint64_t address, size_t length,
+                                   uintptr_t address, size_t length,
                                    uint64_t iova, int atomic, int list_size,
                                    uint32_t mkey_index, const char *reason,
                                    struct mlx5dv_devx_obj **mr_p,
                                    uint32_t *mkey)
 {
-    int i;
-    char *in;
-    void *klm;
     ucs_status_t status;
+    uintptr_t mr_address;
+    void *klm;
+    char *in;
+    int i;
+
+    if (reason != NULL) {
+        ucs_trace("%s: initialize ksm_data_addr list for %s address 0x%lx "
+                  "length %zu iova 0x%" PRIx64,
+                  uct_ib_device_name(&md->super.dev), reason, address, length,
+                  iova);
+    }
 
     status = uct_ib_mlx5_alloc_mkey_inbox(list_size, &in);
     if (status != UCS_OK) {
         return status;
     }
 
     klm = UCT_IB_MLX5DV_ADDR_OF(create_mkey_in, in, klm_pas_mtt);
+    ucs_log_indent(+1);
     for (i = 0; i < list_size; i++) {
+        mr_address = address + (i * UCT_IB_MD_MAX_MR_SIZE);
+        ucs_trace("klm[%d] va 0x%lx mr [addr %p len %zu lkey 0x%x]", i,
+                  mr_address, mr->addr, mr->length, mr->lkey);
+        UCT_IB_MLX5DV_SET64(klm, klm, address, mr_address);
         UCT_IB_MLX5DV_SET(klm, klm, mkey, mr->lkey);
-        UCT_IB_MLX5DV_SET64(klm, klm, address,
-                            address + (i * UCT_IB_MD_MAX_MR_SIZE));
         klm = UCS_PTR_BYTE_OFFSET(klm, UCT_IB_MLX5DV_ST_SZ_BYTES(klm));
     }
+    ucs_log_indent(-1);
 
     status = uct_ib_mlx5_devx_reg_ksm(md, atomic, iova, length, list_size,
                                       UCT_IB_MD_MAX_MR_SIZE, in, mkey_index,