Merge pull request #15021 from daos-stack/juszhan/google/2.6

Merge upstream/release/2.6 into upstream/google/2.6

ci/provisioning/post_provision_config.sh

@@ -20,7 +20,7 @@ source ci/provisioning/post_provision_config_common_functions.sh
 source ci/junit.sh
 
 
-: "${MLNX_VER_NUM:=latest-5.8}"
+: "${MLNX_VER_NUM:=24.04-0.6.6.0}"
 
 : "${DISTRO:=EL_7}"
 DSL_REPO_var="DAOS_STACK_${DISTRO}_LOCAL_REPO"

src/cart/crt_hg.c

@@ -672,6 +672,8 @@ crt_get_info_string(bool primary, crt_provider_t provider, int iface_idx,
 	start_port = crt_provider_ctx0_port_get(primary, provider);
 	domain_str   = crt_provider_domain_str_get(primary, provider, iface_idx);
 
+	D_ASSERTF(provider_str != NULL, "String for provider=%d not found\n", provider);
+
 	/* CXI provider uses domain names for info string */
 	if (provider == CRT_PROV_OFI_CXI)
 		iface_str = NULL;
@@ -690,8 +692,7 @@ crt_get_info_string(bool primary, crt_provider_t provider, int iface_idx,
 		D_GOTO(out, rc);
 	}
 
-	if (provider_str)
-		size += strlen(provider_str);
+	size = strlen(provider_str);
 	if (domain_str)
 		size += strlen(domain_str);
 	if (iface_str)

src/dtx/dtx_rpc.c

@@ -1220,6 +1220,9 @@ dtx_refresh_internal(struct ds_cont_child *cont, int *check_count, d_list_t *che
 	/* Handle the entries whose leaders are on current server. */
 	d_list_for_each_entry_safe(dsp, tmp, &self, dsp_link) {
 		struct dtx_entry	dte;
+		struct dtx_entry	*pdte = &dte;
+		struct dtx_cos_key	 dck;
+
 
 		d_list_del(&dsp->dsp_link);
 
@@ -1228,13 +1231,31 @@ dtx_refresh_internal(struct ds_cont_child *cont, int *check_count, d_list_t *che
 		dte.dte_refs = 1;
 		dte.dte_mbs = dsp->dsp_mbs;
 
+		if (for_io) {
+			rc = vos_dtx_check(cont->sc_hdl, &dsp->dsp_xid, NULL, NULL, NULL, false);
+			switch(rc) {
+			case DTX_ST_COMMITTABLE:
+				dck.oid = dsp->dsp_oid;
+				dck.dkey_hash = dsp->dsp_dkey_hash;
+				rc = dtx_commit(cont, &pdte, &dck, 1);
+				if (rc < 0 && rc != -DER_NONEXIST && for_io)
+					d_list_add_tail(&dsp->dsp_link, cmt_list);
+				else
+					dtx_dsp_free(dsp);
+				continue;
+			case DTX_ST_COMMITTED:
+			case -DER_NONEXIST: /* Aborted */
+				dtx_dsp_free(dsp);
+				continue;
+			default:
+				break;
+			}
+		}
+
 		rc = dtx_status_handle_one(cont, &dte, dsp->dsp_oid, dsp->dsp_dkey_hash,
 					   dsp->dsp_epoch, NULL, NULL);
 		switch (rc) {
 		case DSHR_NEED_COMMIT: {
-			struct dtx_entry	*pdte = &dte;
-			struct dtx_cos_key	 dck;
-
 			dck.oid = dsp->dsp_oid;
 			dck.dkey_hash = dsp->dsp_dkey_hash;
 			rc = dtx_commit(cont, &pdte, &dck, 1);

src/object/srv_obj.c

@@ -5655,7 +5655,8 @@ ds_obj_coll_punch_handler(crt_rpc_t *rpc)
 			   1 /* start, [0] is for current engine */, ocpi->ocpi_disp_width,
 			   &exec_arg.coll_cur);
 
-	rc = dtx_leader_begin(ioc.ioc_vos_coh, &odm->odm_xid, &epoch, 1, version,
+	rc = dtx_leader_begin(ioc.ioc_vos_coh, &odm->odm_xid, &epoch,
+			      dcts[0].dct_shards[dmi->dmi_tgt_id].dcs_nr, version,
 			      &ocpi->ocpi_oid, NULL /* dti_cos */, 0 /* dti_cos_cnt */,
 			      NULL /* tgts */, exec_arg.coll_cur.grp_nr /* tgt_cnt */,
 			      dtx_flags, odm->odm_mbs, dce, &dlh);

src/tests/ftest/telemetry/wal_metrics.py

@@ -100,8 +100,8 @@ def test_wal_replay_metrics(self):
                         # Replay size should be > 0 after pool create for MD on SSD
                         ranges[metric][label] = [1]
                     elif metric.endswith('_replay_time'):
-                        # Replay time should be 10 - 50,000 after pool create for MD on SSD
-                        ranges[metric][label] = [10, 50000]
+                        # Replay time should be 1 - 1,000,000 us after pool create for MD on SSD
+                        ranges[metric][label] = [1, 1000000]
                     elif metric.endswith('_replay_transactions'):
                         # Replay transactions should be > 0 after pool create for MD on SSD
                         ranges[metric][label] = [1]

src/tests/ftest/util/file_count_test_base.py

@@ -117,7 +117,7 @@ def run_file_count(self):
                 try:
                     self.processes = ior_np
                     self.ppn = ior_ppn
-                    if self.ior_cmd.api.value == 'HDF5-VOL':
+                    if api == 'HDF5-VOL':
                         self.ior_cmd.api.update('HDF5')
                         self.run_ior_with_pool(
                             create_pool=False, plugin_path=hdf5_plugin_path, mount_dir=mount_dir)

src/tests/ftest/util/launch_utils.py

@@ -1015,8 +1015,8 @@ def update_test_yaml(self, logger, scm_size, scm_mount, extra_yaml, multiplier,
             if new_yaml_file:
                 if verbose > 0:
                     # Optionally display a diff of the yaml file
-                    if not run_local(logger, f"diff -y {test.yaml_file} {new_yaml_file}").passed:
-                        raise RunException(f"Error diff'ing {test.yaml_file}")
+                    # diff returns rc=1 if the files are different, so ignore errors
+                    run_local(logger, f"diff -y {test.yaml_file} {new_yaml_file}")
                 test.yaml_file = new_yaml_file
 
             # Display the modified yaml file variants with debug

src/vos/vos_common.c

@@ -269,13 +269,15 @@ vos_tx_end(struct vos_container *cont, struct dtx_handle *dth_in,
 	   struct umem_rsrvd_act **rsrvd_scmp, d_list_t *nvme_exts,
 	   bool started, struct bio_desc *biod, int err)
 {
-	struct vos_pool         *pool;
-	struct dtx_handle	*dth = dth_in;
-	struct vos_dtx_act_ent	*dae;
-	struct dtx_rsrvd_uint	*dru;
-	struct vos_dtx_cmt_ent	*dce = NULL;
-	struct dtx_handle	 tmp = {0};
-	int			 rc;
+	struct vos_pool         	*pool;
+	struct umem_instance		*umm;
+	struct dtx_handle		*dth = dth_in;
+	struct vos_dtx_act_ent		*dae;
+	struct vos_dtx_act_ent_df	*dae_df;
+	struct dtx_rsrvd_uint		*dru;
+	struct vos_dtx_cmt_ent		*dce = NULL;
+	struct dtx_handle		 tmp = {0};
+	int				 rc = 0;
 
 	if (!dtx_is_valid_handle(dth)) {
 		/** Created a dummy dth handle for publishing extents */
@@ -287,11 +289,11 @@ vos_tx_end(struct vos_container *cont, struct dtx_handle *dth_in,
 		D_INIT_LIST_HEAD(&tmp.dth_deferred_nvme);
 	}
 
-	if (dth->dth_local) {
+	if (dth->dth_local)
 		pool = vos_hdl2pool(dth_in->dth_poh);
-	} else {
+	else
 		pool = cont->vc_pool;
-	}
+	umm = vos_pool2umm(pool);
 
 	if (rsrvd_scmp != NULL) {
 		D_ASSERT(nvme_exts != NULL);
@@ -300,7 +302,7 @@ vos_tx_end(struct vos_container *cont, struct dtx_handle *dth_in,
 			 * Just do your best to release the SCM reservation. Can't handle another
 			 * error while handling one already anyway.
 			 */
-			(void)vos_publish_scm(vos_pool2umm(pool), *rsrvd_scmp, false /* publish */);
+			(void)vos_publish_scm(umm, *rsrvd_scmp, false /* publish */);
 			D_FREE(*rsrvd_scmp);
 			*rsrvd_scmp = NULL;
 			err         = -DER_NOMEM;
@@ -341,9 +343,9 @@ vos_tx_end(struct vos_container *cont, struct dtx_handle *dth_in,
 	vos_dth_set(NULL, pool->vp_sysdb);
 
 	if (bio_nvme_configured(SMD_DEV_TYPE_META) && biod != NULL)
-		err = umem_tx_end_ex(vos_pool2umm(pool), err, biod);
+		err = umem_tx_end_ex(umm, err, biod);
 	else
-		err = umem_tx_end(vos_pool2umm(pool), err);
+		err = umem_tx_end(umm, err);
 
 cancel:
 	if (dtx_is_valid_handle(dth_in)) {
@@ -409,8 +411,11 @@ vos_tx_end(struct vos_container *cont, struct dtx_handle *dth_in,
 				vos_dtx_post_handle(cont, &dae, &dce, 1, false, err != 0);
 			} else {
 				D_ASSERT(dce == NULL);
-				if (err == 0)
+				if (err == 0) {
 					dae->dae_prepared = 1;
+					dae_df = umem_off2ptr(umm, dae->dae_df_off);
+					D_ASSERT(!(dae_df->dae_flags & DTE_INVALID));
+				}
 			}
 		}
 	}