Merge branch 'master' into bfaccini/DAOS-13194_2nd

To fix conflicts !!...

Required-githooks: true

Signed-off-by: Bruno Faccini <[email protected]>

ci/docker_nlt.sh

@@ -22,7 +22,7 @@ pushd "$TMP_DIR"
 set +e
 
 sudo --preserve-env=VIRTUAL_ENV,PATH ./node_local_test.py \
-    --no-root --memcheck no --system-ram-reserved 32 --server-debug WARN "$@"
+    --no-root --memcheck no --system-ram-reserved 48 --server-debug WARN "$@"
 
 RC=$?
 set -e

ci/provisioning/post_provision_config_common_functions.sh

@@ -190,7 +190,7 @@ set_local_repo() {
     version=${version%%.*}
     if [ "$repo_server" = "artifactory" ] &&
        { [[ $(pr_repos) = *daos@PR-* ]] || [ -z "$(rpm_test_version)" ]; } &&
-       [[ ! ${CHANGE_TARGET:-$BRANCH_NAME} =~ ^[.-0-9A-Za-z]+-testing ]]; then
+       [[ ! ${CHANGE_TARGET:-$BRANCH_NAME} =~ ^[-.0-9A-Za-z]+-testing ]]; then
         # Disable the daos repo so that the Jenkins job repo or a PR-repos*: repo is
         # used for daos packages
         dnf -y config-manager \

ci/test_files_to_stash.txt

@@ -7,6 +7,7 @@ build/*/*/src/gurt/tests/test_gurt_telem_consumer,
 build/*/*/src/rdb/raft/src/tests_main,
 build/*/*/src/common/tests/btree_direct,
 build/*/*/src/common/tests/btree,
+build/*/*/src/common/tests/common_test,
 build/*/*/src/common/tests/sched,
 build/*/*/src/common/tests/drpc_tests,
 build/*/*/src/common/tests/acl_api_tests,
@@ -27,12 +28,16 @@ build/*/*/src/engine/tests/drpc_client_tests,
 build/*/*/src/engine/tests/drpc_handler_tests,
 build/*/*/src/engine/tests/drpc_listener_tests,
 build/*/*/src/mgmt/tests/srv_drpc_tests,
+build/*/*/src/object/tests/cli_checksum_tests,
+build/*/*/src/object/tests/srv_checksum_tests,
 build/*/*/src/security/tests/cli_security_tests,
 build/*/*/src/security/tests/srv_acl_tests,
 build/*/*/src/vos/vea/tests/vea_ut,
+build/*/*/src/vos/tests/pool_scrubbing_tests,
 build/*/*/src/common/tests/umem_test,
 build/*/*/src/common/tests/umem_test_bmem,
 build/*/*/src/bio/smd/tests/smd_ut,
+build/*/*/src/tests/rpc/rpc_tests,
 src/common/tests/btree.sh,
 src/control/run_go_tests.sh,
 src/rdb/raft_tests/raft_tests.py,

debian/changelog

@@ -1,3 +1,15 @@
+daos (2.5.100-3) unstable; urgency=medium
+  [ Wang Shilong ]
+  * Remove lmdb-devel for MD on SSD
+
+ -- Wang Shilong <[email protected]> Wed, 14 Jun 2023 07:58:00 -0600
+
+daos (2.5.100-2) unstable; urgency=medium
+  [ Ryon Jensen ]
+  * Removed unnecessary test files
+
+ -- Ryon Jensen <[email protected]>  Wed, 07 Jun 2023 10:46:00 -0700
+
 daos (2.5.100-1) unstable; urgency=medium
     [ Jeff Olivier ]
     * Bump release to 2.5.100

debian/control

@@ -32,7 +32,6 @@ Build-Depends: debhelper (>= 10),
                libraft-dev (= 0.9.1-1401.gc18bcb8),
                python3-tabulate,
                liblz4-dev,
-               liblmdb-dev,
                libcapstone-dev
 Standards-Version: 4.1.2
 Homepage: https://docs.daos.io/

debian/daos-client-tests.install

@@ -2,7 +2,6 @@ usr/lib/daos/TESTING/*
 usr/bin/hello_drpc
 usr/bin/acl_dump_test
 usr/bin/agent_tests
-usr/bin/common_test
 usr/bin/drpc_engine_test
 usr/bin/drpc_test
 usr/bin/dfuse_test

debian/daos-server-tests.install

@@ -6,9 +6,6 @@ usr/bin/rdbt
 usr/bin/smd_ut
 usr/bin/bio_ut
 usr/bin/vea_ut
-usr/bin/srv_checksum_tests
-usr/bin/pool_scrubbing_tests
-usr/bin/rpc_tests
 usr/bin/vos_tests
 usr/bin/vea_stress
 usr/bin/vos_perf

docs/admin/env_variables.md

@@ -68,6 +68,7 @@ Environment variables in this section only apply to the client side.
 |Variable                 |Description|
 |-------------------------|-----------|
 |FI\_MR\_CACHE\_MAX\_COUNT|Enable MR (Memory Registration) caching in OFI layer. Recommended to be set to 0 (disable) when CRT\_DISABLE\_MEM\_PIN is NOT set to 1. INTEGER. Default to unset.|
+|D\_POLL\_TIMEOUT|Polling timeout passed to network progress for synchronous operations. Default to 0 (busy polling), value in micro-seconds otherwise.|
 
 
 ## Debug System (Client & Server)

docs/overview/data_integrity.md

@@ -148,6 +148,7 @@ Unit and functional testing is performed at many layers.
 commont_test
 vos_tests -z
 srv_checksum_tests
+cli_checksum_tests
 pool_scrubbing_tests
 rpc_tests
 ```

site_scons/components/__init__.py

@@ -218,8 +218,6 @@ def define_common(reqs):
 
     reqs.define('yaml', headers=['yaml.h'], package='libyaml-devel')
 
-    reqs.define('lmdb', headers=['lmdb.h'], libs=['lmdb'], package='lmdb-devel')
-
     reqs.define('event', libs=['event'], package='libevent-devel')
 
     reqs.define('crypto', libs=['crypto'], headers=['openssl/md5.h'], package='openssl-devel')

site_scons/prereq_tools/base.py

@@ -528,7 +528,7 @@ def run_build(self, opts):
         common_reqs = ['argobots', 'ucx', 'ofi', 'hwloc', 'mercury', 'boost', 'uuid',
                        'crypto', 'protobufc', 'lz4', 'isal', 'isal_crypto']
         client_reqs = ['fuse', 'json-c', 'capstone']
-        server_reqs = ['pmdk', 'spdk', 'lmdb']
+        server_reqs = ['pmdk', 'spdk']
         test_reqs = ['cmocka']
 
         reqs = []

site_scons/site_tools/compiler_setup.py

@@ -4,18 +4,19 @@
 from SCons.Script import Configure
 
 
-DESIRED_FLAGS = ['-Wno-gnu-designator',
-                 '-Wno-missing-braces',
-                 '-fstack-usage',
+DESIRED_FLAGS = ['-fstack-usage',
+                 '-Wno-sign-compare',
+                 '-Wno-unused-parameter',
+                 '-Wno-missing-field-initializers',
+                 '-Wno-implicit-fallthrough',
                  '-Wno-ignored-attributes',
                  '-Wno-gnu-zero-variadic-macro-arguments',
                  '-Wno-tautological-constant-out-of-range-compare',
                  '-Wno-unused-command-line-argument',
                  '-Wframe-larger-than=4096']
 
 # Compiler flags to prevent optimizing out security checks
-DESIRED_FLAGS.extend(['-fno-strict-overflow', '-fno-delete-null-pointer-checks',
-                      '-fwrapv'])
+DESIRED_FLAGS.extend(['-fno-strict-overflow', '-fno-delete-null-pointer-checks', '-fwrapv'])
 
 # Compiler flags for stack hardening
 DESIRED_FLAGS.extend(['-fstack-protector-strong', '-fstack-clash-protection'])
@@ -48,10 +49,7 @@ def _base_setup(env):
 
     # Turn on -Wall first, then DESIRED_FLAGS may disable some of the options
     # that this brings in.
-    env.Append(CCFLAGS=['-g',
-                        '-Wshadow',
-                        '-Wall',
-                        '-fpic'])
+    env.Append(CCFLAGS=['-g', '-Wextra', '-Wshadow', '-Wall', '-fpic'])
 
     env.AppendIfSupported(CCFLAGS=DESIRED_FLAGS)
 

src/bio/SConscript

@@ -38,9 +38,6 @@ def scons():
     bio = denv.d_library("bio", tgts, install_off="../..", LIBS=libs)
     denv.Install('$PREFIX/lib64/daos_srv', bio)
 
-    if prereqs.test_requested():
-        SConscript('tests/SConscript', exports='denv')
-
 
 if __name__ == "SCons.Script":
     scons()

src/bio/smd/tests/smd_ut.c

@@ -199,7 +199,6 @@ db_fini(void)
 uuid_t	dev_id1;
 uuid_t	dev_id2;
 
-static bool is_lmdb;
 static int
 smd_ut_setup(void **state)
 {
@@ -210,13 +209,8 @@ smd_ut_setup(void **state)
 		print_error("Error initializing the debug instance\n");
 		return rc;
 	}
-	if (is_lmdb) {
-		lmm_db_init_ex(SMD_STORAGE_PATH, "sys_db", true, false);
-		rc = smd_init(lmm_db_get());
-	} else {
-		db_init();
-		rc = smd_init(&ut_db.ud_db);
-	}
+	db_init();
+	rc = smd_init(&ut_db.ud_db);
 
 	if (rc) {
 		print_error("Error initializing SMD store: %d\n", rc);
@@ -230,13 +224,8 @@ smd_ut_setup(void **state)
 static int
 smd_ut_teardown(void **state)
 {
-	if (is_lmdb) {
-		lmm_db_fini();
-		smd_fini();
-	} else {
-		smd_fini();
-		db_fini();
-	}
+	smd_fini();
+	db_fini();
 	daos_debug_fini();
 	return 0;
 }
@@ -552,14 +541,12 @@ print_usage(char *name)
 	print_message(
 		"\n\nCOMMON TESTS\n==========================\n");
 	print_message("%s -h|--help\n", name);
-	print_message("%s -l|--lmdb\n", name);
 }
 
 const char *s_opts = "hl";
 static int idx;
 static struct option l_opts[] = {
 	{"help", no_argument,	NULL, 'h'},
-	{"lmdb", no_argument,	NULL, 'l'},
 };
 
 int main(int argc, char **argv)
@@ -579,9 +566,6 @@ int main(int argc, char **argv)
 			print_usage(argv[0]);
 			rc = 0;
 			goto out;
-		case 'l':
-			is_lmdb = true;
-			break;
 		default:
 			rc = 1;
 			goto out;

src/cart/crt_init.c

@@ -71,7 +71,8 @@ dump_envariables(void)
 		"CRT_CTX_SHARE_ADDR", "CRT_CTX_NUM", "D_FI_CONFIG",
 		"FI_UNIVERSE_SIZE", "CRT_ENABLE_MEM_PIN",
 		"FI_OFI_RXM_USE_SRX", "D_LOG_FLUSH", "CRT_MRC_ENABLE",
-		"CRT_SECONDARY_PROVIDER", "D_PROVIDER_AUTH_KEY", "D_PORT_AUTO_ADJUST"};
+		"CRT_SECONDARY_PROVIDER", "D_PROVIDER_AUTH_KEY", "D_PORT_AUTO_ADJUST",
+		"D_POOL_TIMEOUT"};
 
 	D_INFO("-- ENVARS: --\n");
 	for (i = 0; i < ARRAY_SIZE(envars); i++) {

src/cart/crt_self_test_client.c

@@ -877,7 +877,7 @@ crt_self_test_start_handler(crt_rpc_t *rpc_req)
 
 	/*
 	 * Set up a bulk descriptor to use later to send the latencies back
-	 * to the self-test requestor
+	 * to the self-test requester
 	 */
 	d_iov_set(&g_data->rep_latencies_iov, g_data->rep_latencies,
 		 g_data->rep_count * sizeof(g_data->rep_latencies[0]));

src/client/api/event.c

@@ -20,6 +20,12 @@
 static __thread daos_event_t	ev_thpriv;
 static __thread bool		ev_thpriv_is_init;
 
+/**
+ * Global progress timeout for synchronous operation
+ * busy-polling by default (0), timeout in us otherwise
+ */
+static uint32_t ev_prog_timeout;
+
 #define EQ_WITH_CRT
 
 #if !defined(EQ_WITH_CRT)
@@ -91,6 +97,8 @@ daos_eq_lib_init()
 
 	eq_ref = 1;
 
+	d_getenv_int("D_POLL_TIMEOUT", &ev_prog_timeout);
+
 unlock:
 	D_MUTEX_UNLOCK(&daos_eq_lock);
 	return rc;
@@ -1262,7 +1270,7 @@ daos_event_priv_wait()
 
 	/* Wait on the event to complete */
 	while (evx->evx_status != DAOS_EVS_READY) {
-		rc = crt_progress_cond(evx->evx_ctx, 0, ev_progress_cb, &epa);
+		rc = crt_progress_cond(evx->evx_ctx, ev_prog_timeout, ev_progress_cb, &epa);
 
 		/** progress succeeded, loop can exit if event completed */
 		if (rc == 0) {

src/client/api/rpc.c

@@ -16,9 +16,10 @@ daos_rpc_cb(const struct crt_cb_info *cb_info)
 	tse_task_t	*task = cb_info->cci_arg;
 	int		rc = cb_info->cci_rc;
 
-	if (cb_info->cci_rc == -DER_TIMEDOUT)
+	if (cb_info->cci_rc == -DER_TIMEDOUT) {
 		/** TODO */
 		;
+	}
 
 	tse_task_complete(task, rc);
 }
@@ -162,7 +163,7 @@ daos_rpc_proto_query(crt_opcode_t base_opc, uint32_t *ver_array, int count, int
 	}
 
 	num_ranks = dc_mgmt_net_get_num_srv_ranks();
-	rproto->ep.ep_rank = rand() % num_ranks;
+	rproto->ep.ep_rank = d_rand() % num_ranks;
 	rproto->ver_array = ver_array;
 	rproto->array_size = count;
 	rproto->ep.ep_grp = sys->sy_group;

src/client/dfs/dfs.c

@@ -4653,7 +4653,8 @@ dfs_read_int(dfs_t *dfs, dfs_obj_t *obj, daos_off_t off, dfs_iod_t *iod,
 	if (rc)
 		D_GOTO(err_params, rc = daos_der2errno(rc));
 
-	return dc_task_schedule(task, true);
+	rc = dc_task_schedule(task, true);
+	return daos_der2errno(rc);
 
 err_params:
 	D_FREE(params);