DAOS-16251 misc: Debug memory access issues

Jenkinsfile

@@ -185,7 +185,7 @@ pipeline {
         //                     'stages of this run (i.e. auto, auto_md_on_ssd, auto:-3DNAND, ' +
         //                     '0000:81:00.0, etc.).  Does not apply to MD on SSD stages.')
         string(name: 'BuildType',
-               defaultValue: '',
+               defaultValue: 'debug',
                description: 'Type of build.  Passed to scons as BUILD_TYPE.  (I.e. dev, release, debug, etc.).  ' +
                             'Defaults to release on an RC or dev otherwise.')
         string(name: 'TestRepeat',

ci/unit/required_packages.sh

@@ -30,7 +30,8 @@ pkgs="argobots                         \
       pmix                             \
       protobuf-c                       \
       spdk-devel                       \
-      valgrind-devel"
+      valgrind-devel                   \
+      libasan"
 
 # output with trailing newline suppressed
 echo  -e "$pkgs\c"

site_scons/components/__init__.py

@@ -281,6 +281,8 @@
 
     if reqs.target_type == 'debug':
         abt_build.append('--enable-debug=most')
+        abt_build.append('--disable-mem-pool')
+        abt_build.append('--disable-aligned-alloc')
     else:
         abt_build.append('--disable-debug')
 
@@ -295,7 +297,7 @@
                           ['make'],
                           ['make', 'install']],
                 requires=['libunwind'],
-                libs=['abt'],
+#               libs=['abt'],
                 headers=['abt.h'])
 
     reqs.define('fuse', libs=['fuse3'], defines=['FUSE_USE_VERSION=35'],

site_scons/site_tools/compiler_setup.py

@@ -3,6 +3,8 @@
 from SCons.Script import Configure, Exit, GetOption
 
 DESIRED_FLAGS = ['-fstack-usage',
+                 '-fsanitize=address',
+                 '-fno-omit-frame-pointer',
                  '-Wno-sign-compare',
                  '-Wno-unused-parameter',
                  '-Wno-missing-field-initializers',
@@ -13,7 +15,7 @@
                  '-Wno-unused-command-line-argument',
                  '-Wmismatched-dealloc',
                  '-Wfree-nonheap-object',
-                 '-Wframe-larger-than=4096']
+                 '-Wframe-larger-than=8192']
 
 # Compiler flags to prevent optimizing out security checks
 DESIRED_FLAGS.extend(['-fno-strict-overflow', '-fno-delete-null-pointer-checks', '-fwrapv'])
@@ -53,6 +55,7 @@ def _base_setup(env):
     env.Append(CCFLAGS=['-g', '-Wextra', '-Wshadow', '-Wall', '-fpic'])
 
     env.AppendIfSupported(CCFLAGS=DESIRED_FLAGS)
+    env.AppendIfSupported(LINKFLAGS=['-fsanitize=address'])
 
     if '-Wmismatched-dealloc' in env['CCFLAGS']:
         env.AppendUnique(CPPDEFINES={'HAVE_DEALLOC': '1'})

src/control/SConscript

@@ -37,7 +37,7 @@ def get_build_flags(benv):
     if is_release_build(benv):
         return '-buildmode=pie'
     # enable race detector for non-release builds
-    return '-race'
+    return ''
 
 
 def gen_build_id():
@@ -140,6 +140,8 @@ def scons():
     # Sets CGO_LDFLAGS for rpath options
     denv.d_add_rpaths("..", True, True)
     denv.AppendENVPath("CGO_CFLAGS", denv.subst("$_CPPINCFLAGS"), sep=" ")
+    denv.AppendENVPath("CGO_CFLAGS", "-fsanitize=address", sep=" ")
+    denv.AppendENVPath("CGO_LDFLAGS", "-fsanitize=address", sep=" ")
     if prereqs.client_requested():
         install_go_bin(denv, "daos_agent")
         install_go_bin(denv, "dmg", install_man=True)

src/control/run_go_tests.sh

@@ -215,6 +215,7 @@ set +e
 LD_LIBRARY_PATH="$LD_LIBRARY_PATH" \
 CGO_LDFLAGS="$CGO_LDFLAGS" \
 CGO_CFLAGS="$CGO_CFLAGS" \
+ASAN_OPTIONS=verify_asan_link_order=0 \
 	$GO_TEST_RUNNER "$GO_TEST_EXTRA_ARGS"
 testrc=$?
 popd >/dev/null

src/mgmt/cli_mgmt.c

@@ -1367,8 +1367,8 @@ dc_mgmt_pool_list(tse_task_t *task)
 	rc = daos_rpc_send_wait(rpc);
 	if (rc != 0) {
 		DL_ERROR(rc, "rpc send failed");
-		crt_req_decref(rpc);
 		wipe_cred_iov(&in->pli_cred);
+		crt_req_decref(rpc);
 		goto rechoose;
 	}
 
@@ -1377,8 +1377,8 @@ dc_mgmt_pool_list(tse_task_t *task)
 
 	rc = rsvc_client_complete_rpc(&ms_client, &ep, rc, out->plo_op.mo_rc, &out->plo_op.mo_hint);
 	if (rc == RSVC_CLIENT_RECHOOSE) {
-		crt_req_decref(rpc);
 		wipe_cred_iov(&in->pli_cred);
+		crt_req_decref(rpc);
 		goto rechoose;
 	}
 
@@ -1430,8 +1430,8 @@ dc_mgmt_pool_list(tse_task_t *task)
 	if (rc != 0)
 		DL_ERROR(rc, "failed to list pools");
 
-	crt_req_decref(rpc);
 	wipe_cred_iov(&in->pli_cred);
+	crt_req_decref(rpc);
 out_client:
 	rsvc_client_fini(&ms_client);
 out_grp:

src/object/tests/srv_checksum_tests.c

@@ -1275,7 +1275,7 @@ fetch_with_hole7(void **state)
 		.layout = {
 			{.data = "ABCDEFGHIJKLM", .sel = {1, 2}, .ful = {1, 13} },
 			{.data = "", .sel = {3, 7}, .ful = {3, 7}, .is_hole = true},
-			{.data = "HIJKLM", .sel = {8, 13}, .ful = {1, 13} },
+			{.data = "HIJKLM", .sel = {8, 13}, .ful = {8, 13} },
 			{.data = NULL}
 		}
 	});

src/utils/ddb/tests/ddb_vos_tests.c

@@ -295,7 +295,7 @@ get_dkey_from_idx_tests(void **state)
 	assert_success(dv_get_dkey(coh, uoid, 0, &dkey));
 	i = 1;
 	while (SUCCESS(dv_get_dkey(coh, uoid, i, &dkey2))) {
-		assert_string_not_equal(dkey.iov_buf, dkey2.iov_buf);
+		assert_key_not_equal(dkey, dkey2);
 		i++;
 		daos_iov_free(&dkey2);
 	}
@@ -330,8 +330,8 @@ get_akey_from_idx_tests(void **state)
 
 	assert_success(dv_get_akey(coh, uoid, &dkey, 0, &akey));
 	i = 1;
-	while (SUCCESS(dv_get_dkey(coh, uoid, i, &akey2))) {
-		assert_string_not_equal(akey.iov_buf, akey2.iov_buf);
+	while (SUCCESS(dv_get_akey(coh, uoid, &dkey, i, &akey2))) {
+		assert_key_not_equal(akey, akey2);
 		i++;
 		daos_iov_free(&akey2);
 	}

src/vos/storage_estimator/common/tests/storage_estimator.sh

@@ -1,5 +1,7 @@
 #!/bin/bash
 
+export ASAN_OPTIONS=verify_asan_link_order=0
+
 TEST_DIR=$(mktemp -d)
 CURRENT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
 PROJECT_DIR="${CURRENT_DIR}/../../../../.."

src/vos/tests/evt_ctl.c

@@ -1770,7 +1770,7 @@ insert_val(struct test_arg *arg, daos_handle_t toh, daos_epoch_t epoch, uint64_t
 
 	memset(&entry.ei_csum, 0, sizeof(entry.ei_csum));
 
-	rc = bio_alloc_init(arg->ta_utx, &entry.ei_addr, &data, length);
+	rc = bio_alloc_init(arg->ta_utx, &entry.ei_addr, data, length);
 	if (rc != 0)
 		return rc;
 

src/vos/tests/vts_pm.c

@@ -1968,7 +1968,7 @@ many_keys(void **state)
 	assert_rc_equal(rc, 0);
 
 	rex.rx_idx = 0;
-	rex.rx_nr = sizeof(w) - 1;
+	rex.rx_nr = strlen(w);
 
 	iod.iod_type = DAOS_IOD_ARRAY;
 	iod.iod_size = 1;

utils/build.config

@@ -2,7 +2,7 @@
 component=daos
 
 [commit_versions]
-argobots=v1.1
+argobots=v1.2
 fuse=fuse-3.16.2
 pmdk=2.0.0
 isal=v2.30.0

utils/rpms/daos.spec

@@ -112,6 +112,9 @@ BuildRequires: systemd-rpm-macros
 %endif
 %endif
 BuildRequires: libuuid-devel
+%if (0%{?rhel} >= 8)
+BuildRequires: libasan
+%endif
 
 %if (0%{?suse_version} > 0)
 BuildRequires: libucp-devel
@@ -127,6 +130,9 @@ Requires: openssl
 # suffice
 Requires: mercury >= %{mercury_version}
 
+%if (0%{?rhel} >= 8)
+Requires: libasan
+%endif
 
 %description
 The Distributed Asynchronous Object Storage (DAOS) is an open-source

utils/scripts/install-el8.sh

@@ -64,7 +64,8 @@ dnf --nodocs install \
     systemd \
     valgrind-devel \
     which \
-    yasm
+    yasm \
+    libasan
 
 # ipmctl is only available on x86_64
 if [ "$arch" = x86_64 ]; then

utils/scripts/install-el9.sh

@@ -62,4 +62,5 @@ dnf --nodocs install \
     sudo \
     valgrind-devel \
     which \
-    yasm
+    yasm \
+    libasan