Merge pull request #15573 from daos-stack/jvolivie/try_jenkins

Backport several fixes

.github/dependabot.yml

@@ -0,0 +1,13 @@
+version: 2
+updates:
+  - package-ecosystem: pip
+    directories:
+      - /utils/ansible/ftest/
+      - /utils/cq/
+      - /
+    schedule:
+      interval: daily
+    groups:
+      python-packages:
+        patterns:
+          - "*"

.github/workflows/ci2.yml

@@ -15,7 +15,7 @@ jobs:
   # from scratch, but do not save them.
   Build-and-test:
     name: Run DAOS/NLT tests
-    runs-on: ubuntu-22.04
+    runs-on: ubuntu-24.04
     permissions:
       # https://github.com/EnricoMi/publish-unit-test-result-action#permissions
       checks: write
@@ -26,7 +26,7 @@ jobs:
         include:
           - distro: ubuntu
             base: ubuntu
-            with: ubuntu:mantic
+            with: ubuntu:oracular
     env:
       DEPS_JOBS: 10
       COMPILER: clang
@@ -76,7 +76,7 @@ jobs:
 
   Build:
     name: Build DAOS
-    runs-on: ubuntu-22.04
+    runs-on: ubuntu-24.04
     strategy:
       fail-fast: false
       matrix:

.github/workflows/landing-builds.yml

@@ -35,7 +35,7 @@ jobs:
   # for distros where we only want to build on master and not PRs see the Build-branch job below.
   Prepare:
     name: Create Docker images
-    runs-on: ubuntu-22.04
+    runs-on: ubuntu-24.04
     strategy:
       fail-fast: false
       max-parallel: 1
@@ -44,7 +44,7 @@ jobs:
         include:
           - distro: ubuntu
             base: ubuntu
-            with: ubuntu:mantic
+            with: ubuntu:oracular
           - distro: rocky
             base: el.9
             with: rockylinux/rockylinux:9
@@ -88,7 +88,7 @@ jobs:
   Build-and-test:
     name: Run DAOS/NLT tests
     needs: Prepare
-    runs-on: ubuntu-22.04
+    runs-on: ubuntu-24.04
     permissions:
       # https://github.com/EnricoMi/publish-unit-test-result-action#permissions
       checks: write
@@ -99,7 +99,7 @@ jobs:
         include:
           - distro: ubuntu
             base: ubuntu
-            with: ubuntu:mantic
+            with: ubuntu:oracular
     env:
       DEPS_JOBS: 10
       BASE_DISTRO: ${{ matrix.with }}
@@ -148,7 +148,7 @@ jobs:
   Build:
     name: Build DAOS in image
     needs: Prepare
-    runs-on: ubuntu-22.04
+    runs-on: ubuntu-24.04
     strategy:
       fail-fast: false
       max-parallel: 2
@@ -158,7 +158,7 @@ jobs:
         include:
           - distro: ubuntu
             base: ubuntu
-            with: ubuntu:mantic
+            with: ubuntu:oracular
           - distro: rocky
             base: el.9
             with: rockylinux/rockylinux:9
@@ -226,7 +226,7 @@ jobs:
 
   Build-branch:
     name: Build DAOS
-    runs-on: ubuntu-22.04
+    runs-on: ubuntu-24.04
     strategy:
       fail-fast: false
       matrix:
@@ -322,7 +322,7 @@ jobs:
         include:
           - distro: ubuntu
             base: ubuntu
-            with: ubuntu:mantic
+            with: ubuntu:oracular
           - distro: alma.8
             base: el.8
             with: almalinux:8

.github/workflows/linting.yml

@@ -36,7 +36,7 @@ jobs:
 
   shell-check:
     name: ShellCheck
-    runs-on: ubuntu-22.04
+    runs-on: ubuntu-24.04
     steps:
       - name: Checkout code
         uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11  # v4.1.1
@@ -54,7 +54,7 @@ jobs:
 
   log-check:
     name: Logging macro checking
-    runs-on: ubuntu-22.04
+    runs-on: ubuntu-24.04
     steps:
       - name: Checkout code
         uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11  # v4.1.1
@@ -65,15 +65,15 @@ jobs:
 
   ftest-tags:
     name: Ftest tag check
-    runs-on: ubuntu-22.04
+    runs-on: ubuntu-24.04
     steps:
       - name: Checkout code
         uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11  # v4.1.1
       - name: Check DAOS ftest tags.
         run: \[ ! -x src/tests/ftest/tags.py \] || ./src/tests/ftest/tags.py lint --verbose
 
   flake8-lint:
-    runs-on: ubuntu-22.04
+    runs-on: ubuntu-24.04
     name: Flake8 check
     steps:
       - name: Check out source repository
@@ -107,7 +107,7 @@ jobs:
 
   doxygen:
     name: Doxygen
-    runs-on: ubuntu-22.04
+    runs-on: ubuntu-24.04
     steps:
       - name: Checkout code
         uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11  # v4.1.1
@@ -128,7 +128,7 @@ jobs:
 
   pylint:
     name: Pylint check
-    runs-on: ubuntu-22.04
+    runs-on: ubuntu-24.04
     steps:
       - name: Checkout code
         uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11  # v4.1.1
@@ -148,7 +148,7 @@ jobs:
 
   codespell:
     name: Codespell
-    runs-on: ubuntu-22.04
+    runs-on: ubuntu-24.04
     steps:
       - name: Checkout code
         uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11  # v4.1.1
@@ -163,7 +163,7 @@ jobs:
 
   linting-summary:
     name: Linting Summary
-    runs-on: ubuntu-22.04
+    runs-on: ubuntu-24.04
     needs: [isort, shell-check, log-check, ftest-tags, flake8-lint, doxygen, pylint, codespell]
     if: (!cancelled())
     steps:

.github/workflows/pr-metadata.yml

@@ -13,7 +13,7 @@ permissions: {}
 
 jobs:
   example_comment_pr:
-    runs-on: ubuntu-22.04
+    runs-on: ubuntu-24.04
     permissions:
       pull-requests: write
     name: Report Jira data to PR comment

.github/workflows/version-checks.yml

@@ -7,13 +7,14 @@ on:
   pull_request:
     paths:
       - 'utils/cq/requirements.txt'
+      - '.github/workflows/version-checks.yml'
 
 permissions: {}
 
 jobs:
   upgrade-check:
     name: Check for updates
-    runs-on: ubuntu-22.04
+    runs-on: ubuntu-24.04
     strategy:
       fail-fast: false
       matrix:

Jenkinsfile

@@ -802,7 +802,7 @@ pipeline {
                             unitTestPost artifacts: ['nlt_logs/'],
                                          testResults: 'nlt-junit.xml',
                                          always_script: 'ci/unit/test_nlt_post.sh',
-					 referenceJobName: 'daos-stack/daos/release%252F2.6',
+					 referenceJobName: 'daos-stack/daos/google%252F2.6',
                                          valgrind_stash: 'el8-gcc-nlt-memcheck'
                             recordIssues enabledForFailure: true,
                                          failOnError: false,
@@ -1039,7 +1039,7 @@ pipeline {
                     }
                     post {
                         always {
-                            discoverGitReferenceBuild referenceJob: 'daos-stack/daos/release%252F2.6',
+                            discoverGitReferenceBuild referenceJob: 'daos-stack/daos/google%252F2.6',
                                                       scm: 'daos-stack/daos',
                                                       requiredResult: hudson.model.Result.UNSTABLE
                             recordIssues enabledForFailure: true,
@@ -1059,7 +1059,8 @@ pipeline {
                             stash name: 'fault-inject-valgrind',
                                   includes: '*.memcheck.xml',
                                   allowEmpty: true
-                            archiveArtifacts artifacts: 'nlt_logs/el8.fault-injection/'
+                            archiveArtifacts artifacts: 'nlt_logs/el8.fault-injection/',
+                                             allowEmptyArchive: true
                             job_status_update()
                         }
                     }

ci/storage/test_main_storage_prepare_node.sh

@@ -40,7 +40,7 @@ if ipmctl show -dimm; then
     ipmctl show -region
     find /dev -name 'pmem*'
 
-    daos_server scm $STORAGE_PREP_OPT --force
+    daos_server scm "$STORAGE_PREP_OPT"  --force
 
     if [ -n "$STORAGE_SCAN" ]; then
       # if we don't have pmem here, then we have a problem.

ci/unit/test_post_always.sh

@@ -37,7 +37,7 @@ case $STAGE_NAME in
 	;;
 esac
 
-mkdir -p $test_log_dir
+mkdir -p "$test_log_dir"
 
 rsync -v -rlpt -z -e "ssh $SSH_KEY_ARGS" jenkins@"$NODE":build/ .
 

docs/admin/env_variables.md

@@ -54,7 +54,7 @@ Environment variables in this section only apply to the server side.
 |DAOS\_DTX\_RPC\_HELPER\_THD|DTX RPC helper threshold. The valid range is [18, unlimited). The default value is 513.|
 |DAOS\_DTX\_BATCHED\_ULT\_MAX|The max count of DTX batched commit ULTs. The valid range is [0, unlimited). 0 means to commit DTX synchronously. The default value is 32.|
 |DAOS\_FORWARD\_NEIGHBOR|Set to enable I/O forwarding on neighbor xstream in the absence of helper threads.|
-|DAOS\_POOL\_RF|Redundancy factor for the pool. The valid range is [1, 4]. The default value is 2.|
+|DAOS\_POOL\_RF|Redundancy factor for the pool. The valid range is [0, 4]. The default value is 2.|
 
 ## Server and Client environment variables
 

site_scons/site_tools/compiler_setup.py

@@ -4,6 +4,7 @@
 
 DESIRED_FLAGS = ['-fstack-usage',
                  '-Wno-sign-compare',
+                 '-Wno-missing-attributes',
                  '-Wno-unused-parameter',
                  '-Wno-missing-field-initializers',
                  '-Wno-implicit-fallthrough',

src/client/dfuse/dfuse_core.c

@@ -1401,7 +1401,7 @@ dfuse_fs_start(struct dfuse_info *dfuse_info, struct dfuse_cont *dfs)
 	 * standard allocation macros
 	 */
 	args.allocated = 1;
-	args.argv = calloc(sizeof(*args.argv), args.argc);
+	args.argv      = calloc(args.argc, sizeof(*args.argv));
 	if (!args.argv)
 		D_GOTO(err, rc = -DER_NOMEM);
 

src/client/java/daos-java/pom.xml

@@ -41,7 +41,7 @@
     <dependency>
       <groupId>io.netty</groupId>
       <artifactId>netty-buffer</artifactId>
-      <version>4.1.77.Final</version>
+      <version>4.1.115.Final</version>
     </dependency>
     <dependency>
       <groupId>com.google.code.findbugs</groupId>

src/gurt/telemetry.c

@@ -830,7 +830,7 @@ d_tm_init_with_name(int id, uint64_t mem_size, int flags, const char *root_name)
 {
 	struct d_tm_shmem_hdr   *new_shmem = NULL;
 	key_t			 key;
-	int                      shmid;
+	int                      shmid = 0;
 	int			 rc = DER_SUCCESS;
 
 	if (root_name == NULL || strnlen(root_name, D_TM_MAX_NAME_LEN) == 0) {

src/placement/tests/benchmark_util.h

@@ -41,8 +41,7 @@ benchmark_alloc(void)
 {
 	struct benchmark_handle *hdl;
 
-	hdl = (struct benchmark_handle *)calloc(sizeof(struct benchmark_handle),
-						1);
+	hdl = (struct benchmark_handle *)calloc(1, sizeof(struct benchmark_handle));
 	if (hdl == NULL)
 		return NULL;
 

src/placement/tests/jump_map_place_obj.c

@@ -1441,7 +1441,7 @@ drain_target_same_shard_repeatedly_for_all_shards(void **state)
 	int			i;
 	uint32_t		shard_id = 0;
 	uint32_t		target;
-	uint32_t		new_target;
+	uint32_t                new_target = 0;
 
 	for (shard_id = 0; shard_id < 18; shard_id++) {
 		jtc_init_with_layout(&ctx, 18 * 2, 1, 4, OC_EC_16P2G1,

src/pool/srv.c

@@ -1,5 +1,5 @@
 /**
- * (C) Copyright 2016-2022 Intel Corporation.
+ * (C) Copyright 2016-2024 Intel Corporation.
  *
  * SPDX-License-Identifier: BSD-2-Clause-Patent
  */
@@ -21,10 +21,29 @@
 #include "srv_layout.h"
 
 bool		ec_agg_disabled;
-uint32_t	pw_rf; /* pool wise RF */
-#define PW_RF_DEFAULT	(2)
-#define PW_RF_MIN	(1)
-#define PW_RF_MAX	(4)
+uint32_t        pw_rf = -1; /* pool wise redundancy factor */
+#define PW_RF_DEFAULT (2)
+#define PW_RF_MIN     (0)
+#define PW_RF_MAX     (4)
+
+static inline bool
+check_pool_redundancy_factor(const char *variable)
+{
+	d_getenv_uint32_t(variable, &pw_rf);
+	if (pw_rf == -1)
+		return false;
+
+	D_INFO("Checked threshold %s=%d\n", variable, pw_rf);
+
+	if (pw_rf <= PW_RF_MAX)
+		return true;
+
+	D_INFO("pw_rf %d is out of range [%d, %d], take default %d\n", pw_rf, PW_RF_MIN, PW_RF_MAX,
+	       PW_RF_DEFAULT);
+	pw_rf = PW_RF_DEFAULT;
+
+	return true;
+}
 
 static int
 init(void)
@@ -52,14 +71,10 @@ init(void)
 	if (unlikely(ec_agg_disabled))
 		D_WARN("EC aggregation is disabled.\n");
 
-	pw_rf = PW_RF_DEFAULT;
-	d_getenv_uint32_t("DAOS_POOL_RF", &pw_rf);
-	if (pw_rf < PW_RF_MIN || pw_rf > PW_RF_MAX) {
-		D_INFO("pw_rf %d is out of range [%d, %d], take default %d\n",
-		       pw_rf, PW_RF_MIN, PW_RF_MAX, PW_RF_DEFAULT);
+	pw_rf = -1;
+	if (!check_pool_redundancy_factor("DAOS_POOL_RF"))
 		pw_rf = PW_RF_DEFAULT;
-	}
-	D_INFO("pool wise RF %d\n", pw_rf);
+	D_INFO("pool redundancy factor %d\n", pw_rf);
 
 	ds_pool_rsvc_class_register();
 

src/tests/ftest/scripts/main.sh

@@ -108,4 +108,4 @@ else
     rc=0
 fi
 
-exit $rc
+exit "$rc"

src/utils/crt_launch/crt_launch.c

@@ -281,13 +281,13 @@ int main(int argc, char **argv)
 	par_rank(PAR_COMM_WORLD, &my_rank);
 	par_size(PAR_COMM_WORLD, &world_size);
 
-	hostbuf = calloc(sizeof(*hostbuf), 1);
+	hostbuf = calloc(1, sizeof(*hostbuf));
 	if (!hostbuf) {
 		D_ERROR("Failed to allocate hostbuf\n");
 		D_GOTO(exit, rc = -1);
 	}
 
-	recv_buf = calloc(sizeof(struct host), world_size);
+	recv_buf = calloc(world_size, sizeof(*recv_buf));
 	if (!recv_buf) {
 		D_ERROR("Failed to allocate recv_buf\n");
 		D_GOTO(exit, rc = -1);