diff --git a/.github/dependabot.yml b/.github/dependabot.yml new file mode 100644 index 00000000000..b9a7d8a8b6a --- /dev/null +++ b/.github/dependabot.yml @@ -0,0 +1,13 @@ +version: 2 +updates: + - package-ecosystem: pip + directories: + - /utils/ansible/ftest/ + - /utils/cq/ + - / + schedule: + interval: daily + groups: + python-packages: + patterns: + - "*" diff --git a/.github/workflows/ci2.yml b/.github/workflows/ci2.yml index 3f9eb05a3b0..50063bbad2b 100644 --- a/.github/workflows/ci2.yml +++ b/.github/workflows/ci2.yml @@ -15,7 +15,7 @@ jobs: # from scratch, but do not save them. Build-and-test: name: Run DAOS/NLT tests - runs-on: ubuntu-22.04 + runs-on: ubuntu-24.04 permissions: # https://github.com/EnricoMi/publish-unit-test-result-action#permissions checks: write @@ -26,7 +26,7 @@ jobs: include: - distro: ubuntu base: ubuntu - with: ubuntu:mantic + with: ubuntu:oracular env: DEPS_JOBS: 10 COMPILER: clang @@ -76,7 +76,7 @@ jobs: Build: name: Build DAOS - runs-on: ubuntu-22.04 + runs-on: ubuntu-24.04 strategy: fail-fast: false matrix: diff --git a/.github/workflows/landing-builds.yml b/.github/workflows/landing-builds.yml index 814c4f0e71f..aeedd80b9d6 100644 --- a/.github/workflows/landing-builds.yml +++ b/.github/workflows/landing-builds.yml @@ -35,7 +35,7 @@ jobs: # for distros where we only want to build on master and not PRs see the Build-branch job below. Prepare: name: Create Docker images - runs-on: ubuntu-22.04 + runs-on: ubuntu-24.04 strategy: fail-fast: false max-parallel: 1 @@ -44,7 +44,7 @@ jobs: include: - distro: ubuntu base: ubuntu - with: ubuntu:mantic + with: ubuntu:oracular - distro: rocky base: el.9 with: rockylinux/rockylinux:9 @@ -88,7 +88,7 @@ jobs: Build-and-test: name: Run DAOS/NLT tests needs: Prepare - runs-on: ubuntu-22.04 + runs-on: ubuntu-24.04 permissions: # https://github.com/EnricoMi/publish-unit-test-result-action#permissions checks: write @@ -99,7 +99,7 @@ jobs: include: - distro: ubuntu base: ubuntu - with: ubuntu:mantic + with: ubuntu:oracular env: DEPS_JOBS: 10 BASE_DISTRO: ${{ matrix.with }} @@ -148,7 +148,7 @@ jobs: Build: name: Build DAOS in image needs: Prepare - runs-on: ubuntu-22.04 + runs-on: ubuntu-24.04 strategy: fail-fast: false max-parallel: 2 @@ -158,7 +158,7 @@ jobs: include: - distro: ubuntu base: ubuntu - with: ubuntu:mantic + with: ubuntu:oracular - distro: rocky base: el.9 with: rockylinux/rockylinux:9 @@ -226,7 +226,7 @@ jobs: Build-branch: name: Build DAOS - runs-on: ubuntu-22.04 + runs-on: ubuntu-24.04 strategy: fail-fast: false matrix: @@ -322,7 +322,7 @@ jobs: include: - distro: ubuntu base: ubuntu - with: ubuntu:mantic + with: ubuntu:oracular - distro: alma.8 base: el.8 with: almalinux:8 diff --git a/.github/workflows/linting.yml b/.github/workflows/linting.yml index 67c57bd3154..753641a9e21 100644 --- a/.github/workflows/linting.yml +++ b/.github/workflows/linting.yml @@ -36,7 +36,7 @@ jobs: shell-check: name: ShellCheck - runs-on: ubuntu-22.04 + runs-on: ubuntu-24.04 steps: - name: Checkout code uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 @@ -54,7 +54,7 @@ jobs: log-check: name: Logging macro checking - runs-on: ubuntu-22.04 + runs-on: ubuntu-24.04 steps: - name: Checkout code uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 @@ -65,7 +65,7 @@ jobs: ftest-tags: name: Ftest tag check - runs-on: ubuntu-22.04 + runs-on: ubuntu-24.04 steps: - name: Checkout code uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 @@ -73,7 +73,7 @@ jobs: run: \[ ! -x src/tests/ftest/tags.py \] || ./src/tests/ftest/tags.py lint --verbose flake8-lint: - runs-on: ubuntu-22.04 + runs-on: ubuntu-24.04 name: Flake8 check steps: - name: Check out source repository @@ -107,7 +107,7 @@ jobs: doxygen: name: Doxygen - runs-on: ubuntu-22.04 + runs-on: ubuntu-24.04 steps: - name: Checkout code uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 @@ -128,7 +128,7 @@ jobs: pylint: name: Pylint check - runs-on: ubuntu-22.04 + runs-on: ubuntu-24.04 steps: - name: Checkout code uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 @@ -148,7 +148,7 @@ jobs: codespell: name: Codespell - runs-on: ubuntu-22.04 + runs-on: ubuntu-24.04 steps: - name: Checkout code uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 @@ -163,7 +163,7 @@ jobs: linting-summary: name: Linting Summary - runs-on: ubuntu-22.04 + runs-on: ubuntu-24.04 needs: [isort, shell-check, log-check, ftest-tags, flake8-lint, doxygen, pylint, codespell] if: (!cancelled()) steps: diff --git a/.github/workflows/pr-metadata.yml b/.github/workflows/pr-metadata.yml index a4d08418ccc..9b8e5bcc0c9 100644 --- a/.github/workflows/pr-metadata.yml +++ b/.github/workflows/pr-metadata.yml @@ -13,7 +13,7 @@ permissions: {} jobs: example_comment_pr: - runs-on: ubuntu-22.04 + runs-on: ubuntu-24.04 permissions: pull-requests: write name: Report Jira data to PR comment diff --git a/.github/workflows/version-checks.yml b/.github/workflows/version-checks.yml index 4463460b2c5..6f9d4cc7327 100644 --- a/.github/workflows/version-checks.yml +++ b/.github/workflows/version-checks.yml @@ -7,13 +7,14 @@ on: pull_request: paths: - 'utils/cq/requirements.txt' + - '.github/workflows/version-checks.yml' permissions: {} jobs: upgrade-check: name: Check for updates - runs-on: ubuntu-22.04 + runs-on: ubuntu-24.04 strategy: fail-fast: false matrix: diff --git a/Jenkinsfile b/Jenkinsfile index 5dbfe704446..72c6f82c98a 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -802,7 +802,7 @@ pipeline { unitTestPost artifacts: ['nlt_logs/'], testResults: 'nlt-junit.xml', always_script: 'ci/unit/test_nlt_post.sh', - referenceJobName: 'daos-stack/daos/release%252F2.6', + referenceJobName: 'daos-stack/daos/google%252F2.6', valgrind_stash: 'el8-gcc-nlt-memcheck' recordIssues enabledForFailure: true, failOnError: false, @@ -1039,7 +1039,7 @@ pipeline { } post { always { - discoverGitReferenceBuild referenceJob: 'daos-stack/daos/release%252F2.6', + discoverGitReferenceBuild referenceJob: 'daos-stack/daos/google%252F2.6', scm: 'daos-stack/daos', requiredResult: hudson.model.Result.UNSTABLE recordIssues enabledForFailure: true, @@ -1059,7 +1059,8 @@ pipeline { stash name: 'fault-inject-valgrind', includes: '*.memcheck.xml', allowEmpty: true - archiveArtifacts artifacts: 'nlt_logs/el8.fault-injection/' + archiveArtifacts artifacts: 'nlt_logs/el8.fault-injection/', + allowEmptyArchive: true job_status_update() } } diff --git a/ci/storage/test_main_storage_prepare_node.sh b/ci/storage/test_main_storage_prepare_node.sh index 2aa67b9deff..0be5a33167c 100755 --- a/ci/storage/test_main_storage_prepare_node.sh +++ b/ci/storage/test_main_storage_prepare_node.sh @@ -40,7 +40,7 @@ if ipmctl show -dimm; then ipmctl show -region find /dev -name 'pmem*' - daos_server scm $STORAGE_PREP_OPT --force + daos_server scm "$STORAGE_PREP_OPT" --force if [ -n "$STORAGE_SCAN" ]; then # if we don't have pmem here, then we have a problem. diff --git a/ci/unit/test_post_always.sh b/ci/unit/test_post_always.sh index dfe94c2f2d9..5d48ab5482d 100755 --- a/ci/unit/test_post_always.sh +++ b/ci/unit/test_post_always.sh @@ -37,7 +37,7 @@ case $STAGE_NAME in ;; esac -mkdir -p $test_log_dir +mkdir -p "$test_log_dir" rsync -v -rlpt -z -e "ssh $SSH_KEY_ARGS" jenkins@"$NODE":build/ . diff --git a/docs/admin/env_variables.md b/docs/admin/env_variables.md index 3de0a079203..008a83ca4a5 100644 --- a/docs/admin/env_variables.md +++ b/docs/admin/env_variables.md @@ -54,7 +54,7 @@ Environment variables in this section only apply to the server side. |DAOS\_DTX\_RPC\_HELPER\_THD|DTX RPC helper threshold. The valid range is [18, unlimited). The default value is 513.| |DAOS\_DTX\_BATCHED\_ULT\_MAX|The max count of DTX batched commit ULTs. The valid range is [0, unlimited). 0 means to commit DTX synchronously. The default value is 32.| |DAOS\_FORWARD\_NEIGHBOR|Set to enable I/O forwarding on neighbor xstream in the absence of helper threads.| -|DAOS\_POOL\_RF|Redundancy factor for the pool. The valid range is [1, 4]. The default value is 2.| +|DAOS\_POOL\_RF|Redundancy factor for the pool. The valid range is [0, 4]. The default value is 2.| ## Server and Client environment variables diff --git a/site_scons/site_tools/compiler_setup.py b/site_scons/site_tools/compiler_setup.py index 9f25ccace71..abee049e3e7 100644 --- a/site_scons/site_tools/compiler_setup.py +++ b/site_scons/site_tools/compiler_setup.py @@ -4,6 +4,7 @@ DESIRED_FLAGS = ['-fstack-usage', '-Wno-sign-compare', + '-Wno-missing-attributes', '-Wno-unused-parameter', '-Wno-missing-field-initializers', '-Wno-implicit-fallthrough', diff --git a/src/client/dfuse/dfuse_core.c b/src/client/dfuse/dfuse_core.c index 6397b283e97..0895451f9f8 100644 --- a/src/client/dfuse/dfuse_core.c +++ b/src/client/dfuse/dfuse_core.c @@ -1401,7 +1401,7 @@ dfuse_fs_start(struct dfuse_info *dfuse_info, struct dfuse_cont *dfs) * standard allocation macros */ args.allocated = 1; - args.argv = calloc(sizeof(*args.argv), args.argc); + args.argv = calloc(args.argc, sizeof(*args.argv)); if (!args.argv) D_GOTO(err, rc = -DER_NOMEM); diff --git a/src/client/java/daos-java/pom.xml b/src/client/java/daos-java/pom.xml index 8921791c942..6b26378ac6f 100644 --- a/src/client/java/daos-java/pom.xml +++ b/src/client/java/daos-java/pom.xml @@ -41,7 +41,7 @@ io.netty netty-buffer - 4.1.77.Final + 4.1.115.Final com.google.code.findbugs diff --git a/src/gurt/telemetry.c b/src/gurt/telemetry.c index 27a0dc42058..33b5e410dca 100644 --- a/src/gurt/telemetry.c +++ b/src/gurt/telemetry.c @@ -830,7 +830,7 @@ d_tm_init_with_name(int id, uint64_t mem_size, int flags, const char *root_name) { struct d_tm_shmem_hdr *new_shmem = NULL; key_t key; - int shmid; + int shmid = 0; int rc = DER_SUCCESS; if (root_name == NULL || strnlen(root_name, D_TM_MAX_NAME_LEN) == 0) { diff --git a/src/placement/tests/benchmark_util.h b/src/placement/tests/benchmark_util.h index f3ccc532a71..9b325fb00b0 100644 --- a/src/placement/tests/benchmark_util.h +++ b/src/placement/tests/benchmark_util.h @@ -41,8 +41,7 @@ benchmark_alloc(void) { struct benchmark_handle *hdl; - hdl = (struct benchmark_handle *)calloc(sizeof(struct benchmark_handle), - 1); + hdl = (struct benchmark_handle *)calloc(1, sizeof(struct benchmark_handle)); if (hdl == NULL) return NULL; diff --git a/src/placement/tests/jump_map_place_obj.c b/src/placement/tests/jump_map_place_obj.c index 158292e2e34..639d1cc8eb7 100644 --- a/src/placement/tests/jump_map_place_obj.c +++ b/src/placement/tests/jump_map_place_obj.c @@ -1441,7 +1441,7 @@ drain_target_same_shard_repeatedly_for_all_shards(void **state) int i; uint32_t shard_id = 0; uint32_t target; - uint32_t new_target; + uint32_t new_target = 0; for (shard_id = 0; shard_id < 18; shard_id++) { jtc_init_with_layout(&ctx, 18 * 2, 1, 4, OC_EC_16P2G1, diff --git a/src/pool/srv.c b/src/pool/srv.c index 7e5548e8508..9c9611c04ef 100644 --- a/src/pool/srv.c +++ b/src/pool/srv.c @@ -1,5 +1,5 @@ /** - * (C) Copyright 2016-2022 Intel Corporation. + * (C) Copyright 2016-2024 Intel Corporation. * * SPDX-License-Identifier: BSD-2-Clause-Patent */ @@ -21,10 +21,29 @@ #include "srv_layout.h" bool ec_agg_disabled; -uint32_t pw_rf; /* pool wise RF */ -#define PW_RF_DEFAULT (2) -#define PW_RF_MIN (1) -#define PW_RF_MAX (4) +uint32_t pw_rf = -1; /* pool wise redundancy factor */ +#define PW_RF_DEFAULT (2) +#define PW_RF_MIN (0) +#define PW_RF_MAX (4) + +static inline bool +check_pool_redundancy_factor(const char *variable) +{ + d_getenv_uint32_t(variable, &pw_rf); + if (pw_rf == -1) + return false; + + D_INFO("Checked threshold %s=%d\n", variable, pw_rf); + + if (pw_rf <= PW_RF_MAX) + return true; + + D_INFO("pw_rf %d is out of range [%d, %d], take default %d\n", pw_rf, PW_RF_MIN, PW_RF_MAX, + PW_RF_DEFAULT); + pw_rf = PW_RF_DEFAULT; + + return true; +} static int init(void) @@ -52,14 +71,10 @@ init(void) if (unlikely(ec_agg_disabled)) D_WARN("EC aggregation is disabled.\n"); - pw_rf = PW_RF_DEFAULT; - d_getenv_uint32_t("DAOS_POOL_RF", &pw_rf); - if (pw_rf < PW_RF_MIN || pw_rf > PW_RF_MAX) { - D_INFO("pw_rf %d is out of range [%d, %d], take default %d\n", - pw_rf, PW_RF_MIN, PW_RF_MAX, PW_RF_DEFAULT); + pw_rf = -1; + if (!check_pool_redundancy_factor("DAOS_POOL_RF")) pw_rf = PW_RF_DEFAULT; - } - D_INFO("pool wise RF %d\n", pw_rf); + D_INFO("pool redundancy factor %d\n", pw_rf); ds_pool_rsvc_class_register(); diff --git a/src/tests/ftest/scripts/main.sh b/src/tests/ftest/scripts/main.sh index e6079637064..706d622479b 100755 --- a/src/tests/ftest/scripts/main.sh +++ b/src/tests/ftest/scripts/main.sh @@ -108,4 +108,4 @@ else rc=0 fi -exit $rc +exit "$rc" diff --git a/src/utils/crt_launch/crt_launch.c b/src/utils/crt_launch/crt_launch.c index e2544d45c0b..527aa581399 100644 --- a/src/utils/crt_launch/crt_launch.c +++ b/src/utils/crt_launch/crt_launch.c @@ -281,13 +281,13 @@ int main(int argc, char **argv) par_rank(PAR_COMM_WORLD, &my_rank); par_size(PAR_COMM_WORLD, &world_size); - hostbuf = calloc(sizeof(*hostbuf), 1); + hostbuf = calloc(1, sizeof(*hostbuf)); if (!hostbuf) { D_ERROR("Failed to allocate hostbuf\n"); D_GOTO(exit, rc = -1); } - recv_buf = calloc(sizeof(struct host), world_size); + recv_buf = calloc(world_size, sizeof(*recv_buf)); if (!recv_buf) { D_ERROR("Failed to allocate recv_buf\n"); D_GOTO(exit, rc = -1); diff --git a/utils/cq/requirements.txt b/utils/cq/requirements.txt index aeccf6ca33b..5a81d115618 100644 --- a/utils/cq/requirements.txt +++ b/utils/cq/requirements.txt @@ -6,7 +6,7 @@ flake8<6.0.0 isort==5.13.2 pylint==3.2.0 yamllint==1.35.1 -codespell==2.2.6 +codespell==2.3.0 # Used by ci/jira_query.py which pip installs it standalone. jira torch>=2.2.0 diff --git a/utils/node_local_test.py b/utils/node_local_test.py index cd850587beb..b3f7d9a457c 100755 --- a/utils/node_local_test.py +++ b/utils/node_local_test.py @@ -6648,7 +6648,17 @@ def run(wf, args): # If the perf-check option is given then re-start everything without much # debugging enabled and run some micro-benchmarks to give numbers for use # as a comparison against other builds. + run_fi = False + if args.perf_check or fi_test or fi_test_dfuse: + fs = subprocess.run([os.path.join(conf['PREFIX'], 'bin', 'fault_status')], check=False) + print(fs) + if fs.returncode == 0: + run_fi = True + else: + print("Unable to detect fault injection feature, skipping testing") + + if run_fi: args.server_debug = 'INFO' args.memcheck = 'no' args.dfuse_debug = 'WARN' diff --git a/utils/scripts/helpers/install_maldet.sh b/utils/scripts/helpers/install_maldet.sh index 7b02511afe9..5f9666d348d 100755 --- a/utils/scripts/helpers/install_maldet.sh +++ b/utils/scripts/helpers/install_maldet.sh @@ -34,7 +34,7 @@ printf "LogSyslog yes\n" >> /etc/clamd.d/scan.conf lmd_tarball='maldetect-current.tar.gz' : "${REPO_FILE_URL:=https://artifactory.dc.hpdd.intel.com/artifactory/repo-files/}" -lmd_base_url="$(dirname $REPO_FILE_URL)" +lmd_base_url="$(dirname "$REPO_FILE_URL")" lmd_base="${lmd_base_url#*://}" lmd_url="${lmd_base_url}/maldetect/downloads/${lmd_tarball}" curl "${lmd_url}" --silent --show-error --fail -o "/var/tmp/${lmd_tarball}"