Merge remote-tracking branch 'origin/release/2.6' into juszhan/google…

…/2.6

Required-githooks: true

Change-Id: I806b547df778ae427c3646b952769bc291359f02

.github/workflows/landing-builds.yml

@@ -16,6 +16,7 @@ on:
       - ci/**
       - requirements-build.txt
       - requirements-utest.txt
+      - utils/build.config
 
 permissions: {}
 

.github/workflows/linting.yml

@@ -26,7 +26,7 @@ jobs:
       - uses: actions/setup-python@82c7e631bb3cdc910f68e0081d67478d79c6982d  # v5.1.0
         with:
           python-version: '3'
-      - uses: isort/isort-action@master
+      - uses: isort/isort-action@f14e57e1d457956c45a19c05a89cccdf087846e5  # v1.1.0
         with:
           requirementsFiles: "requirements.txt"
       - name: Run on SConstruct file.

.github/workflows/trivy.yml

@@ -0,0 +1,70 @@
+name: Trivy scan
+
+on:
+  workflow_dispatch:
+  push:
+    branches: ["master", "release/**"]
+  pull_request:
+    branches: ["master", "release/**"]
+
+# Declare default permissions as nothing.
+permissions: {}
+
+jobs:
+  build:
+    name: Build
+    runs-on: ubuntu-20.04
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11  # v4.1.1
+
+      - name: Run Trivy vulnerability scanner in repo mode
+        uses: aquasecurity/trivy-action@6e7b7d1fd3e4fef0c5fa8cce1229c54b2c9bd0d8  # 0.24.0
+        with:
+          scan-type: 'fs'
+          scan-ref: '.'
+          trivy-config: 'utils/trivy/trivy.yaml'
+
+      - name: Prepare the report to be uploaded to the GitHub artifact store
+        run: |
+          mkdir report
+          cp trivy-report-daos.txt report
+          cp utils/trivy/.trivyignore report/trivyignore.txt
+
+      - name: Upload the report to the GitHub artifact store
+        uses: actions/upload-artifact@65462800fd760344b1a7b4382951275a0abb4808  # v4.3.3
+        with:
+          path: report/*
+          name: trivy-report-daos
+
+      - name: Adjust config file to use sarif format
+        run: |
+          sed -i 's/output: "trivy-report-daos.txt"/output: "trivy-results.sarif"/g' \
+            utils/trivy/trivy.yaml
+          sed -i 's/format: template/format: sarif/g' utils/trivy/trivy.yaml
+
+      - name: Run Trivy vulnerability scanner in repo mode
+        uses: aquasecurity/trivy-action@6e7b7d1fd3e4fef0c5fa8cce1229c54b2c9bd0d8  # 0.24.0
+        with:
+          scan-type: 'fs'
+          scan-ref: '.'
+          trivy-config: 'utils/trivy/trivy.yaml'
+
+      - name: Upload Trivy scan results to GitHub Security tab
+        uses: github/codeql-action/upload-sarif@afb54ba388a7dca6ecae48f608c4ff05ff4cc77a
+        # 3.25.15 (v3)
+        with:
+          sarif_file: 'trivy-results.sarif'
+
+      - name: Adjust config file to show and validate scan results
+        run: |
+          sed -i 's/output: "trivy-results.sarif"//g' utils/trivy/trivy.yaml
+          sed -i 's/format: sarif/format: table/g' utils/trivy/trivy.yaml
+          sed -i 's/exit-code: 0/exit-code: 1/g' utils/trivy/trivy.yaml
+
+      - name: Run Trivy vulnerability scanner in repo mode
+        uses: aquasecurity/trivy-action@6e7b7d1fd3e4fef0c5fa8cce1229c54b2c9bd0d8  # 0.24.0
+        with:
+          scan-type: 'fs'
+          scan-ref: '.'
+          trivy-config: 'utils/trivy/trivy.yaml'

Jenkinsfile

@@ -801,6 +801,7 @@ pipeline {
                             unitTestPost artifacts: ['nlt_logs/'],
                                          testResults: 'nlt-junit.xml',
                                          always_script: 'ci/unit/test_nlt_post.sh',
+					 referenceJobName: 'daos-stack/daos/release%252F2.6',
                                          valgrind_stash: 'el8-gcc-nlt-memcheck'
                             recordIssues enabledForFailure: true,
                                          failOnError: false,
@@ -1037,7 +1038,7 @@ pipeline {
                     }
                     post {
                         always {
-                            discoverGitReferenceBuild referenceJob: 'daos-stack/daos/master',
+                            discoverGitReferenceBuild referenceJob: 'daos-stack/daos/release%252F2.6',
                                                       scm: 'daos-stack/daos',
                                                       requiredResult: hudson.model.Result.UNSTABLE
                             recordIssues enabledForFailure: true,

debian/changelog

@@ -1,3 +1,19 @@
+daos (2.6.1-4) unstable; urgency=medium
+  [ Tomasz Gromadzki ]
+  * Add support of the PMDK package 2.1.0 with NDCTL enabled.
+    * Increase the default ULT stack size to 20KiB if the engine uses
+      the DCPM storage class.
+    * Prevent using the RAM storage class (simulated PMem) when
+      the shutdown state (SDS) is active.
+      * Automatically disable SDS for the RAM storage class on engine startup.
+      * Force explicitly setting the PMEMOBJ_CONF='sds.at_create=0'
+        environment variable to deactivate SDS for the DAOS tools
+        (ddb, daos_perf, vos_perf, etc.) when used WITHOUT DCPM.
+        Otherwise, a user is supposed to be stopped by an error
+        like: "Unsafe shutdown count is not supported for this source".
+
+ -- Tomasz Gromadzki <[email protected]>  Wed, 02 Oct 2024 12:00:00 +0200
+
 daos (2.6.1-3) unstable; urgency=medium
   [ Phillip Henderson ]
   * Third release candidate for 2.6.1

debian/control

@@ -18,7 +18,7 @@ Build-Depends: debhelper (>= 10),
                python3-distro,
                libabt-dev,
                libucx-dev,
-               libpmemobj-dev (>= 2.0.0),
+               libpmemobj-dev (>= 2.1.0),
                libfuse3-dev,
                libprotobuf-c-dev,
                libjson-c-dev,
@@ -118,7 +118,9 @@ Depends: python (>=3.8), python3, python-yaml, python3-yaml,
          daos-client (= ${binary:Version}),
          daos-admin (= ${binary:Version}),
          golang-go (>= 2:1.21),
-         libcapstone-dev
+         libcapstone-dev,
+         libndctl-dev,
+         libdaxctl-dev
 Description: The Distributed Asynchronous Object Storage (DAOS) is an open-source
  software-defined object store designed from the ground up for
  massively distributed Non Volatile Memory (NVM). DAOS takes advantage

docs/admin/env_variables.md

@@ -44,6 +44,7 @@ Environment variables in this section only apply to the server side.
 |DAOS\_MD\_CAP         |Size of a metadata pmem pool/file in MBs. INTEGER. Default to 128 MB.|
 |DAOS\_START\_POOL\_SVC|Determines whether to start existing pool services when starting a daos\_server. BOOL. Default to true.|
 |CRT\_DISABLE\_MEM\_PIN|Disable memory pinning workaround on a server side. BOOL. Default to 0.|
+|CRT\_EVENT\_DELAY|Delay in seconds before handling a set of CaRT events. INTEGER. Default to 10 s. A longer delay enables batching of successive CaRT events, leading to fewer pool map changes when multiple engines become unavailable at around the same time.|
 |DAOS\_SCHED\_PRIO\_DISABLED|Disable server ULT prioritizing. BOOL. Default to 0.|
 |DAOS\_SCHED\_RELAX\_MODE|The mode of CPU relaxing on idle. "disabled":disable relaxing; "net":wait on network request for INTVL; "sleep":sleep for INTVL. STRING. Default to "net"|
 |DAOS\_SCHED\_RELAX\_INTVL|CPU relax interval in milliseconds. INTEGER. Default to 1 ms.|

site_scons/components/__init__.py

@@ -268,7 +268,6 @@ def define_components(reqs):
                 retriever=GitRepoRetriever(),
                 commands=[['make',
                            'all',
-                           'NDCTL_ENABLE=n',
                            'BUILD_EXAMPLES=n',
                            'BUILD_BENCHMARKS=n',
                            'DOC=n',

src/cart/README.env

@@ -1,13 +1,10 @@
 This file lists the environment variables used in CaRT.
 
  . D_PROVIDER (Deprecated: CRT_PHY_ADDR_STR)
-   It determines which mercury NA plugin to be used:
+   It determines which mercury NA plugin and transport to be used:
    - set it as "ofi+verbs;ofi_rxm" to use OFI verbs;ofi_rxm provider
-   - set it as "ofi+gni" to use OFI gni provider
    - set it as "sm" to use SM plugin which only works within single node
-   - set it as "ofi+tcp;ofi_rxm" to use OFI tcp;ofi_rxm provider.
-   - set it as "ofi+sockets" to use OFI sockets provider
-      NOTE: This provider is deprecated in favor of "ofi+tcp;ofi_rxm"
+   - set it as "ofi+tcp" to use OFI tcp provider.
    - by default (not set or set as any other value) it will use ofi tcp
       provider.
 
@@ -186,3 +183,27 @@ This file lists the environment variables used in CaRT.
    Set parent directory for client side metrics. Each client will write its metrics to
    a file with the pattern <D_CLIENT_METRICS_DUMP_DIR>/<DAOS_JOBID>-<pid>.csv. As a
    convenience, setting this variable automatically sets D_CLIENT_METRICS_ENABLE=1.
+
+   D_POST_INIT
+   (server only) Controls the initial number of requests that are posted on context creation.
+   When using a transport that supports multi-recv, also controls the maximum size
+   of buffers (DAOS_RPC_SIZE x D_POST_INIT x D_MRECV_BUF).
+
+   D_POST_INCR
+   (server only) Controls the number of RPC handles that are incrementally posted when the
+   initial number of requests (D_POST_INIT) is exhausted.
+
+   D_MRECV_BUF
+   (server only) When using a transport that supports multi-recv, controls the total number
+   of multi-recv buffers that are posted.
+
+   D_MRECV_BUF_COPY
+   (server only) When using a transport that supports multi-recv, controls when we should
+   start copying data in an effort to release multi-recv buffers. Copy will occur when at
+   most D_MRECV_BUF_COPY buffers remain.
+
+   SWIM_TRAFFIC_CLASS
+   (server only) Select a traffic class for the SWIM protocol to use and prevent potential
+   traffic congestion. Available options are: "unspec" (default), "best_effort",
+   "low_latency", "bulk_data".
+

src/cart/crt_hg.c

@@ -859,10 +859,15 @@ crt_hg_class_init(crt_provider_t provider, int ctx_idx, bool primary, int iface_
 	if (prov_data->cpg_max_unexp_size > 0)
 		init_info.na_init_info.max_unexpected_size = prov_data->cpg_max_unexp_size;
 
-	init_info.request_post_init = crt_gdata.cg_post_init;
-	init_info.request_post_incr = crt_gdata.cg_post_incr;
-
-	hg_class = HG_Init_opt(info_string, crt_is_service(), &init_info);
+	init_info.request_post_init         = crt_gdata.cg_post_init;
+	init_info.request_post_incr         = crt_gdata.cg_post_incr;
+	init_info.multi_recv_op_max         = crt_gdata.cg_mrecv_buf;
+	init_info.multi_recv_copy_threshold = crt_gdata.cg_mrecv_buf_copy;
+	/* Separate SWIM traffic in an effort to prevent potential congestion. */
+	if (crt_is_service() && ctx_idx == crt_gdata.cg_swim_crt_idx)
+		init_info.traffic_class = (enum na_traffic_class)crt_gdata.cg_swim_tc;
+
+	hg_class = HG_Init_opt2(info_string, crt_is_service(), HG_VERSION(2, 4), &init_info);
 	if (hg_class == NULL) {
 		D_ERROR("Could not initialize HG class.\n");
 		D_GOTO(out, rc = -DER_HG);
@@ -1477,6 +1482,16 @@ crt_hg_reply_send(struct crt_rpc_priv *rpc_priv)
 		rc = crt_hgret_2_der(hg_ret);
 	}
 
+	/* Release input buffer */
+	if (rpc_priv->crp_release_input_early && !rpc_priv->crp_forward) {
+		hg_ret = HG_Release_input_buf(rpc_priv->crp_hg_hdl);
+		if (hg_ret != HG_SUCCESS) {
+			RPC_ERROR(rpc_priv, "HG_Release_input_buf failed, hg_ret: " DF_HG_RC "\n",
+				  DP_HG_RC(hg_ret));
+			/* Fall through */
+		}
+	}
+
 	return rc;
 }
 

src/cart/crt_init.c

@@ -18,6 +18,10 @@ static volatile int	gdata_init_flag;
 struct crt_plugin_gdata crt_plugin_gdata;
 static bool		g_prov_settings_applied[CRT_PROV_COUNT];
 
+#define X(a, b) b,
+static const char *const crt_tc_name[] = {CRT_TRAFFIC_CLASSES};
+#undef X
+
 static void
 crt_lib_init(void) __attribute__((__constructor__));
 
@@ -237,16 +241,30 @@ crt_gdata_dump(void)
 	DUMP_GDATA_FIELD("%d", cg_rpc_quota);
 }
 
+static enum crt_traffic_class
+crt_str_to_tc(const char *str)
+{
+	enum crt_traffic_class i = 0;
+
+	while (str != NULL && strcmp(crt_tc_name[i], str) != 0 && i < CRT_TC_UNKNOWN)
+		i++;
+
+	return i == CRT_TC_UNKNOWN ? CRT_TC_UNSPEC : i;
+}
+
 /* first step init - for initializing crt_gdata */
 static int data_init(int server, crt_init_options_t *opt)
 {
-	uint32_t        timeout = 0;
-	uint32_t	credits;
-	uint32_t	fi_univ_size = 0;
-	uint32_t	mem_pin_enable = 0;
-	uint32_t        is_secondary;
-	uint32_t        post_init = CRT_HG_POST_INIT, post_incr = CRT_HG_POST_INCR;
-	int		rc = 0;
+	uint32_t     timeout = 0;
+	uint32_t     credits;
+	uint32_t     fi_univ_size   = 0;
+	uint32_t     mem_pin_enable = 0;
+	uint32_t     is_secondary;
+	uint32_t     post_init = CRT_HG_POST_INIT, post_incr = CRT_HG_POST_INCR;
+	unsigned int mrecv_buf          = CRT_HG_MRECV_BUF;
+	unsigned int mrecv_buf_copy     = 0; /* buf copy disabled by default */
+	char        *swim_traffic_class = NULL;
+	int          rc                 = 0;
 
 	crt_env_dump();
 
@@ -255,6 +273,12 @@ static int data_init(int server, crt_init_options_t *opt)
 	crt_gdata.cg_post_init = post_init;
 	crt_env_get(D_POST_INCR, &post_incr);
 	crt_gdata.cg_post_incr = post_incr;
+	crt_env_get(D_MRECV_BUF, &mrecv_buf);
+	crt_gdata.cg_mrecv_buf = mrecv_buf;
+	crt_env_get(D_MRECV_BUF_COPY, &mrecv_buf_copy);
+	crt_gdata.cg_mrecv_buf_copy = mrecv_buf_copy;
+	crt_env_get(SWIM_TRAFFIC_CLASS, &swim_traffic_class);
+	crt_gdata.cg_swim_tc = crt_str_to_tc(swim_traffic_class);
 
 	is_secondary = 0;
 	/* Apply CART-890 workaround for server side only */

src/cart/crt_internal_types.h

@@ -42,6 +42,17 @@ struct crt_na_config {
 	char            **noc_domain_str; /* Array of domains */
 };
 
+#define CRT_TRAFFIC_CLASSES                                                                        \
+	X(CRT_TC_UNSPEC, "unspec")           /* Leave it upon plugin to choose */                  \
+	X(CRT_TC_BEST_EFFORT, "best_effort") /* Best effort */                                     \
+	X(CRT_TC_LOW_LATENCY, "low_latency") /* Low latency */                                     \
+	X(CRT_TC_BULK_DATA, "bulk_data")     /* Bulk data */                                       \
+	X(CRT_TC_UNKNOWN, "unknown")         /* Unknown */
+
+#define X(a, b) a,
+enum crt_traffic_class { CRT_TRAFFIC_CLASSES };
+#undef X
+
 struct crt_prov_gdata {
 	/** NA plugin type */
 	int			cpg_provider;
@@ -103,6 +114,9 @@ struct crt_gdata {
 	/** global swim index for all servers */
 	int32_t			cg_swim_crt_idx;
 
+	/** traffic class used by SWIM */
+	enum crt_traffic_class   cg_swim_tc;
+
 	/** credits limitation for #in-flight RPCs per target EP CTX */
 	uint32_t		cg_credit_ep_ctx;
 
@@ -216,6 +230,7 @@ struct crt_event_cb_priv {
 	ENV(SWIM_PING_TIMEOUT)                                                                     \
 	ENV(SWIM_PROTOCOL_PERIOD_LEN)                                                              \
 	ENV(SWIM_SUSPECT_TIMEOUT)                                                                  \
+	ENV_STR(SWIM_TRAFFIC_CLASS)                                                                \
 	ENV_STR(UCX_IB_FORK_INIT)
 
 /* uint env */

src/cart/crt_iv.c

@@ -1,5 +1,5 @@
 /*
- * (C) Copyright 2016-2023 Intel Corporation.
+ * (C) Copyright 2016-2024 Intel Corporation.
  *
  * SPDX-License-Identifier: BSD-2-Clause-Patent
  */
@@ -2911,8 +2911,12 @@ bulk_update_transfer_done_aux(const struct crt_bulk_cb_info *info)
 	return rc;
 
 send_error:
-	rc = crt_bulk_free(cb_info->buc_bulk_hdl);
+	/* send back whatever error got us here */
 	output->rc = rc;
+	rc         = crt_bulk_free(cb_info->buc_bulk_hdl);
+	if (rc != 0)
+		DL_ERROR(rc, "crt_bulk_free() failed");
+
 	iv_ops->ivo_on_put(ivns_internal, &cb_info->buc_iv_value,
 			   cb_info->buc_user_priv);
 

src/cart/crt_rpc.c

@@ -1550,6 +1550,26 @@ crt_req_send(crt_rpc_t *req, crt_cb_t complete_cb, void *arg)
 	return rc;
 }
 
+int
+crt_reply_send_input_free(crt_rpc_t *req)
+{
+	struct crt_rpc_priv *rpc_priv = NULL;
+	int                  rc       = 0;
+
+	if (req == NULL) {
+		D_ERROR("invalid parameter (NULL req).\n");
+		D_GOTO(out, rc = -DER_INVAL);
+	}
+
+	rpc_priv                          = container_of(req, struct crt_rpc_priv, crp_pub);
+	rpc_priv->crp_release_input_early = 1;
+
+	return crt_reply_send(req);
+
+out:
+	return rc;
+}
+
 int
 crt_reply_send(crt_rpc_t *req)
 {