mjmac/DAOS 8331 no agent

SConstruct

@@ -363,7 +363,7 @@ MINIMAL_ENV = ('HOME', 'TERM', 'SSH_AUTH_SOCK', 'http_proxy', 'https_proxy', 'PK
 
 # Environment variables that are also kept when LD_PRELOAD is set.
 PRELOAD_ENV = ('LD_PRELOAD', 'D_LOG_FILE', 'DAOS_AGENT_DRPC_DIR', 'D_LOG_MASK', 'DD_MASK',
-               'DD_SUBSYS', 'D_IL_MAX_EQ')
+               'DD_SUBSYS', 'D_IL_MAX_EQ', 'D_IL_ENFORCE_EXEC_ENV', 'D_IL_COMPATIBLE')
 
 
 def scons():

docs/user/filesystem.md

@@ -1020,6 +1020,10 @@ libpil4dfs intercepting summary for ops on DFS:
 [op_sum ]  5003
 ```
 
+### Turn on compatible mode in libpil4dfs
+Fake file descriptor (FD) is used in regular mode in libpil4dfs.so for efficiency. open() returns fake fd to applications. In cases of some APIs are not intercepted, applications could crash with the error "Bad File Descriptor". Compatible mode is provided to work around such situations.
+Setting env "D_IL_COMPATIBLE=1" turns on compatible mode. Kernel fd allocated by dfuse instead of fake fd will be returned to applications. This mode provides better compatibility with degraded performance in open, openat, and opendir, etc. Please start dfuse with "--disable-caching" to disable caching before using compatible mode.
+
 ### Child Process Inheritance
 
 Normally child processes inherit environmental variables from parent processes. In rare cases, e.g.

site_scons/prereq_tools/base.py

@@ -1264,8 +1264,6 @@ def set_environment(self, env, needed_libs):
                 lib_paths.append(full_path)
                 # will adjust this to be a relative rpath later
                 env.AppendUnique(RPATH_FULL=[full_path])
-                # For binaries run during build
-                env.AppendENVPath("LD_LIBRARY_PATH", full_path)
 
             # Ensure RUNPATH is used rather than RPATH.  RPATH is deprecated
             # and this allows LD_LIBRARY_PATH to override RPATH

src/cart/README.env

@@ -153,20 +153,9 @@ This file lists the environment variables used in CaRT.
    If it is not set the default value of 64 is used.
    Setting it to 0 disables quota
 
- . CRT_CTX_SHARE_ADDR
-   Set it to non-zero to make all the contexts share one network address, in
-   this case CaRT will create one SEP and each context maps to one tx/rx
-   context pair.
-   When the ENV not set or set to 0 each context will create one separate SEP.
-
  . CRT_CTX_NUM
-   When in scalable endpoint mode, this envariable specifies the number of contexts
-   that user wants to create. If CRT_CTX_NUM exceeds the OFI provider capability,
-   NA layer will fail to initialize. If user creates more contexts than CRT_CTX_NUM,
-   context creation will fail.
-   For regular (non scalable endpoint) mode:
-    - Maximum number of cart contexts is set to number of cores by default, up to 64.
-    - If CRT_CTX_NUM is set, this value is used instead as a limit.
+   If set, specifies the limit of number of allowed CaRT contexts to be created.
+   Valid range is [1, 64], with default being 64 if unset.
 
  . D_FI_CONFIG
    Specifies the fault injection configuration file. If this variable is not set
@@ -185,12 +174,15 @@ This file lists the environment variables used in CaRT.
 
  . D_CLIENT_METRICS_ENABLE
    When set to 1, client side metrics will be collected on each daos client, which
-   can by retrieved by daos_metrics -j job_id on each client.
+   can by retrieved by daos_metrics -j job_id on each client. Only needed if
+   daos_agent is not configured to enable client metrics for all connected processes.
 
  . D_CLIENT_METRICS_RETAIN
-   when set to 1, client side metrics will be retained even after the job exits, i.e.
-   those metrics can be retrieved by daos_metrics even after job exits.
-
- . D_CLIENT_METRICS_DUMP_PATH
-   Set client side metrics dump path(file) for each client, so these metrics will be
-   dumped to the specified file when the job exits.
+   When set to 1, client side metrics will be retained even after the job exits, i.e.
+   those metrics can be retrieved by daos_metrics even after job exits. Normally
+   managed by daos_agent.
+
+ . D_CLIENT_METRICS_DUMP_DIR
+   Set parent directory for client side metrics. Each client will write its metrics to
+   a file with the pattern <D_CLIENT_METRICS_DUMP_DIR>/<DAOS_JOBID>-<pid>.csv. As a
+   convenience, setting this variable automatically sets D_CLIENT_METRICS_ENABLE=1.

src/cart/crt_init.c

@@ -57,8 +57,7 @@ static const char      *crt_env_names[] = {
     "DAOS_SIGNAL_REGISTER",
     "D_CLIENT_METRICS_ENABLE",
     "D_CLIENT_METRICS_RETAIN",
-    "D_CLIENT_METRICS_DUMP_PATH",
-
+    "D_CLIENT_METRICS_DUMP_DIR",
 };
 
 static void
@@ -204,8 +203,8 @@ prov_data_init(struct crt_prov_gdata *prov_data, crt_provider_t provider,
 		return rc;
 
 	if (crt_is_service()) {
-		ctx_num = CRT_SRV_CONTEXT_NUM;
-		max_num_ctx = CRT_SRV_CONTEXT_NUM;
+		ctx_num		= CRT_SRV_CONTEXT_NUM;
+		max_num_ctx	= CRT_SRV_CONTEXT_NUM;
 	} else {
 		/* Only limit the number of contexts for clients */
 		d_getenv_uint("CRT_CTX_NUM", &ctx_num);
@@ -220,13 +219,19 @@ prov_data_init(struct crt_prov_gdata *prov_data, crt_provider_t provider,
 
 	if (max_num_ctx > CRT_SRV_CONTEXT_NUM)
 		max_num_ctx = CRT_SRV_CONTEXT_NUM;
-
 	/* To be able to run on VMs */
 	if (max_num_ctx < CRT_SRV_CONTEXT_NUM_MIN)
 		max_num_ctx = CRT_SRV_CONTEXT_NUM_MIN;
 
 	D_DEBUG(DB_ALL, "Max number of contexts set to %d\n", max_num_ctx);
 
+	d_getenv_bool("CRT_CTX_SHARE_ADDR", &set_sep);
+	if (set_sep)
+		D_WARN("Unsupported SEP mode requested. Unset CRT_CTX_SHARE_ADDR\n");
+
+	if (opt && opt->cio_sep_override && opt->cio_use_sep)
+		D_WARN("Unsupported SEP mode requested in init options\n");
+
 	if (opt && opt->cio_use_expected_size)
 		max_expect_size = opt->cio_max_expected_size;
 

src/cart/utils/crt_utils.c

@@ -415,7 +415,6 @@ crtu_dc_mgmt_net_cfg_setenv(const char *name)
 {
 	int                      rc;
 	char                    *provider;
-	char                    *crt_ctx_share_addr = NULL;
 	char                    *cli_srx_set        = NULL;
 	char                    *crt_timeout        = NULL;
 	char                    *d_interface;
@@ -443,16 +442,6 @@ crtu_dc_mgmt_net_cfg_setenv(const char *name)
 	if (rc != 0)
 		D_GOTO(cleanup, rc = d_errno2der(errno));
 
-	rc = asprintf(&crt_ctx_share_addr, "%d", crt_net_cfg_info.crt_ctx_share_addr);
-	if (rc < 0) {
-		crt_ctx_share_addr = NULL;
-		D_GOTO(cleanup, rc = -DER_NOMEM);
-	}
-	D_INFO("setenv CRT_CTX_SHARE_ADDR=%s\n", crt_ctx_share_addr);
-	rc = d_setenv("CRT_CTX_SHARE_ADDR", crt_ctx_share_addr, 1);
-	if (rc != 0)
-		D_GOTO(cleanup, rc = d_errno2der(errno));
-
 	/* If the server has set this, the client must use the same value. */
 	if (crt_net_cfg_info.srv_srx_set != -1) {
 		rc = asprintf(&cli_srx_set, "%d", crt_net_cfg_info.srv_srx_set);
@@ -464,8 +453,6 @@ crtu_dc_mgmt_net_cfg_setenv(const char *name)
 		rc = d_setenv("FI_OFI_RXM_USE_SRX", cli_srx_set, 1);
 		if (rc != 0)
 			D_GOTO(cleanup, rc = d_errno2der(errno));
-
-		D_DEBUG(DB_MGMT, "Using server's value for FI_OFI_RXM_USE_SRX: %s\n", cli_srx_set);
 	} else {
 		/* Client may not set it if the server hasn't. */
 		d_agetenv_str(&cli_srx_set, "FI_OFI_RXM_USE_SRX");
@@ -501,9 +488,7 @@ crtu_dc_mgmt_net_cfg_setenv(const char *name)
 			D_GOTO(cleanup, rc = d_errno2der(errno));
 	} else {
 		d_interface = d_interface_env;
-		D_DEBUG(DB_MGMT,
-			"Using client provided D_INTERFACE: %s\n",
-			d_interface);
+		D_DEBUG(DB_MGMT, "Using client provided D_INTERFACE: %s\n", d_interface);
 	}
 
 	d_agetenv_str(&d_domain_env, "D_DOMAIN");
@@ -519,16 +504,14 @@ crtu_dc_mgmt_net_cfg_setenv(const char *name)
 	}
 
 	D_INFO("CaRT env setup with:\n"
-	       "\tD_INTERFACE=%s, D_DOMAIN: %s, D_PROVIDER: %s, "
-	       "CRT_CTX_SHARE_ADDR: %s, CRT_TIMEOUT: %s\n",
-	       d_interface, d_domain, provider, crt_ctx_share_addr, crt_timeout);
+	       "\tD_INTERFACE=%s, D_DOMAIN: %s, D_PROVIDER: %s, CRT_TIMEOUT: %s\n",
+	       d_interface, d_domain, provider, crt_timeout);
 
 cleanup:
 	d_freeenv_str(&d_domain_env);
 	d_freeenv_str(&d_interface_env);
 	d_freeenv_str(&crt_timeout);
 	d_freeenv_str(&cli_srx_set);
-	d_freeenv_str(&crt_ctx_share_addr);
 	dc_put_attach_info(&crt_net_cfg_info, crt_net_cfg_resp);
 
 	return rc;

src/chk/chk_common.c

@@ -1238,6 +1238,8 @@ chk_ins_init(struct chk_instance **p_ins)
 out_init:
 	if (rc == 0)
 		*p_ins = ins;
+	else
+		D_FREE(ins);
 
 	return rc;
 }

src/chk/chk_engine.c

@@ -2933,7 +2933,7 @@ chk_engine_pool_start(uint64_t gen, uuid_t uuid, uint32_t phase, uint32_t flags)
 		D_GOTO(put, rc = (rc == -DER_NONEXIST ? 1 : rc));
 
 	if (cbk->cb_phase < phase) {
-		cbk->cb_phase = cbk->cb_phase;
+		cbk->cb_phase = phase;
 		/* QUEST: How to estimate the left time? */
 		cbk->cb_time.ct_left_time = CHK__CHECK_SCAN_PHASE__CSP_DONE - cbk->cb_phase;
 		rc = chk_bk_update_pool(cbk, uuid_str);

src/chk/chk_leader.c

@@ -1396,7 +1396,7 @@ chk_leader_start_pool_svc(struct chk_pool_rec *cpr)
 
 	rc = ds_rsvc_dist_start(DS_RSVC_CLASS_POOL, &psid, cpr->cpr_uuid, ranks, RDB_NIL_TERM,
 				cpr->cpr_healthy ? DS_RSVC_START : DS_RSVC_DICTATE,
-				false /* bootstrap */, 0 /* size */);
+				false /* bootstrap */, 0 /* size */, 0 /* vos_df_version */);
 
 out:
 	d_rank_list_free(ranks);
@@ -3385,8 +3385,7 @@ chk_leader_prop(chk_prop_cb_t prop_cb, void *buf)
 {
 	struct chk_property	*prop = &chk_leader->ci_prop;
 
-	return prop_cb(buf, (struct chk_policy *)prop->cp_policies,
-		       CHK_POLICY_MAX - 1, prop->cp_flags);
+	return prop_cb(buf, prop->cp_policies, CHK_POLICY_MAX - 1, prop->cp_flags);
 }
 
 static int

src/chk/chk_upcall.c

@@ -94,8 +94,6 @@ chk_report_upcall(uint64_t gen, uint64_t seq, uint32_t cla, uint32_t act, int re
 		D_ASPRINTF(report.pool_uuid, DF_UUIDF, DP_UUID(*pool));
 		if (report.pool_uuid == NULL)
 			D_GOTO(out, rc = -DER_NOMEM);
-	} else {
-		report.pool_uuid = NULL;
 	}
 
 	report.pool_label = pool_label;
@@ -104,8 +102,6 @@ chk_report_upcall(uint64_t gen, uint64_t seq, uint32_t cla, uint32_t act, int re
 		D_ASPRINTF(report.cont_uuid, DF_UUIDF, DP_UUID(*cont));
 		if (report.cont_uuid == NULL)
 			D_GOTO(out, rc = -DER_NOMEM);
-	} else {
-		report.cont_uuid = NULL;
 	}
 
 	report.cont_label = cont_label;
@@ -114,24 +110,18 @@ chk_report_upcall(uint64_t gen, uint64_t seq, uint32_t cla, uint32_t act, int re
 		D_ASPRINTF(report.objid, DF_UOID, DP_UOID(*obj));
 		if (report.objid == NULL)
 			D_GOTO(out, rc = -DER_NOMEM);
-	} else {
-		report.objid = NULL;
 	}
 
 	if (!daos_iov_empty(dkey)) {
 		D_ASPRINTF(report.dkey, DF_KEY, DP_KEY(dkey));
 		if (report.dkey == NULL)
 			D_GOTO(out, rc = -DER_NOMEM);
-	} else {
-		report.dkey = NULL;
 	}
 
 	if (!daos_iov_empty(akey)) {
 		D_ASPRINTF(report.akey, DF_KEY, DP_KEY(akey));
 		if (report.akey == NULL)
 			D_GOTO(out, rc = -DER_NOMEM);
-	} else {
-		report.akey = NULL;
 	}
 
 	D_ASPRINTF(report.timestamp, "%s", ctime(&tm));
@@ -150,20 +140,23 @@ chk_report_upcall(uint64_t gen, uint64_t seq, uint32_t cla, uint32_t act, int re
 			goto out;
 
 		report.n_act_details = rc;
-	} else {
-		report.n_act_details = 0;
-		report.act_details = NULL;
 	}
 
 	rc = ds_chk_report_upcall(&report);
 
 out:
-	D_FREE(report.pool_uuid);
-	D_FREE(report.cont_uuid);
-	D_FREE(report.objid);
-	D_FREE(report.dkey);
-	D_FREE(report.akey);
-	D_FREE(report.timestamp);
+	if (report.pool_uuid != protobuf_c_empty_string)
+		D_FREE(report.pool_uuid);
+	if (report.cont_uuid != protobuf_c_empty_string)
+		D_FREE(report.cont_uuid);
+	if (report.objid != protobuf_c_empty_string)
+		D_FREE(report.objid);
+	if (report.dkey != protobuf_c_empty_string)
+		D_FREE(report.dkey);
+	if (report.akey != protobuf_c_empty_string)
+		D_FREE(report.akey);
+	if (report.timestamp != protobuf_c_empty_string)
+		D_FREE(report.timestamp);
 	chk_sg_free(report.act_details, report.n_act_details);
 
 	D_CDEBUG(rc != 0, DLOG_ERR, DLOG_INFO,

src/client/api/init.c

@@ -147,7 +147,7 @@ daos_init(void)
 {
 	struct d_fault_attr_t *d_fault_init;
 	struct d_fault_attr_t *d_fault_mem = NULL;
-	struct d_fault_attr_t d_fault_mem_saved;
+	struct d_fault_attr_t  d_fault_mem_saved;
 	int rc;
 
 	D_MUTEX_LOCK(&module_lock);
@@ -196,19 +196,24 @@ daos_init(void)
 	if (rc != 0)
 		D_GOTO(out_agent, rc);
 
+	/** get and cache attach info of default system */
+	rc = dc_mgmt_cache_attach_info(NULL);
+	if (rc != 0)
+		D_GOTO(out_job, rc);
+
 	/**
 	 * get CaRT configuration (see mgmtModule.handleGetAttachInfo for the
 	 * handling of NULL system names)
 	 */
 	rc = dc_mgmt_net_cfg(NULL);
 	if (rc != 0)
-		D_GOTO(out_job, rc);
+		D_GOTO(out_attach, rc);
 
 	/** set up event queue */
 	rc = daos_eq_lib_init();
 	if (rc != 0) {
 		D_ERROR("failed to initialize eq_lib: "DF_RC"\n", DP_RC(rc));
-		D_GOTO(out_job, rc);
+		D_GOTO(out_attach, rc);
 	}
 
 	/**
@@ -274,6 +279,8 @@ daos_init(void)
 	pl_fini();
 out_eq:
 	daos_eq_lib_fini();
+out_attach:
+	dc_mgmt_drop_attach_info();
 out_job:
 	dc_job_fini();
 out_agent:
@@ -334,6 +341,7 @@ daos_fini(void)
 			DF_RC"\n", DP_RC(rc));
 
 	dc_tm_fini();
+	dc_mgmt_drop_attach_info();
 	dc_agent_fini();
 	dc_job_fini();