From ea9ae1211a2f496b0baf759dd8ef2c2c34b1d275 Mon Sep 17 00:00:00 2001 From: Joseph Moore <26410038+jgmoore-or@users.noreply.github.com> Date: Thu, 22 Aug 2024 09:39:01 -0600 Subject: [PATCH 01/26] DAOS-15439 cart: Support free-form provider specification for UCX. (#14911) - Support free-form provider specification for ucx Signed-off-by: Joseph Moore --- src/cart/crt_hg.c | 137 ++++++++-------------- src/cart/crt_hg.h | 25 ++-- src/cart/crt_init.c | 270 +++++++++++++++++++++++--------------------- 3 files changed, 199 insertions(+), 233 deletions(-) diff --git a/src/cart/crt_hg.c b/src/cart/crt_hg.c index 8f5687e88b0..c1d1b3a3f9b 100644 --- a/src/cart/crt_hg.c +++ b/src/cart/crt_hg.c @@ -14,97 +14,52 @@ * List of supported CaRT providers. The table is terminated with the last entry * having nad_str = NULL. */ -struct crt_na_dict crt_na_dict[] = { - { - .nad_type = CRT_PROV_SM, - .nad_str = "sm", - .nad_contig_eps = false, - .nad_port_bind = false, - }, { - .nad_type = CRT_PROV_OFI_VERBS_RXM, - .nad_str = "ofi+verbs;ofi_rxm", - .nad_alt_str = "ofi+verbs", - .nad_contig_eps = true, - .nad_port_bind = true, - }, { - .nad_type = CRT_PROV_OFI_TCP, - .nad_str = "ofi+tcp", - .nad_contig_eps = true, - .nad_port_bind = true, - }, { - .nad_type = CRT_PROV_OFI_TCP_RXM, - .nad_str = "ofi+tcp;ofi_rxm", - .nad_contig_eps = true, - .nad_port_bind = true, - }, { - .nad_type = CRT_PROV_OFI_CXI, - .nad_str = "ofi+cxi", - .nad_contig_eps = true, - .nad_port_bind = false, - }, { - .nad_type = CRT_PROV_OFI_OPX, - .nad_str = "ofi+opx", - .nad_contig_eps = false, - .nad_port_bind = true, - }, { - .nad_type = CRT_PROV_UCX_RC, - .nad_str = "ucx+rc_v", - .nad_contig_eps = true, - .nad_port_bind = true, - }, { - .nad_type = CRT_PROV_UCX_UD, - .nad_str = "ucx+ud_v", - .nad_contig_eps = true, - .nad_port_bind = true, - }, { - .nad_type = CRT_PROV_UCX_RC_UD, - .nad_str = "ucx+rc_v,ud_v", - .nad_contig_eps = true, - .nad_port_bind = true, - }, { - .nad_type = CRT_PROV_UCX_RC_O, - .nad_str = "ucx+rc", - .nad_contig_eps = true, - .nad_port_bind = true, - }, { - .nad_type = CRT_PROV_UCX_UD_O, - .nad_str = "ucx+ud", - .nad_contig_eps = true, - .nad_port_bind = true, - }, { - .nad_type = CRT_PROV_UCX_RC_UD_O, - .nad_str = "ucx+rc,ud", - .nad_contig_eps = true, - .nad_port_bind = true, - }, { - .nad_type = CRT_PROV_UCX_RC_X, - .nad_str = "ucx+rc_x", - .nad_contig_eps = true, - .nad_port_bind = true, - }, { - .nad_type = CRT_PROV_UCX_UD_X, - .nad_str = "ucx+ud_x", - .nad_contig_eps = true, - .nad_port_bind = true, - }, { - .nad_type = CRT_PROV_UCX_RC_UD_X, - .nad_str = "ucx+rc_x,ud_x", - .nad_contig_eps = true, - .nad_port_bind = true, - }, { - .nad_type = CRT_PROV_UCX_DC_X, - .nad_str = "ucx+dc_x", - .nad_contig_eps = true, - .nad_port_bind = true, - }, { - .nad_type = CRT_PROV_UCX_TCP, - .nad_str = "ucx+tcp", - .nad_contig_eps = true, - .nad_port_bind = true, - }, { - .nad_str = NULL, - } -}; +struct crt_na_dict crt_na_dict[] = {{ + .nad_type = CRT_PROV_SM, + .nad_str = "sm", + .nad_contig_eps = false, + .nad_port_bind = false, + }, + { + .nad_type = CRT_PROV_OFI_VERBS_RXM, + .nad_str = "ofi+verbs;ofi_rxm", + .nad_alt_str = "ofi+verbs", + .nad_contig_eps = true, + .nad_port_bind = true, + }, + { + .nad_type = CRT_PROV_OFI_TCP, + .nad_str = "ofi+tcp", + .nad_contig_eps = true, + .nad_port_bind = true, + }, + { + .nad_type = CRT_PROV_OFI_TCP_RXM, + .nad_str = "ofi+tcp;ofi_rxm", + .nad_contig_eps = true, + .nad_port_bind = true, + }, + { + .nad_type = CRT_PROV_OFI_CXI, + .nad_str = "ofi+cxi", + .nad_contig_eps = true, + .nad_port_bind = false, + }, + { + .nad_type = CRT_PROV_OFI_OPX, + .nad_str = "ofi+opx", + .nad_contig_eps = false, + .nad_port_bind = true, + }, + { + .nad_type = CRT_PROV_UCX, + .nad_str = "ucx+ud_x", + .nad_contig_eps = true, + .nad_port_bind = true, + }, + { + .nad_str = NULL, + }}; int crt_hg_parse_uri(const char *uri, crt_provider_t *prov, char *addr) diff --git a/src/cart/crt_hg.h b/src/cart/crt_hg.h index e9b4b6511e5..a72b9ae49af 100644 --- a/src/cart/crt_hg.h +++ b/src/cart/crt_hg.h @@ -32,6 +32,8 @@ #define CRT_HG_POST_INCR (512) #define CRT_HG_MRECV_BUF (16) +#define CRT_UCX_STR "ucx" + struct crt_rpc_priv; struct crt_common_hdr; struct crt_corpc_hdr; @@ -40,7 +42,7 @@ struct crt_corpc_hdr; * Enumeration specifying providers supported by the library */ typedef enum { - CRT_PROV_SM = 0, + CRT_PROV_SM = 0, CRT_PROV_OFI_SOCKETS, CRT_PROV_OFI_VERBS_RXM, CRT_PROV_OFI_GNI, @@ -48,19 +50,9 @@ typedef enum { CRT_PROV_OFI_TCP_RXM, CRT_PROV_OFI_CXI, CRT_PROV_OFI_OPX, - CRT_PROV_OFI_LAST = CRT_PROV_OFI_OPX, - CRT_PROV_UCX_RC, - CRT_PROV_UCX_UD, - CRT_PROV_UCX_RC_UD, - CRT_PROV_UCX_RC_O, - CRT_PROV_UCX_UD_O, - CRT_PROV_UCX_RC_UD_O, - CRT_PROV_UCX_RC_X, - CRT_PROV_UCX_UD_X, - CRT_PROV_UCX_RC_UD_X, - CRT_PROV_UCX_DC_X, - CRT_PROV_UCX_TCP, - CRT_PROV_UCX_LAST = CRT_PROV_UCX_TCP, + CRT_PROV_OFI_LAST = CRT_PROV_OFI_OPX, + CRT_PROV_UCX, + CRT_PROV_UCX_LAST = CRT_PROV_UCX, /* Note: This entry should be the last valid one in enum */ CRT_PROV_COUNT, CRT_PROV_UNKNOWN = -1, @@ -75,8 +67,7 @@ crt_hg_parse_uri(const char *uri, crt_provider_t *prov, char *addr); static inline bool crt_provider_is_ucx(crt_provider_t prov) { - return (prov >= CRT_PROV_UCX_RC) && - (prov <= CRT_PROV_UCX_LAST); + return (prov >= CRT_PROV_UCX) && (prov <= CRT_PROV_UCX_LAST); } static inline bool @@ -96,6 +87,8 @@ struct crt_na_dict { bool nad_port_bind; /** a flag to indicate if endpoints are contiguous */ bool nad_contig_eps; + /** a flag to indicate if nad_str is allocated on the heap */ + bool nad_str_alloc; }; extern struct crt_na_dict crt_na_dict[]; diff --git a/src/cart/crt_init.c b/src/cart/crt_init.c index f7e849728aa..d341a2a6dac 100644 --- a/src/cart/crt_init.c +++ b/src/cart/crt_init.c @@ -12,11 +12,11 @@ #include #include "crt_internal.h" -struct crt_gdata crt_gdata; +struct crt_gdata crt_gdata; struct crt_envs crt_genvs; -static volatile int gdata_init_flag; +static volatile int gdata_init_flag; struct crt_plugin_gdata crt_plugin_gdata; -static bool g_prov_settings_applied[CRT_PROV_COUNT]; +static bool g_prov_settings_applied[CRT_PROV_COUNT]; static void crt_lib_init(void) __attribute__((__constructor__)); @@ -28,9 +28,9 @@ crt_lib_fini(void) __attribute__((__destructor__)); static void crt_lib_init(void) { - int rc; - uint64_t start_rpcid; - struct timespec now; + int rc; + uint64_t start_rpcid; + struct timespec now; rc = D_RWLOCK_INIT(&crt_gdata.cg_rwlock, NULL); D_ASSERT(rc == 0); @@ -41,8 +41,8 @@ crt_lib_init(void) */ D_CASSERT(sizeof(uuid_t) == 16); - crt_gdata.cg_refcount = 0; - crt_gdata.cg_inited = 0; + crt_gdata.cg_refcount = 0; + crt_gdata.cg_inited = 0; crt_gdata.cg_primary_prov = CRT_PROV_OFI_TCP_RXM; rc = d_gettime(&now); @@ -50,8 +50,8 @@ crt_lib_init(void) d_srand(now.tv_sec * 1000 * 1000 * 1000 + now.tv_nsec + getpid()); start_rpcid = ((uint64_t)d_rand()) << 32; - crt_gdata.cg_rpcid = start_rpcid; - crt_gdata.cg_num_cores = sysconf(_SC_NPROCESSORS_ONLN); + crt_gdata.cg_rpcid = start_rpcid; + crt_gdata.cg_num_cores = sysconf(_SC_NPROCESSORS_ONLN); crt_gdata.cg_iv_inline_limit = 19456; /* 19KB */ /* envs not inited until crt_init() time */ @@ -90,16 +90,15 @@ dump_opt(crt_init_options_t *opt) } static int -crt_na_config_init(bool primary, crt_provider_t provider, - char *interface, char *domain, char *port, +crt_na_config_init(bool primary, crt_provider_t provider, char *interface, char *domain, char *port, char *auth_key, bool port_auto_adjust); /* Workaround for CART-890 */ static void mem_pin_workaround(void) { - struct rlimit rlim; - int rc = 0; + struct rlimit rlim; + int rc = 0; /* Note: mallopt() returns 1 on success */ /* Prevent malloc from releasing memory via sbrk syscall */ @@ -139,32 +138,32 @@ mem_pin_workaround(void) #define CRT_SRV_CONTEXT_NUM_MIN (16 + 2) static int -prov_data_init(struct crt_prov_gdata *prov_data, crt_provider_t provider, - bool primary, crt_init_options_t *opt) +prov_data_init(struct crt_prov_gdata *prov_data, crt_provider_t provider, bool primary, + crt_init_options_t *opt) { - uint32_t ctx_num = 0; - uint32_t max_expect_size = 0; - uint32_t max_unexpect_size = 0; - uint32_t max_num_ctx = CRT_SRV_CONTEXT_NUM; - int i; - int rc; + uint32_t ctx_num = 0; + uint32_t max_expect_size = 0; + uint32_t max_unexpect_size = 0; + uint32_t max_num_ctx = CRT_SRV_CONTEXT_NUM; + int i; + int rc; rc = D_MUTEX_INIT(&prov_data->cpg_mutex, NULL); if (rc != 0) return rc; if (crt_is_service()) { - ctx_num = CRT_SRV_CONTEXT_NUM; - max_num_ctx = CRT_SRV_CONTEXT_NUM; + ctx_num = CRT_SRV_CONTEXT_NUM; + max_num_ctx = CRT_SRV_CONTEXT_NUM; } else { /* Only limit the number of contexts for clients */ crt_env_get(CRT_CTX_NUM, &ctx_num); /* Default setting to the number of cores */ if (opt) - max_num_ctx = ctx_num ? ctx_num : - max(crt_gdata.cg_num_cores, opt->cio_ctx_max_num); + max_num_ctx = + ctx_num ? ctx_num : max(crt_gdata.cg_num_cores, opt->cio_ctx_max_num); else max_num_ctx = ctx_num ? ctx_num : crt_gdata.cg_num_cores; } @@ -183,15 +182,15 @@ prov_data_init(struct crt_prov_gdata *prov_data, crt_provider_t provider, if (opt && opt->cio_use_unexpected_size) max_unexpect_size = opt->cio_max_unexpected_size; - prov_data->cpg_inited = true; - prov_data->cpg_provider = provider; - prov_data->cpg_ctx_num = 0; + prov_data->cpg_inited = true; + prov_data->cpg_provider = provider; + prov_data->cpg_ctx_num = 0; prov_data->cpg_sep_mode = false; - prov_data->cpg_contig_ports = true; - prov_data->cpg_ctx_max_num = max_num_ctx; - prov_data->cpg_max_exp_size = max_expect_size; + prov_data->cpg_contig_ports = true; + prov_data->cpg_ctx_max_num = max_num_ctx; + prov_data->cpg_max_exp_size = max_expect_size; prov_data->cpg_max_unexp_size = max_unexpect_size; - prov_data->cpg_primary = primary; + prov_data->cpg_primary = primary; for (i = 0; i < CRT_SRV_CONTEXT_NUM; i++) prov_data->cpg_used_idx[i] = false; @@ -230,17 +229,18 @@ crt_gdata_dump(void) } /* first step init - for initializing crt_gdata */ -static int data_init(int server, crt_init_options_t *opt) +static int +data_init(int server, crt_init_options_t *opt) { - uint32_t timeout = 0; - uint32_t credits; - uint32_t fi_univ_size = 0; - uint32_t mem_pin_enable = 0; - uint32_t is_secondary; - uint32_t post_init = CRT_HG_POST_INIT, post_incr = CRT_HG_POST_INCR; - unsigned int mrecv_buf = CRT_HG_MRECV_BUF; - unsigned int mrecv_buf_copy = 0; /* buf copy disabled by default */ - int rc = 0; + uint32_t timeout = 0; + uint32_t credits; + uint32_t fi_univ_size = 0; + uint32_t mem_pin_enable = 0; + uint32_t is_secondary; + uint32_t post_init = CRT_HG_POST_INIT, post_incr = CRT_HG_POST_INCR; + unsigned int mrecv_buf = CRT_HG_MRECV_BUF; + unsigned int mrecv_buf_copy = 0; /* buf copy disabled by default */ + int rc = 0; crt_env_dump(); @@ -308,21 +308,22 @@ static int data_init(int server, crt_init_options_t *opt) /** Enable statistics only for the server side and if requested */ if (opt && opt->cio_use_sensors && server) { - int ret; + int ret; /** enable sensors */ crt_gdata.cg_use_sensors = true; /** set up the global sensors */ ret = d_tm_add_metric(&crt_gdata.cg_uri_self, D_TM_COUNTER, - "total number of URI requests for self", - "", "net/uri/lookup_self"); + "total number of URI requests for self", "", + "net/uri/lookup_self"); if (ret) D_WARN("Failed to create uri self sensor: " DF_RC "\n", DP_RC(ret)); ret = d_tm_add_metric(&crt_gdata.cg_uri_other, D_TM_COUNTER, "total number of URI requests for other " - "ranks", "", "net/uri/lookup_other"); + "ranks", + "", "net/uri/lookup_other"); if (ret) D_WARN("Failed to create uri other sensor: " DF_RC "\n", DP_RC(ret)); } @@ -336,10 +337,10 @@ static int data_init(int server, crt_init_options_t *opt) static int crt_plugin_init(void) { - struct crt_prog_cb_priv *cbs_prog; + struct crt_prog_cb_priv *cbs_prog; struct crt_event_cb_priv *cbs_event; - size_t cbs_size = CRT_CALLBACKS_NUM; - int i, rc; + size_t cbs_size = CRT_CALLBACKS_NUM; + int i, rc; D_ASSERT(crt_plugin_gdata.cpg_inited == 0); @@ -402,30 +403,30 @@ crt_plugin_fini(void) static int __split_arg(char *s_arg_to_split, const char *delim, char **first_arg, char **second_arg) { - char *save_ptr = NULL; - char *arg_to_split; + char *save_ptr = NULL; + char *arg_to_split; D_ASSERT(first_arg != NULL); D_ASSERT(second_arg != NULL); /* no-op, not an error case */ if (s_arg_to_split == NULL) { - *first_arg = NULL; + *first_arg = NULL; *second_arg = NULL; return DER_SUCCESS; } D_STRNDUP(arg_to_split, s_arg_to_split, 255); if (!arg_to_split) { - *first_arg = NULL; + *first_arg = NULL; *second_arg = NULL; return -DER_NOMEM; } - *first_arg = 0; + *first_arg = 0; *second_arg = 0; - *first_arg = strtok_r(arg_to_split, delim, &save_ptr); + *first_arg = strtok_r(arg_to_split, delim, &save_ptr); *second_arg = save_ptr; return DER_SUCCESS; @@ -435,13 +436,13 @@ crt_provider_t crt_str_to_provider(const char *str_provider) { crt_provider_t prov = CRT_PROV_UNKNOWN; - int i; + int i, len; + char *p = NULL; if (str_provider == NULL) return prov; for (i = 0; crt_na_dict[i].nad_str != NULL; i++) { - if (!strncmp(str_provider, crt_na_dict[i].nad_str, strlen(crt_na_dict[i].nad_str) + 1) || (crt_na_dict[i].nad_alt_str && @@ -450,6 +451,21 @@ crt_str_to_provider(const char *str_provider) prov = crt_na_dict[i].nad_type; break; } + if (crt_na_dict[i].nad_type == CRT_PROV_UCX && + !strncmp(str_provider, CRT_UCX_STR, strlen(CRT_UCX_STR))) { + len = strlen(str_provider); + if (len > strlen(CRT_UCX_STR) && strchr(str_provider, '+')) { + D_STRNDUP(p, str_provider, len); + if (!p) { + return prov; + } else { + crt_na_dict[i].nad_str = p; + crt_na_dict[i].nad_str_alloc = true; + } + } + prov = crt_na_dict[i].nad_type; + break; + } } return prov; @@ -485,8 +501,7 @@ prov_settings_apply(bool primary, crt_provider_t prov, crt_init_options_t *opt) if (g_prov_settings_applied[prov] == true) return; - if (prov == CRT_PROV_OFI_VERBS_RXM || - prov == CRT_PROV_OFI_TCP_RXM) { + if (prov == CRT_PROV_OFI_VERBS_RXM || prov == CRT_PROV_OFI_TCP_RXM) { /* Use shared receive queues to avoid large mem consumption */ d_setenv("FI_OFI_RXM_USE_SRX", "1", 0); @@ -528,8 +543,8 @@ crt_protocol_info_free(struct crt_protocol_info *protocol_info) int crt_init_opt(crt_group_id_t grpid, uint32_t flags, crt_init_options_t *opt) { - bool server = flags & CRT_FLAG_BIT_SERVER; - int rc = 0; + bool server = flags & CRT_FLAG_BIT_SERVER; + int rc = 0; crt_provider_t primary_provider; crt_provider_t secondary_provider; crt_provider_t tmp_prov; @@ -590,7 +605,7 @@ crt_init_opt(crt_group_id_t grpid, uint32_t flags, crt_init_options_t *opt) if (gdata_init_flag == 0) { rc = data_init(server, opt); if (rc != 0) { - D_ERROR("data_init failed "DF_RC"\n", DP_RC(rc)); + D_ERROR("data_init failed " DF_RC "\n", DP_RC(rc)); D_GOTO(out, rc); } } @@ -598,9 +613,8 @@ crt_init_opt(crt_group_id_t grpid, uint32_t flags, crt_init_options_t *opt) D_RWLOCK_WRLOCK(&crt_gdata.cg_rwlock); if (crt_gdata.cg_inited == 0) { - crt_gdata.cg_server = server; - crt_gdata.cg_auto_swim_disable = - (flags & CRT_FLAG_BIT_AUTO_SWIM_DISABLE) ? 1 : 0; + crt_gdata.cg_server = server; + crt_gdata.cg_auto_swim_disable = (flags & CRT_FLAG_BIT_AUTO_SWIM_DISABLE) ? 1 : 0; crt_env_get(CRT_ATTACH_INFO_PATH, &path); if (path != NULL && strlen(path) > 0) { @@ -608,7 +622,8 @@ crt_init_opt(crt_group_id_t grpid, uint32_t flags, crt_init_options_t *opt) if (rc != 0) D_ERROR("Got %s from ENV CRT_ATTACH_INFO_PATH, " "but crt_group_config_path_set failed " - "rc: %d, ignore the ENV.\n", path, rc); + "rc: %d, ignore the ENV.\n", + path, rc); else D_DEBUG(DB_ALL, "set group_config_path as %s.\n", path); } @@ -653,7 +668,7 @@ crt_init_opt(crt_group_id_t grpid, uint32_t flags, crt_init_options_t *opt) if (rc != 0) D_GOTO(unlock, rc); - primary_provider = crt_str_to_provider(provider_str0); + primary_provider = crt_str_to_provider(provider_str0); secondary_provider = crt_str_to_provider(provider_str1); if (primary_provider == CRT_PROV_UNKNOWN) { @@ -662,11 +677,11 @@ crt_init_opt(crt_group_id_t grpid, uint32_t flags, crt_init_options_t *opt) } /* - * A coma-separated list of arguments for interfaces, domains, ports, keys is interpreted - * differently, depending whether it is on a client or on a server side. + * A coma-separated list of arguments for interfaces, domains, ports, keys is + * interpreted differently, depending whether it is on a client or on a server side. * - * On a client, a coma-separated list means multi-interface selection, while on a server - * it means a multi-provider selection. + * On a client, a coma-separated list means multi-interface selection, while on a + * server it means a multi-provider selection. */ if (crt_is_service()) { rc = __split_arg(interface, ",", &iface0, &iface1); @@ -722,7 +737,8 @@ crt_init_opt(crt_group_id_t grpid, uint32_t flags, crt_init_options_t *opt) /* Secondary provider needs its own interface or domain */ if (iface1 == NULL && domain1 == NULL) { - D_ERROR("Either a secondary domain or interface must be specified\n"); + D_ERROR( + "Either a secondary domain or interface must be specified\n"); D_GOTO(unlock, rc = -DER_INVAL); } @@ -733,23 +749,22 @@ crt_init_opt(crt_group_id_t grpid, uint32_t flags, crt_init_options_t *opt) if (iface0 == NULL && primary_provider != CRT_PROV_OFI_CXI) D_WARN("No interface specified\n"); - rc = prov_data_init(&crt_gdata.cg_prov_gdata_primary, - primary_provider, true, opt); + rc = prov_data_init(&crt_gdata.cg_prov_gdata_primary, primary_provider, true, opt); if (rc != 0) D_GOTO(unlock, rc); prov_settings_apply(true, primary_provider, opt); crt_gdata.cg_primary_prov = primary_provider; - rc = crt_na_config_init(true, primary_provider, iface0, domain0, - port0, auth_key0, port_auto_adjust); + rc = crt_na_config_init(true, primary_provider, iface0, domain0, port0, auth_key0, + port_auto_adjust); if (rc != 0) { - D_ERROR("crt_na_config_init() failed, "DF_RC"\n", DP_RC(rc)); + D_ERROR("crt_na_config_init() failed, " DF_RC "\n", DP_RC(rc)); D_GOTO(unlock, rc); } if (secondary_provider != CRT_PROV_UNKNOWN) { - num_secondaries = 1; + num_secondaries = 1; crt_gdata.cg_num_secondary_provs = num_secondaries; if (port1 == NULL || port1[0] == '\0') { @@ -767,40 +782,40 @@ crt_init_opt(crt_group_id_t grpid, uint32_t flags, crt_init_options_t *opt) crt_gdata.cg_secondary_provs[0] = secondary_provider; } - for (i = 0; i < num_secondaries; i++) { + for (i = 0; i < num_secondaries; i++) { tmp_prov = crt_gdata.cg_secondary_provs[i]; - rc = prov_data_init(&crt_gdata.cg_prov_gdata_secondary[i], - tmp_prov, false, opt); + rc = prov_data_init(&crt_gdata.cg_prov_gdata_secondary[i], tmp_prov, false, + opt); if (rc != 0) D_GOTO(cleanup, rc); prov_settings_apply(false, tmp_prov, opt); - rc = crt_na_config_init(false, tmp_prov, iface1, domain1, - port1, auth_key1, port_auto_adjust); + rc = crt_na_config_init(false, tmp_prov, iface1, domain1, port1, auth_key1, + port_auto_adjust); if (rc != 0) { - D_ERROR("crt_na_config_init() failed, "DF_RC"\n", DP_RC(rc)); + D_ERROR("crt_na_config_init() failed, " DF_RC "\n", DP_RC(rc)); D_GOTO(cleanup, rc); } } rc = crt_hg_init(); if (rc != 0) { - D_ERROR("crt_hg_init() failed, "DF_RC"\n", DP_RC(rc)); + D_ERROR("crt_hg_init() failed, " DF_RC "\n", DP_RC(rc)); D_GOTO(cleanup, rc); } rc = crt_grp_init(grpid); if (rc != 0) { - D_ERROR("crt_grp_init() failed, "DF_RC"\n", DP_RC(rc)); + D_ERROR("crt_grp_init() failed, " DF_RC "\n", DP_RC(rc)); D_GOTO(cleanup, rc); } if (crt_plugin_gdata.cpg_inited == 0) { rc = crt_plugin_init(); if (rc != 0) { - D_ERROR("crt_plugin_init() failed, "DF_RC"\n", DP_RC(rc)); + D_ERROR("crt_plugin_init() failed, " DF_RC "\n", DP_RC(rc)); D_GOTO(cleanup, rc); } } @@ -810,13 +825,13 @@ crt_init_opt(crt_group_id_t grpid, uint32_t flags, crt_init_options_t *opt) crt_iv_init(opt); rc = crt_opc_map_create(); if (rc != 0) { - D_ERROR("crt_opc_map_create() failed, "DF_RC"\n", DP_RC(rc)); + D_ERROR("crt_opc_map_create() failed, " DF_RC "\n", DP_RC(rc)); D_GOTO(self_test, rc); } rc = crt_internal_rpc_register(server); if (rc != 0) { - D_ERROR("crt_internal_rpc_register() failed, "DF_RC"\n", DP_RC(rc)); + D_ERROR("crt_internal_rpc_register() failed, " DF_RC "\n", DP_RC(rc)); D_GOTO(self_test, rc); } @@ -866,7 +881,7 @@ crt_init_opt(crt_group_id_t grpid, uint32_t flags, crt_init_options_t *opt) D_FREE(auth_key0); if (rc != 0) { - D_ERROR("failed, "DF_RC"\n", DP_RC(rc)); + D_ERROR("failed, " DF_RC "\n", DP_RC(rc)); d_fault_inject_fini(); d_log_fini(); } @@ -904,8 +919,7 @@ crt_finalize(void) if (prov_data->cpg_ctx_num > 0) { D_ASSERT(!crt_context_empty(crt_gdata.cg_primary_prov, CRT_LOCKED)); - D_ERROR("cannot finalize, current ctx_num(%d).\n", - prov_data->cpg_ctx_num); + D_ERROR("cannot finalize, current ctx_num(%d).\n", prov_data->cpg_ctx_num); crt_gdata.cg_refcount++; D_RWLOCK_UNLOCK(&crt_gdata.cg_rwlock); D_GOTO(out, rc = -DER_BUSY); @@ -934,8 +948,8 @@ crt_finalize(void) D_RWLOCK_UNLOCK(&crt_gdata.cg_rwlock); crt_gdata.cg_refcount = 0; - crt_gdata.cg_inited = 0; - gdata_init_flag = 0; + crt_gdata.cg_inited = 0; + gdata_init_flag = 0; crt_na_config_fini(true, crt_gdata.cg_primary_prov); @@ -944,6 +958,10 @@ crt_finalize(void) crt_na_config_fini(false, crt_gdata.cg_secondary_provs[i]); } + for (i = 0; crt_na_dict[i].nad_str != NULL; i++) + if (crt_na_dict[i].nad_str_alloc) + D_FREE(crt_na_dict[i].nad_str); + D_FREE(crt_gdata.cg_secondary_provs); D_FREE(crt_gdata.cg_prov_gdata_secondary); } else { @@ -960,14 +978,15 @@ crt_finalize(void) if (rc == 0) d_log_fini(); /* d_log_fini is reference counted */ else - D_ERROR("failed, rc: "DF_RC"\n", DP_RC(rc)); + D_ERROR("failed, rc: " DF_RC "\n", DP_RC(rc)); crt_env_fini(); return rc; } -static inline bool is_integer_str(char *str) +static inline bool +is_integer_str(char *str) { const char *p; @@ -993,7 +1012,7 @@ crt_get_port_opx(int *port) int rc = 0; uint16_t pid; - pid = getpid(); + pid = getpid(); *port = pid; D_DEBUG(DB_ALL, "got a port: %d.\n", *port); @@ -1005,13 +1024,13 @@ crt_get_port_opx(int *port) static void crt_port_range_verify(int port) { - char proc[] = "/proc/sys/net/ipv4/ip_local_port_range"; - FILE *f; - char buff[PORT_RANGE_STR_SIZE]; - int start_port = -1; - int end_port = -1; - char *p; - int rc; + char proc[] = "/proc/sys/net/ipv4/ip_local_port_range"; + FILE *f; + char buff[PORT_RANGE_STR_SIZE]; + int start_port = -1; + int end_port = -1; + char *p; + int rc; f = fopen(proc, "r"); if (!f) { @@ -1034,7 +1053,7 @@ crt_port_range_verify(int port) /* Data is in the format of */ while (*p != '\0') { if (*p == ' ' || *p == '\t') { - *p = '\0'; + *p = '\0'; start_port = atoi(buff); p++; @@ -1052,26 +1071,25 @@ crt_port_range_verify(int port) if (port >= start_port && port <= end_port) { D_WARN("Requested port %d is inside of the local port range " - "as specified by file '%s'\n", port, proc); + "as specified by file '%s'\n", + port, proc); D_WARN("In order to avoid port conflicts pick a different " "value outside of the %d-%d range\n", start_port, end_port); } } - static int -crt_na_config_init(bool primary, crt_provider_t provider, - char *interface, char *domain, char *port_str, - char *auth_key, bool port_auto_adjust) +crt_na_config_init(bool primary, crt_provider_t provider, char *interface, char *domain, + char *port_str, char *auth_key, bool port_auto_adjust) { - struct crt_na_config *na_cfg; - int rc = 0; - int port = -1; - char *save_ptr = NULL; - char *token = NULL; - int idx = 0; - int count = 0; + struct crt_na_config *na_cfg; + int rc = 0; + int port = -1; + char *save_ptr = NULL; + char *token = NULL; + int idx = 0; + int count = 0; na_cfg = crt_provider_get_na_config(primary, provider); @@ -1101,7 +1119,7 @@ crt_na_config_init(bool primary, crt_provider_t provider, if (na_cfg->noc_interface) { /* count number of ','-separated interfaces */ - count = 1; + count = 1; save_ptr = na_cfg->noc_interface; while (*save_ptr != '\0') { @@ -1117,10 +1135,10 @@ crt_na_config_init(bool primary, crt_provider_t provider, /* store each interface name in the na_cfg->noc_iface_str[] array */ save_ptr = 0; idx = 0; - token = strtok_r(na_cfg->noc_interface, ",", &save_ptr); + token = strtok_r(na_cfg->noc_interface, ",", &save_ptr); while (token != NULL) { na_cfg->noc_iface_str[idx] = token; - token = strtok_r(NULL, ",", &save_ptr); + token = strtok_r(NULL, ",", &save_ptr); idx++; } } else { @@ -1171,8 +1189,7 @@ crt_na_config_init(bool primary, crt_provider_t provider, port = atoi(port_str); if (provider == CRT_PROV_OFI_SOCKETS || - provider == CRT_PROV_OFI_VERBS_RXM || - provider == CRT_PROV_OFI_TCP_RXM) + provider == CRT_PROV_OFI_VERBS_RXM || provider == CRT_PROV_OFI_TCP_RXM) crt_port_range_verify(port); if (provider == CRT_PROV_OFI_CXI && port_auto_adjust) { @@ -1207,7 +1224,8 @@ crt_na_config_init(bool primary, crt_provider_t provider, return rc; } -void crt_na_config_fini(bool primary, crt_provider_t provider) +void +crt_na_config_fini(bool primary, crt_provider_t provider) { struct crt_na_config *na_cfg; @@ -1217,7 +1235,7 @@ void crt_na_config_fini(bool primary, crt_provider_t provider) D_FREE(na_cfg->noc_auth_key); D_FREE(na_cfg->noc_iface_str); D_FREE(na_cfg->noc_domain_str); - na_cfg->noc_port = 0; - na_cfg->noc_iface_total = 0; + na_cfg->noc_port = 0; + na_cfg->noc_iface_total = 0; na_cfg->noc_domain_total = 0; } From dd488d137eda32d2c9df8cef6c5f66c6c7a75bc6 Mon Sep 17 00:00:00 2001 From: Phil Henderson Date: Thu, 22 Aug 2024 12:09:02 -0400 Subject: [PATCH 02/26] DAOS-16265 test: Add debug for common test dir between variants (#14818) Help determine contents of common test dir between test variants. Signed-off-by: Phil Henderson --- src/tests/ftest/launch.py | 3 ++- src/tests/ftest/util/apricot/apricot/test.py | 10 +++++++--- 2 files changed, 9 insertions(+), 4 deletions(-) diff --git a/src/tests/ftest/launch.py b/src/tests/ftest/launch.py index ad5b034301d..11d4a62c006 100755 --- a/src/tests/ftest/launch.py +++ b/src/tests/ftest/launch.py @@ -270,7 +270,8 @@ def _run(self, args): # pylint: disable=unsupported-binary-operation all_hosts = args.test_servers | args.test_clients | self.local_host self.details["installed packages"] = find_packages( - logger, all_hosts, "'^(daos|libfabric|mercury|ior|openmpi|mpifileutils)-'") + logger, all_hosts, + "'^(daos|libfabric|mercury|ior|openmpi|mpifileutils|mlnx-ofed-basic)-'") # Setup the test environment test_env = TestEnvironment() diff --git a/src/tests/ftest/util/apricot/apricot/test.py b/src/tests/ftest/util/apricot/apricot/test.py index 3110f77e30f..0daa30cf1d8 100644 --- a/src/tests/ftest/util/apricot/apricot/test.py +++ b/src/tests/ftest/util/apricot/apricot/test.py @@ -26,7 +26,7 @@ from exception_utils import CommandFailure from fault_config_utils import FaultInjection from general_utils import (dict_to_str, dump_engines_stacks, get_avocado_config_value, - get_default_config_file, get_file_listing, nodeset_append_suffix, + get_default_config_file, nodeset_append_suffix, set_avocado_config_value) from host_utils import HostException, HostInfo, HostRole, get_host_parameters, get_local_host from logger_utils import TestLogger @@ -766,9 +766,13 @@ def setUp(self): # List common test directory contents before running the test self.log.info("-" * 100) - self.log.debug("Common test directory (%s) contents:", self.test_dir) + self.log.debug("Common test directory (%s) contents:", os.path.dirname(self.test_dir)) all_hosts = include_local_host(self.host_info.all_hosts) - get_file_listing(all_hosts, self.test_dir, self.test_env.agent_user).log_output(self.log) + test_dir_parent = os.path.dirname(self.test_dir) + result = run_remote(self.log, all_hosts, f"df -h {test_dir_parent}") + if int(max(re.findall(r" ([\d+])% ", result.joined_stdout) + ["0"])) > 90: + run_remote(self.log, all_hosts, f"du -sh {test_dir_parent}/*") + self.log.info("-" * 100) if not self.start_servers_once or self.name.uid == 1: # Kill commands left running on the hosts (from a previous test) From 5240d0ccaa0cddfcff91391d6f3158b683caef3e Mon Sep 17 00:00:00 2001 From: Ken Cain Date: Thu, 22 Aug 2024 15:22:05 -0400 Subject: [PATCH 03/26] DAOS-15996 test: enhance output of test_daos_rebuild_ec (#14949) for easier debugging when the test fails, to know what subtest it was running, etc. Signed-off-by: Kenneth Cain --- src/tests/suite/daos_rebuild_ec.c | 60 +++++++++++++++++++++++++++++++ 1 file changed, 60 insertions(+) diff --git a/src/tests/suite/daos_rebuild_ec.c b/src/tests/suite/daos_rebuild_ec.c index 77df81730bd..492ec652624 100644 --- a/src/tests/suite/daos_rebuild_ec.c +++ b/src/tests/suite/daos_rebuild_ec.c @@ -112,6 +112,8 @@ rebuild_mixed_stripes(void **state) d_rank_t rank = 0; int size = 8 * CELL_SIZE + 10000; + print_message("BEGIN %s\n", __FUNCTION__); + if (!test_runable(arg, 7)) return; @@ -173,6 +175,8 @@ rebuild_ec_multi_stripes(void **state) uint16_t fail_shards[2]; int i, size = 8 * CELL_SIZE; + print_message("BEGIN %s\n", __FUNCTION__); + if (!test_runable(arg, 7)) return; @@ -314,78 +318,92 @@ rebuild_ec_6nodes_setup(void **state) static void rebuild_partial_fail_data(void **state) { + print_message("BEGIN %s\n", __FUNCTION__); rebuild_ec_internal(state, OC_EC_2P1G1, 1, 0, PARTIAL_UPDATE); + print_message("END %s\n", __FUNCTION__); } static void rebuild_partial_fail_parity(void **state) { + print_message("BEGIN %s\n", __FUNCTION__); rebuild_ec_internal(state, OC_EC_2P1G1, 0, 1, PARTIAL_UPDATE); } static void rebuild_full_fail_data(void **state) { + print_message("BEGIN %s\n", __FUNCTION__); rebuild_ec_internal(state, OC_EC_2P1G1, 1, 0, FULL_UPDATE); } static void rebuild_full_fail_parity(void **state) { + print_message("BEGIN %s\n", __FUNCTION__); rebuild_ec_internal(state, OC_EC_2P1G1, 0, 1, FULL_UPDATE); } static void rebuild_full_partial_fail_data(void **state) { + print_message("BEGIN %s\n", __FUNCTION__); rebuild_ec_internal(state, OC_EC_2P1G1, 1, 0, FULL_PARTIAL_UPDATE); } static void rebuild_full_partial_fail_parity(void **state) { + print_message("BEGIN %s\n", __FUNCTION__); rebuild_ec_internal(state, OC_EC_2P1G1, 0, 1, FULL_PARTIAL_UPDATE); } static void rebuild_partial_full_fail_data(void **state) { + print_message("BEGIN %s\n", __FUNCTION__); rebuild_ec_internal(state, OC_EC_2P1G1, 1, 0, PARTIAL_FULL_UPDATE); } static void rebuild_partial_full_fail_parity(void **state) { + print_message("BEGIN %s\n", __FUNCTION__); rebuild_ec_internal(state, OC_EC_2P1G1, 0, 1, PARTIAL_FULL_UPDATE); } static void rebuild2p_partial_fail_data(void **state) { + print_message("BEGIN %s\n", __FUNCTION__); rebuild_ec_internal(state, OC_EC_4P2G1, 1, 0, FULL_UPDATE); } static void rebuild2p_partial_fail_2data(void **state) { + print_message("BEGIN %s\n", __FUNCTION__); rebuild_ec_internal(state, OC_EC_4P2G1, 2, 0, FULL_UPDATE); } static void rebuild2p_partial_fail_data_parity(void **state) { + print_message("BEGIN %s\n", __FUNCTION__); rebuild_ec_internal(state, OC_EC_4P2G1, 1, 1, FULL_UPDATE); } static void rebuild2p_partial_fail_parity(void **state) { + print_message("BEGIN %s\n", __FUNCTION__); rebuild_ec_internal(state, OC_EC_4P2G1, 0, 1, FULL_UPDATE); } static void rebuild2p_partial_fail_2parity(void **state) { + print_message("BEGIN %s\n", __FUNCTION__); rebuild_ec_internal(state, OC_EC_4P2G1, 0, 2, FULL_UPDATE); } @@ -394,6 +412,7 @@ rebuild_dfs_fail_data_s0(void **state) { int shard = 0; + print_message("BEGIN %s\n", __FUNCTION__); dfs_ec_rebuild_io(state, &shard, 1); } @@ -402,6 +421,7 @@ rebuild_dfs_fail_data_s1(void **state) { int shard = 1; + print_message("BEGIN %s\n", __FUNCTION__); dfs_ec_rebuild_io(state, &shard, 1); } @@ -410,6 +430,7 @@ rebuild_dfs_fail_data_s3(void **state) { int shard = 3; + print_message("BEGIN %s\n", __FUNCTION__); dfs_ec_rebuild_io(state, &shard, 1); } @@ -418,6 +439,7 @@ rebuild_dfs_fail_2data_s0s1(void **state) { int shards[2]; + print_message("BEGIN %s\n", __FUNCTION__); shards[0] = 0; shards[1] = 1; dfs_ec_rebuild_io(state, shards, 2); @@ -428,6 +450,7 @@ rebuild_dfs_fail_2data_s0s2(void **state) { int shards[2]; + print_message("BEGIN %s\n", __FUNCTION__); shards[0] = 0; shards[1] = 2; dfs_ec_rebuild_io(state, shards, 2); @@ -438,6 +461,7 @@ rebuild_dfs_fail_2data_s0s3(void **state) { int shards[2]; + print_message("BEGIN %s\n", __FUNCTION__); shards[0] = 0; shards[1] = 3; dfs_ec_rebuild_io(state, shards, 2); @@ -448,6 +472,7 @@ rebuild_dfs_fail_2data_s1s2(void **state) { int shards[2]; + print_message("BEGIN %s\n", __FUNCTION__); shards[0] = 1; shards[1] = 2; dfs_ec_rebuild_io(state, shards, 2); @@ -458,6 +483,7 @@ rebuild_dfs_fail_2data_s1s3(void **state) { int shards[2]; + print_message("BEGIN %s\n", __FUNCTION__); shards[0] = 1; shards[1] = 3; dfs_ec_rebuild_io(state, shards, 2); @@ -468,6 +494,7 @@ rebuild_dfs_fail_2data_s2s3(void **state) { int shards[2]; + print_message("BEGIN %s\n", __FUNCTION__); shards[0] = 2; shards[1] = 3; dfs_ec_rebuild_io(state, shards, 2); @@ -478,6 +505,7 @@ rebuild_dfs_fail_data_parity_s0p1(void **state) { int shards[2]; + print_message("BEGIN %s\n", __FUNCTION__); shards[0] = 0; shards[1] = 5; dfs_ec_rebuild_io(state, shards, 2); @@ -488,6 +516,7 @@ rebuild_dfs_fail_data_parity_s3p1(void **state) { int shards[2]; + print_message("BEGIN %s\n", __FUNCTION__); shards[0] = 3; shards[1] = 5; dfs_ec_rebuild_io(state, shards, 2); @@ -498,6 +527,7 @@ rebuild_dfs_fail_data_parity_s2p1(void **state) { int shards[2]; + print_message("BEGIN %s\n", __FUNCTION__); shards[0] = 2; shards[1] = 5; dfs_ec_rebuild_io(state, shards, 2); @@ -508,6 +538,7 @@ rebuild_dfs_fail_data_parity_s0p0(void **state) { int shards[2]; + print_message("BEGIN %s\n", __FUNCTION__); shards[0] = 0; shards[1] = 4; dfs_ec_rebuild_io(state, shards, 2); @@ -518,6 +549,7 @@ rebuild_dfs_fail_data_parity_s2p0(void **state) { int shards[2]; + print_message("BEGIN %s\n", __FUNCTION__); shards[0] = 2; shards[1] = 4; dfs_ec_rebuild_io(state, shards, 2); @@ -528,6 +560,7 @@ rebuild_dfs_fail_data_parity_s3p0(void **state) { int shards[2]; + print_message("BEGIN %s\n", __FUNCTION__); shards[0] = 3; shards[1] = 4; dfs_ec_rebuild_io(state, shards, 2); @@ -681,6 +714,7 @@ rebuild_dfs_fail_seq_s0s1(void **state) { int shards[2]; + print_message("BEGIN %s\n", __FUNCTION__); shards[0] = 0; shards[1] = 1; dfs_ec_seq_fail(state, shards, 2); @@ -691,6 +725,7 @@ rebuild_dfs_fail_seq_s1s2(void **state) { int shards[2]; + print_message("BEGIN %s\n", __FUNCTION__); shards[0] = 1; shards[1] = 2; dfs_ec_seq_fail(state, shards, 2); @@ -701,6 +736,7 @@ rebuild_dfs_fail_seq_s2s3(void **state) { int shards[2]; + print_message("BEGIN %s\n", __FUNCTION__); shards[0] = 2; shards[1] = 3; dfs_ec_seq_fail(state, shards, 2); @@ -711,6 +747,7 @@ rebuild_dfs_fail_seq_s0s3(void **state) { int shards[2]; + print_message("BEGIN %s\n", __FUNCTION__); shards[0] = 0; shards[1] = 3; dfs_ec_seq_fail(state, shards, 2); @@ -721,6 +758,7 @@ rebuild_dfs_fail_seq_s0p0(void **state) { int shards[2]; + print_message("BEGIN %s\n", __FUNCTION__); shards[0] = 0; shards[1] = 4; dfs_ec_seq_fail(state, shards, 2); @@ -731,6 +769,7 @@ rebuild_dfs_fail_seq_s3p1(void **state) { int shards[2]; + print_message("BEGIN %s\n", __FUNCTION__); shards[0] = 3; shards[1] = 5; dfs_ec_seq_fail(state, shards, 2); @@ -741,6 +780,7 @@ rebuild_dfs_fail_seq_p0p1(void **state) { int shards[2]; + print_message("BEGIN %s\n", __FUNCTION__); shards[0] = 4; shards[1] = 5; dfs_ec_seq_fail(state, shards, 2); @@ -765,6 +805,8 @@ rebuild_multiple_group_ec_object(void **state) int size = 4 * CELL_SIZE; int rc; + print_message("BEGIN %s\n", __FUNCTION__); + if (!test_runable(arg, 8)) return; @@ -874,6 +916,8 @@ rebuild_ec_dkey_enumeration(void **state) d_rank_t rank; int i; + print_message("BEGIN %s\n", __FUNCTION__); + if (!test_runable(arg, 8)) return; @@ -905,6 +949,8 @@ rebuild_ec_parity_multi_group(void **state) d_rank_t rank; int i; + print_message("BEGIN %s\n", __FUNCTION__); + if (!test_runable(arg, 8)) return; @@ -993,12 +1039,14 @@ rebuild_ec_snapshot(void **state, daos_oclass_id_t oclass, int shard) static void rebuild_ec_snapshot_data_shard(void **state) { + print_message("BEGIN %s\n", __FUNCTION__); rebuild_ec_snapshot(state, OC_EC_4P2G1, 0); } static void rebuild_ec_snapshot_parity_shard(void **state) { + print_message("BEGIN %s\n", __FUNCTION__); rebuild_ec_snapshot(state, OC_EC_4P2G1, 5); } @@ -1014,6 +1062,8 @@ rebuild_ec_parity_overwrite(void **state) int i; int stripe_size = 2 * CELL_SIZE; + print_message("BEGIN %s\n", __FUNCTION__); + if (!test_runable(arg, 8)) return; @@ -1058,6 +1108,8 @@ rebuild_ec_then_aggregation(void **state) char *data; char *verify_data; + print_message("BEGIN %s\n", __FUNCTION__); + if (!test_runable(arg, 8)) return; @@ -1115,6 +1167,8 @@ rebuild_ec_multiple_shards(void **state) uint64_t stripe_size = 4 * CELL_SIZE; daos_recx_t recx; + print_message("BEGIN %s\n", __FUNCTION__); + if (!test_runable(arg, 6)) return; @@ -1198,6 +1252,8 @@ rebuild_ec_multiple_failure_tgts(void **state) char data[20]; char v_data[20]; + print_message("BEGIN %s\n", __FUNCTION__); + if (!test_runable(arg, 6)) return; @@ -1340,6 +1396,7 @@ rebuild_ec_overwrite_fail_parity_data(void **state) { int kill_shards[2]; + print_message("BEGIN %s\n", __FUNCTION__); kill_shards[0] = 0; kill_shards[1] = 1; rebuild_ec_parity_overwrite_fail_parity_internal(state, kill_shards, 2, false); @@ -1350,6 +1407,7 @@ rebuild_ec_overwrite_fail_parity_data_with_parity(void **state) { int kill_shards[2]; + print_message("BEGIN %s\n", __FUNCTION__); kill_shards[0] = 1; kill_shards[1] = 2; rebuild_ec_parity_overwrite_fail_parity_internal(state, kill_shards, 2, true); @@ -1363,6 +1421,7 @@ rebuild_ec_combined_ops(void **state) daos_obj_id_t oid; int rc; + print_message("BEGIN %s\n", __FUNCTION__); if (!test_runable(arg, 6)) return; @@ -1394,6 +1453,7 @@ rebuild_ec_combined_ops(void **state) ioreq_init(&req, arg->coh, oid, DAOS_IOD_ARRAY, arg); verify_ec_full(&req, arg->index, 0); ioreq_fini(&req); + print_message("END %s\n", __FUNCTION__); } /** create a new pool/container for each test */ From 90df6dccb416b67b72e427930ce8b18a273d6264 Mon Sep 17 00:00:00 2001 From: Ken Cain Date: Thu, 22 Aug 2024 15:22:35 -0400 Subject: [PATCH 04/26] DAOS-15137 test: enhance output of test_daos_extend_simple (#14948) for easier debugging when the test fails, to know what it was trying to do (kill rank, extend, reintegrate, etc.), and where it actually failed. Signed-off-by: Kenneth Cain --- src/tests/suite/daos_extend_simple.c | 29 ++++++++++++++++++++++++++++ 1 file changed, 29 insertions(+) diff --git a/src/tests/suite/daos_extend_simple.c b/src/tests/suite/daos_extend_simple.c index 117ea095566..98be6290844 100644 --- a/src/tests/suite/daos_extend_simple.c +++ b/src/tests/suite/daos_extend_simple.c @@ -33,6 +33,8 @@ extend_dkeys(void **state) int j; int rc; + print_message("BEGIN %s\n", __FUNCTION__); + if (!test_runable(arg, 3)) return; @@ -73,6 +75,8 @@ extend_akeys(void **state) int j; int rc; + print_message("BEGIN %s\n", __FUNCTION__); + if (!test_runable(arg, 3)) return; @@ -113,6 +117,8 @@ extend_indexes(void **state) int k; int rc; + print_message("BEGIN %s\n", __FUNCTION__); + if (!test_runable(arg, 3)) return; @@ -155,6 +161,8 @@ extend_large_rec(void **state) int j; int rc; + print_message("BEGIN %s\n", __FUNCTION__); + if (!test_runable(arg, 3)) return; @@ -192,6 +200,8 @@ extend_objects(void **state) daos_obj_id_t oids[OBJ_NR]; int i; + print_message("BEGIN %s\n", __FUNCTION__); + if (!test_runable(arg, 3)) return; @@ -250,6 +260,8 @@ extend_read_check(dfs_t *dfs_mt, dfs_obj_t *dir) buf = malloc(buf_size); verify_buf = malloc(buf_size); + print_message("%s(): allocations buf_size=" DF_U64 ", buf=%p, verify_buf=%p\n", + __FUNCTION__, buf_size, buf, verify_buf); assert_non_null(buf); assert_non_null(verify_buf); d_iov_set(&iov, buf, buf_size); @@ -266,18 +278,23 @@ extend_read_check(dfs_t *dfs_mt, dfs_obj_t *dir) sprintf(filename, "file%d", i); rc = dfs_open(dfs_mt, dir, filename, S_IFREG | S_IWUSR | S_IRUSR, O_RDWR, OC_EC_2P1GX, 1048576, NULL, &obj); + print_message("%s(): dfs_open(filename=%s) rc=%d\n", __FUNCTION__, filename, rc); assert_int_equal(rc, 0); memset(verify_buf, 'a' + i, buf_size); rc = dfs_read(dfs_mt, obj, &sgl, 0, &read_size, NULL); + print_message("%s(): dfs_read() read_size=" DF_U64 ", rc=%d\n", __FUNCTION__, + read_size, rc); assert_int_equal(rc, 0); assert_int_equal((int)read_size, buf_size); assert_memory_equal(buf, verify_buf, read_size); rc = dfs_release(obj); + print_message("%s(): dfs_release() rc=%d\n", __FUNCTION__, rc); assert_int_equal(rc, 0); } free(buf); free(verify_buf); + print_message("%s(): done, freed buf and verify_buf\n", __FUNCTION__); } static void @@ -509,6 +526,7 @@ dfs_extend_punch_kill(void **state) { FAULT_INJECTION_REQUIRED(); + print_message("BEGIN %s\n", __FUNCTION__); dfs_extend_internal(state, EXTEND_PUNCH, extend_cb_internal, true); } @@ -517,6 +535,7 @@ dfs_extend_punch_extend(void **state) { FAULT_INJECTION_REQUIRED(); + print_message("BEGIN %s\n", __FUNCTION__); dfs_extend_internal(state, EXTEND_PUNCH, extend_cb_internal, false); } @@ -525,6 +544,7 @@ dfs_extend_stat_kill(void **state) { FAULT_INJECTION_REQUIRED(); + print_message("BEGIN %s\n", __FUNCTION__); dfs_extend_internal(state, EXTEND_STAT, extend_cb_internal, true); } @@ -533,6 +553,7 @@ dfs_extend_stat_extend(void **state) { FAULT_INJECTION_REQUIRED(); + print_message("BEGIN %s\n", __FUNCTION__); dfs_extend_internal(state, EXTEND_STAT, extend_cb_internal, false); } @@ -541,6 +562,7 @@ dfs_extend_enumerate_kill(void **state) { FAULT_INJECTION_REQUIRED(); + print_message("BEGIN %s\n", __FUNCTION__); dfs_extend_internal(state, EXTEND_ENUMERATE, extend_cb_internal, true); } @@ -549,6 +571,7 @@ dfs_extend_enumerate_extend(void **state) { FAULT_INJECTION_REQUIRED(); + print_message("BEGIN %s\n", __FUNCTION__); dfs_extend_internal(state, EXTEND_ENUMERATE, extend_cb_internal, false); } @@ -557,6 +580,7 @@ dfs_extend_fetch_kill(void **state) { FAULT_INJECTION_REQUIRED(); + print_message("BEGIN %s\n", __FUNCTION__); dfs_extend_internal(state, EXTEND_FETCH, extend_cb_internal, true); } @@ -565,6 +589,7 @@ dfs_extend_fetch_extend(void **state) { FAULT_INJECTION_REQUIRED(); + print_message("BEGIN %s\n", __FUNCTION__); dfs_extend_internal(state, EXTEND_FETCH, extend_cb_internal, false); } @@ -573,6 +598,7 @@ dfs_extend_write_kill(void **state) { FAULT_INJECTION_REQUIRED(); + print_message("BEGIN %s\n", __FUNCTION__); dfs_extend_internal(state, EXTEND_UPDATE, extend_cb_internal, true); } @@ -581,6 +607,7 @@ dfs_extend_write_extend(void **state) { FAULT_INJECTION_REQUIRED(); + print_message("BEGIN %s\n", __FUNCTION__); dfs_extend_internal(state, EXTEND_UPDATE, extend_cb_internal, false); } @@ -598,6 +625,8 @@ dfs_extend_fail_retry(void **state) FAULT_INJECTION_REQUIRED(); + print_message("BEGIN %s\n", __FUNCTION__); + attr.da_props = daos_prop_alloc(1); assert_non_null(attr.da_props); attr.da_props->dpp_entries[0].dpe_type = DAOS_PROP_CO_REDUN_LVL; From b5edb7db4bcfc25de148423d41c972a54a847cb9 Mon Sep 17 00:00:00 2001 From: Dalton Bohning Date: Fri, 23 Aug 2024 12:27:24 -0700 Subject: [PATCH 05/26] DAOS-16391 test: ignore rc from run_local diff (#14978) diff returns rc=1 if the files are different, which is expected. Ignore the rc Signed-off-by: Dalton Bohning --- src/tests/ftest/util/launch_utils.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/tests/ftest/util/launch_utils.py b/src/tests/ftest/util/launch_utils.py index 2ed42ce7a9b..a86df01ea55 100644 --- a/src/tests/ftest/util/launch_utils.py +++ b/src/tests/ftest/util/launch_utils.py @@ -1015,8 +1015,8 @@ def update_test_yaml(self, logger, scm_size, scm_mount, extra_yaml, multiplier, if new_yaml_file: if verbose > 0: # Optionally display a diff of the yaml file - if not run_local(logger, f"diff -y {test.yaml_file} {new_yaml_file}").passed: - raise RunException(f"Error diff'ing {test.yaml_file}") + # diff returns rc=1 if the files are different, so ignore errors + run_local(logger, f"diff -y {test.yaml_file} {new_yaml_file}") test.yaml_file = new_yaml_file # Display the modified yaml file variants with debug From fd9e9457b03fc5c1305bb1b4de747266457fd17c Mon Sep 17 00:00:00 2001 From: Li Wei Date: Mon, 26 Aug 2024 04:17:33 +0900 Subject: [PATCH 06/26] DAOS-14544 rsvc: crt tree type usage fix (#14834) - Change from flat to knomial tree for broadcasts Signed-off-by: shiying Signed-off-by: Li Wei Co-authored-by: shiying --- src/rsvc/srv.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/rsvc/srv.c b/src/rsvc/srv.c index e1b438a2a5d..7d47aa8931f 100644 --- a/src/rsvc/srv.c +++ b/src/rsvc/srv.c @@ -1200,7 +1200,7 @@ bcast_create(crt_opcode_t opc, bool filter_invert, d_rank_list_t *filter_ranks, NULL /* co_bulk_hdl */, NULL /* priv */, filter_invert ? CRT_RPC_FLAG_FILTER_INVERT : 0, - crt_tree_topo(CRT_TREE_FLAT, 0), rpc); + crt_tree_topo(CRT_TREE_KNOMIAL, 2), rpc); } /** From 8a4cec46fde142d879dae0bd171d17f04782531a Mon Sep 17 00:00:00 2001 From: Alexander Oganezov Date: Sun, 25 Aug 2024 22:31:18 -0700 Subject: [PATCH 07/26] DAOS-16407 cart: coverity 2555825 fix (#14994) - Added explicit assert check for provider_str not being null and removed a conditional check which confused coverity. Signed-off-by: Alexander A Oganezov --- src/cart/crt_hg.c | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/cart/crt_hg.c b/src/cart/crt_hg.c index c1d1b3a3f9b..789d75ceb31 100644 --- a/src/cart/crt_hg.c +++ b/src/cart/crt_hg.c @@ -672,6 +672,8 @@ crt_get_info_string(bool primary, crt_provider_t provider, int iface_idx, start_port = crt_provider_ctx0_port_get(primary, provider); domain_str = crt_provider_domain_str_get(primary, provider, iface_idx); + D_ASSERTF(provider_str != NULL, "String for provider=%d not found\n", provider); + /* CXI provider uses domain names for info string */ if (provider == CRT_PROV_OFI_CXI) iface_str = NULL; @@ -690,8 +692,7 @@ crt_get_info_string(bool primary, crt_provider_t provider, int iface_idx, D_GOTO(out, rc); } - if (provider_str) - size += strlen(provider_str); + size = strlen(provider_str); if (domain_str) size += strlen(domain_str); if (iface_str) From 1f1881cb9c804e8d6a2454637205afb3cf78ef0b Mon Sep 17 00:00:00 2001 From: Mohamad Chaarawi Date: Mon, 26 Aug 2024 09:16:14 -0500 Subject: [PATCH 08/26] DAOS-16371 il: do not return unsupported for fputs if ioil is loaded (#14989) Signed-off-by: Mohamad Chaarawi --- src/client/dfuse/il/int_posix.c | 14 ++------------ src/tests/suite/dfuse_test.c | 10 +++++++++- 2 files changed, 11 insertions(+), 13 deletions(-) diff --git a/src/client/dfuse/il/int_posix.c b/src/client/dfuse/il/int_posix.c index 71d0726dda6..3609adcb5fc 100644 --- a/src/client/dfuse/il/int_posix.c +++ b/src/client/dfuse/il/int_posix.c @@ -2522,15 +2522,9 @@ dfuse_fputs(char *__str, FILE *stream) if (drop_reference_if_disabled(entry)) goto do_real_fn; - D_ERROR("Unsupported function\n"); - - entry->fd_err = ENOTSUP; - + DISABLE_STREAM(entry, stream); vector_decref(&fd_table, entry); - errno = ENOTSUP; - return EOF; - do_real_fn: return __real_fputs(__str, stream); } @@ -2553,13 +2547,9 @@ dfuse_fputws(const wchar_t *ws, FILE *stream) if (drop_reference_if_disabled(entry)) goto do_real_fn; - entry->fd_err = ENOTSUP; - + DISABLE_STREAM(entry, stream); vector_decref(&fd_table, entry); - errno = ENOTSUP; - return -1; - do_real_fn: return __real_fputws(ws, stream); } diff --git a/src/tests/suite/dfuse_test.c b/src/tests/suite/dfuse_test.c index acf46e6b7e4..eea40621108 100644 --- a/src/tests/suite/dfuse_test.c +++ b/src/tests/suite/dfuse_test.c @@ -324,9 +324,17 @@ do_stream(void **state) rc = fclose(stream); assert_int_equal(rc, 0); - rc = unlinkat(root, "stream_file", 0); + fd = openat(root, "stream_file", O_RDWR | O_EXCL, S_IWUSR | S_IRUSR); + assert_return_code(fd, errno); + stream = fdopen(fd, "w+"); + assert_non_null(stream); + + rc = fputs("Hello World!\n", stream); assert_return_code(rc, errno); + fclose(stream); + rc = unlinkat(root, "stream_file", 0); + assert_return_code(rc, errno); rc = close(root); assert_return_code(rc, errno); } From de71fb324222e51ce10d2b4487b978f162b0bc0e Mon Sep 17 00:00:00 2001 From: Li Wei Date: Tue, 27 Aug 2024 00:35:55 +0900 Subject: [PATCH 09/26] DAOS-16251 engine: Misc fixes and cleanups (#14983) - ivc_on_get stores random entry_priv_val into priv_entry for many ivc_ent_get implementations. Although not used, this should be avoided. - ds_iv_done stores pointer to stack variable rc in cb_info->future, which outlives the stack frame of ds_iv_done. Although not used, this pointer is confusing. - ds_pool_iv_map_update associates the input map buffer with the map version from ds_pool, rather than the input map version. Although this may be fine, we should really not ask for unnecessary trouble/concern. Signed-off-by: Li Wei --- src/engine/server_iv.c | 4 ++-- src/pool/srv_iv.c | 6 ++---- 2 files changed, 4 insertions(+), 6 deletions(-) diff --git a/src/engine/server_iv.c b/src/engine/server_iv.c index 8349d7bee01..a5275cc2e7f 100644 --- a/src/engine/server_iv.c +++ b/src/engine/server_iv.c @@ -578,7 +578,7 @@ ivc_on_get(crt_iv_namespace_t ivns, crt_iv_key_t *iv_key, struct ds_iv_class *class; struct ds_iv_key key; struct iv_priv_entry *priv_entry; - void *entry_priv_val; + void *entry_priv_val = NULL; bool alloc_entry = false; int rc; @@ -978,7 +978,7 @@ ds_iv_done(crt_iv_namespace_t ivns, uint32_t class_id, iv_value->sg_iovs[0].iov_len); } - ABT_future_set(cb_info->future, &rc); + ABT_future_set(cb_info->future, NULL); return ret; } diff --git a/src/pool/srv_iv.c b/src/pool/srv_iv.c index 99c5cad9127..7e72d30f66e 100644 --- a/src/pool/srv_iv.c +++ b/src/pool/srv_iv.c @@ -1222,8 +1222,7 @@ pool_iv_update(struct ds_iv_ns *ns, int class_id, uuid_t key_uuid, } int -ds_pool_iv_map_update(struct ds_pool *pool, struct pool_buf *buf, - uint32_t map_ver) +ds_pool_iv_map_update(struct ds_pool *pool, struct pool_buf *buf, uint32_t map_ver) { struct pool_iv_entry *iv_entry; uint32_t iv_entry_size; @@ -1240,8 +1239,7 @@ ds_pool_iv_map_update(struct ds_pool *pool, struct pool_buf *buf, return -DER_NOMEM; crt_group_rank(pool->sp_group, &iv_entry->piv_map.piv_master_rank); - iv_entry->piv_map.piv_pool_map_ver = - buf == NULL ? 0 : pool->sp_map_version; + iv_entry->piv_map.piv_pool_map_ver = buf == NULL ? 0 : map_ver; if (buf != NULL) memcpy(&iv_entry->piv_map.piv_pool_buf, buf, pool_buf_size(buf->pb_nr)); From 2ac6f52fd3cf41070f5d1c02bcb5bb1f44aa7920 Mon Sep 17 00:00:00 2001 From: Li Wei Date: Tue, 27 Aug 2024 10:14:08 +0900 Subject: [PATCH 10/26] DAOS-16251 tests: Fix various buffer overflows (#15003) - vos: Fix vtx_pm buffer overflow - vos: Fix evt_ctl stack buffer overflow - object: Fix srv_checksum_tests buffer overflow - utils: Fix ddb_vos_tests buffer overflow etc Signed-off-by: Li Wei --- src/object/tests/srv_checksum_tests.c | 2 +- src/utils/ddb/tests/ddb_vos_tests.c | 6 +++--- src/vos/tests/evt_ctl.c | 2 +- src/vos/tests/vts_pm.c | 2 +- 4 files changed, 6 insertions(+), 6 deletions(-) diff --git a/src/object/tests/srv_checksum_tests.c b/src/object/tests/srv_checksum_tests.c index 97c9d19fa26..dd6f1cb3e54 100644 --- a/src/object/tests/srv_checksum_tests.c +++ b/src/object/tests/srv_checksum_tests.c @@ -1275,7 +1275,7 @@ fetch_with_hole7(void **state) .layout = { {.data = "ABCDEFGHIJKLM", .sel = {1, 2}, .ful = {1, 13} }, {.data = "", .sel = {3, 7}, .ful = {3, 7}, .is_hole = true}, - {.data = "HIJKLM", .sel = {8, 13}, .ful = {1, 13} }, + {.data = "HIJKLM", .sel = {8, 13}, .ful = {8, 13} }, {.data = NULL} } }); diff --git a/src/utils/ddb/tests/ddb_vos_tests.c b/src/utils/ddb/tests/ddb_vos_tests.c index d398303ae98..f0c73a4f367 100644 --- a/src/utils/ddb/tests/ddb_vos_tests.c +++ b/src/utils/ddb/tests/ddb_vos_tests.c @@ -295,7 +295,7 @@ get_dkey_from_idx_tests(void **state) assert_success(dv_get_dkey(coh, uoid, 0, &dkey)); i = 1; while (SUCCESS(dv_get_dkey(coh, uoid, i, &dkey2))) { - assert_string_not_equal(dkey.iov_buf, dkey2.iov_buf); + assert_key_not_equal(dkey, dkey2); i++; daos_iov_free(&dkey2); } @@ -330,8 +330,8 @@ get_akey_from_idx_tests(void **state) assert_success(dv_get_akey(coh, uoid, &dkey, 0, &akey)); i = 1; - while (SUCCESS(dv_get_dkey(coh, uoid, i, &akey2))) { - assert_string_not_equal(akey.iov_buf, akey2.iov_buf); + while (SUCCESS(dv_get_akey(coh, uoid, &dkey, i, &akey2))) { + assert_key_not_equal(akey, akey2); i++; daos_iov_free(&akey2); } diff --git a/src/vos/tests/evt_ctl.c b/src/vos/tests/evt_ctl.c index 138f06d7e23..12bfb27a21b 100644 --- a/src/vos/tests/evt_ctl.c +++ b/src/vos/tests/evt_ctl.c @@ -1770,7 +1770,7 @@ insert_val(struct test_arg *arg, daos_handle_t toh, daos_epoch_t epoch, uint64_t memset(&entry.ei_csum, 0, sizeof(entry.ei_csum)); - rc = bio_alloc_init(arg->ta_utx, &entry.ei_addr, &data, length); + rc = bio_alloc_init(arg->ta_utx, &entry.ei_addr, data, length); if (rc != 0) return rc; diff --git a/src/vos/tests/vts_pm.c b/src/vos/tests/vts_pm.c index 07da1dba3ee..99e53fc71b6 100644 --- a/src/vos/tests/vts_pm.c +++ b/src/vos/tests/vts_pm.c @@ -1968,7 +1968,7 @@ many_keys(void **state) assert_rc_equal(rc, 0); rex.rx_idx = 0; - rex.rx_nr = sizeof(w) - 1; + rex.rx_nr = strlen(w); iod.iod_type = DAOS_IOD_ARRAY; iod.iod_size = 1; From e3869b7465ce97688ec6a49c760105c00b6382b6 Mon Sep 17 00:00:00 2001 From: Makito Kano Date: Tue, 27 Aug 2024 23:46:56 +0900 Subject: [PATCH 11/26] DAOS-16406 test: file_count_test_base.py - Don't obtain dfuse mount_dir from test yaml (#14993) Remove the following from file_count_test_base.py mount_dir = self.params.get("mount_dir", "/run/dfuse/*") dfuse mount_dir used to be defined in the test yaml, but now the harness creates a unique directory per test. mount_dir definitions were removed from the test yaml in PR11453. Affected tests are: deployment/io_sys_admin.py io/large_file_count.py io/small_file_count.py Signed-off-by: Makito Kano --- src/tests/ftest/util/file_count_test_base.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/src/tests/ftest/util/file_count_test_base.py b/src/tests/ftest/util/file_count_test_base.py index 58b1a49f0a5..be21183c97a 100644 --- a/src/tests/ftest/util/file_count_test_base.py +++ b/src/tests/ftest/util/file_count_test_base.py @@ -69,7 +69,6 @@ def run_file_count(self): dir_oclass = None apis = self.params.get("api", "/run/largefilecount/*") hdf5_plugin_path = self.params.get("plugin_path", '/run/hdf5_vol/*') - mount_dir = self.params.get("mount_dir", "/run/dfuse/*") ior_np = self.params.get("np", '/run/ior/client_processes/*', 1) ior_ppn = self.params.get("ppn", '/run/ior/client_processes/*', None) mdtest_np = self.params.get("np", '/run/mdtest/client_processes/*', 1) @@ -119,8 +118,7 @@ def run_file_count(self): self.ppn = ior_ppn if api == 'HDF5-VOL': self.ior_cmd.api.update('HDF5') - self.run_ior_with_pool( - create_pool=False, plugin_path=hdf5_plugin_path, mount_dir=mount_dir) + self.run_ior_with_pool(create_pool=False, plugin_path=hdf5_plugin_path) elif self.ior_cmd.api.value == 'POSIX': self.run_ior_with_pool(create_pool=False, intercept=intercept) else: From 0f811a9d12751ae9016f817679ba88fdd0735bd5 Mon Sep 17 00:00:00 2001 From: Nasf-Fan Date: Wed, 28 Aug 2024 10:12:24 +0800 Subject: [PATCH 12/26] DAOS-16347 object: partial coll-punch because of CPU yield (#15000) If multiple shards for the same object reside on the same vos target and CPU yield for DTX refresh during collectively punch such object, then the local collective punch needs to restarted to avoid missing to punch some object shards on current target. Drop redundant daos_is_zero_dti() check in object RPC hander logic. Signed-off-by: Fan Yong --- src/object/srv_obj.c | 102 +++++++++++++++++++++---------------------- 1 file changed, 51 insertions(+), 51 deletions(-) diff --git a/src/object/srv_obj.c b/src/object/srv_obj.c index 3c950615034..0a246bebdca 100644 --- a/src/object/srv_obj.c +++ b/src/object/srv_obj.c @@ -60,6 +60,9 @@ obj_gen_dtx_mbs(uint32_t flags, uint32_t *tgt_cnt, struct daos_shard_tgt **p_tgt int i; int j; + if (*tgt_cnt == 0) + return 0; + D_ASSERT(tgts != NULL); if (!(flags & ORF_CONTAIN_LEADER)) { @@ -2652,11 +2655,9 @@ ds_obj_tgt_update_handler(crt_rpc_t *rpc) tgts = orw->orw_shard_tgts.ca_arrays; tgt_cnt = orw->orw_shard_tgts.ca_count; - if (!daos_is_zero_dti(&orw->orw_dti) && tgt_cnt != 0) { - rc = obj_gen_dtx_mbs(orw->orw_flags, &tgt_cnt, &tgts, &mbs); - if (rc != 0) - D_GOTO(out, rc); - } + rc = obj_gen_dtx_mbs(orw->orw_flags, &tgt_cnt, &tgts, &mbs); + if (rc != 0) + D_GOTO(out, rc); epoch.oe_value = orw->orw_epoch; epoch.oe_first = orw->orw_epoch_first; @@ -2880,11 +2881,9 @@ ds_obj_rw_handler(crt_rpc_t *rpc) tgts = orw->orw_shard_tgts.ca_arrays; tgt_cnt = orw->orw_shard_tgts.ca_count; - if (!daos_is_zero_dti(&orw->orw_dti) && tgt_cnt != 0) { - rc = obj_gen_dtx_mbs(orw->orw_flags, &tgt_cnt, &tgts, &mbs); - if (rc != 0) - D_GOTO(out, rc); - } + rc = obj_gen_dtx_mbs(orw->orw_flags, &tgt_cnt, &tgts, &mbs); + if (rc != 0) + D_GOTO(out, rc); version = orw->orw_map_ver; max_ver = orw->orw_map_ver; @@ -3004,9 +3003,6 @@ ds_obj_rw_handler(crt_rpc_t *rpc) goto again2; } - /* Standalone fetches do not get -DER_TX_RESTART. */ - D_ASSERT(!daos_is_zero_dti(&orw->orw_dti)); - break; case -DER_AGAIN: orw->orw_flags |= ORF_RESEND; @@ -3426,27 +3422,16 @@ obj_punch_complete(crt_rpc_t *rpc, int status, uint32_t map_version) } static int -obj_local_punch(struct obj_punch_in *opi, crt_opcode_t opc, - struct obj_io_context *ioc, struct dtx_handle *dth) +obj_punch_one(struct obj_punch_in *opi, crt_opcode_t opc, + struct obj_io_context *ioc, struct dtx_handle *dth) { struct ds_cont_child *cont = ioc->ioc_coc; - struct dtx_share_peer *dsp; - uint64_t sched_seq; - uint32_t retry = 0; - int rc = 0; - - if (daos_is_zero_dti(&opi->opi_dti)) { - D_DEBUG(DB_TRACE, "disable dtx\n"); - dth = NULL; - } + int rc; -again: rc = dtx_sub_init(dth, &opi->opi_oid, opi->opi_dkey_hash); if (rc != 0) goto out; - sched_seq = sched_cur_seq(); - switch (opc) { case DAOS_OBJ_RPC_PUNCH: case DAOS_OBJ_RPC_TGT_PUNCH: @@ -3478,7 +3463,32 @@ obj_local_punch(struct obj_punch_in *opi, crt_opcode_t opc, D_GOTO(out, rc = -DER_NOSYS); } - if (dth != NULL && obj_dtx_need_refresh(dth, rc)) { +out: + return rc; +} + +static int +obj_local_punch(struct obj_punch_in *opi, crt_opcode_t opc, uint32_t shard_nr, uint32_t *shards, + struct obj_io_context *ioc, struct dtx_handle *dth) +{ + struct dtx_share_peer *dsp; + uint64_t sched_seq; + uint32_t retry = 0; + int rc = 0; + int i; + +again: + sched_seq = sched_cur_seq(); + + /* There may be multiple shards reside on the same VOS target. */ + for (i = 0; i < shard_nr; i++) { + opi->opi_oid.id_shard = shards[i]; + rc = obj_punch_one(opi, opc, ioc, dth); + if (rc != 0) + break; + } + + if (obj_dtx_need_refresh(dth, rc)) { if (unlikely(++retry % 10 == 3)) { dsp = d_list_entry(dth->dth_share_tbd_list.next, struct dtx_share_peer, dsp_link); @@ -3559,7 +3569,6 @@ obj_tgt_punch(struct obj_tgt_punch_args *otpa, uint32_t *shards, uint32_t count) daos_epoch_t tmp; uint32_t dtx_flags = 0; int rc = 0; - int i; if (p_ioc == NULL) { p_ioc = &ioc; @@ -3616,17 +3625,11 @@ obj_tgt_punch(struct obj_tgt_punch_args *otpa, uint32_t *shards, uint32_t count) D_GOTO(out, rc = -DER_IO); exec: - /* There may be multiple shards reside on the same VOS target. */ - for (i = 0; i < count; i++) { - opi->opi_oid.id_shard = shards[i]; - rc = obj_local_punch(opi, otpa->opc, p_ioc, dth); - if (rc != 0) { - DL_CDEBUG(rc == -DER_INPROGRESS || rc == -DER_TX_RESTART || - (rc == -DER_NONEXIST && (opi->opi_api_flags & DAOS_COND_PUNCH)), - DB_IO, DLOG_ERR, rc, DF_UOID, DP_UOID(opi->opi_oid)); - goto out; - } - } + rc = obj_local_punch(opi, otpa->opc, count, shards, p_ioc, dth); + if (rc != 0) + DL_CDEBUG(rc == -DER_INPROGRESS || rc == -DER_TX_RESTART || + (rc == -DER_NONEXIST && (opi->opi_api_flags & DAOS_COND_PUNCH)), + DB_IO, DLOG_ERR, rc, DF_UOID, DP_UOID(opi->opi_oid)); out: if (otpa->ver != NULL) @@ -3652,11 +3655,9 @@ ds_obj_tgt_punch_handler(crt_rpc_t *rpc) uint32_t version = 0; int rc; - if (!daos_is_zero_dti(&opi->opi_dti) && tgt_cnt != 0) { - rc = obj_gen_dtx_mbs(opi->opi_flags, &tgt_cnt, &tgts, &otpa.mbs); - if (rc != 0) - D_GOTO(out, rc); - } + rc = obj_gen_dtx_mbs(opi->opi_flags, &tgt_cnt, &tgts, &otpa.mbs); + if (rc != 0) + D_GOTO(out, rc); otpa.opc = opc_get(rpc->cr_opc); otpa.opi = opi; @@ -3732,7 +3733,8 @@ obj_tgt_punch_disp(struct dtx_leader_handle *dlh, void *arg, int idx, dtx_sub_co if (dlh->dlh_handle.dth_prepared) goto comp; - rc = obj_local_punch(opi, opc_get(rpc->cr_opc), exec_arg->ioc, &dlh->dlh_handle); + rc = obj_local_punch(opi, opc_get(rpc->cr_opc), 1, &opi->opi_oid.id_shard, + exec_arg->ioc, &dlh->dlh_handle); if (rc != 0) DL_CDEBUG(rc == -DER_INPROGRESS || rc == -DER_TX_RESTART || (rc == -DER_NONEXIST && (opi->opi_api_flags & DAOS_COND_PUNCH)), @@ -3809,11 +3811,9 @@ ds_obj_punch_handler(crt_rpc_t *rpc) tgts = opi->opi_shard_tgts.ca_arrays; tgt_cnt = opi->opi_shard_tgts.ca_count; - if (!daos_is_zero_dti(&opi->opi_dti) && tgt_cnt != 0) { - rc = obj_gen_dtx_mbs(opi->opi_flags, &tgt_cnt, &tgts, &mbs); - if (rc != 0) - D_GOTO(out, rc); - } + rc = obj_gen_dtx_mbs(opi->opi_flags, &tgt_cnt, &tgts, &mbs); + if (rc != 0) + D_GOTO(out, rc); if (tgt_cnt == 0) { if (!(opi->opi_api_flags & DAOS_COND_MASK)) From 9879f07cd861a6f28ed7bb4c3a603795be84aba2 Mon Sep 17 00:00:00 2001 From: Johann Lombardi Date: Wed, 28 Aug 2024 14:16:36 +0200 Subject: [PATCH 13/26] DAOS-15575 dfs: replace DAOS_TX_NONE with dfs th (#14094) This patch lays the ground work for adding support for snapshot to dfs in subsequent patches. It does not change anything from a functionality standpoint and just replaces all occurences of DAOS_TX_NONE in libdfs with dfs->th which can only be set to DAOS_TX_NONE for now. A dfs::th_epoch field is added to the global dfs handle to record the epoch associated with the handle (can only be set to DAOS_EPOCH_MAX for now, which means DAOS_TX_NONE). Signed-off-by: Johann Lombardi Co-authored-by: Mohamad Chaarawi --- src/client/dfs/common.c | 4 ++-- src/client/dfs/cont.c | 4 ++-- src/client/dfs/dfs_internal.h | 6 ++++++ src/client/dfs/dir.c | 2 +- src/client/dfs/file.c | 2 +- src/client/dfs/io.c | 6 +++--- src/client/dfs/lookup.c | 30 ++++++++++++++++-------------- src/client/dfs/mnt.c | 28 +++++++++++++++++++++++++--- src/client/dfs/obj.c | 30 ++++++++++++++++-------------- src/client/dfs/pipeline.c | 2 +- src/client/dfs/readdir.c | 6 +++--- src/client/dfs/xattr.c | 10 +++++----- 12 files changed, 81 insertions(+), 49 deletions(-) diff --git a/src/client/dfs/common.c b/src/client/dfs/common.c index c66de734411..04f2867065e 100644 --- a/src/client/dfs/common.c +++ b/src/client/dfs/common.c @@ -715,8 +715,8 @@ open_dir(dfs_t *dfs, dfs_obj_t *parent, int flags, daos_oclass_id_t cid, struct } /* Check if parent has the dirname entry */ - rc = fetch_entry(dfs->layout_v, parent_oh, DAOS_TX_NONE, dir->name, len, false, &exists, - entry, 0, NULL, NULL, NULL); + rc = fetch_entry(dfs->layout_v, parent_oh, dfs->th, dir->name, len, false, &exists, entry, + 0, NULL, NULL, NULL); if (rc) { D_DEBUG(DB_TRACE, "fetch_entry %s failed %d.\n", dir->name, rc); return rc; diff --git a/src/client/dfs/cont.c b/src/client/dfs/cont.c index 910e1819aa3..ecd9d91e287 100644 --- a/src/client/dfs/cont.c +++ b/src/client/dfs/cont.c @@ -1238,14 +1238,14 @@ dfs_get_size_by_oid(dfs_t *dfs, daos_obj_id_t oid, daos_size_t chunk_size, daos_ return EINVAL; rc = - daos_array_open_with_attr(dfs->coh, oid, DAOS_TX_NONE, DAOS_OO_RO, 1, + daos_array_open_with_attr(dfs->coh, oid, dfs->th, DAOS_OO_RO, 1, chunk_size ? chunk_size : dfs->attr.da_chunk_size, &oh, NULL); if (rc != 0) { D_ERROR("daos_array_open() failed: " DF_RC "\n", DP_RC(rc)); return daos_der2errno(rc); } - rc = daos_array_get_size(oh, DAOS_TX_NONE, size, NULL); + rc = daos_array_get_size(oh, dfs->th, size, NULL); if (rc) { daos_array_close(oh, NULL); D_ERROR("daos_array_get_size() failed: " DF_RC "\n", DP_RC(rc)); diff --git a/src/client/dfs/dfs_internal.h b/src/client/dfs/dfs_internal.h index 4aac7a690de..99f2fb8cde2 100644 --- a/src/client/dfs/dfs_internal.h +++ b/src/client/dfs/dfs_internal.h @@ -108,6 +108,8 @@ typedef uint16_t dfs_layout_ver_t; /** object struct that is instantiated for a DFS open object */ struct dfs_obj { + /** DFS mount point of object */ + dfs_t *dfs; /** DAOS object ID */ daos_obj_id_t oid; /** DAOS object open handle */ @@ -162,6 +164,10 @@ struct dfs { daos_handle_t coh; /** refcount on cont handle that through the DFS API */ uint32_t coh_refcount; + /** Transaction handle epoch. DAOS_EPOCH_MAX for DAOS_TX_NONE */ + daos_epoch_t th_epoch; + /** Transaction handle */ + daos_handle_t th; /** Object ID reserved for this DFS (see oid_gen below) */ daos_obj_id_t oid; /** superblock object OID */ diff --git a/src/client/dfs/dir.c b/src/client/dfs/dir.c index 1e0eca7de99..000c0625f58 100644 --- a/src/client/dfs/dir.c +++ b/src/client/dfs/dir.c @@ -326,6 +326,6 @@ dfs_dir_anchor_set(dfs_obj_t *obj, const char name[], daos_anchor_t *anchor) return rc; d_iov_set(&dkey, (void *)name, len); - rc = daos_obj_key2anchor(obj->oh, DAOS_TX_NONE, &dkey, NULL, anchor, NULL); + rc = daos_obj_key2anchor(obj->oh, obj->dfs->th, &dkey, NULL, anchor, NULL); return daos_der2errno(rc); } diff --git a/src/client/dfs/file.c b/src/client/dfs/file.c index 9d9e27e7a96..2e63310dd96 100644 --- a/src/client/dfs/file.c +++ b/src/client/dfs/file.c @@ -146,6 +146,6 @@ dfs_get_size(dfs_t *dfs, dfs_obj_t *obj, daos_size_t *size) if (obj == NULL || !S_ISREG(obj->mode)) return EINVAL; - rc = daos_array_get_size(obj->oh, DAOS_TX_NONE, size, NULL); + rc = daos_array_get_size(obj->oh, dfs->th, size, NULL); return daos_der2errno(rc); } diff --git a/src/client/dfs/io.c b/src/client/dfs/io.c index 7ebd35ee37c..3919d8cfe19 100644 --- a/src/client/dfs/io.c +++ b/src/client/dfs/io.c @@ -76,7 +76,7 @@ dfs_read_int(dfs_t *dfs, dfs_obj_t *obj, daos_off_t off, dfs_iod_t *iod, d_sg_li args = dc_task_get_args(task); args->oh = obj->oh; - args->th = DAOS_TX_NONE; + args->th = dfs->th; args->sgl = sgl; args->iod = ¶ms->arr_iod; @@ -140,7 +140,7 @@ dfs_read(dfs_t *dfs, dfs_obj_t *obj, d_sg_list_t *sgl, daos_off_t off, daos_size rg.rg_idx = off; iod.arr_rgs = &rg; - rc = daos_array_read(obj->oh, DAOS_TX_NONE, &iod, sgl, NULL); + rc = daos_array_read(obj->oh, dfs->th, &iod, sgl, NULL); if (rc) { D_ERROR("daos_array_read() failed, " DF_RC "\n", DP_RC(rc)); return daos_der2errno(rc); @@ -183,7 +183,7 @@ dfs_readx(dfs_t *dfs, dfs_obj_t *obj, dfs_iod_t *iod, d_sg_list_t *sgl, daos_siz arr_iod.arr_nr = iod->iod_nr; arr_iod.arr_rgs = iod->iod_rgs; - rc = daos_array_read(obj->oh, DAOS_TX_NONE, &arr_iod, sgl, ev); + rc = daos_array_read(obj->oh, dfs->th, &arr_iod, sgl, ev); if (rc) { D_ERROR("daos_array_read() failed (%d)\n", rc); return daos_der2errno(rc); diff --git a/src/client/dfs/lookup.c b/src/client/dfs/lookup.c index 7856952fd06..7e792989245 100644 --- a/src/client/dfs/lookup.c +++ b/src/client/dfs/lookup.c @@ -57,6 +57,7 @@ lookup_rel_path(dfs_t *dfs, dfs_obj_t *root, const char *path, int flags, dfs_ob obj->d.oclass = root->d.oclass; obj->d.chunk_size = root->d.chunk_size; obj->mode = root->mode; + obj->dfs = dfs; strncpy(obj->name, root->name, DFS_MAX_NAME + 1); rc = daos_obj_open(dfs->coh, obj->oid, daos_mode, &obj->oh, NULL); @@ -114,7 +115,7 @@ lookup_rel_path(dfs_t *dfs, dfs_obj_t *root, const char *path, int flags, dfs_ob len = strlen(token); entry.chunk_size = 0; - rc = fetch_entry(dfs->layout_v, parent.oh, DAOS_TX_NONE, token, len, true, &exists, + rc = fetch_entry(dfs->layout_v, parent.oh, dfs->th, token, len, true, &exists, &entry, 0, NULL, NULL, NULL); if (rc) D_GOTO(err_obj, rc); @@ -141,10 +142,10 @@ lookup_rel_path(dfs_t *dfs, dfs_obj_t *root, const char *path, int flags, dfs_ob D_GOTO(err_obj, rc = ENOENT); } - rc = daos_array_open_with_attr( - dfs->coh, entry.oid, DAOS_TX_NONE, daos_mode, 1, - entry.chunk_size ? entry.chunk_size : dfs->attr.da_chunk_size, &obj->oh, - NULL); + rc = daos_array_open_with_attr(dfs->coh, entry.oid, dfs->th, daos_mode, 1, + entry.chunk_size ? entry.chunk_size + : dfs->attr.da_chunk_size, + &obj->oh, NULL); if (rc != 0) { D_ERROR("daos_array_open() Failed (%d)\n", rc); D_GOTO(err_obj, rc = daos_der2errno(rc)); @@ -161,7 +162,7 @@ lookup_rel_path(dfs_t *dfs, dfs_obj_t *root, const char *path, int flags, dfs_ob if (stbuf) { daos_array_stbuf_t array_stbuf = {0}; - rc = daos_array_stat(obj->oh, DAOS_TX_NONE, &array_stbuf, NULL); + rc = daos_array_stat(obj->oh, dfs->th, &array_stbuf, NULL); if (rc) { daos_array_close(obj->oh, NULL); D_GOTO(err_obj, rc = daos_der2errno(rc)); @@ -279,7 +280,7 @@ lookup_rel_path(dfs_t *dfs, dfs_obj_t *root, const char *path, int flags, dfs_ob if (stbuf) { daos_epoch_t ep; - rc = daos_obj_query_max_epoch(obj->oh, DAOS_TX_NONE, &ep, NULL); + rc = daos_obj_query_max_epoch(obj->oh, dfs->th, &ep, NULL); if (rc) { daos_obj_close(obj->oh, NULL); D_GOTO(err_obj, rc = daos_der2errno(rc)); @@ -302,7 +303,7 @@ lookup_rel_path(dfs_t *dfs, dfs_obj_t *root, const char *path, int flags, dfs_ob daos_epoch_t ep; /** refresh possibly stale root stbuf */ - rc = fetch_entry(dfs->layout_v, dfs->super_oh, DAOS_TX_NONE, "/", 1, false, + rc = fetch_entry(dfs->layout_v, dfs->super_oh, dfs->th, "/", 1, false, &exists, &entry, 0, NULL, NULL, NULL); if (rc) { D_ERROR("fetch_entry() failed: %d (%s)\n", rc, strerror(rc)); @@ -321,7 +322,7 @@ lookup_rel_path(dfs_t *dfs, dfs_obj_t *root, const char *path, int flags, dfs_ob dfs->root_stbuf.st_uid = entry.uid; dfs->root_stbuf.st_gid = entry.gid; - rc = daos_obj_query_max_epoch(dfs->root.oh, DAOS_TX_NONE, &ep, NULL); + rc = daos_obj_query_max_epoch(dfs->root.oh, dfs->th, &ep, NULL); if (rc) D_GOTO(err_obj, rc = daos_der2errno(rc)); @@ -420,8 +421,8 @@ lookup_rel_int(dfs_t *dfs, dfs_obj_t *parent, const char *name, int flags, dfs_o if (daos_mode == -1) return EINVAL; - rc = fetch_entry(dfs->layout_v, parent->oh, DAOS_TX_NONE, name, len, true, &exists, &entry, - xnr, xnames, xvals, xsizes); + rc = fetch_entry(dfs->layout_v, parent->oh, dfs->th, name, len, true, &exists, &entry, xnr, + xnames, xvals, xsizes); if (rc) return rc; @@ -441,12 +442,13 @@ lookup_rel_int(dfs_t *dfs, dfs_obj_t *parent, const char *name, int flags, dfs_o oid_cp(&obj->parent_oid, parent->oid); oid_cp(&obj->oid, entry.oid); obj->mode = entry.mode; + obj->dfs = dfs; /** if entry is a file, open the array object and return */ switch (entry.mode & S_IFMT) { case S_IFREG: rc = daos_array_open_with_attr( - dfs->coh, entry.oid, DAOS_TX_NONE, daos_mode, 1, + dfs->coh, entry.oid, dfs->th, daos_mode, 1, entry.chunk_size ? entry.chunk_size : dfs->attr.da_chunk_size, &obj->oh, NULL); if (rc != 0) { D_ERROR("daos_array_open_with_attr() Failed " DF_RC "\n", DP_RC(rc)); @@ -465,7 +467,7 @@ lookup_rel_int(dfs_t *dfs, dfs_obj_t *parent, const char *name, int flags, dfs_o if (stbuf) { daos_array_stbuf_t array_stbuf = {0}; - rc = daos_array_stat(obj->oh, DAOS_TX_NONE, &array_stbuf, NULL); + rc = daos_array_stat(obj->oh, dfs->th, &array_stbuf, NULL); if (rc) { daos_array_close(obj->oh, NULL); D_GOTO(err_obj, rc = daos_der2errno(rc)); @@ -530,7 +532,7 @@ lookup_rel_int(dfs_t *dfs, dfs_obj_t *parent, const char *name, int flags, dfs_o if (stbuf) { daos_epoch_t ep; - rc = daos_obj_query_max_epoch(obj->oh, DAOS_TX_NONE, &ep, NULL); + rc = daos_obj_query_max_epoch(obj->oh, dfs->th, &ep, NULL); if (rc) { daos_obj_close(obj->oh, NULL); D_GOTO(err_obj, rc = daos_der2errno(rc)); diff --git a/src/client/dfs/mnt.c b/src/client/dfs/mnt.c index ecfa6280366..a73fafe34df 100644 --- a/src/client/dfs/mnt.c +++ b/src/client/dfs/mnt.c @@ -584,9 +584,11 @@ dfs_mount(daos_handle_t poh, daos_handle_t coh, int flags, dfs_t **_dfs) if (dfs == NULL) D_GOTO(err_prop, rc = ENOMEM); - dfs->poh = poh; - dfs->coh = coh; - dfs->amode = amode; + dfs->poh = poh; + dfs->coh = coh; + dfs->amode = amode; + dfs->th = DAOS_TX_NONE; + dfs->th_epoch = DAOS_EPOCH_MAX; rc = D_MUTEX_INIT(&dfs->lock, NULL); if (rc != 0) @@ -738,6 +740,9 @@ dfs_umount(dfs_t *dfs) } D_MUTEX_UNLOCK(&dfs->lock); + if (daos_handle_is_valid(dfs->th)) + daos_tx_close(dfs->th, NULL); + daos_obj_close(dfs->root.oh, NULL); daos_obj_close(dfs->super_oh, NULL); @@ -848,6 +853,7 @@ struct dfs_glob { uuid_t coh_uuid; daos_obj_id_t super_oid; daos_obj_id_t root_oid; + daos_epoch_t th_epoch; }; static inline void @@ -937,6 +943,7 @@ dfs_local2global(dfs_t *dfs, d_iov_t *glob) dfs_params->oclass = dfs->attr.da_oclass_id; dfs_params->dir_oclass = dfs->attr.da_dir_oclass_id; dfs_params->file_oclass = dfs->attr.da_file_oclass_id; + dfs_params->th_epoch = dfs->th_epoch; uuid_copy(dfs_params->coh_uuid, coh_uuid); uuid_copy(dfs_params->cont_uuid, cont_uuid); @@ -1043,6 +1050,21 @@ dfs_global2local(daos_handle_t poh, daos_handle_t coh, int flags, d_iov_t glob, D_GOTO(err_dfs, rc = daos_der2errno(rc)); } + /** Create transaction handle */ + if (dfs_params->th_epoch == DAOS_EPOCH_MAX) { + dfs->th_epoch = DAOS_EPOCH_MAX; + dfs->th = DAOS_TX_NONE; + } else { + rc = daos_tx_open_snap(coh, dfs_params->th_epoch, &dfs->th, NULL); + if (rc) { + D_ERROR("daos_tx_open_snap() failed, " DF_RC "\n", DP_RC(rc)); + daos_obj_close(dfs->super_oh, NULL); + daos_obj_close(dfs->root.oh, NULL); + D_GOTO(err_dfs, rc = daos_der2errno(rc)); + } + dfs->th_epoch = dfs_params->th_epoch; + } + dfs->mounted = DFS_MOUNT; *_dfs = dfs; diff --git a/src/client/dfs/obj.c b/src/client/dfs/obj.c index 024a230268d..a9dd6d6f51f 100644 --- a/src/client/dfs/obj.c +++ b/src/client/dfs/obj.c @@ -214,8 +214,8 @@ open_file(dfs_t *dfs, dfs_obj_t *parent, int flags, daos_oclass_id_t cid, daos_s } /* Check if parent has the filename entry */ - rc = fetch_entry(dfs->layout_v, parent->oh, DAOS_TX_NONE, file->name, len, false, &exists, - entry, 0, NULL, NULL, NULL); + rc = fetch_entry(dfs->layout_v, parent->oh, dfs->th, file->name, len, false, &exists, entry, + 0, NULL, NULL, NULL); if (rc) { D_DEBUG(DB_TRACE, "fetch_entry %s failed %d.\n", file->name, rc); return rc; @@ -237,7 +237,7 @@ open_file(dfs_t *dfs, dfs_obj_t *parent, int flags, daos_oclass_id_t cid, daos_s /** Open the byte array */ file->mode = entry->mode; - rc = daos_array_open_with_attr(dfs->coh, entry->oid, DAOS_TX_NONE, daos_mode, 1, + rc = daos_array_open_with_attr(dfs->coh, entry->oid, dfs->th, daos_mode, 1, entry->chunk_size, &file->oh, NULL); if (rc != 0) { D_ERROR("daos_array_open_with_attr() failed, " DF_RC "\n", DP_RC(rc)); @@ -254,7 +254,7 @@ open_file(dfs_t *dfs, dfs_obj_t *parent, int flags, daos_oclass_id_t cid, daos_s if (size) *size = 0; } else if (size) { - rc = daos_array_get_size(file->oh, DAOS_TX_NONE, size, NULL); + rc = daos_array_get_size(file->oh, dfs->th, size, NULL); if (rc != 0) { D_ERROR("daos_array_get_size() failed (%d)\n", rc); daos_array_close(file->oh, NULL); @@ -376,6 +376,7 @@ open_stat(dfs_t *dfs, dfs_obj_t *parent, const char *name, mode_t mode, int flag } strncpy(obj->name, name, len + 1); + obj->dfs = dfs; obj->mode = mode; obj->flags = flags; oid_cp(&obj->parent_oid, parent->oid); @@ -507,6 +508,7 @@ dfs_dup(dfs_t *dfs, dfs_obj_t *obj, int flags, dfs_obj_t **_new_obj) } strncpy(new_obj->name, obj->name, DFS_MAX_NAME + 1); + new_obj->dfs = dfs; new_obj->mode = obj->mode; new_obj->flags = flags; oid_cp(&new_obj->parent_oid, obj->parent_oid); @@ -672,6 +674,7 @@ dfs_obj_global2local(dfs_t *dfs, int flags, d_iov_t glob, dfs_obj_t **_obj) strncpy(obj->name, obj_glob->name, DFS_MAX_NAME + 1); obj->name[DFS_MAX_NAME] = '\0'; obj->mode = obj_glob->mode; + obj->dfs = dfs; obj->flags = flags ? flags : obj_glob->flags; daos_mode = get_daos_obj_mode(obj->flags); @@ -686,7 +689,7 @@ dfs_obj_global2local(dfs_t *dfs, int flags, d_iov_t glob, dfs_obj_t **_obj) return 0; } - rc = daos_array_open_with_attr(dfs->coh, obj->oid, DAOS_TX_NONE, daos_mode, 1, + rc = daos_array_open_with_attr(dfs->coh, obj->oid, dfs->th, daos_mode, 1, obj_glob->chunk_size, &obj->oh, NULL); if (rc) { D_ERROR("daos_array_open_with_attr() failed, " DF_RC "\n", DP_RC(rc)); @@ -784,7 +787,7 @@ dfs_stat(dfs_t *dfs, dfs_obj_t *parent, const char *name, struct stat *stbuf) oh = parent->oh; } - return entry_stat(dfs, DAOS_TX_NONE, oh, name, len, NULL, true, stbuf, NULL); + return entry_stat(dfs, dfs->th, oh, name, len, NULL, true, stbuf, NULL); } int @@ -803,8 +806,7 @@ dfs_ostat(dfs_t *dfs, dfs_obj_t *obj, struct stat *stbuf) if (rc) return daos_der2errno(rc); - rc = - entry_stat(dfs, DAOS_TX_NONE, oh, obj->name, strlen(obj->name), obj, true, stbuf, NULL); + rc = entry_stat(dfs, dfs->th, oh, obj->name, strlen(obj->name), obj, true, stbuf, NULL); if (rc) D_GOTO(out, rc); @@ -938,7 +940,7 @@ statx_task(tse_task_t *task) fetch_arg = daos_task_get_args(fetch_task); fetch_arg->oh = args->parent_oh; - fetch_arg->th = DAOS_TX_NONE; + fetch_arg->th = args->dfs->th; fetch_arg->flags = DAOS_COND_DKEY_FETCH; fetch_arg->dkey = &op_args->dkey; fetch_arg->nr = 1; @@ -957,7 +959,7 @@ statx_task(tse_task_t *task) /** set array_stat parameters */ stat_arg = daos_task_get_args(stat_task); stat_arg->oh = args->obj->oh; - stat_arg->th = DAOS_TX_NONE; + stat_arg->th = args->dfs->th; stat_arg->stbuf = &op_args->array_stbuf; need_stat = true; } else if (S_ISDIR(args->obj->mode)) { @@ -972,7 +974,7 @@ statx_task(tse_task_t *task) /** set obj_query parameters */ stat_arg = daos_task_get_args(stat_task); stat_arg->oh = args->obj->oh; - stat_arg->th = DAOS_TX_NONE; + stat_arg->th = args->dfs->th; stat_arg->max_epoch = &op_args->array_stbuf.st_max_epoch; stat_arg->flags = 0; stat_arg->dkey = NULL; @@ -1087,7 +1089,7 @@ dfs_access(dfs_t *dfs, dfs_obj_t *parent, const char *name, int mask) } /* Check if parent has the entry */ - rc = fetch_entry(dfs->layout_v, oh, DAOS_TX_NONE, name, len, true, &exists, &entry, 0, NULL, + rc = fetch_entry(dfs->layout_v, oh, dfs->th, name, len, true, &exists, &entry, 0, NULL, NULL, NULL); if (rc) return rc; @@ -1123,7 +1125,7 @@ int dfs_chmod(dfs_t *dfs, dfs_obj_t *parent, const char *name, mode_t mode) { daos_handle_t oh; - daos_handle_t th = DAOS_TX_NONE; + daos_handle_t th = dfs->th; bool exists; struct dfs_entry entry = {0}; d_sg_list_t sgl; @@ -1168,7 +1170,7 @@ dfs_chmod(dfs_t *dfs, dfs_obj_t *parent, const char *name, mode_t mode) } /* Check if parent has the entry */ - rc = fetch_entry(dfs->layout_v, oh, DAOS_TX_NONE, name, len, true, &exists, &entry, 0, NULL, + rc = fetch_entry(dfs->layout_v, oh, dfs->th, name, len, true, &exists, &entry, 0, NULL, NULL, NULL); if (rc) return rc; diff --git a/src/client/dfs/pipeline.c b/src/client/dfs/pipeline.c index 79cbe62324e..6131e552f5a 100644 --- a/src/client/dfs/pipeline.c +++ b/src/client/dfs/pipeline.c @@ -320,7 +320,7 @@ dfs_readdir_with_filter(dfs_t *dfs, dfs_obj_t *obj, dfs_pipeline_t *dpipe, daos_ memset(buf_keys, 0, *nr * DFS_MAX_NAME); - rc = daos_pipeline_run(dfs->coh, obj->oh, &dpipe->pipeline, DAOS_TX_NONE, 0, NULL, + rc = daos_pipeline_run(dfs->coh, obj->oh, &dpipe->pipeline, dfs->th, 0, NULL, &nr_iods, &iod, anchor, &nr_kds, kds, &sgl_keys, &sgl_recs, NULL, NULL, &stats, NULL); if (rc) diff --git a/src/client/dfs/readdir.c b/src/client/dfs/readdir.c index bd377fd19c8..a284b92e4c2 100644 --- a/src/client/dfs/readdir.c +++ b/src/client/dfs/readdir.c @@ -54,7 +54,7 @@ readdir_int(dfs_t *dfs, dfs_obj_t *obj, daos_anchor_t *anchor, uint32_t *nr, str d_iov_set(&iov, enum_buf, (*nr) * DFS_MAX_NAME); sgl.sg_iovs = &iov; - rc = daos_obj_list_dkey(obj->oh, DAOS_TX_NONE, &number, kds, &sgl, anchor, NULL); + rc = daos_obj_list_dkey(obj->oh, dfs->th, &number, kds, &sgl, anchor, NULL); if (rc) D_GOTO(out, rc = daos_der2errno(rc)); @@ -65,7 +65,7 @@ readdir_int(dfs_t *dfs, dfs_obj_t *obj, daos_anchor_t *anchor, uint32_t *nr, str /** stat the entry if requested */ if (stbufs) { - rc = entry_stat(dfs, DAOS_TX_NONE, obj->oh, dirs[key_nr].d_name, + rc = entry_stat(dfs, dfs->th, obj->oh, dirs[key_nr].d_name, kds[i].kd_key_len, NULL, true, &stbufs[key_nr], NULL); if (rc) { @@ -147,7 +147,7 @@ dfs_iterate(dfs_t *dfs, dfs_obj_t *obj, daos_anchor_t *anchor, uint32_t *nr, siz * list num or less entries, but not more than we can fit in * enum_buf */ - rc = daos_obj_list_dkey(obj->oh, DAOS_TX_NONE, &num, kds, &sgl, anchor, NULL); + rc = daos_obj_list_dkey(obj->oh, dfs->th, &num, kds, &sgl, anchor, NULL); if (rc) D_GOTO(out, rc = daos_der2errno(rc)); diff --git a/src/client/dfs/xattr.c b/src/client/dfs/xattr.c index 6bfe8b04b6b..49b700e3def 100644 --- a/src/client/dfs/xattr.c +++ b/src/client/dfs/xattr.c @@ -179,13 +179,13 @@ dfs_getxattr(dfs_t *dfs, dfs_obj_t *obj, const char *name, void *value, daos_siz sgl.sg_nr_out = 0; sgl.sg_iovs = &sg_iov; - rc = daos_obj_fetch(oh, DAOS_TX_NONE, DAOS_COND_AKEY_FETCH, &dkey, 1, &iod, &sgl, - NULL, NULL); + rc = daos_obj_fetch(oh, dfs->th, DAOS_COND_AKEY_FETCH, &dkey, 1, &iod, &sgl, NULL, + NULL); } else { iod.iod_size = DAOS_REC_ANY; - rc = daos_obj_fetch(oh, DAOS_TX_NONE, DAOS_COND_AKEY_FETCH, &dkey, 1, &iod, NULL, - NULL, NULL); + rc = daos_obj_fetch(oh, dfs->th, DAOS_COND_AKEY_FETCH, &dkey, 1, &iod, NULL, NULL, + NULL); } if (rc) { DL_CDEBUG(rc == -DER_NONEXIST, DLOG_DBG, DLOG_ERR, rc, "Failed to fetch xattr '%s'", @@ -325,7 +325,7 @@ dfs_listxattr(dfs_t *dfs, dfs_obj_t *obj, char *list, daos_size_t *size) d_iov_set(&iov, enum_buf, ENUM_DESC_BUF); sgl.sg_iovs = &iov; - rc = daos_obj_list_akey(oh, DAOS_TX_NONE, &dkey, &number, kds, &sgl, &anchor, NULL); + rc = daos_obj_list_akey(oh, dfs->th, &dkey, &number, kds, &sgl, &anchor, NULL); if (rc) D_GOTO(out, rc = daos_der2errno(rc)); From f41a173da938a93cf314dcba2505bd5e70925a37 Mon Sep 17 00:00:00 2001 From: Phil Henderson Date: Wed, 28 Aug 2024 15:33:34 -0400 Subject: [PATCH 14/26] DAOS-16334 test: (#15022) Update the --control-metadata-path assignment for dmg config generate command to avoid issues cleaning up the files generated. Signed-off-by: Phil Henderson --- src/tests/ftest/control/config_generate_run.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/tests/ftest/control/config_generate_run.py b/src/tests/ftest/control/config_generate_run.py index b17ad7beb4e..98b57b64f00 100644 --- a/src/tests/ftest/control/config_generate_run.py +++ b/src/tests/ftest/control/config_generate_run.py @@ -1,5 +1,5 @@ ''' - (C) Copyright 2018-2023 Intel Corporation. + (C) Copyright 2018-2024 Intel Corporation. SPDX-License-Identifier: BSD-2-Clause-Patent ''' @@ -47,7 +47,7 @@ def test_config_generate_run(self): # path needs to be set in that case. ext_md_path = "" if use_tmpfs_scm: - ext_md_path = self.test_dir + ext_md_path = self.test_env.log_dir # Call dmg config generate. AP is always the first server host. server_host = self.hostlist_servers[0] From 20f73610170ac65a4c981718a352799deade564f Mon Sep 17 00:00:00 2001 From: Phil Henderson Date: Wed, 28 Aug 2024 17:28:09 -0400 Subject: [PATCH 15/26] DAOS-16366 test: Use agent/server config files from test directory (#14944) Use agent, control, and server config files placed in the common test directory instead of /etc/daos with a systemctl override configuration file. Signed-off-by: Phil Henderson --- src/tests/ftest/daos_test/dfuse.py | 11 +- src/tests/ftest/launch.py | 14 +- src/tests/ftest/pool/destroy.py | 7 +- src/tests/ftest/recovery/ddb.py | 17 +- src/tests/ftest/util/agent_utils.py | 6 +- src/tests/ftest/util/apricot/apricot/test.py | 13 +- src/tests/ftest/util/collection_utils.py | 109 +------- src/tests/ftest/util/command_utils.py | 49 ++-- src/tests/ftest/util/environment_utils.py | 168 ++++++------- src/tests/ftest/util/fault_config_utils.py | 13 +- src/tests/ftest/util/file_utils.py | 147 +++++++++++ src/tests/ftest/util/general_utils.py | 173 +------------ src/tests/ftest/util/launch_utils.py | 79 +++++- src/tests/ftest/util/server_utils.py | 3 +- src/tests/ftest/util/systemctl_utils.py | 249 +++++++++++++++++++ 15 files changed, 644 insertions(+), 414 deletions(-) create mode 100644 src/tests/ftest/util/file_utils.py create mode 100644 src/tests/ftest/util/systemctl_utils.py diff --git a/src/tests/ftest/daos_test/dfuse.py b/src/tests/ftest/daos_test/dfuse.py index 205d1cd11a6..e735e745cab 100644 --- a/src/tests/ftest/daos_test/dfuse.py +++ b/src/tests/ftest/daos_test/dfuse.py @@ -10,7 +10,8 @@ from apricot import TestWithServers from cmocka_utils import CmockaUtils from dfuse_utils import get_dfuse, start_dfuse -from general_utils import create_directory, get_log_file +from file_utils import create_directory +from general_utils import get_log_file from job_manager_utils import get_job_manager @@ -80,7 +81,9 @@ def run_test(self, il_lib=None): else: # Bypass, simply create a remote directory and use that. mount_dir = '/tmp/dfuse-test' - create_directory(self.hostlist_clients, mount_dir) + result = create_directory(self.log, self.hostlist_clients, mount_dir) + if not result.passed: + self.fail(f"Error creating {mount_dir} on {result.failed_hosts}") cmocka_utils = CmockaUtils( self.hostlist_clients, "dfuse", self.outputdir, self.test_dir, self.log) @@ -118,7 +121,9 @@ def run_test(self, il_lib=None): else: # make D_IL_MOUNT_POINT different from mount_dir so it tests a non-DAOS filesystem dummy_dir = '/tmp/dummy' - create_directory(self.hostlist_clients, dummy_dir) + result = create_directory(self.log, self.hostlist_clients, dummy_dir) + if not result.passed: + self.fail(f"Error creating {dummy_dir} on {result.failed_hosts}") daos_test_env['D_IL_MOUNT_POINT'] = dummy_dir if cache_mode != 'writeback': command.append('--metadata') diff --git a/src/tests/ftest/launch.py b/src/tests/ftest/launch.py index 11d4a62c006..f3daa8464d7 100755 --- a/src/tests/ftest/launch.py +++ b/src/tests/ftest/launch.py @@ -21,7 +21,8 @@ from util.code_coverage_utils import CodeCoverage from util.environment_utils import TestEnvironment, TestEnvironmentException, set_test_environment from util.host_utils import get_local_host -from util.launch_utils import LaunchException, TestGroup, setup_fuse_config, summarize_run +from util.launch_utils import (LaunchException, TestGroup, setup_fuse_config, setup_systemctl, + summarize_run) from util.logger_utils import LOG_FILE_FORMAT, get_console_handler, get_file_handler from util.network_utils import PROVIDER_ALIAS, SUPPORTED_PROVIDERS from util.package_utils import find_packages @@ -326,6 +327,15 @@ def _run(self, args): message = "Issue detected setting up the fuse configuration" setup_result.warn_test(logger, "Setup", message, sys.exc_info()) + # Setup override systemctl files + try: + clients = args.test_clients if args.test_clients else args.test_servers + cleanup_files = setup_systemctl( + logger, args.test_servers, clients | self.local_host, test_env) + except LaunchException: + message = "Issue detected setting up the systemctl configuration" + return self.get_exit_status(1, message, "Setup", sys.exc_info()) + # Get the core file pattern information core_files = {} if args.process_cores: @@ -371,7 +381,7 @@ def _run(self, args): logger, self.result, self.repeat, self.slurm_setup, args.sparse, args.failfast, not args.disable_stop_daos, args.archive, args.rename, args.jenkinslog, core_files, args.logs_threshold, args.user_create, code_coverage, self.job_results_dir, - self.logdir, args.clear_mounts) + self.logdir, args.clear_mounts, cleanup_files) # Convert the test status to a launch.py status status |= summarize_run(logger, self.mode, test_status) diff --git a/src/tests/ftest/pool/destroy.py b/src/tests/ftest/pool/destroy.py index 26280a74978..2bcd5f6cea7 100644 --- a/src/tests/ftest/pool/destroy.py +++ b/src/tests/ftest/pool/destroy.py @@ -362,10 +362,11 @@ def test_destroy_wrong_group(self): server_group_b = self.server_group + "_b" # Prepare and configure dmg config files for a and b. - dmg_config_file_a = get_default_config_file(name="control_a") + config_path = os.path.dirname(self.test_env.control_config) + dmg_config_file_a = get_default_config_file(name="control_a", path=config_path) dmg_config_temp_a = self.get_config_file( name=server_group_a, command="dmg", path=self.test_dir) - dmg_config_file_b = get_default_config_file(name="control_b") + dmg_config_file_b = get_default_config_file(name="control_b", path=config_path) dmg_config_temp_b = self.get_config_file( name=server_group_b, command="dmg", path=self.test_dir) @@ -393,7 +394,7 @@ def test_destroy_wrong_group(self): # Get dmg_c instance that uses daos_control_c.yml. Server group is b. cert_dir = os.path.join(os.sep, "etc", "daos", "certs") - dmg_config_file_c = get_default_config_file(name="control_c") + dmg_config_file_c = get_default_config_file(name="control_c", path=config_path) dmg_config_temp_c = self.get_config_file( name=server_group_b, command="dmg", path=self.test_dir) dmg_c = get_dmg_command( diff --git a/src/tests/ftest/recovery/ddb.py b/src/tests/ftest/recovery/ddb.py index 8447db1873a..04df3184984 100644 --- a/src/tests/ftest/recovery/ddb.py +++ b/src/tests/ftest/recovery/ddb.py @@ -10,10 +10,12 @@ from ClusterShell.NodeSet import NodeSet from ddb_utils import DdbCommand from exception_utils import CommandFailure -from general_utils import (DaosTestError, create_string_buffer, distribute_files, - get_clush_command, get_random_string, report_errors, run_command) +from file_utils import distribute_files +from general_utils import (DaosTestError, create_string_buffer, get_random_string, report_errors, + run_command) from pydaos.raw import DaosObjClass, IORequest from recovery_test_base import RecoveryTestBase +from run_utils import get_clush_command def insert_objects(context, container, object_count, dkey_count, akey_count, base_dkey, @@ -507,14 +509,9 @@ def test_recovery_ddb_load(self): file.write(new_data) # Copy the created file to server node. - try: - distribute_files( - hosts=host, source=load_file_path, destination=load_file_path, - mkdir=False) - except DaosTestError as error: - raise CommandFailure( - "ERROR: Copying new_data.txt to {0}: {1}".format(host, error)) \ - from error + result = distribute_files(self.log, host, load_file_path, load_file_path, False) + if not result.passed: + raise CommandFailure(f"ERROR: Copying new_data.txt to {result.failed_hosts}") # The file with the new data is ready. Run ddb load. ddb_command.value_load(component_path="[0]/[0]/[0]/[0]", load_file_path=load_file_path) diff --git a/src/tests/ftest/util/agent_utils.py b/src/tests/ftest/util/agent_utils.py index ff815356563..74b79fb9796 100644 --- a/src/tests/ftest/util/agent_utils.py +++ b/src/tests/ftest/util/agent_utils.py @@ -6,7 +6,6 @@ import os import re import socket -from getpass import getuser from agent_utils_params import DaosAgentTransportCredentials, DaosAgentYamlParameters from ClusterShell.NodeSet import NodeSet @@ -289,7 +288,7 @@ def start(self): # Verify the socket directory exists when using a non-systemctl manager if self.verify_socket_dir: - self.verify_socket_directory(getuser()) + self.verify_socket_directory(self.manager.job.certificate_owner) super().start() @@ -319,7 +318,8 @@ def support_collect_log(self, **kwargs): """ cmd = self.manager.job.copy() cmd.debug.value = False - cmd.config.value = get_default_config_file("agent") + cmd.config.value = get_default_config_file( + "agent", os.path.dirname(self.manager.job.yaml.filename)) cmd.set_command(("support", "collect-log"), **kwargs) self.log.info("Support collect-log on clients: %s", str(cmd)) return run_remote(self.log, self.hosts, cmd.with_exports) diff --git a/src/tests/ftest/util/apricot/apricot/test.py b/src/tests/ftest/util/apricot/apricot/test.py index 0daa30cf1d8..601de353afa 100644 --- a/src/tests/ftest/util/apricot/apricot/test.py +++ b/src/tests/ftest/util/apricot/apricot/test.py @@ -26,8 +26,7 @@ from exception_utils import CommandFailure from fault_config_utils import FaultInjection from general_utils import (dict_to_str, dump_engines_stacks, get_avocado_config_value, - get_default_config_file, nodeset_append_suffix, - set_avocado_config_value) + nodeset_append_suffix, set_avocado_config_value) from host_utils import HostException, HostInfo, HostRole, get_host_parameters, get_local_host from logger_utils import TestLogger from pydaos.raw import DaosApiError, DaosContext, DaosLog @@ -762,7 +761,7 @@ def setUp(self): self.fail(f"Error creating test-specific temporary directory on {result.failed_hosts}") # Copy the fault injection files to the hosts. - self.fault_injection.copy_fault_files(self.host_info.all_hosts) + self.fault_injection.copy_fault_files(self.log, self.host_info.all_hosts) # List common test directory contents before running the test self.log.info("-" * 100) @@ -1067,7 +1066,7 @@ def add_agent_manager(self, group=None, config_file=None, config_temp=None): if group is None: group = self.server_group if config_file is None and self.agent_manager_class == "Systemctl": - config_file = get_default_config_file("agent") + config_file = self.test_env.agent_config config_temp = self.get_config_file(group, "agent", self.test_dir) elif config_file is None: config_file = self.get_config_file(group, "agent") @@ -1117,14 +1116,14 @@ def add_server_manager(self, group=None, svr_config_file=None, if group is None: group = self.server_group if svr_config_file is None and self.server_manager_class == "Systemctl": - svr_config_file = get_default_config_file("server") + svr_config_file = self.test_env.server_config svr_config_temp = self.get_config_file( group, "server", self.test_dir) elif svr_config_file is None: svr_config_file = self.get_config_file(group, "server") svr_config_temp = None if dmg_config_file is None and self.server_manager_class == "Systemctl": - dmg_config_file = get_default_config_file("control") + dmg_config_file = self.test_env.control_config dmg_config_temp = self.get_config_file(group, "dmg", self.test_dir) elif dmg_config_file is None: dmg_config_file = self.get_config_file(group, "dmg") @@ -1673,7 +1672,7 @@ def get_dmg_command(self, index=0): return self.server_managers[index].dmg if self.server_manager_class == "Systemctl": - dmg_config_file = get_default_config_file("control") + dmg_config_file = self.test_env.control_config dmg_config_temp = self.get_config_file("daos", "dmg", self.test_dir) dmg_cert_dir = os.path.join(os.sep, "etc", "daos", "certs") else: diff --git a/src/tests/ftest/util/collection_utils.py b/src/tests/ftest/util/collection_utils.py index 3b054eb19e9..a900769f4d6 100644 --- a/src/tests/ftest/util/collection_utils.py +++ b/src/tests/ftest/util/collection_utils.py @@ -17,6 +17,7 @@ from util.environment_utils import TestEnvironment from util.host_utils import get_local_host from util.run_utils import find_command, run_local, run_remote, stop_processes +from util.systemctl_utils import stop_service from util.user_utils import get_chown_command from util.yaml_utils import get_test_category @@ -66,97 +67,6 @@ def stop_daos_server_service(logger, test): return stop_service(logger, hosts, service) -def stop_service(logger, hosts, service): - """Stop any daos_server.service running on the hosts running servers. - - Args: - logger (Logger): logger for the messages produced by this method - hosts (NodeSet): list of hosts on which to stop the service. - service (str): name of the service - - Returns: - bool: True if the service was successfully stopped; False otherwise - - """ - result = {"status": True} - if hosts: - status_keys = ["reset-failed", "stop", "disable"] - mapping = {"stop": "active", "disable": "enabled", "reset-failed": "failed"} - check_hosts = NodeSet(hosts) - loop = 1 - # Reduce 'max_loops' to 2 once https://jira.hpdd.intel.com/browse/DAOS-7809 - # has been resolved - max_loops = 3 - while check_hosts: - # Check the status of the service on each host - result = get_service_status(logger, check_hosts, service) - check_hosts = NodeSet() - for key in status_keys: - if result[key]: - if loop == max_loops: - # Exit the while loop if the service is still running - logger.error( - " - Error %s still %s on %s", service, mapping[key], result[key]) - result["status"] = False - else: - # Issue the appropriate systemctl command to remedy the - # detected state, e.g. 'stop' for 'active'. - command = ["sudo", "-n", "systemctl", key, service] - run_remote(logger, result[key], " ".join(command)) - - # Run the status check again on this group of hosts - check_hosts.add(result[key]) - loop += 1 - else: - logger.debug(" Skipping stopping %s service - no hosts", service) - - return result["status"] - - -def get_service_status(logger, hosts, service): - """Get the status of the daos_server.service. - - Args: - logger (Logger): logger for the messages produced by this method - hosts (NodeSet): hosts on which to get the service state - service (str): name of the service - - Returns: - dict: a dictionary with the following keys: - - "status": boolean set to True if status was obtained; False otherwise - - "stop": NodeSet where to stop the daos_server.service - - "disable": NodeSet where to disable the daos_server.service - - "reset-failed": NodeSet where to reset the daos_server.service - - """ - status = { - "status": True, - "stop": NodeSet(), - "disable": NodeSet(), - "reset-failed": NodeSet()} - status_states = { - "stop": ["active", "activating", "deactivating"], - "disable": ["active", "activating", "deactivating"], - "reset-failed": ["failed"]} - command = ["systemctl", "is-active", service] - result = run_remote(logger, hosts, " ".join(command)) - for data in result.output: - if data.timeout: - status["status"] = False - status["stop"].add(data.hosts) - status["disable"].add(data.hosts) - status["reset-failed"].add(data.hosts) - logger.debug(" %s: TIMEOUT", data.hosts) - break - logger.debug(" %s: %s", data.hosts, "\n".join(data.stdout)) - for key, state_list in status_states.items(): - for line in data.stdout: - if line in state_list: - status[key].add(data.hosts) - break - return status - - def reset_server_storage(logger, test): """Reset the server storage for the hosts that ran servers in the test. @@ -981,14 +891,15 @@ def collect_test_result(logger, test, test_result, job_results_dir, stop_daos, a "depth": 1, "timeout": 300, } - remote_files["remote configuration files"] = { - "source": os.path.join(os.sep, "etc", "daos"), - "destination": os.path.join(job_results_dir, "latest", TEST_RESULTS_DIRS[0]), - "pattern": "daos_*.yml", - "hosts": test.host_info.all_hosts, - "depth": 1, - "timeout": 300, - } + for index, source in enumerate(test_env.config_file_directories()): + remote_files[f"remote configuration files ({index})"] = { + "source": source, + "destination": os.path.join(job_results_dir, "latest", TEST_RESULTS_DIRS[0]), + "pattern": "daos_*.yml", + "hosts": test.host_info.all_hosts, + "depth": 1, + "timeout": 300, + } remote_files["daos log files"] = { "source": test_env.log_dir, "destination": os.path.join(job_results_dir, "latest", TEST_RESULTS_DIRS[1]), diff --git a/src/tests/ftest/util/command_utils.py b/src/tests/ftest/util/command_utils.py index b22300974fb..06da3b27868 100644 --- a/src/tests/ftest/util/command_utils.py +++ b/src/tests/ftest/util/command_utils.py @@ -19,9 +19,9 @@ from command_utils_base import (BasicParameter, CommandWithParameters, EnvironmentVariables, FormattedParameter, LogParameter, ObjectWithParameters) from exception_utils import CommandFailure -from general_utils import (DaosTestError, change_file_owner, check_file_exists, create_directory, - distribute_files, get_file_listing, get_job_manager_class, - get_subprocess_stdout, run_command) +from file_utils import change_file_owner, create_directory, distribute_files +from general_utils import (DaosTestError, check_file_exists, get_file_listing, + get_job_manager_class, get_subprocess_stdout, run_command) from run_utils import command_as_user, run_remote from user_utils import get_primary_group from yaml_utils import get_yaml_data @@ -1017,19 +1017,17 @@ def copy_certificates(self, source, hosts): self.log.debug("Copying certificates for %s:", self._command) data = yaml.get_certificate_data(yaml.get_attribute_names(LogParameter)) for name in data: - create_directory(hosts, name, verbose=True, raise_exception=False) + create_directory(self.log, hosts, name, verbose=True) for file_name in data[name]: src_file = os.path.join(source, file_name) dst_file = os.path.join(name, file_name) self.log.debug(" %s -> %s", src_file, dst_file) result = distribute_files( - hosts, src_file, dst_file, mkdir=False, - verbose=False, raise_exception=False, sudo=True, - owner=self.certificate_owner) - if result.exit_status != 0: + self.log, hosts, src_file, dst_file, mkdir=False, + verbose=False, sudo=True, owner=self.certificate_owner) + if not result.passed: self.log.info( - " WARNING: %s copy failed on %s:\n%s", - dst_file, hosts, result) + " WARNING: %s copy failed on %s", dst_file, result.failed_hosts) names.add(name) yaml = yaml.other_params @@ -1051,21 +1049,18 @@ def copy_configuration(self, hosts): Raises: CommandFailure: if there is an error copying the configuration file - """ if self.yaml is not None and hasattr(self.yaml, "filename"): if self.temporary_file and hosts: self.log.info( "Copying %s yaml configuration file to %s on %s", self.temporary_file, self.yaml.filename, hosts) - try: - distribute_files( - hosts, self.temporary_file, self.yaml.filename, - verbose=False, sudo=True) - except DaosTestError as error: + result = distribute_files( + self.log, hosts, self.temporary_file, self.yaml.filename, verbose=False, + sudo=True) + if not result.passed: raise CommandFailure( - "ERROR: Copying yaml configuration file to {}: " - "{}".format(hosts, error)) from error + f"ERROR: Copying yaml configuration file to {result.failed_hosts}") def verify_socket_directory(self, user, hosts): """Verify the domain socket directory is present and owned by this user. @@ -1088,15 +1083,17 @@ def verify_socket_directory(self, user, hosts): self.log.info( "%s: creating socket directory %s for user %s on %s", self.command, directory, user, nodes) - try: - create_directory(nodes, directory, sudo=True) - change_file_owner(nodes, directory, user, get_primary_group(user), sudo=True) - except DaosTestError as error: + result = create_directory(self.log, nodes, directory, user="root") + if not result.passed: + raise CommandFailure( + f"{self.command}: error creating socket directory {directory} for user " + f"{user} on {result.failed_hosts}") + result = change_file_owner( + self.log, nodes, directory, user, get_primary_group(user), user="root") + if not result.passed: raise CommandFailure( - "{}: error setting up missing socket directory {} for " - "user {} on {}:\n{}".format( - self.command, directory, user, nodes, - error)) from error + f"{self.command}: error setting socket directory {directory} owner for " + f"user {user} on {result.failed_hosts}") def get_socket_dir(self): """Get the socket directory. diff --git a/src/tests/ftest/util/environment_utils.py b/src/tests/ftest/util/environment_utils.py index 1e295be3815..e36d750500e 100644 --- a/src/tests/ftest/util/environment_utils.py +++ b/src/tests/ftest/util/environment_utils.py @@ -9,6 +9,7 @@ from ClusterShell.NodeSet import NodeSet # pylint: disable=import-error,no-name-in-module +from util.host_utils import get_local_host from util.network_utils import (PROVIDER_ALIAS, SUPPORTED_PROVIDERS, NetworkException, get_common_provider, get_fastest_interface) from util.run_utils import run_remote @@ -109,6 +110,9 @@ class TestEnvironment(): 'daos_prefix': 'DAOS_TEST_PREFIX', 'agent_user': 'DAOS_TEST_AGENT_USER', 'systemd_library_path': 'DAOS_TEST_SYSTEMD_LIBRARY_PATH', + 'control_config': 'DAOS_TEST_CONTROL_CONFIG', + 'agent_config': 'DAOS_TEST_AGENT_CONFIG', + 'server_config': 'DAOS_TEST_SERVER_CONFIG', } def __init__(self): @@ -155,29 +159,34 @@ def set_defaults(self, logger, servers=None, clients=None, provider=None, insecu # Set defaults for any unset values if self.log_dir is None: - self.log_dir = self._default_log_dir() + self.log_dir = os.path.join(os.sep, "var", "tmp", "daos_testing") if self.shared_dir is None: - self.shared_dir = self._default_shared_dir() + self.shared_dir = os.path.expanduser(os.path.join("~", "daos_test")) if self.app_dir is None: - self.app_dir = self._default_app_dir() + self.app_dir = os.path.join(self.shared_dir, "daos_test", "apps") if self.user_dir is None: - self.user_dir = self._default_user_dir() + self.user_dir = os.path.join(self.log_dir, "user") if self.interface is None: self.interface = self._default_interface(logger, all_hosts) if self.provider is None: self.provider = self._default_provider(logger, servers) if self.insecure_mode is None: - self.insecure_mode = self._default_insecure_mode() + self.insecure_mode = "True" if self.bullseye_src is None: - self.bullseye_src = self._default_bullseye_src() + self.bullseye_src = os.path.join( + os.path.dirname(os.path.abspath(__file__)), "..", "test.cov") if self.bullseye_file is None: - self.bullseye_file = self._default_bullseye_file() + self.bullseye_file = os.path.join(os.sep, "tmp", "test.cov") if self.daos_prefix is None: self.daos_prefix = self._default_daos_prefix(logger) if self.agent_user is None: - self.agent_user = self._default_agent_user() - if self.systemd_library_path is None: - self.systemd_library_path = self._default_systemd_library_path() + self.agent_user = 'root' + if self.control_config is None: + self.control_config = os.path.join(self.log_dir, "configs", "daos_control.yml") + if self.agent_config is None: + self.agent_config = os.path.join(self.log_dir, "configs", "daos_agent.yml") + if self.server_config is None: + self.server_config = os.path.join(self.log_dir, "configs", "daos_server.yml") def __set_value(self, key, value): """Set the test environment variable. @@ -209,14 +218,6 @@ def app_dir(self, value): """ self.__set_value('app_dir', value) - def _default_app_dir(self): - """Get the default application directory path. - - Returns: - str: the default application directory path - """ - return os.path.join(self.shared_dir, "daos_test", "apps") - @property def app_src(self): """Get the location from which to copy test applications. @@ -253,15 +254,6 @@ def log_dir(self, value): """ self.__set_value('log_dir', value) - @staticmethod - def _default_log_dir(): - """Get the default local log directory path. - - Returns: - str: the default local log directory path - """ - return os.path.join(os.sep, "var", "tmp", "daos_testing") - @property def shared_dir(self): """Get the shared log directory path. @@ -280,15 +272,6 @@ def shared_dir(self, value): """ self.__set_value('shared_dir', value) - @staticmethod - def _default_shared_dir(): - """Get the default shared log directory path. - - Returns: - str: the default shared log directory path - """ - return os.path.expanduser(os.path.join("~", "daos_test")) - @property def user_dir(self): """Get the user directory path. @@ -307,14 +290,6 @@ def user_dir(self, value): """ self.__set_value('user_dir', value) - def _default_user_dir(self): - """Get the default user directory path. - - Returns: - str: the default user directory path - """ - return os.path.join(self.log_dir, "user") - @property def interface(self): """Get the interface device. @@ -352,7 +327,7 @@ def _default_interface(self, logger, hosts): # Find all the /sys/class/net interfaces on the launch node (excluding lo) logger.debug("Detecting network devices - D_INTERFACE not set") try: - interface = get_fastest_interface(logger, hosts) + interface = get_fastest_interface(logger, hosts | get_local_host()) except NetworkException as error: raise TestEnvironmentException("Error obtaining a default interface!") from error return interface @@ -447,15 +422,6 @@ def insecure_mode(self, value): """ self.__set_value('insecure_mode', value) - @staticmethod - def _default_insecure_mode(): - """Get the default insecure mode. - - Returns: - str: the default insecure mode - """ - return "True" - @property def bullseye_src(self): """Get the bullseye source file. @@ -474,15 +440,6 @@ def bullseye_src(self, value): """ self.__set_value('bullseye_src', value) - @staticmethod - def _default_bullseye_src(): - """Get the default bullseye source file. - - Returns: - str: the default bullseye source file - """ - return os.path.join(os.path.dirname(os.path.abspath(__file__)), "..", "test.cov") - @property def bullseye_file(self): """Get the bullseye file. @@ -501,15 +458,6 @@ def bullseye_file(self, value): """ self.__set_value('bullseye_file', value) - @staticmethod - def _default_bullseye_file(): - """Get the default bullseye file. - - Returns: - str: the default bullseye file - """ - return os.path.join(os.sep, "tmp", "test.cov") - @property def daos_prefix(self): """Get the daos_prefix. @@ -572,15 +520,6 @@ def agent_user(self, value): """ self.__set_value('agent_user', value) - @staticmethod - def _default_agent_user(): - """Get the default daos_agent user. - - Returns: - str: the default daos_agent user - """ - return 'root' - @property def systemd_library_path(self): """Get the systemd LD_LIBRARY_PATH. @@ -599,14 +538,71 @@ def systemd_library_path(self, value): """ self.__set_value('systemd_library_path', value) - @staticmethod - def _default_systemd_library_path(): - """Get the default systemd LD_LIBRARY_PATH. + @property + def control_config(self): + """Get the control config file used in testing. + + Returns: + str: the control config file + """ + return os.environ.get(self.__ENV_VAR_MAP['control_config']) + + @control_config.setter + def control_config(self, value): + """Set the control config file used in testing. + + Args: + value (str): the control config file + """ + self.__set_value('control_config', value) + + @property + def agent_config(self): + """Get the agent config file used in testing. + + Returns: + str: the agent config file + """ + return os.environ.get(self.__ENV_VAR_MAP['agent_config']) + + @agent_config.setter + def agent_config(self, value): + """Set the agent config file used in testing. + + Args: + value (str): the agent config file + """ + self.__set_value('agent_config', value) + + @property + def server_config(self): + """Get the server config file used in testing. + + Returns: + str: the server config file + """ + return os.environ.get(self.__ENV_VAR_MAP['server_config']) + + @server_config.setter + def server_config(self, value): + """Set the server config file used in testing. + + Args: + value (str): the server config file + """ + self.__set_value('server_config', value) + + def config_file_directories(self): + """Get the unique list of directories for the client, control, and server config files. Returns: - str: the default systemd LD_LIBRARY_PATH + list: a list of directories for the client, control, and server config files """ - return None + directories = set() + directories.add(os.path.dirname(self.agent_config)) + directories.add(os.path.dirname(self.control_config)) + directories.add(os.path.dirname(self.server_config)) + return list(directories) def set_test_environment(logger, test_env=None, servers=None, clients=None, provider=None, diff --git a/src/tests/ftest/util/fault_config_utils.py b/src/tests/ftest/util/fault_config_utils.py index b6d437c2e29..5dd3071271f 100644 --- a/src/tests/ftest/util/fault_config_utils.py +++ b/src/tests/ftest/util/fault_config_utils.py @@ -8,7 +8,7 @@ import os import yaml -from general_utils import distribute_files +from file_utils import distribute_files from run_utils import run_local, run_remote # a lookup table of predefined faults @@ -295,15 +295,22 @@ def start(self, fault_list, test_dir): # orterun or something, could re-evaluate this later self.write_fault_file(None) - def copy_fault_files(self, hosts): + def copy_fault_files(self, logger, hosts): """Copy the fault injection file to all test hosts. Args: + logger (Logger): logger for the messages produced by this method hosts (list): list of hosts to copy the fault injection file + + Raises: + FaultInjectionFailed: if there is an error copying the fault injection files """ if self._fault_list: self._hosts = hosts - distribute_files(self._hosts, self.fault_file, self.fault_file) + result = distribute_files(logger, self._hosts, self.fault_file, self.fault_file) + if not result.passed: + raise FaultInjectionFailed( + f"Error copying fault injection files to {result.failed_hosts}") def stop(self): """Remove the fault injection file created during testing. diff --git a/src/tests/ftest/util/file_utils.py b/src/tests/ftest/util/file_utils.py new file mode 100644 index 00000000000..13f7e45a382 --- /dev/null +++ b/src/tests/ftest/util/file_utils.py @@ -0,0 +1,147 @@ +""" + (C) Copyright 2018-2024 Intel Corporation. + + SPDX-License-Identifier: BSD-2-Clause-Patent +""" + +import os + +# pylint: disable=import-error,no-name-in-module +from util.host_utils import get_local_host +from util.run_utils import command_as_user, get_clush_command, run_local, run_remote +from util.user_utils import get_chown_command, get_primary_group + + +def __run_command(logger, hosts, command, verbose=True, timeout=15): + """Run the command locally if there are no remote hosts or remotely. + + Args: + logger (Logger): logger for the messages produced by this method + hosts (NodeSet): hosts on which to run the command + command (str): command to run + verbose (bool, optional): log the command output. Defaults to True. + timeout (int, optional): command timeout. Defaults to 15 seconds. + + Returns: + CommandResult: groups of command results from the same hosts with the same return status + """ + if not hosts.difference(get_local_host()): + return run_local(logger, command, verbose, timeout) + return run_remote(logger, hosts, command, verbose, timeout) + + +def create_directory(logger, hosts, directory, timeout=15, verbose=True, user=None): + """Create the specified directory on the specified hosts. + + Args: + logger (Logger): logger for the messages produced by this method + hosts (NodeSet): hosts on which to create the directory + directory (str): the directory to create + timeout (int, optional): command timeout. Defaults to 15 seconds. + verbose (bool, optional): log the command output. Defaults to True. + user (str, optional): user with which to run the command. Defaults to None. + + Returns: + CommandResult: groups of command results from the same hosts with the same return status + """ + command = command_as_user(f"mkdir -p {directory}", user) + return __run_command(logger, hosts, command, verbose, timeout) + + +def change_file_owner(logger, hosts, filename, owner, group, timeout=15, verbose=True, user=None): + """Create the specified directory on the specified hosts. + + Args: + logger (Logger): logger for the messages produced by this method + hosts (NodeSet): hosts on which to create the directory + filename (str): the file for which to change ownership + owner (str): new owner of the file + group (str): new group owner of the file + timeout (int, optional): command timeout. Defaults to 15 seconds. + verbose (bool, optional): log the command output. Defaults to True. + user (str, optional): user with which to run the command. Defaults to None. + + Returns: + CommandResult: groups of command results from the same hosts with the same return status + """ + command = command_as_user(get_chown_command(owner, group, file=filename), user) + return __run_command(logger, hosts, command, verbose, timeout) + + +def get_file_size(logger, host, file_name): + """Obtain the file size on the specified host. + + Args: + logger (Logger): logger for the messages produced by this method + host (NodeSet): host from which to get the file size + file_name (str): name of remote file + + Returns: + CommandResult: groups of command results from the same hosts with the same return status + """ + return __run_command(logger, host, f"stat -c%s {file_name}") + + +def distribute_files(logger, hosts, source, destination, mkdir=True, timeout=60, + verbose=True, sudo=False, owner=None): + """Copy the source to the destination on each of the specified hosts. + + Optionally (by default) ensure the destination directory exists on each of + the specified hosts prior to copying the source. + + Args: + logger (Logger): logger for the messages produced by this method + hosts (NodeSet): hosts on which to copy the source + source (str): the file to copy to the hosts + destination (str): the host location in which to copy the source + mkdir (bool, optional): whether or not to ensure the destination + directory exists on hosts prior to copying the source. Defaults to + True. + timeout (int, optional): command timeout. Defaults to 60 seconds. + verbose (bool, optional): whether to log the command run and + stdout/stderr. Defaults to True. + raise_exception (bool, optional): whether to raise an exception if the + command returns a non-zero exit status. Defaults to True. + sudo (bool, optional): whether to run the command via sudo. Defaults to + False. + owner (str, optional): if specified the owner to assign as the owner of + the copied file. Defaults to None. + + Returns: + CommandResult: groups of command results from the same hosts with the same return status + """ + result = None + if mkdir: + result = create_directory(logger, hosts, os.path.dirname(destination), timeout, verbose) + + if result is None or result.passed: + if sudo: + # In order to copy a protected file to a remote host in CI the source will first be + # copied as is to the remote host + other_hosts = hosts.difference(get_local_host()) + if other_hosts: + # Existing files with strict file permissions can cause the subsequent non-sudo + # copy to fail, so remove the file first + _rm_command = command_as_user(f"rm -f {source}", "root") + run_remote(logger, other_hosts, _rm_command, verbose, timeout) + result = distribute_files( + logger, other_hosts, source, source, mkdir=True, + timeout=timeout, verbose=verbose, sudo=False, owner=None) + + if result is None or result.passed: + # Then a local sudo copy will be executed on the remote node to copy the source + # to the destination + _cp_command = command_as_user(f"cp {source} {destination}", "root") + result = run_remote(logger, hosts, _cp_command, verbose, timeout) + else: + # Without the sudo requirement copy the source to the destination directly with clush + _command = get_clush_command(hosts, args=f"-S -v --copy {source} --dest {destination}") + result = run_local(logger, _command, verbose, timeout) + + # If requested update the ownership of the destination file + if owner is not None and result.passed: + result = change_file_owner( + logger, hosts, destination, owner, get_primary_group(owner), timeout, verbose, + "root" if sudo else None) + + return result diff --git a/src/tests/ftest/util/general_utils.py b/src/tests/ftest/util/general_utils.py index 7ef3b73572d..84e55601ff2 100644 --- a/src/tests/ftest/util/general_utils.py +++ b/src/tests/ftest/util/general_utils.py @@ -16,15 +16,13 @@ from getpass import getuser from importlib import import_module from logging import getLogger -from socket import gethostname from avocado.core.settings import settings from avocado.core.version import MAJOR from avocado.utils import process from ClusterShell.NodeSet import NodeSet from ClusterShell.Task import task_self -from run_utils import command_as_user, get_clush_command, run_local, run_remote -from user_utils import get_chown_command, get_primary_group +from run_utils import command_as_user, run_local, run_remote class DaosTestError(Exception): @@ -891,177 +889,20 @@ def convert_string(item, separator=","): return item -def create_directory(hosts, directory, timeout=15, verbose=True, - raise_exception=True, sudo=False): - """Create the specified directory on the specified hosts. - - Args: - hosts (NodeSet): hosts on which to create the directory - directory (str): the directory to create - timeout (int, optional): command timeout. Defaults to 15 seconds. - verbose (bool, optional): whether to log the command run and - stdout/stderr. Defaults to True. - raise_exception (bool, optional): whether to raise an exception if the - command returns a non-zero exit status. Defaults to True. - sudo (bool, optional): whether to run the command via sudo. Defaults to - False. - - Raises: - DaosTestError: if there is an error running the command - - Returns: - CmdResult: an avocado.utils.process CmdResult object containing the - result of the command execution. A CmdResult object has the - following properties: - command - command string - exit_status - exit_status of the command - stdout - the stdout - stderr - the stderr - duration - command execution time - interrupted - whether the command completed within timeout - pid - command's pid - - """ - mkdir_command = "/usr/bin/mkdir -p {}".format(directory) - command = get_clush_command(hosts, args="-S -v", command=mkdir_command, command_sudo=sudo) - return run_command(command, timeout=timeout, verbose=verbose, raise_exception=raise_exception) - - -def change_file_owner(hosts, filename, owner, group, timeout=15, verbose=True, - raise_exception=True, sudo=False): - """Create the specified directory on the specified hosts. - - Args: - hosts (NodeSet): hosts on which to create the directory - filename (str): the file for which to change ownership - owner (str): new owner of the file - group (str): new group owner of the file - timeout (int, optional): command timeout. Defaults to 15 seconds. - verbose (bool, optional): whether to log the command run and - stdout/stderr. Defaults to True. - raise_exception (bool, optional): whether to raise an exception if the - command returns a non-zero exit status. Defaults to True. - sudo (bool, optional): whether to run the command via sudo. Defaults to - False. - - Raises: - DaosTestError: if there is an error running the command - - Returns: - CmdResult: an avocado.utils.process CmdResult object containing the - result of the command execution. A CmdResult object has the - following properties: - command - command string - exit_status - exit_status of the command - stdout - the stdout - stderr - the stderr - duration - command execution time - interrupted - whether the command completed within timeout - pid - command's pid - - """ - chown_command = get_chown_command(owner, group, file=filename) - command = get_clush_command(hosts, args="-S -v", command=chown_command, command_sudo=sudo) - return run_command(command, timeout=timeout, verbose=verbose, raise_exception=raise_exception) - - -def distribute_files(hosts, source, destination, mkdir=True, timeout=60, - verbose=True, raise_exception=True, sudo=False, - owner=None): - """Copy the source to the destination on each of the specified hosts. - - Optionally (by default) ensure the destination directory exists on each of - the specified hosts prior to copying the source. - - Args: - hosts (NodeSet): hosts on which to copy the source - source (str): the file to copy to the hosts - destination (str): the host location in which to copy the source - mkdir (bool, optional): whether or not to ensure the destination - directory exists on hosts prior to copying the source. Defaults to - True. - timeout (int, optional): command timeout. Defaults to 60 seconds. - verbose (bool, optional): whether to log the command run and - stdout/stderr. Defaults to True. - raise_exception (bool, optional): whether to raise an exception if the - command returns a non-zero exit status. Defaults to True. - sudo (bool, optional): whether to run the command via sudo. Defaults to - False. - owner (str, optional): if specified the owner to assign as the owner of - the copied file. Defaults to None. - - Raises: - DaosTestError: if there is an error running the command - - Returns: - CmdResult: an avocado.utils.process CmdResult object containing the - result of the command execution. A CmdResult object has the - following properties: - command - command string - exit_status - exit_status of the command - stdout - the stdout - stderr - the stderr - duration - command execution time - interrupted - whether the command completed within timeout - pid - command's pid - - """ - result = None - if mkdir: - result = create_directory( - hosts, os.path.dirname(destination), verbose=verbose, - raise_exception=raise_exception) - if result is None or result.exit_status == 0: - if sudo: - # In order to copy a protected file to a remote host in CI the - # source will first be copied as is to the remote host - localhost = gethostname().split(".")[0] - other_hosts = NodeSet.fromlist([host for host in hosts if host != localhost]) - if other_hosts: - # Existing files with strict file permissions can cause the - # subsequent non-sudo copy to fail, so remove the file first - rm_command = get_clush_command( - other_hosts, args="-S -v", command="rm -f {}".format(source), - command_sudo=True) - run_command(rm_command, verbose=verbose, raise_exception=False) - result = distribute_files( - other_hosts, source, source, mkdir=True, - timeout=timeout, verbose=verbose, - raise_exception=raise_exception, sudo=False, owner=None) - if result is None or result.exit_status == 0: - # Then a local sudo copy will be executed on the remote node to - # copy the source to the destination - command = get_clush_command( - hosts, args="-S -v", command="cp {} {}".format(source, destination), - command_sudo=True) - result = run_command(command, timeout, verbose, raise_exception) - else: - # Without the sudo requirement copy the source to the destination - # directly with clush - command = get_clush_command( - hosts, args="-S -v --copy {} --dest {}".format(source, destination)) - result = run_command(command, timeout, verbose, raise_exception) - - # If requested update the ownership of the destination file - if owner is not None and result.exit_status == 0: - change_file_owner( - hosts, destination, owner, get_primary_group(owner), timeout=timeout, - verbose=verbose, raise_exception=raise_exception, sudo=sudo) - return result - - -def get_default_config_file(name): +def get_default_config_file(name, path=None): """Get the default config file. Args: name (str): daos component name, e.g. server, agent, control + path (str, optional): path to use for the config file. Defaults to None which will use the + /etc/daos default. Returns: str: the default config file - """ - file_name = "".join(["daos_", name, ".yml"]) - return os.path.join(os.sep, "etc", "daos", file_name) + if path is None: + path = os.path.join(os.sep, "etc", "daos") + return os.path.join(path, f"daos_{name}.yml") def get_file_listing(hosts, files, user): diff --git a/src/tests/ftest/util/launch_utils.py b/src/tests/ftest/util/launch_utils.py index a86df01ea55..0f7284c50ef 100644 --- a/src/tests/ftest/util/launch_utils.py +++ b/src/tests/ftest/util/launch_utils.py @@ -20,9 +20,10 @@ from util.host_utils import HostException, HostInfo, get_local_host, get_node_set from util.logger_utils import LOG_FILE_FORMAT, get_file_handler from util.results_utils import LaunchTestName -from util.run_utils import RunException, run_local, run_remote +from util.run_utils import RunException, command_as_user, run_local, run_remote from util.slurm_utils import create_partition, delete_partition, show_partition from util.storage_utils import StorageException, StorageInfo +from util.systemctl_utils import SystemctlFailure, create_override_config from util.user_utils import get_group_id, get_user_groups, groupadd, useradd, userdel from util.yaml_utils import YamlUpdater, get_yaml_data @@ -77,6 +78,66 @@ def setup_fuse_config(logger, hosts): raise LaunchException(f"Failed to setup {config}") +def __add_systemctl_override(logger, hosts, service, user, command, config, path, lib_path): + """Add a systemctl override file for the specified service. + + Args: + logger (Logger): logger for the messages produced by this method + hosts (NodeSet): hosts on which to create the systemctl config + service (str): service for which to issue the command + user (str): user to use to issue the command + service_command (str): full path to the service command + service_config (str): full path to the service config + path (str): the PATH variable to set in the systemd config. + ld_library_path (str): the LD_LIBRARY_PATH variable to set in the systemd config. + + Raises: + LaunchException: if setup fails + + Returns: + dict: a dictionary of the systemctl override config file key with a dictionary value + containing the related host and user information + """ + logger.debug("-" * 80) + logger.info("Setting up systemctl override for %s", service) + try: + systemctl_override = create_override_config( + logger, hosts, service, user, command, config, path, lib_path) + except SystemctlFailure as error: + raise LaunchException(f"Failed to setup systemctl config for {service}") from error + return {systemctl_override: {"hosts": hosts, "user": user}} + + +def setup_systemctl(logger, servers, clients, test_env): + """Set up the systemctl override files for the daos_server and daos_agent. + + Args: + logger (Logger): logger for the messages produced by this method + servers (NodeSet): hosts that may run the daos_server command + clients (NodeSet): hosts that may run the daos_agent command + test_env (TestEnvironment): the test environment + + Raises: + LaunchException: if setup fails + + Returns: + dict: a dictionary of systemctl override config file keys with NodeSet values identifying + the hosts on which to remove the config files at the end of testing + """ + systemctl_configs = {} + systemctl_configs.update( + __add_systemctl_override( + logger, servers, "daos_server.service", "root", + os.path.join(test_env.daos_prefix, "bin", "daos_server"), test_env.server_config, + None, None)) + systemctl_configs.update( + __add_systemctl_override( + logger, clients, "daos_agent.service", test_env.agent_user, + os.path.join(test_env.daos_prefix, "bin", "daos_agent"), test_env.agent_config, + None, None)) + return systemctl_configs + + def display_disk_space(logger, path): """Display disk space of provided path destination. @@ -544,12 +605,13 @@ def _setup_test_directory(self, logger, test): f"sudo -n rm -fr {test_env.log_dir}", f"mkdir -p {test_env.log_dir}", f"chmod a+wrx {test_env.log_dir}", - f"ls -al {test_env.log_dir}", - f"mkdir -p {test_env.user_dir}" ] # Predefine the sub directories used to collect the files process()/_archive_files() + directories = [test_env.user_dir] + test_env.config_file_directories() for directory in TEST_RESULTS_DIRS: - commands.append(f"mkdir -p {test_env.log_dir}/{directory}") + directories.append(os.path.join(test_env.log_dir, directory)) + commands.append(f"mkdir -p {' '.join(directories)}") + commands.append(f"ls -al {test_env.log_dir}") for command in commands: if not run_remote(logger, hosts, command).passed: message = "Error setting up the common test directory on all hosts" @@ -1161,7 +1223,7 @@ def _setup_application_directory(self, logger, result): def run_tests(self, logger, result, repeat, slurm_setup, sparse, fail_fast, stop_daos, archive, rename, jenkins_log, core_files, threshold, user_create, code_coverage, - job_results_dir, logdir, clear_mounts): + job_results_dir, logdir, clear_mounts, cleanup_files): # pylint: disable=too-many-arguments """Run all the tests. @@ -1183,6 +1245,7 @@ def run_tests(self, logger, result, repeat, slurm_setup, sparse, fail_fast, stop job_results_dir (str): avocado job-results directory logdir (str): base directory in which to place the log file clear_mounts (list): mount points to remove before each test + cleanup_files (dict): files to remove on specific hosts at the end of testing Returns: int: status code indicating any issues running tests @@ -1233,6 +1296,12 @@ def run_tests(self, logger, result, repeat, slurm_setup, sparse, fail_fast, stop # Stop logging to the test log file logger.removeHandler(test_file_handler) + # Cleanup any specified files at the end of testing + for file, info in cleanup_files.items(): + command = command_as_user(f"rm -fr {file}", info['user']) + if not run_remote(logger, info['hosts'], command).passed: + return_code |= 16 + # Collect code coverage files after all test have completed if not code_coverage.finalize(logger, job_results_dir, result.tests[0]): return_code |= 16 diff --git a/src/tests/ftest/util/server_utils.py b/src/tests/ftest/util/server_utils.py index 299103fd577..c3b41a69b50 100644 --- a/src/tests/ftest/util/server_utils.py +++ b/src/tests/ftest/util/server_utils.py @@ -459,7 +459,8 @@ def support_collect_log(self, **kwargs): cmd = DaosServerCommand(self.manager.job.command_path) cmd.run_user = "daos_server" cmd.debug.value = False - kwargs['config'] = get_default_config_file("server") + kwargs['config'] = get_default_config_file( + "server", os.path.dirname(self.manager.job.yaml.filename)) cmd.set_command(("support", "collect-log"), **kwargs) self.log.info("Support collect-log on servers: %s", str(cmd)) return run_remote( diff --git a/src/tests/ftest/util/systemctl_utils.py b/src/tests/ftest/util/systemctl_utils.py new file mode 100644 index 00000000000..848b7dc6d31 --- /dev/null +++ b/src/tests/ftest/util/systemctl_utils.py @@ -0,0 +1,249 @@ +""" + (C) Copyright 2018-2024 Intel Corporation. + + SPDX-License-Identifier: BSD-2-Clause-Patent +""" + +import getpass +import os +import tempfile + +from ClusterShell.NodeSet import NodeSet +# pylint: disable=import-error,no-name-in-module +from util.file_utils import create_directory, distribute_files +from util.run_utils import command_as_user, run_remote + + +class SystemctlFailure(Exception): + """Base exception for this module.""" + + +def get_service_status(logger, hosts, service, user="root"): + """Get the status of the daos_server.service. + + Args: + logger (Logger): logger for the messages produced by this method + hosts (NodeSet): hosts on which to get the service state + service (str): name of the service + user (str, optional): user to use to issue the command. Defaults to "root". + + Returns: + dict: a dictionary with the following keys: + - "status": boolean set to True if status was obtained; False otherwise + - "stop": NodeSet where to stop the daos_server.service + - "disable": NodeSet where to disable the daos_server.service + - "reset-failed": NodeSet where to reset the daos_server.service + + """ + status = { + "status": True, + "stop": NodeSet(), + "disable": NodeSet(), + "reset-failed": NodeSet()} + status_states = { + "stop": ["active", "activating", "deactivating"], + "disable": ["active", "activating", "deactivating"], + "reset-failed": ["failed"]} + command = get_systemctl_command("is-active", service, user) + result = run_remote(logger, hosts, command, False) + for data in result.output: + if data.timeout: + status["status"] = False + status["stop"].add(data.hosts) + status["disable"].add(data.hosts) + status["reset-failed"].add(data.hosts) + logger.debug(" %s: TIMEOUT", data.hosts) + break + logger.debug(" %s: %s", data.hosts, "\n".join(data.stdout)) + for key, state_list in status_states.items(): + for line in data.stdout: + if line in state_list: + status[key].add(data.hosts) + break + return status + + +def stop_service(logger, hosts, service, user="root"): + """Stop any daos_server.service running on the hosts running servers. + + Args: + logger (Logger): logger for the messages produced by this method + hosts (NodeSet): list of hosts on which to stop the service. + service (str): name of the service + user (str, optional): user to use to issue the command. Defaults to "root". + + Returns: + bool: True if the service was successfully stopped; False otherwise + + """ + if not hosts: + logger.debug(" Skipping stopping %s service - no hosts", service) + return True + + result = {"status": True} + status_keys = ["reset-failed", "stop", "disable"] + mapping = {"stop": "active", "disable": "enabled", "reset-failed": "failed"} + check_hosts = NodeSet(hosts) + loop = 1 + # Reduce 'max_loops' to 2 once https://jira.hpdd.intel.com/browse/DAOS-7809 has been resolved + max_loops = 3 + while check_hosts: + # Check the status of the service on each host + result = get_service_status(logger, check_hosts, service) + check_hosts = NodeSet() + for key in status_keys: + if result[key]: + if loop == max_loops: + # Exit the while loop if the service is still running + logger.error( + " - Error %s still %s on %s", service, mapping[key], result[key]) + result["status"] = False + else: + # Issue the appropriate systemctl command to remedy the + # detected state, e.g. 'stop' for 'active'. + command = command_as_user(get_systemctl_command(key, service, user), user) + run_remote(logger, result[key], command) + + # Run the status check again on this group of hosts + check_hosts.add(result[key]) + loop += 1 + + return result["status"] + + +def get_systemctl_command(unit_command, service, user="root"): + """Get the systemctl command for the specified inputs. + + Args: + unit_command (str): command to issue for the service + service (str): service for which to issue the command + user (str, optional): user to use to issue the command. Defaults to "root". + + Returns: + str: the systemctl command for the specified service and user + """ + command = ["systemctl"] + if user != "root": + command.append(f"--user {user}") + if unit_command: + command.append(unit_command) + if service: + command.append(service) + return " ".join(command) + + +def get_service_file(logger, hosts, service, user, verbose=True, timeout=120): + """Get the service file. + + Args: + logger (Logger): logger for the messages produced by this method + hosts (NodeSet): hosts on which to run the command + service (str): service for which to issue the command + user (str, optional): user to use to issue the command. Defaults to "root". + verbose (bool, optional): log the command output. Defaults to True. + timeout (int, optional): number of seconds to wait for the command to complete. + Defaults to 120 seconds. + + Raises: + SystemctlFailure: if there is a problem obtaining the service file + + Returns: + str: the service file + """ + command = ' | '.join([ + get_systemctl_command("status", service, user), + "grep 'Loaded:'", + "grep -oE '/.*service'", + "xargs sh -c '[ -e \"$0\" ] && echo \"$0\"'" + ]) + result = run_remote(logger, hosts, command, verbose, timeout) + if not result.passed: + raise SystemctlFailure("Error obtaining the service file path") + if not result.homogeneous: + raise SystemctlFailure("Error obtaining a homogeneous service file path") + return list(result.all_stdout.values())[0].strip() + + +def create_override_config(logger, hosts, service, user, service_command, service_config, path, + ld_library_path, verbose=True, timeout=120): + """Create a systemctl override config file. + + Args: + logger (Logger): logger for the messages produced by this method + hosts (NodeSet): hosts on which to run the command + service (str): service for which to issue the command + user (str): user to use to issue the command + service_command (str): full path to the service command + service_config (str): full path to the service config + path (str): the PATH variable to set in the systemd config. + ld_library_path (str): the LD_LIBRARY_PATH variable to set in the systemd config. + verbose (bool, optional): log the command output. Defaults to True. + timeout (int, optional): number of seconds to wait for the command to complete. + Defaults to 120 seconds. + + Raises: + SystemctlFailure: if there are problems detecting, creating, or distributing the systemctl + override config file + + Returns: + str: the systemctl override config file path + """ + # Get the existing service file + service_file = get_service_file(logger, hosts, service, user, verbose, timeout) + + # Create the override directory + override_file = os.path.join(f"{service_file}.d", "override.conf") + result = create_directory(logger, hosts, os.path.dirname(override_file), timeout, verbose, user) + if not result.passed: + raise SystemctlFailure("Error creating the systemctl override config directory") + + # Create the override file - empty ExecStart clears the existing setting + override_contents = [ + "[Service]", + "ExecStart=", + f"ExecStart={service_command} start -o {service_config}" + ] + if path: + override_contents.append(f'Environment="PATH={path}"') + if ld_library_path: + override_contents.append(f'Environment="LD_LIBRARY_PATH={ld_library_path}"') + override_contents = "\n".join(override_contents) + "\n" + + with tempfile.NamedTemporaryFile() as temp: + temp.write(bytes(override_contents, encoding='utf-8')) + temp.flush() + os.chmod(temp.name, 0o644) + + _sudo = user != getpass.getuser() + _owner = user if _sudo else None + + result = distribute_files( + logger, hosts, temp.name, override_file, mkdir=False, verbose=verbose, sudo=_sudo, + owner=_owner) + if not result.passed: + raise SystemctlFailure( + "Error distributing the systemctl override config directory") + + # Reload on all hosts to pick up changes + if not daemon_reload(logger, hosts, user, verbose, timeout).passed: + raise SystemctlFailure("Error reloading systemctl daemon with override config directory") + + return override_file + + +def daemon_reload(logger, hosts, user, verbose=True, timeout=120): + """Run systemctl daemon-reload. + + Args: + logger (Logger): logger for the messages produced by this method + hosts (NodeSet): hosts on which to run the command + user (str, optional): user to use to issue the command. Defaults to "root". + verbose (bool, optional): log the command output. Defaults to True. + timeout (int, optional): number of seconds to wait for the command to complete. + Defaults to 120 seconds. + + Returns: + CommandResult: groups of command results from the same hosts with the same return status + """ + command = get_systemctl_command("daemon-reload", None, user) + return run_remote(logger, hosts, command_as_user(command, user), verbose, timeout) From 663237cfec6a37487a6ad05fe71733cd56e190aa Mon Sep 17 00:00:00 2001 From: Li Wei Date: Thu, 29 Aug 2024 07:56:06 +0900 Subject: [PATCH 16/26] DAOS-16251 mgmt: Fix use-after-free in pool_list (#15014) In dc_mgmt_pool_list, calling wipe_cred_iov on in->pli_cred after calling crt_req_decref on rpc is a use-after-free. Signed-off-by: Li Wei --- src/mgmt/cli_mgmt.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/mgmt/cli_mgmt.c b/src/mgmt/cli_mgmt.c index c24f4802171..6bb1c7c9513 100644 --- a/src/mgmt/cli_mgmt.c +++ b/src/mgmt/cli_mgmt.c @@ -1367,8 +1367,8 @@ dc_mgmt_pool_list(tse_task_t *task) rc = daos_rpc_send_wait(rpc); if (rc != 0) { DL_ERROR(rc, "rpc send failed"); - crt_req_decref(rpc); wipe_cred_iov(&in->pli_cred); + crt_req_decref(rpc); goto rechoose; } @@ -1377,8 +1377,8 @@ dc_mgmt_pool_list(tse_task_t *task) rc = rsvc_client_complete_rpc(&ms_client, &ep, rc, out->plo_op.mo_rc, &out->plo_op.mo_hint); if (rc == RSVC_CLIENT_RECHOOSE) { - crt_req_decref(rpc); wipe_cred_iov(&in->pli_cred); + crt_req_decref(rpc); goto rechoose; } @@ -1430,8 +1430,8 @@ dc_mgmt_pool_list(tse_task_t *task) if (rc != 0) DL_ERROR(rc, "failed to list pools"); - crt_req_decref(rpc); wipe_cred_iov(&in->pli_cred); + crt_req_decref(rpc); out_client: rsvc_client_fini(&ms_client); out_grp: From 3d129b263f19f8c98d7f9fc32bedad83d626a5bc Mon Sep 17 00:00:00 2001 From: Cedric Koch-Hofer <94527853+knard-intel@users.noreply.github.com> Date: Thu, 29 Aug 2024 16:32:22 +0200 Subject: [PATCH 17/26] DAOS-14348 client: GC for pil4dfs dentry cache (#14995) Hotfix of miscellaneous issues: - invalid type of atomic_flag - invalid usage of DL_WARN macro Signed-off-by: Cedric Koch-Hofer --- src/client/dfuse/pil4dfs/dfs_dcache.c | 12 ++++++------ src/client/dfuse/pil4dfs/int_dfs.c | 12 ++++-------- 2 files changed, 10 insertions(+), 14 deletions(-) diff --git a/src/client/dfuse/pil4dfs/dfs_dcache.c b/src/client/dfuse/pil4dfs/dfs_dcache.c index 96bd96b668b..b0c138a4261 100644 --- a/src/client/dfuse/pil4dfs/dfs_dcache.c +++ b/src/client/dfuse/pil4dfs/dfs_dcache.c @@ -71,7 +71,7 @@ struct dfs_dcache { /** Next Garbage collection date */ struct timespec dd_expire_gc; /** True iff one thread is running the garbage collection */ - _Atomic bool dd_running_gc; + atomic_flag dd_running_gc; /** Destroy a dfs dir-cache */ destroy_fn_t destroy_fn; /** Return the dir-cahe record of a given location and insert it if needed */ @@ -95,7 +95,7 @@ struct dcache_rec { /** Reference counter used to manage memory deallocation */ _Atomic uint32_t dr_ref; /** True iff this record was deleted from the hash table*/ - _Atomic bool dr_deleted; + atomic_flag dr_deleted; /** Entry in the garbage collector list */ d_list_t dr_entry_gc; /** True iff this record is not in the garbage collector list */ @@ -309,7 +309,7 @@ gc_reclaim(dfs_dcache_t *dcache) dcache->dd_count_gc); out_unset: - atomic_store_relaxed(&dcache->dd_running_gc, false); + atomic_flag_clear(&dcache->dd_running_gc); out: return rc; } @@ -417,7 +417,7 @@ dcache_add_root(dfs_dcache_t *dcache, dfs_obj_t *obj) rec->dr_obj = obj; atomic_init(&rec->dr_ref, 0); - atomic_init(&rec->dr_deleted, false); + atomic_flag_clear(&rec->dr_deleted); memcpy(&rec->dr_key_child_prefix[0], &dcache->dd_key_root_prefix[0], DCACHE_KEY_PREF_SIZE); memcpy(&rec->dr_key[0], &dcache->dd_key_root_prefix[0], DCACHE_KEY_PREF_SIZE); rec->dr_key_len = DCACHE_KEY_PREF_SIZE - 1; @@ -470,7 +470,7 @@ dcache_create_act(dfs_t *dfs, uint32_t bits, uint32_t rec_timeout, uint32_t gc_p rc = D_MUTEX_INIT(&dcache_tmp->dd_mutex_gc, NULL); if (rc != 0) D_GOTO(error_mutex, daos_errno2der(rc)); - atomic_init(&dcache_tmp->dd_running_gc, false); + atomic_flag_clear(&dcache_tmp->dd_running_gc); D_INIT_LIST_HEAD(&dcache_tmp->dd_head_gc); rc = d_hash_table_create_inplace(D_HASH_FT_MUTEX | D_HASH_FT_LRU, bits, NULL, @@ -571,7 +571,7 @@ dcache_add(dfs_dcache_t *dcache, dcache_rec_t *parent, const char *name, const c D_GOTO(error, rc = -DER_NOMEM); atomic_init(&rec_tmp->dr_ref, 1); - atomic_init(&rec_tmp->dr_deleted, false); + atomic_flag_clear(&rec_tmp->dr_deleted); rc = dfs_lookup_rel(dcache->dd_dfs, parent->dr_obj, name, O_RDWR, &obj, &mode, NULL); if (rc != 0) diff --git a/src/client/dfuse/pil4dfs/int_dfs.c b/src/client/dfuse/pil4dfs/int_dfs.c index 7e009c70d11..81cd69ccda1 100644 --- a/src/client/dfuse/pil4dfs/int_dfs.c +++ b/src/client/dfuse/pil4dfs/int_dfs.c @@ -6776,26 +6776,22 @@ init_myhook(void) dcache_size_bits = DCACHE_SIZE_BITS; rc = d_getenv_uint32_t("D_IL_DCACHE_SIZE_BITS", &dcache_size_bits); if (rc != -DER_SUCCESS && rc != -DER_NONEXIST) - DS_WARN(daos_der2errno(rc), - "'D_IL_DCACHE_SIZE_BITS' env variable could not be used"); + DL_WARN(rc, "'D_IL_DCACHE_SIZE_BITS' env variable could not be used"); dcache_rec_timeout = DCACHE_REC_TIMEOUT; rc = d_getenv_uint32_t("D_IL_DCACHE_REC_TIMEOUT", &dcache_rec_timeout); if (rc != -DER_SUCCESS && rc != -DER_NONEXIST) - DS_WARN(daos_der2errno(rc), - "'D_IL_DCACHE_REC_TIMEOUT' env variable could not be used"); + DL_WARN(rc, "'D_IL_DCACHE_REC_TIMEOUT' env variable could not be used"); dcache_gc_period = DCACHE_GC_PERIOD; rc = d_getenv_uint32_t("D_IL_DCACHE_GC_PERIOD", &dcache_gc_period); if (rc != -DER_SUCCESS && rc != -DER_NONEXIST) - DS_WARN(daos_der2errno(rc), - "'D_IL_DCACHE_GC_PERIOD' env variable could not be used"); + DL_WARN(rc, "'D_IL_DCACHE_GC_PERIOD' env variable could not be used"); dcache_gc_reclaim_max = DCACHE_GC_RECLAIM_MAX; rc = d_getenv_uint32_t("D_IL_DCACHE_GC_RECLAIM_MAX", &dcache_gc_reclaim_max); if (rc != -DER_SUCCESS && rc != -DER_NONEXIST) - DS_WARN(daos_der2errno(rc), - "'D_IL_DCACHE_GC_RECLAIM_MAX' env variable could not be used"); + DL_WARN(rc, "'D_IL_DCACHE_GC_RECLAIM_MAX' env variable could not be used"); if (dcache_gc_reclaim_max == 0) { D_WARN("'D_IL_DCACHE_GC_RECLAIM_MAX' env variable could not be used: value == 0."); dcache_gc_reclaim_max = DCACHE_GC_RECLAIM_MAX; From 06238f299dcd504f4efc1edc72183e62f20e6123 Mon Sep 17 00:00:00 2001 From: Michael MacDonald Date: Thu, 29 Aug 2024 11:34:26 -0400 Subject: [PATCH 18/26] DAOS-16451 telemetry: Adjust type of (_sum|_sumsquares) (#15018) These stats metrics should be Counter rather than Gauge, as their values will only increase. Also includes a small fix for pem_test.go to set the test dir permissions explicitly irrespective of umask. Signed-off-by: Michael MacDonald --- src/control/lib/telemetry/promexp/util.go | 10 ++++++---- src/control/lib/telemetry/promexp/util_test.go | 14 ++++++++------ src/control/security/pem_test.go | 4 ++++ 3 files changed, 18 insertions(+), 10 deletions(-) diff --git a/src/control/lib/telemetry/promexp/util.go b/src/control/lib/telemetry/promexp/util.go index 6ddc46623d3..269e0c6d45a 100644 --- a/src/control/lib/telemetry/promexp/util.go +++ b/src/control/lib/telemetry/promexp/util.go @@ -141,16 +141,18 @@ func getMetricStats(baseName string, m telemetry.Metric) (stats []*metricStat) { desc: " (mean)", }, "sum": { - fn: func() float64 { return float64(ms.Sum()) }, - desc: " (sum)", + fn: func() float64 { return float64(ms.Sum()) }, + isCounter: true, + desc: " (sum)", }, "stddev": { fn: ms.StdDev, desc: " (std dev)", }, "sumsquares": { - fn: ms.SumSquares, - desc: " (sum of squares)", + fn: ms.SumSquares, + isCounter: true, + desc: " (sum of squares)", }, "samples": { fn: func() float64 { return float64(ms.SampleSize()) }, diff --git a/src/control/lib/telemetry/promexp/util_test.go b/src/control/lib/telemetry/promexp/util_test.go index 104da9ec383..323cdf1aaf6 100644 --- a/src/control/lib/telemetry/promexp/util_test.go +++ b/src/control/lib/telemetry/promexp/util_test.go @@ -88,9 +88,10 @@ func TestPromExp_getMetricStats(t *testing.T) { value: 3.0, }, { - name: "stats_gauge_sum", - desc: " (sum)", - value: 15.0, + name: "stats_gauge_sum", + desc: " (sum)", + value: 15.0, + isCounter: true, }, { name: "stats_gauge_samples", @@ -104,9 +105,10 @@ func TestPromExp_getMetricStats(t *testing.T) { value: 1.58113883, }, { - name: "stats_gauge_sumsquares", - desc: " (sum of squares)", - value: 55, + name: "stats_gauge_sumsquares", + desc: " (sum of squares)", + value: 55, + isCounter: true, }, }, }, diff --git a/src/control/security/pem_test.go b/src/control/security/pem_test.go index 88b501980e6..54990c37f47 100644 --- a/src/control/security/pem_test.go +++ b/src/control/security/pem_test.go @@ -190,6 +190,10 @@ func TestSecurity_Pem_ValidateCertDirectory(t *testing.T) { if err := os.Mkdir(testDir, tc.perms); err != nil { t.Fatal(err) } + // Use chmod to set the permissions regardless of umask. + if err := os.Chmod(testDir, tc.perms); err != nil { + t.Fatal(err) + } testFile := test.CreateTestFile(t, dir, "some content") path := testDir From 9735bf4ae30317e57fe9ebe8503f55cf5c25323e Mon Sep 17 00:00:00 2001 From: Kris Jacque Date: Thu, 29 Aug 2024 09:52:27 -0600 Subject: [PATCH 19/26] DAOS-16245 control: Fix dmg cont set-owner (#14945) - Fix bug where all CONT_PROP_SET RPCs were sent directly to ds_cont_op_hdlr instead of using the custom set_prop handler that detected server vs. client RPCs. ds_cont_op_hdlr now detects server RPCs and calls the appropriate handler. - Add CONT_PROP_SET_BYLABEL RPC to allow for container label usage via dmg (without pool/cont handles). - Update dmg cont set-owner command to accept positional arguments and allow either UUIDs or labels. Signed-off-by: Kris Jacque --- src/container/rpc.c | 3 +- src/container/rpc.h | 58 ++++- src/container/srv_container.c | 163 +++++++----- src/container/srv_internal.h | 3 +- src/control/cmd/daos/acl.go | 14 +- src/control/cmd/dmg/command_test.go | 2 +- src/control/cmd/dmg/cont.go | 45 ++-- src/control/cmd/dmg/cont_test.go | 44 ++-- src/control/cmd/dmg/json_test.go | 3 +- src/control/common/proto/mgmt/addons.go | 4 +- src/control/common/proto/mgmt/cont.pb.go | 122 +++------ src/control/common/proto/mgmt/mgmt.pb.go | 238 +++++++++--------- src/control/common/proto/mgmt/mgmt_grpc.pb.go | 10 +- src/control/lib/control/cont.go | 26 +- src/control/lib/control/cont_test.go | 70 +++--- src/control/server/mgmt_cont.go | 6 +- src/control/server/mgmt_cont_test.go | 24 +- src/include/daos_srv/container.h | 5 +- src/mgmt/cont.pb-c.c | 107 +------- src/mgmt/cont.pb-c.h | 54 +--- src/mgmt/srv_container.c | 39 +-- src/mgmt/srv_drpc.c | 32 +-- src/mgmt/srv_internal.h | 6 +- src/mgmt/tests/mocks.c | 16 +- src/mgmt/tests/mocks.h | 10 +- src/mgmt/tests/srv_drpc_tests.c | 77 +++--- src/proto/mgmt/cont.proto | 17 +- src/proto/mgmt/mgmt.proto | 4 +- src/tests/ftest/security/cont_get_acl.yaml | 1 + src/tests/ftest/security/cont_owner.py | 75 ++++++ .../ftest/util/cont_security_test_base.py | 4 +- src/tests/ftest/util/daos_utils.py | 2 +- src/tests/ftest/util/dmg_utils.py | 8 +- src/tests/ftest/util/dmg_utils_base.py | 8 +- 34 files changed, 628 insertions(+), 672 deletions(-) diff --git a/src/container/rpc.c b/src/container/rpc.c index 6c3e96b57c0..51a13ed256e 100644 --- a/src/container/rpc.c +++ b/src/container/rpc.c @@ -1,5 +1,5 @@ /** - * (C) Copyright 2016-2023 Intel Corporation. + * (C) Copyright 2016-2024 Intel Corporation. * * SPDX-License-Identifier: BSD-2-Clause-Patent */ @@ -134,6 +134,7 @@ CRT_RPC_DEFINE(cont_tgt_snapshot_notify, DAOS_ISEQ_CONT_TGT_SNAPSHOT_NOTIFY, DAOS_OSEQ_CONT_TGT_SNAPSHOT_NOTIFY) CRT_RPC_DEFINE(cont_prop_set, DAOS_ISEQ_CONT_PROP_SET, DAOS_OSEQ_CONT_PROP_SET) CRT_RPC_DEFINE(cont_prop_set_v8, DAOS_ISEQ_CONT_PROP_SET_V8, DAOS_OSEQ_CONT_PROP_SET) +CRT_RPC_DEFINE(cont_prop_set_bylabel, DAOS_ISEQ_CONT_PROP_SET_BYLABEL, DAOS_OSEQ_CONT_PROP_SET) CRT_RPC_DEFINE(cont_acl_update, DAOS_ISEQ_CONT_ACL_UPDATE, DAOS_OSEQ_CONT_ACL_UPDATE) CRT_RPC_DEFINE(cont_acl_update_v8, DAOS_ISEQ_CONT_ACL_UPDATE_V8, DAOS_OSEQ_CONT_ACL_UPDATE) CRT_RPC_DEFINE(cont_acl_delete, DAOS_ISEQ_CONT_ACL_DELETE, DAOS_OSEQ_CONT_ACL_DELETE) diff --git a/src/container/rpc.h b/src/container/rpc.h index 61e66af5ced..7d15dba4c58 100644 --- a/src/container/rpc.h +++ b/src/container/rpc.h @@ -1,5 +1,5 @@ /** - * (C) Copyright 2016-2023 Intel Corporation. + * (C) Copyright 2016-2024 Intel Corporation. * * SPDX-License-Identifier: BSD-2-Clause-Patent */ @@ -74,7 +74,8 @@ X(CONT_TGT_EPOCH_AGGREGATE, 0, &CQF_cont_tgt_epoch_aggregate, \ ds_cont_tgt_epoch_aggregate_handler, &ds_cont_tgt_epoch_aggregate_co_ops) \ X(CONT_TGT_SNAPSHOT_NOTIFY, 0, &CQF_cont_tgt_snapshot_notify, \ - ds_cont_tgt_snapshot_notify_handler, &ds_cont_tgt_snapshot_notify_co_ops) + ds_cont_tgt_snapshot_notify_handler, &ds_cont_tgt_snapshot_notify_co_ops) \ + X(CONT_PROP_SET_BYLABEL, 0, &CQF_cont_prop_set_bylabel, ds_cont_set_prop_srv_handler, NULL) /* Define for RPC enum population below */ #define X(a, ...) a, @@ -919,24 +920,43 @@ CRT_RPC_DECLARE(cont_tgt_snapshot_notify, DAOS_ISEQ_CONT_TGT_SNAPSHOT_NOTIFY, CRT_RPC_DECLARE(cont_prop_set, DAOS_ISEQ_CONT_PROP_SET, DAOS_OSEQ_CONT_PROP_SET) CRT_RPC_DECLARE(cont_prop_set_v8, DAOS_ISEQ_CONT_PROP_SET_V8, DAOS_OSEQ_CONT_PROP_SET) +#define DAOS_ISEQ_CONT_PROP_SET_BYLABEL /* input fields */ \ + DAOS_ISEQ_CONT_PROP_SET_V8 \ + ((d_const_string_t) (cpsi_label) CRT_VAR) + +CRT_RPC_DECLARE(cont_prop_set_bylabel, DAOS_ISEQ_CONT_PROP_SET_BYLABEL, DAOS_OSEQ_CONT_PROP_SET) + /* clang-format on */ static inline void cont_prop_set_in_get_data(crt_rpc_t *rpc, crt_opcode_t opc, int cont_proto_ver, - daos_prop_t **cpsi_propp, uuid_t *cpsi_pool_uuidp) + daos_prop_t **cpsi_propp, uuid_t *cpsi_pool_uuidp, uuid_t *cpsi_co_uuidp, + const char **cont_label) { void *in = crt_req_get(rpc); - if (cont_proto_ver >= CONT_PROTO_VER_WITH_SVC_OP_KEY) { + if (opc == CONT_PROP_SET_BYLABEL) { + *cpsi_propp = ((struct cont_prop_set_bylabel_in *)in)->cpsi_prop; + if (cpsi_pool_uuidp) + uuid_copy(*cpsi_pool_uuidp, + ((struct cont_prop_set_bylabel_in *)in)->cpsi_pool_uuid); + if (cont_label != NULL) + *cont_label = ((struct cont_prop_set_bylabel_in *)in)->cpsi_label; + } else if (cont_proto_ver >= CONT_PROTO_VER_WITH_SVC_OP_KEY) { *cpsi_propp = ((struct cont_prop_set_v8_in *)in)->cpsi_prop; if (cpsi_pool_uuidp) uuid_copy(*cpsi_pool_uuidp, ((struct cont_prop_set_v8_in *)in)->cpsi_pool_uuid); + if (cpsi_co_uuidp) + uuid_copy(*cpsi_co_uuidp, + ((struct cont_prop_set_v8_in *)in)->cpsi_op.ci_uuid); } else { *cpsi_propp = ((struct cont_prop_set_in *)in)->cpsi_prop; if (cpsi_pool_uuidp) uuid_copy(*cpsi_pool_uuidp, ((struct cont_prop_set_in *)in)->cpsi_pool_uuid); + if (cpsi_co_uuidp) + uuid_copy(*cpsi_co_uuidp, ((struct cont_prop_set_in *)in)->cpsi_op.ci_uuid); } } @@ -946,16 +966,42 @@ cont_prop_set_in_set_data(crt_rpc_t *rpc, crt_opcode_t opc, int cont_proto_ver, { void *in = crt_req_get(rpc); - if (cont_proto_ver >= CONT_PROTO_VER_WITH_SVC_OP_KEY) { + if (opc == CONT_PROP_SET_BYLABEL) { + ((struct cont_prop_set_bylabel_in *)in)->cpsi_prop = cpsi_prop; + uuid_copy(((struct cont_prop_set_bylabel_in *)in)->cpsi_pool_uuid, cpsi_pool_uuid); + } else if (cont_proto_ver >= CONT_PROTO_VER_WITH_SVC_OP_KEY) { ((struct cont_prop_set_v8_in *)in)->cpsi_prop = cpsi_prop; uuid_copy(((struct cont_prop_set_v8_in *)in)->cpsi_pool_uuid, cpsi_pool_uuid); - } else { ((struct cont_prop_set_in *)in)->cpsi_prop = cpsi_prop; uuid_copy(((struct cont_prop_set_in *)in)->cpsi_pool_uuid, cpsi_pool_uuid); } } +static inline void +cont_prop_set_in_set_cont_uuid(crt_rpc_t *rpc, crt_opcode_t opc, int cont_proto_ver, + uuid_t cont_uuid) +{ + void *in = crt_req_get(rpc); + + D_ASSERT(opc == CONT_PROP_SET); + if (cont_proto_ver >= CONT_PROTO_VER_WITH_SVC_OP_KEY) + uuid_copy(((struct cont_prop_set_v8_in *)in)->cpsi_op.ci_uuid, cont_uuid); + else + uuid_copy(((struct cont_prop_set_in *)in)->cpsi_op.ci_uuid, cont_uuid); +} + +static inline void +cont_prop_set_bylabel_in_set_label(crt_rpc_t *rpc, crt_opcode_t opc, int cont_proto_ver, + const char *label) +{ + void *in = crt_req_get(rpc); + + D_ASSERT(opc == CONT_PROP_SET_BYLABEL); + /* NB: prop set by label is on the server side only - no version variants */ + ((struct cont_prop_set_bylabel_in *)in)->cpsi_label = label; +} + /* clang-format off */ #define DAOS_ISEQ_CONT_ACL_UPDATE /* input fields */ \ diff --git a/src/container/srv_container.c b/src/container/srv_container.c index 051fd309ece..9071f8f731c 100644 --- a/src/container/srv_container.c +++ b/src/container/srv_container.c @@ -3671,8 +3671,7 @@ check_set_prop_label(struct rdb_tx *tx, struct cont *cont, } static int -set_prop(struct rdb_tx *tx, struct ds_pool *pool, - struct cont *cont, uint64_t sec_capas, uuid_t hdl_uuid, +set_prop(struct rdb_tx *tx, struct ds_pool *pool, struct cont *cont, uint64_t sec_capas, daos_prop_t *prop_in) { int rc; @@ -3739,10 +3738,9 @@ ds_cont_prop_set(struct rdb_tx *tx, struct ds_pool_hdl *pool_hdl, struct cont *c DP_CONT(pool_hdl->sph_pool->sp_uuid, in->cpsi_op.ci_uuid), rpc, DP_UUID(in->cpsi_op.ci_hdl)); - cont_prop_set_in_get_data(rpc, CONT_PROP_SET, cont_proto_ver, &prop_in, NULL); + cont_prop_set_in_get_data(rpc, CONT_PROP_SET, cont_proto_ver, &prop_in, NULL, NULL, NULL); - return set_prop(tx, pool_hdl->sph_pool, cont, hdl->ch_sec_capas, - in->cpsi_op.ci_hdl, prop_in); + return set_prop(tx, pool_hdl->sph_pool, cont, hdl->ch_sec_capas, prop_in); } static int @@ -3796,8 +3794,7 @@ set_acl(struct rdb_tx *tx, struct ds_pool_hdl *pool_hdl, if (prop->dpp_entries[0].dpe_val_ptr == NULL) D_GOTO(out_prop, rc = -DER_NOMEM); - rc = set_prop(tx, pool_hdl->sph_pool, cont, hdl->ch_sec_capas, - hdl_uuid, prop); + rc = set_prop(tx, pool_hdl->sph_pool, cont, hdl->ch_sec_capas, prop); out_prop: daos_prop_free(prop); @@ -5505,6 +5502,8 @@ cont_cli_opc_name(crt_opcode_t opc) case CONT_SNAP_CREATE: return "SNAP_CREATE"; case CONT_SNAP_DESTROY: return "SNAP_DESTROY"; case CONT_PROP_SET: return "PROP_SET"; + case CONT_PROP_SET_BYLABEL: + return "PROP_SET_BYLABEL"; case CONT_ACL_UPDATE: return "ACL_UPDATE"; case CONT_ACL_DELETE: return "ACL_DELETE"; case CONT_OPEN_BYLABEL: return "OPEN_BYLABEL"; @@ -5529,6 +5528,20 @@ ds_cont_op_handler(crt_rpc_t *rpc, int cont_proto_ver) struct cont_svc *svc; int rc; + /* + * Some mgmt RPCs may come from either client or server (admin/dRPC) calls. RPCs from + * servers don't contain pool/cont handles. + */ + if (!daos_rpc_from_client(rpc)) { + switch (opc) { + case CONT_PROP_SET: + return ds_cont_set_prop_srv_handler(rpc); + default: + D_ASSERTF(false, "unexpected server RPC %s (%d)", cont_cli_opc_name(opc), + opc); + } + } + pool_hdl = ds_pool_hdl_lookup(in->ci_pool_hdl); if (pool_hdl == NULL) D_GOTO(out, rc = -DER_NO_HDL); @@ -5675,19 +5688,25 @@ ds_cont_oid_fetch_add(uuid_t po_uuid, uuid_t co_uuid, uint64_t num_oids, uint64_ /* Send the RPC from a DAOS server instance to the container service */ int -ds_cont_svc_set_prop(uuid_t pool_uuid, uuid_t cont_uuid, - d_rank_list_t *ranks, daos_prop_t *prop) +ds_cont_svc_set_prop(uuid_t pool_uuid, const char *cont_id, d_rank_list_t *ranks, daos_prop_t *prop) { - int rc; - struct rsvc_client client; - crt_endpoint_t ep; - uuid_t null_uuid; - struct dss_module_info *info = dss_get_module_info(); - crt_rpc_t *rpc; - struct cont_prop_set_out *out; + int rc; + struct rsvc_client client; + crt_endpoint_t ep; + crt_opcode_t opc = CONT_PROP_SET; + uuid_t cont_uuid; + uuid_t null_uuid; + struct dss_module_info *info = dss_get_module_info(); + crt_rpc_t *rpc; + struct cont_prop_set_out *out; + + D_DEBUG(DB_MGMT, DF_UUID "/%s: Setting container prop\n", DP_UUID(pool_uuid), cont_id); - D_DEBUG(DB_MGMT, DF_CONT": Setting container prop\n", - DP_CONT(pool_uuid, cont_uuid)); + /* cont_id may be a UUID or label */ + if (uuid_parse(cont_id, cont_uuid) != 0) + opc = CONT_PROP_SET_BYLABEL; + + D_DEBUG(DB_MGMT, "using opcode %s (%d)\n", cont_cli_opc_name(opc), opc); uuid_clear(null_uuid); rc = rsvc_client_init(&client, ranks); @@ -5696,30 +5715,31 @@ ds_cont_svc_set_prop(uuid_t pool_uuid, uuid_t cont_uuid, rechoose: ep.ep_grp = NULL; /* primary group */ - rc = rsvc_client_choose(&client, &ep); + rc = rsvc_client_choose(&client, &ep); if (rc != 0) { - D_ERROR(DF_CONT": cannot find pool service: "DF_RC"\n", - DP_CONT(pool_uuid, cont_uuid), DP_RC(rc)); + DL_ERROR(rc, DF_UUID ": cannot find pool service", DP_UUID(pool_uuid)); D_GOTO(out_client, rc); } - rc = cont_req_create(info->dmi_ctx, &ep, CONT_PROP_SET, null_uuid, null_uuid, null_uuid, + rc = cont_req_create(info->dmi_ctx, &ep, opc, null_uuid, null_uuid, null_uuid, NULL /* req_timep */, &rpc); if (rc != 0) { - D_ERROR(DF_CONT": failed to create cont set prop rpc: %d\n", - DP_CONT(pool_uuid, cont_uuid), rc); + DL_ERROR(rc, DF_UUID "/%s: failed to create cont set prop rpc", DP_UUID(pool_uuid), + cont_id); D_GOTO(out_client, rc); } - cont_prop_set_in_set_data(rpc, CONT_PROP_SET, DAOS_CONT_VERSION, prop, pool_uuid); + cont_prop_set_in_set_data(rpc, opc, DAOS_CONT_VERSION, prop, pool_uuid); + if (opc == CONT_PROP_SET_BYLABEL) + cont_prop_set_bylabel_in_set_label(rpc, opc, DAOS_CONT_VERSION, cont_id); + else /* CONT_PROP_SET */ + cont_prop_set_in_set_cont_uuid(rpc, opc, DAOS_CONT_VERSION, cont_uuid); - rc = dss_rpc_send(rpc); + rc = dss_rpc_send(rpc); out = crt_reply_get(rpc); D_ASSERT(out != NULL); - rc = rsvc_client_complete_rpc(&client, &ep, rc, - out->cpso_op.co_rc, - &out->cpso_op.co_hint); + rc = rsvc_client_complete_rpc(&client, &ep, rc, out->cpso_op.co_rc, &out->cpso_op.co_hint); if (rc == RSVC_CLIENT_RECHOOSE) { crt_req_decref(rpc); dss_sleep(1000 /* ms */); @@ -5728,8 +5748,8 @@ ds_cont_svc_set_prop(uuid_t pool_uuid, uuid_t cont_uuid, rc = out->cpso_op.co_rc; if (rc != 0) { - D_ERROR(DF_CONT": failed to set prop for container: %d\n", - DP_CONT(pool_uuid, cont_uuid), rc); + DL_ERROR(rc, DF_UUID ": failed to set prop for container %s", DP_UUID(pool_uuid), + cont_id); } crt_req_decref(rpc); @@ -5740,61 +5760,72 @@ ds_cont_svc_set_prop(uuid_t pool_uuid, uuid_t cont_uuid, } void -ds_cont_set_prop_handler(crt_rpc_t *rpc) +ds_cont_set_prop_srv_handler(crt_rpc_t *rpc) { - int rc; - struct cont_svc *svc; - struct cont_prop_set_in *in = crt_req_get(rpc); - struct cont_prop_set_out *out = crt_reply_get(rpc); - struct rdb_tx tx; - uuid_t pool_uuid; - uuid_t cont_uuid; - daos_prop_t *prop; - struct cont *cont; - - /* Client RPCs go through the regular flow with pool/cont handles */ - if (daos_rpc_from_client(rpc)) { - ds_cont_op_handler(rpc, 7); - return; - } + int rc; + crt_opcode_t opc = opc_get(rpc->cr_opc); + struct cont_svc *svc; + struct cont_prop_set_out *out = crt_reply_get(rpc); + struct rdb_tx tx; + uuid_t pool_uuid; + uuid_t cont_uuid; + const char *cont_label = NULL; + char cont_id[DAOS_PROP_MAX_LABEL_BUF_LEN] = {0}; + daos_prop_t *prop; + struct cont *cont; /* * Server RPCs don't have pool or container handles. Just need the pool - * and container UUIDs. + * and container IDs. */ - cont_prop_set_in_get_data(rpc, CONT_PROP_SET, DAOS_CONT_VERSION, &prop, &pool_uuid); - uuid_copy(cont_uuid, in->cpsi_op.ci_uuid); + cont_prop_set_in_get_data(rpc, opc, DAOS_CONT_VERSION, &prop, &pool_uuid, &cont_uuid, + &cont_label); + if (opc == CONT_PROP_SET_BYLABEL) + strncpy(cont_id, cont_label, sizeof(cont_id) - 1); + else /* CONT_PROP_SET */ + uuid_unparse(cont_uuid, cont_id); - D_DEBUG(DB_MD, DF_CONT": processing cont set prop rpc %p\n", - DP_CONT(pool_uuid, cont_uuid), rpc); + D_DEBUG(DB_MD, DF_UUID "/%s: processing cont set prop (%s) rpc %p\n", DP_UUID(pool_uuid), + cont_id, cont_cli_opc_name(opc), rpc); - rc = cont_svc_lookup_leader(pool_uuid, 0 /* id */, - &svc, &out->cpso_op.co_hint); + rc = cont_svc_lookup_leader(pool_uuid, 0, &svc, &out->cpso_op.co_hint); if (rc != 0) { - D_ERROR(DF_CONT": Failed to look up cont svc: %d\n", - DP_CONT(pool_uuid, cont_uuid), rc); + DL_ERROR(rc, DF_UUID "/%s: failed to look up cont svc", DP_UUID(pool_uuid), + cont_id); D_GOTO(out, rc); } rc = rdb_tx_begin(svc->cs_rsvc->s_db, svc->cs_rsvc->s_term, &tx); - if (rc != 0) + if (rc != 0) { + DL_ERROR(rc, DF_UUID "/%s: failed to start RDB transaction", DP_UUID(pool_uuid), + cont_id); D_GOTO(out_svc, rc); + } ABT_rwlock_wrlock(svc->cs_lock); - rc = cont_lookup(&tx, svc, cont_uuid, &cont); - if (rc != 0) + /* cont_id may be a UUID or label */ + if (opc == CONT_PROP_SET) + rc = cont_lookup(&tx, svc, cont_uuid, &cont); + else /* CONT_PROP_SET_BYLABEL */ + rc = cont_lookup_bylabel(&tx, svc, cont_label, &cont); + if (rc != 0) { + DL_ERROR(rc, DF_UUID ": failed to look up container '%s'", DP_UUID(pool_uuid), + cont_id); D_GOTO(out_lock, rc); + } - rc = set_prop(&tx, svc->cs_pool, cont, ds_sec_get_admin_cont_capabilities(), cont_uuid, - prop); - if (rc != 0) + rc = set_prop(&tx, svc->cs_pool, cont, ds_sec_get_admin_cont_capabilities(), prop); + if (rc != 0) { + DL_ERROR(rc, DF_CONT ": failed to set properties", + DP_CONT(svc->cs_pool_uuid, cont->c_uuid)); D_GOTO(out_cont, rc); + } rc = rdb_tx_commit(&tx); if (rc != 0) - D_ERROR(DF_CONT": Unable to commit RDB transaction\n", - DP_CONT(pool_uuid, cont_uuid)); + DL_ERROR(rc, DF_CONT ": unable to commit RDB transaction", + DP_CONT(svc->cs_pool_uuid, cont->c_uuid)); out_cont: cont_put(cont); @@ -5805,8 +5836,8 @@ ds_cont_set_prop_handler(crt_rpc_t *rpc) ds_rsvc_set_hint(svc->cs_rsvc, &out->cpso_op.co_hint); cont_svc_put_leader(svc); out: - D_DEBUG(DB_MD, DF_CONT ": replying rpc: %p rc=%d\n", - DP_CONT(pool_uuid, in->cpsi_op.ci_uuid), rpc, rc); + D_DEBUG(DB_MD, DF_UUID "/%s: replying rpc: %p rc=%d\n", DP_UUID(pool_uuid), cont_id, rpc, + rc); out->cpso_op.co_rc = rc; crt_reply_send(rpc); diff --git a/src/container/srv_internal.h b/src/container/srv_internal.h index ac94db455ec..81527044b7f 100644 --- a/src/container/srv_internal.h +++ b/src/container/srv_internal.h @@ -187,7 +187,8 @@ void ds_cont_op_handler_v8(crt_rpc_t *rpc); void ds_cont_op_handler_v7(crt_rpc_t *rpc); void ds_cont_op_handler_v6(crt_rpc_t *rpc); -void ds_cont_set_prop_handler(crt_rpc_t *rpc); +void + ds_cont_set_prop_srv_handler(crt_rpc_t *rpc); int ds_cont_bcast_create(crt_context_t ctx, struct cont_svc *svc, crt_opcode_t opcode, crt_rpc_t **rpc); int ds_cont_oid_fetch_add(uuid_t poh_uuid, uuid_t co_uuid, uint64_t num_oids, uint64_t *oid); diff --git a/src/control/cmd/daos/acl.go b/src/control/cmd/daos/acl.go index 7cbd3f98d0e..34d5b8491f4 100644 --- a/src/control/cmd/daos/acl.go +++ b/src/control/cmd/daos/acl.go @@ -351,22 +351,30 @@ func (cmd *containerGetACLCmd) Execute(args []string) error { return errors.Wrapf(err, "failed to query ACL for container %s", cmd.contUUID) } - output := os.Stdout if cmd.File != "" { flags := os.O_CREATE | os.O_WRONLY if !cmd.Force { flags |= os.O_EXCL } - output, err = os.OpenFile(cmd.File, flags, 0644) + output, err := os.OpenFile(cmd.File, flags, 0644) if err != nil { return errors.Wrap(err, "failed to open ACL output file") } defer output.Close() + + if _, err := fmt.Fprint(output, control.FormatACL(acl, cmd.Verbose)); err != nil { + return errors.Wrapf(err, "failed to write ACL output file") + } } - return cmd.outputACL(output, acl, cmd.Verbose) + var buf strings.Builder + if err := cmd.outputACL(&buf, acl, cmd.Verbose); err != nil { + return err + } + cmd.Info(buf.String()) + return nil } type containerSetOwnerCmd struct { diff --git a/src/control/cmd/dmg/command_test.go b/src/control/cmd/dmg/command_test.go index 5121ba4dcae..f56714dfaee 100644 --- a/src/control/cmd/dmg/command_test.go +++ b/src/control/cmd/dmg/command_test.go @@ -147,7 +147,7 @@ func (bci *bridgeConnInvoker) InvokeUnaryRPC(ctx context.Context, uReq control.U case *control.ListPoolsReq: resp = control.MockMSResponse("", nil, &mgmtpb.ListPoolsResp{}) case *control.ContSetOwnerReq: - resp = control.MockMSResponse("", nil, &mgmtpb.ContSetOwnerResp{}) + resp = control.MockMSResponse("", nil, &mgmtpb.DaosResp{}) case *control.PoolQueryReq: resp = control.MockMSResponse("", nil, &mgmtpb.PoolQueryResp{}) case *control.PoolQueryTargetReq: diff --git a/src/control/cmd/dmg/cont.go b/src/control/cmd/dmg/cont.go index 8906773bd1a..afc117d3f38 100644 --- a/src/control/cmd/dmg/cont.go +++ b/src/control/cmd/dmg/cont.go @@ -1,5 +1,5 @@ // -// (C) Copyright 2020-2022 Intel Corporation. +// (C) Copyright 2020-2024 Intel Corporation. // // SPDX-License-Identifier: BSD-2-Clause-Patent // @@ -10,11 +10,24 @@ import ( "github.com/jessevdk/go-flags" "github.com/pkg/errors" - "github.com/daos-stack/daos/src/control/common/cmdutil" "github.com/daos-stack/daos/src/control/lib/control" "github.com/daos-stack/daos/src/control/lib/ui" ) +// ContID is a container label or UUID. +type ContID struct { + ui.LabelOrUUIDFlag +} + +// contCmd is the base struct for all container commands. +type contCmd struct { + poolCmd + + Args struct { + Cont ContID `positional-arg-name:"" required:"1"` + } `positional-args:"yes"` +} + // ContCmd is the struct representing the top-level container subcommand. type ContCmd struct { SetOwner ContSetOwnerCmd `command:"set-owner" description:"Change the owner for a DAOS container"` @@ -22,18 +35,14 @@ type ContCmd struct { // ContSetOwnerCmd is the struct representing the command to change the owner of a DAOS container. type ContSetOwnerCmd struct { - baseCmd - ctlInvokerCmd - cmdutil.JSONOutputCmd + contCmd GroupName ui.ACLPrincipalFlag `short:"g" long:"group" description:"New owner-group for the container, format name@domain"` UserName ui.ACLPrincipalFlag `short:"u" long:"user" description:"New owner-user for the container, format name@domain"` - ContUUID string `short:"c" long:"cont" required:"1" description:"UUID of the DAOS container"` - PoolUUID string `short:"p" long:"pool" required:"1" description:"UUID of the DAOS pool for the container"` } // Execute runs the container set-owner command -func (c *ContSetOwnerCmd) Execute(args []string) error { - if c.GroupName == "" && c.UserName == "" { +func (cmd *ContSetOwnerCmd) Execute(args []string) error { + if cmd.GroupName == "" && cmd.UserName == "" { return &flags.Error{ Type: flags.ErrRequired, Message: "at least one of `--user' or `--group' must be supplied", @@ -41,23 +50,23 @@ func (c *ContSetOwnerCmd) Execute(args []string) error { } msg := "SUCCEEDED" req := &control.ContSetOwnerReq{ - ContUUID: c.ContUUID, - PoolUUID: c.PoolUUID, - User: c.UserName.String(), - Group: c.GroupName.String(), + ContID: cmd.Args.Cont.String(), + PoolID: cmd.poolCmd.Args.Pool.String(), + User: cmd.UserName.String(), + Group: cmd.GroupName.String(), } - ctx := c.MustLogCtx() - err := control.ContSetOwner(ctx, c.ctlInvoker, req) + ctx := cmd.MustLogCtx() + err := control.ContSetOwner(ctx, cmd.ctlInvoker, req) if err != nil { msg = errors.WithMessage(err, "FAILED").Error() } - if c.JSONOutputEnabled() { - return c.OutputJSON(nil, err) + if cmd.JSONOutputEnabled() { + return cmd.OutputJSON(nil, err) } - c.Infof("Container-set-owner command %s\n", msg) + cmd.Infof("Container-set-owner command %s\n", msg) return err } diff --git a/src/control/cmd/dmg/cont_test.go b/src/control/cmd/dmg/cont_test.go index 9240dccdd1e..421f77477c5 100644 --- a/src/control/cmd/dmg/cont_test.go +++ b/src/control/cmd/dmg/cont_test.go @@ -1,5 +1,5 @@ // -// (C) Copyright 2020-2021 Intel Corporation. +// (C) Copyright 2020-2024 Intel Corporation. // // SPDX-License-Identifier: BSD-2-Clause-Patent // @@ -29,58 +29,58 @@ func TestContSetOwnerCommand(t *testing.T) { "Set owner with no arguments", "cont set-owner", "", - errMissingFlag, + errors.New("required arguments"), }, { "Set owner user", - fmt.Sprintf("cont set-owner --pool=%s --cont=%s --user=%s", testPoolUUID, testContUUID, testUser), + fmt.Sprintf("cont set-owner --user=%s %s %s", testUser, testPoolUUID, testContUUID), strings.Join([]string{ printRequest(t, &control.ContSetOwnerReq{ - PoolUUID: testPoolUUID.String(), - ContUUID: testContUUID.String(), - User: testUser, - Group: "", + PoolID: testPoolUUID.String(), + ContID: testContUUID.String(), + User: testUser, + Group: "", }), }, " "), nil, }, { "Set owner group", - fmt.Sprintf("cont set-owner --pool=%s --cont=%s --group=%s", testPoolUUID, testContUUID, testGroup), + fmt.Sprintf("cont set-owner --group=%s %s %s", testGroup, testPoolUUID, testContUUID), strings.Join([]string{ printRequest(t, &control.ContSetOwnerReq{ - PoolUUID: testPoolUUID.String(), - ContUUID: testContUUID.String(), - User: "", - Group: testGroup, + PoolID: testPoolUUID.String(), + ContID: testContUUID.String(), + User: "", + Group: testGroup, }), }, " "), nil, }, { "Set owner user and group", - fmt.Sprintf("cont set-owner --pool=%s --cont=%s --user=%s --group=%s", - testPoolUUID, testContUUID, testUser, testGroup), + fmt.Sprintf("cont set-owner --user=%s --group=%s %s %s", testUser, testGroup, + testPoolUUID, testContUUID), strings.Join([]string{ printRequest(t, &control.ContSetOwnerReq{ - PoolUUID: testPoolUUID.String(), - ContUUID: testContUUID.String(), - User: testUser, - Group: testGroup, + PoolID: testPoolUUID.String(), + ContID: testContUUID.String(), + User: testUser, + Group: testGroup, }), }, " "), nil, }, { "Bad owner principal", - fmt.Sprintf("cont set-owner --pool=%s --cont=%s --user=%s --group=%s", - testPoolUUID, testContUUID, "bad@@", testGroup), + fmt.Sprintf("cont set-owner --user=%s --group=%s %s %s", "bad@@", testGroup, + testPoolUUID, testContUUID), "", errors.New("invalid ACL principal"), }, { "Bad group principal", - fmt.Sprintf("cont set-owner --pool=%s --cont=%s --user=%s --group=%s", - testPoolUUID, testContUUID, testUser, "bad@@"), + fmt.Sprintf("cont set-owner --user=%s --group=%s %s %s", testUser, "bad@@", + testPoolUUID, testContUUID), "", errors.New("invalid ACL principal"), }, }) diff --git a/src/control/cmd/dmg/json_test.go b/src/control/cmd/dmg/json_test.go index 893a312509a..8187f5b1078 100644 --- a/src/control/cmd/dmg/json_test.go +++ b/src/control/cmd/dmg/json_test.go @@ -101,8 +101,7 @@ func TestDmg_JsonOutput(t *testing.T) { case "pool query-targets": testArgs = append(testArgs, test.MockUUID(), "--rank", "0", "--target-idx", "1,3,5,7") case "container set-owner": - testArgs = append(testArgs, "--user", "foo", "--pool", test.MockUUID(), - "--cont", test.MockUUID()) + testArgs = append(testArgs, "--user", "foo", test.MockUUID(), test.MockUUID()) case "telemetry metrics list", "telemetry metrics query": return // These commands query via http directly case "system cleanup": diff --git a/src/control/common/proto/mgmt/addons.go b/src/control/common/proto/mgmt/addons.go index 34299c3096c..7f049b29b39 100644 --- a/src/control/common/proto/mgmt/addons.go +++ b/src/control/common/proto/mgmt/addons.go @@ -197,12 +197,12 @@ func (r *ContSetOwnerReq) SetSvcRanks(rl []uint32) { // SetUUID sets the request's ID to a UUID. func (r *ContSetOwnerReq) SetUUID(id uuid.UUID) { - r.PoolUUID = id.String() + r.PoolId = id.String() } // GetId fetches the pool ID. func (r *ContSetOwnerReq) GetId() string { - return r.PoolUUID + return r.PoolId } // SetSvcRanks sets the request's Pool Service Ranks. diff --git a/src/control/common/proto/mgmt/cont.pb.go b/src/control/common/proto/mgmt/cont.pb.go index 86fcdb3fe41..9d261f47004 100644 --- a/src/control/common/proto/mgmt/cont.pb.go +++ b/src/control/common/proto/mgmt/cont.pb.go @@ -1,12 +1,12 @@ // -// (C) Copyright 2020-2021 Intel Corporation. +// (C) Copyright 2020-2024 Intel Corporation. // // SPDX-License-Identifier: BSD-2-Clause-Patent // // Code generated by protoc-gen-go. DO NOT EDIT. // versions: -// protoc-gen-go v1.28.1 +// protoc-gen-go v1.31.0 // protoc v3.5.0 // source: mgmt/cont.proto @@ -26,17 +26,17 @@ const ( _ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20) ) -// ContSetOwnerReq supplies new pool parameters. +// ContSetOwnerReq changes the ownership of a container. type ContSetOwnerReq struct { state protoimpl.MessageState sizeCache protoimpl.SizeCache unknownFields protoimpl.UnknownFields Sys string `protobuf:"bytes,1,opt,name=sys,proto3" json:"sys,omitempty"` // DAOS system identifier - ContUUID string `protobuf:"bytes,2,opt,name=contUUID,proto3" json:"contUUID,omitempty"` // UUID of the container - PoolUUID string `protobuf:"bytes,3,opt,name=poolUUID,proto3" json:"poolUUID,omitempty"` // UUID of the pool that the container is in - Owneruser string `protobuf:"bytes,4,opt,name=owneruser,proto3" json:"owneruser,omitempty"` // formatted user e.g. "bob@" - Ownergroup string `protobuf:"bytes,5,opt,name=ownergroup,proto3" json:"ownergroup,omitempty"` // formatted group e.g. "builders@" + ContId string `protobuf:"bytes,2,opt,name=cont_id,json=contId,proto3" json:"cont_id,omitempty"` // UUID or label of the container + PoolId string `protobuf:"bytes,3,opt,name=pool_id,json=poolId,proto3" json:"pool_id,omitempty"` // UUID or label of the pool that the container is in + OwnerUser string `protobuf:"bytes,4,opt,name=owner_user,json=ownerUser,proto3" json:"owner_user,omitempty"` // formatted user e.g. "bob@" + OwnerGroup string `protobuf:"bytes,5,opt,name=owner_group,json=ownerGroup,proto3" json:"owner_group,omitempty"` // formatted group e.g. "builders@" SvcRanks []uint32 `protobuf:"varint,6,rep,packed,name=svc_ranks,json=svcRanks,proto3" json:"svc_ranks,omitempty"` // List of pool service ranks } @@ -79,30 +79,30 @@ func (x *ContSetOwnerReq) GetSys() string { return "" } -func (x *ContSetOwnerReq) GetContUUID() string { +func (x *ContSetOwnerReq) GetContId() string { if x != nil { - return x.ContUUID + return x.ContId } return "" } -func (x *ContSetOwnerReq) GetPoolUUID() string { +func (x *ContSetOwnerReq) GetPoolId() string { if x != nil { - return x.PoolUUID + return x.PoolId } return "" } -func (x *ContSetOwnerReq) GetOwneruser() string { +func (x *ContSetOwnerReq) GetOwnerUser() string { if x != nil { - return x.Owneruser + return x.OwnerUser } return "" } -func (x *ContSetOwnerReq) GetOwnergroup() string { +func (x *ContSetOwnerReq) GetOwnerGroup() string { if x != nil { - return x.Ownergroup + return x.OwnerGroup } return "" } @@ -114,73 +114,22 @@ func (x *ContSetOwnerReq) GetSvcRanks() []uint32 { return nil } -// ContSetOwnerResp returns created pool uuid and ranks. -type ContSetOwnerResp struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache - unknownFields protoimpl.UnknownFields - - Status int32 `protobuf:"varint,1,opt,name=status,proto3" json:"status,omitempty"` // DAOS error code -} - -func (x *ContSetOwnerResp) Reset() { - *x = ContSetOwnerResp{} - if protoimpl.UnsafeEnabled { - mi := &file_mgmt_cont_proto_msgTypes[1] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } -} - -func (x *ContSetOwnerResp) String() string { - return protoimpl.X.MessageStringOf(x) -} - -func (*ContSetOwnerResp) ProtoMessage() {} - -func (x *ContSetOwnerResp) ProtoReflect() protoreflect.Message { - mi := &file_mgmt_cont_proto_msgTypes[1] - if protoimpl.UnsafeEnabled && x != nil { - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - if ms.LoadMessageInfo() == nil { - ms.StoreMessageInfo(mi) - } - return ms - } - return mi.MessageOf(x) -} - -// Deprecated: Use ContSetOwnerResp.ProtoReflect.Descriptor instead. -func (*ContSetOwnerResp) Descriptor() ([]byte, []int) { - return file_mgmt_cont_proto_rawDescGZIP(), []int{1} -} - -func (x *ContSetOwnerResp) GetStatus() int32 { - if x != nil { - return x.Status - } - return 0 -} - var File_mgmt_cont_proto protoreflect.FileDescriptor var file_mgmt_cont_proto_rawDesc = []byte{ 0x0a, 0x0f, 0x6d, 0x67, 0x6d, 0x74, 0x2f, 0x63, 0x6f, 0x6e, 0x74, 0x2e, 0x70, 0x72, 0x6f, 0x74, - 0x6f, 0x12, 0x04, 0x6d, 0x67, 0x6d, 0x74, 0x22, 0xb6, 0x01, 0x0a, 0x0f, 0x43, 0x6f, 0x6e, 0x74, + 0x6f, 0x12, 0x04, 0x6d, 0x67, 0x6d, 0x74, 0x22, 0xb2, 0x01, 0x0a, 0x0f, 0x43, 0x6f, 0x6e, 0x74, 0x53, 0x65, 0x74, 0x4f, 0x77, 0x6e, 0x65, 0x72, 0x52, 0x65, 0x71, 0x12, 0x10, 0x0a, 0x03, 0x73, - 0x79, 0x73, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x73, 0x79, 0x73, 0x12, 0x1a, 0x0a, - 0x08, 0x63, 0x6f, 0x6e, 0x74, 0x55, 0x55, 0x49, 0x44, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, - 0x08, 0x63, 0x6f, 0x6e, 0x74, 0x55, 0x55, 0x49, 0x44, 0x12, 0x1a, 0x0a, 0x08, 0x70, 0x6f, 0x6f, - 0x6c, 0x55, 0x55, 0x49, 0x44, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x08, 0x70, 0x6f, 0x6f, - 0x6c, 0x55, 0x55, 0x49, 0x44, 0x12, 0x1c, 0x0a, 0x09, 0x6f, 0x77, 0x6e, 0x65, 0x72, 0x75, 0x73, - 0x65, 0x72, 0x18, 0x04, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x6f, 0x77, 0x6e, 0x65, 0x72, 0x75, - 0x73, 0x65, 0x72, 0x12, 0x1e, 0x0a, 0x0a, 0x6f, 0x77, 0x6e, 0x65, 0x72, 0x67, 0x72, 0x6f, 0x75, - 0x70, 0x18, 0x05, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0a, 0x6f, 0x77, 0x6e, 0x65, 0x72, 0x67, 0x72, - 0x6f, 0x75, 0x70, 0x12, 0x1b, 0x0a, 0x09, 0x73, 0x76, 0x63, 0x5f, 0x72, 0x61, 0x6e, 0x6b, 0x73, - 0x18, 0x06, 0x20, 0x03, 0x28, 0x0d, 0x52, 0x08, 0x73, 0x76, 0x63, 0x52, 0x61, 0x6e, 0x6b, 0x73, - 0x22, 0x2a, 0x0a, 0x10, 0x43, 0x6f, 0x6e, 0x74, 0x53, 0x65, 0x74, 0x4f, 0x77, 0x6e, 0x65, 0x72, - 0x52, 0x65, 0x73, 0x70, 0x12, 0x16, 0x0a, 0x06, 0x73, 0x74, 0x61, 0x74, 0x75, 0x73, 0x18, 0x01, - 0x20, 0x01, 0x28, 0x05, 0x52, 0x06, 0x73, 0x74, 0x61, 0x74, 0x75, 0x73, 0x42, 0x3a, 0x5a, 0x38, + 0x79, 0x73, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x73, 0x79, 0x73, 0x12, 0x17, 0x0a, + 0x07, 0x63, 0x6f, 0x6e, 0x74, 0x5f, 0x69, 0x64, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, + 0x63, 0x6f, 0x6e, 0x74, 0x49, 0x64, 0x12, 0x17, 0x0a, 0x07, 0x70, 0x6f, 0x6f, 0x6c, 0x5f, 0x69, + 0x64, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x70, 0x6f, 0x6f, 0x6c, 0x49, 0x64, 0x12, + 0x1d, 0x0a, 0x0a, 0x6f, 0x77, 0x6e, 0x65, 0x72, 0x5f, 0x75, 0x73, 0x65, 0x72, 0x18, 0x04, 0x20, + 0x01, 0x28, 0x09, 0x52, 0x09, 0x6f, 0x77, 0x6e, 0x65, 0x72, 0x55, 0x73, 0x65, 0x72, 0x12, 0x1f, + 0x0a, 0x0b, 0x6f, 0x77, 0x6e, 0x65, 0x72, 0x5f, 0x67, 0x72, 0x6f, 0x75, 0x70, 0x18, 0x05, 0x20, + 0x01, 0x28, 0x09, 0x52, 0x0a, 0x6f, 0x77, 0x6e, 0x65, 0x72, 0x47, 0x72, 0x6f, 0x75, 0x70, 0x12, + 0x1b, 0x0a, 0x09, 0x73, 0x76, 0x63, 0x5f, 0x72, 0x61, 0x6e, 0x6b, 0x73, 0x18, 0x06, 0x20, 0x03, + 0x28, 0x0d, 0x52, 0x08, 0x73, 0x76, 0x63, 0x52, 0x61, 0x6e, 0x6b, 0x73, 0x42, 0x3a, 0x5a, 0x38, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x64, 0x61, 0x6f, 0x73, 0x2d, 0x73, 0x74, 0x61, 0x63, 0x6b, 0x2f, 0x64, 0x61, 0x6f, 0x73, 0x2f, 0x73, 0x72, 0x63, 0x2f, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2f, 0x63, 0x6f, 0x6d, 0x6d, 0x6f, 0x6e, 0x2f, 0x70, 0x72, @@ -199,10 +148,9 @@ func file_mgmt_cont_proto_rawDescGZIP() []byte { return file_mgmt_cont_proto_rawDescData } -var file_mgmt_cont_proto_msgTypes = make([]protoimpl.MessageInfo, 2) +var file_mgmt_cont_proto_msgTypes = make([]protoimpl.MessageInfo, 1) var file_mgmt_cont_proto_goTypes = []interface{}{ - (*ContSetOwnerReq)(nil), // 0: mgmt.ContSetOwnerReq - (*ContSetOwnerResp)(nil), // 1: mgmt.ContSetOwnerResp + (*ContSetOwnerReq)(nil), // 0: mgmt.ContSetOwnerReq } var file_mgmt_cont_proto_depIdxs = []int32{ 0, // [0:0] is the sub-list for method output_type @@ -230,18 +178,6 @@ func file_mgmt_cont_proto_init() { return nil } } - file_mgmt_cont_proto_msgTypes[1].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*ContSetOwnerResp); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } } type x struct{} out := protoimpl.TypeBuilder{ @@ -249,7 +185,7 @@ func file_mgmt_cont_proto_init() { GoPackagePath: reflect.TypeOf(x{}).PkgPath(), RawDescriptor: file_mgmt_cont_proto_rawDesc, NumEnums: 0, - NumMessages: 2, + NumMessages: 1, NumExtensions: 0, NumServices: 0, }, diff --git a/src/control/common/proto/mgmt/mgmt.pb.go b/src/control/common/proto/mgmt/mgmt.pb.go index 9b0c52ed9a8..0b45d1cae25 100644 --- a/src/control/common/proto/mgmt/mgmt.pb.go +++ b/src/control/common/proto/mgmt/mgmt.pb.go @@ -6,7 +6,7 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: -// protoc-gen-go v1.30.0 +// protoc-gen-go v1.31.0 // protoc v3.5.0 // source: mgmt/mgmt.proto @@ -41,7 +41,7 @@ var file_mgmt_mgmt_proto_rawDesc = []byte{ 0x11, 0x6d, 0x67, 0x6d, 0x74, 0x2f, 0x73, 0x79, 0x73, 0x74, 0x65, 0x6d, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x1a, 0x0d, 0x63, 0x68, 0x6b, 0x2f, 0x63, 0x68, 0x6b, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x1a, 0x10, 0x63, 0x68, 0x6b, 0x2f, 0x66, 0x61, 0x75, 0x6c, 0x74, 0x73, 0x2e, 0x70, 0x72, - 0x6f, 0x74, 0x6f, 0x32, 0x98, 0x15, 0x0a, 0x07, 0x4d, 0x67, 0x6d, 0x74, 0x53, 0x76, 0x63, 0x12, + 0x6f, 0x74, 0x6f, 0x32, 0x90, 0x15, 0x0a, 0x07, 0x4d, 0x67, 0x6d, 0x74, 0x53, 0x76, 0x63, 0x12, 0x27, 0x0a, 0x04, 0x4a, 0x6f, 0x69, 0x6e, 0x12, 0x0d, 0x2e, 0x6d, 0x67, 0x6d, 0x74, 0x2e, 0x4a, 0x6f, 0x69, 0x6e, 0x52, 0x65, 0x71, 0x1a, 0x0e, 0x2e, 0x6d, 0x67, 0x6d, 0x74, 0x2e, 0x4a, 0x6f, 0x69, 0x6e, 0x52, 0x65, 0x73, 0x70, 0x22, 0x00, 0x12, 0x43, 0x0a, 0x0c, 0x43, 0x6c, 0x75, 0x73, @@ -120,102 +120,101 @@ var file_mgmt_mgmt_proto_rawDesc = []byte{ 0x6e, 0x74, 0x61, 0x69, 0x6e, 0x65, 0x72, 0x73, 0x12, 0x11, 0x2e, 0x6d, 0x67, 0x6d, 0x74, 0x2e, 0x4c, 0x69, 0x73, 0x74, 0x43, 0x6f, 0x6e, 0x74, 0x52, 0x65, 0x71, 0x1a, 0x12, 0x2e, 0x6d, 0x67, 0x6d, 0x74, 0x2e, 0x4c, 0x69, 0x73, 0x74, 0x43, 0x6f, 0x6e, 0x74, 0x52, 0x65, 0x73, 0x70, 0x22, - 0x00, 0x12, 0x3f, 0x0a, 0x0c, 0x43, 0x6f, 0x6e, 0x74, 0x53, 0x65, 0x74, 0x4f, 0x77, 0x6e, 0x65, + 0x00, 0x12, 0x37, 0x0a, 0x0c, 0x43, 0x6f, 0x6e, 0x74, 0x53, 0x65, 0x74, 0x4f, 0x77, 0x6e, 0x65, 0x72, 0x12, 0x15, 0x2e, 0x6d, 0x67, 0x6d, 0x74, 0x2e, 0x43, 0x6f, 0x6e, 0x74, 0x53, 0x65, 0x74, - 0x4f, 0x77, 0x6e, 0x65, 0x72, 0x52, 0x65, 0x71, 0x1a, 0x16, 0x2e, 0x6d, 0x67, 0x6d, 0x74, 0x2e, - 0x43, 0x6f, 0x6e, 0x74, 0x53, 0x65, 0x74, 0x4f, 0x77, 0x6e, 0x65, 0x72, 0x52, 0x65, 0x73, 0x70, - 0x22, 0x00, 0x12, 0x3c, 0x0a, 0x0b, 0x53, 0x79, 0x73, 0x74, 0x65, 0x6d, 0x51, 0x75, 0x65, 0x72, - 0x79, 0x12, 0x14, 0x2e, 0x6d, 0x67, 0x6d, 0x74, 0x2e, 0x53, 0x79, 0x73, 0x74, 0x65, 0x6d, 0x51, - 0x75, 0x65, 0x72, 0x79, 0x52, 0x65, 0x71, 0x1a, 0x15, 0x2e, 0x6d, 0x67, 0x6d, 0x74, 0x2e, 0x53, - 0x79, 0x73, 0x74, 0x65, 0x6d, 0x51, 0x75, 0x65, 0x72, 0x79, 0x52, 0x65, 0x73, 0x70, 0x22, 0x00, - 0x12, 0x39, 0x0a, 0x0a, 0x53, 0x79, 0x73, 0x74, 0x65, 0x6d, 0x53, 0x74, 0x6f, 0x70, 0x12, 0x13, - 0x2e, 0x6d, 0x67, 0x6d, 0x74, 0x2e, 0x53, 0x79, 0x73, 0x74, 0x65, 0x6d, 0x53, 0x74, 0x6f, 0x70, - 0x52, 0x65, 0x71, 0x1a, 0x14, 0x2e, 0x6d, 0x67, 0x6d, 0x74, 0x2e, 0x53, 0x79, 0x73, 0x74, 0x65, - 0x6d, 0x53, 0x74, 0x6f, 0x70, 0x52, 0x65, 0x73, 0x70, 0x22, 0x00, 0x12, 0x3c, 0x0a, 0x0b, 0x53, - 0x79, 0x73, 0x74, 0x65, 0x6d, 0x53, 0x74, 0x61, 0x72, 0x74, 0x12, 0x14, 0x2e, 0x6d, 0x67, 0x6d, - 0x74, 0x2e, 0x53, 0x79, 0x73, 0x74, 0x65, 0x6d, 0x53, 0x74, 0x61, 0x72, 0x74, 0x52, 0x65, 0x71, - 0x1a, 0x15, 0x2e, 0x6d, 0x67, 0x6d, 0x74, 0x2e, 0x53, 0x79, 0x73, 0x74, 0x65, 0x6d, 0x53, 0x74, - 0x61, 0x72, 0x74, 0x52, 0x65, 0x73, 0x70, 0x22, 0x00, 0x12, 0x42, 0x0a, 0x0d, 0x53, 0x79, 0x73, - 0x74, 0x65, 0x6d, 0x45, 0x78, 0x63, 0x6c, 0x75, 0x64, 0x65, 0x12, 0x16, 0x2e, 0x6d, 0x67, 0x6d, + 0x4f, 0x77, 0x6e, 0x65, 0x72, 0x52, 0x65, 0x71, 0x1a, 0x0e, 0x2e, 0x6d, 0x67, 0x6d, 0x74, 0x2e, + 0x44, 0x61, 0x6f, 0x73, 0x52, 0x65, 0x73, 0x70, 0x22, 0x00, 0x12, 0x3c, 0x0a, 0x0b, 0x53, 0x79, + 0x73, 0x74, 0x65, 0x6d, 0x51, 0x75, 0x65, 0x72, 0x79, 0x12, 0x14, 0x2e, 0x6d, 0x67, 0x6d, 0x74, + 0x2e, 0x53, 0x79, 0x73, 0x74, 0x65, 0x6d, 0x51, 0x75, 0x65, 0x72, 0x79, 0x52, 0x65, 0x71, 0x1a, + 0x15, 0x2e, 0x6d, 0x67, 0x6d, 0x74, 0x2e, 0x53, 0x79, 0x73, 0x74, 0x65, 0x6d, 0x51, 0x75, 0x65, + 0x72, 0x79, 0x52, 0x65, 0x73, 0x70, 0x22, 0x00, 0x12, 0x39, 0x0a, 0x0a, 0x53, 0x79, 0x73, 0x74, + 0x65, 0x6d, 0x53, 0x74, 0x6f, 0x70, 0x12, 0x13, 0x2e, 0x6d, 0x67, 0x6d, 0x74, 0x2e, 0x53, 0x79, + 0x73, 0x74, 0x65, 0x6d, 0x53, 0x74, 0x6f, 0x70, 0x52, 0x65, 0x71, 0x1a, 0x14, 0x2e, 0x6d, 0x67, + 0x6d, 0x74, 0x2e, 0x53, 0x79, 0x73, 0x74, 0x65, 0x6d, 0x53, 0x74, 0x6f, 0x70, 0x52, 0x65, 0x73, + 0x70, 0x22, 0x00, 0x12, 0x3c, 0x0a, 0x0b, 0x53, 0x79, 0x73, 0x74, 0x65, 0x6d, 0x53, 0x74, 0x61, + 0x72, 0x74, 0x12, 0x14, 0x2e, 0x6d, 0x67, 0x6d, 0x74, 0x2e, 0x53, 0x79, 0x73, 0x74, 0x65, 0x6d, + 0x53, 0x74, 0x61, 0x72, 0x74, 0x52, 0x65, 0x71, 0x1a, 0x15, 0x2e, 0x6d, 0x67, 0x6d, 0x74, 0x2e, + 0x53, 0x79, 0x73, 0x74, 0x65, 0x6d, 0x53, 0x74, 0x61, 0x72, 0x74, 0x52, 0x65, 0x73, 0x70, 0x22, + 0x00, 0x12, 0x42, 0x0a, 0x0d, 0x53, 0x79, 0x73, 0x74, 0x65, 0x6d, 0x45, 0x78, 0x63, 0x6c, 0x75, + 0x64, 0x65, 0x12, 0x16, 0x2e, 0x6d, 0x67, 0x6d, 0x74, 0x2e, 0x53, 0x79, 0x73, 0x74, 0x65, 0x6d, + 0x45, 0x78, 0x63, 0x6c, 0x75, 0x64, 0x65, 0x52, 0x65, 0x71, 0x1a, 0x17, 0x2e, 0x6d, 0x67, 0x6d, 0x74, 0x2e, 0x53, 0x79, 0x73, 0x74, 0x65, 0x6d, 0x45, 0x78, 0x63, 0x6c, 0x75, 0x64, 0x65, 0x52, - 0x65, 0x71, 0x1a, 0x17, 0x2e, 0x6d, 0x67, 0x6d, 0x74, 0x2e, 0x53, 0x79, 0x73, 0x74, 0x65, 0x6d, - 0x45, 0x78, 0x63, 0x6c, 0x75, 0x64, 0x65, 0x52, 0x65, 0x73, 0x70, 0x22, 0x00, 0x12, 0x3c, 0x0a, - 0x0b, 0x53, 0x79, 0x73, 0x74, 0x65, 0x6d, 0x45, 0x72, 0x61, 0x73, 0x65, 0x12, 0x14, 0x2e, 0x6d, - 0x67, 0x6d, 0x74, 0x2e, 0x53, 0x79, 0x73, 0x74, 0x65, 0x6d, 0x45, 0x72, 0x61, 0x73, 0x65, 0x52, - 0x65, 0x71, 0x1a, 0x15, 0x2e, 0x6d, 0x67, 0x6d, 0x74, 0x2e, 0x53, 0x79, 0x73, 0x74, 0x65, 0x6d, - 0x45, 0x72, 0x61, 0x73, 0x65, 0x52, 0x65, 0x73, 0x70, 0x22, 0x00, 0x12, 0x42, 0x0a, 0x0d, 0x53, - 0x79, 0x73, 0x74, 0x65, 0x6d, 0x43, 0x6c, 0x65, 0x61, 0x6e, 0x75, 0x70, 0x12, 0x16, 0x2e, 0x6d, + 0x65, 0x73, 0x70, 0x22, 0x00, 0x12, 0x3c, 0x0a, 0x0b, 0x53, 0x79, 0x73, 0x74, 0x65, 0x6d, 0x45, + 0x72, 0x61, 0x73, 0x65, 0x12, 0x14, 0x2e, 0x6d, 0x67, 0x6d, 0x74, 0x2e, 0x53, 0x79, 0x73, 0x74, + 0x65, 0x6d, 0x45, 0x72, 0x61, 0x73, 0x65, 0x52, 0x65, 0x71, 0x1a, 0x15, 0x2e, 0x6d, 0x67, 0x6d, + 0x74, 0x2e, 0x53, 0x79, 0x73, 0x74, 0x65, 0x6d, 0x45, 0x72, 0x61, 0x73, 0x65, 0x52, 0x65, 0x73, + 0x70, 0x22, 0x00, 0x12, 0x42, 0x0a, 0x0d, 0x53, 0x79, 0x73, 0x74, 0x65, 0x6d, 0x43, 0x6c, 0x65, + 0x61, 0x6e, 0x75, 0x70, 0x12, 0x16, 0x2e, 0x6d, 0x67, 0x6d, 0x74, 0x2e, 0x53, 0x79, 0x73, 0x74, + 0x65, 0x6d, 0x43, 0x6c, 0x65, 0x61, 0x6e, 0x75, 0x70, 0x52, 0x65, 0x71, 0x1a, 0x17, 0x2e, 0x6d, 0x67, 0x6d, 0x74, 0x2e, 0x53, 0x79, 0x73, 0x74, 0x65, 0x6d, 0x43, 0x6c, 0x65, 0x61, 0x6e, 0x75, - 0x70, 0x52, 0x65, 0x71, 0x1a, 0x17, 0x2e, 0x6d, 0x67, 0x6d, 0x74, 0x2e, 0x53, 0x79, 0x73, 0x74, - 0x65, 0x6d, 0x43, 0x6c, 0x65, 0x61, 0x6e, 0x75, 0x70, 0x52, 0x65, 0x73, 0x70, 0x22, 0x00, 0x12, - 0x3b, 0x0a, 0x11, 0x53, 0x79, 0x73, 0x74, 0x65, 0x6d, 0x43, 0x68, 0x65, 0x63, 0x6b, 0x45, 0x6e, - 0x61, 0x62, 0x6c, 0x65, 0x12, 0x14, 0x2e, 0x6d, 0x67, 0x6d, 0x74, 0x2e, 0x43, 0x68, 0x65, 0x63, - 0x6b, 0x45, 0x6e, 0x61, 0x62, 0x6c, 0x65, 0x52, 0x65, 0x71, 0x1a, 0x0e, 0x2e, 0x6d, 0x67, 0x6d, - 0x74, 0x2e, 0x44, 0x61, 0x6f, 0x73, 0x52, 0x65, 0x73, 0x70, 0x22, 0x00, 0x12, 0x3d, 0x0a, 0x12, - 0x53, 0x79, 0x73, 0x74, 0x65, 0x6d, 0x43, 0x68, 0x65, 0x63, 0x6b, 0x44, 0x69, 0x73, 0x61, 0x62, - 0x6c, 0x65, 0x12, 0x15, 0x2e, 0x6d, 0x67, 0x6d, 0x74, 0x2e, 0x43, 0x68, 0x65, 0x63, 0x6b, 0x44, - 0x69, 0x73, 0x61, 0x62, 0x6c, 0x65, 0x52, 0x65, 0x71, 0x1a, 0x0e, 0x2e, 0x6d, 0x67, 0x6d, 0x74, - 0x2e, 0x44, 0x61, 0x6f, 0x73, 0x52, 0x65, 0x73, 0x70, 0x22, 0x00, 0x12, 0x3f, 0x0a, 0x10, 0x53, - 0x79, 0x73, 0x74, 0x65, 0x6d, 0x43, 0x68, 0x65, 0x63, 0x6b, 0x53, 0x74, 0x61, 0x72, 0x74, 0x12, - 0x13, 0x2e, 0x6d, 0x67, 0x6d, 0x74, 0x2e, 0x43, 0x68, 0x65, 0x63, 0x6b, 0x53, 0x74, 0x61, 0x72, - 0x74, 0x52, 0x65, 0x71, 0x1a, 0x14, 0x2e, 0x6d, 0x67, 0x6d, 0x74, 0x2e, 0x43, 0x68, 0x65, 0x63, - 0x6b, 0x53, 0x74, 0x61, 0x72, 0x74, 0x52, 0x65, 0x73, 0x70, 0x22, 0x00, 0x12, 0x3c, 0x0a, 0x0f, - 0x53, 0x79, 0x73, 0x74, 0x65, 0x6d, 0x43, 0x68, 0x65, 0x63, 0x6b, 0x53, 0x74, 0x6f, 0x70, 0x12, - 0x12, 0x2e, 0x6d, 0x67, 0x6d, 0x74, 0x2e, 0x43, 0x68, 0x65, 0x63, 0x6b, 0x53, 0x74, 0x6f, 0x70, - 0x52, 0x65, 0x71, 0x1a, 0x13, 0x2e, 0x6d, 0x67, 0x6d, 0x74, 0x2e, 0x43, 0x68, 0x65, 0x63, 0x6b, - 0x53, 0x74, 0x6f, 0x70, 0x52, 0x65, 0x73, 0x70, 0x22, 0x00, 0x12, 0x3f, 0x0a, 0x10, 0x53, 0x79, - 0x73, 0x74, 0x65, 0x6d, 0x43, 0x68, 0x65, 0x63, 0x6b, 0x51, 0x75, 0x65, 0x72, 0x79, 0x12, 0x13, - 0x2e, 0x6d, 0x67, 0x6d, 0x74, 0x2e, 0x43, 0x68, 0x65, 0x63, 0x6b, 0x51, 0x75, 0x65, 0x72, 0x79, - 0x52, 0x65, 0x71, 0x1a, 0x14, 0x2e, 0x6d, 0x67, 0x6d, 0x74, 0x2e, 0x43, 0x68, 0x65, 0x63, 0x6b, - 0x51, 0x75, 0x65, 0x72, 0x79, 0x52, 0x65, 0x73, 0x70, 0x22, 0x00, 0x12, 0x41, 0x0a, 0x14, 0x53, - 0x79, 0x73, 0x74, 0x65, 0x6d, 0x43, 0x68, 0x65, 0x63, 0x6b, 0x53, 0x65, 0x74, 0x50, 0x6f, 0x6c, - 0x69, 0x63, 0x79, 0x12, 0x17, 0x2e, 0x6d, 0x67, 0x6d, 0x74, 0x2e, 0x43, 0x68, 0x65, 0x63, 0x6b, - 0x53, 0x65, 0x74, 0x50, 0x6f, 0x6c, 0x69, 0x63, 0x79, 0x52, 0x65, 0x71, 0x1a, 0x0e, 0x2e, 0x6d, - 0x67, 0x6d, 0x74, 0x2e, 0x44, 0x61, 0x6f, 0x73, 0x52, 0x65, 0x73, 0x70, 0x22, 0x00, 0x12, 0x4b, - 0x0a, 0x14, 0x53, 0x79, 0x73, 0x74, 0x65, 0x6d, 0x43, 0x68, 0x65, 0x63, 0x6b, 0x47, 0x65, 0x74, - 0x50, 0x6f, 0x6c, 0x69, 0x63, 0x79, 0x12, 0x17, 0x2e, 0x6d, 0x67, 0x6d, 0x74, 0x2e, 0x43, 0x68, - 0x65, 0x63, 0x6b, 0x47, 0x65, 0x74, 0x50, 0x6f, 0x6c, 0x69, 0x63, 0x79, 0x52, 0x65, 0x71, 0x1a, - 0x18, 0x2e, 0x6d, 0x67, 0x6d, 0x74, 0x2e, 0x43, 0x68, 0x65, 0x63, 0x6b, 0x47, 0x65, 0x74, 0x50, - 0x6f, 0x6c, 0x69, 0x63, 0x79, 0x52, 0x65, 0x73, 0x70, 0x22, 0x00, 0x12, 0x3c, 0x0a, 0x11, 0x53, - 0x79, 0x73, 0x74, 0x65, 0x6d, 0x43, 0x68, 0x65, 0x63, 0x6b, 0x52, 0x65, 0x70, 0x61, 0x69, 0x72, - 0x12, 0x11, 0x2e, 0x6d, 0x67, 0x6d, 0x74, 0x2e, 0x43, 0x68, 0x65, 0x63, 0x6b, 0x41, 0x63, 0x74, - 0x52, 0x65, 0x71, 0x1a, 0x12, 0x2e, 0x6d, 0x67, 0x6d, 0x74, 0x2e, 0x43, 0x68, 0x65, 0x63, 0x6b, - 0x41, 0x63, 0x74, 0x52, 0x65, 0x73, 0x70, 0x22, 0x00, 0x12, 0x3c, 0x0a, 0x0b, 0x50, 0x6f, 0x6f, - 0x6c, 0x55, 0x70, 0x67, 0x72, 0x61, 0x64, 0x65, 0x12, 0x14, 0x2e, 0x6d, 0x67, 0x6d, 0x74, 0x2e, - 0x50, 0x6f, 0x6f, 0x6c, 0x55, 0x70, 0x67, 0x72, 0x61, 0x64, 0x65, 0x52, 0x65, 0x71, 0x1a, 0x15, - 0x2e, 0x6d, 0x67, 0x6d, 0x74, 0x2e, 0x50, 0x6f, 0x6f, 0x6c, 0x55, 0x70, 0x67, 0x72, 0x61, 0x64, - 0x65, 0x52, 0x65, 0x73, 0x70, 0x22, 0x00, 0x12, 0x39, 0x0a, 0x0d, 0x53, 0x79, 0x73, 0x74, 0x65, - 0x6d, 0x53, 0x65, 0x74, 0x41, 0x74, 0x74, 0x72, 0x12, 0x16, 0x2e, 0x6d, 0x67, 0x6d, 0x74, 0x2e, - 0x53, 0x79, 0x73, 0x74, 0x65, 0x6d, 0x53, 0x65, 0x74, 0x41, 0x74, 0x74, 0x72, 0x52, 0x65, 0x71, - 0x1a, 0x0e, 0x2e, 0x6d, 0x67, 0x6d, 0x74, 0x2e, 0x44, 0x61, 0x6f, 0x73, 0x52, 0x65, 0x73, 0x70, - 0x22, 0x00, 0x12, 0x42, 0x0a, 0x0d, 0x53, 0x79, 0x73, 0x74, 0x65, 0x6d, 0x47, 0x65, 0x74, 0x41, - 0x74, 0x74, 0x72, 0x12, 0x16, 0x2e, 0x6d, 0x67, 0x6d, 0x74, 0x2e, 0x53, 0x79, 0x73, 0x74, 0x65, - 0x6d, 0x47, 0x65, 0x74, 0x41, 0x74, 0x74, 0x72, 0x52, 0x65, 0x71, 0x1a, 0x17, 0x2e, 0x6d, 0x67, + 0x70, 0x52, 0x65, 0x73, 0x70, 0x22, 0x00, 0x12, 0x3b, 0x0a, 0x11, 0x53, 0x79, 0x73, 0x74, 0x65, + 0x6d, 0x43, 0x68, 0x65, 0x63, 0x6b, 0x45, 0x6e, 0x61, 0x62, 0x6c, 0x65, 0x12, 0x14, 0x2e, 0x6d, + 0x67, 0x6d, 0x74, 0x2e, 0x43, 0x68, 0x65, 0x63, 0x6b, 0x45, 0x6e, 0x61, 0x62, 0x6c, 0x65, 0x52, + 0x65, 0x71, 0x1a, 0x0e, 0x2e, 0x6d, 0x67, 0x6d, 0x74, 0x2e, 0x44, 0x61, 0x6f, 0x73, 0x52, 0x65, + 0x73, 0x70, 0x22, 0x00, 0x12, 0x3d, 0x0a, 0x12, 0x53, 0x79, 0x73, 0x74, 0x65, 0x6d, 0x43, 0x68, + 0x65, 0x63, 0x6b, 0x44, 0x69, 0x73, 0x61, 0x62, 0x6c, 0x65, 0x12, 0x15, 0x2e, 0x6d, 0x67, 0x6d, + 0x74, 0x2e, 0x43, 0x68, 0x65, 0x63, 0x6b, 0x44, 0x69, 0x73, 0x61, 0x62, 0x6c, 0x65, 0x52, 0x65, + 0x71, 0x1a, 0x0e, 0x2e, 0x6d, 0x67, 0x6d, 0x74, 0x2e, 0x44, 0x61, 0x6f, 0x73, 0x52, 0x65, 0x73, + 0x70, 0x22, 0x00, 0x12, 0x3f, 0x0a, 0x10, 0x53, 0x79, 0x73, 0x74, 0x65, 0x6d, 0x43, 0x68, 0x65, + 0x63, 0x6b, 0x53, 0x74, 0x61, 0x72, 0x74, 0x12, 0x13, 0x2e, 0x6d, 0x67, 0x6d, 0x74, 0x2e, 0x43, + 0x68, 0x65, 0x63, 0x6b, 0x53, 0x74, 0x61, 0x72, 0x74, 0x52, 0x65, 0x71, 0x1a, 0x14, 0x2e, 0x6d, + 0x67, 0x6d, 0x74, 0x2e, 0x43, 0x68, 0x65, 0x63, 0x6b, 0x53, 0x74, 0x61, 0x72, 0x74, 0x52, 0x65, + 0x73, 0x70, 0x22, 0x00, 0x12, 0x3c, 0x0a, 0x0f, 0x53, 0x79, 0x73, 0x74, 0x65, 0x6d, 0x43, 0x68, + 0x65, 0x63, 0x6b, 0x53, 0x74, 0x6f, 0x70, 0x12, 0x12, 0x2e, 0x6d, 0x67, 0x6d, 0x74, 0x2e, 0x43, + 0x68, 0x65, 0x63, 0x6b, 0x53, 0x74, 0x6f, 0x70, 0x52, 0x65, 0x71, 0x1a, 0x13, 0x2e, 0x6d, 0x67, + 0x6d, 0x74, 0x2e, 0x43, 0x68, 0x65, 0x63, 0x6b, 0x53, 0x74, 0x6f, 0x70, 0x52, 0x65, 0x73, 0x70, + 0x22, 0x00, 0x12, 0x3f, 0x0a, 0x10, 0x53, 0x79, 0x73, 0x74, 0x65, 0x6d, 0x43, 0x68, 0x65, 0x63, + 0x6b, 0x51, 0x75, 0x65, 0x72, 0x79, 0x12, 0x13, 0x2e, 0x6d, 0x67, 0x6d, 0x74, 0x2e, 0x43, 0x68, + 0x65, 0x63, 0x6b, 0x51, 0x75, 0x65, 0x72, 0x79, 0x52, 0x65, 0x71, 0x1a, 0x14, 0x2e, 0x6d, 0x67, + 0x6d, 0x74, 0x2e, 0x43, 0x68, 0x65, 0x63, 0x6b, 0x51, 0x75, 0x65, 0x72, 0x79, 0x52, 0x65, 0x73, + 0x70, 0x22, 0x00, 0x12, 0x41, 0x0a, 0x14, 0x53, 0x79, 0x73, 0x74, 0x65, 0x6d, 0x43, 0x68, 0x65, + 0x63, 0x6b, 0x53, 0x65, 0x74, 0x50, 0x6f, 0x6c, 0x69, 0x63, 0x79, 0x12, 0x17, 0x2e, 0x6d, 0x67, + 0x6d, 0x74, 0x2e, 0x43, 0x68, 0x65, 0x63, 0x6b, 0x53, 0x65, 0x74, 0x50, 0x6f, 0x6c, 0x69, 0x63, + 0x79, 0x52, 0x65, 0x71, 0x1a, 0x0e, 0x2e, 0x6d, 0x67, 0x6d, 0x74, 0x2e, 0x44, 0x61, 0x6f, 0x73, + 0x52, 0x65, 0x73, 0x70, 0x22, 0x00, 0x12, 0x4b, 0x0a, 0x14, 0x53, 0x79, 0x73, 0x74, 0x65, 0x6d, + 0x43, 0x68, 0x65, 0x63, 0x6b, 0x47, 0x65, 0x74, 0x50, 0x6f, 0x6c, 0x69, 0x63, 0x79, 0x12, 0x17, + 0x2e, 0x6d, 0x67, 0x6d, 0x74, 0x2e, 0x43, 0x68, 0x65, 0x63, 0x6b, 0x47, 0x65, 0x74, 0x50, 0x6f, + 0x6c, 0x69, 0x63, 0x79, 0x52, 0x65, 0x71, 0x1a, 0x18, 0x2e, 0x6d, 0x67, 0x6d, 0x74, 0x2e, 0x43, + 0x68, 0x65, 0x63, 0x6b, 0x47, 0x65, 0x74, 0x50, 0x6f, 0x6c, 0x69, 0x63, 0x79, 0x52, 0x65, 0x73, + 0x70, 0x22, 0x00, 0x12, 0x3c, 0x0a, 0x11, 0x53, 0x79, 0x73, 0x74, 0x65, 0x6d, 0x43, 0x68, 0x65, + 0x63, 0x6b, 0x52, 0x65, 0x70, 0x61, 0x69, 0x72, 0x12, 0x11, 0x2e, 0x6d, 0x67, 0x6d, 0x74, 0x2e, + 0x43, 0x68, 0x65, 0x63, 0x6b, 0x41, 0x63, 0x74, 0x52, 0x65, 0x71, 0x1a, 0x12, 0x2e, 0x6d, 0x67, + 0x6d, 0x74, 0x2e, 0x43, 0x68, 0x65, 0x63, 0x6b, 0x41, 0x63, 0x74, 0x52, 0x65, 0x73, 0x70, 0x22, + 0x00, 0x12, 0x3c, 0x0a, 0x0b, 0x50, 0x6f, 0x6f, 0x6c, 0x55, 0x70, 0x67, 0x72, 0x61, 0x64, 0x65, + 0x12, 0x14, 0x2e, 0x6d, 0x67, 0x6d, 0x74, 0x2e, 0x50, 0x6f, 0x6f, 0x6c, 0x55, 0x70, 0x67, 0x72, + 0x61, 0x64, 0x65, 0x52, 0x65, 0x71, 0x1a, 0x15, 0x2e, 0x6d, 0x67, 0x6d, 0x74, 0x2e, 0x50, 0x6f, + 0x6f, 0x6c, 0x55, 0x70, 0x67, 0x72, 0x61, 0x64, 0x65, 0x52, 0x65, 0x73, 0x70, 0x22, 0x00, 0x12, + 0x39, 0x0a, 0x0d, 0x53, 0x79, 0x73, 0x74, 0x65, 0x6d, 0x53, 0x65, 0x74, 0x41, 0x74, 0x74, 0x72, + 0x12, 0x16, 0x2e, 0x6d, 0x67, 0x6d, 0x74, 0x2e, 0x53, 0x79, 0x73, 0x74, 0x65, 0x6d, 0x53, 0x65, + 0x74, 0x41, 0x74, 0x74, 0x72, 0x52, 0x65, 0x71, 0x1a, 0x0e, 0x2e, 0x6d, 0x67, 0x6d, 0x74, 0x2e, + 0x44, 0x61, 0x6f, 0x73, 0x52, 0x65, 0x73, 0x70, 0x22, 0x00, 0x12, 0x42, 0x0a, 0x0d, 0x53, 0x79, + 0x73, 0x74, 0x65, 0x6d, 0x47, 0x65, 0x74, 0x41, 0x74, 0x74, 0x72, 0x12, 0x16, 0x2e, 0x6d, 0x67, 0x6d, 0x74, 0x2e, 0x53, 0x79, 0x73, 0x74, 0x65, 0x6d, 0x47, 0x65, 0x74, 0x41, 0x74, 0x74, 0x72, - 0x52, 0x65, 0x73, 0x70, 0x22, 0x00, 0x12, 0x39, 0x0a, 0x0d, 0x53, 0x79, 0x73, 0x74, 0x65, 0x6d, - 0x53, 0x65, 0x74, 0x50, 0x72, 0x6f, 0x70, 0x12, 0x16, 0x2e, 0x6d, 0x67, 0x6d, 0x74, 0x2e, 0x53, - 0x79, 0x73, 0x74, 0x65, 0x6d, 0x53, 0x65, 0x74, 0x50, 0x72, 0x6f, 0x70, 0x52, 0x65, 0x71, 0x1a, - 0x0e, 0x2e, 0x6d, 0x67, 0x6d, 0x74, 0x2e, 0x44, 0x61, 0x6f, 0x73, 0x52, 0x65, 0x73, 0x70, 0x22, - 0x00, 0x12, 0x42, 0x0a, 0x0d, 0x53, 0x79, 0x73, 0x74, 0x65, 0x6d, 0x47, 0x65, 0x74, 0x50, 0x72, - 0x6f, 0x70, 0x12, 0x16, 0x2e, 0x6d, 0x67, 0x6d, 0x74, 0x2e, 0x53, 0x79, 0x73, 0x74, 0x65, 0x6d, - 0x47, 0x65, 0x74, 0x50, 0x72, 0x6f, 0x70, 0x52, 0x65, 0x71, 0x1a, 0x17, 0x2e, 0x6d, 0x67, 0x6d, + 0x52, 0x65, 0x71, 0x1a, 0x17, 0x2e, 0x6d, 0x67, 0x6d, 0x74, 0x2e, 0x53, 0x79, 0x73, 0x74, 0x65, + 0x6d, 0x47, 0x65, 0x74, 0x41, 0x74, 0x74, 0x72, 0x52, 0x65, 0x73, 0x70, 0x22, 0x00, 0x12, 0x39, + 0x0a, 0x0d, 0x53, 0x79, 0x73, 0x74, 0x65, 0x6d, 0x53, 0x65, 0x74, 0x50, 0x72, 0x6f, 0x70, 0x12, + 0x16, 0x2e, 0x6d, 0x67, 0x6d, 0x74, 0x2e, 0x53, 0x79, 0x73, 0x74, 0x65, 0x6d, 0x53, 0x65, 0x74, + 0x50, 0x72, 0x6f, 0x70, 0x52, 0x65, 0x71, 0x1a, 0x0e, 0x2e, 0x6d, 0x67, 0x6d, 0x74, 0x2e, 0x44, + 0x61, 0x6f, 0x73, 0x52, 0x65, 0x73, 0x70, 0x22, 0x00, 0x12, 0x42, 0x0a, 0x0d, 0x53, 0x79, 0x73, + 0x74, 0x65, 0x6d, 0x47, 0x65, 0x74, 0x50, 0x72, 0x6f, 0x70, 0x12, 0x16, 0x2e, 0x6d, 0x67, 0x6d, 0x74, 0x2e, 0x53, 0x79, 0x73, 0x74, 0x65, 0x6d, 0x47, 0x65, 0x74, 0x50, 0x72, 0x6f, 0x70, 0x52, - 0x65, 0x73, 0x70, 0x22, 0x00, 0x12, 0x37, 0x0a, 0x11, 0x46, 0x61, 0x75, 0x6c, 0x74, 0x49, 0x6e, - 0x6a, 0x65, 0x63, 0x74, 0x52, 0x65, 0x70, 0x6f, 0x72, 0x74, 0x12, 0x10, 0x2e, 0x63, 0x68, 0x6b, - 0x2e, 0x43, 0x68, 0x65, 0x63, 0x6b, 0x52, 0x65, 0x70, 0x6f, 0x72, 0x74, 0x1a, 0x0e, 0x2e, 0x6d, - 0x67, 0x6d, 0x74, 0x2e, 0x44, 0x61, 0x6f, 0x73, 0x52, 0x65, 0x73, 0x70, 0x22, 0x00, 0x12, 0x34, - 0x0a, 0x14, 0x46, 0x61, 0x75, 0x6c, 0x74, 0x49, 0x6e, 0x6a, 0x65, 0x63, 0x74, 0x50, 0x6f, 0x6f, - 0x6c, 0x46, 0x61, 0x75, 0x6c, 0x74, 0x12, 0x0a, 0x2e, 0x63, 0x68, 0x6b, 0x2e, 0x46, 0x61, 0x75, - 0x6c, 0x74, 0x1a, 0x0e, 0x2e, 0x6d, 0x67, 0x6d, 0x74, 0x2e, 0x44, 0x61, 0x6f, 0x73, 0x52, 0x65, - 0x73, 0x70, 0x22, 0x00, 0x12, 0x38, 0x0a, 0x18, 0x46, 0x61, 0x75, 0x6c, 0x74, 0x49, 0x6e, 0x6a, - 0x65, 0x63, 0x74, 0x4d, 0x67, 0x6d, 0x74, 0x50, 0x6f, 0x6f, 0x6c, 0x46, 0x61, 0x75, 0x6c, 0x74, - 0x12, 0x0a, 0x2e, 0x63, 0x68, 0x6b, 0x2e, 0x46, 0x61, 0x75, 0x6c, 0x74, 0x1a, 0x0e, 0x2e, 0x6d, - 0x67, 0x6d, 0x74, 0x2e, 0x44, 0x61, 0x6f, 0x73, 0x52, 0x65, 0x73, 0x70, 0x22, 0x00, 0x42, 0x3a, - 0x5a, 0x38, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x64, 0x61, 0x6f, - 0x73, 0x2d, 0x73, 0x74, 0x61, 0x63, 0x6b, 0x2f, 0x64, 0x61, 0x6f, 0x73, 0x2f, 0x73, 0x72, 0x63, - 0x2f, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2f, 0x63, 0x6f, 0x6d, 0x6d, 0x6f, 0x6e, 0x2f, - 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2f, 0x6d, 0x67, 0x6d, 0x74, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, - 0x6f, 0x33, + 0x65, 0x71, 0x1a, 0x17, 0x2e, 0x6d, 0x67, 0x6d, 0x74, 0x2e, 0x53, 0x79, 0x73, 0x74, 0x65, 0x6d, + 0x47, 0x65, 0x74, 0x50, 0x72, 0x6f, 0x70, 0x52, 0x65, 0x73, 0x70, 0x22, 0x00, 0x12, 0x37, 0x0a, + 0x11, 0x46, 0x61, 0x75, 0x6c, 0x74, 0x49, 0x6e, 0x6a, 0x65, 0x63, 0x74, 0x52, 0x65, 0x70, 0x6f, + 0x72, 0x74, 0x12, 0x10, 0x2e, 0x63, 0x68, 0x6b, 0x2e, 0x43, 0x68, 0x65, 0x63, 0x6b, 0x52, 0x65, + 0x70, 0x6f, 0x72, 0x74, 0x1a, 0x0e, 0x2e, 0x6d, 0x67, 0x6d, 0x74, 0x2e, 0x44, 0x61, 0x6f, 0x73, + 0x52, 0x65, 0x73, 0x70, 0x22, 0x00, 0x12, 0x34, 0x0a, 0x14, 0x46, 0x61, 0x75, 0x6c, 0x74, 0x49, + 0x6e, 0x6a, 0x65, 0x63, 0x74, 0x50, 0x6f, 0x6f, 0x6c, 0x46, 0x61, 0x75, 0x6c, 0x74, 0x12, 0x0a, + 0x2e, 0x63, 0x68, 0x6b, 0x2e, 0x46, 0x61, 0x75, 0x6c, 0x74, 0x1a, 0x0e, 0x2e, 0x6d, 0x67, 0x6d, + 0x74, 0x2e, 0x44, 0x61, 0x6f, 0x73, 0x52, 0x65, 0x73, 0x70, 0x22, 0x00, 0x12, 0x38, 0x0a, 0x18, + 0x46, 0x61, 0x75, 0x6c, 0x74, 0x49, 0x6e, 0x6a, 0x65, 0x63, 0x74, 0x4d, 0x67, 0x6d, 0x74, 0x50, + 0x6f, 0x6f, 0x6c, 0x46, 0x61, 0x75, 0x6c, 0x74, 0x12, 0x0a, 0x2e, 0x63, 0x68, 0x6b, 0x2e, 0x46, + 0x61, 0x75, 0x6c, 0x74, 0x1a, 0x0e, 0x2e, 0x6d, 0x67, 0x6d, 0x74, 0x2e, 0x44, 0x61, 0x6f, 0x73, + 0x52, 0x65, 0x73, 0x70, 0x22, 0x00, 0x42, 0x3a, 0x5a, 0x38, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, + 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x64, 0x61, 0x6f, 0x73, 0x2d, 0x73, 0x74, 0x61, 0x63, 0x6b, 0x2f, + 0x64, 0x61, 0x6f, 0x73, 0x2f, 0x73, 0x72, 0x63, 0x2f, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, + 0x2f, 0x63, 0x6f, 0x6d, 0x6d, 0x6f, 0x6e, 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2f, 0x6d, 0x67, + 0x6d, 0x74, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, } var file_mgmt_mgmt_proto_goTypes = []interface{}{ @@ -279,22 +278,21 @@ var file_mgmt_mgmt_proto_goTypes = []interface{}{ (*GetAttachInfoResp)(nil), // 57: mgmt.GetAttachInfoResp (*ListPoolsResp)(nil), // 58: mgmt.ListPoolsResp (*ListContResp)(nil), // 59: mgmt.ListContResp - (*ContSetOwnerResp)(nil), // 60: mgmt.ContSetOwnerResp + (*DaosResp)(nil), // 60: mgmt.DaosResp (*SystemQueryResp)(nil), // 61: mgmt.SystemQueryResp (*SystemStopResp)(nil), // 62: mgmt.SystemStopResp (*SystemStartResp)(nil), // 63: mgmt.SystemStartResp (*SystemExcludeResp)(nil), // 64: mgmt.SystemExcludeResp (*SystemEraseResp)(nil), // 65: mgmt.SystemEraseResp (*SystemCleanupResp)(nil), // 66: mgmt.SystemCleanupResp - (*DaosResp)(nil), // 67: mgmt.DaosResp - (*CheckStartResp)(nil), // 68: mgmt.CheckStartResp - (*CheckStopResp)(nil), // 69: mgmt.CheckStopResp - (*CheckQueryResp)(nil), // 70: mgmt.CheckQueryResp - (*CheckGetPolicyResp)(nil), // 71: mgmt.CheckGetPolicyResp - (*CheckActResp)(nil), // 72: mgmt.CheckActResp - (*PoolUpgradeResp)(nil), // 73: mgmt.PoolUpgradeResp - (*SystemGetAttrResp)(nil), // 74: mgmt.SystemGetAttrResp - (*SystemGetPropResp)(nil), // 75: mgmt.SystemGetPropResp + (*CheckStartResp)(nil), // 67: mgmt.CheckStartResp + (*CheckStopResp)(nil), // 68: mgmt.CheckStopResp + (*CheckQueryResp)(nil), // 69: mgmt.CheckQueryResp + (*CheckGetPolicyResp)(nil), // 70: mgmt.CheckGetPolicyResp + (*CheckActResp)(nil), // 71: mgmt.CheckActResp + (*PoolUpgradeResp)(nil), // 72: mgmt.PoolUpgradeResp + (*SystemGetAttrResp)(nil), // 73: mgmt.SystemGetAttrResp + (*SystemGetPropResp)(nil), // 74: mgmt.SystemGetPropResp } var file_mgmt_mgmt_proto_depIdxs = []int32{ 0, // 0: mgmt.MgmtSvc.Join:input_type -> mgmt.JoinReq @@ -362,29 +360,29 @@ var file_mgmt_mgmt_proto_depIdxs = []int32{ 57, // 62: mgmt.MgmtSvc.GetAttachInfo:output_type -> mgmt.GetAttachInfoResp 58, // 63: mgmt.MgmtSvc.ListPools:output_type -> mgmt.ListPoolsResp 59, // 64: mgmt.MgmtSvc.ListContainers:output_type -> mgmt.ListContResp - 60, // 65: mgmt.MgmtSvc.ContSetOwner:output_type -> mgmt.ContSetOwnerResp + 60, // 65: mgmt.MgmtSvc.ContSetOwner:output_type -> mgmt.DaosResp 61, // 66: mgmt.MgmtSvc.SystemQuery:output_type -> mgmt.SystemQueryResp 62, // 67: mgmt.MgmtSvc.SystemStop:output_type -> mgmt.SystemStopResp 63, // 68: mgmt.MgmtSvc.SystemStart:output_type -> mgmt.SystemStartResp 64, // 69: mgmt.MgmtSvc.SystemExclude:output_type -> mgmt.SystemExcludeResp 65, // 70: mgmt.MgmtSvc.SystemErase:output_type -> mgmt.SystemEraseResp 66, // 71: mgmt.MgmtSvc.SystemCleanup:output_type -> mgmt.SystemCleanupResp - 67, // 72: mgmt.MgmtSvc.SystemCheckEnable:output_type -> mgmt.DaosResp - 67, // 73: mgmt.MgmtSvc.SystemCheckDisable:output_type -> mgmt.DaosResp - 68, // 74: mgmt.MgmtSvc.SystemCheckStart:output_type -> mgmt.CheckStartResp - 69, // 75: mgmt.MgmtSvc.SystemCheckStop:output_type -> mgmt.CheckStopResp - 70, // 76: mgmt.MgmtSvc.SystemCheckQuery:output_type -> mgmt.CheckQueryResp - 67, // 77: mgmt.MgmtSvc.SystemCheckSetPolicy:output_type -> mgmt.DaosResp - 71, // 78: mgmt.MgmtSvc.SystemCheckGetPolicy:output_type -> mgmt.CheckGetPolicyResp - 72, // 79: mgmt.MgmtSvc.SystemCheckRepair:output_type -> mgmt.CheckActResp - 73, // 80: mgmt.MgmtSvc.PoolUpgrade:output_type -> mgmt.PoolUpgradeResp - 67, // 81: mgmt.MgmtSvc.SystemSetAttr:output_type -> mgmt.DaosResp - 74, // 82: mgmt.MgmtSvc.SystemGetAttr:output_type -> mgmt.SystemGetAttrResp - 67, // 83: mgmt.MgmtSvc.SystemSetProp:output_type -> mgmt.DaosResp - 75, // 84: mgmt.MgmtSvc.SystemGetProp:output_type -> mgmt.SystemGetPropResp - 67, // 85: mgmt.MgmtSvc.FaultInjectReport:output_type -> mgmt.DaosResp - 67, // 86: mgmt.MgmtSvc.FaultInjectPoolFault:output_type -> mgmt.DaosResp - 67, // 87: mgmt.MgmtSvc.FaultInjectMgmtPoolFault:output_type -> mgmt.DaosResp + 60, // 72: mgmt.MgmtSvc.SystemCheckEnable:output_type -> mgmt.DaosResp + 60, // 73: mgmt.MgmtSvc.SystemCheckDisable:output_type -> mgmt.DaosResp + 67, // 74: mgmt.MgmtSvc.SystemCheckStart:output_type -> mgmt.CheckStartResp + 68, // 75: mgmt.MgmtSvc.SystemCheckStop:output_type -> mgmt.CheckStopResp + 69, // 76: mgmt.MgmtSvc.SystemCheckQuery:output_type -> mgmt.CheckQueryResp + 60, // 77: mgmt.MgmtSvc.SystemCheckSetPolicy:output_type -> mgmt.DaosResp + 70, // 78: mgmt.MgmtSvc.SystemCheckGetPolicy:output_type -> mgmt.CheckGetPolicyResp + 71, // 79: mgmt.MgmtSvc.SystemCheckRepair:output_type -> mgmt.CheckActResp + 72, // 80: mgmt.MgmtSvc.PoolUpgrade:output_type -> mgmt.PoolUpgradeResp + 60, // 81: mgmt.MgmtSvc.SystemSetAttr:output_type -> mgmt.DaosResp + 73, // 82: mgmt.MgmtSvc.SystemGetAttr:output_type -> mgmt.SystemGetAttrResp + 60, // 83: mgmt.MgmtSvc.SystemSetProp:output_type -> mgmt.DaosResp + 74, // 84: mgmt.MgmtSvc.SystemGetProp:output_type -> mgmt.SystemGetPropResp + 60, // 85: mgmt.MgmtSvc.FaultInjectReport:output_type -> mgmt.DaosResp + 60, // 86: mgmt.MgmtSvc.FaultInjectPoolFault:output_type -> mgmt.DaosResp + 60, // 87: mgmt.MgmtSvc.FaultInjectMgmtPoolFault:output_type -> mgmt.DaosResp 44, // [44:88] is the sub-list for method output_type 0, // [0:44] is the sub-list for method input_type 0, // [0:0] is the sub-list for extension type_name diff --git a/src/control/common/proto/mgmt/mgmt_grpc.pb.go b/src/control/common/proto/mgmt/mgmt_grpc.pb.go index 1c976e82251..bf69683f90c 100644 --- a/src/control/common/proto/mgmt/mgmt_grpc.pb.go +++ b/src/control/common/proto/mgmt/mgmt_grpc.pb.go @@ -121,7 +121,7 @@ type MgmtSvcClient interface { // List all containers in a pool ListContainers(ctx context.Context, in *ListContReq, opts ...grpc.CallOption) (*ListContResp, error) // Change the owner of a DAOS container - ContSetOwner(ctx context.Context, in *ContSetOwnerReq, opts ...grpc.CallOption) (*ContSetOwnerResp, error) + ContSetOwner(ctx context.Context, in *ContSetOwnerReq, opts ...grpc.CallOption) (*DaosResp, error) // Query DAOS system status SystemQuery(ctx context.Context, in *SystemQueryReq, opts ...grpc.CallOption) (*SystemQueryResp, error) // Stop DAOS system (shutdown data-plane instances) @@ -365,8 +365,8 @@ func (c *mgmtSvcClient) ListContainers(ctx context.Context, in *ListContReq, opt return out, nil } -func (c *mgmtSvcClient) ContSetOwner(ctx context.Context, in *ContSetOwnerReq, opts ...grpc.CallOption) (*ContSetOwnerResp, error) { - out := new(ContSetOwnerResp) +func (c *mgmtSvcClient) ContSetOwner(ctx context.Context, in *ContSetOwnerReq, opts ...grpc.CallOption) (*DaosResp, error) { + out := new(DaosResp) err := c.cc.Invoke(ctx, MgmtSvc_ContSetOwner_FullMethodName, in, out, opts...) if err != nil { return nil, err @@ -620,7 +620,7 @@ type MgmtSvcServer interface { // List all containers in a pool ListContainers(context.Context, *ListContReq) (*ListContResp, error) // Change the owner of a DAOS container - ContSetOwner(context.Context, *ContSetOwnerReq) (*ContSetOwnerResp, error) + ContSetOwner(context.Context, *ContSetOwnerReq) (*DaosResp, error) // Query DAOS system status SystemQuery(context.Context, *SystemQueryReq) (*SystemQueryResp, error) // Stop DAOS system (shutdown data-plane instances) @@ -735,7 +735,7 @@ func (UnimplementedMgmtSvcServer) ListPools(context.Context, *ListPoolsReq) (*Li func (UnimplementedMgmtSvcServer) ListContainers(context.Context, *ListContReq) (*ListContResp, error) { return nil, status.Errorf(codes.Unimplemented, "method ListContainers not implemented") } -func (UnimplementedMgmtSvcServer) ContSetOwner(context.Context, *ContSetOwnerReq) (*ContSetOwnerResp, error) { +func (UnimplementedMgmtSvcServer) ContSetOwner(context.Context, *ContSetOwnerReq) (*DaosResp, error) { return nil, status.Errorf(codes.Unimplemented, "method ContSetOwner not implemented") } func (UnimplementedMgmtSvcServer) SystemQuery(context.Context, *SystemQueryReq) (*SystemQueryResp, error) { diff --git a/src/control/lib/control/cont.go b/src/control/lib/control/cont.go index a0398c115ae..d109f0e9ae1 100644 --- a/src/control/lib/control/cont.go +++ b/src/control/lib/control/cont.go @@ -1,5 +1,5 @@ // -// (C) Copyright 2020-2022 Intel Corporation. +// (C) Copyright 2020-2024 Intel Corporation. // // SPDX-License-Identifier: BSD-2-Clause-Patent // @@ -20,10 +20,10 @@ import ( type ContSetOwnerReq struct { msRequest unaryRequest - ContUUID string // Container UUID - PoolUUID string // UUID of the pool for the container - User string // User to own the container, or empty if none - Group string // Group to own the container, or empty if none + ContID string // Container UUID or label + PoolID string // UUID or label of the pool for the container + User string // User to own the container, or empty if none + Group string // Group to own the container, or empty if none } // ContSetOwner changes the owner user and/or group of a DAOS container. @@ -32,12 +32,12 @@ func ContSetOwner(ctx context.Context, rpcClient UnaryInvoker, req *ContSetOwner return errors.New("nil request") } - if err := checkUUID(req.ContUUID); err != nil { - return err + if req.PoolID == "" { + return errors.New("no pool label or UUID specified") } - if err := checkUUID(req.PoolUUID); err != nil { - return err + if req.ContID == "" { + return errors.New("no container label or UUID specified") } if req.User == "" && req.Group == "" { @@ -47,10 +47,10 @@ func ContSetOwner(ctx context.Context, rpcClient UnaryInvoker, req *ContSetOwner req.setRPC(func(ctx context.Context, conn *grpc.ClientConn) (proto.Message, error) { return mgmtpb.NewMgmtSvcClient(conn).ContSetOwner(ctx, &mgmtpb.ContSetOwnerReq{ Sys: req.getSystem(rpcClient), - ContUUID: req.ContUUID, - PoolUUID: req.PoolUUID, - Owneruser: req.User, - Ownergroup: req.Group, + ContId: req.ContID, + PoolId: req.PoolID, + OwnerUser: req.User, + OwnerGroup: req.Group, }) }) diff --git a/src/control/lib/control/cont_test.go b/src/control/lib/control/cont_test.go index 6bd6cdd8a10..05b005c547a 100644 --- a/src/control/lib/control/cont_test.go +++ b/src/control/lib/control/cont_test.go @@ -1,5 +1,5 @@ // -// (C) Copyright 2020-2022 Intel Corporation. +// (C) Copyright 2020-2024 Intel Corporation. // // SPDX-License-Identifier: BSD-2-Clause-Patent // @@ -22,10 +22,10 @@ func TestControl_ContSetOwner(t *testing.T) { testContUUID := uuid.New().String() validReq := &ContSetOwnerReq{ - PoolUUID: testPoolUUID, - ContUUID: testContUUID, - User: "someuser@", - Group: "somegroup@", + PoolID: testPoolUUID, + ContID: testContUUID, + User: "someuser@", + Group: "somegroup@", } for name, tc := range map[string]struct { @@ -37,36 +37,22 @@ func TestControl_ContSetOwner(t *testing.T) { req: nil, expErr: errors.New("nil request"), }, - "bad container UUID": { + "no container ID": { req: &ContSetOwnerReq{ - PoolUUID: testPoolUUID, - ContUUID: "junk", + PoolID: testPoolUUID, }, - expErr: errors.New("invalid UUID"), + expErr: errors.New("container label or UUID"), }, - "bad pool UUID": { + "no pool ID": { req: &ContSetOwnerReq{ - PoolUUID: "garbage", - ContUUID: testContUUID, + ContID: testContUUID, }, - expErr: errors.New("invalid UUID"), - }, - "no container UUID": { - req: &ContSetOwnerReq{ - PoolUUID: testPoolUUID, - }, - expErr: errors.New("invalid UUID"), - }, - "no pool UUID": { - req: &ContSetOwnerReq{ - ContUUID: testContUUID, - }, - expErr: errors.New("invalid UUID"), + expErr: errors.New("pool label or UUID"), }, "no user or group": { req: &ContSetOwnerReq{ - PoolUUID: testPoolUUID, - ContUUID: testContUUID, + PoolID: testPoolUUID, + ContID: testContUUID, }, expErr: errors.New("no user or group specified"), }, @@ -74,31 +60,31 @@ func TestControl_ContSetOwner(t *testing.T) { req: validReq, mic: &MockInvokerConfig{ UnaryResponse: MockMSResponse("host1", nil, - &mgmtpb.ContSetOwnerResp{}, + &mgmtpb.DaosResp{}, ), }, }, "user-only success": { req: &ContSetOwnerReq{ - PoolUUID: testPoolUUID, - ContUUID: testContUUID, - User: "someuser@", + PoolID: testPoolUUID, + ContID: testContUUID, + User: "someuser@", }, mic: &MockInvokerConfig{ UnaryResponse: MockMSResponse("host1", nil, - &mgmtpb.ContSetOwnerResp{}, + &mgmtpb.DaosResp{}, ), }, }, "group-only success": { req: &ContSetOwnerReq{ - PoolUUID: testPoolUUID, - ContUUID: testContUUID, - Group: "somegroup@", + PoolID: testPoolUUID, + ContID: testContUUID, + Group: "somegroup@", }, mic: &MockInvokerConfig{ UnaryResponse: MockMSResponse("host1", nil, - &mgmtpb.ContSetOwnerResp{}, + &mgmtpb.DaosResp{}, ), }, }, @@ -116,6 +102,18 @@ func TestControl_ContSetOwner(t *testing.T) { }, expErr: errors.New("remote failed"), }, + "labels OK": { + req: &ContSetOwnerReq{ + PoolID: "pool1", + ContID: "cont1", + User: "someuser@", + }, + mic: &MockInvokerConfig{ + UnaryResponse: MockMSResponse("host1", nil, + &mgmtpb.DaosResp{}, + ), + }, + }, } { t.Run(name, func(t *testing.T) { log, buf := logging.NewTestLogger(t.Name()) diff --git a/src/control/server/mgmt_cont.go b/src/control/server/mgmt_cont.go index 75706a6a1ae..4cc712d44a9 100644 --- a/src/control/server/mgmt_cont.go +++ b/src/control/server/mgmt_cont.go @@ -1,5 +1,5 @@ // -// (C) Copyright 2020-2022 Intel Corporation. +// (C) Copyright 2020-2024 Intel Corporation. // // SPDX-License-Identifier: BSD-2-Clause-Patent // @@ -36,7 +36,7 @@ func (svc *mgmtSvc) ListContainers(ctx context.Context, req *mgmtpb.ListContReq) } // ContSetOwner forwards a gRPC request to the DAOS I/O Engine to change a container's ownership. -func (svc *mgmtSvc) ContSetOwner(ctx context.Context, req *mgmtpb.ContSetOwnerReq) (*mgmtpb.ContSetOwnerResp, error) { +func (svc *mgmtSvc) ContSetOwner(ctx context.Context, req *mgmtpb.ContSetOwnerReq) (*mgmtpb.DaosResp, error) { if err := svc.checkReplicaRequest(req); err != nil { return nil, err } @@ -46,7 +46,7 @@ func (svc *mgmtSvc) ContSetOwner(ctx context.Context, req *mgmtpb.ContSetOwnerRe return nil, err } - resp := &mgmtpb.ContSetOwnerResp{} + resp := &mgmtpb.DaosResp{} if err = proto.Unmarshal(dresp.Body, resp); err != nil { return nil, errors.Wrap(err, "unmarshal ContSetOwner response") } diff --git a/src/control/server/mgmt_cont_test.go b/src/control/server/mgmt_cont_test.go index 5542fe40c52..46ed2002968 100644 --- a/src/control/server/mgmt_cont_test.go +++ b/src/control/server/mgmt_cont_test.go @@ -1,5 +1,5 @@ // -// (C) Copyright 2018-2022 Intel Corporation. +// (C) Copyright 2018-2024 Intel Corporation. // // SPDX-License-Identifier: BSD-2-Clause-Patent // @@ -153,10 +153,10 @@ func TestMgmt_ContSetOwner(t *testing.T) { validContSetOwnerReq := func() *mgmtpb.ContSetOwnerReq { return &mgmtpb.ContSetOwnerReq{ Sys: build.DefaultSystemName, - ContUUID: "contUUID", - PoolUUID: mockUUID, - Owneruser: "user@", - Ownergroup: "group@", + ContId: "contUUID", + PoolId: mockUUID, + OwnerUser: "user@", + OwnerGroup: "group@", } } @@ -164,7 +164,7 @@ func TestMgmt_ContSetOwner(t *testing.T) { createMS func(*testing.T, logging.Logger) *mgmtSvc setupDrpc func(*testing.T, *mgmtSvc) req *mgmtpb.ContSetOwnerReq - expResp *mgmtpb.ContSetOwnerResp + expResp *mgmtpb.DaosResp expErr error }{ "nil req": { @@ -173,10 +173,10 @@ func TestMgmt_ContSetOwner(t *testing.T) { "pool svc not found": { req: &mgmtpb.ContSetOwnerReq{ Sys: build.DefaultSystemName, - ContUUID: "contUUID", - PoolUUID: "fake", - Owneruser: "user@", - Ownergroup: "group@", + ContId: "contUUID", + PoolId: "fake", + OwnerUser: "user@", + OwnerGroup: "group@", }, expErr: errors.New("unable to find pool"), }, @@ -207,10 +207,10 @@ func TestMgmt_ContSetOwner(t *testing.T) { }, "success": { setupDrpc: func(t *testing.T, svc *mgmtSvc) { - setupSvcDrpcClient(svc, 0, getMockDrpcClient(&mgmtpb.ContSetOwnerResp{}, nil)) + setupSvcDrpcClient(svc, 0, getMockDrpcClient(&mgmtpb.DaosResp{}, nil)) }, req: validContSetOwnerReq(), - expResp: &mgmtpb.ContSetOwnerResp{ + expResp: &mgmtpb.DaosResp{ Status: 0, }, }, diff --git a/src/include/daos_srv/container.h b/src/include/daos_srv/container.h index d44931ae4cf..940d02fdc25 100644 --- a/src/include/daos_srv/container.h +++ b/src/include/daos_srv/container.h @@ -32,8 +32,9 @@ int ds_cont_svc_init(struct cont_svc **svcp, const uuid_t pool_uuid, void ds_cont_svc_fini(struct cont_svc **svcp); int ds_cont_svc_step_up(struct cont_svc *svc); void ds_cont_svc_step_down(struct cont_svc *svc); -int ds_cont_svc_set_prop(uuid_t pool_uuid, uuid_t cont_uuid, - d_rank_list_t *ranks, daos_prop_t *prop); +int + ds_cont_svc_set_prop(uuid_t pool_uuid, const char *cont_id, d_rank_list_t *ranks, + daos_prop_t *prop); int ds_cont_list(uuid_t pool_uuid, struct daos_pool_cont_info **conts, uint64_t *ncont); int ds_cont_filter(uuid_t pool_uuid, daos_pool_cont_filter_t *filt, struct daos_pool_cont_info2 **conts, uint64_t *ncont); diff --git a/src/mgmt/cont.pb-c.c b/src/mgmt/cont.pb-c.c index 8517129b4a8..ff2c38b76ea 100644 --- a/src/mgmt/cont.pb-c.c +++ b/src/mgmt/cont.pb-c.c @@ -52,51 +52,6 @@ void mgmt__cont_set_owner_req__free_unpacked assert(message->base.descriptor == &mgmt__cont_set_owner_req__descriptor); protobuf_c_message_free_unpacked ((ProtobufCMessage*)message, allocator); } -void mgmt__cont_set_owner_resp__init - (Mgmt__ContSetOwnerResp *message) -{ - static const Mgmt__ContSetOwnerResp init_value = MGMT__CONT_SET_OWNER_RESP__INIT; - *message = init_value; -} -size_t mgmt__cont_set_owner_resp__get_packed_size - (const Mgmt__ContSetOwnerResp *message) -{ - assert(message->base.descriptor == &mgmt__cont_set_owner_resp__descriptor); - return protobuf_c_message_get_packed_size ((const ProtobufCMessage*)(message)); -} -size_t mgmt__cont_set_owner_resp__pack - (const Mgmt__ContSetOwnerResp *message, - uint8_t *out) -{ - assert(message->base.descriptor == &mgmt__cont_set_owner_resp__descriptor); - return protobuf_c_message_pack ((const ProtobufCMessage*)message, out); -} -size_t mgmt__cont_set_owner_resp__pack_to_buffer - (const Mgmt__ContSetOwnerResp *message, - ProtobufCBuffer *buffer) -{ - assert(message->base.descriptor == &mgmt__cont_set_owner_resp__descriptor); - return protobuf_c_message_pack_to_buffer ((const ProtobufCMessage*)message, buffer); -} -Mgmt__ContSetOwnerResp * - mgmt__cont_set_owner_resp__unpack - (ProtobufCAllocator *allocator, - size_t len, - const uint8_t *data) -{ - return (Mgmt__ContSetOwnerResp *) - protobuf_c_message_unpack (&mgmt__cont_set_owner_resp__descriptor, - allocator, len, data); -} -void mgmt__cont_set_owner_resp__free_unpacked - (Mgmt__ContSetOwnerResp *message, - ProtobufCAllocator *allocator) -{ - if(!message) - return; - assert(message->base.descriptor == &mgmt__cont_set_owner_resp__descriptor); - protobuf_c_message_free_unpacked ((ProtobufCMessage*)message, allocator); -} static const ProtobufCFieldDescriptor mgmt__cont_set_owner_req__field_descriptors[6] = { { @@ -112,48 +67,48 @@ static const ProtobufCFieldDescriptor mgmt__cont_set_owner_req__field_descriptor 0,NULL,NULL /* reserved1,reserved2, etc */ }, { - "contUUID", + "cont_id", 2, PROTOBUF_C_LABEL_NONE, PROTOBUF_C_TYPE_STRING, 0, /* quantifier_offset */ - offsetof(Mgmt__ContSetOwnerReq, contuuid), + offsetof(Mgmt__ContSetOwnerReq, cont_id), NULL, &protobuf_c_empty_string, 0, /* flags */ 0,NULL,NULL /* reserved1,reserved2, etc */ }, { - "poolUUID", + "pool_id", 3, PROTOBUF_C_LABEL_NONE, PROTOBUF_C_TYPE_STRING, 0, /* quantifier_offset */ - offsetof(Mgmt__ContSetOwnerReq, pooluuid), + offsetof(Mgmt__ContSetOwnerReq, pool_id), NULL, &protobuf_c_empty_string, 0, /* flags */ 0,NULL,NULL /* reserved1,reserved2, etc */ }, { - "owneruser", + "owner_user", 4, PROTOBUF_C_LABEL_NONE, PROTOBUF_C_TYPE_STRING, 0, /* quantifier_offset */ - offsetof(Mgmt__ContSetOwnerReq, owneruser), + offsetof(Mgmt__ContSetOwnerReq, owner_user), NULL, &protobuf_c_empty_string, 0, /* flags */ 0,NULL,NULL /* reserved1,reserved2, etc */ }, { - "ownergroup", + "owner_group", 5, PROTOBUF_C_LABEL_NONE, PROTOBUF_C_TYPE_STRING, 0, /* quantifier_offset */ - offsetof(Mgmt__ContSetOwnerReq, ownergroup), + offsetof(Mgmt__ContSetOwnerReq, owner_group), NULL, &protobuf_c_empty_string, 0, /* flags */ @@ -173,10 +128,10 @@ static const ProtobufCFieldDescriptor mgmt__cont_set_owner_req__field_descriptor }, }; static const unsigned mgmt__cont_set_owner_req__field_indices_by_name[] = { - 1, /* field[1] = contUUID */ - 4, /* field[4] = ownergroup */ - 3, /* field[3] = owneruser */ - 2, /* field[2] = poolUUID */ + 1, /* field[1] = cont_id */ + 4, /* field[4] = owner_group */ + 3, /* field[3] = owner_user */ + 2, /* field[2] = pool_id */ 5, /* field[5] = svc_ranks */ 0, /* field[0] = sys */ }; @@ -200,41 +155,3 @@ const ProtobufCMessageDescriptor mgmt__cont_set_owner_req__descriptor = (ProtobufCMessageInit) mgmt__cont_set_owner_req__init, NULL,NULL,NULL /* reserved[123] */ }; -static const ProtobufCFieldDescriptor mgmt__cont_set_owner_resp__field_descriptors[1] = -{ - { - "status", - 1, - PROTOBUF_C_LABEL_NONE, - PROTOBUF_C_TYPE_INT32, - 0, /* quantifier_offset */ - offsetof(Mgmt__ContSetOwnerResp, status), - NULL, - NULL, - 0, /* flags */ - 0,NULL,NULL /* reserved1,reserved2, etc */ - }, -}; -static const unsigned mgmt__cont_set_owner_resp__field_indices_by_name[] = { - 0, /* field[0] = status */ -}; -static const ProtobufCIntRange mgmt__cont_set_owner_resp__number_ranges[1 + 1] = -{ - { 1, 0 }, - { 0, 1 } -}; -const ProtobufCMessageDescriptor mgmt__cont_set_owner_resp__descriptor = -{ - PROTOBUF_C__MESSAGE_DESCRIPTOR_MAGIC, - "mgmt.ContSetOwnerResp", - "ContSetOwnerResp", - "Mgmt__ContSetOwnerResp", - "mgmt", - sizeof(Mgmt__ContSetOwnerResp), - 1, - mgmt__cont_set_owner_resp__field_descriptors, - mgmt__cont_set_owner_resp__field_indices_by_name, - 1, mgmt__cont_set_owner_resp__number_ranges, - (ProtobufCMessageInit) mgmt__cont_set_owner_resp__init, - NULL,NULL,NULL /* reserved[123] */ -}; diff --git a/src/mgmt/cont.pb-c.h b/src/mgmt/cont.pb-c.h index cf435513a91..49f17aca154 100644 --- a/src/mgmt/cont.pb-c.h +++ b/src/mgmt/cont.pb-c.h @@ -16,7 +16,6 @@ PROTOBUF_C__BEGIN_DECLS typedef struct _Mgmt__ContSetOwnerReq Mgmt__ContSetOwnerReq; -typedef struct _Mgmt__ContSetOwnerResp Mgmt__ContSetOwnerResp; /* --- enums --- */ @@ -25,7 +24,7 @@ typedef struct _Mgmt__ContSetOwnerResp Mgmt__ContSetOwnerResp; /* --- messages --- */ /* - * ContSetOwnerReq supplies new pool parameters. + * ContSetOwnerReq changes the ownership of a container. */ struct _Mgmt__ContSetOwnerReq { @@ -35,21 +34,21 @@ struct _Mgmt__ContSetOwnerReq */ char *sys; /* - * UUID of the container + * UUID or label of the container */ - char *contuuid; + char *cont_id; /* - * UUID of the pool that the container is in + * UUID or label of the pool that the container is in */ - char *pooluuid; + char *pool_id; /* * formatted user e.g. "bob@" */ - char *owneruser; + char *owner_user; /* * formatted group e.g. "builders@" */ - char *ownergroup; + char *owner_group; /* * List of pool service ranks */ @@ -61,22 +60,6 @@ struct _Mgmt__ContSetOwnerReq , (char *)protobuf_c_empty_string, (char *)protobuf_c_empty_string, (char *)protobuf_c_empty_string, (char *)protobuf_c_empty_string, (char *)protobuf_c_empty_string, 0,NULL } -/* - * ContSetOwnerResp returns created pool uuid and ranks. - */ -struct _Mgmt__ContSetOwnerResp -{ - ProtobufCMessage base; - /* - * DAOS error code - */ - int32_t status; -}; -#define MGMT__CONT_SET_OWNER_RESP__INIT \ - { PROTOBUF_C_MESSAGE_INIT (&mgmt__cont_set_owner_resp__descriptor) \ - , 0 } - - /* Mgmt__ContSetOwnerReq methods */ void mgmt__cont_set_owner_req__init (Mgmt__ContSetOwnerReq *message); @@ -96,33 +79,11 @@ Mgmt__ContSetOwnerReq * void mgmt__cont_set_owner_req__free_unpacked (Mgmt__ContSetOwnerReq *message, ProtobufCAllocator *allocator); -/* Mgmt__ContSetOwnerResp methods */ -void mgmt__cont_set_owner_resp__init - (Mgmt__ContSetOwnerResp *message); -size_t mgmt__cont_set_owner_resp__get_packed_size - (const Mgmt__ContSetOwnerResp *message); -size_t mgmt__cont_set_owner_resp__pack - (const Mgmt__ContSetOwnerResp *message, - uint8_t *out); -size_t mgmt__cont_set_owner_resp__pack_to_buffer - (const Mgmt__ContSetOwnerResp *message, - ProtobufCBuffer *buffer); -Mgmt__ContSetOwnerResp * - mgmt__cont_set_owner_resp__unpack - (ProtobufCAllocator *allocator, - size_t len, - const uint8_t *data); -void mgmt__cont_set_owner_resp__free_unpacked - (Mgmt__ContSetOwnerResp *message, - ProtobufCAllocator *allocator); /* --- per-message closures --- */ typedef void (*Mgmt__ContSetOwnerReq_Closure) (const Mgmt__ContSetOwnerReq *message, void *closure_data); -typedef void (*Mgmt__ContSetOwnerResp_Closure) - (const Mgmt__ContSetOwnerResp *message, - void *closure_data); /* --- services --- */ @@ -130,7 +91,6 @@ typedef void (*Mgmt__ContSetOwnerResp_Closure) /* --- descriptors --- */ extern const ProtobufCMessageDescriptor mgmt__cont_set_owner_req__descriptor; -extern const ProtobufCMessageDescriptor mgmt__cont_set_owner_resp__descriptor; PROTOBUF_C__END_DECLS diff --git a/src/mgmt/srv_container.c b/src/mgmt/srv_container.c index d3dfcae265b..ec61969def7 100644 --- a/src/mgmt/srv_container.c +++ b/src/mgmt/srv_container.c @@ -1,5 +1,5 @@ /** - * (C) Copyright 2020-2022 Intel Corporation. + * (C) Copyright 2020-2024 Intel Corporation. * * SPDX-License-Identifier: BSD-2-Clause-Patent */ @@ -13,34 +13,19 @@ #include "srv_internal.h" -static int -cont_set_prop(uuid_t pool_uuid, d_rank_list_t *svc_ranks, - uuid_t cont_uuid, daos_prop_t *prop) -{ - int rc = 0; - - rc = ds_cont_svc_set_prop(pool_uuid, cont_uuid, svc_ranks, prop); - if (rc != 0) - goto out; - -out: - return rc; -} - int -ds_mgmt_cont_set_owner(uuid_t pool_uuid, d_rank_list_t *svc_ranks, - uuid_t cont_uuid, const char *user, - const char *group) +ds_mgmt_cont_set_owner(uuid_t pool_uuid, d_rank_list_t *svc_ranks, const char *cont_id, + const char *user, const char *group) { - int rc = 0; - daos_prop_t *prop; - uint32_t prop_nr = 0; - uint32_t i = 0; - bool user_set = false; - bool grp_set = false; + int rc = 0; + daos_prop_t *prop; + uint32_t prop_nr = 0; + uint32_t i = 0; + bool user_set = false; + bool grp_set = false; - D_DEBUG(DB_MGMT, "Setting owner for container "DF_UUID" in pool " - DF_UUID"\n", DP_UUID(cont_uuid), DP_UUID(pool_uuid)); + D_DEBUG(DB_MGMT, DF_UUID ": Setting owner for container '%s'\n", DP_UUID(pool_uuid), + cont_id); user_set = user != NULL && strnlen(user, DAOS_ACL_MAX_PRINCIPAL_LEN) > 0; grp_set = group != NULL && strnlen(group, DAOS_ACL_MAX_PRINCIPAL_LEN) > 0; @@ -76,7 +61,7 @@ ds_mgmt_cont_set_owner(uuid_t pool_uuid, d_rank_list_t *svc_ranks, i++; } - rc = cont_set_prop(pool_uuid, svc_ranks, cont_uuid, prop); + rc = ds_cont_svc_set_prop(pool_uuid, cont_id, svc_ranks, prop); out_prop: daos_prop_free(prop); return rc; diff --git a/src/mgmt/srv_drpc.c b/src/mgmt/srv_drpc.c index 0128eab023d..be1a67a8c54 100644 --- a/src/mgmt/srv_drpc.c +++ b/src/mgmt/srv_drpc.c @@ -2456,11 +2456,11 @@ ds_mgmt_drpc_cont_set_owner(Drpc__Call *drpc_req, Drpc__Response *drpc_resp) { struct drpc_alloc alloc = PROTO_ALLOCATOR_INIT(alloc); Mgmt__ContSetOwnerReq *req = NULL; - Mgmt__ContSetOwnerResp resp = MGMT__CONT_SET_OWNER_RESP__INIT; + Mgmt__DaosResp resp = MGMT__DAOS_RESP__INIT; uint8_t *body; - size_t len; - uuid_t pool_uuid, cont_uuid; + size_t len; d_rank_list_t *svc_ranks = NULL; + uuid_t pool_uuid; int rc = 0; req = mgmt__cont_set_owner_req__unpack(&alloc.alloc, drpc_req->body.len, @@ -2472,39 +2472,33 @@ ds_mgmt_drpc_cont_set_owner(Drpc__Call *drpc_req, Drpc__Response *drpc_resp) return; } - D_INFO("Received request to change container owner\n"); - - if (uuid_parse(req->contuuid, cont_uuid) != 0) { - rc = -DER_INVAL; - DL_ERROR(rc, "Container UUID is invalid"); - goto out; - } + D_INFO("Received request to change owner for container '%s' in pool '%s'\n", req->cont_id, + req->pool_id); - if (uuid_parse(req->pooluuid, pool_uuid) != 0) { - rc = -DER_INVAL; - DL_ERROR(rc, "Pool UUID is invalid"); - goto out; + if (uuid_parse(req->pool_id, pool_uuid) != 0) { + DL_ERROR(-DER_INVAL, "Pool UUID is invalid"); + D_GOTO(out, rc = -DER_INVAL); } svc_ranks = uint32_array_to_rank_list(req->svc_ranks, req->n_svc_ranks); if (svc_ranks == NULL) D_GOTO(out, rc = -DER_NOMEM); - rc = ds_mgmt_cont_set_owner(pool_uuid, svc_ranks, cont_uuid, - req->owneruser, req->ownergroup); + rc = ds_mgmt_cont_set_owner(pool_uuid, svc_ranks, req->cont_id, req->owner_user, + req->owner_group); if (rc != 0) - D_ERROR("Set owner failed: %d\n", rc); + D_ERROR("Container set owner failed: " DF_RC "\n", DP_RC(rc)); d_rank_list_free(svc_ranks); out: resp.status = rc; - len = mgmt__cont_set_owner_resp__get_packed_size(&resp); + len = mgmt__daos_resp__get_packed_size(&resp); D_ALLOC(body, len); if (body == NULL) { drpc_resp->status = DRPC__STATUS__FAILED_MARSHAL; } else { - mgmt__cont_set_owner_resp__pack(&resp, body); + mgmt__daos_resp__pack(&resp, body); drpc_resp->body.len = len; drpc_resp->body.data = body; } diff --git a/src/mgmt/srv_internal.h b/src/mgmt/srv_internal.h index 656f706082e..11fe77c9b1c 100644 --- a/src/mgmt/srv_internal.h +++ b/src/mgmt/srv_internal.h @@ -120,9 +120,9 @@ int int ds_mgmt_pool_query_targets(uuid_t pool_uuid, d_rank_list_t *svc_ranks, d_rank_t rank, d_rank_list_t *tgts, daos_target_info_t **infos); -int ds_mgmt_cont_set_owner(uuid_t pool_uuid, d_rank_list_t *svc_ranks, - uuid_t cont_uuid, const char *user, - const char *group); +int + ds_mgmt_cont_set_owner(uuid_t pool_uuid, d_rank_list_t *svc_ranks, const char *cont_id, + const char *user, const char *group); /** srv_chk.c */ int ds_mgmt_check_start(uint32_t rank_nr, d_rank_t *ranks, uint32_t policy_nr, diff --git a/src/mgmt/tests/mocks.c b/src/mgmt/tests/mocks.c index dcbda6c92f8..912a36f293a 100644 --- a/src/mgmt/tests/mocks.c +++ b/src/mgmt/tests/mocks.c @@ -384,18 +384,17 @@ mock_ds_mgmt_pool_query_targets_teardown(void) } } -int ds_mgmt_cont_set_owner_return; -uuid_t ds_mgmt_cont_set_owner_pool; -uuid_t ds_mgmt_cont_set_owner_cont; +int ds_mgmt_cont_set_owner_return; +uuid_t ds_mgmt_cont_set_owner_pool; +char *ds_mgmt_cont_set_owner_cont; char *ds_mgmt_cont_set_owner_user; char *ds_mgmt_cont_set_owner_group; int -ds_mgmt_cont_set_owner(uuid_t pool_uuid, d_rank_list_t *svc_ranks, - uuid_t cont_uuid, const char *user, - const char *group) +ds_mgmt_cont_set_owner(uuid_t pool_uuid, d_rank_list_t *svc_ranks, const char *cont_id, + const char *user, const char *group) { uuid_copy(ds_mgmt_cont_set_owner_pool, pool_uuid); - uuid_copy(ds_mgmt_cont_set_owner_cont, cont_uuid); + D_STRNDUP(ds_mgmt_cont_set_owner_cont, cont_id, DAOS_PROP_LABEL_MAX_LEN); if (user != NULL) D_STRNDUP(ds_mgmt_cont_set_owner_user, user, DAOS_ACL_MAX_PRINCIPAL_LEN); @@ -412,12 +411,13 @@ mock_ds_mgmt_cont_set_owner_setup(void) ds_mgmt_cont_set_owner_return = 0; uuid_clear(ds_mgmt_cont_set_owner_pool); - uuid_clear(ds_mgmt_cont_set_owner_cont); + ds_mgmt_cont_set_owner_cont = NULL; ds_mgmt_cont_set_owner_user = NULL; ds_mgmt_cont_set_owner_group = NULL; } void mock_ds_mgmt_cont_set_owner_teardown(void) { + D_FREE(ds_mgmt_cont_set_owner_cont); D_FREE(ds_mgmt_cont_set_owner_user); D_FREE(ds_mgmt_cont_set_owner_group); } diff --git a/src/mgmt/tests/mocks.h b/src/mgmt/tests/mocks.h index f8d83b4b680..8bd0ffe998a 100644 --- a/src/mgmt/tests/mocks.h +++ b/src/mgmt/tests/mocks.h @@ -139,11 +139,11 @@ void mock_ds_mgmt_pool_evict_setup(void); /* * Mock ds_mgmt_cont_set_owner */ -extern int ds_mgmt_cont_set_owner_return; -extern uuid_t ds_mgmt_cont_set_owner_pool; -extern uuid_t ds_mgmt_cont_set_owner_cont; -extern char *ds_mgmt_cont_set_owner_user; -extern char *ds_mgmt_cont_set_owner_group; +extern int ds_mgmt_cont_set_owner_return; +extern uuid_t ds_mgmt_cont_set_owner_pool; +extern char *ds_mgmt_cont_set_owner_cont; +extern char *ds_mgmt_cont_set_owner_user; +extern char *ds_mgmt_cont_set_owner_group; void mock_ds_mgmt_cont_set_owner_setup(void); void mock_ds_mgmt_cont_set_owner_teardown(void); void mock_ds_mgmt_pool_query_targets_gen_infos(uint32_t n_infos); diff --git a/src/mgmt/tests/srv_drpc_tests.c b/src/mgmt/tests/srv_drpc_tests.c index 9b034f04d6a..85cd31deefe 100644 --- a/src/mgmt/tests/srv_drpc_tests.c +++ b/src/mgmt/tests/srv_drpc_tests.c @@ -2408,17 +2408,16 @@ static void expect_drpc_cont_set_owner_resp_with_status(Drpc__Response *resp, int expected_err) { - Mgmt__ContSetOwnerResp *payload_resp = NULL; + Mgmt__DaosResp *payload_resp = NULL; assert_int_equal(resp->status, DRPC__STATUS__SUCCESS); assert_non_null(resp->body.data); - payload_resp = mgmt__cont_set_owner_resp__unpack(NULL, resp->body.len, - resp->body.data); + payload_resp = mgmt__daos_resp__unpack(NULL, resp->body.len, resp->body.data); assert_non_null(payload_resp); assert_int_equal(payload_resp->status, expected_err); - mgmt__cont_set_owner_resp__free_unpacked(payload_resp, NULL); + mgmt__daos_resp__free_unpacked(payload_resp, NULL); } static void @@ -2428,10 +2427,10 @@ test_drpc_cont_set_owner_bad_pool_uuid(void **state) Drpc__Response resp = DRPC__RESPONSE__INIT; Mgmt__ContSetOwnerReq req = MGMT__CONT_SET_OWNER_REQ__INIT; - req.pooluuid = "this isn't really a uuid"; - req.contuuid = "11111111-1111-1111-1111-111111111111"; - req.owneruser = "user@"; - req.ownergroup = "group@"; + req.pool_id = "this isn't really a uuid"; + req.cont_id = "11111111-1111-1111-1111-111111111111"; + req.owner_user = "user@"; + req.owner_group = "group@"; setup_cont_set_owner_drpc_call(&call, &req); @@ -2444,22 +2443,32 @@ test_drpc_cont_set_owner_bad_pool_uuid(void **state) } static void -test_drpc_cont_set_owner_bad_cont_uuid(void **state) +test_drpc_cont_set_owner_cont_label(void **state) { Drpc__Call call = DRPC__CALL__INIT; Drpc__Response resp = DRPC__RESPONSE__INIT; Mgmt__ContSetOwnerReq req = MGMT__CONT_SET_OWNER_REQ__INIT; + uuid_t pool_uuid; - req.pooluuid = "11111111-1111-1111-1111-111111111111"; - req.contuuid = "this isn't really a uuid"; - req.owneruser = "user@"; - req.ownergroup = "group@"; + req.pool_id = "11111111-1111-1111-1111-111111111111"; + req.cont_id = "cont_label"; + req.owner_user = "user@"; + req.owner_group = "group@"; setup_cont_set_owner_drpc_call(&call, &req); ds_mgmt_drpc_cont_set_owner(&call, &resp); - expect_drpc_cont_set_owner_resp_with_status(&resp, -DER_INVAL); + expect_drpc_cont_set_owner_resp_with_status(&resp, 0); + + /* Verify mgmt_cont_set_owner called with correct params */ + assert_int_equal(uuid_parse(req.pool_id, pool_uuid), 0); + assert_int_equal(uuid_compare(pool_uuid, ds_mgmt_cont_set_owner_pool), 0); + assert_string_equal(req.cont_id, ds_mgmt_cont_set_owner_cont); + assert_non_null(ds_mgmt_cont_set_owner_user); + assert_string_equal(req.owner_user, ds_mgmt_cont_set_owner_user); + assert_non_null(ds_mgmt_cont_set_owner_group); + assert_string_equal(req.owner_group, ds_mgmt_cont_set_owner_group); D_FREE(call.body.data); D_FREE(resp.body.data); @@ -2473,10 +2482,10 @@ test_drpc_cont_set_owner_failed(void **state) Mgmt__ContSetOwnerReq req = MGMT__CONT_SET_OWNER_REQ__INIT; int exp_rc = -DER_MISC; - req.pooluuid = "11111111-1111-1111-1111-111111111111"; - req.contuuid = "22222222-2222-2222-2222-222222222222"; - req.owneruser = "user@"; - req.ownergroup = "group@"; + req.pool_id = "11111111-1111-1111-1111-111111111111"; + req.cont_id = "22222222-2222-2222-2222-222222222222"; + req.owner_user = "user@"; + req.owner_group = "group@"; setup_cont_set_owner_drpc_call(&call, &req); ds_mgmt_cont_set_owner_return = exp_rc; @@ -2495,22 +2504,17 @@ test_drpc_cont_set_owner_success(void **state) Drpc__Call call = DRPC__CALL__INIT; Drpc__Response resp = DRPC__RESPONSE__INIT; Mgmt__ContSetOwnerReq req = MGMT__CONT_SET_OWNER_REQ__INIT; - char *pool_uuid_str; - uuid_t pool_uuid; - char *cont_uuid_str; - uuid_t cont_uuid; + char *pool_uuid_str; + char *cont_uuid_str; + uuid_t pool_uuid; pool_uuid_str = "11111111-1111-1111-1111-111111111111"; - if (uuid_parse(pool_uuid_str, pool_uuid)) - return; cont_uuid_str = "22222222-2222-2222-2222-222222222222"; - if (uuid_parse(cont_uuid_str, cont_uuid)) - return; - req.pooluuid = pool_uuid_str; - req.contuuid = cont_uuid_str; - req.owneruser = "user@"; - req.ownergroup = "group@"; + req.pool_id = pool_uuid_str; + req.cont_id = cont_uuid_str; + req.owner_user = "user@"; + req.owner_group = "group@"; setup_cont_set_owner_drpc_call(&call, &req); @@ -2519,14 +2523,13 @@ test_drpc_cont_set_owner_success(void **state) expect_drpc_cont_set_owner_resp_with_status(&resp, 0); /* Verify mgmt_cont_set_owner called with correct params */ - assert_int_equal(uuid_compare(pool_uuid, ds_mgmt_cont_set_owner_pool), - 0); - assert_int_equal(uuid_compare(cont_uuid, ds_mgmt_cont_set_owner_cont), - 0); + assert_int_equal(uuid_parse(pool_uuid_str, pool_uuid), 0); + assert_int_equal(uuid_compare(pool_uuid, ds_mgmt_cont_set_owner_pool), 0); + assert_string_equal(cont_uuid_str, ds_mgmt_cont_set_owner_cont); assert_non_null(ds_mgmt_cont_set_owner_user); - assert_string_equal(req.owneruser, ds_mgmt_cont_set_owner_user); + assert_string_equal(req.owner_user, ds_mgmt_cont_set_owner_user); assert_non_null(ds_mgmt_cont_set_owner_group); - assert_string_equal(req.ownergroup, ds_mgmt_cont_set_owner_group); + assert_string_equal(req.owner_group, ds_mgmt_cont_set_owner_group); D_FREE(call.body.data); D_FREE(resp.body.data); @@ -3112,7 +3115,7 @@ main(void) PING_RANK_TEST(test_drpc_ping_rank_success), PREP_SHUTDOWN_TEST(test_drpc_prep_shutdown_success), SET_LOG_MASKS_TEST(test_drpc_set_log_masks_success), - CONT_SET_OWNER_TEST(test_drpc_cont_set_owner_bad_cont_uuid), + CONT_SET_OWNER_TEST(test_drpc_cont_set_owner_cont_label), CONT_SET_OWNER_TEST(test_drpc_cont_set_owner_bad_pool_uuid), CONT_SET_OWNER_TEST(test_drpc_cont_set_owner_failed), CONT_SET_OWNER_TEST(test_drpc_cont_set_owner_success), diff --git a/src/proto/mgmt/cont.proto b/src/proto/mgmt/cont.proto index f90a35528d6..4e6c2a52a57 100644 --- a/src/proto/mgmt/cont.proto +++ b/src/proto/mgmt/cont.proto @@ -1,5 +1,5 @@ // -// (C) Copyright 2020-2021 Intel Corporation. +// (C) Copyright 2020-2024 Intel Corporation. // // SPDX-License-Identifier: BSD-2-Clause-Patent // @@ -12,17 +12,12 @@ option go_package = "github.com/daos-stack/daos/src/control/common/proto/mgmt"; // Management Service Protobuf Definitions related to interactions between // DAOS control server and DAOS containers. -// ContSetOwnerReq supplies new pool parameters. +// ContSetOwnerReq changes the ownership of a container. message ContSetOwnerReq { string sys = 1; // DAOS system identifier - string contUUID = 2; // UUID of the container - string poolUUID = 3; // UUID of the pool that the container is in - string owneruser = 4; // formatted user e.g. "bob@" - string ownergroup = 5; // formatted group e.g. "builders@" + string cont_id = 2; // UUID or label of the container + string pool_id = 3; // UUID or label of the pool that the container is in + string owner_user = 4; // formatted user e.g. "bob@" + string owner_group = 5; // formatted group e.g. "builders@" repeated uint32 svc_ranks = 6; // List of pool service ranks } - -// ContSetOwnerResp returns created pool uuid and ranks. -message ContSetOwnerResp { - int32 status = 1; // DAOS error code -} diff --git a/src/proto/mgmt/mgmt.proto b/src/proto/mgmt/mgmt.proto index d58c8115fa8..82a83aa36f9 100644 --- a/src/proto/mgmt/mgmt.proto +++ b/src/proto/mgmt/mgmt.proto @@ -1,5 +1,5 @@ // -// (C) Copyright 2019-2022 Intel Corporation. +// (C) Copyright 2019-2024 Intel Corporation. // // SPDX-License-Identifier: BSD-2-Clause-Patent // @@ -70,7 +70,7 @@ service MgmtSvc { // List all containers in a pool rpc ListContainers(ListContReq) returns (ListContResp) {} // Change the owner of a DAOS container - rpc ContSetOwner(ContSetOwnerReq) returns (ContSetOwnerResp) {} + rpc ContSetOwner(ContSetOwnerReq) returns (DaosResp) {} // Query DAOS system status rpc SystemQuery(SystemQueryReq) returns(SystemQueryResp) {} // Stop DAOS system (shutdown data-plane instances) diff --git a/src/tests/ftest/security/cont_get_acl.yaml b/src/tests/ftest/security/cont_get_acl.yaml index c82bcaddb70..afc21101c9b 100644 --- a/src/tests/ftest/security/cont_get_acl.yaml +++ b/src/tests/ftest/security/cont_get_acl.yaml @@ -15,6 +15,7 @@ server_config: 0: class: ram scm_mount: /mnt/daos + scm_size: 6 system_ram_reserved: 1 pool: control_method: dmg diff --git a/src/tests/ftest/security/cont_owner.py b/src/tests/ftest/security/cont_owner.py index 7935a6b9bc5..b4ec4f0d526 100644 --- a/src/tests/ftest/security/cont_owner.py +++ b/src/tests/ftest/security/cont_owner.py @@ -30,6 +30,18 @@ def _create_cont_with_acl(self, cont_type): self.container = self.create_container_with_daos(self.pool, acl_file=acl_file_name, cont_type=cont_type) + def _get_ownership(self): + result = self.container.get_acl() + return { + "user": result['response']['owner_user'], + "group": result['response']['owner_group'], + } + + def _check_ownership(self, exp_user, exp_group): + ownership = self._get_ownership() + self.assertEqual(ownership["user"], exp_user, "user owner doesn't match") + self.assertEqual(ownership["group"], exp_group, "group owner doesn't match") + def test_container_set_owner_no_check_non_posix(self): """ Description: @@ -125,3 +137,66 @@ def test_container_set_owner_no_check_posix(self): no_check=True) self.verify_daos_pool_cont_result(result, "set owner with uid/gid with no-check", "pass", None) + + def test_dmg_cont_set_owner(self): + """ + Description: + Verify dmg container set-owner + + :avocado: tags=all,full_regression + :avocado: tags=vm + :avocado: tags=security,control,container,cont_sec,cont_set_owner + :avocado: tags=DaosContainerOwnerTest,test_dmg_cont_set_owner + """ + test_user = "fakeuser@" + test_group = "fakegroup@" + + self._create_cont_with_acl(cont_type="python") + orig_owner = self._get_ownership() + + # No user/group + with self.dmg.no_exception(): + result = self.get_dmg_command().cont_set_owner(self.pool.identifier, + self.container.identifier) + self.verify_daos_pool_cont_result(result, "set owner with no user or group", "fail", + "at least one") + self._check_ownership(orig_owner["user"], orig_owner["group"]) # expect unchanged + + # User only - not locally checked + with self.dmg.no_exception(): + result = self.get_dmg_command().cont_set_owner(self.pool.identifier, + self.container.identifier, + user=test_user) + self.verify_daos_pool_cont_result(result, "set owner user", "pass", None) + self._check_ownership(test_user, orig_owner["group"]) + + # Group only - not locally checked + with self.dmg.no_exception(): + result = self.get_dmg_command().cont_set_owner(self.pool.identifier, + self.container.identifier, + group=test_group) + self.verify_daos_pool_cont_result(result, "set owner group", "pass", None) + self._check_ownership(test_user, test_group) + + # User and group + with self.dmg.no_exception(): + result = self.get_dmg_command().cont_set_owner(self.pool.identifier, + self.container.identifier, + user="fakeuser2", group="fakegroup2") + self.verify_daos_pool_cont_result(result, "set owner user and group", "pass", None) + self._check_ownership("fakeuser2@", "fakegroup2@") + + # Labels as IDs + with self.dmg.no_exception(): + result = self.get_dmg_command().cont_set_owner(self.pool.label.value, + self.container.label.value, + user=orig_owner["user"]) + self.verify_daos_pool_cont_result(result, "set owner user with labels", "pass", None) + self._check_ownership(orig_owner["user"], "fakegroup2@") + + # UUIDs as IDs + with self.dmg.no_exception(): + result = self.get_dmg_command().cont_set_owner(self.pool.uuid, self.container.uuid, + group=orig_owner["group"]) + self.verify_daos_pool_cont_result(result, "set owner group with UUIDs", "pass", None) + self._check_ownership(orig_owner["user"], orig_owner["group"]) diff --git a/src/tests/ftest/util/cont_security_test_base.py b/src/tests/ftest/util/cont_security_test_base.py index 28e714052a5..0a9058f87d8 100644 --- a/src/tests/ftest/util/cont_security_test_base.py +++ b/src/tests/ftest/util/cont_security_test_base.py @@ -83,9 +83,7 @@ def get_container_acl_list(self, container, verbose=False, outfile=None): result = container.get_acl(verbose, outfile) cont_permission_list = [] - for line in result.stdout_text.splitlines(): - if not line.startswith("A:"): - continue + for line in result['response']['entries']: if line.startswith("A::"): found_user = re.search(r"A::(.+)@:(.*)", line) if found_user: diff --git a/src/tests/ftest/util/daos_utils.py b/src/tests/ftest/util/daos_utils.py index efe76b199d9..c916761b25d 100644 --- a/src/tests/ftest/util/daos_utils.py +++ b/src/tests/ftest/util/daos_utils.py @@ -194,7 +194,7 @@ def container_get_acl(self, pool, cont, verbose=False, outfile=None): CommandFailure: if the daos container get-acl command fails. """ - return self._get_result( + return self._get_json_result( ("container", "get-acl"), pool=pool, cont=cont, verbose=verbose, outfile=outfile) diff --git a/src/tests/ftest/util/dmg_utils.py b/src/tests/ftest/util/dmg_utils.py index 0fbc91b38da..34e21f66d0a 100644 --- a/src/tests/ftest/util/dmg_utils.py +++ b/src/tests/ftest/util/dmg_utils.py @@ -957,12 +957,12 @@ def pool_reintegrate(self, pool, rank, tgt_idx=None): return self._get_result( ("pool", "reintegrate"), pool=pool, rank=rank, tgt_idx=tgt_idx) - def cont_set_owner(self, pool, cont, user, group): + def cont_set_owner(self, pool, cont, user=None, group=None): """Dmg container set-owner to the specified new user/group. Args: - pool (str): Pool uuid. - cont (str): Container uuid. + pool (str): Pool label or UUID. + cont (str): Container label or UUID. user (str): new user for the container. group (str): new group for the container. @@ -974,7 +974,7 @@ def cont_set_owner(self, pool, cont, user, group): CommandFailure: if the dmg pool reintegrate command fails. """ - return self._get_result( + return self._get_json_result( ("cont", "set-owner"), pool=pool, cont=cont, user=user, group=group) def system_cleanup(self, machinename=None, verbose=True): diff --git a/src/tests/ftest/util/dmg_utils_base.py b/src/tests/ftest/util/dmg_utils_base.py index 43272eac510..31ba1b50f33 100644 --- a/src/tests/ftest/util/dmg_utils_base.py +++ b/src/tests/ftest/util/dmg_utils_base.py @@ -254,18 +254,18 @@ def get_sub_command_class(self): # pylint: disable=redefined-variable-type """Get the dmg cont sub command object.""" if self.sub_command.value == "set-owner": - self.sub_command_class = self.SetownerSubCommand() + self.sub_command_class = self.SetOwnerSubCommand() else: self.sub_command_class = None - class SetownerSubCommand(CommandWithParameters): + class SetOwnerSubCommand(CommandWithParameters): """Defines an object for the dmg cont set-owner command.""" def __init__(self): """Create a dmg cont set-owner command object.""" super().__init__("/run/dmg/cont/set-owner/*", "set-owner") - self.pool = FormattedParameter("--pool={}", None) - self.cont = FormattedParameter("--cont={}", None) + self.pool = BasicParameter(None, position=1) + self.cont = BasicParameter(None, position=2) self.user = FormattedParameter("--user={}", None) self.group = FormattedParameter("--group={}", None) From a5552da465665cb7852eb02d5a8aa11b9a67a8ff Mon Sep 17 00:00:00 2001 From: wiliamhuang Date: Thu, 29 Aug 2024 10:55:20 -0500 Subject: [PATCH 20/26] DAOS-16365 client: intercept MPI_Init() to avoid nested call (#14992) We observed deadlock in MPI applications on Aurora due to nested calls of zeInit() inside MPI_Init(). daos_init() is involved in such nested calls. This PR intercepts MPI_Init() and avoid running daos_init() inside MPI_Init(). Signed-off-by: Lei Huang --- src/client/dfuse/pil4dfs/int_dfs.c | 29 +++++++++++++++++++++++++++++ 1 file changed, 29 insertions(+) diff --git a/src/client/dfuse/pil4dfs/int_dfs.c b/src/client/dfuse/pil4dfs/int_dfs.c index 81cd69ccda1..0ff8d0d8f19 100644 --- a/src/client/dfuse/pil4dfs/int_dfs.c +++ b/src/client/dfuse/pil4dfs/int_dfs.c @@ -160,6 +160,8 @@ static long int page_size; #define DAOS_INIT_NOT_RUNNING 0 #define DAOS_INIT_RUNNING 1 +static _Atomic uint64_t mpi_init_count; + static long int daos_initing; _Atomic bool d_daos_inited; static bool daos_debug_inited; @@ -467,6 +469,8 @@ static int (*next_posix_fallocate64)(int fd, off64_t offset, off64_t len); static int (*next_tcgetattr)(int fd, void *termios_p); /* end NOT supported by DAOS */ +static int (*next_mpi_init)(int *argc, char ***argv); + /* to do!! */ /** * static char * (*org_realpath)(const char *pathname, char *resolved_path); @@ -1020,6 +1024,22 @@ consume_low_fd(void) return rc; } +int +MPI_Init(int *argc, char ***argv) +{ + int rc; + + if (next_mpi_init == NULL) { + next_mpi_init = dlsym(RTLD_NEXT, "MPI_Init"); + D_ASSERT(next_mpi_init != NULL); + } + + atomic_fetch_add_relaxed(&mpi_init_count, 1); + rc = next_mpi_init(argc, argv); + atomic_fetch_add_relaxed(&mpi_init_count, -1); + return rc; +} + /** determine whether a path (both relative and absolute) is on DAOS or not. If yes, * returns parent object, item name, full path of parent dir, full absolute path, and * the pointer to struct dfs_mt. @@ -1117,6 +1137,15 @@ query_path(const char *szInput, int *is_target_path, struct dcache_rec **parent, uint64_t status_old = DAOS_INIT_NOT_RUNNING; bool rc_cmp_swap; + /* Check whether MPI_Init() is running. If yes, pass to the original + * libc functions. Avoid possible zeInit reentrancy/nested call. + */ + + if (atomic_load_relaxed(&mpi_init_count) > 0) { + *is_target_path = 0; + goto out_normal; + } + /* daos_init() is expensive to call. We call it only when necessary. */ /* Check whether daos_init() is running. If yes, pass to the original From 5db61e8ffd1457c60707d0ed9f2133f37f346bc7 Mon Sep 17 00:00:00 2001 From: Dalton Bohning Date: Thu, 29 Aug 2024 09:32:48 -0700 Subject: [PATCH 21/26] DAOS-16462 test: remove server manager srv_timeout (#15029) remove server manager srv_timeout since it is unused Signed-off-by: Dalton Bohning --- src/tests/ftest/aggregation/dfuse_space_check.yaml | 3 --- src/tests/ftest/dfuse/sparse_file.yaml | 3 --- src/tests/ftest/osa/offline_drain.yaml | 3 --- src/tests/ftest/osa/offline_parallel_test.yaml | 3 --- src/tests/ftest/osa/online_drain.yaml | 2 -- src/tests/ftest/util/yaml_utils.py | 3 +-- 6 files changed, 1 insertion(+), 16 deletions(-) diff --git a/src/tests/ftest/aggregation/dfuse_space_check.yaml b/src/tests/ftest/aggregation/dfuse_space_check.yaml index 47ae21d4de5..dc8e9013174 100644 --- a/src/tests/ftest/aggregation/dfuse_space_check.yaml +++ b/src/tests/ftest/aggregation/dfuse_space_check.yaml @@ -1,5 +1,3 @@ -server_manager: - srv_timeout: 500 hosts: test_servers: 1 test_clients: 1 @@ -14,7 +12,6 @@ server_config: pool: scm_size: 200MB nvme_size: 1GiB # Minimum for 1 target - svcn: 1 container: type: POSIX control_method: daos diff --git a/src/tests/ftest/dfuse/sparse_file.yaml b/src/tests/ftest/dfuse/sparse_file.yaml index 0e77acfe41a..5b30160a012 100644 --- a/src/tests/ftest/dfuse/sparse_file.yaml +++ b/src/tests/ftest/dfuse/sparse_file.yaml @@ -1,5 +1,3 @@ -server_manager: - srv_timeout: 500 hosts: test_servers: 1 test_clients: 1 @@ -14,7 +12,6 @@ server_config: pool: scm_size: 200000000 nvme_size: 1073741824 - svcn: 1 container: type: POSIX control_method: daos diff --git a/src/tests/ftest/osa/offline_drain.yaml b/src/tests/ftest/osa/offline_drain.yaml index 5e4651e499f..a8776edd4a4 100644 --- a/src/tests/ftest/osa/offline_drain.yaml +++ b/src/tests/ftest/osa/offline_drain.yaml @@ -1,5 +1,3 @@ -server_manager: - srv_timeout: 500 hosts: test_servers: 3 test_clients: 1 @@ -35,7 +33,6 @@ pool: scm_size: 12000000000 nvme_size: 108000000000 svcn: 4 - control_method: dmg rebuild_timeout: 240 properties: scrub:timed container: diff --git a/src/tests/ftest/osa/offline_parallel_test.yaml b/src/tests/ftest/osa/offline_parallel_test.yaml index 4f0af968d3a..e8dd4563339 100644 --- a/src/tests/ftest/osa/offline_parallel_test.yaml +++ b/src/tests/ftest/osa/offline_parallel_test.yaml @@ -1,5 +1,3 @@ -server_manager: - srv_timeout: 500 hosts: test_servers: server-[1-2] test_clients: 1 @@ -37,7 +35,6 @@ pool: scm_size: 6000000000 nvme_size: 54000000000 svcn: 4 - control_method: dmg container: type: POSIX control_method: daos diff --git a/src/tests/ftest/osa/online_drain.yaml b/src/tests/ftest/osa/online_drain.yaml index a671e290398..738683694b1 100644 --- a/src/tests/ftest/osa/online_drain.yaml +++ b/src/tests/ftest/osa/online_drain.yaml @@ -1,5 +1,3 @@ -server_manager: - srv_timeout: 500 hosts: test_servers: 3 test_clients: 1 diff --git a/src/tests/ftest/util/yaml_utils.py b/src/tests/ftest/util/yaml_utils.py index 1bb969e4049..c5a21847881 100644 --- a/src/tests/ftest/util/yaml_utils.py +++ b/src/tests/ftest/util/yaml_utils.py @@ -1,5 +1,5 @@ """ -(C) Copyright 2020-2023 Intel Corporation. +(C) Copyright 2020-2024 Intel Corporation. SPDX-License-Identifier: BSD-2-Clause-Patent """ @@ -87,7 +87,6 @@ class YamlUpdater(): ("pool_query_timeout", "_timeout", int), ("pool_query_delay", "_timeout", int), ("rebuild_timeout", "_timeout", int), - ("srv_timeout", "_timeout", int), ("storage_prepare_timeout", "_timeout", int), ("storage_format_timeout", "_timeout", int), ] From ccf7c143dfd8a37e6143cb63d42e08476d03f5e7 Mon Sep 17 00:00:00 2001 From: Dalton Bohning Date: Thu, 29 Aug 2024 09:35:42 -0700 Subject: [PATCH 22/26] DAOS-16463 test: remove get_host_log_files (#15030) Remove get_host_log_files since it is no longer needed. Also remove search_log which used get_host_log_files and is unused. Signed-off-by: Dalton Bohning --- src/tests/ftest/util/apricot/apricot/test.py | 1 - src/tests/ftest/util/server_utils.py | 79 -------------------- 2 files changed, 80 deletions(-) diff --git a/src/tests/ftest/util/apricot/apricot/test.py b/src/tests/ftest/util/apricot/apricot/test.py index 601de353afa..6f0ac266f9e 100644 --- a/src/tests/ftest/util/apricot/apricot/test.py +++ b/src/tests/ftest/util/apricot/apricot/test.py @@ -1323,7 +1323,6 @@ def _list_server_manager_info(self): self.log.info("-" * 100) self.log.info("--- SERVER INFORMATION ---") for manager in self.server_managers: - manager.get_host_log_files() try: manager.dmg.storage_query_list_devices() except CommandFailure: diff --git a/src/tests/ftest/util/server_utils.py b/src/tests/ftest/util/server_utils.py index c3b41a69b50..6ae05af94e9 100644 --- a/src/tests/ftest/util/server_utils.py +++ b/src/tests/ftest/util/server_utils.py @@ -7,9 +7,7 @@ import os import random -import re import time -from collections import defaultdict from getpass import getuser from avocado import fail_on @@ -1179,80 +1177,3 @@ def get_daos_metrics(self, verbose=False, timeout=60): command="sudo {} -S {} --csv".format(daos_metrics_exe, engine)) engines.append(results) return engines - - def get_host_log_files(self): - """Get the active engine log file names on each host. - - Returns: - dict: host keys with lists of log files on that host values - - """ - self.log.debug("Determining the current %s log files", self.manager.job.command) - - # Get a list of engine pids from all of the hosts - host_engine_pids = defaultdict(list) - result = run_remote(self.log, self.hosts, "pgrep daos_engine", False) - for data in result.output: - if data.passed: - # Search each individual line of output independently to ensure a pid match - for line in data.stdout: - match = re.findall(r'(^[0-9]+)', line) - for host in data.hosts: - host_engine_pids[host].extend(match) - - # Find the log files that match the engine pids on each host - host_log_files = defaultdict(list) - log_files = self.manager.job.get_engine_values("log_file") - for host, pid_list in host_engine_pids.items(): - # Generate a list of all of the possible log files that could exist on this host - file_search = [] - for log_file in log_files: - for pid in pid_list: - file_search.append(".".join([log_file, pid])) - # Determine which of those log files actually do exist on this host - # This matches the engine pid to the engine log file name - command = f"ls -1 {' '.join(file_search)} 2>&1 | grep -v 'No such file or directory'" - result = run_remote(self.log, host, command, False) - for data in result.output: - for line in data.stdout: - match = re.findall(fr"^({'|'.join(file_search)})", line) - if match: - host_log_files[host].append(match[0]) - - self.log.debug("Engine log files per host") - for host in sorted(host_log_files): - self.log.debug(" %s:", host) - for log_file in sorted(host_log_files[host]): - self.log.debug(" %s", log_file) - - return host_log_files - - def search_log(self, pattern): - """Search the server log files on the remote hosts for the specified pattern. - - Args: - pattern (str): the grep -E pattern to use to search the server log files - - Returns: - int: number of patterns found - - """ - self.log.debug("Searching %s logs for '%s'", self.manager.job.command, pattern) - host_log_files = self.get_host_log_files() - - # Search for the pattern in the remote log files - matches = 0 - for host, log_files in host_log_files.items(): - log_file_matches = 0 - self.log.debug("Searching for '%s' in %s on %s", pattern, log_files, host) - result = run_remote(self.log, host, f"grep -E '{pattern}' {' '.join(log_files)}") - for data in result.output: - if data.returncode == 0: - matches = re.findall(fr'{pattern}', '\n'.join(data.stdout)) - log_file_matches += len(matches) - self.log.debug("Found %s matches on %s", log_file_matches, host) - matches += log_file_matches - self.log.debug( - "Found %s total matches for '%s' in the %s logs", - matches, pattern, self.manager.job.command) - return matches From 041962bc5d18f37c1af548e95f19178bfc6f5acf Mon Sep 17 00:00:00 2001 From: Dalton Bohning Date: Thu, 29 Aug 2024 09:52:14 -0700 Subject: [PATCH 23/26] DAOS-16457 test: remove display_memory_info (#15031) display_memory_info was added to debug an issue when starting the servers, but resolved by #14295. It is no longer needed and consumes too much log space and time. Signed-off-by: Dalton Bohning --- src/tests/ftest/util/server_utils.py | 14 -------------- 1 file changed, 14 deletions(-) diff --git a/src/tests/ftest/util/server_utils.py b/src/tests/ftest/util/server_utils.py index 6ae05af94e9..752473021a3 100644 --- a/src/tests/ftest/util/server_utils.py +++ b/src/tests/ftest/util/server_utils.py @@ -464,14 +464,6 @@ def support_collect_log(self, **kwargs): return run_remote( self.log, self._hosts, cmd.with_exports, timeout=self.collect_log_timeout.value) - def display_memory_info(self): - """Display server hosts memory info.""" - self.log.debug("#" * 80) - self.log.debug(" Collection debug memory info") - run_remote(self.log, self._hosts, "free -m && df -h --type=tmpfs") - run_remote(self.log, self._hosts, "ps -eo size,pid,user,command --sort -size | head -n 6") - self.log.debug("#" * 80) - def detect_format_ready(self, reformat=False): """Detect when all the daos_servers are ready for storage format. @@ -664,14 +656,11 @@ def start(self): self.prepare() # Start the servers and wait for them to be ready for storage format - self.display_memory_info() self.detect_format_ready() # Collect storage and network information from the servers. - self.display_memory_info() self.information.collect_storage_information() self.information.collect_network_information() - self.display_memory_info() # Format storage and wait for server to change ownership self.log.info(" Formatting hosts: <%s>", self.dmg.hostlist) @@ -711,9 +700,6 @@ def stop(self): # Make sure the mount directory belongs to non-root user self.set_scm_mount_ownership() - # Collective memory usage after stop. - self.display_memory_info() - # Report any errors after all stop actions have been attempted if messages: raise ServerFailed("Failed to stop servers:\n {}".format("\n ".join(messages))) From 222cdee6c7bd350f027a388ea250a55a123519c0 Mon Sep 17 00:00:00 2001 From: Jerome Soumagne Date: Thu, 29 Aug 2024 12:03:39 -0500 Subject: [PATCH 24/26] DAOS-16315 mercury: update to 2.4.0rc5 (#15015) Signed-off-by: Jerome Soumagne --- utils/build.config | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/utils/build.config b/utils/build.config index 3748bbd6e4e..174df687036 100644 --- a/utils/build.config +++ b/utils/build.config @@ -9,7 +9,7 @@ isal=v2.30.0 isal_crypto=v2.23.0 spdk=v22.01.2 ofi=v1.19.1 -mercury=v2.4.0rc4 +mercury=v2.4.0rc5 protobufc=v1.3.3 ucx=v1.14.1 From c33318be0851e46ee99a99dc30a330371664825c Mon Sep 17 00:00:00 2001 From: Niu Yawei Date: Fri, 30 Aug 2024 09:39:17 +0800 Subject: [PATCH 25/26] DAOS-16465 vos: fix misused ip_hdl (#15034) vos_iter_param_t.ip_hdl is for passing dkey or akey tree open handle in vos_iterate_key() case, don't always regard it as 'vos_pool' or 'vos_container'. Signed-off-by: Niu Yawei --- src/include/daos_srv/vos_types.h | 4 +++- src/vos/vos_iterator.c | 5 +++++ src/vos/vos_obj.c | 13 ++++++++++--- 3 files changed, 18 insertions(+), 4 deletions(-) diff --git a/src/include/daos_srv/vos_types.h b/src/include/daos_srv/vos_types.h index 3d0e8f1e9b2..194b2434c28 100644 --- a/src/include/daos_srv/vos_types.h +++ b/src/include/daos_srv/vos_types.h @@ -413,7 +413,9 @@ typedef int (*vos_iter_filter_cb_t)(daos_handle_t ih, vos_iter_desc_t *desc, * Parameters for initializing VOS iterator */ typedef struct { - /** pool connection handle or container open handle */ + /** pool connection handle or container open handle, + * for vos_iterate_key(), it's used for passing dkey or akey tree open handle. + */ daos_handle_t ip_hdl; /** standalone prepare: DAOS_HDL_INVAL * nested prepare: parent iterator handle diff --git a/src/vos/vos_iterator.c b/src/vos/vos_iterator.c index 360664dee38..c2394d8f64f 100644 --- a/src/vos/vos_iterator.c +++ b/src/vos/vos_iterator.c @@ -148,6 +148,11 @@ is_sysdb_pool(vos_iter_type_t type, vos_iter_param_t *param) struct vos_pool *vos_pool; struct vos_container *vos_cont; + /* vos_iterate_key() will never be called upon sysdb */ + if (param->ip_flags == VOS_IT_KEY_TREE) + return false; + + D_ASSERT(!(param->ip_flags & VOS_IT_KEY_TREE)); if (type == VOS_ITER_COUUID) { vos_pool = vos_hdl2pool(param->ip_hdl); D_ASSERT(vos_pool != NULL); diff --git a/src/vos/vos_obj.c b/src/vos/vos_obj.c index ee56a01b629..b4e772b1e29 100644 --- a/src/vos/vos_obj.c +++ b/src/vos/vos_obj.c @@ -1690,9 +1690,9 @@ vos_obj_iter_prep(vos_iter_type_t type, vos_iter_param_t *param, struct vos_ts_set *ts_set) { struct vos_obj_iter *oiter; - struct vos_container *cont = vos_hdl2cont(param->ip_hdl); - bool is_sysdb = cont->vc_pool->vp_sysdb; - struct dtx_handle *dth = vos_dth_get(is_sysdb); + struct vos_container *cont = NULL; + bool is_sysdb = false; + struct dtx_handle *dth = NULL; daos_epoch_t bound; int rc; @@ -1700,6 +1700,13 @@ vos_obj_iter_prep(vos_iter_type_t type, vos_iter_param_t *param, if (oiter == NULL) return -DER_NOMEM; + /* ip_hdl is dkey or akey tree open handle for vos_iterate_key() */ + if (!(param->ip_flags & VOS_IT_KEY_TREE)) { + cont = vos_hdl2cont(param->ip_hdl); + is_sysdb = cont->vc_pool->vp_sysdb; + dth = vos_dth_get(is_sysdb); + } + bound = dtx_is_valid_handle(dth) ? dth->dth_epoch_bound : param->ip_epr.epr_hi; oiter->it_iter.it_bound = MAX(bound, param->ip_epr.epr_hi); From 741fb9ff35f6eb55a9564fdf9411127c19833911 Mon Sep 17 00:00:00 2001 From: Niu Yawei Date: Tue, 3 Sep 2024 14:58:34 +0800 Subject: [PATCH 26/26] DAOS-14317 vos: initial changes for the phase2 object pre-load (#15001) - Introduced new durable format 'vos_obj_p2_df' for the md-on-ssd phase2 object, at most 4 evict-able bucket IDs could be stored. - Changed vos_obj_hold() & vos_obj_release() to pin or unpin object respectively. - Changed the private data of VOS dkey/akey/value trees from 'vos_pool' to 'vos_object', the private data will be used for allocating/reserving from the evict-able bucket. - Move the vos_obj_hold() call from vos_update_end() to vos_update_begin() for the phase2 pool, reserve value from the object evict-able bucket. TODO: - Reorg GC code to pre-load objects before starting transaction. - Reorg DTX commit code to pre-load objects before starting transaction. - Reorg multiple-objects tx code to pre-load objects. - Improve engine scheduler to take both non-evict-able & evict-able zones space pressure into account. Signed-off-by: Niu Yawei --- src/common/btree.c | 9 +++- src/include/daos/btree.h | 2 + src/include/daos/mem.h | 2 - src/include/daos_srv/evtree.h | 6 ++- src/tests/ftest/util/telemetry_utils.py | 2 +- src/vos/evtree.c | 10 ++-- src/vos/tests/vts_io.c | 54 +++++++++---------- src/vos/vos_aggregate.c | 2 +- src/vos/vos_gc.c | 4 +- src/vos/vos_internal.h | 48 ++++++++++++++++- src/vos/vos_io.c | 44 ++++++++++------ src/vos/vos_layout.h | 13 +++++ src/vos/vos_obj.c | 11 ++-- src/vos/vos_obj.h | 8 ++- src/vos/vos_obj_cache.c | 70 +++++++++++++++++++++++-- src/vos/vos_obj_index.c | 31 +++++++---- src/vos/vos_query.c | 6 +-- src/vos/vos_tree.c | 44 ++++++++++------ 18 files changed, 270 insertions(+), 96 deletions(-) diff --git a/src/common/btree.c b/src/common/btree.c index 6bf1bdb2b15..0657bf384dc 100644 --- a/src/common/btree.c +++ b/src/common/btree.c @@ -945,8 +945,12 @@ btr_root_alloc(struct btr_context *tcx) struct btr_instance *tins = &tcx->tc_tins; struct btr_root *root; - tins->ti_root_off = umem_zalloc(btr_umm(tcx), - sizeof(struct btr_root)); + if (btr_ops(tcx)->to_node_alloc != NULL) + tins->ti_root_off = btr_ops(tcx)->to_node_alloc(&tcx->tc_tins, + sizeof(struct btr_root)); + else + tins->ti_root_off = umem_zalloc(btr_umm(tcx), sizeof(struct btr_root)); + if (UMOFF_IS_NULL(tins->ti_root_off)) return btr_umm(tcx)->umm_nospc_rc; @@ -3884,6 +3888,7 @@ btr_tree_destroy(struct btr_context *tcx, void *args, bool *destroyed) tcx->tc_tins.ti_root_off, tcx->tc_order); root = tcx->tc_tins.ti_root; + tcx->tc_tins.ti_destroy = 1; if (root && !UMOFF_IS_NULL(root->tr_node)) { /* destroy the root and all descendants */ rc = btr_node_destroy(tcx, root->tr_node, args, &empty); diff --git a/src/include/daos/btree.h b/src/include/daos/btree.h index 24c7b95cbe4..51f7999e47a 100644 --- a/src/include/daos/btree.h +++ b/src/include/daos/btree.h @@ -429,6 +429,8 @@ struct btr_instance { struct btr_root *ti_root; /** Customized operations for the tree */ btr_ops_t *ti_ops; + /** The context is used for tree destroy */ + unsigned int ti_destroy : 1; }; /** diff --git a/src/include/daos/mem.h b/src/include/daos/mem.h index 7a7ca3dcf7c..b94f75b22c4 100644 --- a/src/include/daos/mem.h +++ b/src/include/daos/mem.h @@ -451,8 +451,6 @@ typedef void umem_cache_wait_cb_t(void *arg, uint64_t chkpt_tx, uint64_t *committed_tx); /** - * Write all dirty pages before @wal_tx to MD blob. (XXX: not yet implemented) - * * This function can yield internally, it is called by checkpoint service of upper level stack. * * \param[in] store The umem store diff --git a/src/include/daos_srv/evtree.h b/src/include/daos_srv/evtree.h index 63224259ccc..292c8848c87 100644 --- a/src/include/daos_srv/evtree.h +++ b/src/include/daos_srv/evtree.h @@ -1,5 +1,5 @@ /** - * (C) Copyright 2017-2023 Intel Corporation. + * (C) Copyright 2017-2024 Intel Corporation. * * SPDX-License-Identifier: BSD-2-Clause-Patent */ @@ -70,6 +70,10 @@ struct evt_desc_cbs { struct evt_desc *desc, daos_size_t nob, void *args); void *dc_bio_free_args; + /** + * Argument for allocation. + */ + void *dc_alloc_arg; /** * Availability check, it is for data tracked by DTX undo log. * It is optional, EVTree always treats data extent is available if diff --git a/src/tests/ftest/util/telemetry_utils.py b/src/tests/ftest/util/telemetry_utils.py index b782173e3f4..d42b08470e6 100644 --- a/src/tests/ftest/util/telemetry_utils.py +++ b/src/tests/ftest/util/telemetry_utils.py @@ -424,7 +424,7 @@ class TelemetryUtils(): ENGINE_NVME_INTEL_VENDOR_METRICS ENGINE_MEM_USAGE_METRICS = [ "engine_mem_vos_dtx_cmt_ent_48", - "engine_mem_vos_vos_obj_360", + "engine_mem_vos_vos_obj_384", "engine_mem_vos_vos_lru_size", "engine_mem_dtx_dtx_leader_handle_360"] ENGINE_MEM_TOTAL_USAGE_METRICS = [ diff --git a/src/vos/evtree.c b/src/vos/evtree.c index d635453f8b2..59f8855c3c1 100644 --- a/src/vos/evtree.c +++ b/src/vos/evtree.c @@ -1443,8 +1443,9 @@ evt_node_alloc(struct evt_context *tcx, unsigned int flags, struct evt_node *nd; umem_off_t nd_off; bool leaf = (flags & EVT_NODE_LEAF); + struct vos_object *obj = tcx->tc_desc_cbs.dc_alloc_arg; - nd_off = umem_zalloc(evt_umm(tcx), evt_node_size(tcx, leaf)); + nd_off = vos_obj_alloc(evt_umm(tcx), obj, evt_node_size(tcx, leaf), true); if (UMOFF_IS_NULL(nd_off)) return -DER_NOSPACE; @@ -3249,8 +3250,9 @@ evt_common_insert(struct evt_context *tcx, struct evt_node *nd, } if (leaf) { - umem_off_t desc_off; - uint32_t csum_buf_size = 0; + umem_off_t desc_off; + uint32_t csum_buf_size = 0; + struct vos_object *obj = tcx->tc_desc_cbs.dc_alloc_arg; if (ci_is_valid(&ent->ei_csum)) csum_buf_size = ci_csums_len(ent->ei_csum); @@ -3263,7 +3265,7 @@ evt_common_insert(struct evt_context *tcx, struct evt_node *nd, D_DEBUG(DB_TRACE, "Allocating an extra %d bytes " "for checksum", csum_buf_size); } - desc_off = umem_zalloc(evt_umm(tcx), desc_size); + desc_off = vos_obj_alloc(evt_umm(tcx), obj, desc_size, true); if (UMOFF_IS_NULL(desc_off)) return -DER_NOSPACE; diff --git a/src/vos/tests/vts_io.c b/src/vos/tests/vts_io.c index 9cb37e41700..08d097b35ed 100644 --- a/src/vos/tests/vts_io.c +++ b/src/vos/tests/vts_io.c @@ -898,7 +898,7 @@ io_update_and_fetch_dkey(struct io_test_args *arg, daos_epoch_t update_epoch, static inline int hold_obj(struct vos_container *cont, daos_unit_oid_t oid, daos_epoch_range_t *epr, daos_epoch_t bound, uint64_t flags, uint32_t intent, struct vos_object **obj_p, - struct vos_ts_set *ts_set) + struct vos_ts_set *ts_set, struct umem_instance *umm) { int rc; @@ -908,7 +908,16 @@ hold_obj(struct vos_container *cont, daos_unit_oid_t oid, daos_epoch_range_t *ep if (flags & VOS_OBJ_CREATE) { assert_ptr_not_equal(*obj_p, NULL); + + if (umm != NULL) { + rc = umem_tx_begin(umm, NULL); + assert_rc_equal(rc, 0); + } + rc = vos_obj_incarnate(*obj_p, epr, bound, flags, intent, ts_set); + + if (umm != NULL) + rc = umem_tx_end(umm, rc); } return rc; @@ -926,7 +935,8 @@ hold_objects(struct vos_object **objs, daos_handle_t *coh, daos_unit_oid_t *oid, hold_flags |= VOS_OBJ_VISIBLE; for (i = start; i < end; i++) { rc = hold_obj(vos_hdl2cont(*coh), *oid, &epr, 0, hold_flags, - no_create ? DAOS_INTENT_DEFAULT : DAOS_INTENT_UPDATE, &objs[i], 0); + no_create ? DAOS_INTENT_DEFAULT : DAOS_INTENT_UPDATE, + &objs[i], 0, NULL); if (rc != exp_rc) return 1; } @@ -1006,82 +1016,72 @@ io_obj_cache_test(void **state) ummg = vos_cont2umm(vos_hdl2cont(ctx->tc_co_hdl)); umml = vos_cont2umm(vos_hdl2cont(l_coh)); - rc = umem_tx_begin(ummg, NULL); - assert_rc_equal(rc, 0); rc = hold_obj(vos_hdl2cont(ctx->tc_co_hdl), oids[0], &epr, 0, - VOS_OBJ_CREATE | VOS_OBJ_VISIBLE, DAOS_INTENT_UPDATE, &objs[0], 0); + VOS_OBJ_CREATE | VOS_OBJ_VISIBLE, DAOS_INTENT_UPDATE, &objs[0], 0, ummg); assert_rc_equal(rc, 0); /** Hold object for discard */ rc = hold_obj(vos_hdl2cont(ctx->tc_co_hdl), oids[0], &epr, 0, VOS_OBJ_DISCARD, - DAOS_INTENT_DISCARD, &obj1, 0); + DAOS_INTENT_DISCARD, &obj1, 0, ummg); assert_rc_equal(rc, 0); /** Second discard should fail */ rc = hold_obj(vos_hdl2cont(ctx->tc_co_hdl), oids[0], &epr, 0, VOS_OBJ_DISCARD, - DAOS_INTENT_DISCARD, &obj2, 0); + DAOS_INTENT_DISCARD, &obj2, 0, ummg); assert_rc_equal(rc, -DER_BUSY); /** Should prevent simultaneous aggregation */ rc = hold_obj(vos_hdl2cont(ctx->tc_co_hdl), oids[0], &epr, 0, VOS_OBJ_AGGREGATE, - DAOS_INTENT_PURGE, &obj2, 0); + DAOS_INTENT_PURGE, &obj2, 0, ummg); assert_rc_equal(rc, -DER_BUSY); /** Should prevent simultaneous hold for create as well */ rc = hold_obj(vos_hdl2cont(ctx->tc_co_hdl), oids[0], &epr, 0, - VOS_OBJ_CREATE | VOS_OBJ_VISIBLE, DAOS_INTENT_UPDATE, &obj2, 0); + VOS_OBJ_CREATE | VOS_OBJ_VISIBLE, DAOS_INTENT_UPDATE, &obj2, + 0, ummg); assert_rc_equal(rc, -DER_UPDATE_AGAIN); /** Need to be able to hold for read though or iteration won't work */ rc = hold_obj(vos_hdl2cont(ctx->tc_co_hdl), oids[0], &epr, 0, VOS_OBJ_VISIBLE, - DAOS_INTENT_DEFAULT, &obj2, 0); + DAOS_INTENT_DEFAULT, &obj2, 0, ummg); vos_obj_release(obj2, 0, false); vos_obj_release(obj1, VOS_OBJ_DISCARD, false); /** Hold object for aggregation */ rc = hold_obj(vos_hdl2cont(ctx->tc_co_hdl), oids[0], &epr, 0, VOS_OBJ_AGGREGATE, - DAOS_INTENT_PURGE, &obj1, 0); + DAOS_INTENT_PURGE, &obj1, 0, ummg); assert_rc_equal(rc, 0); /** Discard should fail */ rc = hold_obj(vos_hdl2cont(ctx->tc_co_hdl), oids[0], &epr, 0, VOS_OBJ_DISCARD, - DAOS_INTENT_DISCARD, &obj2, 0); + DAOS_INTENT_DISCARD, &obj2, 0, ummg); assert_rc_equal(rc, -DER_BUSY); /** Second aggregation should fail */ rc = hold_obj(vos_hdl2cont(ctx->tc_co_hdl), oids[0], &epr, 0, VOS_OBJ_AGGREGATE, - DAOS_INTENT_PURGE, &obj2, 0); + DAOS_INTENT_PURGE, &obj2, 0, ummg); assert_rc_equal(rc, -DER_BUSY); /** Simultaneous create should work */ rc = hold_obj(vos_hdl2cont(ctx->tc_co_hdl), oids[0], &epr, 0, - VOS_OBJ_CREATE | VOS_OBJ_VISIBLE, DAOS_INTENT_UPDATE, &obj2, 0); + VOS_OBJ_CREATE | VOS_OBJ_VISIBLE, DAOS_INTENT_UPDATE, &obj2, 0, ummg); assert_rc_equal(rc, 0); vos_obj_release(obj2, 0, false); /** Need to be able to hold for read though or iteration won't work */ rc = hold_obj(vos_hdl2cont(ctx->tc_co_hdl), oids[0], &epr, 0, VOS_OBJ_VISIBLE, - DAOS_INTENT_DEFAULT, &obj2, 0); + DAOS_INTENT_DEFAULT, &obj2, 0, ummg); vos_obj_release(obj2, 0, false); vos_obj_release(obj1, VOS_OBJ_AGGREGATE, false); /** Now that other one is done, this should work */ rc = hold_obj(vos_hdl2cont(ctx->tc_co_hdl), oids[0], &epr, 0, VOS_OBJ_DISCARD, - DAOS_INTENT_DISCARD, &obj2, 0); + DAOS_INTENT_DISCARD, &obj2, 0, ummg); assert_rc_equal(rc, 0); vos_obj_release(obj2, VOS_OBJ_DISCARD, false); - rc = umem_tx_end(ummg, 0); - assert_rc_equal(rc, 0); - vos_obj_release(objs[0], 0, false); - rc = umem_tx_begin(umml, NULL); - assert_rc_equal(rc, 0); - rc = hold_obj(vos_hdl2cont(l_coh), oids[1], &epr, 0, - VOS_OBJ_CREATE | VOS_OBJ_VISIBLE, DAOS_INTENT_UPDATE, &objs[0], 0); + VOS_OBJ_CREATE | VOS_OBJ_VISIBLE, DAOS_INTENT_UPDATE, &objs[0], 0, umml); assert_rc_equal(rc, 0); vos_obj_release(objs[0], 0, false); - rc = umem_tx_end(umml, 0); - assert_rc_equal(rc, 0); - rc = hold_objects(objs, &ctx->tc_co_hdl, &oids[0], 0, 10, true, 0); assert_int_equal(rc, 0); @@ -1091,7 +1091,7 @@ io_obj_cache_test(void **state) rc = hold_objects(objs, &l_coh, &oids[1], 10, 15, true, 0); assert_int_equal(rc, 0); rc = hold_obj(vos_hdl2cont(l_coh), oids[1], &epr, 0, VOS_OBJ_VISIBLE, - DAOS_INTENT_DEFAULT, &objs[16], 0); + DAOS_INTENT_DEFAULT, &objs[16], 0, NULL); assert_rc_equal(rc, 0); vos_obj_release(objs[16], 0, false); diff --git a/src/vos/vos_aggregate.c b/src/vos/vos_aggregate.c index 65d70dd7762..5064e74d730 100644 --- a/src/vos/vos_aggregate.c +++ b/src/vos/vos_aggregate.c @@ -984,7 +984,7 @@ reserve_segment(struct vos_object *obj, struct agg_io_context *io, if (vos_io_scm(vos_obj2pool(obj), DAOS_IOD_ARRAY, size, VOS_IOS_AGGREGATION)) { /** Store on SCM */ - off = vos_reserve_scm(obj->obj_cont, io->ic_rsrvd_scm, size); + off = vos_reserve_scm(obj->obj_cont, io->ic_rsrvd_scm, size, obj); if (UMOFF_IS_NULL(off)) { now = daos_gettime_coarse(); if (now - obj->obj_cont->vc_agg_nospc_ts > VOS_NOSPC_ERROR_INTVL) { diff --git a/src/vos/vos_gc.c b/src/vos/vos_gc.c index 0937b883f33..b90cd5d1199 100644 --- a/src/vos/vos_gc.c +++ b/src/vos/vos_gc.c @@ -1,5 +1,5 @@ /** - * (C) Copyright 2019-2023 Intel Corporation. + * (C) Copyright 2019-2024 Intel Corporation. * * SPDX-License-Identifier: BSD-2-Clause-Patent */ @@ -115,7 +115,7 @@ gc_drain_evt(struct vos_gc *gc, struct vos_pool *pool, daos_handle_t coh, daos_handle_t toh; int rc; - vos_evt_desc_cbs_init(&cbs, pool, coh); + vos_evt_desc_cbs_init(&cbs, pool, coh, NULL); rc = evt_open(root, &pool->vp_uma, &cbs, &toh); if (rc == -DER_NONEXIST) { *empty = true; diff --git a/src/vos/vos_internal.h b/src/vos/vos_internal.h index f6a74fce7e6..72275c1d41c 100644 --- a/src/vos/vos_internal.h +++ b/src/vos/vos_internal.h @@ -1258,7 +1258,7 @@ vos_bio_addr_free(struct vos_pool *pool, bio_addr_t *addr, daos_size_t nob); void vos_evt_desc_cbs_init(struct evt_desc_cbs *cbs, struct vos_pool *pool, - daos_handle_t coh); + daos_handle_t coh, struct vos_object *obj); int vos_tx_begin(struct dtx_handle *dth, struct umem_instance *umm, bool is_sysdb); @@ -1312,7 +1312,7 @@ vos_dedup_invalidate(struct vos_pool *pool); umem_off_t vos_reserve_scm(struct vos_container *cont, struct umem_rsrvd_act *rsrvd_scm, - daos_size_t size); + daos_size_t size, struct vos_object *obj); int vos_publish_scm(struct umem_instance *umm, struct umem_rsrvd_act *rsrvd_scm, bool publish); int @@ -1329,6 +1329,12 @@ vos_pool2umm(struct vos_pool *pool) return &pool->vp_umm; } +static inline struct umem_store * +vos_pool2store(struct vos_pool *pool) +{ + return &pool->vp_umm.umm_pool->up_store; +} + static inline struct umem_instance * vos_cont2umm(struct vos_container *cont) { @@ -1844,4 +1850,42 @@ vos_io_scm(struct vos_pool *pool, daos_iod_type_t type, daos_size_t size, enum v int vos_insert_oid(struct dtx_handle *dth, struct vos_container *cont, daos_unit_oid_t *oid); +static inline bool +vos_pool_is_p2(struct vos_pool *pool) +{ + struct umem_store *store = vos_pool2store(pool); + + return store->store_type == DAOS_MD_BMEM_V2; +} + +static inline umem_off_t +vos_obj_alloc(struct umem_instance *umm, struct vos_object *obj, size_t size, bool zeroing) +{ + + if (obj != NULL && vos_pool_is_p2(vos_obj2pool(obj))) { + D_ASSERT(obj->obj_bkt_allot == 1); + if (zeroing) + return umem_zalloc_from_bucket(umm, size, obj->obj_bkt_ids[0]); + + return umem_alloc_from_bucket(umm, size, obj->obj_bkt_ids[0]); + } + + if (zeroing) + return umem_zalloc(umm, size); + + return umem_alloc(umm, size); +} + +static inline umem_off_t +vos_obj_reserve(struct umem_instance *umm, struct vos_object *obj, + struct umem_rsrvd_act *rsrvd_scm, daos_size_t size) +{ + if (obj != NULL && vos_pool_is_p2(vos_obj2pool(obj))) { + D_ASSERT(obj->obj_bkt_allot == 1); + return umem_reserve_from_bucket(umm, rsrvd_scm, size, obj->obj_bkt_ids[0]); + } + + return umem_reserve(umm, rsrvd_scm, size); +} + #endif /* __VOS_INTERNAL_H__ */ diff --git a/src/vos/vos_io.c b/src/vos/vos_io.c index 4d452f50d6a..e1435801ee5 100644 --- a/src/vos/vos_io.c +++ b/src/vos/vos_io.c @@ -2040,17 +2040,16 @@ vos_recx2irec_size(daos_size_t rsize, struct dcs_csum_info *csum) umem_off_t vos_reserve_scm(struct vos_container *cont, struct umem_rsrvd_act *rsrvd_scm, - daos_size_t size) + daos_size_t size, struct vos_object *obj) { - umem_off_t umoff; + umem_off_t umoff; + struct umem_instance *umm = vos_cont2umm(cont); D_ASSERT(size > 0); - - if (vos_cont2umm(cont)->umm_ops->mo_reserve != NULL) { - umoff = umem_reserve(vos_cont2umm(cont), rsrvd_scm, size); - } else { - umoff = umem_alloc(vos_cont2umm(cont), size); - } + if (umm->umm_ops->mo_reserve != NULL) + umoff = vos_obj_reserve(umm, obj, rsrvd_scm, size); + else + umoff = vos_obj_alloc(umm, obj, size, false); return umoff; } @@ -2096,7 +2095,7 @@ reserve_space(struct vos_io_context *ioc, uint16_t media, daos_size_t size, if (media == DAOS_MEDIA_SCM) { umem_off_t umoff; - umoff = vos_reserve_scm(ioc->ic_cont, ioc->ic_rsrvd_scm, size); + umoff = vos_reserve_scm(ioc->ic_cont, ioc->ic_rsrvd_scm, size, ioc->ic_obj); if (!UMOFF_IS_NULL(umoff)) { ioc->ic_umoffs[ioc->ic_umoffs_cnt] = umoff; ioc->ic_umoffs_cnt++; @@ -2430,13 +2429,16 @@ vos_update_end(daos_handle_t ioh, uint32_t pm_ver, daos_key_t *dkey, int err, if (err != 0) goto abort; - err = vos_ts_set_add(ioc->ic_ts_set, ioc->ic_cont->vc_ts_idx, NULL, 0); - D_ASSERT(err == 0); + /* The object is already held for md-on-ssd phase2 pool */ + if (ioc->ic_obj == NULL) { + err = vos_ts_set_add(ioc->ic_ts_set, ioc->ic_cont->vc_ts_idx, NULL, 0); + D_ASSERT(err == 0); - err = vos_obj_hold(ioc->ic_cont, ioc->ic_oid, &ioc->ic_epr, ioc->ic_bound, - flags, DAOS_INTENT_UPDATE, &ioc->ic_obj, ioc->ic_ts_set); - if (err != 0) - goto abort; + err = vos_obj_hold(ioc->ic_cont, ioc->ic_oid, &ioc->ic_epr, ioc->ic_bound, + flags, DAOS_INTENT_UPDATE, &ioc->ic_obj, ioc->ic_ts_set); + if (err != 0) + goto abort; + } err = vos_tx_begin(dth, umem, ioc->ic_cont->vc_pool->vp_sysdb); if (err != 0) @@ -2612,6 +2614,18 @@ vos_update_begin(daos_handle_t coh, daos_unit_oid_t oid, daos_epoch_t epoch, goto error; } + /* Hold the object for md-on-ssd phase2 pool */ + if (vos_pool_is_p2(vos_cont2pool(ioc->ic_cont))) { + rc = vos_ts_set_add(ioc->ic_ts_set, ioc->ic_cont->vc_ts_idx, NULL, 0); + D_ASSERT(rc == 0); + + rc = vos_obj_hold(ioc->ic_cont, ioc->ic_oid, &ioc->ic_epr, ioc->ic_bound, + VOS_OBJ_VISIBLE | VOS_OBJ_CREATE, DAOS_INTENT_UPDATE, + &ioc->ic_obj, ioc->ic_ts_set); + if (rc != 0) + goto error; + } + rc = dkey_update_begin(ioc); if (rc != 0) { D_ERROR(DF_UOID ": dkey update begin failed. " DF_RC "\n", DP_UOID(oid), DP_RC(rc)); diff --git a/src/vos/vos_layout.h b/src/vos/vos_layout.h index 72459544c27..0400a351175 100644 --- a/src/vos/vos_layout.h +++ b/src/vos/vos_layout.h @@ -377,4 +377,17 @@ struct vos_obj_df { struct btr_root vo_tree; }; +#define VOS_OBJ_BKTS_MAX 4 + +/* + * VOS object durable format for md-on-ssd phase2. The size is fit to the 128 bytes + * slab (see slab_map[] defined in mem.c). + */ +struct vos_obj_p2_df { + struct vos_obj_df p2_obj_df; + uint32_t p2_bkt_ids[VOS_OBJ_BKTS_MAX]; + uint64_t p2_reserved; +}; +D_CASSERT(sizeof(struct vos_obj_p2_df) == D_ALIGNUP(sizeof(struct vos_obj_df), 32)); + #endif diff --git a/src/vos/vos_obj.c b/src/vos/vos_obj.c index b4e772b1e29..4101c3fa261 100644 --- a/src/vos/vos_obj.c +++ b/src/vos/vos_obj.c @@ -1065,7 +1065,8 @@ key_iter_fetch_root(struct vos_obj_iter *oiter, vos_iter_type_t type, * subtree */ if (krec->kr_bmap & KREC_BF_EVT) { - vos_evt_desc_cbs_init(&cbs, vos_obj2pool(obj), vos_cont2hdl(obj->obj_cont)); + vos_evt_desc_cbs_init(&cbs, vos_obj2pool(obj), vos_cont2hdl(obj->obj_cont), + obj); rc = evt_open(&krec->kr_evt, info->ii_uma, &cbs, &info->ii_tree_hdl); if (rc) { D_DEBUG(DB_TRACE, @@ -1077,7 +1078,7 @@ key_iter_fetch_root(struct vos_obj_iter *oiter, vos_iter_type_t type, info->ii_fake_akey_flag = VOS_IT_DKEY_EV; } else { rc = dbtree_open_inplace_ex(&krec->kr_btr, info->ii_uma, - vos_cont2hdl(obj->obj_cont), vos_obj2pool(obj), + vos_cont2hdl(obj->obj_cont), obj, &info->ii_tree_hdl); if (rc) { D_DEBUG(DB_TRACE, @@ -2037,7 +2038,7 @@ vos_obj_akey_iter_nested_prep(vos_iter_type_t type, struct vos_iter_info *info, } rc = dbtree_open_inplace_ex(info->ii_btr, info->ii_uma, vos_cont2hdl(obj->obj_cont), - vos_obj2pool(obj), &toh); + obj, &toh); if (rc) { D_DEBUG(DB_TRACE, "Failed to open tree for iterator:" @@ -2094,7 +2095,7 @@ vos_obj_iter_sv_nested_prep(vos_iter_type_t type, struct vos_iter_info *info, } rc = dbtree_open_inplace_ex(info->ii_btr, info->ii_uma, vos_cont2hdl(obj->obj_cont), - vos_obj2pool(obj), &toh); + obj, &toh); if (rc) { D_DEBUG(DB_TRACE, "Failed to open tree for iterator:" @@ -2154,7 +2155,7 @@ vos_obj_ev_iter_nested_prep(vos_iter_type_t type, struct vos_iter_info *info, goto prepare; } - vos_evt_desc_cbs_init(&cbs, vos_obj2pool(obj), vos_cont2hdl(obj->obj_cont)); + vos_evt_desc_cbs_init(&cbs, vos_obj2pool(obj), vos_cont2hdl(obj->obj_cont), obj); rc = evt_open(info->ii_evt, info->ii_uma, &cbs, &toh); if (rc) { D_DEBUG(DB_TRACE, diff --git a/src/vos/vos_obj.h b/src/vos/vos_obj.h index 2ccc8d71988..7b254ad2d75 100644 --- a/src/vos/vos_obj.h +++ b/src/vos/vos_obj.h @@ -47,12 +47,18 @@ struct vos_object { struct vos_obj_df *obj_df; /** backref to container */ struct vos_container *obj_cont; + /* Handle for the pinned object */ + struct umem_pin_handle *obj_pin_hdl; + /** Bucket IDs for the object */ + uint32_t obj_bkt_ids[VOS_OBJ_BKTS_MAX]; /** nobody should access this object */ bool obj_zombie; /** Object is held for discard */ uint32_t obj_discard : 1, /** If non-zero, object is held for aggregation */ - obj_aggregate : 1; + obj_aggregate : 1, + /** Evict-able bucket is already allocated */ + obj_bkt_allot : 1; }; enum { diff --git a/src/vos/vos_obj_cache.c b/src/vos/vos_obj_cache.c index 8845eae0085..502aafa6306 100644 --- a/src/vos/vos_obj_cache.c +++ b/src/vos/vos_obj_cache.c @@ -245,12 +245,58 @@ obj_get(struct daos_lru_cache *occ, struct vos_container *cont, daos_unit_oid_t return rc; } +static inline void +vos_obj_unpin(struct vos_object *obj) +{ + struct vos_pool *pool = vos_obj2pool(obj); + struct umem_store *store = vos_pool2store(pool); + + if (obj->obj_pin_hdl != NULL) { + umem_cache_unpin(store, obj->obj_pin_hdl); + obj->obj_pin_hdl = NULL; + } +} + +/* Support single evict-able bucket for this moment */ +static inline int +vos_obj_pin(struct vos_object *obj) +{ + struct vos_pool *pool = vos_obj2pool(obj); + struct umem_store *store = vos_pool2store(pool); + struct umem_cache_range rg; + + if (!vos_pool_is_p2(pool)) + return 0; + + if (!obj->obj_bkt_allot) { + if (!obj->obj_df) { + /* TODO: Revise all vos_obj_hold() callers to move it out of tx! */ + /* obj->obj_bkt_ids[0] = umem_allot_mb_evictable(vos_pool2umm(pool), 0); */ + obj->obj_bkt_ids[0] = UMEM_DEFAULT_MBKT_ID; + } else { + struct vos_obj_p2_df *p2 = (struct vos_obj_p2_df *)obj->obj_df; + + obj->obj_bkt_ids[0] = p2->p2_bkt_ids[0]; + } + obj->obj_bkt_allot = 1; + } + + D_ASSERT(obj->obj_pin_hdl == NULL); + if (obj->obj_bkt_ids[0] == UMEM_DEFAULT_MBKT_ID) + return 0; + + rg.cr_off = umem_get_mb_base_offset(vos_pool2umm(pool), obj->obj_bkt_ids[0]); + rg.cr_size = store->cache->ca_page_sz; + + return umem_cache_pin(store, &rg, 1, false, &obj->obj_pin_hdl); +} + static inline void obj_release(struct daos_lru_cache *occ, struct vos_object *obj, bool evict) { D_ASSERT(obj != NULL); - /* TODO: Unpin the object in md-on-ssd phase II */ + vos_obj_unpin(obj); if (obj == &obj_local) { clean_object(obj); @@ -301,6 +347,8 @@ cache_object(struct daos_lru_cache *occ, struct vos_object **objp) obj_new->obj_sync_epoch = obj_local.obj_sync_epoch; obj_new->obj_df = obj_local.obj_df; obj_new->obj_zombie = obj_local.obj_zombie; + obj_new->obj_bkt_allot = obj_local.obj_bkt_allot; + obj_new->obj_pin_hdl = obj_local.obj_pin_hdl; obj_local.obj_toh = DAOS_HDL_INVAL; obj_local.obj_ih = DAOS_HDL_INVAL; @@ -363,7 +411,9 @@ vos_obj_check_discard(struct vos_container *cont, daos_unit_oid_t oid, uint64_t if (rc) return rc; - /* TODO: Pin object in memory */ + rc = vos_obj_pin(obj); + if (rc) + return rc; if (check_discard(obj, flags)) /* Update request will retry with this error */ @@ -453,6 +503,10 @@ vos_obj_hold(struct vos_container *cont, daos_unit_oid_t oid, daos_epoch_range_t D_ASSERT(cont != NULL); D_ASSERT(cont->vc_pool); D_ASSERT(obj_p != NULL); + + /* TODO: Revise all vos_obj_hold() callers to move it out of tx! */ + /* D_ASSERT(!vos_pool_is_p2(cont->vc_pool) || umem_tx_none(vos_pool2umm(cont->vc_pool))); */ + *obj_p = NULL; occ = vos_obj_cache_get(cont->vc_pool->vp_sysdb); @@ -507,8 +561,16 @@ vos_obj_hold(struct vos_container *cont, daos_unit_oid_t oid, daos_epoch_range_t D_ASSERT(tmprc == 0); /* Non-zero only valid for akey */ } - /* TODO: Pin the object in memory in md-on-ssd phase II. Revise the 'obj_local' implementation - * then, since this function could yield. */ + /* For md-on-ssd phase2 pool, add object to cache before yield in vos_obj_pin() */ + if (obj == &obj_local && vos_pool_is_p2(cont->vc_pool)) { + rc = cache_object(occ, &obj); + if (rc != 0) + goto failed; + } + + rc = vos_obj_pin(obj); + if (rc) + goto failed; /* It's done for DAOS_INTENT_UPDATE or DAOS_INTENT_PUNCH or DAOS_INTENT_KILL */ if (intent == DAOS_INTENT_UPDATE || intent == DAOS_INTENT_PUNCH || diff --git a/src/vos/vos_obj_index.c b/src/vos/vos_obj_index.c index 72870cfd76e..e6111cd0746 100644 --- a/src/vos/vos_obj_index.c +++ b/src/vos/vos_obj_index.c @@ -47,7 +47,8 @@ oi_hkey_size(void) static int oi_rec_msize(int alloc_overhead) { - return alloc_overhead + sizeof(struct vos_obj_df); + /* This function is only used for metadata overhead estimation. */ + return alloc_overhead + D_ALIGNUP(sizeof(struct vos_obj_df), 32); } static void @@ -67,6 +68,15 @@ oi_hkey_cmp(struct btr_instance *tins, struct btr_record *rec, void *hkey) return dbtree_key_cmp_rc(memcmp(oid1, oid2, sizeof(*oid1))); } +static inline unsigned int +vos_obj_df_size(struct vos_pool *pool) +{ + if (vos_pool_is_p2(pool)) + return sizeof(struct vos_obj_p2_df); + + return sizeof(struct vos_obj_df); +} + static int oi_rec_alloc(struct btr_instance *tins, d_iov_t *key_iov, d_iov_t *val_iov, struct btr_record *rec, d_iov_t *val_out) @@ -76,10 +86,11 @@ oi_rec_alloc(struct btr_instance *tins, d_iov_t *key_iov, struct vos_obj_df *obj; daos_unit_oid_t *key; umem_off_t obj_off; + struct vos_pool *pool = (struct vos_pool *)tins->ti_priv; int rc; /* Allocate a PMEM value of type vos_obj_df */ - obj_off = umem_zalloc(&tins->ti_umm, sizeof(struct vos_obj_df)); + obj_off = umem_zalloc(&tins->ti_umm, vos_obj_df_size(pool)); if (UMOFF_IS_NULL(obj_off)) return -DER_NOSPACE; @@ -100,11 +111,11 @@ oi_rec_alloc(struct btr_instance *tins, d_iov_t *key_iov, } else { struct vos_obj_df *new_obj = val_out->iov_buf; - memcpy(obj, new_obj, sizeof(*obj)); + memcpy(obj, new_obj, vos_obj_df_size(pool)); obj->vo_id = *key; } - d_iov_set(val_iov, obj, sizeof(struct vos_obj_df)); + d_iov_set(val_iov, obj, vos_obj_df_size(pool)); rec->rec_off = obj_off; /* For new created object, commit it synchronously to reduce @@ -176,7 +187,7 @@ oi_rec_fetch(struct btr_instance *tins, struct btr_record *rec, DP_UOID(obj->vo_id), rec->rec_off); D_ASSERT(val_iov != NULL); - d_iov_set(val_iov, obj, sizeof(struct vos_obj_df)); + d_iov_set(val_iov, obj, vos_obj_df_size((struct vos_pool *)tins->ti_priv)); return 0; } @@ -504,7 +515,7 @@ oi_iter_nested_tree_fetch(struct vos_iterator *iter, vos_iter_type_t type, return rc; } - D_ASSERT(rec_iov.iov_len == sizeof(struct vos_obj_df)); + D_ASSERT(rec_iov.iov_len == vos_obj_df_size(oiter->oit_cont->vc_pool)); obj = (struct vos_obj_df *)rec_iov.iov_buf; rc = oi_iter_ilog_check(obj, oiter, &info->ii_epr, false); @@ -610,7 +621,7 @@ oi_iter_match_probe(struct vos_iterator *iter, daos_anchor_t *anchor, uint32_t f goto failed; } - D_ASSERT(iov.iov_len == sizeof(struct vos_obj_df)); + D_ASSERT(iov.iov_len == vos_obj_df_size(oiter->oit_cont->vc_pool)); obj = (struct vos_obj_df *)iov.iov_buf; if (iter->it_filter_cb != NULL && (flags & VOS_ITER_PROBE_AGAIN) == 0) { @@ -767,7 +778,7 @@ oi_iter_fetch(struct vos_iterator *iter, vos_iter_entry_t *it_entry, return rc; } - D_ASSERT(rec_iov.iov_len == sizeof(struct vos_obj_df)); + D_ASSERT(rec_iov.iov_len == vos_obj_df_size(oiter->oit_cont->vc_pool)); return oi_iter_fill(rec_iov.iov_buf, oiter, false, it_entry); } @@ -818,7 +829,7 @@ oi_iter_check_punch(daos_handle_t ih) "Probe should be done before aggregation\n"); if (rc != 0) return rc; - D_ASSERT(rec_iov.iov_len == sizeof(struct vos_obj_df)); + D_ASSERT(rec_iov.iov_len == vos_obj_df_size(oiter->oit_cont->vc_pool)); obj = (struct vos_obj_df *)rec_iov.iov_buf; oid = obj->vo_id; @@ -873,7 +884,7 @@ oi_iter_aggregate(daos_handle_t ih, bool range_discard) "Probe should be done before aggregation\n"); if (rc != 0) return rc; - D_ASSERT(rec_iov.iov_len == sizeof(struct vos_obj_df)); + D_ASSERT(rec_iov.iov_len == vos_obj_df_size(oiter->oit_cont->vc_pool)); obj = (struct vos_obj_df *)rec_iov.iov_buf; oid = obj->vo_id; diff --git a/src/vos/vos_query.c b/src/vos/vos_query.c index e924e4016b6..b4d414012e5 100644 --- a/src/vos/vos_query.c +++ b/src/vos/vos_query.c @@ -162,7 +162,7 @@ query_normal_recx(struct open_query *query, daos_recx_t *recx) uint32_t inob; - vos_evt_desc_cbs_init(&cbs, query->qt_pool, query->qt_coh); + vos_evt_desc_cbs_init(&cbs, query->qt_pool, query->qt_coh, query->qt_obj); rc = evt_open(query->qt_recx_root, &query->qt_pool->vp_uma, &cbs, &toh); if (rc != 0) return rc; @@ -344,7 +344,7 @@ query_ec_recx(struct open_query *query, daos_recx_t *recx) bool prefresh = true; - vos_evt_desc_cbs_init(&cbs, query->qt_pool, query->qt_coh); + vos_evt_desc_cbs_init(&cbs, query->qt_pool, query->qt_coh, query->qt_obj); rc = evt_open(query->qt_recx_root, &query->qt_pool->vp_uma, &cbs, &toh); if (rc != 0) return rc; @@ -517,7 +517,7 @@ open_and_query_key(struct open_query *query, daos_key_t *key, return -DER_NONEXIST; rc = dbtree_open_inplace_ex(to_open, &query->qt_pool->vp_uma, - query->qt_coh, query->qt_pool, toh); + query->qt_coh, query->qt_obj, toh); if (rc != 0) return rc; diff --git a/src/vos/vos_tree.c b/src/vos/vos_tree.c index c36fcaa88c5..b2245e626d9 100644 --- a/src/vos/vos_tree.c +++ b/src/vos/vos_tree.c @@ -154,8 +154,10 @@ ktr_hkey_gen(struct btr_instance *tins, d_iov_t *key_iov, void *hkey) { struct ktr_hkey *kkey = (struct ktr_hkey *)hkey; struct umem_pool *umm_pool = tins->ti_umm.umm_pool; - struct vos_pool *pool = (struct vos_pool *)tins->ti_priv; + struct vos_pool *pool; + D_ASSERT(tins->ti_destroy == 0); + pool = vos_obj2pool(tins->ti_priv); D_ASSERT(key_iov->iov_len < pool->vp_pool_df->pd_scm_sz); hkey_common_gen(key_iov, hkey); @@ -255,7 +257,8 @@ ktr_rec_alloc(struct btr_instance *tins, d_iov_t *key_iov, rbund = iov2rec_bundle(val_iov); - rec->rec_off = umem_zalloc(&tins->ti_umm, vos_krec_size(rbund)); + D_ASSERT(tins->ti_destroy == 0); + rec->rec_off = vos_obj_alloc(&tins->ti_umm, tins->ti_priv, vos_krec_size(rbund), true); if (UMOFF_IS_NULL(rec->rec_off)) return -DER_NOSPACE; @@ -298,11 +301,15 @@ ktr_rec_free(struct btr_instance *tins, struct btr_record *rec, void *args) if (rc != 0) return rc; - pool = (struct vos_pool *)tins->ti_priv; + D_ASSERT(tins->ti_priv); + if (tins->ti_destroy) + pool = (struct vos_pool *)tins->ti_priv; + else + pool = vos_obj2pool(tins->ti_priv); + vos_ilog_ts_evict(&krec->kr_ilog, (krec->kr_bmap & KREC_BF_DKEY) ? VOS_TS_TYPE_DKEY : VOS_TS_TYPE_AKEY, pool->vp_sysdb); - D_ASSERT(tins->ti_priv); gc = (krec->kr_bmap & KREC_BF_DKEY) ? GC_DKEY : GC_AKEY; coh = vos_cont2hdl(args); return gc_add_item(pool, coh, gc, rec->rec_off, 0); @@ -351,7 +358,8 @@ ktr_rec_update(struct btr_instance *tins, struct btr_record *rec, static umem_off_t ktr_node_alloc(struct btr_instance *tins, int size) { - return umem_zalloc(&tins->ti_umm, size); + D_ASSERT(tins->ti_destroy == 0); + return vos_obj_alloc(&tins->ti_umm, tins->ti_priv, size, true); } static btr_ops_t key_btr_ops = { @@ -620,9 +628,13 @@ svt_rec_free_internal(struct btr_instance *tins, struct btr_record *rec, if (!overwrite) { /* SCM value is stored together with vos_irec_df */ if (addr->ba_type == DAOS_MEDIA_NVME) { - struct vos_pool *pool = tins->ti_priv; + struct vos_pool *pool; - D_ASSERT(pool != NULL); + D_ASSERT(tins->ti_priv != NULL); + if (tins->ti_destroy) + pool = (struct vos_pool *)tins->ti_priv; + else + pool = vos_obj2pool(tins->ti_priv); rc = vos_bio_addr_free(pool, addr, irec->ir_size); if (rc) return rc; @@ -714,7 +726,7 @@ svt_check_availability(struct btr_instance *tins, struct btr_record *rec, static umem_off_t svt_node_alloc(struct btr_instance *tins, int size) { - return umem_zalloc(&tins->ti_umm, size); + return vos_obj_alloc(&tins->ti_umm, tins->ti_priv, size, true); } static btr_ops_t singv_btr_ops = { @@ -802,12 +814,13 @@ evt_dop_log_del(struct umem_instance *umm, daos_epoch_t epoch, } void -vos_evt_desc_cbs_init(struct evt_desc_cbs *cbs, struct vos_pool *pool, - daos_handle_t coh) +vos_evt_desc_cbs_init(struct evt_desc_cbs *cbs, struct vos_pool *pool, daos_handle_t coh, + struct vos_object *obj) { /* NB: coh is not required for destroy */ cbs->dc_bio_free_cb = evt_dop_bio_free; cbs->dc_bio_free_args = (void *)pool; + cbs->dc_alloc_arg = (void *)obj; cbs->dc_log_status_cb = evt_dop_log_status; cbs->dc_log_status_args = (void *)(unsigned long)coh.cookie; cbs->dc_log_add_cb = evt_dop_log_add; @@ -829,7 +842,7 @@ tree_open_create(struct vos_object *obj, enum vos_tree_class tclass, int flags, int unexpected_flag; int rc = 0; - vos_evt_desc_cbs_init(&cbs, pool, coh); + vos_evt_desc_cbs_init(&cbs, pool, coh, obj); if ((krec->kr_bmap & (KREC_BF_BTR | KREC_BF_EVT)) == 0) goto create; @@ -855,7 +868,7 @@ tree_open_create(struct vos_object *obj, enum vos_tree_class tclass, int flags, if (expected_flag == KREC_BF_EVT) { rc = evt_open(&krec->kr_evt, uma, &cbs, sub_toh); } else { - rc = dbtree_open_inplace_ex(&krec->kr_btr, uma, coh, pool, sub_toh); + rc = dbtree_open_inplace_ex(&krec->kr_btr, uma, coh, obj, sub_toh); } if (rc != 0) D_ERROR("Failed to open tree: " DF_RC "\n", DP_RC(rc)); @@ -924,7 +937,7 @@ tree_open_create(struct vos_object *obj, enum vos_tree_class tclass, int flags, rc = dbtree_create_inplace_ex(ta->ta_class, tree_feats, ta->ta_order, uma, &krec->kr_btr, - coh, pool, sub_toh); + coh, obj, sub_toh); if (rc != 0) { D_ERROR("Failed to create btree: "DF_RC"\n", DP_RC(rc)); goto out; @@ -1206,14 +1219,13 @@ obj_tree_init(struct vos_object *obj) ta->ta_order, vos_obj2uma(obj), &obj->obj_df->vo_tree, vos_cont2hdl(obj->obj_cont), - vos_obj2pool(obj), - &obj->obj_toh); + obj, &obj->obj_toh); } else { D_DEBUG(DB_DF, "Open btree for object\n"); rc = dbtree_open_inplace_ex(&obj->obj_df->vo_tree, vos_obj2uma(obj), vos_cont2hdl(obj->obj_cont), - vos_obj2pool(obj), &obj->obj_toh); + obj, &obj->obj_toh); } if (rc)