From b3041bde5161a1fd6846cc2251f5102abf3d4af7 Mon Sep 17 00:00:00 2001 From: Jeff Olivier Date: Tue, 10 Dec 2024 18:12:32 -0700 Subject: [PATCH 1/7] Partial backport of debug macro patch Required-githooks: true Change-Id: Ifd3f793661ea9f64aa47162a791b17b4987164ba Signed-off-by: Jeff Olivier --- src/include/daos_srv/rebuild.h | 33 ++++++++++++++++++++++++++++++++- 1 file changed, 32 insertions(+), 1 deletion(-) diff --git a/src/include/daos_srv/rebuild.h b/src/include/daos_srv/rebuild.h index 8a856c70bb1..30ad5d8a18f 100644 --- a/src/include/daos_srv/rebuild.h +++ b/src/include/daos_srv/rebuild.h @@ -1,5 +1,5 @@ /** - * (C) Copyright 2017-2023 Intel Corporation. + * (C) Copyright 2017-2024 Intel Corporation. * * SPDX-License-Identifier: BSD-2-Clause-Patent */ @@ -34,6 +34,37 @@ typedef enum { (rb_op) == RB_OP_UPGRADE ? "Upgrade" : \ (rb_op) == RB_OP_NONE ? "None" : \ "Unknown") +/* Common rebuild identifying information for INFO/DEBUG logging: + * rb=/// + */ +#define DF_RB "rb=" DF_UUID "/%u/%u/%s" + +/* Full rebuild identifying information includes / + * Instead of this, use DF_RB most of the time (use this for leader change scenarios, etc.) + */ +#define DF_RBF DF_RB " ld=%u/" DF_U64 + +/* arguments for log rebuild identifier given a struct rebuild_global_pool_tracker * */ +#define DP_RB_RGT(rgt) \ + DP_UUID((rgt)->rgt_pool_uuid), (rgt)->rgt_rebuild_ver, (rgt)->rgt_rebuild_gen, \ + RB_OP_STR((rgt)->rgt_opc) + +/* arguments for log rebuild identifier given a struct rebuild_tgt_pool_tracker *rpt */ +#define DP_RB_RPT(rpt) \ + DP_UUID((rpt)->rt_pool_uuid), (rpt)->rt_rebuild_ver, (rpt)->rt_rebuild_gen, \ + RB_OP_STR((rpt)->rt_rebuild_op) +#define DP_RBF_RPT(rpt) DP_RB_RPT(rpt), (rpt)->rt_leader_rank, (rpt)->rt_leader_term + +/* arguments for log rebuild identifier given a struct rebuild_scan_in *rsin */ +#define DP_RB_RSI(rsi) \ + DP_UUID((rsi)->rsi_pool_uuid), (rsi)->rsi_rebuild_ver, (rsi)->rsi_rebuild_gen, \ + RB_OP_STR((rsi)->rsi_rebuild_op) +#define DP_RBF_RSI(rsi) DP_RB_RSI(rsi), (rsi)->rsi_master_rank, (rsi)->rsi_leader_term + +/* arguments for log rebuild identifier given a struct migrate_query_arg * */ +#define DP_RB_MQA(mqa) \ + DP_UUID((mqa)->pool_uuid), (mqa)->version, (mqa)->generation, RB_OP_STR((mqa)->rebuild_op) +#define DP_RBF_MQA(mqa) DP_RB_MQA(mqa), (mqa)->leader_rank, (mqa)->leader_term int ds_rebuild_schedule(struct ds_pool *pool, uint32_t map_ver, daos_epoch_t stable_eph, uint32_t layout_version, From f945f17dbe5d6ffbc94de447260628def74bf1ae Mon Sep 17 00:00:00 2001 From: Liu Xuezhao Date: Tue, 29 Oct 2024 23:30:54 +0800 Subject: [PATCH 2/7] DAOS-16702 rebuild: restart rebuild for a massive failure case (#15343) * DAOS-16702 rebuild: restart rebuild for a massive failure case In special massive failure case - 1. some engines down and triggered rebuild. 2. one engine participated the rebuild, not finished yet, it down again, the #failures exceeds pool RF and will not change pool map. 3. That engine restarted by administrator. In that case should recover the rebuild task on the engine, to simplify it now just abort and retry the global rebuild task. No such issue by the typical recover approach that restart the whole system including the PS leader. Signed-off-by: Xuezhao Liu --- src/include/daos_srv/rebuild.h | 1 + src/pool/srv_pool.c | 44 +++++++++++++++++++++++++++++++++- src/rebuild/scan.c | 24 ++++++++++++++++--- src/rebuild/srv.c | 39 ++++++++++++++++++++++++++++++ 4 files changed, 104 insertions(+), 4 deletions(-) diff --git a/src/include/daos_srv/rebuild.h b/src/include/daos_srv/rebuild.h index 30ad5d8a18f..5deed3f3a6a 100644 --- a/src/include/daos_srv/rebuild.h +++ b/src/include/daos_srv/rebuild.h @@ -70,6 +70,7 @@ int ds_rebuild_schedule(struct ds_pool *pool, uint32_t map_ver, daos_epoch_t stable_eph, uint32_t layout_version, struct pool_target_id_list *tgts, daos_rebuild_opc_t rebuild_op, uint64_t delay_sec); +void ds_rebuild_restart_if_rank_wip(uuid_t pool_uuid, d_rank_t rank); int ds_rebuild_query(uuid_t pool_uuid, struct daos_rebuild_status *status); void ds_rebuild_running_query(uuid_t pool_uuid, uint32_t opc, uint32_t *rebuild_ver, diff --git a/src/pool/srv_pool.c b/src/pool/srv_pool.c index 6e3a01379fa..7fa0f33ea5f 100644 --- a/src/pool/srv_pool.c +++ b/src/pool/srv_pool.c @@ -1458,6 +1458,43 @@ resume_event_handling(struct pool_svc *svc) ABT_mutex_unlock(events->pse_mutex); } +/* + * Restart rebuild if the rank is UPIN in pool map and is in rebuilding. + * + * This function only used when PS leader gets CRT_EVT_ALIVE event of engine \a rank, + * if that rank is UPIN in pool map and with unfinished rebuilding should be massive + * failure case - + * 1. some engines down and triggered rebuild. + * 2. the engine \a rank participated the rebuild, not finished yet, it became down again, + * the #failures exceeds pool RF and will not change pool map. + * 3. That engine restarted by administrator. + * + * In that case should recover the rebuild task on engine \a rank, to simplify it now just + * abort and retry the global rebuild task. + */ +static void +pool_restart_rebuild_if_rank_wip(struct ds_pool *pool, d_rank_t rank) +{ + struct pool_domain *dom; + + dom = pool_map_find_dom_by_rank(pool->sp_map, rank); + if (dom == NULL) { + D_DEBUG(DB_MD, DF_UUID": rank %d non-exist on pool map.\n", + DP_UUID(pool->sp_uuid), rank); + return; + } + + if (dom->do_comp.co_status != PO_COMP_ST_UPIN) { + D_INFO(DF_UUID": rank %d status %d in pool map, got CRT_EVT_ALIVE.\n", + DP_UUID(pool->sp_uuid), rank, dom->do_comp.co_status); + return; + } + + ds_rebuild_restart_if_rank_wip(pool->sp_uuid, rank); + + return; +} + static int pool_svc_exclude_ranks(struct pool_svc *svc, struct pool_svc_event_set *event_set); static int @@ -1487,8 +1524,13 @@ handle_event(struct pool_svc *svc, struct pool_svc_event_set *event_set) for (i = 0; i < event_set->pss_len; i++) { struct pool_svc_event *event = &event_set->pss_buf[i]; - if (event->psv_src != CRT_EVS_SWIM || event->psv_type != CRT_EVT_ALIVE) + if (event->psv_type != CRT_EVT_ALIVE) continue; + + D_DEBUG(DB_MD, DF_UUID ": got CRT_EVT_ALIVE event, psv_src %d, psv_rank %d\n", + DP_UUID(svc->ps_uuid), event->psv_src, event->psv_rank); + pool_restart_rebuild_if_rank_wip(svc->ps_pool, event->psv_rank); + if (ds_pool_map_rank_up(svc->ps_pool->sp_map, event->psv_rank)) { /* * The rank is up in the pool map. Request a pool map diff --git a/src/rebuild/scan.c b/src/rebuild/scan.c index 4b9a9108fe1..09472e78d97 100644 --- a/src/rebuild/scan.c +++ b/src/rebuild/scan.c @@ -1169,9 +1169,26 @@ rebuild_tgt_scan_handler(crt_rpc_t *rpc) "rsi_rebuild_ver %d != rt_rebuild_ver %d\n", rsi->rsi_rebuild_ver, rpt->rt_rebuild_ver); - D_DEBUG(DB_REBUILD, DF_UUID" already started, req "DF_U64" master %u/"DF_U64"\n", - DP_UUID(rsi->rsi_pool_uuid), rsi->rsi_leader_term, rsi->rsi_master_rank, - rpt->rt_leader_term); + /* The same PS leader request rebuild with higher rsi_rebuild_gen. + * Is the case of massive failure case, see pool_restart_rebuild_if_rank_wip(). + */ + if (rpt->rt_leader_rank == rsi->rsi_master_rank && + rpt->rt_leader_term == rsi->rsi_leader_term && + rpt->rt_rebuild_gen < rsi->rsi_rebuild_gen) { + /* rebuild_leader_status_notify(LAZY rebuild_iv_update), + * it will set rpt->rt_global_done to abort rpt. + * set rt_abort here just for safe. + */ + rpt->rt_abort = 1; + D_INFO(DF_RBF ", start new rebuild, gen %d -> %d.\n", + DP_RBF_RPT(rpt), rpt->rt_rebuild_gen, rsi->rsi_rebuild_gen); + rpt_put(rpt); + rpt = NULL; + goto tls_lookup; + } + + D_DEBUG(DB_REBUILD, "already started, existing " DF_RBF ", req " DF_RBF "\n", + DP_RBF_RPT(rpt), DP_RBF_RSI(rsi)); /* Ignore the rebuild trigger request if it comes from * an old or same leader. @@ -1203,6 +1220,7 @@ rebuild_tgt_scan_handler(crt_rpc_t *rpc) rpt = NULL; } +tls_lookup: tls = rebuild_pool_tls_lookup(rsi->rsi_pool_uuid, rsi->rsi_rebuild_ver, rsi->rsi_rebuild_gen); if (tls != NULL) { diff --git a/src/rebuild/srv.c b/src/rebuild/srv.c index 5c463af9e90..8f0696a22f0 100644 --- a/src/rebuild/srv.c +++ b/src/rebuild/srv.c @@ -490,6 +490,45 @@ ds_rebuild_running_query(uuid_t pool_uuid, uint32_t opc, uint32_t *upper_ver, rpt_put(rpt); } +/* + * Restart rebuild if \a rank's rebuild not finished. + * Only used for massive failure recovery case, see pool_restart_rebuild_if_rank_wip(). + */ +void +ds_rebuild_restart_if_rank_wip(uuid_t pool_uuid, d_rank_t rank) +{ + struct rebuild_global_pool_tracker *rgt; + int i; + + rgt = rebuild_global_pool_tracker_lookup(pool_uuid, -1, -1); + if (rgt == NULL) + return; + + if (rgt->rgt_status.rs_state != DRS_IN_PROGRESS) { + rgt_put(rgt); + return; + } + + for (i = 0; i < rgt->rgt_servers_number; i++) { + if (rgt->rgt_servers[i].rank == rank) { + if (!rgt->rgt_servers[i].pull_done) { + rgt->rgt_status.rs_errno = -DER_STALE; + rgt->rgt_abort = 1; + rgt->rgt_status.rs_fail_rank = rank; + D_INFO(DF_RB ": abort rebuild because rank %d WIP\n", + DP_RB_RGT(rgt), rank); + } + rgt_put(rgt); + return; + } + } + + D_INFO(DF_RB ": rank %d not in rgt_servers, rgt_servers_number %d\n", + DP_RB_RGT(rgt), rank, rgt->rgt_servers_number); + rgt_put(rgt); + return; +} + /* TODO: Add something about what the current operation is for output status */ int ds_rebuild_query(uuid_t pool_uuid, struct daos_rebuild_status *status) From 05c2ad908c73d2d5df07c6095c74081c50a748e9 Mon Sep 17 00:00:00 2001 From: Cedric Koch-Hofer <94527853+knard-intel@users.noreply.github.com> Date: Tue, 11 Jun 2024 14:59:09 +0200 Subject: [PATCH 3/7] DAOS-10250 control: Get enabled and disabled ranks with dmg pool query (#14436) Allow enabled and disabled ranks option to be used simultaneously (DAOS-10250). Update and add cmocka unit tests of engine management related functions (DAOS-10253). Fix memory leaks of ranks string in function ds_mgmt_drpc_pool_query(). Required-githooks: true Signed-off-by: Cedric Koch-Hofer --- src/common/tests_dmg_helpers.c | 6 +- src/control/cmd/dmg/pool.go | 4 - src/control/cmd/dmg/pool_test.go | 32 ++- src/gurt/misc.c | 101 +++++----- src/gurt/tests/test_gurt.c | 189 +++++++++++++++++- src/include/daos_srv/pool.h | 8 +- src/include/gurt/common.h | 6 +- src/mgmt/srv_drpc.c | 130 ++++++------ src/mgmt/srv_internal.h | 7 +- src/mgmt/srv_pool.c | 32 +-- src/mgmt/tests/mocks.c | 48 +++-- src/mgmt/tests/mocks.h | 16 +- src/mgmt/tests/srv_drpc_tests.c | 24 +-- src/pool/srv_cli.c | 138 ++++++++----- .../ftest/control/dmg_pool_query_ranks.py | 122 ++++++----- 15 files changed, 563 insertions(+), 300 deletions(-) diff --git a/src/common/tests_dmg_helpers.c b/src/common/tests_dmg_helpers.c index f894b9f861d..0cf9a50aef3 100644 --- a/src/common/tests_dmg_helpers.c +++ b/src/common/tests_dmg_helpers.c @@ -1118,9 +1118,9 @@ dmg_pool_extend(const char *dmg_config_file, const uuid_t uuid, rank_list.rl_ranks = ranks; rank_list.rl_nr = rank_nr; - rank_str = d_rank_list_to_str(&rank_list); - if (rank_str == NULL) - D_GOTO(out, rc = -DER_NOMEM); + rc = d_rank_list_to_str(&rank_list, &rank_str); + if (rc != 0) + D_GOTO(out, rc); uuid_unparse_lower(uuid, uuid_str); args = cmd_push_arg(args, &argcount, "%s ", uuid_str); diff --git a/src/control/cmd/dmg/pool.go b/src/control/cmd/dmg/pool.go index b9d05d3eeaf..7edf2c6652f 100644 --- a/src/control/cmd/dmg/pool.go +++ b/src/control/cmd/dmg/pool.go @@ -621,10 +621,6 @@ func (cmd *PoolQueryCmd) Execute(args []string) error { if cmd.HealthOnly { req.QueryMask = daos.HealthOnlyPoolQueryMask } - // TODO (DAOS-10250) The two options should not be incompatible (i.e. engine limitation) - if cmd.ShowEnabledRanks && cmd.ShowDisabledRanks { - return errIncompatFlags("show-enabled-ranks", "show-disabled-ranks") - } if cmd.ShowEnabledRanks { req.QueryMask.SetOptions(daos.PoolQueryOptionEnabledEngines) } diff --git a/src/control/cmd/dmg/pool_test.go b/src/control/cmd/dmg/pool_test.go index 55936b926a0..67bf72bf54e 100644 --- a/src/control/cmd/dmg/pool_test.go +++ b/src/control/cmd/dmg/pool_test.go @@ -1047,6 +1047,32 @@ func TestPoolCommands(t *testing.T) { }, " "), nil, }, + { + "Query pool with UUID, enabled ranks and disabled ranks", + "pool query --show-disabled --show-enabled 12345678-1234-1234-1234-1234567890ab", + strings.Join([]string{ + printRequest(t, &control.PoolQueryReq{ + ID: "12345678-1234-1234-1234-1234567890ab", + QueryMask: setQueryMask(func(qm *daos.PoolQueryMask) { + qm.SetOptions(daos.PoolQueryOptionEnabledEngines, daos.PoolQueryOptionDisabledEngines) + }), + }), + }, " "), + nil, + }, + { + "Query pool with UUID, enabled ranks and disabled ranks", + "pool query -b -e 12345678-1234-1234-1234-1234567890ab", + strings.Join([]string{ + printRequest(t, &control.PoolQueryReq{ + ID: "12345678-1234-1234-1234-1234567890ab", + QueryMask: setQueryMask(func(qm *daos.PoolQueryMask) { + qm.SetOptions(daos.PoolQueryOptionEnabledEngines, daos.PoolQueryOptionDisabledEngines) + }), + }), + }, " "), + nil, + }, { "Query pool for health only", "pool query --health-only 12345678-1234-1234-1234-1234567890ab", @@ -1091,12 +1117,6 @@ func TestPoolCommands(t *testing.T) { "", fmt.Errorf("Unknown command"), }, - { - "Query pool with incompatible arguments", - "pool query --show-disabled --show-enabled 12345678-1234-1234-1234-1234567890ab", - "", - errors.New("may not be mixed"), - }, }) } diff --git a/src/gurt/misc.c b/src/gurt/misc.c index 29cc6d3bc24..641869db9ba 100644 --- a/src/gurt/misc.c +++ b/src/gurt/misc.c @@ -744,24 +744,39 @@ d_rank_list_dump(d_rank_list_t *rank_list, d_string_t name, int name_len) * Create a ranged string representation of a rank list. * * \param[in] rank_list the rank list to represent + * \param[out] ranks_str Returned ranged string (caller must free) * - * \return a ranged string (caller must free) + * \return 0 on success, a negative value on error */ -char * -d_rank_list_to_str(d_rank_list_t *rank_list) +int +d_rank_list_to_str(d_rank_list_t *ranks, char **ranks_str) { - char *str; - bool truncated = false; - d_rank_range_list_t *range_list; + d_rank_range_list_t *range_list = NULL; + char *range_list_str; + int rc; + + D_ASSERT(ranks_str != NULL); - range_list = d_rank_range_list_create_from_ranks(rank_list); + if (ranks == NULL) { + range_list_str = NULL; + D_GOTO(out, rc = -DER_SUCCESS); + } + + range_list = d_rank_range_list_create_from_ranks(ranks); if (range_list == NULL) - return NULL; - str = d_rank_range_list_str(range_list, &truncated); + D_GOTO(error, rc = -DER_NOMEM); + rc = d_rank_range_list_str(range_list, &range_list_str); + if (rc != 0) + D_GOTO(error, rc); + +out: + *ranks_str = range_list_str; + +error: d_rank_range_list_free(range_list); - return str; + return rc; } d_rank_list_t * @@ -844,7 +859,6 @@ d_rank_range_list_realloc(d_rank_range_list_t *range_list, uint32_t size) return range_list; } -/* TODO (DAOS-10253) Add unit tests for this function */ d_rank_range_list_t * d_rank_range_list_create_from_ranks(d_rank_list_t *rank_list) { @@ -890,56 +904,51 @@ d_rank_range_list_create_from_ranks(d_rank_list_t *rank_list) return range_list; } -/* TODO (DAOS-10253) Add unit tests for this function */ -char * -d_rank_range_list_str(d_rank_range_list_t *list, bool *truncated) +int +d_rank_range_list_str(d_rank_range_list_t *list, char **ranks_str) { - const size_t MAXBYTES = 512; - char *line; - char *linepos; - int ret = 0; - size_t remaining = MAXBYTES - 2u; - int i; - int err = 0; + const size_t MAXBYTES = 512u; + size_t remaining = MAXBYTES - 2u; + char *line; + char *linepos; + int i; + int len; + int rc; + + D_ASSERT(list != NULL); - *truncated = false; D_ALLOC(line, MAXBYTES); if (line == NULL) - return NULL; + D_GOTO(error, rc = -DER_NOMEM); *line = '['; linepos = line + 1; for (i = 0; i < list->rrl_nr; i++) { - uint32_t lo = list->rrl_ranges[i].lo; - uint32_t hi = list->rrl_ranges[i].hi; - bool lastrange = (i == (list->rrl_nr - 1)); + uint32_t lo = list->rrl_ranges[i].lo; + uint32_t hi = list->rrl_ranges[i].hi; + bool lastrange = (i == (list->rrl_nr - 1)); if (lo == hi) - ret = snprintf(linepos, remaining, "%u%s", lo, lastrange ? "" : ","); + len = snprintf(linepos, remaining, "%u%s", lo, lastrange ? "" : ","); else - ret = snprintf(linepos, remaining, "%u-%u%s", lo, hi, lastrange ? "" : ","); - - if (ret < 0) { - err = errno; - D_ERROR("rank set could not be serialized: %s (%d)\n", strerror(err), err); - break; - } - - if (ret >= remaining) { - err = EOVERFLOW; - D_WARN("rank set has been partially serialized\n"); - break; - } - - remaining -= ret; - linepos += ret; + len = snprintf(linepos, remaining, "%u-%u%s", lo, hi, lastrange ? "" : ","); + if (len < 0) + D_GOTO(error, rc = -DER_INVAL); + if (len >= remaining) + D_GOTO(error, rc = -DER_TRUNC); + + remaining -= len; + linepos += len; } memcpy(linepos, "]", 2u); - if (err != 0) - *truncated = true; + *ranks_str = line; + D_GOTO(out, rc = -DER_SUCCESS); - return line; +error: + D_FREE(line); +out: + return rc; } void diff --git a/src/gurt/tests/test_gurt.c b/src/gurt/tests/test_gurt.c index 40f9256e724..901e4d7514f 100644 --- a/src/gurt/tests/test_gurt.c +++ b/src/gurt/tests/test_gurt.c @@ -2510,6 +2510,190 @@ test_d_setenv(void **state) assert_false(d_isenv_def("foo")); } +static void +test_d_rank_list_to_str(void **state) +{ + d_rank_list_t *ranks; + char *ranks_str = NULL; + int i; + int rc; + + // Test with null list + rc = d_rank_list_to_str(NULL, &ranks_str); + assert_int_equal(rc, -DER_SUCCESS); + assert_null(ranks_str); + + // Test with empty list + ranks = d_rank_list_alloc(0); + assert_non_null(ranks); + + rc = d_rank_list_to_str(ranks, &ranks_str); + assert_int_equal(rc, -DER_SUCCESS); + assert_string_equal(ranks_str, "[]"); + + D_FREE(ranks_str); + d_rank_list_free(ranks); + + // Test with one rank + ranks = d_rank_list_alloc(1); + assert_non_null(ranks); + ranks->rl_ranks[0] = 2; + + rc = d_rank_list_to_str(ranks, &ranks_str); + assert_int_equal(rc, -DER_SUCCESS); + assert_string_equal(ranks_str, "[2]"); + + D_FREE(ranks_str); + d_rank_list_free(ranks); + + // Test with 4 ranks and two ranges + ranks = d_rank_list_alloc(4); + assert_non_null(ranks); + ranks->rl_ranks[0] = 2; + ranks->rl_ranks[1] = 1; + ranks->rl_ranks[2] = 5; + ranks->rl_ranks[3] = 3; + + rc = d_rank_list_to_str(ranks, &ranks_str); + assert_int_equal(rc, -DER_SUCCESS); + assert_string_equal(ranks_str, "[1-3,5]"); + + D_FREE(ranks_str); + d_rank_list_free(ranks); + + // Test truncate error + ranks = d_rank_list_alloc(1024); + assert_non_null(ranks); + for (i = 0; i < 1024; ++i) + ranks->rl_ranks[i] = 2 * i + 1; + + rc = d_rank_list_to_str(ranks, &ranks_str); + assert_int_equal(rc, -DER_TRUNC); + assert_null(ranks_str); + + d_rank_list_free(ranks); +} + +static void +test_d_rank_range_list_create_from_ranks(void **state) +{ + d_rank_list_t *ranks; + d_rank_range_list_t *range_list; + + // Test with null list + range_list = d_rank_range_list_create_from_ranks(NULL); + assert_non_null(range_list); + assert_int_equal(range_list->rrl_nr, 0); + + d_rank_range_list_free(range_list); + + // Test with empty list + ranks = d_rank_list_alloc(0); + assert_non_null(ranks); + + range_list = d_rank_range_list_create_from_ranks(ranks); + assert_non_null(range_list); + assert_int_equal(range_list->rrl_nr, 0); + + d_rank_range_list_free(range_list); + d_rank_list_free(ranks); + + // Test with one rank + ranks = d_rank_list_alloc(1); + assert_non_null(ranks); + ranks->rl_ranks[0] = 2; + + range_list = d_rank_range_list_create_from_ranks(ranks); + assert_non_null(range_list); + assert_int_equal(range_list->rrl_nr, 1); + assert_int_equal(range_list->rrl_ranges[0].lo, 2); + assert_int_equal(range_list->rrl_ranges[0].hi, 2); + + d_rank_range_list_free(range_list); + d_rank_list_free(ranks); + + // Test with 4 ranks and two ranges + ranks = d_rank_list_alloc(4); + assert_non_null(ranks); + ranks->rl_ranks[0] = 2; + ranks->rl_ranks[1] = 1; + ranks->rl_ranks[2] = 5; + ranks->rl_ranks[3] = 3; + + range_list = d_rank_range_list_create_from_ranks(ranks); + assert_non_null(range_list); + assert_int_equal(range_list->rrl_nr, 2); + assert_int_equal(range_list->rrl_ranges[0].lo, 1); + assert_int_equal(range_list->rrl_ranges[0].hi, 3); + assert_int_equal(range_list->rrl_ranges[1].lo, 5); + assert_int_equal(range_list->rrl_ranges[1].hi, 5); + + d_rank_range_list_free(range_list); + d_rank_list_free(ranks); +} + +static void +test_d_rank_range_list_str(void **state) +{ + d_rank_range_list_t *range_list; + char *ranks_str = NULL; + int i; + int rc; + + // Test with empty list + range_list = d_rank_range_list_alloc(0); + assert_non_null(range_list); + + rc = d_rank_range_list_str(range_list, &ranks_str); + assert_int_equal(rc, 0); + assert_string_equal(ranks_str, "[]"); + + D_FREE(ranks_str); + d_rank_range_list_free(range_list); + + // Test with one rank + range_list = d_rank_range_list_alloc(1); + assert_non_null(range_list); + range_list->rrl_ranges[0].lo = 2; + range_list->rrl_ranges[0].hi = 2; + + rc = d_rank_range_list_str(range_list, &ranks_str); + assert_int_equal(rc, 0); + assert_string_equal(ranks_str, "[2]"); + + D_FREE(ranks_str); + d_rank_range_list_free(range_list); + + // Test with 4 ranks and two ranges + range_list = d_rank_range_list_alloc(2); + assert_non_null(range_list); + range_list->rrl_ranges[0].lo = 1; + range_list->rrl_ranges[0].hi = 3; + range_list->rrl_ranges[1].lo = 5; + range_list->rrl_ranges[1].hi = 5; + + rc = d_rank_range_list_str(range_list, &ranks_str); + assert_int_equal(rc, 0); + assert_string_equal(ranks_str, "[1-3,5]"); + + D_FREE(ranks_str); + d_rank_range_list_free(range_list); + + // Test truncate error + range_list = d_rank_range_list_alloc(1024); + assert_non_null(range_list); + for (i = 0; i < 1024; ++i) { + range_list->rrl_ranges[i].lo = i; + range_list->rrl_ranges[i].hi = i; + } + + rc = d_rank_range_list_str(range_list, &ranks_str); + assert_int_equal(rc, -DER_TRUNC); + assert_null(ranks_str); + + d_rank_range_list_free(range_list); +} + int main(int argc, char **argv) { @@ -2546,7 +2730,10 @@ main(int argc, char **argv) teardown_getenv_mocks), cmocka_unit_test_setup_teardown(test_d_getenv_uint64_t, setup_getenv_mocks, teardown_getenv_mocks), - cmocka_unit_test(test_d_setenv)}; + cmocka_unit_test(test_d_setenv), + cmocka_unit_test(test_d_rank_list_to_str), + cmocka_unit_test(test_d_rank_range_list_create_from_ranks), + cmocka_unit_test(test_d_rank_range_list_str)}; d_register_alt_assert(mock_assert); diff --git a/src/include/daos_srv/pool.h b/src/include/daos_srv/pool.h index de22d55ed5d..985c6346a34 100644 --- a/src/include/daos_srv/pool.h +++ b/src/include/daos_srv/pool.h @@ -304,9 +304,11 @@ int dsc_pool_svc_delete_acl(uuid_t pool_uuid, d_rank_list_t *ranks, uint64_t dea enum daos_acl_principal_type principal_type, const char *principal_name); -int dsc_pool_svc_query(uuid_t pool_uuid, d_rank_list_t *ps_ranks, uint64_t deadline, - d_rank_list_t **ranks, daos_pool_info_t *pool_info, - uint32_t *pool_layout_ver, uint32_t *upgrade_layout_ver); +int + dsc_pool_svc_query(uuid_t pool_uuid, d_rank_list_t *ps_ranks, uint64_t deadline, + d_rank_list_t **enabled_ranks, d_rank_list_t **disabled_ranks, + daos_pool_info_t *pool_info, uint32_t *pool_layout_ver, + uint32_t *upgrade_layout_ver); int dsc_pool_svc_query_target(uuid_t pool_uuid, d_rank_list_t *ps_ranks, uint64_t deadline, d_rank_t rank, uint32_t tgt_idx, daos_target_info_t *ti); diff --git a/src/include/gurt/common.h b/src/include/gurt/common.h index 8dec9ea9871..060cdc86c18 100644 --- a/src/include/gurt/common.h +++ b/src/include/gurt/common.h @@ -481,12 +481,14 @@ int d_rank_list_append(d_rank_list_t *rank_list, d_rank_t rank); int d_rank_list_dump(d_rank_list_t *rank_list, d_string_t name, int name_len); d_rank_list_t *uint32_array_to_rank_list(uint32_t *ints, size_t len); int rank_list_to_uint32_array(d_rank_list_t *rl, uint32_t **ints, size_t *len); -char *d_rank_list_to_str(d_rank_list_t *rank_list); +int + d_rank_list_to_str(d_rank_list_t *rank_list, char **rank_str); d_rank_range_list_t *d_rank_range_list_alloc(uint32_t size); d_rank_range_list_t *d_rank_range_list_realloc(d_rank_range_list_t *range_list, uint32_t size); d_rank_range_list_t *d_rank_range_list_create_from_ranks(d_rank_list_t *rank_list); -char *d_rank_range_list_str(d_rank_range_list_t *list, bool *truncated); +int + d_rank_range_list_str(d_rank_range_list_t *list, char **ranks_str); void d_rank_range_list_free(d_rank_range_list_t *range_list); #ifdef FAULT_INJECTION diff --git a/src/mgmt/srv_drpc.c b/src/mgmt/srv_drpc.c index de4a951a050..0ddedeabc59 100644 --- a/src/mgmt/srv_drpc.c +++ b/src/mgmt/srv_drpc.c @@ -394,7 +394,7 @@ static int pool_create_fill_resp(Mgmt__PoolCreateResp *resp, uuid_t uuid, d_rank D_DEBUG(DB_MGMT, "%d service replicas\n", svc_ranks->rl_nr); - rc = ds_mgmt_pool_query(uuid, svc_ranks, &enabled_ranks, &pool_info, NULL, NULL); + rc = ds_mgmt_pool_query(uuid, svc_ranks, &enabled_ranks, NULL, &pool_info, NULL, NULL); if (rc != 0) { D_ERROR("Failed to query created pool: rc=%d\n", rc); D_GOTO(out, rc); @@ -1165,10 +1165,10 @@ add_props_to_resp(daos_prop_t *prop, Mgmt__PoolGetPropResp *resp) D_ERROR("svc rank list unset\n"); D_GOTO(out, rc = -DER_INVAL); } - resp_props[j]->strval = d_rank_list_to_str( - (d_rank_list_t *)entry->dpe_val_ptr); - if (resp_props[j]->strval == NULL) - D_GOTO(out, rc = -DER_NOMEM); + rc = d_rank_list_to_str((d_rank_list_t *)entry->dpe_val_ptr, + &resp_props[j]->strval); + if (rc != 0) + D_GOTO(out, rc); resp_props[j]->value_case = MGMT__POOL_PROPERTY__VALUE_STRVAL; break; @@ -1734,22 +1734,22 @@ pool_query_free_tier_stats(Mgmt__PoolQueryResp *resp) void ds_mgmt_drpc_pool_query(Drpc__Call *drpc_req, Drpc__Response *drpc_resp) { - struct drpc_alloc alloc = PROTO_ALLOCATOR_INIT(alloc); - int rc = 0; - Mgmt__PoolQueryReq *req; - Mgmt__PoolQueryResp resp = MGMT__POOL_QUERY_RESP__INIT; - Mgmt__StorageUsageStats scm = MGMT__STORAGE_USAGE_STATS__INIT; - Mgmt__StorageUsageStats nvme = MGMT__STORAGE_USAGE_STATS__INIT; - Mgmt__PoolRebuildStatus rebuild = MGMT__POOL_REBUILD_STATUS__INIT; - uuid_t uuid; - daos_pool_info_t pool_info = {0}; - d_rank_list_t *svc_ranks; - d_rank_list_t *ranks; - d_rank_range_list_t *range_list; - char *range_list_str = NULL; - bool truncated; - size_t len; - uint8_t *body; + struct drpc_alloc alloc = PROTO_ALLOCATOR_INIT(alloc); + int rc = 0; + Mgmt__PoolQueryReq *req; + Mgmt__PoolQueryResp resp = MGMT__POOL_QUERY_RESP__INIT; + Mgmt__StorageUsageStats scm = MGMT__STORAGE_USAGE_STATS__INIT; + Mgmt__StorageUsageStats nvme = MGMT__STORAGE_USAGE_STATS__INIT; + Mgmt__PoolRebuildStatus rebuild = MGMT__POOL_REBUILD_STATUS__INIT; + uuid_t uuid; + daos_pool_info_t pool_info = {0}; + d_rank_list_t *svc_ranks = NULL; + d_rank_list_t *enabled_ranks = NULL; + d_rank_list_t *disabled_ranks = NULL; + char *enabled_ranks_str = NULL; + char *disabled_ranks_str = NULL; + size_t len; + uint8_t *body; req = mgmt__pool_query_req__unpack(&alloc.alloc, drpc_req->body.len, drpc_req->body.data); @@ -1762,58 +1762,61 @@ ds_mgmt_drpc_pool_query(Drpc__Call *drpc_req, Drpc__Response *drpc_resp) D_INFO("Received request to query DAOS pool %s\n", req->id); if (uuid_parse(req->id, uuid) != 0) { - rc = -DER_INVAL; - DL_ERROR(rc, "Pool UUID is invalid"); - goto out; - } - - /* TODO (DAOS-10250) Enabled and disabled engines should be retrieve both if needed */ - if (req->query_mask & DPI_ENGINES_ENABLED && req->query_mask & DPI_ENGINES_DISABLED) { - D_ERROR("cannot query enabled and disabled engines in the same request\n"); - D_GOTO(out, rc = -DER_NOTSUPPORTED); + DL_ERROR(-DER_INVAL, "Pool UUID is invalid"); + D_GOTO(error, rc = -DER_INVAL); } svc_ranks = uint32_array_to_rank_list(req->svc_ranks, req->n_svc_ranks); if (svc_ranks == NULL) - D_GOTO(out, rc = -DER_NOMEM); + D_GOTO(error, rc = -DER_NOMEM); pool_info.pi_bits = req->query_mask; - rc = ds_mgmt_pool_query(uuid, svc_ranks, &ranks, &pool_info, &resp.pool_layout_ver, - &resp.upgrade_layout_ver); + rc = ds_mgmt_pool_query(uuid, svc_ranks, &enabled_ranks, &disabled_ranks, &pool_info, + &resp.pool_layout_ver, &resp.upgrade_layout_ver); if (rc != 0) { - D_ERROR("Failed to query the pool, rc=%d\n", rc); - goto out_svc_ranks; + DL_ERROR(rc, DF_UUID ": Failed to query the pool", DP_UUID(uuid)); + D_GOTO(error, rc); } - /* Calculate and stringify rank ranges to return to control plane for display */ - range_list = d_rank_range_list_create_from_ranks(ranks); - if (range_list == NULL) - D_GOTO(out_ranks, rc = -DER_NOMEM); - range_list_str = d_rank_range_list_str(range_list, &truncated); - if (range_list_str == NULL) - D_GOTO(out_ranges, rc = -DER_NOMEM); - D_DEBUG(DB_MGMT, DF_UUID": %s ranks: %s%s\n", DP_UUID(uuid), - pool_info.pi_bits & DPI_ENGINES_ENABLED ? "ENABLED" : "DISABLED", range_list_str, - truncated ? " ...(TRUNCATED)" : ""); + rc = d_rank_list_to_str(enabled_ranks, &enabled_ranks_str); + if (rc != 0) { + DL_ERROR(rc, DF_UUID ": Failed to serialize the list of enabled ranks", + DP_UUID(uuid)); + D_GOTO(error, rc); + } + if (enabled_ranks_str != NULL) + D_DEBUG(DB_MGMT, DF_UUID ": list of enabled ranks: %s\n", DP_UUID(uuid), + enabled_ranks_str); + + rc = d_rank_list_to_str(disabled_ranks, &disabled_ranks_str); + if (rc != 0) { + DL_ERROR(rc, DF_UUID ": Failed to serialize the list of disabled ranks", + DP_UUID(uuid)); + D_GOTO(error, rc); + } + if (disabled_ranks_str != NULL) + D_DEBUG(DB_MGMT, DF_UUID ": list of disabled ranks: %s\n", DP_UUID(uuid), + disabled_ranks_str); /* Populate the response */ resp.query_mask = pool_info.pi_bits; - resp.uuid = req->id; - resp.total_targets = pool_info.pi_ntargets; + resp.uuid = req->id; + resp.total_targets = pool_info.pi_ntargets; resp.disabled_targets = pool_info.pi_ndisabled; - resp.active_targets = pool_info.pi_space.ps_ntargets; - resp.total_engines = pool_info.pi_nnodes; - resp.svc_ldr = pool_info.pi_leader; + resp.active_targets = pool_info.pi_space.ps_ntargets; + resp.total_engines = pool_info.pi_nnodes; + resp.svc_ldr = pool_info.pi_leader; resp.svc_reps = req->svc_ranks; resp.n_svc_reps = req->n_svc_ranks; - resp.version = pool_info.pi_map_ver; - resp.enabled_ranks = (req->query_mask & DPI_ENGINES_ENABLED) ? range_list_str : ""; - resp.disabled_ranks = (req->query_mask & DPI_ENGINES_DISABLED) ? range_list_str : ""; + resp.version = pool_info.pi_map_ver; + if (enabled_ranks_str != NULL) + resp.enabled_ranks = enabled_ranks_str; + if (disabled_ranks_str != NULL) + resp.disabled_ranks = disabled_ranks_str; D_ALLOC_ARRAY(resp.tier_stats, DAOS_MEDIA_MAX); - if (resp.tier_stats == NULL) { - D_GOTO(out_ranges, rc = -DER_NOMEM); - } + if (resp.tier_stats == NULL) + D_GOTO(error, rc = -DER_NOMEM); storage_usage_stats_from_pool_space(&scm, &pool_info.pi_space, DAOS_MEDIA_SCM); @@ -1828,13 +1831,7 @@ ds_mgmt_drpc_pool_query(Drpc__Call *drpc_req, Drpc__Response *drpc_resp) pool_rebuild_status_from_info(&rebuild, &pool_info.pi_rebuild_st); resp.rebuild = &rebuild; -out_ranges: - d_rank_range_list_free(range_list); -out_ranks: - d_rank_list_free(ranks); -out_svc_ranks: - d_rank_list_free(svc_ranks); -out: +error: resp.status = rc; len = mgmt__pool_query_resp__get_packed_size(&resp); @@ -1847,10 +1844,13 @@ ds_mgmt_drpc_pool_query(Drpc__Call *drpc_req, Drpc__Response *drpc_resp) drpc_resp->body.data = body; } - D_FREE(range_list_str); - mgmt__pool_query_req__free_unpacked(req, &alloc.alloc); + d_rank_list_free(enabled_ranks); + D_FREE(enabled_ranks_str); + d_rank_list_free(disabled_ranks); + D_FREE(disabled_ranks_str); + d_rank_list_free(svc_ranks); pool_query_free_tier_stats(&resp); } diff --git a/src/mgmt/srv_internal.h b/src/mgmt/srv_internal.h index 88f7e901008..d9c1a83bc8e 100644 --- a/src/mgmt/srv_internal.h +++ b/src/mgmt/srv_internal.h @@ -113,9 +113,10 @@ int ds_mgmt_pool_delete_acl(uuid_t pool_uuid, d_rank_list_t *svc_ranks, int ds_mgmt_pool_list_cont(uuid_t uuid, d_rank_list_t *svc_ranks, struct daos_pool_cont_info **containers, uint64_t *ncontainers); -int ds_mgmt_pool_query(uuid_t pool_uuid, d_rank_list_t *svc_ranks, d_rank_list_t **ranks, - daos_pool_info_t *pool_info, uint32_t *pool_layout_ver, - uint32_t *upgrade_layout_ver); +int + ds_mgmt_pool_query(uuid_t pool_uuid, d_rank_list_t *svc_ranks, d_rank_list_t **enabled_ranks, + d_rank_list_t **disabled_ranks, daos_pool_info_t *pool_info, + uint32_t *pool_layout_ver, uint32_t *upgrade_layout_ver); int ds_mgmt_pool_query_targets(uuid_t pool_uuid, d_rank_list_t *svc_ranks, d_rank_t rank, d_rank_list_t *tgts, daos_target_info_t **infos); diff --git a/src/mgmt/srv_pool.c b/src/mgmt/srv_pool.c index c1eb018cdb1..b64297c117a 100644 --- a/src/mgmt/srv_pool.c +++ b/src/mgmt/srv_pool.c @@ -196,18 +196,13 @@ ds_mgmt_create_pool(uuid_t pool_uuid, const char *group, char *tgt_dev, d_rank_l if (!d_rank_list_identical(pg_targets, targets)) { char *pg_str, *tgt_str; - pg_str = d_rank_list_to_str(pg_ranks); - if (pg_str == NULL) { - rc = -DER_NOMEM; + rc = d_rank_list_to_str(pg_ranks, &pg_str); + if (rc != 0) D_GOTO(out, rc); - } - tgt_str = d_rank_list_to_str(targets); - if (tgt_str == NULL) { - D_FREE(pg_str); - rc = -DER_NOMEM; + rc = d_rank_list_to_str(targets, &tgt_str); + if (rc != 0) D_GOTO(out, rc); - } D_ERROR(DF_UUID": targets (%s) contains ranks not in pg (%s)\n", DP_UUID(pool_uuid), tgt_str, pg_str); @@ -389,13 +384,8 @@ ds_mgmt_pool_list_cont(uuid_t uuid, d_rank_list_t *svc_ranks, * * \param[in] pool_uuid UUID of the pool. * \param[in] svc_ranks Ranks of pool svc replicas. - * \param[out] ranks Optional, returned storage ranks in this pool. - * If #pool_info is NULL, engines with disabled targets. - * If #pool_info is passed, engines with enabled or - * disabled targets according to - * #pi_bits (DPI_ENGINES_ENABLED bit). - * Note: ranks may be empty (i.e., *ranks->rl_nr may be 0). - * The caller must free the list with d_rank_list_free(). + * \param[out] enabled_ranks Optional, returned storage ranks with enabled targets. + * \param[out] disabled_ranks Optional, returned storage ranks with disabled targets. * \param[in][out] pool_info Query results * \param[in][out] pool_layout_ver Pool global version * \param[in][out] upgrade_layout_ver Latest pool global version this pool might be upgraded @@ -405,9 +395,9 @@ ds_mgmt_pool_list_cont(uuid_t uuid, d_rank_list_t *svc_ranks, * Negative value Other error */ int -ds_mgmt_pool_query(uuid_t pool_uuid, d_rank_list_t *svc_ranks, d_rank_list_t **ranks, - daos_pool_info_t *pool_info, uint32_t *pool_layout_ver, - uint32_t *upgrade_layout_ver) +ds_mgmt_pool_query(uuid_t pool_uuid, d_rank_list_t *svc_ranks, d_rank_list_t **enabled_ranks, + d_rank_list_t **disabled_ranks, daos_pool_info_t *pool_info, + uint32_t *pool_layout_ver, uint32_t *upgrade_layout_ver) { if (pool_info == NULL) { D_ERROR("pool_info was NULL\n"); @@ -416,8 +406,8 @@ ds_mgmt_pool_query(uuid_t pool_uuid, d_rank_list_t *svc_ranks, d_rank_list_t **r D_DEBUG(DB_MGMT, "Querying pool "DF_UUID"\n", DP_UUID(pool_uuid)); - return dsc_pool_svc_query(pool_uuid, svc_ranks, mgmt_ps_call_deadline(), ranks, pool_info, - pool_layout_ver, upgrade_layout_ver); + return dsc_pool_svc_query(pool_uuid, svc_ranks, mgmt_ps_call_deadline(), enabled_ranks, + disabled_ranks, pool_info, pool_layout_ver, upgrade_layout_ver); } /** diff --git a/src/mgmt/tests/mocks.c b/src/mgmt/tests/mocks.c index eace651e60b..912a36f293a 100644 --- a/src/mgmt/tests/mocks.c +++ b/src/mgmt/tests/mocks.c @@ -1,5 +1,5 @@ /* - * (C) Copyright 2019-2023 Intel Corporation. + * (C) Copyright 2019-2024 Intel Corporation. * * SPDX-License-Identifier: BSD-2-Clause-Patent */ @@ -274,17 +274,18 @@ void mock_ds_mgmt_pool_list_cont_teardown(void) } } -int ds_mgmt_pool_query_return; -uuid_t ds_mgmt_pool_query_uuid; -daos_pool_info_t ds_mgmt_pool_query_info_out; -daos_pool_info_t ds_mgmt_pool_query_info_in; -void *ds_mgmt_pool_query_info_ptr; -d_rank_list_t *ds_mgmt_pool_query_ranks_out; +int ds_mgmt_pool_query_return; +uuid_t ds_mgmt_pool_query_uuid; +daos_pool_info_t ds_mgmt_pool_query_info_out; +daos_pool_info_t ds_mgmt_pool_query_info_in; +void *ds_mgmt_pool_query_info_ptr; +d_rank_list_t *ds_mgmt_pool_query_enabled_ranks_out; +d_rank_list_t *ds_mgmt_pool_query_disabled_ranks_out; int -ds_mgmt_pool_query(uuid_t pool_uuid, d_rank_list_t *svc_ranks, d_rank_list_t **ranks, - daos_pool_info_t *pool_info, uint32_t *pool_layout_ver, - uint32_t *upgrade_layout_ver) +ds_mgmt_pool_query(uuid_t pool_uuid, d_rank_list_t *svc_ranks, d_rank_list_t **enabled_ranks, + d_rank_list_t **disabled_ranks, daos_pool_info_t *pool_info, + uint32_t *pool_layout_ver, uint32_t *upgrade_layout_ver) { /* If function is to return with an error, pool_info and ranks will not be filled. */ if (ds_mgmt_pool_query_return != 0) @@ -292,14 +293,26 @@ ds_mgmt_pool_query(uuid_t pool_uuid, d_rank_list_t *svc_ranks, d_rank_list_t **r uuid_copy(ds_mgmt_pool_query_uuid, pool_uuid); ds_mgmt_pool_query_info_ptr = (void *)pool_info; - if (pool_info != NULL) { - ds_mgmt_pool_query_info_in = *pool_info; - *pool_info = ds_mgmt_pool_query_info_out; + + if (pool_info == NULL) + return ds_mgmt_pool_query_return; + + if ((pool_info->pi_bits & DPI_ENGINES_ENABLED) != 0) { + D_ASSERT(enabled_ranks != NULL); + + *enabled_ranks = d_rank_list_alloc(8); /* 0-7 ; caller must free this */ + ds_mgmt_pool_query_enabled_ranks_out = *enabled_ranks; } - if (ranks != NULL) { - *ranks = d_rank_list_alloc(8); /* 0-7 ; caller must free this */ - ds_mgmt_pool_query_ranks_out = *ranks; + if ((pool_info->pi_bits & DPI_ENGINES_DISABLED) != 0) { + D_ASSERT(disabled_ranks != NULL); + + *disabled_ranks = d_rank_list_alloc(4); /* 0-3 ; caller must free this */ + ds_mgmt_pool_query_disabled_ranks_out = *disabled_ranks; } + + ds_mgmt_pool_query_info_in = *pool_info; + *pool_info = ds_mgmt_pool_query_info_out; + return ds_mgmt_pool_query_return; /* 0 */ } @@ -310,7 +323,8 @@ mock_ds_mgmt_pool_query_setup(void) uuid_clear(ds_mgmt_pool_query_uuid); ds_mgmt_pool_query_info_ptr = NULL; memset(&ds_mgmt_pool_query_info_out, 0, sizeof(daos_pool_info_t)); - ds_mgmt_pool_query_ranks_out = NULL; + ds_mgmt_pool_query_enabled_ranks_out = NULL; + ds_mgmt_pool_query_disabled_ranks_out = NULL; } int ds_mgmt_pool_query_targets_return; diff --git a/src/mgmt/tests/mocks.h b/src/mgmt/tests/mocks.h index 4f4ddbc9522..8bd0ffe998a 100644 --- a/src/mgmt/tests/mocks.h +++ b/src/mgmt/tests/mocks.h @@ -1,5 +1,5 @@ /* - * (C) Copyright 2019-2022 Intel Corporation. + * (C) Copyright 2019-2024 Intel Corporation. * * SPDX-License-Identifier: BSD-2-Clause-Patent */ @@ -103,12 +103,14 @@ void mock_ds_mgmt_pool_extend_setup(void); /* * Mock ds_mgmt_pool_query */ -extern int ds_mgmt_pool_query_return; -extern uuid_t ds_mgmt_pool_query_uuid; -extern daos_pool_info_t ds_mgmt_pool_query_info_in; -extern daos_pool_info_t ds_mgmt_pool_query_info_out; -extern void *ds_mgmt_pool_query_info_ptr; -extern d_rank_list_t *ds_mgmt_pool_query_ranks_out; +extern int ds_mgmt_pool_query_return; +extern uuid_t ds_mgmt_pool_query_uuid; +extern daos_pool_info_t ds_mgmt_pool_query_info_in; +extern daos_pool_info_t ds_mgmt_pool_query_info_out; +extern void *ds_mgmt_pool_query_info_ptr; +extern d_rank_list_t *ds_mgmt_pool_query_enabled_ranks_out; +extern d_rank_list_t *ds_mgmt_pool_query_disabled_ranks_out; + void mock_ds_mgmt_pool_query_setup(void); /* diff --git a/src/mgmt/tests/srv_drpc_tests.c b/src/mgmt/tests/srv_drpc_tests.c index 599eb8db77e..2b5e0fb7dd4 100644 --- a/src/mgmt/tests/srv_drpc_tests.c +++ b/src/mgmt/tests/srv_drpc_tests.c @@ -1269,22 +1269,6 @@ expect_drpc_pool_query_resp_with_error(Drpc__Response *resp, int expected_err) mgmt__pool_query_resp__free_unpacked(pq_resp, NULL); } -static void -test_drpc_pool_query_incompat_ranks_flags(void **state) -{ - Drpc__Call call = DRPC__CALL__INIT; - Drpc__Response resp = DRPC__RESPONSE__INIT; - - setup_pool_query_drpc_call(&call, TEST_UUID, DPI_ENGINES_DISABLED | DPI_ENGINES_ENABLED); - - ds_mgmt_drpc_pool_query(&call, &resp); - - expect_drpc_pool_query_resp_with_error(&resp, -DER_NOTSUPPORTED); - - D_FREE(call.body.data); - D_FREE(resp.body.data); -} - static void test_drpc_pool_query_bad_uuid(void **state) { @@ -1429,7 +1413,7 @@ test_drpc_pool_query_success(void **state) init_test_rebuild_status(&exp_info.pi_rebuild_st); ds_mgmt_pool_query_info_out = exp_info; - setup_pool_query_drpc_call(&call, TEST_UUID, DPI_ENGINES_ENABLED); + setup_pool_query_drpc_call(&call, TEST_UUID, DPI_ENGINES_ENABLED | DPI_ENGINES_DISABLED); ds_mgmt_drpc_pool_query(&call, &resp); @@ -1438,9 +1422,10 @@ test_drpc_pool_query_success(void **state) return; assert_int_equal(uuid_compare(exp_uuid, ds_mgmt_pool_query_uuid), 0); assert_non_null(ds_mgmt_pool_query_info_ptr); - assert_non_null(ds_mgmt_pool_query_ranks_out); + assert_non_null(ds_mgmt_pool_query_enabled_ranks_out); + assert_non_null(ds_mgmt_pool_query_disabled_ranks_out); assert_int_equal(ds_mgmt_pool_query_info_in.pi_bits, - DEFAULT_QUERY_BITS | DPI_ENGINES_ENABLED); + DEFAULT_QUERY_BITS | DPI_ENGINES_ENABLED | DPI_ENGINES_DISABLED); expect_query_resp_with_info(&exp_info, MGMT__POOL_REBUILD_STATUS__STATE__IDLE, @@ -3114,7 +3099,6 @@ main(void) POOL_EXTEND_TEST(test_drpc_extend_mgmt_svc_fails), POOL_EXTEND_TEST(test_drpc_extend_success), REINTEGRATE_TEST(test_drpc_reintegrate_bad_uuid), - QUERY_TEST(test_drpc_pool_query_incompat_ranks_flags), QUERY_TEST(test_drpc_pool_query_bad_uuid), QUERY_TEST(test_drpc_pool_query_mgmt_svc_fails), QUERY_TEST(test_drpc_pool_query_success), diff --git a/src/pool/srv_cli.c b/src/pool/srv_cli.c index 1decb57b3ec..0545480fe29 100644 --- a/src/pool/srv_cli.c +++ b/src/pool/srv_cli.c @@ -1,5 +1,5 @@ /* - * (C) Copyright 2017-2023 Intel Corporation. + * (C) Copyright 2017-2024 Intel Corporation. * * SPDX-License-Identifier: BSD-2-Clause-Patent */ @@ -335,13 +335,14 @@ dsc_pool_svc_call(uuid_t uuid, d_rank_list_t *ranks, struct dsc_pool_svc_call_cb } struct pool_query_arg { - d_rank_list_t **pqa_ranks; - daos_pool_info_t *pqa_info; - uint32_t *pqa_layout_ver; - uint32_t *pqa_upgrade_layout_ver; - crt_bulk_t pqa_bulk; - struct pool_buf *pqa_map_buf; - uint32_t pqa_map_size; + d_rank_list_t **pqa_enabled_ranks; + d_rank_list_t **pqa_disabled_ranks; + daos_pool_info_t *pqa_info; + uint32_t *pqa_layout_ver; + uint32_t *pqa_upgrade_layout_ver; + crt_bulk_t pqa_bulk; + struct pool_buf *pqa_map_buf; + uint32_t pqa_map_size; }; static int @@ -367,46 +368,77 @@ pool_query_init(uuid_t pool_uuid, crt_rpc_t *rpc, void *varg) } static int -process_query_result(d_rank_list_t **ranks, daos_pool_info_t *info, uuid_t pool_uuid, - uint32_t map_version, uint32_t leader_rank, struct daos_pool_space *ps, - struct daos_rebuild_status *rs, struct pool_buf *map_buf) +process_query_result(d_rank_list_t **enabled_ranks, d_rank_list_t **disabled_ranks, + daos_pool_info_t *info, uuid_t pool_uuid, uint32_t map_version, + uint32_t leader_rank, struct daos_pool_space *ps, + struct daos_rebuild_status *rs, struct pool_buf *map_buf, uint64_t pi_bits) { - struct pool_map *map; - int rc; - unsigned int num_disabled = 0; + struct pool_map *map = NULL; + unsigned int num_disabled = 0; + d_rank_list_t *enabled_rank_list = NULL; + d_rank_list_t *disabled_rank_list = NULL; + int rc; rc = pool_map_create(map_buf, map_version, &map); if (rc != 0) { - D_ERROR(DF_UUID": failed to create local pool map, "DF_RC"\n", - DP_UUID(pool_uuid), DP_RC(rc)); - return rc; + DL_ERROR(rc, DF_UUID ": failed to create local pool map", DP_UUID(pool_uuid)); + D_GOTO(error, rc); } rc = pool_map_find_failed_tgts(map, NULL, &num_disabled); if (rc != 0) { - D_ERROR(DF_UUID": failed to get num disabled tgts, "DF_RC"\n", - DP_UUID(pool_uuid), DP_RC(rc)); - goto out; + DL_ERROR(rc, DF_UUID ": failed to get num disabled tgts", DP_UUID(pool_uuid)); + D_GOTO(error, rc); } - info->pi_ndisabled = num_disabled; - if (ranks != NULL) { - bool get_enabled = (info ? ((info->pi_bits & DPI_ENGINES_ENABLED) != 0) : false); + if ((pi_bits & DPI_ENGINES_ENABLED) != 0) { + if (enabled_ranks == NULL) { + DL_ERROR(-DER_INVAL, + DF_UUID ": query pool requested enabled ranks, but ptr is NULL", + DP_UUID(pool_uuid)); + D_GOTO(error, rc = -DER_INVAL); + } - rc = pool_map_get_ranks(pool_uuid, map, get_enabled, ranks); + rc = pool_map_get_ranks(pool_uuid, map, true, &enabled_rank_list); if (rc != 0) { - D_ERROR(DF_UUID": pool_map_get_ranks() failed, "DF_RC"\n", - DP_UUID(pool_uuid), DP_RC(rc)); - goto out; + DL_ERROR(rc, DF_UUID ": pool_map_get_ranks() failed", DP_UUID(pool_uuid)); + D_GOTO(error, rc); } - D_DEBUG(DB_MD, DF_UUID": found %u %s ranks in pool map\n", - DP_UUID(pool_uuid), (*ranks)->rl_nr, get_enabled ? "ENABLED" : "DISABLED"); + D_DEBUG(DB_MD, DF_UUID ": found %" PRIu32 " enabled ranks in pool map\n", + DP_UUID(pool_uuid), enabled_rank_list->rl_nr); } - pool_query_reply_to_info(pool_uuid, map_buf, map_version, leader_rank, ps, rs, info); + if ((pi_bits & DPI_ENGINES_DISABLED) != 0) { + if (disabled_ranks == NULL) { + DL_ERROR(-DER_INVAL, + DF_UUID ": query pool requested disabled ranks, but ptr is NULL", + DP_UUID(pool_uuid)); + D_GOTO(error, rc = -DER_INVAL); + } + rc = pool_map_get_ranks(pool_uuid, map, false, &disabled_rank_list); + if (rc != 0) { + DL_ERROR(rc, DF_UUID ": pool_map_get_ranks() failed", DP_UUID(pool_uuid)); + D_GOTO(error, rc); + } + D_DEBUG(DB_MD, DF_UUID ": found %" PRIu32 " disabled ranks in pool map\n", + DP_UUID(pool_uuid), disabled_rank_list->rl_nr); + } + + pool_query_reply_to_info(pool_uuid, map_buf, map_version, leader_rank, ps, rs, info); + info->pi_ndisabled = num_disabled; + if (enabled_rank_list != NULL) + *enabled_ranks = enabled_rank_list; + if (disabled_rank_list != NULL) + *disabled_ranks = disabled_rank_list; + D_GOTO(out, rc = -DER_SUCCESS); + +error: + d_rank_list_free(disabled_rank_list); + d_rank_list_free(enabled_rank_list); out: - pool_map_decref(map); + if (map != NULL) + pool_map_decref(map); return rc; } @@ -432,9 +464,10 @@ pool_query_consume(uuid_t pool_uuid, crt_rpc_t *rpc, void *varg) D_DEBUG(DB_MGMT, DF_UUID": Successfully queried pool\n", DP_UUID(pool_uuid)); - rc = process_query_result(arg->pqa_ranks, arg->pqa_info, pool_uuid, - out->pqo_op.po_map_version, out->pqo_op.po_hint.sh_rank, - &out->pqo_space, &out->pqo_rebuild_st, arg->pqa_map_buf); + rc = process_query_result(arg->pqa_enabled_ranks, arg->pqa_disabled_ranks, arg->pqa_info, + pool_uuid, out->pqo_op.po_map_version, + out->pqo_op.po_hint.sh_rank, &out->pqo_space, + &out->pqo_rebuild_st, arg->pqa_map_buf, arg->pqa_info->pi_bits); if (arg->pqa_layout_ver) *arg->pqa_layout_ver = out->pqo_pool_layout_ver; if (arg->pqa_upgrade_layout_ver) @@ -465,34 +498,35 @@ static struct dsc_pool_svc_call_cbs pool_query_cbs = { /** * Query the pool without holding a pool handle. * - * \param[in] pool_uuid UUID of the pool - * \param[in] ps_ranks Ranks of pool svc replicas - * \param[in] deadline Unix time deadline in milliseconds - * \param[out] ranks Optional, returned storage ranks in this pool. - * If #pool_info is NULL, engines with disabled targets. - * If #pool_info is passed, engines with enabled or disabled - * targets according to #pi_bits (DPI_ENGINES_ENABLED bit). - * Note: ranks may be empty (i.e., *ranks->rl_nr may be 0). - * The caller must free the list with d_rank_list_free(). - * \param[out] pool_info Results of the pool query - * \param[out] pool_layout_ver Results of the current pool global version - * \param[out] pool_upgrade_layout_ver Results of the target latest pool global version + * \param[in] pool_uuid UUID of the pool + * \param[in] ps_ranks Ranks of pool svc replicas + * \param[in] deadline Unix time deadline in milliseconds + * \param[out] enabled_ranks Optional, storage ranks with enabled targets. + * \param[out] disabled_ranks Optional, storage ranks with disabled targets. + * \param[in][out] pool_info Results of the pool query + * \param[in][out] pool_layout_ver Results of the current pool global version + * \param[in][out] upgrade_layout_ver Results of the target latest pool global version * * \return 0 Success * -DER_INVAL Invalid input * Negative value Error + * + * \note The enabled_ranks and disabled_ranks may be empty (i.e., *ranks->rl_nr may be 0). + * \note The caller must free the lists enabled_ranks and disabled_ranks with d_rank_list_free(). */ int dsc_pool_svc_query(uuid_t pool_uuid, d_rank_list_t *ps_ranks, uint64_t deadline, - d_rank_list_t **ranks, daos_pool_info_t *pool_info, uint32_t *pool_layout_ver, + d_rank_list_t **enabled_ranks, d_rank_list_t **disabled_ranks, + daos_pool_info_t *pool_info, uint32_t *pool_layout_ver, uint32_t *upgrade_layout_ver) { struct pool_query_arg arg = { - .pqa_ranks = ranks, - .pqa_info = pool_info, - .pqa_layout_ver = pool_layout_ver, - .pqa_upgrade_layout_ver = upgrade_layout_ver, - .pqa_map_size = 127 /* 4 KB */ + .pqa_enabled_ranks = enabled_ranks, + .pqa_disabled_ranks = disabled_ranks, + .pqa_info = pool_info, + .pqa_layout_ver = pool_layout_ver, + .pqa_upgrade_layout_ver = upgrade_layout_ver, + .pqa_map_size = 127 /* 4 KB */ }; return dsc_pool_svc_call(pool_uuid, ps_ranks, &pool_query_cbs, &arg, deadline); diff --git a/src/tests/ftest/control/dmg_pool_query_ranks.py b/src/tests/ftest/control/dmg_pool_query_ranks.py index cb23d2c5af1..2522e3fad3a 100644 --- a/src/tests/ftest/control/dmg_pool_query_ranks.py +++ b/src/tests/ftest/control/dmg_pool_query_ranks.py @@ -3,6 +3,7 @@ SPDX-License-Identifier: BSD-2-Clause-Patent """ + import time from control_test_base import ControlTestBase @@ -41,57 +42,68 @@ def test_pool_query_ranks_basic(self): self.log.debug("Checking without ranks state information") data = self.dmg.pool_query(self.pool.identifier) self.assertIsNone( - data['response'].get('enabled_ranks'), + data["response"].get("enabled_ranks"), "Invalid enabled_ranks field: want=None, got={}".format( - data['response'].get('enabled_ranks'))) + data["response"].get("enabled_ranks") + ), + ) self.assertIsNone( - data['response'].get('disabled_ranks'), + data["response"].get("disabled_ranks"), "Invalid disabled_ranks field: want=None, got={}".format( - data['response'].get('disabled_ranks'))) + data["response"].get("disabled_ranks") + ), + ) self.log.debug("Checking enabled ranks state information") data = self.dmg.pool_query(self.pool.identifier, show_enabled=True) self.assertListEqual( - data['response'].get('enabled_ranks'), [0, 1, 2], + data["response"].get("enabled_ranks"), + [0, 1, 2], "Invalid enabled_ranks field: want=[0, 1, 2], got={}".format( - data['response'].get('enabled_ranks'))) + data["response"].get("enabled_ranks") + ), + ) self.assertIsNone( - data['response'].get('disabled_ranks'), + data["response"].get("disabled_ranks"), "Invalid disabled_ranks field: want=None, got={}".format( - data['response'].get('disabled_ranks'))) + data["response"].get("disabled_ranks") + ), + ) self.log.debug("Checking disabled ranks state information") data = self.dmg.pool_query(self.pool.identifier, show_disabled=True) self.assertIsNone( - data['response'].get('enabled_ranks'), + data["response"].get("enabled_ranks"), "Invalid enabled_ranks field: want=None, got={}".format( - data['response'].get('enabled_ranks'))) + data["response"].get("enabled_ranks") + ), + ) self.assertListEqual( - data['response'].get('disabled_ranks'), [], + data["response"].get("disabled_ranks"), + [], "Invalid disabled_ranks field: want=[], got={}".format( - data['response'].get('disabled_ranks'))) - - def test_pool_query_ranks_error(self): - """Test that ranks state option are mutually exclusive. - - Test Description: - Check that options '--show-enabled' and '--show-disabled" are mutually exclusive. - - :avocado: tags=all,daily_regression - :avocado: tags=vm - :avocado: tags=dmg,control,pool_query,pool_query_ranks - :avocado: tags=DmgPoolQueryRanks,test_pool_query_ranks_error - """ - self.log.info("Tests of pool query with incompatible options") - - # Disable raising an exception if the dmg command fails - self.dmg.exit_status_exception = False - try: - data = self.dmg.pool_query(self.pool.identifier, show_enabled=True, show_disabled=True) - self.assertIsNotNone(data["error"], "Expected error not returned") - self.assertIn(r'may not be mixed with', str(data['error']), "Invalid error message") - finally: - self.dmg.exit_status_exception = True + data["response"].get("disabled_ranks") + ), + ) + + self.log.debug("Checking enabled and disabled ranks state information") + data = self.dmg.pool_query( + self.pool.identifier, show_enabled=True, show_disabled=True + ) + self.assertListEqual( + data["response"]["enabled_ranks"], + [0, 1, 2], + "Invalid enabled_ranks field: want=[0, 1, 2], got={}".format( + data["response"]["enabled_ranks"] + ), + ) + self.assertListEqual( + data["response"]["disabled_ranks"], + [], + "Invalid disabled_ranks field: want=[], got={}".format( + data["response"]["disabled_ranks"] + ), + ) def test_pool_query_ranks_mgmt(self): """Test the state of ranks after excluding and reintegrate them. @@ -121,18 +133,23 @@ def test_pool_query_ranks_mgmt(self): disabled_ranks = sorted(disabled_ranks + [rank]) self.log.debug("Checking enabled ranks state information") - data = self.dmg.pool_query(self.pool.identifier, show_enabled=True) + data = self.dmg.pool_query( + self.pool.identifier, show_enabled=True, show_disabled=True + ) self.assertListEqual( - data['response'].get('enabled_ranks'), enabled_ranks, + data["response"].get("enabled_ranks"), + enabled_ranks, "Invalid enabled_ranks field: want={}, got={}".format( - enabled_ranks, data['response'].get('enabled_ranks'))) - - self.log.debug("Checking disabled ranks state information") - data = self.dmg.pool_query(self.pool.identifier, show_disabled=True) + enabled_ranks, data["response"]["enabled_ranks"] + ), + ) self.assertListEqual( - data['response'].get('disabled_ranks'), disabled_ranks, + data["response"].get("disabled_ranks"), + disabled_ranks, "Invalid disabled_ranks field: want={}, got={}".format( - disabled_ranks, data['response'].get('disabled_ranks'))) + disabled_ranks, data["response"].get("disabled_ranks") + ), + ) self.log.debug("Waiting for pool to be rebuild") self.pool.wait_for_rebuild_to_start() @@ -158,18 +175,23 @@ def test_pool_query_ranks_mgmt(self): disabled_ranks.remove(rank) self.log.debug("Checking enabled ranks state information") - data = self.dmg.pool_query(self.pool.identifier, show_enabled=True) + data = self.dmg.pool_query( + self.pool.identifier, show_enabled=True, show_disabled=True + ) self.assertListEqual( - data['response'].get('enabled_ranks'), enabled_ranks, + data["response"].get("enabled_ranks"), + enabled_ranks, "Invalid enabled_ranks field: want={}, got={}".format( - enabled_ranks, data['response'].get('enabled_ranks'))) - - self.log.debug("Checking disabled ranks state information") - data = self.dmg.pool_query(self.pool.identifier, show_disabled=True) + enabled_ranks, data["response"]["enabled_ranks"] + ), + ) self.assertListEqual( - data['response'].get('disabled_ranks'), disabled_ranks, + data["response"].get("disabled_ranks"), + disabled_ranks, "Invalid disabled_ranks field: want={}, got={}".format( - disabled_ranks, data['response'].get('disabled_ranks'))) + disabled_ranks, data["response"].get("disabled_ranks") + ), + ) self.log.debug("Waiting for pool to be rebuild") self.pool.wait_for_rebuild_to_start() From 03ff4ad771b29531515faccf576c2fc4c6a3f2c3 Mon Sep 17 00:00:00 2001 From: Cedric Koch-Hofer <94527853+knard-intel@users.noreply.github.com> Date: Thu, 3 Oct 2024 15:22:56 +0200 Subject: [PATCH 4/7] DAOS-14419 control: Display disabled ranks by default (#15112) Always display the disabled targets and remove the old associated options. Required-githooks: true Signed-off-by: Cedric Koch-Hofer --- src/control/cmd/daos/pool.go | 12 ++--- src/control/cmd/daos/pretty/pool.go | 2 +- src/control/cmd/dmg/pool.go | 9 ++-- src/control/cmd/dmg/pool_test.go | 48 ------------------- src/control/lib/control/pool_test.go | 4 +- src/control/lib/daos/pool.go | 2 +- src/control/lib/daos/pool_test.go | 2 +- .../ftest/control/dmg_pool_query_ranks.py | 45 ++--------------- .../ftest/control/dmg_pool_query_test.py | 1 + .../ftest/control/dmg_pool_query_test.yaml | 3 +- src/tests/ftest/deployment/network_failure.py | 2 +- .../ftest/deployment/server_rank_failure.py | 2 +- src/tests/ftest/util/dmg_utils.py | 6 +-- src/tests/ftest/util/dmg_utils_base.py | 1 - src/tests/ftest/util/test_utils_pool.py | 10 ++-- 15 files changed, 26 insertions(+), 123 deletions(-) diff --git a/src/control/cmd/daos/pool.go b/src/control/cmd/daos/pool.go index 2aae717766e..7a917b1a9e4 100644 --- a/src/control/cmd/daos/pool.go +++ b/src/control/cmd/daos/pool.go @@ -224,9 +224,8 @@ type poolCmd struct { type poolQueryCmd struct { poolBaseCmd - ShowEnabledRanks bool `short:"e" long:"show-enabled" description:"Show engine unique identifiers (ranks) which are enabled"` - ShowDisabledRanks bool `short:"b" long:"show-disabled" description:"Show engine unique identifiers (ranks) which are disabled"` - HealthOnly bool `short:"t" long:"health-only" description:"Only perform pool health related queries"` + ShowEnabledRanks bool `short:"e" long:"show-enabled" description:"Show engine unique identifiers (ranks) which are enabled"` + HealthOnly bool `short:"t" long:"health-only" description:"Only perform pool health related queries"` } func convertPoolSpaceInfo(in *C.struct_daos_pool_space, mt C.uint) *daos.StorageUsageStats { @@ -340,15 +339,10 @@ func (cmd *poolQueryCmd) Execute(_ []string) error { if cmd.HealthOnly { queryMask = daos.HealthOnlyPoolQueryMask } - if cmd.ShowEnabledRanks && cmd.ShowDisabledRanks { - return errors.New("show-enabled and show-disabled can't be used at the same time.") - } if cmd.ShowEnabledRanks { queryMask.SetOptions(daos.PoolQueryOptionEnabledEngines) } - if cmd.ShowDisabledRanks { - queryMask.SetOptions(daos.PoolQueryOptionDisabledEngines) - } + queryMask.SetOptions(daos.PoolQueryOptionDisabledEngines) cleanup, err := cmd.resolveAndConnect(C.DAOS_PC_RO, nil) if err != nil { diff --git a/src/control/cmd/daos/pretty/pool.go b/src/control/cmd/daos/pretty/pool.go index aa70115152d..a9f685b536f 100644 --- a/src/control/cmd/daos/pretty/pool.go +++ b/src/control/cmd/daos/pretty/pool.go @@ -50,7 +50,7 @@ func PrintPoolInfo(pi *daos.PoolInfo, out io.Writer) error { if pi.EnabledRanks != nil && pi.EnabledRanks.Count() > 0 { fmt.Fprintf(w, "- Enabled ranks: %s\n", pi.EnabledRanks) } - if pi.DisabledRanks != nil && pi.DisabledRanks.Count() > 0 { + if pi.DisabledRanks.Count() > 0 { fmt.Fprintf(w, "- Disabled ranks: %s\n", pi.DisabledRanks) } if pi.Rebuild != nil { diff --git a/src/control/cmd/dmg/pool.go b/src/control/cmd/dmg/pool.go index 7edf2c6652f..400ef237ca0 100644 --- a/src/control/cmd/dmg/pool.go +++ b/src/control/cmd/dmg/pool.go @@ -606,9 +606,8 @@ func (cmd *PoolReintegrateCmd) Execute(args []string) error { // PoolQueryCmd is the struct representing the command to query a DAOS pool. type PoolQueryCmd struct { poolCmd - ShowEnabledRanks bool `short:"e" long:"show-enabled" description:"Show engine unique identifiers (ranks) which are enabled"` - ShowDisabledRanks bool `short:"b" long:"show-disabled" description:"Show engine unique identifiers (ranks) which are disabled"` - HealthOnly bool `short:"t" long:"health-only" description:"Only perform pool health related queries"` + ShowEnabledRanks bool `short:"e" long:"show-enabled" description:"Show engine unique identifiers (ranks) which are enabled"` + HealthOnly bool `short:"t" long:"health-only" description:"Only perform pool health related queries"` } // Execute is run when PoolQueryCmd subcommand is activated @@ -624,9 +623,7 @@ func (cmd *PoolQueryCmd) Execute(args []string) error { if cmd.ShowEnabledRanks { req.QueryMask.SetOptions(daos.PoolQueryOptionEnabledEngines) } - if cmd.ShowDisabledRanks { - req.QueryMask.SetOptions(daos.PoolQueryOptionDisabledEngines) - } + req.QueryMask.SetOptions(daos.PoolQueryOptionDisabledEngines) resp, err := control.PoolQuery(cmd.MustLogCtx(), cmd.ctlInvoker, req) if cmd.JSONOutputEnabled() { diff --git a/src/control/cmd/dmg/pool_test.go b/src/control/cmd/dmg/pool_test.go index 67bf72bf54e..84e5dda9521 100644 --- a/src/control/cmd/dmg/pool_test.go +++ b/src/control/cmd/dmg/pool_test.go @@ -1025,54 +1025,6 @@ func TestPoolCommands(t *testing.T) { }, " "), nil, }, - { - "Query pool with UUID and disabled ranks", - "pool query --show-disabled 12345678-1234-1234-1234-1234567890ab", - strings.Join([]string{ - printRequest(t, &control.PoolQueryReq{ - ID: "12345678-1234-1234-1234-1234567890ab", - QueryMask: setQueryMask(func(qm *daos.PoolQueryMask) { qm.SetOptions(daos.PoolQueryOptionDisabledEngines) }), - }), - }, " "), - nil, - }, - { - "Query pool with UUID and disabled ranks", - "pool query -b 12345678-1234-1234-1234-1234567890ab", - strings.Join([]string{ - printRequest(t, &control.PoolQueryReq{ - ID: "12345678-1234-1234-1234-1234567890ab", - QueryMask: setQueryMask(func(qm *daos.PoolQueryMask) { qm.SetOptions(daos.PoolQueryOptionDisabledEngines) }), - }), - }, " "), - nil, - }, - { - "Query pool with UUID, enabled ranks and disabled ranks", - "pool query --show-disabled --show-enabled 12345678-1234-1234-1234-1234567890ab", - strings.Join([]string{ - printRequest(t, &control.PoolQueryReq{ - ID: "12345678-1234-1234-1234-1234567890ab", - QueryMask: setQueryMask(func(qm *daos.PoolQueryMask) { - qm.SetOptions(daos.PoolQueryOptionEnabledEngines, daos.PoolQueryOptionDisabledEngines) - }), - }), - }, " "), - nil, - }, - { - "Query pool with UUID, enabled ranks and disabled ranks", - "pool query -b -e 12345678-1234-1234-1234-1234567890ab", - strings.Join([]string{ - printRequest(t, &control.PoolQueryReq{ - ID: "12345678-1234-1234-1234-1234567890ab", - QueryMask: setQueryMask(func(qm *daos.PoolQueryMask) { - qm.SetOptions(daos.PoolQueryOptionEnabledEngines, daos.PoolQueryOptionDisabledEngines) - }), - }), - }, " "), - nil, - }, { "Query pool for health only", "pool query --health-only 12345678-1234-1234-1234-1234567890ab", diff --git a/src/control/lib/control/pool_test.go b/src/control/lib/control/pool_test.go index 6849b88b053..c111ba183cc 100644 --- a/src/control/lib/control/pool_test.go +++ b/src/control/lib/control/pool_test.go @@ -807,7 +807,7 @@ func TestControl_PoolQueryResp_MarshalJSON(t *testing.T) { UpgradeLayoutVer: 8, }, }, - exp: `{"query_mask":"rebuild,space","state":"Ready","uuid":"` + poolUUID.String() + `","total_targets":1,"active_targets":2,"total_engines":3,"disabled_targets":4,"version":5,"svc_ldr":6,"svc_reps":[0,1,2],"rebuild":null,"tier_stats":null,"pool_layout_ver":7,"upgrade_layout_ver":8,"status":42}`, + exp: `{"query_mask":"disabled_engines,rebuild,space","state":"Ready","uuid":"` + poolUUID.String() + `","total_targets":1,"active_targets":2,"total_engines":3,"disabled_targets":4,"version":5,"svc_ldr":6,"svc_reps":[0,1,2],"rebuild":null,"tier_stats":null,"pool_layout_ver":7,"upgrade_layout_ver":8,"status":42}`, }, "valid rankset": { pqr: &PoolQueryResp{ @@ -829,7 +829,7 @@ func TestControl_PoolQueryResp_MarshalJSON(t *testing.T) { UpgradeLayoutVer: 8, }, }, - exp: `{"query_mask":"rebuild,space","state":"Ready","uuid":"` + poolUUID.String() + `","total_targets":1,"active_targets":2,"total_engines":3,"disabled_targets":4,"version":5,"svc_ldr":6,"svc_reps":[0,1,2],"rebuild":null,"tier_stats":null,"enabled_ranks":[0,1,2,3,5],"disabled_ranks":[],"pool_layout_ver":7,"upgrade_layout_ver":8,"status":42}`, + exp: `{"query_mask":"disabled_engines,rebuild,space","state":"Ready","uuid":"` + poolUUID.String() + `","total_targets":1,"active_targets":2,"total_engines":3,"disabled_targets":4,"version":5,"svc_ldr":6,"svc_reps":[0,1,2],"rebuild":null,"tier_stats":null,"enabled_ranks":[0,1,2,3,5],"disabled_ranks":[],"pool_layout_ver":7,"upgrade_layout_ver":8,"status":42}`, }, } { t.Run(name, func(t *testing.T) { diff --git a/src/control/lib/daos/pool.go b/src/control/lib/daos/pool.go index fe44a00e210..e47e6e2b23d 100644 --- a/src/control/lib/daos/pool.go +++ b/src/control/lib/daos/pool.go @@ -104,7 +104,7 @@ type ( const ( // DefaultPoolQueryMask defines the default pool query mask. - DefaultPoolQueryMask = PoolQueryMask(^uint64(0) &^ (C.DPI_ENGINES_ENABLED | C.DPI_ENGINES_DISABLED)) + DefaultPoolQueryMask = PoolQueryMask(^uint64(0) &^ C.DPI_ENGINES_ENABLED) // HealthOnlyPoolQueryMask defines the mask for health-only queries. HealthOnlyPoolQueryMask = PoolQueryMask(^uint64(0) &^ (C.DPI_ENGINES_ENABLED | C.DPI_SPACE)) diff --git a/src/control/lib/daos/pool_test.go b/src/control/lib/daos/pool_test.go index e76f33f4c25..3ff1ca098d2 100644 --- a/src/control/lib/daos/pool_test.go +++ b/src/control/lib/daos/pool_test.go @@ -130,7 +130,7 @@ func TestDaos_PoolQueryMask(t *testing.T) { testMask: genTestMask(func(pqm *PoolQueryMask) { *pqm = DefaultPoolQueryMask }), - expString: genOptsStr(PoolQueryOptionRebuild, PoolQueryOptionSpace), + expString: genOptsStr(PoolQueryOptionDisabledEngines, PoolQueryOptionRebuild, PoolQueryOptionSpace), }, "health-only query mask": { testMask: genTestMask(func(pqm *PoolQueryMask) { diff --git a/src/tests/ftest/control/dmg_pool_query_ranks.py b/src/tests/ftest/control/dmg_pool_query_ranks.py index 2522e3fad3a..c84d2dc3c80 100644 --- a/src/tests/ftest/control/dmg_pool_query_ranks.py +++ b/src/tests/ftest/control/dmg_pool_query_ranks.py @@ -47,37 +47,6 @@ def test_pool_query_ranks_basic(self): data["response"].get("enabled_ranks") ), ) - self.assertIsNone( - data["response"].get("disabled_ranks"), - "Invalid disabled_ranks field: want=None, got={}".format( - data["response"].get("disabled_ranks") - ), - ) - - self.log.debug("Checking enabled ranks state information") - data = self.dmg.pool_query(self.pool.identifier, show_enabled=True) - self.assertListEqual( - data["response"].get("enabled_ranks"), - [0, 1, 2], - "Invalid enabled_ranks field: want=[0, 1, 2], got={}".format( - data["response"].get("enabled_ranks") - ), - ) - self.assertIsNone( - data["response"].get("disabled_ranks"), - "Invalid disabled_ranks field: want=None, got={}".format( - data["response"].get("disabled_ranks") - ), - ) - - self.log.debug("Checking disabled ranks state information") - data = self.dmg.pool_query(self.pool.identifier, show_disabled=True) - self.assertIsNone( - data["response"].get("enabled_ranks"), - "Invalid enabled_ranks field: want=None, got={}".format( - data["response"].get("enabled_ranks") - ), - ) self.assertListEqual( data["response"].get("disabled_ranks"), [], @@ -86,10 +55,8 @@ def test_pool_query_ranks_basic(self): ), ) - self.log.debug("Checking enabled and disabled ranks state information") - data = self.dmg.pool_query( - self.pool.identifier, show_enabled=True, show_disabled=True - ) + self.log.debug("Checking enabled ranks state information") + data = self.dmg.pool_query(self.pool.identifier, show_enabled=True) self.assertListEqual( data["response"]["enabled_ranks"], [0, 1, 2], @@ -133,9 +100,7 @@ def test_pool_query_ranks_mgmt(self): disabled_ranks = sorted(disabled_ranks + [rank]) self.log.debug("Checking enabled ranks state information") - data = self.dmg.pool_query( - self.pool.identifier, show_enabled=True, show_disabled=True - ) + data = self.dmg.pool_query(self.pool.identifier, show_enabled=True) self.assertListEqual( data["response"].get("enabled_ranks"), enabled_ranks, @@ -175,9 +140,7 @@ def test_pool_query_ranks_mgmt(self): disabled_ranks.remove(rank) self.log.debug("Checking enabled ranks state information") - data = self.dmg.pool_query( - self.pool.identifier, show_enabled=True, show_disabled=True - ) + data = self.dmg.pool_query(self.pool.identifier, show_enabled=True) self.assertListEqual( data["response"].get("enabled_ranks"), enabled_ranks, diff --git a/src/tests/ftest/control/dmg_pool_query_test.py b/src/tests/ftest/control/dmg_pool_query_test.py index ec8c56c6260..b7c83b59b55 100644 --- a/src/tests/ftest/control/dmg_pool_query_test.py +++ b/src/tests/ftest/control/dmg_pool_query_test.py @@ -62,6 +62,7 @@ def test_pool_query_basic(self): "uuid": self.pool.uuid.lower(), "total_targets": self.params.get("total_targets", path="/run/exp_vals/*"), "active_targets": self.params.get("active_targets", path="/run/exp_vals/*"), + "disabled_ranks": self.params.get("disabled_ranks", path="/run/exp_vals/*"), "total_engines": self.params.get("total_engines", path="/run/exp_vals/*"), "disabled_targets": self.params.get("disabled_targets", path="/run/exp_vals/*"), "version": self.params.get("version", path="/run/exp_vals/*"), diff --git a/src/tests/ftest/control/dmg_pool_query_test.yaml b/src/tests/ftest/control/dmg_pool_query_test.yaml index 8c92ae36945..9b105da40ff 100644 --- a/src/tests/ftest/control/dmg_pool_query_test.yaml +++ b/src/tests/ftest/control/dmg_pool_query_test.yaml @@ -28,12 +28,13 @@ exp_vals: pool_state: "Ready" total_targets: 4 active_targets: 4 + disabled_ranks: [] total_engines: 1 disabled_targets: 0 version: 1 leader: 0 replicas: [0] - query_mask: "rebuild,space" + query_mask: "disabled_engines,rebuild,space" scm: total: 16000008192 nvme: diff --git a/src/tests/ftest/deployment/network_failure.py b/src/tests/ftest/deployment/network_failure.py index ec67962c3d5..84c7b259370 100644 --- a/src/tests/ftest/deployment/network_failure.py +++ b/src/tests/ftest/deployment/network_failure.py @@ -227,7 +227,7 @@ def verify_network_failure(self, ior_namespace, container_namespace): # 6. Call dmg pool query -b to find the disabled ranks. self.log_step("Find the disabled ranks.") - output = dmg_cmd.pool_query(pool=self.pool.identifier, show_disabled=True) + output = dmg_cmd.pool_query(pool=self.pool.identifier) disabled_ranks = output["response"].get("disabled_ranks") self.log.info("Disabled ranks = %s", disabled_ranks) diff --git a/src/tests/ftest/deployment/server_rank_failure.py b/src/tests/ftest/deployment/server_rank_failure.py index d2156d8fc8d..d1637fe54eb 100644 --- a/src/tests/ftest/deployment/server_rank_failure.py +++ b/src/tests/ftest/deployment/server_rank_failure.py @@ -183,7 +183,7 @@ def verify_rank_failure(self, ior_namespace): errors.append("Server rank {} state isn't joined!".format(member["rank"])) # 9. Call dmg pool query -b to find the disabled ranks. - output = self.get_dmg_command().pool_query(pool=self.pool.identifier, show_disabled=True) + output = self.get_dmg_command().pool_query(pool=self.pool.identifier) disabled_ranks = output["response"].get("disabled_ranks") self.log.info("Disabled ranks = %s", disabled_ranks) diff --git a/src/tests/ftest/util/dmg_utils.py b/src/tests/ftest/util/dmg_utils.py index effc3172bac..a48b45e59dd 100644 --- a/src/tests/ftest/util/dmg_utils.py +++ b/src/tests/ftest/util/dmg_utils.py @@ -625,13 +625,12 @@ def pool_create(self, scm_size, uid=None, gid=None, nvme_size=None, return data - def pool_query(self, pool, show_enabled=False, show_disabled=False): + def pool_query(self, pool, show_enabled=False): """Query a pool with the dmg command. Args: pool (str): Pool UUID or label to query. show_enabled (bool, optional): Display enabled ranks. - show_disabled (bool, optional): Display disabled ranks. Raises: CommandFailure: if the dmg pool query command fails. @@ -677,8 +676,7 @@ def pool_query(self, pool, show_enabled=False, show_disabled=False): # "error": null, # "status": 0 # } - return self._get_json_result(("pool", "query"), pool=pool, - show_enabled=show_enabled, show_disabled=show_disabled) + return self._get_json_result(("pool", "query"), pool=pool, show_enabled=show_enabled) def pool_query_targets(self, pool, rank=None, target_idx=None): """Call dmg pool query-targets. diff --git a/src/tests/ftest/util/dmg_utils_base.py b/src/tests/ftest/util/dmg_utils_base.py index 39109320af1..951694a2251 100644 --- a/src/tests/ftest/util/dmg_utils_base.py +++ b/src/tests/ftest/util/dmg_utils_base.py @@ -533,7 +533,6 @@ def __init__(self): super().__init__("/run/dmg/pool/query/*", "query") self.pool = BasicParameter(None, position=1) self.show_enabled = FormattedParameter("--show-enabled", False) - self.show_disabled = FormattedParameter("--show-disabled", False) class QueryTargetsSubCommand(CommandWithParameters): """Defines an object for the dmg pool query-targets command.""" diff --git a/src/tests/ftest/util/test_utils_pool.py b/src/tests/ftest/util/test_utils_pool.py index fbb6484e292..7c6b5758f87 100644 --- a/src/tests/ftest/util/test_utils_pool.py +++ b/src/tests/ftest/util/test_utils_pool.py @@ -700,12 +700,11 @@ def overwrite_acl(self): self.log.error("self.acl_file isn't defined!") @fail_on(CommandFailure) - def query(self, show_enabled=False, show_disabled=False): + def query(self, show_enabled=False): """Execute dmg pool query. Args: show_enabled (bool, optional): Display enabled ranks. - show_disabled (bool, optional): Display disabled ranks. Returns: dict: the dmg json command output converted to a python dictionary @@ -720,7 +719,7 @@ def query(self, show_enabled=False, show_disabled=False): while True: try: - return self.dmg.pool_query(self.identifier, show_enabled, show_disabled) + return self.dmg.pool_query(self.identifier, show_enabled) except CommandFailure as error: if end_time is None: @@ -1119,19 +1118,18 @@ def pool_percentage_used(self): } return pool_percent - def set_query_data(self, show_enabled=False, show_disabled=False): + def set_query_data(self, show_enabled=False): """Execute dmg pool query and store the results. Args: show_enabled (bool, optional): Display enabled ranks. - show_disabled (bool, optional): Display disabled ranks. Raises: TestFail: if the dmg pool query command failed """ self.query_data = {} - self.query_data = self.query(show_enabled, show_disabled) + self.query_data = self.query(show_enabled) def _get_query_data_keys(self, *keys, refresh=False): """Get the pool version from the dmg pool query output. From b4c94c29600edbf6229222c5b6b2550f45df6681 Mon Sep 17 00:00:00 2001 From: Cedric Koch-Hofer <94527853+knard-intel@users.noreply.github.com> Date: Fri, 25 Oct 2024 14:49:50 +0200 Subject: [PATCH 5/7] DAOS-16669 test: fix pool list ftest (#15373) Fix regression on pool/list_verbose.py functional test introduced with DAOS-14419. Signed-off-by: Cedric Koch-Hofer --- src/tests/ftest/pool/list_verbose.py | 33 ++++++++++++++++++---------- 1 file changed, 22 insertions(+), 11 deletions(-) diff --git a/src/tests/ftest/pool/list_verbose.py b/src/tests/ftest/pool/list_verbose.py index 68df5f876a4..370ca81ad40 100644 --- a/src/tests/ftest/pool/list_verbose.py +++ b/src/tests/ftest/pool/list_verbose.py @@ -23,7 +23,8 @@ class ListVerboseTest(IorTestBase): def create_expected(self, pool, scm_free, nvme_free, scm_imbalance, nvme_imbalance, targets_disabled=0, scm_size=None, - nvme_size=None, state=None, rebuild_state=None): + nvme_size=None, state=None, rebuild_state=None, + ranks_disabled=None): # pylint: disable=too-many-arguments """Create expected dmg pool list output to compare against the actual. @@ -39,6 +40,7 @@ def create_expected(self, pool, scm_free, nvme_free, scm_imbalance, nvme_size (int, optional): NVMe size to fill in the output. Defaults to None. state (str, optional): Expected pool state. Defaults to None. rebuild_state (str, optional): Expected pool rebuild state. Defaults to None. + ranks_disabled (list, optional): List of disabled ranks. Defaults to None. Returns: dict: Expected in the same format of actual. @@ -61,7 +63,7 @@ def create_expected(self, pool, scm_free, nvme_free, scm_imbalance, upgrade_layout_ver = p_query["response"]["upgrade_layout_ver"] return { - "query_mask": "rebuild,space", + "query_mask": "disabled_engines,rebuild,space", "state": state, "uuid": pool.uuid.lower(), "label": pool.label.value, @@ -69,6 +71,7 @@ def create_expected(self, pool, scm_free, nvme_free, scm_imbalance, "active_targets": targets_total - targets_disabled, "total_engines": rank_count, "disabled_targets": targets_disabled, + "disabled_ranks": ranks_disabled if ranks_disabled else [], "svc_ldr": pool.svc_leader, "svc_reps": pool.svc_ranks, "upgrade_layout_ver": upgrade_layout_ver, @@ -171,7 +174,8 @@ def verify_scm_size(self, actual, created, rank_count): threshold, diff) self.assertTrue(diff < threshold, msg) - def verify_pool_lists(self, targets_disabled, scm_size, nvme_size, state, rebuild_state): + def verify_pool_lists(self, targets_disabled, scm_size, nvme_size, state, rebuild_state, + ranks_disabled): """Call dmg pool list and verify. self.pool should be a list. The elements of the inputs should @@ -183,6 +187,7 @@ def verify_pool_lists(self, targets_disabled, scm_size, nvme_size, state, rebuil nvme_size (list): List of NVMe size for pools. state (list): List of pool state for pools. rebuild_state (list): List of pool rebuild state for pools. + ranks_disabled (list): List of disabled ranks for pools. Returns: list: a list of dictionaries containing information for each pool from the dmg @@ -228,7 +233,8 @@ def verify_pool_lists(self, targets_disabled, scm_size, nvme_size, state, rebuil scm_size=pool_free_data["scm_size"], nvme_size=nvme_size[index], state=state[index], - rebuild_state=rebuild_state[index])) + rebuild_state=rebuild_state[index], + ranks_disabled=ranks_disabled[index])) # Sort pools by UUID. actual_pools.sort(key=lambda item: item.get("uuid")) @@ -294,9 +300,10 @@ def test_fields_basic(self): nvme_size = [None] state = ["Ready"] rebuild_state = ["idle"] + ranks_disabled = [[]] self.verify_pool_lists( targets_disabled=targets_disabled, scm_size=scm_size, nvme_size=nvme_size, - state=state, rebuild_state=rebuild_state) + state=state, rebuild_state=rebuild_state, ranks_disabled=ranks_disabled) # 3. Create second pool. self.log_step("Create second pool") @@ -310,13 +317,15 @@ def test_fields_basic(self): nvme_size.append(None) state.append("Ready") rebuild_state.append("idle") + ranks_disabled.append([]) self.verify_pool_lists( targets_disabled=targets_disabled, scm_size=scm_size, nvme_size=nvme_size, - state=state, rebuild_state=rebuild_state) + state=state, rebuild_state=rebuild_state, ranks_disabled=ranks_disabled) # 5. Exclude target 7 in rank 1 of pool 1. self.log_step("Exclude target 7 in rank 1 of pool 1") - self.pool[0].exclude(ranks=[1], tgt_idx="7") + ranks_disabled[0].append(1) + self.pool[0].exclude(ranks=ranks_disabled[0], tgt_idx="7") # Sizes are reduced by 1/8. reduced_scm_size = self.pool[0].scm_size.value * 0.875 @@ -332,7 +341,7 @@ def test_fields_basic(self): self.verify_pool_lists( targets_disabled=targets_disabled, scm_size=scm_size, nvme_size=nvme_size, - state=state, rebuild_state=rebuild_state) + state=state, rebuild_state=rebuild_state, ranks_disabled=ranks_disabled) # 7-11. Destroy and verify until the pools are gone. while self.pool: @@ -344,10 +353,11 @@ def test_fields_basic(self): targets_disabled.pop() scm_size.pop() nvme_size.pop() + ranks_disabled.pop() self.verify_pool_lists( targets_disabled=targets_disabled, scm_size=scm_size, nvme_size=nvme_size, - state=state, rebuild_state=rebuild_state) + state=state, rebuild_state=rebuild_state, ranks_disabled=ranks_disabled) def verify_used_imbalance(self, storage): """Verification steps for test_used_imbalance. @@ -374,9 +384,10 @@ def verify_used_imbalance(self, storage): scm_size = [None] state = ["Ready"] rebuild_state = ["idle"] + ranks_disabled = [[]] actual_pools_before = self.verify_pool_lists( targets_disabled=targets_disabled, scm_size=scm_size, nvme_size=nvme_size, - state=state, rebuild_state=rebuild_state) + state=state, rebuild_state=rebuild_state, ranks_disabled=ranks_disabled) # 3. Store free. free_before, _ = self.get_free_imbalance(actual_pools_before[0], storage) @@ -394,7 +405,7 @@ def verify_used_imbalance(self, storage): # obtained from actual. actual_pools_after = self.verify_pool_lists( targets_disabled=targets_disabled, scm_size=scm_size, nvme_size=nvme_size, - state=state, rebuild_state=rebuild_state) + state=state, rebuild_state=rebuild_state, ranks_disabled=ranks_disabled) # Obtain the new free and imbalance. free_after, imbalance_after = self.get_free_imbalance( From 771727920d6bf64f5d5e10ca352f6b2d84df1aa7 Mon Sep 17 00:00:00 2001 From: Wang Shilong Date: Sat, 16 Nov 2024 14:18:09 +0800 Subject: [PATCH 6/7] DAOS-16477 mgmt: return suspect engines for pool healthy query (#15458) After significant failures, the system may leave behind some suspect engines that were marked as DEAD by the SWIM protocol, but were not excluded from the system to prevent data loss. An administrator can bring these ranks back online by restarting them. This PR aims to provide an administrative interface for querying suspect engines following a massive failure. These suspect engines can be retrieved using the daos/dmg --health-only command. An example of output of dmg pool query --health-only: Pool 6f450a68-8c7d-4da9-8900-02691650f6a2, ntarget=8, disabled=2, leader=3, version=4, state=Degraded Pool health info: - Disabled ranks: 1 - Suspect ranks: 2 - Rebuild busy, 0 objs, 0 recs Required-githooks: true Signed-off-by: Wang Shilong Signed-off-by: Phil Henderson Co-authored-by: Phil Henderson --- docs/admin/pool_operations.md | 22 + src/control/cmd/daos/health.go | 19 +- src/control/cmd/daos/pool.go | 67 ++- src/control/cmd/daos/pretty/health.go | 7 + src/control/cmd/daos/pretty/pool.go | 4 + src/control/cmd/daos/pretty/pool_test.go | 39 ++ src/control/common/proto/mgmt/pool.pb.go | 196 ++++---- src/control/lib/control/pool_test.go | 101 ++++- src/control/lib/daos/pool.go | 6 +- src/control/lib/daos/pool_test.go | 21 +- src/include/daos_pool.h | 2 + src/include/daos_srv/pool.h | 4 +- src/mgmt/pool.pb-c.c | 422 +++++++----------- src/mgmt/pool.pb-c.h | 16 +- src/mgmt/srv_drpc.c | 22 +- src/mgmt/srv_internal.h | 5 +- src/mgmt/srv_pool.c | 8 +- src/mgmt/tests/mocks.c | 14 +- src/mgmt/tests/mocks.h | 1 + src/mgmt/tests/srv_drpc_tests.c | 8 +- src/pool/srv_cli.c | 90 +++- src/proto/mgmt/pool.proto | 1 + .../ftest/control/dmg_pool_query_ranks.py | 180 ++++---- .../ftest/control/dmg_pool_query_ranks.yaml | 9 +- src/tests/ftest/util/dmg_utils.py | 6 +- src/tests/ftest/util/dmg_utils_base.py | 1 + src/tests/ftest/util/server_utils_params.py | 6 +- src/tests/ftest/util/test_utils_pool.py | 27 ++ 28 files changed, 787 insertions(+), 517 deletions(-) diff --git a/docs/admin/pool_operations.md b/docs/admin/pool_operations.md index 36907a2e31f..efc827ae1e7 100644 --- a/docs/admin/pool_operations.md +++ b/docs/admin/pool_operations.md @@ -286,6 +286,28 @@ The example below shows a rebuild in progress and NVMe space allocated. Rebuild busy, 75 objs, 9722 recs ``` +After experiencing significant failures, the pool may retain some suspect +engines that have been marked as DEAD by the SWIM protocol but were not excluded +from the pool to prevent potential data inconsistency. An administrator can bring +these engines back online by restarting them. The example below illustrates the +system’s status with suspect and disabled engines. + +```bash +$ dmg pool query tank -t +``` + +NB: The --health-only/-t option is necessary to conduct pool health-related queries only. +This is important because suspect ranks may cause commands to hang and timeout so identifying +and restarting them is a useful procedure. + +```bash +Pool 6f450a68-8c7d-4da9-8900-02691650f6a2, ntarget=8, disabled=2, leader=3, version=4, state=Degraded + Pool health info: + - Disabled ranks: 1 + - Suspect ranks: 2 + - Rebuild busy, 0 objs, 0 recs +``` + Additional status and telemetry data is planned to be exported through management tools and will be documented here once available. diff --git a/src/control/cmd/daos/health.go b/src/control/cmd/daos/health.go index 70e54213084..dbeaacde0a0 100644 --- a/src/control/cmd/daos/health.go +++ b/src/control/cmd/daos/health.go @@ -99,24 +99,19 @@ func (cmd *healthCheckCmd) Execute([]string) error { } }() - queryMask := daos.MustNewPoolQueryMask(daos.PoolQueryOptionEnabledEngines) + queryMask := daos.MustNewPoolQueryMask(daos.PoolQueryOptionEnabledEngines, + daos.PoolQueryOptionSuspectEngines) + if pool.DisabledTargets > 0 { + queryMask.SetOptions(daos.PoolQueryOptionDisabledEngines) + } tpi, err := queryPool(poolHdl, queryMask) if err != nil { cmd.Errorf("failed to query pool %s: %v", pool.Label, err) continue } pool.EnabledRanks = tpi.EnabledRanks - - if pool.DisabledTargets > 0 { - queryMask.ClearAll() - queryMask.SetOptions(daos.PoolQueryOptionDisabledEngines) - tpi, err = queryPool(poolHdl, queryMask) - if err != nil { - cmd.Errorf("failed to query pool %s: %v", pool.Label, err) - continue - } - pool.DisabledRanks = tpi.DisabledRanks - } + pool.DisabledRanks = tpi.DisabledRanks + pool.SuspectRanks = tpi.SuspectRanks poolConts, err := listContainers(poolHdl) if err != nil { diff --git a/src/control/cmd/daos/pool.go b/src/control/cmd/daos/pool.go index 7a917b1a9e4..085a78b3b73 100644 --- a/src/control/cmd/daos/pool.go +++ b/src/control/cmd/daos/pool.go @@ -295,11 +295,12 @@ func convertPoolInfo(pinfo *C.daos_pool_info_t) (*daos.PoolInfo, error) { return poolInfo, nil } -func queryPool(poolHdl C.daos_handle_t, queryMask daos.PoolQueryMask) (*daos.PoolInfo, error) { +func queryPoolRankLists(poolHdl C.daos_handle_t, queryMask daos.PoolQueryMask) (*daos.PoolInfo, error) { var rlPtr **C.d_rank_list_t = nil var rl *C.d_rank_list_t = nil - if queryMask.HasOption(daos.PoolQueryOptionEnabledEngines) || queryMask.HasOption(daos.PoolQueryOptionDisabledEngines) { + if queryMask.HasOption(daos.PoolQueryOptionEnabledEngines) || queryMask.HasOption(daos.PoolQueryOptionDisabledEngines) || + queryMask.HasOption(daos.PoolQueryOptionSuspectEngines) { rlPtr = &rl } @@ -329,6 +330,68 @@ func queryPool(poolHdl C.daos_handle_t, queryMask daos.PoolQueryMask) (*daos.Poo if queryMask.HasOption(daos.PoolQueryOptionDisabledEngines) { poolInfo.DisabledRanks = rs } + if queryMask.HasOption(daos.PoolQueryOptionSuspectEngines) { + poolInfo.SuspectRanks = rs + } + } + + return poolInfo, nil +} +func queryPool(poolHdl C.daos_handle_t, queryMask daos.PoolQueryMask) (*daos.PoolInfo, error) { + poolInfo := &daos.PoolInfo{} + originalMask := queryMask // Save the original queryMask + + // Function to handle the query and return a single RankList + queryAndUpdate := func(option string) error { + // Clear previous options and set new option + queryMask.ClearAll() + queryMask.SetOptions(option) + + poolInfo1, err := queryPoolRankLists(poolHdl, queryMask) + if err != nil { + return err + } + + switch option { + case daos.PoolQueryOptionEnabledEngines: + poolInfo.EnabledRanks = poolInfo1.EnabledRanks + case daos.PoolQueryOptionDisabledEngines: + poolInfo.DisabledRanks = poolInfo1.DisabledRanks + case daos.PoolQueryOptionSuspectEngines: + poolInfo.SuspectRanks = poolInfo1.SuspectRanks + } + return nil + } + + // Preprocess queryMask, select one option for the first query + var firstOption string + if originalMask.HasOption(daos.PoolQueryOptionEnabledEngines) { + firstOption = daos.PoolQueryOptionEnabledEngines + } else if originalMask.HasOption(daos.PoolQueryOptionDisabledEngines) { + firstOption = daos.PoolQueryOptionDisabledEngines + } else if originalMask.HasOption(daos.PoolQueryOptionSuspectEngines) { + firstOption = daos.PoolQueryOptionSuspectEngines + } + + // Perform the first query to get basic information + if err := queryAndUpdate(firstOption); err != nil { + return nil, err + } + + // Check the original query mask and update fields as needed + queryOptions := []string{ + daos.PoolQueryOptionEnabledEngines, + daos.PoolQueryOptionDisabledEngines, + daos.PoolQueryOptionSuspectEngines, + } + + // Process each option sequentially + for _, opt := range queryOptions { + if originalMask.HasOption(opt) && opt != firstOption { + if err := queryAndUpdate(opt); err != nil { + return nil, err + } + } } return poolInfo, nil diff --git a/src/control/cmd/daos/pretty/health.go b/src/control/cmd/daos/pretty/health.go index 25c94e10f99..ee77cd72371 100644 --- a/src/control/cmd/daos/pretty/health.go +++ b/src/control/cmd/daos/pretty/health.go @@ -61,6 +61,13 @@ func printPoolHealth(out io.Writer, pi *daos.PoolInfo, verbose bool) { } var healthStrings []string + if pi.SuspectRanks != nil && pi.SuspectRanks.Count() > 0 { + degStr := "Suspect" + if verbose { + degStr += fmt.Sprintf(" %s", pi.SuspectRanks) + } + healthStrings = append(healthStrings, degStr) + } if pi.DisabledTargets > 0 { degStr := "Degraded" if verbose { diff --git a/src/control/cmd/daos/pretty/pool.go b/src/control/cmd/daos/pretty/pool.go index a9f685b536f..697009396ce 100644 --- a/src/control/cmd/daos/pretty/pool.go +++ b/src/control/cmd/daos/pretty/pool.go @@ -53,6 +53,10 @@ func PrintPoolInfo(pi *daos.PoolInfo, out io.Writer) error { if pi.DisabledRanks.Count() > 0 { fmt.Fprintf(w, "- Disabled ranks: %s\n", pi.DisabledRanks) } + if pi.QueryMask.HasOption(daos.PoolQueryOptionSuspectEngines) && + pi.SuspectRanks != nil && pi.SuspectRanks.Count() > 0 { + fmt.Fprintf(w, "- Suspect ranks: %s\n", pi.SuspectRanks) + } if pi.Rebuild != nil { if pi.Rebuild.Status == 0 { fmt.Fprintf(w, "- Rebuild %s, %d objs, %d recs\n", diff --git a/src/control/cmd/daos/pretty/pool_test.go b/src/control/cmd/daos/pretty/pool_test.go index 3a1724e1dda..11f50c0e6be 100644 --- a/src/control/cmd/daos/pretty/pool_test.go +++ b/src/control/cmd/daos/pretty/pool_test.go @@ -122,6 +122,45 @@ Pool space info: - Storage tier 1 (NVMe): Total size: 2 B Free: 1 B, min:0 B, max:0 B, mean:0 B +`, poolUUID.String()), + }, + "normal response; suspect ranks": { + pi: &daos.PoolInfo{ + QueryMask: daos.HealthOnlyPoolQueryMask, + State: daos.PoolServiceStateDegraded, + UUID: poolUUID, + TotalTargets: 2, + DisabledTargets: 1, + ActiveTargets: 1, + ServiceLeader: 42, + Version: 100, + PoolLayoutVer: 1, + UpgradeLayoutVer: 2, + DisabledRanks: ranklist.MustCreateRankSet("[0,1,3]"), + SuspectRanks: ranklist.MustCreateRankSet("[2]"), + Rebuild: &daos.PoolRebuildStatus{ + State: daos.PoolRebuildStateBusy, + Objects: 42, + Records: 21, + }, + TierStats: []*daos.StorageUsageStats{ + { + Total: 2, + Free: 1, + }, + { + Total: 2, + Free: 1, + }, + }, + }, + expPrintStr: fmt.Sprintf(` +Pool %s, ntarget=2, disabled=1, leader=42, version=100, state=Degraded +Pool layout out of date (1 < 2) -- see `+backtickStr+` for details. +Pool health info: +- Disabled ranks: 0-1,3 +- Suspect ranks: 2 +- Rebuild busy, 42 objs, 21 recs `, poolUUID.String()), }, "normal response; disabled ranks": { diff --git a/src/control/common/proto/mgmt/pool.pb.go b/src/control/common/proto/mgmt/pool.pb.go index b699d1f55dc..4634d33b9af 100644 --- a/src/control/common/proto/mgmt/pool.pb.go +++ b/src/control/common/proto/mgmt/pool.pb.go @@ -1842,6 +1842,7 @@ type PoolQueryResp struct { SvcLdr uint32 `protobuf:"varint,18,opt,name=svc_ldr,json=svcLdr,proto3" json:"svc_ldr,omitempty"` // current raft leader (2.6+) SvcReps []uint32 `protobuf:"varint,19,rep,packed,name=svc_reps,json=svcReps,proto3" json:"svc_reps,omitempty"` // service replica ranks QueryMask uint64 `protobuf:"varint,20,opt,name=query_mask,json=queryMask,proto3" json:"query_mask,omitempty"` // Bitmask of pool query options used + SuspectRanks string `protobuf:"bytes,21,opt,name=suspect_ranks,json=suspectRanks,proto3" json:"suspect_ranks,omitempty"` // optional set of suspect ranks } func (x *PoolQueryResp) Reset() { @@ -2009,6 +2010,13 @@ func (x *PoolQueryResp) GetQueryMask() uint64 { return 0 } +func (x *PoolQueryResp) GetSuspectRanks() string { + if x != nil { + return x.SuspectRanks + } + return "" +} + type PoolProperty struct { state protoimpl.MessageState sizeCache protoimpl.SizeCache @@ -3032,7 +3040,7 @@ var file_mgmt_pool_proto_rawDesc = []byte{ 0x04, 0x52, 0x07, 0x72, 0x65, 0x63, 0x6f, 0x72, 0x64, 0x73, 0x22, 0x25, 0x0a, 0x05, 0x53, 0x74, 0x61, 0x74, 0x65, 0x12, 0x08, 0x0a, 0x04, 0x49, 0x44, 0x4c, 0x45, 0x10, 0x00, 0x12, 0x08, 0x0a, 0x04, 0x44, 0x4f, 0x4e, 0x45, 0x10, 0x01, 0x12, 0x08, 0x0a, 0x04, 0x42, 0x55, 0x53, 0x59, 0x10, - 0x02, 0x22, 0xc0, 0x05, 0x0a, 0x0d, 0x50, 0x6f, 0x6f, 0x6c, 0x51, 0x75, 0x65, 0x72, 0x79, 0x52, + 0x02, 0x22, 0xe5, 0x05, 0x0a, 0x0d, 0x50, 0x6f, 0x6f, 0x6c, 0x51, 0x75, 0x65, 0x72, 0x79, 0x52, 0x65, 0x73, 0x70, 0x12, 0x16, 0x0a, 0x06, 0x73, 0x74, 0x61, 0x74, 0x75, 0x73, 0x18, 0x01, 0x20, 0x01, 0x28, 0x05, 0x52, 0x06, 0x73, 0x74, 0x61, 0x74, 0x75, 0x73, 0x12, 0x12, 0x0a, 0x04, 0x75, 0x75, 0x69, 0x64, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x75, 0x75, 0x69, 0x64, 0x12, @@ -3075,103 +3083,105 @@ var file_mgmt_pool_proto_rawDesc = []byte{ 0x65, 0x70, 0x73, 0x18, 0x13, 0x20, 0x03, 0x28, 0x0d, 0x52, 0x07, 0x73, 0x76, 0x63, 0x52, 0x65, 0x70, 0x73, 0x12, 0x1d, 0x0a, 0x0a, 0x71, 0x75, 0x65, 0x72, 0x79, 0x5f, 0x6d, 0x61, 0x73, 0x6b, 0x18, 0x14, 0x20, 0x01, 0x28, 0x04, 0x52, 0x09, 0x71, 0x75, 0x65, 0x72, 0x79, 0x4d, 0x61, 0x73, - 0x6b, 0x4a, 0x04, 0x08, 0x09, 0x10, 0x0a, 0x52, 0x0b, 0x74, 0x6f, 0x74, 0x61, 0x6c, 0x5f, 0x6e, - 0x6f, 0x64, 0x65, 0x73, 0x22, 0x63, 0x0a, 0x0c, 0x50, 0x6f, 0x6f, 0x6c, 0x50, 0x72, 0x6f, 0x70, - 0x65, 0x72, 0x74, 0x79, 0x12, 0x16, 0x0a, 0x06, 0x6e, 0x75, 0x6d, 0x62, 0x65, 0x72, 0x18, 0x01, - 0x20, 0x01, 0x28, 0x0d, 0x52, 0x06, 0x6e, 0x75, 0x6d, 0x62, 0x65, 0x72, 0x12, 0x18, 0x0a, 0x06, - 0x73, 0x74, 0x72, 0x76, 0x61, 0x6c, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x48, 0x00, 0x52, 0x06, - 0x73, 0x74, 0x72, 0x76, 0x61, 0x6c, 0x12, 0x18, 0x0a, 0x06, 0x6e, 0x75, 0x6d, 0x76, 0x61, 0x6c, - 0x18, 0x03, 0x20, 0x01, 0x28, 0x04, 0x48, 0x00, 0x52, 0x06, 0x6e, 0x75, 0x6d, 0x76, 0x61, 0x6c, - 0x42, 0x07, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x22, 0x83, 0x01, 0x0a, 0x0e, 0x50, 0x6f, - 0x6f, 0x6c, 0x53, 0x65, 0x74, 0x50, 0x72, 0x6f, 0x70, 0x52, 0x65, 0x71, 0x12, 0x10, 0x0a, 0x03, - 0x73, 0x79, 0x73, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x73, 0x79, 0x73, 0x12, 0x0e, - 0x0a, 0x02, 0x69, 0x64, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x02, 0x69, 0x64, 0x12, 0x32, - 0x0a, 0x0a, 0x70, 0x72, 0x6f, 0x70, 0x65, 0x72, 0x74, 0x69, 0x65, 0x73, 0x18, 0x03, 0x20, 0x03, - 0x28, 0x0b, 0x32, 0x12, 0x2e, 0x6d, 0x67, 0x6d, 0x74, 0x2e, 0x50, 0x6f, 0x6f, 0x6c, 0x50, 0x72, - 0x6f, 0x70, 0x65, 0x72, 0x74, 0x79, 0x52, 0x0a, 0x70, 0x72, 0x6f, 0x70, 0x65, 0x72, 0x74, 0x69, - 0x65, 0x73, 0x12, 0x1b, 0x0a, 0x09, 0x73, 0x76, 0x63, 0x5f, 0x72, 0x61, 0x6e, 0x6b, 0x73, 0x18, - 0x04, 0x20, 0x03, 0x28, 0x0d, 0x52, 0x08, 0x73, 0x76, 0x63, 0x52, 0x61, 0x6e, 0x6b, 0x73, 0x22, - 0x29, 0x0a, 0x0f, 0x50, 0x6f, 0x6f, 0x6c, 0x53, 0x65, 0x74, 0x50, 0x72, 0x6f, 0x70, 0x52, 0x65, - 0x73, 0x70, 0x12, 0x16, 0x0a, 0x06, 0x73, 0x74, 0x61, 0x74, 0x75, 0x73, 0x18, 0x01, 0x20, 0x01, - 0x28, 0x05, 0x52, 0x06, 0x73, 0x74, 0x61, 0x74, 0x75, 0x73, 0x22, 0x83, 0x01, 0x0a, 0x0e, 0x50, - 0x6f, 0x6f, 0x6c, 0x47, 0x65, 0x74, 0x50, 0x72, 0x6f, 0x70, 0x52, 0x65, 0x71, 0x12, 0x10, 0x0a, - 0x03, 0x73, 0x79, 0x73, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x73, 0x79, 0x73, 0x12, - 0x0e, 0x0a, 0x02, 0x69, 0x64, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x02, 0x69, 0x64, 0x12, - 0x32, 0x0a, 0x0a, 0x70, 0x72, 0x6f, 0x70, 0x65, 0x72, 0x74, 0x69, 0x65, 0x73, 0x18, 0x03, 0x20, - 0x03, 0x28, 0x0b, 0x32, 0x12, 0x2e, 0x6d, 0x67, 0x6d, 0x74, 0x2e, 0x50, 0x6f, 0x6f, 0x6c, 0x50, - 0x72, 0x6f, 0x70, 0x65, 0x72, 0x74, 0x79, 0x52, 0x0a, 0x70, 0x72, 0x6f, 0x70, 0x65, 0x72, 0x74, - 0x69, 0x65, 0x73, 0x12, 0x1b, 0x0a, 0x09, 0x73, 0x76, 0x63, 0x5f, 0x72, 0x61, 0x6e, 0x6b, 0x73, - 0x18, 0x04, 0x20, 0x03, 0x28, 0x0d, 0x52, 0x08, 0x73, 0x76, 0x63, 0x52, 0x61, 0x6e, 0x6b, 0x73, - 0x22, 0x5d, 0x0a, 0x0f, 0x50, 0x6f, 0x6f, 0x6c, 0x47, 0x65, 0x74, 0x50, 0x72, 0x6f, 0x70, 0x52, - 0x65, 0x73, 0x70, 0x12, 0x16, 0x0a, 0x06, 0x73, 0x74, 0x61, 0x74, 0x75, 0x73, 0x18, 0x01, 0x20, - 0x01, 0x28, 0x05, 0x52, 0x06, 0x73, 0x74, 0x61, 0x74, 0x75, 0x73, 0x12, 0x32, 0x0a, 0x0a, 0x70, - 0x72, 0x6f, 0x70, 0x65, 0x72, 0x74, 0x69, 0x65, 0x73, 0x18, 0x02, 0x20, 0x03, 0x28, 0x0b, 0x32, - 0x12, 0x2e, 0x6d, 0x67, 0x6d, 0x74, 0x2e, 0x50, 0x6f, 0x6f, 0x6c, 0x50, 0x72, 0x6f, 0x70, 0x65, - 0x72, 0x74, 0x79, 0x52, 0x0a, 0x70, 0x72, 0x6f, 0x70, 0x65, 0x72, 0x74, 0x69, 0x65, 0x73, 0x22, - 0x4f, 0x0a, 0x0e, 0x50, 0x6f, 0x6f, 0x6c, 0x55, 0x70, 0x67, 0x72, 0x61, 0x64, 0x65, 0x52, 0x65, + 0x6b, 0x12, 0x23, 0x0a, 0x0d, 0x73, 0x75, 0x73, 0x70, 0x65, 0x63, 0x74, 0x5f, 0x72, 0x61, 0x6e, + 0x6b, 0x73, 0x18, 0x15, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0c, 0x73, 0x75, 0x73, 0x70, 0x65, 0x63, + 0x74, 0x52, 0x61, 0x6e, 0x6b, 0x73, 0x4a, 0x04, 0x08, 0x09, 0x10, 0x0a, 0x52, 0x0b, 0x74, 0x6f, + 0x74, 0x61, 0x6c, 0x5f, 0x6e, 0x6f, 0x64, 0x65, 0x73, 0x22, 0x63, 0x0a, 0x0c, 0x50, 0x6f, 0x6f, + 0x6c, 0x50, 0x72, 0x6f, 0x70, 0x65, 0x72, 0x74, 0x79, 0x12, 0x16, 0x0a, 0x06, 0x6e, 0x75, 0x6d, + 0x62, 0x65, 0x72, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x06, 0x6e, 0x75, 0x6d, 0x62, 0x65, + 0x72, 0x12, 0x18, 0x0a, 0x06, 0x73, 0x74, 0x72, 0x76, 0x61, 0x6c, 0x18, 0x02, 0x20, 0x01, 0x28, + 0x09, 0x48, 0x00, 0x52, 0x06, 0x73, 0x74, 0x72, 0x76, 0x61, 0x6c, 0x12, 0x18, 0x0a, 0x06, 0x6e, + 0x75, 0x6d, 0x76, 0x61, 0x6c, 0x18, 0x03, 0x20, 0x01, 0x28, 0x04, 0x48, 0x00, 0x52, 0x06, 0x6e, + 0x75, 0x6d, 0x76, 0x61, 0x6c, 0x42, 0x07, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x22, 0x83, + 0x01, 0x0a, 0x0e, 0x50, 0x6f, 0x6f, 0x6c, 0x53, 0x65, 0x74, 0x50, 0x72, 0x6f, 0x70, 0x52, 0x65, 0x71, 0x12, 0x10, 0x0a, 0x03, 0x73, 0x79, 0x73, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x73, 0x79, 0x73, 0x12, 0x0e, 0x0a, 0x02, 0x69, 0x64, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, - 0x02, 0x69, 0x64, 0x12, 0x1b, 0x0a, 0x09, 0x73, 0x76, 0x63, 0x5f, 0x72, 0x61, 0x6e, 0x6b, 0x73, - 0x18, 0x03, 0x20, 0x03, 0x28, 0x0d, 0x52, 0x08, 0x73, 0x76, 0x63, 0x52, 0x61, 0x6e, 0x6b, 0x73, - 0x22, 0x29, 0x0a, 0x0f, 0x50, 0x6f, 0x6f, 0x6c, 0x55, 0x70, 0x67, 0x72, 0x61, 0x64, 0x65, 0x52, - 0x65, 0x73, 0x70, 0x12, 0x16, 0x0a, 0x06, 0x73, 0x74, 0x61, 0x74, 0x75, 0x73, 0x18, 0x01, 0x20, - 0x01, 0x28, 0x05, 0x52, 0x06, 0x73, 0x74, 0x61, 0x74, 0x75, 0x73, 0x22, 0x81, 0x01, 0x0a, 0x12, - 0x50, 0x6f, 0x6f, 0x6c, 0x51, 0x75, 0x65, 0x72, 0x79, 0x54, 0x61, 0x72, 0x67, 0x65, 0x74, 0x52, + 0x02, 0x69, 0x64, 0x12, 0x32, 0x0a, 0x0a, 0x70, 0x72, 0x6f, 0x70, 0x65, 0x72, 0x74, 0x69, 0x65, + 0x73, 0x18, 0x03, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x12, 0x2e, 0x6d, 0x67, 0x6d, 0x74, 0x2e, 0x50, + 0x6f, 0x6f, 0x6c, 0x50, 0x72, 0x6f, 0x70, 0x65, 0x72, 0x74, 0x79, 0x52, 0x0a, 0x70, 0x72, 0x6f, + 0x70, 0x65, 0x72, 0x74, 0x69, 0x65, 0x73, 0x12, 0x1b, 0x0a, 0x09, 0x73, 0x76, 0x63, 0x5f, 0x72, + 0x61, 0x6e, 0x6b, 0x73, 0x18, 0x04, 0x20, 0x03, 0x28, 0x0d, 0x52, 0x08, 0x73, 0x76, 0x63, 0x52, + 0x61, 0x6e, 0x6b, 0x73, 0x22, 0x29, 0x0a, 0x0f, 0x50, 0x6f, 0x6f, 0x6c, 0x53, 0x65, 0x74, 0x50, + 0x72, 0x6f, 0x70, 0x52, 0x65, 0x73, 0x70, 0x12, 0x16, 0x0a, 0x06, 0x73, 0x74, 0x61, 0x74, 0x75, + 0x73, 0x18, 0x01, 0x20, 0x01, 0x28, 0x05, 0x52, 0x06, 0x73, 0x74, 0x61, 0x74, 0x75, 0x73, 0x22, + 0x83, 0x01, 0x0a, 0x0e, 0x50, 0x6f, 0x6f, 0x6c, 0x47, 0x65, 0x74, 0x50, 0x72, 0x6f, 0x70, 0x52, 0x65, 0x71, 0x12, 0x10, 0x0a, 0x03, 0x73, 0x79, 0x73, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x73, 0x79, 0x73, 0x12, 0x0e, 0x0a, 0x02, 0x69, 0x64, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, - 0x52, 0x02, 0x69, 0x64, 0x12, 0x12, 0x0a, 0x04, 0x72, 0x61, 0x6e, 0x6b, 0x18, 0x03, 0x20, 0x01, - 0x28, 0x0d, 0x52, 0x04, 0x72, 0x61, 0x6e, 0x6b, 0x12, 0x18, 0x0a, 0x07, 0x74, 0x61, 0x72, 0x67, - 0x65, 0x74, 0x73, 0x18, 0x04, 0x20, 0x03, 0x28, 0x0d, 0x52, 0x07, 0x74, 0x61, 0x72, 0x67, 0x65, - 0x74, 0x73, 0x12, 0x1b, 0x0a, 0x09, 0x73, 0x76, 0x63, 0x5f, 0x72, 0x61, 0x6e, 0x6b, 0x73, 0x18, - 0x05, 0x20, 0x03, 0x28, 0x0d, 0x52, 0x08, 0x73, 0x76, 0x63, 0x52, 0x61, 0x6e, 0x6b, 0x73, 0x22, - 0x75, 0x0a, 0x12, 0x53, 0x74, 0x6f, 0x72, 0x61, 0x67, 0x65, 0x54, 0x61, 0x72, 0x67, 0x65, 0x74, - 0x55, 0x73, 0x61, 0x67, 0x65, 0x12, 0x14, 0x0a, 0x05, 0x74, 0x6f, 0x74, 0x61, 0x6c, 0x18, 0x01, - 0x20, 0x01, 0x28, 0x04, 0x52, 0x05, 0x74, 0x6f, 0x74, 0x61, 0x6c, 0x12, 0x12, 0x0a, 0x04, 0x66, - 0x72, 0x65, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x04, 0x52, 0x04, 0x66, 0x72, 0x65, 0x65, 0x12, - 0x35, 0x0a, 0x0a, 0x6d, 0x65, 0x64, 0x69, 0x61, 0x5f, 0x74, 0x79, 0x70, 0x65, 0x18, 0x03, 0x20, - 0x01, 0x28, 0x0e, 0x32, 0x16, 0x2e, 0x6d, 0x67, 0x6d, 0x74, 0x2e, 0x53, 0x74, 0x6f, 0x72, 0x61, - 0x67, 0x65, 0x4d, 0x65, 0x64, 0x69, 0x61, 0x54, 0x79, 0x70, 0x65, 0x52, 0x09, 0x6d, 0x65, 0x64, - 0x69, 0x61, 0x54, 0x79, 0x70, 0x65, 0x22, 0xda, 0x02, 0x0a, 0x13, 0x50, 0x6f, 0x6f, 0x6c, 0x51, - 0x75, 0x65, 0x72, 0x79, 0x54, 0x61, 0x72, 0x67, 0x65, 0x74, 0x49, 0x6e, 0x66, 0x6f, 0x12, 0x38, - 0x0a, 0x04, 0x74, 0x79, 0x70, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x24, 0x2e, 0x6d, + 0x52, 0x02, 0x69, 0x64, 0x12, 0x32, 0x0a, 0x0a, 0x70, 0x72, 0x6f, 0x70, 0x65, 0x72, 0x74, 0x69, + 0x65, 0x73, 0x18, 0x03, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x12, 0x2e, 0x6d, 0x67, 0x6d, 0x74, 0x2e, + 0x50, 0x6f, 0x6f, 0x6c, 0x50, 0x72, 0x6f, 0x70, 0x65, 0x72, 0x74, 0x79, 0x52, 0x0a, 0x70, 0x72, + 0x6f, 0x70, 0x65, 0x72, 0x74, 0x69, 0x65, 0x73, 0x12, 0x1b, 0x0a, 0x09, 0x73, 0x76, 0x63, 0x5f, + 0x72, 0x61, 0x6e, 0x6b, 0x73, 0x18, 0x04, 0x20, 0x03, 0x28, 0x0d, 0x52, 0x08, 0x73, 0x76, 0x63, + 0x52, 0x61, 0x6e, 0x6b, 0x73, 0x22, 0x5d, 0x0a, 0x0f, 0x50, 0x6f, 0x6f, 0x6c, 0x47, 0x65, 0x74, + 0x50, 0x72, 0x6f, 0x70, 0x52, 0x65, 0x73, 0x70, 0x12, 0x16, 0x0a, 0x06, 0x73, 0x74, 0x61, 0x74, + 0x75, 0x73, 0x18, 0x01, 0x20, 0x01, 0x28, 0x05, 0x52, 0x06, 0x73, 0x74, 0x61, 0x74, 0x75, 0x73, + 0x12, 0x32, 0x0a, 0x0a, 0x70, 0x72, 0x6f, 0x70, 0x65, 0x72, 0x74, 0x69, 0x65, 0x73, 0x18, 0x02, + 0x20, 0x03, 0x28, 0x0b, 0x32, 0x12, 0x2e, 0x6d, 0x67, 0x6d, 0x74, 0x2e, 0x50, 0x6f, 0x6f, 0x6c, + 0x50, 0x72, 0x6f, 0x70, 0x65, 0x72, 0x74, 0x79, 0x52, 0x0a, 0x70, 0x72, 0x6f, 0x70, 0x65, 0x72, + 0x74, 0x69, 0x65, 0x73, 0x22, 0x4f, 0x0a, 0x0e, 0x50, 0x6f, 0x6f, 0x6c, 0x55, 0x70, 0x67, 0x72, + 0x61, 0x64, 0x65, 0x52, 0x65, 0x71, 0x12, 0x10, 0x0a, 0x03, 0x73, 0x79, 0x73, 0x18, 0x01, 0x20, + 0x01, 0x28, 0x09, 0x52, 0x03, 0x73, 0x79, 0x73, 0x12, 0x0e, 0x0a, 0x02, 0x69, 0x64, 0x18, 0x02, + 0x20, 0x01, 0x28, 0x09, 0x52, 0x02, 0x69, 0x64, 0x12, 0x1b, 0x0a, 0x09, 0x73, 0x76, 0x63, 0x5f, + 0x72, 0x61, 0x6e, 0x6b, 0x73, 0x18, 0x03, 0x20, 0x03, 0x28, 0x0d, 0x52, 0x08, 0x73, 0x76, 0x63, + 0x52, 0x61, 0x6e, 0x6b, 0x73, 0x22, 0x29, 0x0a, 0x0f, 0x50, 0x6f, 0x6f, 0x6c, 0x55, 0x70, 0x67, + 0x72, 0x61, 0x64, 0x65, 0x52, 0x65, 0x73, 0x70, 0x12, 0x16, 0x0a, 0x06, 0x73, 0x74, 0x61, 0x74, + 0x75, 0x73, 0x18, 0x01, 0x20, 0x01, 0x28, 0x05, 0x52, 0x06, 0x73, 0x74, 0x61, 0x74, 0x75, 0x73, + 0x22, 0x81, 0x01, 0x0a, 0x12, 0x50, 0x6f, 0x6f, 0x6c, 0x51, 0x75, 0x65, 0x72, 0x79, 0x54, 0x61, + 0x72, 0x67, 0x65, 0x74, 0x52, 0x65, 0x71, 0x12, 0x10, 0x0a, 0x03, 0x73, 0x79, 0x73, 0x18, 0x01, + 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x73, 0x79, 0x73, 0x12, 0x0e, 0x0a, 0x02, 0x69, 0x64, 0x18, + 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x02, 0x69, 0x64, 0x12, 0x12, 0x0a, 0x04, 0x72, 0x61, 0x6e, + 0x6b, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x04, 0x72, 0x61, 0x6e, 0x6b, 0x12, 0x18, 0x0a, + 0x07, 0x74, 0x61, 0x72, 0x67, 0x65, 0x74, 0x73, 0x18, 0x04, 0x20, 0x03, 0x28, 0x0d, 0x52, 0x07, + 0x74, 0x61, 0x72, 0x67, 0x65, 0x74, 0x73, 0x12, 0x1b, 0x0a, 0x09, 0x73, 0x76, 0x63, 0x5f, 0x72, + 0x61, 0x6e, 0x6b, 0x73, 0x18, 0x05, 0x20, 0x03, 0x28, 0x0d, 0x52, 0x08, 0x73, 0x76, 0x63, 0x52, + 0x61, 0x6e, 0x6b, 0x73, 0x22, 0x75, 0x0a, 0x12, 0x53, 0x74, 0x6f, 0x72, 0x61, 0x67, 0x65, 0x54, + 0x61, 0x72, 0x67, 0x65, 0x74, 0x55, 0x73, 0x61, 0x67, 0x65, 0x12, 0x14, 0x0a, 0x05, 0x74, 0x6f, + 0x74, 0x61, 0x6c, 0x18, 0x01, 0x20, 0x01, 0x28, 0x04, 0x52, 0x05, 0x74, 0x6f, 0x74, 0x61, 0x6c, + 0x12, 0x12, 0x0a, 0x04, 0x66, 0x72, 0x65, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x04, 0x52, 0x04, + 0x66, 0x72, 0x65, 0x65, 0x12, 0x35, 0x0a, 0x0a, 0x6d, 0x65, 0x64, 0x69, 0x61, 0x5f, 0x74, 0x79, + 0x70, 0x65, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x16, 0x2e, 0x6d, 0x67, 0x6d, 0x74, 0x2e, + 0x53, 0x74, 0x6f, 0x72, 0x61, 0x67, 0x65, 0x4d, 0x65, 0x64, 0x69, 0x61, 0x54, 0x79, 0x70, 0x65, + 0x52, 0x09, 0x6d, 0x65, 0x64, 0x69, 0x61, 0x54, 0x79, 0x70, 0x65, 0x22, 0xda, 0x02, 0x0a, 0x13, + 0x50, 0x6f, 0x6f, 0x6c, 0x51, 0x75, 0x65, 0x72, 0x79, 0x54, 0x61, 0x72, 0x67, 0x65, 0x74, 0x49, + 0x6e, 0x66, 0x6f, 0x12, 0x38, 0x0a, 0x04, 0x74, 0x79, 0x70, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, + 0x0e, 0x32, 0x24, 0x2e, 0x6d, 0x67, 0x6d, 0x74, 0x2e, 0x50, 0x6f, 0x6f, 0x6c, 0x51, 0x75, 0x65, + 0x72, 0x79, 0x54, 0x61, 0x72, 0x67, 0x65, 0x74, 0x49, 0x6e, 0x66, 0x6f, 0x2e, 0x54, 0x61, 0x72, + 0x67, 0x65, 0x74, 0x54, 0x79, 0x70, 0x65, 0x52, 0x04, 0x74, 0x79, 0x70, 0x65, 0x12, 0x3b, 0x0a, + 0x05, 0x73, 0x74, 0x61, 0x74, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x25, 0x2e, 0x6d, 0x67, 0x6d, 0x74, 0x2e, 0x50, 0x6f, 0x6f, 0x6c, 0x51, 0x75, 0x65, 0x72, 0x79, 0x54, 0x61, 0x72, - 0x67, 0x65, 0x74, 0x49, 0x6e, 0x66, 0x6f, 0x2e, 0x54, 0x61, 0x72, 0x67, 0x65, 0x74, 0x54, 0x79, - 0x70, 0x65, 0x52, 0x04, 0x74, 0x79, 0x70, 0x65, 0x12, 0x3b, 0x0a, 0x05, 0x73, 0x74, 0x61, 0x74, - 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x25, 0x2e, 0x6d, 0x67, 0x6d, 0x74, 0x2e, 0x50, - 0x6f, 0x6f, 0x6c, 0x51, 0x75, 0x65, 0x72, 0x79, 0x54, 0x61, 0x72, 0x67, 0x65, 0x74, 0x49, 0x6e, - 0x66, 0x6f, 0x2e, 0x54, 0x61, 0x72, 0x67, 0x65, 0x74, 0x53, 0x74, 0x61, 0x74, 0x65, 0x52, 0x05, - 0x73, 0x74, 0x61, 0x74, 0x65, 0x12, 0x2e, 0x0a, 0x05, 0x73, 0x70, 0x61, 0x63, 0x65, 0x18, 0x03, - 0x20, 0x03, 0x28, 0x0b, 0x32, 0x18, 0x2e, 0x6d, 0x67, 0x6d, 0x74, 0x2e, 0x53, 0x74, 0x6f, 0x72, - 0x61, 0x67, 0x65, 0x54, 0x61, 0x72, 0x67, 0x65, 0x74, 0x55, 0x73, 0x61, 0x67, 0x65, 0x52, 0x05, - 0x73, 0x70, 0x61, 0x63, 0x65, 0x22, 0x3b, 0x0a, 0x0a, 0x54, 0x61, 0x72, 0x67, 0x65, 0x74, 0x54, - 0x79, 0x70, 0x65, 0x12, 0x0b, 0x0a, 0x07, 0x55, 0x4e, 0x4b, 0x4e, 0x4f, 0x57, 0x4e, 0x10, 0x00, - 0x12, 0x07, 0x0a, 0x03, 0x48, 0x44, 0x44, 0x10, 0x01, 0x12, 0x07, 0x0a, 0x03, 0x53, 0x53, 0x44, - 0x10, 0x02, 0x12, 0x06, 0x0a, 0x02, 0x50, 0x4d, 0x10, 0x03, 0x12, 0x06, 0x0a, 0x02, 0x56, 0x4d, - 0x10, 0x04, 0x22, 0x5f, 0x0a, 0x0b, 0x54, 0x61, 0x72, 0x67, 0x65, 0x74, 0x53, 0x74, 0x61, 0x74, - 0x65, 0x12, 0x11, 0x0a, 0x0d, 0x53, 0x54, 0x41, 0x54, 0x45, 0x5f, 0x55, 0x4e, 0x4b, 0x4e, 0x4f, - 0x57, 0x4e, 0x10, 0x00, 0x12, 0x0c, 0x0a, 0x08, 0x44, 0x4f, 0x57, 0x4e, 0x5f, 0x4f, 0x55, 0x54, - 0x10, 0x01, 0x12, 0x08, 0x0a, 0x04, 0x44, 0x4f, 0x57, 0x4e, 0x10, 0x02, 0x12, 0x06, 0x0a, 0x02, - 0x55, 0x50, 0x10, 0x03, 0x12, 0x09, 0x0a, 0x05, 0x55, 0x50, 0x5f, 0x49, 0x4e, 0x10, 0x04, 0x12, - 0x07, 0x0a, 0x03, 0x4e, 0x45, 0x57, 0x10, 0x05, 0x12, 0x09, 0x0a, 0x05, 0x44, 0x52, 0x41, 0x49, - 0x4e, 0x10, 0x06, 0x22, 0x5e, 0x0a, 0x13, 0x50, 0x6f, 0x6f, 0x6c, 0x51, 0x75, 0x65, 0x72, 0x79, - 0x54, 0x61, 0x72, 0x67, 0x65, 0x74, 0x52, 0x65, 0x73, 0x70, 0x12, 0x16, 0x0a, 0x06, 0x73, 0x74, - 0x61, 0x74, 0x75, 0x73, 0x18, 0x01, 0x20, 0x01, 0x28, 0x05, 0x52, 0x06, 0x73, 0x74, 0x61, 0x74, - 0x75, 0x73, 0x12, 0x2f, 0x0a, 0x05, 0x69, 0x6e, 0x66, 0x6f, 0x73, 0x18, 0x02, 0x20, 0x03, 0x28, - 0x0b, 0x32, 0x19, 0x2e, 0x6d, 0x67, 0x6d, 0x74, 0x2e, 0x50, 0x6f, 0x6f, 0x6c, 0x51, 0x75, 0x65, - 0x72, 0x79, 0x54, 0x61, 0x72, 0x67, 0x65, 0x74, 0x49, 0x6e, 0x66, 0x6f, 0x52, 0x05, 0x69, 0x6e, - 0x66, 0x6f, 0x73, 0x2a, 0x25, 0x0a, 0x10, 0x53, 0x74, 0x6f, 0x72, 0x61, 0x67, 0x65, 0x4d, 0x65, - 0x64, 0x69, 0x61, 0x54, 0x79, 0x70, 0x65, 0x12, 0x07, 0x0a, 0x03, 0x53, 0x43, 0x4d, 0x10, 0x00, - 0x12, 0x08, 0x0a, 0x04, 0x4e, 0x56, 0x4d, 0x45, 0x10, 0x01, 0x2a, 0x56, 0x0a, 0x10, 0x50, 0x6f, - 0x6f, 0x6c, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x53, 0x74, 0x61, 0x74, 0x65, 0x12, 0x0c, - 0x0a, 0x08, 0x43, 0x72, 0x65, 0x61, 0x74, 0x69, 0x6e, 0x67, 0x10, 0x00, 0x12, 0x09, 0x0a, 0x05, - 0x52, 0x65, 0x61, 0x64, 0x79, 0x10, 0x01, 0x12, 0x0e, 0x0a, 0x0a, 0x44, 0x65, 0x73, 0x74, 0x72, - 0x6f, 0x79, 0x69, 0x6e, 0x67, 0x10, 0x02, 0x12, 0x0c, 0x0a, 0x08, 0x44, 0x65, 0x67, 0x72, 0x61, - 0x64, 0x65, 0x64, 0x10, 0x03, 0x12, 0x0b, 0x0a, 0x07, 0x55, 0x6e, 0x6b, 0x6e, 0x6f, 0x77, 0x6e, - 0x10, 0x04, 0x42, 0x3a, 0x5a, 0x38, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, - 0x2f, 0x64, 0x61, 0x6f, 0x73, 0x2d, 0x73, 0x74, 0x61, 0x63, 0x6b, 0x2f, 0x64, 0x61, 0x6f, 0x73, - 0x2f, 0x73, 0x72, 0x63, 0x2f, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2f, 0x63, 0x6f, 0x6d, - 0x6d, 0x6f, 0x6e, 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2f, 0x6d, 0x67, 0x6d, 0x74, 0x62, 0x06, - 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, + 0x67, 0x65, 0x74, 0x49, 0x6e, 0x66, 0x6f, 0x2e, 0x54, 0x61, 0x72, 0x67, 0x65, 0x74, 0x53, 0x74, + 0x61, 0x74, 0x65, 0x52, 0x05, 0x73, 0x74, 0x61, 0x74, 0x65, 0x12, 0x2e, 0x0a, 0x05, 0x73, 0x70, + 0x61, 0x63, 0x65, 0x18, 0x03, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x18, 0x2e, 0x6d, 0x67, 0x6d, 0x74, + 0x2e, 0x53, 0x74, 0x6f, 0x72, 0x61, 0x67, 0x65, 0x54, 0x61, 0x72, 0x67, 0x65, 0x74, 0x55, 0x73, + 0x61, 0x67, 0x65, 0x52, 0x05, 0x73, 0x70, 0x61, 0x63, 0x65, 0x22, 0x3b, 0x0a, 0x0a, 0x54, 0x61, + 0x72, 0x67, 0x65, 0x74, 0x54, 0x79, 0x70, 0x65, 0x12, 0x0b, 0x0a, 0x07, 0x55, 0x4e, 0x4b, 0x4e, + 0x4f, 0x57, 0x4e, 0x10, 0x00, 0x12, 0x07, 0x0a, 0x03, 0x48, 0x44, 0x44, 0x10, 0x01, 0x12, 0x07, + 0x0a, 0x03, 0x53, 0x53, 0x44, 0x10, 0x02, 0x12, 0x06, 0x0a, 0x02, 0x50, 0x4d, 0x10, 0x03, 0x12, + 0x06, 0x0a, 0x02, 0x56, 0x4d, 0x10, 0x04, 0x22, 0x5f, 0x0a, 0x0b, 0x54, 0x61, 0x72, 0x67, 0x65, + 0x74, 0x53, 0x74, 0x61, 0x74, 0x65, 0x12, 0x11, 0x0a, 0x0d, 0x53, 0x54, 0x41, 0x54, 0x45, 0x5f, + 0x55, 0x4e, 0x4b, 0x4e, 0x4f, 0x57, 0x4e, 0x10, 0x00, 0x12, 0x0c, 0x0a, 0x08, 0x44, 0x4f, 0x57, + 0x4e, 0x5f, 0x4f, 0x55, 0x54, 0x10, 0x01, 0x12, 0x08, 0x0a, 0x04, 0x44, 0x4f, 0x57, 0x4e, 0x10, + 0x02, 0x12, 0x06, 0x0a, 0x02, 0x55, 0x50, 0x10, 0x03, 0x12, 0x09, 0x0a, 0x05, 0x55, 0x50, 0x5f, + 0x49, 0x4e, 0x10, 0x04, 0x12, 0x07, 0x0a, 0x03, 0x4e, 0x45, 0x57, 0x10, 0x05, 0x12, 0x09, 0x0a, + 0x05, 0x44, 0x52, 0x41, 0x49, 0x4e, 0x10, 0x06, 0x22, 0x5e, 0x0a, 0x13, 0x50, 0x6f, 0x6f, 0x6c, + 0x51, 0x75, 0x65, 0x72, 0x79, 0x54, 0x61, 0x72, 0x67, 0x65, 0x74, 0x52, 0x65, 0x73, 0x70, 0x12, + 0x16, 0x0a, 0x06, 0x73, 0x74, 0x61, 0x74, 0x75, 0x73, 0x18, 0x01, 0x20, 0x01, 0x28, 0x05, 0x52, + 0x06, 0x73, 0x74, 0x61, 0x74, 0x75, 0x73, 0x12, 0x2f, 0x0a, 0x05, 0x69, 0x6e, 0x66, 0x6f, 0x73, + 0x18, 0x02, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x19, 0x2e, 0x6d, 0x67, 0x6d, 0x74, 0x2e, 0x50, 0x6f, + 0x6f, 0x6c, 0x51, 0x75, 0x65, 0x72, 0x79, 0x54, 0x61, 0x72, 0x67, 0x65, 0x74, 0x49, 0x6e, 0x66, + 0x6f, 0x52, 0x05, 0x69, 0x6e, 0x66, 0x6f, 0x73, 0x2a, 0x25, 0x0a, 0x10, 0x53, 0x74, 0x6f, 0x72, + 0x61, 0x67, 0x65, 0x4d, 0x65, 0x64, 0x69, 0x61, 0x54, 0x79, 0x70, 0x65, 0x12, 0x07, 0x0a, 0x03, + 0x53, 0x43, 0x4d, 0x10, 0x00, 0x12, 0x08, 0x0a, 0x04, 0x4e, 0x56, 0x4d, 0x45, 0x10, 0x01, 0x2a, + 0x56, 0x0a, 0x10, 0x50, 0x6f, 0x6f, 0x6c, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x53, 0x74, + 0x61, 0x74, 0x65, 0x12, 0x0c, 0x0a, 0x08, 0x43, 0x72, 0x65, 0x61, 0x74, 0x69, 0x6e, 0x67, 0x10, + 0x00, 0x12, 0x09, 0x0a, 0x05, 0x52, 0x65, 0x61, 0x64, 0x79, 0x10, 0x01, 0x12, 0x0e, 0x0a, 0x0a, + 0x44, 0x65, 0x73, 0x74, 0x72, 0x6f, 0x79, 0x69, 0x6e, 0x67, 0x10, 0x02, 0x12, 0x0c, 0x0a, 0x08, + 0x44, 0x65, 0x67, 0x72, 0x61, 0x64, 0x65, 0x64, 0x10, 0x03, 0x12, 0x0b, 0x0a, 0x07, 0x55, 0x6e, + 0x6b, 0x6e, 0x6f, 0x77, 0x6e, 0x10, 0x04, 0x42, 0x3a, 0x5a, 0x38, 0x67, 0x69, 0x74, 0x68, 0x75, + 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x64, 0x61, 0x6f, 0x73, 0x2d, 0x73, 0x74, 0x61, 0x63, 0x6b, + 0x2f, 0x64, 0x61, 0x6f, 0x73, 0x2f, 0x73, 0x72, 0x63, 0x2f, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, + 0x6c, 0x2f, 0x63, 0x6f, 0x6d, 0x6d, 0x6f, 0x6e, 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2f, 0x6d, + 0x67, 0x6d, 0x74, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, } var ( diff --git a/src/control/lib/control/pool_test.go b/src/control/lib/control/pool_test.go index c111ba183cc..aace70eb067 100644 --- a/src/control/lib/control/pool_test.go +++ b/src/control/lib/control/pool_test.go @@ -809,7 +809,7 @@ func TestControl_PoolQueryResp_MarshalJSON(t *testing.T) { }, exp: `{"query_mask":"disabled_engines,rebuild,space","state":"Ready","uuid":"` + poolUUID.String() + `","total_targets":1,"active_targets":2,"total_engines":3,"disabled_targets":4,"version":5,"svc_ldr":6,"svc_reps":[0,1,2],"rebuild":null,"tier_stats":null,"pool_layout_ver":7,"upgrade_layout_ver":8,"status":42}`, }, - "valid rankset": { + "valid rankset default query": { pqr: &PoolQueryResp{ Status: 42, PoolInfo: daos.PoolInfo{ @@ -831,6 +831,28 @@ func TestControl_PoolQueryResp_MarshalJSON(t *testing.T) { }, exp: `{"query_mask":"disabled_engines,rebuild,space","state":"Ready","uuid":"` + poolUUID.String() + `","total_targets":1,"active_targets":2,"total_engines":3,"disabled_targets":4,"version":5,"svc_ldr":6,"svc_reps":[0,1,2],"rebuild":null,"tier_stats":null,"enabled_ranks":[0,1,2,3,5],"disabled_ranks":[],"pool_layout_ver":7,"upgrade_layout_ver":8,"status":42}`, }, + "valid rankset health query": { + pqr: &PoolQueryResp{ + Status: 42, + PoolInfo: daos.PoolInfo{ + QueryMask: daos.HealthOnlyPoolQueryMask, + State: daos.PoolServiceStateReady, + UUID: poolUUID, + TotalTargets: 1, + ActiveTargets: 2, + TotalEngines: 3, + DisabledTargets: 4, + Version: 5, + ServiceLeader: 6, + ServiceReplicas: []ranklist.Rank{0, 1, 2}, + DisabledRanks: &ranklist.RankSet{}, + SuspectRanks: ranklist.MustCreateRankSet("[7,8,9]"), + PoolLayoutVer: 7, + UpgradeLayoutVer: 8, + }, + }, + exp: `{"query_mask":"disabled_engines,rebuild,suspect_engines","state":"Ready","uuid":"` + poolUUID.String() + `","total_targets":1,"active_targets":2,"total_engines":3,"disabled_targets":4,"version":5,"svc_ldr":6,"svc_reps":[0,1,2],"rebuild":null,"tier_stats":null,"disabled_ranks":[],"suspect_ranks":[7,8,9],"pool_layout_ver":7,"upgrade_layout_ver":8,"mem_file_bytes":0,"status":42}`, + }, } { t.Run(name, func(t *testing.T) { got, err := json.Marshal(tc.pqr) @@ -871,7 +893,7 @@ func TestControl_PoolQueryResp_UnmarshalJSON(t *testing.T) { }, }, "valid rankset": { - data: `{"enabled_ranks":"[0,1-3,5]","disabled_ranks":"[]","status":0,"uuid":"` + poolUUID.String() + `","total_targets":1,"active_targets":2,"total_engines":3,"disabled_targets":4,"version":5,"svc_ldr":6,"svc_reps":null,"rebuild":null,"tier_stats":null,"pool_layout_ver":7,"upgrade_layout_ver":8}`, + data: `{"enabled_ranks":"[0,1-3,5]","disabled_ranks":"[]","suspect_ranks":"[4]","status":0,"uuid":"` + poolUUID.String() + `","total_targets":1,"active_targets":2,"total_engines":3,"disabled_targets":4,"version":5,"svc_ldr":6,"svc_reps":null,"rebuild":null,"tier_stats":null,"pool_layout_ver":7,"upgrade_layout_ver":8}`, expResp: PoolQueryResp{ Status: 0, PoolInfo: daos.PoolInfo{ @@ -884,6 +906,7 @@ func TestControl_PoolQueryResp_UnmarshalJSON(t *testing.T) { ServiceLeader: 6, EnabledRanks: ranklist.MustCreateRankSet("[0-3,5]"), DisabledRanks: &ranklist.RankSet{}, + SuspectRanks: ranklist.MustCreateRankSet("[4]"), PoolLayoutVer: 7, UpgradeLayoutVer: 8, }, @@ -1154,6 +1177,80 @@ func TestControl_PoolQuery(t *testing.T) { }, }, }, + "query succeeds suspect ranks": { + mic: &MockInvokerConfig{ + UnaryResponse: MockMSResponse("host1", nil, + &mgmtpb.PoolQueryResp{ + Uuid: poolUUID.String(), + TotalTargets: 42, + ActiveTargets: 16, + DisabledTargets: 17, + PoolLayoutVer: 1, + UpgradeLayoutVer: 2, + State: mgmtpb.PoolServiceState_Degraded, + Rebuild: &mgmtpb.PoolRebuildStatus{ + State: mgmtpb.PoolRebuildStatus_BUSY, + Objects: 1, + Records: 2, + }, + TierStats: []*mgmtpb.StorageUsageStats{ + { + Total: 123456, + Free: 0, + Min: 1, + Max: 2, + Mean: 3, + MediaType: mgmtpb.StorageMediaType(daos.StorageMediaTypeScm), + }, + { + Total: 123456, + Free: 0, + Min: 1, + Max: 2, + Mean: 3, + MediaType: mgmtpb.StorageMediaType(daos.StorageMediaTypeNvme), + }, + }, + SuspectRanks: "[1,2,3,7]", + }, + ), + }, + expResp: &PoolQueryResp{ + PoolInfo: daos.PoolInfo{ + UUID: poolUUID, + TotalTargets: 42, + ActiveTargets: 16, + DisabledTargets: 17, + PoolLayoutVer: 1, + UpgradeLayoutVer: 2, + State: daos.PoolServiceStateDegraded, + Rebuild: &daos.PoolRebuildStatus{ + State: daos.PoolRebuildStateBusy, + Objects: 1, + Records: 2, + }, + TierStats: []*daos.StorageUsageStats{ + { + Total: 123456, + Free: 0, + Min: 1, + Max: 2, + Mean: 3, + MediaType: daos.StorageMediaTypeScm, + }, + { + Total: 123456, + Free: 0, + Min: 1, + Max: 2, + Mean: 3, + MediaType: daos.StorageMediaTypeNvme, + }, + }, + SuspectRanks: ranklist.MustCreateRankSet("[1-3,7]"), + }, + }, + }, } { t.Run(name, func(t *testing.T) { log, buf := logging.NewTestLogger(t.Name()) diff --git a/src/control/lib/daos/pool.go b/src/control/lib/daos/pool.go index e47e6e2b23d..b06a44c9025 100644 --- a/src/control/lib/daos/pool.go +++ b/src/control/lib/daos/pool.go @@ -77,6 +77,7 @@ type ( TierStats []*StorageUsageStats `json:"tier_stats"` EnabledRanks *ranklist.RankSet `json:"enabled_ranks,omitempty"` DisabledRanks *ranklist.RankSet `json:"disabled_ranks,omitempty"` + SuspectRanks *ranklist.RankSet `json:"suspect_ranks,omitempty"` PoolLayoutVer uint32 `json:"pool_layout_ver"` UpgradeLayoutVer uint32 `json:"upgrade_layout_ver"` } @@ -104,7 +105,7 @@ type ( const ( // DefaultPoolQueryMask defines the default pool query mask. - DefaultPoolQueryMask = PoolQueryMask(^uint64(0) &^ C.DPI_ENGINES_ENABLED) + DefaultPoolQueryMask = PoolQueryMask(^uint64(0) &^ (C.DPI_ENGINES_ENABLED | C.DPI_ENGINES_SUSPECT)) // HealthOnlyPoolQueryMask defines the mask for health-only queries. HealthOnlyPoolQueryMask = PoolQueryMask(^uint64(0) &^ (C.DPI_ENGINES_ENABLED | C.DPI_SPACE)) @@ -116,6 +117,8 @@ const ( PoolQueryOptionEnabledEngines = "enabled_engines" // PoolQueryOptionDisabledEngines retrieves disabled engines as part of the pool query. PoolQueryOptionDisabledEngines = "disabled_engines" + // PoolQueryOptionSuspectEngines retrieves suspect engines as part of the pool query. + PoolQueryOptionSuspectEngines = "suspect_engines" // PoolConnectFlagReadOnly indicates that the connection is read-only. PoolConnectFlagReadOnly = C.DAOS_PC_RO @@ -130,6 +133,7 @@ var poolQueryOptMap = map[C.int]string{ C.DPI_REBUILD_STATUS: PoolQueryOptionRebuild, C.DPI_ENGINES_ENABLED: PoolQueryOptionEnabledEngines, C.DPI_ENGINES_DISABLED: PoolQueryOptionDisabledEngines, + C.DPI_ENGINES_SUSPECT: PoolQueryOptionSuspectEngines, } func resolvePoolQueryOpt(name string) (C.int, error) { diff --git a/src/control/lib/daos/pool_test.go b/src/control/lib/daos/pool_test.go index 3ff1ca098d2..1b91b9f5520 100644 --- a/src/control/lib/daos/pool_test.go +++ b/src/control/lib/daos/pool_test.go @@ -136,13 +136,14 @@ func TestDaos_PoolQueryMask(t *testing.T) { testMask: genTestMask(func(pqm *PoolQueryMask) { *pqm = HealthOnlyPoolQueryMask }), - expString: genOptsStr(PoolQueryOptionDisabledEngines, PoolQueryOptionRebuild), + expString: genOptsStr(PoolQueryOptionDisabledEngines, PoolQueryOptionRebuild, PoolQueryOptionSuspectEngines), }, "set query all=true": { testMask: genTestMask(func(pqm *PoolQueryMask) { pqm.SetAll() }), - expString: genOptsStr(PoolQueryOptionDisabledEngines, PoolQueryOptionEnabledEngines, PoolQueryOptionRebuild, PoolQueryOptionSpace), + expString: genOptsStr(PoolQueryOptionDisabledEngines, PoolQueryOptionEnabledEngines, + PoolQueryOptionRebuild, PoolQueryOptionSpace, PoolQueryOptionSuspectEngines), }, "set query all=false": { testMask: genTestMask(func(pqm *PoolQueryMask) { @@ -162,7 +163,8 @@ func TestDaos_PoolQueryMask(t *testing.T) { pqm.SetAll() pqm.ClearOptions(PoolQueryOptionSpace) }), - expString: genOptsStr(PoolQueryOptionDisabledEngines, PoolQueryOptionEnabledEngines, PoolQueryOptionRebuild), + expString: genOptsStr(PoolQueryOptionDisabledEngines, PoolQueryOptionEnabledEngines, + PoolQueryOptionRebuild, PoolQueryOptionSuspectEngines), }, "set query space=false (already false)": { testMask: genTestMask(func(pqm *PoolQueryMask) { @@ -181,7 +183,8 @@ func TestDaos_PoolQueryMask(t *testing.T) { pqm.SetAll() pqm.ClearOptions(PoolQueryOptionRebuild) }), - expString: genOptsStr(PoolQueryOptionDisabledEngines, PoolQueryOptionEnabledEngines, PoolQueryOptionSpace), + expString: genOptsStr(PoolQueryOptionDisabledEngines, PoolQueryOptionEnabledEngines, PoolQueryOptionSpace, + PoolQueryOptionSuspectEngines), }, "set query enabled_engines=true": { testMask: genTestMask(func(pqm *PoolQueryMask) { @@ -194,7 +197,8 @@ func TestDaos_PoolQueryMask(t *testing.T) { pqm.SetAll() pqm.ClearOptions(PoolQueryOptionEnabledEngines) }), - expString: genOptsStr(PoolQueryOptionDisabledEngines, PoolQueryOptionRebuild, PoolQueryOptionSpace), + expString: genOptsStr(PoolQueryOptionDisabledEngines, PoolQueryOptionRebuild, PoolQueryOptionSpace, + PoolQueryOptionSuspectEngines), }, "set query disabled_engines=true": { testMask: genTestMask(func(pqm *PoolQueryMask) { @@ -207,7 +211,8 @@ func TestDaos_PoolQueryMask(t *testing.T) { pqm.SetAll() pqm.ClearOptions(PoolQueryOptionDisabledEngines) }), - expString: genOptsStr(PoolQueryOptionEnabledEngines, PoolQueryOptionRebuild, PoolQueryOptionSpace), + expString: genOptsStr(PoolQueryOptionEnabledEngines, PoolQueryOptionRebuild, PoolQueryOptionSpace, + PoolQueryOptionSuspectEngines), }, } { t.Run(name, func(t *testing.T) { @@ -232,7 +237,7 @@ func TestDaos_PoolQueryMaskMarshalJSON(t *testing.T) { testMask: genTestMask(func(pqm *PoolQueryMask) { pqm.SetAll() }), - expJSON: []byte(`"disabled_engines,enabled_engines,rebuild,space"`), + expJSON: []byte(`"disabled_engines,enabled_engines,rebuild,space,suspect_engines"`), }, } { t.Run(name, func(t *testing.T) { @@ -262,7 +267,7 @@ func TestDaos_PoolQueryMaskUnmarshalJSON(t *testing.T) { }, "uint64 value": { testData: []byte("18446744073709551603"), - expString: "rebuild,space", + expString: "rebuild,space,suspect_engines", }, "string values": { testData: []byte("rebuild,disabled_engines"), diff --git a/src/include/daos_pool.h b/src/include/daos_pool.h index 73f44368913..a8ab2e6c6a2 100644 --- a/src/include/daos_pool.h +++ b/src/include/daos_pool.h @@ -162,6 +162,8 @@ enum daos_pool_info_bit { DPI_ENGINES_ENABLED = 1ULL << 2, /** true to include (in \a ranks) engines with some or all targets disabled (down). */ DPI_ENGINES_DISABLED = 1ULL << 3, + /** true to include (in \a ranks) suspect engines. */ + DPI_ENGINES_SUSPECT = 1ULL << 4, /** query all above optional info */ DPI_ALL = -1, }; diff --git a/src/include/daos_srv/pool.h b/src/include/daos_srv/pool.h index 985c6346a34..a8ea070446b 100644 --- a/src/include/daos_srv/pool.h +++ b/src/include/daos_srv/pool.h @@ -307,8 +307,8 @@ int dsc_pool_svc_delete_acl(uuid_t pool_uuid, d_rank_list_t *ranks, uint64_t dea int dsc_pool_svc_query(uuid_t pool_uuid, d_rank_list_t *ps_ranks, uint64_t deadline, d_rank_list_t **enabled_ranks, d_rank_list_t **disabled_ranks, - daos_pool_info_t *pool_info, uint32_t *pool_layout_ver, - uint32_t *upgrade_layout_ver); + d_rank_list_t **suspect_ranks, daos_pool_info_t *pool_info, + uint32_t *pool_layout_ver, uint32_t *upgrade_layout_ver); int dsc_pool_svc_query_target(uuid_t pool_uuid, d_rank_list_t *ps_ranks, uint64_t deadline, d_rank_t rank, uint32_t tgt_idx, daos_target_info_t *ti); diff --git a/src/mgmt/pool.pb-c.c b/src/mgmt/pool.pb-c.c index 3859da6374c..2137c1f5f53 100644 --- a/src/mgmt/pool.pb-c.c +++ b/src/mgmt/pool.pb-c.c @@ -3292,278 +3292,162 @@ const ProtobufCMessageDescriptor mgmt__pool_rebuild_status__descriptor = (ProtobufCMessageInit) mgmt__pool_rebuild_status__init, NULL,NULL,NULL /* reserved[123] */ }; -static const ProtobufCFieldDescriptor mgmt__pool_query_resp__field_descriptors[19] = -{ - { - "status", - 1, - PROTOBUF_C_LABEL_NONE, - PROTOBUF_C_TYPE_INT32, - 0, /* quantifier_offset */ - offsetof(Mgmt__PoolQueryResp, status), - NULL, - NULL, - 0, /* flags */ - 0,NULL,NULL /* reserved1,reserved2, etc */ - }, - { - "uuid", - 2, - PROTOBUF_C_LABEL_NONE, - PROTOBUF_C_TYPE_STRING, - 0, /* quantifier_offset */ - offsetof(Mgmt__PoolQueryResp, uuid), - NULL, - &protobuf_c_empty_string, - 0, /* flags */ - 0,NULL,NULL /* reserved1,reserved2, etc */ - }, - { - "label", - 3, - PROTOBUF_C_LABEL_NONE, - PROTOBUF_C_TYPE_STRING, - 0, /* quantifier_offset */ - offsetof(Mgmt__PoolQueryResp, label), - NULL, - &protobuf_c_empty_string, - 0, /* flags */ - 0,NULL,NULL /* reserved1,reserved2, etc */ - }, - { - "total_targets", - 4, - PROTOBUF_C_LABEL_NONE, - PROTOBUF_C_TYPE_UINT32, - 0, /* quantifier_offset */ - offsetof(Mgmt__PoolQueryResp, total_targets), - NULL, - NULL, - 0, /* flags */ - 0,NULL,NULL /* reserved1,reserved2, etc */ - }, - { - "active_targets", - 5, - PROTOBUF_C_LABEL_NONE, - PROTOBUF_C_TYPE_UINT32, - 0, /* quantifier_offset */ - offsetof(Mgmt__PoolQueryResp, active_targets), - NULL, - NULL, - 0, /* flags */ - 0,NULL,NULL /* reserved1,reserved2, etc */ - }, - { - "disabled_targets", - 6, - PROTOBUF_C_LABEL_NONE, - PROTOBUF_C_TYPE_UINT32, - 0, /* quantifier_offset */ - offsetof(Mgmt__PoolQueryResp, disabled_targets), - NULL, - NULL, - 0, /* flags */ - 0,NULL,NULL /* reserved1,reserved2, etc */ - }, - { - "rebuild", - 7, - PROTOBUF_C_LABEL_NONE, - PROTOBUF_C_TYPE_MESSAGE, - 0, /* quantifier_offset */ - offsetof(Mgmt__PoolQueryResp, rebuild), - &mgmt__pool_rebuild_status__descriptor, - NULL, - 0, /* flags */ - 0,NULL,NULL /* reserved1,reserved2, etc */ - }, - { - "tier_stats", - 8, - PROTOBUF_C_LABEL_REPEATED, - PROTOBUF_C_TYPE_MESSAGE, - offsetof(Mgmt__PoolQueryResp, n_tier_stats), - offsetof(Mgmt__PoolQueryResp, tier_stats), - &mgmt__storage_usage_stats__descriptor, - NULL, - 0, /* flags */ - 0,NULL,NULL /* reserved1,reserved2, etc */ - }, - { - "version", - 10, - PROTOBUF_C_LABEL_NONE, - PROTOBUF_C_TYPE_UINT32, - 0, /* quantifier_offset */ - offsetof(Mgmt__PoolQueryResp, version), - NULL, - NULL, - 0, /* flags */ - 0,NULL,NULL /* reserved1,reserved2, etc */ - }, - { - "leader", - 11, - PROTOBUF_C_LABEL_NONE, - PROTOBUF_C_TYPE_UINT32, - 0, /* quantifier_offset */ - offsetof(Mgmt__PoolQueryResp, leader), - NULL, - NULL, - 0, /* flags */ - 0,NULL,NULL /* reserved1,reserved2, etc */ - }, - { - "enabled_ranks", - 12, - PROTOBUF_C_LABEL_NONE, - PROTOBUF_C_TYPE_STRING, - 0, /* quantifier_offset */ - offsetof(Mgmt__PoolQueryResp, enabled_ranks), - NULL, - &protobuf_c_empty_string, - 0, /* flags */ - 0,NULL,NULL /* reserved1,reserved2, etc */ - }, - { - "disabled_ranks", - 13, - PROTOBUF_C_LABEL_NONE, - PROTOBUF_C_TYPE_STRING, - 0, /* quantifier_offset */ - offsetof(Mgmt__PoolQueryResp, disabled_ranks), - NULL, - &protobuf_c_empty_string, - 0, /* flags */ - 0,NULL,NULL /* reserved1,reserved2, etc */ - }, - { - "total_engines", - 14, - PROTOBUF_C_LABEL_NONE, - PROTOBUF_C_TYPE_UINT32, - 0, /* quantifier_offset */ - offsetof(Mgmt__PoolQueryResp, total_engines), - NULL, - NULL, - 0, /* flags */ - 0,NULL,NULL /* reserved1,reserved2, etc */ - }, - { - "pool_layout_ver", - 15, - PROTOBUF_C_LABEL_NONE, - PROTOBUF_C_TYPE_UINT32, - 0, /* quantifier_offset */ - offsetof(Mgmt__PoolQueryResp, pool_layout_ver), - NULL, - NULL, - 0, /* flags */ - 0,NULL,NULL /* reserved1,reserved2, etc */ - }, - { - "upgrade_layout_ver", - 16, - PROTOBUF_C_LABEL_NONE, - PROTOBUF_C_TYPE_UINT32, - 0, /* quantifier_offset */ - offsetof(Mgmt__PoolQueryResp, upgrade_layout_ver), - NULL, - NULL, - 0, /* flags */ - 0,NULL,NULL /* reserved1,reserved2, etc */ - }, - { - "state", - 17, - PROTOBUF_C_LABEL_NONE, - PROTOBUF_C_TYPE_ENUM, - 0, /* quantifier_offset */ - offsetof(Mgmt__PoolQueryResp, state), - &mgmt__pool_service_state__descriptor, - NULL, - 0, /* flags */ - 0,NULL,NULL /* reserved1,reserved2, etc */ - }, - { - "svc_ldr", - 18, - PROTOBUF_C_LABEL_NONE, - PROTOBUF_C_TYPE_UINT32, - 0, /* quantifier_offset */ - offsetof(Mgmt__PoolQueryResp, svc_ldr), - NULL, - NULL, - 0, /* flags */ - 0,NULL,NULL /* reserved1,reserved2, etc */ - }, - { - "svc_reps", - 19, - PROTOBUF_C_LABEL_REPEATED, - PROTOBUF_C_TYPE_UINT32, - offsetof(Mgmt__PoolQueryResp, n_svc_reps), - offsetof(Mgmt__PoolQueryResp, svc_reps), - NULL, - NULL, - 0, /* flags */ - 0,NULL,NULL /* reserved1,reserved2, etc */ - }, - { - "query_mask", +static const ProtobufCFieldDescriptor mgmt__pool_query_resp__field_descriptors[20] = { + { + "status", 1, PROTOBUF_C_LABEL_NONE, PROTOBUF_C_TYPE_INT32, 0, /* quantifier_offset */ + offsetof(Mgmt__PoolQueryResp, status), NULL, NULL, 0, /* flags */ + 0, NULL, NULL /* reserved1,reserved2, etc */ + }, + { + "uuid", 2, PROTOBUF_C_LABEL_NONE, PROTOBUF_C_TYPE_STRING, 0, /* quantifier_offset */ + offsetof(Mgmt__PoolQueryResp, uuid), NULL, &protobuf_c_empty_string, 0, /* flags */ + 0, NULL, NULL /* reserved1,reserved2, etc */ + }, + { + "label", 3, PROTOBUF_C_LABEL_NONE, PROTOBUF_C_TYPE_STRING, 0, /* quantifier_offset */ + offsetof(Mgmt__PoolQueryResp, label), NULL, &protobuf_c_empty_string, 0, /* flags */ + 0, NULL, NULL /* reserved1,reserved2, etc */ + }, + { + "total_targets", 4, PROTOBUF_C_LABEL_NONE, PROTOBUF_C_TYPE_UINT32, + 0, /* quantifier_offset */ + offsetof(Mgmt__PoolQueryResp, total_targets), NULL, NULL, 0, /* flags */ + 0, NULL, NULL /* reserved1,reserved2, etc */ + }, + { + "active_targets", 5, PROTOBUF_C_LABEL_NONE, PROTOBUF_C_TYPE_UINT32, + 0, /* quantifier_offset */ + offsetof(Mgmt__PoolQueryResp, active_targets), NULL, NULL, 0, /* flags */ + 0, NULL, NULL /* reserved1,reserved2, etc */ + }, + { + "disabled_targets", 6, PROTOBUF_C_LABEL_NONE, PROTOBUF_C_TYPE_UINT32, + 0, /* quantifier_offset */ + offsetof(Mgmt__PoolQueryResp, disabled_targets), NULL, NULL, 0, /* flags */ + 0, NULL, NULL /* reserved1,reserved2, etc */ + }, + { + "rebuild", 7, PROTOBUF_C_LABEL_NONE, PROTOBUF_C_TYPE_MESSAGE, 0, /* quantifier_offset */ + offsetof(Mgmt__PoolQueryResp, rebuild), &mgmt__pool_rebuild_status__descriptor, NULL, + 0, /* flags */ + 0, NULL, NULL /* reserved1,reserved2, etc */ + }, + { + "tier_stats", 8, PROTOBUF_C_LABEL_REPEATED, PROTOBUF_C_TYPE_MESSAGE, + offsetof(Mgmt__PoolQueryResp, n_tier_stats), offsetof(Mgmt__PoolQueryResp, tier_stats), + &mgmt__storage_usage_stats__descriptor, NULL, 0, /* flags */ + 0, NULL, NULL /* reserved1,reserved2, etc */ + }, + { + "version", 10, PROTOBUF_C_LABEL_NONE, PROTOBUF_C_TYPE_UINT32, 0, /* quantifier_offset */ + offsetof(Mgmt__PoolQueryResp, version), NULL, NULL, 0, /* flags */ + 0, NULL, NULL /* reserved1,reserved2, etc */ + }, + { + "leader", 11, PROTOBUF_C_LABEL_NONE, PROTOBUF_C_TYPE_UINT32, 0, /* quantifier_offset */ + offsetof(Mgmt__PoolQueryResp, leader), NULL, NULL, 0, /* flags */ + 0, NULL, NULL /* reserved1,reserved2, etc */ + }, + { + "enabled_ranks", 12, PROTOBUF_C_LABEL_NONE, PROTOBUF_C_TYPE_STRING, + 0, /* quantifier_offset */ + offsetof(Mgmt__PoolQueryResp, enabled_ranks), NULL, &protobuf_c_empty_string, 0, /* flags */ + 0, NULL, NULL /* reserved1,reserved2, etc */ + }, + { + "disabled_ranks", 13, PROTOBUF_C_LABEL_NONE, PROTOBUF_C_TYPE_STRING, + 0, /* quantifier_offset */ + offsetof(Mgmt__PoolQueryResp, disabled_ranks), NULL, &protobuf_c_empty_string, + 0, /* flags */ + 0, NULL, NULL /* reserved1,reserved2, etc */ + }, + { + "total_engines", 14, PROTOBUF_C_LABEL_NONE, PROTOBUF_C_TYPE_UINT32, + 0, /* quantifier_offset */ + offsetof(Mgmt__PoolQueryResp, total_engines), NULL, NULL, 0, /* flags */ + 0, NULL, NULL /* reserved1,reserved2, etc */ + }, + { + "pool_layout_ver", 15, PROTOBUF_C_LABEL_NONE, PROTOBUF_C_TYPE_UINT32, + 0, /* quantifier_offset */ + offsetof(Mgmt__PoolQueryResp, pool_layout_ver), NULL, NULL, 0, /* flags */ + 0, NULL, NULL /* reserved1,reserved2, etc */ + }, + { + "upgrade_layout_ver", 16, PROTOBUF_C_LABEL_NONE, PROTOBUF_C_TYPE_UINT32, + 0, /* quantifier_offset */ + offsetof(Mgmt__PoolQueryResp, upgrade_layout_ver), NULL, NULL, 0, /* flags */ + 0, NULL, NULL /* reserved1,reserved2, etc */ + }, + { + "state", 17, PROTOBUF_C_LABEL_NONE, PROTOBUF_C_TYPE_ENUM, 0, /* quantifier_offset */ + offsetof(Mgmt__PoolQueryResp, state), &mgmt__pool_service_state__descriptor, NULL, + 0, /* flags */ + 0, NULL, NULL /* reserved1,reserved2, etc */ + }, + { + "svc_ldr", 18, PROTOBUF_C_LABEL_NONE, PROTOBUF_C_TYPE_UINT32, 0, /* quantifier_offset */ + offsetof(Mgmt__PoolQueryResp, svc_ldr), NULL, NULL, 0, /* flags */ + 0, NULL, NULL /* reserved1,reserved2, etc */ + }, + { + "svc_reps", 19, PROTOBUF_C_LABEL_REPEATED, PROTOBUF_C_TYPE_UINT32, + offsetof(Mgmt__PoolQueryResp, n_svc_reps), offsetof(Mgmt__PoolQueryResp, svc_reps), NULL, + NULL, 0, /* flags */ + 0, NULL, NULL /* reserved1,reserved2, etc */ + }, + { + "query_mask", 20, PROTOBUF_C_LABEL_NONE, PROTOBUF_C_TYPE_UINT64, 0, /* quantifier_offset */ + offsetof(Mgmt__PoolQueryResp, query_mask), NULL, NULL, 0, /* flags */ + 0, NULL, NULL /* reserved1,reserved2, etc */ + }, + { + "suspect_ranks", 21, PROTOBUF_C_LABEL_NONE, PROTOBUF_C_TYPE_STRING, + 0, /* quantifier_offset */ + offsetof(Mgmt__PoolQueryResp, suspect_ranks), NULL, &protobuf_c_empty_string, 0, /* flags */ + 0, NULL, NULL /* reserved1,reserved2, etc */ + }, +}; +static const unsigned mgmt__pool_query_resp__field_indices_by_name[] = { + 4, /* field[4] = active_targets */ + 11, /* field[11] = disabled_ranks */ + 5, /* field[5] = disabled_targets */ + 10, /* field[10] = enabled_ranks */ + 2, /* field[2] = label */ + 9, /* field[9] = leader */ + 13, /* field[13] = pool_layout_ver */ + 18, /* field[18] = query_mask */ + 6, /* field[6] = rebuild */ + 15, /* field[15] = state */ + 0, /* field[0] = status */ + 19, /* field[19] = suspect_ranks */ + 16, /* field[16] = svc_ldr */ + 17, /* field[17] = svc_reps */ + 7, /* field[7] = tier_stats */ + 12, /* field[12] = total_engines */ + 3, /* field[3] = total_targets */ + 14, /* field[14] = upgrade_layout_ver */ + 1, /* field[1] = uuid */ + 8, /* field[8] = version */ +}; +static const ProtobufCIntRange mgmt__pool_query_resp__number_ranges[2 + 1] = { + {1, 0}, {10, 8}, {0, 20}}; +const ProtobufCMessageDescriptor mgmt__pool_query_resp__descriptor = { + PROTOBUF_C__MESSAGE_DESCRIPTOR_MAGIC, + "mgmt.PoolQueryResp", + "PoolQueryResp", + "Mgmt__PoolQueryResp", + "mgmt", + sizeof(Mgmt__PoolQueryResp), 20, - PROTOBUF_C_LABEL_NONE, - PROTOBUF_C_TYPE_UINT64, - 0, /* quantifier_offset */ - offsetof(Mgmt__PoolQueryResp, query_mask), + mgmt__pool_query_resp__field_descriptors, + mgmt__pool_query_resp__field_indices_by_name, + 2, + mgmt__pool_query_resp__number_ranges, + (ProtobufCMessageInit)mgmt__pool_query_resp__init, NULL, NULL, - 0, /* flags */ - 0,NULL,NULL /* reserved1,reserved2, etc */ - }, -}; -static const unsigned mgmt__pool_query_resp__field_indices_by_name[] = { - 4, /* field[4] = active_targets */ - 11, /* field[11] = disabled_ranks */ - 5, /* field[5] = disabled_targets */ - 10, /* field[10] = enabled_ranks */ - 2, /* field[2] = label */ - 9, /* field[9] = leader */ - 13, /* field[13] = pool_layout_ver */ - 18, /* field[18] = query_mask */ - 6, /* field[6] = rebuild */ - 15, /* field[15] = state */ - 0, /* field[0] = status */ - 16, /* field[16] = svc_ldr */ - 17, /* field[17] = svc_reps */ - 7, /* field[7] = tier_stats */ - 12, /* field[12] = total_engines */ - 3, /* field[3] = total_targets */ - 14, /* field[14] = upgrade_layout_ver */ - 1, /* field[1] = uuid */ - 8, /* field[8] = version */ -}; -static const ProtobufCIntRange mgmt__pool_query_resp__number_ranges[2 + 1] = -{ - { 1, 0 }, - { 10, 8 }, - { 0, 19 } -}; -const ProtobufCMessageDescriptor mgmt__pool_query_resp__descriptor = -{ - PROTOBUF_C__MESSAGE_DESCRIPTOR_MAGIC, - "mgmt.PoolQueryResp", - "PoolQueryResp", - "Mgmt__PoolQueryResp", - "mgmt", - sizeof(Mgmt__PoolQueryResp), - 19, - mgmt__pool_query_resp__field_descriptors, - mgmt__pool_query_resp__field_indices_by_name, - 2, mgmt__pool_query_resp__number_ranges, - (ProtobufCMessageInit) mgmt__pool_query_resp__init, - NULL,NULL,NULL /* reserved[123] */ + NULL /* reserved[123] */ }; static const ProtobufCFieldDescriptor mgmt__pool_property__field_descriptors[3] = { diff --git a/src/mgmt/pool.pb-c.h b/src/mgmt/pool.pb-c.h index ad8dee1e9d9..7d8e18c96c3 100644 --- a/src/mgmt/pool.pb-c.h +++ b/src/mgmt/pool.pb-c.h @@ -861,11 +861,19 @@ struct _Mgmt__PoolQueryResp * Bitmask of pool query options used */ uint64_t query_mask; + /* + * optional set of suspect ranks + */ + char *suspect_ranks; }; -#define MGMT__POOL_QUERY_RESP__INIT \ - { PROTOBUF_C_MESSAGE_INIT (&mgmt__pool_query_resp__descriptor) \ - , 0, (char *)protobuf_c_empty_string, (char *)protobuf_c_empty_string, 0, 0, 0, NULL, 0,NULL, 0, 0, (char *)protobuf_c_empty_string, (char *)protobuf_c_empty_string, 0, 0, 0, MGMT__POOL_SERVICE_STATE__Creating, 0, 0,NULL, 0 } - +#define MGMT__POOL_QUERY_RESP__INIT \ + { \ + PROTOBUF_C_MESSAGE_INIT(&mgmt__pool_query_resp__descriptor) \ + , 0, (char *)protobuf_c_empty_string, (char *)protobuf_c_empty_string, 0, 0, 0, \ + NULL, 0, NULL, 0, 0, (char *)protobuf_c_empty_string, \ + (char *)protobuf_c_empty_string, 0, 0, 0, MGMT__POOL_SERVICE_STATE__Creating, \ + 0, 0, NULL, 0, (char *)protobuf_c_empty_string \ + } typedef enum { MGMT__POOL_PROPERTY__VALUE__NOT_SET = 0, diff --git a/src/mgmt/srv_drpc.c b/src/mgmt/srv_drpc.c index 0ddedeabc59..0a176dbccc2 100644 --- a/src/mgmt/srv_drpc.c +++ b/src/mgmt/srv_drpc.c @@ -394,7 +394,8 @@ static int pool_create_fill_resp(Mgmt__PoolCreateResp *resp, uuid_t uuid, d_rank D_DEBUG(DB_MGMT, "%d service replicas\n", svc_ranks->rl_nr); - rc = ds_mgmt_pool_query(uuid, svc_ranks, &enabled_ranks, NULL, &pool_info, NULL, NULL); + rc = + ds_mgmt_pool_query(uuid, svc_ranks, &enabled_ranks, NULL, NULL, &pool_info, NULL, NULL); if (rc != 0) { D_ERROR("Failed to query created pool: rc=%d\n", rc); D_GOTO(out, rc); @@ -1746,8 +1747,10 @@ ds_mgmt_drpc_pool_query(Drpc__Call *drpc_req, Drpc__Response *drpc_resp) d_rank_list_t *svc_ranks = NULL; d_rank_list_t *enabled_ranks = NULL; d_rank_list_t *disabled_ranks = NULL; + d_rank_list_t *suspect_ranks = NULL; char *enabled_ranks_str = NULL; char *disabled_ranks_str = NULL; + char *suspect_ranks_str = NULL; size_t len; uint8_t *body; @@ -1771,8 +1774,8 @@ ds_mgmt_drpc_pool_query(Drpc__Call *drpc_req, Drpc__Response *drpc_resp) D_GOTO(error, rc = -DER_NOMEM); pool_info.pi_bits = req->query_mask; - rc = ds_mgmt_pool_query(uuid, svc_ranks, &enabled_ranks, &disabled_ranks, &pool_info, - &resp.pool_layout_ver, &resp.upgrade_layout_ver); + rc = ds_mgmt_pool_query(uuid, svc_ranks, &enabled_ranks, &disabled_ranks, &suspect_ranks, + &pool_info, &resp.pool_layout_ver, &resp.upgrade_layout_ver); if (rc != 0) { DL_ERROR(rc, DF_UUID ": Failed to query the pool", DP_UUID(uuid)); D_GOTO(error, rc); @@ -1794,9 +1797,18 @@ ds_mgmt_drpc_pool_query(Drpc__Call *drpc_req, Drpc__Response *drpc_resp) DP_UUID(uuid)); D_GOTO(error, rc); } + rc = d_rank_list_to_str(suspect_ranks, &suspect_ranks_str); + if (rc != 0) { + DL_ERROR(rc, DF_UUID ": Failed to serialize the list of suspect ranks", + DP_UUID(uuid)); + D_GOTO(error, rc); + } if (disabled_ranks_str != NULL) D_DEBUG(DB_MGMT, DF_UUID ": list of disabled ranks: %s\n", DP_UUID(uuid), disabled_ranks_str); + if (suspect_ranks_str != NULL) + D_DEBUG(DB_MGMT, DF_UUID ": list of suspect ranks: %s\n", DP_UUID(uuid), + suspect_ranks_str); /* Populate the response */ resp.query_mask = pool_info.pi_bits; @@ -1813,6 +1825,8 @@ ds_mgmt_drpc_pool_query(Drpc__Call *drpc_req, Drpc__Response *drpc_resp) resp.enabled_ranks = enabled_ranks_str; if (disabled_ranks_str != NULL) resp.disabled_ranks = disabled_ranks_str; + if (suspect_ranks_str != NULL) + resp.suspect_ranks = suspect_ranks_str; D_ALLOC_ARRAY(resp.tier_stats, DAOS_MEDIA_MAX); if (resp.tier_stats == NULL) @@ -1850,6 +1864,8 @@ ds_mgmt_drpc_pool_query(Drpc__Call *drpc_req, Drpc__Response *drpc_resp) D_FREE(enabled_ranks_str); d_rank_list_free(disabled_ranks); D_FREE(disabled_ranks_str); + d_rank_list_free(suspect_ranks); + D_FREE(suspect_ranks_str); d_rank_list_free(svc_ranks); pool_query_free_tier_stats(&resp); } diff --git a/src/mgmt/srv_internal.h b/src/mgmt/srv_internal.h index d9c1a83bc8e..a2cbaf0890e 100644 --- a/src/mgmt/srv_internal.h +++ b/src/mgmt/srv_internal.h @@ -115,8 +115,9 @@ int ds_mgmt_pool_list_cont(uuid_t uuid, d_rank_list_t *svc_ranks, uint64_t *ncontainers); int ds_mgmt_pool_query(uuid_t pool_uuid, d_rank_list_t *svc_ranks, d_rank_list_t **enabled_ranks, - d_rank_list_t **disabled_ranks, daos_pool_info_t *pool_info, - uint32_t *pool_layout_ver, uint32_t *upgrade_layout_ver); + d_rank_list_t **disabled_ranks, d_rank_list_t **suspect_ranks, + daos_pool_info_t *pool_info, uint32_t *pool_layout_ver, + uint32_t *upgrade_layout_ver); int ds_mgmt_pool_query_targets(uuid_t pool_uuid, d_rank_list_t *svc_ranks, d_rank_t rank, d_rank_list_t *tgts, daos_target_info_t **infos); diff --git a/src/mgmt/srv_pool.c b/src/mgmt/srv_pool.c index b64297c117a..024965d739d 100644 --- a/src/mgmt/srv_pool.c +++ b/src/mgmt/srv_pool.c @@ -396,8 +396,9 @@ ds_mgmt_pool_list_cont(uuid_t uuid, d_rank_list_t *svc_ranks, */ int ds_mgmt_pool_query(uuid_t pool_uuid, d_rank_list_t *svc_ranks, d_rank_list_t **enabled_ranks, - d_rank_list_t **disabled_ranks, daos_pool_info_t *pool_info, - uint32_t *pool_layout_ver, uint32_t *upgrade_layout_ver) + d_rank_list_t **disabled_ranks, d_rank_list_t **suspect_ranks, + daos_pool_info_t *pool_info, uint32_t *pool_layout_ver, + uint32_t *upgrade_layout_ver) { if (pool_info == NULL) { D_ERROR("pool_info was NULL\n"); @@ -407,7 +408,8 @@ ds_mgmt_pool_query(uuid_t pool_uuid, d_rank_list_t *svc_ranks, d_rank_list_t **e D_DEBUG(DB_MGMT, "Querying pool "DF_UUID"\n", DP_UUID(pool_uuid)); return dsc_pool_svc_query(pool_uuid, svc_ranks, mgmt_ps_call_deadline(), enabled_ranks, - disabled_ranks, pool_info, pool_layout_ver, upgrade_layout_ver); + disabled_ranks, suspect_ranks, pool_info, pool_layout_ver, + upgrade_layout_ver); } /** diff --git a/src/mgmt/tests/mocks.c b/src/mgmt/tests/mocks.c index 912a36f293a..c22df3846e0 100644 --- a/src/mgmt/tests/mocks.c +++ b/src/mgmt/tests/mocks.c @@ -281,11 +281,13 @@ daos_pool_info_t ds_mgmt_pool_query_info_in; void *ds_mgmt_pool_query_info_ptr; d_rank_list_t *ds_mgmt_pool_query_enabled_ranks_out; d_rank_list_t *ds_mgmt_pool_query_disabled_ranks_out; +d_rank_list_t *ds_mgmt_pool_query_suspect_ranks_out; int ds_mgmt_pool_query(uuid_t pool_uuid, d_rank_list_t *svc_ranks, d_rank_list_t **enabled_ranks, - d_rank_list_t **disabled_ranks, daos_pool_info_t *pool_info, - uint32_t *pool_layout_ver, uint32_t *upgrade_layout_ver) + d_rank_list_t **disabled_ranks, d_rank_list_t **suspect_ranks, + daos_pool_info_t *pool_info, uint32_t *pool_layout_ver, + uint32_t *upgrade_layout_ver) { /* If function is to return with an error, pool_info and ranks will not be filled. */ if (ds_mgmt_pool_query_return != 0) @@ -310,6 +312,13 @@ ds_mgmt_pool_query(uuid_t pool_uuid, d_rank_list_t *svc_ranks, d_rank_list_t **e ds_mgmt_pool_query_disabled_ranks_out = *disabled_ranks; } + if ((pool_info->pi_bits & DPI_ENGINES_SUSPECT) != 0) { + D_ASSERT(suspect_ranks != NULL); + + *suspect_ranks = d_rank_list_alloc(2); /* 0-1 ; caller must free this */ + ds_mgmt_pool_query_suspect_ranks_out = *suspect_ranks; + } + ds_mgmt_pool_query_info_in = *pool_info; *pool_info = ds_mgmt_pool_query_info_out; @@ -325,6 +334,7 @@ mock_ds_mgmt_pool_query_setup(void) memset(&ds_mgmt_pool_query_info_out, 0, sizeof(daos_pool_info_t)); ds_mgmt_pool_query_enabled_ranks_out = NULL; ds_mgmt_pool_query_disabled_ranks_out = NULL; + ds_mgmt_pool_query_suspect_ranks_out = NULL; } int ds_mgmt_pool_query_targets_return; diff --git a/src/mgmt/tests/mocks.h b/src/mgmt/tests/mocks.h index 8bd0ffe998a..2ecc03d8ed9 100644 --- a/src/mgmt/tests/mocks.h +++ b/src/mgmt/tests/mocks.h @@ -110,6 +110,7 @@ extern daos_pool_info_t ds_mgmt_pool_query_info_out; extern void *ds_mgmt_pool_query_info_ptr; extern d_rank_list_t *ds_mgmt_pool_query_enabled_ranks_out; extern d_rank_list_t *ds_mgmt_pool_query_disabled_ranks_out; +extern d_rank_list_t *ds_mgmt_pool_query_suspect_ranks_out; void mock_ds_mgmt_pool_query_setup(void); diff --git a/src/mgmt/tests/srv_drpc_tests.c b/src/mgmt/tests/srv_drpc_tests.c index 2b5e0fb7dd4..559f9ea08ae 100644 --- a/src/mgmt/tests/srv_drpc_tests.c +++ b/src/mgmt/tests/srv_drpc_tests.c @@ -1408,12 +1408,13 @@ test_drpc_pool_query_success(void **state) Drpc__Response resp = DRPC__RESPONSE__INIT; uuid_t exp_uuid; daos_pool_info_t exp_info = {0}; + uint64_t flags = DPI_ENGINES_ENABLED | DPI_ENGINES_DISABLED | DPI_ENGINES_SUSPECT; init_test_pool_info(&exp_info); init_test_rebuild_status(&exp_info.pi_rebuild_st); ds_mgmt_pool_query_info_out = exp_info; - setup_pool_query_drpc_call(&call, TEST_UUID, DPI_ENGINES_ENABLED | DPI_ENGINES_DISABLED); + setup_pool_query_drpc_call(&call, TEST_UUID, flags); ds_mgmt_drpc_pool_query(&call, &resp); @@ -1424,8 +1425,9 @@ test_drpc_pool_query_success(void **state) assert_non_null(ds_mgmt_pool_query_info_ptr); assert_non_null(ds_mgmt_pool_query_enabled_ranks_out); assert_non_null(ds_mgmt_pool_query_disabled_ranks_out); - assert_int_equal(ds_mgmt_pool_query_info_in.pi_bits, - DEFAULT_QUERY_BITS | DPI_ENGINES_ENABLED | DPI_ENGINES_DISABLED); + assert_non_null(ds_mgmt_pool_query_suspect_ranks_out); + flags |= DEFAULT_QUERY_BITS; + assert_int_equal(ds_mgmt_pool_query_info_in.pi_bits, DEFAULT_QUERY_BITS | flags); expect_query_resp_with_info(&exp_info, MGMT__POOL_REBUILD_STATUS__STATE__IDLE, diff --git a/src/pool/srv_cli.c b/src/pool/srv_cli.c index 0545480fe29..5630394b154 100644 --- a/src/pool/srv_cli.c +++ b/src/pool/srv_cli.c @@ -337,6 +337,7 @@ dsc_pool_svc_call(uuid_t uuid, d_rank_list_t *ranks, struct dsc_pool_svc_call_cb struct pool_query_arg { d_rank_list_t **pqa_enabled_ranks; d_rank_list_t **pqa_disabled_ranks; + d_rank_list_t **pqa_suspect_ranks; daos_pool_info_t *pqa_info; uint32_t *pqa_layout_ver; uint32_t *pqa_upgrade_layout_ver; @@ -367,16 +368,65 @@ pool_query_init(uuid_t pool_uuid, crt_rpc_t *rpc, void *varg) return rc; } +static int +pool_map_get_suspect_ranks(struct pool_map *map, d_rank_list_t **ranks) +{ + crt_group_t *primary_grp; + struct pool_domain *doms; + int doms_cnt; + int i; + int rc = 0; + d_rank_list_t *rank_list = NULL; + + doms_cnt = pool_map_find_ranks(map, PO_COMP_ID_ALL, &doms); + D_ASSERT(doms_cnt >= 0); + primary_grp = crt_group_lookup(NULL); + D_ASSERT(primary_grp != NULL); + + rank_list = d_rank_list_alloc(0); + if (!rank_list) + return -DER_NOMEM; + + for (i = 0; i < doms_cnt; i++) { + struct swim_member_state state; + + if (!(doms[i].do_comp.co_status & PO_COMP_ST_UPIN)) + continue; + + rc = crt_rank_state_get(primary_grp, doms[i].do_comp.co_rank, &state); + if (rc != 0 && rc != -DER_NONEXIST) { + D_ERROR("failed to get status of rank %u: %d\n", doms[i].do_comp.co_rank, + rc); + break; + } + + D_DEBUG(DB_MD, "rank/state %d/%d\n", doms[i].do_comp.co_rank, + rc == -DER_NONEXIST ? -1 : state.sms_status); + if (rc == -DER_NONEXIST || state.sms_status == SWIM_MEMBER_DEAD) { + rc = d_rank_list_append(rank_list, doms[i].do_comp.co_rank); + if (rc) + D_GOTO(err, rc); + } + } +err: + if (rc == 0) + *ranks = rank_list; + else + d_rank_list_free(rank_list); + return rc; +} + static int process_query_result(d_rank_list_t **enabled_ranks, d_rank_list_t **disabled_ranks, - daos_pool_info_t *info, uuid_t pool_uuid, uint32_t map_version, - uint32_t leader_rank, struct daos_pool_space *ps, + d_rank_list_t **suspect_ranks, daos_pool_info_t *info, uuid_t pool_uuid, + uint32_t map_version, uint32_t leader_rank, struct daos_pool_space *ps, struct daos_rebuild_status *rs, struct pool_buf *map_buf, uint64_t pi_bits) { struct pool_map *map = NULL; unsigned int num_disabled = 0; d_rank_list_t *enabled_rank_list = NULL; d_rank_list_t *disabled_rank_list = NULL; + d_rank_list_t *suspect_rank_list = NULL; int rc; rc = pool_map_create(map_buf, map_version, &map); @@ -424,6 +474,22 @@ process_query_result(d_rank_list_t **enabled_ranks, d_rank_list_t **disabled_ran D_DEBUG(DB_MD, DF_UUID ": found %" PRIu32 " disabled ranks in pool map\n", DP_UUID(pool_uuid), disabled_rank_list->rl_nr); } + if ((pi_bits & DPI_ENGINES_SUSPECT) != 0) { + if (suspect_ranks == NULL) { + DL_ERROR(-DER_INVAL, + DF_UUID ": query pool requested suspect ranks, but ptr is NULL", + DP_UUID(pool_uuid)); + D_GOTO(error, rc = -DER_INVAL); + } + + rc = pool_map_get_suspect_ranks(map, &suspect_rank_list); + if (rc != 0) { + DL_ERROR(rc, DF_UUID ": pool_map_get_ranks() failed", DP_UUID(pool_uuid)); + D_GOTO(error, rc); + } + D_DEBUG(DB_MD, DF_UUID ": found %" PRIu32 " suspect ranks in pool map\n", + DP_UUID(pool_uuid), suspect_rank_list->rl_nr); + } pool_query_reply_to_info(pool_uuid, map_buf, map_version, leader_rank, ps, rs, info); info->pi_ndisabled = num_disabled; @@ -431,11 +497,14 @@ process_query_result(d_rank_list_t **enabled_ranks, d_rank_list_t **disabled_ran *enabled_ranks = enabled_rank_list; if (disabled_rank_list != NULL) *disabled_ranks = disabled_rank_list; + if (suspect_rank_list != NULL) + *suspect_ranks = suspect_rank_list; D_GOTO(out, rc = -DER_SUCCESS); error: d_rank_list_free(disabled_rank_list); d_rank_list_free(enabled_rank_list); + d_rank_list_free(suspect_rank_list); out: if (map != NULL) pool_map_decref(map); @@ -464,10 +533,10 @@ pool_query_consume(uuid_t pool_uuid, crt_rpc_t *rpc, void *varg) D_DEBUG(DB_MGMT, DF_UUID": Successfully queried pool\n", DP_UUID(pool_uuid)); - rc = process_query_result(arg->pqa_enabled_ranks, arg->pqa_disabled_ranks, arg->pqa_info, - pool_uuid, out->pqo_op.po_map_version, - out->pqo_op.po_hint.sh_rank, &out->pqo_space, - &out->pqo_rebuild_st, arg->pqa_map_buf, arg->pqa_info->pi_bits); + rc = process_query_result( + arg->pqa_enabled_ranks, arg->pqa_disabled_ranks, arg->pqa_suspect_ranks, arg->pqa_info, + pool_uuid, out->pqo_op.po_map_version, out->pqo_op.po_hint.sh_rank, &out->pqo_space, + &out->pqo_rebuild_st, arg->pqa_map_buf, arg->pqa_info->pi_bits); if (arg->pqa_layout_ver) *arg->pqa_layout_ver = out->pqo_pool_layout_ver; if (arg->pqa_upgrade_layout_ver) @@ -502,7 +571,9 @@ static struct dsc_pool_svc_call_cbs pool_query_cbs = { * \param[in] ps_ranks Ranks of pool svc replicas * \param[in] deadline Unix time deadline in milliseconds * \param[out] enabled_ranks Optional, storage ranks with enabled targets. - * \param[out] disabled_ranks Optional, storage ranks with disabled targets. + * \param[out] disabled_ranks Optional, storage ranks with disabled ranks. + * \param[out] suspect_ranks Optional, suspect ranks marked as DEAD by the SWIM + * protocol, but were not excluded from the system. * \param[in][out] pool_info Results of the pool query * \param[in][out] pool_layout_ver Results of the current pool global version * \param[in][out] upgrade_layout_ver Results of the target latest pool global version @@ -517,12 +588,13 @@ static struct dsc_pool_svc_call_cbs pool_query_cbs = { int dsc_pool_svc_query(uuid_t pool_uuid, d_rank_list_t *ps_ranks, uint64_t deadline, d_rank_list_t **enabled_ranks, d_rank_list_t **disabled_ranks, - daos_pool_info_t *pool_info, uint32_t *pool_layout_ver, - uint32_t *upgrade_layout_ver) + d_rank_list_t **suspect_ranks, daos_pool_info_t *pool_info, + uint32_t *pool_layout_ver, uint32_t *upgrade_layout_ver) { struct pool_query_arg arg = { .pqa_enabled_ranks = enabled_ranks, .pqa_disabled_ranks = disabled_ranks, + .pqa_suspect_ranks = suspect_ranks, .pqa_info = pool_info, .pqa_layout_ver = pool_layout_ver, .pqa_upgrade_layout_ver = upgrade_layout_ver, diff --git a/src/proto/mgmt/pool.proto b/src/proto/mgmt/pool.proto index d54ceca4280..8b6361f594d 100644 --- a/src/proto/mgmt/pool.proto +++ b/src/proto/mgmt/pool.proto @@ -238,6 +238,7 @@ message PoolQueryResp { uint32 svc_ldr = 18; // current raft leader (2.6+) repeated uint32 svc_reps = 19; // service replica ranks uint64 query_mask = 20; // Bitmask of pool query options used + string suspect_ranks = 21; // optional set of suspect ranks } message PoolProperty { diff --git a/src/tests/ftest/control/dmg_pool_query_ranks.py b/src/tests/ftest/control/dmg_pool_query_ranks.py index c84d2dc3c80..7f266a26f10 100644 --- a/src/tests/ftest/control/dmg_pool_query_ranks.py +++ b/src/tests/ftest/control/dmg_pool_query_ranks.py @@ -39,46 +39,28 @@ def test_pool_query_ranks_basic(self): """ self.log.info("Basic tests of pool query with ranks state") - self.log.debug("Checking without ranks state information") + self.log_step("Checking pool query without ranks state information") data = self.dmg.pool_query(self.pool.identifier) - self.assertIsNone( - data["response"].get("enabled_ranks"), - "Invalid enabled_ranks field: want=None, got={}".format( - data["response"].get("enabled_ranks") - ), - ) - self.assertListEqual( - data["response"].get("disabled_ranks"), - [], - "Invalid disabled_ranks field: want=[], got={}".format( - data["response"].get("disabled_ranks") - ), - ) - - self.log.debug("Checking enabled ranks state information") + self._verify_ranks(None, data, "enabled_ranks") + self._verify_ranks([], data, "disabled_ranks") + + self.log_step("Checking pool query with enabled ranks state information") data = self.dmg.pool_query(self.pool.identifier, show_enabled=True) - self.assertListEqual( - data["response"]["enabled_ranks"], - [0, 1, 2], - "Invalid enabled_ranks field: want=[0, 1, 2], got={}".format( - data["response"]["enabled_ranks"] - ), - ) - self.assertListEqual( - data["response"]["disabled_ranks"], - [], - "Invalid disabled_ranks field: want=[], got={}".format( - data["response"]["disabled_ranks"] - ), - ) + self._verify_ranks([0, 1, 2, 3, 4], data, "enabled_ranks") + self._verify_ranks([], data, "disabled_ranks") + + self.log_step("Checking pool query with suspect ranks state information") + data = self.dmg.pool_query(self.pool.identifier, health_only=True) + self._verify_ranks([], data, "suspect_ranks") def test_pool_query_ranks_mgmt(self): """Test the state of ranks after excluding and reintegrate them. Test Description: - Create a pool with some engines exclude them one by one and check the consistency of the - list of enabled and disabled ranks. Then, reintegrate them and check the consistency of - the list of enabled and disabled ranks. + Create a pool with 5 engines, first excluded engine marked as "Disabled" + second stopped one as “Suspect,” restarting it, ensuring rebuild completes, + clearing the “Suspect” status, reintegrating the excluded first engine, and + finally verifying that all engines are enabled with the excluded rank now empty. :avocado: tags=all,daily_regression :avocado: tags=vm @@ -92,70 +74,74 @@ def test_pool_query_ranks_mgmt(self): all_ranks = enabled_ranks.copy() self.random.shuffle(all_ranks) - self.log.info("Starting excluding ranks: all_ranks=%s", all_ranks) - for rank in all_ranks: - self.log.debug("Excluding rank %d", rank) - self.pool.exclude([rank]) - enabled_ranks.remove(rank) - disabled_ranks = sorted(disabled_ranks + [rank]) - - self.log.debug("Checking enabled ranks state information") - data = self.dmg.pool_query(self.pool.identifier, show_enabled=True) - self.assertListEqual( - data["response"].get("enabled_ranks"), - enabled_ranks, - "Invalid enabled_ranks field: want={}, got={}".format( - enabled_ranks, data["response"]["enabled_ranks"] - ), - ) - self.assertListEqual( - data["response"].get("disabled_ranks"), - disabled_ranks, - "Invalid disabled_ranks field: want={}, got={}".format( - disabled_ranks, data["response"].get("disabled_ranks") - ), - ) + exclude_rank = all_ranks[0] + suspect_rank = all_ranks[1] + self.log_step(f"Excluding pool rank:{exclude_rank} all_ranks={all_ranks}") + self.pool.exclude([exclude_rank]) + enabled_ranks.remove(exclude_rank) + disabled_ranks = sorted(disabled_ranks + [exclude_rank]) + + self.log_step("Checking enabled ranks state information") + data = self.dmg.pool_query(self.pool.identifier, show_enabled=True) + self._verify_ranks(enabled_ranks, data, "enabled_ranks") + self._verify_ranks(disabled_ranks, data, "disabled_ranks") - self.log.debug("Waiting for pool to be rebuild") - self.pool.wait_for_rebuild_to_start() - self.pool.wait_for_rebuild_to_end() + self.log_step(f"Waiting for rebuild to start after excluding pool rank {exclude_rank}") + self.pool.wait_for_rebuild_to_start() - self.random.shuffle(all_ranks) - self.log.info("Starting reintegrating ranks: all_ranks=%s", all_ranks) - for rank in all_ranks: - self.log.debug("Reintegrating rank %d", rank) - - cmd_succeed = False - for _ in range(3): - try: - result = self.pool.reintegrate(rank) - cmd_succeed = True - break - except CommandFailure: - self.log.debug("dmg command failed retry") - time.sleep(3) - - self.assertTrue(cmd_succeed, "pool reintegrate failed: {}".format(result)) - enabled_ranks = sorted(enabled_ranks + [rank]) - disabled_ranks.remove(rank) - - self.log.debug("Checking enabled ranks state information") - data = self.dmg.pool_query(self.pool.identifier, show_enabled=True) - self.assertListEqual( - data["response"].get("enabled_ranks"), - enabled_ranks, - "Invalid enabled_ranks field: want={}, got={}".format( - enabled_ranks, data["response"]["enabled_ranks"] - ), - ) + # kill second rank. + self.log_step(f"Stopping rank:{suspect_rank} all_ranks={all_ranks}") + self.server_managers[0].stop_ranks([suspect_rank], self.d_log) + + self.log_step(f"Waiting for pool rank {suspect_rank} to be suspected") + self.pool.wait_pool_suspect_ranks([suspect_rank], timeout=30) + self._verify_ranks(disabled_ranks, data, "disabled_ranks") + + self.log_step(f"Starting rank {suspect_rank}") + self.server_managers[0].start_ranks([suspect_rank], self.d_log) + + self.log_step("Waiting for pool ranks to no longer be suspected") + self.pool.wait_pool_suspect_ranks([], timeout=30) + + self.log_step("Waiting for rebuild to complete") + self.pool.wait_for_rebuild_to_end() + + self.log_step(f"Reintegrating rank {exclude_rank}") + cmd_succeed = False + for _ in range(3): + try: + self.pool.reintegrate(exclude_rank) + cmd_succeed = True + break + except CommandFailure: + self.log.debug("dmg command failed retry") + time.sleep(3) + + self.assertTrue(cmd_succeed, "pool reintegrate failed") + self.log_step(f"Waiting for rebuild to complete after reintegrating rank {exclude_rank}") + self.pool.wait_for_rebuild_to_start() + self.pool.wait_for_rebuild_to_end() + + enabled_ranks = sorted(enabled_ranks + [exclude_rank]) + disabled_ranks.remove(exclude_rank) + + self.log_step("Checking enabled ranks state information") + data = self.dmg.pool_query(self.pool.identifier, show_enabled=True) + self._verify_ranks(enabled_ranks, data, "enabled_ranks") + self._verify_ranks(disabled_ranks, data, "disabled_ranks") + + def _verify_ranks(self, expect, data, key): + """Verify the expected and actual rank lists are equal. + + Args: + expect (list): list of ranks to expect + data (dict): dmg json response containing actual list of ranks + key (str): the dmg json response key used to access the actual list of ranks + """ + actual = data["response"].get(key) + if expect is None: + self.assertIsNone(actual, f"Invalid {key} field: want=None, got={actual}") + else: self.assertListEqual( - data["response"].get("disabled_ranks"), - disabled_ranks, - "Invalid disabled_ranks field: want={}, got={}".format( - disabled_ranks, data["response"].get("disabled_ranks") - ), - ) - - self.log.debug("Waiting for pool to be rebuild") - self.pool.wait_for_rebuild_to_start() - self.pool.wait_for_rebuild_to_end() + actual, expect, f"Invalid {key} field: want={expect}, got={actual}") + self.log.debug("Check of %s passed: %s == %s", key, expect, actual) diff --git a/src/tests/ftest/control/dmg_pool_query_ranks.yaml b/src/tests/ftest/control/dmg_pool_query_ranks.yaml index 78edac3bd2c..6e4f7eae7ea 100644 --- a/src/tests/ftest/control/dmg_pool_query_ranks.yaml +++ b/src/tests/ftest/control/dmg_pool_query_ranks.yaml @@ -1,16 +1,21 @@ hosts: - test_servers: 3 + test_servers: 5 timeouts: test_pool_query_ranks_basic: 120 test_pool_query_ranks_error: 120 test_pool_query_ranks_mgmt: 480 server_config: name: daos_server + crt_timeout: 5 engines_per_host: 1 engines: 0: targets: 4 nr_xs_helpers: 0 + env_vars: + - SWIM_SUSPECT_TIMEOUT=10000 + - DAOS_POOL_RF=1 + - DD_MASK=io,epc,rebuild storage: 0: class: ram @@ -19,3 +24,5 @@ server_config: pool: control_method: dmg size: 4GB + svcn: 5 + properties: rd_fac:1 diff --git a/src/tests/ftest/util/dmg_utils.py b/src/tests/ftest/util/dmg_utils.py index a48b45e59dd..41de535cc0d 100644 --- a/src/tests/ftest/util/dmg_utils.py +++ b/src/tests/ftest/util/dmg_utils.py @@ -625,12 +625,13 @@ def pool_create(self, scm_size, uid=None, gid=None, nvme_size=None, return data - def pool_query(self, pool, show_enabled=False): + def pool_query(self, pool, show_enabled=False, health_only=False): """Query a pool with the dmg command. Args: pool (str): Pool UUID or label to query. show_enabled (bool, optional): Display enabled ranks. + health_only (bool, optional): Only perform pool health related queries. Raises: CommandFailure: if the dmg pool query command fails. @@ -676,7 +677,8 @@ def pool_query(self, pool, show_enabled=False): # "error": null, # "status": 0 # } - return self._get_json_result(("pool", "query"), pool=pool, show_enabled=show_enabled) + return self._get_json_result(("pool", "query"), pool=pool, + show_enabled=show_enabled, health_only=health_only) def pool_query_targets(self, pool, rank=None, target_idx=None): """Call dmg pool query-targets. diff --git a/src/tests/ftest/util/dmg_utils_base.py b/src/tests/ftest/util/dmg_utils_base.py index 951694a2251..7e5d2300a53 100644 --- a/src/tests/ftest/util/dmg_utils_base.py +++ b/src/tests/ftest/util/dmg_utils_base.py @@ -533,6 +533,7 @@ def __init__(self): super().__init__("/run/dmg/pool/query/*", "query") self.pool = BasicParameter(None, position=1) self.show_enabled = FormattedParameter("--show-enabled", False) + self.health_only = FormattedParameter("--health-only", False) class QueryTargetsSubCommand(CommandWithParameters): """Defines an object for the dmg pool query-targets command.""" diff --git a/src/tests/ftest/util/server_utils_params.py b/src/tests/ftest/util/server_utils_params.py index 7cda958d242..46db4891220 100644 --- a/src/tests/ftest/util/server_utils_params.py +++ b/src/tests/ftest/util/server_utils_params.py @@ -539,9 +539,11 @@ def get_params(self, test): # Update the env vars with any missing or different required setting update = False - env_var_dict = {env.split("=")[0]: env.split("=")[1] for env in self.env_vars.value} + env_var_dict = { + env.split("=", maxsplit=1)[0]: env.split("=", maxsplit=1)[1] + for env in self.env_vars.value} for key in sorted(required_env_vars): - if key not in env_var_dict or env_var_dict[key] != required_env_vars[key]: + if key not in env_var_dict: env_var_dict[key] = required_env_vars[key] update = True if update: diff --git a/src/tests/ftest/util/test_utils_pool.py b/src/tests/ftest/util/test_utils_pool.py index 7c6b5758f87..f5e88d2c26c 100644 --- a/src/tests/ftest/util/test_utils_pool.py +++ b/src/tests/ftest/util/test_utils_pool.py @@ -1448,6 +1448,33 @@ def check_pool_files(self, hosts, uuid, scm_mount): status = False return status + def wait_pool_suspect_ranks(self, expected, interval=1, timeout=30): + """Wait for the pool suspect ranks. + + Args: + expected (list): suspect ranks check to wait. + interval (int, optional): number of seconds to wait in between pool query checks + timeout(int, optional): time to fail test if it could not match + expected values. + + Raises: + DaosTestError: if waiting for timeout. + + """ + self.log.info("waiting for pool ranks %s to be suspected", expected) + + start = time() + data = self.dmg.pool_query(self.identifier, health_only=True) + while data['response'].get('suspect_ranks') != expected: + self.log.info(" suspect ranks is %s ...", data['response'].get('suspect_ranks')) + if time() - start > timeout: + raise DaosTestError("TIMEOUT detected after {} seconds while for waiting " + "for ranks {} suspect".format(timeout, expected)) + sleep(interval) + data = self.dmg.pool_query(self.identifier, health_only=True) + + self.log.info("Wait for suspect ranks complete: suspect ranks %s", expected) + def verify_uuid_directory(self, host, scm_mount): """Check if pool folder exist on server. From c57c67456519245c68efe0a5cedba7d9e9ad4363 Mon Sep 17 00:00:00 2001 From: Michael MacDonald Date: Wed, 11 Dec 2024 01:54:45 +0000 Subject: [PATCH 7/7] DAOS-16477 pool: Rename Suspect state to Dead Change the name to more closely reflect the underlying SWIM status, and reduce user confusion. An engine that has been marked DEAD by SWIM cannot participate in pool services, and has most likely already SIGKILL-ed itself. Allow-unstable-test: true Features: pool Required-githooks: true Signed-off-by: Michael MacDonald --- docs/admin/pool_operations.md | 8 +- src/control/cmd/daos/health.go | 4 +- src/control/cmd/daos/pool.go | 16 +- src/control/cmd/daos/pretty/health.go | 8 +- src/control/cmd/daos/pretty/pool.go | 6 +- src/control/cmd/daos/pretty/pool_test.go | 6 +- src/control/common/proto/mgmt/pool.pb.go | 200 +++++++++--------- src/control/lib/control/pool_test.go | 14 +- src/control/lib/daos/pool.go | 10 +- src/control/lib/daos/pool_test.go | 23 +- src/include/daos_pool.h | 4 +- src/include/daos_srv/pool.h | 2 +- src/mgmt/pool.pb-c.c | 7 +- src/mgmt/pool.pb-c.h | 4 +- src/mgmt/srv_drpc.c | 25 ++- src/mgmt/srv_internal.h | 2 +- src/mgmt/srv_pool.c | 5 +- src/mgmt/tests/mocks.c | 14 +- src/mgmt/tests/mocks.h | 2 +- src/mgmt/tests/srv_drpc_tests.c | 4 +- src/pool/srv_cli.c | 34 +-- src/proto/mgmt/pool.proto | 2 +- .../ftest/control/dmg_pool_query_ranks.py | 26 +-- src/tests/ftest/util/test_utils_pool.py | 16 +- 24 files changed, 219 insertions(+), 223 deletions(-) diff --git a/docs/admin/pool_operations.md b/docs/admin/pool_operations.md index efc827ae1e7..bd9bdfc5aa3 100644 --- a/docs/admin/pool_operations.md +++ b/docs/admin/pool_operations.md @@ -286,25 +286,25 @@ The example below shows a rebuild in progress and NVMe space allocated. Rebuild busy, 75 objs, 9722 recs ``` -After experiencing significant failures, the pool may retain some suspect +After experiencing significant failures, the pool may retain some "dead" engines that have been marked as DEAD by the SWIM protocol but were not excluded from the pool to prevent potential data inconsistency. An administrator can bring these engines back online by restarting them. The example below illustrates the -system’s status with suspect and disabled engines. +system’s status with dead and disabled engines. ```bash $ dmg pool query tank -t ``` NB: The --health-only/-t option is necessary to conduct pool health-related queries only. -This is important because suspect ranks may cause commands to hang and timeout so identifying +This is important because dead ranks may cause commands to hang and timeout so identifying and restarting them is a useful procedure. ```bash Pool 6f450a68-8c7d-4da9-8900-02691650f6a2, ntarget=8, disabled=2, leader=3, version=4, state=Degraded Pool health info: - Disabled ranks: 1 - - Suspect ranks: 2 + - Dead ranks: 2 - Rebuild busy, 0 objs, 0 recs ``` diff --git a/src/control/cmd/daos/health.go b/src/control/cmd/daos/health.go index dbeaacde0a0..61f1d1df142 100644 --- a/src/control/cmd/daos/health.go +++ b/src/control/cmd/daos/health.go @@ -100,7 +100,7 @@ func (cmd *healthCheckCmd) Execute([]string) error { }() queryMask := daos.MustNewPoolQueryMask(daos.PoolQueryOptionEnabledEngines, - daos.PoolQueryOptionSuspectEngines) + daos.PoolQueryOptionDeadEngines) if pool.DisabledTargets > 0 { queryMask.SetOptions(daos.PoolQueryOptionDisabledEngines) } @@ -111,7 +111,7 @@ func (cmd *healthCheckCmd) Execute([]string) error { } pool.EnabledRanks = tpi.EnabledRanks pool.DisabledRanks = tpi.DisabledRanks - pool.SuspectRanks = tpi.SuspectRanks + pool.DeadRanks = tpi.DeadRanks poolConts, err := listContainers(poolHdl) if err != nil { diff --git a/src/control/cmd/daos/pool.go b/src/control/cmd/daos/pool.go index 085a78b3b73..0066f3959b7 100644 --- a/src/control/cmd/daos/pool.go +++ b/src/control/cmd/daos/pool.go @@ -300,7 +300,7 @@ func queryPoolRankLists(poolHdl C.daos_handle_t, queryMask daos.PoolQueryMask) ( var rl *C.d_rank_list_t = nil if queryMask.HasOption(daos.PoolQueryOptionEnabledEngines) || queryMask.HasOption(daos.PoolQueryOptionDisabledEngines) || - queryMask.HasOption(daos.PoolQueryOptionSuspectEngines) { + queryMask.HasOption(daos.PoolQueryOptionDeadEngines) { rlPtr = &rl } @@ -330,8 +330,8 @@ func queryPoolRankLists(poolHdl C.daos_handle_t, queryMask daos.PoolQueryMask) ( if queryMask.HasOption(daos.PoolQueryOptionDisabledEngines) { poolInfo.DisabledRanks = rs } - if queryMask.HasOption(daos.PoolQueryOptionSuspectEngines) { - poolInfo.SuspectRanks = rs + if queryMask.HasOption(daos.PoolQueryOptionDeadEngines) { + poolInfo.DeadRanks = rs } } @@ -357,8 +357,8 @@ func queryPool(poolHdl C.daos_handle_t, queryMask daos.PoolQueryMask) (*daos.Poo poolInfo.EnabledRanks = poolInfo1.EnabledRanks case daos.PoolQueryOptionDisabledEngines: poolInfo.DisabledRanks = poolInfo1.DisabledRanks - case daos.PoolQueryOptionSuspectEngines: - poolInfo.SuspectRanks = poolInfo1.SuspectRanks + case daos.PoolQueryOptionDeadEngines: + poolInfo.DeadRanks = poolInfo1.DeadRanks } return nil } @@ -369,8 +369,8 @@ func queryPool(poolHdl C.daos_handle_t, queryMask daos.PoolQueryMask) (*daos.Poo firstOption = daos.PoolQueryOptionEnabledEngines } else if originalMask.HasOption(daos.PoolQueryOptionDisabledEngines) { firstOption = daos.PoolQueryOptionDisabledEngines - } else if originalMask.HasOption(daos.PoolQueryOptionSuspectEngines) { - firstOption = daos.PoolQueryOptionSuspectEngines + } else if originalMask.HasOption(daos.PoolQueryOptionDeadEngines) { + firstOption = daos.PoolQueryOptionDeadEngines } // Perform the first query to get basic information @@ -382,7 +382,7 @@ func queryPool(poolHdl C.daos_handle_t, queryMask daos.PoolQueryMask) (*daos.Poo queryOptions := []string{ daos.PoolQueryOptionEnabledEngines, daos.PoolQueryOptionDisabledEngines, - daos.PoolQueryOptionSuspectEngines, + daos.PoolQueryOptionDeadEngines, } // Process each option sequentially diff --git a/src/control/cmd/daos/pretty/health.go b/src/control/cmd/daos/pretty/health.go index ee77cd72371..3d3f8ca035d 100644 --- a/src/control/cmd/daos/pretty/health.go +++ b/src/control/cmd/daos/pretty/health.go @@ -61,12 +61,12 @@ func printPoolHealth(out io.Writer, pi *daos.PoolInfo, verbose bool) { } var healthStrings []string - if pi.SuspectRanks != nil && pi.SuspectRanks.Count() > 0 { - degStr := "Suspect" + if pi.DeadRanks != nil && pi.DeadRanks.Count() > 0 { + deadStr := "Dead" if verbose { - degStr += fmt.Sprintf(" %s", pi.SuspectRanks) + deadStr += fmt.Sprintf(" %s", pi.DeadRanks) } - healthStrings = append(healthStrings, degStr) + healthStrings = append(healthStrings, deadStr) } if pi.DisabledTargets > 0 { degStr := "Degraded" diff --git a/src/control/cmd/daos/pretty/pool.go b/src/control/cmd/daos/pretty/pool.go index 697009396ce..380d0fc66df 100644 --- a/src/control/cmd/daos/pretty/pool.go +++ b/src/control/cmd/daos/pretty/pool.go @@ -53,9 +53,9 @@ func PrintPoolInfo(pi *daos.PoolInfo, out io.Writer) error { if pi.DisabledRanks.Count() > 0 { fmt.Fprintf(w, "- Disabled ranks: %s\n", pi.DisabledRanks) } - if pi.QueryMask.HasOption(daos.PoolQueryOptionSuspectEngines) && - pi.SuspectRanks != nil && pi.SuspectRanks.Count() > 0 { - fmt.Fprintf(w, "- Suspect ranks: %s\n", pi.SuspectRanks) + if pi.QueryMask.HasOption(daos.PoolQueryOptionDeadEngines) && + pi.DeadRanks != nil && pi.DeadRanks.Count() > 0 { + fmt.Fprintf(w, "- Dead ranks: %s\n", pi.DeadRanks) } if pi.Rebuild != nil { if pi.Rebuild.Status == 0 { diff --git a/src/control/cmd/daos/pretty/pool_test.go b/src/control/cmd/daos/pretty/pool_test.go index 11f50c0e6be..6126be98651 100644 --- a/src/control/cmd/daos/pretty/pool_test.go +++ b/src/control/cmd/daos/pretty/pool_test.go @@ -124,7 +124,7 @@ Pool space info: Free: 1 B, min:0 B, max:0 B, mean:0 B `, poolUUID.String()), }, - "normal response; suspect ranks": { + "normal response; dead ranks": { pi: &daos.PoolInfo{ QueryMask: daos.HealthOnlyPoolQueryMask, State: daos.PoolServiceStateDegraded, @@ -137,7 +137,7 @@ Pool space info: PoolLayoutVer: 1, UpgradeLayoutVer: 2, DisabledRanks: ranklist.MustCreateRankSet("[0,1,3]"), - SuspectRanks: ranklist.MustCreateRankSet("[2]"), + DeadRanks: ranklist.MustCreateRankSet("[2]"), Rebuild: &daos.PoolRebuildStatus{ State: daos.PoolRebuildStateBusy, Objects: 42, @@ -159,7 +159,7 @@ Pool %s, ntarget=2, disabled=1, leader=42, version=100, state=Degraded Pool layout out of date (1 < 2) -- see `+backtickStr+` for details. Pool health info: - Disabled ranks: 0-1,3 -- Suspect ranks: 2 +- Dead ranks: 2 - Rebuild busy, 42 objs, 21 recs `, poolUUID.String()), }, diff --git a/src/control/common/proto/mgmt/pool.pb.go b/src/control/common/proto/mgmt/pool.pb.go index 4634d33b9af..c6783774d8c 100644 --- a/src/control/common/proto/mgmt/pool.pb.go +++ b/src/control/common/proto/mgmt/pool.pb.go @@ -1842,7 +1842,7 @@ type PoolQueryResp struct { SvcLdr uint32 `protobuf:"varint,18,opt,name=svc_ldr,json=svcLdr,proto3" json:"svc_ldr,omitempty"` // current raft leader (2.6+) SvcReps []uint32 `protobuf:"varint,19,rep,packed,name=svc_reps,json=svcReps,proto3" json:"svc_reps,omitempty"` // service replica ranks QueryMask uint64 `protobuf:"varint,20,opt,name=query_mask,json=queryMask,proto3" json:"query_mask,omitempty"` // Bitmask of pool query options used - SuspectRanks string `protobuf:"bytes,21,opt,name=suspect_ranks,json=suspectRanks,proto3" json:"suspect_ranks,omitempty"` // optional set of suspect ranks + DeadRanks string `protobuf:"bytes,21,opt,name=dead_ranks,json=deadRanks,proto3" json:"dead_ranks,omitempty"` // optional set of dead ranks } func (x *PoolQueryResp) Reset() { @@ -2010,9 +2010,9 @@ func (x *PoolQueryResp) GetQueryMask() uint64 { return 0 } -func (x *PoolQueryResp) GetSuspectRanks() string { +func (x *PoolQueryResp) GetDeadRanks() string { if x != nil { - return x.SuspectRanks + return x.DeadRanks } return "" } @@ -3040,7 +3040,7 @@ var file_mgmt_pool_proto_rawDesc = []byte{ 0x04, 0x52, 0x07, 0x72, 0x65, 0x63, 0x6f, 0x72, 0x64, 0x73, 0x22, 0x25, 0x0a, 0x05, 0x53, 0x74, 0x61, 0x74, 0x65, 0x12, 0x08, 0x0a, 0x04, 0x49, 0x44, 0x4c, 0x45, 0x10, 0x00, 0x12, 0x08, 0x0a, 0x04, 0x44, 0x4f, 0x4e, 0x45, 0x10, 0x01, 0x12, 0x08, 0x0a, 0x04, 0x42, 0x55, 0x53, 0x59, 0x10, - 0x02, 0x22, 0xe5, 0x05, 0x0a, 0x0d, 0x50, 0x6f, 0x6f, 0x6c, 0x51, 0x75, 0x65, 0x72, 0x79, 0x52, + 0x02, 0x22, 0xdf, 0x05, 0x0a, 0x0d, 0x50, 0x6f, 0x6f, 0x6c, 0x51, 0x75, 0x65, 0x72, 0x79, 0x52, 0x65, 0x73, 0x70, 0x12, 0x16, 0x0a, 0x06, 0x73, 0x74, 0x61, 0x74, 0x75, 0x73, 0x18, 0x01, 0x20, 0x01, 0x28, 0x05, 0x52, 0x06, 0x73, 0x74, 0x61, 0x74, 0x75, 0x73, 0x12, 0x12, 0x0a, 0x04, 0x75, 0x75, 0x69, 0x64, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x75, 0x75, 0x69, 0x64, 0x12, @@ -3083,105 +3083,105 @@ var file_mgmt_pool_proto_rawDesc = []byte{ 0x65, 0x70, 0x73, 0x18, 0x13, 0x20, 0x03, 0x28, 0x0d, 0x52, 0x07, 0x73, 0x76, 0x63, 0x52, 0x65, 0x70, 0x73, 0x12, 0x1d, 0x0a, 0x0a, 0x71, 0x75, 0x65, 0x72, 0x79, 0x5f, 0x6d, 0x61, 0x73, 0x6b, 0x18, 0x14, 0x20, 0x01, 0x28, 0x04, 0x52, 0x09, 0x71, 0x75, 0x65, 0x72, 0x79, 0x4d, 0x61, 0x73, - 0x6b, 0x12, 0x23, 0x0a, 0x0d, 0x73, 0x75, 0x73, 0x70, 0x65, 0x63, 0x74, 0x5f, 0x72, 0x61, 0x6e, - 0x6b, 0x73, 0x18, 0x15, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0c, 0x73, 0x75, 0x73, 0x70, 0x65, 0x63, - 0x74, 0x52, 0x61, 0x6e, 0x6b, 0x73, 0x4a, 0x04, 0x08, 0x09, 0x10, 0x0a, 0x52, 0x0b, 0x74, 0x6f, - 0x74, 0x61, 0x6c, 0x5f, 0x6e, 0x6f, 0x64, 0x65, 0x73, 0x22, 0x63, 0x0a, 0x0c, 0x50, 0x6f, 0x6f, - 0x6c, 0x50, 0x72, 0x6f, 0x70, 0x65, 0x72, 0x74, 0x79, 0x12, 0x16, 0x0a, 0x06, 0x6e, 0x75, 0x6d, - 0x62, 0x65, 0x72, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x06, 0x6e, 0x75, 0x6d, 0x62, 0x65, - 0x72, 0x12, 0x18, 0x0a, 0x06, 0x73, 0x74, 0x72, 0x76, 0x61, 0x6c, 0x18, 0x02, 0x20, 0x01, 0x28, - 0x09, 0x48, 0x00, 0x52, 0x06, 0x73, 0x74, 0x72, 0x76, 0x61, 0x6c, 0x12, 0x18, 0x0a, 0x06, 0x6e, - 0x75, 0x6d, 0x76, 0x61, 0x6c, 0x18, 0x03, 0x20, 0x01, 0x28, 0x04, 0x48, 0x00, 0x52, 0x06, 0x6e, - 0x75, 0x6d, 0x76, 0x61, 0x6c, 0x42, 0x07, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x22, 0x83, - 0x01, 0x0a, 0x0e, 0x50, 0x6f, 0x6f, 0x6c, 0x53, 0x65, 0x74, 0x50, 0x72, 0x6f, 0x70, 0x52, 0x65, + 0x6b, 0x12, 0x1d, 0x0a, 0x0a, 0x64, 0x65, 0x61, 0x64, 0x5f, 0x72, 0x61, 0x6e, 0x6b, 0x73, 0x18, + 0x15, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x64, 0x65, 0x61, 0x64, 0x52, 0x61, 0x6e, 0x6b, 0x73, + 0x4a, 0x04, 0x08, 0x09, 0x10, 0x0a, 0x52, 0x0b, 0x74, 0x6f, 0x74, 0x61, 0x6c, 0x5f, 0x6e, 0x6f, + 0x64, 0x65, 0x73, 0x22, 0x63, 0x0a, 0x0c, 0x50, 0x6f, 0x6f, 0x6c, 0x50, 0x72, 0x6f, 0x70, 0x65, + 0x72, 0x74, 0x79, 0x12, 0x16, 0x0a, 0x06, 0x6e, 0x75, 0x6d, 0x62, 0x65, 0x72, 0x18, 0x01, 0x20, + 0x01, 0x28, 0x0d, 0x52, 0x06, 0x6e, 0x75, 0x6d, 0x62, 0x65, 0x72, 0x12, 0x18, 0x0a, 0x06, 0x73, + 0x74, 0x72, 0x76, 0x61, 0x6c, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x48, 0x00, 0x52, 0x06, 0x73, + 0x74, 0x72, 0x76, 0x61, 0x6c, 0x12, 0x18, 0x0a, 0x06, 0x6e, 0x75, 0x6d, 0x76, 0x61, 0x6c, 0x18, + 0x03, 0x20, 0x01, 0x28, 0x04, 0x48, 0x00, 0x52, 0x06, 0x6e, 0x75, 0x6d, 0x76, 0x61, 0x6c, 0x42, + 0x07, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x22, 0x83, 0x01, 0x0a, 0x0e, 0x50, 0x6f, 0x6f, + 0x6c, 0x53, 0x65, 0x74, 0x50, 0x72, 0x6f, 0x70, 0x52, 0x65, 0x71, 0x12, 0x10, 0x0a, 0x03, 0x73, + 0x79, 0x73, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x73, 0x79, 0x73, 0x12, 0x0e, 0x0a, + 0x02, 0x69, 0x64, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x02, 0x69, 0x64, 0x12, 0x32, 0x0a, + 0x0a, 0x70, 0x72, 0x6f, 0x70, 0x65, 0x72, 0x74, 0x69, 0x65, 0x73, 0x18, 0x03, 0x20, 0x03, 0x28, + 0x0b, 0x32, 0x12, 0x2e, 0x6d, 0x67, 0x6d, 0x74, 0x2e, 0x50, 0x6f, 0x6f, 0x6c, 0x50, 0x72, 0x6f, + 0x70, 0x65, 0x72, 0x74, 0x79, 0x52, 0x0a, 0x70, 0x72, 0x6f, 0x70, 0x65, 0x72, 0x74, 0x69, 0x65, + 0x73, 0x12, 0x1b, 0x0a, 0x09, 0x73, 0x76, 0x63, 0x5f, 0x72, 0x61, 0x6e, 0x6b, 0x73, 0x18, 0x04, + 0x20, 0x03, 0x28, 0x0d, 0x52, 0x08, 0x73, 0x76, 0x63, 0x52, 0x61, 0x6e, 0x6b, 0x73, 0x22, 0x29, + 0x0a, 0x0f, 0x50, 0x6f, 0x6f, 0x6c, 0x53, 0x65, 0x74, 0x50, 0x72, 0x6f, 0x70, 0x52, 0x65, 0x73, + 0x70, 0x12, 0x16, 0x0a, 0x06, 0x73, 0x74, 0x61, 0x74, 0x75, 0x73, 0x18, 0x01, 0x20, 0x01, 0x28, + 0x05, 0x52, 0x06, 0x73, 0x74, 0x61, 0x74, 0x75, 0x73, 0x22, 0x83, 0x01, 0x0a, 0x0e, 0x50, 0x6f, + 0x6f, 0x6c, 0x47, 0x65, 0x74, 0x50, 0x72, 0x6f, 0x70, 0x52, 0x65, 0x71, 0x12, 0x10, 0x0a, 0x03, + 0x73, 0x79, 0x73, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x73, 0x79, 0x73, 0x12, 0x0e, + 0x0a, 0x02, 0x69, 0x64, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x02, 0x69, 0x64, 0x12, 0x32, + 0x0a, 0x0a, 0x70, 0x72, 0x6f, 0x70, 0x65, 0x72, 0x74, 0x69, 0x65, 0x73, 0x18, 0x03, 0x20, 0x03, + 0x28, 0x0b, 0x32, 0x12, 0x2e, 0x6d, 0x67, 0x6d, 0x74, 0x2e, 0x50, 0x6f, 0x6f, 0x6c, 0x50, 0x72, + 0x6f, 0x70, 0x65, 0x72, 0x74, 0x79, 0x52, 0x0a, 0x70, 0x72, 0x6f, 0x70, 0x65, 0x72, 0x74, 0x69, + 0x65, 0x73, 0x12, 0x1b, 0x0a, 0x09, 0x73, 0x76, 0x63, 0x5f, 0x72, 0x61, 0x6e, 0x6b, 0x73, 0x18, + 0x04, 0x20, 0x03, 0x28, 0x0d, 0x52, 0x08, 0x73, 0x76, 0x63, 0x52, 0x61, 0x6e, 0x6b, 0x73, 0x22, + 0x5d, 0x0a, 0x0f, 0x50, 0x6f, 0x6f, 0x6c, 0x47, 0x65, 0x74, 0x50, 0x72, 0x6f, 0x70, 0x52, 0x65, + 0x73, 0x70, 0x12, 0x16, 0x0a, 0x06, 0x73, 0x74, 0x61, 0x74, 0x75, 0x73, 0x18, 0x01, 0x20, 0x01, + 0x28, 0x05, 0x52, 0x06, 0x73, 0x74, 0x61, 0x74, 0x75, 0x73, 0x12, 0x32, 0x0a, 0x0a, 0x70, 0x72, + 0x6f, 0x70, 0x65, 0x72, 0x74, 0x69, 0x65, 0x73, 0x18, 0x02, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x12, + 0x2e, 0x6d, 0x67, 0x6d, 0x74, 0x2e, 0x50, 0x6f, 0x6f, 0x6c, 0x50, 0x72, 0x6f, 0x70, 0x65, 0x72, + 0x74, 0x79, 0x52, 0x0a, 0x70, 0x72, 0x6f, 0x70, 0x65, 0x72, 0x74, 0x69, 0x65, 0x73, 0x22, 0x4f, + 0x0a, 0x0e, 0x50, 0x6f, 0x6f, 0x6c, 0x55, 0x70, 0x67, 0x72, 0x61, 0x64, 0x65, 0x52, 0x65, 0x71, + 0x12, 0x10, 0x0a, 0x03, 0x73, 0x79, 0x73, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x73, + 0x79, 0x73, 0x12, 0x0e, 0x0a, 0x02, 0x69, 0x64, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x02, + 0x69, 0x64, 0x12, 0x1b, 0x0a, 0x09, 0x73, 0x76, 0x63, 0x5f, 0x72, 0x61, 0x6e, 0x6b, 0x73, 0x18, + 0x03, 0x20, 0x03, 0x28, 0x0d, 0x52, 0x08, 0x73, 0x76, 0x63, 0x52, 0x61, 0x6e, 0x6b, 0x73, 0x22, + 0x29, 0x0a, 0x0f, 0x50, 0x6f, 0x6f, 0x6c, 0x55, 0x70, 0x67, 0x72, 0x61, 0x64, 0x65, 0x52, 0x65, + 0x73, 0x70, 0x12, 0x16, 0x0a, 0x06, 0x73, 0x74, 0x61, 0x74, 0x75, 0x73, 0x18, 0x01, 0x20, 0x01, + 0x28, 0x05, 0x52, 0x06, 0x73, 0x74, 0x61, 0x74, 0x75, 0x73, 0x22, 0x81, 0x01, 0x0a, 0x12, 0x50, + 0x6f, 0x6f, 0x6c, 0x51, 0x75, 0x65, 0x72, 0x79, 0x54, 0x61, 0x72, 0x67, 0x65, 0x74, 0x52, 0x65, 0x71, 0x12, 0x10, 0x0a, 0x03, 0x73, 0x79, 0x73, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x73, 0x79, 0x73, 0x12, 0x0e, 0x0a, 0x02, 0x69, 0x64, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, - 0x02, 0x69, 0x64, 0x12, 0x32, 0x0a, 0x0a, 0x70, 0x72, 0x6f, 0x70, 0x65, 0x72, 0x74, 0x69, 0x65, - 0x73, 0x18, 0x03, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x12, 0x2e, 0x6d, 0x67, 0x6d, 0x74, 0x2e, 0x50, - 0x6f, 0x6f, 0x6c, 0x50, 0x72, 0x6f, 0x70, 0x65, 0x72, 0x74, 0x79, 0x52, 0x0a, 0x70, 0x72, 0x6f, - 0x70, 0x65, 0x72, 0x74, 0x69, 0x65, 0x73, 0x12, 0x1b, 0x0a, 0x09, 0x73, 0x76, 0x63, 0x5f, 0x72, - 0x61, 0x6e, 0x6b, 0x73, 0x18, 0x04, 0x20, 0x03, 0x28, 0x0d, 0x52, 0x08, 0x73, 0x76, 0x63, 0x52, - 0x61, 0x6e, 0x6b, 0x73, 0x22, 0x29, 0x0a, 0x0f, 0x50, 0x6f, 0x6f, 0x6c, 0x53, 0x65, 0x74, 0x50, - 0x72, 0x6f, 0x70, 0x52, 0x65, 0x73, 0x70, 0x12, 0x16, 0x0a, 0x06, 0x73, 0x74, 0x61, 0x74, 0x75, - 0x73, 0x18, 0x01, 0x20, 0x01, 0x28, 0x05, 0x52, 0x06, 0x73, 0x74, 0x61, 0x74, 0x75, 0x73, 0x22, - 0x83, 0x01, 0x0a, 0x0e, 0x50, 0x6f, 0x6f, 0x6c, 0x47, 0x65, 0x74, 0x50, 0x72, 0x6f, 0x70, 0x52, - 0x65, 0x71, 0x12, 0x10, 0x0a, 0x03, 0x73, 0x79, 0x73, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, - 0x03, 0x73, 0x79, 0x73, 0x12, 0x0e, 0x0a, 0x02, 0x69, 0x64, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, - 0x52, 0x02, 0x69, 0x64, 0x12, 0x32, 0x0a, 0x0a, 0x70, 0x72, 0x6f, 0x70, 0x65, 0x72, 0x74, 0x69, - 0x65, 0x73, 0x18, 0x03, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x12, 0x2e, 0x6d, 0x67, 0x6d, 0x74, 0x2e, - 0x50, 0x6f, 0x6f, 0x6c, 0x50, 0x72, 0x6f, 0x70, 0x65, 0x72, 0x74, 0x79, 0x52, 0x0a, 0x70, 0x72, - 0x6f, 0x70, 0x65, 0x72, 0x74, 0x69, 0x65, 0x73, 0x12, 0x1b, 0x0a, 0x09, 0x73, 0x76, 0x63, 0x5f, - 0x72, 0x61, 0x6e, 0x6b, 0x73, 0x18, 0x04, 0x20, 0x03, 0x28, 0x0d, 0x52, 0x08, 0x73, 0x76, 0x63, - 0x52, 0x61, 0x6e, 0x6b, 0x73, 0x22, 0x5d, 0x0a, 0x0f, 0x50, 0x6f, 0x6f, 0x6c, 0x47, 0x65, 0x74, - 0x50, 0x72, 0x6f, 0x70, 0x52, 0x65, 0x73, 0x70, 0x12, 0x16, 0x0a, 0x06, 0x73, 0x74, 0x61, 0x74, - 0x75, 0x73, 0x18, 0x01, 0x20, 0x01, 0x28, 0x05, 0x52, 0x06, 0x73, 0x74, 0x61, 0x74, 0x75, 0x73, - 0x12, 0x32, 0x0a, 0x0a, 0x70, 0x72, 0x6f, 0x70, 0x65, 0x72, 0x74, 0x69, 0x65, 0x73, 0x18, 0x02, - 0x20, 0x03, 0x28, 0x0b, 0x32, 0x12, 0x2e, 0x6d, 0x67, 0x6d, 0x74, 0x2e, 0x50, 0x6f, 0x6f, 0x6c, - 0x50, 0x72, 0x6f, 0x70, 0x65, 0x72, 0x74, 0x79, 0x52, 0x0a, 0x70, 0x72, 0x6f, 0x70, 0x65, 0x72, - 0x74, 0x69, 0x65, 0x73, 0x22, 0x4f, 0x0a, 0x0e, 0x50, 0x6f, 0x6f, 0x6c, 0x55, 0x70, 0x67, 0x72, - 0x61, 0x64, 0x65, 0x52, 0x65, 0x71, 0x12, 0x10, 0x0a, 0x03, 0x73, 0x79, 0x73, 0x18, 0x01, 0x20, - 0x01, 0x28, 0x09, 0x52, 0x03, 0x73, 0x79, 0x73, 0x12, 0x0e, 0x0a, 0x02, 0x69, 0x64, 0x18, 0x02, - 0x20, 0x01, 0x28, 0x09, 0x52, 0x02, 0x69, 0x64, 0x12, 0x1b, 0x0a, 0x09, 0x73, 0x76, 0x63, 0x5f, - 0x72, 0x61, 0x6e, 0x6b, 0x73, 0x18, 0x03, 0x20, 0x03, 0x28, 0x0d, 0x52, 0x08, 0x73, 0x76, 0x63, - 0x52, 0x61, 0x6e, 0x6b, 0x73, 0x22, 0x29, 0x0a, 0x0f, 0x50, 0x6f, 0x6f, 0x6c, 0x55, 0x70, 0x67, - 0x72, 0x61, 0x64, 0x65, 0x52, 0x65, 0x73, 0x70, 0x12, 0x16, 0x0a, 0x06, 0x73, 0x74, 0x61, 0x74, - 0x75, 0x73, 0x18, 0x01, 0x20, 0x01, 0x28, 0x05, 0x52, 0x06, 0x73, 0x74, 0x61, 0x74, 0x75, 0x73, - 0x22, 0x81, 0x01, 0x0a, 0x12, 0x50, 0x6f, 0x6f, 0x6c, 0x51, 0x75, 0x65, 0x72, 0x79, 0x54, 0x61, - 0x72, 0x67, 0x65, 0x74, 0x52, 0x65, 0x71, 0x12, 0x10, 0x0a, 0x03, 0x73, 0x79, 0x73, 0x18, 0x01, - 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x73, 0x79, 0x73, 0x12, 0x0e, 0x0a, 0x02, 0x69, 0x64, 0x18, - 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x02, 0x69, 0x64, 0x12, 0x12, 0x0a, 0x04, 0x72, 0x61, 0x6e, - 0x6b, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x04, 0x72, 0x61, 0x6e, 0x6b, 0x12, 0x18, 0x0a, - 0x07, 0x74, 0x61, 0x72, 0x67, 0x65, 0x74, 0x73, 0x18, 0x04, 0x20, 0x03, 0x28, 0x0d, 0x52, 0x07, - 0x74, 0x61, 0x72, 0x67, 0x65, 0x74, 0x73, 0x12, 0x1b, 0x0a, 0x09, 0x73, 0x76, 0x63, 0x5f, 0x72, - 0x61, 0x6e, 0x6b, 0x73, 0x18, 0x05, 0x20, 0x03, 0x28, 0x0d, 0x52, 0x08, 0x73, 0x76, 0x63, 0x52, - 0x61, 0x6e, 0x6b, 0x73, 0x22, 0x75, 0x0a, 0x12, 0x53, 0x74, 0x6f, 0x72, 0x61, 0x67, 0x65, 0x54, - 0x61, 0x72, 0x67, 0x65, 0x74, 0x55, 0x73, 0x61, 0x67, 0x65, 0x12, 0x14, 0x0a, 0x05, 0x74, 0x6f, - 0x74, 0x61, 0x6c, 0x18, 0x01, 0x20, 0x01, 0x28, 0x04, 0x52, 0x05, 0x74, 0x6f, 0x74, 0x61, 0x6c, - 0x12, 0x12, 0x0a, 0x04, 0x66, 0x72, 0x65, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x04, 0x52, 0x04, - 0x66, 0x72, 0x65, 0x65, 0x12, 0x35, 0x0a, 0x0a, 0x6d, 0x65, 0x64, 0x69, 0x61, 0x5f, 0x74, 0x79, - 0x70, 0x65, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x16, 0x2e, 0x6d, 0x67, 0x6d, 0x74, 0x2e, - 0x53, 0x74, 0x6f, 0x72, 0x61, 0x67, 0x65, 0x4d, 0x65, 0x64, 0x69, 0x61, 0x54, 0x79, 0x70, 0x65, - 0x52, 0x09, 0x6d, 0x65, 0x64, 0x69, 0x61, 0x54, 0x79, 0x70, 0x65, 0x22, 0xda, 0x02, 0x0a, 0x13, - 0x50, 0x6f, 0x6f, 0x6c, 0x51, 0x75, 0x65, 0x72, 0x79, 0x54, 0x61, 0x72, 0x67, 0x65, 0x74, 0x49, - 0x6e, 0x66, 0x6f, 0x12, 0x38, 0x0a, 0x04, 0x74, 0x79, 0x70, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, - 0x0e, 0x32, 0x24, 0x2e, 0x6d, 0x67, 0x6d, 0x74, 0x2e, 0x50, 0x6f, 0x6f, 0x6c, 0x51, 0x75, 0x65, - 0x72, 0x79, 0x54, 0x61, 0x72, 0x67, 0x65, 0x74, 0x49, 0x6e, 0x66, 0x6f, 0x2e, 0x54, 0x61, 0x72, - 0x67, 0x65, 0x74, 0x54, 0x79, 0x70, 0x65, 0x52, 0x04, 0x74, 0x79, 0x70, 0x65, 0x12, 0x3b, 0x0a, - 0x05, 0x73, 0x74, 0x61, 0x74, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x25, 0x2e, 0x6d, - 0x67, 0x6d, 0x74, 0x2e, 0x50, 0x6f, 0x6f, 0x6c, 0x51, 0x75, 0x65, 0x72, 0x79, 0x54, 0x61, 0x72, - 0x67, 0x65, 0x74, 0x49, 0x6e, 0x66, 0x6f, 0x2e, 0x54, 0x61, 0x72, 0x67, 0x65, 0x74, 0x53, 0x74, - 0x61, 0x74, 0x65, 0x52, 0x05, 0x73, 0x74, 0x61, 0x74, 0x65, 0x12, 0x2e, 0x0a, 0x05, 0x73, 0x70, - 0x61, 0x63, 0x65, 0x18, 0x03, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x18, 0x2e, 0x6d, 0x67, 0x6d, 0x74, - 0x2e, 0x53, 0x74, 0x6f, 0x72, 0x61, 0x67, 0x65, 0x54, 0x61, 0x72, 0x67, 0x65, 0x74, 0x55, 0x73, - 0x61, 0x67, 0x65, 0x52, 0x05, 0x73, 0x70, 0x61, 0x63, 0x65, 0x22, 0x3b, 0x0a, 0x0a, 0x54, 0x61, - 0x72, 0x67, 0x65, 0x74, 0x54, 0x79, 0x70, 0x65, 0x12, 0x0b, 0x0a, 0x07, 0x55, 0x4e, 0x4b, 0x4e, - 0x4f, 0x57, 0x4e, 0x10, 0x00, 0x12, 0x07, 0x0a, 0x03, 0x48, 0x44, 0x44, 0x10, 0x01, 0x12, 0x07, - 0x0a, 0x03, 0x53, 0x53, 0x44, 0x10, 0x02, 0x12, 0x06, 0x0a, 0x02, 0x50, 0x4d, 0x10, 0x03, 0x12, - 0x06, 0x0a, 0x02, 0x56, 0x4d, 0x10, 0x04, 0x22, 0x5f, 0x0a, 0x0b, 0x54, 0x61, 0x72, 0x67, 0x65, - 0x74, 0x53, 0x74, 0x61, 0x74, 0x65, 0x12, 0x11, 0x0a, 0x0d, 0x53, 0x54, 0x41, 0x54, 0x45, 0x5f, - 0x55, 0x4e, 0x4b, 0x4e, 0x4f, 0x57, 0x4e, 0x10, 0x00, 0x12, 0x0c, 0x0a, 0x08, 0x44, 0x4f, 0x57, - 0x4e, 0x5f, 0x4f, 0x55, 0x54, 0x10, 0x01, 0x12, 0x08, 0x0a, 0x04, 0x44, 0x4f, 0x57, 0x4e, 0x10, - 0x02, 0x12, 0x06, 0x0a, 0x02, 0x55, 0x50, 0x10, 0x03, 0x12, 0x09, 0x0a, 0x05, 0x55, 0x50, 0x5f, - 0x49, 0x4e, 0x10, 0x04, 0x12, 0x07, 0x0a, 0x03, 0x4e, 0x45, 0x57, 0x10, 0x05, 0x12, 0x09, 0x0a, - 0x05, 0x44, 0x52, 0x41, 0x49, 0x4e, 0x10, 0x06, 0x22, 0x5e, 0x0a, 0x13, 0x50, 0x6f, 0x6f, 0x6c, - 0x51, 0x75, 0x65, 0x72, 0x79, 0x54, 0x61, 0x72, 0x67, 0x65, 0x74, 0x52, 0x65, 0x73, 0x70, 0x12, - 0x16, 0x0a, 0x06, 0x73, 0x74, 0x61, 0x74, 0x75, 0x73, 0x18, 0x01, 0x20, 0x01, 0x28, 0x05, 0x52, - 0x06, 0x73, 0x74, 0x61, 0x74, 0x75, 0x73, 0x12, 0x2f, 0x0a, 0x05, 0x69, 0x6e, 0x66, 0x6f, 0x73, - 0x18, 0x02, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x19, 0x2e, 0x6d, 0x67, 0x6d, 0x74, 0x2e, 0x50, 0x6f, + 0x02, 0x69, 0x64, 0x12, 0x12, 0x0a, 0x04, 0x72, 0x61, 0x6e, 0x6b, 0x18, 0x03, 0x20, 0x01, 0x28, + 0x0d, 0x52, 0x04, 0x72, 0x61, 0x6e, 0x6b, 0x12, 0x18, 0x0a, 0x07, 0x74, 0x61, 0x72, 0x67, 0x65, + 0x74, 0x73, 0x18, 0x04, 0x20, 0x03, 0x28, 0x0d, 0x52, 0x07, 0x74, 0x61, 0x72, 0x67, 0x65, 0x74, + 0x73, 0x12, 0x1b, 0x0a, 0x09, 0x73, 0x76, 0x63, 0x5f, 0x72, 0x61, 0x6e, 0x6b, 0x73, 0x18, 0x05, + 0x20, 0x03, 0x28, 0x0d, 0x52, 0x08, 0x73, 0x76, 0x63, 0x52, 0x61, 0x6e, 0x6b, 0x73, 0x22, 0x75, + 0x0a, 0x12, 0x53, 0x74, 0x6f, 0x72, 0x61, 0x67, 0x65, 0x54, 0x61, 0x72, 0x67, 0x65, 0x74, 0x55, + 0x73, 0x61, 0x67, 0x65, 0x12, 0x14, 0x0a, 0x05, 0x74, 0x6f, 0x74, 0x61, 0x6c, 0x18, 0x01, 0x20, + 0x01, 0x28, 0x04, 0x52, 0x05, 0x74, 0x6f, 0x74, 0x61, 0x6c, 0x12, 0x12, 0x0a, 0x04, 0x66, 0x72, + 0x65, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x04, 0x52, 0x04, 0x66, 0x72, 0x65, 0x65, 0x12, 0x35, + 0x0a, 0x0a, 0x6d, 0x65, 0x64, 0x69, 0x61, 0x5f, 0x74, 0x79, 0x70, 0x65, 0x18, 0x03, 0x20, 0x01, + 0x28, 0x0e, 0x32, 0x16, 0x2e, 0x6d, 0x67, 0x6d, 0x74, 0x2e, 0x53, 0x74, 0x6f, 0x72, 0x61, 0x67, + 0x65, 0x4d, 0x65, 0x64, 0x69, 0x61, 0x54, 0x79, 0x70, 0x65, 0x52, 0x09, 0x6d, 0x65, 0x64, 0x69, + 0x61, 0x54, 0x79, 0x70, 0x65, 0x22, 0xda, 0x02, 0x0a, 0x13, 0x50, 0x6f, 0x6f, 0x6c, 0x51, 0x75, + 0x65, 0x72, 0x79, 0x54, 0x61, 0x72, 0x67, 0x65, 0x74, 0x49, 0x6e, 0x66, 0x6f, 0x12, 0x38, 0x0a, + 0x04, 0x74, 0x79, 0x70, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x24, 0x2e, 0x6d, 0x67, + 0x6d, 0x74, 0x2e, 0x50, 0x6f, 0x6f, 0x6c, 0x51, 0x75, 0x65, 0x72, 0x79, 0x54, 0x61, 0x72, 0x67, + 0x65, 0x74, 0x49, 0x6e, 0x66, 0x6f, 0x2e, 0x54, 0x61, 0x72, 0x67, 0x65, 0x74, 0x54, 0x79, 0x70, + 0x65, 0x52, 0x04, 0x74, 0x79, 0x70, 0x65, 0x12, 0x3b, 0x0a, 0x05, 0x73, 0x74, 0x61, 0x74, 0x65, + 0x18, 0x02, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x25, 0x2e, 0x6d, 0x67, 0x6d, 0x74, 0x2e, 0x50, 0x6f, 0x6f, 0x6c, 0x51, 0x75, 0x65, 0x72, 0x79, 0x54, 0x61, 0x72, 0x67, 0x65, 0x74, 0x49, 0x6e, 0x66, - 0x6f, 0x52, 0x05, 0x69, 0x6e, 0x66, 0x6f, 0x73, 0x2a, 0x25, 0x0a, 0x10, 0x53, 0x74, 0x6f, 0x72, - 0x61, 0x67, 0x65, 0x4d, 0x65, 0x64, 0x69, 0x61, 0x54, 0x79, 0x70, 0x65, 0x12, 0x07, 0x0a, 0x03, - 0x53, 0x43, 0x4d, 0x10, 0x00, 0x12, 0x08, 0x0a, 0x04, 0x4e, 0x56, 0x4d, 0x45, 0x10, 0x01, 0x2a, - 0x56, 0x0a, 0x10, 0x50, 0x6f, 0x6f, 0x6c, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x53, 0x74, - 0x61, 0x74, 0x65, 0x12, 0x0c, 0x0a, 0x08, 0x43, 0x72, 0x65, 0x61, 0x74, 0x69, 0x6e, 0x67, 0x10, - 0x00, 0x12, 0x09, 0x0a, 0x05, 0x52, 0x65, 0x61, 0x64, 0x79, 0x10, 0x01, 0x12, 0x0e, 0x0a, 0x0a, - 0x44, 0x65, 0x73, 0x74, 0x72, 0x6f, 0x79, 0x69, 0x6e, 0x67, 0x10, 0x02, 0x12, 0x0c, 0x0a, 0x08, - 0x44, 0x65, 0x67, 0x72, 0x61, 0x64, 0x65, 0x64, 0x10, 0x03, 0x12, 0x0b, 0x0a, 0x07, 0x55, 0x6e, - 0x6b, 0x6e, 0x6f, 0x77, 0x6e, 0x10, 0x04, 0x42, 0x3a, 0x5a, 0x38, 0x67, 0x69, 0x74, 0x68, 0x75, - 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x64, 0x61, 0x6f, 0x73, 0x2d, 0x73, 0x74, 0x61, 0x63, 0x6b, - 0x2f, 0x64, 0x61, 0x6f, 0x73, 0x2f, 0x73, 0x72, 0x63, 0x2f, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, - 0x6c, 0x2f, 0x63, 0x6f, 0x6d, 0x6d, 0x6f, 0x6e, 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2f, 0x6d, - 0x67, 0x6d, 0x74, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, + 0x6f, 0x2e, 0x54, 0x61, 0x72, 0x67, 0x65, 0x74, 0x53, 0x74, 0x61, 0x74, 0x65, 0x52, 0x05, 0x73, + 0x74, 0x61, 0x74, 0x65, 0x12, 0x2e, 0x0a, 0x05, 0x73, 0x70, 0x61, 0x63, 0x65, 0x18, 0x03, 0x20, + 0x03, 0x28, 0x0b, 0x32, 0x18, 0x2e, 0x6d, 0x67, 0x6d, 0x74, 0x2e, 0x53, 0x74, 0x6f, 0x72, 0x61, + 0x67, 0x65, 0x54, 0x61, 0x72, 0x67, 0x65, 0x74, 0x55, 0x73, 0x61, 0x67, 0x65, 0x52, 0x05, 0x73, + 0x70, 0x61, 0x63, 0x65, 0x22, 0x3b, 0x0a, 0x0a, 0x54, 0x61, 0x72, 0x67, 0x65, 0x74, 0x54, 0x79, + 0x70, 0x65, 0x12, 0x0b, 0x0a, 0x07, 0x55, 0x4e, 0x4b, 0x4e, 0x4f, 0x57, 0x4e, 0x10, 0x00, 0x12, + 0x07, 0x0a, 0x03, 0x48, 0x44, 0x44, 0x10, 0x01, 0x12, 0x07, 0x0a, 0x03, 0x53, 0x53, 0x44, 0x10, + 0x02, 0x12, 0x06, 0x0a, 0x02, 0x50, 0x4d, 0x10, 0x03, 0x12, 0x06, 0x0a, 0x02, 0x56, 0x4d, 0x10, + 0x04, 0x22, 0x5f, 0x0a, 0x0b, 0x54, 0x61, 0x72, 0x67, 0x65, 0x74, 0x53, 0x74, 0x61, 0x74, 0x65, + 0x12, 0x11, 0x0a, 0x0d, 0x53, 0x54, 0x41, 0x54, 0x45, 0x5f, 0x55, 0x4e, 0x4b, 0x4e, 0x4f, 0x57, + 0x4e, 0x10, 0x00, 0x12, 0x0c, 0x0a, 0x08, 0x44, 0x4f, 0x57, 0x4e, 0x5f, 0x4f, 0x55, 0x54, 0x10, + 0x01, 0x12, 0x08, 0x0a, 0x04, 0x44, 0x4f, 0x57, 0x4e, 0x10, 0x02, 0x12, 0x06, 0x0a, 0x02, 0x55, + 0x50, 0x10, 0x03, 0x12, 0x09, 0x0a, 0x05, 0x55, 0x50, 0x5f, 0x49, 0x4e, 0x10, 0x04, 0x12, 0x07, + 0x0a, 0x03, 0x4e, 0x45, 0x57, 0x10, 0x05, 0x12, 0x09, 0x0a, 0x05, 0x44, 0x52, 0x41, 0x49, 0x4e, + 0x10, 0x06, 0x22, 0x5e, 0x0a, 0x13, 0x50, 0x6f, 0x6f, 0x6c, 0x51, 0x75, 0x65, 0x72, 0x79, 0x54, + 0x61, 0x72, 0x67, 0x65, 0x74, 0x52, 0x65, 0x73, 0x70, 0x12, 0x16, 0x0a, 0x06, 0x73, 0x74, 0x61, + 0x74, 0x75, 0x73, 0x18, 0x01, 0x20, 0x01, 0x28, 0x05, 0x52, 0x06, 0x73, 0x74, 0x61, 0x74, 0x75, + 0x73, 0x12, 0x2f, 0x0a, 0x05, 0x69, 0x6e, 0x66, 0x6f, 0x73, 0x18, 0x02, 0x20, 0x03, 0x28, 0x0b, + 0x32, 0x19, 0x2e, 0x6d, 0x67, 0x6d, 0x74, 0x2e, 0x50, 0x6f, 0x6f, 0x6c, 0x51, 0x75, 0x65, 0x72, + 0x79, 0x54, 0x61, 0x72, 0x67, 0x65, 0x74, 0x49, 0x6e, 0x66, 0x6f, 0x52, 0x05, 0x69, 0x6e, 0x66, + 0x6f, 0x73, 0x2a, 0x25, 0x0a, 0x10, 0x53, 0x74, 0x6f, 0x72, 0x61, 0x67, 0x65, 0x4d, 0x65, 0x64, + 0x69, 0x61, 0x54, 0x79, 0x70, 0x65, 0x12, 0x07, 0x0a, 0x03, 0x53, 0x43, 0x4d, 0x10, 0x00, 0x12, + 0x08, 0x0a, 0x04, 0x4e, 0x56, 0x4d, 0x45, 0x10, 0x01, 0x2a, 0x56, 0x0a, 0x10, 0x50, 0x6f, 0x6f, + 0x6c, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x53, 0x74, 0x61, 0x74, 0x65, 0x12, 0x0c, 0x0a, + 0x08, 0x43, 0x72, 0x65, 0x61, 0x74, 0x69, 0x6e, 0x67, 0x10, 0x00, 0x12, 0x09, 0x0a, 0x05, 0x52, + 0x65, 0x61, 0x64, 0x79, 0x10, 0x01, 0x12, 0x0e, 0x0a, 0x0a, 0x44, 0x65, 0x73, 0x74, 0x72, 0x6f, + 0x79, 0x69, 0x6e, 0x67, 0x10, 0x02, 0x12, 0x0c, 0x0a, 0x08, 0x44, 0x65, 0x67, 0x72, 0x61, 0x64, + 0x65, 0x64, 0x10, 0x03, 0x12, 0x0b, 0x0a, 0x07, 0x55, 0x6e, 0x6b, 0x6e, 0x6f, 0x77, 0x6e, 0x10, + 0x04, 0x42, 0x3a, 0x5a, 0x38, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, + 0x64, 0x61, 0x6f, 0x73, 0x2d, 0x73, 0x74, 0x61, 0x63, 0x6b, 0x2f, 0x64, 0x61, 0x6f, 0x73, 0x2f, + 0x73, 0x72, 0x63, 0x2f, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2f, 0x63, 0x6f, 0x6d, 0x6d, + 0x6f, 0x6e, 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2f, 0x6d, 0x67, 0x6d, 0x74, 0x62, 0x06, 0x70, + 0x72, 0x6f, 0x74, 0x6f, 0x33, } var ( diff --git a/src/control/lib/control/pool_test.go b/src/control/lib/control/pool_test.go index aace70eb067..65a0b12c6ad 100644 --- a/src/control/lib/control/pool_test.go +++ b/src/control/lib/control/pool_test.go @@ -846,12 +846,12 @@ func TestControl_PoolQueryResp_MarshalJSON(t *testing.T) { ServiceLeader: 6, ServiceReplicas: []ranklist.Rank{0, 1, 2}, DisabledRanks: &ranklist.RankSet{}, - SuspectRanks: ranklist.MustCreateRankSet("[7,8,9]"), + DeadRanks: ranklist.MustCreateRankSet("[7,8,9]"), PoolLayoutVer: 7, UpgradeLayoutVer: 8, }, }, - exp: `{"query_mask":"disabled_engines,rebuild,suspect_engines","state":"Ready","uuid":"` + poolUUID.String() + `","total_targets":1,"active_targets":2,"total_engines":3,"disabled_targets":4,"version":5,"svc_ldr":6,"svc_reps":[0,1,2],"rebuild":null,"tier_stats":null,"disabled_ranks":[],"suspect_ranks":[7,8,9],"pool_layout_ver":7,"upgrade_layout_ver":8,"mem_file_bytes":0,"status":42}`, + exp: `{"query_mask":"dead_engines,disabled_engines,rebuild","state":"Ready","uuid":"` + poolUUID.String() + `","total_targets":1,"active_targets":2,"total_engines":3,"disabled_targets":4,"version":5,"svc_ldr":6,"svc_reps":[0,1,2],"rebuild":null,"tier_stats":null,"disabled_ranks":[],"dead_ranks":[7,8,9],"pool_layout_ver":7,"upgrade_layout_ver":8,"status":42}`, }, } { t.Run(name, func(t *testing.T) { @@ -893,7 +893,7 @@ func TestControl_PoolQueryResp_UnmarshalJSON(t *testing.T) { }, }, "valid rankset": { - data: `{"enabled_ranks":"[0,1-3,5]","disabled_ranks":"[]","suspect_ranks":"[4]","status":0,"uuid":"` + poolUUID.String() + `","total_targets":1,"active_targets":2,"total_engines":3,"disabled_targets":4,"version":5,"svc_ldr":6,"svc_reps":null,"rebuild":null,"tier_stats":null,"pool_layout_ver":7,"upgrade_layout_ver":8}`, + data: `{"enabled_ranks":"[0,1-3,5]","dead_ranks":"[4]","disabled_ranks":"[]","status":0,"uuid":"` + poolUUID.String() + `","total_targets":1,"active_targets":2,"total_engines":3,"disabled_targets":4,"version":5,"svc_ldr":6,"svc_reps":null,"rebuild":null,"tier_stats":null,"pool_layout_ver":7,"upgrade_layout_ver":8}`, expResp: PoolQueryResp{ Status: 0, PoolInfo: daos.PoolInfo{ @@ -906,7 +906,7 @@ func TestControl_PoolQueryResp_UnmarshalJSON(t *testing.T) { ServiceLeader: 6, EnabledRanks: ranklist.MustCreateRankSet("[0-3,5]"), DisabledRanks: &ranklist.RankSet{}, - SuspectRanks: ranklist.MustCreateRankSet("[4]"), + DeadRanks: ranklist.MustCreateRankSet("[4]"), PoolLayoutVer: 7, UpgradeLayoutVer: 8, }, @@ -1177,7 +1177,7 @@ func TestControl_PoolQuery(t *testing.T) { }, }, }, - "query succeeds suspect ranks": { + "query succeeds dead ranks": { mic: &MockInvokerConfig{ UnaryResponse: MockMSResponse("host1", nil, &mgmtpb.PoolQueryResp{ @@ -1211,7 +1211,7 @@ func TestControl_PoolQuery(t *testing.T) { MediaType: mgmtpb.StorageMediaType(daos.StorageMediaTypeNvme), }, }, - SuspectRanks: "[1,2,3,7]", + DeadRanks: "[1,2,3,7]", }, ), }, @@ -1247,7 +1247,7 @@ func TestControl_PoolQuery(t *testing.T) { MediaType: daos.StorageMediaTypeNvme, }, }, - SuspectRanks: ranklist.MustCreateRankSet("[1-3,7]"), + DeadRanks: ranklist.MustCreateRankSet("[1-3,7]"), }, }, }, diff --git a/src/control/lib/daos/pool.go b/src/control/lib/daos/pool.go index b06a44c9025..ba67b1e91bf 100644 --- a/src/control/lib/daos/pool.go +++ b/src/control/lib/daos/pool.go @@ -77,7 +77,7 @@ type ( TierStats []*StorageUsageStats `json:"tier_stats"` EnabledRanks *ranklist.RankSet `json:"enabled_ranks,omitempty"` DisabledRanks *ranklist.RankSet `json:"disabled_ranks,omitempty"` - SuspectRanks *ranklist.RankSet `json:"suspect_ranks,omitempty"` + DeadRanks *ranklist.RankSet `json:"dead_ranks,omitempty"` PoolLayoutVer uint32 `json:"pool_layout_ver"` UpgradeLayoutVer uint32 `json:"upgrade_layout_ver"` } @@ -105,7 +105,7 @@ type ( const ( // DefaultPoolQueryMask defines the default pool query mask. - DefaultPoolQueryMask = PoolQueryMask(^uint64(0) &^ (C.DPI_ENGINES_ENABLED | C.DPI_ENGINES_SUSPECT)) + DefaultPoolQueryMask = PoolQueryMask(^uint64(0) &^ (C.DPI_ENGINES_ENABLED | C.DPI_ENGINES_DEAD)) // HealthOnlyPoolQueryMask defines the mask for health-only queries. HealthOnlyPoolQueryMask = PoolQueryMask(^uint64(0) &^ (C.DPI_ENGINES_ENABLED | C.DPI_SPACE)) @@ -117,8 +117,8 @@ const ( PoolQueryOptionEnabledEngines = "enabled_engines" // PoolQueryOptionDisabledEngines retrieves disabled engines as part of the pool query. PoolQueryOptionDisabledEngines = "disabled_engines" - // PoolQueryOptionSuspectEngines retrieves suspect engines as part of the pool query. - PoolQueryOptionSuspectEngines = "suspect_engines" + // PoolQueryOptionDeadEngines retrieves dead engines as part of the pool query. + PoolQueryOptionDeadEngines = "dead_engines" // PoolConnectFlagReadOnly indicates that the connection is read-only. PoolConnectFlagReadOnly = C.DAOS_PC_RO @@ -133,7 +133,7 @@ var poolQueryOptMap = map[C.int]string{ C.DPI_REBUILD_STATUS: PoolQueryOptionRebuild, C.DPI_ENGINES_ENABLED: PoolQueryOptionEnabledEngines, C.DPI_ENGINES_DISABLED: PoolQueryOptionDisabledEngines, - C.DPI_ENGINES_SUSPECT: PoolQueryOptionSuspectEngines, + C.DPI_ENGINES_DEAD: PoolQueryOptionDeadEngines, } func resolvePoolQueryOpt(name string) (C.int, error) { diff --git a/src/control/lib/daos/pool_test.go b/src/control/lib/daos/pool_test.go index 1b91b9f5520..8cf5f71312f 100644 --- a/src/control/lib/daos/pool_test.go +++ b/src/control/lib/daos/pool_test.go @@ -136,14 +136,14 @@ func TestDaos_PoolQueryMask(t *testing.T) { testMask: genTestMask(func(pqm *PoolQueryMask) { *pqm = HealthOnlyPoolQueryMask }), - expString: genOptsStr(PoolQueryOptionDisabledEngines, PoolQueryOptionRebuild, PoolQueryOptionSuspectEngines), + expString: genOptsStr(PoolQueryOptionDeadEngines, PoolQueryOptionDisabledEngines, PoolQueryOptionRebuild), }, "set query all=true": { testMask: genTestMask(func(pqm *PoolQueryMask) { pqm.SetAll() }), - expString: genOptsStr(PoolQueryOptionDisabledEngines, PoolQueryOptionEnabledEngines, - PoolQueryOptionRebuild, PoolQueryOptionSpace, PoolQueryOptionSuspectEngines), + expString: genOptsStr(PoolQueryOptionDeadEngines, PoolQueryOptionDisabledEngines, PoolQueryOptionEnabledEngines, + PoolQueryOptionRebuild, PoolQueryOptionSpace), }, "set query all=false": { testMask: genTestMask(func(pqm *PoolQueryMask) { @@ -163,8 +163,8 @@ func TestDaos_PoolQueryMask(t *testing.T) { pqm.SetAll() pqm.ClearOptions(PoolQueryOptionSpace) }), - expString: genOptsStr(PoolQueryOptionDisabledEngines, PoolQueryOptionEnabledEngines, - PoolQueryOptionRebuild, PoolQueryOptionSuspectEngines), + expString: genOptsStr(PoolQueryOptionDeadEngines, PoolQueryOptionDisabledEngines, PoolQueryOptionEnabledEngines, + PoolQueryOptionRebuild), }, "set query space=false (already false)": { testMask: genTestMask(func(pqm *PoolQueryMask) { @@ -183,8 +183,7 @@ func TestDaos_PoolQueryMask(t *testing.T) { pqm.SetAll() pqm.ClearOptions(PoolQueryOptionRebuild) }), - expString: genOptsStr(PoolQueryOptionDisabledEngines, PoolQueryOptionEnabledEngines, PoolQueryOptionSpace, - PoolQueryOptionSuspectEngines), + expString: genOptsStr(PoolQueryOptionDeadEngines, PoolQueryOptionDisabledEngines, PoolQueryOptionEnabledEngines, PoolQueryOptionSpace), }, "set query enabled_engines=true": { testMask: genTestMask(func(pqm *PoolQueryMask) { @@ -197,8 +196,7 @@ func TestDaos_PoolQueryMask(t *testing.T) { pqm.SetAll() pqm.ClearOptions(PoolQueryOptionEnabledEngines) }), - expString: genOptsStr(PoolQueryOptionDisabledEngines, PoolQueryOptionRebuild, PoolQueryOptionSpace, - PoolQueryOptionSuspectEngines), + expString: genOptsStr(PoolQueryOptionDeadEngines, PoolQueryOptionDisabledEngines, PoolQueryOptionRebuild, PoolQueryOptionSpace), }, "set query disabled_engines=true": { testMask: genTestMask(func(pqm *PoolQueryMask) { @@ -211,8 +209,7 @@ func TestDaos_PoolQueryMask(t *testing.T) { pqm.SetAll() pqm.ClearOptions(PoolQueryOptionDisabledEngines) }), - expString: genOptsStr(PoolQueryOptionEnabledEngines, PoolQueryOptionRebuild, PoolQueryOptionSpace, - PoolQueryOptionSuspectEngines), + expString: genOptsStr(PoolQueryOptionDeadEngines, PoolQueryOptionEnabledEngines, PoolQueryOptionRebuild, PoolQueryOptionSpace), }, } { t.Run(name, func(t *testing.T) { @@ -237,7 +234,7 @@ func TestDaos_PoolQueryMaskMarshalJSON(t *testing.T) { testMask: genTestMask(func(pqm *PoolQueryMask) { pqm.SetAll() }), - expJSON: []byte(`"disabled_engines,enabled_engines,rebuild,space,suspect_engines"`), + expJSON: []byte(`"dead_engines,disabled_engines,enabled_engines,rebuild,space"`), }, } { t.Run(name, func(t *testing.T) { @@ -267,7 +264,7 @@ func TestDaos_PoolQueryMaskUnmarshalJSON(t *testing.T) { }, "uint64 value": { testData: []byte("18446744073709551603"), - expString: "rebuild,space,suspect_engines", + expString: "dead_engines,rebuild,space", }, "string values": { testData: []byte("rebuild,disabled_engines"), diff --git a/src/include/daos_pool.h b/src/include/daos_pool.h index a8ab2e6c6a2..24746a70ff7 100644 --- a/src/include/daos_pool.h +++ b/src/include/daos_pool.h @@ -162,8 +162,8 @@ enum daos_pool_info_bit { DPI_ENGINES_ENABLED = 1ULL << 2, /** true to include (in \a ranks) engines with some or all targets disabled (down). */ DPI_ENGINES_DISABLED = 1ULL << 3, - /** true to include (in \a ranks) suspect engines. */ - DPI_ENGINES_SUSPECT = 1ULL << 4, + /** true to include (in \a ranks) engines marked DEAD by SWIM. */ + DPI_ENGINES_DEAD = 1ULL << 4, /** query all above optional info */ DPI_ALL = -1, }; diff --git a/src/include/daos_srv/pool.h b/src/include/daos_srv/pool.h index a8ea070446b..b9abe5dfa55 100644 --- a/src/include/daos_srv/pool.h +++ b/src/include/daos_srv/pool.h @@ -307,7 +307,7 @@ int dsc_pool_svc_delete_acl(uuid_t pool_uuid, d_rank_list_t *ranks, uint64_t dea int dsc_pool_svc_query(uuid_t pool_uuid, d_rank_list_t *ps_ranks, uint64_t deadline, d_rank_list_t **enabled_ranks, d_rank_list_t **disabled_ranks, - d_rank_list_t **suspect_ranks, daos_pool_info_t *pool_info, + d_rank_list_t **dead_ranks, daos_pool_info_t *pool_info, uint32_t *pool_layout_ver, uint32_t *upgrade_layout_ver); int dsc_pool_svc_query_target(uuid_t pool_uuid, d_rank_list_t *ps_ranks, uint64_t deadline, d_rank_t rank, uint32_t tgt_idx, daos_target_info_t *ti); diff --git a/src/mgmt/pool.pb-c.c b/src/mgmt/pool.pb-c.c index 2137c1f5f53..b083468bbc1 100644 --- a/src/mgmt/pool.pb-c.c +++ b/src/mgmt/pool.pb-c.c @@ -3402,14 +3402,14 @@ static const ProtobufCFieldDescriptor mgmt__pool_query_resp__field_descriptors[2 0, NULL, NULL /* reserved1,reserved2, etc */ }, { - "suspect_ranks", 21, PROTOBUF_C_LABEL_NONE, PROTOBUF_C_TYPE_STRING, - 0, /* quantifier_offset */ - offsetof(Mgmt__PoolQueryResp, suspect_ranks), NULL, &protobuf_c_empty_string, 0, /* flags */ + "dead_ranks", 21, PROTOBUF_C_LABEL_NONE, PROTOBUF_C_TYPE_STRING, 0, /* quantifier_offset */ + offsetof(Mgmt__PoolQueryResp, dead_ranks), NULL, &protobuf_c_empty_string, 0, /* flags */ 0, NULL, NULL /* reserved1,reserved2, etc */ }, }; static const unsigned mgmt__pool_query_resp__field_indices_by_name[] = { 4, /* field[4] = active_targets */ + 19, /* field[19] = dead_ranks */ 11, /* field[11] = disabled_ranks */ 5, /* field[5] = disabled_targets */ 10, /* field[10] = enabled_ranks */ @@ -3420,7 +3420,6 @@ static const unsigned mgmt__pool_query_resp__field_indices_by_name[] = { 6, /* field[6] = rebuild */ 15, /* field[15] = state */ 0, /* field[0] = status */ - 19, /* field[19] = suspect_ranks */ 16, /* field[16] = svc_ldr */ 17, /* field[17] = svc_reps */ 7, /* field[7] = tier_stats */ diff --git a/src/mgmt/pool.pb-c.h b/src/mgmt/pool.pb-c.h index 7d8e18c96c3..c0c265f3911 100644 --- a/src/mgmt/pool.pb-c.h +++ b/src/mgmt/pool.pb-c.h @@ -862,9 +862,9 @@ struct _Mgmt__PoolQueryResp */ uint64_t query_mask; /* - * optional set of suspect ranks + * optional set of dead ranks */ - char *suspect_ranks; + char *dead_ranks; }; #define MGMT__POOL_QUERY_RESP__INIT \ { \ diff --git a/src/mgmt/srv_drpc.c b/src/mgmt/srv_drpc.c index 0a176dbccc2..4fcf8fe3beb 100644 --- a/src/mgmt/srv_drpc.c +++ b/src/mgmt/srv_drpc.c @@ -1747,10 +1747,10 @@ ds_mgmt_drpc_pool_query(Drpc__Call *drpc_req, Drpc__Response *drpc_resp) d_rank_list_t *svc_ranks = NULL; d_rank_list_t *enabled_ranks = NULL; d_rank_list_t *disabled_ranks = NULL; - d_rank_list_t *suspect_ranks = NULL; + d_rank_list_t *dead_ranks = NULL; char *enabled_ranks_str = NULL; char *disabled_ranks_str = NULL; - char *suspect_ranks_str = NULL; + char *dead_ranks_str = NULL; size_t len; uint8_t *body; @@ -1774,7 +1774,7 @@ ds_mgmt_drpc_pool_query(Drpc__Call *drpc_req, Drpc__Response *drpc_resp) D_GOTO(error, rc = -DER_NOMEM); pool_info.pi_bits = req->query_mask; - rc = ds_mgmt_pool_query(uuid, svc_ranks, &enabled_ranks, &disabled_ranks, &suspect_ranks, + rc = ds_mgmt_pool_query(uuid, svc_ranks, &enabled_ranks, &disabled_ranks, &dead_ranks, &pool_info, &resp.pool_layout_ver, &resp.upgrade_layout_ver); if (rc != 0) { DL_ERROR(rc, DF_UUID ": Failed to query the pool", DP_UUID(uuid)); @@ -1797,18 +1797,17 @@ ds_mgmt_drpc_pool_query(Drpc__Call *drpc_req, Drpc__Response *drpc_resp) DP_UUID(uuid)); D_GOTO(error, rc); } - rc = d_rank_list_to_str(suspect_ranks, &suspect_ranks_str); + rc = d_rank_list_to_str(dead_ranks, &dead_ranks_str); if (rc != 0) { - DL_ERROR(rc, DF_UUID ": Failed to serialize the list of suspect ranks", - DP_UUID(uuid)); + DL_ERROR(rc, DF_UUID ": Failed to serialize the list of dead ranks", DP_UUID(uuid)); D_GOTO(error, rc); } if (disabled_ranks_str != NULL) D_DEBUG(DB_MGMT, DF_UUID ": list of disabled ranks: %s\n", DP_UUID(uuid), disabled_ranks_str); - if (suspect_ranks_str != NULL) - D_DEBUG(DB_MGMT, DF_UUID ": list of suspect ranks: %s\n", DP_UUID(uuid), - suspect_ranks_str); + if (dead_ranks_str != NULL) + D_DEBUG(DB_MGMT, DF_UUID ": list of dead ranks: %s\n", DP_UUID(uuid), + dead_ranks_str); /* Populate the response */ resp.query_mask = pool_info.pi_bits; @@ -1825,8 +1824,8 @@ ds_mgmt_drpc_pool_query(Drpc__Call *drpc_req, Drpc__Response *drpc_resp) resp.enabled_ranks = enabled_ranks_str; if (disabled_ranks_str != NULL) resp.disabled_ranks = disabled_ranks_str; - if (suspect_ranks_str != NULL) - resp.suspect_ranks = suspect_ranks_str; + if (dead_ranks_str != NULL) + resp.dead_ranks = dead_ranks_str; D_ALLOC_ARRAY(resp.tier_stats, DAOS_MEDIA_MAX); if (resp.tier_stats == NULL) @@ -1864,8 +1863,8 @@ ds_mgmt_drpc_pool_query(Drpc__Call *drpc_req, Drpc__Response *drpc_resp) D_FREE(enabled_ranks_str); d_rank_list_free(disabled_ranks); D_FREE(disabled_ranks_str); - d_rank_list_free(suspect_ranks); - D_FREE(suspect_ranks_str); + d_rank_list_free(dead_ranks); + D_FREE(dead_ranks_str); d_rank_list_free(svc_ranks); pool_query_free_tier_stats(&resp); } diff --git a/src/mgmt/srv_internal.h b/src/mgmt/srv_internal.h index a2cbaf0890e..638276dac59 100644 --- a/src/mgmt/srv_internal.h +++ b/src/mgmt/srv_internal.h @@ -115,7 +115,7 @@ int ds_mgmt_pool_list_cont(uuid_t uuid, d_rank_list_t *svc_ranks, uint64_t *ncontainers); int ds_mgmt_pool_query(uuid_t pool_uuid, d_rank_list_t *svc_ranks, d_rank_list_t **enabled_ranks, - d_rank_list_t **disabled_ranks, d_rank_list_t **suspect_ranks, + d_rank_list_t **disabled_ranks, d_rank_list_t **dead_ranks, daos_pool_info_t *pool_info, uint32_t *pool_layout_ver, uint32_t *upgrade_layout_ver); int ds_mgmt_pool_query_targets(uuid_t pool_uuid, d_rank_list_t *svc_ranks, d_rank_t rank, diff --git a/src/mgmt/srv_pool.c b/src/mgmt/srv_pool.c index 024965d739d..9f3052d63ea 100644 --- a/src/mgmt/srv_pool.c +++ b/src/mgmt/srv_pool.c @@ -386,6 +386,7 @@ ds_mgmt_pool_list_cont(uuid_t uuid, d_rank_list_t *svc_ranks, * \param[in] svc_ranks Ranks of pool svc replicas. * \param[out] enabled_ranks Optional, returned storage ranks with enabled targets. * \param[out] disabled_ranks Optional, returned storage ranks with disabled targets. + * \param[out] dead_ranks Optional, returned storage ranks marked DEAD by SWIM. * \param[in][out] pool_info Query results * \param[in][out] pool_layout_ver Pool global version * \param[in][out] upgrade_layout_ver Latest pool global version this pool might be upgraded @@ -396,7 +397,7 @@ ds_mgmt_pool_list_cont(uuid_t uuid, d_rank_list_t *svc_ranks, */ int ds_mgmt_pool_query(uuid_t pool_uuid, d_rank_list_t *svc_ranks, d_rank_list_t **enabled_ranks, - d_rank_list_t **disabled_ranks, d_rank_list_t **suspect_ranks, + d_rank_list_t **disabled_ranks, d_rank_list_t **dead_ranks, daos_pool_info_t *pool_info, uint32_t *pool_layout_ver, uint32_t *upgrade_layout_ver) { @@ -408,7 +409,7 @@ ds_mgmt_pool_query(uuid_t pool_uuid, d_rank_list_t *svc_ranks, d_rank_list_t **e D_DEBUG(DB_MGMT, "Querying pool "DF_UUID"\n", DP_UUID(pool_uuid)); return dsc_pool_svc_query(pool_uuid, svc_ranks, mgmt_ps_call_deadline(), enabled_ranks, - disabled_ranks, suspect_ranks, pool_info, pool_layout_ver, + disabled_ranks, dead_ranks, pool_info, pool_layout_ver, upgrade_layout_ver); } diff --git a/src/mgmt/tests/mocks.c b/src/mgmt/tests/mocks.c index c22df3846e0..575e7f9e97a 100644 --- a/src/mgmt/tests/mocks.c +++ b/src/mgmt/tests/mocks.c @@ -281,11 +281,11 @@ daos_pool_info_t ds_mgmt_pool_query_info_in; void *ds_mgmt_pool_query_info_ptr; d_rank_list_t *ds_mgmt_pool_query_enabled_ranks_out; d_rank_list_t *ds_mgmt_pool_query_disabled_ranks_out; -d_rank_list_t *ds_mgmt_pool_query_suspect_ranks_out; +d_rank_list_t *ds_mgmt_pool_query_dead_ranks_out; int ds_mgmt_pool_query(uuid_t pool_uuid, d_rank_list_t *svc_ranks, d_rank_list_t **enabled_ranks, - d_rank_list_t **disabled_ranks, d_rank_list_t **suspect_ranks, + d_rank_list_t **disabled_ranks, d_rank_list_t **dead_ranks, daos_pool_info_t *pool_info, uint32_t *pool_layout_ver, uint32_t *upgrade_layout_ver) { @@ -312,11 +312,11 @@ ds_mgmt_pool_query(uuid_t pool_uuid, d_rank_list_t *svc_ranks, d_rank_list_t **e ds_mgmt_pool_query_disabled_ranks_out = *disabled_ranks; } - if ((pool_info->pi_bits & DPI_ENGINES_SUSPECT) != 0) { - D_ASSERT(suspect_ranks != NULL); + if ((pool_info->pi_bits & DPI_ENGINES_DEAD) != 0) { + D_ASSERT(dead_ranks != NULL); - *suspect_ranks = d_rank_list_alloc(2); /* 0-1 ; caller must free this */ - ds_mgmt_pool_query_suspect_ranks_out = *suspect_ranks; + *dead_ranks = d_rank_list_alloc(2); /* 0-1 ; caller must free this */ + ds_mgmt_pool_query_dead_ranks_out = *dead_ranks; } ds_mgmt_pool_query_info_in = *pool_info; @@ -334,7 +334,7 @@ mock_ds_mgmt_pool_query_setup(void) memset(&ds_mgmt_pool_query_info_out, 0, sizeof(daos_pool_info_t)); ds_mgmt_pool_query_enabled_ranks_out = NULL; ds_mgmt_pool_query_disabled_ranks_out = NULL; - ds_mgmt_pool_query_suspect_ranks_out = NULL; + ds_mgmt_pool_query_dead_ranks_out = NULL; } int ds_mgmt_pool_query_targets_return; diff --git a/src/mgmt/tests/mocks.h b/src/mgmt/tests/mocks.h index 2ecc03d8ed9..ea93e7199b1 100644 --- a/src/mgmt/tests/mocks.h +++ b/src/mgmt/tests/mocks.h @@ -110,7 +110,7 @@ extern daos_pool_info_t ds_mgmt_pool_query_info_out; extern void *ds_mgmt_pool_query_info_ptr; extern d_rank_list_t *ds_mgmt_pool_query_enabled_ranks_out; extern d_rank_list_t *ds_mgmt_pool_query_disabled_ranks_out; -extern d_rank_list_t *ds_mgmt_pool_query_suspect_ranks_out; +extern d_rank_list_t *ds_mgmt_pool_query_dead_ranks_out; void mock_ds_mgmt_pool_query_setup(void); diff --git a/src/mgmt/tests/srv_drpc_tests.c b/src/mgmt/tests/srv_drpc_tests.c index 559f9ea08ae..cde0f58f10e 100644 --- a/src/mgmt/tests/srv_drpc_tests.c +++ b/src/mgmt/tests/srv_drpc_tests.c @@ -1408,7 +1408,7 @@ test_drpc_pool_query_success(void **state) Drpc__Response resp = DRPC__RESPONSE__INIT; uuid_t exp_uuid; daos_pool_info_t exp_info = {0}; - uint64_t flags = DPI_ENGINES_ENABLED | DPI_ENGINES_DISABLED | DPI_ENGINES_SUSPECT; + uint64_t flags = DPI_ENGINES_ENABLED | DPI_ENGINES_DISABLED | DPI_ENGINES_DEAD; init_test_pool_info(&exp_info); init_test_rebuild_status(&exp_info.pi_rebuild_st); @@ -1425,7 +1425,7 @@ test_drpc_pool_query_success(void **state) assert_non_null(ds_mgmt_pool_query_info_ptr); assert_non_null(ds_mgmt_pool_query_enabled_ranks_out); assert_non_null(ds_mgmt_pool_query_disabled_ranks_out); - assert_non_null(ds_mgmt_pool_query_suspect_ranks_out); + assert_non_null(ds_mgmt_pool_query_dead_ranks_out); flags |= DEFAULT_QUERY_BITS; assert_int_equal(ds_mgmt_pool_query_info_in.pi_bits, DEFAULT_QUERY_BITS | flags); diff --git a/src/pool/srv_cli.c b/src/pool/srv_cli.c index 5630394b154..bbc228dd2b1 100644 --- a/src/pool/srv_cli.c +++ b/src/pool/srv_cli.c @@ -337,7 +337,7 @@ dsc_pool_svc_call(uuid_t uuid, d_rank_list_t *ranks, struct dsc_pool_svc_call_cb struct pool_query_arg { d_rank_list_t **pqa_enabled_ranks; d_rank_list_t **pqa_disabled_ranks; - d_rank_list_t **pqa_suspect_ranks; + d_rank_list_t **pqa_dead_ranks; daos_pool_info_t *pqa_info; uint32_t *pqa_layout_ver; uint32_t *pqa_upgrade_layout_ver; @@ -369,7 +369,7 @@ pool_query_init(uuid_t pool_uuid, crt_rpc_t *rpc, void *varg) } static int -pool_map_get_suspect_ranks(struct pool_map *map, d_rank_list_t **ranks) +pool_map_get_dead_ranks(struct pool_map *map, d_rank_list_t **ranks) { crt_group_t *primary_grp; struct pool_domain *doms; @@ -418,7 +418,7 @@ pool_map_get_suspect_ranks(struct pool_map *map, d_rank_list_t **ranks) static int process_query_result(d_rank_list_t **enabled_ranks, d_rank_list_t **disabled_ranks, - d_rank_list_t **suspect_ranks, daos_pool_info_t *info, uuid_t pool_uuid, + d_rank_list_t **dead_ranks, daos_pool_info_t *info, uuid_t pool_uuid, uint32_t map_version, uint32_t leader_rank, struct daos_pool_space *ps, struct daos_rebuild_status *rs, struct pool_buf *map_buf, uint64_t pi_bits) { @@ -426,7 +426,7 @@ process_query_result(d_rank_list_t **enabled_ranks, d_rank_list_t **disabled_ran unsigned int num_disabled = 0; d_rank_list_t *enabled_rank_list = NULL; d_rank_list_t *disabled_rank_list = NULL; - d_rank_list_t *suspect_rank_list = NULL; + d_rank_list_t *dead_rank_list = NULL; int rc; rc = pool_map_create(map_buf, map_version, &map); @@ -474,21 +474,21 @@ process_query_result(d_rank_list_t **enabled_ranks, d_rank_list_t **disabled_ran D_DEBUG(DB_MD, DF_UUID ": found %" PRIu32 " disabled ranks in pool map\n", DP_UUID(pool_uuid), disabled_rank_list->rl_nr); } - if ((pi_bits & DPI_ENGINES_SUSPECT) != 0) { - if (suspect_ranks == NULL) { + if ((pi_bits & DPI_ENGINES_DEAD) != 0) { + if (dead_ranks == NULL) { DL_ERROR(-DER_INVAL, - DF_UUID ": query pool requested suspect ranks, but ptr is NULL", + DF_UUID ": query pool requested dead ranks, but ptr is NULL", DP_UUID(pool_uuid)); D_GOTO(error, rc = -DER_INVAL); } - rc = pool_map_get_suspect_ranks(map, &suspect_rank_list); + rc = pool_map_get_dead_ranks(map, &dead_rank_list); if (rc != 0) { DL_ERROR(rc, DF_UUID ": pool_map_get_ranks() failed", DP_UUID(pool_uuid)); D_GOTO(error, rc); } - D_DEBUG(DB_MD, DF_UUID ": found %" PRIu32 " suspect ranks in pool map\n", - DP_UUID(pool_uuid), suspect_rank_list->rl_nr); + D_DEBUG(DB_MD, DF_UUID ": found %" PRIu32 " dead ranks in pool map\n", + DP_UUID(pool_uuid), dead_rank_list->rl_nr); } pool_query_reply_to_info(pool_uuid, map_buf, map_version, leader_rank, ps, rs, info); @@ -497,14 +497,14 @@ process_query_result(d_rank_list_t **enabled_ranks, d_rank_list_t **disabled_ran *enabled_ranks = enabled_rank_list; if (disabled_rank_list != NULL) *disabled_ranks = disabled_rank_list; - if (suspect_rank_list != NULL) - *suspect_ranks = suspect_rank_list; + if (dead_rank_list != NULL) + *dead_ranks = dead_rank_list; D_GOTO(out, rc = -DER_SUCCESS); error: d_rank_list_free(disabled_rank_list); d_rank_list_free(enabled_rank_list); - d_rank_list_free(suspect_rank_list); + d_rank_list_free(dead_rank_list); out: if (map != NULL) pool_map_decref(map); @@ -534,7 +534,7 @@ pool_query_consume(uuid_t pool_uuid, crt_rpc_t *rpc, void *varg) D_DEBUG(DB_MGMT, DF_UUID": Successfully queried pool\n", DP_UUID(pool_uuid)); rc = process_query_result( - arg->pqa_enabled_ranks, arg->pqa_disabled_ranks, arg->pqa_suspect_ranks, arg->pqa_info, + arg->pqa_enabled_ranks, arg->pqa_disabled_ranks, arg->pqa_dead_ranks, arg->pqa_info, pool_uuid, out->pqo_op.po_map_version, out->pqo_op.po_hint.sh_rank, &out->pqo_space, &out->pqo_rebuild_st, arg->pqa_map_buf, arg->pqa_info->pi_bits); if (arg->pqa_layout_ver) @@ -572,7 +572,7 @@ static struct dsc_pool_svc_call_cbs pool_query_cbs = { * \param[in] deadline Unix time deadline in milliseconds * \param[out] enabled_ranks Optional, storage ranks with enabled targets. * \param[out] disabled_ranks Optional, storage ranks with disabled ranks. - * \param[out] suspect_ranks Optional, suspect ranks marked as DEAD by the SWIM + * \param[out] dead_ranks Optional, storage ranks marked as DEAD by the SWIM * protocol, but were not excluded from the system. * \param[in][out] pool_info Results of the pool query * \param[in][out] pool_layout_ver Results of the current pool global version @@ -588,13 +588,13 @@ static struct dsc_pool_svc_call_cbs pool_query_cbs = { int dsc_pool_svc_query(uuid_t pool_uuid, d_rank_list_t *ps_ranks, uint64_t deadline, d_rank_list_t **enabled_ranks, d_rank_list_t **disabled_ranks, - d_rank_list_t **suspect_ranks, daos_pool_info_t *pool_info, + d_rank_list_t **dead_ranks, daos_pool_info_t *pool_info, uint32_t *pool_layout_ver, uint32_t *upgrade_layout_ver) { struct pool_query_arg arg = { .pqa_enabled_ranks = enabled_ranks, .pqa_disabled_ranks = disabled_ranks, - .pqa_suspect_ranks = suspect_ranks, + .pqa_dead_ranks = dead_ranks, .pqa_info = pool_info, .pqa_layout_ver = pool_layout_ver, .pqa_upgrade_layout_ver = upgrade_layout_ver, diff --git a/src/proto/mgmt/pool.proto b/src/proto/mgmt/pool.proto index 8b6361f594d..b204b5c9e94 100644 --- a/src/proto/mgmt/pool.proto +++ b/src/proto/mgmt/pool.proto @@ -238,7 +238,7 @@ message PoolQueryResp { uint32 svc_ldr = 18; // current raft leader (2.6+) repeated uint32 svc_reps = 19; // service replica ranks uint64 query_mask = 20; // Bitmask of pool query options used - string suspect_ranks = 21; // optional set of suspect ranks + string dead_ranks = 21; // optional set of dead ranks } message PoolProperty { diff --git a/src/tests/ftest/control/dmg_pool_query_ranks.py b/src/tests/ftest/control/dmg_pool_query_ranks.py index 7f266a26f10..f3469d3a6b4 100644 --- a/src/tests/ftest/control/dmg_pool_query_ranks.py +++ b/src/tests/ftest/control/dmg_pool_query_ranks.py @@ -49,17 +49,17 @@ def test_pool_query_ranks_basic(self): self._verify_ranks([0, 1, 2, 3, 4], data, "enabled_ranks") self._verify_ranks([], data, "disabled_ranks") - self.log_step("Checking pool query with suspect ranks state information") + self.log_step("Checking pool query with dead ranks state information") data = self.dmg.pool_query(self.pool.identifier, health_only=True) - self._verify_ranks([], data, "suspect_ranks") + self._verify_ranks([], data, "dead_ranks") def test_pool_query_ranks_mgmt(self): """Test the state of ranks after excluding and reintegrate them. Test Description: Create a pool with 5 engines, first excluded engine marked as "Disabled" - second stopped one as “Suspect,” restarting it, ensuring rebuild completes, - clearing the “Suspect” status, reintegrating the excluded first engine, and + second stopped one as “Dead,” restarting it, ensuring rebuild completes, + clearing the “Dead” status, reintegrating the excluded first engine, and finally verifying that all engines are enabled with the excluded rank now empty. :avocado: tags=all,daily_regression @@ -75,7 +75,7 @@ def test_pool_query_ranks_mgmt(self): all_ranks = enabled_ranks.copy() self.random.shuffle(all_ranks) exclude_rank = all_ranks[0] - suspect_rank = all_ranks[1] + dead_rank = all_ranks[1] self.log_step(f"Excluding pool rank:{exclude_rank} all_ranks={all_ranks}") self.pool.exclude([exclude_rank]) enabled_ranks.remove(exclude_rank) @@ -90,18 +90,18 @@ def test_pool_query_ranks_mgmt(self): self.pool.wait_for_rebuild_to_start() # kill second rank. - self.log_step(f"Stopping rank:{suspect_rank} all_ranks={all_ranks}") - self.server_managers[0].stop_ranks([suspect_rank], self.d_log) + self.log_step(f"Stopping rank:{dead_rank} all_ranks={all_ranks}") + self.server_managers[0].stop_ranks([dead_rank], self.d_log) - self.log_step(f"Waiting for pool rank {suspect_rank} to be suspected") - self.pool.wait_pool_suspect_ranks([suspect_rank], timeout=30) + self.log_step(f"Waiting for pool rank {dead_rank} to be dead") + self.pool.wait_pool_dead_ranks([dead_rank], timeout=30) self._verify_ranks(disabled_ranks, data, "disabled_ranks") - self.log_step(f"Starting rank {suspect_rank}") - self.server_managers[0].start_ranks([suspect_rank], self.d_log) + self.log_step(f"Starting rank {dead_rank}") + self.server_managers[0].start_ranks([dead_rank], self.d_log) - self.log_step("Waiting for pool ranks to no longer be suspected") - self.pool.wait_pool_suspect_ranks([], timeout=30) + self.log_step("Waiting for pool ranks to no longer be dead") + self.pool.wait_pool_dead_ranks([], timeout=30) self.log_step("Waiting for rebuild to complete") self.pool.wait_for_rebuild_to_end() diff --git a/src/tests/ftest/util/test_utils_pool.py b/src/tests/ftest/util/test_utils_pool.py index f5e88d2c26c..e75510ec8b5 100644 --- a/src/tests/ftest/util/test_utils_pool.py +++ b/src/tests/ftest/util/test_utils_pool.py @@ -1448,11 +1448,11 @@ def check_pool_files(self, hosts, uuid, scm_mount): status = False return status - def wait_pool_suspect_ranks(self, expected, interval=1, timeout=30): - """Wait for the pool suspect ranks. + def wait_pool_dead_ranks(self, expected, interval=1, timeout=30): + """Wait for the pool dead ranks. Args: - expected (list): suspect ranks check to wait. + expected (list): dead ranks check to wait. interval (int, optional): number of seconds to wait in between pool query checks timeout(int, optional): time to fail test if it could not match expected values. @@ -1461,19 +1461,19 @@ def wait_pool_suspect_ranks(self, expected, interval=1, timeout=30): DaosTestError: if waiting for timeout. """ - self.log.info("waiting for pool ranks %s to be suspected", expected) + self.log.info("waiting for pool ranks %s to be marked dead", expected) start = time() data = self.dmg.pool_query(self.identifier, health_only=True) - while data['response'].get('suspect_ranks') != expected: - self.log.info(" suspect ranks is %s ...", data['response'].get('suspect_ranks')) + while data['response'].get('dead_ranks') != expected: + self.log.info(" dead ranks is %s ...", data['response'].get('dead_ranks')) if time() - start > timeout: raise DaosTestError("TIMEOUT detected after {} seconds while for waiting " - "for ranks {} suspect".format(timeout, expected)) + "for ranks {} dead".format(timeout, expected)) sleep(interval) data = self.dmg.pool_query(self.identifier, health_only=True) - self.log.info("Wait for suspect ranks complete: suspect ranks %s", expected) + self.log.info("Wait for dead ranks complete: dead ranks %s", expected) def verify_uuid_directory(self, host, scm_mount): """Check if pool folder exist on server.