Skip to content

Commit

Permalink
DAOS-13047 chk: misc fixes for chk related issues - X
Browse files Browse the repository at this point in the history 
Mainly contains the following fixes:

1. Do not start pool service for orphan pool after check.

2. Enable CR25 to test "for-all" option for check repair.

3. CR26 and CR27 to verify how the checker can handle the case that
   some check engine failed to report some pool when start checker.

4. Some cleanup for former review feedback.

Signed-off-by: Fan Yong <[email protected]>
  • Loading branch information
@Nasf-Fan
Nasf-Fan committed Sep 14, 2023
1 parent b33e84d commit 2fcf848
Show file tree
Hide file tree
Showing 10 changed files with 478 additions and 225 deletions.
47 changes: 41 additions & 6 deletions src/chk/chk_common.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -72,6 +72,7 @@ chk_pool_alloc(struct btr_instance *tins, d_iov_t *key_iov, d_iov_t *val_iov,
if (rc != 0)
D_GOTO(out, rc = dss_abterr2der(rc));

D_INIT_LIST_HEAD(&cpr->cpr_shutdown_link);
D_INIT_LIST_HEAD(&cpr->cpr_shard_list);
D_INIT_LIST_HEAD(&cpr->cpr_pending_list);
cpr->cpr_refs = 1;
Expand Down Expand Up @@ -498,7 +499,8 @@ chk_pool_stop_one(struct chk_instance *ins, uuid_t uuid, int status, uint32_t ph
}

if (!ins->ci_is_leader &&
cpr->cpr_bk.cb_pool_status != CHK__CHECK_POOL_STATUS__CPS_CHECKED)
(cpr->cpr_bk.cb_pool_status != CHK__CHECK_POOL_STATUS__CPS_CHECKED ||
cpr->cpr_not_export_ps))
chk_pool_shutdown(cpr, false);

/* Drop the reference that is held when create in chk_pool_alloc(). */
Expand All @@ -509,6 +511,26 @@ chk_pool_stop_one(struct chk_instance *ins, uuid_t uuid, int status, uint32_t ph
*ret = rc;
}

void
chk_pool_stop_all(struct chk_instance *ins, uint32_t status, int *ret)
{
struct chk_pool_rec *cpr;
struct chk_pool_rec *tmp;

/*
* Hold reference on each before stop one to guarantee that the next
* 'tmp' will not be unlinked from the list during stop current cpr.
*/
d_list_for_each_entry(cpr, &ins->ci_pool_list, cpr_link)
chk_pool_get(cpr);

d_list_for_each_entry_safe(cpr, tmp, &ins->ci_pool_list, cpr_link) {
if (ret == NULL || *ret == 0)
chk_pool_stop_one(ins, cpr->cpr_uuid, status, CHK_INVAL_PHASE, ret);
chk_pool_put(cpr);
}
}

int
chk_pools_pause_cb(struct sys_db *db, char *table, d_iov_t *key, void *args)
{
Expand Down Expand Up @@ -771,10 +793,15 @@ chk_pool_handle_notify(struct chk_instance *ins, struct chk_iv *iv)
if (cpr->cpr_stop || unlikely(iv->ci_phase < cbk->cb_phase))
D_GOTO(out, rc = -DER_NOTAPPLICABLE);

if (cpr->cpr_done)
goto out;

if (iv->ci_pool_status == CHK__CHECK_POOL_STATUS__CPS_CHECKED) {
cpr->cpr_done = 1;
if (iv->ci_pool_destroyed)
if (iv->ci_pool_destroyed) {
cpr->cpr_destroyed = 1;
cpr->cpr_not_export_ps = 1;
}
} else if (iv->ci_pool_status == CHK__CHECK_POOL_STATUS__CPS_FAILED ||
iv->ci_pool_status == CHK__CHECK_POOL_STATUS__CPS_IMPLICATED) {
cpr->cpr_skip = 1;
Expand All @@ -784,6 +811,16 @@ chk_pool_handle_notify(struct chk_instance *ins, struct chk_iv *iv)
D_GOTO(out, rc = -DER_NOTAPPLICABLE);
}

if (!ins->ci_is_leader && !cpr->cpr_destroyed && cpr->cpr_done) {
if (iv->ci_pool_status == CHK__CHECK_POOL_STATUS__CPS_CHECKED &&
!cpr->cpr_not_export_ps) {
chk_pool_start_svc(cpr, NULL);
} else if (ins->ci_sched_running && !ins->ci_sched_exiting) {
chk_pool_get(cpr);
d_list_add_tail(&cpr->cpr_shutdown_link, &ins->ci_pool_shutdown_list);
}
}

if (iv->ci_phase != cbk->cb_phase || iv->ci_pool_status != cbk->cb_pool_status ||
cpr->cpr_destroyed) {
cbk->cb_phase = iv->ci_phase;
Expand All @@ -792,10 +829,6 @@ chk_pool_handle_notify(struct chk_instance *ins, struct chk_iv *iv)
rc = chk_bk_update_pool(cbk, uuid_str);
}

if (rc == 0 && !ins->ci_is_leader && !cpr->cpr_destroyed &&
cbk->cb_pool_status == CHK__CHECK_POOL_STATUS__CPS_CHECKED)
chk_pool_start_svc(cpr, NULL);

out:
if (cpr != NULL)
chk_pool_put(cpr);
Expand Down Expand Up @@ -1173,6 +1206,7 @@ chk_ins_init(struct chk_instance **p_ins)
D_INIT_LIST_HEAD(&ins->ci_pool_list);

ins->ci_pending_hdl = DAOS_HDL_INVAL;
D_INIT_LIST_HEAD(&ins->ci_pool_shutdown_list);

rc = ABT_rwlock_create(&ins->ci_abt_lock);
if (rc != ABT_SUCCESS)
Expand Down Expand Up @@ -1227,6 +1261,7 @@ chk_ins_fini(struct chk_instance **p_ins)
D_ASSERT(d_list_empty(&ins->ci_pool_list));

D_ASSERT(daos_handle_is_inval(ins->ci_pending_hdl));
D_ASSERT(d_list_empty(&ins->ci_pool_shutdown_list));

if (ins->ci_sched != ABT_THREAD_NULL)
ABT_thread_free(&ins->ci_sched);
Expand Down
63 changes: 36 additions & 27 deletions src/chk/chk_engine.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -168,12 +168,18 @@ chk_engine_exit(struct chk_instance *ins, uint32_t ins_phase, uint32_t ins_statu
{
struct chk_bookmark *cbk = &ins->ci_bk;
struct chk_pool_rec *cpr;
struct chk_pool_rec *tmp;
struct chk_iv iv = { 0 };
int rc;

d_list_for_each_entry_safe(cpr, tmp, &ins->ci_pool_list, cpr_link)
chk_pool_stop_one(ins, cpr->cpr_uuid, pool_status, CHK_INVAL_PHASE, NULL);
ins->ci_sched_exiting = 1;

while ((cpr = d_list_pop_entry(&ins->ci_pool_shutdown_list, struct chk_pool_rec,
cpr_shutdown_link)) != NULL) {
chk_pool_shutdown(cpr, false);
chk_pool_put(cpr);
}

chk_pool_stop_all(ins, pool_status, NULL);

chk_destroy_pending_tree(ins);
chk_destroy_pool_tree(ins);
Expand All @@ -193,7 +199,7 @@ chk_engine_exit(struct chk_instance *ins, uint32_t ins_phase, uint32_t ins_statu
ins_status != CHK__CHECK_INST_STATUS__CIS_STOPPED &&
ins_status != CHK__CHECK_INST_STATUS__CIS_IMPLICATED && ins->ci_iv_ns != NULL) {
if (DAOS_FAIL_CHECK(DAOS_CHK_PS_NOTIFY_LEADER))
return;
goto out;

iv.ci_gen = cbk->cb_gen;
iv.ci_phase = cbk->cb_phase;
Expand All @@ -207,6 +213,9 @@ chk_engine_exit(struct chk_instance *ins, uint32_t ins_phase, uint32_t ins_statu
DF_ENGINE" on rank %u notify leader for its exit, status %u: rc = %d\n",
DP_ENGINE(ins), dss_self_rank(), ins_status, rc);
}

out:
ins->ci_sched_exiting = 0;
}

static int
Expand Down Expand Up @@ -1795,7 +1804,8 @@ chk_engine_pool_ult(void *args)
if (likely(update))
rc1 = chk_bk_update_pool(cbk, uuid_str);

if (cbk->cb_pool_status == CHK__CHECK_POOL_STATUS__CPS_CHECKED) {
if (cbk->cb_pool_status == CHK__CHECK_POOL_STATUS__CPS_CHECKED &&
!cpr->cpr_not_export_ps) {
chk_pool_start_svc(cpr, &rc2);
if (cpr->cpr_started && cpr->cpr_start_post)
/*
Expand All @@ -1813,7 +1823,11 @@ chk_engine_pool_ult(void *args)

ds_pool_svc_put_leader(svc);
cpr->cpr_done = 1;
chk_pool_put(cpr);
if (ins->ci_sched_running && !ins->ci_sched_exiting &&
(cbk->cb_pool_status != CHK__CHECK_POOL_STATUS__CPS_CHECKED || cpr->cpr_not_export_ps))
d_list_add_tail(&cpr->cpr_shutdown_link, &ins->ci_pool_shutdown_list);
else
chk_pool_put(cpr);
D_FREE(cpma);
}

Expand All @@ -1822,6 +1836,7 @@ chk_engine_sched(void *args)
{
struct chk_instance *ins = args;
struct chk_bookmark *cbk = &ins->ci_bk;
struct chk_pool_rec *cpr;
uint32_t ins_phase;
uint32_t ins_status;
uint32_t pool_status;
Expand Down Expand Up @@ -1868,6 +1883,12 @@ chk_engine_sched(void *args)
if (rc != 0)
goto out;
}

while ((cpr = d_list_pop_entry(&ins->ci_pool_shutdown_list, struct chk_pool_rec,
cpr_shutdown_link)) != NULL) {
chk_pool_shutdown(cpr, false);
chk_pool_put(cpr);
}
}

out:
Expand Down Expand Up @@ -2128,8 +2149,6 @@ chk_engine_start(uint64_t gen, uint32_t rank_nr, d_rank_t *ranks, uint32_t polic
{
struct chk_instance *ins = chk_engine;
struct chk_bookmark *cbk = &ins->ci_bk;
struct chk_pool_rec *cpr;
struct chk_pool_rec *tmp;
struct umem_attr uma = { 0 };
char uuid_str[DAOS_UUID_STR_SIZE];
d_rank_t myrank = dss_self_rank();
Expand Down Expand Up @@ -2219,9 +2238,7 @@ chk_engine_start(uint64_t gen, uint32_t rank_nr, d_rank_t *ranks, uint32_t polic
goto out_done;

out_stop:
d_list_for_each_entry_safe(cpr, tmp, &ins->ci_pool_list, cpr_link)
chk_pool_stop_one(ins, cpr->cpr_uuid, CHK__CHECK_POOL_STATUS__CPS_IMPLICATED,
CHK_INVAL_PHASE, NULL);
chk_pool_stop_all(ins, CHK__CHECK_POOL_STATUS__CPS_IMPLICATED, NULL);
if (cbk->cb_ins_status == CHK__CHECK_INST_STATUS__CIS_RUNNING) {
cbk->cb_time.ct_stop_time = time(NULL);
cbk->cb_ins_status = CHK__CHECK_INST_STATUS__CIS_FAILED;
Expand Down Expand Up @@ -2265,8 +2282,6 @@ chk_engine_stop(uint64_t gen, int pool_nr, uuid_t pools[], uint32_t *flags)
{
struct chk_instance *ins = chk_engine;
struct chk_bookmark *cbk = &ins->ci_bk;
struct chk_pool_rec *cpr;
struct chk_pool_rec *tmp;
d_rank_t myrank = dss_self_rank();
int rc = 0;
int i;
Expand All @@ -2291,12 +2306,9 @@ chk_engine_stop(uint64_t gen, int pool_nr, uuid_t pools[], uint32_t *flags)
D_INFO(DF_ENGINE" stopping on rank %u with %d pools\n", DP_ENGINE(ins), myrank, pool_nr);

if (pool_nr == 0) {
d_list_for_each_entry_safe(cpr, tmp, &ins->ci_pool_list, cpr_link) {
chk_pool_stop_one(ins, cpr->cpr_uuid, CHK__CHECK_POOL_STATUS__CPS_STOPPED,
CHK_INVAL_PHASE, &rc);
if (rc != 0)
D_GOTO(out, rc);
}
chk_pool_stop_all(ins, CHK__CHECK_POOL_STATUS__CPS_STOPPED, &rc);
if (rc != 0)
D_GOTO(out, rc);
} else {
for (i = 0; i < pool_nr; i++) {
chk_pool_stop_one(ins, pools[i], CHK__CHECK_POOL_STATUS__CPS_STOPPED,
Expand Down Expand Up @@ -2630,9 +2642,7 @@ chk_engine_act(uint64_t gen, uint64_t seq, uint32_t cla, uint32_t act, uint32_t

if (likely(prop->cp_policies[cla] != act)) {
prop->cp_policies[cla] = act;
rc = chk_prop_update(prop, NULL);
if (rc != 0)
goto out;
chk_prop_update(prop, NULL);
}

/*
Expand Down Expand Up @@ -2884,6 +2894,9 @@ chk_engine_pool_start(uint64_t gen, uuid_t uuid, uint32_t phase, uint32_t flags)
if (unlikely(cpr->cpr_started))
D_GOTO(out, rc = -DER_ALREADY);

if (flags & CPSF_NOT_EXPORT_PS)
cpr->cpr_not_export_ps = 1;

cbk = &cpr->cpr_bk;
chk_pool_get(cpr);

Expand Down Expand Up @@ -3240,8 +3253,6 @@ chk_engine_rejoin(void *args)
struct chk_instance *ins = chk_engine;
struct chk_property *prop = &ins->ci_prop;
struct chk_bookmark *cbk = &ins->ci_bk;
struct chk_pool_rec *cpr;
struct chk_pool_rec *tmp;
uuid_t *pools = NULL;
struct chk_iv iv = { 0 };
struct umem_attr uma = { 0 };
Expand Down Expand Up @@ -3349,9 +3360,7 @@ chk_engine_rejoin(void *args)
goto out_log;

out_stop:
d_list_for_each_entry_safe(cpr, tmp, &ins->ci_pool_list, cpr_link)
chk_pool_stop_one(ins, cpr->cpr_uuid, CHK__CHECK_POOL_STATUS__CPS_IMPLICATED,
CHK_INVAL_PHASE, NULL);
chk_pool_stop_all(ins, CHK__CHECK_POOL_STATUS__CPS_IMPLICATED, NULL);
if (cbk->cb_ins_status == CHK__CHECK_INST_STATUS__CIS_RUNNING) {
cbk->cb_time.ct_stop_time = time(NULL);
cbk->cb_ins_status = CHK__CHECK_INST_STATUS__CIS_FAILED;
Expand Down
38 changes: 18 additions & 20 deletions src/chk/chk_internal.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -354,7 +354,10 @@ enum chk_mbs_flags {
};

enum chk_pool_start_flags {
/* The pool is not in check list, but it is reported by engine for potential orphan pool. */
CPSF_FOR_ORPHAN = 1,
/* Do not export pool service after check done. */
CPSF_NOT_EXPORT_PS = 2,
};

enum chk_rejoin_flags {
Expand Down Expand Up @@ -518,6 +521,8 @@ struct chk_instance {
struct btr_root ci_pending_btr;
daos_handle_t ci_pending_hdl;

d_list_t ci_pool_shutdown_list;

/* The slowest phase for the failed pool or rank. */
uint32_t ci_slowest_fail_phase;

Expand All @@ -540,6 +545,7 @@ struct chk_instance {

uint32_t ci_is_leader:1,
ci_sched_running:1,
ci_sched_exiting:1,
ci_for_orphan:1,
ci_orphan_done:1, /* leader has processed orphan pools. */
ci_pool_stopped:1, /* check on some pools have been stopped. */
Expand Down Expand Up @@ -577,6 +583,8 @@ struct chk_pool_shard {
struct chk_pool_rec {
/* Link into chk_instance::ci_pool_list. */
d_list_t cpr_link;
/* Link into chk_instance::ci_pool_shutdown_list. */
d_list_t cpr_shutdown_link;
/* The list of chk_pool_shard. */
d_list_t cpr_shard_list;
/* The list of chk_pending_rec. */
Expand All @@ -594,6 +602,7 @@ struct chk_pool_rec {
cpr_healthy:1,
cpr_delay_label:1,
cpr_exist_on_ms:1,
cpr_not_export_ps:1,
cpr_map_refreshed:1;
int cpr_advice;
int cpr_refs;
Expand Down Expand Up @@ -689,6 +698,8 @@ void chk_pool_start_svc(struct chk_pool_rec *cpr, int *ret);

void chk_pool_stop_one(struct chk_instance *ins, uuid_t uuid, int status, uint32_t phase, int *ret);

void chk_pool_stop_all(struct chk_instance *ins, uint32_t status, int *ret);

int chk_pools_pause_cb(struct sys_db *db, char *table, d_iov_t *key, void *args);

int chk_pools_cleanup_cb(struct sys_db *db, char *table, d_iov_t *key, void *args);
Expand Down Expand Up @@ -1002,6 +1013,7 @@ chk_pool_put(struct chk_pool_rec *cpr)
d_list_del(&cpr->cpr_link);
D_ASSERT(cpr->cpr_thread == ABT_THREAD_NULL);
D_ASSERT(d_list_empty(&cpr->cpr_pending_list));
D_ASSERT(d_list_empty(&cpr->cpr_shutdown_link));

while ((cps = d_list_pop_entry(&cpr->cpr_shard_list, struct chk_pool_shard,
cps_link)) != NULL) {
Expand Down Expand Up @@ -1042,11 +1054,15 @@ chk_pool_shutdown(struct chk_pool_rec *cpr, bool locked)

d_iov_set(&psid, cpr->cpr_uuid, sizeof(uuid_t));
rc = ds_rsvc_stop(DS_RSVC_CLASS_POOL, &psid, RDB_NIL_TERM, false);
D_DEBUG(DB_MD, "Stop PS for "DF_UUIDF": "DF_RC"\n",
D_DEBUG(DB_MD, "Shutdown PS for "DF_UUIDF": "DF_RC"\n",
DP_UUID(cpr->cpr_uuid), DP_RC(rc));
cpr->cpr_start_post = 0;

ds_pool_stop(cpr->cpr_uuid);
cpr->cpr_started = 0;
cpr->cpr_start_post = 0;

D_DEBUG(DB_MD, "Stop pool for "DF_UUIDF" with locked %s\n",
DP_UUID(cpr->cpr_uuid), locked ? "true" : "false");

if (!locked)
ABT_mutex_unlock(cpr->cpr_mutex);
Expand All @@ -1070,24 +1086,6 @@ chk_pool_in_zombie(struct chk_pool_rec *cpr)
return found;
}

static inline int
chk_pool_has_err(struct chk_pool_rec *cpr)
{
struct chk_pool_shard *cps;
struct ds_pool_clue *clue;
int rc = 0;

d_list_for_each_entry(cps, &cpr->cpr_shard_list, cps_link) {
clue = cps->cps_data;
if (clue->pc_rc < 0) {
rc = clue->pc_rc;
break;
}
}

return rc >= 0 ? 0 : rc;
}

static inline int
chk_pools_add_from_dir(uuid_t uuid, void *args)
{
Expand Down
Loading

0 comments on commit 2fcf848

Please sign in to comment.