Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

DAOS-12470 chk: misc fixes for chk related issues - I #11440

Merged
merged 1 commit into from
Feb 17, 2023
Merged

DAOS-12470 chk: misc fixes for chk related issues - I #11440

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 6 additions & 2 deletions src/chk/chk_common.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -391,6 +391,8 @@ chk_pool_start_svc(struct chk_pool_rec *cpr, int *ret)
{
int rc = 0;

ABT_mutex_lock(cpr->cpr_mutex);

if (!cpr->cpr_started) {
rc = ds_pool_start_with_svc(cpr->cpr_uuid);
if (rc == 0)
Expand All @@ -407,12 +409,14 @@ chk_pool_start_svc(struct chk_pool_rec *cpr, int *ret)
DF_UUIDF" after check: "DF_RC"\n",
DP_UUID(cpr->cpr_uuid), DP_RC(rc));
/* Failed to post handle pool start, have to stop it. */
chk_pool_shutdown(cpr);
chk_pool_shutdown(cpr, true);
} else {
cpr->cpr_start_post = 1;
}
}

ABT_mutex_unlock(cpr->cpr_mutex);

if (ret != NULL)
*ret = rc;
}
Expand Down Expand Up @@ -469,7 +473,7 @@ chk_pool_stop_one(struct chk_instance *ins, uuid_t uuid, int status, uint32_t ph
cpr->cpr_bk.cb_pool_status == CHK__CHECK_POOL_STATUS__CPS_CHECKED)
chk_pool_start_svc(cpr, &rc);
else if (cpr->cpr_started)
chk_pool_shutdown(cpr);
chk_pool_shutdown(cpr, false);

/* Drop the reference that is held when create in chk_pool_alloc(). */
chk_pool_put(cpr);
Expand Down
11 changes: 5 additions & 6 deletions src/chk/chk_engine.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -195,7 +195,7 @@ chk_engine_exit(struct chk_instance *ins, uint32_t ins_status, uint32_t pool_sta

/* Notify the leader that check instance exit on the engine. */
rc = chk_iv_update(ins->ci_iv_ns, &iv, CRT_IV_SHORTCUT_TO_ROOT,
CRT_IV_SYNC_EAGER, true);
CRT_IV_SYNC_NONE, true);
if (rc != 0)
D_ERROR(DF_ENGINE" on rank %u failed to notify leader for its exit, "
"status %u: "DF_RC"\n",
Expand Down Expand Up @@ -1614,7 +1614,7 @@ chk_engine_pool_notify(struct chk_pool_rec *cpr, struct ds_pool *pool)
iv.ci_to_leader = 1;
/* Synchronously notify the check leader with the new check status/phase. */
rc = chk_iv_update(ins->ci_iv_ns, &iv, CRT_IV_SHORTCUT_TO_ROOT,
CRT_IV_SYNC_EAGER, true);
CRT_IV_SYNC_NONE, true);
if (rc != 0)
D_WARN(DF_ENGINE" on rank %u failed to notify check leader for "
DF_UUIDF", phase %u, ins_status %u, pool_status %u: "DF_RC"\n",
Expand Down Expand Up @@ -1942,7 +1942,7 @@ chk_engine_start_prep(struct chk_instance *ins, uint32_t rank_nr, d_rank_t *rank
goto out;
}

if (d_list_empty(&ins->ci_pool_list) && !(flags & CHK__CHECK_FLAG__CF_ORPHAN_POOL))
if (d_list_empty(&ins->ci_pool_list) && !(api_flags & CHK__CHECK_FLAG__CF_ORPHAN_POOL))
D_GOTO(out, rc = 1);

goto init;
Expand Down Expand Up @@ -2061,7 +2061,7 @@ chk_engine_start_post(struct chk_instance *ins)
if (rc == 0) {
d_list_for_each_entry(cpr, &ins->ci_pool_list, cpr_link)
/* Shutdown former instance left opened pool. */
chk_pool_shutdown(cpr);
chk_pool_shutdown(cpr, false);
}
}

Expand Down Expand Up @@ -3170,8 +3170,7 @@ chk_engine_rejoin(void *args)
iv.ci_to_leader = 1;

/* Notify the leader that check instance exit on the engine. */
rc1 = chk_iv_update(ins->ci_iv_ns, &iv, CRT_IV_SHORTCUT_TO_ROOT,
CRT_IV_SYNC_EAGER, true);
rc1 = chk_iv_update(ins->ci_iv_ns, &iv, CRT_IV_SHORTCUT_TO_ROOT, CRT_IV_SYNC_NONE, true);
if (rc1 != 0)
D_ERROR(DF_ENGINE" on rank %u failed to notify leader for its exit, status %u: "
DF_RC"\n", DP_ENGINE(ins), myrank, cbk->cb_ins_status, DP_RC(rc1));
Expand Down
8 changes: 7 additions & 1 deletion src/chk/chk_internal.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -975,20 +975,26 @@ chk_pool_put(struct chk_pool_rec *cpr)
}

static inline void
chk_pool_shutdown(struct chk_pool_rec *cpr)
chk_pool_shutdown(struct chk_pool_rec *cpr, bool locked)
{
d_iov_t psid;
int rc;

D_ASSERT(cpr->cpr_refs > 0);

if (!locked)
ABT_mutex_lock(cpr->cpr_mutex);

d_iov_set(&psid, cpr->cpr_uuid, sizeof(uuid_t));
rc = ds_rsvc_stop(DS_RSVC_CLASS_POOL, &psid, RDB_NIL_TERM, false);
D_DEBUG(DB_MD, "Stop PS for "DF_UUIDF": "DF_RC"\n",
DP_UUID(cpr->cpr_uuid), DP_RC(rc));
ds_pool_stop(cpr->cpr_uuid);
cpr->cpr_started = 0;
cpr->cpr_start_post = 0;

if (!locked)
ABT_mutex_unlock(cpr->cpr_mutex);
}

static inline void
Expand Down
44 changes: 26 additions & 18 deletions src/chk/chk_iv.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -92,46 +92,50 @@ chk_iv_ent_update(struct ds_iv_entry *entry, struct ds_iv_key *key,
if (src_iv->ci_to_leader) {
/*
* The case of the check engine sending IV message to the check leader
* on the same rank has already been handled via chk_iv_update().
* on the same rank has already been handled via chk_iv_update(). Then
* only need to handle the case that the check leader resides on other
* rank (trigger RPC to the check leader - the IV parent via returning
* -DER_IVCB_FORWARD.
*/
D_ASSERTF(!chk_is_on_leader(src_iv->ci_gen, -1, false),
"Invalid IV forward path for gen "DF_X64"/"DF_X64", rank %u, "
"phase %u, status %d/%d, from_psl %s\n",
src_iv->ci_gen, src_iv->ci_seq, src_iv->ci_rank, src_iv->ci_phase,
src_iv->ci_ins_status, src_iv->ci_pool_status,
src_iv->ci_from_psl ? "yes" : "no");

/* Trigger RPC to the leader (IV parent) via returning -DER_IVCB_FORWARD. */
rc = -DER_IVCB_FORWARD;
} else {
/*
* If it is message to engine, then it may be triggered by check leader,
* but it also may be from the pool service leader to other pool shards.
* Return zero that will trigger IV_SYNC to other check engines.
*
* NOTE: Currently, IV refresh from root node is always direct to leaves,
* it does not need some internal nodes to forward. So here, if it
* is not for PS leader notification, then it must be triggered by
* the check leader.
*/
if (!src_iv->ci_from_psl)
D_ASSERTF(chk_is_on_leader(src_iv->ci_gen, -1, false),
"Invalid IV forward path for gen "DF_X64"/"DF_X64
", rank %u, phase %u, status %d/%d\n", src_iv->ci_gen,
src_iv->ci_seq, src_iv->ci_rank, src_iv->ci_phase,
src_iv->ci_ins_status, src_iv->ci_pool_status);

rc = 0;
}
} else if (src_iv->ci_to_leader) {
*dst_iv = *src_iv;
rc = chk_leader_notify(dst_iv);
} else {
/*
* We got an IV SYNC (refresh) RPC from some engine. But because the engine
* always set CRT_IV_SHORTCUT_TO_ROOT for sync, then this should not happen.
*/
D_ERROR("Got invalid IV SYNC with gen "DF_X64"/"DF_X64", rank %u, phase %u, "
"status %d/%d, to_leader %s, from_psl %s\n",
src_iv->ci_gen, src_iv->ci_seq, src_iv->ci_rank, src_iv->ci_phase,
src_iv->ci_ins_status, src_iv->ci_pool_status,
src_iv->ci_to_leader ? "yes" : "no", src_iv->ci_from_psl ? "yes" : "no");
rc = -DER_IO;
D_ASSERTF(src_iv->ci_to_leader,
"Got invalid IV SYNC with gen "DF_X64"/"DF_X64", rank %u, phase %u, "
"status %d/%d, to_leader no, from_psl %s\n",
src_iv->ci_gen, src_iv->ci_seq, src_iv->ci_rank, src_iv->ci_phase,
src_iv->ci_ins_status, src_iv->ci_pool_status,
src_iv->ci_from_psl ? "yes" : "no");
*dst_iv = *src_iv;
rc = chk_leader_notify(dst_iv);
}

return rc;
Expand All @@ -147,10 +151,13 @@ chk_iv_ent_refresh(struct ds_iv_entry *entry, struct ds_iv_key *key,
int rc = 0;

/*
* Do not need local refresh from the pool service leader.
* But we need local refresh from the check leader on the same rank.
* For the notification from pool service leader, skip the local pool shard that will
* be handled by the pool service leader (including the @cpr status and pool service).
*
* For the notification from the check leader to check engines, do not skip the local
* check engine.
*/
if (src_iv->ci_rank != dss_self_rank()) {
if (!src_iv->ci_to_leader && (src_iv->ci_rank != dss_self_rank() || !src_iv->ci_from_psl)) {
*dst_iv = *src_iv;
rc = chk_engine_notify(dst_iv);
}
Expand Down Expand Up @@ -206,8 +213,9 @@ chk_iv_update(void *ns, struct chk_iv *iv, uint32_t shortcut, uint32_t sync_mode
}

D_CDEBUG(rc != 0, DLOG_ERR, DLOG_INFO,
"CHK iv "DF_X64"/"DF_X64" on rank %u: "DF_RC"\n",
iv->ci_gen, iv->ci_seq, iv->ci_rank, DP_RC(rc));
"CHK iv "DF_X64"/"DF_X64" on rank %u, to_leader %s, from_psl %s: "DF_RC"\n",
iv->ci_gen, iv->ci_seq, iv->ci_rank, iv->ci_to_leader ? "yes" : "no",
iv->ci_from_psl ? "yes" : "no", DP_RC(rc));

return rc;
}
Expand Down
8 changes: 7 additions & 1 deletion src/pool/srv_target.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -198,6 +198,9 @@ start_gc_ult(struct ds_pool_child *child)
D_ASSERT(child != NULL);
D_ASSERT(child->spc_gc_req == NULL);

D_DEBUG(DB_MGMT, DF_UUID"[%d]: starting GC ULT\n",
DP_UUID(child->spc_uuid), dmi->dmi_tgt_id);

sched_req_attr_init(&attr, SCHED_REQ_GC, &child->spc_uuid);
attr.sra_flags = SCHED_REQ_FL_NO_DELAY;

Expand Down Expand Up @@ -830,6 +833,9 @@ ds_pool_chk_post(uuid_t uuid)

D_ASSERT(dss_get_module_info()->dmi_xs_id == 0);

D_DEBUG(DB_MGMT, "Post handle pool starting for "DF_UUIDF" after DAOS check: "DF_RC"\n",
DP_UUID(uuid), DP_RC(rc));

/* The pool must has been opened. */
rc = daos_lru_ref_hold(pool_cache, (void *)uuid, sizeof(uuid_t),
NULL /* create_args */, &llink);
Expand All @@ -848,7 +854,7 @@ ds_pool_chk_post(uuid_t uuid)
daos_lru_ref_release(pool_cache, &pool->sp_entry);

D_CDEBUG(rc != 0, DLOG_ERR, DLOG_INFO,
"Post handle pool start for "DF_UUIDF" after DAOS check: "DF_RC"\n",
"Post handle pool started for "DF_UUIDF" after DAOS check: "DF_RC"\n",
DP_UUID(uuid), DP_RC(rc));

return rc;
Expand Down