DAOS-16477 mgmt: return suspect engines for pool healthy query (#15458)

docs/admin/pool_operations.md

@@ -286,6 +286,28 @@ The example below shows a rebuild in progress and NVMe space allocated.
     Rebuild busy, 75 objs, 9722 recs
 ```
 
+After experiencing significant failures, the pool may retain some suspect
+engines that have been marked as DEAD by the SWIM protocol but were not excluded
+from the pool to prevent potential data inconsistency. An administrator can bring
+these engines back online by restarting them. The example below illustrates the
+system’s status with suspect and disabled engines.
+
+```bash
+$ dmg pool query tank -t
+```
+
+NB: The --health-only/-t option is necessary to conduct pool health-related queries only.
+This is important because suspect ranks may cause commands to hang and timeout so identifying
+and restarting them is a useful procedure.
+
+```bash
+Pool 6f450a68-8c7d-4da9-8900-02691650f6a2, ntarget=8, disabled=2, leader=3, version=4, state=Degraded
+    Pool health info:
+    - Disabled ranks: 1
+    - Suspect ranks: 2
+    - Rebuild busy, 0 objs, 0 recs
+```
+
 Additional status and telemetry data is planned to be exported through
 management tools and will be documented here once available.
 

src/control/cmd/daos/health.go

@@ -95,24 +95,19 @@ func (cmd *healthCheckCmd) Execute([]string) error {
 			}
 		}()
 
-		queryMask := daos.MustNewPoolQueryMask(daos.PoolQueryOptionEnabledEngines)
+		queryMask := daos.MustNewPoolQueryMask(daos.PoolQueryOptionEnabledEngines,
+			daos.PoolQueryOptionSuspectEngines)
+		if pool.DisabledTargets > 0 {
+			queryMask.SetOptions(daos.PoolQueryOptionDisabledEngines)
+		}
 		tpi, err := queryPool(poolHdl, queryMask)
 		if err != nil {
 			cmd.Errorf("failed to query pool %s: %v", pool.Label, err)
 			continue
 		}
 		pool.EnabledRanks = tpi.EnabledRanks
-
-		if pool.DisabledTargets > 0 {
-			queryMask.ClearAll()
-			queryMask.SetOptions(daos.PoolQueryOptionDisabledEngines)
-			tpi, err = queryPool(poolHdl, queryMask)
-			if err != nil {
-				cmd.Errorf("failed to query pool %s: %v", pool.Label, err)
-				continue
-			}
-			pool.DisabledRanks = tpi.DisabledRanks
-		}
+		pool.DisabledRanks = tpi.DisabledRanks
+		pool.SuspectRanks = tpi.SuspectRanks
 
 		poolConts, err := listContainers(poolHdl)
 		if err != nil {

src/control/cmd/daos/pool.go

@@ -296,11 +296,12 @@ func convertPoolInfo(pinfo *C.daos_pool_info_t) (*daos.PoolInfo, error) {
 	return poolInfo, nil
 }
 
-func queryPool(poolHdl C.daos_handle_t, queryMask daos.PoolQueryMask) (*daos.PoolInfo, error) {
+func queryPoolRankLists(poolHdl C.daos_handle_t, queryMask daos.PoolQueryMask) (*daos.PoolInfo, error) {
 	var rlPtr **C.d_rank_list_t = nil
 	var rl *C.d_rank_list_t = nil
 
-	if queryMask.HasOption(daos.PoolQueryOptionEnabledEngines) || queryMask.HasOption(daos.PoolQueryOptionDisabledEngines) {
+	if queryMask.HasOption(daos.PoolQueryOptionEnabledEngines) || queryMask.HasOption(daos.PoolQueryOptionDisabledEngines) ||
+		queryMask.HasOption(daos.PoolQueryOptionSuspectEngines) {
 		rlPtr = &rl
 	}
 
@@ -330,6 +331,68 @@ func queryPool(poolHdl C.daos_handle_t, queryMask daos.PoolQueryMask) (*daos.Poo
 		if queryMask.HasOption(daos.PoolQueryOptionDisabledEngines) {
 			poolInfo.DisabledRanks = rs
 		}
+		if queryMask.HasOption(daos.PoolQueryOptionSuspectEngines) {
+			poolInfo.SuspectRanks = rs
+		}
+	}
+
+	return poolInfo, nil
+}
+func queryPool(poolHdl C.daos_handle_t, queryMask daos.PoolQueryMask) (*daos.PoolInfo, error) {
+	poolInfo := &daos.PoolInfo{}
+	originalMask := queryMask // Save the original queryMask
+
+	// Function to handle the query and return a single RankList
+	queryAndUpdate := func(option string) error {
+		// Clear previous options and set new option
+		queryMask.ClearAll()
+		queryMask.SetOptions(option)
+
+		poolInfo1, err := queryPoolRankLists(poolHdl, queryMask)
+		if err != nil {
+			return err
+		}
+
+		switch option {
+		case daos.PoolQueryOptionEnabledEngines:
+			poolInfo.EnabledRanks = poolInfo1.EnabledRanks
+		case daos.PoolQueryOptionDisabledEngines:
+			poolInfo.DisabledRanks = poolInfo1.DisabledRanks
+		case daos.PoolQueryOptionSuspectEngines:
+			poolInfo.SuspectRanks = poolInfo1.SuspectRanks
+		}
+		return nil
+	}
+
+	// Preprocess queryMask, select one option for the first query
+	var firstOption string
+	if originalMask.HasOption(daos.PoolQueryOptionEnabledEngines) {
+		firstOption = daos.PoolQueryOptionEnabledEngines
+	} else if originalMask.HasOption(daos.PoolQueryOptionDisabledEngines) {
+		firstOption = daos.PoolQueryOptionDisabledEngines
+	} else if originalMask.HasOption(daos.PoolQueryOptionSuspectEngines) {
+		firstOption = daos.PoolQueryOptionSuspectEngines
+	}
+
+	// Perform the first query to get basic information
+	if err := queryAndUpdate(firstOption); err != nil {
+		return nil, err
+	}
+
+	// Check the original query mask and update fields as needed
+	queryOptions := []string{
+		daos.PoolQueryOptionEnabledEngines,
+		daos.PoolQueryOptionDisabledEngines,
+		daos.PoolQueryOptionSuspectEngines,
+	}
+
+	// Process each option sequentially
+	for _, opt := range queryOptions {
+		if originalMask.HasOption(opt) && opt != firstOption {
+			if err := queryAndUpdate(opt); err != nil {
+				return nil, err
+			}
+		}
 	}
 
 	return poolInfo, nil

src/control/cmd/daos/pretty/health.go

@@ -61,6 +61,13 @@ func printPoolHealth(out io.Writer, pi *daos.PoolInfo, verbose bool) {
 	}
 
 	var healthStrings []string
+	if pi.SuspectRanks != nil && pi.SuspectRanks.Count() > 0 {
+		degStr := "Suspect"
+		if verbose {
+			degStr += fmt.Sprintf(" %s", pi.SuspectRanks)
+		}
+		healthStrings = append(healthStrings, degStr)
+	}
 	if pi.DisabledTargets > 0 {
 		degStr := "Degraded"
 		if verbose {

src/control/cmd/daos/pretty/pool.go

@@ -53,6 +53,10 @@ func PrintPoolInfo(pi *daos.PoolInfo, out io.Writer) error {
 	if pi.DisabledRanks != nil && pi.DisabledRanks.Count() > 0 {
 		fmt.Fprintf(w, "- Disabled ranks: %s\n", pi.DisabledRanks)
 	}
+	if pi.QueryMask.HasOption(daos.PoolQueryOptionSuspectEngines) &&
+		pi.SuspectRanks != nil && pi.SuspectRanks.Count() > 0 {
+		fmt.Fprintf(w, "- Suspect ranks: %s\n", pi.SuspectRanks)
+	}
 	if pi.Rebuild != nil {
 		if pi.Rebuild.Status == 0 {
 			fmt.Fprintf(w, "- Rebuild %s, %d objs, %d recs\n",

src/control/cmd/daos/pretty/pool_test.go

@@ -122,6 +122,45 @@ Pool space info:
 - Storage tier 1 (NVMe):
   Total size: 2 B
   Free: 1 B, min:0 B, max:0 B, mean:0 B
+`, poolUUID.String()),
+		},
+		"normal response; suspect ranks": {
+			pi: &daos.PoolInfo{
+				QueryMask:        daos.HealthOnlyPoolQueryMask,
+				State:            daos.PoolServiceStateDegraded,
+				UUID:             poolUUID,
+				TotalTargets:     2,
+				DisabledTargets:  1,
+				ActiveTargets:    1,
+				ServiceLeader:    42,
+				Version:          100,
+				PoolLayoutVer:    1,
+				UpgradeLayoutVer: 2,
+				DisabledRanks:    ranklist.MustCreateRankSet("[0,1,3]"),
+				SuspectRanks:     ranklist.MustCreateRankSet("[2]"),
+				Rebuild: &daos.PoolRebuildStatus{
+					State:   daos.PoolRebuildStateBusy,
+					Objects: 42,
+					Records: 21,
+				},
+				TierStats: []*daos.StorageUsageStats{
+					{
+						Total: 2,
+						Free:  1,
+					},
+					{
+						Total: 2,
+						Free:  1,
+					},
+				},
+			},
+			expPrintStr: fmt.Sprintf(`
+Pool %s, ntarget=2, disabled=1, leader=42, version=100, state=Degraded
+Pool layout out of date (1 < 2) -- see `+backtickStr+` for details.
+Pool health info:
+- Disabled ranks: 0-1,3
+- Suspect ranks: 2
+- Rebuild busy, 42 objs, 21 recs
 `, poolUUID.String()),
 		},
 		"normal response; disabled ranks": {