DAOS-16838 control: Fix dmg storage query usage with emulated NVMe

Features: control Required-githooks: true Signed-off-by: Tom Nabarro <[email protected]>
daos-stack · Dec 2, 2024 · c09dc7b · c09dc7b
1 parent eb1d707
commit c09dc7b
Show file tree

Hide file tree

Showing 2 changed files with 92 additions and 5 deletions.
diff --git a/src/control/server/instance_storage_rpc.go b/src/control/server/instance_storage_rpc.go
@@ -332,6 +332,7 @@ func scanEngineBdevsOverDrpc(ctx context.Context, engine Engine, pbReq *ctlpb.Sc
 	if scanSmdResp == nil {
 		return nil, errors.New("nil smd scan resp")
 	}
+	engine.Tracef("smd scan devices: %+v", scanSmdResp.Devices)
 
 	// Re-link SMD devices inside NVMe controller structures and populate scan response.
 
@@ -340,12 +341,16 @@ func scanEngineBdevsOverDrpc(ctx context.Context, engine Engine, pbReq *ctlpb.Sc
 	}
 	seenCtrlrs := make(map[string]*ctlpb.NvmeController)
 
-	for _, sd := range scanSmdResp.Devices {
+	for i, sd := range scanSmdResp.Devices {
 		if sd.Ctrlr == nil {
 			return nil, errors.Errorf("smd %q has no ctrlr ref", sd.Uuid)
 		}
 
 		addr := sd.Ctrlr.PciAddr
+		if addr == "" {
+			// Mock identifier for emulated NVMe mode where devices have no PCI-address.
+			addr = fmt.Sprintf("0000:00:0.%d", i)
+		}
 
 		if _, exists := seenCtrlrs[addr]; !exists {
 			c := new(ctlpb.NvmeController)
@@ -460,6 +465,7 @@ func bdevScanEngineAssigned(ctx context.Context, engine Engine, req *ctlpb.ScanN
 	return scanEngineBdevsOverDrpc(ctx, engine, req)
 }
 
+// Accommodate for VMD backing devices and emulated NVMe (AIO).
 func getEffCtrlrCount(ctrlrs []*ctlpb.NvmeController) (int, error) {
 	pas := hardware.MustNewPCIAddressSet()
 	for _, c := range ctrlrs {
@@ -471,11 +477,13 @@ func getEffCtrlrCount(ctrlrs []*ctlpb.NvmeController) (int, error) {
 		if npas, err := pas.BackingToVMDAddresses(); err != nil {
 			return 0, err
 		} else {
-			pas = npas
+			return npas.Len(), nil
 		}
 	}
 
-	return pas.Len(), nil
+	// Return inputted number of controllers rather than number of parsed addresses to cater for
+	// the case of emulated NVMe where there will be no valid PCI address.
+	return len(ctrlrs), nil
 }
 
 // bdevScanEngine calls either in to the private engine storage provider to scan bdevs if engine process

diff --git a/src/control/server/instance_storage_rpc_test.go b/src/control/server/instance_storage_rpc_test.go
@@ -44,7 +44,7 @@ func (mp *mockPCIeLinkStatsProvider) PCIeCapsFromConfig(cfgBytes []byte, dev *ha
 	return nil
 }
 
-func TestIOEngineInstance_populateCtrlrHealth(t *testing.T) {
+func TestServer_populateCtrlrHealth(t *testing.T) {
 	healthWithLinkStats := func(maxSpd, spd float32, maxWdt, wdt uint32) *ctlpb.BioHealthResp {
 		bhr := proto.MockNvmeHealth()
 		bhr.LinkMaxSpeed = maxSpd
@@ -459,7 +459,7 @@ func TestIOEngineInstance_populateCtrlrHealth(t *testing.T) {
 	}
 }
 
-func TestIOEngineInstance_bdevScanEngine(t *testing.T) {
+func TestServer_bdevScanEngine(t *testing.T) {
 	c := storage.MockNvmeController(2)
 	withState := func(ctrlr *ctlpb.NvmeController, state ctlpb.NvmeDevState) *ctlpb.NvmeController {
 		ctrlr.DevState = state
@@ -793,6 +793,85 @@ func TestIOEngineInstance_bdevScanEngine(t *testing.T) {
 			// Prove link stat provider gets called without Meta flag.
 			expErr: errors.New("link stats provider fail"),
 		},
+		"scan over drpc; aio file emulated nvme; no pci addresses": {
+			smdRes: &ctlpb.SmdDevResp{
+				Devices: []*ctlpb.SmdDevice{
+					{
+						Uuid: "b80b4653-af58-47b3-aa3b-8ad12965f440",
+						TgtIds: []int32{
+							1024, 1024, 0, 0, 0, 4, 4, 4, 8, 8, 8, 12,
+							12, 12,
+						},
+						RoleBits: 7,
+						Ctrlr: &ctlpb.NvmeController{
+							DevState: ctlpb.NvmeDevState_NORMAL,
+						},
+					},
+					{
+						Uuid: "3c7a8e22-38c0-4e6f-8776-d703d049ae6f",
+						TgtIds: []int32{
+							1024, 1024, 1, 1, 1, 5, 5, 5, 9, 9, 9, 13,
+							13, 13,
+						},
+						RoleBits: 7,
+						Ctrlr: &ctlpb.NvmeController{
+							DevState: ctlpb.NvmeDevState_NORMAL,
+						},
+					},
+					{
+						Uuid: "40ba7b18-3a0e-4d68-9e8f-c4fc556eb506",
+						TgtIds: []int32{
+							1024, 1024, 2, 2, 2, 6, 6, 6, 10, 10, 10,
+							14, 14, 14,
+						},
+						RoleBits: 7,
+						Ctrlr: &ctlpb.NvmeController{
+							DevState: ctlpb.NvmeDevState_NORMAL,
+						},
+					},
+					{
+						Uuid: "d78c849f-fd9e-4a20-9746-b0335e3618da",
+						TgtIds: []int32{
+							1024, 1024, 3, 3, 3, 7, 7, 7, 11, 11, 11,
+							15, 15, 15,
+						},
+						RoleBits: 7,
+						Ctrlr: &ctlpb.NvmeController{
+							DevState: ctlpb.NvmeDevState_NORMAL,
+						},
+					},
+				},
+			},
+			expResp: &ctlpb.ScanNvmeResp{
+				Ctrlrs: proto.NvmeControllers{
+					&ctlpb.NvmeController{
+						DevState: ctlpb.NvmeDevState_NORMAL,
+						SmdDevices: []*ctlpb.SmdDevice{
+							{RoleBits: 7},
+						},
+					},
+					&ctlpb.NvmeController{
+						DevState: ctlpb.NvmeDevState_NORMAL,
+						SmdDevices: []*ctlpb.SmdDevice{
+							{RoleBits: 7},
+						},
+					},
+					&ctlpb.NvmeController{
+						DevState: ctlpb.NvmeDevState_NORMAL,
+						SmdDevices: []*ctlpb.SmdDevice{
+							{RoleBits: 7},
+						},
+					},
+					&ctlpb.NvmeController{
+						DevState: ctlpb.NvmeDevState_NORMAL,
+						SmdDevices: []*ctlpb.SmdDevice{
+							{RoleBits: 7},
+						},
+					},
+				},
+				State: new(ctlpb.ResponseState),
+			},
+		},
 	} {
 		t.Run(name, func(t *testing.T) {
 			log, buf := logging.NewTestLogger(t.Name())