diff --git a/src/control/server/ctl_storage_rpc.go b/src/control/server/ctl_storage_rpc.go index 8076487f272..832dee3eab7 100644 --- a/src/control/server/ctl_storage_rpc.go +++ b/src/control/server/ctl_storage_rpc.go @@ -362,23 +362,6 @@ func (cs *ControlService) scanScm(ctx context.Context, req *ctlpb.ScanScmReq) (* return resp, nil } -// Returns the engine configuration managing the given NVMe controller -func (cs *ControlService) getEngineCfgFromNvmeCtl(nc *ctlpb.NvmeController) (*engine.Config, error) { - pciAddrStr, err := ctrlrToPciStr(nc) - if err != nil { - return nil, err - } - - for index := range cs.srvCfg.Engines { - if findBdevTier(pciAddrStr, cs.srvCfg.Engines[index].Storage.Tiers) != nil { - return cs.srvCfg.Engines[index], nil - } - } - - return nil, errors.Errorf("unknown PCI device, scanned ctrlr %q not found in cfg", - pciAddrStr) -} - // Returns the engine configuration managing the given SCM name-space func (cs *ControlService) getEngineCfgFromScmNsp(nsp *ctlpb.ScmNamespace) (*engine.Config, error) { mountPoint := nsp.GetMount().Path @@ -531,7 +514,7 @@ func getClusterCount(sizeBytes uint64, tgtCount int, clusterSize uint64) uint64 return clusterCount * uint64(tgtCount) } -func (cs *ControlService) getMetaClusterCount(engineCfg *engine.Config, devToAdjust deviceToAdjust) (subtrClusterCount uint64) { +func (cs *ControlService) getMetaClusterCount(devToAdjust deviceToAdjust) (subtrClusterCount uint64) { dev := devToAdjust.ctlr.GetSmdDevices()[devToAdjust.idx] clusterSize := uint64(dev.GetClusterSize()) // Calculate MD cluster overhead based on the number of targets allocated to the device @@ -540,15 +523,21 @@ func (cs *ControlService) getMetaClusterCount(engineCfg *engine.Config, devToAdj if dev.GetRoleBits()&storage.BdevRoleMeta != 0 { clusterCount := getClusterCount(dev.GetMetaSize(), devTgtCount, clusterSize) - cs.log.Tracef("Removing %d Metadata clusters (cluster size: %d, dev tgts: %d) from the usable size of the SMD device %s (rank %d, ctlr %s): ", - clusterCount, clusterSize, devTgtCount, dev.GetUuid(), devToAdjust.rank, devToAdjust.ctlr.GetPciAddr()) + cs.log.Tracef("Removing %d Metadata clusters (meta_size: %s, cluster size: %s, dev tgts: %d - "+ + "%s) from the usable size of the SMD device %s (rank %d, ctlr %q): ", clusterCount, + humanize.IBytes(dev.GetMetaSize()), humanize.IBytes(clusterSize), devTgtCount, + humanize.IBytes(clusterSize*clusterCount), dev.GetUuid(), devToAdjust.rank, + devToAdjust.ctlr.GetPciAddr()) subtrClusterCount += clusterCount } if dev.GetRoleBits()&storage.BdevRoleWAL != 0 { clusterCount := getClusterCount(dev.GetMetaWalSize(), devTgtCount, clusterSize) - cs.log.Tracef("Removing %d Metadata WAL clusters (cluster size: %d, dev tgts: %d) from the usable size of the SMD device %s (rank %d, ctlr %s): ", - clusterCount, clusterSize, devTgtCount, dev.GetUuid(), devToAdjust.rank, devToAdjust.ctlr.GetPciAddr()) + cs.log.Tracef("Removing %d Metadata WAL clusters (meta_wal_size: %s, cluster size: %s, "+ + "dev tgts: %d - %s) from the usable size of the SMD device %s (rank %d, ctlr %q): ", + clusterCount, humanize.IBytes(dev.GetMetaWalSize()), humanize.IBytes(clusterSize), + devTgtCount, humanize.IBytes(clusterSize*clusterCount), dev.GetUuid(), devToAdjust.rank, + devToAdjust.ctlr.GetPciAddr()) subtrClusterCount += clusterCount } @@ -558,15 +547,21 @@ func (cs *ControlService) getMetaClusterCount(engineCfg *engine.Config, devToAdj if dev.GetRoleBits()&storage.BdevRoleMeta != 0 { clusterCount := getClusterCount(dev.GetRdbSize(), 1, clusterSize) - cs.log.Tracef("Removing %d RDB clusters (cluster size: %d) from the usable size of the SMD device %s (rank %d, ctlr %s)", - clusterCount, clusterSize, dev.GetUuid(), devToAdjust.rank, devToAdjust.ctlr.GetPciAddr()) + cs.log.Tracef("Removing %d RDB clusters (rdb_size: %s, cluster size: %s - %s) from the usable "+ + "size of the SMD device %s (rank %d, ctlr %q): ", clusterCount, + humanize.IBytes(dev.GetRdbSize()), humanize.IBytes(clusterSize), + humanize.IBytes(clusterSize*clusterCount), dev.GetUuid(), devToAdjust.rank, + devToAdjust.ctlr.GetPciAddr()) subtrClusterCount += clusterCount } if dev.GetRoleBits()&storage.BdevRoleWAL != 0 { clusterCount := getClusterCount(dev.GetRdbWalSize(), 1, clusterSize) - cs.log.Tracef("Removing %d RDB WAL clusters (cluster size: %d) from the usable size of the SMD device %s (rank %d, ctlr %s)", - clusterCount, clusterSize, dev.GetUuid(), devToAdjust.rank, devToAdjust.ctlr.GetPciAddr()) + cs.log.Tracef("Removing %d RDB WAL clusters (rdb_size: %s, cluster size: %s - %s) from the usable "+ + "size of the SMD device %s (rank %d, ctlr %q): ", clusterCount, + humanize.IBytes(dev.GetRdbWalSize()), humanize.IBytes(clusterSize), + humanize.IBytes(clusterSize*clusterCount), dev.GetUuid(), devToAdjust.rank, + devToAdjust.ctlr.GetPciAddr()) subtrClusterCount += clusterCount } @@ -578,12 +573,6 @@ func (cs *ControlService) getMetaClusterCount(engineCfg *engine.Config, devToAdj func (cs *ControlService) adjustNvmeSize(resp *ctlpb.ScanNvmeResp) { devsStat := make(map[uint32]*deviceSizeStat, 0) for _, ctlr := range resp.GetCtrlrs() { - engineCfg, err := cs.getEngineCfgFromNvmeCtl(ctlr) - if err != nil { - cs.log.Noticef("Skipping NVME controller %s: %s", ctlr.GetPciAddr(), err.Error()) - continue - } - for idx, dev := range ctlr.GetSmdDevices() { rank := dev.GetRank() devTgtCount := getSmdTgtCount(cs.log, dev) @@ -612,7 +601,8 @@ func (cs *ControlService) adjustNvmeSize(resp *ctlpb.ScanNvmeResp) { } cs.log.Tracef("Initial available size of SMD device %s (rank %d, ctlr %s): %s (%d bytes)", - dev.GetUuid(), rank, ctlr.GetPciAddr(), humanize.IBytes(dev.GetAvailBytes()), dev.GetAvailBytes()) + dev.GetUuid(), rank, ctlr.GetPciAddr(), humanize.IBytes(dev.GetAvailBytes()), + dev.GetAvailBytes()) clusterSize := uint64(dev.GetClusterSize()) availBytes := (dev.GetAvailBytes() / clusterSize) * clusterSize @@ -638,7 +628,7 @@ func (cs *ControlService) adjustNvmeSize(resp *ctlpb.ScanNvmeResp) { continue } - subtrClusterCount := cs.getMetaClusterCount(engineCfg, devToAdjust) + subtrClusterCount := cs.getMetaClusterCount(devToAdjust) if subtrClusterCount >= dataClusterCount { cs.log.Debugf("No more usable space in SMD device %s (rank %d, ctlr %s)", dev.GetUuid(), rank, ctlr.GetPciAddr()) diff --git a/src/control/server/ctl_storage_rpc_test.go b/src/control/server/ctl_storage_rpc_test.go index f6fe2bd7eb1..a922f4a914a 100644 --- a/src/control/server/ctl_storage_rpc_test.go +++ b/src/control/server/ctl_storage_rpc_test.go @@ -3974,107 +3974,6 @@ func TestServer_CtlSvc_adjustScmSize(t *testing.T) { } } -func TestServer_CtlSvc_getEngineCfgFromNvmeCtl(t *testing.T) { - type dataInput struct { - tierCfgs storage.TierConfigs - nvmeCtlr *ctl.NvmeController - } - type expectedOutput struct { - res bool - msg string - } - - newTierCfgs := func(tierCfgsSize int32) storage.TierConfigs { - tierCfgs := make(storage.TierConfigs, tierCfgsSize) - for idx := range tierCfgs { - tierCfgs[idx] = storage.NewTierConfig(). - WithStorageClass(storage.ClassNvme.String()). - WithBdevDeviceList(test.MockPCIAddr(int32(idx + 1))) - } - - return tierCfgs - } - - for name, tc := range map[string]struct { - input dataInput - output expectedOutput - }{ - "find NVME Ctlr": { - input: dataInput{ - tierCfgs: newTierCfgs(5), - nvmeCtlr: &ctl.NvmeController{ - PciAddr: test.MockPCIAddr(3), - }, - }, - output: expectedOutput{res: true}, - }, - "not find NVME Ctlr": { - input: dataInput{ - tierCfgs: newTierCfgs(5), - nvmeCtlr: &ctl.NvmeController{ - PciAddr: test.MockPCIAddr(13), - }, - }, - output: expectedOutput{ - res: false, - msg: "unknown PCI device", - }, - }, - "find VMD device": { - input: dataInput{ - tierCfgs: storage.TierConfigs{ - storage.NewTierConfig(). - WithStorageClass(storage.ClassNvme.String()). - WithBdevDeviceList("0000:04:06.3"), - }, - nvmeCtlr: &ctl.NvmeController{ - PciAddr: "040603:02:00.0", - }, - }, - output: expectedOutput{res: true}, - }, - "Invalid address": { - input: dataInput{ - tierCfgs: storage.TierConfigs{ - storage.NewTierConfig(). - WithStorageClass(storage.ClassNvme.String()). - WithBdevDeviceList("0000:04:06.3"), - }, - nvmeCtlr: &ctl.NvmeController{ - PciAddr: "666", - }, - }, - output: expectedOutput{ - res: false, - msg: "Invalid PCI address", - }, - }, - } { - t.Run(name, func(t *testing.T) { - log, buf := logging.NewTestLogger(t.Name()) - defer test.ShowBufferOnFailure(t, buf) - - engineCfg := engine.MockConfig().WithStorage(tc.input.tierCfgs...) - serverCfg := config.DefaultServer().WithEngines(engineCfg) - cs := mockControlService(t, log, serverCfg, nil, nil, nil) - - ec, err := cs.getEngineCfgFromNvmeCtl(tc.input.nvmeCtlr) - - if tc.output.res { - test.AssertEqual(t, engineCfg, ec, - fmt.Sprintf("Invalid engine config: want=%v got=%v", engineCfg, ec)) - return - } - - test.AssertEqual(t, (*engine.Config)(nil), ec, - fmt.Sprintf("Invalid engine config: wait nil")) - test.AssertTrue(t, - strings.Contains(err.Error(), tc.output.msg), - fmt.Sprintf("Invalid error message: %q not contains %q", err, tc.output.msg)) - }) - } -} - func TestServer_CtlSvc_getEngineCfgFromScmNsp(t *testing.T) { type dataInput struct { tierCfgs storage.TierConfigs diff --git a/src/control/server/instance_storage_rpc.go b/src/control/server/instance_storage_rpc.go index 3097440f500..d8bfb581790 100644 --- a/src/control/server/instance_storage_rpc.go +++ b/src/control/server/instance_storage_rpc.go @@ -332,6 +332,7 @@ func scanEngineBdevsOverDrpc(ctx context.Context, engine Engine, pbReq *ctlpb.Sc if scanSmdResp == nil { return nil, errors.New("nil smd scan resp") } + engine.Tracef("smd scan devices: %+v", scanSmdResp.Devices) // Re-link SMD devices inside NVMe controller structures and populate scan response. @@ -340,12 +341,20 @@ func scanEngineBdevsOverDrpc(ctx context.Context, engine Engine, pbReq *ctlpb.Sc } seenCtrlrs := make(map[string]*ctlpb.NvmeController) - for _, sd := range scanSmdResp.Devices { + for i, sd := range scanSmdResp.Devices { if sd.Ctrlr == nil { return nil, errors.Errorf("smd %q has no ctrlr ref", sd.Uuid) } addr := sd.Ctrlr.PciAddr + if addr == "" { + // Mock identifier for emulated NVMe mode where devices have no PCI-address. + // Allows for 256 unique identifiers per-host and formatted string template + // ensures no collisions with real device addresses. Note that this mock + // identifier address is not used outside of this loop and is only used for + // the purpose of mapping SMD records to NVMe (emulated) device details. + addr = fmt.Sprintf("FFFF:00:%X.F", i) + } if _, exists := seenCtrlrs[addr]; !exists { c := new(ctlpb.NvmeController) @@ -460,6 +469,7 @@ func bdevScanEngineAssigned(ctx context.Context, engine Engine, req *ctlpb.ScanN return scanEngineBdevsOverDrpc(ctx, engine, req) } +// Accommodate for VMD backing devices and emulated NVMe (AIO). func getEffCtrlrCount(ctrlrs []*ctlpb.NvmeController) (int, error) { pas := hardware.MustNewPCIAddressSet() for _, c := range ctrlrs { @@ -471,11 +481,13 @@ func getEffCtrlrCount(ctrlrs []*ctlpb.NvmeController) (int, error) { if npas, err := pas.BackingToVMDAddresses(); err != nil { return 0, err } else { - pas = npas + return npas.Len(), nil } } - return pas.Len(), nil + // Return inputted number of controllers rather than number of parsed addresses to cater for + // the case of emulated NVMe where there will be no valid PCI address. + return len(ctrlrs), nil } // bdevScanEngine calls either in to the private engine storage provider to scan bdevs if engine process diff --git a/src/control/server/instance_storage_rpc_test.go b/src/control/server/instance_storage_rpc_test.go index 7fa74f3bd64..73389dc9905 100644 --- a/src/control/server/instance_storage_rpc_test.go +++ b/src/control/server/instance_storage_rpc_test.go @@ -44,7 +44,7 @@ func (mp *mockPCIeLinkStatsProvider) PCIeCapsFromConfig(cfgBytes []byte, dev *ha return nil } -func TestIOEngineInstance_populateCtrlrHealth(t *testing.T) { +func TestServer_populateCtrlrHealth(t *testing.T) { healthWithLinkStats := func(maxSpd, spd float32, maxWdt, wdt uint32) *ctlpb.BioHealthResp { bhr := proto.MockNvmeHealth() bhr.LinkMaxSpeed = maxSpd @@ -459,7 +459,7 @@ func TestIOEngineInstance_populateCtrlrHealth(t *testing.T) { } } -func TestIOEngineInstance_bdevScanEngine(t *testing.T) { +func TestServer_bdevScanEngine(t *testing.T) { c := storage.MockNvmeController(2) withState := func(ctrlr *ctlpb.NvmeController, state ctlpb.NvmeDevState) *ctlpb.NvmeController { ctrlr.DevState = state @@ -793,6 +793,85 @@ func TestIOEngineInstance_bdevScanEngine(t *testing.T) { // Prove link stat provider gets called without Meta flag. expErr: errors.New("link stats provider fail"), }, + "scan over drpc; aio file emulated nvme; no pci addresses": { + smdRes: &ctlpb.SmdDevResp{ + Devices: []*ctlpb.SmdDevice{ + { + Uuid: "b80b4653-af58-47b3-aa3b-8ad12965f440", + TgtIds: []int32{ + 1024, 1024, 0, 0, 0, 4, 4, 4, 8, 8, 8, 12, + 12, 12, + }, + RoleBits: 7, + Ctrlr: &ctlpb.NvmeController{ + DevState: ctlpb.NvmeDevState_NORMAL, + }, + }, + { + Uuid: "3c7a8e22-38c0-4e6f-8776-d703d049ae6f", + TgtIds: []int32{ + 1024, 1024, 1, 1, 1, 5, 5, 5, 9, 9, 9, 13, + 13, 13, + }, + RoleBits: 7, + Ctrlr: &ctlpb.NvmeController{ + DevState: ctlpb.NvmeDevState_NORMAL, + }, + }, + { + Uuid: "40ba7b18-3a0e-4d68-9e8f-c4fc556eb506", + TgtIds: []int32{ + 1024, 1024, 2, 2, 2, 6, 6, 6, 10, 10, 10, + 14, 14, 14, + }, + RoleBits: 7, + Ctrlr: &ctlpb.NvmeController{ + DevState: ctlpb.NvmeDevState_NORMAL, + }, + }, + { + Uuid: "d78c849f-fd9e-4a20-9746-b0335e3618da", + TgtIds: []int32{ + 1024, 1024, 3, 3, 3, 7, 7, 7, 11, 11, 11, + 15, 15, 15, + }, + RoleBits: 7, + Ctrlr: &ctlpb.NvmeController{ + DevState: ctlpb.NvmeDevState_NORMAL, + }, + }, + }, + }, + expResp: &ctlpb.ScanNvmeResp{ + Ctrlrs: proto.NvmeControllers{ + &ctlpb.NvmeController{ + DevState: ctlpb.NvmeDevState_NORMAL, + SmdDevices: []*ctlpb.SmdDevice{ + {RoleBits: 7}, + }, + }, + &ctlpb.NvmeController{ + DevState: ctlpb.NvmeDevState_NORMAL, + SmdDevices: []*ctlpb.SmdDevice{ + {RoleBits: 7}, + }, + }, + &ctlpb.NvmeController{ + DevState: ctlpb.NvmeDevState_NORMAL, + SmdDevices: []*ctlpb.SmdDevice{ + {RoleBits: 7}, + }, + }, + &ctlpb.NvmeController{ + DevState: ctlpb.NvmeDevState_NORMAL, + SmdDevices: []*ctlpb.SmdDevice{ + {RoleBits: 7}, + }, + }, + }, + State: new(ctlpb.ResponseState), + }, + }, } { t.Run(name, func(t *testing.T) { log, buf := logging.NewTestLogger(t.Name()) diff --git a/src/control/server/storage/config.go b/src/control/server/storage/config.go index 6346d422dbc..1468d2e7bdb 100644 --- a/src/control/server/storage/config.go +++ b/src/control/server/storage/config.go @@ -483,8 +483,8 @@ func (tcs TierConfigs) AssignBdevTierRoles(extMetadataPath string) error { if scs[0].Class == ClassDcpm { return errors.New("external metadata path for md-on-ssd invalid with dcpm scm-class") } - // Skip role assignment and validation if no real NVMe tiers exist. - if !tcs.HaveRealNVMe() { + // Skip role assignment and validation if no bdev tiers exist. + if !tcs.HaveBdevs() { return nil }