From fdac84d9495fc93317a4299b51305731bd2213aa Mon Sep 17 00:00:00 2001 From: Blake Devcich Date: Wed, 4 Dec 2024 13:25:45 -0600 Subject: [PATCH] PostMount Env Variables (e.g. NUM_OSTS) There is no way to get the number of OSTs, etc when using `PostMount` commands. For instance, when setting the striping. This change adds the following environment variables for use in the `NnfStorageProfiles` when using `*CmdLines`: - NUM_MDTS - NUM_MGTS - NUM_MGTMDTS - NUM_OSTS - NUM_NNFNODES To support this, a list of nodes for each component type has been added to the status of `NnfStorage`, which in turn is copied to the `NnfNodeStorage` resource's spec. This info can then be turned into the environment variables for use when running the commands. The `.nnf-servers.json` file created to store this information on the compute node has been updated to this new structure. An example with 8 OSTs and 2 MGTMDTs on 1 rabbit: ```json { "mdt": [], "mgt": [], "mgtmdt": [ "rabbit-node-1", "rabbit-node-1" ], "nnfNode": [ "rabbit-node-1" ], "ost": [ "rabbit-node-1", "rabbit-node-1", "rabbit-node-1", "rabbit-node-1", "rabbit-node-1", "rabbit-node-1", "rabbit-node-1", "rabbit-node-1" ] } ``` Signed-off-by: Blake Devcich --- api/v1alpha2/conversion.go | 29 ++++++- api/v1alpha2/zz_generated.conversion.go | 80 +++++++++++++------ api/v1alpha3/conversion.go | 31 ++++++- api/v1alpha3/zz_generated.conversion.go | 80 +++++++++++++------ api/v1alpha4/nnfnodestorage_types.go | 5 ++ api/v1alpha4/nnfstorage_types.go | 25 ++++++ api/v1alpha4/zz_generated.deepcopy.go | 48 ++++++++++- .../nnf.cray.hpe.com_nnfnodestorages.yaml | 35 ++++++++ .../bases/nnf.cray.hpe.com_nnfstorages.yaml | 34 ++++++++ internal/controller/filesystem_helpers.go | 21 ++--- .../controller/nnf_clientmount_controller.go | 64 ++++++++------- .../nnf_clientmount_controller_test.go | 62 ++++++++------ internal/controller/nnf_storage_controller.go | 49 ++++++++++++ .../controller/nnf_storage_controller_test.go | 77 ++++++++++++++++++ 14 files changed, 522 insertions(+), 118 deletions(-) create mode 100644 internal/controller/nnf_storage_controller_test.go diff --git a/api/v1alpha2/conversion.go b/api/v1alpha2/conversion.go index 8806dd0d..1049d3bf 100644 --- a/api/v1alpha2/conversion.go +++ b/api/v1alpha2/conversion.go @@ -343,12 +343,20 @@ func (src *NnfNodeStorage) ConvertTo(dstRaw conversion.Hub) error { // Manually restore data. restored := &nnfv1alpha4.NnfNodeStorage{} - if ok, err := utilconversion.UnmarshalData(src, restored); err != nil || !ok { + hasAnno, err := utilconversion.UnmarshalData(src, restored) + if err != nil { return err } // EDIT THIS FUNCTION! If the annotation is holding anything that is // hub-specific then copy it into 'dst' from 'restored'. // Otherwise, you may comment out UnmarshalData() until it's needed. + if hasAnno { + dst.Spec.LustreStorage.LustreComponents.MDTs = append([]string(nil), restored.Spec.LustreStorage.LustreComponents.MDTs...) + dst.Spec.LustreStorage.LustreComponents.MGTs = append([]string(nil), restored.Spec.LustreStorage.LustreComponents.MGTs...) + dst.Spec.LustreStorage.LustreComponents.MGTMDTs = append([]string(nil), restored.Spec.LustreStorage.LustreComponents.MGTMDTs...) + dst.Spec.LustreStorage.LustreComponents.OSTs = append([]string(nil), restored.Spec.LustreStorage.LustreComponents.OSTs...) + dst.Spec.LustreStorage.LustreComponents.NNFNodes = append([]string(nil), restored.Spec.LustreStorage.LustreComponents.NNFNodes...) + } return nil } @@ -407,12 +415,21 @@ func (src *NnfStorage) ConvertTo(dstRaw conversion.Hub) error { // Manually restore data. restored := &nnfv1alpha4.NnfStorage{} - if ok, err := utilconversion.UnmarshalData(src, restored); err != nil || !ok { + hasAnno, err := utilconversion.UnmarshalData(src, restored) + if err != nil { return err } + // EDIT THIS FUNCTION! If the annotation is holding anything that is // hub-specific then copy it into 'dst' from 'restored'. // Otherwise, you may comment out UnmarshalData() until it's needed. + if hasAnno { + dst.Status.LustreComponents.MDTs = append([]string(nil), restored.Status.LustreComponents.MDTs...) + dst.Status.LustreComponents.MGTs = append([]string(nil), restored.Status.LustreComponents.MGTs...) + dst.Status.LustreComponents.MGTMDTs = append([]string(nil), restored.Status.LustreComponents.MGTMDTs...) + dst.Status.LustreComponents.OSTs = append([]string(nil), restored.Status.LustreComponents.OSTs...) + dst.Status.LustreComponents.NNFNodes = append([]string(nil), restored.Status.LustreComponents.NNFNodes...) + } return nil } @@ -671,3 +688,11 @@ func Convert_v1alpha4_NnfAccessSpec_To_v1alpha2_NnfAccessSpec(in *nnfv1alpha4.Nn func Convert_v1alpha4_NnfDataMovementProfileData_To_v1alpha2_NnfDataMovementProfileData(in *nnfv1alpha4.NnfDataMovementProfileData, out *NnfDataMovementProfileData, s apiconversion.Scope) error { return autoConvert_v1alpha4_NnfDataMovementProfileData_To_v1alpha2_NnfDataMovementProfileData(in, out, s) } + +func Convert_v1alpha4_LustreStorageSpec_To_v1alpha2_LustreStorageSpec(in *nnfv1alpha4.LustreStorageSpec, out *LustreStorageSpec, s apiconversion.Scope) error { + return autoConvert_v1alpha4_LustreStorageSpec_To_v1alpha2_LustreStorageSpec(in, out, s) +} + +func Convert_v1alpha4_NnfStorageLustreStatus_To_v1alpha2_NnfStorageLustreStatus(in *nnfv1alpha4.NnfStorageLustreStatus, out *NnfStorageLustreStatus, s apiconversion.Scope) error { + return autoConvert_v1alpha4_NnfStorageLustreStatus_To_v1alpha2_NnfStorageLustreStatus(in, out, s) +} diff --git a/api/v1alpha2/zz_generated.conversion.go b/api/v1alpha2/zz_generated.conversion.go index 3dd14989..bd69100d 100644 --- a/api/v1alpha2/zz_generated.conversion.go +++ b/api/v1alpha2/zz_generated.conversion.go @@ -48,11 +48,6 @@ func RegisterConversions(s *runtime.Scheme) error { }); err != nil { return err } - if err := s.AddGeneratedConversionFunc((*v1alpha4.LustreStorageSpec)(nil), (*LustreStorageSpec)(nil), func(a, b interface{}, scope conversion.Scope) error { - return Convert_v1alpha4_LustreStorageSpec_To_v1alpha2_LustreStorageSpec(a.(*v1alpha4.LustreStorageSpec), b.(*LustreStorageSpec), scope) - }); err != nil { - return err - } if err := s.AddGeneratedConversionFunc((*NnfAccess)(nil), (*v1alpha4.NnfAccess)(nil), func(a, b interface{}, scope conversion.Scope) error { return Convert_v1alpha2_NnfAccess_To_v1alpha4_NnfAccess(a.(*NnfAccess), b.(*v1alpha4.NnfAccess), scope) }); err != nil { @@ -678,11 +673,6 @@ func RegisterConversions(s *runtime.Scheme) error { }); err != nil { return err } - if err := s.AddGeneratedConversionFunc((*v1alpha4.NnfStorageLustreStatus)(nil), (*NnfStorageLustreStatus)(nil), func(a, b interface{}, scope conversion.Scope) error { - return Convert_v1alpha4_NnfStorageLustreStatus_To_v1alpha2_NnfStorageLustreStatus(a.(*v1alpha4.NnfStorageLustreStatus), b.(*NnfStorageLustreStatus), scope) - }); err != nil { - return err - } if err := s.AddGeneratedConversionFunc((*NnfStorageProfile)(nil), (*v1alpha4.NnfStorageProfile)(nil), func(a, b interface{}, scope conversion.Scope) error { return Convert_v1alpha2_NnfStorageProfile_To_v1alpha4_NnfStorageProfile(a.(*NnfStorageProfile), b.(*v1alpha4.NnfStorageProfile), scope) }); err != nil { @@ -848,6 +838,11 @@ func RegisterConversions(s *runtime.Scheme) error { }); err != nil { return err } + if err := s.AddConversionFunc((*v1alpha4.LustreStorageSpec)(nil), (*LustreStorageSpec)(nil), func(a, b interface{}, scope conversion.Scope) error { + return Convert_v1alpha4_LustreStorageSpec_To_v1alpha2_LustreStorageSpec(a.(*v1alpha4.LustreStorageSpec), b.(*LustreStorageSpec), scope) + }); err != nil { + return err + } if err := s.AddConversionFunc((*v1alpha4.NnfAccessSpec)(nil), (*NnfAccessSpec)(nil), func(a, b interface{}, scope conversion.Scope) error { return Convert_v1alpha4_NnfAccessSpec_To_v1alpha2_NnfAccessSpec(a.(*v1alpha4.NnfAccessSpec), b.(*NnfAccessSpec), scope) }); err != nil { @@ -858,6 +853,11 @@ func RegisterConversions(s *runtime.Scheme) error { }); err != nil { return err } + if err := s.AddConversionFunc((*v1alpha4.NnfStorageLustreStatus)(nil), (*NnfStorageLustreStatus)(nil), func(a, b interface{}, scope conversion.Scope) error { + return Convert_v1alpha4_NnfStorageLustreStatus_To_v1alpha2_NnfStorageLustreStatus(a.(*v1alpha4.NnfStorageLustreStatus), b.(*NnfStorageLustreStatus), scope) + }); err != nil { + return err + } if err := s.AddConversionFunc((*v1alpha4.NnfStorageProfileCmdLines)(nil), (*NnfStorageProfileCmdLines)(nil), func(a, b interface{}, scope conversion.Scope) error { return Convert_v1alpha4_NnfStorageProfileCmdLines_To_v1alpha2_NnfStorageProfileCmdLines(a.(*v1alpha4.NnfStorageProfileCmdLines), b.(*NnfStorageProfileCmdLines), scope) }); err != nil { @@ -896,14 +896,10 @@ func autoConvert_v1alpha4_LustreStorageSpec_To_v1alpha2_LustreStorageSpec(in *v1 out.StartIndex = in.StartIndex out.MgsAddress = in.MgsAddress out.BackFs = in.BackFs + // WARNING: in.LustreComponents requires manual conversion: does not exist in peer-type return nil } -// Convert_v1alpha4_LustreStorageSpec_To_v1alpha2_LustreStorageSpec is an autogenerated conversion function. -func Convert_v1alpha4_LustreStorageSpec_To_v1alpha2_LustreStorageSpec(in *v1alpha4.LustreStorageSpec, out *LustreStorageSpec, s conversion.Scope) error { - return autoConvert_v1alpha4_LustreStorageSpec_To_v1alpha2_LustreStorageSpec(in, out, s) -} - func autoConvert_v1alpha2_NnfAccess_To_v1alpha4_NnfAccess(in *NnfAccess, out *v1alpha4.NnfAccess, s conversion.Scope) error { out.ObjectMeta = in.ObjectMeta if err := Convert_v1alpha2_NnfAccessSpec_To_v1alpha4_NnfAccessSpec(&in.Spec, &out.Spec, s); err != nil { @@ -2188,7 +2184,17 @@ func Convert_v1alpha4_NnfNodeStorageAllocationStatus_To_v1alpha2_NnfNodeStorageA func autoConvert_v1alpha2_NnfNodeStorageList_To_v1alpha4_NnfNodeStorageList(in *NnfNodeStorageList, out *v1alpha4.NnfNodeStorageList, s conversion.Scope) error { out.ListMeta = in.ListMeta - out.Items = *(*[]v1alpha4.NnfNodeStorage)(unsafe.Pointer(&in.Items)) + if in.Items != nil { + in, out := &in.Items, &out.Items + *out = make([]v1alpha4.NnfNodeStorage, len(*in)) + for i := range *in { + if err := Convert_v1alpha2_NnfNodeStorage_To_v1alpha4_NnfNodeStorage(&(*in)[i], &(*out)[i], s); err != nil { + return err + } + } + } else { + out.Items = nil + } return nil } @@ -2199,7 +2205,17 @@ func Convert_v1alpha2_NnfNodeStorageList_To_v1alpha4_NnfNodeStorageList(in *NnfN func autoConvert_v1alpha4_NnfNodeStorageList_To_v1alpha2_NnfNodeStorageList(in *v1alpha4.NnfNodeStorageList, out *NnfNodeStorageList, s conversion.Scope) error { out.ListMeta = in.ListMeta - out.Items = *(*[]NnfNodeStorage)(unsafe.Pointer(&in.Items)) + if in.Items != nil { + in, out := &in.Items, &out.Items + *out = make([]NnfNodeStorage, len(*in)) + for i := range *in { + if err := Convert_v1alpha4_NnfNodeStorage_To_v1alpha2_NnfNodeStorage(&(*in)[i], &(*out)[i], s); err != nil { + return err + } + } + } else { + out.Items = nil + } return nil } @@ -2578,7 +2594,17 @@ func Convert_v1alpha4_NnfStorageAllocationSetStatus_To_v1alpha2_NnfStorageAlloca func autoConvert_v1alpha2_NnfStorageList_To_v1alpha4_NnfStorageList(in *NnfStorageList, out *v1alpha4.NnfStorageList, s conversion.Scope) error { out.ListMeta = in.ListMeta - out.Items = *(*[]v1alpha4.NnfStorage)(unsafe.Pointer(&in.Items)) + if in.Items != nil { + in, out := &in.Items, &out.Items + *out = make([]v1alpha4.NnfStorage, len(*in)) + for i := range *in { + if err := Convert_v1alpha2_NnfStorage_To_v1alpha4_NnfStorage(&(*in)[i], &(*out)[i], s); err != nil { + return err + } + } + } else { + out.Items = nil + } return nil } @@ -2589,7 +2615,17 @@ func Convert_v1alpha2_NnfStorageList_To_v1alpha4_NnfStorageList(in *NnfStorageLi func autoConvert_v1alpha4_NnfStorageList_To_v1alpha2_NnfStorageList(in *v1alpha4.NnfStorageList, out *NnfStorageList, s conversion.Scope) error { out.ListMeta = in.ListMeta - out.Items = *(*[]NnfStorage)(unsafe.Pointer(&in.Items)) + if in.Items != nil { + in, out := &in.Items, &out.Items + *out = make([]NnfStorage, len(*in)) + for i := range *in { + if err := Convert_v1alpha4_NnfStorage_To_v1alpha2_NnfStorage(&(*in)[i], &(*out)[i], s); err != nil { + return err + } + } + } else { + out.Items = nil + } return nil } @@ -2640,14 +2676,10 @@ func autoConvert_v1alpha4_NnfStorageLustreStatus_To_v1alpha2_NnfStorageLustreSta out.MgsAddress = in.MgsAddress out.FileSystemName = in.FileSystemName out.LustreMgtReference = in.LustreMgtReference + // WARNING: in.LustreComponents requires manual conversion: does not exist in peer-type return nil } -// Convert_v1alpha4_NnfStorageLustreStatus_To_v1alpha2_NnfStorageLustreStatus is an autogenerated conversion function. -func Convert_v1alpha4_NnfStorageLustreStatus_To_v1alpha2_NnfStorageLustreStatus(in *v1alpha4.NnfStorageLustreStatus, out *NnfStorageLustreStatus, s conversion.Scope) error { - return autoConvert_v1alpha4_NnfStorageLustreStatus_To_v1alpha2_NnfStorageLustreStatus(in, out, s) -} - func autoConvert_v1alpha2_NnfStorageProfile_To_v1alpha4_NnfStorageProfile(in *NnfStorageProfile, out *v1alpha4.NnfStorageProfile, s conversion.Scope) error { out.ObjectMeta = in.ObjectMeta if err := Convert_v1alpha2_NnfStorageProfileData_To_v1alpha4_NnfStorageProfileData(&in.Data, &out.Data, s); err != nil { diff --git a/api/v1alpha3/conversion.go b/api/v1alpha3/conversion.go index 14e5819c..23682f1f 100644 --- a/api/v1alpha3/conversion.go +++ b/api/v1alpha3/conversion.go @@ -342,12 +342,20 @@ func (src *NnfNodeStorage) ConvertTo(dstRaw conversion.Hub) error { // Manually restore data. restored := &nnfv1alpha4.NnfNodeStorage{} - if ok, err := utilconversion.UnmarshalData(src, restored); err != nil || !ok { + hasAnno, err := utilconversion.UnmarshalData(src, restored) + if err != nil { return err } // EDIT THIS FUNCTION! If the annotation is holding anything that is // hub-specific then copy it into 'dst' from 'restored'. // Otherwise, you may comment out UnmarshalData() until it's needed. + if hasAnno { + dst.Spec.LustreStorage.LustreComponents.MDTs = append([]string(nil), restored.Spec.LustreStorage.LustreComponents.MDTs...) + dst.Spec.LustreStorage.LustreComponents.MGTs = append([]string(nil), restored.Spec.LustreStorage.LustreComponents.MGTs...) + dst.Spec.LustreStorage.LustreComponents.MGTMDTs = append([]string(nil), restored.Spec.LustreStorage.LustreComponents.MGTMDTs...) + dst.Spec.LustreStorage.LustreComponents.OSTs = append([]string(nil), restored.Spec.LustreStorage.LustreComponents.OSTs...) + dst.Spec.LustreStorage.LustreComponents.NNFNodes = append([]string(nil), restored.Spec.LustreStorage.LustreComponents.NNFNodes...) + } return nil } @@ -406,12 +414,21 @@ func (src *NnfStorage) ConvertTo(dstRaw conversion.Hub) error { // Manually restore data. restored := &nnfv1alpha4.NnfStorage{} - if ok, err := utilconversion.UnmarshalData(src, restored); err != nil || !ok { + hasAnno, err := utilconversion.UnmarshalData(src, restored) + if err != nil { return err } + // EDIT THIS FUNCTION! If the annotation is holding anything that is // hub-specific then copy it into 'dst' from 'restored'. // Otherwise, you may comment out UnmarshalData() until it's needed. + if hasAnno { + dst.Status.LustreComponents.MDTs = append([]string(nil), restored.Status.LustreComponents.MDTs...) + dst.Status.LustreComponents.MGTs = append([]string(nil), restored.Status.LustreComponents.MGTs...) + dst.Status.LustreComponents.MGTMDTs = append([]string(nil), restored.Status.LustreComponents.MGTMDTs...) + dst.Status.LustreComponents.OSTs = append([]string(nil), restored.Status.LustreComponents.OSTs...) + dst.Status.LustreComponents.NNFNodes = append([]string(nil), restored.Status.LustreComponents.NNFNodes...) + } return nil } @@ -684,3 +701,13 @@ func Convert_v1alpha4_NnfAccessSpec_To_v1alpha3_NnfAccessSpec(in *nnfv1alpha4.Nn func Convert_v1alpha4_NnfDataMovementProfileData_To_v1alpha3_NnfDataMovementProfileData(in *nnfv1alpha4.NnfDataMovementProfileData, out *NnfDataMovementProfileData, s apiconversion.Scope) error { return autoConvert_v1alpha4_NnfDataMovementProfileData_To_v1alpha3_NnfDataMovementProfileData(in, out, s) } + +// Convert_v1alpha4_LustreStorageSpec_To_v1alpha3_LustreStorageSpec is an autogenerated conversion function. +func Convert_v1alpha4_LustreStorageSpec_To_v1alpha3_LustreStorageSpec(in *nnfv1alpha4.LustreStorageSpec, out *LustreStorageSpec, s apiconversion.Scope) error { + return autoConvert_v1alpha4_LustreStorageSpec_To_v1alpha3_LustreStorageSpec(in, out, s) +} + +// Convert_v1alpha4_NnfStorageLustreStatus_To_v1alpha3_NnfStorageLustreStatus is an autogenerated conversion function. +func Convert_v1alpha4_NnfStorageLustreStatus_To_v1alpha3_NnfStorageLustreStatus(in *nnfv1alpha4.NnfStorageLustreStatus, out *NnfStorageLustreStatus, s apiconversion.Scope) error { + return autoConvert_v1alpha4_NnfStorageLustreStatus_To_v1alpha3_NnfStorageLustreStatus(in, out, s) +} diff --git a/api/v1alpha3/zz_generated.conversion.go b/api/v1alpha3/zz_generated.conversion.go index cd3af0a6..98b09515 100644 --- a/api/v1alpha3/zz_generated.conversion.go +++ b/api/v1alpha3/zz_generated.conversion.go @@ -48,11 +48,6 @@ func RegisterConversions(s *runtime.Scheme) error { }); err != nil { return err } - if err := s.AddGeneratedConversionFunc((*v1alpha4.LustreStorageSpec)(nil), (*LustreStorageSpec)(nil), func(a, b interface{}, scope conversion.Scope) error { - return Convert_v1alpha4_LustreStorageSpec_To_v1alpha3_LustreStorageSpec(a.(*v1alpha4.LustreStorageSpec), b.(*LustreStorageSpec), scope) - }); err != nil { - return err - } if err := s.AddGeneratedConversionFunc((*NnfAccess)(nil), (*v1alpha4.NnfAccess)(nil), func(a, b interface{}, scope conversion.Scope) error { return Convert_v1alpha3_NnfAccess_To_v1alpha4_NnfAccess(a.(*NnfAccess), b.(*v1alpha4.NnfAccess), scope) }); err != nil { @@ -678,11 +673,6 @@ func RegisterConversions(s *runtime.Scheme) error { }); err != nil { return err } - if err := s.AddGeneratedConversionFunc((*v1alpha4.NnfStorageLustreStatus)(nil), (*NnfStorageLustreStatus)(nil), func(a, b interface{}, scope conversion.Scope) error { - return Convert_v1alpha4_NnfStorageLustreStatus_To_v1alpha3_NnfStorageLustreStatus(a.(*v1alpha4.NnfStorageLustreStatus), b.(*NnfStorageLustreStatus), scope) - }); err != nil { - return err - } if err := s.AddGeneratedConversionFunc((*NnfStorageProfile)(nil), (*v1alpha4.NnfStorageProfile)(nil), func(a, b interface{}, scope conversion.Scope) error { return Convert_v1alpha3_NnfStorageProfile_To_v1alpha4_NnfStorageProfile(a.(*NnfStorageProfile), b.(*v1alpha4.NnfStorageProfile), scope) }); err != nil { @@ -848,6 +838,11 @@ func RegisterConversions(s *runtime.Scheme) error { }); err != nil { return err } + if err := s.AddConversionFunc((*v1alpha4.LustreStorageSpec)(nil), (*LustreStorageSpec)(nil), func(a, b interface{}, scope conversion.Scope) error { + return Convert_v1alpha4_LustreStorageSpec_To_v1alpha3_LustreStorageSpec(a.(*v1alpha4.LustreStorageSpec), b.(*LustreStorageSpec), scope) + }); err != nil { + return err + } if err := s.AddConversionFunc((*v1alpha4.NnfAccessSpec)(nil), (*NnfAccessSpec)(nil), func(a, b interface{}, scope conversion.Scope) error { return Convert_v1alpha4_NnfAccessSpec_To_v1alpha3_NnfAccessSpec(a.(*v1alpha4.NnfAccessSpec), b.(*NnfAccessSpec), scope) }); err != nil { @@ -858,6 +853,11 @@ func RegisterConversions(s *runtime.Scheme) error { }); err != nil { return err } + if err := s.AddConversionFunc((*v1alpha4.NnfStorageLustreStatus)(nil), (*NnfStorageLustreStatus)(nil), func(a, b interface{}, scope conversion.Scope) error { + return Convert_v1alpha4_NnfStorageLustreStatus_To_v1alpha3_NnfStorageLustreStatus(a.(*v1alpha4.NnfStorageLustreStatus), b.(*NnfStorageLustreStatus), scope) + }); err != nil { + return err + } if err := s.AddConversionFunc((*v1alpha4.NnfStorageProfileCmdLines)(nil), (*NnfStorageProfileCmdLines)(nil), func(a, b interface{}, scope conversion.Scope) error { return Convert_v1alpha4_NnfStorageProfileCmdLines_To_v1alpha3_NnfStorageProfileCmdLines(a.(*v1alpha4.NnfStorageProfileCmdLines), b.(*NnfStorageProfileCmdLines), scope) }); err != nil { @@ -896,14 +896,10 @@ func autoConvert_v1alpha4_LustreStorageSpec_To_v1alpha3_LustreStorageSpec(in *v1 out.StartIndex = in.StartIndex out.MgsAddress = in.MgsAddress out.BackFs = in.BackFs + // WARNING: in.LustreComponents requires manual conversion: does not exist in peer-type return nil } -// Convert_v1alpha4_LustreStorageSpec_To_v1alpha3_LustreStorageSpec is an autogenerated conversion function. -func Convert_v1alpha4_LustreStorageSpec_To_v1alpha3_LustreStorageSpec(in *v1alpha4.LustreStorageSpec, out *LustreStorageSpec, s conversion.Scope) error { - return autoConvert_v1alpha4_LustreStorageSpec_To_v1alpha3_LustreStorageSpec(in, out, s) -} - func autoConvert_v1alpha3_NnfAccess_To_v1alpha4_NnfAccess(in *NnfAccess, out *v1alpha4.NnfAccess, s conversion.Scope) error { out.ObjectMeta = in.ObjectMeta if err := Convert_v1alpha3_NnfAccessSpec_To_v1alpha4_NnfAccessSpec(&in.Spec, &out.Spec, s); err != nil { @@ -2188,7 +2184,17 @@ func Convert_v1alpha4_NnfNodeStorageAllocationStatus_To_v1alpha3_NnfNodeStorageA func autoConvert_v1alpha3_NnfNodeStorageList_To_v1alpha4_NnfNodeStorageList(in *NnfNodeStorageList, out *v1alpha4.NnfNodeStorageList, s conversion.Scope) error { out.ListMeta = in.ListMeta - out.Items = *(*[]v1alpha4.NnfNodeStorage)(unsafe.Pointer(&in.Items)) + if in.Items != nil { + in, out := &in.Items, &out.Items + *out = make([]v1alpha4.NnfNodeStorage, len(*in)) + for i := range *in { + if err := Convert_v1alpha3_NnfNodeStorage_To_v1alpha4_NnfNodeStorage(&(*in)[i], &(*out)[i], s); err != nil { + return err + } + } + } else { + out.Items = nil + } return nil } @@ -2199,7 +2205,17 @@ func Convert_v1alpha3_NnfNodeStorageList_To_v1alpha4_NnfNodeStorageList(in *NnfN func autoConvert_v1alpha4_NnfNodeStorageList_To_v1alpha3_NnfNodeStorageList(in *v1alpha4.NnfNodeStorageList, out *NnfNodeStorageList, s conversion.Scope) error { out.ListMeta = in.ListMeta - out.Items = *(*[]NnfNodeStorage)(unsafe.Pointer(&in.Items)) + if in.Items != nil { + in, out := &in.Items, &out.Items + *out = make([]NnfNodeStorage, len(*in)) + for i := range *in { + if err := Convert_v1alpha4_NnfNodeStorage_To_v1alpha3_NnfNodeStorage(&(*in)[i], &(*out)[i], s); err != nil { + return err + } + } + } else { + out.Items = nil + } return nil } @@ -2578,7 +2594,17 @@ func Convert_v1alpha4_NnfStorageAllocationSetStatus_To_v1alpha3_NnfStorageAlloca func autoConvert_v1alpha3_NnfStorageList_To_v1alpha4_NnfStorageList(in *NnfStorageList, out *v1alpha4.NnfStorageList, s conversion.Scope) error { out.ListMeta = in.ListMeta - out.Items = *(*[]v1alpha4.NnfStorage)(unsafe.Pointer(&in.Items)) + if in.Items != nil { + in, out := &in.Items, &out.Items + *out = make([]v1alpha4.NnfStorage, len(*in)) + for i := range *in { + if err := Convert_v1alpha3_NnfStorage_To_v1alpha4_NnfStorage(&(*in)[i], &(*out)[i], s); err != nil { + return err + } + } + } else { + out.Items = nil + } return nil } @@ -2589,7 +2615,17 @@ func Convert_v1alpha3_NnfStorageList_To_v1alpha4_NnfStorageList(in *NnfStorageLi func autoConvert_v1alpha4_NnfStorageList_To_v1alpha3_NnfStorageList(in *v1alpha4.NnfStorageList, out *NnfStorageList, s conversion.Scope) error { out.ListMeta = in.ListMeta - out.Items = *(*[]NnfStorage)(unsafe.Pointer(&in.Items)) + if in.Items != nil { + in, out := &in.Items, &out.Items + *out = make([]NnfStorage, len(*in)) + for i := range *in { + if err := Convert_v1alpha4_NnfStorage_To_v1alpha3_NnfStorage(&(*in)[i], &(*out)[i], s); err != nil { + return err + } + } + } else { + out.Items = nil + } return nil } @@ -2640,14 +2676,10 @@ func autoConvert_v1alpha4_NnfStorageLustreStatus_To_v1alpha3_NnfStorageLustreSta out.MgsAddress = in.MgsAddress out.FileSystemName = in.FileSystemName out.LustreMgtReference = in.LustreMgtReference + // WARNING: in.LustreComponents requires manual conversion: does not exist in peer-type return nil } -// Convert_v1alpha4_NnfStorageLustreStatus_To_v1alpha3_NnfStorageLustreStatus is an autogenerated conversion function. -func Convert_v1alpha4_NnfStorageLustreStatus_To_v1alpha3_NnfStorageLustreStatus(in *v1alpha4.NnfStorageLustreStatus, out *NnfStorageLustreStatus, s conversion.Scope) error { - return autoConvert_v1alpha4_NnfStorageLustreStatus_To_v1alpha3_NnfStorageLustreStatus(in, out, s) -} - func autoConvert_v1alpha3_NnfStorageProfile_To_v1alpha4_NnfStorageProfile(in *NnfStorageProfile, out *v1alpha4.NnfStorageProfile, s conversion.Scope) error { out.ObjectMeta = in.ObjectMeta if err := Convert_v1alpha3_NnfStorageProfileData_To_v1alpha4_NnfStorageProfileData(&in.Data, &out.Data, s); err != nil { diff --git a/api/v1alpha4/nnfnodestorage_types.go b/api/v1alpha4/nnfnodestorage_types.go index 6b6b014b..b31af19d 100644 --- a/api/v1alpha4/nnfnodestorage_types.go +++ b/api/v1alpha4/nnfnodestorage_types.go @@ -88,6 +88,11 @@ type LustreStorageSpec struct { // BackFs is the type of backing filesystem to use. // +kubebuilder:validation:Enum=ldiskfs;zfs BackFs string `json:"backFs,omitempty"` + + // LustreComponents defines that list of NNF Nodes that are used for the components (e.g. OSTs) + // in the lustre filesystem. This information is helpful when creating the lustre filesystem and + // using PostMount commands (e.g. to set the striping). + LustreComponents NnfStorageLustreComponents `json:"lustreComponents,omitempty"` } // NnfNodeStorageStatus defines the status for NnfNodeStorage diff --git a/api/v1alpha4/nnfstorage_types.go b/api/v1alpha4/nnfstorage_types.go index b9ab275c..95c96ccd 100644 --- a/api/v1alpha4/nnfstorage_types.go +++ b/api/v1alpha4/nnfstorage_types.go @@ -60,6 +60,27 @@ type NnfStorageLustreSpec struct { PersistentMgsReference corev1.ObjectReference `json:"persistentMgsReference,omitempty"` } +// NnfStorageLustreComponents identifies which NNF nodes are used for each lustre component used by +// the lustre filesystem. Each list can include an NNF node multiple times if that is how it is +// being used (except for NNFNodes). +type NnfStorageLustreComponents struct { + // MTDs is the list of NNF nodes being used as MDTs. + MDTs []string `json:"mdts,omitempty"` + + // MGTs is the list of NNF nodes being used as MGTs. + MGTs []string `json:"mgts,omitempty"` + + // MGTMDTs is the list of NNF nodes being used as combined MGTMDTs. + MGTMDTs []string `json:"mgtmdts,omitempty"` + + // OSTs is the list of NNF nodes being used as OSTs. + OSTs []string `json:"osts,omitempty"` + + // NNfNodes is the list of NNF nodes being used for this filesystem. This is a unique list of + // node names. + NNFNodes []string `json:"nnfNodes,omitempty"` +} + // NnfStorageAllocationSetSpec defines the details for an allocation set type NnfStorageAllocationSetSpec struct { // Name is a human readable label for this set of allocations (e.g., xfs) @@ -124,6 +145,10 @@ type NnfStorageLustreStatus struct { // LustgreMgtReference is an object reference to the NnfLustreMGT resource used // by the NnfStorage LustreMgtReference corev1.ObjectReference `json:"lustreMgtReference,omitempty"` + + // LustreComponents defines that list of NNF Nodes that are used for the components (e.g. OSTs) + // in the lustre filesystem. + LustreComponents NnfStorageLustreComponents `json:"lustreComponents,omitempty"` } // NnfStorageStatus defines the observed status of NNF Storage. diff --git a/api/v1alpha4/zz_generated.deepcopy.go b/api/v1alpha4/zz_generated.deepcopy.go index dbbf2b92..a581ab62 100644 --- a/api/v1alpha4/zz_generated.deepcopy.go +++ b/api/v1alpha4/zz_generated.deepcopy.go @@ -32,6 +32,7 @@ import ( // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *LustreStorageSpec) DeepCopyInto(out *LustreStorageSpec) { *out = *in + in.LustreComponents.DeepCopyInto(&out.LustreComponents) } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new LustreStorageSpec. @@ -1188,7 +1189,7 @@ func (in *NnfNodeStorage) DeepCopyInto(out *NnfNodeStorage) { *out = *in out.TypeMeta = in.TypeMeta in.ObjectMeta.DeepCopyInto(&out.ObjectMeta) - out.Spec = in.Spec + in.Spec.DeepCopyInto(&out.Spec) in.Status.DeepCopyInto(&out.Status) } @@ -1260,7 +1261,7 @@ func (in *NnfNodeStorageList) DeepCopyObject() runtime.Object { // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *NnfNodeStorageSpec) DeepCopyInto(out *NnfNodeStorageSpec) { *out = *in - out.LustreStorage = in.LustreStorage + in.LustreStorage.DeepCopyInto(&out.LustreStorage) out.BlockReference = in.BlockReference } @@ -1583,6 +1584,46 @@ func (in *NnfStorageList) DeepCopyObject() runtime.Object { return nil } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *NnfStorageLustreComponents) DeepCopyInto(out *NnfStorageLustreComponents) { + *out = *in + if in.MDTs != nil { + in, out := &in.MDTs, &out.MDTs + *out = make([]string, len(*in)) + copy(*out, *in) + } + if in.MGTs != nil { + in, out := &in.MGTs, &out.MGTs + *out = make([]string, len(*in)) + copy(*out, *in) + } + if in.MGTMDTs != nil { + in, out := &in.MGTMDTs, &out.MGTMDTs + *out = make([]string, len(*in)) + copy(*out, *in) + } + if in.OSTs != nil { + in, out := &in.OSTs, &out.OSTs + *out = make([]string, len(*in)) + copy(*out, *in) + } + if in.NNFNodes != nil { + in, out := &in.NNFNodes, &out.NNFNodes + *out = make([]string, len(*in)) + copy(*out, *in) + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new NnfStorageLustreComponents. +func (in *NnfStorageLustreComponents) DeepCopy() *NnfStorageLustreComponents { + if in == nil { + return nil + } + out := new(NnfStorageLustreComponents) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *NnfStorageLustreSpec) DeepCopyInto(out *NnfStorageLustreSpec) { *out = *in @@ -1603,6 +1644,7 @@ func (in *NnfStorageLustreSpec) DeepCopy() *NnfStorageLustreSpec { func (in *NnfStorageLustreStatus) DeepCopyInto(out *NnfStorageLustreStatus) { *out = *in out.LustreMgtReference = in.LustreMgtReference + in.LustreComponents.DeepCopyInto(&out.LustreComponents) } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new NnfStorageLustreStatus. @@ -1915,7 +1957,7 @@ func (in *NnfStorageSpec) DeepCopy() *NnfStorageSpec { // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *NnfStorageStatus) DeepCopyInto(out *NnfStorageStatus) { *out = *in - out.NnfStorageLustreStatus = in.NnfStorageLustreStatus + in.NnfStorageLustreStatus.DeepCopyInto(&out.NnfStorageLustreStatus) if in.AllocationSets != nil { in, out := &in.AllocationSets, &out.AllocationSets *out = make([]NnfStorageAllocationSetStatus, len(*in)) diff --git a/config/crd/bases/nnf.cray.hpe.com_nnfnodestorages.yaml b/config/crd/bases/nnf.cray.hpe.com_nnfnodestorages.yaml index 2b0ddcb1..8a7aad79 100644 --- a/config/crd/bases/nnf.cray.hpe.com_nnfnodestorages.yaml +++ b/config/crd/bases/nnf.cray.hpe.com_nnfnodestorages.yaml @@ -576,6 +576,41 @@ spec: filesystem. maxLength: 8 type: string + lustreComponents: + description: |- + LustreComponents defines that list of NNF Nodes that are used for the components (e.g. OSTs) + in the lustre filesystem. This information is helpful when creating the lustre filesystem and + using PostMount commands (e.g. to set the striping). + properties: + mdts: + description: MTDs is the list of NNF nodes being used as MDTs. + items: + type: string + type: array + mgtmdts: + description: MGTMDTs is the list of NNF nodes being used as + combined MGTMDTs. + items: + type: string + type: array + mgts: + description: MGTs is the list of NNF nodes being used as MGTs. + items: + type: string + type: array + nnfNodes: + description: |- + NNfNodes is the list of NNF nodes being used for this filesystem. This is a unique list of + node names. + items: + type: string + type: array + osts: + description: OSTs is the list of NNF nodes being used as OSTs. + items: + type: string + type: array + type: object mgsAddress: description: |- MgsAddress is the NID of the MGS to use. This is used only when diff --git a/config/crd/bases/nnf.cray.hpe.com_nnfstorages.yaml b/config/crd/bases/nnf.cray.hpe.com_nnfstorages.yaml index 3bad19ad..0c3abe1b 100644 --- a/config/crd/bases/nnf.cray.hpe.com_nnfstorages.yaml +++ b/config/crd/bases/nnf.cray.hpe.com_nnfstorages.yaml @@ -841,6 +841,40 @@ spec: filesystem. maxLength: 8 type: string + lustreComponents: + description: |- + LustreComponents defines that list of NNF Nodes that are used for the components (e.g. OSTs) + in the lustre filesystem. + properties: + mdts: + description: MTDs is the list of NNF nodes being used as MDTs. + items: + type: string + type: array + mgtmdts: + description: MGTMDTs is the list of NNF nodes being used as combined + MGTMDTs. + items: + type: string + type: array + mgts: + description: MGTs is the list of NNF nodes being used as MGTs. + items: + type: string + type: array + nnfNodes: + description: |- + NNfNodes is the list of NNF nodes being used for this filesystem. This is a unique list of + node names. + items: + type: string + type: array + osts: + description: OSTs is the list of NNF nodes being used as OSTs. + items: + type: string + type: array + type: object lustreMgtReference: description: |- LustgreMgtReference is an object reference to the NnfLustreMGT resource used diff --git a/internal/controller/filesystem_helpers.go b/internal/controller/filesystem_helpers.go index 4e34caa2..169a40ce 100644 --- a/internal/controller/filesystem_helpers.go +++ b/internal/controller/filesystem_helpers.go @@ -125,21 +125,17 @@ func getBlockDeviceAndFileSystem(ctx context.Context, c client.Client, nnfNodeSt return blockDevice, fileSystem, nil case "lustre": - commandLines := nnfv1alpha4.NnfStorageProfileLustreCmdLines{} + var commandLines nnfv1alpha4.NnfStorageProfileLustreCmdLines switch nnfNodeStorage.Spec.LustreStorage.TargetType { case "mgt": commandLines = nnfStorageProfile.Data.LustreStorage.MgtCmdLines - break case "mgtmdt": commandLines = nnfStorageProfile.Data.LustreStorage.MgtMdtCmdLines - break case "mdt": commandLines = nnfStorageProfile.Data.LustreStorage.MdtCmdLines - break case "ost": commandLines = nnfStorageProfile.Data.LustreStorage.OstCmdLines - break default: return nil, nil, dwsv1alpha2.NewResourceError("invalid Lustre target type %s", nnfNodeStorage.Spec.LustreStorage.TargetType).WithFatal() } @@ -225,7 +221,7 @@ func newZpoolBlockDevice(ctx context.Context, c client.Client, nnfNodeStorage *n return nil, dwsv1alpha2.NewResourceError("could not get NnfNodeBlockStorage: %v", client.ObjectKeyFromObject(nnfNodeBlockStorage)).WithError(err).WithUserMessage("could not find storage allocation").WithMajor() } - if nnfNodeBlockStorage.Status.Ready == false { + if !nnfNodeBlockStorage.Status.Ready { return nil, dwsv1alpha2.NewResourceError("NnfNodeBlockStorage: %v not ready", client.ObjectKeyFromObject(nnfNodeBlockStorage)) } @@ -277,7 +273,7 @@ func newLvmBlockDevice(ctx context.Context, c client.Client, nnfNodeStorage *nnf return nil, dwsv1alpha2.NewResourceError("could not get NnfNodeBlockStorage: %v", client.ObjectKeyFromObject(nnfNodeBlockStorage)).WithError(err).WithUserMessage("could not find storage allocation").WithMajor() } - if nnfNodeBlockStorage.Status.Ready == false { + if !nnfNodeBlockStorage.Status.Ready { return nil, dwsv1alpha2.NewResourceError("NnfNodeBlockStorage: %v not ready", client.ObjectKeyFromObject(nnfNodeBlockStorage)) } @@ -454,9 +450,16 @@ func newLustreFileSystem(ctx context.Context, c client.Client, nnfNodeStorage *n fs.CommandArgs.PreDeactivate = cmdLines.PreDeactivate fs.TempDir = fmt.Sprintf("/mnt/temp/%s-%d", nnfNodeStorage.Name, index) + components := nnfNodeStorage.Spec.LustreStorage.LustreComponents + fs.CommandArgs.Vars = map[string]string{ - "$USERID": fmt.Sprintf("%d", nnfNodeStorage.Spec.UserID), - "$GROUPID": fmt.Sprintf("%d", nnfNodeStorage.Spec.GroupID), + "$USERID": fmt.Sprintf("%d", nnfNodeStorage.Spec.UserID), + "$GROUPID": fmt.Sprintf("%d", nnfNodeStorage.Spec.GroupID), + "$NUM_MDTS": fmt.Sprintf("%d", len(components.MDTs)), + "$NUM_MGTS": fmt.Sprintf("%d", len(components.MGTs)), + "$NUM_MGTMDTS": fmt.Sprintf("%d", len(components.MGTMDTs)), + "$NUM_OSTS": fmt.Sprintf("%d", len(components.OSTs)), + "$NUM_NNFNODES": fmt.Sprintf("%d", len(components.NNFNodes)), } return &fs, nil diff --git a/internal/controller/nnf_clientmount_controller.go b/internal/controller/nnf_clientmount_controller.go index 36292d01..3132a4f9 100644 --- a/internal/controller/nnf_clientmount_controller.go +++ b/internal/controller/nnf_clientmount_controller.go @@ -318,7 +318,9 @@ func (r *NnfClientMountReconciler) dumpServersToFile(ctx context.Context, client defer file.Close() encoder := json.NewEncoder(file) - err = encoder.Encode(createLustreMapping(server)) + + components := getLustreMappingFromServer(server) + err = encoder.Encode(components) if err != nil { return dwsv1alpha2.NewResourceError("could not write JSON to file").WithError(err).WithMajor() } @@ -350,9 +352,9 @@ func (r *NnfClientMountReconciler) getServerForClientMount(ctx context.Context, ownerNS, ownerNSExists := clientMount.Labels[dwsv1alpha2.OwnerNamespaceLabel] _, idxExists := clientMount.Labels[nnfv1alpha4.DirectiveIndexLabel] - // We should expect the owner of the ClientMount to be NnfStorage and have the expected labels + // We should expect the owner to be NnfStorage and have the expected labels if !ownerExists || !ownerNameExists || !ownerNSExists || !idxExists || ownerKind != storageKind { - return nil, dwsv1alpha2.NewResourceError("expected ClientMount owner to be of kind NnfStorage and have the expected labels").WithMajor() + return nil, dwsv1alpha2.NewResourceError("expected owner to be of kind NnfStorage and have the expected labels").WithMajor() } // Retrieve the NnfStorage resource @@ -375,7 +377,7 @@ func (r *NnfClientMountReconciler) getServerForClientMount(ctx context.Context, // We should expect the owner of the NnfStorage to be Workflow or PersistentStorageInstance and // have the expected labels if !ownerExists || !ownerNameExists || !ownerNSExists || !idxExists || (ownerKind != workflowKind && ownerKind != persistentKind) { - return nil, dwsv1alpha2.NewResourceError("expected NnfStorage owner to be of kind Workflow or PersistentStorageInstance and have the expected labels").WithMajor() + return nil, dwsv1alpha2.NewResourceError("expected owner to be of kind Workflow or PersistentStorageInstance and have the expected labels").WithMajor() } // If the owner is a workflow, then we can use the workflow labels and directive index to get @@ -414,36 +416,40 @@ func (r *NnfClientMountReconciler) getServerForClientMount(ctx context.Context, return &serversList.Items[0], nil } -/* -Flatten the AllocationSets to create mapping for lustre information. Example: - - { - "ost": [ - "rabbit-node-1", - "rabbit-node=2" - ] - "mdt": [ - "rabbit-node-1", - "rabbit-node=2" - ] - } -*/ -func createLustreMapping(server *dwsv1alpha2.Servers) map[string][]string { - - m := map[string][]string{} - - for _, allocationSet := range server.Status.AllocationSets { +// Go through the Server's allocation sets to determine the number of Lustre components and rabbit +// nodes. Returns a map with keys for each lustre component type and also the nnf nodes involved. The +// list of nnf nodes is kept unique, but mdts, osts, etc can include a node multiple times. +func getLustreMappingFromServer(server *dwsv1alpha2.Servers) map[string][]string { + nnfNodeKey := "nnfNode" + components := map[string][]string{ + "mdt": []string{}, + "mgt": []string{}, + "mgtmdt": []string{}, + "ost": []string{}, + nnfNodeKey: []string{}, + } + rabbitMap := make(map[string]bool) // use a map to keep the list unique + + // Gather the info from the allocation set + for _, allocationSet := range server.Spec.AllocationSets { label := allocationSet.Label - if _, found := m[label]; !found { - m[label] = []string{} - } + for _, storage := range allocationSet.Storage { + node := storage.Name - for nnfNode, _ := range allocationSet.Storage { - m[label] = append(m[label], nnfNode) + // add to the list for that lustre component for each allocationCount + for i := 0; i < storage.AllocationCount; i++ { + components[label] = append(components[label], node) + } + + // add to the unique list of rabbits + if _, found := rabbitMap[node]; !found { + rabbitMap[node] = true + components[nnfNodeKey] = append(components[nnfNodeKey], node) + } } } - return m + return components } // fakeNnfNodeStorage creates an NnfNodeStorage resource filled in with only the fields diff --git a/internal/controller/nnf_clientmount_controller_test.go b/internal/controller/nnf_clientmount_controller_test.go index 4eb2066b..6443d324 100644 --- a/internal/controller/nnf_clientmount_controller_test.go +++ b/internal/controller/nnf_clientmount_controller_test.go @@ -30,36 +30,48 @@ var _ = Describe("Clientmount Controller Test", func() { It("It should correctly create a human-readable lustre mapping for Servers ", func() { s := dwsv1alpha2.Servers{ - Status: dwsv1alpha2.ServersStatus{ - AllocationSets: []dwsv1alpha2.ServersStatusAllocationSet{ - {Label: "ost", Storage: map[string]dwsv1alpha2.ServersStatusStorage{ - "rabbit-node-1": dwsv1alpha2.ServersStatusStorage{ - AllocationSize: 123345, - }, - "rabbit-node-2": dwsv1alpha2.ServersStatusStorage{ - AllocationSize: 123345, - }, - }}, - {Label: "mdt", Storage: map[string]dwsv1alpha2.ServersStatusStorage{ - "rabbit-node-3": dwsv1alpha2.ServersStatusStorage{ - AllocationSize: 123345, - }, - "rabbit-node-4": dwsv1alpha2.ServersStatusStorage{ - AllocationSize: 123345, - }, - "rabbit-node-8": dwsv1alpha2.ServersStatusStorage{ - AllocationSize: 123345, - }, - }}, + Spec: dwsv1alpha2.ServersSpec{ + AllocationSets: []dwsv1alpha2.ServersSpecAllocationSet{ + {Label: "ost", Storage: []dwsv1alpha2.ServersSpecStorage{ + {Name: "rabbit-node-1", AllocationCount: 2}, + {Name: "rabbit-node-2", AllocationCount: 1}}, + }, + // throw another OST on rabbit-node-2 + {Label: "ost", Storage: []dwsv1alpha2.ServersSpecStorage{ + {Name: "rabbit-node-2", AllocationCount: 1}}, + }, + {Label: "mdt", Storage: []dwsv1alpha2.ServersSpecStorage{ + {Name: "rabbit-node-3", AllocationCount: 1}, + {Name: "rabbit-node-4", AllocationCount: 1}, + {Name: "rabbit-node-8", AllocationCount: 1}}, + }, + {Label: "mgt", Storage: []dwsv1alpha2.ServersSpecStorage{ + {Name: "rabbit-node-3", AllocationCount: 1}}, + }, + {Label: "mgtmdt", Storage: []dwsv1alpha2.ServersSpecStorage{ + {Name: "rabbit-node-4", AllocationCount: 1}}, + }, }, }, } - m := createLustreMapping(&s) - Expect(m).To(HaveLen(2)) - Expect(m["ost"]).To(HaveLen(2)) - Expect(m["ost"]).Should(ContainElements("rabbit-node-1", "rabbit-node-2")) + Expect(s.Spec.AllocationSets).To(HaveLen(5)) + m := getLustreMappingFromServer(&s) + Expect(m).To(HaveLen(5)) // should have keys for 4 lustre components (i.e. ost, mdt, mgt, mgtmdt) + rabbits + + Expect(m["ost"]).To(HaveLen(4)) + Expect(m["ost"]).Should(ContainElements("rabbit-node-1", "rabbit-node-1", "rabbit-node-2", "rabbit-node-2")) + Expect(m["mdt"]).To(HaveLen(3)) Expect(m["mdt"]).Should(ContainElements("rabbit-node-3", "rabbit-node-4", "rabbit-node-8")) + + Expect(m["mgt"]).To(HaveLen(1)) + Expect(m["mgt"]).Should(ContainElements("rabbit-node-3")) + + Expect(m["mgtmdt"]).To(HaveLen(1)) + Expect(m["mgtmdt"]).Should(ContainElements("rabbit-node-4")) + + Expect(m["nnfNode"]).To(HaveLen(5)) + Expect(m["nnfNode"]).Should(ContainElements("rabbit-node-1", "rabbit-node-2", "rabbit-node-3", "rabbit-node-4", "rabbit-node-8")) }) }) diff --git a/internal/controller/nnf_storage_controller.go b/internal/controller/nnf_storage_controller.go index d9cd6e3e..a6a506b1 100644 --- a/internal/controller/nnf_storage_controller.go +++ b/internal/controller/nnf_storage_controller.go @@ -192,6 +192,18 @@ func (r *NnfStorageReconciler) Reconcile(ctx context.Context, req ctrl.Request) } } + // Collect the lists of nodes for each lustre component used for the filesystem + if storage.Spec.FileSystemType == "lustre" { + components := getLustreMappingFromStorage(storage) + storage.Status.LustreComponents = nnfv1alpha4.NnfStorageLustreComponents{ + MDTs: components["mdt"], + MGTs: components["mgt"], + MGTMDTs: components["mgtmdt"], + OSTs: components["ost"], + NNFNodes: components["nnfNode"], + } + } + // For each allocation, create the NnfNodeStorage resources to fan out to the Rabbit nodes for i, allocationSet := range storage.Spec.AllocationSets { // Add a reference to the external MGS PersistentStorageInstance if necessary @@ -639,6 +651,7 @@ func (r *NnfStorageReconciler) createNodeStorage(ctx context.Context, storage *n nnfNodeStorage.Spec.LustreStorage.TargetType = allocationSet.TargetType nnfNodeStorage.Spec.LustreStorage.FileSystemName = storage.Status.FileSystemName nnfNodeStorage.Spec.LustreStorage.MgsAddress = storage.Status.MgsAddress + nnfNodeStorage.Spec.LustreStorage.LustreComponents = storage.Status.LustreComponents // If this isn't the first allocation, then change MGTMDT to MDT so that we only get a single MGT if allocationSet.TargetType == "mgtmdt" && startIndex != 0 { @@ -1272,6 +1285,42 @@ func (r *NnfStorageReconciler) getLustreOST0(ctx context.Context, storage *nnfv1 return nil, nil } +// Go through the Storage's allocation sets to determine the number of Lustre components and rabbit +// nodes. Returns a map with keys for each lustre component type and also the nnf nodes involved. +// The list of nnf nodes is kept unique, but mdts, osts, etc can include a node multiple times. +func getLustreMappingFromStorage(storage *nnfv1alpha4.NnfStorage) map[string][]string { + nnfNodeKey := "nnfNode" + componentMap := map[string][]string{ + "mdt": {}, + "mgt": {}, + "mgtmdt": {}, + "ost": {}, + nnfNodeKey: {}, + } + rabbitMap := make(map[string]bool) // use a map to keep the list unique + + // Gather the info from the allocation set + for _, allocationSet := range storage.Spec.AllocationSets { + name := allocationSet.Name + for _, storage := range allocationSet.Nodes { + node := storage.Name + + // add to the list for that lustre component for each Count + for i := 0; i < storage.Count; i++ { + componentMap[name] = append(componentMap[name], node) + } + + // add to the unique list of rabbits + if _, found := rabbitMap[node]; !found { + rabbitMap[node] = true + componentMap[nnfNodeKey] = append(componentMap[nnfNodeKey], node) + } + } + } + + return componentMap +} + // SetupWithManager sets up the controller with the Manager. func (r *NnfStorageReconciler) SetupWithManager(mgr ctrl.Manager) error { r.ChildObjects = []dwsv1alpha2.ObjectList{ diff --git a/internal/controller/nnf_storage_controller_test.go b/internal/controller/nnf_storage_controller_test.go new file mode 100644 index 00000000..c31bc80b --- /dev/null +++ b/internal/controller/nnf_storage_controller_test.go @@ -0,0 +1,77 @@ +/* + * Copyright 2024 Hewlett Packard Enterprise Development LP + * Other additional copyright holders may be indicated within. + * + * The entirety of this work is licensed under the Apache License, + * Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. + * + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package controller + +import ( + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + nnfv1alpha4 "github.com/NearNodeFlash/nnf-sos/api/v1alpha4" +) + +var _ = Describe("NNFStorage Controller Test", func() { + + It("It should correctly create a human-readable lustre mapping for NnfStorage", func() { + s := nnfv1alpha4.NnfStorage{ + Spec: nnfv1alpha4.NnfStorageSpec{ + AllocationSets: []nnfv1alpha4.NnfStorageAllocationSetSpec{ + {Name: "ost", Nodes: []nnfv1alpha4.NnfStorageAllocationNodes{ + {Name: "rabbit-node-1", Count: 2}, + {Name: "rabbit-node-2", Count: 1}}, + }, + // throw another OST on rabbit-node-2 + {Name: "ost", Nodes: []nnfv1alpha4.NnfStorageAllocationNodes{ + {Name: "rabbit-node-2", Count: 1}}, + }, + {Name: "mdt", Nodes: []nnfv1alpha4.NnfStorageAllocationNodes{ + {Name: "rabbit-node-3", Count: 1}, + {Name: "rabbit-node-4", Count: 1}, + {Name: "rabbit-node-8", Count: 1}}, + }, + {Name: "mgt", Nodes: []nnfv1alpha4.NnfStorageAllocationNodes{ + {Name: "rabbit-node-3", Count: 1}}, + }, + {Name: "mgtmdt", Nodes: []nnfv1alpha4.NnfStorageAllocationNodes{ + {Name: "rabbit-node-4", Count: 1}}, + }, + }, + }, + } + + Expect(s.Spec.AllocationSets).To(HaveLen(5)) + m := getLustreMappingFromStorage(&s) + Expect(m).To(HaveLen(5)) // should have keys for 4 lustre components (i.e. ost, mdt, mgt, mgtmdt) + rabbits + + Expect(m["ost"]).To(HaveLen(4)) + Expect(m["ost"]).Should(ContainElements("rabbit-node-1", "rabbit-node-1", "rabbit-node-2", "rabbit-node-2")) + + Expect(m["mdt"]).To(HaveLen(3)) + Expect(m["mdt"]).Should(ContainElements("rabbit-node-3", "rabbit-node-4", "rabbit-node-8")) + + Expect(m["mgt"]).To(HaveLen(1)) + Expect(m["mgt"]).Should(ContainElements("rabbit-node-3")) + + Expect(m["mgtmdt"]).To(HaveLen(1)) + Expect(m["mgtmdt"]).Should(ContainElements("rabbit-node-4")) + + Expect(m["nnfNode"]).To(HaveLen(5)) + Expect(m["nnfNode"]).Should(ContainElements("rabbit-node-1", "rabbit-node-2", "rabbit-node-3", "rabbit-node-4", "rabbit-node-8")) + }) +})