diff --git a/config/kind/manager_volumes_patch.yaml b/config/kind/manager_volumes_patch.yaml index a6bf6487..3da27b6c 100644 --- a/config/kind/manager_volumes_patch.yaml +++ b/config/kind/manager_volumes_patch.yaml @@ -10,15 +10,16 @@ spec: - name: manager workingDir: /localdisk volumeMounts: - - mountPath: /mnt - name: mnt-dir + - mountPath: /mnt/nnf + name: mock-filesystems mountPropagation: Bidirectional - mountPath: /localdisk name: localdisk volumes: - - name: mnt-dir + - name: mock-filesystems hostPath: - path: /mnt + path: /mnt/nnf + type: DirectoryOrCreate - name: localdisk hostPath: type: DirectoryOrCreate diff --git a/internal/controller/filesystem_helpers.go b/internal/controller/filesystem_helpers.go index cb176cb3..4e34caa2 100644 --- a/internal/controller/filesystem_helpers.go +++ b/internal/controller/filesystem_helpers.go @@ -42,6 +42,21 @@ import ( //+kubebuilder:rbac:groups=nnf.cray.hpe.com,resources=nnfnodestorages/finalizers,verbs=update //+kubebuilder:rbac:groups=nnf.cray.hpe.com,resources=nnfstorageprofiles,verbs=get;create;list;watch;update;patch;delete;deletecollection +func getBlockDeviceAndFileSystemForMock(ctx context.Context, c client.Client, nnfNodeStorage *nnfv1alpha4.NnfNodeStorage, index int, log logr.Logger) (blockdevice.BlockDevice, filesystem.FileSystem, error) { + + blockDevice, err := newMockBlockDevice(ctx, c, nnfNodeStorage, index, log) + if err != nil { + return nil, nil, dwsv1alpha2.NewResourceError("could not create mock block device").WithError(err).WithMajor() + } + + fileSystem, err := newMockFileSystem(nnfNodeStorage, index, log) + if err != nil { + return nil, nil, dwsv1alpha2.NewResourceError("could not create mock file system").WithError(err).WithMajor() + } + + return blockDevice, fileSystem, nil +} + func getBlockDeviceAndFileSystemForKind(ctx context.Context, c client.Client, nnfNodeStorage *nnfv1alpha4.NnfNodeStorage, index int, log logr.Logger) (blockdevice.BlockDevice, filesystem.FileSystem, error) { blockDevice, err := newMockBlockDevice(ctx, c, nnfNodeStorage, index, log) @@ -49,7 +64,7 @@ func getBlockDeviceAndFileSystemForKind(ctx context.Context, c client.Client, nn return nil, nil, dwsv1alpha2.NewResourceError("could not create mock block device").WithError(err).WithMajor() } - fileSystem, err := newMockFileSystem(ctx, c, nnfNodeStorage, blockDevice, index, log) + fileSystem, err := newKindFileSystem(nnfNodeStorage, index, log) if err != nil { return nil, nil, dwsv1alpha2.NewResourceError("could not create mock file system").WithError(err).WithMajor() } @@ -59,8 +74,11 @@ func getBlockDeviceAndFileSystemForKind(ctx context.Context, c client.Client, nn // getBlockDeviceAndFileSystem returns blockdevice and filesystem interfaces based on the allocation type and NnfStorageProfile. func getBlockDeviceAndFileSystem(ctx context.Context, c client.Client, nnfNodeStorage *nnfv1alpha4.NnfNodeStorage, index int, log logr.Logger) (blockdevice.BlockDevice, filesystem.FileSystem, error) { - _, found := os.LookupEnv("NNF_TEST_ENVIRONMENT") - if found || os.Getenv("ENVIRONMENT") == "kind" { + if _, found := os.LookupEnv("NNF_TEST_ENVIRONMENT"); found { + return getBlockDeviceAndFileSystemForMock(ctx, c, nnfNodeStorage, index, log) + + } + if os.Getenv("ENVIRONMENT") == "kind" { return getBlockDeviceAndFileSystemForKind(ctx, c, nnfNodeStorage, index, log) } @@ -444,7 +462,7 @@ func newLustreFileSystem(ctx context.Context, c client.Client, nnfNodeStorage *n return &fs, nil } -func newMockFileSystem(ctx context.Context, c client.Client, nnfNodeStorage *nnfv1alpha4.NnfNodeStorage, blockDevice blockdevice.BlockDevice, index int, log logr.Logger) (filesystem.FileSystem, error) { +func newMockFileSystem(nnfNodeStorage *nnfv1alpha4.NnfNodeStorage, index int, log logr.Logger) (filesystem.FileSystem, error) { path := os.Getenv("MOCK_FILE_SYSTEM_PATH") if len(path) == 0 { path = "/mnt/filesystems" @@ -454,8 +472,21 @@ func newMockFileSystem(ctx context.Context, c client.Client, nnfNodeStorage *nnf Log: log, Path: fmt.Sprintf("/%s/%s-%d", path, nnfNodeStorage.GetName(), index), } + return &fs, nil +} +func newKindFileSystem(nnfNodeStorage *nnfv1alpha4.NnfNodeStorage, index int, log logr.Logger) (filesystem.FileSystem, error) { + path := os.Getenv("MOCK_FILE_SYSTEM_PATH") + if len(path) == 0 { + path = "/mnt/nnf" + } + + fs := filesystem.KindFileSystem{ + Log: log, + Path: fmt.Sprintf("/%s/%s-%d", path, nnfNodeStorage.GetName(), index), + } return &fs, nil + } func lustreTargetPath(ctx context.Context, c client.Client, nnfNodeStorage *nnfv1alpha4.NnfNodeStorage, targetType string, index int) (string, error) { diff --git a/internal/controller/nnf_workflow_controller_container_helpers.go b/internal/controller/nnf_workflow_controller_container_helpers.go index a0579dcb..a8f2533a 100644 --- a/internal/controller/nnf_workflow_controller_container_helpers.go +++ b/internal/controller/nnf_workflow_controller_container_helpers.go @@ -538,10 +538,12 @@ func (c *nnfUserContainer) addNnfVolumes(spec *corev1.PodSpec) { MountPath: vol.mountPath, }) - container.Env = append(container.Env, corev1.EnvVar{ - Name: vol.envVarName, - Value: vol.mountPath, - }) + if vol.envVarName != "" { + container.Env = append(container.Env, corev1.EnvVar{ + Name: vol.envVarName, + Value: vol.mountPath, + }) + } } } } diff --git a/internal/controller/nnf_workflow_controller_helpers.go b/internal/controller/nnf_workflow_controller_helpers.go index de045059..1ec5eb04 100644 --- a/internal/controller/nnf_workflow_controller_helpers.go +++ b/internal/controller/nnf_workflow_controller_helpers.go @@ -1905,6 +1905,17 @@ func (r *NnfWorkflowReconciler) getContainerJobs(ctx context.Context, workflow * return jobList, nil } +func (r *NnfWorkflowReconciler) getNnfNodeStorages(ctx context.Context, workflow *dwsv1alpha2.Workflow) (*nnfv1alpha4.NnfNodeStorageList, error) { + matchLabels := dwsv1alpha2.MatchingWorkflow(workflow) + + nodeStorages := &nnfv1alpha4.NnfNodeStorageList{} + if err := r.List(ctx, nodeStorages, matchLabels); err != nil { + return nil, dwsv1alpha2.NewResourceError("could not retrieve NnfNodeStorages").WithError(err).WithMajor() + } + + return nodeStorages, nil +} + // Create a list of volumes to be mounted inside of the containers based on the DW_JOB/DW_PERSISTENT arguments func (r *NnfWorkflowReconciler) getContainerVolumes(ctx context.Context, workflow *dwsv1alpha2.Workflow, dwArgs map[string]string, profile *nnfv1alpha4.NnfContainerProfile) ([]nnfContainerVolume, *result, error) { volumes := []nnfContainerVolume{} @@ -1991,9 +2002,45 @@ func (r *NnfWorkflowReconciler) getContainerVolumes(ctx context.Context, workflo volumes = append(volumes, vol) } + if os.Getenv("ENVIRONMENT") == "kind" { + devVolumes, err := r.findMockDevicesForKind(ctx, workflow) + if err != nil { + return nil, nil, err + } + volumes = append(volumes, devVolumes...) + } return volumes, nil, nil } +// If we're using the KIND mock storage then we also have to create a volume +// mount for the path that represents the device beneath the filesystem. +func (r *NnfWorkflowReconciler) findMockDevicesForKind(ctx context.Context, workflow *dwsv1alpha2.Workflow) ([]nnfContainerVolume, error) { + volumes := []nnfContainerVolume{} + + nodeStoragesList, err := r.getNnfNodeStorages(ctx, workflow) + if err != nil { + return nil, dwsv1alpha2.NewResourceError("could not find devices for KIND environment").WithError(err) + } + // On GFS2, the same device is visible on multiple rabbits. Track dupes + // and add a mount for only one of them. + devNames := make(map[string]struct{}) + devCount := 0 + for _, nodeStorage := range nodeStoragesList.Items { + if _, found := devNames[nodeStorage.GetName()]; !found { + for idx := 0; idx < nodeStorage.Spec.Count; idx++ { + vol := nnfContainerVolume{ + name: fmt.Sprintf("kind-device-%d", devCount), + mountPath: fmt.Sprintf("/mnt/nnf/%s-%d", nodeStorage.GetName(), idx), + } + volumes = append(volumes, vol) + devCount += 1 + } + devNames[nodeStorage.GetName()] = struct{}{} + } + } + return volumes, nil +} + // Use the container profile to determine how many ports are needed and request them from the default NnfPortManager func (r *NnfWorkflowReconciler) getContainerPorts(ctx context.Context, workflow *dwsv1alpha2.Workflow, index int) (*result, error) { profile, err := getContainerProfile(ctx, r.Client, workflow, index) diff --git a/pkg/filesystem/kind.go b/pkg/filesystem/kind.go index 7aae05ba..aa799fb4 100644 --- a/pkg/filesystem/kind.go +++ b/pkg/filesystem/kind.go @@ -23,6 +23,7 @@ import ( "context" "fmt" "os" + "path/filepath" "github.com/NearNodeFlash/nnf-sos/pkg/blockdevice" "github.com/go-logr/logr" @@ -47,7 +48,7 @@ func (m *KindFileSystem) Create(ctx context.Context, complete bool) (bool, error return false, fmt.Errorf("could not create mount directory %s: %w", m.Path, err) } - m.Log.Info("Created mock file system", "path", m.Path) + m.Log.Info("Created mock file system in kind", "path", m.Path) return true, nil } @@ -55,7 +56,7 @@ func (m *KindFileSystem) Destroy(ctx context.Context) (bool, error) { // Remove the directory. If it fails don't worry about it. _ = os.RemoveAll(m.Path) - m.Log.Info("Destroyed mock file system") + m.Log.Info("Destroyed mock file system in kind") return true, nil } @@ -64,12 +65,12 @@ func (m *KindFileSystem) Activate(ctx context.Context, complete bool) (bool, err return false, nil } - m.Log.Info("Activated mock file system") + m.Log.Info("Activated mock file system in kind") return true, nil } func (m *KindFileSystem) Deactivate(ctx context.Context) (bool, error) { - m.Log.Info("Deactivated mock file system") + m.Log.Info("Deactivated mock file system in kind") return true, nil } @@ -78,11 +79,16 @@ func (m *KindFileSystem) Mount(ctx context.Context, path string, complete bool) return false, nil } + bn := filepath.Dir(path) + if err := os.MkdirAll(bn, 0755); err != nil { + return false, fmt.Errorf("could not create directory for symlink %s: %w", bn, err) + } + if err := os.Symlink(m.Path, path); err != nil { return false, fmt.Errorf("could not create symlink mount %s: %w", path, err) } - m.Log.Info("Mounted mock file system", "filesystem", m.Path, "mount", path) + m.Log.Info("Mounted mock file system in kind", "filesystem", m.Path, "mount", path) return true, nil } @@ -90,7 +96,7 @@ func (m *KindFileSystem) Unmount(ctx context.Context, path string) (bool, error) // Remove the directory. If it fails don't worry about it. _ = os.Remove(path) - m.Log.Info("Unmounted mock file system") + m.Log.Info("Unmounted mock file system in kind") return true, nil } @@ -99,13 +105,13 @@ func (m *KindFileSystem) PostActivate(ctx context.Context, complete bool) (bool, return false, nil } - m.Log.Info("Ran PostActivate") + m.Log.Info("Ran PostActivate in kind") return true, nil } func (m *KindFileSystem) PreDeactivate(ctx context.Context) (bool, error) { - m.Log.Info("Ran PreDeactivate") + m.Log.Info("Ran PreDeactivate in kind") return true, nil }