From b84304da5e720df11ebb2093c68a6f7e8a684b34 Mon Sep 17 00:00:00 2001 From: Alban Bedel Date: Wed, 18 Nov 2020 20:42:37 +0100 Subject: [PATCH 1/2] Prepare support in kube play for other volume types than hostPath Replace the simple map of names to paths with a map of names to a struct to allow passing more parameters. Also move the code to parse the volumes to its own file to avoid making the playKubePod() function overly complex. Finally rework the kube volumes test to also be ready to support more volume types. Signed-off-by: Alban Bedel --- pkg/domain/infra/abi/play.go | 67 +---------------- pkg/specgen/generate/kube/kube.go | 31 ++++---- pkg/specgen/generate/kube/volume.go | 113 ++++++++++++++++++++++++++++ test/e2e/play_kube_test.go | 48 +++++++----- 4 files changed, 163 insertions(+), 96 deletions(-) create mode 100644 pkg/specgen/generate/kube/volume.go diff --git a/pkg/domain/infra/abi/play.go b/pkg/domain/infra/abi/play.go index 4bcc6469c4..3aeb6a2ee7 100644 --- a/pkg/domain/infra/abi/play.go +++ b/pkg/domain/infra/abi/play.go @@ -8,7 +8,6 @@ import ( "os" "strings" - "github.com/containers/buildah/pkg/parse" "github.com/containers/image/v5/types" "github.com/containers/podman/v2/libpod" "github.com/containers/podman/v2/libpod/image" @@ -24,13 +23,6 @@ import ( v1 "k8s.io/api/core/v1" ) -const ( - // https://kubernetes.io/docs/concepts/storage/volumes/#hostpath - kubeDirectoryPermission = 0755 - // https://kubernetes.io/docs/concepts/storage/volumes/#hostpath - kubeFilePermission = 0644 -) - func (ic *ContainerEngine) PlayKube(ctx context.Context, path string, options entities.PlayKubeOptions) (*entities.PlayKubeReport, error) { var ( kubeObject v1.ObjectReference @@ -168,62 +160,9 @@ func (ic *ContainerEngine) playKubePod(ctx context.Context, podName string, podY DockerInsecureSkipTLSVerify: options.SkipTLSVerify, } - // map from name to mount point - volumes := make(map[string]string) - for _, volume := range podYAML.Spec.Volumes { - hostPath := volume.VolumeSource.HostPath - if hostPath == nil { - return nil, errors.Errorf("HostPath is currently the only supported VolumeSource") - } - if hostPath.Type != nil { - switch *hostPath.Type { - case v1.HostPathDirectoryOrCreate: - if _, err := os.Stat(hostPath.Path); os.IsNotExist(err) { - if err := os.Mkdir(hostPath.Path, kubeDirectoryPermission); err != nil { - return nil, err - } - } - // Label a newly created volume - if err := libpod.LabelVolumePath(hostPath.Path); err != nil { - return nil, errors.Wrapf(err, "error giving %s a label", hostPath.Path) - } - case v1.HostPathFileOrCreate: - if _, err := os.Stat(hostPath.Path); os.IsNotExist(err) { - f, err := os.OpenFile(hostPath.Path, os.O_RDONLY|os.O_CREATE, kubeFilePermission) - if err != nil { - return nil, errors.Wrap(err, "error creating HostPath") - } - if err := f.Close(); err != nil { - logrus.Warnf("Error in closing newly created HostPath file: %v", err) - } - } - // unconditionally label a newly created volume - if err := libpod.LabelVolumePath(hostPath.Path); err != nil { - return nil, errors.Wrapf(err, "error giving %s a label", hostPath.Path) - } - case v1.HostPathSocket: - st, err := os.Stat(hostPath.Path) - if err != nil { - return nil, errors.Wrap(err, "error checking HostPathSocket") - } - if st.Mode()&os.ModeSocket != os.ModeSocket { - return nil, errors.Errorf("error checking HostPathSocket: path %s is not a socket", hostPath.Path) - } - - case v1.HostPathDirectory: - case v1.HostPathFile: - case v1.HostPathUnset: - // do nothing here because we will verify the path exists in validateVolumeHostDir - break - default: - return nil, errors.Errorf("Invalid HostPath type %v", hostPath.Type) - } - } - - if err := parse.ValidateVolumeHostDir(hostPath.Path); err != nil { - return nil, errors.Wrapf(err, "error in parsing HostPath in YAML") - } - volumes[volume.Name] = hostPath.Path + volumes, err := kube.InitializeVolumes(podYAML.Spec.Volumes) + if err != nil { + return nil, err } seccompPaths, err := kube.InitializeSeccompPaths(podYAML.ObjectMeta.Annotations, options.SeccompProfileRoot) diff --git a/pkg/specgen/generate/kube/kube.go b/pkg/specgen/generate/kube/kube.go index e1202956c8..aee30b6525 100644 --- a/pkg/specgen/generate/kube/kube.go +++ b/pkg/specgen/generate/kube/kube.go @@ -47,7 +47,7 @@ func ToPodGen(ctx context.Context, podName string, podYAML *v1.PodTemplateSpec) return p, nil } -func ToSpecGen(ctx context.Context, containerYAML v1.Container, iid string, newImage *image.Image, volumes map[string]string, podID, podName, infraID string, configMaps []v1.ConfigMap, seccompPaths *KubeSeccompPaths, restartPolicy string) (*specgen.SpecGenerator, error) { +func ToSpecGen(ctx context.Context, containerYAML v1.Container, iid string, newImage *image.Image, volumes map[string]*KubeVolume, podID, podName, infraID string, configMaps []v1.ConfigMap, seccompPaths *KubeSeccompPaths, restartPolicy string) (*specgen.SpecGenerator, error) { s := specgen.NewSpecGenerator(iid, false) // podName should be non-empty for Deployment objects to be able to create @@ -163,22 +163,27 @@ func ToSpecGen(ctx context.Context, containerYAML v1.Container, iid string, newI s.Env = envs for _, volume := range containerYAML.VolumeMounts { - hostPath, exists := volumes[volume.Name] + volumeSource, exists := volumes[volume.Name] if !exists { return nil, errors.Errorf("Volume mount %s specified for container but not configured in volumes", volume.Name) } - if err := parse.ValidateVolumeCtrDir(volume.MountPath); err != nil { - return nil, errors.Wrapf(err, "error in parsing MountPath") - } - mount := spec.Mount{ - Destination: volume.MountPath, - Source: hostPath, - Type: "bind", - } - if volume.ReadOnly { - mount.Options = []string{"ro"} + switch volumeSource.Type { + case KubeVolumeTypeBindMount: + if err := parse.ValidateVolumeCtrDir(volume.MountPath); err != nil { + return nil, errors.Wrapf(err, "error in parsing MountPath") + } + mount := spec.Mount{ + Destination: volume.MountPath, + Source: volumeSource.Source, + Type: "bind", + } + if volume.ReadOnly { + mount.Options = []string{"ro"} + } + s.Mounts = append(s.Mounts, mount) + default: + return nil, errors.Errorf("Unsupported volume source type") } - s.Mounts = append(s.Mounts, mount) } s.RestartPolicy = restartPolicy diff --git a/pkg/specgen/generate/kube/volume.go b/pkg/specgen/generate/kube/volume.go new file mode 100644 index 0000000000..b24583e8e4 --- /dev/null +++ b/pkg/specgen/generate/kube/volume.go @@ -0,0 +1,113 @@ +package kube + +import ( + "os" + + "github.com/containers/buildah/pkg/parse" + "github.com/containers/podman/v2/libpod" + "github.com/pkg/errors" + "github.com/sirupsen/logrus" + v1 "k8s.io/api/core/v1" +) + +const ( + // https://kubernetes.io/docs/concepts/storage/volumes/#hostpath + kubeDirectoryPermission = 0755 + // https://kubernetes.io/docs/concepts/storage/volumes/#hostpath + kubeFilePermission = 0644 +) + +type KubeVolumeType int + +const ( + KubeVolumeTypeBindMount KubeVolumeType = iota +) + +type KubeVolume struct { + // Type of volume to create + Type KubeVolumeType + // Path for bind mount + Source string +} + +// Create a KubeVolume from an HostPathVolumeSource +func VolumeFromHostPath(hostPath *v1.HostPathVolumeSource) (*KubeVolume, error) { + if hostPath.Type != nil { + switch *hostPath.Type { + case v1.HostPathDirectoryOrCreate: + if _, err := os.Stat(hostPath.Path); os.IsNotExist(err) { + if err := os.Mkdir(hostPath.Path, kubeDirectoryPermission); err != nil { + return nil, err + } + } + // Label a newly created volume + if err := libpod.LabelVolumePath(hostPath.Path); err != nil { + return nil, errors.Wrapf(err, "error giving %s a label", hostPath.Path) + } + case v1.HostPathFileOrCreate: + if _, err := os.Stat(hostPath.Path); os.IsNotExist(err) { + f, err := os.OpenFile(hostPath.Path, os.O_RDONLY|os.O_CREATE, kubeFilePermission) + if err != nil { + return nil, errors.Wrap(err, "error creating HostPath") + } + if err := f.Close(); err != nil { + logrus.Warnf("Error in closing newly created HostPath file: %v", err) + } + } + // unconditionally label a newly created volume + if err := libpod.LabelVolumePath(hostPath.Path); err != nil { + return nil, errors.Wrapf(err, "error giving %s a label", hostPath.Path) + } + case v1.HostPathSocket: + st, err := os.Stat(hostPath.Path) + if err != nil { + return nil, errors.Wrap(err, "error checking HostPathSocket") + } + if st.Mode()&os.ModeSocket != os.ModeSocket { + return nil, errors.Errorf("error checking HostPathSocket: path %s is not a socket", hostPath.Path) + } + + case v1.HostPathDirectory: + case v1.HostPathFile: + case v1.HostPathUnset: + // do nothing here because we will verify the path exists in validateVolumeHostDir + break + default: + return nil, errors.Errorf("Invalid HostPath type %v", hostPath.Type) + } + } + + if err := parse.ValidateVolumeHostDir(hostPath.Path); err != nil { + return nil, errors.Wrapf(err, "error in parsing HostPath in YAML") + } + + return &KubeVolume{ + Type: KubeVolumeTypeBindMount, + Source: hostPath.Path, + }, nil +} + +// Create a KubeVolume from one of the supported VolumeSource +func VolumeFromSource(volumeSource v1.VolumeSource) (*KubeVolume, error) { + if volumeSource.HostPath != nil { + return VolumeFromHostPath(volumeSource.HostPath) + } else { + return nil, errors.Errorf("HostPath is currently the only supported VolumeSource") + } +} + +// Create a map of volume name to KubeVolume +func InitializeVolumes(specVolumes []v1.Volume) (map[string]*KubeVolume, error) { + volumes := make(map[string]*KubeVolume) + + for _, specVolume := range specVolumes { + volume, err := VolumeFromSource(specVolume.VolumeSource) + if err != nil { + return nil, err + } + + volumes[specVolume.Name] = volume + } + + return volumes, nil +} diff --git a/test/e2e/play_kube_test.go b/test/e2e/play_kube_test.go index 92e4544f9d..eb82b3862f 100644 --- a/test/e2e/play_kube_test.go +++ b/test/e2e/play_kube_test.go @@ -164,9 +164,11 @@ spec: volumes: {{ range . }} - name: {{ .Name }} + {{- if (eq .VolumeType "HostPath") }} hostPath: - path: {{ .Path }} - type: {{ .Type }} + path: {{ .HostPath.Path }} + type: {{ .HostPath.Type }} + {{- end }} {{ end }} {{ end }} status: {} @@ -692,19 +694,27 @@ func getCtrNameInPod(pod *Pod) string { return fmt.Sprintf("%s-%s", pod.Name, defaultCtrName) } -type Volume struct { - Name string +type HostPath struct { Path string Type string } -// getVolume takes a type and a location for a volume -// giving it a default name of volName -func getVolume(vType, vPath string) *Volume { +type Volume struct { + VolumeType string + Name string + HostPath +} + +// getHostPathVolume takes a type and a location for a HostPath +// volume giving it a default name of volName +func getHostPathVolume(vType, vPath string) *Volume { return &Volume{ - Name: defaultVolName, - Path: vPath, - Type: vType, + VolumeType: "HostPath", + Name: defaultVolName, + HostPath: HostPath{ + Path: vPath, + Type: vType, + }, } } @@ -1257,7 +1267,7 @@ spec: It("podman play kube test with non-existent empty HostPath type volume", func() { hostPathLocation := filepath.Join(tempdir, "file") - pod := getPod(withVolume(getVolume(`""`, hostPathLocation))) + pod := getPod(withVolume(getHostPathVolume(`""`, hostPathLocation))) err := generateKubeYaml("pod", pod, kubeYaml) Expect(err).To(BeNil()) @@ -1272,7 +1282,7 @@ spec: Expect(err).To(BeNil()) f.Close() - pod := getPod(withVolume(getVolume(`""`, hostPathLocation))) + pod := getPod(withVolume(getHostPathVolume(`""`, hostPathLocation))) err = generateKubeYaml("pod", pod, kubeYaml) Expect(err).To(BeNil()) @@ -1284,7 +1294,7 @@ spec: It("podman play kube test with non-existent File HostPath type volume", func() { hostPathLocation := filepath.Join(tempdir, "file") - pod := getPod(withVolume(getVolume("File", hostPathLocation))) + pod := getPod(withVolume(getHostPathVolume("File", hostPathLocation))) err := generateKubeYaml("pod", pod, kubeYaml) Expect(err).To(BeNil()) @@ -1299,7 +1309,7 @@ spec: Expect(err).To(BeNil()) f.Close() - pod := getPod(withVolume(getVolume("File", hostPathLocation))) + pod := getPod(withVolume(getHostPathVolume("File", hostPathLocation))) err = generateKubeYaml("pod", pod, kubeYaml) Expect(err).To(BeNil()) @@ -1311,7 +1321,7 @@ spec: It("podman play kube test with FileOrCreate HostPath type volume", func() { hostPathLocation := filepath.Join(tempdir, "file") - pod := getPod(withVolume(getVolume("FileOrCreate", hostPathLocation))) + pod := getPod(withVolume(getHostPathVolume("FileOrCreate", hostPathLocation))) err := generateKubeYaml("pod", pod, kubeYaml) Expect(err).To(BeNil()) @@ -1327,7 +1337,7 @@ spec: It("podman play kube test with DirectoryOrCreate HostPath type volume", func() { hostPathLocation := filepath.Join(tempdir, "file") - pod := getPod(withVolume(getVolume("DirectoryOrCreate", hostPathLocation))) + pod := getPod(withVolume(getHostPathVolume("DirectoryOrCreate", hostPathLocation))) err := generateKubeYaml("pod", pod, kubeYaml) Expect(err).To(BeNil()) @@ -1347,7 +1357,7 @@ spec: Expect(err).To(BeNil()) f.Close() - pod := getPod(withVolume(getVolume("Socket", hostPathLocation))) + pod := getPod(withVolume(getHostPathVolume("Socket", hostPathLocation))) err = generateKubeYaml("pod", pod, kubeYaml) Expect(err).To(BeNil()) @@ -1356,14 +1366,14 @@ spec: Expect(kube.ExitCode()).NotTo(Equal(0)) }) - It("podman play kube test with read only volume", func() { + It("podman play kube test with read only HostPath volume", func() { hostPathLocation := filepath.Join(tempdir, "file") f, err := os.Create(hostPathLocation) Expect(err).To(BeNil()) f.Close() ctr := getCtr(withVolumeMount(hostPathLocation, true), withImage(BB)) - pod := getPod(withVolume(getVolume("File", hostPathLocation)), withCtr(ctr)) + pod := getPod(withVolume(getHostPathVolume("File", hostPathLocation)), withCtr(ctr)) err = generateKubeYaml("pod", pod, kubeYaml) Expect(err).To(BeNil()) From 66944baad657aded4aba07fac28ea590e120afac Mon Sep 17 00:00:00 2001 From: Alban Bedel Date: Wed, 18 Nov 2020 20:51:59 +0100 Subject: [PATCH 2/2] Add support for persistent volume claims in kube files In k8s a persistent volume claim (PVC) allow pods to define a volume by referencing the name of a PVC. The PVC basically contains criterias that k8s then use to select which storage source it will use for the volume. Podman only provide one abtracted storage, the named volumes, and create them if they don't exists yet. So this patch simply use a volume with the name of the PVC. Signed-off-by: Alban Bedel --- pkg/specgen/generate/kube/kube.go | 9 +++++++ pkg/specgen/generate/kube/volume.go | 15 +++++++++-- test/e2e/play_kube_test.go | 41 +++++++++++++++++++++++++++++ 3 files changed, 63 insertions(+), 2 deletions(-) diff --git a/pkg/specgen/generate/kube/kube.go b/pkg/specgen/generate/kube/kube.go index aee30b6525..5f72d28bbe 100644 --- a/pkg/specgen/generate/kube/kube.go +++ b/pkg/specgen/generate/kube/kube.go @@ -181,6 +181,15 @@ func ToSpecGen(ctx context.Context, containerYAML v1.Container, iid string, newI mount.Options = []string{"ro"} } s.Mounts = append(s.Mounts, mount) + case KubeVolumeTypeNamed: + namedVolume := specgen.NamedVolume{ + Dest: volume.MountPath, + Name: volumeSource.Source, + } + if volume.ReadOnly { + namedVolume.Options = []string{"ro"} + } + s.Volumes = append(s.Volumes, &namedVolume) default: return nil, errors.Errorf("Unsupported volume source type") } diff --git a/pkg/specgen/generate/kube/volume.go b/pkg/specgen/generate/kube/volume.go index b24583e8e4..2ef0f4c23b 100644 --- a/pkg/specgen/generate/kube/volume.go +++ b/pkg/specgen/generate/kube/volume.go @@ -21,12 +21,13 @@ type KubeVolumeType int const ( KubeVolumeTypeBindMount KubeVolumeType = iota + KubeVolumeTypeNamed KubeVolumeType = iota ) type KubeVolume struct { // Type of volume to create Type KubeVolumeType - // Path for bind mount + // Path for bind mount or volume name for named volume Source string } @@ -87,12 +88,22 @@ func VolumeFromHostPath(hostPath *v1.HostPathVolumeSource) (*KubeVolume, error) }, nil } +// Create a KubeVolume from a PersistentVolumeClaimVolumeSource +func VolumeFromPersistentVolumeClaim(claim *v1.PersistentVolumeClaimVolumeSource) (*KubeVolume, error) { + return &KubeVolume{ + Type: KubeVolumeTypeNamed, + Source: claim.ClaimName, + }, nil +} + // Create a KubeVolume from one of the supported VolumeSource func VolumeFromSource(volumeSource v1.VolumeSource) (*KubeVolume, error) { if volumeSource.HostPath != nil { return VolumeFromHostPath(volumeSource.HostPath) + } else if volumeSource.PersistentVolumeClaim != nil { + return VolumeFromPersistentVolumeClaim(volumeSource.PersistentVolumeClaim) } else { - return nil, errors.Errorf("HostPath is currently the only supported VolumeSource") + return nil, errors.Errorf("HostPath and PersistentVolumeClaim are currently the conly supported VolumeSource") } } diff --git a/test/e2e/play_kube_test.go b/test/e2e/play_kube_test.go index eb82b3862f..5ecfdd6b50 100644 --- a/test/e2e/play_kube_test.go +++ b/test/e2e/play_kube_test.go @@ -169,6 +169,10 @@ spec: path: {{ .HostPath.Path }} type: {{ .HostPath.Type }} {{- end }} + {{- if (eq .VolumeType "PersistentVolumeClaim") }} + persistentVolumeClaim: + claimName: {{ .PersistentVolumeClaim.ClaimName }} + {{- end }} {{ end }} {{ end }} status: {} @@ -699,10 +703,15 @@ type HostPath struct { Type string } +type PersistentVolumeClaim struct { + ClaimName string +} + type Volume struct { VolumeType string Name string HostPath + PersistentVolumeClaim } // getHostPathVolume takes a type and a location for a HostPath @@ -718,6 +727,18 @@ func getHostPathVolume(vType, vPath string) *Volume { } } +// getHostPathVolume takes a name for a Persistentvolumeclaim +// volume giving it a default name of volName +func getPersistentVolumeClaimVolume(vName string) *Volume { + return &Volume{ + VolumeType: "PersistentVolumeClaim", + Name: defaultVolName, + PersistentVolumeClaim: PersistentVolumeClaim{ + ClaimName: vName, + }, + } +} + type Env struct { Name string Value string @@ -1389,6 +1410,26 @@ spec: Expect(inspect.OutputToString()).To(ContainSubstring(correct)) }) + It("podman play kube test with PersistentVolumeClaim volume", func() { + volumeName := "namedVolume" + + ctr := getCtr(withVolumeMount("/test", false), withImage(BB)) + pod := getPod(withVolume(getPersistentVolumeClaimVolume(volumeName)), withCtr(ctr)) + err = generateKubeYaml("pod", pod, kubeYaml) + Expect(err).To(BeNil()) + + kube := podmanTest.Podman([]string{"play", "kube", kubeYaml}) + kube.WaitWithDefaultTimeout() + Expect(kube.ExitCode()).To(Equal(0)) + + inspect := podmanTest.Podman([]string{"inspect", getCtrNameInPod(pod), "--format", "{{ (index .Mounts 0).Type }}:{{ (index .Mounts 0).Name }}"}) + inspect.WaitWithDefaultTimeout() + Expect(inspect.ExitCode()).To(Equal(0)) + + correct := fmt.Sprintf("volume:%s", volumeName) + Expect(inspect.OutputToString()).To(Equal(correct)) + }) + It("podman play kube applies labels to pods", func() { var numReplicas int32 = 5 expectedLabelKey := "key1"