From 1b49c6fe629019f4a8a8cf3a67cf39ef63f5f676 Mon Sep 17 00:00:00 2001 From: Tomas Tormo Date: Mon, 8 May 2023 15:48:15 +0200 Subject: [PATCH 1/4] Add support for Persistent Volumes --- modules/k8s/errors.go | 15 ++++ modules/k8s/persistent_volume.go | 102 +++++++++++++++++++++++++ modules/k8s/persistent_volume_test.go | 103 ++++++++++++++++++++++++++ 3 files changed, 220 insertions(+) create mode 100644 modules/k8s/persistent_volume.go create mode 100644 modules/k8s/persistent_volume_test.go diff --git a/modules/k8s/errors.go b/modules/k8s/errors.go index 5f969355c..90a0dcdb7 100644 --- a/modules/k8s/errors.go +++ b/modules/k8s/errors.go @@ -132,6 +132,21 @@ type UnknownServiceType struct { service *corev1.Service } +// PersistentVolumeNotAvailable is returned when a Kubernetes PersistentVolume is not available +type PersistentVolumeNotAvailable struct { + pv *corev1.PersistentVolume +} + +// Error is a simple function to return a formatted error message as a string +func (err PersistentVolumeNotAvailable) Error() string { + return fmt.Sprintf("Pv %s is not available", err.pv.Name) +} + +// NewPersistentVolumeNotAvailableError returns a PersistentVolumeNotAvailable struct when the given Persistent Volume is not available +func NewPersistentVolumeNotAvailableError(pv *corev1.PersistentVolume) PersistentVolumeNotAvailable { + return PersistentVolumeNotAvailable{pv} +} + // Error is a simple function to return a formatted error message as a string func (err UnknownServiceType) Error() string { return fmt.Sprintf("Service %s has an unknown service type", err.service.Name) diff --git a/modules/k8s/persistent_volume.go b/modules/k8s/persistent_volume.go new file mode 100644 index 000000000..7c1fe4142 --- /dev/null +++ b/modules/k8s/persistent_volume.go @@ -0,0 +1,102 @@ +package k8s + +import ( + "context" + "fmt" + "time" + + "github.com/stretchr/testify/require" + + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + "github.com/gruntwork-io/terratest/modules/logger" + "github.com/gruntwork-io/terratest/modules/retry" + "github.com/gruntwork-io/terratest/modules/testing" +) + +// ListPersistentVolumes will look for PersistentVolumes in the given namespace that match the given filters and return them. This will fail the +// test if there is an error. +func ListPersistentVolumes(t testing.TestingT, options *KubectlOptions, filters metav1.ListOptions) []corev1.PersistentVolume { + pvs, err := ListPersistentVolumesE(t, options, filters) + require.NoError(t, err) + return pvs +} + +// ListPersistentVolumesE will look for PersistentVolumes that match the given filters and return them. +func ListPersistentVolumesE(t testing.TestingT, options *KubectlOptions, filters metav1.ListOptions) ([]corev1.PersistentVolume, error) { + clientset, err := GetKubernetesClientFromOptionsE(t, options) + if err != nil { + return nil, err + } + + resp, err := clientset.CoreV1().PersistentVolumes().List(context.Background(), filters) + if err != nil { + return nil, err + } + return resp.Items, nil +} + +// GetPersistentVolume returns a Kubernetes PersistentVolume resource with the given name. This will fail the test if there is an error. +func GetPersistentVolume(t testing.TestingT, options *KubectlOptions, name string) *corev1.PersistentVolume { + pv, err := GetPersistentVolumeE(t, options, name) + require.NoError(t, err) + return pv +} + +// GetPersistentVolumeE returns a Kubernetes PersistentVolume resource with the given name. +func GetPersistentVolumeE(t testing.TestingT, options *KubectlOptions, name string) (*corev1.PersistentVolume, error) { + clientset, err := GetKubernetesClientFromOptionsE(t, options) + if err != nil { + return nil, err + } + return clientset.CoreV1().PersistentVolumes().Get(context.Background(), name, metav1.GetOptions{}) +} + +// WaitUntilPersistentVolumeAvailableE waits until the given Persistent Volume is the 'Available' status, +// retrying the check for the specified amount of times, sleeping +// for the provided duration between each try. +// This will fail the test if there is an error. +func WaitUntilPersistentVolumeAvailable(t testing.TestingT, options *KubectlOptions, pvName string, retries int, sleepBetweenRetries time.Duration) { + require.NoError(t, WaitUntilPersistentVolumeAvailableE(t, options, pvName, retries, sleepBetweenRetries)) +} + +// WaitUntilPersistentVolumeAvailableE waits until the given PersistentVolume is in the 'Available' status, +// retrying the check for the specified amount of times, sleeping +// for the provided duration between each try. +func WaitUntilPersistentVolumeAvailableE( + t testing.TestingT, + options *KubectlOptions, + pvName string, + retries int, + sleepBetweenRetries time.Duration, +) error { + statusMsg := fmt.Sprintf("Wait for Persistent Volume %s to be available", pvName) + message, err := retry.DoWithRetryE( + t, + statusMsg, + retries, + sleepBetweenRetries, + func() (string, error) { + pv, err := GetPersistentVolumeE(t, options, pvName) + if err != nil { + return "", err + } + if !IsPersistentVolumeAvailable(pv) { + return "", NewPersistentVolumeNotAvailableError(pv) + } + return "Persistent Volume is now available", nil + }, + ) + if err != nil { + logger.Logf(t, "Timedout waiting for PersistentVolume to be available: %s", err) + return err + } + logger.Logf(t, message) + return nil +} + +// IsPersistentVolume returns true if the given PersistentVolume is available +func IsPersistentVolumeAvailable(pv *corev1.PersistentVolume) bool { + return pv != nil && pv.Status.Phase == corev1.VolumeAvailable +} diff --git a/modules/k8s/persistent_volume_test.go b/modules/k8s/persistent_volume_test.go new file mode 100644 index 000000000..d321673e5 --- /dev/null +++ b/modules/k8s/persistent_volume_test.go @@ -0,0 +1,103 @@ +// NOTE: we have build tags to differentiate kubernetes tests from non-kubernetes tests. This is done because minikube +// is heavy and can interfere with docker related tests in terratest. Specifically, many of the tests start to fail with +// `connection refused` errors from `minikube`. To avoid overloading the system, we run the kubernetes tests and helm +// tests separately from the others. This may not be necessary if you have a sufficiently powerful machine. We +// recommend at least 4 cores and 16GB of RAM if you want to run all the tests together. + +package k8s + +import ( + "fmt" + "strings" + "testing" + "time" + + "github.com/stretchr/testify/require" + + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + _ "k8s.io/client-go/plugin/pkg/client/auth" + + "github.com/gruntwork-io/terratest/modules/random" +) + +func TestListPersistentVolumesReturnsAllPersistentVolumes(t *testing.T) { + t.Parallel() + + numPvFound := 0 + pvNames := map[string]struct{}{ + strings.ToLower(random.UniqueId()): {}, + strings.ToLower(random.UniqueId()): {}, + strings.ToLower(random.UniqueId()): {}, + } + + options := NewKubectlOptions("", "", "") + for pvName := range pvNames { + pv := fmt.Sprintf(PvFixtureYamlTemplate, pvName, pvName) + defer KubectlDeleteFromString(t, options, pv) + KubectlApplyFromString(t, options, pv) + } + + pvs := ListPersistentVolumes(t, options, metav1.ListOptions{}) + for _, pv := range pvs { + if _, ok := pvNames[pv.Name]; ok { + numPvFound++ + } + } + + require.Equal(t, numPvFound, len(pvNames)) +} + +func TestListPersistentVolumesReturnsZeroPersistentVolumesIfNoneCreated(t *testing.T) { + t.Parallel() + + options := NewKubectlOptions("", "", "") + pvs := ListPersistentVolumes(t, options, metav1.ListOptions{}) + require.Equal(t, 0, len(pvs)) +} + +func TestGetPersistentVolumeEReturnsErrorForNonExistentPersistentVolumes(t *testing.T) { + t.Parallel() + + options := NewKubectlOptions("", "", "") + _, err := GetPersistentVolumeE(t, options, "non-existent") + require.Error(t, err) +} + +func TestGetPersistentVolumeReturnsCorrectPersistentVolume(t *testing.T) { + t.Parallel() + + pvName := strings.ToLower(random.UniqueId()) + options := NewKubectlOptions("", "", "") + configData := fmt.Sprintf(PvFixtureYamlTemplate, pvName, pvName) + defer KubectlDeleteFromString(t, options, configData) + KubectlApplyFromString(t, options, configData) + + pv := GetPersistentVolume(t, options, pvName) + require.Equal(t, pv.Name, pvName) +} + +func TestWaitUntilPersistentVolumeAvailable(t *testing.T) { + t.Parallel() + + pvName := strings.ToLower(random.UniqueId()) + options := NewKubectlOptions("", "", pvName) + configData := fmt.Sprintf(PvFixtureYamlTemplate, pvName, pvName) + KubectlApplyFromString(t, options, configData) + defer KubectlDeleteFromString(t, options, configData) + + WaitUntilPersistentVolumeAvailable(t, options, pvName, 60, 1*time.Second) +} + +const PvFixtureYamlTemplate = `--- +apiVersion: v1 +kind: PersistentVolume +metadata: + name: %s +spec: + capacity: + storage: 10Mi + accessModes: + - ReadWriteOnce + hostPath: + path: "/tmp/%s" +` From 4582cfdd702cbb780623ac0eb70c95852c1c1dc1 Mon Sep 17 00:00:00 2001 From: Tomas Tormo Date: Mon, 8 May 2023 17:40:23 +0200 Subject: [PATCH 2/4] Add Kubernetes tags to the persistent volume tests --- modules/k8s/persistent_volume_test.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/modules/k8s/persistent_volume_test.go b/modules/k8s/persistent_volume_test.go index d321673e5..2048351e7 100644 --- a/modules/k8s/persistent_volume_test.go +++ b/modules/k8s/persistent_volume_test.go @@ -1,3 +1,6 @@ +//go:build kubeall || kubernetes +// +build kubeall kubernetes + // NOTE: we have build tags to differentiate kubernetes tests from non-kubernetes tests. This is done because minikube // is heavy and can interfere with docker related tests in terratest. Specifically, many of the tests start to fail with // `connection refused` errors from `minikube`. To avoid overloading the system, we run the kubernetes tests and helm From 98ab07c463a52b65de427a632f6e3604b5734d27 Mon Sep 17 00:00:00 2001 From: Tomas Tormo Date: Mon, 8 May 2023 18:26:15 +0200 Subject: [PATCH 3/4] Fix typos --- modules/k8s/persistent_volume.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/modules/k8s/persistent_volume.go b/modules/k8s/persistent_volume.go index 7c1fe4142..ba1947842 100644 --- a/modules/k8s/persistent_volume.go +++ b/modules/k8s/persistent_volume.go @@ -53,7 +53,7 @@ func GetPersistentVolumeE(t testing.TestingT, options *KubectlOptions, name stri return clientset.CoreV1().PersistentVolumes().Get(context.Background(), name, metav1.GetOptions{}) } -// WaitUntilPersistentVolumeAvailableE waits until the given Persistent Volume is the 'Available' status, +// WaitUntilPersistentVolumeAvailable waits until the given Persistent Volume is the 'Available' status, // retrying the check for the specified amount of times, sleeping // for the provided duration between each try. // This will fail the test if there is an error. @@ -89,14 +89,14 @@ func WaitUntilPersistentVolumeAvailableE( }, ) if err != nil { - logger.Logf(t, "Timedout waiting for PersistentVolume to be available: %s", err) + logger.Logf(t, "Timeout waiting for PersistentVolume to be available: %s", err) return err } logger.Logf(t, message) return nil } -// IsPersistentVolume returns true if the given PersistentVolume is available +// IsPersistentVolumeAvailable returns true if the given PersistentVolume is available func IsPersistentVolumeAvailable(pv *corev1.PersistentVolume) bool { return pv != nil && pv.Status.Phase == corev1.VolumeAvailable } From 69a9ee91b754a306459b93d89b66753516b816d1 Mon Sep 17 00:00:00 2001 From: Tomas Tormo Date: Wed, 10 May 2023 09:48:38 +0200 Subject: [PATCH 4/4] Provide a generic function to wait and test if a PV is in a given status --- modules/k8s/errors.go | 17 +++++++++-------- modules/k8s/persistent_volume.go | 27 ++++++++++++++------------- modules/k8s/persistent_volume_test.go | 7 +++++-- 3 files changed, 28 insertions(+), 23 deletions(-) diff --git a/modules/k8s/errors.go b/modules/k8s/errors.go index 90a0dcdb7..9bd0d7a03 100644 --- a/modules/k8s/errors.go +++ b/modules/k8s/errors.go @@ -132,19 +132,20 @@ type UnknownServiceType struct { service *corev1.Service } -// PersistentVolumeNotAvailable is returned when a Kubernetes PersistentVolume is not available -type PersistentVolumeNotAvailable struct { - pv *corev1.PersistentVolume +// PersistentVolumeNotInStatus is returned when a Kubernetes PersistentVolume is not in the expected status phase +type PersistentVolumeNotInStatus struct { + pv *corev1.PersistentVolume + pvStatusPhase *corev1.PersistentVolumePhase } // Error is a simple function to return a formatted error message as a string -func (err PersistentVolumeNotAvailable) Error() string { - return fmt.Sprintf("Pv %s is not available", err.pv.Name) +func (err PersistentVolumeNotInStatus) Error() string { + return fmt.Sprintf("Pv %s is not '%s'", err.pv.Name, *err.pvStatusPhase) } -// NewPersistentVolumeNotAvailableError returns a PersistentVolumeNotAvailable struct when the given Persistent Volume is not available -func NewPersistentVolumeNotAvailableError(pv *corev1.PersistentVolume) PersistentVolumeNotAvailable { - return PersistentVolumeNotAvailable{pv} +// NewPersistentVolumeNotInStatusError returns a PersistentVolumeNotInStatus struct when the given Persistent Volume is not in the expected status phase +func NewPersistentVolumeNotInStatusError(pv *corev1.PersistentVolume, pvStatusPhase *corev1.PersistentVolumePhase) PersistentVolumeNotInStatus { + return PersistentVolumeNotInStatus{pv, pvStatusPhase} } // Error is a simple function to return a formatted error message as a string diff --git a/modules/k8s/persistent_volume.go b/modules/k8s/persistent_volume.go index ba1947842..dcab514a2 100644 --- a/modules/k8s/persistent_volume.go +++ b/modules/k8s/persistent_volume.go @@ -53,25 +53,26 @@ func GetPersistentVolumeE(t testing.TestingT, options *KubectlOptions, name stri return clientset.CoreV1().PersistentVolumes().Get(context.Background(), name, metav1.GetOptions{}) } -// WaitUntilPersistentVolumeAvailable waits until the given Persistent Volume is the 'Available' status, +// WaitUntilPersistentVolumeInStatus waits until the given Persistent Volume is the given status phase, // retrying the check for the specified amount of times, sleeping // for the provided duration between each try. // This will fail the test if there is an error. -func WaitUntilPersistentVolumeAvailable(t testing.TestingT, options *KubectlOptions, pvName string, retries int, sleepBetweenRetries time.Duration) { - require.NoError(t, WaitUntilPersistentVolumeAvailableE(t, options, pvName, retries, sleepBetweenRetries)) +func WaitUntilPersistentVolumeInStatus(t testing.TestingT, options *KubectlOptions, pvName string, pvStatusPhase *corev1.PersistentVolumePhase, retries int, sleepBetweenRetries time.Duration) { + require.NoError(t, WaitUntilPersistentVolumeInStatusE(t, options, pvName, pvStatusPhase, retries, sleepBetweenRetries)) } -// WaitUntilPersistentVolumeAvailableE waits until the given PersistentVolume is in the 'Available' status, +// WaitUntilPersistentVolumeInStatusE waits until the given PersistentVolume is in the given status phase, // retrying the check for the specified amount of times, sleeping // for the provided duration between each try. -func WaitUntilPersistentVolumeAvailableE( +func WaitUntilPersistentVolumeInStatusE( t testing.TestingT, options *KubectlOptions, pvName string, + pvStatusPhase *corev1.PersistentVolumePhase, retries int, sleepBetweenRetries time.Duration, ) error { - statusMsg := fmt.Sprintf("Wait for Persistent Volume %s to be available", pvName) + statusMsg := fmt.Sprintf("Wait for Persistent Volume %s to be '%s'", pvName, *pvStatusPhase) message, err := retry.DoWithRetryE( t, statusMsg, @@ -82,21 +83,21 @@ func WaitUntilPersistentVolumeAvailableE( if err != nil { return "", err } - if !IsPersistentVolumeAvailable(pv) { - return "", NewPersistentVolumeNotAvailableError(pv) + if !IsPersistentVolumeInStatus(pv, pvStatusPhase) { + return "", NewPersistentVolumeNotInStatusError(pv, pvStatusPhase) } - return "Persistent Volume is now available", nil + return fmt.Sprintf("Persistent Volume is now '%s'", *pvStatusPhase), nil }, ) if err != nil { - logger.Logf(t, "Timeout waiting for PersistentVolume to be available: %s", err) + logger.Logf(t, "Timeout waiting for PersistentVolume to be '%s': %s", *pvStatusPhase, err) return err } logger.Logf(t, message) return nil } -// IsPersistentVolumeAvailable returns true if the given PersistentVolume is available -func IsPersistentVolumeAvailable(pv *corev1.PersistentVolume) bool { - return pv != nil && pv.Status.Phase == corev1.VolumeAvailable +// IsPersistentVolumeInStatus returns true if the given PersistentVolume is in the given status phase +func IsPersistentVolumeInStatus(pv *corev1.PersistentVolume, pvStatusPhase *corev1.PersistentVolumePhase) bool { + return pv != nil && pv.Status.Phase == *pvStatusPhase } diff --git a/modules/k8s/persistent_volume_test.go b/modules/k8s/persistent_volume_test.go index 2048351e7..fd261c5e9 100644 --- a/modules/k8s/persistent_volume_test.go +++ b/modules/k8s/persistent_volume_test.go @@ -17,6 +17,7 @@ import ( "github.com/stretchr/testify/require" + corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" _ "k8s.io/client-go/plugin/pkg/client/auth" @@ -79,16 +80,18 @@ func TestGetPersistentVolumeReturnsCorrectPersistentVolume(t *testing.T) { require.Equal(t, pv.Name, pvName) } -func TestWaitUntilPersistentVolumeAvailable(t *testing.T) { +func TestWaitUntilPersistentVolumeInTheGivenStatusPhase(t *testing.T) { t.Parallel() pvName := strings.ToLower(random.UniqueId()) + pvAvailableStatusPhase := corev1.VolumeAvailable + options := NewKubectlOptions("", "", pvName) configData := fmt.Sprintf(PvFixtureYamlTemplate, pvName, pvName) KubectlApplyFromString(t, options, configData) defer KubectlDeleteFromString(t, options, configData) - WaitUntilPersistentVolumeAvailable(t, options, pvName, 60, 1*time.Second) + WaitUntilPersistentVolumeInStatus(t, options, pvName, &pvAvailableStatusPhase, 60, 1*time.Second) } const PvFixtureYamlTemplate = `---