Skip to content

Commit

Permalink
chore: refactor e2e tests
Browse files Browse the repository at this point in the history
  • Loading branch information
jakobmoellerdev committed Sep 19, 2023
1 parent 9ae17da commit a2e74f1
Show file tree
Hide file tree
Showing 9 changed files with 114 additions and 226 deletions.
9 changes: 6 additions & 3 deletions test/e2e/aws_disk_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -22,12 +22,15 @@ import (
"strings"
"time"

. "github.com/onsi/ginkgo/v2"

"github.com/go-logr/logr"

"github.com/aws/aws-sdk-go/aws"
"github.com/aws/aws-sdk-go/aws/credentials"
"github.com/aws/aws-sdk-go/aws/session"
"github.com/aws/aws-sdk-go/service/ec2"
"github.com/go-logr/logr"
"github.com/onsi/ginkgo/v2"

corev1 "k8s.io/api/core/v1"
"k8s.io/apimachinery/pkg/types"
"k8s.io/apimachinery/pkg/util/wait"
Expand Down Expand Up @@ -211,7 +214,7 @@ func getEC2Client(ctx context.Context, region string) (*ec2.EC2, error) {
Region: aws.String(region),
Credentials: credentials.NewStaticCredentials(string(id), string(key), ""),
Logger: aws.LoggerFunc(func(args ...interface{}) {
ginkgo.GinkgoLogr.Info(fmt.Sprint(args), "source", "aws")
GinkgoLogr.Info(fmt.Sprint(args), "source", "aws")
}),
})
if err != nil {
Expand Down
5 changes: 4 additions & 1 deletion test/e2e/config_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -24,16 +24,19 @@ import (
snapapi "github.com/kubernetes-csi/external-snapshotter/client/v4/apis/volumesnapshot/v1"
configv1 "github.com/openshift/api/config/v1"
secv1 "github.com/openshift/api/security/v1"
lvmv1 "github.com/openshift/lvm-operator/api/v1alpha1"

operatorv1 "github.com/operator-framework/api/pkg/operators/v1"
operatorv1alpha1 "github.com/operator-framework/api/pkg/operators/v1alpha1"

"k8s.io/apimachinery/pkg/runtime"
"k8s.io/apimachinery/pkg/runtime/serializer"
utilruntime "k8s.io/apimachinery/pkg/util/runtime"
k8sscheme "k8s.io/client-go/kubernetes/scheme"
"k8s.io/client-go/rest"
"k8s.io/client-go/tools/clientcmd"
crclient "sigs.k8s.io/controller-runtime/pkg/client"

lvmv1 "github.com/openshift/lvm-operator/api/v1alpha1"
)

const (
Expand Down
40 changes: 7 additions & 33 deletions test/e2e/helper_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -18,23 +18,22 @@ package e2e

import (
"context"
"errors"
"fmt"
"sync"

. "github.com/onsi/ginkgo/v2"
. "github.com/onsi/gomega"
"k8s.io/client-go/discovery"
"sigs.k8s.io/controller-runtime/pkg/client/apiutil"

"github.com/openshift/lvm-operator/api/v1alpha1"
"k8s.io/apimachinery/pkg/api/meta"

k8sv1 "k8s.io/api/core/v1"
storagev1 "k8s.io/api/storage/v1"
k8serrors "k8s.io/apimachinery/pkg/api/errors"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/types"
"sigs.k8s.io/controller-runtime/pkg/client"
"sigs.k8s.io/controller-runtime/pkg/client/apiutil"

"github.com/openshift/lvm-operator/api/v1alpha1"
)

const (
Expand Down Expand Up @@ -105,7 +104,7 @@ func DeleteResource(ctx context.Context, obj client.Object) {
if k8serrors.IsNotFound(err) {
return
}
if discovery.IsGroupDiscoveryFailedError(errors.Unwrap(err)) {
if meta.IsNoMatchError(err) {
GinkgoLogr.Info("deletion was requested for resource that is not supported on the cluster",
"gvk", gvk, "obj", client.ObjectKeyFromObject(obj))
return
Expand All @@ -116,27 +115,6 @@ func DeleteResource(ctx context.Context, obj client.Object) {
}, timeout, interval).WithContext(ctx).Should(Satisfy(k8serrors.IsNotFound))
}

func lvmNamespaceSetup(ctx context.Context) {
label := make(map[string]string)
// label required for monitoring this namespace
label["openshift.io/cluster-monitoring"] = "true"
label["pod-security.kubernetes.io/enforce"] = "privileged"
label["security.openshift.io/scc.podSecurityLabelSync"] = "false"

annotations := make(map[string]string)
// annotation required for workload partitioning
annotations["workload.openshift.io/allowed"] = "management"

ns := &k8sv1.Namespace{
ObjectMeta: metav1.ObjectMeta{
Name: testNamespace,
Annotations: annotations,
Labels: label,
},
}
CreateResource(ctx, ns)
}

func lvmNamespaceCleanup(ctx context.Context) {
DeleteResource(ctx, &k8sv1.Namespace{ObjectMeta: metav1.ObjectMeta{Name: testNamespace}})
}
Expand Down Expand Up @@ -181,7 +159,7 @@ func GetDefaultTestLVMClusterTemplate() *v1alpha1.LVMCluster {
}

// createNamespace creates a namespace in the cluster, ignoring if it already exists.
func createNamespace(ctx context.Context, namespace string) error {
func createNamespace(ctx context.Context, namespace string) {
label := make(map[string]string)
// label required for monitoring this namespace
label["openshift.io/cluster-monitoring"] = "true"
Expand All @@ -199,11 +177,7 @@ func createNamespace(ctx context.Context, namespace string) error {
Labels: label,
},
}
err := crClient.Create(ctx, ns)
if err != nil && !k8serrors.IsAlreadyExists(err) {
return err
}
return nil
CreateResource(ctx, ns)
}

// DeleteResources in concurrent rows with sequential elements in each row
Expand Down
4 changes: 3 additions & 1 deletion test/e2e/lvm_cluster_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -21,8 +21,10 @@ import (

. "github.com/onsi/ginkgo/v2"
. "github.com/onsi/gomega"
"github.com/openshift/lvm-operator/api/v1alpha1"

"k8s.io/apimachinery/pkg/types"

"github.com/openshift/lvm-operator/api/v1alpha1"
)

func lvmClusterTest() {
Expand Down
134 changes: 50 additions & 84 deletions test/e2e/lvm_pvc_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -19,22 +19,23 @@ package e2e
import (
"context"
_ "embed"
"errors"
"fmt"
"strings"

"github.com/openshift/lvm-operator/api/v1alpha1"
. "github.com/onsi/ginkgo/v2"
. "github.com/onsi/gomega"
"k8s.io/apimachinery/pkg/api/meta"

snapapi "github.com/kubernetes-csi/external-snapshotter/client/v4/apis/volumesnapshot/v1"

k8sv1 "k8s.io/api/core/v1"
"k8s.io/apimachinery/pkg/api/resource"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/client-go/discovery"
"k8s.io/utils/ptr"
"sigs.k8s.io/controller-runtime/pkg/client"
"sigs.k8s.io/controller-runtime/pkg/client/apiutil"

snapapi "github.com/kubernetes-csi/external-snapshotter/client/v4/apis/volumesnapshot/v1"
. "github.com/onsi/ginkgo/v2"
. "github.com/onsi/gomega"
k8sv1 "k8s.io/api/core/v1"
"sigs.k8s.io/controller-runtime/pkg/client"
"github.com/openshift/lvm-operator/api/v1alpha1"
)

const (
Expand All @@ -49,6 +50,13 @@ var (
PodCommandForPVCTests = []string{"sh", "-c", "tail -f /dev/null"}
)

type pvcType string

const (
pvcTypeEphemeral pvcType = "ephemeral"
pvcTypeStatic pvcType = "static"
)

func pvcTest() {
var cluster *v1alpha1.LVMCluster
BeforeAll(func(ctx SpecContext) {
Expand All @@ -60,22 +68,28 @@ func pvcTest() {
})
})

for _, mode := range []k8sv1.PersistentVolumeMode{
volumeModes := []k8sv1.PersistentVolumeMode{
k8sv1.PersistentVolumeBlock,
k8sv1.PersistentVolumeFilesystem,
} {
Context(fmt.Sprintf("PersistentVolumeMode: %s", string(mode)), func() {
Context("static", func() {
pvcTestsForMode(mode)
})
Context("ephemeral", func() {
ephemeralPVCTestsForMode(mode)
})
}

pvcTypes := []pvcType{
pvcTypeEphemeral,
pvcTypeStatic,
}

for _, pvMode := range volumeModes {
Context(fmt.Sprintf("PersistentVolumeMode: %s", string(pvMode)), func() {
for _, pvcType := range pvcTypes {
Context(fmt.Sprintf("PVC Type: %s", pvcType), func() {
pvcTestsForMode(pvMode, pvcType)
})
}
})
}
}

func pvcTestsForMode(volumeMode k8sv1.PersistentVolumeMode) {
func pvcTestsForMode(volumeMode k8sv1.PersistentVolumeMode, pvcType pvcType) {
var contentMode ContentMode
switch volumeMode {
case k8sv1.PersistentVolumeBlock:
Expand All @@ -84,8 +98,20 @@ func pvcTestsForMode(volumeMode k8sv1.PersistentVolumeMode) {
contentMode = ContentModeFile
}

pvc := generatePVC(volumeMode)
pod := generatePodConsumingPVC(pvc)
var pod *k8sv1.Pod
var pvc *k8sv1.PersistentVolumeClaim
switch pvcType {
case pvcTypeStatic:
pvc = generatePVC(volumeMode)
pod = generatePodConsumingPVC(pvc)
case pvcTypeEphemeral:
pod = generatePodWithEphemeralVolume(volumeMode)
// recreates locally what will be created as an ephemeral volume
pvc = &k8sv1.PersistentVolumeClaim{}
pvc.SetName(fmt.Sprintf("%s-%s", pod.GetName(), pod.Spec.Volumes[0].Name))
pvc.SetNamespace(pod.GetNamespace())
pvc.Spec.VolumeMode = &volumeMode
}

clonePVC := generatePVCCloneFromPVC(pvc)
clonePod := generatePodConsumingPVC(clonePVC)
Expand All @@ -102,7 +128,9 @@ func pvcTestsForMode(volumeMode k8sv1.PersistentVolumeMode) {

expectedData := "TESTDATA"
It("PVC and Pod", func(ctx SpecContext) {
CreateResource(ctx, pvc)
if pvcType == pvcTypeStatic {
CreateResource(ctx, pvc)
}
CreateResource(ctx, pod)
validatePodIsRunning(ctx, client.ObjectKeyFromObject(pod))
validatePVCIsBound(ctx, client.ObjectKeyFromObject(pvc))
Expand Down Expand Up @@ -133,68 +161,6 @@ func pvcTestsForMode(volumeMode k8sv1.PersistentVolumeMode) {
})
}

func ephemeralPVCTestsForMode(volumeMode k8sv1.PersistentVolumeMode) {
var contentMode ContentMode
switch volumeMode {
case k8sv1.PersistentVolumeBlock:
contentMode = ContentModeBlock
case k8sv1.PersistentVolumeFilesystem:
contentMode = ContentModeFile
}

pod := generatePodWithEphemeralVolume(volumeMode)

// recreates locally what will be created as an ephemeral volume
pvc := &k8sv1.PersistentVolumeClaim{}
pvc.SetName(fmt.Sprintf("%s-%s", pod.GetName(), pod.Spec.Volumes[0].Name))
pvc.SetNamespace(pod.GetNamespace())
pvc.Spec.VolumeMode = &volumeMode

clonePVC := generatePVCCloneFromPVC(pvc)
clonePod := generatePodConsumingPVC(clonePVC)

snapshot := generateVolumeSnapshot(pvc, snapshotClass)
snapshotPVC := generatePVCFromSnapshot(volumeMode, snapshot)
snapshotPod := generatePodConsumingPVC(snapshotPVC)

AfterAll(DeleteResources([][]client.Object{
{snapshotPod, snapshotPVC, snapshot},
{clonePod, clonePVC},
{pod},
}))

expectedData := "TESTDATA"
It("Pod (with ephemeral Volume)", func(ctx SpecContext) {
CreateResource(ctx, pod)
validatePodIsRunning(ctx, client.ObjectKeyFromObject(pod))
validatePVCIsBound(ctx, client.ObjectKeyFromObject(pvc))

Expect(contentTester.WriteDataInPod(ctx, pod, expectedData, contentMode)).To(Succeed())
validatePodData(ctx, pod, expectedData, contentMode)
})

It("PVC Cloning", func(ctx SpecContext) {
CreateResource(ctx, clonePVC)
CreateResource(ctx, clonePod)

validatePodIsRunning(ctx, client.ObjectKeyFromObject(clonePod))
validatePVCIsBound(ctx, client.ObjectKeyFromObject(clonePVC))
validatePodData(ctx, clonePod, expectedData, contentMode)
})

It("Snapshots", func(ctx SpecContext) {
createVolumeSnapshotFromPVCOrSkipIfUnsupported(ctx, snapshot)
validateSnapshotReadyToUse(ctx, client.ObjectKeyFromObject(snapshot))

CreateResource(ctx, snapshotPVC)
CreateResource(ctx, snapshotPod)

validatePodIsRunning(ctx, client.ObjectKeyFromObject(snapshotPod))
validatePVCIsBound(ctx, client.ObjectKeyFromObject(snapshotPVC))
validatePodData(ctx, snapshotPod, expectedData, contentMode)
})
}

func generatePVCSpec(mode k8sv1.PersistentVolumeMode) k8sv1.PersistentVolumeClaimSpec {
return k8sv1.PersistentVolumeClaimSpec{
VolumeMode: ptr.To(mode),
Expand Down Expand Up @@ -320,7 +286,7 @@ func createVolumeSnapshotFromPVCOrSkipIfUnsupported(ctx context.Context, snapsho
GinkgoHelper()
By(fmt.Sprintf("Creating VolumeSnapshot %q", snapshot.GetName()))
err := crClient.Create(ctx, snapshot)
if discovery.IsGroupDiscoveryFailedError(errors.Unwrap(err)) {
if meta.IsNoMatchError(err) {
Skip("Skipping Testing of VolumeSnapshot Operations due to lack of volume snapshot support")
}
Expect(err).ToNot(HaveOccurred(), "PVC should be created successfully")
Expand Down
8 changes: 4 additions & 4 deletions test/e2e/lvm_suite_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -21,15 +21,15 @@ import (
"flag"
"testing"

. "github.com/onsi/ginkgo/v2"
. "github.com/onsi/gomega"

"go.uber.org/zap"
"go.uber.org/zap/zapcore"

"github.com/go-logr/zapr"
"sigs.k8s.io/controller-runtime/pkg/log"
ctrlZap "sigs.k8s.io/controller-runtime/pkg/log/zap"

. "github.com/onsi/ginkgo/v2"
. "github.com/onsi/gomega"
)

func TestLvm(t *testing.T) {
Expand All @@ -50,7 +50,7 @@ var _ = BeforeSuite(func(ctx context.Context) {

// Configure the disk and install the operator
beforeTestSuiteSetup(ctx)
lvmNamespaceSetup(ctx)
createNamespace(ctx, testNamespace)
})

var _ = AfterSuite(func(ctx context.Context) {
Expand Down
5 changes: 3 additions & 2 deletions test/e2e/pod_runner_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,8 @@ import (
"net/http"
"strings"

"github.com/onsi/gomega"
. "github.com/onsi/gomega"

k8sv1 "k8s.io/api/core/v1"
"k8s.io/apimachinery/pkg/runtime"
"k8s.io/client-go/kubernetes"
Expand Down Expand Up @@ -129,7 +130,7 @@ func (t *PodRunner) ExecCommandInFirstPodContainer(ctx context.Context, pod *k8s

// WriteDataInPod writes the data to pod.
func (t *PodRunner) WriteDataInPod(ctx context.Context, pod *k8sv1.Pod, content string, mode ContentMode) error {
gomega.Expect(pod.Spec.Containers).NotTo(gomega.BeEmpty())
Expect(pod.Spec.Containers).NotTo(BeEmpty())
var filePath string
if mode == "file" {
filePath = pod.Spec.Containers[0].VolumeMounts[0].MountPath + "/test"
Expand Down
Loading

0 comments on commit a2e74f1

Please sign in to comment.