Skip to content

Commit

Permalink
e2e: create test namespace earlier to avoid race with pull secret con…
Browse files Browse the repository at this point in the history 
…troller
  • Loading branch information
@alebedev87
alebedev87 committed Oct 23, 2023
1 parent 6f8f180 commit b9932fb
Showing 1 changed file with 128 additions and 53 deletions.
181 changes: 128 additions & 53 deletions test/e2e/operator_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -163,6 +163,16 @@ func TestOperatorAvailable(t *testing.T) {
// TestAWSLoadBalancerControllerWithDefaultIngressClass tests the basic happy flow for the operator, mostly
// using the default values.
func TestAWSLoadBalancerControllerWithDefaultIngressClass(t *testing.T) {
// The test namespace should be created earlier
// to let the pull secret for internal registry images to be created.
// The test workload uses the tools image from the internal image registry.
testWorkloadNamespace := "aws-load-balancer-test-default-ing"
t.Logf("Creating test namespace %q", testWorkloadNamespace)
echoNs := createTestNamespace(t, testWorkloadNamespace)
defer func() {
waitForDeletion(context.TODO(), t, kubeClient, echoNs, defaultTimeout)
}()

t.Log("Creating aws load balancer controller instance with default ingress class")

alb := newALBCBuilder().withRoleARNIf(stsModeRequested(), controllerRoleARN).build()
Expand All @@ -181,11 +191,8 @@ func TestAWSLoadBalancerControllerWithDefaultIngressClass(t *testing.T) {
t.Fatalf("did not get expected available condition for deployment: %v", err)
}

testWorkloadNamespace := "aws-load-balancer-test-default-ing"
echoSvc, echoNs := createTestWorkload(t, testWorkloadNamespace)
defer func() {
waitForDeletion(context.TODO(), t, kubeClient, echoNs, defaultTimeout)
}()
t.Logf("Creating test workload in %q namespace", testWorkloadNamespace)
echoSvc := createTestWorkload(t, testWorkloadNamespace)

t.Log("Creating Ingress Resource with default ingress class")
ingName := types.NamespacedName{Name: "echoserver", Namespace: testWorkloadNamespace}
Expand Down Expand Up @@ -234,6 +241,13 @@ func TestAWSLoadBalancerControllerWithDefaultIngressClass(t *testing.T) {

// TestAWSLoadBalancerControllersV1Alpha1 tests the basic happy flow for the operator using v1alpha1 ALBC.
func TestAWSLoadBalancerControllersV1Alpha1(t *testing.T) {
testWorkloadNamespace := "aws-load-balancer-test-v1alpha1"
t.Logf("Creating test namespace %q", testWorkloadNamespace)
echoNs := createTestNamespace(t, testWorkloadNamespace)
defer func() {
waitForDeletion(context.TODO(), t, kubeClient, echoNs, defaultTimeout)
}()

t.Log("Creating v1alpha1 aws load balancer controller instance with default ingress class, additional resource tags and credentials secret")

// The additional resource tags and the credentials secret are added to ALBC
Expand All @@ -254,10 +268,10 @@ func TestAWSLoadBalancerControllersV1Alpha1(t *testing.T) {
t.Fatalf("did not get expected available condition for deployment: %v", err)
}

testWorkloadNamespace := "aws-load-balancer-test-default-ing"
echoSvc, echoNs := createTestWorkload(t, testWorkloadNamespace)
t.Logf("Creating test workload in %q namespace", testWorkloadNamespace)
echoSvc := createTestWorkload(t, testWorkloadNamespace)
defer func() {
waitForDeletion(context.TODO(), t, kubeClient, echoNs, defaultTimeout)
waitForDeletion(context.TODO(), t, kubeClient, echoSvc, defaultTimeout)
}()

t.Log("Creating Ingress Resource with default ingress class")
Expand Down Expand Up @@ -308,6 +322,13 @@ func TestAWSLoadBalancerControllersV1Alpha1(t *testing.T) {
// TestAWSLoadBalancerControllerWithCredentialsSecret tests the basic happy flow for the operator
// using the explicitly specified credentials secret.
func TestAWSLoadBalancerControllerWithCredentialsSecret(t *testing.T) {
testWorkloadNamespace := "aws-load-balancer-test-cred-secret"
t.Logf("Creating test namespace %q", testWorkloadNamespace)
echoNs := createTestNamespace(t, testWorkloadNamespace)
defer func() {
waitForDeletion(context.TODO(), t, kubeClient, echoNs, defaultTimeout)
}()

t.Log("Creating aws load balancer controller instance with credentials secret")
alb := newALBCBuilder().withCredSecret(controllerSecretName).build()
if err := kubeClient.Create(context.TODO(), alb); err != nil {
Expand All @@ -325,10 +346,10 @@ func TestAWSLoadBalancerControllerWithCredentialsSecret(t *testing.T) {
t.Fatalf("did not get expected available condition for deployment: %v", err)
}

testWorkloadNamespace := "aws-load-balancer-test-cred-secret"
echoSvc, echoNs := createTestWorkload(t, testWorkloadNamespace)
t.Logf("Creating test workload in %q namespace", testWorkloadNamespace)
echoSvc := createTestWorkload(t, testWorkloadNamespace)
defer func() {
waitForDeletion(context.TODO(), t, kubeClient, echoNs, defaultTimeout)
waitForDeletion(context.TODO(), t, kubeClient, echoSvc, defaultTimeout)
}()

t.Log("Creating Ingress Resource with default ingress class")
Expand Down Expand Up @@ -377,6 +398,13 @@ func TestAWSLoadBalancerControllerWithCredentialsSecret(t *testing.T) {
}

func TestAWSLoadBalancerControllerWithCustomIngressClass(t *testing.T) {
testWorkloadNamespace := "aws-load-balancer-test-custom-ing"
t.Logf("Creating test namespace %q", testWorkloadNamespace)
echoNs := createTestNamespace(t, testWorkloadNamespace)
defer func() {
waitForDeletion(context.TODO(), t, kubeClient, echoNs, defaultTimeout)
}()

t.Log("Creating a custom ingress class")
ingclassName := types.NamespacedName{Name: "custom-alb", Namespace: "aws-load-balancer-operator"}
ingclass := buildIngressClass(ingclassName, "ingress.k8s.aws/alb")
Expand Down Expand Up @@ -405,10 +433,10 @@ func TestAWSLoadBalancerControllerWithCustomIngressClass(t *testing.T) {
t.Fatalf("did not get expected available condition for deployment: %v", err)
}

testWorkloadNamespace := "aws-load-balancer-test-custom-ing"
echoSvc, echoNs := createTestWorkload(t, testWorkloadNamespace)
t.Logf("Creating test workload in %q namespace", testWorkloadNamespace)
echoSvc := createTestWorkload(t, testWorkloadNamespace)
defer func() {
waitForDeletion(context.TODO(), t, kubeClient, echoNs, defaultTimeout)
waitForDeletion(context.TODO(), t, kubeClient, echoSvc, defaultTimeout)
}()

t.Log("Creating Ingress Resource with custom ingress class")
Expand Down Expand Up @@ -457,6 +485,13 @@ func TestAWSLoadBalancerControllerWithCustomIngressClass(t *testing.T) {
}

func TestAWSLoadBalancerControllerWithInternalLoadBalancer(t *testing.T) {
testWorkloadNamespace := "aws-load-balancer-test-internal-ing"
t.Logf("Creating test namespace %q", testWorkloadNamespace)
echoNs := createTestNamespace(t, testWorkloadNamespace)
defer func() {
waitForDeletion(context.TODO(), t, kubeClient, echoNs, defaultTimeout)
}()

t.Log("Creating aws load balancer controller instance with default ingress class")

alb := newALBCBuilder().withRoleARNIf(stsModeRequested(), controllerRoleARN).build()
Expand All @@ -475,10 +510,10 @@ func TestAWSLoadBalancerControllerWithInternalLoadBalancer(t *testing.T) {
t.Fatalf("did not get expected available condition for deployment: %v", err)
}

testWorkloadNamespace := "aws-load-balancer-test-internal-ing"
echoSvc, echoNs := createTestWorkload(t, testWorkloadNamespace)
t.Logf("Creating test workload in %q namespace", testWorkloadNamespace)
echoSvc := createTestWorkload(t, testWorkloadNamespace)
defer func() {
waitForDeletion(context.TODO(), t, kubeClient, echoNs, defaultTimeout)
waitForDeletion(context.TODO(), t, kubeClient, echoSvc, defaultTimeout)
}()

t.Log("Creating Internal Ingress Resource with default ingress class")
Expand Down Expand Up @@ -548,6 +583,13 @@ func TestAWSLoadBalancerControllerWithInternalLoadBalancer(t *testing.T) {
}

func TestAWSLoadBalancerControllerWithWAFv2(t *testing.T) {
testWorkloadNamespace := "aws-load-balancer-test-wafv2"
t.Logf("Creating test namespace %q", testWorkloadNamespace)
echoNs := createTestNamespace(t, testWorkloadNamespace)
defer func() {
waitForDeletion(context.TODO(), t, kubeClient, echoNs, defaultTimeout)
}()

t.Log("Creating aws load balancer controller instance with default ingress class")

alb := newALBCBuilder().withAddons(albo.AWSAddonWAFv2).withRoleARNIf(stsModeRequested(), controllerRoleARN).build()
Expand All @@ -566,10 +608,10 @@ func TestAWSLoadBalancerControllerWithWAFv2(t *testing.T) {
t.Fatalf("did not get expected available condition for deployment: %v", err)
}

testWorkloadNamespace := "aws-load-balancer-test-wafv2"
echoSvc, echoNs := createTestWorkload(t, testWorkloadNamespace)
t.Logf("Creating test workload in %q namespace", testWorkloadNamespace)
echoSvc := createTestWorkload(t, testWorkloadNamespace)
defer func() {
waitForDeletion(context.TODO(), t, kubeClient, echoNs, defaultTimeout)
waitForDeletion(context.TODO(), t, kubeClient, echoSvc, defaultTimeout)
}()

var aclARN string
Expand Down Expand Up @@ -669,6 +711,13 @@ func TestAWSLoadBalancerControllerWithWAFv2(t *testing.T) {
}

func TestAWSLoadBalancerControllerWithWAFRegional(t *testing.T) {
testWorkloadNamespace := "aws-load-balancer-test-wafregional"
t.Logf("Creating test namespace %q", testWorkloadNamespace)
echoNs := createTestNamespace(t, testWorkloadNamespace)
defer func() {
waitForDeletion(context.TODO(), t, kubeClient, echoNs, defaultTimeout)
}()

t.Log("Creating aws load balancer controller instance with default ingress class")

alb := newALBCBuilder().withAddons(albo.AWSAddonWAFv1).withRoleARNIf(stsModeRequested(), controllerRoleARN).build()
Expand All @@ -687,10 +736,9 @@ func TestAWSLoadBalancerControllerWithWAFRegional(t *testing.T) {
t.Fatalf("did not get expected available condition for deployment: %v", err)
}

testWorkloadNamespace := "aws-load-balancer-test-wafregional"
echoSvc, echoNs := createTestWorkload(t, testWorkloadNamespace)
echoSvc := createTestWorkload(t, testWorkloadNamespace)
defer func() {
waitForDeletion(context.TODO(), t, kubeClient, echoNs, defaultTimeout)
waitForDeletion(context.TODO(), t, kubeClient, echoSvc, defaultTimeout)
}()

var webACLID string
Expand Down Expand Up @@ -782,7 +830,13 @@ func TestAWSLoadBalancerControllerWithWAFRegional(t *testing.T) {
}

func TestIngressGroup(t *testing.T) {
ctx := context.TODO()
testWorkloadNamespace := "aws-load-balancer-test-ing-group"
t.Logf("Creating test namespace %q", testWorkloadNamespace)
echoNs := createTestNamespace(t, testWorkloadNamespace)
defer func() {
waitForDeletion(context.TODO(), t, kubeClient, echoNs, defaultTimeout)
}()

t.Logf("Creating a custom IngressClassParams")
ingressClassParams := &elbv1beta1.IngressClassParams{
ObjectMeta: v1.ObjectMeta{
Expand All @@ -795,7 +849,7 @@ func TestIngressGroup(t *testing.T) {
},
}

if err := kubeClient.Create(ctx, ingressClassParams); err != nil {
if err := kubeClient.Create(context.TODO(), ingressClassParams); err != nil {
t.Fatalf("failed to create IngressClassParams %s: %v", ingressClassParams.Name, err)
}
defer func() {
Expand Down Expand Up @@ -835,10 +889,10 @@ func TestIngressGroup(t *testing.T) {
t.Fatalf("did not get expected available condition for deployment: %v", err)
}

testWorkloadNamespace := "aws-load-balancer-test-custom-ing"
echoSvc, echoNs := createTestWorkload(t, testWorkloadNamespace)
t.Logf("Creating test workload in %q namespace", testWorkloadNamespace)
echoSvc := createTestWorkload(t, testWorkloadNamespace)
defer func() {
waitForDeletion(context.TODO(), t, kubeClient, echoNs, defaultTimeout)
waitForDeletion(context.TODO(), t, kubeClient, echoSvc, defaultTimeout)
}()

t.Log("Creating Ingress Resource 1 with custom ingress class")
Expand Down Expand Up @@ -921,6 +975,13 @@ func TestIngressGroup(t *testing.T) {
// "service.k8s.aws/nlb" load balancer class is used as the default for
// the service reconciliation done by aws-load-balancer-controller.
func TestAWSLoadBalancerControllerWithDefaultLoadBalancerClass(t *testing.T) {
testWorkloadNamespace := "aws-load-balancer-test-default-lb-class"
t.Logf("Creating test namespace %q", testWorkloadNamespace)
echoNs := createTestNamespace(t, testWorkloadNamespace)
defer func() {
waitForDeletion(context.TODO(), t, kubeClient, echoNs, defaultTimeout)
}()

t.Log("Creating aws load balancer controller instance")

alb := newALBCBuilder().withRoleARNIf(stsModeRequested(), controllerRoleARN).build()
Expand All @@ -939,8 +1000,7 @@ func TestAWSLoadBalancerControllerWithDefaultLoadBalancerClass(t *testing.T) {
t.Fatalf("did not get expected available condition for deployment: %v", err)
}

t.Log("Creating test workload")
testWorkloadNamespace := "aws-load-balancer-test-default-lb-class"
t.Logf("Creating test workload in %q namespace", testWorkloadNamespace)
customize := func(svc *corev1.Service) {
svc.Spec.Type = corev1.ServiceTypeLoadBalancer
svc.Spec.LoadBalancerClass = pointer.String("service.k8s.aws/nlb")
Expand All @@ -949,9 +1009,9 @@ func TestAWSLoadBalancerControllerWithDefaultLoadBalancerClass(t *testing.T) {
}
// by default ALBC uses instance target type if there is LoadBalancerClass
}
echoSvc, echoNs := createTestWorkloadWithCustomize(t, testWorkloadNamespace, customize)
echoSvc := createTestWorkloadWithCustomize(t, testWorkloadNamespace, customize)
defer func() {
waitForDeletion(context.TODO(), t, kubeClient, echoNs, defaultTimeout)
waitForDeletion(context.TODO(), t, kubeClient, echoSvc, defaultTimeout)
}()

address, err := getService(context.TODO(), t, kubeClient, defaultTimeout, types.NamespacedName{
Expand Down Expand Up @@ -980,6 +1040,13 @@ func TestAWSLoadBalancerControllerWithDefaultLoadBalancerClass(t *testing.T) {
// "service.k8s.aws/nlb" load balancer class is used as the default for
// the service reconciliation done by aws-load-balancer-controller.
func TestAWSLoadBalancerControllerWithInternalNLB(t *testing.T) {
testWorkloadNamespace := "aws-load-balancer-test-internal-nlb"
t.Logf("Creating test namespace %q", testWorkloadNamespace)
echoNs := createTestNamespace(t, testWorkloadNamespace)
defer func() {
waitForDeletion(context.TODO(), t, kubeClient, echoNs, defaultTimeout)
}()

t.Log("Creating aws load balancer controller instance")
alb := newALBCBuilder().withRoleARNIf(stsModeRequested(), controllerRoleARN).build()
if err := kubeClient.Create(context.TODO(), alb); err != nil {
Expand All @@ -997,17 +1064,16 @@ func TestAWSLoadBalancerControllerWithInternalNLB(t *testing.T) {
t.Fatalf("did not get expected available condition for deployment: %v", err)
}

t.Log("Creating test workload")
testWorkloadNamespace := "aws-load-balancer-test-internal-nlb"
t.Logf("Creating test workload in %q namespace", testWorkloadNamespace)
customize := func(svc *corev1.Service) {
svc.Spec.Type = corev1.ServiceTypeLoadBalancer
svc.Spec.LoadBalancerClass = pointer.String("service.k8s.aws/nlb")
// by default ALBC uses instance target type if there is LoadBalancerClass
// by default ALBC creates internal NLB
}
echoSvc, echoNs := createTestWorkloadWithCustomize(t, testWorkloadNamespace, customize)
echoSvc := createTestWorkloadWithCustomize(t, testWorkloadNamespace, customize)
defer func() {
waitForDeletion(context.TODO(), t, kubeClient, echoNs, defaultTimeout)
waitForDeletion(context.TODO(), t, kubeClient, echoSvc, defaultTimeout)
}()

address, err := getService(context.TODO(), t, kubeClient, defaultTimeout, types.NamespacedName{
Expand Down Expand Up @@ -1050,6 +1116,13 @@ func TestAWSLoadBalancerControllerWithInternalNLB(t *testing.T) {
// which uses the legacy "service.beta.kubernetes.io/aws-load-balancer-type" annotation as well as
// the usage of the service port different from the standard HTTP (80).
func TestAWSLoadBalancerControllerWithExternalTypeNLBAndNonStandardPort(t *testing.T) {
testWorkloadNamespace := "aws-load-balancer-test-lb-nonstd-port"
t.Logf("Creating test namespace %q", testWorkloadNamespace)
echoNs := createTestNamespace(t, testWorkloadNamespace)
defer func() {
waitForDeletion(context.TODO(), t, kubeClient, echoNs, defaultTimeout)
}()

t.Log("Creating aws load balancer controller instance")
alb := newALBCBuilder().withRoleARNIf(stsModeRequested(), controllerRoleARN).build()
if err := kubeClient.Create(context.TODO(), alb); err != nil {
Expand All @@ -1067,8 +1140,7 @@ func TestAWSLoadBalancerControllerWithExternalTypeNLBAndNonStandardPort(t *testi
t.Fatalf("did not get expected available condition for deployment: %v", err)
}

t.Log("Creating test workload")
testWorkloadNamespace := "aws-load-balancer-test-default-lb-class"
t.Logf("Creating test workload in %q namespace", testWorkloadNamespace)
nonStandardPort := int32(8880)
customize := func(svc *corev1.Service) {
svc.Spec.Type = corev1.ServiceTypeLoadBalancer
Expand All @@ -1079,9 +1151,9 @@ func TestAWSLoadBalancerControllerWithExternalTypeNLBAndNonStandardPort(t *testi
"service.beta.kubernetes.io/aws-load-balancer-scheme": "internet-facing",
}
}
echoSvc, echoNs := createTestWorkloadWithCustomize(t, testWorkloadNamespace, customize)
echoSvc := createTestWorkloadWithCustomize(t, testWorkloadNamespace, customize)
defer func() {
waitForDeletion(context.TODO(), t, kubeClient, echoNs, defaultTimeout)
waitForDeletion(context.TODO(), t, kubeClient, echoSvc, defaultTimeout)
}()

address, err := getService(context.TODO(), t, kubeClient, defaultTimeout, types.NamespacedName{
Expand Down Expand Up @@ -1167,24 +1239,17 @@ func ensureCredentialsRequest(secret types.NamespacedName) error {
return nil
}

func createTestWorkload(t *testing.T, namespace string) (*corev1.Service, *corev1.Namespace) {
func createTestWorkload(t *testing.T, namespace string) *corev1.Service {
t.Helper()
return createTestWorkloadWithCustomize(t, namespace, nil)
}

func createTestWorkloadWithCustomize(t *testing.T, namespace string, customize func(*corev1.Service)) (*corev1.Service, *corev1.Namespace) {
func createTestWorkloadWithCustomize(t *testing.T, namespace string, customize func(*corev1.Service)) *corev1.Service {
t.Helper()
t.Logf("Ensuring test workload namespace %s", namespace)
ns := &corev1.Namespace{ObjectMeta: v1.ObjectMeta{Name: namespace}}
err := kubeClient.Create(context.TODO(), ns)
if err != nil && !errors.IsAlreadyExists(err) {
t.Fatalf("failed to ensure namespace %s: %v", namespace, err)
}

echopod := buildEchoPod("echoserver", namespace)
err = kubeClient.Create(context.TODO(), echopod)
err := kubeClient.Create(context.TODO(), echopod)
if err != nil && !errors.IsAlreadyExists(err) {
t.Fatalf("failed to ensure pod %s: %v", echopod.Name, err)
t.Fatalf("failed to create pod %s: %v", echopod.Name, err)
}

echosvc := buildEchoService("echoserver", namespace)
Expand All @@ -1193,10 +1258,20 @@ func createTestWorkloadWithCustomize(t *testing.T, namespace string, customize f
}
err = kubeClient.Create(context.TODO(), echosvc)
if err != nil && !errors.IsAlreadyExists(err) {
t.Fatalf("failed to ensure service %s: %v", echosvc.Name, err)
t.Fatalf("failed to create service %s: %v", echosvc.Name, err)
}

return echosvc, ns
return echosvc
}

func createTestNamespace(t *testing.T, namespace string) *corev1.Namespace {
t.Helper()
ns := &corev1.Namespace{ObjectMeta: v1.ObjectMeta{Name: namespace}}
err := kubeClient.Create(context.TODO(), ns)
if err != nil && !errors.IsAlreadyExists(err) {
t.Fatalf("failed to create namespace %s: %v", namespace, err)
}
return ns
}

// stsModeRequested returns true if the specified e2e platform is STS enabled.
Expand Down

0 comments on commit b9932fb

Please sign in to comment.