Migrate upgrade tests to use the upgrade framework

kubernetes · Nov 20, 2019 · c98c394 · c98c394
1 parent a2e96e0
commit c98c394
Show file tree

Hide file tree

Showing 10 changed files with 386 additions and 280 deletions.
diff --git a/cmd/e2e-test/basic_https_test.go b/cmd/e2e-test/basic_https_test.go
@@ -22,6 +22,7 @@ import (
 	"testing"
 
 	"k8s.io/apimachinery/pkg/util/intstr"
+	"k8s.io/ingress-gce/cmd/e2e-test/common"
 	"k8s.io/ingress-gce/pkg/e2e"
 	"k8s.io/ingress-gce/pkg/fuzz"
 )
@@ -111,7 +112,7 @@ func TestBasicHTTPS(t *testing.T) {
 			t.Logf("GCLB resources created (%s/%s)", s.Namespace, ing.Name)
 
 			// Perform whitebox testing.
-			gclb := whiteboxTest(ing, s, t, "")
+			gclb := common.WhiteboxTest(ing, s, t, Framework.Cloud, "")
 
 			deleteOptions := &fuzz.GCLBDeleteOptions{
 				SkipDefaultBackend: true,

diff --git a/cmd/e2e-test/basic_test.go b/cmd/e2e-test/basic_test.go
@@ -23,6 +23,7 @@ import (
 
 	"k8s.io/api/networking/v1beta1"
 	"k8s.io/apimachinery/pkg/util/intstr"
+	"k8s.io/ingress-gce/cmd/e2e-test/common"
 	"k8s.io/ingress-gce/pkg/e2e"
 	"k8s.io/ingress-gce/pkg/fuzz"
 	"k8s.io/ingress-gce/pkg/fuzz/features"
@@ -83,7 +84,7 @@ func TestBasic(t *testing.T) {
 			t.Logf("GCLB resources createdd (%s/%s)", s.Namespace, tc.ing.Name)
 
 			// Perform whitebox testing.
-			gclb := whiteboxTest(ing, s, t, "")
+			gclb := common.WhiteboxTest(ing, s, t, Framework.Cloud, "")
 
 			deleteOptions := &fuzz.GCLBDeleteOptions{
 				SkipDefaultBackend: true,
@@ -191,7 +192,7 @@ func TestEdge(t *testing.T) {
 			t.Logf("GCLB resources createdd (%s/%s)", s.Namespace, tc.ing.Name)
 
 			// Perform whitebox testing.
-			gclb := whiteboxTest(ing, s, t, "")
+			gclb := common.WhiteboxTest(ing, s, t, Framework.Cloud, "")
 
 			deleteOptions := &fuzz.GCLBDeleteOptions{
 				SkipDefaultBackend: true,
@@ -202,40 +203,3 @@ func TestEdge(t *testing.T) {
 		})
 	}
 }
-
-func waitForStableIngress(expectUnreachable bool, ing *v1beta1.Ingress, s *e2e.Sandbox, t *testing.T) *v1beta1.Ingress {
-	options := &e2e.WaitForIngressOptions{
-		ExpectUnreachable: expectUnreachable,
-	}
-
-	ing, err := e2e.WaitForIngress(s, ing, options)
-	if err != nil {
-		t.Fatalf("error waiting for Ingress to stabilize: %v", err)
-	}
-
-	s.PutStatus(e2e.Stable)
-	return ing
-}
-
-func whiteboxTest(ing *v1beta1.Ingress, s *e2e.Sandbox, t *testing.T, region string) *fuzz.GCLB {
-	if len(ing.Status.LoadBalancer.Ingress) < 1 {
-		t.Fatalf("Ingress does not have an IP: %+v", ing.Status)
-	}
-
-	vip := ing.Status.LoadBalancer.Ingress[0].IP
-	t.Logf("Ingress %s/%s VIP = %s", s.Namespace, ing.Name, vip)
-	params := &fuzz.GCLBForVIPParams{
-		VIP:        vip,
-		Region:     region,
-		Validators: fuzz.FeatureValidators(features.All),
-	}
-	gclb, err := fuzz.GCLBForVIP(context.Background(), Framework.Cloud, params)
-	if err != nil {
-		t.Fatalf("Error getting GCP resources for LB with IP = %q: %v", vip, err)
-	}
-
-	if err := e2e.PerformWhiteboxTests(s, ing, gclb); err != nil {
-		t.Fatalf("Error performing whitebox tests: %v", err)
-	}
-	return gclb
-}
diff --git a/cmd/e2e-test/common/common.go b/cmd/e2e-test/common/common.go
@@ -0,0 +1,65 @@
+/*
+Copyright 2019 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package common
+
+import (
+	"context"
+	"testing"
+
+	"github.com/GoogleCloudPlatform/k8s-cloud-provider/pkg/cloud"
+	"k8s.io/api/networking/v1beta1"
+	"k8s.io/ingress-gce/pkg/e2e"
+	"k8s.io/ingress-gce/pkg/fuzz"
+	"k8s.io/ingress-gce/pkg/fuzz/features"
+)
+
+func WaitForStableIngress(expectUnreachable bool, ing *v1beta1.Ingress, s *e2e.Sandbox, t *testing.T) *v1beta1.Ingress {
+	options := &e2e.WaitForIngressOptions{
+		ExpectUnreachable: expectUnreachable,
+	}
+
+	ing, err := e2e.WaitForIngress(s, ing, options)
+	if err != nil {
+		t.Fatalf("error waiting for Ingress to stabilize: %v", err)
+	}
+
+	s.PutStatus(e2e.Stable)
+	return ing
+}
+
+func WhiteboxTest(ing *v1beta1.Ingress, s *e2e.Sandbox, t *testing.T, cloud cloud.Cloud, region string) *fuzz.GCLB {
+	if len(ing.Status.LoadBalancer.Ingress) < 1 {
+		t.Fatalf("Ingress does not have an IP: %+v", ing.Status)
+	}
+
+	vip := ing.Status.LoadBalancer.Ingress[0].IP
+	t.Logf("Ingress %s/%s VIP = %s", s.Namespace, ing.Name, vip)
+	params := &fuzz.GCLBForVIPParams{
+		VIP:        vip,
+		Region:     region,
+		Validators: fuzz.FeatureValidators(features.All),
+	}
+	gclb, err := fuzz.GCLBForVIP(context.Background(), cloud, params)
+	if err != nil {
+		t.Fatalf("Error getting GCP resources for LB with IP = %q: %v", vip, err)
+	}
+
+	if err := e2e.PerformWhiteboxTests(s, ing, gclb); err != nil {
+		t.Fatalf("Error performing whitebox tests: %v", err)
+	}
+	return gclb
+}
diff --git a/cmd/e2e-test/finalizer_test.go b/cmd/e2e-test/finalizer_test.go
@@ -18,14 +18,13 @@ package main
 
 import (
 	"context"
-	"k8s.io/api/networking/v1beta1"
-	"k8s.io/ingress-gce/pkg/fuzz/features"
 	"testing"
 
 	"k8s.io/apimachinery/pkg/util/intstr"
+	"k8s.io/ingress-gce/cmd/e2e-test/common"
 	"k8s.io/ingress-gce/pkg/e2e"
 	"k8s.io/ingress-gce/pkg/fuzz"
-	"k8s.io/ingress-gce/pkg/utils/common"
+	finalizer "k8s.io/ingress-gce/pkg/utils/common"
 )
 
 // TestFinalizer asserts that finalizer is added/ removed appropriately during the life cycle
@@ -53,15 +52,15 @@ func TestFinalizer(t *testing.T) {
 			t.Fatalf("create(%s/%s) = %v, want nil; Ingress: %v", ing.Namespace, ing.Name, err, ing)
 		}
 		t.Logf("Ingress created (%s/%s)", s.Namespace, ing.Name)
-		ing = waitForStableIngress(true, ing, s, t)
+		ing = common.WaitForStableIngress(true, ing, s, t)
 
 		ingFinalizers := ing.GetFinalizers()
-		if len(ingFinalizers) != 1 || ingFinalizers[0] != common.FinalizerKey {
-			t.Fatalf("GetFinalizers() = %+v, want [%q]", ingFinalizers, common.FinalizerKey)
+		if len(ingFinalizers) != 1 || ingFinalizers[0] != finalizer.FinalizerKey {
+			t.Fatalf("GetFinalizers() = %+v, want [%q]", ingFinalizers, finalizer.FinalizerKey)
 		}
 
 		// Perform whitebox testing.
-		gclb := whiteboxTest(ing, s, t, "")
+		gclb := common.WhiteboxTest(ing, s, t, Framework.Cloud, "")
 
 		deleteOptions := &fuzz.GCLBDeleteOptions{
 			SkipDefaultBackend: true,
@@ -97,15 +96,15 @@ func TestFinalizerIngressClassChange(t *testing.T) {
 			t.Fatalf("create(%s/%s) = %v, want nil; Ingress: %v", ing.Namespace, ing.Name, err, ing)
 		}
 		t.Logf("Ingress created (%s/%s)", s.Namespace, ing.Name)
-		ing = waitForStableIngress(true, ing, s, t)
+		ing = common.WaitForStableIngress(true, ing, s, t)
 
 		ingFinalizers := ing.GetFinalizers()
-		if len(ingFinalizers) != 1 || ingFinalizers[0] != common.FinalizerKey {
-			t.Fatalf("GetFinalizers() = %+v, want [%q]", ingFinalizers, common.FinalizerKey)
+		if len(ingFinalizers) != 1 || ingFinalizers[0] != finalizer.FinalizerKey {
+			t.Fatalf("GetFinalizers() = %+v, want [%q]", ingFinalizers, finalizer.FinalizerKey)
 		}
 
 		// Perform whitebox testing.
-		gclb := whiteboxTest(ing, s, t, "")
+		gclb := common.WhiteboxTest(ing, s, t, Framework.Cloud, "")
 
 		// Change Ingress class
 		newIngClass := "nginx"
@@ -162,21 +161,21 @@ func TestFinalizerIngressesWithSharedResources(t *testing.T) {
 		}
 		t.Logf("Ingress created (%s/%s)", s.Namespace, otherIng.Name)
 
-		ing = waitForStableIngress(true, ing, s, t)
-		otherIng = waitForStableIngress(true, otherIng, s, t)
+		ing = common.WaitForStableIngress(true, ing, s, t)
+		otherIng = common.WaitForStableIngress(true, otherIng, s, t)
 
 		ingFinalizers := ing.GetFinalizers()
-		if len(ingFinalizers) != 1 || ingFinalizers[0] != common.FinalizerKey {
-			t.Fatalf("GetFinalizers() = %+v, want [%q]", ingFinalizers, common.FinalizerKey)
+		if len(ingFinalizers) != 1 || ingFinalizers[0] != finalizer.FinalizerKey {
+			t.Fatalf("GetFinalizers() = %+v, want [%q]", ingFinalizers, finalizer.FinalizerKey)
 		}
 		otherIngFinalizers := otherIng.GetFinalizers()
-		if len(otherIngFinalizers) != 1 || otherIngFinalizers[0] != common.FinalizerKey {
-			t.Fatalf("GetFinalizers() = %+v, want [%q]", otherIngFinalizers, common.FinalizerKey)
+		if len(otherIngFinalizers) != 1 || otherIngFinalizers[0] != finalizer.FinalizerKey {
+			t.Fatalf("GetFinalizers() = %+v, want [%q]", otherIngFinalizers, finalizer.FinalizerKey)
 		}
 
 		// Perform whitebox testing.
-		gclb := whiteboxTest(ing, s, t, "")
-		otherGclb := whiteboxTest(otherIng, s, t, "")
+		gclb := common.WhiteboxTest(ing, s, t, Framework.Cloud, "")
+		otherGclb := common.WhiteboxTest(otherIng, s, t, Framework.Cloud, "")
 
 		// SkipBackends ensure that we dont wait on deletion of shared backends.
 		deleteOptions := &fuzz.GCLBDeleteOptions{
@@ -194,92 +193,3 @@ func TestFinalizerIngressesWithSharedResources(t *testing.T) {
 		}
 	})
 }
-
-// TestUpdateTo1dot7 asserts that finalizer is added to an ingress when upgraded from a version
-// without finalizer to version 1.7. Also, verifies that ingress is deleted with finalizer enabled.
-// Note: The test is named in such a way that it does run as a normal test or an upgrade test for
-// other versions.
-func TestUpdateTo1dot7(t *testing.T) {
-	port80 := intstr.FromInt(80)
-	svcName := "service-1"
-	ing := fuzz.NewIngressBuilder("", "ingress-1", "").
-		AddPath("foo.com", "/", svcName, port80).
-		SetIngressClass("gce").
-		Build()
-	Framework.RunWithSandbox("finalizer-master-upgrade", t, func(t *testing.T, s *e2e.Sandbox) {
-		t.Parallel()
-
-		_, err := e2e.CreateEchoService(s, svcName, nil)
-		if err != nil {
-			t.Fatalf("CreateEchoService(_, %q, nil): %v, want nil", svcName, err)
-		}
-		t.Logf("Echo service created (%s/%s)", s.Namespace, svcName)
-
-		crud := e2e.IngressCRUD{C: Framework.Clientset}
-		ing.Namespace = s.Namespace
-		if _, err := crud.Create(ing); err != nil {
-			t.Fatalf("create(%s/%s) = %v, want nil; Ingress: %v", ing.Namespace, ing.Name, err, ing)
-		}
-		t.Logf("Ingress created (%s/%s)", s.Namespace, ing.Name)
-		ing = waitForStableIngress(true, ing, s, t)
-
-		// Check that finalizer is not added in old version in which finalizer add is not enabled.
-		ingFinalizers := ing.GetFinalizers()
-		if l := len(ingFinalizers); l != 0 {
-			t.Fatalf("GetFinalizers() = %d, want 0", l)
-		}
-		// Perform whitebox testing.
-		whiteboxTest(ing, s, t, "")
-
-		for {
-			// While k8s master is upgrading, it will return a connection refused
-			// error for any k8s resource we try to hit. We loop until the
-			// master upgrade has finished.
-			if s.MasterUpgrading() {
-				continue
-			}
-
-			if s.MasterUpgraded() {
-				t.Logf("Detected master upgrade")
-				break
-			}
-		}
-
-		// Wait for finalizer to be added and verify that correct finalizer is added to the ingress after the upgrade.
-		if err := e2e.WaitForFinalizer(s, ing.Name); err != nil {
-			t.Errorf("e2e.WaitForFinalizer(_, %q) = %v, want nil", ing.Name, err)
-		}
-
-		// Perform whitebox testing.
-		gclb := whiteboxTest(ing, s, t, "")
-
-		// If the Master has upgraded and the Ingress is stable,
-		// we delete the Ingress and exit out of the loop to indicate that
-		// the test is done.
-		deleteOptions := &fuzz.GCLBDeleteOptions{
-			SkipDefaultBackend: true,
-		}
-		if err := e2e.WaitForIngressDeletion(context.Background(), gclb, s, ing, deleteOptions); err != nil {
-			t.Errorf("e2e.WaitForIngressDeletion(..., %q, nil) = %v, want nil", ing.Name, err)
-		}
-	})
-}
-
-func checkGCLB(t *testing.T, s *e2e.Sandbox, ing *v1beta1.Ingress, numForwardingRules, numBackendServices int) *fuzz.GCLB {
-	// Perform whitebox testing.
-	if len(ing.Status.LoadBalancer.Ingress) < 1 {
-		t.Fatalf("Ingress does not have an IP: %+v", ing.Status)
-	}
-	vip := ing.Status.LoadBalancer.Ingress[0].IP
-	t.Logf("Ingress %s/%s VIP = %s", s.Namespace, ing.Name, vip)
-	params := &fuzz.GCLBForVIPParams{VIP: vip, Validators: fuzz.FeatureValidators(features.All)}
-	gclb, err := fuzz.GCLBForVIP(context.Background(), Framework.Cloud, params)
-	if err != nil {
-		t.Fatalf("GCLBForVIP(..., %q, _) = %v, want nil; error getting GCP resources for LB with IP", vip, err)
-	}
-
-	if err = e2e.CheckGCLB(gclb, numForwardingRules, numBackendServices); err != nil {
-		t.Error(err)
-	}
-	return gclb
-}
diff --git a/cmd/e2e-test/neg_test.go b/cmd/e2e-test/neg_test.go
@@ -25,6 +25,7 @@ import (
 	"k8s.io/api/networking/v1beta1"
 	"k8s.io/apimachinery/pkg/util/intstr"
 	"k8s.io/apimachinery/pkg/util/sets"
+	"k8s.io/ingress-gce/cmd/e2e-test/common"
 	"k8s.io/ingress-gce/pkg/annotations"
 	"k8s.io/ingress-gce/pkg/e2e"
 	"k8s.io/ingress-gce/pkg/fuzz"
@@ -116,7 +117,7 @@ func TestNEG(t *testing.T) {
 			t.Logf("GCLB resources created (%s/%s)", s.Namespace, ing.Name)
 
 			// Perform whitebox testing.
-			gclb := whiteboxTest(ing, s, t, "")
+			gclb := common.WhiteboxTest(ing, s, t, Framework.Cloud, "")
 
 			// TODO(mixia): The below checks should be merged into PerformWhiteboxTests().
 			if (len(gclb.NetworkEndpointGroup) > 0) != tc.expectNegBackend {