Update e2e framework for ILB

cmd/e2e-test/ilb_test.go

@@ -0,0 +1,109 @@
+/*
+Copyright 2019 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package main
+
+import (
+	"k8s.io/api/networking/v1beta1"
+	"k8s.io/apimachinery/pkg/util/intstr"
+	"k8s.io/ingress-gce/pkg/annotations"
+	"k8s.io/ingress-gce/pkg/e2e"
+	"k8s.io/ingress-gce/pkg/fuzz"
+	"testing"
+)
+
+func TestILBBasic(t *testing.T) {
+	t.Parallel()
+
+	port80 := intstr.FromInt(80)
+
+	for _, tc := range []struct {
+		desc string
+		ing  *v1beta1.Ingress
+
+		numForwardingRules int
+		numBackendServices int
+	}{
+		{
+			desc: "http ILB default backend",
+			ing: fuzz.NewIngressBuilder("", "ingress-1", "").
+				DefaultBackend("service-1", port80).
+				ConfigureForILB().
+				Build(),
+			numForwardingRules: 1,
+			numBackendServices: 1,
+		},
+		{
+			desc: "http ILB one path",
+			ing: fuzz.NewIngressBuilder("", "ingress-1", "").
+				AddPath("test.com", "/", "service-1", port80).
+				ConfigureForILB().
+				Build(),
+			numForwardingRules: 1,
+			numBackendServices: 2,
+		},
+		{
+			desc: "http ILB multiple paths",
+			ing: fuzz.NewIngressBuilder("", "ingress-1", "").
+				AddPath("test.com", "/foo", "service-1", port80).
+				AddPath("test.com", "/bar", "service-1", port80).
+				ConfigureForILB().
+				Build(),
+			numForwardingRules: 1,
+			numBackendServices: 2,
+		},
+	} {
+		tc := tc // Capture tc as we are running this in parallel.
+		Framework.RunWithSandbox(tc.desc, t, func(t *testing.T, s *e2e.Sandbox) {
+			t.Parallel()
+
+			t.Logf("Ingress = %s", tc.ing.String())
+
+			negAnnotation := annotations.NegAnnotation{Ingress: true}
+			annotation := map[string]string{annotations.NEGAnnotationKey: negAnnotation.String()}
+
+			_, err := e2e.CreateEchoService(s, "service-1", annotation)
+			if err != nil {
+				t.Fatalf("error creating echo service: %v", err)
+			}
+			t.Logf("Echo service created (%s/%s)", s.Namespace, "service-1")
+
+			if _, err := Framework.Clientset.NetworkingV1beta1().Ingresses(s.Namespace).Create(tc.ing); err != nil {
+				t.Fatalf("error creating Ingress spec: %v", err)
+			}
+			t.Logf("Ingress created (%s/%s)", s.Namespace, tc.ing.Name)
+
+			ing, err := e2e.WaitForIngress(s, tc.ing, nil)
+			if err != nil {
+				t.Fatalf("error waiting for Ingress to stabilize: %v", err)
+			}
+			t.Logf("GCLB resources createdd (%s/%s)", s.Namespace, tc.ing.Name)
+
+			// Perform whitebox testing.
+			if len(ing.Status.LoadBalancer.Ingress) < 1 {
+				t.Fatalf("Ingress does not have an IP: %+v", ing.Status)
+			}
+
+			vip := ing.Status.LoadBalancer.Ingress[0].IP
+			t.Logf("Ingress %s/%s VIP = %s", s.Namespace, tc.ing.Name, vip)
+			if !e2e.IsRfc1918Addr(vip) {
+				t.Fatalf("got %v, want RFC1918 address, ing: %v", vip, ing)
+			}
+
+			// TODO(shance): update gcp.go for regional resources so that we can check GC here
+		})
+	}
+}

pkg/e2e/helpers.go

@@ -19,6 +19,7 @@ package e2e
 import (
 	"context"
 	"fmt"
+	"net"
 	"time"
 
 	"encoding/json"
@@ -64,6 +65,28 @@ type WaitForIngressOptions struct {
 	ExpectUnreachable bool
 }
 
+// IsRfc1918Addr returns true if the address supplied is an RFC1918 address
+func IsRfc1918Addr(addr string) bool {
+	ip := net.ParseIP(addr)
+	var ipBlocks []*net.IPNet
+	for _, cidr := range []string{
+		"10.0.0.0/8",
+		"172.16.0.0/12",
+		"192.168.0.0/16",
+	} {
+		_, block, _ := net.ParseCIDR(cidr)
+		ipBlocks = append(ipBlocks, block)
+	}
+
+	for _, block := range ipBlocks {
+		if block.Contains(ip) {
+			return true
+		}
+	}
+
+	return false
+}
+
 // WaitForIngress to stabilize.
 // We expect the ingress to be unreachable at first as LB is
 // still programming itself (i.e 404's / 502's)

pkg/fuzz/features/features.go

@@ -31,4 +31,5 @@ var All = []fuzz.Feature{
 	Affinity,
 	NEG,
 	AppProtocol,
+	ILB,
 }

pkg/fuzz/features/ilb.go

@@ -0,0 +1,65 @@
+/*
+Copyright 2019 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package features
+
+import (
+	"k8s.io/api/networking/v1beta1"
+	"k8s.io/ingress-gce/pkg/fuzz"
+	"net/http"
+)
+
+// ILB is an internal load balancer
+var ILB = &ILBFeature{}
+
+// ILBFeature implements the associated feature
+type ILBFeature struct{}
+
+// NewValidator implements fuzz.Feature.
+func (*ILBFeature) NewValidator() fuzz.FeatureValidator {
+	return &ILBValidator{}
+}
+
+// Name implements fuzz.Feature.
+func (*ILBFeature) Name() string {
+	return "ILB"
+}
+
+// ILBValidator is an example validator.
+type ILBValidator struct {
+	fuzz.NullValidator
+
+	ing *v1beta1.Ingress
+	env fuzz.ValidatorEnv
+}
+
+// Name implements fuzz.FeatureValidator.
+func (*ILBValidator) Name() string {
+	return "ILB"
+}
+
+// ConfigureAttributes implements fuzz.FeatureValidator.
+func (v *ILBValidator) ConfigureAttributes(env fuzz.ValidatorEnv, ing *v1beta1.Ingress, a *fuzz.IngressValidatorAttributes) error {
+	// Capture the env for use later in CheckResponse.
+	v.ing = ing
+	v.env = env
+	return nil
+}
+
+// CheckResponse implements fuzz.FeatureValidator.
+func (v *ILBValidator) CheckResponse(host, path string, resp *http.Response, body []byte) (fuzz.CheckResponseAction, error) {
+	return fuzz.CheckResponseContinue, nil
+}

pkg/fuzz/features/neg.go

@@ -23,6 +23,7 @@ package features
 import (
 	"context"
 	"fmt"
+	"k8s.io/klog"
 	"net/http"
 	"strconv"
 	"strings"
@@ -92,6 +93,9 @@ func (v *negValidator) CheckResponse(host, path string, resp *http.Response, bod
 
 	urlMapName := v.env.Namer().UrlMap(v.env.Namer().LoadBalancer(key))
 	if negEnabled {
+		if utils.IsGCEL7ILBIngress(v.ing) {
+			return fuzz.CheckResponseContinue, verifyNegRegionBackend(v.env, negName, negName, urlMapName)
+		}
 		return fuzz.CheckResponseContinue, verifyNegBackend(v.env, negName, urlMapName)
 	} else {
 		return fuzz.CheckResponseContinue, verifyIgBackend(v.env, v.env.Namer().IGBackend(int64(svcPort.NodePort)), urlMapName)
@@ -143,6 +147,7 @@ func verifyIgBackend(env fuzz.ValidatorEnv, bsName string, urlMapName string) er
 
 // verifyBackend verifies the backend service and check if the corresponding backend group has the keyword
 func verifyBackend(env fuzz.ValidatorEnv, bsName string, backendKeyword string, urlMapName string) error {
+	klog.V(3).Info("Verifying NEG Global Backend")
 	ctx := context.Background()
 	beService, err := env.Cloud().BackendServices().Get(ctx, &meta.Key{Name: bsName})
 	if err != nil {
@@ -178,3 +183,42 @@ func verifyBackend(env fuzz.ValidatorEnv, bsName string, backendKeyword string,
 
 	return fmt.Errorf("backend service %q is not used by UrlMap %q", bsName, urlMapName)
 }
+
+// verifyBackend verifies the backend service and check if the corresponding backend group has the keyword
+func verifyNegRegionBackend(env fuzz.ValidatorEnv, bsName string, backendKeyword string, urlMapName string) error {
+	klog.V(3).Info("Verifying NEG Regional Backend")
+	ctx := context.Background()
+	beService, err := env.Cloud().AlphaRegionBackendServices().Get(ctx, &meta.Key{Name: bsName, Region: "us-central1"})
+	if err != nil {
+		return err
+	}
+
+	if beService == nil {
+		return fmt.Errorf("no backend service returned for name %s", bsName)
+	}
+
+	for _, be := range beService.Backends {
+		if !strings.Contains(be.Group, backendKeyword) {
+			return fmt.Errorf("backend group %q of backend service %q does not contain keyword %q", be.Group, bsName, backendKeyword)
+		}
+	}
+
+	// Examine if ingress url map is targeting the backend service
+	urlMap, err := env.Cloud().AlphaRegionUrlMaps().Get(ctx, &meta.Key{Name: urlMapName, Region: "us-central1"})
+	if err != nil {
+		return err
+	}
+
+	if strings.Contains(urlMap.DefaultService, beService.Name) {
+		return nil
+	}
+	for _, pathMatcher := range urlMap.PathMatchers {
+		for _, rule := range pathMatcher.PathRules {
+			if strings.Contains(rule.Service, beService.Name) {
+				return nil
+			}
+		}
+	}
+
+	return fmt.Errorf("backend service %q is not used by UrlMap %q", bsName, urlMapName)
+}

pkg/fuzz/helpers.go

@@ -267,6 +267,15 @@ func (i *IngressBuilder) SetIngressClass(name string) *IngressBuilder {
 	return i
 }
 
+// Configure for ILB adds the ILB ingress class annotation
+func (i *IngressBuilder) ConfigureForILB() *IngressBuilder {
+	if i.ing.Annotations == nil {
+		i.ing.Annotations = make(map[string]string)
+	}
+	i.ing.Annotations[annotations.IngressClassKey] = annotations.GceL7ILBIngressClass
+	return i
+}
+
 // BackendConfigBuilder is syntactic sugar for creating BackendConfig specs for testing
 // purposes.
 //

pkg/fuzz/validator.go

@@ -210,9 +210,9 @@ type IngressValidator struct {
 // the right SSL certificate is presented
 // each path, each host returns the right contents
 
-// vip for the load balancer. This currently uses the first entry, returns nil
+// Vip for the load balancer. This currently uses the first entry, returns nil
 // if the VIP is not available.
-func (v *IngressValidator) vip() *string {
+func (v *IngressValidator) Vip() *string {
 	statuses := v.ing.Status.LoadBalancer.Ingress
 	if len(statuses) == 0 {
 		return nil
@@ -292,10 +292,10 @@ func (v *IngressValidator) CheckPaths(ctx context.Context, vr *IngressResult) er
 
 // checkPath performs a check for scheme://host/path.
 func (v *IngressValidator) checkPath(ctx context.Context, scheme, host, path string) error {
-	if v.vip() == nil {
+	if v.Vip() == nil {
 		return fmt.Errorf("ingress %s/%s does not have a VIP", v.ing.Namespace, v.ing.Name)
 	}
-	vip := *v.vip()
+	vip := *v.Vip()
 
 	url := fmt.Sprintf("%s://%s%s%s", scheme, vip, portStr(v.attribs, scheme), path)
 	klog.V(3).Infof("Checking Ingress %s/%s url=%q", v.ing.Namespace, v.ing.Name, url)