From 40dfd94e126bcc5e22c1d3f8e178d946b7bc5911 Mon Sep 17 00:00:00 2001 From: Spencer Hance Date: Thu, 8 Aug 2019 21:16:26 -0700 Subject: [PATCH] Update e2e framework for ILB --- cmd/e2e-test/ilb_test.go | 109 ++++++++++++++++++++++++++++++++++ pkg/e2e/helpers.go | 23 +++++++ pkg/fuzz/features/features.go | 1 + pkg/fuzz/features/ilb.go | 65 ++++++++++++++++++++ pkg/fuzz/features/neg.go | 44 ++++++++++++++ pkg/fuzz/helpers.go | 9 +++ pkg/fuzz/validator.go | 8 +-- 7 files changed, 255 insertions(+), 4 deletions(-) create mode 100644 cmd/e2e-test/ilb_test.go create mode 100644 pkg/fuzz/features/ilb.go diff --git a/cmd/e2e-test/ilb_test.go b/cmd/e2e-test/ilb_test.go new file mode 100644 index 0000000000..21c0e97b40 --- /dev/null +++ b/cmd/e2e-test/ilb_test.go @@ -0,0 +1,109 @@ +/* +Copyright 2019 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package main + +import ( + "k8s.io/api/networking/v1beta1" + "k8s.io/apimachinery/pkg/util/intstr" + "k8s.io/ingress-gce/pkg/annotations" + "k8s.io/ingress-gce/pkg/e2e" + "k8s.io/ingress-gce/pkg/fuzz" + "testing" +) + +func TestILBBasic(t *testing.T) { + t.Parallel() + + port80 := intstr.FromInt(80) + + for _, tc := range []struct { + desc string + ing *v1beta1.Ingress + + numForwardingRules int + numBackendServices int + }{ + { + desc: "http ILB default backend", + ing: fuzz.NewIngressBuilder("", "ingress-1", ""). + DefaultBackend("service-1", port80). + ConfigureForILB(). + Build(), + numForwardingRules: 1, + numBackendServices: 1, + }, + { + desc: "http ILB one path", + ing: fuzz.NewIngressBuilder("", "ingress-1", ""). + AddPath("test.com", "/", "service-1", port80). + ConfigureForILB(). + Build(), + numForwardingRules: 1, + numBackendServices: 2, + }, + { + desc: "http ILB multiple paths", + ing: fuzz.NewIngressBuilder("", "ingress-1", ""). + AddPath("test.com", "/foo", "service-1", port80). + AddPath("test.com", "/bar", "service-1", port80). + ConfigureForILB(). + Build(), + numForwardingRules: 1, + numBackendServices: 2, + }, + } { + tc := tc // Capture tc as we are running this in parallel. + Framework.RunWithSandbox(tc.desc, t, func(t *testing.T, s *e2e.Sandbox) { + t.Parallel() + + t.Logf("Ingress = %s", tc.ing.String()) + + negAnnotation := annotations.NegAnnotation{Ingress: true} + annotation := map[string]string{annotations.NEGAnnotationKey: negAnnotation.String()} + + _, err := e2e.CreateEchoService(s, "service-1", annotation) + if err != nil { + t.Fatalf("error creating echo service: %v", err) + } + t.Logf("Echo service created (%s/%s)", s.Namespace, "service-1") + + if _, err := Framework.Clientset.NetworkingV1beta1().Ingresses(s.Namespace).Create(tc.ing); err != nil { + t.Fatalf("error creating Ingress spec: %v", err) + } + t.Logf("Ingress created (%s/%s)", s.Namespace, tc.ing.Name) + + ing, err := e2e.WaitForIngress(s, tc.ing, nil) + if err != nil { + t.Fatalf("error waiting for Ingress to stabilize: %v", err) + } + t.Logf("GCLB resources createdd (%s/%s)", s.Namespace, tc.ing.Name) + + // Perform whitebox testing. + if len(ing.Status.LoadBalancer.Ingress) < 1 { + t.Fatalf("Ingress does not have an IP: %+v", ing.Status) + } + + vip := ing.Status.LoadBalancer.Ingress[0].IP + t.Logf("Ingress %s/%s VIP = %s", s.Namespace, tc.ing.Name, vip) + if !e2e.IsRfc1918Addr(vip) { + t.Fatalf("got %v, want RFC1918 address, ing: %v", vip, ing) + } + + // TODO(shance): update gcp.go for regional resources so that we can check GC here + }) + } +} diff --git a/pkg/e2e/helpers.go b/pkg/e2e/helpers.go index 6e64705e1f..f437f10451 100644 --- a/pkg/e2e/helpers.go +++ b/pkg/e2e/helpers.go @@ -19,6 +19,7 @@ package e2e import ( "context" "fmt" + "net" "time" "encoding/json" @@ -64,6 +65,28 @@ type WaitForIngressOptions struct { ExpectUnreachable bool } +// IsRfc1918Addr returns true if the address supplied is an RFC1918 address +func IsRfc1918Addr(addr string) bool { + ip := net.ParseIP(addr) + var ipBlocks []*net.IPNet + for _, cidr := range []string{ + "10.0.0.0/8", + "172.16.0.0/12", + "192.168.0.0/16", + } { + _, block, _ := net.ParseCIDR(cidr) + ipBlocks = append(ipBlocks, block) + } + + for _, block := range ipBlocks { + if block.Contains(ip) { + return true + } + } + + return false +} + // WaitForIngress to stabilize. // We expect the ingress to be unreachable at first as LB is // still programming itself (i.e 404's / 502's) diff --git a/pkg/fuzz/features/features.go b/pkg/fuzz/features/features.go index cdb9410d40..9299ff0c86 100644 --- a/pkg/fuzz/features/features.go +++ b/pkg/fuzz/features/features.go @@ -31,4 +31,5 @@ var All = []fuzz.Feature{ Affinity, NEG, AppProtocol, + ILB, } diff --git a/pkg/fuzz/features/ilb.go b/pkg/fuzz/features/ilb.go new file mode 100644 index 0000000000..0e54644772 --- /dev/null +++ b/pkg/fuzz/features/ilb.go @@ -0,0 +1,65 @@ +/* +Copyright 2019 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package features + +import ( + "k8s.io/api/networking/v1beta1" + "k8s.io/ingress-gce/pkg/fuzz" + "net/http" +) + +// ILB is an internal load balancer +var ILB = &ILBFeature{} + +// ILBFeature implements the associated feature +type ILBFeature struct{} + +// NewValidator implements fuzz.Feature. +func (*ILBFeature) NewValidator() fuzz.FeatureValidator { + return &ILBValidator{} +} + +// Name implements fuzz.Feature. +func (*ILBFeature) Name() string { + return "ILB" +} + +// ILBValidator is an example validator. +type ILBValidator struct { + fuzz.NullValidator + + ing *v1beta1.Ingress + env fuzz.ValidatorEnv +} + +// Name implements fuzz.FeatureValidator. +func (*ILBValidator) Name() string { + return "ILB" +} + +// ConfigureAttributes implements fuzz.FeatureValidator. +func (v *ILBValidator) ConfigureAttributes(env fuzz.ValidatorEnv, ing *v1beta1.Ingress, a *fuzz.IngressValidatorAttributes) error { + // Capture the env for use later in CheckResponse. + v.ing = ing + v.env = env + return nil +} + +// CheckResponse implements fuzz.FeatureValidator. +func (v *ILBValidator) CheckResponse(host, path string, resp *http.Response, body []byte) (fuzz.CheckResponseAction, error) { + return fuzz.CheckResponseContinue, nil +} diff --git a/pkg/fuzz/features/neg.go b/pkg/fuzz/features/neg.go index 651e938897..a49b4478fa 100644 --- a/pkg/fuzz/features/neg.go +++ b/pkg/fuzz/features/neg.go @@ -23,6 +23,7 @@ package features import ( "context" "fmt" + "k8s.io/klog" "net/http" "strconv" "strings" @@ -92,6 +93,9 @@ func (v *negValidator) CheckResponse(host, path string, resp *http.Response, bod urlMapName := v.env.Namer().UrlMap(v.env.Namer().LoadBalancer(key)) if negEnabled { + if utils.IsGCEL7ILBIngress(v.ing) { + return fuzz.CheckResponseContinue, verifyNegRegionBackend(v.env, negName, negName, urlMapName) + } return fuzz.CheckResponseContinue, verifyNegBackend(v.env, negName, urlMapName) } else { return fuzz.CheckResponseContinue, verifyIgBackend(v.env, v.env.Namer().IGBackend(int64(svcPort.NodePort)), urlMapName) @@ -143,6 +147,7 @@ func verifyIgBackend(env fuzz.ValidatorEnv, bsName string, urlMapName string) er // verifyBackend verifies the backend service and check if the corresponding backend group has the keyword func verifyBackend(env fuzz.ValidatorEnv, bsName string, backendKeyword string, urlMapName string) error { + klog.V(3).Info("Verifying NEG Global Backend") ctx := context.Background() beService, err := env.Cloud().BackendServices().Get(ctx, &meta.Key{Name: bsName}) if err != nil { @@ -178,3 +183,42 @@ func verifyBackend(env fuzz.ValidatorEnv, bsName string, backendKeyword string, return fmt.Errorf("backend service %q is not used by UrlMap %q", bsName, urlMapName) } + +// verifyBackend verifies the backend service and check if the corresponding backend group has the keyword +func verifyNegRegionBackend(env fuzz.ValidatorEnv, bsName string, backendKeyword string, urlMapName string) error { + klog.V(3).Info("Verifying NEG Regional Backend") + ctx := context.Background() + beService, err := env.Cloud().AlphaRegionBackendServices().Get(ctx, &meta.Key{Name: bsName, Region: "us-central1"}) + if err != nil { + return err + } + + if beService == nil { + return fmt.Errorf("no backend service returned for name %s", bsName) + } + + for _, be := range beService.Backends { + if !strings.Contains(be.Group, backendKeyword) { + return fmt.Errorf("backend group %q of backend service %q does not contain keyword %q", be.Group, bsName, backendKeyword) + } + } + + // Examine if ingress url map is targeting the backend service + urlMap, err := env.Cloud().AlphaRegionUrlMaps().Get(ctx, &meta.Key{Name: urlMapName, Region: "us-central1"}) + if err != nil { + return err + } + + if strings.Contains(urlMap.DefaultService, beService.Name) { + return nil + } + for _, pathMatcher := range urlMap.PathMatchers { + for _, rule := range pathMatcher.PathRules { + if strings.Contains(rule.Service, beService.Name) { + return nil + } + } + } + + return fmt.Errorf("backend service %q is not used by UrlMap %q", bsName, urlMapName) +} diff --git a/pkg/fuzz/helpers.go b/pkg/fuzz/helpers.go index 455de3199d..8556c9cd6e 100644 --- a/pkg/fuzz/helpers.go +++ b/pkg/fuzz/helpers.go @@ -267,6 +267,15 @@ func (i *IngressBuilder) SetIngressClass(name string) *IngressBuilder { return i } +// Configure for ILB adds the ILB ingress class annotation +func (i *IngressBuilder) ConfigureForILB() *IngressBuilder { + if i.ing.Annotations == nil { + i.ing.Annotations = make(map[string]string) + } + i.ing.Annotations[annotations.IngressClassKey] = annotations.GceL7ILBIngressClass + return i +} + // BackendConfigBuilder is syntactic sugar for creating BackendConfig specs for testing // purposes. // diff --git a/pkg/fuzz/validator.go b/pkg/fuzz/validator.go index 1185d39d2c..f324dbff76 100644 --- a/pkg/fuzz/validator.go +++ b/pkg/fuzz/validator.go @@ -210,9 +210,9 @@ type IngressValidator struct { // the right SSL certificate is presented // each path, each host returns the right contents -// vip for the load balancer. This currently uses the first entry, returns nil +// Vip for the load balancer. This currently uses the first entry, returns nil // if the VIP is not available. -func (v *IngressValidator) vip() *string { +func (v *IngressValidator) Vip() *string { statuses := v.ing.Status.LoadBalancer.Ingress if len(statuses) == 0 { return nil @@ -292,10 +292,10 @@ func (v *IngressValidator) CheckPaths(ctx context.Context, vr *IngressResult) er // checkPath performs a check for scheme://host/path. func (v *IngressValidator) checkPath(ctx context.Context, scheme, host, path string) error { - if v.vip() == nil { + if v.Vip() == nil { return fmt.Errorf("ingress %s/%s does not have a VIP", v.ing.Namespace, v.ing.Name) } - vip := *v.vip() + vip := *v.Vip() url := fmt.Sprintf("%s://%s%s%s", scheme, vip, portStr(v.attribs, scheme), path) klog.V(3).Infof("Checking Ingress %s/%s url=%q", v.ing.Namespace, v.ing.Name, url)