diff --git a/cmd/main.go b/cmd/main.go index 4607f6d..b3f4c22 100644 --- a/cmd/main.go +++ b/cmd/main.go @@ -16,6 +16,7 @@ import ( "k8s.io/client-go/tools/leaderelection/resourcelock" "github.com/mesosphere/dklb/pkg/admission" + "github.com/mesosphere/dklb/pkg/backends" "github.com/mesosphere/dklb/pkg/cache" "github.com/mesosphere/dklb/pkg/constants" "github.com/mesosphere/dklb/pkg/controllers" @@ -65,8 +66,8 @@ var ( podName string // resyncPeriod is the maximum amount of time that may elapse between two consecutive synchronizations of Ingress/Service resources and the status of EdgeLB pools. resyncPeriod time.Duration - // whWaitGroup is a WaitGroup used to wait for the admission webhook server to shutdown. - whWaitGroup sync.WaitGroup + // srvWaitGroup is a WaitGroup used to wait for the default backend and admission webhook servers to shutdown. + srvWaitGroup sync.WaitGroup ) func init() { @@ -154,6 +155,16 @@ func main() { log.Fatalf("failed to build kubernetes client: %v", err) } + // Launch the default backend. + srvWaitGroup.Add(1) + go func() { + defer srvWaitGroup.Done() + // Create and start the default backend. + if err := backends.NewDefaultBackend().Run(stopCh); err != nil { + log.Fatalf("failed to serve the default backend: %v", err) + } + }() + // Launch the admission webhook if the "ServeAdmissionWebhook" feature is enabled. if featureMap.IsEnabled(features.ServeAdmissionWebhook) { if admissionTLSCertFile == "" { @@ -162,9 +173,9 @@ func main() { if admissionTLSPrivateKeyFile == "" { log.Fatalf("--%s must be set since the %q feature is enabled", admissionTLSPrivateKeyFlagName, features.ServeAdmissionWebhook) } - whWaitGroup.Add(1) + srvWaitGroup.Add(1) go func() { - defer whWaitGroup.Done() + defer srvWaitGroup.Done() // Try to load the provided TLS certificate and private key. p, err := tls.LoadX509KeyPair(admissionTLSCertFile, admissionTLSPrivateKeyFile) if err != nil { @@ -258,8 +269,8 @@ func run(ctx context.Context, kubeClient kubernetes.Interface, edgelbManager man // Wait for the controllers to stop. wg.Wait() - // Wait for the admission webhook to stop. - whWaitGroup.Wait() + // Wait for the default backend and admission webhook servers to stop. + srvWaitGroup.Wait() // Confirm successful shutdown. log.WithField("version", version.Version).Infof("%s is shutting down", constants.ComponentName) // There is a goroutine in the background trying to renew the leader election lock. diff --git a/docs/deployment/10-deployment.yaml b/docs/deployment/10-deployment.yaml index ca86081..0dfbde7 100644 --- a/docs/deployment/10-deployment.yaml +++ b/docs/deployment/10-deployment.yaml @@ -9,10 +9,13 @@ spec: selector: app: dklb ports: - - name: admission - port: 443 - targetPort: 8443 - type: ClusterIP + - name: default-backend + port: 80 + targetPort: 8080 + - name: admission + port: 443 + targetPort: 8443 + type: NodePort --- apiVersion: apps/v1 kind: Deployment @@ -56,6 +59,8 @@ spec: - --log-level - debug ports: + - name: default-backend + containerPort: 8080 - name: admission containerPort: 8443 - name: metrics diff --git a/docs/usage/20-provisioning-ingresses.adoc b/docs/usage/20-provisioning-ingresses.adoc index cc1070b..a64f82f 100644 --- a/docs/usage/20-provisioning-ingresses.adoc +++ b/docs/usage/20-provisioning-ingresses.adoc @@ -38,6 +38,15 @@ kubernetes.io/ingress.class: edgelb All Kubernetes services used as backends in an `Ingress` resource annotated for provisioning with EdgeLB **MUST** be of type `NodePort` or `LoadBalancer`. In particular, services of type `ClusterIP` and headless services cannot be used as the backends for `Ingress` resources to be provisioned by EdgeLB. + +==== `dklb` as the default backend + +In case an invalid `Service` resource is specified as a backend for a given `Ingress` resource, or whenever a default backend is not explicitly defined, `dklb` will be used as the (default) backend instead. +`dklb` will respond to requests arriving at the default backend with `503 SERVICE UNAVAILABLE` and with a short error message. + +Whenever `dklb` gets to be used as a backend, a Kubernetes event will be emitted and associated with the `Ingress` resource being processed. +This event contains useful information about the reason why `dklb` is being used instead of the intended backend, and may be used for diagnosing problems. + === Customizing the name of the EdgeLB pool By default, `dklb` uses the MKE cluster's name and the `Ingress` resource's namespace and name to compute the name of the target EdgeLB pool. diff --git a/hack/skaffold/dklb/pod.yaml b/hack/skaffold/dklb/pod.yaml index c4e5659..eddaad7 100644 --- a/hack/skaffold/dklb/pod.yaml +++ b/hack/skaffold/dklb/pod.yaml @@ -9,10 +9,13 @@ spec: selector: app: dklb ports: + - name: default-backend + port: 80 + targetPort: 8080 - name: admission port: 443 targetPort: 8443 - type: ClusterIP + type: NodePort --- apiVersion: v1 kind: Pod @@ -50,6 +53,8 @@ spec: - --log-level - trace ports: + - name: default-backend + containerPort: 8080 - name: admission containerPort: 8443 - name: metrics diff --git a/hack/skaffold/dklb/skaffold.yaml b/hack/skaffold/dklb/skaffold.yaml index 2e5b579..15b417e 100644 --- a/hack/skaffold/dklb/skaffold.yaml +++ b/hack/skaffold/dklb/skaffold.yaml @@ -1,4 +1,4 @@ -apiVersion: skaffold/v1beta2 +apiVersion: skaffold/v1beta3 kind: Config build: artifacts: diff --git a/pkg/backends/default_backend.go b/pkg/backends/default_backend.go new file mode 100644 index 0000000..5293050 --- /dev/null +++ b/pkg/backends/default_backend.go @@ -0,0 +1,58 @@ +package backends + +import ( + "context" + "net/http" + "time" + + log "github.com/sirupsen/logrus" +) + +const ( + // bindAddress is the address ("host:port") which to bind to. + bindAddress = "0.0.0.0:8080" +) + +// DefaultBackend represents the default backend. +type DefaultBackend struct { +} + +// NewDefaultBackend creates a new instance of the default backend. +func NewDefaultBackend() *DefaultBackend { + return &DefaultBackend{} +} + +// Run starts the HTTP server that backs the default backend. +func (db *DefaultBackend) Run(stopCh chan struct{}) error { + // Configure the HTTP server. + mux := http.NewServeMux() + mux.HandleFunc("/", handle) + srv := http.Server{ + Addr: bindAddress, + Handler: mux, + } + + // Shutdown the server when stopCh is closed. + go func() { + <-stopCh + ctx, fn := context.WithTimeout(context.Background(), 5*time.Second) + defer fn() + if err := srv.Shutdown(ctx); err != nil { + log.Errorf("failed to shutdown the default backend: %v", err) + } else { + log.Debug("the default backend has been shutdown") + } + }() + + // Start listening and serving requests. + log.Debug("starting the default backend") + if err := srv.ListenAndServe(); err != nil && err != http.ErrServerClosed { + return err + } + return nil +} + +// handle handles the specified request by responding with "503 SERVICE UNAVAILABLE" and an error message. +func handle(res http.ResponseWriter, _ *http.Request) { + http.Error(res, "No backend is available to service this request.", http.StatusServiceUnavailable) +} diff --git a/pkg/constants/events.go b/pkg/constants/events.go index 3bb3e44..3b75142 100644 --- a/pkg/constants/events.go +++ b/pkg/constants/events.go @@ -1,6 +1,10 @@ package constants const ( + // ReasonNoDefaultBackendSpecified is the reason used in Kubernetes events emitted whenever an Ingress resource doesn't define a default backend. + ReasonNoDefaultBackendSpecified = "NoDefaultBackendSpecified" + // ReasonInvalidBackendService is the reason used in Kubernetes events emitted due to a missing or otherwise invalid Service resource referenced by an Ingress resource. + ReasonInvalidBackendService = "InvalidBackendService" // ReasonInvalidAnnotations is the reason used in Kubernetes events emitted due to missing/invalid annotations on a Service/Ingress resource. ReasonInvalidAnnotations = "InvalidAnnotations" // ReasonTranslationError is the reason used in Kubernetes events emitted due to failed translation of a Service/Ingress resource into an EdgeLB pool. diff --git a/pkg/constants/generic.go b/pkg/constants/generic.go index 4ccb659..b68f468 100644 --- a/pkg/constants/generic.go +++ b/pkg/constants/generic.go @@ -7,6 +7,10 @@ import ( const ( // ComponentName is the component name to report when performing leader election and emitting Kubernetes events. ComponentName = "dklb" + // DefaultBackendServiceName is the name of the Service resource that exposes dklb as a default backend for Ingress resources. + DefaultBackendServiceName = "dklb" + // DefaultBackendServicePort is the service port defined in the Service resource that exposes dklb as a default backend for Ingress resources. + DefaultBackendServicePort = 80 // DefaultEdgeLBHost is the default host at which the EdgeLB API server can be reached. DefaultEdgeLBHost = "api.edgelb.marathon.l4lb.thisdcos.directory" // DefaultEdgeLBPath is the default path at which the EdgeLB API server can be reached. diff --git a/pkg/controllers/ingress.go b/pkg/controllers/ingress.go index bfd431e..e09aa6f 100644 --- a/pkg/controllers/ingress.go +++ b/pkg/controllers/ingress.go @@ -160,7 +160,7 @@ func (c *IngressController) processQueueItem(workItem WorkItem) error { prettyprint.LogfSpew(log.Tracef, options, "computed ingress translation options for %q", workItem.Key) // Perform translation of the Ingress resource into an EdgeLB pool. - if err := translator.NewIngressTranslator(c.clusterName, ingress, *options, c.kubeCache, c.edgelbManager).Translate(); err != nil { + if err := translator.NewIngressTranslator(c.clusterName, ingress, *options, c.kubeCache, c.edgelbManager, er).Translate(); err != nil { er.Eventf(ingress, corev1.EventTypeWarning, constants.ReasonTranslationError, "failed to translate ingress: %v", err) c.logger.Errorf("failed to translate ingress %q: %v", workItem.Key, err) return err diff --git a/pkg/translator/ingress_translator.go b/pkg/translator/ingress_translator.go index 4613319..005fbbb 100644 --- a/pkg/translator/ingress_translator.go +++ b/pkg/translator/ingress_translator.go @@ -10,6 +10,8 @@ import ( log "github.com/sirupsen/logrus" corev1 "k8s.io/api/core/v1" extsv1beta1 "k8s.io/api/extensions/v1beta1" + "k8s.io/apimachinery/pkg/util/intstr" + "k8s.io/client-go/tools/record" dklbcache "github.com/mesosphere/dklb/pkg/cache" "github.com/mesosphere/dklb/pkg/constants" @@ -20,6 +22,15 @@ import ( "github.com/mesosphere/dklb/pkg/util/prettyprint" ) +var ( + // defaultBackendServiceName is the value used internally as ".serviceName" to signal the fact that dklb should be used as the default backend. + // It will also end up being used as part of the name of an EdgeLB backend whenever an Ingress resource doesn't define a default backend or a referenced Service resource is missing or otherwise invalid. + defaultBackendServiceName = "default-backend" + // defaultBackendServicePort is the value used internally as ".servicePort" to signal the fact that dklb should be used as the default backend. + // It will also end up being used as part of the name of an EdgeLB backend whenever an Ingress resource doesn't define a default backend or a referenced Service resource is missing or otherwise invalid. + defaultBackendServicePort = intstr.FromInt(0) +) + // IngressTranslator is the base implementation of IngressTranslator. type IngressTranslator struct { // clusterName is the name of the Mesos framework that corresponds to the current Kubernetes cluster. @@ -34,22 +45,32 @@ type IngressTranslator struct { manager manager.EdgeLBManager // logger is the logger to use when performing translation. logger *log.Entry + // recorder is the event recorder used to emit events associated with a given Ingress resource. + recorder record.EventRecorder } // NewIngressTranslator returns an ingress translator that can be used to translate the specified Ingress resource into an EdgeLB pool. -func NewIngressTranslator(clusterName string, ingress *extsv1beta1.Ingress, options IngressTranslationOptions, kubeCache dklbcache.KubernetesResourceCache, manager manager.EdgeLBManager) *IngressTranslator { +func NewIngressTranslator(clusterName string, ingress *extsv1beta1.Ingress, options IngressTranslationOptions, kubeCache dklbcache.KubernetesResourceCache, manager manager.EdgeLBManager, recorder record.EventRecorder) *IngressTranslator { return &IngressTranslator{ clusterName: clusterName, - ingress: ingress, - options: options, - kubeCache: kubeCache, - manager: manager, - logger: log.WithField("ingress", kubernetesutil.Key(ingress)), + // Use a clone of the Ingress resource as we may need to modify it in order to inject the default backend. + ingress: ingress.DeepCopy(), + options: options, + kubeCache: kubeCache, + manager: manager, + logger: log.WithField("ingress", kubernetesutil.Key(ingress)), + recorder: recorder, } } // Translate performs translation of the associated Ingress resource into an EdgeLB pool. func (it *IngressTranslator) Translate() error { + // Attempt to determine the node port at which the default backend is exposed. + defaultBackendNodePort, err := it.determineDefaultBackendNodePort() + if err != nil { + return err + } + // Return immediately if translation is paused. if it.options.EdgeLBPoolTranslationPaused { it.logger.Warnf("skipping translation of %q as translation is paused for the resource", kubernetesutil.Key(it.ingress)) @@ -57,10 +78,7 @@ func (it *IngressTranslator) Translate() error { } // Compute the mapping between Ingress backends defined on the current Ingress resource and their target node ports. - backendMap, err := it.computeIngressBackendNodePortMap() - if err != nil { - return err - } + backendMap := it.computeIngressBackendNodePortMap(defaultBackendNodePort) // Check whether an EdgeLB pool with the requested name already exists in EdgeLB. ctx, fn := context.WithTimeout(context.Background(), defaultEdgeLBManagerTimeout) @@ -79,11 +97,37 @@ func (it *IngressTranslator) Translate() error { return it.updateOrDeleteEdgeLBPool(pool, backendMap) } +// determineDefaultBackendNodePort attempts to determine the node port at which the default backend is exposed. +func (it *IngressTranslator) determineDefaultBackendNodePort() (int32, error) { + s, err := it.kubeCache.GetService(constants.KubeSystemNamespaceName, constants.DefaultBackendServiceName) + if err != nil { + return 0, fmt.Errorf("failed to read the \"%s/%s\" service: %v", constants.KubeSystemNamespaceName, constants.DefaultBackendServiceName, err) + } + if s.Spec.Type != corev1.ServiceTypeNodePort && s.Spec.Type != corev1.ServiceTypeLoadBalancer { + return 0, fmt.Errorf("service %q is of unexpected type %q", kubernetesutil.Key(s), s.Spec.Type) + } + for _, port := range s.Spec.Ports { + if port.Port == constants.DefaultBackendServicePort && port.NodePort > 0 { + return port.NodePort, nil + } + } + return 0, fmt.Errorf("no valid node port has been assigned to the default backend") +} + // computeIngressBackendNodePortMap computes the mapping between (unique) Ingress backends defined on the current Ingress resource and their target node ports. // It starts by compiling a set of all (possibly duplicate) Ingress backends defined on the Ingress resource. -// Then, it iterates over said set and checks whether the referenced service port exists, adding them to the map or returning errors as appropriate. +// In case a default backend hasn't been specified, dklb's default backend is injected as the default one. +// Then, it iterates over said set and checks whether the referenced service port exists, adding them to the map or using the default backend's node port instead. // As the returned object is in fact a map, duplicate Ingress backends are automatically removed. -func (it *IngressTranslator) computeIngressBackendNodePortMap() (IngressBackendNodePortMap, error) { +func (it *IngressTranslator) computeIngressBackendNodePortMap(defaultBackendNodePort int32) IngressBackendNodePortMap { + // Inject dklb as the default backend in case none is specified. + if it.ingress.Spec.Backend == nil { + it.ingress.Spec.Backend = &extsv1beta1.IngressBackend{ + ServiceName: defaultBackendServiceName, + ServicePort: defaultBackendServicePort, + } + it.recorder.Eventf(it.ingress, corev1.EventTypeWarning, constants.ReasonNoDefaultBackendSpecified, "%s will be used as the default backend since none was specified", constants.ComponentName) + } // backends is the slice containing all Ingress backends present in the current Ingress resource. backends := make([]extsv1beta1.IngressBackend, 0) // Iterate over all Ingress backends, adding them to the slice of results. @@ -94,14 +138,25 @@ func (it *IngressTranslator) computeIngressBackendNodePortMap() (IngressBackendN res := make(IngressBackendNodePortMap, len(backends)) // Iterate over the set of Ingress backends, computing the target node port. for _, backend := range backends { + // If the target service's name corresponds to "defaultBackendServiceName", we use the default backend's node port. + if backend.ServiceName == defaultBackendServiceName && backend.ServicePort == defaultBackendServicePort { + res[backend] = defaultBackendNodePort + continue + } if nodePort, err := it.computeNodePortForIngressBackend(backend); err == nil { res[backend] = nodePort } else { - return nil, err + // We've failed to compute the target node port for the current backend. + // This may be caused by the specified Service resource being absent or not being of NodePort/LoadBalancer type. + // Hence, we use the default backend's node port and report the error as an event, but do not fail. + msg := fmt.Sprintf("using the default backend in place of \"%s:%s\": %v", backend.ServiceName, backend.ServicePort.String(), err) + it.recorder.Eventf(it.ingress, corev1.EventTypeWarning, constants.ReasonInvalidBackendService, msg) + it.logger.Warn(msg) + res[backend] = defaultBackendNodePort } } // Return the populated map. - return res, nil + return res } // computeNodePortForIngressBackend computes the node port targeted by the specified Ingress backend. diff --git a/pkg/translator/ingress_translator_internal_test.go b/pkg/translator/ingress_translator_internal_test.go index 58d1f19..61f00d9 100644 --- a/pkg/translator/ingress_translator_internal_test.go +++ b/pkg/translator/ingress_translator_internal_test.go @@ -12,6 +12,7 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/util/intstr" + "k8s.io/client-go/tools/record" "github.com/mesosphere/dklb/pkg/constants" "github.com/mesosphere/dklb/pkg/util/pointers" @@ -22,6 +23,8 @@ import ( ) var ( + // defaultBackendNodePort is used as the node port at which the default backend is exposed. + defaultBackendNodePort = int32(32000) // dummyIngress1 is a dummy Kubernetes Ingress resource. dummyIngress1 = ingresstestutil.DummyIngressResource("foo", "bar", func(ingress *extsv1beta1.Ingress) { ingress.Annotations = map[string]string{ @@ -57,6 +60,13 @@ var ( }, } }) + // dummyIngress2 is a dummy Kubernetes Ingress resource that doesn't define a default backend and has no rules. + dummyIngress2 = ingresstestutil.DummyIngressResource("foo", "bar", func(ingress *extsv1beta1.Ingress) { + ingress.Annotations = map[string]string{ + constants.EdgeLBIngressClassAnnotationKey: constants.EdgeLBIngressClassAnnotationValue, + } + ingress.Spec.Rules = []extsv1beta1.IngressRule{} + }) // dummyIngress1WithoutDummyIngress1BackendBaz is a dummy Kubernetes Ingress resource built from "dummyIngress1" by removing the "dummyIngress1BackendBaz" backend.. dummyIngress1WithoutDummyIngress1BackendBaz = ingresstestutil.DummyIngressResource("foo", "bar", func(ingress *extsv1beta1.Ingress) { ingress.Annotations = map[string]string{ @@ -172,6 +182,8 @@ var ( }, EdgeLBPoolPort: 18080, } + // dummyIngress2TranslationOptions represents a set of translation options that can be used to translate "dummyIngress2". + dummyIngress2TranslationOptions = dummyIngress1TranslationOptions // backendForIngress1Foo is the computed (expected) backend for path "/bar" of "dummyIngress1. backendForDummyIngress1Bar = computeEdgeLBBackendForIngressBackend(testClusterName, dummyIngress1, dummyIngress1.Spec.Rules[0].HTTP.Paths[0].Backend, dummyIngress1BackendBar.Spec.Ports[0].NodePort) @@ -179,23 +191,41 @@ var ( backendForDummyIngress1Baz = computeEdgeLBBackendForIngressBackend(testClusterName, dummyIngress1, dummyIngress1.Spec.Rules[0].HTTP.Paths[1].Backend, dummyIngress1BackendBaz.Spec.Ports[0].NodePort) // defaultBackendForDummyIngress1 is the computed (expected) default backend for "dummyIngress1". defaultBackendForDummyIngress1 = computeEdgeLBBackendForIngressBackend(testClusterName, dummyIngress1, *dummyIngress1.Spec.Backend, dummyIngress1BackendFoo.Spec.Ports[0].NodePort) + // defaultBackendForDummyIngress2 is the computed (expected) default backend for "dummyIngress1". + // For this Ingress resource, we expect the default backend to be injected and used. + defaultBackendForDummyIngress2 = computeEdgeLBBackendForIngressBackend(testClusterName, dummyIngress2, extsv1beta1.IngressBackend{ + ServiceName: defaultBackendServiceName, + ServicePort: defaultBackendServicePort, + }, defaultBackendNodePort) // frontendForDummyIngress1 is the computed (expected) frontend for "dummyIngress1". frontendForDummyIngress1 = computeEdgeLBFrontendForIngress(testClusterName, dummyIngress1, dummyIngress1TranslationOptions) + // frontendForDummyIngress2 is the computed (expected) frontend for "dummyIngress2". + // For this Ingress resource, we expect the default backend to be injected and used. + frontendForDummyIngress2 = &models.V2Frontend{ + BindAddress: constants.EdgeLBFrontendBindAddress, + BindPort: pointers.NewInt32(dummyIngress2TranslationOptions.EdgeLBPoolPort), + LinkBackend: &models.V2FrontendLinkBackend{ + DefaultBackend: "dev.kubernetes01:foo:bar:default-backend:0", + }, + Name: "dev.kubernetes01:foo:bar", + Protocol: models.V2ProtocolHTTP, + } ) func TestCreateEdgeLBPoolObjectForIngress(t *testing.T) { tests := []struct { - description string - resources []runtime.Object - ingress *extsv1beta1.Ingress - options IngressTranslationOptions - expectedName string - expectedRole string - expectedCpus float64 - expectedMem int32 - expectedSize int - expectedBackends []*models.V2Backend - expectedFrontends []*models.V2Frontend + description string + resources []runtime.Object + ingress *extsv1beta1.Ingress + options IngressTranslationOptions + expectedName string + expectedRole string + expectedCpus float64 + expectedMem int32 + expectedSize int + expectedBackends []*models.V2Backend + expectedFrontends []*models.V2Frontend + expectedEventCount int }{ { description: "create an edgelb pool based on valid translation options", @@ -219,18 +249,40 @@ func TestCreateEdgeLBPoolObjectForIngress(t *testing.T) { expectedFrontends: []*models.V2Frontend{ frontendForDummyIngress1, }, + expectedEventCount: 0, + }, + { + description: "create an edgelb pool based on valid translation options for an ingress resource that doesn't define a default backend", + resources: []runtime.Object{ + dummyIngress1BackendBar, + dummyIngress1BackendBaz, + }, + ingress: dummyIngress2, + options: dummyIngress2TranslationOptions, + expectedName: "baz", + expectedRole: "custom_role", + expectedCpus: 5010.203, + expectedMem: 3724, + expectedSize: 3, + expectedBackends: []*models.V2Backend{ + defaultBackendForDummyIngress2, + }, + expectedFrontends: []*models.V2Frontend{ + frontendForDummyIngress2, + }, + expectedEventCount: 1, }, } for _, test := range tests { t.Logf("test case: %s", test.description) // Create a mock KubernetesResourceCache. kubeCache := cachetestutil.NewFakeKubernetesResourceCache(test.resources...) + // Create a new fake event recorder. + recorder := record.NewFakeRecorder(1) // Create a new instance of the Ingress translator. - translator := NewIngressTranslator(testClusterName, test.ingress, test.options, kubeCache, nil) + translator := NewIngressTranslator(testClusterName, test.ingress, test.options, kubeCache, nil, recorder) // Compute the mapping between Ingress backends and Service node ports. - m, err := translator.computeIngressBackendNodePortMap() - // Make sure no error occurred. - assert.NoError(t, err) + m := translator.computeIngressBackendNodePortMap(defaultBackendNodePort) // Create the target EdgeLB pool object. pool := translator.createEdgeLBPoolObject(m) // Make sure the resulting EdgeLB pool object meets our expectations. @@ -241,6 +293,7 @@ func TestCreateEdgeLBPoolObjectForIngress(t *testing.T) { assert.Equal(t, pointers.NewInt32(int32(test.expectedSize)), pool.Count) assert.Equal(t, test.expectedBackends, pool.Haproxy.Backends) assert.Equal(t, test.expectedFrontends, pool.Haproxy.Frontends) + assert.Equal(t, test.expectedEventCount, len(recorder.Events)) } } @@ -461,12 +514,12 @@ func TestUpdateEdgeLBPoolObjectForIngress(t *testing.T) { t.Logf("test case: %s", test.description) // Create a mock KubernetesResourceCache. kubeCache := cachetestutil.NewFakeKubernetesResourceCache(test.resources...) + // Create a new fake event recorder. + recorder := record.NewFakeRecorder(1) // Create a new instance of the Ingress translator. - translator := NewIngressTranslator(testClusterName, test.ingress, test.options, kubeCache, nil) + translator := NewIngressTranslator(testClusterName, test.ingress, test.options, kubeCache, nil, recorder) // Compute the mapping between Ingress backends and Service node ports. - m, err := translator.computeIngressBackendNodePortMap() - // Make sure no error occurred. - assert.NoError(t, err) + m := translator.computeIngressBackendNodePortMap(defaultBackendNodePort) // Update the EdgeLB pool object in-place. wasChanged, _ := translator.updateEdgeLBPoolObject(test.pool, m) // Check that the need for a pool update was adequately detected. diff --git a/pkg/translator/ingress_translator_test.go b/pkg/translator/ingress_translator_test.go index 1171549..672cb88 100644 --- a/pkg/translator/ingress_translator_test.go +++ b/pkg/translator/ingress_translator_test.go @@ -12,6 +12,7 @@ import ( extsv1beta1 "k8s.io/api/extensions/v1beta1" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/util/intstr" + "k8s.io/client-go/tools/record" "github.com/mesosphere/dklb/pkg/constants" dklberrors "github.com/mesosphere/dklb/pkg/errors" @@ -23,6 +24,16 @@ import ( ) var ( + // defaultBackendService represents the Service resource used to expose the default backend. + defaultBackendService = servicetestutil.DummyServiceResource(constants.KubeSystemNamespaceName, constants.DefaultBackendServiceName, func(service *corev1.Service) { + service.Spec.Type = corev1.ServiceTypeNodePort + service.Spec.Ports = []corev1.ServicePort{ + { + NodePort: 32000, + Port: constants.DefaultBackendServicePort, + }, + } + }) // dummyService1 is a dummy Service resource exposing a single port. dummyService1 = servicetestutil.DummyServiceResource("foo", "bar", func(service *corev1.Service) { service.Spec.Type = corev1.ServiceTypeNodePort @@ -50,7 +61,9 @@ func TestIngressTranslator_Translate(t *testing.T) { // Tests that a pool is created whenever it doesn't exist and the pool creation strategy is set to "IfNotPresent". { description: "pool is created whenever it doesn't exist and the pool creation strategy is set to \"IfNotPresent\"", - resources: []runtime.Object{}, + resources: []runtime.Object{ + defaultBackendService, + }, ingress: ingresstestutil.DummyIngressResource("foo", "bar", func(ingress *extsv1beta1.Ingress) { ingress.Annotations = map[string]string{ constants.EdgeLBIngressClassAnnotationKey: constants.EdgeLBIngressClassAnnotationValue, @@ -73,7 +86,9 @@ func TestIngressTranslator_Translate(t *testing.T) { // Tests that a pool is not created when it doesn't exist and the pool creation strategy is set to "Never". { description: "pool is not created when it doesn't exist and the pool creation strategy is set to \"Never\"", - resources: []runtime.Object{}, + resources: []runtime.Object{ + defaultBackendService, + }, ingress: ingresstestutil.DummyIngressResource("foo", "bar", func(ingress *extsv1beta1.Ingress) { ingress.Annotations = map[string]string{ constants.EdgeLBIngressClassAnnotationKey: constants.EdgeLBIngressClassAnnotationValue, @@ -95,7 +110,9 @@ func TestIngressTranslator_Translate(t *testing.T) { // Tests that a pool is not created when it doesn't exist, the target Ingress resource has a non-empty status field and the pool creation strategy is set to "Once". { description: "pool is not created when it doesn't exist, the target Ingress resource has a non-empty status field and the pool creation strategy is set to \"Once\"", - resources: []runtime.Object{}, + resources: []runtime.Object{ + defaultBackendService, + }, ingress: ingresstestutil.DummyIngressResource("foo", "bar", func(ingress *extsv1beta1.Ingress) { ingress.Annotations = map[string]string{ constants.EdgeLBIngressClassAnnotationKey: constants.EdgeLBIngressClassAnnotationValue, @@ -123,6 +140,7 @@ func TestIngressTranslator_Translate(t *testing.T) { { description: "pool is updated whenever it exists but is not in sync with the target Ingress resource", resources: []runtime.Object{ + defaultBackendService, dummyService1, }, ingress: ingresstestutil.DummyIngressResource("foo", "bar", func(ingress *extsv1beta1.Ingress) { @@ -172,8 +190,10 @@ func TestIngressTranslator_Translate(t *testing.T) { // Create and customize a mock EdgeLB manager. m := new(edgelbmanagertestutil.MockEdgeLBManager) test.mockCustomizer(m) + // Create a new fake event recorder. + recorder := record.NewFakeRecorder(1) // Perform translation of the Ingress resource. - err := translator.NewIngressTranslator(testClusterName, test.ingress, test.options, k, m).Translate() + err := translator.NewIngressTranslator(testClusterName, test.ingress, test.options, k, m, recorder).Translate() if test.expectedError != nil { // Make sure we've got the expected error. assert.Equal(t, test.expectedError, err) diff --git a/test/e2e/framework/http.go b/test/e2e/framework/http.go index b741e83..8f412ad 100644 --- a/test/e2e/framework/http.go +++ b/test/e2e/framework/http.go @@ -3,6 +3,7 @@ package framework import ( "encoding/json" "fmt" + "io/ioutil" "net/http" "strings" ) @@ -36,6 +37,31 @@ func (r *EchoResponse) XForwardedForContains(v string) bool { return false } +// Request performs a "method" request to the specified host and path, returning the status code and the response's body. +// TODO (@bcustodio) Add support for HTTPS if/when necessary. +func (f *Framework) Request(method, host, path string) (int, string, error) { + // Build the HTTP request. + req, err := http.NewRequest(method, fmt.Sprintf("http://%s%s", host, path), nil) + if err != nil { + return 0, "", err + } + // Perform the request. + res, err := f.HTTPClient.Do(req) + if err != nil { + return 0, "", err + } + // Read the response's body. + b, err := ioutil.ReadAll(res.Body) + if err != nil { + return 0, "", err + } + // Close the response's body. + if err := res.Body.Close(); err != nil { + return 0, "", err + } + return res.StatusCode, string(b), nil +} + // EchoRequest performs a "method" request to the specified host and path, returning the resulting "echo" response or an error. // TODO (@bcustodio) Add support for HTTPS if/when necessary. func (f *Framework) EchoRequest(method, host, path string, headers map[string]string) (*EchoResponse, error) { diff --git a/test/e2e/ingress_test.go b/test/e2e/ingress_test.go index 754ac57..7ff2709 100644 --- a/test/e2e/ingress_test.go +++ b/test/e2e/ingress_test.go @@ -579,5 +579,131 @@ var _ = Describe("Ingress", func() { }) }) }) + + It("uses dklb as its default backend whenever one is not specified or a service is missing [HTTP] [Public]", func() { + // Create a temporary namespace for the test. + f.WithTemporaryNamespace(func(namespace *corev1.Namespace) { + var ( + echoPod1 *corev1.Pod + echoSvc1 *corev1.Service + err error + ingress *extsv1beta1.Ingress + ) + + // Create the first "echo" pod. + echoPod1, err = f.CreateEchoPod(namespace.Name, "http-echo-1") + Expect(err).NotTo(HaveOccurred(), "failed to create echo pod") + // Create the first "echo" service. + echoSvc1, err = f.CreateServiceForEchoPod(echoPod1) + Expect(err).NotTo(HaveOccurred(), "failed to create service for echo pod %q", kubernetes.Key(echoPod1)) + + // Create an Ingress resource targeting the service above, annotating it to be provisioned by EdgeLB. + ingress, err = f.CreateEdgeLBIngress(namespace.Name, "http-echo", func(ingress *extsv1beta1.Ingress) { + ingress.Annotations = map[string]string{ + // Request for the EdgeLB pool to be called "". + constants.EdgeLBPoolNameAnnotationKey: namespace.Name, + // Request for the EdgeLB pool to be deployed to an agent with the "slave_public" role. + constants.EdgeLBPoolRoleAnnotationKey: constants.EdgeLBRolePublic, + // Request for the EdgeLB pool to be given 0.2 CPUs. + constants.EdgeLBPoolCpusAnnotationKey: "200m", + // Request for the EdgeLB pool to be given 256MiB of RAM. + constants.EdgeLBPoolMemAnnotationKey: "256Mi", + // Request for the EdgeLB pool to be deployed into a single agent. + constants.EdgeLBPoolSizeAnnotationKey: "1", + } + ingress.Spec.Rules = []extsv1beta1.IngressRule{ + { + IngressRuleValue: extsv1beta1.IngressRuleValue{ + HTTP: &extsv1beta1.HTTPIngressRuleValue{ + Paths: []extsv1beta1.HTTPIngressPath{ + { + Path: "/foo(/.*)?", + Backend: extsv1beta1.IngressBackend{ + ServiceName: echoSvc1.Name, + ServicePort: intstr.FromInt(int(echoSvc1.Spec.Ports[0].Port)), + }, + }, + { + Path: "/bar(/.*)?", + Backend: extsv1beta1.IngressBackend{ + ServiceName: "missing-service", + ServicePort: intstr.FromString("http"), + }, + }, + }, + }, + }, + }, + } + }) + Expect(err).NotTo(HaveOccurred(), "failed to create ingress") + + // Wait for EdgeLB to acknowledge the pool's creation. + err = retry.WithTimeout(framework.DefaultRetryTimeout, framework.DefaultRetryInterval, func() (bool, error) { + ctx, fn := context.WithTimeout(context.Background(), framework.DefaultRetryInterval/2) + defer fn() + _, err = f.EdgeLBManager.GetPoolByName(ctx, ingress.Annotations[constants.EdgeLBPoolNameAnnotationKey]) + return err == nil, nil + }) + Expect(err).NotTo(HaveOccurred(), "timed out while waiting for the edgelb api server to acknowledge the pool's creation") + + // TODO (@bcustodio) Wait for the pool's IP(s) to be reported. + + // Wait for the Ingress to respond with the default backend at "http:///foo". + err = retry.WithTimeout(framework.DefaultRetryTimeout, framework.DefaultRetryInterval, func() (bool, error) { + r, err := f.HTTPClient.Get(fmt.Sprintf("http://%s/foo", publicIP)) + if err != nil { + log.Debugf("waiting for the ingress to be reachable at %s", publicIP) + return false, nil + } + log.Debugf("the ingress is reachable at %s", publicIP) + return r.StatusCode == 200, nil + }) + Expect(err).NotTo(HaveOccurred(), "timed out while waiting for the ingress to be reachable") + + // Make sure that requests are directed towards the expected backend. + tests := []struct { + description string + path string + expectedStatusCode int + expectedBodyRegex string + }{ + // Test that requests made to "/foo" are directed towards "http-echo-1". + { + description: "%s request to path /foo is directed towards http-echo-1", + path: "/foo", + expectedStatusCode: 200, + expectedBodyRegex: "http-echo-1", + }, + // Test that requests made to "/" are directed towards "dklb". + { + description: "%s request to path / is directed towards \"dklb\"", + path: "/", + expectedStatusCode: 503, + expectedBodyRegex: "No backend is available to service this request.", + }, + // Test that requests made to "/bar" are directed towards "dklb". + { + description: "%s request to path /bar is directed towards \"dklb\"", + path: "/bar", + expectedStatusCode: 503, + expectedBodyRegex: "No backend is available to service this request.", + }, + } + for _, test := range tests { + for _, method := range []string{"GET", "POST", "PUT", "PATCH", "DELETE"} { + log.Debugf("test case: %s", fmt.Sprintf(test.description, method)) + status, body, err := f.Request(method, publicIP, test.path) + Expect(err).NotTo(HaveOccurred(), "failed to perform http request") + Expect(status).To(Equal(test.expectedStatusCode), "the response's status code doesn't match the expectation") + Expect(body).To(MatchRegexp(test.expectedBodyRegex), "the response's body doesn't match the expectations") + } + } + + // Manually delete the Ingress resource now in order to prevent the EdgeLB pool from being re-created during namespace deletion. + err = f.KubeClient.ExtensionsV1beta1().Ingresses(ingress.Namespace).Delete(ingress.Name, metav1.NewDeleteOptions(0)) + Expect(err).NotTo(HaveOccurred(), "failed to delete ingress %q", kubernetes.Key(ingress)) + }) + }) }) })