http: use dklb as the default backend when necessary

cmd/main.go

@@ -16,6 +16,7 @@ import (
 	"k8s.io/client-go/tools/leaderelection/resourcelock"
 
 	"github.com/mesosphere/dklb/pkg/admission"
+	"github.com/mesosphere/dklb/pkg/backends"
 	"github.com/mesosphere/dklb/pkg/cache"
 	"github.com/mesosphere/dklb/pkg/constants"
 	"github.com/mesosphere/dklb/pkg/controllers"
@@ -65,8 +66,8 @@ var (
 	podName string
 	// resyncPeriod is the maximum amount of time that may elapse between two consecutive synchronizations of Ingress/Service resources and the status of EdgeLB pools.
 	resyncPeriod time.Duration
-	// whWaitGroup is a WaitGroup used to wait for the admission webhook server to shutdown.
-	whWaitGroup sync.WaitGroup
+	// srvWaitGroup is a WaitGroup used to wait for the default backend and admission webhook servers to shutdown.
+	srvWaitGroup sync.WaitGroup
 )
 
 func init() {
@@ -154,6 +155,16 @@ func main() {
 		log.Fatalf("failed to build kubernetes client: %v", err)
 	}
 
+	// Launch the default backend.
+	srvWaitGroup.Add(1)
+	go func() {
+		defer srvWaitGroup.Done()
+		// Create and start the default backend.
+		if err := backends.NewDefaultBackend().Run(stopCh); err != nil {
+			log.Fatalf("failed to serve the default backend: %v", err)
+		}
+	}()
+
 	// Launch the admission webhook if the "ServeAdmissionWebhook" feature is enabled.
 	if featureMap.IsEnabled(features.ServeAdmissionWebhook) {
 		if admissionTLSCertFile == "" {
@@ -162,9 +173,9 @@ func main() {
 		if admissionTLSPrivateKeyFile == "" {
 			log.Fatalf("--%s must be set since the %q feature is enabled", admissionTLSPrivateKeyFlagName, features.ServeAdmissionWebhook)
 		}
-		whWaitGroup.Add(1)
+		srvWaitGroup.Add(1)
 		go func() {
-			defer whWaitGroup.Done()
+			defer srvWaitGroup.Done()
 			// Try to load the provided TLS certificate and private key.
 			p, err := tls.LoadX509KeyPair(admissionTLSCertFile, admissionTLSPrivateKeyFile)
 			if err != nil {
@@ -258,8 +269,8 @@ func run(ctx context.Context, kubeClient kubernetes.Interface, edgelbManager man
 
 	// Wait for the controllers to stop.
 	wg.Wait()
-	// Wait for the admission webhook to stop.
-	whWaitGroup.Wait()
+	// Wait for the default backend and admission webhook servers to stop.
+	srvWaitGroup.Wait()
 	// Confirm successful shutdown.
 	log.WithField("version", version.Version).Infof("%s is shutting down", constants.ComponentName)
 	// There is a goroutine in the background trying to renew the leader election lock.

docs/deployment/10-deployment.yaml

@@ -9,10 +9,13 @@ spec:
   selector:
     app: dklb
   ports:
-    - name: admission
-      port: 443
-      targetPort: 8443
-  type: ClusterIP
+  - name: default-backend
+    port: 80
+    targetPort: 8080
+  - name: admission
+    port: 443
+    targetPort: 8443
+  type: NodePort
 ---
 apiVersion: apps/v1
 kind: Deployment
@@ -56,6 +59,8 @@ spec:
         - --log-level
         - debug
         ports:
+        - name: default-backend
+          containerPort: 8080
         - name: admission
           containerPort: 8443
         - name: metrics

docs/usage/20-provisioning-ingresses.adoc

@@ -38,6 +38,15 @@ kubernetes.io/ingress.class: edgelb
 All Kubernetes services used as backends in an `Ingress` resource annotated for provisioning with EdgeLB **MUST** be of type `NodePort` or `LoadBalancer`.
 In particular, services of type `ClusterIP` and headless services cannot be used as the backends for `Ingress` resources to be provisioned by EdgeLB.
 
+
+==== `dklb` as the default backend
+
+In case an invalid `Service` resource is specified as a backend for a given `Ingress` resource, or whenever a default backend is not explicitly defined, `dklb` will be used as the (default) backend instead.
+`dklb` will respond to requests arriving at the default backend with `503 SERVICE UNAVAILABLE` and with a short error message.
+
+Whenever `dklb` gets to be used as a backend, a Kubernetes event will be emitted and associated with the `Ingress` resource being processed.
+This event contains useful information about the reason why `dklb` is being used instead of the intended backend, and may be used for diagnosing problems.
+
 === Customizing the name of the EdgeLB pool
 
 By default, `dklb` uses the MKE cluster's name and the `Ingress` resource's namespace and name to compute the name of the target EdgeLB pool.

hack/skaffold/dklb/pod.yaml

@@ -9,10 +9,13 @@ spec:
   selector:
     app: dklb
   ports:
+  - name: default-backend
+    port: 80
+    targetPort: 8080
   - name: admission
     port: 443
     targetPort: 8443
-  type: ClusterIP
+  type: NodePort
 ---
 apiVersion: v1
 kind: Pod
@@ -50,6 +53,8 @@ spec:
     - --log-level
     - trace
     ports:
+    - name: default-backend
+      containerPort: 8080
     - name: admission
       containerPort: 8443
     - name: metrics

hack/skaffold/dklb/skaffold.yaml

@@ -1,4 +1,4 @@
-apiVersion: skaffold/v1beta2
+apiVersion: skaffold/v1beta3
 kind: Config
 build:
   artifacts:

pkg/backends/default_backend.go

@@ -0,0 +1,58 @@
+package backends
+
+import (
+	"context"
+	"net/http"
+	"time"
+
+	log "github.com/sirupsen/logrus"
+)
+
+const (
+	// bindAddress is the address ("host:port") which to bind to.
+	bindAddress = "0.0.0.0:8080"
+)
+
+// DefaultBackend represents the default backend.
+type DefaultBackend struct {
+}
+
+// NewDefaultBackend creates a new instance of the default backend.
+func NewDefaultBackend() *DefaultBackend {
+	return &DefaultBackend{}
+}
+
+// Run starts the HTTP server that backs the default backend.
+func (db *DefaultBackend) Run(stopCh chan struct{}) error {
+	// Configure the HTTP server.
+	mux := http.NewServeMux()
+	mux.HandleFunc("/", handle)
+	srv := http.Server{
+		Addr:    bindAddress,
+		Handler: mux,
+	}
+
+	// Shutdown the server when stopCh is closed.
+	go func() {
+		<-stopCh
+		ctx, fn := context.WithTimeout(context.Background(), 5*time.Second)
+		defer fn()
+		if err := srv.Shutdown(ctx); err != nil {
+			log.Errorf("failed to shutdown the default backend: %v", err)
+		} else {
+			log.Debug("the default backend has been shutdown")
+		}
+	}()
+
+	// Start listening and serving requests.
+	log.Debug("starting the default backend")
+	if err := srv.ListenAndServe(); err != nil && err != http.ErrServerClosed {
+		return err
+	}
+	return nil
+}
+
+// handle handles the specified request by responding with "503 SERVICE UNAVAILABLE" and an error message.
+func handle(res http.ResponseWriter, _ *http.Request) {
+	http.Error(res, "No backend is available to service this request.", http.StatusServiceUnavailable)
+}

pkg/constants/events.go

@@ -1,6 +1,10 @@
 package constants
 
 const (
+	// ReasonNoDefaultBackendSpecified is the reason used in Kubernetes events emitted whenever an Ingress resource doesn't define a default backend.
+	ReasonNoDefaultBackendSpecified = "NoDefaultBackendSpecified"
+	// ReasonInvalidBackendService is the reason used in Kubernetes events emitted due to a missing or otherwise invalid Service resource referenced by an Ingress resource.
+	ReasonInvalidBackendService = "InvalidBackendService"
 	// ReasonInvalidAnnotations is the reason used in Kubernetes events emitted due to missing/invalid annotations on a Service/Ingress resource.
 	ReasonInvalidAnnotations = "InvalidAnnotations"
 	// ReasonTranslationError is the reason used in Kubernetes events emitted due to failed translation of a Service/Ingress resource into an EdgeLB pool.

pkg/constants/generic.go

@@ -7,6 +7,10 @@ import (
 const (
 	// ComponentName is the component name to report when performing leader election and emitting Kubernetes events.
 	ComponentName = "dklb"
+	// DefaultBackendServiceName is the name of the Service resource that exposes dklb as a default backend for Ingress resources.
+	DefaultBackendServiceName = "dklb"
+	// DefaultBackendServicePort is the service port defined in the Service resource that exposes dklb as a default backend for Ingress resources.
+	DefaultBackendServicePort = 80
 	// DefaultEdgeLBHost is the default host at which the EdgeLB API server can be reached.
 	DefaultEdgeLBHost = "api.edgelb.marathon.l4lb.thisdcos.directory"
 	// DefaultEdgeLBPath is the default path at which the EdgeLB API server can be reached.

pkg/controllers/ingress.go

@@ -160,7 +160,7 @@ func (c *IngressController) processQueueItem(workItem WorkItem) error {
 	prettyprint.LogfSpew(log.Tracef, options, "computed ingress translation options for %q", workItem.Key)
 
 	// Perform translation of the Ingress resource into an EdgeLB pool.
-	if err := translator.NewIngressTranslator(c.clusterName, ingress, *options, c.kubeCache, c.edgelbManager).Translate(); err != nil {
+	if err := translator.NewIngressTranslator(c.clusterName, ingress, *options, c.kubeCache, c.edgelbManager, er).Translate(); err != nil {
 		er.Eventf(ingress, corev1.EventTypeWarning, constants.ReasonTranslationError, "failed to translate ingress: %v", err)
 		c.logger.Errorf("failed to translate ingress %q: %v", workItem.Key, err)
 		return err

pkg/translator/ingress_translator.go

@@ -10,6 +10,8 @@ import (
 	log "github.com/sirupsen/logrus"
 	corev1 "k8s.io/api/core/v1"
 	extsv1beta1 "k8s.io/api/extensions/v1beta1"
+	"k8s.io/apimachinery/pkg/util/intstr"
+	"k8s.io/client-go/tools/record"
 
 	dklbcache "github.com/mesosphere/dklb/pkg/cache"
 	"github.com/mesosphere/dklb/pkg/constants"
@@ -20,6 +22,15 @@ import (
 	"github.com/mesosphere/dklb/pkg/util/prettyprint"
 )
 
+var (
+	// defaultBackendServiceName is the value used internally as ".serviceName" to signal the fact that dklb should be used as the default backend.
+	// It will also end up being used as part of the name of an EdgeLB backend whenever an Ingress resource doesn't define a default backend or a referenced Service resource is missing or otherwise invalid.
+	defaultBackendServiceName = "default-backend"
+	// defaultBackendServicePort is the value used internally as ".servicePort" to signal the fact that dklb should be used as the default backend.
+	// It will also end up being used as part of the name of an EdgeLB backend whenever an Ingress resource doesn't define a default backend or a referenced Service resource is missing or otherwise invalid.
+	defaultBackendServicePort = intstr.FromInt(0)
+)
+
 // IngressTranslator is the base implementation of IngressTranslator.
 type IngressTranslator struct {
 	// clusterName is the name of the Mesos framework that corresponds to the current Kubernetes cluster.
@@ -34,33 +45,40 @@ type IngressTranslator struct {
 	manager manager.EdgeLBManager
 	// logger is the logger to use when performing translation.
 	logger *log.Entry
+	// recorder is the event recorder used to emit events associated with a given Ingress resource.
+	recorder record.EventRecorder
 }
 
 // NewIngressTranslator returns an ingress translator that can be used to translate the specified Ingress resource into an EdgeLB pool.
-func NewIngressTranslator(clusterName string, ingress *extsv1beta1.Ingress, options IngressTranslationOptions, kubeCache dklbcache.KubernetesResourceCache, manager manager.EdgeLBManager) *IngressTranslator {
+func NewIngressTranslator(clusterName string, ingress *extsv1beta1.Ingress, options IngressTranslationOptions, kubeCache dklbcache.KubernetesResourceCache, manager manager.EdgeLBManager, recorder record.EventRecorder) *IngressTranslator {
 	return &IngressTranslator{
 		clusterName: clusterName,
-		ingress:     ingress,
-		options:     options,
-		kubeCache:   kubeCache,
-		manager:     manager,
-		logger:      log.WithField("ingress", kubernetesutil.Key(ingress)),
+		// Use a clone of the Ingress resource as we may need to modify it in order to inject the default backend.
+		ingress:   ingress.DeepCopy(),
+		options:   options,
+		kubeCache: kubeCache,
+		manager:   manager,
+		logger:    log.WithField("ingress", kubernetesutil.Key(ingress)),
+		recorder:  recorder,
 	}
 }
 
 // Translate performs translation of the associated Ingress resource into an EdgeLB pool.
 func (it *IngressTranslator) Translate() error {
+	// Attempt to determine the node port at which the default backend is exposed.
+	defaultBackendNodePort, err := it.determineDefaultBackendNodePort()
+	if err != nil {
+		return err
+	}
+
 	// Return immediately if translation is paused.
 	if it.options.EdgeLBPoolTranslationPaused {
 		it.logger.Warnf("skipping translation of %q as translation is paused for the resource", kubernetesutil.Key(it.ingress))
 		return nil
 	}
 
 	// Compute the mapping between Ingress backends defined on the current Ingress resource and their target node ports.
-	backendMap, err := it.computeIngressBackendNodePortMap()
-	if err != nil {
-		return err
-	}
+	backendMap := it.computeIngressBackendNodePortMap(defaultBackendNodePort)
 
 	// Check whether an EdgeLB pool with the requested name already exists in EdgeLB.
 	ctx, fn := context.WithTimeout(context.Background(), defaultEdgeLBManagerTimeout)
@@ -79,11 +97,37 @@ func (it *IngressTranslator) Translate() error {
 	return it.updateOrDeleteEdgeLBPool(pool, backendMap)
 }
 
+// determineDefaultBackendNodePort attempts to determine the node port at which the default backend is exposed.
+func (it *IngressTranslator) determineDefaultBackendNodePort() (int32, error) {
+	s, err := it.kubeCache.GetService(constants.KubeSystemNamespaceName, constants.DefaultBackendServiceName)
+	if err != nil {
+		return 0, fmt.Errorf("failed to read the \"%s/%s\" service: %v", constants.KubeSystemNamespaceName, constants.DefaultBackendServiceName, err)
+	}
+	if s.Spec.Type != corev1.ServiceTypeNodePort && s.Spec.Type != corev1.ServiceTypeLoadBalancer {
+		return 0, fmt.Errorf("service %q is of unexpected type %q", kubernetesutil.Key(s), s.Spec.Type)
+	}
+	for _, port := range s.Spec.Ports {
+		if port.Port == constants.DefaultBackendServicePort && port.NodePort > 0 {
+			return port.NodePort, nil
+		}
+	}
+	return 0, fmt.Errorf("no valid node port has been assigned to the default backend")
+}
+
 // computeIngressBackendNodePortMap computes the mapping between (unique) Ingress backends defined on the current Ingress resource and their target node ports.
 // It starts by compiling a set of all (possibly duplicate) Ingress backends defined on the Ingress resource.
-// Then, it iterates over said set and checks whether the referenced service port exists, adding them to the map or returning errors as appropriate.
+// In case a default backend hasn't been specified, dklb's default backend is injected as the default one.
+// Then, it iterates over said set and checks whether the referenced service port exists, adding them to the map or using the default backend's node port instead.
 // As the returned object is in fact a map, duplicate Ingress backends are automatically removed.
-func (it *IngressTranslator) computeIngressBackendNodePortMap() (IngressBackendNodePortMap, error) {
+func (it *IngressTranslator) computeIngressBackendNodePortMap(defaultBackendNodePort int32) IngressBackendNodePortMap {
+	// Inject dklb as the default backend in case none is specified.
+	if it.ingress.Spec.Backend == nil {
+		it.ingress.Spec.Backend = &extsv1beta1.IngressBackend{
+			ServiceName: defaultBackendServiceName,
+			ServicePort: defaultBackendServicePort,
+		}
+		it.recorder.Eventf(it.ingress, corev1.EventTypeWarning, constants.ReasonNoDefaultBackendSpecified, "%s will be used as the default backend since none was specified", constants.ComponentName)
+	}
 	// backends is the slice containing all Ingress backends present in the current Ingress resource.
 	backends := make([]extsv1beta1.IngressBackend, 0)
 	// Iterate over all Ingress backends, adding them to the slice of results.
@@ -94,14 +138,25 @@ func (it *IngressTranslator) computeIngressBackendNodePortMap() (IngressBackendN
 	res := make(IngressBackendNodePortMap, len(backends))
 	// Iterate over the set of Ingress backends, computing the target node port.
 	for _, backend := range backends {
+		// If the target service's name corresponds to "defaultBackendServiceName", we use the default backend's node port.
+		if backend.ServiceName == defaultBackendServiceName && backend.ServicePort == defaultBackendServicePort {
+			res[backend] = defaultBackendNodePort
+			continue
+		}
 		if nodePort, err := it.computeNodePortForIngressBackend(backend); err == nil {
 			res[backend] = nodePort
 		} else {
-			return nil, err
+			// We've failed to compute the target node port for the current backend.
+			// This may be caused by the specified Service resource being absent or not being of NodePort/LoadBalancer type.
+			// Hence, we use the default backend's node port and report the error as an event, but do not fail.
+			msg := fmt.Sprintf("using the default backend in place of \"%s:%s\": %v", backend.ServiceName, backend.ServicePort.String(), err)
+			it.recorder.Eventf(it.ingress, corev1.EventTypeWarning, constants.ReasonInvalidBackendService, msg)
+			it.logger.Warn(msg)
+			res[backend] = defaultBackendNodePort
 		}
 	}
 	// Return the populated map.
-	return res, nil
+	return res
 }
 
 // computeNodePortForIngressBackend computes the node port targeted by the specified Ingress backend.