Merge pull request #1244 from aledbf/add-custom-backend-annotation

Add custom default backend annotation

controllers/nginx/configuration.md

@@ -50,6 +50,7 @@ The following annotations are supported:
 |[ingress.kubernetes.io/base-url-scheme](#rewrite)|string|
 |[ingress.kubernetes.io/client-body-buffer-size](#client-body-buffer-size)|string|
 |[ingress.kubernetes.io/configuration-snippet](#configuration-snippet)|string|
+|[ingress.kubernetes.io/default-backend](#default-backend)|string|
 |[ingress.kubernetes.io/enable-cors](#enable-cors)|true or false|
 |[ingress.kubernetes.io/force-ssl-redirect](#server-side-https-enforcement-through-redirect)|true or false|
 |[ingress.kubernetes.io/from-to-www-redirect](#redirect-from-to-www)|true or false|
@@ -158,6 +159,10 @@ Using this annotation you can add additional configuration to the NGINX location
 ingress.kubernetes.io/configuration-snippet: |
   more_set_headers "Request-Id: $request_id";
 ```
+### Default Backend
+
+The ingress controller requires a default backend. This service is handle the response when the service in the Ingress rule does not have endpoints.
+This is a global configuration for the ingress controller. In some cases could be required to return a custom content or format. In this scenario we can use the annotation `ingress.kubernetes.io/default-backend: <svc name>` to specify a custom default backend.
 
 ### Enable CORS
 

controllers/nginx/rootfs/etc/nginx/template/nginx.tmpl

@@ -415,7 +415,8 @@ http {
             fastcgi_param       HTTP_X_Endpoints {{ .DefaultBackendEndpoints }};
             fastcgi_pass        unix:/var/run/go-fastcgi.sock;
             {{ else }}
-            return 404;
+            set $proxy_upstream_name "upstream-default-backend";
+            proxy_pass               http://upstream-default-backend;
             {{ end }}
         }
     }

core/pkg/ingress/annotations/defaultbackend/main.go

@@ -0,0 +1,57 @@
+/*
+Copyright 2015 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package defaultbackend
+
+import (
+	"fmt"
+
+	"github.com/pkg/errors"
+	extensions "k8s.io/api/extensions/v1beta1"
+
+	"k8s.io/ingress/core/pkg/ingress/annotations/parser"
+	"k8s.io/ingress/core/pkg/ingress/resolver"
+)
+
+const (
+	defaultBackend = "ingress.kubernetes.io/default-backend"
+)
+
+type backend struct {
+	serviceResolver resolver.Service
+}
+
+// NewParser creates a new default backend annotation parser
+func NewParser(sr resolver.Service) parser.IngressAnnotation {
+	return backend{sr}
+}
+
+// Parse parses the annotations contained in the ingress to use
+// a custom default backend
+func (db backend) Parse(ing *extensions.Ingress) (interface{}, error) {
+	s, err := parser.GetStringAnnotation(defaultBackend, ing)
+	if err != nil {
+		return nil, err
+	}
+
+	name := fmt.Sprintf("%v/%v", ing.Namespace, s)
+	svc, err := db.serviceResolver.GetService(s)
+	if err != nil {
+		return nil, errors.Wrapf(err, "unexpected error reading service %v", name)
+	}
+
+	return svc, nil
+}

core/pkg/ingress/annotations/secureupstream/main_test.go

@@ -24,6 +24,7 @@ import (
 	meta_v1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 
 	"fmt"
+
 	"k8s.io/apimachinery/pkg/util/intstr"
 	"k8s.io/ingress/core/pkg/ingress/resolver"
 )

core/pkg/ingress/controller/annotations.go

@@ -25,6 +25,7 @@ import (
 	"k8s.io/ingress/core/pkg/ingress/annotations/authtls"
 	"k8s.io/ingress/core/pkg/ingress/annotations/clientbodybuffersize"
 	"k8s.io/ingress/core/pkg/ingress/annotations/cors"
+	"k8s.io/ingress/core/pkg/ingress/annotations/defaultbackend"
 	"k8s.io/ingress/core/pkg/ingress/annotations/healthcheck"
 	"k8s.io/ingress/core/pkg/ingress/annotations/ipwhitelist"
 	"k8s.io/ingress/core/pkg/ingress/annotations/parser"
@@ -46,6 +47,7 @@ type extractorConfig interface {
 	resolver.AuthCertificate
 	resolver.DefaultBackend
 	resolver.Secret
+	resolver.Service
 }
 
 type annotationExtractor struct {
@@ -75,6 +77,7 @@ func newAnnotationExtractor(cfg extractorConfig) annotationExtractor {
 			"ConfigurationSnippet": snippet.NewParser(),
 			"Alias":                alias.NewParser(),
 			"ClientBodyBufferSize": clientbodybuffersize.NewParser(),
+			"DefaultBackend":       defaultbackend.NewParser(cfg),
 		},
 	}
 }
@@ -89,6 +92,10 @@ func (e *annotationExtractor) Extract(ing *extensions.Ingress) map[string]interf
 				continue
 			}
 
+			if !errors.IsLocationDenied(err) {
+				continue
+			}
+
 			_, alreadyDenied := anns[DeniedKeyName]
 			if !alreadyDenied {
 				anns[DeniedKeyName] = err

core/pkg/ingress/controller/annotations_test.go

@@ -40,7 +40,8 @@ const (
 )
 
 type mockCfg struct {
-	MockSecrets map[string]*api.Secret
+	MockSecrets  map[string]*api.Secret
+	MockServices map[string]*api.Service
 }
 
 func (m mockCfg) GetDefaultBackend() defaults.Backend {
@@ -51,6 +52,10 @@ func (m mockCfg) GetSecret(name string) (*api.Secret, error) {
 	return m.MockSecrets[name], nil
 }
 
+func (m mockCfg) GetService(name string) (*api.Service, error) {
+	return m.MockServices[name], nil
+}
+
 func (m mockCfg) GetAuthCertificate(name string) (*resolver.AuthSSLCert, error) {
 	if secret, _ := m.GetSecret(name); secret != nil {
 		return &resolver.AuthSSLCert{

core/pkg/ingress/controller/controller.go

@@ -31,6 +31,7 @@ import (
 
 	api "k8s.io/api/core/v1"
 	extensions "k8s.io/api/extensions/v1beta1"
+	"k8s.io/apimachinery/pkg/conversion"
 	"k8s.io/apimachinery/pkg/fields"
 	"k8s.io/apimachinery/pkg/util/intstr"
 	"k8s.io/apimachinery/pkg/util/runtime"
@@ -66,6 +67,8 @@ const (
 var (
 	// list of ports that cannot be used by TCP or UDP services
 	reservedPorts = []string{"80", "443", "8181", "18080"}
+
+	cloner = conversion.NewCloner()
 )
 
 // GenericController holds the boilerplate code required to build an Ingress controlller.
@@ -321,6 +324,8 @@ func newIngressController(config *Configuration) *GenericController {
 		ConfigMap: ic.mapLister,
 	})
 
+	cloner.RegisterDeepCopyFunc(ingress.GetGeneratedDeepCopyFuncs)
+
 	return &ic
 }
 
@@ -340,7 +345,7 @@ func (ic GenericController) GetDefaultBackend() defaults.Backend {
 }
 
 // GetRecorder returns the event recorder
-func (ic GenericController) GetRecoder() record.EventRecorder {
+func (ic GenericController) GetRecorder() record.EventRecorder {
 	return ic.recorder
 }
 
@@ -356,6 +361,18 @@ func (ic GenericController) GetSecret(name string) (*api.Secret, error) {
 	return s.(*api.Secret), nil
 }
 
+// GetService searches for a service in the local secrets Store
+func (ic GenericController) GetService(name string) (*api.Service, error) {
+	s, exists, err := ic.svcLister.Store.GetByKey(name)
+	if err != nil {
+		return nil, err
+	}
+	if !exists {
+		return nil, fmt.Errorf("service %v was not found", name)
+	}
+	return s.(*api.Service), nil
+}
+
 func (ic *GenericController) getConfigMap(ns, name string) (*api.ConfigMap, error) {
 	s, exists, err := ic.mapLister.Store.GetByKey(fmt.Sprintf("%v/%v", ns, name))
 	if err != nil {
@@ -688,6 +705,7 @@ func (ic *GenericController) getBackendServers() ([]*ingress.Backend, []*ingress
 						loc.Backend = ups.Name
 						loc.Port = ups.Port
 						loc.Service = ups.Service
+						loc.Ingress = ing
 						mergeLocationAnnotations(loc, anns)
 						if loc.Redirect.FromToWWW {
 							server.RedirectFromToWWW = true
@@ -704,6 +722,7 @@ func (ic *GenericController) getBackendServers() ([]*ingress.Backend, []*ingress
 						IsDefBackend: false,
 						Service:      ups.Service,
 						Port:         ups.Port,
+						Ingress:      ing,
 					}
 					mergeLocationAnnotations(loc, anns)
 					if loc.Redirect.FromToWWW {
@@ -731,12 +750,38 @@ func (ic *GenericController) getBackendServers() ([]*ingress.Backend, []*ingress
 		}
 	}
 
-	// Configure Backends[].SSLPassthrough
+	aUpstreams := make([]*ingress.Backend, 0, len(upstreams))
+
 	for _, upstream := range upstreams {
 		isHTTPSfrom := []*ingress.Server{}
 		for _, server := range servers {
 			for _, location := range server.Locations {
 				if upstream.Name == location.Backend {
+					if len(upstream.Endpoints) == 0 {
+						glog.V(3).Infof("upstream %v does not have any active endpoints. Using default backend", upstream.Name)
+						location.Backend = "upstream-default-backend"
+
+						// check if the location contains endpoints and a custom default backend
+						if location.DefaultBackend != nil {
+							sp := location.DefaultBackend.Spec.Ports[0]
+							endps := ic.getEndpoints(location.DefaultBackend, &sp, api.ProtocolTCP, &healthcheck.Upstream{})
+							if len(endps) > 0 {
+								glog.V(3).Infof("using custom default backend in server %v location %v (service %v/%v)",
+									server.Hostname, location.Path, location.DefaultBackend.Namespace, location.DefaultBackend.Name)
+								b, err := cloner.DeepCopy(upstream)
+								if err == nil {
+									name := fmt.Sprintf("custom-default-backend-%v", upstream.Name)
+									nb := b.(*ingress.Backend)
+									nb.Name = name
+									nb.Endpoints = endps
+									aUpstreams = append(aUpstreams, nb)
+									location.Backend = name
+								}
+							}
+						}
+					}
+
+					// Configure Backends[].SSLPassthrough
 					if server.SSLPassthrough {
 						if location.Path == rootLocation {
 							if location.Backend == defUpstreamName {
@@ -746,24 +791,24 @@ func (ic *GenericController) getBackendServers() ([]*ingress.Backend, []*ingress
 
 							isHTTPSfrom = append(isHTTPSfrom, server)
 						}
-						continue
 					}
 				}
 			}
 		}
+
 		if len(isHTTPSfrom) > 0 {
 			upstream.SSLPassthrough = true
 		}
 	}
 
-	aUpstreams := make([]*ingress.Backend, 0, len(upstreams))
-	for _, value := range upstreams {
-		if len(value.Endpoints) == 0 {
-			glog.V(3).Infof("upstream %v does not have any active endpoints. Using default backend", value.Name)
-			value.Endpoints = append(value.Endpoints, ic.cfg.Backend.DefaultEndpoint())
+	// create the list of upstreams and skip those without endpoints
+	for _, upstream := range upstreams {
+		if len(upstream.Endpoints) == 0 {
+			continue
 		}
-		aUpstreams = append(aUpstreams, value)
+		aUpstreams = append(aUpstreams, upstream)
 	}
+
 	if ic.cfg.SortBackends {
 		sort.Sort(ingress.BackendByNameServers(aUpstreams))
 	}

core/pkg/ingress/resolver/main.go

@@ -41,6 +41,12 @@ type AuthCertificate interface {
 	GetAuthCertificate(string) (*AuthSSLCert, error)
 }
 
+// Service has a method that searches for services contenating
+// the namespace and name using a the character /
+type Service interface {
+	GetService(string) (*api.Service, error)
+}
+
 // AuthSSLCert contains the necessary information to do certificate based
 // authentication of an ingress location
 type AuthSSLCert struct {

core/pkg/ingress/types.go

@@ -150,6 +150,7 @@ type Configuration struct {
 }
 
 // Backend describes one or more remote server/s (endpoints) associated with a service
+// +k8s:deepcopy-gen=true
 type Backend struct {
 	// Name represents an unique api.Service name formatted as <namespace>-<name>-<port>
 	Name    string             `json:"name"`
@@ -177,19 +178,22 @@ type Backend struct {
 // restarts. Exactly one of these values will be set on the upstream, since multiple
 // affinity values are incompatible. Once set, the backend makes no guarantees
 // about honoring updates.
+// +k8s:deepcopy-gen=true
 type SessionAffinityConfig struct {
 	AffinityType          string                `json:"name"`
 	CookieSessionAffinity CookieSessionAffinity `json:"cookieSessionAffinity"`
 }
 
 // CookieSessionAffinity defines the structure used in Affinity configured by Cookies.
+// +k8s:deepcopy-gen=true
 type CookieSessionAffinity struct {
 	Name      string              `json:"name"`
 	Hash      string              `json:"hash"`
 	Locations map[string][]string `json:"locations,omitempty"`
 }
 
 // Endpoint describes a kubernetes endpoint in a backend
+// +k8s:deepcopy-gen=true
 type Endpoint struct {
 	// Address IP address of the endpoint
 	Address string `json:"address"`
@@ -261,11 +265,14 @@ type Location struct {
 	// contains active endpoints or not. Returning true means the location
 	// uses the default backend.
 	IsDefBackend bool `json:"isDefBackend"`
+	// Ingress returns the ingress from which this location was generated
+	Ingress *extensions.Ingress `json:"ingress"`
 	// Backend describes the name of the backend to use.
 	Backend string `json:"backend"`
-
-	Service *api.Service       `json:"service,omitempty"`
-	Port    intstr.IntOrString `json:"port"`
+	// Service describes the referenced services from the ingress
+	Service *api.Service `json:"service,omitempty"`
+	// Port describes to which port from the service
+	Port intstr.IntOrString `json:"port"`
 	// BasicDigestAuth returns authentication configuration for
 	// an Ingress rule.
 	// +optional
@@ -301,14 +308,17 @@ type Location struct {
 	Proxy proxy.Configuration `json:"proxy,omitempty"`
 	// UsePortInRedirects indicates if redirects must specify the port
 	// +optional
-	UsePortInRedirects bool `json:"use-port-in-redirects"`
+	UsePortInRedirects bool `json:"usePortInRedirects"`
 	// ConfigurationSnippet contains additional configuration for the backend
 	// to be considered in the configuration of the location
-	ConfigurationSnippet string `json:"configuration-snippet"`
+	ConfigurationSnippet string `json:"configurationSnippet"`
 	// ClientBodyBufferSize allows for the configuration of the client body
 	// buffer size for a specific location.
 	// +optional
-	ClientBodyBufferSize string `json:"client-body-buffer-size,omitempty"`
+	ClientBodyBufferSize string `json:"clientBodyBufferSize,omitempty"`
+	// DefaultBackend allows the use of a custom default backend for this location.
+	// +optional
+	DefaultBackend *api.Service `json:"defaultBackend,omitempty"`
 }
 
 // SSLPassthroughBackend describes a SSL upstream server configured