New option to set a shutdown grace period

cmd/nginx/flags.go

@@ -171,6 +171,8 @@ Takes the form "<host>:port". If not provided, no admission controller is starte
 		profilerPort = flags.Int("profiler-port", 10245, "Port to use for expose the ingress controller Go profiler when it is enabled.")
 
 		statusUpdateInterval = flags.Int("status-update-interval", status.UpdateInterval, "Time interval in seconds in which the status should check if an update is required. Default is 60 seconds")
+
+		shutdownGracePeriod = flags.Int("shutdown-grace-period", 0, "Seconds to wait after receiving the shutdown signal, before stopping the nginx process.")
 	)
 
 	flags.StringVar(&nginx.MaxmindMirror, "maxmind-mirror", "", `Maxmind mirror url (example: http://geoip.local/databases`)
@@ -281,6 +283,7 @@ https://blog.maxmind.com/2019/12/18/significant-changes-to-accessing-and-using-g
 		PublishService:         *publishSvc,
 		PublishStatusAddress:   *publishStatusAddress,
 		UpdateStatusOnShutdown: *updateStatusOnShutdown,
+		ShutdownGracePeriod:    *shutdownGracePeriod,
 		UseNodeInternalIP:      *useNodeInternalIP,
 		SyncRateLimit:          *syncRateLimit,
 		ListenPorts: &ngx_config.ListenPorts{

docs/user-guide/cli-arguments.md

@@ -53,6 +53,7 @@ They are set in the container spec of the `nginx-ingress-controller` Deployment
 | `--udp-services-configmap`         | Name of the ConfigMap containing the definition of the UDP services to expose. The key in the map indicates the external port to be used. The value is a reference to a Service in the form "namespace/name:port", where "port" can either be a port name or number. |
 | `--update-status`                  | Update the load-balancer status of Ingress objects this controller satisfies. Requires setting the publish-service parameter to a valid Service reference. (default true) |
 | `--update-status-on-shutdown`      | Update the load-balancer status of Ingress objects when the controller shuts down. Requires the update-status parameter. (default true) |
+| `--shutdown-grace-period`          | Seconds to wait after receiving the shutdown signal, before stopping the nginx process. |
 | `-v, --v Level`                    | number for the log level verbosity |
 | `--validating-webhook`             | The address to start an admission controller on to validate incoming ingresses. Takes the form "<host>:port". If not provided, no admission controller is started. |
 | `--validating-webhook-certificate` | The path of the validating webhook certificate PEM. |

internal/ingress/controller/controller.go

@@ -106,6 +106,8 @@ type Configuration struct {
 	MaxmindEditionFiles []string
 
 	MonitorMaxBatchSize int
+
+	ShutdownGracePeriod int
 }
 
 // GetPublishService returns the Service used to set the load-balancer status of Ingresses.

internal/ingress/controller/nginx.go

@@ -367,6 +367,8 @@ func (n *NGINXController) Stop() error {
 		return fmt.Errorf("shutdown already in progress")
 	}
 
+	time.Sleep(time.Duration(n.cfg.ShutdownGracePeriod) * time.Second)
+
 	klog.InfoS("Shutting down controller queues")
 	close(n.stopCh)
 	go n.syncQueue.Shutdown()

test/e2e/framework/healthz.go

@@ -0,0 +1,44 @@
+/*
+Copyright 2020 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package framework
+
+import (
+	"fmt"
+	"net/http"
+)
+
+// VerifyHealthz verifies the status code of the healthz endpoint
+func (f *Framework) VerifyHealthz(ip string, statusCode int) error {
+	url := fmt.Sprintf("http://%v:10254/healthz", ip)
+
+	client := &http.Client{}
+	req, err := http.NewRequest("GET", url, nil)
+	if err != nil {
+		return fmt.Errorf("creating GET request for URL %q failed: %v", url, err)
+	}
+	resp, err := client.Do(req)
+	if err != nil {
+		return fmt.Errorf("executing GET request for URL %q failed: %v", url, err)
+	}
+	defer resp.Body.Close()
+
+	if resp.StatusCode != statusCode {
+		return fmt.Errorf("GET request for URL %q returned HTTP status %s", url, resp.Status)
+	}
+
+	return nil
+}

test/e2e/gracefulshutdown/grace_period.go

@@ -0,0 +1,88 @@
+/*
+Copyright 2020 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package gracefulshutdown
+
+import (
+	"context"
+	"net/http"
+	"strings"
+	"time"
+
+	"github.com/onsi/ginkgo"
+	"github.com/stretchr/testify/assert"
+	appsv1 "k8s.io/api/apps/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/ingress-nginx/test/e2e/framework"
+)
+
+var _ = framework.IngressNginxDescribe("[Shutdown] Grace period shutdown", func() {
+	f := framework.NewDefaultFramework("shutdown-grace-period")
+
+	ginkgo.It("/healthz should return status code 500 during shutdown grace period", func() {
+
+		f.NewSlowEchoDeployment()
+
+		err := f.UpdateIngressControllerDeployment(func(deployment *appsv1.Deployment) error {
+			args := []string{}
+			for _, v := range deployment.Spec.Template.Spec.Containers[0].Args {
+				if strings.Contains(v, "--shutdown-grace-period") {
+					continue
+				}
+
+				args = append(args, v)
+			}
+
+			args = append(args, "--shutdown-grace-period=90")
+			deployment.Spec.Template.Spec.Containers[0].Args = args
+			cmds := []string{"/wait-shutdown"}
+			deployment.Spec.Template.Spec.Containers[0].Lifecycle.PreStop.Exec.Command = cmds
+			grace := int64(3600)
+			deployment.Spec.Template.Spec.TerminationGracePeriodSeconds = &grace
+			_, err := f.KubeClientSet.AppsV1().Deployments(f.Namespace).Update(context.TODO(), deployment, metav1.UpdateOptions{})
+			return err
+		})
+
+		assert.Nil(ginkgo.GinkgoT(), err, "updating ingress controller deployment flags")
+
+		ip := f.GetNginxPodIP()
+
+		err = f.VerifyHealthz(ip, http.StatusOK)
+		assert.Nil(ginkgo.GinkgoT(), err)
+
+		result := make(chan []error)
+		go func(c chan []error) {
+			defer ginkgo.GinkgoRecover()
+			errors := []error{}
+
+			framework.Sleep(60 * time.Second)
+
+			err = f.VerifyHealthz(ip, http.StatusInternalServerError)
+			if err != nil {
+				errors = append(errors, err)
+			}
+
+			c <- errors
+		}(result)
+
+		f.ScaleDeploymentToZero("nginx-ingress-controller")
+
+		for _, err := range <-result {
+			assert.Nil(ginkgo.GinkgoT(), err)
+		}
+
+	})
+})