Merge pull request #157 from nginxinc/graceful-termination

Add graceful termination

nginx-controller/controller/controller.go

@@ -260,6 +260,17 @@ func (lbc *LoadBalancerController) Run() {
 	<-lbc.stopCh
 }
 
+// Stop shutdowns the load balancer controller
+func (lbc *LoadBalancerController) Stop() {
+	close(lbc.stopCh)
+
+	lbc.ingQueue.shutdown()
+	if lbc.watchNginxConfigMaps {
+		lbc.cfgmQueue.shutdown()
+	}
+	lbc.endpQueue.shutdown()
+}
+
 func (lbc *LoadBalancerController) syncEndp(key string) {
 	glog.V(3).Infof("Syncing endpoints %v", key)
 

nginx-controller/main.go

@@ -2,6 +2,9 @@ package main
 
 import (
 	"flag"
+	"os"
+	"os/signal"
+	"syscall"
 	"time"
 
 	"github.com/golang/glog"
@@ -81,11 +84,13 @@ func main() {
 		nginxIngressTemplatePath = "nginx-plus.ingress.tmpl"
 	}
 	ngxc, _ := nginx.NewNginxController("/etc/nginx/", local, *healthStatus, nginxConfTemplatePath, nginxIngressTemplatePath)
-	ngxc.Start()
+	nginxDone := make(chan error, 1)
+	ngxc.Start(nginxDone)
 
 	nginxConfig := nginx.NewDefaultConfig()
 	var nginxAPI *plus.NginxAPIController
 	if *nginxPlus {
+		time.Sleep(500 * time.Millisecond)
 		nginxAPI, err = plus.NewNginxAPIController("http://127.0.0.1:8080/upstream_conf", "http://127.0.0.1:8080/status", local)
 		if err != nil {
 			glog.Fatalf("Failed to create NginxAPIController: %v", err)
@@ -94,5 +99,44 @@ func main() {
 	cnf := nginx.NewConfigurator(ngxc, nginxConfig, nginxAPI)
 
 	lbc, _ := controller.NewLoadBalancerController(kubeClient, 30*time.Second, *watchNamespace, cnf, *nginxConfigMaps, *nginxPlus)
+	go handleTermination(lbc, ngxc, nginxDone)
 	lbc.Run()
+
+	for {
+		glog.Info("Waiting for the controller to exit...")
+		time.Sleep(30 * time.Second)
+	}
+}
+
+func handleTermination(lbc *controller.LoadBalancerController, ngxc *nginx.NginxController, nginxDone chan error) {
+	signalChan := make(chan os.Signal, 1)
+	signal.Notify(signalChan, syscall.SIGTERM)
+
+	exitStatus := 0
+	exited := false
+
+	select {
+	case err := <-nginxDone:
+		if err != nil {
+			glog.Errorf("nginx command exited with an error: %v", err)
+			exitStatus = 1
+		} else {
+			glog.Info("nginx command exited succesfully")
+		}
+		exited = true
+	case <-signalChan:
+		glog.Infof("Received SIGTERM, shutting down")
+	}
+
+	glog.Infof("Shutting down the controller")
+	lbc.Stop()
+
+	if !exited {
+		glog.Infof("Shutting down NGINX")
+		ngxc.Quit()
+		<-nginxDone
+	}
+
+	glog.Infof("Exiting with a status: %v", exitStatus)
+	os.Exit(exitStatus)
 }

nginx-controller/nginx/nginx.go

@@ -258,16 +258,33 @@ func (nginx *NginxController) Reload() error {
 }
 
 // Start starts NGINX
-func (nginx *NginxController) Start() {
+func (nginx *NginxController) Start(done chan error) {
 	if !nginx.local {
-		if err := shellOut("nginx"); err != nil {
+		cmd := exec.Command("nginx")
+		cmd.Stdout = os.Stdout
+		cmd.Stderr = os.Stderr
+		if err := cmd.Start(); err != nil {
 			glog.Fatalf("Failed to start nginx: %v", err)
 		}
+		go func() {
+			done <- cmd.Wait()
+		}()
 	} else {
 		glog.V(3).Info("Starting nginx")
 	}
 }
 
+// Quit shutdowns NGINX gracefully
+func (nginx *NginxController) Quit() {
+	if !nginx.local {
+		if err := shellOut("nginx -s quit"); err != nil {
+			glog.Fatalf("Failed to quit nginx: %v", err)
+		}
+	} else {
+		glog.V(3).Info("Quitting nginx")
+	}
+}
+
 func createDir(path string) {
 	if err := os.Mkdir(path, os.ModeDir); err != nil {
 		glog.Fatalf("Couldn't create directory %v: %v", path, err)

nginx-controller/nginx/templates/nginx-plus.tmpl

@@ -2,6 +2,8 @@
 user  nginx;
 worker_processes  auto;
 
+daemon off;
+
 error_log  /var/log/nginx/error.log notice;
 pid        /var/run/nginx.pid;
 

nginx-controller/nginx/templates/nginx.tmpl

@@ -2,6 +2,8 @@
 user  nginx;
 worker_processes  auto;
 
+daemon off;
+
 error_log  /var/log/nginx/error.log warn;
 pid        /var/run/nginx.pid;