From 7794dd8415a5bcfc4d38112c9bb530205a55700e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C3=98yvind=20Ingebrigtsen=20=C3=98vergaard?= Date: Mon, 11 Apr 2016 22:56:22 +0200 Subject: [PATCH 1/4] Move nginx.conf from base image into git It should be possible to modify the default nginx configuration. --- nginx-controller/Dockerfile | 1 + nginx-controller/nginx/nginx.conf | 32 +++++++++++++++++++++++++++++++ 2 files changed, 33 insertions(+) create mode 100644 nginx-controller/nginx/nginx.conf diff --git a/nginx-controller/Dockerfile b/nginx-controller/Dockerfile index 5f4277287d..69a635dee3 100644 --- a/nginx-controller/Dockerfile +++ b/nginx-controller/Dockerfile @@ -14,6 +14,7 @@ EXPOSE 80 443 COPY nginx-ingress / COPY nginx/ingress.tmpl / +COPY nginx/nginx.conf /etc/nginx/nginx.conf RUN rm /etc/nginx/conf.d/* diff --git a/nginx-controller/nginx/nginx.conf b/nginx-controller/nginx/nginx.conf new file mode 100644 index 0000000000..e4bad8dbc5 --- /dev/null +++ b/nginx-controller/nginx/nginx.conf @@ -0,0 +1,32 @@ + +user nginx; +worker_processes 1; + +error_log /var/log/nginx/error.log warn; +pid /var/run/nginx.pid; + + +events { + worker_connections 1024; +} + + +http { + include /etc/nginx/mime.types; + default_type application/octet-stream; + + log_format main '$remote_addr - $remote_user [$time_local] "$request" ' + '$status $body_bytes_sent "$http_referer" ' + '"$http_user_agent" "$http_x_forwarded_for"'; + + access_log /var/log/nginx/access.log main; + + sendfile on; + #tcp_nopush on; + + keepalive_timeout 65; + + #gzip on; + + include /etc/nginx/conf.d/*.conf; +} From 6dd11e5cead23974d0dfbda753e8399f8dcda479 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C3=98yvind=20Ingebrigtsen=20=C3=98vergaard?= Date: Fri, 15 Apr 2016 15:54:18 +0200 Subject: [PATCH 2/4] access_log to stdout, error_log to stderr I borrowed this idea from https://github.com/kubernetes/contrib/tree/master/ingress/controllers/nginx: * Symlink the access and error log files to respectively /dev/stdout and /dev/stderr. * When we start the nginx master process, attach its stdout and stderr to those of the controller process. Since the controller process is the entrypoint for the container, this is the containers stdout and stderr. * Run nginx in the foreground (daemon off), because in daemon mode it seems to detach its stdout. This also means that we have to start the nginx master process in a goroutine since it will no longer fork(). * nginx worker processes will inherit the master's stdout and stderr such that access_log will be written to the containers stdout and error_log will be written to the containers stderr. --- nginx-controller/Dockerfile | 6 ++++++ nginx-controller/main.go | 2 +- nginx-controller/nginx/nginx.conf | 1 + nginx-controller/nginx/nginx.go | 12 +++++++++++- 4 files changed, 19 insertions(+), 2 deletions(-) diff --git a/nginx-controller/Dockerfile b/nginx-controller/Dockerfile index 69a635dee3..59484de520 100644 --- a/nginx-controller/Dockerfile +++ b/nginx-controller/Dockerfile @@ -18,4 +18,10 @@ COPY nginx/nginx.conf /etc/nginx/nginx.conf RUN rm /etc/nginx/conf.d/* +# This works only if the nginx master process is started from the process which is the Docker entrypoint, and has its +# stdout and stderr set to the stdout and stderr of its parent (entrypoint). nginx must also run with `daemon off`, +# otherwise it will detach its stdout. +RUN ln -sf /dev/stdout /var/log/nginx/access.log +RUN ln -sf /dev/stderr /var/log/nginx/error.log + CMD ["/nginx-ingress"] diff --git a/nginx-controller/main.go b/nginx-controller/main.go index 7abdbc1462..5c1f26dff8 100644 --- a/nginx-controller/main.go +++ b/nginx-controller/main.go @@ -45,7 +45,7 @@ func main() { resolver := getKubeDNSIP(kubeClient) ngxc, _ := nginx.NewNGINXController(resolver, "/etc/nginx/", local) - ngxc.Start() + go ngxc.Start() lbc, _ := controller.NewLoadBalancerController(kubeClient, 30*time.Second, *watchNamespace, ngxc) lbc.Run() } diff --git a/nginx-controller/nginx/nginx.conf b/nginx-controller/nginx/nginx.conf index e4bad8dbc5..f563d3650f 100644 --- a/nginx-controller/nginx/nginx.conf +++ b/nginx-controller/nginx/nginx.conf @@ -1,4 +1,5 @@ +daemon off; user nginx; worker_processes 1; diff --git a/nginx-controller/nginx/nginx.go b/nginx-controller/nginx/nginx.go index 03b160554f..6f1525d914 100644 --- a/nginx-controller/nginx/nginx.go +++ b/nginx-controller/nginx/nginx.go @@ -170,7 +170,17 @@ func (nginx *NGINXController) Reload() { // Start starts NGINX func (nginx *NGINXController) Start() { if !nginx.local { - shellOut("nginx") + command := exec.Command("nginx") + command.Stdout = os.Stdout + command.Stderr = os.Stderr + err := command.Start() + if err != nil { + glog.Fatalf("Error while starting nginx: %v", err) + } + err = command.Wait() + if err != nil { + glog.Fatalf("Error while waiting for nginx: %v", err) + } } } From eefbb65baa5bbdd67ed3c11990d896062a8a6355 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C3=98yvind=20Ingebrigtsen=20=C3=98vergaard?= Date: Mon, 18 Apr 2016 13:51:16 +0200 Subject: [PATCH 3/4] Use stdout and stderr directly Don't symlink /dev/std{out,err} to /var/log/nginx/{access,error}.log. I think this makes it less convoluted. --- nginx-controller/Dockerfile | 6 ------ nginx-controller/nginx/nginx.conf | 4 ++-- 2 files changed, 2 insertions(+), 8 deletions(-) diff --git a/nginx-controller/Dockerfile b/nginx-controller/Dockerfile index 59484de520..69a635dee3 100644 --- a/nginx-controller/Dockerfile +++ b/nginx-controller/Dockerfile @@ -18,10 +18,4 @@ COPY nginx/nginx.conf /etc/nginx/nginx.conf RUN rm /etc/nginx/conf.d/* -# This works only if the nginx master process is started from the process which is the Docker entrypoint, and has its -# stdout and stderr set to the stdout and stderr of its parent (entrypoint). nginx must also run with `daemon off`, -# otherwise it will detach its stdout. -RUN ln -sf /dev/stdout /var/log/nginx/access.log -RUN ln -sf /dev/stderr /var/log/nginx/error.log - CMD ["/nginx-ingress"] diff --git a/nginx-controller/nginx/nginx.conf b/nginx-controller/nginx/nginx.conf index f563d3650f..82c4954104 100644 --- a/nginx-controller/nginx/nginx.conf +++ b/nginx-controller/nginx/nginx.conf @@ -3,7 +3,7 @@ daemon off; user nginx; worker_processes 1; -error_log /var/log/nginx/error.log warn; +error_log stderr warn; pid /var/run/nginx.pid; @@ -20,7 +20,7 @@ http { '$status $body_bytes_sent "$http_referer" ' '"$http_user_agent" "$http_x_forwarded_for"'; - access_log /var/log/nginx/access.log main; + access_log /dev/stdout main; sendfile on; #tcp_nopush on; From 2acd6fddbf4d4ca9be2aac50c6653f3bbe8a0af3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C3=98yvind=20Ingebrigtsen=20=C3=98vergaard?= Date: Mon, 25 Apr 2016 10:58:15 +0200 Subject: [PATCH 4/4] Add $host and $request_time to access_log format --- nginx-controller/nginx/nginx.conf | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/nginx-controller/nginx/nginx.conf b/nginx-controller/nginx/nginx.conf index 82c4954104..3ff91487e7 100644 --- a/nginx-controller/nginx/nginx.conf +++ b/nginx-controller/nginx/nginx.conf @@ -16,8 +16,8 @@ http { include /etc/nginx/mime.types; default_type application/octet-stream; - log_format main '$remote_addr - $remote_user [$time_local] "$request" ' - '$status $body_bytes_sent "$http_referer" ' + log_format main '$remote_addr - $remote_user [$time_local] $host "$request" ' + '$status $body_bytes_sent $request_time "$http_referer" ' '"$http_user_agent" "$http_x_forwarded_for"'; access_log /dev/stdout main;