Add support for Opentracing

build/DockerfileWithOpentracing

@@ -0,0 +1,83 @@
+ARG NGINX_VERSION=1.17.1
+ARG OPENTRACING_CPP_VERSION=1.5.1
+
+FROM nginx:${NGINX_VERSION} AS opentracing-builder
+ARG OPENTRACING_CPP_VERSION
+
+RUN set -x \
+    && apt-get update \
+    && apt-get install --no-install-recommends --no-install-suggests -y \
+    libcurl4-openssl-dev \
+    build-essential \
+    cmake \
+    git \
+    ca-certificates \
+    pkg-config \
+    wget \
+    libz-dev \
+    automake \
+    autogen \
+    autoconf \
+    libtool \
+    libpcre3 \
+    libpcre3-dev \
+    && tempDir="$(mktemp -d)" \
+    && chmod 777 "$tempDir" \
+    && git clone https://github.com/opentracing-contrib/nginx-opentracing src \
+    && cd "$tempDir" \
+    && git clone -b v$OPENTRACING_CPP_VERSION https://github.com/opentracing/opentracing-cpp.git \
+    && cd opentracing-cpp \
+    && mkdir .build && cd .build \
+    && cmake -DCMAKE_BUILD_TYPE=Release \
+    -DBUILD_TESTING=OFF .. \
+    && make && make install \
+    && cd "$tempDir" \
+    && NGINX_VERSION_BUILD=`nginx -v 2>&1` && NGINX_VERSION_BUILD=${NGINX_VERSION_BUILD#*nginx/} \
+    && echo "deb-src http://nginx.org/packages/mainline/debian/ stretch nginx" >> /etc/apt/sources.list \
+    && apt-get update \
+    && apt-get build-dep -y nginx=${NGINX_VERSION_BUILD} \
+    && wget -O nginx-release-${NGINX_VERSION_BUILD}.tar.gz https://github.com/nginx/nginx/archive/release-${NGINX_VERSION_BUILD}.tar.gz \
+    && tar zxf nginx-release-${NGINX_VERSION_BUILD}.tar.gz \
+    && cd nginx-release-${NGINX_VERSION_BUILD} \
+    && auto/configure \
+    --with-compat \
+    --add-dynamic-module=/src/opentracing \
+    --with-debug \
+    && make modules \
+    && cp objs/ngx_http_opentracing_module.so  /ngx_http_opentracing_module.so
+
+
+FROM debian:stretch-slim AS tracer-downloader
+ARG JAEGER_VERSION=v0.4.2
+RUN set -x \
+    && apt-get update \
+    && apt-get install --no-install-recommends --no-install-suggests -y ca-certificates apt-transport-https wget \
+    && wget https://github.com/jaegertracing/jaeger-client-cpp/releases/download/${JAEGER_VERSION}/libjaegertracing_plugin.linux_amd64.so -O /usr/local/lib/libjaegertracing_plugin.so
+
+
+# Final Image
+FROM nginx:${NGINX_VERSION}
+ARG OPENTRACING_CPP_VERSION
+# forward nginx access and error logs to stdout and stderr of the ingress
+# controller process
+RUN ln -sf /proc/1/fd/1 /var/log/nginx/access.log \
+    && ln -sf /proc/1/fd/1 /var/log/nginx/stream-access.log \
+    && ln -sf /proc/1/fd/2 /var/log/nginx/error.log
+
+COPY nginx-ingress internal/configs/version1/nginx.ingress.tmpl internal/configs/version1/nginx.tmpl internal/configs/version2/nginx.virtualserver.tmpl /
+
+COPY --from=opentracing-builder /ngx_http_opentracing_module.so /usr/lib/nginx/modules/ngx_http_opentracing_module.so
+COPY --from=opentracing-builder /usr/local/lib/libopentracing.so.${OPENTRACING_CPP_VERSION} /usr/local/lib/libopentracing.so.1
+# Edit the line below to use a different tracer
+COPY --from=tracer-downloader /usr/local/lib/libjaegertracing_plugin.so /usr/local/lib/libjaegertracing_plugin.so
+RUN ldconfig
+
+RUN rm /etc/nginx/conf.d/*
+
+RUN mkdir -p /etc/nginx/secrets
+
+# Uncomment the line below if you would like to add the default.pem to the image
+# and use it as a certificate and key for the default server
+# ADD default.pem /etc/nginx/secrets/default
+
+ENTRYPOINT ["/nginx-ingress"]

build/DockerfileWithOpentracingForPlus

@@ -0,0 +1,79 @@
+FROM debian:stretch-slim AS tracer-downloader
+ARG JAEGER_VERSION=v0.4.2
+
+RUN set -x \
+    && apt-get update \
+    && apt-get install --no-install-recommends --no-install-suggests -y ca-certificates apt-transport-https wget \
+    && wget https://github.com/jaegertracing/jaeger-client-cpp/releases/download/${JAEGER_VERSION}/libjaegertracing_plugin.linux_amd64.so -O /usr/local/lib/libjaegertracing_plugin.so
+
+
+# Final Image
+FROM debian:stretch-slim
+LABEL maintainer="NGINX Docker Maintainers <[email protected]>"
+
+ENV NGINX_PLUS_VERSION 18-1~stretch
+ARG IC_VERSION
+
+# Download certificate and key from the customer portal (https://cs.nginx.com)
+# and copy to the build context
+COPY nginx-repo.crt /etc/ssl/nginx/
+COPY nginx-repo.key /etc/ssl/nginx/
+
+# Make sure the certificate and key have correct permissions
+RUN chmod 644 /etc/ssl/nginx/*
+
+# Install NGINX Plus
+RUN set -x \
+    && apt-get update \
+    && apt-get install --no-install-recommends --no-install-suggests -y apt-transport-https ca-certificates gnupg1 \
+    && \
+    NGINX_GPGKEY=573BFD6B3D8FBC641079A6ABABF5BD827BD9BF62; \
+    found=''; \
+    for server in \
+    ha.pool.sks-keyservers.net \
+    hkp://keyserver.ubuntu.com:80 \
+    hkp://p80.pool.sks-keyservers.net:80 \
+    pgp.mit.edu \
+    ; do \
+    echo "Fetching GPG key $NGINX_GPGKEY from $server"; \
+    apt-key adv --keyserver "$server" --keyserver-options timeout=10 --recv-keys "$NGINX_GPGKEY" && found=yes && break; \
+    done; \
+    test -z "$found" && echo >&2 "error: failed to fetch GPG key $NGINX_GPGKEY" && exit 1; \
+    echo "Acquire::https::plus-pkgs.nginx.com::Verify-Peer \"true\";" >> /etc/apt/apt.conf.d/90nginx \
+    && echo "Acquire::https::plus-pkgs.nginx.com::Verify-Host \"true\";" >> /etc/apt/apt.conf.d/90nginx \
+    && echo "Acquire::https::plus-pkgs.nginx.com::SslCert     \"/etc/ssl/nginx/nginx-repo.crt\";" >> /etc/apt/apt.conf.d/90nginx \
+    && echo "Acquire::https::plus-pkgs.nginx.com::SslKey      \"/etc/ssl/nginx/nginx-repo.key\";" >> /etc/apt/apt.conf.d/90nginx \
+    && echo "Acquire::https::plus-pkgs.nginx.com::User-Agent  \"k8s-ic-$IC_VERSION-apt\";" >> /etc/apt/apt.conf.d/90nginx \
+    && printf "deb https://plus-pkgs.nginx.com/debian stretch nginx-plus\n" > /etc/apt/sources.list.d/nginx-plus.list \
+    && apt-get update && apt-get install -y \
+    nginx-plus=${NGINX_PLUS_VERSION} \
+    # Install OpenTracing module
+    nginx-plus-module-opentracing \ 
+    && apt-get remove --purge --auto-remove -y gnupg1 \
+    && rm -rf /var/lib/apt/lists/* \
+    && rm -rf /etc/ssl/nginx \
+    && rm /etc/apt/apt.conf.d/90nginx /etc/apt/sources.list.d/nginx-plus.list
+
+
+# forward nginx access and error logs to stdout and stderr of the ingress
+# controller process
+RUN ln -sf /proc/1/fd/1 /var/log/nginx/access.log \
+    && ln -sf /proc/1/fd/1 /var/log/nginx/stream-access.log \
+    && ln -sf /proc/1/fd/2 /var/log/nginx/error.log
+
+
+# Edit the line below to use a different tracer
+COPY --from=tracer-downloader /usr/local/lib/libjaegertracing_plugin.so /usr/local/lib/libjaegertracing_plugin.so
+
+EXPOSE 80 443
+
+COPY nginx-ingress internal/configs/version1/nginx-plus.ingress.tmpl internal/configs/version1/nginx-plus.tmpl internal/configs/version2/nginx-plus.virtualserver.tmpl  /
+
+RUN rm /etc/nginx/conf.d/* \
+    && mkdir -p /etc/nginx/secrets
+
+# Uncomment the line below if you would like to add the default.pem to the image
+# and use it as a certificate and key for the default server
+# ADD default.pem /etc/nginx/secrets/default
+
+ENTRYPOINT ["/nginx-ingress"]

build/README.md

@@ -74,10 +74,12 @@ The **Makefile** contains the following main variables for you to customize (eit
 * **VERSION** -- the current version of the controller.
 * **TAG** -- the tag added to the image. It's set to the value of the `VERSION` variable by default.
 * **PUSH_TO_GCR**. If you’re running your Kubernetes in GCE and using Google Container Registry, make sure that `PUSH_TO_GCR = 1`. This means using the `gcloud docker push` command to push the image, which is convenient when pushing images to GCR. By default, the variable is unset and the regular `docker push` command is used to push the image to the registry.
-* **DOCKERFILE** -- the path to a Dockerfile. We provide three Dockerfiles:
+* **DOCKERFILE** -- the path to a Dockerfile. We provide five Dockerfiles:
   1. `Dockerfile`, for building a debian-based image with NGINX. It's used by default.
   1. `DockerfileForAlpine`, for building an alpine-based image with NGINX.
   1. `DockerfileForPlus`, for building an debian-based image with NGINX Plus.
+  1. `DockerfileWithOpentracing`, for building a debian-based image with NGINX, [opentracing](https://github.com/opentracing-contrib/nginx-opentracing) module and the [Jaeger](https://www.jaegertracing.io/) tracer.
+  1. `DockerfileWithOpentracingForPlus`, for building a debian-based image with NGINX Plus, [opentracing](https://github.com/opentracing-contrib/nginx-opentracing) module and the [Jaeger](https://www.jaegertracing.io/) tracer.
 * **GENERATE_DEFAULT_CERT_AND_KEY** - The Ingress controller requires a certificate and a key for the default HTTP/HTTPS server. You can reference them in a TLS Secret in a command-line argument to the Ingress controller. As an alternative, you can add a file in the PEM format with your certificate and key to the image as `/etc/nginx/secrets/default`. Optionally, you can generate a self-signed certificate and a key during the build process. Set `GENERATE_DEFAULT_CERT_AND_KEY` to `1` to generate a certificate and a key in the `default.pem` file. Note that you must add the `ADD` instruction in the Dockerfile to copy the cert and the key to the image. The default value of `GENERATE_DEFAULT_CERT_AND_KEY` is `0`.
 * **DOCKER_BUILD_OPTIONS** -- the [options](https://docs.docker.com/engine/reference/commandline/build/#options) for the `docker build` command. For example, `--pull`.
 * **BUILD_IN_CONTAINER** -- By default, to compile the controller we use the [golang](https://hub.docker.com/_/golang/) container that we run as part of the building process. If you want to compile the controller using your local golang environment:

cmd/nginx-ingress/main.go

@@ -327,7 +327,17 @@ func main() {
 		glog.Fatalf("Error generating NGINX main config: %v", err)
 	}
 	nginxManager.CreateMainConfig(content)
-	nginxManager.UpdateConfigVersionFile()
+
+	nginxManager.UpdateConfigVersionFile(ngxConfig.OpenTracingLoadModule)
+
+	nginxManager.SetOpenTracing(ngxConfig.OpenTracingLoadModule)
+
+	if ngxConfig.OpenTracingLoadModule {
+		err := nginxManager.CreateOpenTracingTracerConfig(cfgParams.MainOpenTracingTracerConfig)
+		if err != nil {
+			glog.Fatalf("Error creating OpenTracing tracer config file: %v", err)
+		}
+	}
 
 	nginxDone := make(chan error, 1)
 	nginxManager.Start(nginxDone)

docs/configmap-and-annotations.md

@@ -184,3 +184,11 @@ spec:
 | N/A | `stream-snippets` | Sets a custom snippet in stream context. | N/A | [Support for  TCP/UDP Load Balancing](../examples/tcp-udp). |
 | N/A | `main-template` | Sets the main NGINX configuration template. | By default the template is read from the file in the container. | [Custom Templates](../examples/custom-templates). |
 | N/A | `ingress-template` | Sets the NGINX configuration template for an Ingress resource. | By default the template is read from the file on the container. | [Custom Templates](../examples/custom-templates). |
+
+
+### Modules
+| Annotation | ConfigMap Key | Description | Default | Example |
+| ---------- | -------------- | ----------- | ------- | ------- |
+| N/A | `opentracing` | Enables [OpenTracing](https://opentracing.io) globally (for all Ingress, VirtualServer and VirtualServerRoute resources). Note: requires the Ingress Controller image with OpenTracing module and a tracer. See the [docs](./opentracing.md) for more information. | `False` | [Support for OpenTracing](../examples/opentracing/README.md). |
+| N/A | `opentracing-tracer` | Sets the path to the vendor tracer binary plugin.| N/A | [Support for OpenTracing](../examples/opentracing/README.md). |
+| N/A | `opentracing-tracer-config` | Sets the tracer configuration in JSON format.| N/A | [Support for OpenTracing](../examples/opentracing/README.md). |

docs/opentracing.md

@@ -0,0 +1,80 @@
+# OpenTracing Support
+The Ingress Controller supports [OpenTracing](http://opentracing.io/) with the third-party module [opentracing-contrib/nginx-opentracing](https://github.com/opentracing-contrib/nginx-opentracing).
+
+This document explains how to use OpenTracing with the Ingress Controller. Additionally, we have an [example](../examples/opentracing/README.md) on how to enable OpenTracing for a simple web application using Jaeger as a tracer.
+
+## Prerequisites
+1. **Use the Ingress Controller image with OpenTracing.** The default Ingress Controller images don’t include the OpenTraing module. To use OpenTracing, you need to build the image with that module. Follow the build instructions to build the image using `DockerfileWithOpentracing` for NGINX or `DockerfileWithOpentracingForPlus` for NGINX Plus. 
+By default, the Dockerfiles install Jaeger as a tracer. However, it is possible to replace Jaeger with other supported [tracers](https://github.com/opentracing-contrib/nginx-opentracing#building-from-source). For that, please modify the Dockerfile accordingly:
+   1. Change the download line in the tracer-downloader stage of the Dockerfile to download the right tracer.
+   1. Edit the COPY line of the final image to copy the previously downloaded tracer to the image
+
+1. **Load the OpenTracing module.** You need to load the module with the configuration for the chosen tracer using the following ConfigMap keys:
+   * `opentracing-tracer`: sets the path to the vendor tracer binary plugin. This is the path you used in the COPY line of step *ii* above.
+   * `opentracing-tracer-config`: sets the tracer configuration in JSON format.
+
+   Below an example on how to use those keys to load the module with Jaeger tracer:
+    ```yaml
+    opentracing-tracer: "/usr/local/lib/libjaegertracing_plugin.so"
+    opentracing-tracer-config: |
+            {
+                "service_name": "nginx-ingress",
+                "sampler": {
+                    "type": "const",
+                    "param": 1
+                },
+                "reporter": {
+                    "localAgentHostPort": "jaeger-agent.default.svc.cluster.local:6831"
+                }
+            }
+    ```
+
+## Enable OpenTracing Globally
+To enable OpenTracing globally (for all Ingress, VirtualServer and VirtualServerRoute resources), set the `opentracing` ConfigMap key to `True`:
+
+```yaml
+opentracing: True
+```
+
+## Enable/Disable OpenTracing per Ingress Resource
+
+It is possible to use annotations to enable or disable OpenTracing for a specific Ingress Resource. As mentioned in the prerequisites section, both `opentracing-tracer` and `opentracing-tracer-config` must be configured.
+
+Consider the following two cases:
+1. OpenTracing is globally disabled.
+   1. To enable OpenTracing for a specific Ingress Resource, use the server snippet annotation:
+        ```yaml
+        nginx.org/server-snippets: |
+            opentracing on;
+        ```
+   1. To enable OpenTracing for specific paths, (1) you need to use [Mergeable Ingress resources](../examples/mergeable-ingress-types/README.md) and (2) use the location snippets annotation to enable OpenTracing for the paths of a specific Minion Ingress resource:
+        ```yaml
+        nginx.org/location-snippets: |
+            opentracing on;
+        ```
+
+2. OpenTracing is globally enabled:
+   1. To disable OpenTracing for a specific Ingress Resource, use the server snippet annotation:  
+        ```yaml
+        nginx.org/server-snippets: |
+            opentracing off;
+        ```
+
+   1. To disable OpenTracing for specific paths, (1) you need to use [Mergeable Ingress resources](../examples/mergeable-ingress-types/README.md) and (2) use the location snippets annotation to disable OpenTracing for the paths of a specific Minion Ingress resource:
+        ```yaml
+        nginx.org/location-snippets: |
+            opentracing off;
+        ```
+
+## Customize OpenTracing
+
+You can customize OpenTracing though the supported [OpenTracing module directives](https://github.com/opentracing-contrib/nginx-opentracing/blob/master/doc/Reference.md). Use the snippets ConfigMap keys or annotations to insert those directives into the http, server or location contexts of the generated NGINX configuration. 
+
+For example, to propagate the active span context for upstream requests, it is required to set the `opentracing_propagate_context` directive, which you can add to an Ingress resource using the location snippets annotation: 
+
+```yaml
+nginx.org/location-snippets: |
+   opentracing_propagate_context;
+```
+
+**Note**: `opentracing_propagate_context` and `opentracing_grpc_propagate_context` directives can be used in http, server or location contexts according to the [module documentation](https://github.com/opentracing-contrib/nginx-opentracing/blob/master/doc/Reference.md#opentracing_propagate_context). However, because of the way the module works and how the Ingress Controller generates the NGINX configuration, it is only possible to use the directive in the location context.

examples/opentracing/README.md

@@ -0,0 +1,48 @@
+
+# OpenTracing Support
+
+In this example we deploy the NGINX or NGINX Plus Ingress Controller and a simple web application. Then we enable OpenTracing and use a tracer (Jaeger) for tracing the requests that go through NGINX or NGINX Plus to the web application.
+
+## Prerequisites
+
+The default Ingress Controller images don’t include the OpenTracing module required for this example. See Step 1 of the [Prerequisites section](../../docs/opentracing.md#Prerequisites) in the OpenTracing doc for the instructions on how to get the right image with Jaeger tracer. 
+
+## Step 1 - Deploy Ingress Controller and the Cafe App
+
+Follow steps 1-3 of the [complete example](../complete-example/README.md) to deploy the Ingress Controller and the cafe app. Make sure to use the Ingress Controller  image with the OpenTracing module and the tracer installed.
+
+## Step 2 - Deploy a Tracer
+
+1. Use the [all-in-one dev template](https://github.com/jaegertracing/jaeger-kubernetes#development-setup) to deploy Jaeger in the default namespace. **Note:** This template should be only used for development or testing.
+   ```
+   kubectl create -f https://raw.githubusercontent.com/jaegertracing/jaeger-kubernetes/master/all-in-one/jaeger-all-in-one-template.yml
+   ```
+
+2. Wait for the jaeger pod to be ready:
+   ```
+   $ kubectl get pod
+
+   NAME                      READY     STATUS   
+   jaeger-6c996dbcd9-j5jzf   1/1       Running
+   ```
+
+## Step 3 - Enable OpenTracing
+1. Update the ConfigMap with the keys required to load OpenTracing module with Jaeger and enable  OpenTracing for all Ingress resources.
+   ```
+   kubectl apply -f nginx-config.yaml
+   ```
+
+## Step 4 - Test Tracing
+1. Make a request to the app. 
+
+   **Note:** $IC_HTTPS_PORT and $IC_IP env variables should have been set from the Prerequisites step in the complete-example installation instructions.
+   ```
+   curl --resolve cafe.example.com:$IC_HTTPS_PORT:$IC_IP https://cafe.example.com:$IC_HTTPS_PORT/coffee --insecure
+   ```
+1. Forward a local port to the Jaeger UI port on the Jaeger pod:
+   ```
+   kubectl port-forward <YOUR_JAEGER_POD> 16686:16686
+   ``` 
+1. Open Jaeger dashboard in your browser available via http://localhost:16686. Search for the traces by specifying the name of the service to `nginx-ingress` and clicking `Find Traces`. You will see:
+
+![Jaeger UI](./jaeger-ui.png)

examples/opentracing/nginx-config.yaml

@@ -0,0 +1,20 @@
+kind: ConfigMap
+apiVersion: v1
+metadata:
+  name: nginx-config
+  namespace: nginx-ingress
+data:
+  opentracing: "True"
+  opentracing-tracer: "/usr/local/lib/libjaegertracing_plugin.so"
+  opentracing-tracer-config: |
+        {
+            "service_name": "nginx-ingress",
+            "sampler": {
+                "type": "const",
+                "param": 1
+            },
+            "reporter": {
+                "localAgentHostPort": "jaeger-agent.default.svc.cluster.local:6831"
+            }
+        }
+

internal/configs/config_params.go

@@ -53,6 +53,10 @@ type ConfigParams struct {
 	MainKeepaliveRequests         int64
 	VariablesHashBucketSize       uint64
 	VariablesHashMaxSize          uint64
+	MainOpenTracingLoadModule     bool
+	MainOpenTracingEnabled        bool
+	MainOpenTracingTracer         string
+	MainOpenTracingTracerConfig   string
 
 	RealIPHeader    string
 	SetRealIPFrom   []string