diff --git a/build/DockerfileWithOpentracing b/build/DockerfileWithOpentracing new file mode 100644 index 0000000000..8734a30f78 --- /dev/null +++ b/build/DockerfileWithOpentracing @@ -0,0 +1,83 @@ +ARG NGINX_VERSION=1.17.1 +ARG OPENTRACING_CPP_VERSION=1.5.1 + +FROM nginx:${NGINX_VERSION} AS opentracing-builder +ARG OPENTRACING_CPP_VERSION + +RUN set -x \ + && apt-get update \ + && apt-get install --no-install-recommends --no-install-suggests -y \ + libcurl4-openssl-dev \ + build-essential \ + cmake \ + git \ + ca-certificates \ + pkg-config \ + wget \ + libz-dev \ + automake \ + autogen \ + autoconf \ + libtool \ + libpcre3 \ + libpcre3-dev \ + && tempDir="$(mktemp -d)" \ + && chmod 777 "$tempDir" \ + && git clone https://github.com/opentracing-contrib/nginx-opentracing src \ + && cd "$tempDir" \ + && git clone -b v$OPENTRACING_CPP_VERSION https://github.com/opentracing/opentracing-cpp.git \ + && cd opentracing-cpp \ + && mkdir .build && cd .build \ + && cmake -DCMAKE_BUILD_TYPE=Release \ + -DBUILD_TESTING=OFF .. \ + && make && make install \ + && cd "$tempDir" \ + && NGINX_VERSION_BUILD=`nginx -v 2>&1` && NGINX_VERSION_BUILD=${NGINX_VERSION_BUILD#*nginx/} \ + && echo "deb-src http://nginx.org/packages/mainline/debian/ stretch nginx" >> /etc/apt/sources.list \ + && apt-get update \ + && apt-get build-dep -y nginx=${NGINX_VERSION_BUILD} \ + && wget -O nginx-release-${NGINX_VERSION_BUILD}.tar.gz https://github.com/nginx/nginx/archive/release-${NGINX_VERSION_BUILD}.tar.gz \ + && tar zxf nginx-release-${NGINX_VERSION_BUILD}.tar.gz \ + && cd nginx-release-${NGINX_VERSION_BUILD} \ + && auto/configure \ + --with-compat \ + --add-dynamic-module=/src/opentracing \ + --with-debug \ + && make modules \ + && cp objs/ngx_http_opentracing_module.so /ngx_http_opentracing_module.so + + +FROM debian:stretch-slim AS tracer-downloader +ARG JAEGER_VERSION=v0.4.2 +RUN set -x \ + && apt-get update \ + && apt-get install --no-install-recommends --no-install-suggests -y ca-certificates apt-transport-https wget \ + && wget https://github.com/jaegertracing/jaeger-client-cpp/releases/download/${JAEGER_VERSION}/libjaegertracing_plugin.linux_amd64.so -O /usr/local/lib/libjaegertracing_plugin.so + + +# Final Image +FROM nginx:${NGINX_VERSION} +ARG OPENTRACING_CPP_VERSION +# forward nginx access and error logs to stdout and stderr of the ingress +# controller process +RUN ln -sf /proc/1/fd/1 /var/log/nginx/access.log \ + && ln -sf /proc/1/fd/1 /var/log/nginx/stream-access.log \ + && ln -sf /proc/1/fd/2 /var/log/nginx/error.log + +COPY nginx-ingress internal/configs/version1/nginx.ingress.tmpl internal/configs/version1/nginx.tmpl internal/configs/version2/nginx.virtualserver.tmpl / + +COPY --from=opentracing-builder /ngx_http_opentracing_module.so /usr/lib/nginx/modules/ngx_http_opentracing_module.so +COPY --from=opentracing-builder /usr/local/lib/libopentracing.so.${OPENTRACING_CPP_VERSION} /usr/local/lib/libopentracing.so.1 +# Edit the line below to use a different tracer +COPY --from=tracer-downloader /usr/local/lib/libjaegertracing_plugin.so /usr/local/lib/libjaegertracing_plugin.so +RUN ldconfig + +RUN rm /etc/nginx/conf.d/* + +RUN mkdir -p /etc/nginx/secrets + +# Uncomment the line below if you would like to add the default.pem to the image +# and use it as a certificate and key for the default server +# ADD default.pem /etc/nginx/secrets/default + +ENTRYPOINT ["/nginx-ingress"] \ No newline at end of file diff --git a/build/DockerfileWithOpentracingForPlus b/build/DockerfileWithOpentracingForPlus new file mode 100644 index 0000000000..1c7af72f4e --- /dev/null +++ b/build/DockerfileWithOpentracingForPlus @@ -0,0 +1,79 @@ +FROM debian:stretch-slim AS tracer-downloader +ARG JAEGER_VERSION=v0.4.2 + +RUN set -x \ + && apt-get update \ + && apt-get install --no-install-recommends --no-install-suggests -y ca-certificates apt-transport-https wget \ + && wget https://github.com/jaegertracing/jaeger-client-cpp/releases/download/${JAEGER_VERSION}/libjaegertracing_plugin.linux_amd64.so -O /usr/local/lib/libjaegertracing_plugin.so + + +# Final Image +FROM debian:stretch-slim +LABEL maintainer="NGINX Docker Maintainers " + +ENV NGINX_PLUS_VERSION 18-1~stretch +ARG IC_VERSION + +# Download certificate and key from the customer portal (https://cs.nginx.com) +# and copy to the build context +COPY nginx-repo.crt /etc/ssl/nginx/ +COPY nginx-repo.key /etc/ssl/nginx/ + +# Make sure the certificate and key have correct permissions +RUN chmod 644 /etc/ssl/nginx/* + +# Install NGINX Plus +RUN set -x \ + && apt-get update \ + && apt-get install --no-install-recommends --no-install-suggests -y apt-transport-https ca-certificates gnupg1 \ + && \ + NGINX_GPGKEY=573BFD6B3D8FBC641079A6ABABF5BD827BD9BF62; \ + found=''; \ + for server in \ + ha.pool.sks-keyservers.net \ + hkp://keyserver.ubuntu.com:80 \ + hkp://p80.pool.sks-keyservers.net:80 \ + pgp.mit.edu \ + ; do \ + echo "Fetching GPG key $NGINX_GPGKEY from $server"; \ + apt-key adv --keyserver "$server" --keyserver-options timeout=10 --recv-keys "$NGINX_GPGKEY" && found=yes && break; \ + done; \ + test -z "$found" && echo >&2 "error: failed to fetch GPG key $NGINX_GPGKEY" && exit 1; \ + echo "Acquire::https::plus-pkgs.nginx.com::Verify-Peer \"true\";" >> /etc/apt/apt.conf.d/90nginx \ + && echo "Acquire::https::plus-pkgs.nginx.com::Verify-Host \"true\";" >> /etc/apt/apt.conf.d/90nginx \ + && echo "Acquire::https::plus-pkgs.nginx.com::SslCert \"/etc/ssl/nginx/nginx-repo.crt\";" >> /etc/apt/apt.conf.d/90nginx \ + && echo "Acquire::https::plus-pkgs.nginx.com::SslKey \"/etc/ssl/nginx/nginx-repo.key\";" >> /etc/apt/apt.conf.d/90nginx \ + && echo "Acquire::https::plus-pkgs.nginx.com::User-Agent \"k8s-ic-$IC_VERSION-apt\";" >> /etc/apt/apt.conf.d/90nginx \ + && printf "deb https://plus-pkgs.nginx.com/debian stretch nginx-plus\n" > /etc/apt/sources.list.d/nginx-plus.list \ + && apt-get update && apt-get install -y \ + nginx-plus=${NGINX_PLUS_VERSION} \ + # Install OpenTracing module + nginx-plus-module-opentracing \ + && apt-get remove --purge --auto-remove -y gnupg1 \ + && rm -rf /var/lib/apt/lists/* \ + && rm -rf /etc/ssl/nginx \ + && rm /etc/apt/apt.conf.d/90nginx /etc/apt/sources.list.d/nginx-plus.list + + +# forward nginx access and error logs to stdout and stderr of the ingress +# controller process +RUN ln -sf /proc/1/fd/1 /var/log/nginx/access.log \ + && ln -sf /proc/1/fd/1 /var/log/nginx/stream-access.log \ + && ln -sf /proc/1/fd/2 /var/log/nginx/error.log + + +# Edit the line below to use a different tracer +COPY --from=tracer-downloader /usr/local/lib/libjaegertracing_plugin.so /usr/local/lib/libjaegertracing_plugin.so + +EXPOSE 80 443 + +COPY nginx-ingress internal/configs/version1/nginx-plus.ingress.tmpl internal/configs/version1/nginx-plus.tmpl internal/configs/version2/nginx-plus.virtualserver.tmpl / + +RUN rm /etc/nginx/conf.d/* \ + && mkdir -p /etc/nginx/secrets + +# Uncomment the line below if you would like to add the default.pem to the image +# and use it as a certificate and key for the default server +# ADD default.pem /etc/nginx/secrets/default + +ENTRYPOINT ["/nginx-ingress"] diff --git a/build/README.md b/build/README.md index 907cd522ae..6adc3379ac 100644 --- a/build/README.md +++ b/build/README.md @@ -74,10 +74,12 @@ The **Makefile** contains the following main variables for you to customize (eit * **VERSION** -- the current version of the controller. * **TAG** -- the tag added to the image. It's set to the value of the `VERSION` variable by default. * **PUSH_TO_GCR**. If you’re running your Kubernetes in GCE and using Google Container Registry, make sure that `PUSH_TO_GCR = 1`. This means using the `gcloud docker push` command to push the image, which is convenient when pushing images to GCR. By default, the variable is unset and the regular `docker push` command is used to push the image to the registry. -* **DOCKERFILE** -- the path to a Dockerfile. We provide three Dockerfiles: +* **DOCKERFILE** -- the path to a Dockerfile. We provide five Dockerfiles: 1. `Dockerfile`, for building a debian-based image with NGINX. It's used by default. 1. `DockerfileForAlpine`, for building an alpine-based image with NGINX. 1. `DockerfileForPlus`, for building an debian-based image with NGINX Plus. + 1. `DockerfileWithOpentracing`, for building a debian-based image with NGINX, [opentracing](https://github.com/opentracing-contrib/nginx-opentracing) module and the [Jaeger](https://www.jaegertracing.io/) tracer. + 1. `DockerfileWithOpentracingForPlus`, for building a debian-based image with NGINX Plus, [opentracing](https://github.com/opentracing-contrib/nginx-opentracing) module and the [Jaeger](https://www.jaegertracing.io/) tracer. * **GENERATE_DEFAULT_CERT_AND_KEY** - The Ingress controller requires a certificate and a key for the default HTTP/HTTPS server. You can reference them in a TLS Secret in a command-line argument to the Ingress controller. As an alternative, you can add a file in the PEM format with your certificate and key to the image as `/etc/nginx/secrets/default`. Optionally, you can generate a self-signed certificate and a key during the build process. Set `GENERATE_DEFAULT_CERT_AND_KEY` to `1` to generate a certificate and a key in the `default.pem` file. Note that you must add the `ADD` instruction in the Dockerfile to copy the cert and the key to the image. The default value of `GENERATE_DEFAULT_CERT_AND_KEY` is `0`. * **DOCKER_BUILD_OPTIONS** -- the [options](https://docs.docker.com/engine/reference/commandline/build/#options) for the `docker build` command. For example, `--pull`. * **BUILD_IN_CONTAINER** -- By default, to compile the controller we use the [golang](https://hub.docker.com/_/golang/) container that we run as part of the building process. If you want to compile the controller using your local golang environment: diff --git a/cmd/nginx-ingress/main.go b/cmd/nginx-ingress/main.go index 5c65d127cd..e3158e5901 100644 --- a/cmd/nginx-ingress/main.go +++ b/cmd/nginx-ingress/main.go @@ -327,7 +327,17 @@ func main() { glog.Fatalf("Error generating NGINX main config: %v", err) } nginxManager.CreateMainConfig(content) - nginxManager.UpdateConfigVersionFile() + + nginxManager.UpdateConfigVersionFile(ngxConfig.OpenTracingLoadModule) + + nginxManager.SetOpenTracing(ngxConfig.OpenTracingLoadModule) + + if ngxConfig.OpenTracingLoadModule { + err := nginxManager.CreateOpenTracingTracerConfig(cfgParams.MainOpenTracingTracerConfig) + if err != nil { + glog.Fatalf("Error creating OpenTracing tracer config file: %v", err) + } + } nginxDone := make(chan error, 1) nginxManager.Start(nginxDone) diff --git a/docs/configmap-and-annotations.md b/docs/configmap-and-annotations.md index a97113ba07..9c1a666d71 100644 --- a/docs/configmap-and-annotations.md +++ b/docs/configmap-and-annotations.md @@ -184,3 +184,11 @@ spec: | N/A | `stream-snippets` | Sets a custom snippet in stream context. | N/A | [Support for TCP/UDP Load Balancing](../examples/tcp-udp). | | N/A | `main-template` | Sets the main NGINX configuration template. | By default the template is read from the file in the container. | [Custom Templates](../examples/custom-templates). | | N/A | `ingress-template` | Sets the NGINX configuration template for an Ingress resource. | By default the template is read from the file on the container. | [Custom Templates](../examples/custom-templates). | + + +### Modules +| Annotation | ConfigMap Key | Description | Default | Example | +| ---------- | -------------- | ----------- | ------- | ------- | +| N/A | `opentracing` | Enables [OpenTracing](https://opentracing.io) globally (for all Ingress, VirtualServer and VirtualServerRoute resources). Note: requires the Ingress Controller image with OpenTracing module and a tracer. See the [docs](./opentracing.md) for more information. | `False` | [Support for OpenTracing](../examples/opentracing/README.md). | +| N/A | `opentracing-tracer` | Sets the path to the vendor tracer binary plugin.| N/A | [Support for OpenTracing](../examples/opentracing/README.md). | +| N/A | `opentracing-tracer-config` | Sets the tracer configuration in JSON format.| N/A | [Support for OpenTracing](../examples/opentracing/README.md). | \ No newline at end of file diff --git a/docs/opentracing.md b/docs/opentracing.md new file mode 100644 index 0000000000..012ca64126 --- /dev/null +++ b/docs/opentracing.md @@ -0,0 +1,80 @@ +# OpenTracing Support +The Ingress Controller supports [OpenTracing](http://opentracing.io/) with the third-party module [opentracing-contrib/nginx-opentracing](https://github.com/opentracing-contrib/nginx-opentracing). + +This document explains how to use OpenTracing with the Ingress Controller. Additionally, we have an [example](../examples/opentracing/README.md) on how to enable OpenTracing for a simple web application using Jaeger as a tracer. + +## Prerequisites +1. **Use the Ingress Controller image with OpenTracing.** The default Ingress Controller images don’t include the OpenTraing module. To use OpenTracing, you need to build the image with that module. Follow the build instructions to build the image using `DockerfileWithOpentracing` for NGINX or `DockerfileWithOpentracingForPlus` for NGINX Plus. +By default, the Dockerfiles install Jaeger as a tracer. However, it is possible to replace Jaeger with other supported [tracers](https://github.com/opentracing-contrib/nginx-opentracing#building-from-source). For that, please modify the Dockerfile accordingly: + 1. Change the download line in the tracer-downloader stage of the Dockerfile to download the right tracer. + 1. Edit the COPY line of the final image to copy the previously downloaded tracer to the image + +1. **Load the OpenTracing module.** You need to load the module with the configuration for the chosen tracer using the following ConfigMap keys: + * `opentracing-tracer`: sets the path to the vendor tracer binary plugin. This is the path you used in the COPY line of step *ii* above. + * `opentracing-tracer-config`: sets the tracer configuration in JSON format. + + Below an example on how to use those keys to load the module with Jaeger tracer: + ```yaml + opentracing-tracer: "/usr/local/lib/libjaegertracing_plugin.so" + opentracing-tracer-config: | + { + "service_name": "nginx-ingress", + "sampler": { + "type": "const", + "param": 1 + }, + "reporter": { + "localAgentHostPort": "jaeger-agent.default.svc.cluster.local:6831" + } + } + ``` + +## Enable OpenTracing Globally +To enable OpenTracing globally (for all Ingress, VirtualServer and VirtualServerRoute resources), set the `opentracing` ConfigMap key to `True`: + +```yaml +opentracing: True +``` + +## Enable/Disable OpenTracing per Ingress Resource + +It is possible to use annotations to enable or disable OpenTracing for a specific Ingress Resource. As mentioned in the prerequisites section, both `opentracing-tracer` and `opentracing-tracer-config` must be configured. + +Consider the following two cases: +1. OpenTracing is globally disabled. + 1. To enable OpenTracing for a specific Ingress Resource, use the server snippet annotation: + ```yaml + nginx.org/server-snippets: | + opentracing on; + ``` + 1. To enable OpenTracing for specific paths, (1) you need to use [Mergeable Ingress resources](../examples/mergeable-ingress-types/README.md) and (2) use the location snippets annotation to enable OpenTracing for the paths of a specific Minion Ingress resource: + ```yaml + nginx.org/location-snippets: | + opentracing on; + ``` + +2. OpenTracing is globally enabled: + 1. To disable OpenTracing for a specific Ingress Resource, use the server snippet annotation: + ```yaml + nginx.org/server-snippets: | + opentracing off; + ``` + + 1. To disable OpenTracing for specific paths, (1) you need to use [Mergeable Ingress resources](../examples/mergeable-ingress-types/README.md) and (2) use the location snippets annotation to disable OpenTracing for the paths of a specific Minion Ingress resource: + ```yaml + nginx.org/location-snippets: | + opentracing off; + ``` + +## Customize OpenTracing + +You can customize OpenTracing though the supported [OpenTracing module directives](https://github.com/opentracing-contrib/nginx-opentracing/blob/master/doc/Reference.md). Use the snippets ConfigMap keys or annotations to insert those directives into the http, server or location contexts of the generated NGINX configuration. + +For example, to propagate the active span context for upstream requests, it is required to set the `opentracing_propagate_context` directive, which you can add to an Ingress resource using the location snippets annotation: + +```yaml +nginx.org/location-snippets: | + opentracing_propagate_context; +``` + +**Note**: `opentracing_propagate_context` and `opentracing_grpc_propagate_context` directives can be used in http, server or location contexts according to the [module documentation](https://github.com/opentracing-contrib/nginx-opentracing/blob/master/doc/Reference.md#opentracing_propagate_context). However, because of the way the module works and how the Ingress Controller generates the NGINX configuration, it is only possible to use the directive in the location context. \ No newline at end of file diff --git a/examples/opentracing/README.md b/examples/opentracing/README.md new file mode 100644 index 0000000000..87e6b9de6a --- /dev/null +++ b/examples/opentracing/README.md @@ -0,0 +1,48 @@ + +# OpenTracing Support + +In this example we deploy the NGINX or NGINX Plus Ingress Controller and a simple web application. Then we enable OpenTracing and use a tracer (Jaeger) for tracing the requests that go through NGINX or NGINX Plus to the web application. + +## Prerequisites + +The default Ingress Controller images don’t include the OpenTracing module required for this example. See Step 1 of the [Prerequisites section](../../docs/opentracing.md#Prerequisites) in the OpenTracing doc for the instructions on how to get the right image with Jaeger tracer. + +## Step 1 - Deploy Ingress Controller and the Cafe App + +Follow steps 1-3 of the [complete example](../complete-example/README.md) to deploy the Ingress Controller and the cafe app. Make sure to use the Ingress Controller image with the OpenTracing module and the tracer installed. + +## Step 2 - Deploy a Tracer + +1. Use the [all-in-one dev template](https://github.com/jaegertracing/jaeger-kubernetes#development-setup) to deploy Jaeger in the default namespace. **Note:** This template should be only used for development or testing. + ``` + kubectl create -f https://raw.githubusercontent.com/jaegertracing/jaeger-kubernetes/master/all-in-one/jaeger-all-in-one-template.yml + ``` + +2. Wait for the jaeger pod to be ready: + ``` + $ kubectl get pod + + NAME READY STATUS + jaeger-6c996dbcd9-j5jzf 1/1 Running + ``` + +## Step 3 - Enable OpenTracing +1. Update the ConfigMap with the keys required to load OpenTracing module with Jaeger and enable OpenTracing for all Ingress resources. + ``` + kubectl apply -f nginx-config.yaml + ``` + +## Step 4 - Test Tracing +1. Make a request to the app. + + **Note:** $IC_HTTPS_PORT and $IC_IP env variables should have been set from the Prerequisites step in the complete-example installation instructions. + ``` + curl --resolve cafe.example.com:$IC_HTTPS_PORT:$IC_IP https://cafe.example.com:$IC_HTTPS_PORT/coffee --insecure + ``` +1. Forward a local port to the Jaeger UI port on the Jaeger pod: + ``` + kubectl port-forward 16686:16686 + ``` +1. Open Jaeger dashboard in your browser available via http://localhost:16686. Search for the traces by specifying the name of the service to `nginx-ingress` and clicking `Find Traces`. You will see: + +![Jaeger UI](./jaeger-ui.png) diff --git a/examples/opentracing/jaeger-ui.png b/examples/opentracing/jaeger-ui.png new file mode 100644 index 0000000000..d4c4fa06fe Binary files /dev/null and b/examples/opentracing/jaeger-ui.png differ diff --git a/examples/opentracing/nginx-config.yaml b/examples/opentracing/nginx-config.yaml new file mode 100644 index 0000000000..43b06e536d --- /dev/null +++ b/examples/opentracing/nginx-config.yaml @@ -0,0 +1,20 @@ +kind: ConfigMap +apiVersion: v1 +metadata: + name: nginx-config + namespace: nginx-ingress +data: + opentracing: "True" + opentracing-tracer: "/usr/local/lib/libjaegertracing_plugin.so" + opentracing-tracer-config: | + { + "service_name": "nginx-ingress", + "sampler": { + "type": "const", + "param": 1 + }, + "reporter": { + "localAgentHostPort": "jaeger-agent.default.svc.cluster.local:6831" + } + } + diff --git a/internal/configs/config_params.go b/internal/configs/config_params.go index fa3fda4198..f006825ad0 100644 --- a/internal/configs/config_params.go +++ b/internal/configs/config_params.go @@ -53,6 +53,10 @@ type ConfigParams struct { MainKeepaliveRequests int64 VariablesHashBucketSize uint64 VariablesHashMaxSize uint64 + MainOpenTracingLoadModule bool + MainOpenTracingEnabled bool + MainOpenTracingTracer string + MainOpenTracingTracerConfig string RealIPHeader string SetRealIPFrom []string diff --git a/internal/configs/configmaps.go b/internal/configs/configmaps.go index eddf0e3739..61a60b2f19 100644 --- a/internal/configs/configmaps.go +++ b/internal/configs/configmaps.go @@ -391,6 +391,30 @@ func ParseConfigMap(cfgm *v1.ConfigMap, nginxPlus bool) *ConfigParams { } } + if openTracingTracer, exists := cfgm.Data["opentracing-tracer"]; exists { + cfgParams.MainOpenTracingTracer = openTracingTracer + } + + if openTracingTracerConfig, exists := cfgm.Data["opentracing-tracer-config"]; exists { + cfgParams.MainOpenTracingTracerConfig = openTracingTracerConfig + } + + if cfgParams.MainOpenTracingTracer != "" || cfgParams.MainOpenTracingTracerConfig != "" { + cfgParams.MainOpenTracingLoadModule = true + } + + if openTracing, exists, err := GetMapKeyAsBool(cfgm.Data, "opentracing", cfgm); exists { + if err != nil { + glog.Error(err) + } else { + if cfgParams.MainOpenTracingLoadModule { + cfgParams.MainOpenTracingEnabled = openTracing + } else { + glog.Error("ConfigMap Key 'opentracing' requires both 'opentracing-tracer' and 'opentracing-tracer-config' Keys configured, Opentracing will be disabled") + } + } + } + return cfgParams } @@ -431,6 +455,10 @@ func GenerateNginxMainConfig(staticCfgParams *StaticConfigParams, config *Config KeepaliveRequests: config.MainKeepaliveRequests, VariablesHashBucketSize: config.VariablesHashBucketSize, VariablesHashMaxSize: config.VariablesHashMaxSize, + OpenTracingLoadModule: config.MainOpenTracingLoadModule, + OpenTracingEnabled: config.MainOpenTracingEnabled, + OpenTracingTracer: config.MainOpenTracingTracer, + OpenTracingTracerConfig: config.MainOpenTracingTracerConfig, } return nginxCfg } diff --git a/internal/configs/configurator.go b/internal/configs/configurator.go index 23c9defae7..d102afc954 100644 --- a/internal/configs/configurator.go +++ b/internal/configs/configurator.go @@ -149,6 +149,11 @@ func (cnf *Configurator) AddOrUpdateVirtualServer(virtualServerEx *VirtualServer return nil } +func (cnf *Configurator) addOrUpdateOpenTracingTracerConfig(content string) error { + err := cnf.nginxManager.CreateOpenTracingTracerConfig(content) + return err +} + func (cnf *Configurator) addOrUpdateVirtualServer(virtualServerEx *VirtualServerEx) error { tlsPemFileName := "" if virtualServerEx.TLSSecret != nil { @@ -546,6 +551,13 @@ func (cnf *Configurator) UpdateConfig(cfgParams *ConfigParams, ingExes []*Ingres } } + if mainCfg.OpenTracingLoadModule { + if err := cnf.addOrUpdateOpenTracingTracerConfig(mainCfg.OpenTracingTracerConfig); err != nil { + return fmt.Errorf("Error when updating OpenTracing tracer config: %v", err) + } + } + + cnf.nginxManager.SetOpenTracing(mainCfg.OpenTracingLoadModule) if err := cnf.nginxManager.Reload(); err != nil { return fmt.Errorf("Error when updating config from ConfigMap: %v", err) } diff --git a/internal/configs/version1/config.go b/internal/configs/version1/config.go index 0d245f3eb5..b219f05e24 100644 --- a/internal/configs/version1/config.go +++ b/internal/configs/version1/config.go @@ -155,6 +155,10 @@ type MainConfig struct { KeepaliveRequests int64 VariablesHashBucketSize uint64 VariablesHashMaxSize uint64 + OpenTracingLoadModule bool + OpenTracingEnabled bool + OpenTracingTracer string + OpenTracingTracerConfig string } // NewUpstreamWithDefaultServer creates an upstream with the default server. diff --git a/internal/configs/version1/nginx-plus.tmpl b/internal/configs/version1/nginx-plus.tmpl index 8855ec2b2a..a3e051607c 100644 --- a/internal/configs/version1/nginx-plus.tmpl +++ b/internal/configs/version1/nginx-plus.tmpl @@ -13,6 +13,10 @@ daemon off; error_log /var/log/nginx/error.log {{.ErrorLogLevel}}; pid /var/run/nginx.pid; +{{- if .OpenTracingLoadModule}} +load_module modules/ngx_http_opentracing_module.so; +{{- end}} + {{- if .MainSnippets}} {{range $value := .MainSnippets}} {{$value}}{{end}} @@ -69,6 +73,13 @@ http { {{if .SSLPreferServerCiphers}}ssl_prefer_server_ciphers on;{{end}} {{if .SSLDHParam}}ssl_dhparam {{.SSLDHParam}};{{end}} + {{if .OpenTracingEnabled}} + opentracing on; + {{end}} + {{if .OpenTracingLoadModule}} + opentracing_load_tracer {{ .OpenTracingTracer }} /etc/tracer-config.json; + {{end}} + {{if .ResolverAddresses}} resolver {{range $resolver := .ResolverAddresses}}{{$resolver}}{{end}}{{if .ResolverValid}} valid={{.ResolverValid}}{{end}}{{if not .ResolverIPV6}} ipv6=off{{end}}; {{if .ResolverTimeout}}resolver_timeout {{.ResolverTimeout}};{{end}} @@ -85,6 +96,10 @@ http { server_tokens "{{.ServerTokens}}"; access_log off; + {{if .OpenTracingEnabled}} + opentracing off; + {{end}} + {{if .HealthStatus}} location /nginx-health { default_type text/plain; @@ -106,6 +121,10 @@ http { access_log off; + {{if .OpenTracingEnabled}} + opentracing off; + {{end}} + location = /dashboard.html { } {{range $value := .NginxStatusAllowCIDRs}} @@ -123,6 +142,10 @@ http { listen unix:/var/run/nginx-plus-api.sock; access_log off; + {{if .OpenTracingEnabled}} + opentracing off; + {{end}} + # $config_version_mismatch is defined in /etc/nginx/config-version.conf location /configVersionCheck { if ($config_version_mismatch) { diff --git a/internal/configs/version1/nginx.tmpl b/internal/configs/version1/nginx.tmpl index b9e412d310..33180c6ede 100644 --- a/internal/configs/version1/nginx.tmpl +++ b/internal/configs/version1/nginx.tmpl @@ -12,6 +12,10 @@ daemon off; error_log /var/log/nginx/error.log {{.ErrorLogLevel}}; pid /var/run/nginx.pid; +{{- if .OpenTracingLoadModule}} +load_module modules/ngx_http_opentracing_module.so; +{{- end}} + {{- if .MainSnippets}} {{range $value := .MainSnippets}} {{$value}}{{end}} @@ -67,6 +71,13 @@ http { {{if .SSLPreferServerCiphers}}ssl_prefer_server_ciphers on;{{end}} {{if .SSLDHParam}}ssl_dhparam {{.SSLDHParam}};{{end}} + {{if .OpenTracingEnabled}} + opentracing on; + {{end}} + {{if .OpenTracingLoadModule}} + opentracing_load_tracer {{ .OpenTracingTracer }} /etc/tracer-config.json; + {{end}} + server { listen 80 default_server{{if .ProxyProtocol}} proxy_protocol{{end}}; listen 443 ssl default_server{{if .HTTP2}} http2{{end}}{{if .ProxyProtocol}} proxy_protocol{{end}}; @@ -78,6 +89,10 @@ http { server_tokens "{{.ServerTokens}}"; access_log off; + {{if .OpenTracingEnabled}} + opentracing off; + {{end}} + {{if .HealthStatus}} location /nginx-health { default_type text/plain; @@ -97,6 +112,9 @@ http { {{range $value := .NginxStatusAllowCIDRs}} allow {{$value}};{{end}} deny all; + {{if .OpenTracingEnabled}} + opentracing off; + {{end}} location /stub_status { stub_status; } @@ -108,6 +126,10 @@ http { listen unix:/var/run/nginx-status.sock; access_log off; + {{if .OpenTracingEnabled}} + opentracing off; + {{end}} + location /stub_status { stub_status; } @@ -121,6 +143,10 @@ http { listen unix:/var/run/nginx-502-server.sock; access_log off; + {{if .OpenTracingEnabled}} + opentracing off; + {{end}} + location / { return 502; } diff --git a/internal/nginx/fake_manager.go b/internal/nginx/fake_manager.go index cbabc68531..6cf4894313 100644 --- a/internal/nginx/fake_manager.go +++ b/internal/nginx/fake_manager.go @@ -81,7 +81,7 @@ func (*FakeManager) Quit() { } // UpdateConfigVersionFile provides a fake implementation of UpdateConfigVersionFile. -func (*FakeManager) UpdateConfigVersionFile() { +func (*FakeManager) UpdateConfigVersionFile(openTracing bool) { glog.V(3).Infof("Writing config version") } @@ -94,3 +94,14 @@ func (*FakeManager) UpdateServersInPlus(upstream string, servers []string, confi glog.V(3).Infof("Updating servers of %v: %v", upstream, servers) return nil } + +// CreateOpenTracingTracerConfig creates a fake implementation of CreateOpenTracingTracerConfig. +func (*FakeManager) CreateOpenTracingTracerConfig(content string) error { + glog.V(3).Infof("Writing OpenTracing tracer config file") + + return nil +} + +// SetOpenTracing creates a fake implementation of SetOpenTracing. +func (*FakeManager) SetOpenTracing(openTracing bool) { +} diff --git a/internal/nginx/manager.go b/internal/nginx/manager.go index 9eca6efa1d..5c1c51d614 100644 --- a/internal/nginx/manager.go +++ b/internal/nginx/manager.go @@ -21,6 +21,7 @@ const TLSSecretFileMode = 0600 const JWKSecretFileMode = 0644 const configFileMode = 0644 +const jsonFileForOpenTracingTracer = "/etc/tracer-config.json" // ServerConfig holds the config data for an upstream server in NGINX Plus. type ServerConfig struct { @@ -39,12 +40,14 @@ type Manager interface { DeleteSecret(name string) GetFilenameForSecret(name string) string CreateDHParam(content string) (string, error) + CreateOpenTracingTracerConfig(content string) error Start(done chan error) Reload() error Quit() - UpdateConfigVersionFile() + UpdateConfigVersionFile(openTracing bool) SetPlusClients(plusClient *client.NginxClient, plusConfigVersionCheckClient *http.Client) UpdateServersInPlus(upstream string, servers []string, config ServerConfig) error + SetOpenTracing(openTracing bool) } // LocalManager updates NGINX configuration, starts, reloads and quits NGINX, @@ -64,6 +67,7 @@ type LocalManager struct { plusClient *client.NginxClient plusConfigVersionCheckClient *http.Client metricsCollector collectors.ManagerCollector + OpenTracing bool } // NewLocalManager creates a LocalManager. @@ -195,7 +199,7 @@ func (lm *LocalManager) Start(done chan error) { func (lm *LocalManager) Reload() error { // write a new config version lm.configVersion++ - lm.UpdateConfigVersionFile() + lm.UpdateConfigVersionFile(lm.OpenTracing) glog.V(3).Infof("Reloading nginx with configVersion: %v", lm.configVersion) @@ -228,8 +232,8 @@ func (lm *LocalManager) Quit() { } // UpdateConfigVersionFile writes the config version file. -func (lm *LocalManager) UpdateConfigVersionFile() { - cfg, err := lm.verifyConfigGenerator.GenerateVersionConfig(lm.configVersion) +func (lm *LocalManager) UpdateConfigVersionFile(openTracing bool) { + cfg, err := lm.verifyConfigGenerator.GenerateVersionConfig(lm.configVersion, openTracing) if err != nil { glog.Fatalf("Error generating config version content: %v", err) } @@ -277,6 +281,17 @@ func (lm *LocalManager) UpdateServersInPlus(upstream string, servers []string, c return nil } +// CreateOpenTracingTracerConfig creates a json configuration file for the OpenTracing tracer with the content of the string. +func (lm *LocalManager) CreateOpenTracingTracerConfig(content string) error { + glog.V(3).Infof("Writing OpenTracing tracer config file to %v", jsonFileForOpenTracingTracer) + err := createFileAndWrite(jsonFileForOpenTracingTracer, []byte(content)) + if err != nil { + return fmt.Errorf("Failed to write config file: %v", err) + } + + return nil +} + // verifyConfigVersion is used to check if the worker process that the API client is connected // to is using the latest version of nginx config. This way we avoid making changes on // a worker processes that is being shut down. @@ -300,3 +315,8 @@ func verifyConfigVersion(httpClient *http.Client, configVersion int) error { return nil } + +// SetOpenTracing sets the value of OpenTracing for the Manager +func (lm *LocalManager) SetOpenTracing(openTracing bool) { + lm.OpenTracing = openTracing +} diff --git a/internal/nginx/verify.go b/internal/nginx/verify.go index ec573df38b..bbaca8ec0e 100644 --- a/internal/nginx/verify.go +++ b/internal/nginx/verify.go @@ -80,7 +80,11 @@ func (c *verifyClient) WaitForCorrectVersion(expectedVersion int) error { const configVersionTemplateString = `server { listen unix:/var/run/nginx-config-version.sock; - access_log off; + access_log off; + + {{if .OpenTracingLoadModule}} + opentracing off; + {{end}} location /configVersion { return 200 {{.ConfigVersion}}; @@ -108,12 +112,14 @@ func newVerifyConfigGenerator() (*verifyConfigGenerator, error) { } // GenerateVersionConfig generates the config version file. -func (c *verifyConfigGenerator) GenerateVersionConfig(configVersion int) ([]byte, error) { +func (c *verifyConfigGenerator) GenerateVersionConfig(configVersion int, openTracing bool) ([]byte, error) { var configBuffer bytes.Buffer templateValues := struct { - ConfigVersion int + ConfigVersion int + OpenTracingLoadModule bool }{ configVersion, + openTracing, } err := c.configVersionTemplate.Execute(&configBuffer, templateValues) if err != nil { diff --git a/internal/nginx/verify_test.go b/internal/nginx/verify_test.go index f2ff35a606..58412af0ef 100644 --- a/internal/nginx/verify_test.go +++ b/internal/nginx/verify_test.go @@ -56,11 +56,14 @@ func TestConfigWriter(t *testing.T) { if err != nil { t.Fatalf("error instantiating ConfigWriter: %v", err) } - config, err := cw.GenerateVersionConfig(1) + config, err := cw.GenerateVersionConfig(1, true) if err != nil { t.Errorf("error generating version config: %v", err) } if !strings.Contains(string(config), "configVersion") { t.Errorf("configVersion endpoint not set. config contents: %v", string(config)) } + if !strings.Contains(string(config), "opentracing off") { + t.Errorf("opentracing directive missing when is enabled. config contents: %v", string(config)) + } }