Skip to content

Commit

Permalink
First sidecar module: OpenTelemetry (kubernetes#8013)
Browse files Browse the repository at this point in the history
* remove opentelemetry from main nginx image

* add opentelemetry sidecar image

* handle extra modules in helm chart

* fix running helm chart

* mount the modules volume in the init container

* merge the mounted folder

* fix the otel image

* fix licence year

* fix cloudbuild image

* use the same nginx version as in the main image

* only retrieve /etc/nginx/modules for now
  • Loading branch information
dmathieu authored and rchshld committed May 17, 2023
1 parent d1b1a52 commit d6d2d1b
Show file tree
Hide file tree
Showing 14 changed files with 353 additions and 53 deletions.
10 changes: 10 additions & 0 deletions charts/ingress-nginx/ci/daemonset-extra-modules.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,10 @@
controller:
kind: DaemonSet
image:
repository: ingress-controller/controller
tag: 1.0.0-dev
service:
type: ClusterIP
extraModules:
- name: opentelemetry
image: busybox
10 changes: 10 additions & 0 deletions charts/ingress-nginx/ci/deployment-extra-modules.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,10 @@
controller:
image:
repository: ingress-controller/controller
tag: 1.0.0-dev
digest: null
service:
type: ClusterIP
extraModules:
- name: opentelemetry
image: busybox
28 changes: 24 additions & 4 deletions charts/ingress-nginx/templates/controller-daemonset.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -143,8 +143,12 @@ spec:
hostPort: {{ $key }}
{{- end }}
{{- end }}
{{- if (or .Values.controller.customTemplate.configMapName .Values.controller.extraVolumeMounts .Values.controller.admissionWebhooks.enabled) }}
{{- if (or .Values.controller.customTemplate.configMapName .Values.controller.extraVolumeMounts .Values.controller.admissionWebhooks.enabled .Values.controller.extraModules) }}
volumeMounts:
{{- if .Values.controller.extraModules }}
- name: modules
mountPath: /modules_mount
{{- end }}
{{- if .Values.controller.customTemplate.configMapName }}
- mountPath: /etc/nginx/template
name: nginx-template-volume
Expand All @@ -165,8 +169,20 @@ spec:
{{- if .Values.controller.extraContainers }}
{{ toYaml .Values.controller.extraContainers | nindent 8 }}
{{- end }}
{{- if .Values.controller.extraInitContainers }}
initContainers: {{ toYaml .Values.controller.extraInitContainers | nindent 8 }}


{{- if (or .Values.controller.extraInitContainers .Values.controller.extraModules) }}
initContainers:
{{- if .Values.controller.extraInitContainers }}
{{ toYaml .Values.controller.extraInitContainers | nindent 8 }}
{{- end }}
{{- if .Values.controller.extraModules }}
{{- range .Values.controller.extraModules }}
- name: {{ .Name }}
image: {{ .Image }}
command: ['sh', '-c', '/usr/local/bin/init_module.sh']
{{- end }}
{{- end }}
{{- end }}
{{- if .Values.controller.hostNetwork }}
hostNetwork: {{ .Values.controller.hostNetwork }}
Expand All @@ -185,8 +201,12 @@ spec:
{{- end }}
serviceAccountName: {{ template "ingress-nginx.serviceAccountName" . }}
terminationGracePeriodSeconds: {{ .Values.controller.terminationGracePeriodSeconds }}
{{- if (or .Values.controller.customTemplate.configMapName .Values.controller.extraVolumeMounts .Values.controller.admissionWebhooks.enabled .Values.controller.extraVolumes) }}
{{- if (or .Values.controller.customTemplate.configMapName .Values.controller.extraVolumeMounts .Values.controller.admissionWebhooks.enabled .Values.controller.extraVolumes .Values.controller.extraModules) }}
volumes:
{{- if .Values.controller.extraModules }}
- name: modules
emptyDir: {}
{{- end }}
{{- if .Values.controller.customTemplate.configMapName }}
- name: nginx-template-volume
configMap:
Expand Down
29 changes: 25 additions & 4 deletions charts/ingress-nginx/templates/controller-deployment.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -140,8 +140,12 @@ spec:
hostPort: {{ $key }}
{{- end }}
{{- end }}
{{- if (or .Values.controller.customTemplate.configMapName .Values.controller.extraVolumeMounts .Values.controller.admissionWebhooks.enabled) }}
{{- if (or .Values.controller.customTemplate.configMapName .Values.controller.extraVolumeMounts .Values.controller.admissionWebhooks.enabled .Values.controller.extraModules) }}
volumeMounts:
{{- if .Values.controller.extraModules }}
- name: modules
mountPath: /modules_mount
{{- end }}
{{- if .Values.controller.customTemplate.configMapName }}
- mountPath: /etc/nginx/template
name: nginx-template-volume
Expand All @@ -162,8 +166,21 @@ spec:
{{- if .Values.controller.extraContainers }}
{{ toYaml .Values.controller.extraContainers | nindent 8 }}
{{- end }}
{{- if .Values.controller.extraInitContainers }}
initContainers: {{ toYaml .Values.controller.extraInitContainers | nindent 8 }}
{{- if (or .Values.controller.extraInitContainers .Values.controller.extraModules) }}
initContainers:
{{- if .Values.controller.extraInitContainers }}
{{ toYaml .Values.controller.extraInitContainers | nindent 8 }}
{{- end }}
{{- if .Values.controller.extraModules }}
{{- range .Values.controller.extraModules }}
- name: {{ .name }}
image: {{ .image }}
command: ['sh', '-c', '/usr/local/bin/init_module.sh']
volumeMounts:
- name: modules
mountPath: /modules_mount
{{- end }}
{{- end }}
{{- end }}
{{- if .Values.controller.hostNetwork }}
hostNetwork: {{ .Values.controller.hostNetwork }}
Expand All @@ -182,8 +199,12 @@ spec:
{{- end }}
serviceAccountName: {{ template "ingress-nginx.serviceAccountName" . }}
terminationGracePeriodSeconds: {{ .Values.controller.terminationGracePeriodSeconds }}
{{- if (or .Values.controller.customTemplate.configMapName .Values.controller.extraVolumeMounts .Values.controller.admissionWebhooks.enabled .Values.controller.extraVolumes) }}
{{- if (or .Values.controller.customTemplate.configMapName .Values.controller.extraVolumeMounts .Values.controller.admissionWebhooks.enabled .Values.controller.extraVolumes .Values.controller.extraModules) }}
volumes:
{{- if .Values.controller.extraModules }}
- name: modules
emptyDir: {}
{{- end }}
{{- if .Values.controller.customTemplate.configMapName }}
- name: nginx-template-volume
configMap:
Expand Down
9 changes: 9 additions & 0 deletions charts/ingress-nginx/values.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -559,6 +559,15 @@ controller:
# image: busybox
# command: ['sh', '-c', 'until nslookup myservice; do echo waiting for myservice; sleep 2; done;']

extraModules: []
## Modules, which are mounted into the core nginx image
# - name: opentelemetry
# image: busybox
#
# The image must contain a `/usr/local/bin/init_module.sh` executable, which
# will be executed as initContainers, to move its config files within the
# mounted volume.

admissionWebhooks:
annotations: {}
# ignore-check.kube-linter.io/no-read-only-rootfs: "This deployment needs write access to root filesystem".
Expand Down
2 changes: 2 additions & 0 deletions images/nginx/rootfs/Dockerfile
Original file line number Diff line number Diff line change
Expand Up @@ -33,6 +33,7 @@ ENV LUA_CPATH="/usr/local/lib/lua/?/?.so;/usr/local/lib/lua/?.so;;"
COPY --from=builder /usr/local /usr/local
COPY --from=builder /opt /opt
COPY --from=builder /etc/nginx /etc/nginx
COPY --from=builder entrypoint.sh /usr/local/entrypoint.sh

RUN apk update \
&& apk upgrade \
Expand Down Expand Up @@ -72,4 +73,5 @@ RUN apk update \

EXPOSE 80 443

ENTRYPOINT ["/usr/local/entrypoint.sh"]
CMD ["nginx", "-g", "daemon off;"]
45 changes: 0 additions & 45 deletions images/nginx/rootfs/build.sh
Original file line number Diff line number Diff line change
Expand Up @@ -35,12 +35,6 @@ export NGINX_DIGEST_AUTH=1.0.0
# Check for recent changes: https://github.com/yaoweibin/ngx_http_substitutions_filter_module/compare/v0.6.4...master
export NGINX_SUBSTITUTIONS=b8a71eacc7f986ba091282ab8b1bbbc6ae1807e0

# Check for recent changes: https://github.com/open-telemetry/opentelemetry-cpp/compare/v1.0.0...main
export OPENTELEMETRY_CPP_VERSION=1.0.0

# Check for recent changes: https://github.com/open-telemetry/opentelemetry-cpp-contrib/compare/c655b8...main
export OPENTELEMETRY_CONTRIB_COMMIT=c655b849f017a5363085a4b4e6fcae8a4b7621ad

# Check for recent changes: https://github.com/opentracing-contrib/nginx-opentracing/compare/v0.19.0...master
export NGINX_OPENTRACING_VERSION=0.19.0

Expand Down Expand Up @@ -144,11 +138,6 @@ if [[ ${ARCH} == "s390x" ]]; then
export LUA_STREAM_NGX_VERSION=0.0.7
fi

export USE_OPENTELEMETRY=true
if [[ ${ARCH} == "s390x" ]] || [[ ${ARCH} == "armv7l" ]]; then
export USE_OPENTELEMETRY=false
fi

get_src()
{
hash="$1"
Expand Down Expand Up @@ -226,9 +215,6 @@ get_src f09851e6309560a8ff3e901548405066c83f1f6ff88aa7171e0763bd9514762b \
get_src a98b48947359166326d58700ccdc27256d2648218072da138ab6b47de47fbd8f \
"https://github.com/yaoweibin/ngx_http_substitutions_filter_module/archive/$NGINX_SUBSTITUTIONS.tar.gz"

get_src 37b2a2abf75e865449ff1425cee96dbd74659ac0c612c84ee5f381244360cab2 \
"https://github.com/open-telemetry/opentelemetry-cpp-contrib/archive/$OPENTELEMETRY_CONTRIB_COMMIT.tar.gz"

get_src 6f97776ebdf019b105a755c7736b70bdbd7e575c7f0d39db5fe127873c7abf17 \
"https://github.com/opentracing-contrib/nginx-opentracing/archive/v$NGINX_OPENTRACING_VERSION.tar.gz"

Expand Down Expand Up @@ -482,32 +468,6 @@ cmake -DCMAKE_BUILD_TYPE=Release \
make
make install

if [ $USE_OPENTELEMETRY = true ]; then
# build opentelemetry lib
apk add protobuf-dev \
grpc \
grpc-dev \
gtest-dev \
c-ares-dev

cd $BUILD_PATH
git clone --recursive https://github.com/open-telemetry/opentelemetry-cpp opentelemetry-cpp-$OPENTELEMETRY_CPP_VERSION
cd "opentelemetry-cpp-$OPENTELEMETRY_CPP_VERSION"
git checkout v$OPENTELEMETRY_CPP_VERSION
mkdir .build
cd .build

cmake -DCMAKE_BUILD_TYPE=Release \
-DBUILD_TESTING=OFF \
-DWITH_EXAMPLES=OFF \
-DCMAKE_POSITION_INDEPENDENT_CODE=ON \
-DWITH_OTLP=ON \
-DWITH_OTLP_HTTP=OFF \
..
make
make install
fi

# Get Brotli source and deps
cd "$BUILD_PATH"
git clone --depth=1 https://github.com/google/ngx_brotli.git
Expand Down Expand Up @@ -675,11 +635,6 @@ WITH_MODULES=" \
--add-dynamic-module=$BUILD_PATH/ngx_http_geoip2_module-${GEOIP2_VERSION} \
--add-dynamic-module=$BUILD_PATH/ngx_brotli"

if [ $USE_OPENTELEMETRY = true ]; then
WITH_MODULES+=" \
--add-dynamic-module=$BUILD_PATH/opentelemetry-cpp-contrib-$OPENTELEMETRY_CONTRIB_COMMIT/instrumentation/nginx"
fi

./configure \
--prefix=/usr/local/nginx \
--conf-path=/etc/nginx/nginx.conf \
Expand Down
27 changes: 27 additions & 0 deletions images/nginx/rootfs/entrypoint.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,27 @@
#!/bin/bash

# Copyright 2022 The Kubernetes Authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

set -o errexit
set -o nounset
set -o pipefail

if [ -d "/modules_mount/etc/nginx/modules" ]; then
for dir in /modules_mount/etc/nginx/modules/*; do
cp "$dir"/* "/etc/nginx/modules/$(basename "$dir")"
done
fi

exec "$@"
57 changes: 57 additions & 0 deletions images/opentelemetry/Makefile
Original file line number Diff line number Diff line change
@@ -0,0 +1,57 @@
# Copyright 2021 The Kubernetes Authors. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

.DEFAULT_GOAL:=build

# set default shell
SHELL=/bin/bash -o pipefail -o errexit

DIR:=$(strip $(shell dirname $(realpath $(lastword $(MAKEFILE_LIST)))))
INIT_BUILDX=$(DIR)/../../hack/init-buildx.sh

# 0.0.0 shouldn't clobber any released builds
TAG ?= 0.0
REGISTRY ?= gcr.io/k8s-staging-ingress-nginx

IMAGE = $(REGISTRY)/opentelemetry

# required to enable buildx
export DOCKER_CLI_EXPERIMENTAL=enabled

# build with buildx
PLATFORMS?=linux/amd64,linux/arm
OUTPUT=
PROGRESS=plain
build: ensure-buildx
docker buildx build \
--platform=${PLATFORMS} $(OUTPUT) \
--progress=$(PROGRESS) \
--pull \
--tag $(IMAGE):$(TAG) rootfs

# push the cross built image
push: OUTPUT=--push
push: build

# enable buildx
ensure-buildx:
# this is required for cloudbuild
ifeq ("$(wildcard $(INIT_BUILDX))","")
@curl -sSL https://raw.githubusercontent.com/kubernetes/ingress-nginx/main/hack/init-buildx.sh | bash
else
@exec $(INIT_BUILDX)
endif
@echo "done"

.PHONY: build push ensure-buildx
5 changes: 5 additions & 0 deletions images/opentelemetry/README.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,5 @@
# OpenTelemetry library builder

**How to use this image:**
This image only contains the necessary files in /usr/local and /etc/nginx/opentelemetry to
be copied to Ingress Controller deployment when OpenTelemetry is enabled
24 changes: 24 additions & 0 deletions images/opentelemetry/cloudbuild.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,24 @@
timeout: 10800s
options:
substitution_option: ALLOW_LOOSE
# job builds a multi-arch docker image for amd64,arm,arm64 and s390x.
machineType: N1_HIGHCPU_32
steps:
- name: gcr.io/k8s-staging-test-infra/gcb-docker-gcloud:v20211118-2f2d816b90
entrypoint: bash
env:
- DOCKER_CLI_EXPERIMENTAL=enabled
- TAG=$_GIT_TAG
- BASE_REF=$_PULL_BASE_REF
- REGISTRY=gcr.io/k8s-staging-ingress-nginx
# default cloudbuild has HOME=/builder/home and docker buildx is in /root/.docker/cli-plugins/docker-buildx
# set the home to /root explicitly to if using docker buildx
- HOME=/root
args:
- -c
- |
gcloud auth configure-docker \
&& make push
substitutions:
_GIT_TAG: "12345"
_PULL_BASE_REF: "master"
28 changes: 28 additions & 0 deletions images/opentelemetry/rootfs/Dockerfile
Original file line number Diff line number Diff line change
@@ -0,0 +1,28 @@
# Copyright 2021 The Kubernetes Authors. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.


FROM alpine:3.14.2 as builder

COPY . /

RUN apk update \
&& apk upgrade \
&& apk add -U bash \
&& /build.sh

FROM busybox:latest

COPY --from=builder init_module.sh /usr/local/bin/init_module.sh
COPY --from=builder /etc/nginx/modules /etc/nginx/modules
Loading

0 comments on commit d6d2d1b

Please sign in to comment.