Inherit NET_BIND_SERVICE from IC to Nginx (#3722) (#3849)

8be0144: Rework port binding logic without privileges caused issues for host networking configurations. The Kubernetes documentation states that the `net.*` sysctls can be used with container networking, which was misinterpreted. This commit reverts the change, bringing back NET_BIND_SERVICE to the Nginx process, as well as reverts the libcap package removal done in a later commit. In order to avoid privilege escalation being re-introduced, the IC process is also receiving NET_BIND_SERVICE, so that it can be inherited over to Nginx. This change aims to restore host networking as functional for the Helm chart. A future change is recommended to harden security for the IC process (to drop the capability after executing Nginx) as well as Nginx itself (to drop the capability after binding). OBS! To use a 3.1.0 image, you should manually install the `setcap` binary and add `+ep` on `/nginx-ingress` and `+eip` on `nginx` binary. (cherry picked from commit 5d56f71) Co-authored-by: Valters Jansons <[email protected]>
nginx · May 4, 2023 · 2d0558d · 2d0558d
1 parent 69e071f
commit 2d0558d
Show file tree

Hide file tree

Showing 7 changed files with 42 additions and 23 deletions.
diff --git a/build/Dockerfile b/build/Dockerfile
@@ -13,7 +13,10 @@ FROM opentracing/nginx-opentracing:nginx-1.23.3-alpine as alpine-opentracing-lib
 FROM nginx:1.23.3 AS debian
 
 RUN --mount=type=bind,from=opentracing-lib,target=/tmp/ot/ \
-	cp -av /tmp/ot/usr/local/lib/libopentracing.so* /tmp/ot/usr/local/lib/libjaegertracing*so* /tmp/ot/usr/local/lib/libzipkin*so* /tmp/ot/usr/local/lib/libdd*so* /tmp/ot/usr/local/lib/libyaml*so* /usr/local/lib/ \
+	apt-get update \
+	&& apt-get install --no-install-recommends --no-install-suggests -y libcap2-bin \
+	&& rm -rf /var/lib/apt/lists/* \
+	&& cp -av /tmp/ot/usr/local/lib/libopentracing.so* /tmp/ot/usr/local/lib/libjaegertracing*so* /tmp/ot/usr/local/lib/libzipkin*so* /tmp/ot/usr/local/lib/libdd*so* /tmp/ot/usr/local/lib/libyaml*so* /usr/local/lib/ \
 	&& cp -av /tmp/ot/usr/lib/nginx/modules/ngx_http_opentracing_module.so /usr/lib/nginx/modules/ \
 	&& ldconfig
 
@@ -22,7 +25,7 @@ RUN --mount=type=bind,from=opentracing-lib,target=/tmp/ot/ \
 FROM nginx:1.23.3-alpine AS alpine
 
 RUN --mount=type=bind,from=alpine-opentracing-lib,target=/tmp/ot/ \
-	apk add --no-cache libstdc++ \
+	apk add --no-cache libcap libstdc++ \
 	# temp fix for CVE-2023-23916
 	&& apk upgrade --no-cache curl libcurl \
 	&& cp -av /tmp/ot/usr/local/lib/libopentracing.so* /tmp/ot/usr/local/lib/libjaegertracing*so* /tmp/ot/usr/local/lib/libzipkin*so* /tmp/ot/usr/local/lib/libdd*so* /tmp/ot/usr/local/lib/libyaml*so* /usr/local/lib/ \
@@ -39,7 +42,7 @@ RUN --mount=type=secret,id=nginx-repo.crt,dst=/etc/apk/cert.pem,mode=0644 \
 	--mount=type=bind,from=alpine-opentracing-lib,target=/tmp/ot/ \
 	wget -nv -O /etc/apk/keys/nginx_signing.rsa.pub https://cs.nginx.com/static/keys/nginx_signing.rsa.pub \
 	&& printf "%s\n" "https://pkgs.nginx.com/plus/${NGINX_PLUS_VERSION}/alpine/v$(grep -E -o '^[0-9]+\.[0-9]+' /etc/alpine-release)/main" >> /etc/apk/repositories \
-	&& apk add --no-cache nginx-plus nginx-plus-module-njs nginx-plus-module-opentracing libcurl \
+	&& apk add --no-cache nginx-plus nginx-plus-module-njs nginx-plus-module-opentracing libcap libcurl \
 	&& cp -av /tmp/ot/usr/local/lib/libjaegertracing*so* /tmp/ot/usr/local/lib/libzipkin*so* /tmp/ot/usr/local/lib/libdd*so* /tmp/ot/usr/local/lib/libyaml*so* /usr/local/lib/ \
 	&& ldconfig /usr/local/lib/
 
@@ -62,7 +65,7 @@ RUN --mount=type=secret,id=nginx-repo.crt,dst=/etc/ssl/nginx/nginx-repo.crt,mode
 	&& printf "%s\n" "Acquire::https::pkgs.nginx.com::User-Agent \"k8s-ic-$IC_VERSION${BUILD_OS##debian-plus}-apt\";" >> /etc/apt/apt.conf.d/90pkgs-nginx \
 	&& printf "%s\n" "deb https://pkgs.nginx.com/plus/${NGINX_PLUS_VERSION}/debian ${DEBIAN_VERSION} nginx-plus" > /etc/apt/sources.list.d/nginx-plus.list \
 	&& apt-get update \
-	&& apt-get install --no-install-recommends --no-install-suggests -y nginx-plus nginx-plus-module-njs nginx-plus-module-opentracing libcurl4 \
+	&& apt-get install --no-install-recommends --no-install-suggests -y nginx-plus nginx-plus-module-njs nginx-plus-module-opentracing libcap2-bin libcurl4 \
 	&& apt-get purge --auto-remove -y apt-transport-https gnupg curl \
 	&& cp -av /tmp/ot/usr/local/lib/libjaegertracing*so* /tmp/ot/usr/local/lib/libzipkin*so* /tmp/ot/usr/local/lib/libdd*so* /tmp/ot/usr/local/lib/libyaml*so* /usr/local/lib/ \
 	&& ldconfig \
@@ -185,7 +188,7 @@ RUN --mount=type=secret,id=nginx-repo.crt,dst=/etc/ssl/nginx/nginx-repo.crt,mode
 # RUN update-ca-trust extract
 
 
-############################################# Create common files and permissions #############################################
+############################################# Create common files, permissions and setcap #############################################
 FROM ${BUILD_OS} as common
 
 ARG BUILD_OS
@@ -208,6 +211,8 @@ RUN [ -n "${NAP_MODULES##*dos*}" ] && exit 0; mkdir -p /root/app_protect_dos /et
 	&& chmod 777 /shared/cores /var/log/adm /var/run/adm /etc/app_protect_dos
 
 RUN --mount=type=bind,target=/tmp mkdir -p /var/lib/nginx /etc/nginx/secrets /etc/nginx/stream-conf.d \
+	&& setcap 'cap_net_bind_service=+eip' /usr/sbin/nginx 'cap_net_bind_service=+eip' /usr/sbin/nginx-debug \
+	&& setcap -v 'cap_net_bind_service=+eip' /usr/sbin/nginx 'cap_net_bind_service=+eip' /usr/sbin/nginx-debug \
 	&& [ -z "${BUILD_OS##*plus*}" ] && PLUS=-plus; cp -a /tmp/internal/configs/version1/nginx$PLUS.ingress.tmpl /tmp/internal/configs/version1/nginx$PLUS.tmpl \
 	/tmp/internal/configs/version2/nginx$PLUS.virtualserver.tmpl /tmp/internal/configs/version2/nginx$PLUS.transportserver.tmpl / \
 	&& chown -R 101:0 /etc/nginx /var/cache/nginx /var/lib/nginx /*.tmpl \
@@ -250,6 +255,11 @@ FROM common AS container
 LABEL org.nginx.kic.image.build.version="container"
 
 COPY --link --from=builder --chown=101:0 /nginx-ingress /
+# root is required for `setcap` invocation
+USER 0
+RUN setcap 'cap_net_bind_service=+ep' /nginx-ingress && setcap -v 'cap_net_bind_service=+ep' /nginx-ingress
+# 101 is nginx, defined above
+USER 101
 
 
 ############################################# Create image with nginx-ingress built locally #############################################
@@ -258,6 +268,11 @@ FROM common AS local
 LABEL org.nginx.kic.image.build.version="local"
 
 COPY --link --chown=101:0 nginx-ingress /
+# root is required for `setcap` invocation
+USER 0
+RUN setcap 'cap_net_bind_service=+ep' /nginx-ingress && setcap -v 'cap_net_bind_service=+ep' /nginx-ingress
+# 101 is nginx, defined above
+USER 101
 
 
 ############################################# Create image with nginx-ingress built by GoReleaser #############################################
@@ -267,6 +282,11 @@ ARG TARGETARCH
 LABEL org.nginx.kic.image.build.version="goreleaser"
 
 COPY --link --chown=101:0 dist/kubernetes-ingress_linux_${TARGETARCH}*/nginx-ingress /
+# root is required for `setcap` invocation
+USER 0
+RUN setcap 'cap_net_bind_service=+ep' /nginx-ingress && setcap -v 'cap_net_bind_service=+ep' /nginx-ingress
+# 101 is nginx, defined above
+USER 101
 
 
 ############################################# Create image with nginx-ingress built by GoReleaser for AWS Marketplace #############################################
@@ -277,6 +297,11 @@ ARG NAP_MODULES_AWS
 LABEL org.nginx.kic.image.build.version="aws"
 
 COPY --link --chown=101:0 dist/aws*${NAP_MODULES_AWS}_linux_${TARGETARCH}*/nginx-ingress /
+# root is required for `setcap` invocation
+USER 0
+RUN setcap 'cap_net_bind_service=+ep' /nginx-ingress && setcap -v 'cap_net_bind_service=+ep' /nginx-ingress
+# 101 is nginx, defined above
+USER 101
 
 
 ############################################# Create image with nginx-ingress extracted from image on Docker Hub #############################################

diff --git a/deployments/daemon-set/nginx-ingress.yaml b/deployments/daemon-set/nginx-ingress.yaml
@@ -23,9 +23,6 @@ spec:
         seccompProfile:
           type: RuntimeDefault
 #        fsGroup: 101 #nginx
-        sysctls:
-          - name: "net.ipv4.ip_unprivileged_port_start"
-            value: "0"
 #      volumes:
 #      - name: nginx-etc
 #        emptyDir: {}
@@ -69,6 +66,8 @@ spec:
           capabilities:
             drop:
             - ALL
+            add:
+            - NET_BIND_SERVICE
 #        volumeMounts:
 #        - mountPath: /etc/nginx
 #          name: nginx-etc

diff --git a/deployments/daemon-set/nginx-plus-ingress.yaml b/deployments/daemon-set/nginx-plus-ingress.yaml
@@ -23,9 +23,6 @@ spec:
         seccompProfile:
           type: RuntimeDefault
 #        fsGroup: 101 #nginx
-        sysctls:
-          - name: "net.ipv4.ip_unprivileged_port_start"
-            value: "0"
 #      volumes:
 #      - name: nginx-etc
 #        emptyDir: {}
@@ -69,6 +66,8 @@ spec:
           capabilities:
             drop:
             - ALL
+            add:
+            - NET_BIND_SERVICE
 #        volumeMounts:
 #        - mountPath: /etc/nginx
 #          name: nginx-etc

diff --git a/deployments/deployment/nginx-ingress.yaml b/deployments/deployment/nginx-ingress.yaml
@@ -24,9 +24,6 @@ spec:
         seccompProfile:
           type: RuntimeDefault
 #        fsGroup: 101 #nginx
-        sysctls:
-          - name: "net.ipv4.ip_unprivileged_port_start"
-            value: "0"
 #      volumes:
 #      - name: nginx-etc
 #        emptyDir: {}
@@ -69,6 +66,8 @@ spec:
           capabilities:
             drop:
             - ALL
+            add:
+            - NET_BIND_SERVICE
 #        volumeMounts:
 #        - mountPath: /etc/nginx
 #          name: nginx-etc

diff --git a/deployments/deployment/nginx-plus-ingress.yaml b/deployments/deployment/nginx-plus-ingress.yaml
@@ -24,9 +24,6 @@ spec:
         seccompProfile:
           type: RuntimeDefault
 #        fsGroup: 101 #nginx
-        sysctls:
-          - name: "net.ipv4.ip_unprivileged_port_start"
-            value: "0"
 #      volumes:
 #      - name: nginx-etc
 #        emptyDir: {}
@@ -71,6 +68,8 @@ spec:
           capabilities:
             drop:
             - ALL
+            add:
+            - NET_BIND_SERVICE
 #        volumeMounts:
 #        - mountPath: /etc/nginx
 #          name: nginx-etc

diff --git a/deployments/helm-chart/templates/controller-daemonset.yaml b/deployments/helm-chart/templates/controller-daemonset.yaml
@@ -45,9 +45,6 @@ spec:
 {{- if .Values.controller.readOnlyRootFilesystem }}
         fsGroup: 101 #nginx
 {{- end }}
-        sysctls:
-          - name: "net.ipv4.ip_unprivileged_port_start"
-            value: "0"
       terminationGracePeriodSeconds: {{ .Values.controller.terminationGracePeriodSeconds }}
 {{- if .Values.controller.nodeSelector }}
       nodeSelector:
@@ -126,6 +123,8 @@ spec:
           capabilities:
             drop:
             - ALL
+            add:
+            - NET_BIND_SERVICE
 {{- if or .Values.controller.readOnlyRootFilesystem .Values.controller.volumeMounts }}
         volumeMounts:
 {{- end }}

diff --git a/deployments/helm-chart/templates/controller-deployment.yaml b/deployments/helm-chart/templates/controller-deployment.yaml
@@ -83,9 +83,6 @@ spec:
 {{- if .Values.controller.readOnlyRootFilesystem }}
         fsGroup: 101 #nginx
 {{- end }}
-        sysctls:
-          - name: "net.ipv4.ip_unprivileged_port_start"
-            value: "0"
       terminationGracePeriodSeconds: {{ .Values.controller.terminationGracePeriodSeconds }}
       hostNetwork: {{ .Values.controller.hostNetwork }}
       dnsPolicy: {{ .Values.controller.dnsPolicy }}
@@ -133,6 +130,8 @@ spec:
           capabilities:
             drop:
             - ALL
+            add:
+            - NET_BIND_SERVICE
 {{- if or .Values.controller.readOnlyRootFilesystem .Values.controller.volumeMounts }}
         volumeMounts:
 {{- end }}