Inherit NET_BIND_SERVICE from IC to Nginx

8be0144: Rework port binding logic without privileges caused issues for host networking configurations. The Kubernetes documentation states that the `net.*` sysctls can be used with container networking, which was misinterpreted. This commit reverts the change, bringing back NET_BIND_SERVICE to the Nginx process, as well as reverts the libcap package removal done in a later commit. In order to avoid privilege escalation being re-introduced, the IC process is also receiving NET_BIND_SERVICE, so that it can be inherited over to Nginx. This change aims to restore host networking as functional for the Helm chart. A future change is recommended to harden security for the IC process (to drop the capability after executing Nginx) as well as Nginx itself (to drop the capability after binding). OBS! To use a 3.1.0 image, you should manually install the `setcap` binary and add `+ep` on `/nginx-ingress` and `+eip` on `nginx` binary.
nginx · Apr 5, 2023 · a8bd2d3 · a8bd2d3
1 parent 9d57002
commit a8bd2d3
Show file tree

Hide file tree

Showing 7 changed files with 39 additions and 24 deletions.
diff --git a/build/Dockerfile b/build/Dockerfile
@@ -13,7 +13,10 @@ FROM opentracing/nginx-opentracing:nginx-1.23.4-alpine as alpine-opentracing-lib
 FROM nginx:1.23.4 AS debian
 
 RUN --mount=type=bind,from=opentracing-lib,target=/tmp/ot/ \
-	cp -av /tmp/ot/usr/local/lib/libopentracing.so* /tmp/ot/usr/local/lib/libjaegertracing*so* /tmp/ot/usr/local/lib/libzipkin*so* /tmp/ot/usr/local/lib/libdd*so* /tmp/ot/usr/local/lib/libyaml*so* /usr/local/lib/ \
+	apt-get update \
+	&& apt-get install --no-install-recommends --no-install-suggests -y libcap2-bin \
+	&& rm -rf /var/lib/apt/lists/* \
+	&& cp -av /tmp/ot/usr/local/lib/libopentracing.so* /tmp/ot/usr/local/lib/libjaegertracing*so* /tmp/ot/usr/local/lib/libzipkin*so* /tmp/ot/usr/local/lib/libdd*so* /tmp/ot/usr/local/lib/libyaml*so* /usr/local/lib/ \
 	&& cp -av /tmp/ot/usr/lib/nginx/modules/ngx_http_opentracing_module.so /usr/lib/nginx/modules/ \
 	&& ldconfig
 
@@ -22,7 +25,7 @@ RUN --mount=type=bind,from=opentracing-lib,target=/tmp/ot/ \
 FROM nginx:1.23.4-alpine AS alpine
 
 RUN --mount=type=bind,from=alpine-opentracing-lib,target=/tmp/ot/ \
-	apk add --no-cache libstdc++ \
+	apk add --no-cache libcap libstdc++ \
 	# temp fix for CVE-2023-23916
 	&& apk upgrade --no-cache curl libcurl \
 	&& cp -av /tmp/ot/usr/local/lib/libopentracing.so* /tmp/ot/usr/local/lib/libjaegertracing*so* /tmp/ot/usr/local/lib/libzipkin*so* /tmp/ot/usr/local/lib/libdd*so* /tmp/ot/usr/local/lib/libyaml*so* /usr/local/lib/ \
@@ -39,7 +42,7 @@ RUN --mount=type=secret,id=nginx-repo.crt,dst=/etc/apk/cert.pem,mode=0644 \
 	--mount=type=bind,from=alpine-opentracing-lib,target=/tmp/ot/ \
 	wget -nv -O /etc/apk/keys/nginx_signing.rsa.pub https://cs.nginx.com/static/keys/nginx_signing.rsa.pub \
 	&& printf "%s\n" "https://pkgs.nginx.com/plus/${NGINX_PLUS_VERSION}/alpine/v$(grep -E -o '^[0-9]+\.[0-9]+' /etc/alpine-release)/main" >> /etc/apk/repositories \
-	&& apk add --no-cache nginx-plus nginx-plus-module-njs nginx-plus-module-opentracing libcurl \
+	&& apk add --no-cache nginx-plus nginx-plus-module-njs nginx-plus-module-opentracing libcap libcurl \
 	&& cp -av /tmp/ot/usr/local/lib/libjaegertracing*so* /tmp/ot/usr/local/lib/libzipkin*so* /tmp/ot/usr/local/lib/libdd*so* /tmp/ot/usr/local/lib/libyaml*so* /usr/local/lib/ \
 	&& ldconfig /usr/local/lib/
 
@@ -62,7 +65,7 @@ RUN --mount=type=secret,id=nginx-repo.crt,dst=/etc/ssl/nginx/nginx-repo.crt,mode
 	&& printf "%s\n" "Acquire::https::pkgs.nginx.com::User-Agent \"k8s-ic-$IC_VERSION${BUILD_OS##debian-plus}-apt\";" >> /etc/apt/apt.conf.d/90pkgs-nginx \
 	&& printf "%s\n" "deb https://pkgs.nginx.com/plus/${NGINX_PLUS_VERSION}/debian ${DEBIAN_VERSION} nginx-plus" > /etc/apt/sources.list.d/nginx-plus.list \
 	&& apt-get update \
-	&& apt-get install --no-install-recommends --no-install-suggests -y nginx-plus nginx-plus-module-njs nginx-plus-module-opentracing libcurl4 \
+	&& apt-get install --no-install-recommends --no-install-suggests -y nginx-plus nginx-plus-module-njs nginx-plus-module-opentracing libcap2-bin libcurl4 \
 	&& apt-get purge --auto-remove -y apt-transport-https gnupg curl \
 	&& cp -av /tmp/ot/usr/local/lib/libjaegertracing*so* /tmp/ot/usr/local/lib/libzipkin*so* /tmp/ot/usr/local/lib/libdd*so* /tmp/ot/usr/local/lib/libyaml*so* /usr/local/lib/ \
 	&& ldconfig \
@@ -185,7 +188,7 @@ RUN --mount=type=secret,id=nginx-repo.crt,dst=/etc/ssl/nginx/nginx-repo.crt,mode
 # RUN update-ca-trust extract
 
 
-############################################# Create common files and permissions #############################################
+############################################# Create common files, permissions and setcap #############################################
 FROM ${BUILD_OS} as common
 
 ARG BUILD_OS
@@ -208,6 +211,8 @@ RUN [ -n "${NAP_MODULES##*dos*}" ] && exit 0; mkdir -p /root/app_protect_dos /et
 	&& chmod 777 /shared/cores /var/log/adm /var/run/adm /etc/app_protect_dos
 
 RUN --mount=type=bind,target=/tmp mkdir -p /var/lib/nginx /etc/nginx/secrets /etc/nginx/stream-conf.d \
+	&& setcap 'cap_net_bind_service=+eip' /usr/sbin/nginx 'cap_net_bind_service=+eip' /usr/sbin/nginx-debug \
+	&& setcap -v 'cap_net_bind_service=+eip' /usr/sbin/nginx 'cap_net_bind_service=+eip' /usr/sbin/nginx-debug \
 	&& [ -z "${BUILD_OS##*plus*}" ] && PLUS=-plus; cp -a /tmp/internal/configs/version1/nginx$PLUS.ingress.tmpl /tmp/internal/configs/version1/nginx$PLUS.tmpl \
 	/tmp/internal/configs/version2/nginx$PLUS.virtualserver.tmpl /tmp/internal/configs/version2/nginx$PLUS.transportserver.tmpl / \
 	&& chown -R 101:0 /etc/nginx /var/cache/nginx /var/lib/nginx /*.tmpl \
@@ -237,11 +242,12 @@ ARG IC_VERSION
 ARG TARGETARCH
 
 WORKDIR /go/src/github.com/nginxinc/kubernetes-ingress/
-RUN apk add --no-cache git
+RUN apk add --no-cache git libcap
 RUN --mount=type=bind,target=/go/src/github.com/nginxinc/kubernetes-ingress/ --mount=type=cache,target=/root/.cache/go-build \
 	go mod download
 RUN --mount=type=bind,target=/go/src/github.com/nginxinc/kubernetes-ingress/ --mount=type=cache,target=/root/.cache/go-build \
 	CGO_ENABLED=0 GOOS=linux GOARCH=$TARGETARCH go build -trimpath -ldflags "-s -w -X main.version=${IC_VERSION}" -o /nginx-ingress github.com/nginxinc/kubernetes-ingress/cmd/nginx-ingress
+RUN --mount=type=bind,target=/tmp setcap 'cap_net_bind_service=+ep' /nginx-ingress && setcap -v 'cap_net_bind_service=+ep' /nginx-ingress
 
 
 ############################################# Create image with nginx-ingress built in container #############################################
@@ -258,6 +264,11 @@ FROM common AS local
 LABEL org.nginx.kic.image.build.version="local"
 
 COPY --link --chown=101:0 nginx-ingress /
+# root is required for `setcap` invocation
+USER 0
+RUN --mount=type=bind,target=/tmp setcap 'cap_net_bind_service=+ep' /nginx-ingress && setcap -v 'cap_net_bind_service=+ep' /nginx-ingress
+# 101 is nginx, defined above
+USER 101
 
 
 ############################################# Create image with nginx-ingress built by GoReleaser #############################################
@@ -267,6 +278,11 @@ ARG TARGETARCH
 LABEL org.nginx.kic.image.build.version="goreleaser"
 
 COPY --link --chown=101:0 dist/kubernetes-ingress_linux_${TARGETARCH}*/nginx-ingress /
+# root is required for `setcap` invocation
+USER 0
+RUN --mount=type=bind,target=/tmp setcap 'cap_net_bind_service=+ep' /nginx-ingress && setcap -v 'cap_net_bind_service=+ep' /nginx-ingress
+# 101 is nginx, defined above
+USER 101
 
 
 ############################################# Create image with nginx-ingress built by GoReleaser for AWS Marketplace #############################################
@@ -277,6 +293,11 @@ ARG NAP_MODULES_AWS
 LABEL org.nginx.kic.image.build.version="aws"
 
 COPY --link --chown=101:0 dist/aws*${NAP_MODULES_AWS}_linux_${TARGETARCH}*/nginx-ingress /
+# root is required for `setcap` invocation
+USER 0
+RUN --mount=type=bind,target=/tmp setcap 'cap_net_bind_service=+ep' /nginx-ingress && setcap -v 'cap_net_bind_service=+ep' /nginx-ingress
+# 101 is nginx, defined above
+USER 101
 
 
 ############################################# Create image with nginx-ingress extracted from image on Docker Hub #############################################

diff --git a/deployments/daemon-set/nginx-ingress.yaml b/deployments/daemon-set/nginx-ingress.yaml
@@ -23,9 +23,6 @@ spec:
         seccompProfile:
           type: RuntimeDefault
 #        fsGroup: 101 #nginx
-        sysctls:
-          - name: "net.ipv4.ip_unprivileged_port_start"
-            value: "0"
 #      volumes:
 #      - name: nginx-etc
 #        emptyDir: {}
@@ -69,6 +66,8 @@ spec:
           capabilities:
             drop:
             - ALL
+            add:
+            - NET_BIND_SERVICE
 #        volumeMounts:
 #        - mountPath: /etc/nginx
 #          name: nginx-etc

diff --git a/deployments/daemon-set/nginx-plus-ingress.yaml b/deployments/daemon-set/nginx-plus-ingress.yaml
@@ -23,9 +23,6 @@ spec:
         seccompProfile:
           type: RuntimeDefault
 #        fsGroup: 101 #nginx
-        sysctls:
-          - name: "net.ipv4.ip_unprivileged_port_start"
-            value: "0"
 #      volumes:
 #      - name: nginx-etc
 #        emptyDir: {}
@@ -69,6 +66,8 @@ spec:
           capabilities:
             drop:
             - ALL
+            add:
+            - NET_BIND_SERVICE
 #        volumeMounts:
 #        - mountPath: /etc/nginx
 #          name: nginx-etc

diff --git a/deployments/deployment/nginx-ingress.yaml b/deployments/deployment/nginx-ingress.yaml
@@ -24,9 +24,6 @@ spec:
         seccompProfile:
           type: RuntimeDefault
 #        fsGroup: 101 #nginx
-        sysctls:
-          - name: "net.ipv4.ip_unprivileged_port_start"
-            value: "0"
 #      volumes:
 #      - name: nginx-etc
 #        emptyDir: {}
@@ -69,6 +66,8 @@ spec:
           capabilities:
             drop:
             - ALL
+            add:
+            - NET_BIND_SERVICE
 #        volumeMounts:
 #        - mountPath: /etc/nginx
 #          name: nginx-etc

diff --git a/deployments/deployment/nginx-plus-ingress.yaml b/deployments/deployment/nginx-plus-ingress.yaml
@@ -24,9 +24,6 @@ spec:
         seccompProfile:
           type: RuntimeDefault
 #        fsGroup: 101 #nginx
-        sysctls:
-          - name: "net.ipv4.ip_unprivileged_port_start"
-            value: "0"
 #      volumes:
 #      - name: nginx-etc
 #        emptyDir: {}
@@ -71,6 +68,8 @@ spec:
           capabilities:
             drop:
             - ALL
+            add:
+            - NET_BIND_SERVICE
 #        volumeMounts:
 #        - mountPath: /etc/nginx
 #          name: nginx-etc

diff --git a/deployments/helm-chart/templates/controller-daemonset.yaml b/deployments/helm-chart/templates/controller-daemonset.yaml
@@ -48,9 +48,6 @@ spec:
 {{- if .Values.controller.readOnlyRootFilesystem }}
         fsGroup: 101 #nginx
 {{- end }}
-        sysctls:
-          - name: "net.ipv4.ip_unprivileged_port_start"
-            value: "0"
       terminationGracePeriodSeconds: {{ .Values.controller.terminationGracePeriodSeconds }}
 {{- if .Values.controller.nodeSelector }}
       nodeSelector:
@@ -135,6 +132,8 @@ spec:
           capabilities:
             drop:
             - ALL
+            add:
+            - NET_BIND_SERVICE
 {{- if or .Values.controller.readOnlyRootFilesystem .Values.nginxServiceMesh.enable .Values.controller.volumeMounts }}
         volumeMounts:
 {{- end }}

diff --git a/deployments/helm-chart/templates/controller-deployment.yaml b/deployments/helm-chart/templates/controller-deployment.yaml
@@ -90,9 +90,6 @@ spec:
 {{- if .Values.controller.readOnlyRootFilesystem }}
         fsGroup: 101 #nginx
 {{- end }}
-        sysctls:
-          - name: "net.ipv4.ip_unprivileged_port_start"
-            value: "0"
       terminationGracePeriodSeconds: {{ .Values.controller.terminationGracePeriodSeconds }}
       hostNetwork: {{ .Values.controller.hostNetwork }}
       dnsPolicy: {{ .Values.controller.dnsPolicy }}
@@ -140,6 +137,8 @@ spec:
           capabilities:
             drop:
             - ALL
+            add:
+            - NET_BIND_SERVICE
 {{- if or .Values.controller.readOnlyRootFilesystem .Values.nginxServiceMesh.enable .Values.controller.volumeMounts }}
         volumeMounts:
 {{- end }}