From 8be01446762dcaae9a2916b3d59ca78c6ca5670f Mon Sep 17 00:00:00 2001 From: Valters Jansons Date: Thu, 2 Mar 2023 18:48:28 +0200 Subject: [PATCH] Rework port binding logic without privileges (#3573) This is a breaking change! To start Nginx Ingress Controller with this commit applied, you must use an updated image. Historically, the Ingress Controller entrypoint launched with restricted capabilities. Then Nginx process escalated privileges (NET_BIND_SERVICE) to bind ports 80 and 443 as non-root user. Allowing privilege escalation is generally frowned upon in various policies. The Nginx binary in old images was adding NET_BIND_SERVICE to its Permitted capability set and also setting the Effective bit, to enforce the Permitted capability during launch. (That's the escalation there.) With this change, privilege escalation is no longer allowed and the NET_BIND_SERVICE capability is removed. To allow the binary to start, the capabilities are no longer being adjusted on the binary file. This works because Kubernetes v1.22+ allows Pods to independently lower unprivileged port range to start with zero without affecting other ports (namespaced/"safe" sysctls). OBS! An old image may be used if the binary's Effective bit is removed: FROM nginx/nginx-ingress:3.0.2 USER root RUN setcap 'cap_net_bind_service=-e' /usr/sbin/nginx 'cap_net_bind_service=-e' /usr/sbin/nginx-debug \ && setcap -v 'cap_net_bind_service=-e' /usr/sbin/nginx 'cap_net_bind_service=-e' /usr/sbin/nginx-debug # 101 is nginx USER 101 --- .github/workflows/ci.yml | 2 +- build/Dockerfile | 4 +--- cmd/nginx-ingress/main.go | 2 +- deployments/daemon-set/nginx-ingress.yaml | 8 +++++--- deployments/daemon-set/nginx-plus-ingress.yaml | 8 +++++--- deployments/deployment/nginx-ingress.yaml | 8 +++++--- deployments/deployment/nginx-plus-ingress.yaml | 8 +++++--- deployments/helm-chart-dos-arbitrator/Chart.yaml | 2 +- deployments/helm-chart/Chart.yaml | 2 +- .../helm-chart/templates/controller-daemonset.yaml | 8 +++++--- .../helm-chart/templates/controller-deployment.yaml | 8 +++++--- docs/content/technical-specifications.md | 3 ++- 12 files changed, 37 insertions(+), 26 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 69e895323a..9ee06cc41b 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -296,7 +296,7 @@ jobs: {\"image\": \"debian-plus-nap\", \"marker\": \"appprotect\"}], \ \"k8s\": [\"${{ needs.checks.outputs.k8s_latest }}\"]}" >> $GITHUB_OUTPUT else - echo "matrix={\"k8s\": [\"1.21.14\", \"1.22.15\", \"1.23.13\", \"1.24.7\", \"1.25.3\", \"${{ needs.checks.outputs.k8s_latest }}\"], \ + echo "matrix={\"k8s\": [\"1.22.15\", \"1.23.13\", \"1.24.7\", \"1.25.3\", \"${{ needs.checks.outputs.k8s_latest }}\"], \ \"images\": [{\"image\": \"debian\"}, {\"image\": \"debian-plus\"}]}" >> $GITHUB_OUTPUT fi diff --git a/build/Dockerfile b/build/Dockerfile index 26ec9b3cb5..74897dce7d 100644 --- a/build/Dockerfile +++ b/build/Dockerfile @@ -186,7 +186,7 @@ RUN --mount=type=secret,id=nginx-repo.crt,dst=/etc/ssl/nginx/nginx-repo.crt,mode # RUN update-ca-trust extract -############################################# Create common files, permissions and setcap ############################################# +############################################# Create common files and permissions ############################################# FROM ${BUILD_OS} as common ARG BUILD_OS @@ -209,8 +209,6 @@ RUN [ -n "${NAP_MODULES##*dos*}" ] && exit 0; mkdir -p /root/app_protect_dos /et && chmod 777 /shared/cores /var/log/adm /var/run/adm /etc/app_protect_dos RUN --mount=type=bind,target=/tmp mkdir -p /var/lib/nginx /etc/nginx/secrets /etc/nginx/stream-conf.d \ - && setcap 'cap_net_bind_service=+ep' /usr/sbin/nginx 'cap_net_bind_service=+ep' /usr/sbin/nginx-debug \ - && setcap -v 'cap_net_bind_service=+ep' /usr/sbin/nginx 'cap_net_bind_service=+ep' /usr/sbin/nginx-debug \ && [ -z "${BUILD_OS##*plus*}" ] && PLUS=-plus; cp -a /tmp/internal/configs/version1/nginx$PLUS.ingress.tmpl /tmp/internal/configs/version1/nginx$PLUS.tmpl \ /tmp/internal/configs/version2/nginx$PLUS.virtualserver.tmpl /tmp/internal/configs/version2/nginx$PLUS.transportserver.tmpl / \ && chown -R 101:0 /etc/nginx /var/cache/nginx /var/lib/nginx /*.tmpl \ diff --git a/cmd/nginx-ingress/main.go b/cmd/nginx-ingress/main.go index dbe4886b35..914ef7618e 100644 --- a/cmd/nginx-ingress/main.go +++ b/cmd/nginx-ingress/main.go @@ -233,7 +233,7 @@ func kubernetesVersionInfo(kubeClient kubernetes.Interface) { } glog.Infof("Kubernetes version: %v", k8sVersion) - minK8sVersion, err := util_version.ParseGeneric("1.21.0") + minK8sVersion, err := util_version.ParseGeneric("1.22.0") if err != nil { glog.Fatalf("unexpected error parsing minimum supported version: %v", err) } diff --git a/deployments/daemon-set/nginx-ingress.yaml b/deployments/daemon-set/nginx-ingress.yaml index f13769734e..58f988ac5d 100644 --- a/deployments/daemon-set/nginx-ingress.yaml +++ b/deployments/daemon-set/nginx-ingress.yaml @@ -19,6 +19,10 @@ spec: spec: serviceAccountName: nginx-ingress automountServiceAccountToken: true + securityContext: + sysctls: + - name: "net.ipv4.ip_unprivileged_port_start" + value: "0" containers: - image: nginx/nginx-ingress:3.0.2 imagePullPolicy: IfNotPresent @@ -47,13 +51,11 @@ spec: # cpu: "1" # memory: "1Gi" securityContext: - allowPrivilegeEscalation: true + allowPrivilegeEscalation: false runAsUser: 101 #nginx capabilities: drop: - ALL - add: - - NET_BIND_SERVICE env: - name: POD_NAMESPACE valueFrom: diff --git a/deployments/daemon-set/nginx-plus-ingress.yaml b/deployments/daemon-set/nginx-plus-ingress.yaml index 6c78956d5b..3d2ee496d1 100644 --- a/deployments/daemon-set/nginx-plus-ingress.yaml +++ b/deployments/daemon-set/nginx-plus-ingress.yaml @@ -19,6 +19,10 @@ spec: spec: serviceAccountName: nginx-ingress automountServiceAccountToken: true + securityContext: + sysctls: + - name: "net.ipv4.ip_unprivileged_port_start" + value: "0" containers: - image: nginx-plus-ingress:3.0.2 imagePullPolicy: IfNotPresent @@ -47,13 +51,11 @@ spec: # cpu: "1" # memory: "1Gi" securityContext: - allowPrivilegeEscalation: true + allowPrivilegeEscalation: false runAsUser: 101 #nginx capabilities: drop: - ALL - add: - - NET_BIND_SERVICE env: - name: POD_NAMESPACE valueFrom: diff --git a/deployments/deployment/nginx-ingress.yaml b/deployments/deployment/nginx-ingress.yaml index 9276a8b8f3..95faf6a256 100644 --- a/deployments/deployment/nginx-ingress.yaml +++ b/deployments/deployment/nginx-ingress.yaml @@ -20,6 +20,10 @@ spec: spec: serviceAccountName: nginx-ingress automountServiceAccountToken: true + securityContext: + sysctls: + - name: "net.ipv4.ip_unprivileged_port_start" + value: "0" containers: - image: nginx/nginx-ingress:3.0.2 imagePullPolicy: IfNotPresent @@ -46,14 +50,12 @@ spec: # cpu: "1" # memory: "1Gi" securityContext: - allowPrivilegeEscalation: true + allowPrivilegeEscalation: false runAsUser: 101 #nginx runAsNonRoot: true capabilities: drop: - ALL - add: - - NET_BIND_SERVICE env: - name: POD_NAMESPACE valueFrom: diff --git a/deployments/deployment/nginx-plus-ingress.yaml b/deployments/deployment/nginx-plus-ingress.yaml index eb19e0ecb9..3151d1ff8e 100644 --- a/deployments/deployment/nginx-plus-ingress.yaml +++ b/deployments/deployment/nginx-plus-ingress.yaml @@ -20,6 +20,10 @@ spec: spec: serviceAccountName: nginx-ingress automountServiceAccountToken: true + securityContext: + sysctls: + - name: "net.ipv4.ip_unprivileged_port_start" + value: "0" containers: - image: nginx-plus-ingress:3.0.2 imagePullPolicy: IfNotPresent @@ -48,14 +52,12 @@ spec: # cpu: "1" # memory: "1Gi" securityContext: - allowPrivilegeEscalation: true + allowPrivilegeEscalation: false runAsUser: 101 #nginx runAsNonRoot: true capabilities: drop: - ALL - add: - - NET_BIND_SERVICE env: - name: POD_NAMESPACE valueFrom: diff --git a/deployments/helm-chart-dos-arbitrator/Chart.yaml b/deployments/helm-chart-dos-arbitrator/Chart.yaml index 1a0285af04..0aabca8340 100644 --- a/deployments/helm-chart-dos-arbitrator/Chart.yaml +++ b/deployments/helm-chart-dos-arbitrator/Chart.yaml @@ -2,7 +2,7 @@ name: nginx-appprotect-dos-arbitrator version: 0.1.0 appVersion: 1.1.0 apiVersion: v1 -kubeVersion: ">= 1.21.0-0" +kubeVersion: ">= 1.22.0-0" description: NGINX App Protect Dos arbitrator icon: https://raw.githubusercontent.com/nginxinc/kubernetes-ingress/v3.0.2/deployments/helm-chart-dos-arbitrator/chart-icon.png home: https://github.com/nginxinc/kubernetes-ingress diff --git a/deployments/helm-chart/Chart.yaml b/deployments/helm-chart/Chart.yaml index af234a87bf..39fcdf3f9d 100644 --- a/deployments/helm-chart/Chart.yaml +++ b/deployments/helm-chart/Chart.yaml @@ -2,7 +2,7 @@ apiVersion: v2 name: nginx-ingress version: 0.16.2 appVersion: 3.0.2 -kubeVersion: ">= 1.21.0-0" +kubeVersion: ">= 1.22.0-0" type: application description: NGINX Ingress Controller icon: https://raw.githubusercontent.com/nginxinc/kubernetes-ingress/v3.0.2/deployments/helm-chart/chart-icon.png diff --git a/deployments/helm-chart/templates/controller-daemonset.yaml b/deployments/helm-chart/templates/controller-daemonset.yaml index 6e2f371894..64663d28ae 100644 --- a/deployments/helm-chart/templates/controller-daemonset.yaml +++ b/deployments/helm-chart/templates/controller-daemonset.yaml @@ -43,6 +43,10 @@ spec: spec: serviceAccountName: {{ include "nginx-ingress.serviceAccountName" . }} automountServiceAccountToken: true + securityContext: + sysctls: + - name: "net.ipv4.ip_unprivileged_port_start" + value: "0" terminationGracePeriodSeconds: {{ .Values.controller.terminationGracePeriodSeconds }} {{- if .Values.controller.nodeSelector }} nodeSelector: @@ -110,14 +114,12 @@ spec: initialDelaySeconds: {{ .Values.controller.readyStatus.initialDelaySeconds }} {{- end }} securityContext: - allowPrivilegeEscalation: true + allowPrivilegeEscalation: false runAsUser: 101 #nginx runAsNonRoot: true capabilities: drop: - ALL - add: - - NET_BIND_SERVICE {{- if or .Values.controller.volumeMounts .Values.nginxServiceMesh.enable }} volumeMounts: {{- end }} diff --git a/deployments/helm-chart/templates/controller-deployment.yaml b/deployments/helm-chart/templates/controller-deployment.yaml index f8d08780ed..d0a65e127b 100644 --- a/deployments/helm-chart/templates/controller-deployment.yaml +++ b/deployments/helm-chart/templates/controller-deployment.yaml @@ -76,6 +76,10 @@ spec: {{- end }} serviceAccountName: {{ include "nginx-ingress.serviceAccountName" . }} automountServiceAccountToken: true + securityContext: + sysctls: + - name: "net.ipv4.ip_unprivileged_port_start" + value: "0" terminationGracePeriodSeconds: {{ .Values.controller.terminationGracePeriodSeconds }} hostNetwork: {{ .Values.controller.hostNetwork }} dnsPolicy: {{ .Values.controller.dnsPolicy }} @@ -116,14 +120,12 @@ spec: resources: {{ toYaml .Values.controller.resources | indent 10 }} securityContext: - allowPrivilegeEscalation: true + allowPrivilegeEscalation: false runAsUser: 101 #nginx runAsNonRoot: true capabilities: drop: - ALL - add: - - NET_BIND_SERVICE {{- if or (.Values.controller.volumeMounts) (.Values.nginxServiceMesh.enable) }} volumeMounts: {{- end }} diff --git a/docs/content/technical-specifications.md b/docs/content/technical-specifications.md index 97469f72db..148b5e926c 100644 --- a/docs/content/technical-specifications.md +++ b/docs/content/technical-specifications.md @@ -12,7 +12,8 @@ docs: "DOCS-617" We advise users to run the most recent release of the NGINX Ingress Controller, and we issue software updates to the most recent release. We provide technical support for F5 customers who are using the most recent version of the NGINX Ingress Controller, and any version released within two years of the current release. -The current release version 3.x supports `discovery.k8s.io/v1` API version of EndpointSlice, available from Kubernetes 1.21 onwards. +The current 3.x release does not use `NET_BIND_SERVICE` capability, and instead relies on `net.ipv4.ip_unprivileged_port_start` sysctl to allow port binding. Kubernetes 1.22 or later is required for this sysctl to be [classified as safe](https://kubernetes.io/docs/tasks/administer-cluster/sysctl-cluster/#safe-and-unsafe-sysctls). +The 3.0.0 release supports `discovery.k8s.io/v1` API version of EndpointSlice, available from Kubernetes 1.21 onwards. The 2.4.2 release is compatible with the Kubernetes Ingress v1 API. Therefore Kubernetes 1.19 and later. The 1.12 release supports the Ingress v1beta1 API and continues to receive security fixes to support those unable to upgrade to Kubernetes 1.19 or later. The v1beta1 Ingress API was deprecated with Kubernetes release 1.19 and removed with the Kubernetes 1.22 release.