Upgrade to ingress-nginx v0.40.2 (#122)

* Upgrade to ingress-nginx v0.40.1 * Cleanup trialing spaces * Upgrade architect-orb from 0.10.0 to 0.12.0 * Update e2e integration test dependencies * Align with API change in appcatalog * Add back v1beta1 to admissionReviewVersions * Adjust hook delete policy * Use architect-orb 0.13.0 * Small indenting fix * Cover Ingress API related validation breaking change and migration options * Temporarily disable validating webhook, to check rest of the perf tests works * Reenable validating webhook * Use ingress-nginx-controller 0.40.2 * Fix a typo * Tune helm hook weights * Update changelog with new findings * Add recommendation to change Ingress resources API group
giantswarm · Oct 7, 2020 · ecfda79 · ecfda79
1 parent b7f304c
commit ecfda79
Show file tree

Hide file tree

Showing 18 changed files with 148 additions and 44 deletions.
diff --git a/.circleci/config.yml b/.circleci/config.yml
@@ -1,6 +1,6 @@
 version: 2.1
 orbs:
-  architect: giantswarm/architect@0.10.0
+  architect: giantswarm/architect@0.13.0
 
 jobs:
   run-perf-tests:

diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -7,6 +7,24 @@ and this project's packages adheres to [Semantic Versioning](http://semver.org/s
 
 ## [Unreleased]
 
+### Changed
+
+- Upgrade ingress-nginx-controller from v0.35.0 to [v0.40.2](https://github.com/kubernetes/ingress-nginx/blob/master/Changelog.md#0402).
+
+  **Important** upstream changes to pay special attention to:
+
+  - App/chart requires Kubernetes 1.16+ based platform release
+    - It is recommended to change API group of Ingress resources from `extensions/v1beta1` to `networking.k8s.io/v1beta1` (available since Kubernetes 1.14)
+  - Default configuration changes:
+    - [`gzip-level`](https://github.com/kubernetes/ingress-nginx/blob/master/docs/user-guide/nginx-configuration/configmap.md#gzip-level) default changed from `5` to `1`
+    - [`ssl-session-tickets`](https://github.com/kubernetes/ingress-nginx/blob/master/docs/user-guide/nginx-configuration/configmap.md#ssl-session-tickets) default changed from `true` to `false`
+    - [`use-gzip`](https://github.com/kubernetes/ingress-nginx/blob/master/docs/user-guide/nginx-configuration/configmap.md#use-gzip) default changed from `true` to `false`
+    - [`upstream-keepalive-connections`](https://github.com/kubernetes/ingress-nginx/blob/master/docs/user-guide/nginx-configuration/configmap.md#upstream-keepalive-connections) changed from `32` to `320`
+    - [`upstream-keepalive-requests`](https://github.com/kubernetes/ingress-nginx/blob/master/docs/user-guide/nginx-configuration/configmap.md#upstream-keepalive-requests) changed from `100` to `10000`
+- Support and enable by default [mimalloc](https://github.com/microsoft/mimalloc) as a drop-in malloc replacement to reduce nginx memory utilization.
+- Support configuring additional environment variables for NGINX Ingress Controller container, to support configuring additional mimalloc [options](https://github.com/microsoft/mimalloc#environment-options).
+- Adjust Helm `hook-delete-policy` and `hook-weight` to make admission webhook management more reliable.
+
 ## [1.9.2] - 2020-09-02
 
 ### Added

diff --git a/go.mod b/go.mod
@@ -1,14 +1,14 @@
 module github.com/giantswarm/nginx-ingress-controller-app
 
-go 1.14
+go 1.15
 
 require (
-	github.com/giantswarm/apiextensions/v2 v2.0.1 // indirect
-	github.com/giantswarm/appcatalog v0.1.11
-	github.com/giantswarm/e2esetup/v2 v2.0.0
+	github.com/giantswarm/apiextensions/v2 v2.5.3 // indirect
+	github.com/giantswarm/appcatalog v0.2.7
+	github.com/giantswarm/e2esetup/v2 v2.1.0
 	github.com/giantswarm/e2etests/v2 v2.0.0
-	github.com/giantswarm/helmclient/v2 v2.0.0
+	github.com/giantswarm/helmclient/v2 v2.1.4
 	github.com/giantswarm/k8sclient/v4 v4.0.0
-	github.com/giantswarm/micrologger v0.3.1
-	k8s.io/apimachinery v0.18.5
+	github.com/giantswarm/micrologger v0.3.3
+	k8s.io/apimachinery v0.18.9
 )
diff --git a/go.sum b/go.sum
diff --git a/helm/nginx-ingress-controller-app/Chart.yaml b/helm/nginx-ingress-controller-app/Chart.yaml
@@ -1,8 +1,9 @@
 apiVersion: v1
-appVersion: v0.35.0
+appVersion: v0.40.2
 description: A Helm chart for the nginx ingress-controller
 home: https://github.com/giantswarm/nginx-ingress-controller-app
 icon: https://upload.wikimedia.org/wikipedia/commons/thumb/c/c5/Nginx_logo.svg/500px-Nginx_logo.svg.png
+kubeVersion: ">=1.16.0-0"
 name: nginx-ingress-controller-app
 namespace: kube-system
 version: [[ .Version ]]

diff --git a/helm/nginx-ingress-controller-app/templates/admission-webhooks/job-patch/clusterrole.yaml b/helm/nginx-ingress-controller-app/templates/admission-webhooks/job-patch/clusterrole.yaml
@@ -5,8 +5,8 @@ metadata:
   name: {{ include "resource.default.name" . }}-admission
   annotations:
     "helm.sh/hook": pre-install,pre-upgrade,post-install,post-upgrade
-    "helm.sh/hook-delete-policy": before-hook-creation,hook-succeeded
-    "helm.sh/hook-weight": "-1"
+    "helm.sh/hook-delete-policy": "before-hook-creation,hook-succeeded,hook-failed"
+    "helm.sh/hook-weight": "-2"
   labels:
     {{- include "labels.common" . | nindent 4 }}
     app.kubernetes.io/component: admission-webhook
@@ -20,7 +20,7 @@ rules:
       - update
 {{- if .Values.podSecurityPolicy.enabled }}
   - apiGroups: 
-       - extensions
+      - extensions
     resources:
       - podsecuritypolicies
     verbs:

diff --git a/...inx-ingress-controller-app/templates/admission-webhooks/job-patch/clusterrolebinding.yaml b/...inx-ingress-controller-app/templates/admission-webhooks/job-patch/clusterrolebinding.yaml
@@ -5,7 +5,7 @@ metadata:
   name:  {{ include "resource.default.name" . }}-admission
   annotations:
     "helm.sh/hook": pre-install,pre-upgrade,post-install,post-upgrade
-    "helm.sh/hook-delete-policy": before-hook-creation,hook-succeeded
+    "helm.sh/hook-delete-policy": "before-hook-creation,hook-succeeded,hook-failed"
     "helm.sh/hook-weight": "-1"
   labels:
     {{- include "labels.common" . | nindent 4 }}

diff --git a/...nginx-ingress-controller-app/templates/admission-webhooks/job-patch/job-createSecret.yaml b/...nginx-ingress-controller-app/templates/admission-webhooks/job-patch/job-createSecret.yaml
@@ -6,7 +6,7 @@ metadata:
   namespace: {{ .Release.Namespace }}
   annotations:
     "helm.sh/hook": pre-install,pre-upgrade
-    "helm.sh/hook-delete-policy": before-hook-creation,hook-succeeded
+    "helm.sh/hook-delete-policy": "before-hook-creation,hook-succeeded,hook-failed"
     "helm.sh/hook-weight": "0"
   labels:
     {{- include "labels.common" . | nindent 4 }}

diff --git a/...nginx-ingress-controller-app/templates/admission-webhooks/job-patch/job-patchWebhook.yaml b/...nginx-ingress-controller-app/templates/admission-webhooks/job-patch/job-patchWebhook.yaml
@@ -6,7 +6,7 @@ metadata:
   namespace: {{ .Release.Namespace }}
   annotations:
     "helm.sh/hook": post-install,post-upgrade
-    "helm.sh/hook-delete-policy": before-hook-creation,hook-succeeded
+    "helm.sh/hook-delete-policy": "before-hook-creation,hook-succeeded,hook-failed"
     "helm.sh/hook-weight": "0"
   labels:
     {{- include "labels.common" . | nindent 4 }}

diff --git a/helm/nginx-ingress-controller-app/templates/admission-webhooks/job-patch/np.yaml b/helm/nginx-ingress-controller-app/templates/admission-webhooks/job-patch/np.yaml
@@ -5,7 +5,7 @@ metadata:
   namespace: {{ .Release.Namespace }}
   annotations:
     "helm.sh/hook": pre-install,pre-upgrade
-    "helm.sh/hook-delete-policy": before-hook-creation,hook-succeeded
+    "helm.sh/hook-delete-policy": "before-hook-creation,hook-succeeded,hook-failed"
     "helm.sh/hook-weight": "-1"
   labels:
     {{- include "labels.common" . | nindent 4 }}

diff --git a/helm/nginx-ingress-controller-app/templates/admission-webhooks/job-patch/psp.yaml b/helm/nginx-ingress-controller-app/templates/admission-webhooks/job-patch/psp.yaml
@@ -5,8 +5,8 @@ metadata:
   name: {{ include "resource.default.name" . }}-admission
   annotations:
     "helm.sh/hook": pre-install,pre-upgrade,post-install,post-upgrade
-    "helm.sh/hook-delete-policy": before-hook-creation,hook-succeeded
-    "helm.sh/hook-weight": "-1"
+    "helm.sh/hook-delete-policy": "before-hook-creation,hook-succeeded,hook-failed"
+    "helm.sh/hook-weight": "-3"
   labels:
     {{- include "labels.common" . | nindent 4 }}
     app.kubernetes.io/component: admission-webhook

diff --git a/helm/nginx-ingress-controller-app/templates/admission-webhooks/job-patch/role.yaml b/helm/nginx-ingress-controller-app/templates/admission-webhooks/job-patch/role.yaml
@@ -5,8 +5,8 @@ metadata:
   name:  {{ include "resource.default.name" . }}-admission
   annotations:
     "helm.sh/hook": pre-install,pre-upgrade,post-install,post-upgrade
-    "helm.sh/hook-delete-policy": before-hook-creation,hook-succeeded
-    "helm.sh/hook-weight": "-1"
+    "helm.sh/hook-delete-policy": "before-hook-creation,hook-succeeded,hook-failed"
+    "helm.sh/hook-weight": "-2"
   labels:
     {{- include "labels.common" . | nindent 4 }}
     app.kubernetes.io/component: admission-webhook

diff --git a/helm/nginx-ingress-controller-app/templates/admission-webhooks/job-patch/rolebinding.yaml b/helm/nginx-ingress-controller-app/templates/admission-webhooks/job-patch/rolebinding.yaml
@@ -5,7 +5,7 @@ metadata:
   name: {{ include "resource.default.name" . }}-admission
   annotations:
     "helm.sh/hook": pre-install,pre-upgrade,post-install,post-upgrade
-    "helm.sh/hook-delete-policy": before-hook-creation,hook-succeeded
+    "helm.sh/hook-delete-policy": "before-hook-creation,hook-succeeded,hook-failed"
     "helm.sh/hook-weight": "-1"
   labels:
     {{- include "labels.common" . | nindent 4 }}

diff --git a/helm/nginx-ingress-controller-app/templates/admission-webhooks/job-patch/serviceaccount.yaml b/helm/nginx-ingress-controller-app/templates/admission-webhooks/job-patch/serviceaccount.yaml
@@ -5,8 +5,8 @@ metadata:
   name: {{ include "resource.default.name" . }}-admission
   annotations:
     "helm.sh/hook": pre-install,pre-upgrade,post-install,post-upgrade
-    "helm.sh/hook-delete-policy": before-hook-creation,hook-succeeded
-    "helm.sh/hook-weight": "-1"
+    "helm.sh/hook-delete-policy": "before-hook-creation,hook-succeeded,hook-failed"
+    "helm.sh/hook-weight": "-2"
   labels:
     {{- include "labels.common" . | nindent 4 }}
     app.kubernetes.io/component: admission-webhook

diff --git a/helm/nginx-ingress-controller-app/templates/admission-webhooks/validating-webhook.yaml b/helm/nginx-ingress-controller-app/templates/admission-webhooks/validating-webhook.yaml
@@ -1,7 +1,7 @@
 {{- if .Values.controller.admissionWebhooks.enabled -}}
 # before changing this value, check the required kubernetes version
 # https://kubernetes.io/docs/reference/access-authn-authz/extensible-admission-controllers/#prerequisites
-apiVersion: admissionregistration.k8s.io/v1beta1
+apiVersion: admissionregistration.k8s.io/v1
 kind: ValidatingWebhookConfiguration
 metadata:
   name: {{ include "resource.default.name" . }}-admission
@@ -12,10 +12,10 @@ webhooks:
 - name: validate.nginx.ingress.kubernetes.io
   rules:
   - apiGroups:
-    - extensions
     - networking.k8s.io
     apiVersions:
     - v1beta1
+    - v1
     operations:
     - CREATE
     - UPDATE
@@ -24,11 +24,11 @@ webhooks:
   failurePolicy: Fail
   sideEffects: None
   admissionReviewVersions:
-    - v1
     - v1beta1
+    - v1
   clientConfig:
     service:
       namespace: {{ .Release.Namespace }}
       name: {{ include "resource.default.name" . }}-admission
-      path: /extensions/v1beta1/ingresses
+      path: /networking/v1beta1/ingresses
 {{- end }}
diff --git a/helm/nginx-ingress-controller-app/templates/controller-deployment.yaml b/helm/nginx-ingress-controller-app/templates/controller-deployment.yaml
@@ -115,6 +115,13 @@ spec:
           valueFrom:
             fieldRef:
               fieldPath: metadata.namespace
+        {{- if .Values.controller.enableMimalloc }}
+        - name: LD_PRELOAD
+          value: /usr/local/lib/libmimalloc.so
+        {{- end }}
+        {{- if .Values.controller.extraEnvs }}
+          {{- toYaml .Values.controller.extraEnvs | nindent 8 }}
+        {{- end }}
         {{- if .Values.cluster.profile }}
         {{- if gt (.Values.cluster.profile | int) 2 }}
         resources:

diff --git a/helm/nginx-ingress-controller-app/values.yaml b/helm/nginx-ingress-controller-app/values.yaml
@@ -33,11 +33,6 @@ configmap:
   # Increase hash table size to allow more server names for stability reasons
   server-name-hash-bucket-size: "1024"
 
-  # configmap.server-tokens
-  # Controls whether to send NGINX Server header in responses and display NGINX
-  # version in error pages.
-  server-tokens: "false"
-
   # configmap.worker-processes
   # Sets the number of worker processes.
   worker-processes: "4"
@@ -79,7 +74,7 @@ controller:
 
     # controller.image.tag
     # When updating tag make sure to also keep appVersion in Chart.yaml in sync
-    tag: v0.35.0
+    tag: v0.40.2
 
   # controller.containerPort
   containerPort:
@@ -314,6 +309,21 @@ controller:
       tolerations: []
       runAsUser: 2000
 
+  # controller.enableMimalloc
+  # Enables mimalloc as a drop-in replacement for malloc, reduces the memory utilization.
+  # For additional options it provides see https://github.com/microsoft/mimalloc#environment-options
+  enableMimalloc: true
+
+  # controller.extraEnvs
+  # Additional environment variables to set for the controller.
+  extraEnvs: []
+  # extraEnvs:
+  #   - name: FOO
+  #     valueFrom:
+  #       secretKeyRef:
+  #         key: FOO
+  #         name: secret-resource
+
 # image
 image:
   registry: quay.io

diff --git a/integration/test/basic/main_test.go b/integration/test/basic/main_test.go
@@ -41,7 +41,7 @@ func init() {
 
 	var latestRelease string
 	{
-		latestRelease, err = appcatalog.GetLatestVersion(ctx, catalogURL, appName)
+		latestRelease, err = appcatalog.GetLatestVersion(ctx, catalogURL, appName, "")
 		if err != nil {
 			panic(err.Error())
 		}
@@ -132,7 +132,7 @@ func init() {
 							"app.kubernetes.io/instance":         name,
 							"app.kubernetes.io/managed-by":       "Helm",
 							"app.kubernetes.io/name":             name,
-							"app.kubernetes.io/version":          "v0.35.0",
+							"app.kubernetes.io/version":          "v0.40.2",
 							"giantswarm.io/monitoring_basic_sli": "true",
 							"giantswarm.io/service-type":         "managed",
 							"helm.sh/chart":                      helmChartLabel,
@@ -147,7 +147,7 @@ func init() {
 							"app.kubernetes.io/instance":   name,
 							"app.kubernetes.io/managed-by": "Helm",
 							"app.kubernetes.io/name":       name,
-							"app.kubernetes.io/version":    "v0.35.0",
+							"app.kubernetes.io/version":    "v0.40.2",
 							"giantswarm.io/service-type":   "managed",
 							"helm.sh/chart":                helmChartLabel,
 							"k8s-app":                      name,