chart(breaking change): enable TLS and default annotations for ingress (

#2326)

docs for #2321

Signed-off-by: Viet Nguyen Duc <[email protected]>

.github/workflows/build-test.yml

@@ -26,6 +26,7 @@ permissions: write-all
 
 env:
   GH_CLI_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+  GH_CLI_TOKEN_PR: ${{ secrets.SELENIUM_CI_TOKEN || secrets.GITHUB_TOKEN }}
   RUN_ID: ${{ github.run_id }}
   RERUN_FAILED_ONLY: ${{ github.event.inputs.rerunFailedOnly || true }}
   RUN_ATTEMPT: ${{ github.run_attempt }}
@@ -57,7 +58,12 @@ jobs:
         run: |
           sudo apt update
           sudo apt install gh
+      - name: Authenticate GitHub CLI for PR
+        if: github.event_name == 'pull_request'
+        run: |
+          echo "$GH_CLI_TOKEN_PR" | gh auth login --with-token
       - name: Authenticate GitHub CLI
+        if: github.event_name != 'pull_request'
         run: |
           echo "$GH_CLI_TOKEN" | gh auth login --with-token
       - name: Rerun workflow when failure

Makefile

@@ -798,7 +798,7 @@ chart_test_autoscaling_job_hostname:
 
 chart_test_autoscaling_job:
 	PLATFORMS=$(PLATFORMS) TEST_CHROMIUM=true RELEASE_NAME=selenium CHART_ENABLE_TRACING=true CHART_FULL_DISTRIBUTED_MODE=true \
-	SECURE_INGRESS_ONLY_GENERATE=true CHART_ENABLE_INGRESS_HOSTNAME=true SELENIUM_GRID_PROTOCOL=https SELENIUM_GRID_HOST=selenium-grid.prod SUB_PATH=/ SELENIUM_GRID_PORT=443 \
+	SECURE_INGRESS_ONLY_CONFIG_INLINE=true SECURE_USE_EXTERNAL_CERT=true CHART_ENABLE_INGRESS_HOSTNAME=true SELENIUM_GRID_PROTOCOL=https SELENIUM_GRID_HOST=selenium-grid.prod SUB_PATH=/ SELENIUM_GRID_PORT=443 \
 	VERSION=$(TAG_VERSION) VIDEO_TAG=$(FFMPEG_TAG_VERSION)-$(BUILD_DATE) NAMESPACE=$(NAMESPACE) BINDING_VERSION=$(BINDING_VERSION) \
 	./tests/charts/make/chart_test.sh JobAutoscaling
 

charts/selenium-grid/README.md

@@ -281,25 +281,23 @@ nginx.ingress.kubernetes.io/client-body-buffer-size
 
 # `ingress.nginx.proxyBuffer.number` pass value to annotation(s)
 nginx.ingress.kubernetes.io/proxy-buffers-number
-```
-
-You can generate a test double self-signed certificate specify for your `hostname`, assign it to spec `ingress.tls` and NGINX ingress controller default certificate (if it is enabled inline). For example:
 
-```yaml
-tls:
-  ingress:
-    generateTLS: true
+# `ingress.nginx.websocket` pass boolean value to add backend service has WebSocket request (Hub/Router - noVNC, CDP, etc.)
+nginx.org/websocket-services: "{{ template ($.Values.isolateComponents | ternary "seleniumGrid.router.fullname" "seleniumGrid.hub.fullname") $ }}"
 
-ingress:
-  hostname: "your.domain.com"
+# `ingress.nginx.sslPassthrough` pass boolean value to enable SSL Passthrough (when secure connection is enabled in Grid server backend)
+nginx.ingress.kubernetes.io/ssl-passthrough: "true"
+nginx.ingress.kubernetes.io/backend-protocol: "HTTPS"
 
-ingress-nginx:
-  enabled: true
-  controller:
-    extraArgs:
-      default-ssl-certificate: '$(POD_NAMESPACE)/selenium-tls-secret'
+# `ingress.nginx.sslSecret` to specify a Secret with the certificate `tls.crt`, key `tls.key`, the name in the form "namespace/secretName"
+# By default, it is empty, the chart will use internal TLS secret resource (or the first `secretName` under `ingress.tls` if set)
+nginx.ingress.kubernetes.io/proxy-ssl-secret: {{ template "seleniumGrid.tls.fullname" $ }}
 ```
 
+Refer to [NGINX Ingress Controller Annotations](https://github.com/kubernetes/ingress-nginx/blob/main/docs/user-guide/nginx-configuration/annotations.md) for more details.
+
+Refer to below section [Configuration of Secure Communication] for more details on how to configure secure communication to Ingress proxy.
+
 ## Configuration
 
 ### Configuration global
@@ -656,7 +654,7 @@ There are multiple ways to insert your certificate, private key, truststore to t
    ```bash
    # Steps to prepare your self-signed certificate
    ./certs/cert.sh -d /path/to/your/
-   # Create TLS Secret with your certificate, private key, truststore
+   # Create TLS Secret with your certificate, private key, truststore (or a Secret type kubernetes.io/tls)
    kubectl create secret generic -n $NAMESPACE my-external-tls-secret \
        --from-file=tls.crt=/path/to/your/tls.crt \
        --from-file=tls.key=/path/to/your/tls.key \
@@ -688,18 +686,16 @@ tls:
 ```
 
 In additional, if the ingress is enabled, and approach SSL Passthrough is used to ensure the request forwards to the backend components via an encrypted connection.
-With `ingress.hostname` is set, the default server TLS secret is also used for hosts TLS secretName when `ingress.tls` is empty. Once you specify `ingress.tls`, your specified secret will be used for hosts TLS secretName.
+With `ingress.hostname` is set, the default server TLS secret is also used for hosts TLS secretName when `ingress.tls` is empty. Once you specify `ingress.tls`, your specified secret will be used for hosts TLS secretName. For example
 
 ![SeleniumGrid_TLS_SSL-Passthrough](./images/SeleniumGrid_TLS_SSL-Passthrough.png)
 
-Moreover, when sub-chart `ingress-nginx` is enabled (deploy Ingress NGINX Controller together), the default server TLS secret can also be assigned via `ingress-nginx.controller.extraArgs.default-ssl-certificate`.
-For example (replace `$RELEASENAME` and `$NAMESPACE` with your values):
+```yaml
+tls:
+  enabled: true
 
-```bash
-helm upgrade -i $RELEASENAME -n $NAMESPACE docker-selenium/selenium-grid \
-    --set tls.enabled=true \
-    --set ingress-nginx.enabled=true \
-    --set ingress-nginx.controller.extraArgs.default-ssl-certificate=$NAMESPACE/$RELEASENAME-selenium-tls-secret
+ingress-ngnix:
+  enabled: true
 ```
 
 Below is an example of Grid UI accessible via NodePort with secure connection, and using external TLS Secret (replace `$RELEASENAME` and `$NAMESPACE` with your values):
@@ -729,7 +725,7 @@ tls:
 
 ![SeleniumGrid_TLS_SSL-Termination](./images/SeleniumGrid_TLS_SSL-Termination.png)
 
-In additional, a self-signed certificate and private key can be generated runtime during the chart deployment for Ingress TLS by setting these values (replace `$RELEASENAME` with your value):
+In additional, a self-signed certificate and private key can be generated runtime during the chart deployment for Ingress TLS by setting these values:
 
 ```yaml
 tls:
@@ -747,9 +743,6 @@ tls:
 
 ingress-ngnix:
   enabled: true
-  controller:
-    extraArgs:
-      default-ssl-certificate: $(POD_NAMESPACE)/$RELEASENAME-selenium-tls-secret
 ```
 
 You can get the `tls.crt` and `tls.key` from the Secret after the chart is deployed. For example (replace `$RELEASENAME` and `$NAMESPACE` with your values):
@@ -767,12 +760,38 @@ helm upgrade -i $RELEASENAME -n $NAMESPACE docker-selenium/selenium-grid \
   --set ingress.hostname="selenium-grid.prod.domain.com" \
   --set tls.ingress.enabled=true \
   --set tls.nameOverride=my-external-tls-secret \
-  --set ingress-nginx.enabled=true \
-  --set ingress-nginx.controller.extraArgs.default-ssl-certificate=$NAMESPACE/my-external-tls-secret
+  --set ingress-nginx.enabled=true
 ```
 
 Grid UI can be accessed via HTTPS address `https://selenium-grid.prod.domain.com`.
 
+Inline config TLS for the Ingress resource is also considered as enable secure connection to the Ingress proxy.
+For example, below is the config with using external TLS Secret for the Ingress resource and enable sub-chart NGINX Ingress Controller:
+
+```yaml
+ingress:
+  hostname: selenium-grid.prod.domain.com
+  tls:
+    - secretName: my-external-tls-secret
+      hosts:
+        - selenium-grid.prod.domain.com
+
+ingress-ngnix:
+    enabled: true
+```
+
+In case the Ingress resource is configured without `hostname` and `tls`, the incoming traffic access via `global.K8S_PUBLIC_IP`. When sub-chart `ingress-nginx` is enabled (deploy Ingress NGINX Controller together), the default TLS secret can also be assigned via `ingress-nginx.controller.extraArgs.default-ssl-certificate`.
+For example (replace `$RELEASENAME` and `$NAMESPACE` with your values):
+
+```bash
+helm upgrade -i $RELEASENAME -n $NAMESPACE docker-selenium/selenium-grid \
+  --set global.K8S_PUBLIC_IP=$(hostname -i) \
+  --set tls.ingress.enabled=true \
+  --set tls.nameOverride=my-external-tls-secret \
+  --set ingress-nginx.enabled=true \
+  --set ingress-nginx.controller.extraArgs.default-ssl-certificate=$NAMESPACE/my-external-tls-secret
+```
+
 ### Node Registration
 
 To enable secure in the node registration to make sure that the node is one you control and not a rouge node, you can enable and provide a registration secret string to Distributor, Router and

charts/selenium-grid/templates/_helpers.tpl

@@ -9,7 +9,7 @@ Server secure connection
 Ingress proxy forward secure connection
 */}}
 {{- define "seleniumGrid.ingress.secureConnection" -}}
-{{- or $.Values.tls.enabled $.Values.tls.ingress.enabled $.Values.tls.ingress.generateTLS | ternary "true" "" -}}
+{{- or $.Values.tls.enabled $.Values.tls.ingress.enabled $.Values.tls.ingress.generateTLS (not (empty $.Values.ingress.tls)) | ternary "true" "" -}}
 {{- end -}}
 
 {{/*
@@ -122,10 +122,21 @@ nginx.ingress.kubernetes.io/proxy-buffers-number: {{ . | quote }}
   {{- if .websocket }}
 nginx.org/websocket-services: {{ include ($.Values.isolateComponents | ternary "seleniumGrid.router.fullname" "seleniumGrid.hub.fullname") $ | quote }}
   {{- end }}
-{{- end }}
-{{- if eq (include "seleniumGrid.server.secureConnection" $) "true" }}
+  {{- if eq (include "seleniumGrid.server.secureConnection" $) "true" }}
+    {{- if .sslPassthrough }}
 nginx.ingress.kubernetes.io/ssl-passthrough: "true"
 nginx.ingress.kubernetes.io/backend-protocol: "HTTPS"
+    {{- end }}
+  {{- end }}
+  {{- if eq (include "seleniumGrid.ingress.secureConnection" $) "true" }}
+    {{- if not (empty .sslSecret) }}
+nginx.ingress.kubernetes.io/proxy-ssl-secret: {{ tpl .sslSecret $ | quote }}
+    {{- else if (empty $.Values.ingress.tls) }}
+nginx.ingress.kubernetes.io/proxy-ssl-secret: {{ tpl (printf "%s/%s" $.Release.Namespace (include "seleniumGrid.tls.fullname" $)) $ | quote }}
+    {{- else }}
+nginx.ingress.kubernetes.io/proxy-ssl-secret: {{ tpl (printf "%s/%s" $.Release.Namespace (index $.Values.ingress.tls 0).secretName) $ | quote }}
+    {{- end }}
+  {{- end }}
 {{- end }}
 {{- end -}}
 

charts/selenium-grid/templates/ingress.yaml

@@ -36,15 +36,15 @@ spec:
   tls:
     - hosts:
         - {{ tpl .Values.ingress.hostname $ | quote }}
-      secretName: {{ include "seleniumGrid.tls.fullname" . | quote }}
+      secretName: {{ include "seleniumGrid.tls.fullname" $ | quote }}
   {{- else if .Values.ingress.tls }}
   tls:
     {{- range .Values.ingress.tls }}
     - hosts:
         {{- range .hosts }}
         - {{ tpl . $ | quote }}
         {{- end }}
-      secretName: {{ tpl (default "" .secretName) $ | quote }}
+      secretName: {{ tpl (default (include "seleniumGrid.tls.fullname" $) .secretName) $ | quote }}
     {{- end }}
   {{- end }}
   rules:

charts/selenium-grid/templates/jaeger-ingress.yaml

@@ -25,16 +25,16 @@ spec:
   {{- if and (or .Values.tls.enabled .Values.tls.ingress.generateTLS) (tpl .Values.ingress.hostname $) (not .Values.ingress.tls) }}
   tls:
     - hosts:
-        - {{ tpl .Values.ingress.hostname $ | quote }}
-      secretName: {{ include "seleniumGrid.tls.fullname" . | quote }}
+        - {{ tpl .Values.ingress.hostname $ }}
+      secretName: {{ include "seleniumGrid.tls.fullname" $ | quote }}
   {{- else if .Values.ingress.tls }}
   tls:
     {{- range .Values.ingress.tls }}
     - hosts:
         {{- range .hosts }}
-        - {{ tpl . $ | quote }}
+        - {{ tpl . $ }}
         {{- end }}
-      secretName: {{ tpl (.secretName) $ | quote }}
+      secretName: {{ tpl (default (include "seleniumGrid.tls.fullname" $) .secretName) $ | quote }}
     {{- end }}
   {{- end }}
   rules:

charts/selenium-grid/values.yaml

@@ -121,6 +121,8 @@ ingress:
     proxyBuffer:
       size: 512M
       number: 4
+    sslPassthrough: true
+    sslSecret: ""
   ports:
     http: 80
     https: 443

tests/SmokeTests/__init__.py

@@ -2,7 +2,7 @@
 import unittest
 import time
 import json
-from ssl import _create_unverified_context
+import ssl
 import requests
 from requests.auth import HTTPBasicAuth
 
@@ -50,6 +50,13 @@ def smoke_test_container(self, port):
             self.assertFalse(status_json['value']['ready'], "Container is autoscaling with min replica set to 0")
 
 
+    def client_verify_cert(self, port):
+        grid_url_status = '%s://%s:%s/status' % (SELENIUM_GRID_PROTOCOL, SELENIUM_GRID_HOST, port)
+        cert_path = os.environ.get("REQUESTS_CA_BUNDLE")
+        response = requests.get(grid_url_status, verify=cert_path)
+
 class GridTest(SmokeTests):
     def test_grid_is_up(self):
         self.smoke_test_container('%s' % SELENIUM_GRID_PORT)
+        if SELENIUM_GRID_PROTOCOL == "https":
+            self.client_verify_cert('%s' % SELENIUM_GRID_PORT)

tests/charts/make/chart_test.sh

@@ -213,6 +213,7 @@ fi
 if [ "${SECURE_USE_EXTERNAL_CERT}" = "true" ]; then
   HELM_COMMAND_SET_IMAGES="${HELM_COMMAND_SET_IMAGES} \
   --set tls.nameOverride=${EXTERNAL_TLS_SECRET_NAME} \
+  --set ingress.nginx.sslSecret="${SELENIUM_NAMESPACE}/${EXTERNAL_TLS_SECRET_NAME}" \
   "
   cert_dir="./tests/tests"
   ADD_IP_ADDRESS=hostname ./${CHART_PATH}/certs/cert.sh -d ${cert_dir}
@@ -222,14 +223,29 @@ if [ "${SECURE_USE_EXTERNAL_CERT}" = "true" ]; then
   CHART_CERT_PATH="./tests/tests/tls.crt"
 fi
 
-if [ "${SECURE_USE_EXTERNAL_CERT}" = "true" ]; then
-  HELM_COMMAND_SET_IMAGES="${HELM_COMMAND_SET_IMAGES} \
-  --set ingress-nginx.controller.extraArgs.default-ssl-certificate=${SELENIUM_NAMESPACE}/${EXTERNAL_TLS_SECRET_NAME} \
-  "
-else
-  HELM_COMMAND_SET_IMAGES="${HELM_COMMAND_SET_IMAGES} \
-  --set ingress-nginx.controller.extraArgs.default-ssl-certificate=${SELENIUM_NAMESPACE}/${SELENIUM_TLS_SECRET_NAME} \
-  "
+if [ "${SECURE_INGRESS_ONLY_CONFIG_INLINE}" = "true" ]; then
+  if [ "${SECURE_USE_EXTERNAL_CERT}" = "true" ]; then
+    HELM_COMMAND_SET_IMAGES="${HELM_COMMAND_SET_IMAGES} \
+    --set ingress.tls[0].hosts[0]=${SELENIUM_GRID_HOST} \
+    --set ingress.tls[0].secretName=${EXTERNAL_TLS_SECRET_NAME} \
+    "
+  else
+    HELM_COMMAND_SET_IMAGES="${HELM_COMMAND_SET_IMAGES} \
+    --set ingress.tls[0].hosts[0]=${SELENIUM_TLS_SECRET_NAME} \
+    "
+  fi
+fi
+
+if [ "${SELENIUM_GRID_PROTOCOL}" = "https" ] && [ "${CHART_ENABLE_INGRESS_HOSTNAME}" != "true" ]; then
+  if [ "${SECURE_USE_EXTERNAL_CERT}" = "true" ]; then
+    HELM_COMMAND_SET_IMAGES="${HELM_COMMAND_SET_IMAGES} \
+    --set ingress-nginx.controller.extraArgs.default-ssl-certificate=${SELENIUM_NAMESPACE}/${EXTERNAL_TLS_SECRET_NAME} \
+    "
+  else
+    HELM_COMMAND_SET_IMAGES="${HELM_COMMAND_SET_IMAGES} \
+    --set ingress-nginx.controller.extraArgs.default-ssl-certificate=${SELENIUM_NAMESPACE}/${SELENIUM_TLS_SECRET_NAME} \
+    "
+  fi
 fi
 
 if [ "${SELENIUM_GRID_AUTOSCALING}" = "true" ]; then

tests/charts/refValues/simplex-minikube.yaml

@@ -87,9 +87,6 @@ videoRecorder:
 ingress-nginx:
   enabled: true
   controller:
-    # Set controller default certificate use the same with Selenium Grid
-    extraArgs:
-      default-ssl-certificate: '$(POD_NAMESPACE)/selenium-tls-secret'
     hostPort:
       enabled: true
     kind: DaemonSet