From e15df42f8f1f2f4447288afebf08362c4a16291a Mon Sep 17 00:00:00 2001 From: Viet Nguyen Duc Date: Sat, 27 Jul 2024 02:37:10 +0700 Subject: [PATCH] chart(breaking change): enable TLS and default annotations for ingress (#2326) docs for https://github.com/SeleniumHQ/docker-selenium/issues/2321 Signed-off-by: Viet Nguyen Duc --- .github/workflows/build-test.yml | 6 ++ Makefile | 2 +- charts/selenium-grid/README.md | 77 ++++++++++++------- charts/selenium-grid/templates/_helpers.tpl | 17 +++- charts/selenium-grid/templates/ingress.yaml | 4 +- .../templates/jaeger-ingress.yaml | 8 +- charts/selenium-grid/values.yaml | 2 + tests/SmokeTests/__init__.py | 9 ++- tests/charts/make/chart_test.sh | 32 ++++++-- tests/charts/refValues/simplex-minikube.yaml | 3 - 10 files changed, 109 insertions(+), 51 deletions(-) diff --git a/.github/workflows/build-test.yml b/.github/workflows/build-test.yml index 02a3852d8..1e2f7fa54 100644 --- a/.github/workflows/build-test.yml +++ b/.github/workflows/build-test.yml @@ -26,6 +26,7 @@ permissions: write-all env: GH_CLI_TOKEN: ${{ secrets.GITHUB_TOKEN }} + GH_CLI_TOKEN_PR: ${{ secrets.SELENIUM_CI_TOKEN || secrets.GITHUB_TOKEN }} RUN_ID: ${{ github.run_id }} RERUN_FAILED_ONLY: ${{ github.event.inputs.rerunFailedOnly || true }} RUN_ATTEMPT: ${{ github.run_attempt }} @@ -57,7 +58,12 @@ jobs: run: | sudo apt update sudo apt install gh + - name: Authenticate GitHub CLI for PR + if: github.event_name == 'pull_request' + run: | + echo "$GH_CLI_TOKEN_PR" | gh auth login --with-token - name: Authenticate GitHub CLI + if: github.event_name != 'pull_request' run: | echo "$GH_CLI_TOKEN" | gh auth login --with-token - name: Rerun workflow when failure diff --git a/Makefile b/Makefile index 240ea3220..3ebc81931 100644 --- a/Makefile +++ b/Makefile @@ -798,7 +798,7 @@ chart_test_autoscaling_job_hostname: chart_test_autoscaling_job: PLATFORMS=$(PLATFORMS) TEST_CHROMIUM=true RELEASE_NAME=selenium CHART_ENABLE_TRACING=true CHART_FULL_DISTRIBUTED_MODE=true \ - SECURE_INGRESS_ONLY_GENERATE=true CHART_ENABLE_INGRESS_HOSTNAME=true SELENIUM_GRID_PROTOCOL=https SELENIUM_GRID_HOST=selenium-grid.prod SUB_PATH=/ SELENIUM_GRID_PORT=443 \ + SECURE_INGRESS_ONLY_CONFIG_INLINE=true SECURE_USE_EXTERNAL_CERT=true CHART_ENABLE_INGRESS_HOSTNAME=true SELENIUM_GRID_PROTOCOL=https SELENIUM_GRID_HOST=selenium-grid.prod SUB_PATH=/ SELENIUM_GRID_PORT=443 \ VERSION=$(TAG_VERSION) VIDEO_TAG=$(FFMPEG_TAG_VERSION)-$(BUILD_DATE) NAMESPACE=$(NAMESPACE) BINDING_VERSION=$(BINDING_VERSION) \ ./tests/charts/make/chart_test.sh JobAutoscaling diff --git a/charts/selenium-grid/README.md b/charts/selenium-grid/README.md index b32739be8..17623d1ac 100644 --- a/charts/selenium-grid/README.md +++ b/charts/selenium-grid/README.md @@ -281,25 +281,23 @@ nginx.ingress.kubernetes.io/client-body-buffer-size # `ingress.nginx.proxyBuffer.number` pass value to annotation(s) nginx.ingress.kubernetes.io/proxy-buffers-number -``` - -You can generate a test double self-signed certificate specify for your `hostname`, assign it to spec `ingress.tls` and NGINX ingress controller default certificate (if it is enabled inline). For example: -```yaml -tls: - ingress: - generateTLS: true +# `ingress.nginx.websocket` pass boolean value to add backend service has WebSocket request (Hub/Router - noVNC, CDP, etc.) +nginx.org/websocket-services: "{{ template ($.Values.isolateComponents | ternary "seleniumGrid.router.fullname" "seleniumGrid.hub.fullname") $ }}" -ingress: - hostname: "your.domain.com" +# `ingress.nginx.sslPassthrough` pass boolean value to enable SSL Passthrough (when secure connection is enabled in Grid server backend) +nginx.ingress.kubernetes.io/ssl-passthrough: "true" +nginx.ingress.kubernetes.io/backend-protocol: "HTTPS" -ingress-nginx: - enabled: true - controller: - extraArgs: - default-ssl-certificate: '$(POD_NAMESPACE)/selenium-tls-secret' +# `ingress.nginx.sslSecret` to specify a Secret with the certificate `tls.crt`, key `tls.key`, the name in the form "namespace/secretName" +# By default, it is empty, the chart will use internal TLS secret resource (or the first `secretName` under `ingress.tls` if set) +nginx.ingress.kubernetes.io/proxy-ssl-secret: {{ template "seleniumGrid.tls.fullname" $ }} ``` +Refer to [NGINX Ingress Controller Annotations](https://github.com/kubernetes/ingress-nginx/blob/main/docs/user-guide/nginx-configuration/annotations.md) for more details. + +Refer to below section [Configuration of Secure Communication] for more details on how to configure secure communication to Ingress proxy. + ## Configuration ### Configuration global @@ -656,7 +654,7 @@ There are multiple ways to insert your certificate, private key, truststore to t ```bash # Steps to prepare your self-signed certificate ./certs/cert.sh -d /path/to/your/ - # Create TLS Secret with your certificate, private key, truststore + # Create TLS Secret with your certificate, private key, truststore (or a Secret type kubernetes.io/tls) kubectl create secret generic -n $NAMESPACE my-external-tls-secret \ --from-file=tls.crt=/path/to/your/tls.crt \ --from-file=tls.key=/path/to/your/tls.key \ @@ -688,18 +686,16 @@ tls: ``` In additional, if the ingress is enabled, and approach SSL Passthrough is used to ensure the request forwards to the backend components via an encrypted connection. -With `ingress.hostname` is set, the default server TLS secret is also used for hosts TLS secretName when `ingress.tls` is empty. Once you specify `ingress.tls`, your specified secret will be used for hosts TLS secretName. +With `ingress.hostname` is set, the default server TLS secret is also used for hosts TLS secretName when `ingress.tls` is empty. Once you specify `ingress.tls`, your specified secret will be used for hosts TLS secretName. For example ![SeleniumGrid_TLS_SSL-Passthrough](./images/SeleniumGrid_TLS_SSL-Passthrough.png) -Moreover, when sub-chart `ingress-nginx` is enabled (deploy Ingress NGINX Controller together), the default server TLS secret can also be assigned via `ingress-nginx.controller.extraArgs.default-ssl-certificate`. -For example (replace `$RELEASENAME` and `$NAMESPACE` with your values): +```yaml +tls: + enabled: true -```bash -helm upgrade -i $RELEASENAME -n $NAMESPACE docker-selenium/selenium-grid \ - --set tls.enabled=true \ - --set ingress-nginx.enabled=true \ - --set ingress-nginx.controller.extraArgs.default-ssl-certificate=$NAMESPACE/$RELEASENAME-selenium-tls-secret +ingress-ngnix: + enabled: true ``` Below is an example of Grid UI accessible via NodePort with secure connection, and using external TLS Secret (replace `$RELEASENAME` and `$NAMESPACE` with your values): @@ -729,7 +725,7 @@ tls: ![SeleniumGrid_TLS_SSL-Termination](./images/SeleniumGrid_TLS_SSL-Termination.png) -In additional, a self-signed certificate and private key can be generated runtime during the chart deployment for Ingress TLS by setting these values (replace `$RELEASENAME` with your value): +In additional, a self-signed certificate and private key can be generated runtime during the chart deployment for Ingress TLS by setting these values: ```yaml tls: @@ -747,9 +743,6 @@ tls: ingress-ngnix: enabled: true - controller: - extraArgs: - default-ssl-certificate: $(POD_NAMESPACE)/$RELEASENAME-selenium-tls-secret ``` You can get the `tls.crt` and `tls.key` from the Secret after the chart is deployed. For example (replace `$RELEASENAME` and `$NAMESPACE` with your values): @@ -767,12 +760,38 @@ helm upgrade -i $RELEASENAME -n $NAMESPACE docker-selenium/selenium-grid \ --set ingress.hostname="selenium-grid.prod.domain.com" \ --set tls.ingress.enabled=true \ --set tls.nameOverride=my-external-tls-secret \ - --set ingress-nginx.enabled=true \ - --set ingress-nginx.controller.extraArgs.default-ssl-certificate=$NAMESPACE/my-external-tls-secret + --set ingress-nginx.enabled=true ``` Grid UI can be accessed via HTTPS address `https://selenium-grid.prod.domain.com`. +Inline config TLS for the Ingress resource is also considered as enable secure connection to the Ingress proxy. +For example, below is the config with using external TLS Secret for the Ingress resource and enable sub-chart NGINX Ingress Controller: + +```yaml +ingress: + hostname: selenium-grid.prod.domain.com + tls: + - secretName: my-external-tls-secret + hosts: + - selenium-grid.prod.domain.com + +ingress-ngnix: + enabled: true +``` + +In case the Ingress resource is configured without `hostname` and `tls`, the incoming traffic access via `global.K8S_PUBLIC_IP`. When sub-chart `ingress-nginx` is enabled (deploy Ingress NGINX Controller together), the default TLS secret can also be assigned via `ingress-nginx.controller.extraArgs.default-ssl-certificate`. +For example (replace `$RELEASENAME` and `$NAMESPACE` with your values): + +```bash +helm upgrade -i $RELEASENAME -n $NAMESPACE docker-selenium/selenium-grid \ + --set global.K8S_PUBLIC_IP=$(hostname -i) \ + --set tls.ingress.enabled=true \ + --set tls.nameOverride=my-external-tls-secret \ + --set ingress-nginx.enabled=true \ + --set ingress-nginx.controller.extraArgs.default-ssl-certificate=$NAMESPACE/my-external-tls-secret +``` + ### Node Registration To enable secure in the node registration to make sure that the node is one you control and not a rouge node, you can enable and provide a registration secret string to Distributor, Router and diff --git a/charts/selenium-grid/templates/_helpers.tpl b/charts/selenium-grid/templates/_helpers.tpl index ed241d769..c259a40fd 100644 --- a/charts/selenium-grid/templates/_helpers.tpl +++ b/charts/selenium-grid/templates/_helpers.tpl @@ -9,7 +9,7 @@ Server secure connection Ingress proxy forward secure connection */}} {{- define "seleniumGrid.ingress.secureConnection" -}} -{{- or $.Values.tls.enabled $.Values.tls.ingress.enabled $.Values.tls.ingress.generateTLS | ternary "true" "" -}} +{{- or $.Values.tls.enabled $.Values.tls.ingress.enabled $.Values.tls.ingress.generateTLS (not (empty $.Values.ingress.tls)) | ternary "true" "" -}} {{- end -}} {{/* @@ -122,10 +122,21 @@ nginx.ingress.kubernetes.io/proxy-buffers-number: {{ . | quote }} {{- if .websocket }} nginx.org/websocket-services: {{ include ($.Values.isolateComponents | ternary "seleniumGrid.router.fullname" "seleniumGrid.hub.fullname") $ | quote }} {{- end }} -{{- end }} -{{- if eq (include "seleniumGrid.server.secureConnection" $) "true" }} + {{- if eq (include "seleniumGrid.server.secureConnection" $) "true" }} + {{- if .sslPassthrough }} nginx.ingress.kubernetes.io/ssl-passthrough: "true" nginx.ingress.kubernetes.io/backend-protocol: "HTTPS" + {{- end }} + {{- end }} + {{- if eq (include "seleniumGrid.ingress.secureConnection" $) "true" }} + {{- if not (empty .sslSecret) }} +nginx.ingress.kubernetes.io/proxy-ssl-secret: {{ tpl .sslSecret $ | quote }} + {{- else if (empty $.Values.ingress.tls) }} +nginx.ingress.kubernetes.io/proxy-ssl-secret: {{ tpl (printf "%s/%s" $.Release.Namespace (include "seleniumGrid.tls.fullname" $)) $ | quote }} + {{- else }} +nginx.ingress.kubernetes.io/proxy-ssl-secret: {{ tpl (printf "%s/%s" $.Release.Namespace (index $.Values.ingress.tls 0).secretName) $ | quote }} + {{- end }} + {{- end }} {{- end }} {{- end -}} diff --git a/charts/selenium-grid/templates/ingress.yaml b/charts/selenium-grid/templates/ingress.yaml index c45c86fd0..37c54003c 100644 --- a/charts/selenium-grid/templates/ingress.yaml +++ b/charts/selenium-grid/templates/ingress.yaml @@ -36,7 +36,7 @@ spec: tls: - hosts: - {{ tpl .Values.ingress.hostname $ | quote }} - secretName: {{ include "seleniumGrid.tls.fullname" . | quote }} + secretName: {{ include "seleniumGrid.tls.fullname" $ | quote }} {{- else if .Values.ingress.tls }} tls: {{- range .Values.ingress.tls }} @@ -44,7 +44,7 @@ spec: {{- range .hosts }} - {{ tpl . $ | quote }} {{- end }} - secretName: {{ tpl (default "" .secretName) $ | quote }} + secretName: {{ tpl (default (include "seleniumGrid.tls.fullname" $) .secretName) $ | quote }} {{- end }} {{- end }} rules: diff --git a/charts/selenium-grid/templates/jaeger-ingress.yaml b/charts/selenium-grid/templates/jaeger-ingress.yaml index db086138b..54747a8a3 100644 --- a/charts/selenium-grid/templates/jaeger-ingress.yaml +++ b/charts/selenium-grid/templates/jaeger-ingress.yaml @@ -25,16 +25,16 @@ spec: {{- if and (or .Values.tls.enabled .Values.tls.ingress.generateTLS) (tpl .Values.ingress.hostname $) (not .Values.ingress.tls) }} tls: - hosts: - - {{ tpl .Values.ingress.hostname $ | quote }} - secretName: {{ include "seleniumGrid.tls.fullname" . | quote }} + - {{ tpl .Values.ingress.hostname $ }} + secretName: {{ include "seleniumGrid.tls.fullname" $ | quote }} {{- else if .Values.ingress.tls }} tls: {{- range .Values.ingress.tls }} - hosts: {{- range .hosts }} - - {{ tpl . $ | quote }} + - {{ tpl . $ }} {{- end }} - secretName: {{ tpl (.secretName) $ | quote }} + secretName: {{ tpl (default (include "seleniumGrid.tls.fullname" $) .secretName) $ | quote }} {{- end }} {{- end }} rules: diff --git a/charts/selenium-grid/values.yaml b/charts/selenium-grid/values.yaml index 54deb8491..d14704ee5 100644 --- a/charts/selenium-grid/values.yaml +++ b/charts/selenium-grid/values.yaml @@ -121,6 +121,8 @@ ingress: proxyBuffer: size: 512M number: 4 + sslPassthrough: true + sslSecret: "" ports: http: 80 https: 443 diff --git a/tests/SmokeTests/__init__.py b/tests/SmokeTests/__init__.py index 8652eea6d..d427eec72 100644 --- a/tests/SmokeTests/__init__.py +++ b/tests/SmokeTests/__init__.py @@ -2,7 +2,7 @@ import unittest import time import json -from ssl import _create_unverified_context +import ssl import requests from requests.auth import HTTPBasicAuth @@ -50,6 +50,13 @@ def smoke_test_container(self, port): self.assertFalse(status_json['value']['ready'], "Container is autoscaling with min replica set to 0") + def client_verify_cert(self, port): + grid_url_status = '%s://%s:%s/status' % (SELENIUM_GRID_PROTOCOL, SELENIUM_GRID_HOST, port) + cert_path = os.environ.get("REQUESTS_CA_BUNDLE") + response = requests.get(grid_url_status, verify=cert_path) + class GridTest(SmokeTests): def test_grid_is_up(self): self.smoke_test_container('%s' % SELENIUM_GRID_PORT) + if SELENIUM_GRID_PROTOCOL == "https": + self.client_verify_cert('%s' % SELENIUM_GRID_PORT) diff --git a/tests/charts/make/chart_test.sh b/tests/charts/make/chart_test.sh index 263f09d9a..092a41ed2 100755 --- a/tests/charts/make/chart_test.sh +++ b/tests/charts/make/chart_test.sh @@ -213,6 +213,7 @@ fi if [ "${SECURE_USE_EXTERNAL_CERT}" = "true" ]; then HELM_COMMAND_SET_IMAGES="${HELM_COMMAND_SET_IMAGES} \ --set tls.nameOverride=${EXTERNAL_TLS_SECRET_NAME} \ + --set ingress.nginx.sslSecret="${SELENIUM_NAMESPACE}/${EXTERNAL_TLS_SECRET_NAME}" \ " cert_dir="./tests/tests" ADD_IP_ADDRESS=hostname ./${CHART_PATH}/certs/cert.sh -d ${cert_dir} @@ -222,14 +223,29 @@ if [ "${SECURE_USE_EXTERNAL_CERT}" = "true" ]; then CHART_CERT_PATH="./tests/tests/tls.crt" fi -if [ "${SECURE_USE_EXTERNAL_CERT}" = "true" ]; then - HELM_COMMAND_SET_IMAGES="${HELM_COMMAND_SET_IMAGES} \ - --set ingress-nginx.controller.extraArgs.default-ssl-certificate=${SELENIUM_NAMESPACE}/${EXTERNAL_TLS_SECRET_NAME} \ - " -else - HELM_COMMAND_SET_IMAGES="${HELM_COMMAND_SET_IMAGES} \ - --set ingress-nginx.controller.extraArgs.default-ssl-certificate=${SELENIUM_NAMESPACE}/${SELENIUM_TLS_SECRET_NAME} \ - " +if [ "${SECURE_INGRESS_ONLY_CONFIG_INLINE}" = "true" ]; then + if [ "${SECURE_USE_EXTERNAL_CERT}" = "true" ]; then + HELM_COMMAND_SET_IMAGES="${HELM_COMMAND_SET_IMAGES} \ + --set ingress.tls[0].hosts[0]=${SELENIUM_GRID_HOST} \ + --set ingress.tls[0].secretName=${EXTERNAL_TLS_SECRET_NAME} \ + " + else + HELM_COMMAND_SET_IMAGES="${HELM_COMMAND_SET_IMAGES} \ + --set ingress.tls[0].hosts[0]=${SELENIUM_TLS_SECRET_NAME} \ + " + fi +fi + +if [ "${SELENIUM_GRID_PROTOCOL}" = "https" ] && [ "${CHART_ENABLE_INGRESS_HOSTNAME}" != "true" ]; then + if [ "${SECURE_USE_EXTERNAL_CERT}" = "true" ]; then + HELM_COMMAND_SET_IMAGES="${HELM_COMMAND_SET_IMAGES} \ + --set ingress-nginx.controller.extraArgs.default-ssl-certificate=${SELENIUM_NAMESPACE}/${EXTERNAL_TLS_SECRET_NAME} \ + " + else + HELM_COMMAND_SET_IMAGES="${HELM_COMMAND_SET_IMAGES} \ + --set ingress-nginx.controller.extraArgs.default-ssl-certificate=${SELENIUM_NAMESPACE}/${SELENIUM_TLS_SECRET_NAME} \ + " + fi fi if [ "${SELENIUM_GRID_AUTOSCALING}" = "true" ]; then diff --git a/tests/charts/refValues/simplex-minikube.yaml b/tests/charts/refValues/simplex-minikube.yaml index 9fc5162e8..25f722e17 100644 --- a/tests/charts/refValues/simplex-minikube.yaml +++ b/tests/charts/refValues/simplex-minikube.yaml @@ -87,9 +87,6 @@ videoRecorder: ingress-nginx: enabled: true controller: - # Set controller default certificate use the same with Selenium Grid - extraArgs: - default-ssl-certificate: '$(POD_NAMESPACE)/selenium-tls-secret' hostPort: enabled: true kind: DaemonSet