From f2280e985e701d53a1282631b166198752090ebc Mon Sep 17 00:00:00 2001 From: Viet Nguyen Duc Date: Thu, 30 May 2024 07:59:00 +0700 Subject: [PATCH] feat(chart): probe checks for Distributor and Router (#2272) --- charts/selenium-grid/README.md | 53 +++++++++++----- .../configs/distributor/distributorProbe.sh | 39 ++++++++++++ .../configs/router/routerProbe.sh | 27 ++++++++ .../selenium-grid/templates/_nameHelpers.tpl | 14 +++++ .../templates/distributor-configmap.yaml | 31 +++++++++ .../templates/distributor-deployment.yaml | 63 +++++++++++++++++++ .../templates/hub-deployment.yaml | 15 +++++ .../templates/router-configmap.yaml | 30 +++++++++ .../templates/router-deployment.yaml | 13 ++++ .../templates/server-configmap.yaml | 3 + charts/selenium-grid/values.yaml | 61 ++++++++++++++++++ generate_release_notes.sh | 7 +-- tests/SeleniumTests/__init__.py | 2 +- tests/charts/ci/base-resources-values.yaml | 6 ++ tests/charts/make/chart_test.sh | 21 +++++-- 15 files changed, 360 insertions(+), 25 deletions(-) create mode 100644 charts/selenium-grid/configs/distributor/distributorProbe.sh create mode 100644 charts/selenium-grid/configs/router/routerProbe.sh create mode 100644 charts/selenium-grid/templates/distributor-configmap.yaml create mode 100644 charts/selenium-grid/templates/router-configmap.yaml diff --git a/charts/selenium-grid/README.md b/charts/selenium-grid/README.md index 12eaf9b8a..d12d3b6e1 100644 --- a/charts/selenium-grid/README.md +++ b/charts/selenium-grid/README.md @@ -23,7 +23,10 @@ This chart enables the creation of a Selenium Grid Server in Kubernetes. * [Configuration `global.K8S_PUBLIC_IP`](#configuration-globalk8s_public_ip) * [Configuration of Nodes](#configuration-of-nodes) * [Container ports and Service ports](#container-ports-and-service-ports) - * [Probes](#probes) + * [Configuration of Probes](#configuration-of-probes) + * [Node Probes](#node-probes) + * [Distributor Probes](#distributor-probes) + * [Router Probes](#router-probes) * [Configuration extra scripts mount to container](#configuration-extra-scripts-mount-to-container) * [Configuration of video recorder and video uploader](#configuration-of-video-recorder-and-video-uploader) * [Video recorder](#video-recorder) @@ -299,20 +302,20 @@ ingress-nginx: ### Configuration global For now, global configuration supported is: -| Parameter | Default | Description | -|------------------------------------------------|-------------------------|------------------------------------------| -| `global.K8S_PUBLIC_IP` | `""` | Public IP of the host running K8s | -| `global.seleniumGrid.imageRegistry` | `selenium` | Distribution registry to pull images | -| `global.seleniumGrid.imageTag` | `4.21.0-20240522` | Image tag for all selenium components | -| `global.seleniumGrid.nodesImageTag` | `4.21.0-20240522` | Image tag for browser's nodes | -| `global.seleniumGrid.videoImageTag` | `ffmpeg-6.1.1-20240522` | Image tag for browser's video recorder | -| `global.seleniumGrid.imagePullSecret` | `""` | Pull secret to be used for all images | -| `global.seleniumGrid.imagePullSecret` | `""` | Pull secret to be used for all images | -| `global.seleniumGrid.affinity` | `{}` | Affinity assigned globally | -| `global.seleniumGrid.logLevel` | `INFO` | Set log level for all components | -| `global.seleniumGrid.defaultNodeStartupProbe` | `exec` | Default startup probe method in Nodes | -| `global.seleniumGrid.defaultNodeLivenessProbe` | `exec` | Default liveness probe method in Nodes | -| `global.seleniumGrid.stdoutProbeLog` | `true` | Enable probe logs output in kubectl logs | +| Parameter | Default | Description | +|-----------------------------------------------------|-------------------------|---------------------------------------------| +| `global.K8S_PUBLIC_IP` | `""` | Public IP of the host running K8s | +| `global.seleniumGrid.imageRegistry` | `selenium` | Distribution registry to pull images | +| `global.seleniumGrid.imageTag` | `4.21.0-20240522` | Image tag for all selenium components | +| `global.seleniumGrid.nodesImageTag` | `4.21.0-20240522` | Image tag for browser's nodes | +| `global.seleniumGrid.videoImageTag` | `ffmpeg-6.1.1-20240522` | Image tag for browser's video recorder | +| `global.seleniumGrid.imagePullSecret` | `""` | Pull secret to be used for all images | +| `global.seleniumGrid.affinity` | `{}` | Affinity assigned globally | +| `global.seleniumGrid.logLevel` | `INFO` | Set log level for all components | +| `global.seleniumGrid.defaultNodeStartupProbe` | `exec` | Default startup probe method in Nodes | +| `global.seleniumGrid.defaultNodeLivenessProbe` | `exec` | Default liveness probe method in Nodes | +| `global.seleniumGrid.defaultComponentLivenessProbe` | `exec` | Default liveness probe method in Components | +| `global.seleniumGrid.stdoutProbeLog` | `true` | Enable probe logs output in kubectl logs | #### Configuration `global.K8S_PUBLIC_IP` @@ -379,7 +382,9 @@ edgeNode: protocol: TCP ``` -#### Probes +### Configuration of Probes + +#### Node Probes By default, `startupProbe` is enabled and `readinessProbe` and `livenessProbe` are disabled. You can enable/disable them via `.startupProbe.enabled` `.readinessProbe.enabled` `.livenessProbe.enabled` in respective node type. @@ -411,6 +416,22 @@ edgeNode: periodSeconds: 5 ``` +#### Distributor Probes + +By default, `startupProbe`, `readinessProbe` and `livenessProbe` are enabled for this component in both full distributed and Hub-Nodes mode. + +There is a script in chart `configs/distributor/distributorProbe.sh` is loaded into ConfigMap and mounted to the container is used by `livenessProbe`. You can customize the script via `--set-file distributorConfigMap.extraScripts.distributorProbe\.sh=/path/to/your_script.sh` or set via YAML values. + +There are some reports on a scenario that would be difficult to reproduce or rare: `Grid UI is accessible but no nodes can be fetched or registered. Or something like there are few requests in session queue but could not be accepted. After restarting the Distributor, the issue is resolved`. Based on that, a proactive approach to do automatic restart whenever detecting it is not healthy via `livenessProbe` and the condition check is executed. The script queries GraphQL endpoint to get `sessionCount`, and `sessionQueueSize`. If the `sessionQueueSize` is greater than 0 and `sessionCount` is 0 until the `failureThreshold`, the Distributor will be restarted. You can adjust the threshold as well as interval via probe settings. + +#### Router Probes + +By default, `startupProbe`, `readinessProbe` and `livenessProbe` are enabled for this component in full distributed mode. + +There is a script in chart `configs/router/routerProbe.sh` loaded into ConfigMap and mounted to the container is used by `livenessProbe`. You can customize the script via `--set-file routerConfigMap.extraScripts.routerProbe\.sh=/path/to/your_script.sh` or set via YAML values. + +The script checks GraphQL endpoint is reachable. If the `http_code` is not `200` until the `failureThreshold`, the Router will be restarted. You can adjust the threshold as well as interval via probe settings. + ### Configuration extra scripts mount to container This is supported for containers of browser node, video recorder and video uploader. By default, in these containers, there are scripts, config files implemented. In case you want to customize or replace them with your own implementation. Instead of forking the chart, use volume mount. Now, from your external files, you can insert them into ConfigMap via Helm CLI `--set-file` or compose them in your own YAML values file and pass to Helm CLI `--values` when deploying chart. Any files name that you defined will be picked up into ConfigMap and mounted to the container. diff --git a/charts/selenium-grid/configs/distributor/distributorProbe.sh b/charts/selenium-grid/configs/distributor/distributorProbe.sh new file mode 100644 index 000000000..2ff44c527 --- /dev/null +++ b/charts/selenium-grid/configs/distributor/distributorProbe.sh @@ -0,0 +1,39 @@ +#!/bin/bash + +max_time=3 +retry_time=3 +probe_name="Probe.${1:-"Liveness"}" +ts_format=${SE_LOG_TIMESTAMP_FORMAT:-"+%T.%3N"} + +if [ -n "${ROUTER_USERNAME}" ] && [ -n "${ROUTER_PASSWORD}" ]; then + BASIC_AUTH="${ROUTER_USERNAME}:${ROUTER_PASSWORD}@" +fi + +if [ -z "${SE_GRID_GRAPHQL_URL}" ] && [ -n "${SE_HUB_HOST:-${SE_ROUTER_HOST}}" ] && [ -n "${SE_HUB_PORT:-${SE_ROUTER_PORT}}" ]; then + SE_GRID_GRAPHQL_URL="${SE_SERVER_PROTOCOL}://${BASIC_AUTH}${SE_HUB_HOST:-${SE_ROUTER_HOST}}:${SE_HUB_PORT:-${SE_ROUTER_PORT}}${SE_SUB_PATH}/graphql" +elif [ -z "${SE_GRID_GRAPHQL_URL}" ]; then + echo "$(date ${ts_format}) DEBUG [${probe_name}] - Could not construct GraphQL endpoint, it can be set directly via SE_GRID_GRAPHQL_URL. Bypass the probe checks for now." + exit 0 +fi + +GRAPHQL_PRE_CHECK=$(curl --noproxy "*" -m ${max_time} -k -X POST -H "Content-Type: application/json" --data '{"query":"{ grid { sessionCount } }"}' -s -o /dev/null -w "%{http_code}" ${SE_GRID_GRAPHQL_URL}) + +if [ ${GRAPHQL_PRE_CHECK} -ne 200 ]; then + echo "$(date ${ts_format}) DEBUG [${probe_name}] - GraphQL endpoint ${SE_GRID_GRAPHQL_URL} is not reachable. Status code: ${GRAPHQL_PRE_CHECK}." + exit 1 +fi + +SESSION_QUEUE_SIZE=$(curl --noproxy "*" --retry ${retry_time} -m ${max_time} -k -X POST -H "Content-Type: application/json" --data '{"query":"{ grid { sessionQueueSize } }"}' -s ${SE_GRID_GRAPHQL_URL} | jq -r '.data.grid.sessionQueueSize') + +SESSION_COUNT=$(curl --noproxy "*" --retry ${retry_time} -m ${max_time} -k -X POST -H "Content-Type: application/json" --data '{"query": "{ grid { sessionCount } }"}' -s ${SE_GRID_GRAPHQL_URL} | jq -r '.data.grid.sessionCount') + +MAX_SESSION=$(curl --noproxy "*" --retry ${retry_time} -m ${max_time} -k -X POST -H "Content-Type: application/json" --data '{"query":"{ grid { maxSession } }"}' -s ${SE_GRID_GRAPHQL_URL} | jq -r '.data.grid.maxSession') + +if [ ${SESSION_QUEUE_SIZE} -gt 0 ] && [ ${SESSION_COUNT} -eq 0 ]; then + echo "$(date ${ts_format}) DEBUG [${probe_name}] - Session Queue Size: ${SESSION_QUEUE_SIZE}, Session Count: ${SESSION_COUNT}, Max Session: ${MAX_SESSION}" + echo "$(date ${ts_format}) DEBUG [${probe_name}] - It seems the Distributor is delayed in processing a new session in the queue. Probe checks failed." + exit 1 +else + echo "$(date ${ts_format}) DEBUG [${probe_name}] - Distributor is healthy." + exit 0 +fi diff --git a/charts/selenium-grid/configs/router/routerProbe.sh b/charts/selenium-grid/configs/router/routerProbe.sh new file mode 100644 index 000000000..167d78721 --- /dev/null +++ b/charts/selenium-grid/configs/router/routerProbe.sh @@ -0,0 +1,27 @@ +#!/bin/bash + +max_time=3 +retry_time=3 +probe_name="Probe.${1:-"Liveness"}" +ts_format=${SE_LOG_TIMESTAMP_FORMAT:-"+%T.%3N"} + +if [ -n "${ROUTER_USERNAME}" ] && [ -n "${ROUTER_PASSWORD}" ]; then + BASIC_AUTH="${ROUTER_USERNAME}:${ROUTER_PASSWORD}@" +fi + +if [ -z "${SE_GRID_GRAPHQL_URL}" ] && [ -n "${SE_HUB_HOST:-${SE_ROUTER_HOST}}" ] && [ -n "${SE_HUB_PORT:-${SE_ROUTER_PORT}}" ]; then + SE_GRID_GRAPHQL_URL="${SE_SERVER_PROTOCOL}://${BASIC_AUTH}${SE_HUB_HOST:-${SE_ROUTER_HOST}}:${SE_HUB_PORT:-${SE_ROUTER_PORT}}${SE_SUB_PATH}/graphql" +elif [ -z "${SE_GRID_GRAPHQL_URL}" ]; then + echo "$(date ${ts_format}) DEBUG [${probe_name}] - Could not construct GraphQL endpoint, it can be set directly via SE_GRID_GRAPHQL_URL. Bypass the probe checks for now." + exit 0 +fi + +GRAPHQL_PRE_CHECK=$(curl --noproxy "*" -m ${max_time} -k -X POST -H "Content-Type: application/json" --data '{"query":"{ grid { sessionCount } }"}' -s -o /dev/null -w "%{http_code}" ${SE_GRID_GRAPHQL_URL}) + +if [ ${GRAPHQL_PRE_CHECK} -ne 200 ]; then + echo "$(date ${ts_format}) DEBUG [${probe_name}] - GraphQL endpoint ${SE_GRID_GRAPHQL_URL} is not reachable. Status code: ${GRAPHQL_PRE_CHECK}." + exit 1 +else + echo "$(date ${ts_format}) DEBUG [${probe_name}] - GraphQL endpoint is healthy." + exit 0 +fi diff --git a/charts/selenium-grid/templates/_nameHelpers.tpl b/charts/selenium-grid/templates/_nameHelpers.tpl index 055e7559a..fd9916a9a 100644 --- a/charts/selenium-grid/templates/_nameHelpers.tpl +++ b/charts/selenium-grid/templates/_nameHelpers.tpl @@ -154,6 +154,20 @@ Service Account fullname {{- tpl (default (include "seleniumGrid.component.name" (list "selenium-serviceaccount" $)) .Values.serviceAccount.nameOverride) $ | trunc 63 | trimSuffix "-" -}} {{- end -}} +{{/* +Distributor ConfigMap fullname +*/}} +{{- define "seleniumGrid.distributor.configmap.fullname" -}} +{{- tpl (default (include "seleniumGrid.component.name" (list "selenium-distributor-config" $)) .Values.distributorConfigMap.nameOverride) $ | trunc 63 | trimSuffix "-" -}} +{{- end -}} + +{{/* +Router ConfigMap fullname +*/}} +{{- define "seleniumGrid.router.configmap.fullname" -}} +{{- tpl (default (include "seleniumGrid.component.name" (list "selenium-router-config" $)) .Values.routerConfigMap.nameOverride) $ | trunc 63 | trimSuffix "-" -}} +{{- end -}} + {{/* Recorder ConfigMap fullname */}} diff --git a/charts/selenium-grid/templates/distributor-configmap.yaml b/charts/selenium-grid/templates/distributor-configmap.yaml new file mode 100644 index 000000000..0aea7d9dc --- /dev/null +++ b/charts/selenium-grid/templates/distributor-configmap.yaml @@ -0,0 +1,31 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ template "seleniumGrid.distributor.configmap.fullname" $ }} + namespace: {{ .Release.Namespace }} +{{- with .Values.distributorConfigMap.annotations }} + annotations: {{- toYaml . | nindent 4 }} +{{- end }} + labels: + {{- include "seleniumGrid.commonLabels" . | nindent 4 }} + {{- with .Values.customLabels }} + {{- toYaml . | nindent 4 }} + {{- end }} +data: + SE_GRID_GRAPHQL_URL: '{{ include "seleniumGrid.graphqlURL" $ }}' +{{- $fileProceeded := list -}} +{{- range $path, $_ := .Files.Glob $.Values.distributorConfigMap.extraScriptsImportFrom }} + {{- $fileName := base $path -}} + {{- $value := index $.Values.distributorConfigMap.extraScripts $fileName -}} + {{- if empty $value }} +{{- $fileName | nindent 2 -}}: {{- toYaml ($.Files.Get $path) | indent 4 }} + {{- else }} +{{- $fileName | nindent 2 -}}: {{- toYaml $value | indent 4 }} + {{- end }} + {{- $fileProceeded = append $fileProceeded $fileName -}} +{{- end }} +{{- range $fileName, $value := .Values.distributorConfigMap.extraScripts }} + {{- if not (has $fileName $fileProceeded) }} +{{- $fileName | nindent 2 -}}: {{- toYaml (default "" $value) | indent 4 }} + {{- end }} +{{- end }} diff --git a/charts/selenium-grid/templates/distributor-deployment.yaml b/charts/selenium-grid/templates/distributor-deployment.yaml index ec0994fe4..f727581b1 100644 --- a/charts/selenium-grid/templates/distributor-deployment.yaml +++ b/charts/selenium-grid/templates/distributor-deployment.yaml @@ -24,6 +24,7 @@ spec: checksum/event-bus-configmap: {{ include (print $.Template.BasePath "/event-bus-configmap.yaml") . | sha256sum }} checksum/logging-configmap: {{ include (print $.Template.BasePath "/logging-configmap.yaml") . | sha256sum }} checksum/server-configmap: {{ include (print $.Template.BasePath "/server-configmap.yaml") . | sha256sum }} + checksum/distributor-configmap: {{ include (print $.Template.BasePath "/distributor-configmap.yaml") . | sha256sum }} checksum/secrets: {{ include (print $.Template.BasePath "/secrets.yaml") . | sha256sum }} {{- with .Values.components.distributor.annotations }} {{- toYaml . | nindent 8 }} @@ -63,6 +64,8 @@ spec: {{- tpl (toYaml .) $ | nindent 12 }} {{- end }} envFrom: + - configMapRef: + name: {{ template "seleniumGrid.distributor.configmap.fullname" . }} - configMapRef: name: {{ template "seleniumGrid.eventBus.configmap.fullname" . }} - configMapRef: @@ -75,6 +78,11 @@ spec: {{- toYaml . | nindent 12 }} {{- end }} volumeMounts: + {{- range $fileName, $value := $.Values.distributorConfigMap.extraScripts }} + - name: {{ tpl (default (include "seleniumGrid.distributor.configmap.fullname" $) $.Values.distributorConfigMap.scriptVolumeMountName) $ | quote }} + mountPath: {{ $.Values.distributorConfigMap.extraScriptsDirectory }}/{{ $fileName }} + subPath: {{ $fileName }} + {{- end }} {{- if .Values.tls.enabled }} - name: {{ include "seleniumGrid.tls.fullname" . | quote }} mountPath: {{ .Values.serverConfigMap.certVolumeMountPath | quote }} @@ -83,6 +91,57 @@ spec: ports: - containerPort: {{ .Values.components.distributor.port }} protocol: TCP + {{- if .Values.components.distributor.startupProbe.enabled }} + {{- with .Values.components.distributor.startupProbe }} + startupProbe: + {{- if (ne (include "seleniumGrid.probe.fromUserDefine" (dict "values" . "root" $)) "{}") }} + {{- include "seleniumGrid.probe.fromUserDefine" (dict "values" . "root" $) | nindent 10 }} + {{- else }} + httpGet: + scheme: {{ default (include "seleniumGrid.probe.httpGet.schema" $) .schema }} + path: {{ .path }} + port: {{ default ($.Values.components.distributor.port) .port }} + {{- end }} + {{- if (ne (include "seleniumGrid.probe.settings" .) "{}") }} + {{- include "seleniumGrid.probe.settings" . | nindent 12 }} + {{- end }} + {{- end }} + {{- end }} + {{- if .Values.components.distributor.readinessProbe.enabled }} + {{- with .Values.components.distributor.readinessProbe }} + readinessProbe: + {{- if (ne (include "seleniumGrid.probe.fromUserDefine" (dict "values" . "root" $)) "{}") }} + {{- include "seleniumGrid.probe.fromUserDefine" (dict "values" . "root" $) | nindent 10 }} + {{- else }} + httpGet: + scheme: {{ default (include "seleniumGrid.probe.httpGet.schema" $) .schema }} + path: {{ .path }} + port: {{ default ($.Values.components.distributor.port) .port }} + {{- end }} + {{- if (ne (include "seleniumGrid.probe.settings" .) "{}") }} + {{- include "seleniumGrid.probe.settings" . | nindent 12 }} + {{- end }} + {{- end }} + {{- end }} + {{- if .Values.components.distributor.livenessProbe.enabled }} + {{- with .Values.components.distributor.livenessProbe }} + livenessProbe: + {{- if (ne (include "seleniumGrid.probe.fromUserDefine" (dict "values" . "root" $)) "{}") }} + {{- include "seleniumGrid.probe.fromUserDefine" (dict "values" . "root" $) | nindent 10 }} + {{- else if eq $.Values.global.seleniumGrid.defaultComponentLivenessProbe "exec" }} + exec: + command: ["bash", "-c", "{{ $.Values.distributorConfigMap.extraScriptsDirectory }}/distributorProbe.sh Liveness {{ include "seleniumGrid.probe.stdout" $ }}"] + {{- else }} + httpGet: + scheme: {{ default (include "seleniumGrid.probe.httpGet.schema" $) .schema }} + path: {{ .path }} + port: {{ default ($.Values.components.distributor.port) .port }} + {{- end }} + {{- if (ne (include "seleniumGrid.probe.settings" .) "{}") }} + {{- include "seleniumGrid.probe.settings" . | nindent 12 }} + {{- end }} + {{- end }} + {{- end }} {{- with .Values.components.distributor.resources }} resources: {{- toYaml . | nindent 12 }} {{- end }} @@ -107,6 +166,10 @@ spec: priorityClassName: {{ . }} {{- end }} volumes: + - name: {{ tpl (default (include "seleniumGrid.distributor.configmap.fullname" $) $.Values.distributorConfigMap.scriptVolumeMountName) $ | quote }} + configMap: + name: {{ template "seleniumGrid.distributor.configmap.fullname" $ }} + defaultMode: {{ $.Values.distributorConfigMap.defaultMode }} {{- if .Values.tls.enabled }} - name: {{ include "seleniumGrid.tls.fullname" . | quote }} secret: diff --git a/charts/selenium-grid/templates/hub-deployment.yaml b/charts/selenium-grid/templates/hub-deployment.yaml index c611d62e4..0a231e8a2 100644 --- a/charts/selenium-grid/templates/hub-deployment.yaml +++ b/charts/selenium-grid/templates/hub-deployment.yaml @@ -26,6 +26,7 @@ spec: annotations: checksum/logging-configmap: {{ include (print $.Template.BasePath "/logging-configmap.yaml") . | sha256sum }} checksum/server-configmap: {{ include (print $.Template.BasePath "/server-configmap.yaml") . | sha256sum }} + checksum/distributor-configmap: {{ include (print $.Template.BasePath "/distributor-configmap.yaml") . | sha256sum }} checksum/secrets: {{ include (print $.Template.BasePath "/secrets.yaml") . | sha256sum }} {{- with .Values.hub.annotations }} {{- toYaml . | nindent 8 }} @@ -83,6 +84,9 @@ spec: livenessProbe: {{- if (ne (include "seleniumGrid.probe.fromUserDefine" (dict "values" . "root" $)) "{}") }} {{- include "seleniumGrid.probe.fromUserDefine" (dict "values" . "root" $) | nindent 10 }} + {{- else if eq $.Values.global.seleniumGrid.defaultComponentLivenessProbe "exec" }} + exec: + command: ["bash", "-c", "{{ $.Values.distributorConfigMap.extraScriptsDirectory }}/distributorProbe.sh Liveness {{ include "seleniumGrid.probe.stdout" $ }}"] {{- else }} httpGet: scheme: {{ default (include "seleniumGrid.probe.httpGet.schema" $) .schema }} @@ -120,6 +124,8 @@ spec: {{- tpl (toYaml .) $ | nindent 12 }} {{- end }} envFrom: + - configMapRef: + name: {{ template "seleniumGrid.distributor.configmap.fullname" . }} - configMapRef: name: {{ template "seleniumGrid.logging.configmap.fullname" . }} - configMapRef: @@ -130,6 +136,11 @@ spec: {{- toYaml . | nindent 12 }} {{- end }} volumeMounts: + {{- range $fileName, $value := $.Values.distributorConfigMap.extraScripts }} + - name: {{ tpl (default (include "seleniumGrid.distributor.configmap.fullname" $) $.Values.distributorConfigMap.scriptVolumeMountName) $ }} + mountPath: {{ $.Values.distributorConfigMap.extraScriptsDirectory }}/{{ $fileName }} + subPath: {{ $fileName }} + {{- end }} {{- if .Values.tls.enabled }} - name: {{ include "seleniumGrid.tls.fullname" . | quote }} mountPath: {{ .Values.serverConfigMap.certVolumeMountPath | quote }} @@ -162,6 +173,10 @@ spec: priorityClassName: {{ . }} {{- end }} volumes: + - name: {{ tpl (default (include "seleniumGrid.distributor.configmap.fullname" $) $.Values.distributorConfigMap.scriptVolumeMountName) $ | quote }} + configMap: + name: {{ template "seleniumGrid.distributor.configmap.fullname" $ }} + defaultMode: {{ $.Values.distributorConfigMap.defaultMode }} {{- if .Values.tls.enabled }} - name: {{ include "seleniumGrid.tls.fullname" . | quote }} secret: diff --git a/charts/selenium-grid/templates/router-configmap.yaml b/charts/selenium-grid/templates/router-configmap.yaml new file mode 100644 index 000000000..bb8ec158a --- /dev/null +++ b/charts/selenium-grid/templates/router-configmap.yaml @@ -0,0 +1,30 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ template "seleniumGrid.router.configmap.fullname" $ }} + namespace: {{ .Release.Namespace }} +{{- with .Values.routerConfigMap.annotations }} + annotations: {{- toYaml . | nindent 4 }} +{{- end }} + labels: + {{- include "seleniumGrid.commonLabels" . | nindent 4 }} + {{- with .Values.customLabels }} + {{- toYaml . | nindent 4 }} + {{- end }} +data: +{{- $fileProceeded := list -}} +{{- range $path, $_ := .Files.Glob $.Values.routerConfigMap.extraScriptsImportFrom }} + {{- $fileName := base $path -}} + {{- $value := index $.Values.routerConfigMap.extraScripts $fileName -}} + {{- if empty $value }} +{{- $fileName | nindent 2 -}}: {{- toYaml ($.Files.Get $path) | indent 4 }} + {{- else }} +{{- $fileName | nindent 2 -}}: {{- toYaml $value | indent 4 }} + {{- end }} + {{- $fileProceeded = append $fileProceeded $fileName -}} +{{- end }} +{{- range $fileName, $value := .Values.routerConfigMap.extraScripts }} + {{- if not (has $fileName $fileProceeded) }} +{{- $fileName | nindent 2 -}}: {{- toYaml (default "" $value) | indent 4 }} + {{- end }} +{{- end }} diff --git a/charts/selenium-grid/templates/router-deployment.yaml b/charts/selenium-grid/templates/router-deployment.yaml index 8b57395ae..0c76bbfc7 100644 --- a/charts/selenium-grid/templates/router-deployment.yaml +++ b/charts/selenium-grid/templates/router-deployment.yaml @@ -23,6 +23,7 @@ spec: annotations: checksum/logging-configmap: {{ include (print $.Template.BasePath "/logging-configmap.yaml") . | sha256sum }} checksum/server-configmap: {{ include (print $.Template.BasePath "/server-configmap.yaml") . | sha256sum }} + checksum/router-configmap: {{ include (print $.Template.BasePath "/router-configmap.yaml") . | sha256sum }} checksum/secrets: {{ include (print $.Template.BasePath "/secrets.yaml") . | sha256sum }} {{- with .Values.components.router.annotations }} {{- toYaml . | nindent 8 }} @@ -77,6 +78,11 @@ spec: {{- toYaml . | nindent 12 }} {{- end }} volumeMounts: + {{- range $fileName, $value := $.Values.routerConfigMap.extraScripts }} + - name: {{ tpl (default (include "seleniumGrid.router.configmap.fullname" $) $.Values.routerConfigMap.scriptVolumeMountName) $ | quote }} + mountPath: {{ $.Values.routerConfigMap.extraScriptsDirectory }}/{{ $fileName }} + subPath: {{ $fileName }} + {{- end }} {{- if .Values.tls.enabled }} - name: {{ include "seleniumGrid.tls.fullname" . | quote }} mountPath: {{ .Values.serverConfigMap.certVolumeMountPath | quote }} @@ -122,6 +128,9 @@ spec: {{- with .Values.components.router.livenessProbe }} {{- if (ne (include "seleniumGrid.probe.fromUserDefine" (dict "values" . "root" $)) "{}") }} {{- include "seleniumGrid.probe.fromUserDefine" (dict "values" . "root" $) | nindent 10 }} + {{- else if eq $.Values.global.seleniumGrid.defaultComponentLivenessProbe "exec" }} + exec: + command: ["bash", "-c", "{{ $.Values.routerConfigMap.extraScriptsDirectory }}/routerProbe.sh Liveness {{ include "seleniumGrid.probe.stdout" $ }}"] {{- else }} httpGet: scheme: {{ default (include "seleniumGrid.probe.httpGet.schema" $) .schema }} @@ -157,6 +166,10 @@ spec: priorityClassName: {{ . }} {{- end }} volumes: + - name: {{ tpl (default (include "seleniumGrid.router.configmap.fullname" $) $.Values.routerConfigMap.scriptVolumeMountName) $ | quote }} + configMap: + name: {{ template "seleniumGrid.router.configmap.fullname" $ }} + defaultMode: {{ $.Values.routerConfigMap.defaultMode }} {{- if .Values.tls.enabled }} - name: {{ include "seleniumGrid.tls.fullname" . | quote }} secret: diff --git a/charts/selenium-grid/templates/server-configmap.yaml b/charts/selenium-grid/templates/server-configmap.yaml index fbf161195..b72e9c43d 100644 --- a/charts/selenium-grid/templates/server-configmap.yaml +++ b/charts/selenium-grid/templates/server-configmap.yaml @@ -13,6 +13,9 @@ metadata: {{- end }} data: SE_SERVER_PROTOCOL: {{ include "seleniumGrid.server.protocol" . | quote }} +{{- with $.Values.serverConfigMap.env }} + {{- toYaml . | nindent 2 }} +{{- end }} {{- if .Values.tls.enabled }} SE_HTTPS_CERTIFICATE: {{ printf "%s/%s" .Values.serverConfigMap.certVolumeMountPath .Values.serverConfigMap.certificateFile | quote }} SE_HTTPS_PRIVATE_KEY: {{ printf "%s/%s" .Values.serverConfigMap.certVolumeMountPath .Values.serverConfigMap.privateKeyFile | quote }} diff --git a/charts/selenium-grid/values.yaml b/charts/selenium-grid/values.yaml index 1cb16462f..c1b2686e1 100644 --- a/charts/selenium-grid/values.yaml +++ b/charts/selenium-grid/values.yaml @@ -23,6 +23,8 @@ global: defaultNodeStartupProbe: exec # liveness probe method `exec.command` is using a script is mounted from `nodeConfigMap.extraScripts.nodeProbe.sh` defaultNodeLivenessProbe: exec + # liveness probe method `exec.command` is using a script is mounted from ConfigMap to component container + defaultComponentLivenessProbe: exec # probe logs output can be retrieved using `kubectl logs` stdoutProbeLog: false @@ -135,6 +137,36 @@ busConfigMap: # Custom annotations for configmap annotations: {} +distributorConfigMap: + # nameOverride: + # Default mode for ConfigMap is mounted as file + defaultMode: 0755 + # Directory where the extra scripts are imported to ConfigMap by default (if given a relative path, it should be in chart's directory) + extraScriptsImportFrom: "configs/distributor/**" + # Directory where the extra scripts are mounted to + extraScriptsDirectory: "/opt/selenium" + extraScripts: + distributorProbe.sh: "" + # Name of volume mount is used to mount scripts in the ConfigMap + scriptVolumeMountName: + # Custom annotations for configmap + annotations: {} + +routerConfigMap: + # nameOverride: + # Default mode for ConfigMap is mounted as file + defaultMode: 0755 + # Directory where the extra scripts are imported to ConfigMap by default (if given a relative path, it should be in chart's directory) + extraScriptsImportFrom: "configs/router/**" + # Directory where the extra scripts are mounted to + extraScriptsDirectory: "/opt/selenium" + extraScripts: + routerProbe.sh: "" + # Name of volume mount is used to mount scripts in the ConfigMap + scriptVolumeMountName: + # Custom annotations for configmap + annotations: {} + # ConfigMap that contains common environment variables for browser nodes nodeConfigMap: # nameOverride: @@ -212,6 +244,8 @@ serverConfigMap: trustStoreFile: selenium.jks # Disable verification the hostname included in the server's TLS/SSL certificates matches the hostnames provided disableHostnameVerification: true + env: + SE_JAVA_OPTS: "-XX:+UseZGC" # Custom annotations for configmap annotations: {} @@ -310,6 +344,33 @@ components: # Distributor port port: 5553 nodePort: 30553 + # Wait for pod startup + startupProbe: + enabled: true + path: /readyz + initialDelaySeconds: 5 + failureThreshold: 10 + timeoutSeconds: 10 + periodSeconds: 10 + successThreshold: 1 + # Readiness probe settings + readinessProbe: + enabled: true + path: /readyz + initialDelaySeconds: 12 + failureThreshold: 10 + timeoutSeconds: 10 + periodSeconds: 10 + successThreshold: 1 + # Liveness probe settings + livenessProbe: + enabled: true + path: /readyz + initialDelaySeconds: 10 + failureThreshold: 10 + timeoutSeconds: 10 + periodSeconds: 10 + successThreshold: 1 # Resources for Distributor container resources: {} # SecurityContext for Distributor container diff --git a/generate_release_notes.sh b/generate_release_notes.sh index f531b6a59..f0aeda948 100755 --- a/generate_release_notes.sh +++ b/generate_release_notes.sh @@ -25,7 +25,7 @@ RCLONE_VERSION=$(docker run --entrypoint="" --rm ${NAMESPACE}/video:${FFMPEG_TAG JRE_VERSION=$(docker run --entrypoint="" --rm ${NAMESPACE}/base:${TAG_VERSION} java --version | grep -oP '\b\d+\.\d+\.\d+\b' | head -1) FIREFOX_ARM64_VERSION=$(docker run --rm --platform linux/arm64 ${NAMESPACE}/node-firefox:${TAG_VERSION} firefox --version | awk '{print $3}') CHROMIUM_VERSION=$(docker run --rm ${NAMESPACE}/node-chromium:${TAG_VERSION} chromium --version | awk '{print $2}') -CHROMIUMDRIVER_VERSION=$(docker run --rm ${NAMESPACE}/node-chrome:${TAG_VERSION} chromedriver --version | awk '{print $2}') +CHROMIUMDRIVER_VERSION=$(docker run --rm ${NAMESPACE}/node-chromium:${TAG_VERSION} chromedriver --version | awk '{print $2}') echo "" >> release_notes.md @@ -33,12 +33,11 @@ echo "### Released versions" >> release_notes.md echo "| Components | x86_64 (amd64) | aarch64 (arm64/armv8) |" >> release_notes.md echo "|:----------:|:--------------:|:---------------------:|" >> release_notes.md echo "| Selenium | ${GRID_VERSION} | ${GRID_VERSION} |" >> release_notes.md +echo "| Chromium | ${CHROMIUM_VERSION} | ${CHROMIUM_VERSION} |" >> release_notes.md echo "| Chrome | ${CHROME_VERSION} | x |" >> release_notes.md -echo "| ChromeDriver | ${CHROMEDRIVER_VERSION} | x |" >> release_notes.md +echo "| ChromeDriver | ${CHROMEDRIVER_VERSION} | ${CHROMIUMDRIVER_VERSION} |" >> release_notes.md echo "| Edge | ${EDGE_VERSION} | x |" >> release_notes.md echo "| EdgeDriver | ${EDGEDRIVER_VERSION} | x |" >> release_notes.md -echo "| Chromium | ${CHROMIUM_VERSION} | ${CHROMIUM_VERSION} |" >> release_notes.md -echo "| ChromiumDriver | ${CHROMIUMDRIVER_VERSION} | ${CHROMIUMDRIVER_VERSION} |" >> release_notes.md echo "| Firefox | ${FIREFOX_VERSION} | ${FIREFOX_ARM64_VERSION} |" >> release_notes.md echo "| GeckoDriver | ${GECKODRIVER_VERSION} | ${GECKODRIVER_VERSION} |" >> release_notes.md echo "| ffmpeg | ${FFMPEG_VERSION} | ${FFMPEG_VERSION} |" >> release_notes.md diff --git a/tests/SeleniumTests/__init__.py b/tests/SeleniumTests/__init__.py index 4273e2685..1a3f8e271 100644 --- a/tests/SeleniumTests/__init__.py +++ b/tests/SeleniumTests/__init__.py @@ -23,7 +23,7 @@ TEST_DELAY_AFTER_TEST = int(os.environ.get('TEST_DELAY_AFTER_TEST', 0)) TEST_NODE_RELAY = os.environ.get('TEST_NODE_RELAY', 'false') TEST_ANDROID_PLATFORM_API = os.environ.get('ANDROID_PLATFORM_API') -TEST_PLATFORMS = os.environ.get('TEST_PLATFORMS', 'linux/adm64') +TEST_PLATFORMS = os.environ.get('TEST_PLATFORMS', 'linux/amd64') if SELENIUM_GRID_USERNAME and SELENIUM_GRID_PASSWORD: SELENIUM_GRID_HOST = f"{SELENIUM_GRID_USERNAME}:{SELENIUM_GRID_PASSWORD}@{SELENIUM_GRID_HOST}" diff --git a/tests/charts/ci/base-resources-values.yaml b/tests/charts/ci/base-resources-values.yaml index fbeb383ff..b1592bf79 100644 --- a/tests/charts/ci/base-resources-values.yaml +++ b/tests/charts/ci/base-resources-values.yaml @@ -1,5 +1,7 @@ components: router: + livenessProbe: + failureThreshold: 6 resources: requests: cpu: 100m @@ -8,6 +10,8 @@ components: cpu: 500m memory: 2500Mi distributor: + livenessProbe: + failureThreshold: 6 resources: requests: cpu: "1" @@ -41,6 +45,8 @@ components: memory: 512Mi hub: + livenessProbe: + failureThreshold: 6 resources: requests: cpu: "1" diff --git a/tests/charts/make/chart_test.sh b/tests/charts/make/chart_test.sh index 37c8f6286..76efcb8aa 100755 --- a/tests/charts/make/chart_test.sh +++ b/tests/charts/make/chart_test.sh @@ -28,7 +28,7 @@ SSL_CERT_DIR=${SSL_CERT_DIR:-"/etc/ssl/certs"} VIDEO_TAG=${VIDEO_TAG:-"latest"} CHART_ENABLE_TRACING=${CHART_ENABLE_TRACING:-"false"} CHART_FULL_DISTRIBUTED_MODE=${CHART_FULL_DISTRIBUTED_MODE:-"false"} -HOSTNAME_ADDRESS=${HOSTNAME_ADDRESS:-"selenium-grid.prod"} +HOSTNAME_ADDRESS=${HOSTNAME_ADDRESS:-${SELENIUM_GRID_HOST}} CHART_ENABLE_INGRESS_HOSTNAME=${CHART_ENABLE_INGRESS_HOSTNAME:-"false"} CHART_ENABLE_BASIC_AUTH=${CHART_ENABLE_BASIC_AUTH:-"false"} BASIC_AUTH_USERNAME=${BASIC_AUTH_USERNAME:-"sysAdminUser"} @@ -38,6 +38,7 @@ TEST_EXISTING_KEDA=${TEST_EXISTING_KEDA:-"true"} TEST_UPGRADE_CHART=${TEST_UPGRADE_CHART:-"false"} TEST_PV_CLAIM_NAME=${TEST_PV_CLAIM_NAME:-"selenium-grid-pvc-local"} LIMIT_RESOURCES=${LIMIT_RESOURCES:-"true"} +TEST_PLATFORMS=${PLATFORMS:-"linux/amd64"} cleanup() { # Get the list of pods @@ -165,7 +166,7 @@ if [ "${CHART_ENABLE_BASIC_AUTH}" = "true" ]; then export SELENIUM_GRID_PASSWORD=${BASIC_AUTH_PASSWORD} fi -if [ "${PLATFORMS}" != "linux/amd64" ]; then +if [ "${TEST_PLATFORMS}" != "linux/amd64" ]; then HELM_COMMAND_SET_IMAGES="${HELM_COMMAND_SET_IMAGES} \ --set edgeNode.enabled=false \ --set chromeNode.imageName=node-chromium \ @@ -241,10 +242,10 @@ export HUB_CHECKS_MAX_ATTEMPTS=${HUB_CHECKS_MAX_ATTEMPTS} export WEB_DRIVER_WAIT_TIMEOUT=${WEB_DRIVER_WAIT_TIMEOUT} export SELENIUM_GRID_TEST_HEADLESS=${SELENIUM_GRID_TEST_HEADLESS:-"false"} export TEST_DELAY_AFTER_TEST=${TEST_DELAY_AFTER_TEST:-"10"} -export PLATFORMS=${PLATFORMS:-"linux/amd64"} +export TEST_PLATFORMS=${TEST_PLATFORMS} if [ "${MATRIX_BROWSER}" = "NoAutoscaling" ]; then ./tests/bootstrap.sh NodeFirefox - if [ "${PLATFORMS}" = "linux/amd64" ]; then + if [ "${TEST_PLATFORMS}" = "linux/amd64" ]; then ./tests/bootstrap.sh NodeChrome ./tests/bootstrap.sh NodeEdge else @@ -257,6 +258,18 @@ fi echo "Get pods status" kubectl get pods -n ${SELENIUM_NAMESPACE} +# Wait until no pods are in "Terminating" state +while true; do + terminating_pods=$(kubectl get pods -n ${SELENIUM_NAMESPACE} --no-headers | grep Terminating | wc -l) + if [ $terminating_pods -eq 0 ]; then + echo "No pods in 'Terminating' state." + break + else + echo "Waiting for $terminating_pods pod(s) to terminate..." + sleep 2 + fi +done + echo "Get all resources in all namespaces" kubectl get all -A >> tests/tests/describe_all_resources_${MATRIX_BROWSER}.txt