From 2bb6bdac431b41e674a06adb5d3fccbc8d9fd46e Mon Sep 17 00:00:00 2001 From: Viet Nguyen Duc Date: Fri, 27 Sep 2024 08:39:36 +0000 Subject: [PATCH] chart(add): add component metrics exporter Signed-off-by: Viet Nguyen Duc --- charts/selenium-grid/CONFIGURATION.md | 21 +++++++--- charts/selenium-grid/templates/_helpers.tpl | 7 ++++ .../selenium-grid/templates/_nameHelpers.tpl | 7 ++++ .../graphql-exporter-deployment.yaml | 33 ++++++++++++++++ .../templates/graphql-exporter-service.yaml | 25 ++++++++++++ .../patch-keda/patch-keda-objects-job.yaml | 10 +++-- charts/selenium-grid/values.yaml | 39 ++++++++++++------- tests/SeleniumTests/__init__.py | 15 ++++--- tests/charts/ci/base-tls-values.yaml | 21 +++++++--- tests/charts/refValues/sample-aws.yaml | 2 - tests/charts/refValues/simplex-minikube.yaml | 2 - 11 files changed, 146 insertions(+), 36 deletions(-) create mode 100644 charts/selenium-grid/templates/graphql-exporter-deployment.yaml create mode 100644 charts/selenium-grid/templates/graphql-exporter-service.yaml diff --git a/charts/selenium-grid/CONFIGURATION.md b/charts/selenium-grid/CONFIGURATION.md index 326577712c..796309a45e 100644 --- a/charts/selenium-grid/CONFIGURATION.md +++ b/charts/selenium-grid/CONFIGURATION.md @@ -307,15 +307,26 @@ A Helm chart for creating a Selenium Grid Server in Kubernetes | tracing.ingress.paths | list | `[{"backend":{"service":{"name":"{{ .Release.Name }}-jaeger-query","port":{"number":16686}}},"path":"/jaeger","pathType":"Prefix"}]` | Configure paths for Jaeger ingress resource | | monitoring.enabled | bool | `false` | | | monitoring.enabledWithExistingAgent | bool | `false` | | +| monitoring.exporter.imageRegistry | string | `"ricardbejarano"` | | +| monitoring.exporter.imageName | string | `"graphql_exporter"` | | +| monitoring.exporter.imageTag | string | `"latest"` | | +| monitoring.exporter.imagePullSecret | string | `""` | Custom pull secret for container in patch job | +| monitoring.exporter.annotations | object | `{}` | | +| monitoring.exporter.port | int | `9199` | | +| monitoring.exporter.service.enabled | bool | `true` | Create a service for exporter | +| monitoring.exporter.service.type | string | `"ClusterIP"` | Service type | +| monitoring.exporter.service.loadBalancerIP | string | `""` | Set specific loadBalancerIP when serviceType is LoadBalancer (see https://kubernetes.io/docs/concepts/services-networking/service/#loadbalancer) | +| monitoring.exporter.service.nodePort | int | `30199` | Node port for service | +| monitoring.exporter.service.annotations | object | `{}` | Annotations for exporter service | | autoscaling.enabled | bool | `false` | Enable autoscaling. Implies installing KEDA | | autoscaling.enableWithExistingKEDA | bool | `false` | Enable autoscaling without automatically installing KEDA | | autoscaling.scalingType | string | `"job"` | Which type of KEDA scaling to use: job or deployment | -| autoscaling.authenticationRef | object | `{"annotations":{"helm.sh/hook":"post-install,post-upgrade,post-rollback","helm.sh/hook-weight":"-2"},"name":""}` | Specify an external KEDA TriggerAuthentication resource is used for scaler triggers config. Apply for all browser nodes | -| autoscaling.annotations | object | `{"helm.sh/hook":"post-install,post-upgrade,post-rollback","helm.sh/hook-weight":"1"}` | Annotations for KEDA resources: ScaledObject and ScaledJob | +| autoscaling.authenticationRef | object | `{"annotations":{},"name":""}` | Specify an external KEDA TriggerAuthentication resource is used for scaler triggers config. Apply for all browser nodes | +| autoscaling.annotations | object | `{}` | Annotations for KEDA resources: ScaledObject and ScaledJob | | autoscaling.patchObjectFinalizers.nameOverride | string | `nil` | Override the name of the patch job | -| autoscaling.patchObjectFinalizers.enabled | bool | `true` | Enable patching finalizers for KEDA scaled resources. Workaround for Hook post-upgrade selenium-grid/templates/x-node-hpa.yaml failed: object is being deleted: scaledobjects.keda.sh "x" already exists | +| autoscaling.patchObjectFinalizers.enabled | bool | `false` | Enable patching finalizers for KEDA scaled resources. Workaround for Hook post-upgrade selenium-grid/templates/x-node-hpa.yaml failed: object is being deleted: scaledobjects.keda.sh "x" already exists | | autoscaling.patchObjectFinalizers.activeDeadlineSeconds | int | `120` | Deadline (in seconds) for patch job to complete | -| autoscaling.patchObjectFinalizers.annotations | object | `{"helm.sh/hook":"post-install,post-upgrade,post-rollback,pre-delete","helm.sh/hook-delete-policy":"hook-succeeded,before-hook-creation","helm.sh/hook-weight":"-1"}` | Annotations for patch job | +| autoscaling.patchObjectFinalizers.annotations | object | `{}` | Annotations for patch job | | autoscaling.patchObjectFinalizers.serviceAccount | string | `""` | Define an external service account name contains permissions to patch KEDA scaled resources | | autoscaling.patchObjectFinalizers.imagePullSecret | string | `""` | Custom pull secret for container in patch job | | autoscaling.patchObjectFinalizers.resources | object | `{"limits":{"cpu":"50m","memory":"50Mi"},"requests":{"cpu":"10m","memory":"10Mi"}}` | Define resources for container in patch job | @@ -508,7 +519,7 @@ A Helm chart for creating a Selenium Grid Server in Kubernetes | videoRecorder.s3 | object | `{"args":[],"command":[],"extraEnvironmentVariables":null,"imageName":"aws-cli","imagePullPolicy":"IfNotPresent","imageRegistry":"bitnami","imageTag":"latest","securityContext":{"runAsUser":0}}` | Container spec for the uploader if above it is defined as "uploader.name: s3" | | customLabels | object | `{}` | Custom labels for k8s resources | | keda.image | object | `{"keda":{"registry":"selenium","repository":"keda","tag":"2.15.1-selenium-grid-20240922"},"metricsApiServer":{"registry":"selenium","repository":"keda-metrics-apiserver","tag":"2.15.1-selenium-grid-20240922"},"webhooks":{"registry":"selenium","repository":"keda-admission-webhooks","tag":"2.15.1-selenium-grid-20240922"}}` | Specify image for KEDA components | -| keda.additionalAnnotations | string | `nil` | Annotations for KEDA resources | +| keda.crds.additionalAnnotations."helm.sh/hook" | string | `"pre-install,pre-upgrade,pre-rollback,post-delete"` | | | keda.http.timeout | int | `60000` | | | keda.webhooks | object | `{"enabled":false}` | Enable KEDA admission webhooks component | | ingress-nginx | object | `{"controller":{"admissionWebhooks":{"enabled":false}}}` | Configuration for dependency chart ingress-nginx | diff --git a/charts/selenium-grid/templates/_helpers.tpl b/charts/selenium-grid/templates/_helpers.tpl index 855fc0b85a..cf43bc2450 100644 --- a/charts/selenium-grid/templates/_helpers.tpl +++ b/charts/selenium-grid/templates/_helpers.tpl @@ -659,6 +659,13 @@ Graphql Url of the hub or the router {{- printf "%s/graphql" (include "seleniumGrid.server.url" $) -}} {{- end -}} +{{/* +Graphql Url for internal monitoring exporter +*/}} +{{- define "seleniumGrid.monitoring.graphqlURL" -}} +{{- printf "%s://%s%s%s%s/graphql" (include "seleniumGrid.server.url.schema" $) (include "seleniumGrid.url.basicAuth" $) (include "seleniumGrid.server.url.host" $) (include "seleniumGrid.server.url.port" $) (include "seleniumGrid.url.subPath" $) -}} +{{- end -}} + {{- define "seleniumGrid.url.schema" -}} {{- $schema := "http" -}} {{- if or (eq (include "seleniumGrid.server.secureConnection" $) "true") (eq (include "seleniumGrid.ingress.secureConnection" $) "true") -}} diff --git a/charts/selenium-grid/templates/_nameHelpers.tpl b/charts/selenium-grid/templates/_nameHelpers.tpl index 67c020c402..baeae466f0 100644 --- a/charts/selenium-grid/templates/_nameHelpers.tpl +++ b/charts/selenium-grid/templates/_nameHelpers.tpl @@ -56,6 +56,13 @@ component.autoscaling: "{{ .Release.Name }}" {{- end -}} {{- end -}} +{{/* +Selenium metrics exporter fullname +*/}} +{{- define "seleniumGrid.monitoring.exporter.fullname" -}} +{{- tpl (default (include "seleniumGrid.component.name" (list "selenium-metrics-exporter" $)) $.Values.monitoring.nameOverride) $ | trunc 63 | trimSuffix "-" -}} +{{- end -}} + {{/* Selenium Hub fullname */}} diff --git a/charts/selenium-grid/templates/graphql-exporter-deployment.yaml b/charts/selenium-grid/templates/graphql-exporter-deployment.yaml new file mode 100644 index 0000000000..729449e17b --- /dev/null +++ b/charts/selenium-grid/templates/graphql-exporter-deployment.yaml @@ -0,0 +1,33 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ template "seleniumGrid.monitoring.exporter.fullname" $ }} + namespace: {{ .Release.Namespace }} + labels: &exporter_labels + app: {{ template "seleniumGrid.monitoring.exporter.fullname" $ }} + app.kubernetes.io/name: {{ template "seleniumGrid.monitoring.exporter.fullname" $ }} + {{- include "seleniumGrid.commonLabels" . | nindent 4 }} +spec: + replicas: 1 + selector: + matchLabels: + app: {{ template "seleniumGrid.monitoring.exporter.fullname" $ }} + app.kubernetes.io/name: {{ template "seleniumGrid.monitoring.exporter.fullname" $ }} + template: + metadata: + labels: *exporter_labels + annotations: + {{- with .Values.monitoring.exporter.annotations }} + {{- toYaml . | nindent 8 }} + {{- end }} + spec: + {{- if or .Values.global.seleniumGrid.imagePullSecret .Values.monitoring.exporter.imagePullSecret }} + imagePullSecrets: + - name: {{ default .Values.global.seleniumGrid.imagePullSecret .Values.monitoring.exporter.imagePullSecret }} + {{- end }} + containers: + - name: graphql-exporter + {{- $imageRegistry := default .Values.global.seleniumGrid.imageRegistry .Values.monitoring.exporter.imageRegistry }} + image: {{ printf "%s/%s:%s" $imageRegistry .Values.monitoring.exporter.imageName .Values.monitoring.exporter.imageTag | quote }} + ports: + - containerPort: {{ .Values.monitoring.exporter.port }} diff --git a/charts/selenium-grid/templates/graphql-exporter-service.yaml b/charts/selenium-grid/templates/graphql-exporter-service.yaml new file mode 100644 index 0000000000..9e57895649 --- /dev/null +++ b/charts/selenium-grid/templates/graphql-exporter-service.yaml @@ -0,0 +1,25 @@ +apiVersion: v1 +kind: Service +metadata: + name: {{ template "seleniumGrid.monitoring.exporter.fullname" $ }} + namespace: {{ .Release.Namespace }} + labels: + app: {{ template "seleniumGrid.monitoring.exporter.fullname" $ }} + app.kubernetes.io/name: {{ template "seleniumGrid.monitoring.exporter.fullname" $ }} + {{- include "seleniumGrid.commonLabels" . | nindent 4 }} +spec: + selector: + app: {{ template "seleniumGrid.monitoring.exporter.fullname" $ }} + app.kubernetes.io/name: {{ template "seleniumGrid.monitoring.exporter.fullname" $ }} + type: {{ .Values.monitoring.exporter.service.type }} + {{- if and (eq .Values.monitoring.exporter.service.type "LoadBalancer") ( .Values.monitoring.exporter.service.loadBalancerIP ) }} + loadBalancerIP: {{ .Values.monitoring.exporter.service.loadBalancerIP }} + {{- end }} + ports: + - name: http-port + protocol: TCP + port: {{ .Values.monitoring.exporter.port }} + targetPort: {{ .Values.monitoring.exporter.port }} + {{- if and (eq .Values.monitoring.exporter.service.type "NodePort") .Values.monitoring.exporter.service.nodePort }} + nodePort: {{ .Values.monitoring.exporter.service.nodePort }} + {{- end }} diff --git a/charts/selenium-grid/templates/patch-keda/patch-keda-objects-job.yaml b/charts/selenium-grid/templates/patch-keda/patch-keda-objects-job.yaml index 326f52fcbb..eeda16aa1c 100644 --- a/charts/selenium-grid/templates/patch-keda/patch-keda-objects-job.yaml +++ b/charts/selenium-grid/templates/patch-keda/patch-keda-objects-job.yaml @@ -34,9 +34,13 @@ spec: args: - | echo "Cleaning up ScaledObjects, ScaledJobs and HPAs for {{ .Release.Name }} when upgrading or disabling autoscaling." - kubectl patch ScaledObjects,ScaledJobs -n {{ .Release.Namespace }} -l component.autoscaling={{ .Release.Name }} -p '{"metadata":{"finalizers":null}}' || true ; - kubectl delete ScaledObjects,ScaledJobs -n {{ .Release.Namespace }} -l component.autoscaling={{ .Release.Name }} --wait || true ; - kubectl delete hpa -n {{ .Release.Namespace }} -l component.autoscaling={{ .Release.Name }} --wait || true ; + kubectl patch ScaledObjects -n {{ .Release.Namespace }} -l component.autoscaling={{ .Release.Name }} -p '{"metadata":{"finalizers":null}}' || true ; + kubectl patch ScaledJobs -n {{ .Release.Namespace }} -l component.autoscaling={{ .Release.Name }} -p '{"metadata":{"finalizers":null}}' || true ; + kubectl patch TriggerAuthentication -n {{ .Release.Namespace }} -l component.autoscaling={{ .Release.Name }} -p '{"metadata":{"finalizers":null}}' || true ; + kubectl delete ScaledObjects -n {{ .Release.Namespace }} -l component.autoscaling={{ .Release.Name }} || true ; + kubectl delete ScaledJobs -n {{ .Release.Namespace }} -l component.autoscaling={{ .Release.Name }} || true ; + kubectl delete TriggerAuthentication -n {{ .Release.Namespace }} -l component.autoscaling={{ .Release.Name }} || true ; + kubectl delete hpa -n {{ .Release.Namespace }} -l component.autoscaling={{ .Release.Name }} || true ; {{- with $.Values.autoscaling.patchObjectFinalizers.resources }} resources: {{ toYaml . | nindent 12 }} {{- end }} diff --git a/charts/selenium-grid/values.yaml b/charts/selenium-grid/values.yaml index b2da6ad2da..5b38f9a95f 100644 --- a/charts/selenium-grid/values.yaml +++ b/charts/selenium-grid/values.yaml @@ -791,6 +791,25 @@ tracing: monitoring: enabled: false enabledWithExistingAgent: false + exporter: + imageRegistry: "ricardbejarano" + imageName: "graphql_exporter" + imageTag: "latest" + # -- Custom pull secret for container in patch job + imagePullSecret: "" + annotations: {} + port: 9199 + service: + # -- Create a service for exporter + enabled: true + # -- Service type + type: ClusterIP + # -- Set specific loadBalancerIP when serviceType is LoadBalancer (see https://kubernetes.io/docs/concepts/services-networking/service/#loadbalancer) + loadBalancerIP: "" + # -- Node port for service + nodePort: 30199 + # -- Annotations for exporter service + annotations: {} # Keda scaled object configuration autoscaling: @@ -803,25 +822,18 @@ autoscaling: # -- Specify an external KEDA TriggerAuthentication resource is used for scaler triggers config. Apply for all browser nodes authenticationRef: name: "" - annotations: - "helm.sh/hook": post-install,post-upgrade,post-rollback - "helm.sh/hook-weight": "-2" + annotations: {} # -- Annotations for KEDA resources: ScaledObject and ScaledJob - annotations: - "helm.sh/hook": post-install,post-upgrade,post-rollback - "helm.sh/hook-weight": "1" + annotations: {} patchObjectFinalizers: # -- Override the name of the patch job nameOverride: # -- Enable patching finalizers for KEDA scaled resources. Workaround for Hook post-upgrade selenium-grid/templates/x-node-hpa.yaml failed: object is being deleted: scaledobjects.keda.sh "x" already exists - enabled: true + enabled: false # -- Deadline (in seconds) for patch job to complete activeDeadlineSeconds: 120 # -- Annotations for patch job - annotations: - "helm.sh/hook": post-install,post-upgrade,post-rollback,pre-delete - "helm.sh/hook-delete-policy": hook-succeeded,before-hook-creation - "helm.sh/hook-weight": "-1" + annotations: {} # -- Define an external service account name contains permissions to patch KEDA scaled resources serviceAccount: "" # -- Custom pull secret for container in patch job @@ -1562,8 +1574,9 @@ keda: registry: selenium repository: keda-admission-webhooks tag: "2.15.1-selenium-grid-20240922" - # -- Annotations for KEDA resources - additionalAnnotations: + crds: + additionalAnnotations: + "helm.sh/hook": pre-install,pre-upgrade,pre-rollback,post-delete http: timeout: 60000 # -- Enable KEDA admission webhooks component diff --git a/tests/SeleniumTests/__init__.py b/tests/SeleniumTests/__init__.py index f18c31c4a3..daca99a0d5 100644 --- a/tests/SeleniumTests/__init__.py +++ b/tests/SeleniumTests/__init__.py @@ -129,7 +129,8 @@ def setUp(self): try: options = ChromeOptions() options.enable_downloads = SELENIUM_ENABLE_MANAGED_DOWNLOADS - options.add_argument('disable-features=DownloadBubble,DownloadBubbleV2') + if not SELENIUM_ENABLE_MANAGED_DOWNLOADS: + options.add_argument('disable-features=DownloadBubble,DownloadBubbleV2') if TEST_ADD_CAPS_RECORD_VIDEO: options.set_capability('se:recordVideo', True) options.set_capability('se:name', f"{self._testMethodName} ({self.__class__.__name__})") @@ -166,7 +167,8 @@ def setUp(self): try: options = EdgeOptions() options.enable_downloads = SELENIUM_ENABLE_MANAGED_DOWNLOADS - options.add_argument('disable-features=DownloadBubble,DownloadBubbleV2') + if not SELENIUM_ENABLE_MANAGED_DOWNLOADS: + options.add_argument('disable-features=DownloadBubble,DownloadBubbleV2') if TEST_ADD_CAPS_RECORD_VIDEO: options.set_capability('se:recordVideo', True) options.set_capability('se:name', f"{self._testMethodName} ({self.__class__.__name__})") @@ -189,13 +191,14 @@ class FirefoxTests(SeleniumGenericTests): def setUp(self): try: profile = webdriver.FirefoxProfile() - profile.set_preference("browser.download.manager.showWhenStarting", False) - profile.set_preference("browser.helperApps.neverAsk.saveToDisk", "*/*") + options = FirefoxOptions() + options.enable_downloads = SELENIUM_ENABLE_MANAGED_DOWNLOADS + if not SELENIUM_ENABLE_MANAGED_DOWNLOADS: + profile.set_preference("browser.download.manager.showWhenStarting", False) + profile.set_preference("browser.helperApps.neverAsk.saveToDisk", "*/*") profile.set_preference('intl.accept_languages', 'vi-VN,vi') profile.set_preference('intl.locale.requested', 'vi-VN,vi') - options = FirefoxOptions() options.profile = profile - options.enable_downloads = SELENIUM_ENABLE_MANAGED_DOWNLOADS if TEST_ADD_CAPS_RECORD_VIDEO: options.set_capability('se:recordVideo', True) options.set_capability('se:name', f"{self._testMethodName} ({self.__class__.__name__})") diff --git a/tests/charts/ci/base-tls-values.yaml b/tests/charts/ci/base-tls-values.yaml index 1d34b11e63..ed82343be7 100644 --- a/tests/charts/ci/base-tls-values.yaml +++ b/tests/charts/ci/base-tls-values.yaml @@ -3,18 +3,17 @@ registrationSecret: value: "HappyTestOps" monitoring: - enabled: false + enabled: true kube-prometheus-stack: cleanPrometheusOperatorObjectNames: true defaultRules: create: true annotations: - "helm.sh/hook": post-install,post-upgrade,post-rollback + "helm.sh/hook": pre-install,pre-upgrade,pre-rollback,post-delete alertmanager: enabled: true annotations: - "helm.sh/hook": post-install,post-upgrade,post-rollback ingress: enabled: true ingressClassName: nginx @@ -26,7 +25,6 @@ kube-prometheus-stack: forceDeployDatasources: true forceDeployDashboards: true annotations: - "helm.sh/hook": post-install,post-upgrade,post-rollback ingress: enabled: true ingressClassName: nginx @@ -35,13 +33,26 @@ kube-prometheus-stack: prometheus: enabled: true annotations: - "helm.sh/hook": post-install,post-upgrade,post-rollback ingress: enabled: true ingressClassName: nginx hosts: - pts.selenium-grid.prod prometheusSpec: + additionalScrapeConfigs: + - job_name: "selenium-grid-analytics" + metrics_path: "/query" + bearer_token: "" + params: + endpoint: + - 'http://selenium-router.selenium:4444/graphql' + query: + - | + { grid { sessionCount, maxSession, totalSlots, nodeCount, sessionQueueSize }, nodesInfo { nodes { id, status, sessionCount, maxSession, slotCount, stereotypes, sessions { id, capabilities, sessionDurationMillis, slot { id, stereotype } } } }, sessionsInfo { sessionQueueRequests } } + zoneTag: [] + static_configs: + - targets: + - 'selenium-metrics-exporter.selenium:9199' storageSpec: volumeClaimTemplate: spec: diff --git a/tests/charts/refValues/sample-aws.yaml b/tests/charts/refValues/sample-aws.yaml index dc1eac9468..7a02e76567 100644 --- a/tests/charts/refValues/sample-aws.yaml +++ b/tests/charts/refValues/sample-aws.yaml @@ -33,8 +33,6 @@ isolateComponents: true autoscaling: enabled: true scalingType: job - annotations: - helm.sh/hook: post-install,post-upgrade,post-rollback scaledOptions: minReplicaCount: 0 maxReplicaCount: 8 diff --git a/tests/charts/refValues/simplex-minikube.yaml b/tests/charts/refValues/simplex-minikube.yaml index 66ef712452..d85a2354de 100644 --- a/tests/charts/refValues/simplex-minikube.yaml +++ b/tests/charts/refValues/simplex-minikube.yaml @@ -45,8 +45,6 @@ autoscaling: # enabled: true enableWithExistingKEDA: true scalingType: job - annotations: - helm.sh/hook: post-install,post-upgrade,post-rollback scaledOptions: minReplicaCount: 0 maxReplicaCount: 8