From e25636416e476713ee377d3558da80150f5ec892 Mon Sep 17 00:00:00 2001 From: Viet Nguyen Duc Date: Fri, 27 Sep 2024 08:39:36 +0000 Subject: [PATCH] chart(add): add component metrics exporter Signed-off-by: Viet Nguyen Duc --- Makefile | 2 +- charts/selenium-grid/templates/_helpers.tpl | 7 ++++ .../selenium-grid/templates/_nameHelpers.tpl | 7 ++++ .../graphql-exporter-deployment.yaml | 33 ++++++++++++++++ .../templates/graphql-exporter-service.yaml | 25 ++++++++++++ .../patch-keda/patch-keda-objects-job.yaml | 10 +++-- charts/selenium-grid/values.yaml | 39 ++++++++++++------- tests/SeleniumTests/__init__.py | 15 ++++--- tests/charts/ci/base-tls-values.yaml | 21 +++++++--- tests/charts/refValues/sample-aws.yaml | 2 - tests/charts/refValues/simplex-minikube.yaml | 2 - 11 files changed, 131 insertions(+), 32 deletions(-) create mode 100644 charts/selenium-grid/templates/graphql-exporter-deployment.yaml create mode 100644 charts/selenium-grid/templates/graphql-exporter-service.yaml diff --git a/Makefile b/Makefile index e1d68af871..fc4ebf3c55 100644 --- a/Makefile +++ b/Makefile @@ -1,6 +1,6 @@ NAME := $(or $(NAME),$(NAME),selenium) CURRENT_DATE := $(shell date '+%Y%m%d') -BUILD_DATE := $(or $(BUILD_DATE),$(BUILD_DATE),$(CURRENT_DATE)) +BUILD_DATE := $(or $(BUILD_DATE),$(BUILD_DATE),20240923) BASE_RELEASE := $(or $(BASE_RELEASE),$(BASE_RELEASE),selenium-4.25.0) BASE_VERSION := $(or $(BASE_VERSION),$(BASE_VERSION),4.25.0) BINDING_VERSION := $(or $(BINDING_VERSION),$(BINDING_VERSION),4.25.0) diff --git a/charts/selenium-grid/templates/_helpers.tpl b/charts/selenium-grid/templates/_helpers.tpl index 855fc0b85a..cf43bc2450 100644 --- a/charts/selenium-grid/templates/_helpers.tpl +++ b/charts/selenium-grid/templates/_helpers.tpl @@ -659,6 +659,13 @@ Graphql Url of the hub or the router {{- printf "%s/graphql" (include "seleniumGrid.server.url" $) -}} {{- end -}} +{{/* +Graphql Url for internal monitoring exporter +*/}} +{{- define "seleniumGrid.monitoring.graphqlURL" -}} +{{- printf "%s://%s%s%s%s/graphql" (include "seleniumGrid.server.url.schema" $) (include "seleniumGrid.url.basicAuth" $) (include "seleniumGrid.server.url.host" $) (include "seleniumGrid.server.url.port" $) (include "seleniumGrid.url.subPath" $) -}} +{{- end -}} + {{- define "seleniumGrid.url.schema" -}} {{- $schema := "http" -}} {{- if or (eq (include "seleniumGrid.server.secureConnection" $) "true") (eq (include "seleniumGrid.ingress.secureConnection" $) "true") -}} diff --git a/charts/selenium-grid/templates/_nameHelpers.tpl b/charts/selenium-grid/templates/_nameHelpers.tpl index 67c020c402..baeae466f0 100644 --- a/charts/selenium-grid/templates/_nameHelpers.tpl +++ b/charts/selenium-grid/templates/_nameHelpers.tpl @@ -56,6 +56,13 @@ component.autoscaling: "{{ .Release.Name }}" {{- end -}} {{- end -}} +{{/* +Selenium metrics exporter fullname +*/}} +{{- define "seleniumGrid.monitoring.exporter.fullname" -}} +{{- tpl (default (include "seleniumGrid.component.name" (list "selenium-metrics-exporter" $)) $.Values.monitoring.nameOverride) $ | trunc 63 | trimSuffix "-" -}} +{{- end -}} + {{/* Selenium Hub fullname */}} diff --git a/charts/selenium-grid/templates/graphql-exporter-deployment.yaml b/charts/selenium-grid/templates/graphql-exporter-deployment.yaml new file mode 100644 index 0000000000..729449e17b --- /dev/null +++ b/charts/selenium-grid/templates/graphql-exporter-deployment.yaml @@ -0,0 +1,33 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ template "seleniumGrid.monitoring.exporter.fullname" $ }} + namespace: {{ .Release.Namespace }} + labels: &exporter_labels + app: {{ template "seleniumGrid.monitoring.exporter.fullname" $ }} + app.kubernetes.io/name: {{ template "seleniumGrid.monitoring.exporter.fullname" $ }} + {{- include "seleniumGrid.commonLabels" . | nindent 4 }} +spec: + replicas: 1 + selector: + matchLabels: + app: {{ template "seleniumGrid.monitoring.exporter.fullname" $ }} + app.kubernetes.io/name: {{ template "seleniumGrid.monitoring.exporter.fullname" $ }} + template: + metadata: + labels: *exporter_labels + annotations: + {{- with .Values.monitoring.exporter.annotations }} + {{- toYaml . | nindent 8 }} + {{- end }} + spec: + {{- if or .Values.global.seleniumGrid.imagePullSecret .Values.monitoring.exporter.imagePullSecret }} + imagePullSecrets: + - name: {{ default .Values.global.seleniumGrid.imagePullSecret .Values.monitoring.exporter.imagePullSecret }} + {{- end }} + containers: + - name: graphql-exporter + {{- $imageRegistry := default .Values.global.seleniumGrid.imageRegistry .Values.monitoring.exporter.imageRegistry }} + image: {{ printf "%s/%s:%s" $imageRegistry .Values.monitoring.exporter.imageName .Values.monitoring.exporter.imageTag | quote }} + ports: + - containerPort: {{ .Values.monitoring.exporter.port }} diff --git a/charts/selenium-grid/templates/graphql-exporter-service.yaml b/charts/selenium-grid/templates/graphql-exporter-service.yaml new file mode 100644 index 0000000000..9e57895649 --- /dev/null +++ b/charts/selenium-grid/templates/graphql-exporter-service.yaml @@ -0,0 +1,25 @@ +apiVersion: v1 +kind: Service +metadata: + name: {{ template "seleniumGrid.monitoring.exporter.fullname" $ }} + namespace: {{ .Release.Namespace }} + labels: + app: {{ template "seleniumGrid.monitoring.exporter.fullname" $ }} + app.kubernetes.io/name: {{ template "seleniumGrid.monitoring.exporter.fullname" $ }} + {{- include "seleniumGrid.commonLabels" . | nindent 4 }} +spec: + selector: + app: {{ template "seleniumGrid.monitoring.exporter.fullname" $ }} + app.kubernetes.io/name: {{ template "seleniumGrid.monitoring.exporter.fullname" $ }} + type: {{ .Values.monitoring.exporter.service.type }} + {{- if and (eq .Values.monitoring.exporter.service.type "LoadBalancer") ( .Values.monitoring.exporter.service.loadBalancerIP ) }} + loadBalancerIP: {{ .Values.monitoring.exporter.service.loadBalancerIP }} + {{- end }} + ports: + - name: http-port + protocol: TCP + port: {{ .Values.monitoring.exporter.port }} + targetPort: {{ .Values.monitoring.exporter.port }} + {{- if and (eq .Values.monitoring.exporter.service.type "NodePort") .Values.monitoring.exporter.service.nodePort }} + nodePort: {{ .Values.monitoring.exporter.service.nodePort }} + {{- end }} diff --git a/charts/selenium-grid/templates/patch-keda/patch-keda-objects-job.yaml b/charts/selenium-grid/templates/patch-keda/patch-keda-objects-job.yaml index 326f52fcbb..eeda16aa1c 100644 --- a/charts/selenium-grid/templates/patch-keda/patch-keda-objects-job.yaml +++ b/charts/selenium-grid/templates/patch-keda/patch-keda-objects-job.yaml @@ -34,9 +34,13 @@ spec: args: - | echo "Cleaning up ScaledObjects, ScaledJobs and HPAs for {{ .Release.Name }} when upgrading or disabling autoscaling." - kubectl patch ScaledObjects,ScaledJobs -n {{ .Release.Namespace }} -l component.autoscaling={{ .Release.Name }} -p '{"metadata":{"finalizers":null}}' || true ; - kubectl delete ScaledObjects,ScaledJobs -n {{ .Release.Namespace }} -l component.autoscaling={{ .Release.Name }} --wait || true ; - kubectl delete hpa -n {{ .Release.Namespace }} -l component.autoscaling={{ .Release.Name }} --wait || true ; + kubectl patch ScaledObjects -n {{ .Release.Namespace }} -l component.autoscaling={{ .Release.Name }} -p '{"metadata":{"finalizers":null}}' || true ; + kubectl patch ScaledJobs -n {{ .Release.Namespace }} -l component.autoscaling={{ .Release.Name }} -p '{"metadata":{"finalizers":null}}' || true ; + kubectl patch TriggerAuthentication -n {{ .Release.Namespace }} -l component.autoscaling={{ .Release.Name }} -p '{"metadata":{"finalizers":null}}' || true ; + kubectl delete ScaledObjects -n {{ .Release.Namespace }} -l component.autoscaling={{ .Release.Name }} || true ; + kubectl delete ScaledJobs -n {{ .Release.Namespace }} -l component.autoscaling={{ .Release.Name }} || true ; + kubectl delete TriggerAuthentication -n {{ .Release.Namespace }} -l component.autoscaling={{ .Release.Name }} || true ; + kubectl delete hpa -n {{ .Release.Namespace }} -l component.autoscaling={{ .Release.Name }} || true ; {{- with $.Values.autoscaling.patchObjectFinalizers.resources }} resources: {{ toYaml . | nindent 12 }} {{- end }} diff --git a/charts/selenium-grid/values.yaml b/charts/selenium-grid/values.yaml index b2da6ad2da..5b38f9a95f 100644 --- a/charts/selenium-grid/values.yaml +++ b/charts/selenium-grid/values.yaml @@ -791,6 +791,25 @@ tracing: monitoring: enabled: false enabledWithExistingAgent: false + exporter: + imageRegistry: "ricardbejarano" + imageName: "graphql_exporter" + imageTag: "latest" + # -- Custom pull secret for container in patch job + imagePullSecret: "" + annotations: {} + port: 9199 + service: + # -- Create a service for exporter + enabled: true + # -- Service type + type: ClusterIP + # -- Set specific loadBalancerIP when serviceType is LoadBalancer (see https://kubernetes.io/docs/concepts/services-networking/service/#loadbalancer) + loadBalancerIP: "" + # -- Node port for service + nodePort: 30199 + # -- Annotations for exporter service + annotations: {} # Keda scaled object configuration autoscaling: @@ -803,25 +822,18 @@ autoscaling: # -- Specify an external KEDA TriggerAuthentication resource is used for scaler triggers config. Apply for all browser nodes authenticationRef: name: "" - annotations: - "helm.sh/hook": post-install,post-upgrade,post-rollback - "helm.sh/hook-weight": "-2" + annotations: {} # -- Annotations for KEDA resources: ScaledObject and ScaledJob - annotations: - "helm.sh/hook": post-install,post-upgrade,post-rollback - "helm.sh/hook-weight": "1" + annotations: {} patchObjectFinalizers: # -- Override the name of the patch job nameOverride: # -- Enable patching finalizers for KEDA scaled resources. Workaround for Hook post-upgrade selenium-grid/templates/x-node-hpa.yaml failed: object is being deleted: scaledobjects.keda.sh "x" already exists - enabled: true + enabled: false # -- Deadline (in seconds) for patch job to complete activeDeadlineSeconds: 120 # -- Annotations for patch job - annotations: - "helm.sh/hook": post-install,post-upgrade,post-rollback,pre-delete - "helm.sh/hook-delete-policy": hook-succeeded,before-hook-creation - "helm.sh/hook-weight": "-1" + annotations: {} # -- Define an external service account name contains permissions to patch KEDA scaled resources serviceAccount: "" # -- Custom pull secret for container in patch job @@ -1562,8 +1574,9 @@ keda: registry: selenium repository: keda-admission-webhooks tag: "2.15.1-selenium-grid-20240922" - # -- Annotations for KEDA resources - additionalAnnotations: + crds: + additionalAnnotations: + "helm.sh/hook": pre-install,pre-upgrade,pre-rollback,post-delete http: timeout: 60000 # -- Enable KEDA admission webhooks component diff --git a/tests/SeleniumTests/__init__.py b/tests/SeleniumTests/__init__.py index f18c31c4a3..daca99a0d5 100644 --- a/tests/SeleniumTests/__init__.py +++ b/tests/SeleniumTests/__init__.py @@ -129,7 +129,8 @@ def setUp(self): try: options = ChromeOptions() options.enable_downloads = SELENIUM_ENABLE_MANAGED_DOWNLOADS - options.add_argument('disable-features=DownloadBubble,DownloadBubbleV2') + if not SELENIUM_ENABLE_MANAGED_DOWNLOADS: + options.add_argument('disable-features=DownloadBubble,DownloadBubbleV2') if TEST_ADD_CAPS_RECORD_VIDEO: options.set_capability('se:recordVideo', True) options.set_capability('se:name', f"{self._testMethodName} ({self.__class__.__name__})") @@ -166,7 +167,8 @@ def setUp(self): try: options = EdgeOptions() options.enable_downloads = SELENIUM_ENABLE_MANAGED_DOWNLOADS - options.add_argument('disable-features=DownloadBubble,DownloadBubbleV2') + if not SELENIUM_ENABLE_MANAGED_DOWNLOADS: + options.add_argument('disable-features=DownloadBubble,DownloadBubbleV2') if TEST_ADD_CAPS_RECORD_VIDEO: options.set_capability('se:recordVideo', True) options.set_capability('se:name', f"{self._testMethodName} ({self.__class__.__name__})") @@ -189,13 +191,14 @@ class FirefoxTests(SeleniumGenericTests): def setUp(self): try: profile = webdriver.FirefoxProfile() - profile.set_preference("browser.download.manager.showWhenStarting", False) - profile.set_preference("browser.helperApps.neverAsk.saveToDisk", "*/*") + options = FirefoxOptions() + options.enable_downloads = SELENIUM_ENABLE_MANAGED_DOWNLOADS + if not SELENIUM_ENABLE_MANAGED_DOWNLOADS: + profile.set_preference("browser.download.manager.showWhenStarting", False) + profile.set_preference("browser.helperApps.neverAsk.saveToDisk", "*/*") profile.set_preference('intl.accept_languages', 'vi-VN,vi') profile.set_preference('intl.locale.requested', 'vi-VN,vi') - options = FirefoxOptions() options.profile = profile - options.enable_downloads = SELENIUM_ENABLE_MANAGED_DOWNLOADS if TEST_ADD_CAPS_RECORD_VIDEO: options.set_capability('se:recordVideo', True) options.set_capability('se:name', f"{self._testMethodName} ({self.__class__.__name__})") diff --git a/tests/charts/ci/base-tls-values.yaml b/tests/charts/ci/base-tls-values.yaml index 1d34b11e63..ed82343be7 100644 --- a/tests/charts/ci/base-tls-values.yaml +++ b/tests/charts/ci/base-tls-values.yaml @@ -3,18 +3,17 @@ registrationSecret: value: "HappyTestOps" monitoring: - enabled: false + enabled: true kube-prometheus-stack: cleanPrometheusOperatorObjectNames: true defaultRules: create: true annotations: - "helm.sh/hook": post-install,post-upgrade,post-rollback + "helm.sh/hook": pre-install,pre-upgrade,pre-rollback,post-delete alertmanager: enabled: true annotations: - "helm.sh/hook": post-install,post-upgrade,post-rollback ingress: enabled: true ingressClassName: nginx @@ -26,7 +25,6 @@ kube-prometheus-stack: forceDeployDatasources: true forceDeployDashboards: true annotations: - "helm.sh/hook": post-install,post-upgrade,post-rollback ingress: enabled: true ingressClassName: nginx @@ -35,13 +33,26 @@ kube-prometheus-stack: prometheus: enabled: true annotations: - "helm.sh/hook": post-install,post-upgrade,post-rollback ingress: enabled: true ingressClassName: nginx hosts: - pts.selenium-grid.prod prometheusSpec: + additionalScrapeConfigs: + - job_name: "selenium-grid-analytics" + metrics_path: "/query" + bearer_token: "" + params: + endpoint: + - 'http://selenium-router.selenium:4444/graphql' + query: + - | + { grid { sessionCount, maxSession, totalSlots, nodeCount, sessionQueueSize }, nodesInfo { nodes { id, status, sessionCount, maxSession, slotCount, stereotypes, sessions { id, capabilities, sessionDurationMillis, slot { id, stereotype } } } }, sessionsInfo { sessionQueueRequests } } + zoneTag: [] + static_configs: + - targets: + - 'selenium-metrics-exporter.selenium:9199' storageSpec: volumeClaimTemplate: spec: diff --git a/tests/charts/refValues/sample-aws.yaml b/tests/charts/refValues/sample-aws.yaml index dc1eac9468..7a02e76567 100644 --- a/tests/charts/refValues/sample-aws.yaml +++ b/tests/charts/refValues/sample-aws.yaml @@ -33,8 +33,6 @@ isolateComponents: true autoscaling: enabled: true scalingType: job - annotations: - helm.sh/hook: post-install,post-upgrade,post-rollback scaledOptions: minReplicaCount: 0 maxReplicaCount: 8 diff --git a/tests/charts/refValues/simplex-minikube.yaml b/tests/charts/refValues/simplex-minikube.yaml index 66ef712452..d85a2354de 100644 --- a/tests/charts/refValues/simplex-minikube.yaml +++ b/tests/charts/refValues/simplex-minikube.yaml @@ -45,8 +45,6 @@ autoscaling: # enabled: true enableWithExistingKEDA: true scalingType: job - annotations: - helm.sh/hook: post-install,post-upgrade,post-rollback scaledOptions: minReplicaCount: 0 maxReplicaCount: 8