chart(add): add component metrics exporter

Signed-off-by: Viet Nguyen Duc <[email protected]>
NDViet · Sep 27, 2024 · 2bb6bda · 2bb6bda
1 parent e61f20c
commit 2bb6bda
Show file tree

Hide file tree

Showing 11 changed files with 146 additions and 36 deletions.
diff --git a/charts/selenium-grid/CONFIGURATION.md b/charts/selenium-grid/CONFIGURATION.md
@@ -307,15 +307,26 @@ A Helm chart for creating a Selenium Grid Server in Kubernetes
 | tracing.ingress.paths | list | `[{"backend":{"service":{"name":"{{ .Release.Name }}-jaeger-query","port":{"number":16686}}},"path":"/jaeger","pathType":"Prefix"}]` | Configure paths for Jaeger ingress resource |
 | monitoring.enabled | bool | `false` |  |
 | monitoring.enabledWithExistingAgent | bool | `false` |  |
+| monitoring.exporter.imageRegistry | string | `"ricardbejarano"` |  |
+| monitoring.exporter.imageName | string | `"graphql_exporter"` |  |
+| monitoring.exporter.imageTag | string | `"latest"` |  |
+| monitoring.exporter.imagePullSecret | string | `""` | Custom pull secret for container in patch job |
+| monitoring.exporter.annotations | object | `{}` |  |
+| monitoring.exporter.port | int | `9199` |  |
+| monitoring.exporter.service.enabled | bool | `true` | Create a service for exporter |
+| monitoring.exporter.service.type | string | `"ClusterIP"` | Service type |
+| monitoring.exporter.service.loadBalancerIP | string | `""` | Set specific loadBalancerIP when serviceType is LoadBalancer (see https://kubernetes.io/docs/concepts/services-networking/service/#loadbalancer) |
+| monitoring.exporter.service.nodePort | int | `30199` | Node port for service |
+| monitoring.exporter.service.annotations | object | `{}` | Annotations for exporter service |
 | autoscaling.enabled | bool | `false` | Enable autoscaling. Implies installing KEDA |
 | autoscaling.enableWithExistingKEDA | bool | `false` | Enable autoscaling without automatically installing KEDA |
 | autoscaling.scalingType | string | `"job"` | Which type of KEDA scaling to use: job or deployment |
-| autoscaling.authenticationRef | object | `{"annotations":{"helm.sh/hook":"post-install,post-upgrade,post-rollback","helm.sh/hook-weight":"-2"},"name":""}` | Specify an external KEDA TriggerAuthentication resource is used for scaler triggers config. Apply for all browser nodes |
-| autoscaling.annotations | object | `{"helm.sh/hook":"post-install,post-upgrade,post-rollback","helm.sh/hook-weight":"1"}` | Annotations for KEDA resources: ScaledObject and ScaledJob |
+| autoscaling.authenticationRef | object | `{"annotations":{},"name":""}` | Specify an external KEDA TriggerAuthentication resource is used for scaler triggers config. Apply for all browser nodes |
+| autoscaling.annotations | object | `{}` | Annotations for KEDA resources: ScaledObject and ScaledJob |
 | autoscaling.patchObjectFinalizers.nameOverride | string | `nil` | Override the name of the patch job |
-| autoscaling.patchObjectFinalizers.enabled | bool | `true` | Enable patching finalizers for KEDA scaled resources. Workaround for Hook post-upgrade selenium-grid/templates/x-node-hpa.yaml failed: object is being deleted: scaledobjects.keda.sh "x" already exists |
+| autoscaling.patchObjectFinalizers.enabled | bool | `false` | Enable patching finalizers for KEDA scaled resources. Workaround for Hook post-upgrade selenium-grid/templates/x-node-hpa.yaml failed: object is being deleted: scaledobjects.keda.sh "x" already exists |
 | autoscaling.patchObjectFinalizers.activeDeadlineSeconds | int | `120` | Deadline (in seconds) for patch job to complete |
-| autoscaling.patchObjectFinalizers.annotations | object | `{"helm.sh/hook":"post-install,post-upgrade,post-rollback,pre-delete","helm.sh/hook-delete-policy":"hook-succeeded,before-hook-creation","helm.sh/hook-weight":"-1"}` | Annotations for patch job |
+| autoscaling.patchObjectFinalizers.annotations | object | `{}` | Annotations for patch job |
 | autoscaling.patchObjectFinalizers.serviceAccount | string | `""` | Define an external service account name contains permissions to patch KEDA scaled resources |
 | autoscaling.patchObjectFinalizers.imagePullSecret | string | `""` | Custom pull secret for container in patch job |
 | autoscaling.patchObjectFinalizers.resources | object | `{"limits":{"cpu":"50m","memory":"50Mi"},"requests":{"cpu":"10m","memory":"10Mi"}}` | Define resources for container in patch job |
@@ -508,7 +519,7 @@ A Helm chart for creating a Selenium Grid Server in Kubernetes
 | videoRecorder.s3 | object | `{"args":[],"command":[],"extraEnvironmentVariables":null,"imageName":"aws-cli","imagePullPolicy":"IfNotPresent","imageRegistry":"bitnami","imageTag":"latest","securityContext":{"runAsUser":0}}` | Container spec for the uploader if above it is defined as "uploader.name: s3" |
 | customLabels | object | `{}` | Custom labels for k8s resources |
 | keda.image | object | `{"keda":{"registry":"selenium","repository":"keda","tag":"2.15.1-selenium-grid-20240922"},"metricsApiServer":{"registry":"selenium","repository":"keda-metrics-apiserver","tag":"2.15.1-selenium-grid-20240922"},"webhooks":{"registry":"selenium","repository":"keda-admission-webhooks","tag":"2.15.1-selenium-grid-20240922"}}` | Specify image for KEDA components |
-| keda.additionalAnnotations | string | `nil` | Annotations for KEDA resources |
+| keda.crds.additionalAnnotations."helm.sh/hook" | string | `"pre-install,pre-upgrade,pre-rollback,post-delete"` |  |
 | keda.http.timeout | int | `60000` |  |
 | keda.webhooks | object | `{"enabled":false}` | Enable KEDA admission webhooks component |
 | ingress-nginx | object | `{"controller":{"admissionWebhooks":{"enabled":false}}}` | Configuration for dependency chart ingress-nginx |

diff --git a/charts/selenium-grid/templates/_helpers.tpl b/charts/selenium-grid/templates/_helpers.tpl
@@ -659,6 +659,13 @@ Graphql Url of the hub or the router
 {{- printf "%s/graphql" (include "seleniumGrid.server.url" $) -}}
 {{- end -}}
 
+{{/*
+Graphql Url for internal monitoring exporter
+*/}}
+{{- define "seleniumGrid.monitoring.graphqlURL" -}}
+{{- printf "%s://%s%s%s%s/graphql" (include "seleniumGrid.server.url.schema" $) (include "seleniumGrid.url.basicAuth" $) (include "seleniumGrid.server.url.host" $) (include "seleniumGrid.server.url.port" $) (include "seleniumGrid.url.subPath" $) -}}
+{{- end -}}
+
 {{- define "seleniumGrid.url.schema" -}}
 {{- $schema := "http" -}}
 {{- if or (eq (include "seleniumGrid.server.secureConnection" $) "true") (eq (include "seleniumGrid.ingress.secureConnection" $) "true") -}}

diff --git a/charts/selenium-grid/templates/_nameHelpers.tpl b/charts/selenium-grid/templates/_nameHelpers.tpl
@@ -56,6 +56,13 @@ component.autoscaling: "{{ .Release.Name }}"
 {{- end -}}
 {{- end -}}
 
+{{/*
+Selenium metrics exporter fullname
+*/}}
+{{- define "seleniumGrid.monitoring.exporter.fullname" -}}
+{{- tpl (default (include "seleniumGrid.component.name" (list "selenium-metrics-exporter" $)) $.Values.monitoring.nameOverride) $ | trunc 63 | trimSuffix "-" -}}
+{{- end -}}
+
 {{/*
 Selenium Hub fullname
 */}}

diff --git a/charts/selenium-grid/templates/graphql-exporter-deployment.yaml b/charts/selenium-grid/templates/graphql-exporter-deployment.yaml
@@ -0,0 +1,33 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: {{ template "seleniumGrid.monitoring.exporter.fullname" $ }}
+  namespace: {{ .Release.Namespace }}
+  labels: &exporter_labels
+    app: {{ template "seleniumGrid.monitoring.exporter.fullname" $ }}
+    app.kubernetes.io/name: {{ template "seleniumGrid.monitoring.exporter.fullname" $ }}
+    {{- include "seleniumGrid.commonLabels" . | nindent 4 }}
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app: {{ template "seleniumGrid.monitoring.exporter.fullname" $ }}
+      app.kubernetes.io/name: {{ template "seleniumGrid.monitoring.exporter.fullname" $ }}
+  template:
+    metadata:
+      labels: *exporter_labels
+      annotations:
+        {{- with .Values.monitoring.exporter.annotations }}
+            {{- toYaml . | nindent 8 }}
+        {{- end }}
+    spec:
+    {{- if or .Values.global.seleniumGrid.imagePullSecret .Values.monitoring.exporter.imagePullSecret }}
+      imagePullSecrets:
+        - name: {{ default .Values.global.seleniumGrid.imagePullSecret .Values.monitoring.exporter.imagePullSecret }}
+    {{- end }}
+      containers:
+        - name: graphql-exporter
+          {{- $imageRegistry := default .Values.global.seleniumGrid.imageRegistry .Values.monitoring.exporter.imageRegistry }}
+          image: {{ printf "%s/%s:%s" $imageRegistry .Values.monitoring.exporter.imageName .Values.monitoring.exporter.imageTag | quote }}
+          ports:
+            - containerPort: {{ .Values.monitoring.exporter.port }}
diff --git a/charts/selenium-grid/templates/graphql-exporter-service.yaml b/charts/selenium-grid/templates/graphql-exporter-service.yaml
@@ -0,0 +1,25 @@
+apiVersion: v1
+kind: Service
+metadata:
+  name: {{ template "seleniumGrid.monitoring.exporter.fullname" $ }}
+  namespace: {{ .Release.Namespace }}
+  labels:
+    app: {{ template "seleniumGrid.monitoring.exporter.fullname" $ }}
+    app.kubernetes.io/name: {{ template "seleniumGrid.monitoring.exporter.fullname" $ }}
+    {{- include "seleniumGrid.commonLabels" . | nindent 4 }}
+spec:
+  selector:
+    app: {{ template "seleniumGrid.monitoring.exporter.fullname" $ }}
+    app.kubernetes.io/name: {{ template "seleniumGrid.monitoring.exporter.fullname" $ }}
+  type: {{ .Values.monitoring.exporter.service.type }}
+  {{- if and (eq .Values.monitoring.exporter.service.type "LoadBalancer") ( .Values.monitoring.exporter.service.loadBalancerIP ) }}
+  loadBalancerIP: {{ .Values.monitoring.exporter.service.loadBalancerIP }}
+  {{- end }}
+  ports:
+    - name: http-port
+      protocol: TCP
+      port: {{ .Values.monitoring.exporter.port }}
+      targetPort: {{ .Values.monitoring.exporter.port }}
+    {{- if and (eq .Values.monitoring.exporter.service.type "NodePort") .Values.monitoring.exporter.service.nodePort }}
+      nodePort: {{ .Values.monitoring.exporter.service.nodePort }}
+    {{- end }}
diff --git a/charts/selenium-grid/templates/patch-keda/patch-keda-objects-job.yaml b/charts/selenium-grid/templates/patch-keda/patch-keda-objects-job.yaml
@@ -34,9 +34,13 @@ spec:
           args:
             - |
               echo "Cleaning up ScaledObjects, ScaledJobs and HPAs for {{ .Release.Name }} when upgrading or disabling autoscaling."
-              kubectl patch ScaledObjects,ScaledJobs -n {{ .Release.Namespace }} -l component.autoscaling={{ .Release.Name }} -p '{"metadata":{"finalizers":null}}' || true ;
-              kubectl delete ScaledObjects,ScaledJobs -n {{ .Release.Namespace }} -l component.autoscaling={{ .Release.Name }} --wait || true ;
-              kubectl delete hpa -n {{ .Release.Namespace }} -l component.autoscaling={{ .Release.Name }} --wait || true ;
+              kubectl patch ScaledObjects -n {{ .Release.Namespace }} -l component.autoscaling={{ .Release.Name }} -p '{"metadata":{"finalizers":null}}' || true ;
+              kubectl patch ScaledJobs -n {{ .Release.Namespace }} -l component.autoscaling={{ .Release.Name }} -p '{"metadata":{"finalizers":null}}' || true ;
+              kubectl patch TriggerAuthentication -n {{ .Release.Namespace }} -l component.autoscaling={{ .Release.Name }} -p '{"metadata":{"finalizers":null}}' || true ;
+              kubectl delete ScaledObjects -n {{ .Release.Namespace }} -l component.autoscaling={{ .Release.Name }} || true ;
+              kubectl delete ScaledJobs -n {{ .Release.Namespace }} -l component.autoscaling={{ .Release.Name }} || true ;
+              kubectl delete TriggerAuthentication -n {{ .Release.Namespace }} -l component.autoscaling={{ .Release.Name }} || true ;
+              kubectl delete hpa -n {{ .Release.Namespace }} -l component.autoscaling={{ .Release.Name }} || true ;
         {{- with $.Values.autoscaling.patchObjectFinalizers.resources }}
           resources: {{ toYaml . | nindent 12 }}
         {{- end }}

diff --git a/charts/selenium-grid/values.yaml b/charts/selenium-grid/values.yaml
@@ -791,6 +791,25 @@ tracing:
 monitoring:
   enabled: false
   enabledWithExistingAgent: false
+  exporter:
+    imageRegistry: "ricardbejarano"
+    imageName: "graphql_exporter"
+    imageTag: "latest"
+    # -- Custom pull secret for container in patch job
+    imagePullSecret: ""
+    annotations: {}
+    port: 9199
+    service:
+      # -- Create a service for exporter
+      enabled: true
+      # -- Service type
+      type: ClusterIP
+      # -- Set specific loadBalancerIP when serviceType is LoadBalancer (see https://kubernetes.io/docs/concepts/services-networking/service/#loadbalancer)
+      loadBalancerIP: ""
+      # -- Node port for service
+      nodePort: 30199
+      # -- Annotations for exporter service
+      annotations: {}
 
 # Keda scaled object configuration
 autoscaling:
@@ -803,25 +822,18 @@ autoscaling:
   # -- Specify an external KEDA TriggerAuthentication resource is used for scaler triggers config. Apply for all browser nodes
   authenticationRef:
     name: ""
-    annotations:
-      "helm.sh/hook": post-install,post-upgrade,post-rollback
-      "helm.sh/hook-weight": "-2"
+    annotations: {}
   # -- Annotations for KEDA resources: ScaledObject and ScaledJob
-  annotations:
-    "helm.sh/hook": post-install,post-upgrade,post-rollback
-    "helm.sh/hook-weight": "1"
+  annotations: {}
   patchObjectFinalizers:
     # -- Override the name of the patch job
     nameOverride:
     # -- Enable patching finalizers for KEDA scaled resources. Workaround for Hook post-upgrade selenium-grid/templates/x-node-hpa.yaml failed: object is being deleted: scaledobjects.keda.sh "x" already exists
-    enabled: true
+    enabled: false
     # -- Deadline (in seconds) for patch job to complete
     activeDeadlineSeconds: 120
     # -- Annotations for patch job
-    annotations:
-      "helm.sh/hook": post-install,post-upgrade,post-rollback,pre-delete
-      "helm.sh/hook-delete-policy": hook-succeeded,before-hook-creation
-      "helm.sh/hook-weight": "-1"
+    annotations: {}
     # -- Define an external service account name contains permissions to patch KEDA scaled resources
     serviceAccount: ""
     # -- Custom pull secret for container in patch job
@@ -1562,8 +1574,9 @@ keda:
       registry: selenium
       repository: keda-admission-webhooks
       tag: "2.15.1-selenium-grid-20240922"
-  # -- Annotations for KEDA resources
-  additionalAnnotations:
+  crds:
+    additionalAnnotations:
+      "helm.sh/hook": pre-install,pre-upgrade,pre-rollback,post-delete
   http:
     timeout: 60000
   # -- Enable KEDA admission webhooks component

diff --git a/tests/SeleniumTests/__init__.py b/tests/SeleniumTests/__init__.py
@@ -129,7 +129,8 @@ def setUp(self):
         try:
             options = ChromeOptions()
             options.enable_downloads = SELENIUM_ENABLE_MANAGED_DOWNLOADS
-            options.add_argument('disable-features=DownloadBubble,DownloadBubbleV2')
+            if not SELENIUM_ENABLE_MANAGED_DOWNLOADS:
+                options.add_argument('disable-features=DownloadBubble,DownloadBubbleV2')
             if TEST_ADD_CAPS_RECORD_VIDEO:
                 options.set_capability('se:recordVideo', True)
             options.set_capability('se:name', f"{self._testMethodName} ({self.__class__.__name__})")
@@ -166,7 +167,8 @@ def setUp(self):
         try:
             options = EdgeOptions()
             options.enable_downloads = SELENIUM_ENABLE_MANAGED_DOWNLOADS
-            options.add_argument('disable-features=DownloadBubble,DownloadBubbleV2')
+            if not SELENIUM_ENABLE_MANAGED_DOWNLOADS:
+                options.add_argument('disable-features=DownloadBubble,DownloadBubbleV2')
             if TEST_ADD_CAPS_RECORD_VIDEO:
                 options.set_capability('se:recordVideo', True)
             options.set_capability('se:name', f"{self._testMethodName} ({self.__class__.__name__})")
@@ -189,13 +191,14 @@ class FirefoxTests(SeleniumGenericTests):
     def setUp(self):
         try:
             profile = webdriver.FirefoxProfile()
-            profile.set_preference("browser.download.manager.showWhenStarting", False)
-            profile.set_preference("browser.helperApps.neverAsk.saveToDisk", "*/*")
+            options = FirefoxOptions()
+            options.enable_downloads = SELENIUM_ENABLE_MANAGED_DOWNLOADS
+            if not SELENIUM_ENABLE_MANAGED_DOWNLOADS:
+                profile.set_preference("browser.download.manager.showWhenStarting", False)
+                profile.set_preference("browser.helperApps.neverAsk.saveToDisk", "*/*")
             profile.set_preference('intl.accept_languages', 'vi-VN,vi')
             profile.set_preference('intl.locale.requested', 'vi-VN,vi')
-            options = FirefoxOptions()
             options.profile = profile
-            options.enable_downloads = SELENIUM_ENABLE_MANAGED_DOWNLOADS
             if TEST_ADD_CAPS_RECORD_VIDEO:
                 options.set_capability('se:recordVideo', True)
             options.set_capability('se:name', f"{self._testMethodName} ({self.__class__.__name__})")

diff --git a/tests/charts/ci/base-tls-values.yaml b/tests/charts/ci/base-tls-values.yaml
@@ -3,18 +3,17 @@ registrationSecret:
   value: "HappyTestOps"
 
 monitoring:
-  enabled: false
+  enabled: true
 
 kube-prometheus-stack:
   cleanPrometheusOperatorObjectNames: true
   defaultRules:
     create: true
     annotations:
-      "helm.sh/hook": post-install,post-upgrade,post-rollback
+      "helm.sh/hook": pre-install,pre-upgrade,pre-rollback,post-delete
   alertmanager:
     enabled: true
     annotations:
-      "helm.sh/hook": post-install,post-upgrade,post-rollback
     ingress:
       enabled: true
       ingressClassName: nginx
@@ -26,7 +25,6 @@ kube-prometheus-stack:
     forceDeployDatasources: true
     forceDeployDashboards: true
     annotations:
-      "helm.sh/hook": post-install,post-upgrade,post-rollback
     ingress:
       enabled: true
       ingressClassName: nginx
@@ -35,13 +33,26 @@ kube-prometheus-stack:
   prometheus:
     enabled: true
     annotations:
-      "helm.sh/hook": post-install,post-upgrade,post-rollback
     ingress:
       enabled: true
       ingressClassName: nginx
       hosts:
         - pts.selenium-grid.prod
     prometheusSpec:
+      additionalScrapeConfigs:
+        - job_name: "selenium-grid-analytics"
+          metrics_path: "/query"
+          bearer_token: ""
+          params:
+            endpoint:
+              - 'http://selenium-router.selenium:4444/graphql'
+            query:
+              - |
+                { grid { sessionCount, maxSession, totalSlots, nodeCount, sessionQueueSize }, nodesInfo { nodes { id, status, sessionCount, maxSession, slotCount, stereotypes, sessions { id, capabilities, sessionDurationMillis, slot { id, stereotype } } } }, sessionsInfo { sessionQueueRequests } }
+            zoneTag: []
+          static_configs:
+            - targets:
+                - 'selenium-metrics-exporter.selenium:9199'
       storageSpec:
         volumeClaimTemplate:
           spec:

diff --git a/tests/charts/refValues/sample-aws.yaml b/tests/charts/refValues/sample-aws.yaml
@@ -33,8 +33,6 @@ isolateComponents: true
 autoscaling:
   enabled: true
   scalingType: job
-  annotations:
-    helm.sh/hook: post-install,post-upgrade,post-rollback
   scaledOptions:
     minReplicaCount: 0
     maxReplicaCount: 8

diff --git a/tests/charts/refValues/simplex-minikube.yaml b/tests/charts/refValues/simplex-minikube.yaml
@@ -45,8 +45,6 @@ autoscaling:
 #  enabled: true
   enableWithExistingKEDA: true
   scalingType: job
-  annotations:
-    helm.sh/hook: post-install,post-upgrade,post-rollback
   scaledOptions:
     minReplicaCount: 0
     maxReplicaCount: 8