chart(feat): add graphql metrics exporter for monitoring

Signed-off-by: Viet Nguyen Duc <[email protected]>

Base/Dockerfile

@@ -1,4 +1,4 @@
-FROM ubuntu:noble-20240827.1
+FROM ubuntu:noble-20240904.1
 ARG AUTHORS=SeleniumHQ
 LABEL authors="${AUTHORS} <[email protected]>"
 LABEL org.opencontainers.image.source="https://github.com/${AUTHORS}/docker-selenium"

Makefile

@@ -29,7 +29,7 @@ SBOM_OUTPUT := $(or $(SBOM_OUTPUT),$(SBOM_OUTPUT),package_versions.txt)
 KEDA_TAG_PREV_VERSION := $(or $(KEDA_TAG_PREV_VERSION),$(KEDA_TAG_PREV_VERSION),2.15.1-selenium-grid)
 KEDA_TAG_VERSION := $(or $(KEDA_TAG_VERSION),$(KEDA_TAG_VERSION),2.15.1-selenium-grid)
 KEDA_BASED_NAME := $(or $(KEDA_BASED_NAME),$(KEDA_BASED_NAME),ndviet)
-KEDA_BASED_TAG := $(or $(KEDA_BASED_TAG),$(KEDA_BASED_TAG),2.15.1-selenium-grid)
+KEDA_BASED_TAG := $(or $(KEDA_BASED_TAG),$(KEDA_BASED_TAG),2.15.1-selenium-grid-20241007)
 
 all: hub \
 	distributor \

NodeBase/Dockerfile

@@ -118,6 +118,7 @@ RUN --mount=type=secret,id=SEL_PASSWD \
   && unzip -x websockify.zip \
   && rm websockify.zip \
   && mv websockify-${WEBSOCKIFY_VERSION} /opt/bin/noVNC/utils/websockify \
+  && chmod +x /opt/bin/noVNC/utils/websockify/run \
   && rm -rf /opt/bin/noVNC/utils/websockify/docker /opt/bin/noVNC/utils/websockify/tests \
 #========================================================================
 # Run this command for executable file permissions for /dev/shm when    #

charts/selenium-grid/CONFIGURATION.md

@@ -79,7 +79,7 @@ A Helm chart for creating a Selenium Grid Server in Kubernetes
 | serviceAccount.create | bool | `true` | Create a service account for all components. If using an external service account, set to false and provide its name in `nameOverride` below |
 | serviceAccount.nameOverride | string | `nil` | Override to use an external service account |
 | serviceAccount.annotations | object | `{}` | Annotations for the service account |
-| rbacRole | object | `{"annotations":{},"create":true,"nameOverride":null,"rules":[{"apiGroups":["keda.sh"],"resources":["scaledjobs"],"verbs":["get","list","patch","update","delete"]},{"apiGroups":["keda.sh"],"resources":["scaledobjects"],"verbs":["get","list","patch","update","delete"]},{"apiGroups":["autoscaling"],"resources":["horizontalpodautoscalers"],"verbs":["get","list","patch","update","delete"]}]}` | RBAC settings for patching finalizers KEDA scaled resources |
+| rbacRole | object | `{"annotations":{},"create":true,"nameOverride":null,"rules":[{"apiGroups":["keda.sh"],"resources":["scaledjobs"],"verbs":["get","list","patch","update","delete"]},{"apiGroups":["keda.sh"],"resources":["scaledobjects"],"verbs":["get","list","patch","update","delete"]},{"apiGroups":["keda.sh"],"resources":["triggerauthentication"],"verbs":["get","list","patch","update","delete"]},{"apiGroups":["autoscaling"],"resources":["horizontalpodautoscalers"],"verbs":["get","list","patch","update","delete"]}]}` | RBAC settings for patching finalizers KEDA scaled resources |
 | rbacRole.create | bool | `true` | Enable to create RBAC role to access few KEDA resources. If using an external role, set to false and provide its name in `nameOverride` below |
 | rbacRole.nameOverride | string | `nil` | Override resource name or provide an external role name |
 | rbacRoleBinding | object | `{"annotations":{},"create":true,"nameOverride":null,"roleRef":{"apiGroup":"rbac.authorization.k8s.io","kind":"Role"},"subjects":[{"kind":"ServiceAccount"}]}` | RBAC role binding settings for patching finalizers KEDA scaled resources |
@@ -307,6 +307,22 @@ A Helm chart for creating a Selenium Grid Server in Kubernetes
 | tracing.ingress.paths | list | `[{"backend":{"service":{"name":"{{ .Release.Name }}-jaeger-query","port":{"number":16686}}},"path":"/jaeger","pathType":"Prefix"}]` | Configure paths for Jaeger ingress resource |
 | monitoring.enabled | bool | `false` |  |
 | monitoring.enabledWithExistingAgent | bool | `false` |  |
+| monitoring.exporter.nameOverride | string | `"selenium-metrics-exporter"` |  |
+| monitoring.exporter.imageRegistry | string | `"ricardbejarano"` |  |
+| monitoring.exporter.imageName | string | `"graphql_exporter"` |  |
+| monitoring.exporter.imageTag | string | `"latest"` |  |
+| monitoring.exporter.imagePullSecret | string | `""` | Custom pull secret for container in patch job |
+| monitoring.exporter.annotations | object | `{}` |  |
+| monitoring.exporter.port | int | `9199` |  |
+| monitoring.exporter.service.enabled | bool | `true` | Create a service for exporter |
+| monitoring.exporter.service.type | string | `"ClusterIP"` | Service type |
+| monitoring.exporter.service.loadBalancerIP | string | `""` | Set specific loadBalancerIP when serviceType is LoadBalancer (see https://kubernetes.io/docs/concepts/services-networking/service/#loadbalancer) |
+| monitoring.exporter.service.nodePort | int | `30199` | Node port for service |
+| monitoring.exporter.service.annotations | object | `{}` | Annotations for exporter service |
+| monitoring.exporter.replicas | int | `1` |  |
+| monitoring.additionalScrapeConfigs.key | string | `"selenium-grid.yaml"` |  |
+| monitoring.additionalScrapeConfigs.value | string | `""` |  |
+| monitoring.annotations | object | `{}` |  |
 | autoscaling.enabled | bool | `false` | Enable autoscaling. Implies installing KEDA |
 | autoscaling.enableWithExistingKEDA | bool | `false` | Enable autoscaling without automatically installing KEDA |
 | autoscaling.scalingType | string | `"job"` | Which type of KEDA scaling to use: job or deployment |
@@ -512,6 +528,6 @@ A Helm chart for creating a Selenium Grid Server in Kubernetes
 | keda.http.timeout | int | `60000` |  |
 | keda.webhooks | object | `{"enabled":false}` | Enable KEDA admission webhooks component |
 | ingress-nginx | object | `{"controller":{"admissionWebhooks":{"enabled":false}}}` | Configuration for dependency chart ingress-nginx |
-| kube-prometheus-stack | object | `{"cleanPrometheusOperatorObjectNames":true}` | Configuration for dependency chart kube-prometheus-stack |
+| kube-prometheus-stack | object | `{"cleanPrometheusOperatorObjectNames":true,"prometheus":{"prometheusSpec":{"additionalScrapeConfigsSecret":{"enabled":true,"key":"selenium-grid.yaml","name":"selenium-metrics-exporter"}}}}` | Configuration for dependency chart kube-prometheus-stack |
 | jaeger | object | `{"agent":{"enabled":false},"allInOne":{"enabled":true,"extraEnv":[{"name":"QUERY_BASE_PATH","value":"/jaeger"}]},"collector":{"enabled":false},"provisionDataStore":{"cassandra":false},"query":{"enabled":false},"storage":{"type":"badger"}}` | Configuration for dependency chart jaeger |
 

charts/selenium-grid/configs/scrape/selenium-grid.yaml

@@ -0,0 +1,15 @@
+- job_name: "selenium-grid-analytics"
+  metrics_path: "/query"
+  bearer_token: ""
+  params:
+    endpoint:
+      - '{{ template "seleniumGrid.monitoring.graphqlURL" $ }}'
+    query:
+      - |
+        { grid { sessionCount, maxSession, totalSlots, nodeCount, sessionQueueSize }, nodesInfo { nodes { id, status, sessionCount, maxSession, slotCount, stereotypes, sessions { id, capabilities, sessionDurationMillis, slot { id, stereotype } } } }, sessionsInfo { sessionQueueRequests } }
+    zoneTag: []
+  tls_config:
+    insecure_skip_verify: true
+  static_configs:
+    - targets:
+        - '{{ template "seleniumGrid.monitoring.exporter.fullname" $ }}.{{ .Release.Namespace }}:{{ $.Values.monitoring.exporter.port }}'

charts/selenium-grid/templates/_helpers.tpl

@@ -659,6 +659,13 @@ Graphql Url of the hub or the router
 {{- printf "%s/graphql" (include "seleniumGrid.server.url" $) -}}
 {{- end -}}
 
+{{/*
+Graphql Url for internal monitoring exporter
+*/}}
+{{- define "seleniumGrid.monitoring.graphqlURL" -}}
+{{- printf "%s://%s%s%s%s/graphql" (include "seleniumGrid.server.url.schema" .) (include "seleniumGrid.url.basicAuth" .) (include "seleniumGrid.server.url.host" .) (include "seleniumGrid.server.url.port" .) (include "seleniumGrid.url.subPath" .) -}}
+{{- end -}}
+
 {{- define "seleniumGrid.url.schema" -}}
 {{- $schema := "http" -}}
 {{- if or (eq (include "seleniumGrid.server.secureConnection" $) "true") (eq (include "seleniumGrid.ingress.secureConnection" $) "true") -}}

charts/selenium-grid/templates/_nameHelpers.tpl

@@ -56,6 +56,13 @@ component.autoscaling: "{{ .Release.Name }}"
 {{- end -}}
 {{- end -}}
 
+{{/*
+Selenium metrics exporter fullname
+*/}}
+{{- define "seleniumGrid.monitoring.exporter.fullname" -}}
+{{- tpl (default (include "seleniumGrid.component.name" (list "selenium-metrics-exporter" $)) $.Values.monitoring.exporter.nameOverride) $ | trunc 63 | trimSuffix "-" -}}
+{{- end -}}
+
 {{/*
 Selenium Hub fullname
 */}}

charts/selenium-grid/templates/monitoring-exporter-deployment.yaml

@@ -0,0 +1,35 @@
+{{- if $.Values.monitoring.enabled }}
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: {{ template "seleniumGrid.monitoring.exporter.fullname" $ }}
+  namespace: {{ .Release.Namespace }}
+  labels: &exporter_labels
+    app: {{ template "seleniumGrid.monitoring.exporter.fullname" $ }}
+    app.kubernetes.io/name: {{ template "seleniumGrid.monitoring.exporter.fullname" $ }}
+    {{- include "seleniumGrid.commonLabels" . | nindent 4 }}
+spec:
+  replicas: {{ .Values.monitoring.exporter.replicas }}
+  selector:
+    matchLabels:
+      app: {{ template "seleniumGrid.monitoring.exporter.fullname" $ }}
+      app.kubernetes.io/name: {{ template "seleniumGrid.monitoring.exporter.fullname" $ }}
+  template:
+    metadata:
+      labels: *exporter_labels
+      annotations:
+        {{- with .Values.monitoring.exporter.annotations }}
+            {{- toYaml . | nindent 8 }}
+        {{- end }}
+    spec:
+    {{- if or .Values.global.seleniumGrid.imagePullSecret .Values.monitoring.exporter.imagePullSecret }}
+      imagePullSecrets:
+        - name: {{ default .Values.global.seleniumGrid.imagePullSecret .Values.monitoring.exporter.imagePullSecret }}
+    {{- end }}
+      containers:
+        - name: graphql-exporter
+          {{- $imageRegistry := default .Values.global.seleniumGrid.imageRegistry .Values.monitoring.exporter.imageRegistry }}
+          image: {{ printf "%s/%s:%s" $imageRegistry .Values.monitoring.exporter.imageName .Values.monitoring.exporter.imageTag | quote }}
+          ports:
+            - containerPort: {{ .Values.monitoring.exporter.port }}
+  {{- end }}

charts/selenium-grid/templates/monitoring-exporter-service.yaml

@@ -0,0 +1,27 @@
+{{- if $.Values.monitoring.enabled }}
+apiVersion: v1
+kind: Service
+metadata:
+  name: {{ template "seleniumGrid.monitoring.exporter.fullname" $ }}
+  namespace: {{ .Release.Namespace }}
+  labels:
+    app: {{ template "seleniumGrid.monitoring.exporter.fullname" $ }}
+    app.kubernetes.io/name: {{ template "seleniumGrid.monitoring.exporter.fullname" $ }}
+    {{- include "seleniumGrid.commonLabels" . | nindent 4 }}
+spec:
+  selector:
+    app: {{ template "seleniumGrid.monitoring.exporter.fullname" $ }}
+    app.kubernetes.io/name: {{ template "seleniumGrid.monitoring.exporter.fullname" $ }}
+  type: {{ .Values.monitoring.exporter.service.type }}
+  {{- if and (eq .Values.monitoring.exporter.service.type "LoadBalancer") ( .Values.monitoring.exporter.service.loadBalancerIP ) }}
+  loadBalancerIP: {{ .Values.monitoring.exporter.service.loadBalancerIP }}
+  {{- end }}
+  ports:
+    - name: http-port
+      protocol: TCP
+      port: {{ .Values.monitoring.exporter.port }}
+      targetPort: {{ .Values.monitoring.exporter.port }}
+    {{- if and (eq .Values.monitoring.exporter.service.type "NodePort") .Values.monitoring.exporter.service.nodePort }}
+      nodePort: {{ .Values.monitoring.exporter.service.nodePort }}
+    {{- end }}
+  {{- end }}

charts/selenium-grid/templates/monitoring-scape-secret.yaml

@@ -0,0 +1,29 @@
+{{- if $.Values.monitoring.enabled }}
+apiVersion: v1
+kind: Secret
+metadata:
+  name: {{ template "seleniumGrid.monitoring.exporter.fullname" $ }}
+  namespace: {{ .Release.Namespace }}
+  annotations:
+  {{- with .Values.monitoring.annotations }}
+    {{- toYaml . | nindent 4 }}
+  {{- end }}
+  labels:
+    {{- include "seleniumGrid.commonLabels" $ | nindent 4 }}
+    {{- with .Values.customLabels }}
+      {{- toYaml . | nindent 4 }}
+    {{- end }}
+type: Opaque
+data:
+  {{- range $path, $_ :=  .Files.Glob "configs/scrape/*.yaml" }}
+    {{- $fileName := base $path -}}
+    {{- $value := $.Values.monitoring.additionalScrapeConfigs.value -}}
+    {{- if empty $value }}
+  {{- $fileName | nindent 2 -}}: |
+    {{ toYaml (tpl ($.Files.Get $path) $ | b64enc) }}
+    {{- else }}
+  {{- $fileName | nindent 2 -}}: |
+    {{ toYaml ($value | b64enc) }}
+    {{- end }}
+  {{- end }}
+  {{- end }}

charts/selenium-grid/templates/patch-keda/patch-keda-objects-job.yaml

@@ -34,8 +34,8 @@ spec:
           args:
             - |
               echo "Cleaning up ScaledObjects, ScaledJobs and HPAs for {{ .Release.Name }} when upgrading or disabling autoscaling."
-              kubectl patch ScaledObjects,ScaledJobs -n {{ .Release.Namespace }} -l component.autoscaling={{ .Release.Name }} -p '{"metadata":{"finalizers":null}}' || true ;
-              kubectl delete ScaledObjects,ScaledJobs -n {{ .Release.Namespace }} -l component.autoscaling={{ .Release.Name }} --wait || true ;
+              kubectl get ScaledObjects,ScaledJobs,TriggerAuthentication -n {{ .Release.Namespace }} -l component.autoscaling={{ .Release.Name }} -o=json | jq '.metadata.finalizers = null' | kubectl apply -f - || true ;
+              kubectl delete ScaledObjects,ScaledJobs,TriggerAuthentication -n {{ .Release.Namespace }} -l component.autoscaling={{ .Release.Name }} --wait || true ;
               kubectl delete hpa -n {{ .Release.Namespace }} -l component.autoscaling={{ .Release.Name }} --wait || true ;
         {{- with $.Values.autoscaling.patchObjectFinalizers.resources }}
           resources: {{ toYaml . | nindent 12 }}

charts/selenium-grid/templates/trigger-auth.yaml

@@ -9,6 +9,8 @@ metadata:
       {{- toYaml . | nindent 4 }}
     {{- end }}
   labels:
+    app: {{ template "seleniumGrid.autoscaling.authenticationRef.fullname" $ }}
+    app.kubernetes.io/name: {{ template "seleniumGrid.autoscaling.authenticationRef.fullname" $ }}
     {{- include "seleniumGrid.commonLabels" $ | nindent 4 }}
     {{- include "seleniumGrid.autoscalingLabels" $ | nindent 4 }}
 spec:

charts/selenium-grid/values.yaml

@@ -153,6 +153,11 @@ rbacRole:
       resources:
         - scaledobjects
       verbs: [get, list, patch, update, delete]
+    - apiGroups:
+        - keda.sh
+      resources:
+        - triggerauthentication
+      verbs: [get, list, patch, update, delete]
     - apiGroups:
         - autoscaling
       resources:
@@ -791,6 +796,31 @@ tracing:
 monitoring:
   enabled: false
   enabledWithExistingAgent: false
+  exporter:
+    nameOverride: &scrape_name "selenium-metrics-exporter"
+    imageRegistry: "ricardbejarano"
+    imageName: "graphql_exporter"
+    imageTag: "latest"
+    # -- Custom pull secret for container in patch job
+    imagePullSecret: ""
+    annotations: {}
+    port: 9199
+    service:
+      # -- Create a service for exporter
+      enabled: true
+      # -- Service type
+      type: ClusterIP
+      # -- Set specific loadBalancerIP when serviceType is LoadBalancer (see https://kubernetes.io/docs/concepts/services-networking/service/#loadbalancer)
+      loadBalancerIP: ""
+      # -- Node port for service
+      nodePort: 30199
+      # -- Annotations for exporter service
+      annotations: {}
+    replicas: 1
+  additionalScrapeConfigs:
+    key: &scrape_key "selenium-grid.yaml"
+    value: ""
+  annotations: {}
 
 # Keda scaled object configuration
 autoscaling:
@@ -1581,6 +1611,12 @@ ingress-nginx:
 kube-prometheus-stack:
   # enabled: false
   cleanPrometheusOperatorObjectNames: true
+  prometheus:
+    prometheusSpec:
+      additionalScrapeConfigsSecret:
+        enabled: true
+        name: *scrape_name
+        key: *scrape_key
 
 # -- Configuration for dependency chart jaeger
 jaeger:

tests/SeleniumTests/__init__.py

@@ -129,7 +129,8 @@ def setUp(self):
         try:
             options = ChromeOptions()
             options.enable_downloads = SELENIUM_ENABLE_MANAGED_DOWNLOADS
-            options.add_argument('disable-features=DownloadBubble,DownloadBubbleV2')
+            if not SELENIUM_ENABLE_MANAGED_DOWNLOADS:
+                options.add_argument('disable-features=DownloadBubble,DownloadBubbleV2')
             if TEST_ADD_CAPS_RECORD_VIDEO:
                 options.set_capability('se:recordVideo', True)
             options.set_capability('se:name', f"{self._testMethodName} ({self.__class__.__name__})")
@@ -166,7 +167,8 @@ def setUp(self):
         try:
             options = EdgeOptions()
             options.enable_downloads = SELENIUM_ENABLE_MANAGED_DOWNLOADS
-            options.add_argument('disable-features=DownloadBubble,DownloadBubbleV2')
+            if not SELENIUM_ENABLE_MANAGED_DOWNLOADS:
+                options.add_argument('disable-features=DownloadBubble,DownloadBubbleV2')
             if TEST_ADD_CAPS_RECORD_VIDEO:
                 options.set_capability('se:recordVideo', True)
             options.set_capability('se:name', f"{self._testMethodName} ({self.__class__.__name__})")
@@ -189,13 +191,14 @@ class FirefoxTests(SeleniumGenericTests):
     def setUp(self):
         try:
             profile = webdriver.FirefoxProfile()
-            profile.set_preference("browser.download.manager.showWhenStarting", False)
-            profile.set_preference("browser.helperApps.neverAsk.saveToDisk", "*/*")
+            options = FirefoxOptions()
+            options.enable_downloads = SELENIUM_ENABLE_MANAGED_DOWNLOADS
+            if not SELENIUM_ENABLE_MANAGED_DOWNLOADS:
+                profile.set_preference("browser.download.manager.showWhenStarting", False)
+                profile.set_preference("browser.helperApps.neverAsk.saveToDisk", "*/*")
             profile.set_preference('intl.accept_languages', 'vi-VN,vi')
             profile.set_preference('intl.locale.requested', 'vi-VN,vi')
-            options = FirefoxOptions()
             options.profile = profile
-            options.enable_downloads = SELENIUM_ENABLE_MANAGED_DOWNLOADS
             if TEST_ADD_CAPS_RECORD_VIDEO:
                 options.set_capability('se:recordVideo', True)
             options.set_capability('se:name', f"{self._testMethodName} ({self.__class__.__name__})")

tests/charts/ci/base-tls-values.yaml

@@ -3,18 +3,17 @@ registrationSecret:
   value: "HappyTestOps"
 
 monitoring:
-  enabled: false
+  enabled: true
 
 kube-prometheus-stack:
   cleanPrometheusOperatorObjectNames: true
   defaultRules:
     create: true
     annotations:
-      "helm.sh/hook": post-install,post-upgrade,post-rollback
+      "helm.sh/hook": pre-install,pre-upgrade,pre-rollback,post-delete
   alertmanager:
     enabled: true
     annotations:
-      "helm.sh/hook": post-install,post-upgrade,post-rollback
     ingress:
       enabled: true
       ingressClassName: nginx
@@ -26,7 +25,6 @@ kube-prometheus-stack:
     forceDeployDatasources: true
     forceDeployDashboards: true
     annotations:
-      "helm.sh/hook": post-install,post-upgrade,post-rollback
     ingress:
       enabled: true
       ingressClassName: nginx
@@ -35,7 +33,6 @@ kube-prometheus-stack:
   prometheus:
     enabled: true
     annotations:
-      "helm.sh/hook": post-install,post-upgrade,post-rollback
     ingress:
       enabled: true
       ingressClassName: nginx

tests/charts/make/chart_setup_env.sh

@@ -83,7 +83,7 @@ elif [ "${CLUSTER}" = "minikube" ]; then
     rm -rf minikube-linux-$(dpkg --print-architecture)
     echo "==============================="
     echo "Installing Go"
-    GO_VERSION="1.22.3"
+    GO_VERSION="1.23.2"
     curl -sLO https://go.dev/dl/go$GO_VERSION.linux-$(dpkg --print-architecture).tar.gz
     sudo tar -xf go$GO_VERSION.linux-$(dpkg --print-architecture).tar.gz -C /usr/local
     rm -rf go$GO_VERSION.linux-$(dpkg --print-architecture).tar.gz*