update: handle graceful shutdown in Node container (#2337)

Signed-off-by: Viet Nguyen Duc <[email protected]>

.gitignore

@@ -24,6 +24,7 @@ StandaloneC*/generate_config
 StandaloneF*/generate_config
 StandaloneE*/generate_config
 videos
+Base/configs
 
 # Created by https://www.gitignore.io/api/virtualenv
 

Base/Dockerfile

@@ -127,7 +127,7 @@ RUN  if [ "$(dpkg --print-architecture)" = "amd64" ]; then \
 #======================================
 # Add Grid check script
 #======================================
-COPY --chown="${SEL_UID}:${SEL_GID}" check-grid.sh entry_point.sh /opt/bin/
+COPY --chown="${SEL_UID}:${SEL_GID}" check-grid.sh entry_point.sh configs/node/nodeGridUrl.sh configs/node/nodePreStop.sh /opt/bin/
 
 #======================================
 # Add Supervisor configuration file
@@ -149,6 +149,7 @@ ENV SE_BIND_HOST=false \
     SE_REJECT_UNSUPPORTED_CAPS=false \
     SE_OTEL_JAVA_GLOBAL_AUTOCONFIGURE_ENABLED=true \
     SE_OTEL_TRACES_EXPORTER="otlp" \
-    SE_SUPERVISORD_LOG_LEVEL="info"
+    SE_SUPERVISORD_LOG_LEVEL="info" \
+    SE_OPT_BIN="/opt/bin"
 
 CMD ["/opt/bin/entry_point.sh"]

Base/entry_point.sh

@@ -1,5 +1,6 @@
 #!/usr/bin/env bash
 
+NODE_CONFIG_DIRECTORY=${NODE_CONFIG_DIRECTORY:-$SE_OPT_BIN}
 #==============================================
 # OpenShift or non-sudo environments support
 # https://docs.openshift.com/container-platform/3.11/creating_images/guidelines.html#openshift-specific-guidelines
@@ -17,6 +18,10 @@ SUPERVISOR_PID=$!
 
 function shutdown {
     echo "Trapped SIGTERM/SIGINT/x so shutting down supervisord..."
+    if [ "${SE_NODE_GRACEFUL_SHUTDOWN}" = "true" ]; then
+        echo "Waiting for Selenium Node to shutdown gracefully..."
+        bash ${NODE_CONFIG_DIRECTORY}/nodePreStop.sh
+    fi
     kill -s SIGTERM ${SUPERVISOR_PID}
     wait ${SUPERVISOR_PID}
     echo "Shutdown complete"

Makefile

@@ -68,10 +68,11 @@ build: all
 ci: build test
 
 base:
+	rm -rf ./Base/configs/node && mkdir -p ./Base/configs/node && cp -r ./charts/selenium-grid/configs/node ./Base/configs
 	cd ./Base && docker buildx build --platform $(PLATFORMS) $(BUILD_ARGS) --build-arg VERSION=$(BASE_VERSION) --build-arg RELEASE=$(BASE_RELEASE) --build-arg AUTHORS=$(AUTHORS) -t $(NAME)/base:$(TAG_VERSION) .
 
 base_nightly:
-	cd ./Base && docker buildx build --platform $(PLATFORMS) $(BUILD_ARGS) --build-arg VERSION=$(BASE_VERSION_NIGHTLY) --build-arg RELEASE=$(BASE_RELEASE_NIGHTLY) --build-arg AUTHORS=$(AUTHORS) -t $(NAME)/base:$(TAG_VERSION) .
+	BASE_VERSION=$(BASE_VERSION_NIGHTLY) BASE_RELEASE=$(BASE_RELEASE_NIGHTLY) make base
 
 hub: base
 	cd ./Hub && docker buildx build --platform $(PLATFORMS) $(BUILD_ARGS) $(FROM_IMAGE_ARGS) -t $(NAME)/hub:$(TAG_VERSION) .
@@ -557,7 +558,8 @@ test_parallel: hub chrome firefox edge chromium
 			cd ./tests || true ; \
 			echo TAG=$(TAG_VERSION) > .env ; \
 			echo TEST_DRAIN_AFTER_SESSION_COUNT=$(or $(TEST_DRAIN_AFTER_SESSION_COUNT), 0) >> .env ; \
-			echo TEST_PARALLEL_HARDENING=$(or $(TEST_PARALLEL_HARDENING), "false") >> .env ; \
+			echo TEST_PARALLEL_HARDENING=$(or $(TEST_PARALLEL_HARDENING), "true") >> .env ; \
+			echo TEST_PARALLEL_COUNT=$(or $(TEST_PARALLEL_COUNT), 5) >> .env ; \
 			echo LOG_LEVEL=$(or $(LOG_LEVEL), "INFO") >> .env ; \
 			echo REQUEST_TIMEOUT=$(or $(REQUEST_TIMEOUT), 300) >> .env ; \
 			echo NODE=$$node >> .env ; \
@@ -569,11 +571,14 @@ test_parallel: hub chrome firefox edge chromium
 					echo NODE_CHROME=chromium >> .env ; \
 			fi; \
 			echo TEST_PLATFORMS=$(PLATFORMS) >> .env ; \
-			DOCKER_DEFAULT_PLATFORM=$(PLATFORMS) docker compose --profile $(PLATFORMS) -f docker-compose-v3-test-parallel.yml up --no-log-prefix --exit-code-from tests ; \
-	done
+			export $$(cat .env | xargs) ; \
+			DOCKER_DEFAULT_PLATFORM=$(PLATFORMS) docker compose --profile $(PLATFORMS) -f docker-compose-v3-test-parallel.yml up -d --no-log-prefix ; \
+			RUN_IN_DOCKER_COMPOSE=true bash ./bootstrap.sh $$node ; \
+	done ; \
+	docker compose -f docker-compose-v3-test-parallel.yml down
 
 test_video_dynamic_name:
-	VIDEO_FILE_NAME=auto TEST_DELAY_AFTER_TEST=10 \
+	VIDEO_FILE_NAME=auto TEST_DELAY_AFTER_TEST=0 \
 	make test_video
 
 # This should run on its own CI job. There is no need to combine it with the other tests.
@@ -636,7 +641,7 @@ test_node_relay: hub node_base standalone_firefox
 			echo ANDROID_BASED_IMAGE=$(or $(ANDROID_BASED_IMAGE),docker-android) >> .env ; \
 			echo ANDROID_BASED_TAG=$(or $(ANDROID_BASED_TAG),emulator_14.0) >> .env ; \
 			echo ANDROID_PLATFORM_API=$(or $(ANDROID_PLATFORM_API),14) >> .env ; \
-			echo TEST_DELAY_AFTER_TEST=$(or $(TEST_DELAY_AFTER_TEST), 15) >> .env ; \
+			echo TEST_DELAY_AFTER_TEST=$(or $(TEST_DELAY_AFTER_TEST), 0) >> .env ; \
 			echo NODE=$$node >> .env ; \
 			echo TEST_NODE_RELAY=$$node >> .env ; \
 			echo UID=$$(id -u) >> .env ; \
@@ -688,7 +693,7 @@ test_node_docker: hub standalone_docker standalone_chrome standalone_firefox sta
 			echo LOG_LEVEL=$(or $(LOG_LEVEL), "INFO") >> .env ; \
 			echo REQUEST_TIMEOUT=$(or $(REQUEST_TIMEOUT), 300) >> .env ; \
 			echo SELENIUM_ENABLE_MANAGED_DOWNLOADS=$(or $(SELENIUM_ENABLE_MANAGED_DOWNLOADS), "false") >> .env ; \
-			echo TEST_DELAY_AFTER_TEST=$(or $(TEST_DELAY_AFTER_TEST), 5) >> .env ; \
+			echo TEST_DELAY_AFTER_TEST=$(or $(TEST_DELAY_AFTER_TEST), 0) >> .env ; \
 			echo NODE=$$node >> .env ; \
 			echo UID=$$(id -u) >> .env ; \
 			echo BINDING_VERSION=$(BINDING_VERSION) >> .env ; \
@@ -765,7 +770,7 @@ chart_test_template:
 	./tests/charts/bootstrap.sh
 
 chart_test_autoscaling_disabled:
-	PLATFORMS=$(PLATFORMS) TEST_CHROMIUM=true RELEASE_NAME=selenium SELENIUM_GRID_AUTOSCALING=false TEST_DELAY_AFTER_TEST=15 CHART_ENABLE_TRACING=true \
+	PLATFORMS=$(PLATFORMS) TEST_CHROMIUM=true RELEASE_NAME=selenium SELENIUM_GRID_AUTOSCALING=false TEST_DELAY_AFTER_TEST=0 CHART_ENABLE_TRACING=true \
 	SECURE_INGRESS_ONLY_GENERATE=true SELENIUM_GRID_PROTOCOL=https SELENIUM_GRID_HOST=$$(hostname -i) SELENIUM_GRID_PORT=443 \
 	VERSION=$(TAG_VERSION) VIDEO_TAG=$(FFMPEG_TAG_VERSION)-$(BUILD_DATE) NAMESPACE=$(NAMESPACE) BINDING_VERSION=$(BINDING_VERSION) \
 	./tests/charts/make/chart_test.sh NoAutoscaling

NodeBase/selenium.conf

@@ -57,6 +57,7 @@ stderr_capture_maxbytes=50MB
 priority=15
 command=bash -c "/opt/bin/start-selenium-node.sh; EXIT_CODE=$?; kill -s SIGINT `cat /var/run/supervisor/supervisord.pid`; exit $EXIT_CODE"
 stopasgroup = true
+killasgroup=true
 autostart=true
 autorestart=false
 startsecs=0

NodeChromium/Dockerfile

@@ -7,9 +7,9 @@ LABEL authors=${AUTHORS}
 USER root
 
 # Install Chromium
-RUN apt-get update -qqy \
-  && apt-get install -y software-properties-common \
-  && add-apt-repository ppa:xtradeb/apps \
+RUN echo "deb http://deb.debian.org/debian/ sid main" >> /etc/apt/sources.list \
+  && apt-key adv --keyserver hkp://keyserver.ubuntu.com:80 --recv-keys 0E98404D386FA1D9 6ED0E7B82643E131 \
+  && apt-get update -qqy \
   && apt-get -qqy install chromium chromium-l10n \
   && rm -rf /var/lib/apt/lists/* /var/cache/apt/*
 

Video/supervisord.conf

@@ -14,9 +14,11 @@ minprocs=200                                  ; (min. avail process descriptors;
 [program:video-recording]
 priority=0
 command=/opt/bin/video.sh
+stopasgroup = true
+killasgroup=true
 autostart=true
+startsecs=0
 autorestart=true
-stopsignal=INT
 
 ;Logs (all activity redirected to stdout so it can be seen through "docker logs"
 redirect_stderr=true
@@ -28,7 +30,6 @@ priority=5
 command=python3 /opt/bin/video_ready.py
 autostart=true
 autorestart=true
-stopsignal=INT
 
 ;Logs (all activity redirected to stdout so it can be seen through "docker logs"
 redirect_stderr=true
@@ -40,7 +41,6 @@ priority=10
 command=bash -c "if [ ${SE_VIDEO_INTERNAL_UPLOAD} = "true" ]; then /opt/bin/upload.sh; fi"
 autostart=%(ENV_SE_VIDEO_INTERNAL_UPLOAD)s
 autorestart=%(ENV_SE_VIDEO_INTERNAL_UPLOAD)s
-stopsignal=INT
 
 ;Logs (all activity redirected to stdout so it can be seen through "docker logs"
 redirect_stderr=true

charts/selenium-grid/configs/distributor/distributorProbe.sh

@@ -4,7 +4,7 @@ max_time=3
 retry_time=3
 probe_name="Probe.${1:-"Liveness"}"
 ts_format=${SE_LOG_TIMESTAMP_FORMAT:-"+%T.%3N"}
-ROUTER_CONFIG_DIRECTORY=${ROUTER_CONFIG_DIRECTORY:-"/opt/selenium"}
+ROUTER_CONFIG_DIRECTORY=${ROUTER_CONFIG_DIRECTORY:-$SE_OPT_BIN}
 
 GRID_GRAPHQL_URL=$(bash ${ROUTER_CONFIG_DIRECTORY}/routerGraphQLUrl.sh)
 

charts/selenium-grid/configs/node/nodeGridUrl.sh

@@ -2,15 +2,16 @@
 
 max_time=3
 
+if [ -n "${SE_ROUTER_USERNAME}" ] && [ -n "${SE_ROUTER_PASSWORD}" ]; then
+  BASIC_AUTH="${SE_ROUTER_USERNAME}:${SE_ROUTER_PASSWORD}@"
+fi
+if [ "${SE_SUB_PATH}" = "/" ]; then
+  SE_SUB_PATH=""
+fi
+
 if [ -z "${SE_HUB_HOST:-$SE_ROUTER_HOST}" ] || [ -z "${SE_HUB_PORT:-$SE_ROUTER_PORT}" ]; then
   grid_url=""
 else
-  if [ -n "${SE_ROUTER_USERNAME}" ] && [ -n "${SE_ROUTER_PASSWORD}" ]; then
-    BASIC_AUTH="${SE_ROUTER_USERNAME}:${SE_ROUTER_PASSWORD}@"
-  fi
-  if [ "${SE_SUB_PATH}" = "/" ]; then
-    SE_SUB_PATH=""
-  fi
   grid_url=${SE_SERVER_PROTOCOL}://${BASIC_AUTH}${SE_HUB_HOST:-$SE_ROUTER_HOST}:${SE_HUB_PORT:-$SE_ROUTER_PORT}${SE_SUB_PATH}
 fi
 
@@ -19,7 +20,7 @@ if [ -z "${grid_url}" ]; then
 fi
 
 if [ -z "${grid_url}" ]; then
-  return 0
+  grid_url="${SE_SERVER_PROTOCOL}://${BASIC_AUTH}127.0.0.1:4444${SE_SUB_PATH}" # For standalone mode
 fi
 
 grid_url_checks=$(curl --noproxy "*" -m ${max_time} -s -k -o /dev/null -w "%{http_code}" ${grid_url})

charts/selenium-grid/configs/node/nodePreStop.sh

@@ -1,7 +1,8 @@
 #!/bin/bash
 
 probe_name="lifecycle.${1:-"preStop"}"
-NODE_CONFIG_DIRECTORY=${NODE_CONFIG_DIRECTORY:-"/opt/selenium"}
+SE_NODE_PORT=${SE_NODE_PORT:-"5555"}
+NODE_CONFIG_DIRECTORY=${NODE_CONFIG_DIRECTORY:-$SE_OPT_BIN}
 
 max_time=3
 retry_time=5

charts/selenium-grid/configs/node/nodeProbe.sh

@@ -2,7 +2,8 @@
 
 max_time=3
 probe_name="Probe.${1:-"Startup"}"
-NODE_CONFIG_DIRECTORY=${NODE_CONFIG_DIRECTORY:-"/opt/selenium"}
+SE_NODE_PORT=${SE_NODE_PORT:-"5555"}
+NODE_CONFIG_DIRECTORY=${NODE_CONFIG_DIRECTORY:-$SE_OPT_BIN}
 
 ID=$(echo $RANDOM)
 tmp_node_file="/tmp/nodeProbe${ID}"

charts/selenium-grid/configs/router/routerProbe.sh

@@ -4,7 +4,7 @@ max_time=3
 retry_time=3
 probe_name="Probe.${1:-"Liveness"}"
 ts_format=${SE_LOG_TIMESTAMP_FORMAT:-"+%T.%3N"}
-ROUTER_CONFIG_DIRECTORY=${ROUTER_CONFIG_DIRECTORY:-"/opt/selenium"}
+ROUTER_CONFIG_DIRECTORY=${ROUTER_CONFIG_DIRECTORY:-$SE_OPT_BIN}
 
 GRID_GRAPHQL_URL=$(bash ${ROUTER_CONFIG_DIRECTORY}/routerGraphQLUrl.sh)
 

charts/selenium-grid/values.yaml

@@ -160,7 +160,7 @@ distributorConfigMap:
   # Directory where the extra scripts are imported to ConfigMap by default (if given a relative path, it should be in chart's directory)
   extraScriptsImportFrom: "configs/distributor/**"
   # Directory where the extra scripts are mounted to
-  extraScriptsDirectory: "/opt/selenium"
+  extraScriptsDirectory: "/opt/bin"
   extraScripts:
     distributorProbe.sh: ""
   # Name of volume mount is used to mount scripts in the ConfigMap
@@ -175,7 +175,7 @@ routerConfigMap:
   # Directory where the extra scripts are imported to ConfigMap by default (if given a relative path, it should be in chart's directory)
   extraScriptsImportFrom: "configs/router/**"
   # Directory where the extra scripts are mounted to
-  extraScriptsDirectory: "/opt/selenium"
+  extraScriptsDirectory: "/opt/bin"
   extraScripts:
     routerGraphQLUrl.sh: ""
     routerProbe.sh: ""
@@ -192,7 +192,7 @@ nodeConfigMap:
   # Directory where the extra scripts are imported to ConfigMap by default (if given a relative path, it should be in chart's directory)
   extraScriptsImportFrom: "configs/node/**"
   # Directory where the extra scripts are mounted to
-  extraScriptsDirectory: "/opt/selenium"
+  extraScriptsDirectory: "/opt/bin"
   extraScripts:
     nodeGridUrl.sh: ""
     nodePreStop.sh: ""
@@ -695,7 +695,7 @@ autoscaling:
   deregisterLifecycle:
   #  preStop:
   #    exec:
-  #      command: [ "bash", "-c", "/opt/selenium/nodePreStop.sh" ]
+  #      command: [ "bash", "-c", "/opt/bin/nodePreStop.sh" ]
 
 # Configuration for chrome nodes
 chromeNode:

tests/SeleniumTests/__init__.py

@@ -20,6 +20,7 @@
 SELENIUM_ENABLE_MANAGED_DOWNLOADS = os.environ.get('SELENIUM_ENABLE_MANAGED_DOWNLOADS', 'true').lower() == 'true'
 WEB_DRIVER_WAIT_TIMEOUT = int(os.environ.get('WEB_DRIVER_WAIT_TIMEOUT', 60))
 TEST_PARALLEL_HARDENING = os.environ.get('TEST_PARALLEL_HARDENING', 'false').lower() == 'true'
+TEST_PARALLEL_COUNT = int(os.environ.get('TEST_PARALLEL_COUNT', 5))
 TEST_DELAY_AFTER_TEST = int(os.environ.get('TEST_DELAY_AFTER_TEST', 0))
 TEST_NODE_RELAY = os.environ.get('TEST_NODE_RELAY', 'false')
 TEST_ANDROID_PLATFORM_API = os.environ.get('ANDROID_PLATFORM_API')
@@ -256,7 +257,7 @@ def test_parallel_autoscaling(self):
         if not TEST_PARALLEL_HARDENING:
             runner.run(platform.add_test_based_platform(1))
         else:
-            runner.run(platform.add_test_based_platform(9))
+            runner.run(platform.add_test_based_platform(TEST_PARALLEL_COUNT))
 
 class JobAutoscalingTests(unittest.TestCase):
     def test_parallel_autoscaling(self):
@@ -265,7 +266,7 @@ def test_parallel_autoscaling(self):
         if not TEST_PARALLEL_HARDENING:
             runner.run(platform.add_test_based_platform(1))
         else:
-            runner.run(platform.add_test_based_platform(9))
+            runner.run(platform.add_test_based_platform(TEST_PARALLEL_COUNT))
 
 class TestPlatform:
     def add_test_based_platform(self, repeat):

tests/charts/make/chart_test.sh

@@ -327,7 +327,7 @@ export HUB_CHECKS_INTERVAL=${HUB_CHECKS_INTERVAL}
 export HUB_CHECKS_MAX_ATTEMPTS=${HUB_CHECKS_MAX_ATTEMPTS}
 export WEB_DRIVER_WAIT_TIMEOUT=${WEB_DRIVER_WAIT_TIMEOUT}
 export SELENIUM_GRID_TEST_HEADLESS=${SELENIUM_GRID_TEST_HEADLESS:-"false"}
-export TEST_DELAY_AFTER_TEST=${TEST_DELAY_AFTER_TEST:-"10"}
+export TEST_DELAY_AFTER_TEST=${TEST_DELAY_AFTER_TEST:-"0"}
 export TEST_PLATFORMS=${TEST_PLATFORMS}
 if [ "${MATRIX_BROWSER}" = "NoAutoscaling" ]; then
   ./tests/bootstrap.sh NodeFirefox

tests/docker-compose-v3-test-parallel.yml

@@ -24,6 +24,9 @@ services:
       - SE_BROWSER_ARGS_INCOGNITO=--incognito --incognito
       - SE_LOG_LEVEL=${LOG_LEVEL}
       - SE_SUPERVISORD_LOG_LEVEL=error
+      - SE_NODE_GRACEFUL_SHUTDOWN=true
+      - SE_DRAIN_AFTER_SESSION_COUNT=1
+    restart: always
 
   firefox:
     profiles:
@@ -45,6 +48,9 @@ services:
       - SE_NODE_ENABLE_MANAGED_DOWNLOADS=true
       - SE_LOG_LEVEL=${LOG_LEVEL}
       - SE_SUPERVISORD_LOG_LEVEL=error
+      - SE_NODE_GRACEFUL_SHUTDOWN=true
+      - SE_DRAIN_AFTER_SESSION_COUNT=3
+    restart: always
 
   edge:
     profiles:
@@ -65,6 +71,9 @@ services:
       - SE_NODE_ENABLE_MANAGED_DOWNLOADS=true
       - SE_LOG_LEVEL=${LOG_LEVEL}
       - SE_SUPERVISORD_LOG_LEVEL=error
+      - SE_NODE_GRACEFUL_SHUTDOWN=true
+      - SE_DRAIN_AFTER_SESSION_COUNT=2
+    restart: always
 
   selenium-hub:
     image: selenium/hub:${TAG}
@@ -79,17 +88,17 @@ services:
       - SE_SESSION_REQUEST_TIMEOUT=${REQUEST_TIMEOUT}
       - SE_SUPERVISORD_LOG_LEVEL=error
 
-  tests:
-    image: docker-selenium-tests:latest
-    build:
-      context: ./
-      dockerfile: ./Dockerfile
-    depends_on:
-      - selenium-hub
-    environment:
-      - RUN_IN_DOCKER_COMPOSE=true
-      - SELENIUM_GRID_HOST=selenium-hub
-      - TEST_PARALLEL_HARDENING=${TEST_PARALLEL_HARDENING}
-      - BINDING_VERSION=${BINDING_VERSION}
-      - TEST_PLATFORMS=${PLATFORMS}
-    command: ["/bin/bash", "-c", "./bootstrap.sh ${NODE}"]
+#  tests:
+#    image: docker-selenium-tests:latest
+#    build:
+#      context: ./
+#      dockerfile: ./Dockerfile
+#    depends_on:
+#      - selenium-hub
+#    environment:
+#      - RUN_IN_DOCKER_COMPOSE=true
+#      - SELENIUM_GRID_HOST=selenium-hub
+#      - TEST_PARALLEL_HARDENING=${TEST_PARALLEL_HARDENING}
+#      - BINDING_VERSION=${BINDING_VERSION}
+#      - TEST_PLATFORMS=${PLATFORMS}
+#    command: ["/bin/bash", "-c", "./bootstrap.sh ${NODE}"]