Skip to content

Commit

Permalink
update: FFmpeg 7.0.2 and fix video container termination
Browse files Browse the repository at this point in the history
Signed-off-by: Viet Nguyen Duc <[email protected]>
  • Loading branch information
VietND96 committed Aug 28, 2024
1 parent 918765f commit 55c6bb8
Show file tree
Hide file tree
Showing 21 changed files with 255 additions and 287 deletions.
2 changes: 1 addition & 1 deletion .github/workflows/helm-chart-test.yml
Original file line number Diff line number Diff line change
Expand Up @@ -156,7 +156,7 @@ jobs:
max_attempts: 3
command: |
NAME=${IMAGE_REGISTRY} VERSION=${BRANCH} BUILD_DATE=${BUILD_DATE} TEST_UPGRADE_CHART=false make chart_test_autoscaling_${{ matrix.test-strategy }} \
&& make test_video_integrity
&& NAME=${IMAGE_REGISTRY} VERSION=${BRANCH} BUILD_DATE=${BUILD_DATE} make test_video_integrity
- name: Test chart upgrade
if: (matrix.test-upgrade == true)
run: |
Expand Down
30 changes: 17 additions & 13 deletions Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -18,9 +18,9 @@ MAJOR := $(word 1,$(subst ., ,$(TAG_VERSION)))
MINOR := $(word 2,$(subst ., ,$(TAG_VERSION)))
MAJOR_MINOR_PATCH := $(word 1,$(subst -, ,$(TAG_VERSION)))
FFMPEG_TAG_PREV_VERSION := $(or $(FFMPEG_TAG_PREV_VERSION),$(FFMPEG_TAG_PREV_VERSION),ffmpeg-7.0.1)
FFMPEG_TAG_VERSION := $(or $(FFMPEG_TAG_VERSION),$(FFMPEG_TAG_VERSION),ffmpeg-7.0.1)
FFMPEG_TAG_VERSION := $(or $(FFMPEG_TAG_VERSION),$(FFMPEG_TAG_VERSION),ffmpeg-7.0.2)
FFMPEG_BASED_NAME := $(or $(FFMPEG_BASED_NAME),$(FFMPEG_BASED_NAME),linuxserver)
FFMPEG_BASED_TAG := $(or $(FFMPEG_BASED_TAG),$(FFMPEG_BASED_TAG),7.0.1)
FFMPEG_BASED_TAG := $(or $(FFMPEG_BASED_TAG),$(FFMPEG_BASED_TAG),7.0.2)
PLATFORMS := $(or $(PLATFORMS),$(shell echo $$PLATFORMS),linux/amd64)
SEL_PASSWD := $(or $(SEL_PASSWD),$(SEL_PASSWD),secret)

Expand Down Expand Up @@ -560,18 +560,21 @@ test_chromium:
test_chromium_standalone:
PLATFORMS=$(PLATFORMS) VERSION=$(TAG_VERSION) NAMESPACE=$(NAMESPACE) BASE_RELEASE=$(BASE_RELEASE) BASE_VERSION=$(BASE_VERSION) BINDING_VERSION=$(BINDING_VERSION) SKIP_BUILD=true ./tests/bootstrap.sh StandaloneChromium

test_parallel: hub chrome firefox edge chromium
test_parallel: hub chrome firefox edge chromium video
sudo rm -rf ./tests/tests
sudo rm -rf ./tests/videos; mkdir -p ./tests/videos
sudo cp -r ./charts/selenium-grid/certs ./tests/videos
for node in DeploymentAutoscaling JobAutoscaling ; do \
cd ./tests || true ; \
echo TAG=$(TAG_VERSION) > .env ; \
echo TEST_DRAIN_AFTER_SESSION_COUNT=$(or $(TEST_DRAIN_AFTER_SESSION_COUNT), 0) >> .env ; \
echo VIDEO_TAG=$(FFMPEG_TAG_VERSION)-$(BUILD_DATE) >> .env ; \
echo TEST_DELAY_AFTER_TEST=$(or $(TEST_DELAY_AFTER_TEST), 2) >> .env ; \
echo TEST_DRAIN_AFTER_SESSION_COUNT=$(or $(TEST_DRAIN_AFTER_SESSION_COUNT), 2) >> .env ; \
echo TEST_PARALLEL_HARDENING=$(or $(TEST_PARALLEL_HARDENING), "true") >> .env ; \
echo TEST_PARALLEL_COUNT=$(or $(TEST_PARALLEL_COUNT), 5) >> .env ; \
echo HUB_CHECKS_INTERVAL=$(or $(HUB_CHECKS_INTERVAL), 45) >> .env ; \
echo LOG_LEVEL=$(or $(LOG_LEVEL), "INFO") >> .env ; \
echo REQUEST_TIMEOUT=$(or $(REQUEST_TIMEOUT), 300) >> .env ; \
echo REQUEST_TIMEOUT=$(or $(REQUEST_TIMEOUT), 600) >> .env ; \
echo NODE=$$node >> .env ; \
echo UID=$$(id -u) >> .env ; \
echo BINDING_VERSION=$(BINDING_VERSION) >> .env ; \
Expand All @@ -586,14 +589,15 @@ test_parallel: hub chrome firefox edge chromium
export $$(cat .env | xargs) ; \
DOCKER_DEFAULT_PLATFORM=$(PLATFORMS) docker compose --profile $(PLATFORMS) -f docker-compose-v3-test-parallel.yml up -d --remove-orphans --no-log-prefix ; \
RUN_IN_DOCKER_COMPOSE=true bash ./bootstrap.sh $$node ; \
done ; \
docker compose -f docker-compose-v3-test-parallel.yml down
docker compose -f docker-compose-v3-test-parallel.yml down ; \
done
make test_video_integrity

test_video_standalone: standalone_chrome standalone_chromium standalone_firefox standalone_edge
DOCKER_COMPOSE_FILE=docker-compose-v3-test-standalone.yml make test_video
DOCKER_COMPOSE_FILE=docker-compose-v3-test-standalone.yml TEST_DELAY_AFTER_TEST=2 make test_video

test_video_dynamic_name:
VIDEO_FILE_NAME=auto \
VIDEO_FILE_NAME=auto TEST_DELAY_AFTER_TEST=2 \
make test_video

# This should run on its own CI job. There is no need to combine it with the other tests.
Expand All @@ -617,7 +621,7 @@ test_video: video hub chrome firefox edge chromium
echo NODE=$$node >> .env ; \
echo UID=$$(id -u) >> .env ; \
echo BINDING_VERSION=$(BINDING_VERSION) >> .env ; \
echo TEST_DELAY_AFTER_TEST=$(or $(TEST_DELAY_AFTER_TEST), 0) >> .env ; \
echo TEST_DELAY_AFTER_TEST=$(or $(TEST_DELAY_AFTER_TEST), 2) >> .env ; \
echo SELENIUM_ENABLE_MANAGED_DOWNLOADS=$(or $(SELENIUM_ENABLE_MANAGED_DOWNLOADS), "true") >> .env ; \
echo BASIC_AUTH_USERNAME=$(or $(BASIC_AUTH_USERNAME), "admin") >> .env ; \
echo BASIC_AUTH_PASSWORD=$(or $(BASIC_AUTH_PASSWORD), "admin") >> .env ; \
Expand Down Expand Up @@ -697,7 +701,7 @@ test_node_relay: hub node_base standalone_firefox

test_standalone_docker: standalone_docker
DOCKER_COMPOSE_FILE=docker-compose-v3-test-standalone-docker.yaml CONFIG_FILE=standalone_docker_config.toml \
RECORD_STANDALONE=true GRID_URL=http://0.0.0.0:4444 LIST_OF_TESTS_AMD64="DeploymentAutoscaling" TEST_PARALLEL_HARDENING=true \
RECORD_STANDALONE=true GRID_URL=http://0.0.0.0:4444 LIST_OF_TESTS_AMD64="DeploymentAutoscaling" TEST_PARALLEL_HARDENING=true TEST_DELAY_AFTER_TEST=2 \
SELENIUM_ENABLE_MANAGED_DOWNLOADS=true LOG_LEVEL=SEVERE SKIP_CHECK_DOWNLOADS_VOLUME=true make test_node_docker

test_node_docker: hub standalone_docker standalone_chrome standalone_firefox standalone_edge standalone_chromium video
Expand Down Expand Up @@ -725,7 +729,7 @@ test_node_docker: hub standalone_docker standalone_chrome standalone_firefox sta
echo LOG_LEVEL=$(or $(LOG_LEVEL), "INFO") >> .env ; \
echo REQUEST_TIMEOUT=$(or $(REQUEST_TIMEOUT), 300) >> .env ; \
echo SELENIUM_ENABLE_MANAGED_DOWNLOADS=$(or $(SELENIUM_ENABLE_MANAGED_DOWNLOADS), "false") >> .env ; \
echo TEST_DELAY_AFTER_TEST=$(or $(TEST_DELAY_AFTER_TEST), 0) >> .env ; \
echo TEST_DELAY_AFTER_TEST=$(or $(TEST_DELAY_AFTER_TEST), 2) >> .env ; \
echo RECORD_STANDALONE=$(or $(RECORD_STANDALONE), "true") >> .env ; \
echo GRID_URL=$(or $(GRID_URL), "") >> .env ; \
echo NODE=$$node >> .env ; \
Expand Down Expand Up @@ -790,7 +794,7 @@ test_video_integrity:
fi; \
for file in $$list_files; do \
echo "Checking video file: $$file"; \
docker run -u $$(id -u) -v $$(pwd):$$(pwd) -w $$(pwd) --entrypoint="" $(FFMPEG_BASED_NAME)/ffmpeg:$(FFMPEG_BASED_TAG) ffmpeg -v error -i "$$file" -f null - ; \
docker run -u $$(id -u) -v $$(pwd):$$(pwd) -w $$(pwd) --entrypoint="" $(NAME)/video:$(FFMPEG_TAG_VERSION)-$(BUILD_DATE) ffmpeg -v error -i "$$file" -f null - ; \
if [ $$? -ne 0 ]; then \
echo "Video file $$file is corrupted"; \
number_corrupted_files=$$((number_corrupted_files+1)); \
Expand Down
29 changes: 20 additions & 9 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -49,6 +49,7 @@ Talk to us at https://www.selenium.dev/support/
* [Execution with Docker Compose](#execution-with-docker-compose)
* [Configuring the child containers](#configuring-the-child-containers)
* [Video recording, screen resolution, and time zones in a Dynamic Grid](#video-recording-screen-resolution-and-time-zones-in-a-dynamic-grid)
* [Time zone configuration via env variable](#time-zone-configuration-via-env-variable)
* [Deploying to Kubernetes](#deploying-to-kubernetes)
* [Configuring the containers](#configuring-the-containers)
* [SE_OPTS Selenium Configuration Options](#se_opts-selenium-configuration-options)
Expand Down Expand Up @@ -712,13 +713,13 @@ When using in Dynamic Grid, those variables should be combined with the prefix `

### Environment variables and default values for upload feature

| Environment variable | Default value | Description |
|-------------------------------|------------------------------------|-------------------------------------------------------------------------------------------|
| `SE_UPLOAD_RETAIN_LOCAL_FILE` | `false` | Keep local file after uploading successfully |
| `SE_UPLOAD_COMMAND` | `copy` | RCLONE command is used to transfer file. Enforce `move` when retain local file is `false` |
| `SE_UPLOAD_OPTS` | `-P --cutoff-mode SOFT --metadata` | Other options belong to RCLONE command can be set. |
| `SE_UPLOAD_CONFIG_FILE_NAME` | `upload.conf` | Config file for remote host instead of set via env variable prefix SE_RCLONE_* |
| `SE_UPLOAD_CONFIG_DIRECTORY` | `/opt/bin` | Directory of config file (change it when conf file in another directory is mounted) |
| Environment variable | Default value | Description |
|-------------------------------|---------------------------------------------|-------------------------------------------------------------------------------------------|
| `SE_UPLOAD_RETAIN_LOCAL_FILE` | `false` | Keep local file after uploading successfully |
| `SE_UPLOAD_COMMAND` | `copy` | RCLONE command is used to transfer file. Enforce `move` when retain local file is `false` |
| `SE_UPLOAD_OPTS` | `-P --cutoff-mode SOFT --metadata--inplace` | Other options belong to RCLONE command can be set. |
| `SE_UPLOAD_CONFIG_FILE_NAME` | `upload.conf` | Config file for remote host instead of set via env variable prefix SE_RCLONE_* |
| `SE_UPLOAD_CONFIG_DIRECTORY` | `/opt/bin` | Directory of config file (change it when conf file in another directory is mounted) |

___

Expand Down Expand Up @@ -971,8 +972,6 @@ docker run --rm --name selenium-docker -p 4444:4444 `
selenium/standalone-docker:4.23.1-20240820
```



### Video recording, screen resolution, and time zones in a Dynamic Grid
To record your WebDriver session, you need to add a `se:recordVideo`
field set to `true`. You can also set a time zone and a screen resolution,
Expand Down Expand Up @@ -1009,6 +1008,18 @@ driver.quit()
After test executed, under (`${PWD}/assets`) you can see the video file name in path `/<sessionId>/test_visit_basic_auth_secured_page_ChromeTests.mp4`

The file name will be trimmed to 255 characters to avoid long file names. Moreover, the `space` character will be replaced by `_`, and only the characters alphabets, numbers, `-` (hyphen), and `_` (underscore) are retained in the file name. (This feat is available once this [PR](https://github.com/SeleniumHQ/selenium/pull/13907) merged)

### Time zone configuration via env variable

`tzdata` is installed in based images, and you can set the time zone in container by using the env variable `TZ`.
By default, the time zone is set to `UTC`.
List of supported time zones can be found [here](https://en.wikipedia.org/wiki/List_of_tz_database_time_zones). For example:

```bash
$ docker run --rm --entrypoint="" -e TZ=Asia/Ho_Chi_Minh selenium/node-chromium:latest date +%FT%T%Z
2024-08-28T18:19:26+07
```

___

## Deploying to Kubernetes
Expand Down
3 changes: 2 additions & 1 deletion Video/Dockerfile
Original file line number Diff line number Diff line change
Expand Up @@ -83,7 +83,7 @@ RUN curl -fLo /tmp/rclone.zip https://downloads.rclone.org/rclone-${RCLONE_VERSI
&& rm -rf /tmp/rclone-*
COPY --chown="${SEL_UID}:${SEL_GID}" upload.sh upload.conf /opt/bin/
ENV SE_VIDEO_UPLOAD_ENABLED=false \
SE_VIDEO_INTERNAL_UPLOAD=false \
SE_VIDEO_INTERNAL_UPLOAD=true \
SE_UPLOAD_DESTINATION_PREFIX=""

RUN mkdir -p /var/run/supervisor /var/log/supervisor ${VIDEO_FOLDER} \
Expand All @@ -101,6 +101,7 @@ CMD ["/opt/bin/entry_point.sh"]
ENV DISPLAY_NUM=99 \
DISPLAY_CONTAINER_NAME=selenium \
SE_SERVER_PROTOCOL="http" \
SE_VIDEO_POLL_INTERVAL=1 \
SE_SCREEN_WIDTH=1920 \
SE_SCREEN_HEIGHT=1080 \
SE_FRAME_RATE=15 \
Expand Down
13 changes: 10 additions & 3 deletions Video/supervisord.conf
Original file line number Diff line number Diff line change
Expand Up @@ -12,36 +12,43 @@ minfds=1024 ; (min. avail startup file descrip
minprocs=200 ; (min. avail process descriptors;default 200)

[program:video-recording]
priority=0
priority=10
command=/opt/bin/video.sh
killasgroup=true
autostart=true
startsecs=0
autorestart=true
stopsignal=TERM
stopwaitsecs=30

;Logs (all activity redirected to stdout so it can be seen through "docker logs"
redirect_stderr=true
stdout_logfile=/dev/stdout
stdout_logfile_maxbytes=0

[program:video-ready]
priority=5
priority=0
command=python3 /opt/bin/video_ready.py
killasgroup=true
autostart=true
startsecs=0
autorestart=true
stopsignal=KILL

;Logs (all activity redirected to stdout so it can be seen through "docker logs"
redirect_stderr=true
stdout_logfile=/dev/stdout
stdout_logfile_maxbytes=0

[program:video-upload]
priority=10
priority=5
command=/opt/bin/upload.sh
killasgroup=true
autostart=%(ENV_SE_VIDEO_INTERNAL_UPLOAD)s
startsecs=0
autorestart=%(ENV_SE_VIDEO_INTERNAL_UPLOAD)s
stopsignal=TERM
stopwaitsecs=30

;Logs (all activity redirected to stdout so it can be seen through "docker logs"
redirect_stderr=true
Expand Down
107 changes: 50 additions & 57 deletions Video/upload.sh
Original file line number Diff line number Diff line change
Expand Up @@ -4,14 +4,14 @@ VIDEO_FOLDER=${VIDEO_FOLDER}
UPLOAD_CONFIG_DIRECTORY=${SE_UPLOAD_CONFIG_DIRECTORY:-"/opt/bin"}
UPLOAD_CONFIG_FILE_NAME=${SE_UPLOAD_CONFIG_FILE_NAME:-"upload.conf"}
UPLOAD_COMMAND=${SE_UPLOAD_COMMAND:-"copy"}
UPLOAD_OPTS=${SE_UPLOAD_OPTS:-"-P --cutoff-mode SOFT --metadata"}
UPLOAD_OPTS=${SE_UPLOAD_OPTS:-"-P --cutoff-mode SOFT --metadata --inplace"}
UPLOAD_RETAIN_LOCAL_FILE=${SE_UPLOAD_RETAIN_LOCAL_FILE:-"false"}
UPLOAD_PIPE_FILE_NAME=${SE_UPLOAD_PIPE_FILE_NAME:-"uploadpipe"}
SE_VIDEO_INTERNAL_UPLOAD=${SE_VIDEO_INTERNAL_UPLOAD:-"false"}
VIDEO_INTERNAL_UPLOAD=${VIDEO_INTERNAL_UPLOAD:-$SE_VIDEO_INTERNAL_UPLOAD}
VIDEO_UPLOAD_BATCH_CHECK=${SE_VIDEO_UPLOAD_BATCH_CHECK:-"10"}
process_name="video.uploader"

if [ "${SE_VIDEO_INTERNAL_UPLOAD}" = "true" ];
if [ "${VIDEO_INTERNAL_UPLOAD}" = "true" ];
then
# If using RCLONE in the same container, write signal to /tmp internally
UPLOAD_PIPE_FILE="/tmp/${UPLOAD_PIPE_FILE_NAME}"
Expand Down Expand Up @@ -65,78 +65,71 @@ function rclone_upload() {
check_and_clear_background
}

function consume_pipe_file() {
function check_if_pid_alive() {
local pid=$1
if kill -0 "${pid}" > /dev/null 2>&1; then
return 0
fi
return 1
}

function consume_pipe_file_in_background() {
echo "$(date +%FT%T%Z) [${process_name}] - Start consuming pipe file to upload"
while read FILE DESTINATION < ${UPLOAD_PIPE_FILE};
do
if [ "${FILE}" = "exit" ];
then
FORCE_EXIT=true
exit
echo "$(date +%FT%T%Z) [${process_name}] - Received exit signal. Aborting upload process"
return 0
elif [ "$FILE" != "" ] && [ "$DESTINATION" != "" ];
then
rclone_upload "${FILE}" "${DESTINATION}"
elif [ -f ${FORCE_EXIT_FILE} ];
then
echo "$(date +%FT%T%Z) [${process_name}] - Force exit signal detected"
exit
fi
done
echo "$(date +%FT%T%Z) [${process_name}] - Stopped consuming pipe file. Upload process is done"
return 0
}

function graceful_exit() {
echo "$(date +%FT%T%Z) [${process_name}] - Uploader is shutting down"
if [ "${FORCE_EXIT}" != "true" ]; then
consume_pipe_file
# Function to check if the named pipe exists
check_if_pipefile_exists() {
if [ -p "${UPLOAD_PIPE_FILE}" ]; then
echo "$(date +%FT%T%Z) [${process_name}] - Named pipe ${UPLOAD_PIPE_FILE} exists"
return 0
fi
echo "$(date +%FT%T%Z) [${process_name}] - Uploader consumed all files in the pipe"
rm -rf ${FORCE_EXIT_FILE}
echo "$(date +%FT%T%Z) [${process_name}] - Uploader is ready to shutdown"
return 1
}
trap graceful_exit SIGTERM SIGINT EXIT

# Function to create the named pipe if it doesn't exist
function create_named_pipe() {
if [ ! -p "${UPLOAD_PIPE_FILE}" ];
then
if [ -e "${UPLOAD_PIPE_FILE}" ];
then
rm -f "${UPLOAD_PIPE_FILE}"
fi
mkfifo "${UPLOAD_PIPE_FILE}"
echo "$(date +%FT%T%Z) [${process_name}] - Created named pipe ${UPLOAD_PIPE_FILE}"
function wait_until_pipefile_exists() {
echo "$(date +%FT%T%Z) [${process_name}] - Waiting for ${UPLOAD_PIPE_FILE} to be present"
until check_if_pipefile_exists; do
sleep 1
done
}

function graceful_exit() {
echo "$(date +%FT%T%Z) [${process_name}] - Trapped SIGTERM/SIGINT/x so shutting down uploader"
if ! check_if_pid_alive "${UPLOAD_PID}"; then
consume_pipe_file_in_background &
UPLOAD_PID=$!
fi
echo "exit" >> "${UPLOAD_PIPE_FILE}" &
wait "${UPLOAD_PID}"
echo "$(date +%FT%T%Z) [${process_name}] - Uploader consumed all files in the pipe"
rm -rf "${FORCE_EXIT_FILE}"
echo "$(date +%FT%T%Z) [${process_name}] - Uploader is ready to shutdown"
exit 0
}

TIMEOUT=300 # Timeout in seconds (5 minutes)
START_TIME=$(date +%s)
rename_rclone_env
trap graceful_exit SIGTERM SIGINT EXIT

while true; do
if [ -e "${UPLOAD_PIPE_FILE}" ];
then
if [ -p "${UPLOAD_PIPE_FILE}" ];
then
break
else
echo "$(date +%FT%T%Z) [${process_name}] - ${UPLOAD_PIPE_FILE} exists but is not a named pipe"
create_named_pipe
fi
else
create_named_pipe
fi

CURRENT_TIME=$(date +%s)
ELAPSED_TIME=$((CURRENT_TIME - START_TIME))
if [ ${ELAPSED_TIME} -ge ${TIMEOUT} ];
then
echo "$(date +%FT%T%Z) [${process_name}] - Timeout waiting for ${UPLOAD_PIPE_FILE} to be created"
exit 1
wait_until_pipefile_exists
if ! check_if_pid_alive "${UPLOAD_PID}"; then
consume_pipe_file_in_background &
UPLOAD_PID=$!
fi

echo "$(date +%FT%T%Z) [${process_name}] - Waiting for ${UPLOAD_PIPE_FILE} to be created"
sleep 1
while check_if_pid_alive "${UPLOAD_PID}"; do
sleep 1
done
done

echo "$(date +%FT%T%Z) [${process_name}] - Waiting for video files put into pipe for proceeding to upload"

rename_rclone_env
consume_pipe_file
Loading

0 comments on commit 55c6bb8

Please sign in to comment.