Enable Tracing in Selenium Grid (#1711)

Closes #1561

Base/Dockerfile

@@ -78,6 +78,19 @@ RUN  mkdir -p /opt/selenium /opt/selenium/assets /var/run/supervisor /var/log/su
   && chgrp -R 0 /opt/selenium ${HOME} /opt/selenium/assets /var/run/supervisor /var/log/supervisor \
   && chmod -R g=u /opt/selenium ${HOME} /opt/selenium/assets /var/run/supervisor /var/log/supervisor
 
+#=====
+# Download observability related jaegar jars and make them available in a separate directory
+# so that the container can skip downloading them everytime it comes up
+#=====
+RUN curl -fL https://github.com/coursier/launchers/raw/master/cs-x86_64-pc-linux.gz | gzip -d > /tmp/cs \
+  && chmod +x /tmp/cs \
+  && mkdir -p /external_jars \
+  && chmod -R 777 /external_jars
+
+RUN /tmp/cs fetch --classpath --cache /external_jars io.opentelemetry:opentelemetry-exporter-jaeger:1.19.0 io.grpc:grpc-netty:1.50.2 > /external_jars/.classpath.txt
+
+RUN chmod 777 /external_jars/.classpath.txt
+
 #===================================================
 # Run the following commands as non-privileged user
 #===================================================
@@ -86,6 +99,4 @@ USER 1200:1201
 # Boolean value, maps "--bind-host"
 ENV SE_BIND_HOST false
 
-
-
 CMD ["/opt/bin/entry_point.sh"]

Distributor/start-selenium-grid-distributor.sh

@@ -54,10 +54,20 @@ if [ ! -z "$SE_DISTRIBUTOR_PORT" ]; then
   PORT_CONFIG="--port ${SE_DISTRIBUTOR_PORT}"
 fi
 
+EXTRA_LIBS="/opt/selenium/selenium-http-jdk-client.jar"
+
+if [ ! -z "$SE_ENABLE_TRACING" ]; then
+  EXTERNAL_JARS=$(</external_jars/.classpath.txt)
+  EXTRA_LIBS=${EXTRA_LIBS}:${EXTERNAL_JARS}
+  echo "Tracing is enabled"
+  echo "Classpath will be enriched with these external jars : " ${EXTRA_LIBS}
+else
+  echo "Tracing is disabled"
+fi
 
 java ${JAVA_OPTS:-$SE_JAVA_OPTS} -Dwebdriver.http.factory=jdk-http-client \
   -jar /opt/selenium/selenium-server.jar \
-  --ext /opt/selenium/selenium-http-jdk-client.jar distributor \
+  --ext ${EXTRA_LIBS} distributor \
   --sessions-host "${SE_SESSIONS_MAP_HOST}" --sessions-port "${SE_SESSIONS_MAP_PORT}" \
   --sessionqueue-host "${SE_SESSION_QUEUE_HOST}" --sessionqueue-port "${SE_SESSION_QUEUE_PORT}" \
   --publish-events tcp://"${SE_EVENT_BUS_HOST}":"${SE_EVENT_BUS_PUBLISH_PORT}" \

EventBus/start-selenium-grid-eventbus.sh

@@ -19,9 +19,20 @@ if [ ! -z "$SE_OPTS" ]; then
   echo "Appending Selenium options: ${SE_OPTS}"
 fi
 
+EXTRA_LIBS="/opt/selenium/selenium-http-jdk-client.jar"
+
+if [ ! -z "$SE_ENABLE_TRACING" ]; then
+  EXTERNAL_JARS=$(</external_jars/.classpath.txt)
+  EXTRA_LIBS=${EXTRA_LIBS}:${EXTERNAL_JARS}
+  echo "Tracing is enabled"
+  echo "Classpath will be enriched with these external jars : " ${EXTRA_LIBS}
+else
+  echo "Tracing is disabled"
+fi
+
 java ${JAVA_OPTS:-$SE_JAVA_OPTS} -Dwebdriver.http.factory=jdk-http-client \
   -jar /opt/selenium/selenium-server.jar \
-  --ext /opt/selenium/selenium-http-jdk-client.jar event-bus \
+  --ext ${EXTRA_LIBS} event-bus \
   --bind-host ${SE_BIND_HOST} \
   ${HOST_CONFIG} \
   ${PORT_CONFIG} \

Hub/start-selenium-grid-hub.sh

@@ -17,9 +17,21 @@ if [ ! -z "$SE_HUB_PORT" ]; then
   PORT_CONFIG="--port ${SE_HUB_PORT}"
 fi
 
+EXTRA_LIBS="/opt/selenium/selenium-http-jdk-client.jar"
+
+if [ ! -z "$SE_ENABLE_TRACING" ]; then
+  EXTERNAL_JARS=$(</external_jars/.classpath.txt)
+  EXTRA_LIBS=${EXTRA_LIBS}:${EXTERNAL_JARS}
+  echo "Tracing is enabled"
+  echo "Classpath will be enriched with these external jars : " ${EXTRA_LIBS}
+else
+  echo "Tracing is disabled"
+fi
+
+
 java ${JAVA_OPTS:-$SE_JAVA_OPTS} -Dwebdriver.http.factory=jdk-http-client \
   -jar /opt/selenium/selenium-server.jar \
-  --ext /opt/selenium/selenium-http-jdk-client.jar hub \
+  --ext ${EXTRA_LIBS} hub \
   --session-request-timeout ${SE_SESSION_REQUEST_TIMEOUT} \
   --session-retry-interval ${SE_SESSION_RETRY_INTERVAL} \
   --relax-checks ${SE_RELAX_CHECKS} \

NodeBase/start-selenium-node.sh

@@ -35,13 +35,25 @@ if [ "$GENERATE_CONFIG" = true ]; then
   echo "Generating Selenium Config"
   /opt/bin/generate_config
 fi
+
+EXTRA_LIBS="/opt/selenium/selenium-http-jdk-client.jar"
+
+if [ ! -z "$SE_ENABLE_TRACING" ]; then
+  EXTERNAL_JARS=$(</external_jars/.classpath.txt)
+  EXTRA_LIBS=${EXTRA_LIBS}:${EXTERNAL_JARS}
+  echo "Tracing is enabled"
+  echo "Classpath will be enriched with these external jars : " ${EXTRA_LIBS}
+else
+  echo "Tracing is disabled"
+fi
+
 echo "Selenium Grid Node configuration: "
 cat "$CONFIG_FILE"
 echo "Starting Selenium Grid Node..."
 
 java ${JAVA_OPTS:-$SE_JAVA_OPTS} -Dwebdriver.http.factory=jdk-http-client \
   -jar /opt/selenium/selenium-server.jar \
-  --ext /opt/selenium/selenium-http-jdk-client.jar node \
+  --ext ${EXTRA_LIBS} node \
   --bind-host ${SE_BIND_HOST} \
   --config "$CONFIG_FILE" \
   ${SE_OPTS}

NodeDocker/start-selenium-grid-docker.sh

@@ -29,9 +29,20 @@ if [ ! -z "$SE_NODE_GRID_URL" ]; then
   SE_GRID_URL="--grid-url ${SE_NODE_GRID_URL}"
 fi
 
+EXTRA_LIBS="/opt/selenium/selenium-http-jdk-client.jar"
+
+if [ ! -z "$SE_ENABLE_TRACING" ]; then
+  EXTERNAL_JARS=$(</external_jars/.classpath.txt)
+  EXTRA_LIBS=${EXTRA_LIBS}:${EXTERNAL_JARS}
+  echo "Tracing is enabled"
+  echo "Classpath will be enriched with these external jars : " ${EXTRA_LIBS}
+else
+  echo "Tracing is disabled"
+fi
+
 java ${JAVA_OPTS:-$SE_JAVA_OPTS} -Dwebdriver.http.factory=jdk-http-client \
   -jar /opt/selenium/selenium-server.jar \
-  --ext /opt/selenium/selenium-http-jdk-client.jar node \
+  --ext ${EXTRA_LIBS} node \
   --publish-events tcp://"${SE_EVENT_BUS_HOST}":${SE_EVENT_BUS_PUBLISH_PORT} \
   --subscribe-events tcp://"${SE_EVENT_BUS_HOST}":${SE_EVENT_BUS_SUBSCRIBE_PORT} \
   --bind-host ${SE_BIND_HOST} \

README.md

@@ -54,6 +54,7 @@ Talk to us at https://www.selenium.dev/support/
 * [Waiting for the Grid to be ready](#waiting-for-the-grid-to-be-ready)
 * [Debugging](#debugging)
 * [Install certificates for Chromium based browsers](#install-certificates-for-Chromium-based-browsers)
+* [Tracing in Grid](#tracing-in-grid)
 * [Troubleshooting](#troubleshooting)
 
 
@@ -1132,6 +1133,46 @@ If you get a prompt asking for a password, it is: `secret`.
 
 ___
 
+## Tracing in Grid
+
+In order to enable tracing in Selenium Grid container, the following commands can be executed:
+
+```bash
+docker network create grid
+docker run -d -p 16686:16686 -p 14250:14250 --net grid --name jaeger jaegertracing/all-in-one:1.17
+docker run -d -p 4442-4444:4442-4444 --net grid --name selenium-hub selenium/hub:4.5.3-20221024
+docker run -d --net grid -e SE_EVENT_BUS_HOST=selenium-hub \
+    --shm-size="2g" \
+	-e SE_ENABLE_TRACING=true \
+	-e JAVA_OPTS="-Dotel.traces.exporter=jaeger -Dotel.exporter.jaeger.endpoint=http://jaegar:14250 -Dotel.resource.attributes=service.name=selenium-hub" \
+    -e SE_EVENT_BUS_PUBLISH_PORT=4442 \
+    -e SE_EVENT_BUS_SUBSCRIBE_PORT=4443 \
+    selenium/node-chrome:4.5.3-20221024
+docker run -d --net grid -e SE_EVENT_BUS_HOST=selenium-hub \
+    --shm-size="2g" \
+	-e SE_ENABLE_TRACING=true \
+	-e JAVA_OPTS="-Dotel.traces.exporter=jaeger -Dotel.exporter.jaeger.endpoint=http://jaegar:14250 -Dotel.resource.attributes=service.name=selenium-node-edge" \
+    -e SE_EVENT_BUS_PUBLISH_PORT=4442 \
+    -e SE_EVENT_BUS_SUBSCRIBE_PORT=4443 \
+    selenium/node-edge:4.5.3-20221024
+docker run -d --net grid -e SE_EVENT_BUS_HOST=selenium-hub \
+    --shm-size="2g" \
+	-e SE_ENABLE_TRACING=true \
+	-e JAVA_OPTS="-Dotel.traces.exporter=jaeger -Dotel.exporter.jaeger.endpoint=http://jaegar:14250 -Dotel.resource.attributes=service.name=selenium-node-firefox" \
+    -e SE_EVENT_BUS_PUBLISH_PORT=4442 \
+    -e SE_EVENT_BUS_SUBSCRIBE_PORT=4443 \
+    selenium/node-firefox:4.5.3-20221024
+```
+
+You can also refer to the below docker-compose yaml files to be able to start a simple grid (or) a dynamic grid.
+
+* Simple Grid [v3 yaml file](docker-compose-v3-tracing.yml)
+* Simple Grid [v2 yaml file](docker-compose-v2-tracing.yml)
+* Dynamic Grid [v3 yaml file](docker-compose-v3-full-grid-tracing.yml)
+
+You can view the [Jaegar UI](http://localhost:16686/) and trace your request.
+___
+
 ## Troubleshooting
 
 All output gets sent to stdout, so it can be inspected by running:

Router/start-selenium-grid-router.sh

@@ -49,9 +49,20 @@ if [ ! -z "$SE_ROUTER_PORT" ]; then
   PORT_CONFIG="--port ${SE_ROUTER_PORT}"
 fi
 
+EXTRA_LIBS="/opt/selenium/selenium-http-jdk-client.jar"
+
+if [ ! -z "$SE_ENABLE_TRACING" ]; then
+  EXTERNAL_JARS=$(</external_jars/.classpath.txt)
+  EXTRA_LIBS=${EXTRA_LIBS}:${EXTERNAL_JARS}
+  echo "Tracing is enabled"
+  echo "Classpath will be enriched with these external jars : " ${EXTRA_LIBS}
+else
+  echo "Tracing is disabled"
+fi
+
 java ${JAVA_OPTS:-$SE_JAVA_OPTS} -Dwebdriver.http.factory=jdk-http-client \
   -jar /opt/selenium/selenium-server.jar \
-  --ext /opt/selenium/selenium-http-jdk-client.jar router \
+  --ext ${EXTRA_LIBS} router \
   --sessions-host "${SE_SESSIONS_MAP_HOST}" --sessions-port "${SE_SESSIONS_MAP_PORT}" \
   --distributor-host "${SE_DISTRIBUTOR_HOST}" --distributor-port "${SE_DISTRIBUTOR_PORT}" \
   --sessionqueue-host "${SE_SESSION_QUEUE_HOST}" --sessionqueue-port "${SE_SESSION_QUEUE_PORT}" \

SessionQueue/start-selenium-grid-session-queue.sh

@@ -19,9 +19,20 @@ if [ ! -z "$SE_SESSION_QUEUE_PORT" ]; then
   PORT_CONFIG="--port ${SE_SESSION_QUEUE_PORT}"
 fi
 
+EXTRA_LIBS="/opt/selenium/selenium-http-jdk-client.jar"
+
+if [ ! -z "$SE_ENABLE_TRACING" ]; then
+  EXTERNAL_JARS=$(</external_jars/.classpath.txt)
+  EXTRA_LIBS=${EXTRA_LIBS}:${EXTERNAL_JARS}
+  echo "Tracing is enabled"
+  echo "Classpath will be enriched with these external jars : " ${EXTRA_LIBS}
+else
+  echo "Tracing is disabled"
+fi
+
 java ${JAVA_OPTS:-$SE_JAVA_OPTS} -Dwebdriver.http.factory=jdk-http-client \
   -jar /opt/selenium/selenium-server.jar \
-  --ext /opt/selenium/selenium-http-jdk-client.jar sessionqueue \
+  --ext ${EXTRA_LIBS} sessionqueue \
   --session-request-timeout ${SE_SESSION_REQUEST_TIMEOUT} \
   --session-retry-interval ${SE_SESSION_RETRY_INTERVAL} \
   --bind-host ${SE_BIND_HOST} \

Sessions/start-selenium-grid-sessions.sh

@@ -34,9 +34,20 @@ if [ ! -z "$SE_SESSIONS_PORT" ]; then
   PORT_CONFIG="--port ${SE_SESSIONS_PORT}"
 fi
 
+EXTRA_LIBS="/opt/selenium/selenium-http-jdk-client.jar"
+
+if [ ! -z "$SE_ENABLE_TRACING" ]; then
+  EXTERNAL_JARS=$(</external_jars/.classpath.txt)
+  EXTRA_LIBS=${EXTRA_LIBS}:${EXTERNAL_JARS}
+  echo "Tracing is enabled"
+  echo "Classpath will be enriched with these external jars : " ${EXTRA_LIBS}
+else
+  echo "Tracing is disabled"
+fi
+
 java ${JAVA_OPTS:-$SE_JAVA_OPTS} -Dwebdriver.http.factory=jdk-http-client \
   -jar /opt/selenium/selenium-server.jar \
-  --ext /opt/selenium/selenium-http-jdk-client.jar sessions \
+  --ext ${EXTRA_LIBS} sessions \
   --publish-events tcp://"${SE_EVENT_BUS_HOST}":${SE_EVENT_BUS_PUBLISH_PORT} \
   --subscribe-events tcp://"${SE_EVENT_BUS_HOST}":${SE_EVENT_BUS_SUBSCRIBE_PORT} \
   --bind-host ${SE_BIND_HOST} \

Standalone/start-selenium-standalone.sh

@@ -12,9 +12,20 @@ echo "Selenium Grid Standalone configuration: "
 cat /opt/selenium/config.toml
 echo "Starting Selenium Grid Standalone..."
 
+EXTRA_LIBS="/opt/selenium/selenium-http-jdk-client.jar"
+
+if [ ! -z "$SE_ENABLE_TRACING" ]; then
+  EXTERNAL_JARS=$(</external_jars/.classpath.txt)
+  EXTRA_LIBS=${EXTRA_LIBS}:${EXTERNAL_JARS}
+  echo "Tracing is enabled"
+  echo "Classpath will be enriched with these external jars : " ${EXTRA_LIBS}
+else
+  echo "Tracing is disabled"
+fi
+
 java ${JAVA_OPTS:-$SE_JAVA_OPTS} -Dwebdriver.http.factory=jdk-http-client \
   -jar /opt/selenium/selenium-server.jar \
-  --ext /opt/selenium/selenium-http-jdk-client.jar standalone \
+  --ext ${EXTRA_LIBS} standalone \
   --bind-host ${SE_BIND_HOST} \
   --config /opt/selenium/config.toml \
   ${SE_OPTS}

StandaloneDocker/start-selenium-grid-docker.sh

@@ -14,9 +14,20 @@ if [ ! -z "$SE_NODE_GRID_URL" ]; then
   SE_GRID_URL="--grid-url ${SE_NODE_GRID_URL}"
 fi
 
+EXTRA_LIBS="/opt/selenium/selenium-http-jdk-client.jar"
+
+if [ ! -z "$SE_ENABLE_TRACING" ]; then
+  EXTERNAL_JARS=$(</external_jars/.classpath.txt)
+  EXTRA_LIBS=${EXTRA_LIBS}:${EXTERNAL_JARS}
+  echo "Tracing is enabled"
+  echo "Classpath will be enriched with these external jars : " ${EXTRA_LIBS}
+else
+  echo "Tracing is disabled"
+fi
+
 java ${JAVA_OPTS:-$SE_JAVA_OPTS} -Dwebdriver.http.factory=jdk-http-client \
   -jar /opt/selenium/selenium-server.jar \
-  --ext /opt/selenium/selenium-http-jdk-client.jar standalone \
+  --ext ${EXTRA_LIBS} standalone \
   --relax-checks ${SE_RELAX_CHECKS} \
   --detect-drivers false \
   --bind-host ${SE_BIND_HOST} \

docker-compose-v2-tracing.yml

@@ -0,0 +1,63 @@
+# To execute this docker-compose yml file use `docker-compose -f docker-compose-v2-tracing.yml up`
+# Add the `-d` flag at the end for detached execution
+# To stop the execution, hit Ctrl+C, and then `docker-compose -f docker-compose-v2-tracing.yml down`
+version: '2'
+services:
+  jaegar:
+    image: jaegertracing/all-in-one:1.17
+    ports:
+      - "16686:16686"
+      - "14250:14250"
+  chrome:
+    image: selenium/node-chrome:4.5.3-20221024
+    shm_size: 2gb
+    depends_on:
+      - selenium-hub
+    environment:
+      - SE_EVENT_BUS_HOST=selenium-hub
+      - SE_EVENT_BUS_PUBLISH_PORT=4442
+      - SE_EVENT_BUS_SUBSCRIBE_PORT=4443
+      - SE_ENABLE_TRACING=true
+      - JAVA_OPTS=-Dotel.traces.exporter=jaeger -Dotel.exporter.jaeger.endpoint=http://jaegar:14250 -Dotel.resource.attributes=service.name=selenium-node-chrome
+    ports:
+      - "6900:5900"
+
+  edge:
+    image: selenium/node-edge:4.5.3-20221024
+    shm_size: 2gb
+    depends_on:
+      - selenium-hub
+    environment:
+      - SE_EVENT_BUS_HOST=selenium-hub
+      - SE_EVENT_BUS_PUBLISH_PORT=4442
+      - SE_EVENT_BUS_SUBSCRIBE_PORT=4443
+      - SE_ENABLE_TRACING=true
+      - JAVA_OPTS=-Dotel.traces.exporter=jaeger -Dotel.exporter.jaeger.endpoint=http://jaegar:14250 -Dotel.resource.attributes=service.name=selenium-node-edge
+    ports:
+      - "6901:5900"
+
+  firefox:
+    image: selenium/node-firefox:4.5.3-20221024
+    shm_size: 2gb
+    depends_on:
+      - selenium-hub
+    environment:
+      - SE_EVENT_BUS_HOST=selenium-hub
+      - SE_EVENT_BUS_PUBLISH_PORT=4442
+      - SE_EVENT_BUS_SUBSCRIBE_PORT=4443
+      - SE_ENABLE_TRACING=true
+      - JAVA_OPTS=-Dotel.traces.exporter=jaeger -Dotel.exporter.jaeger.endpoint=http://jaegar:14250 -Dotel.resource.attributes=service.name=selenium-node-firefox
+    ports:
+      - "6902:5900"
+
+  selenium-hub:
+    image: selenium/hub:4.5.3-20221024
+    ports:
+      - "4442:4442"
+      - "4443:4443"
+      - "4444:4444"
+    depends_on:
+      - jaegar
+    environment:
+      - SE_ENABLE_TRACING=true
+      - JAVA_OPTS=-Dotel.traces.exporter=jaeger -Dotel.exporter.jaeger.endpoint=http://jaegar:14250 -Dotel.resource.attributes=service.name=selenium-hub