chart(breaking change): update config key to enable ingress (#2349)

Signed-off-by: Viet Nguyen Duc <[email protected]>
SeleniumHQ · Aug 12, 2024 · 91233c4 · 91233c4
1 parent 22b2f55
commit 91233c4
Show file tree

Hide file tree

Showing 17 changed files with 111 additions and 37 deletions.
diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml
@@ -23,6 +23,11 @@ on:
         required: false
         type: boolean
         default: false
+      skip-build-push-image:
+        description: 'Skip the build & push images'
+        required: false
+        type: boolean
+        default: false
   push:
     branches:
       - trunk
@@ -99,13 +104,6 @@ jobs:
         run: ./update_tag_in_docs_and_files.sh ${LATEST_TAG} ${NEXT_TAG}
       - name: Update chart CHANGELOG
         run: ./generate_chart_changelog.sh
-      - name: Build images
-        uses: nick-invision/retry@master
-        with:
-          timeout_minutes: 90
-          max_attempts: 3
-          retry_wait_seconds: 60
-          command: PLATFORMS="${PLATFORMS}" VERSION="${GRID_VERSION}" BUILD_DATE=${BUILD_DATE} make build
       - name: Build Helm chart
         uses: nick-invision/retry@master
         with:
@@ -116,6 +114,18 @@ jobs:
             SET_VERSION=false make chart_build
             echo "CHART_PACKAGE_PATH=$(cat /tmp/selenium_chart_version)" >> $GITHUB_ENV
             echo "CHART_FILE_NAME=$(basename $(cat /tmp/selenium_chart_version))" >> $GITHUB_ENV
+      - name: Render chart templates
+        run: |
+          make chart_render_template
+          echo "PUBLISH_YAML_MANIFESTS=$(find ./tests/tests -name "k8s_*.yaml" | tr '\n' ',')" >> $GITHUB_ENV
+      - name: Build images
+        if: github.event.inputs.skip-build-push-image != 'true'
+        uses: nick-invision/retry@master
+        with:
+          timeout_minutes: 90
+          max_attempts: 3
+          retry_wait_seconds: 60
+          command: PLATFORMS="${PLATFORMS}" VERSION="${GRID_VERSION}" BUILD_DATE=${BUILD_DATE} make build
       - name: Login Docker Hub
         run: |
           docker login -u="$DOCKER_USERNAME" -p="$DOCKER_PASSWORD"
@@ -124,22 +134,26 @@ jobs:
           DOCKER_USERNAME: ${{secrets.DOCKER_USERNAME}}
           DOCKER_PASSWORD: ${{secrets.DOCKER_PASSWORD}}
       - name: Deploy new images
+        if: github.event.inputs.skip-build-push-image != 'true'
         uses: nick-invision/retry@master
         with:
           timeout_minutes: 20
           max_attempts: 3
           retry_wait_seconds: 120
           command: VERSION="${GRID_VERSION}" BUILD_DATE=${BUILD_DATE} make release
       - name: Tag images as latest
+        if: github.event.inputs.skip-build-push-image != 'true'
         run: VERSION="${GRID_VERSION}" BUILD_DATE=${BUILD_DATE} make tag_latest
       - name: Deploy latest tag
+        if: github.event.inputs.skip-build-push-image != 'true'
         uses: nick-invision/retry@master
         with:
           timeout_minutes: 20
           max_attempts: 3
           retry_wait_seconds: 120
           command: VERSION="${GRID_VERSION}" BUILD_DATE=${BUILD_DATE} make release_latest
       - name: Tag browser images
+        if: github.event.inputs.skip-build-push-image != 'true'
         uses: nick-invision/retry@master
         with:
           timeout_minutes: 20
@@ -180,7 +194,7 @@ jobs:
           path: ./release_notes.md
           if-no-files-found: ignore
       - name: Create Release
-        if: env.LATEST_TAG != env.NEXT_TAG && github.event.inputs.skip-commit != 'true'
+        if: env.LATEST_TAG != env.NEXT_TAG
         id: create_release
         uses: softprops/action-gh-release@master
         with:
@@ -192,3 +206,4 @@ jobs:
           prerelease: ${{ env.PRERELEASE }}
           draft: false
           append_body: false
+          files: ${{ env.PUBLISH_YAML_MANIFESTS }}
diff --git a/.github/workflows/docker-test.yml b/.github/workflows/docker-test.yml
@@ -63,6 +63,11 @@ jobs:
             test-video: false
             build-all: false
     steps:
+      - name: Free Disk Space (Ubuntu)
+        uses: jlumbroso/free-disk-space@main
+        with:
+          tool-cache: false
+          large-packages: false
       - name: Checkout code
         uses: actions/checkout@main
       - name: Set up containerd image store feature

diff --git a/.github/workflows/helm-chart-test.yml b/.github/workflows/helm-chart-test.yml
@@ -78,6 +78,11 @@ jobs:
       TEST_EXISTING_KEDA: ${{ matrix.test-existing-keda }}
       TEST_UPGRADE_CHART: ${{ matrix.test-upgrade }}
     steps:
+      - name: Free Disk Space (Ubuntu)
+        uses: jlumbroso/free-disk-space@main
+        with:
+          tool-cache: false
+          large-packages: false
       - name: Checkout code
         uses: actions/checkout@main
       - name: Set up containerd image store feature

diff --git a/.github/workflows/rerun-failed.yml b/.github/workflows/rerun-failed.yml
@@ -13,6 +13,8 @@ on:
         type: boolean
         default: true
 
+permissions: write-all
+
 env:
   GH_CLI_TOKEN: ${{ secrets.GITHUB_TOKEN }}
   RUN_ID: ${{ github.event.inputs.runId }}

diff --git a/Makefile b/Makefile
@@ -777,42 +777,51 @@ test_video_integrity:
 chart_test_template:
 	./tests/charts/bootstrap.sh
 
+chart_render_template:
+	RENDER_HELM_TEMPLATE_ONLY=true make chart_test_autoscaling_disabled chart_test_autoscaling_deployment_https chart_test_autoscaling_deployment chart_test_autoscaling_job_https chart_test_autoscaling_job_hostname chart_test_autoscaling_job
+
 chart_test_autoscaling_disabled:
 	PLATFORMS=$(PLATFORMS) TEST_CHROMIUM=true RELEASE_NAME=selenium SELENIUM_GRID_AUTOSCALING=false TEST_DELAY_AFTER_TEST=0 CHART_ENABLE_TRACING=true \
 	SECURE_INGRESS_ONLY_GENERATE=true SELENIUM_GRID_PROTOCOL=https SELENIUM_GRID_HOST=$$(hostname -i) SELENIUM_GRID_PORT=443 \
 	VERSION=$(TAG_VERSION) VIDEO_TAG=$(FFMPEG_TAG_VERSION)-$(BUILD_DATE) NAMESPACE=$(NAMESPACE) BINDING_VERSION=$(BINDING_VERSION) \
+	TEMPLATE_OUTPUT_FILENAME="k8s_nodeChromium_enableTracing_secureIngress_generateCerts_ingressPublicIP_subPath.yaml" \
 	./tests/charts/make/chart_test.sh NoAutoscaling
 
 chart_test_autoscaling_deployment_https:
 	PLATFORMS=$(PLATFORMS) CHART_FULL_DISTRIBUTED_MODE=true CHART_ENABLE_BASIC_AUTH=true \
 	SECURE_INGRESS_ONLY_DEFAULT=true INGRESS_DISABLE_USE_HTTP2=true SELENIUM_GRID_PROTOCOL=https CHART_ENABLE_INGRESS_HOSTNAME=true SELENIUM_GRID_PORT=443 \
 	SELENIUM_GRID_AUTOSCALING_MIN_REPLICA=1 \
 	VERSION=$(TAG_VERSION) VIDEO_TAG=$(FFMPEG_TAG_VERSION)-$(BUILD_DATE) NAMESPACE=$(NAMESPACE) BINDING_VERSION=$(BINDING_VERSION) \
+	TEMPLATE_OUTPUT_FILENAME="k8s_fullDistributed_basicAuth_secureIngress_defaultCerts_ingressHostName_disableHttp2_autoScaling_scaledObject_subPath.yaml" \
 	./tests/charts/make/chart_test.sh DeploymentAutoscaling
 
 chart_test_autoscaling_deployment:
 	PLATFORMS=$(PLATFORMS) RELEASE_NAME=selenium CHART_ENABLE_TRACING=true \
 	SECURE_CONNECTION_SERVER=true SECURE_USE_EXTERNAL_CERT=true SERVICE_TYPE_NODEPORT=true SELENIUM_GRID_PROTOCOL=https SELENIUM_GRID_HOST=$$(hostname -i) SELENIUM_GRID_PORT=31444 \
 	SELENIUM_GRID_AUTOSCALING_MIN_REPLICA=1 \
 	VERSION=$(TAG_VERSION) VIDEO_TAG=$(FFMPEG_TAG_VERSION)-$(BUILD_DATE) NAMESPACE=$(NAMESPACE) BINDING_VERSION=$(BINDING_VERSION) \
+	TEMPLATE_OUTPUT_FILENAME="k8s_prefixSelenium_enableTracing_secureServer_externalCerts_nodePort_autoScaling_scaledObject_subPath.yaml" \
 	./tests/charts/make/chart_test.sh DeploymentAutoscaling
 
 chart_test_autoscaling_job_https:
 	PLATFORMS=$(PLATFORMS) RELEASE_NAME=selenium CHART_ENABLE_BASIC_AUTH=true \
 	SECURE_CONNECTION_SERVER=true SELENIUM_GRID_PROTOCOL=https SELENIUM_GRID_PORT=443 SUB_PATH=/ \
 	VERSION=$(TAG_VERSION) VIDEO_TAG=$(FFMPEG_TAG_VERSION)-$(BUILD_DATE) NAMESPACE=$(NAMESPACE) BINDING_VERSION=$(BINDING_VERSION) \
+	TEMPLATE_OUTPUT_FILENAME="k8s_prefixSelenium_basicAuth_secureServer_autoScaling_scaledJob.yaml" \
 	./tests/charts/make/chart_test.sh JobAutoscaling
 
 chart_test_autoscaling_job_hostname:
 	PLATFORMS=$(PLATFORMS) CHART_ENABLE_TRACING=true CHART_ENABLE_BASIC_AUTH=true \
 	SECURE_INGRESS_ONLY_DEFAULT=true SECURE_USE_EXTERNAL_CERT=true SELENIUM_GRID_PROTOCOL=https SELENIUM_GRID_HOST=$$(hostname -i) SELENIUM_GRID_PORT=443 \
 	VERSION=$(TAG_VERSION) VIDEO_TAG=$(FFMPEG_TAG_VERSION)-$(BUILD_DATE) NAMESPACE=$(NAMESPACE) BINDING_VERSION=$(BINDING_VERSION) \
+	TEMPLATE_OUTPUT_FILENAME="k8s_enableTracing_basicAuth_secureIngress_externalCerts_ingressPublicIP_autoScaling_scaledJob_subPath.yaml" \
 	./tests/charts/make/chart_test.sh JobAutoscaling
 
 chart_test_autoscaling_job:
 	PLATFORMS=$(PLATFORMS) TEST_CHROMIUM=true RELEASE_NAME=selenium CHART_ENABLE_TRACING=true CHART_FULL_DISTRIBUTED_MODE=true \
 	SECURE_INGRESS_ONLY_CONFIG_INLINE=true SECURE_USE_EXTERNAL_CERT=true CHART_ENABLE_INGRESS_HOSTNAME=true SELENIUM_GRID_PROTOCOL=https SELENIUM_GRID_HOST=selenium-grid.prod SUB_PATH=/ SELENIUM_GRID_PORT=443 \
 	VERSION=$(TAG_VERSION) VIDEO_TAG=$(FFMPEG_TAG_VERSION)-$(BUILD_DATE) NAMESPACE=$(NAMESPACE) BINDING_VERSION=$(BINDING_VERSION) \
+	TEMPLATE_OUTPUT_FILENAME="k8s_prefixSelenium_nodeChromium_enableTracing_fullDistributed_secureIngress_externalCerts_ingressHostName_ingressTLSInline_autoScaling_scaledJob_.yaml" \
 	./tests/charts/make/chart_test.sh JobAutoscaling
 
 chart_test_language_bindings:

diff --git a/charts/selenium-grid/Chart.yaml b/charts/selenium-grid/Chart.yaml
@@ -9,20 +9,20 @@ dependencies:
 - repository: https://kedacore.github.io/charts
   version: 2.15.0
   name: keda
-  condition: autoscaling.enabled
+  condition: autoscaling.enabled, keda.enabled
 - repository: https://kubernetes.github.io/ingress-nginx
   version: 4.11.1
   name: ingress-nginx
-  condition: ingress-nginx.enabled
+  condition: ingress.enabled, ingress-nginx.enabled
 - repository: https://jaegertracing.github.io/helm-charts
   version: 3.1.2
   name: jaeger
-  condition: tracing.enabled
+  condition: tracing.enabled, jaeger.enabled
 - repository: https://prometheus-community.github.io/helm-charts
   version: 61.8.0
   name: kube-prometheus-stack
-  condition: monitoring.enabled
   alias: prometheus-stack
+  condition: monitoring.enabled, prometheus-stack.enabled
 maintainers:
   - name: SeleniumHQ
     email: [email protected]

diff --git a/charts/selenium-grid/README.md b/charts/selenium-grid/README.md
@@ -719,7 +719,7 @@ Below is an example of Grid UI accessible via NodePort with secure connection, a
 
 ```bash
 helm upgrade -i $RELEASENAME -n $NAMESPACE docker-selenium/selenium-grid \
-  --set ingress.enabled=false \
+  --set ingress.enableWithExistingController=false \
   --set isolateComponents=true \
   --set components.router.serviceType=NodePort \
   --set tls.enabled=true \
@@ -776,8 +776,7 @@ helm upgrade -i $RELEASENAME -n $NAMESPACE docker-selenium/selenium-grid \
   --set ingress.enabled=true \
   --set ingress.hostname="selenium-grid.prod.domain.com" \
   --set tls.ingress.enabled=true \
-  --set tls.nameOverride=my-external-tls-secret \
-  --set ingress-nginx.enabled=true
+  --set tls.nameOverride=my-external-tls-secret
 ```
 
 Grid UI can be accessed via HTTPS address `https://selenium-grid.prod.domain.com`.
@@ -805,7 +804,6 @@ helm upgrade -i $RELEASENAME -n $NAMESPACE docker-selenium/selenium-grid \
   --set global.K8S_PUBLIC_IP=$(hostname -i) \
   --set tls.ingress.enabled=true \
   --set tls.nameOverride=my-external-tls-secret \
-  --set ingress-nginx.enabled=true \
   --set ingress-nginx.controller.extraArgs.default-ssl-certificate=$NAMESPACE/my-external-tls-secret
 ```
 
@@ -877,6 +875,9 @@ tracing:
     exporterEndpoint: 'http://jaeger.domain.com:4317'
 ```
 
+By default, the exporter is set to `otlp`. It is wide compatibility with many tracing backends.
+Read more: [vendors](https://opentelemetry.io/ecosystem/vendors/) native support OpenTelemetry and guidelines on [integration](https://opentelemetry.io/ecosystem/integrations/)
+
 ### Configuration of Selenium Grid chart
 This table contains the configuration parameters of the chart and their default values:
 
@@ -893,7 +894,8 @@ This table contains the configuration parameters of the chart and their default
 | `busConfigMap.annotations`                    | `{}`                                        | Custom annotations for configmap                                                                                           |
 | `nodeConfigMap.nameOverride`                  | ``                                          | Name of the configmap that contains common environment variables for browser nodes                                         |
 | `nodeConfigMap.annotations`                   | `{}`                                        | Custom annotations for configmap                                                                                           |
-| `ingress.enabled`                             | `true`                                      | Enable or disable ingress resource                                                                                         |
+| `ingress.enabled`                             | `false`                                     | Enable ingress. Implies installing Ingress NGINX Controller                                                                |
+| `ingress.enableWithExistingController`        | `true`                                      | Enable ingress without automatically installing Ingress NGINX Controller                                                   |
 | `ingress.className`                           | `""`                                        | Name of ingress class to select which controller will implement ingress resource                                           |
 | `ingress.annotations`                         | `{}`                                        | Custom annotations for ingress resource                                                                                    |
 | `ingress.nginx.proxyTimeout`                  | `3600`                                      | Value is used to set for NGINX ingress annotations related to proxy timeout                                                |

diff --git a/charts/selenium-grid/templates/NOTES.txt b/charts/selenium-grid/templates/NOTES.txt
@@ -6,7 +6,7 @@ Selenium Grid Server deployed successfully.
 {{- $port := ternary .Values.components.router.port .Values.hub.port .Values.isolateComponents }}
 {{- $localUrl := ternary "http://localhost:PORT" "http://localhost:PORT/wd/hub" .Values.isolateComponents }}
 
-{{- if .Values.ingress.enabled }}
+{{- if eq (include "seleniumGrid.ingress.enabled" $) "true" }}
     {{- if .Values.ingress.hostname }}
 1. Ingress is enabled, and it exposes the Grid Hub or Grid Router with the hostname you supplied.
     To access Selenium from outside of Kubernetes, simply open {{ include "seleniumGrid.url" .}}.

diff --git a/charts/selenium-grid/templates/_helpers.tpl b/charts/selenium-grid/templates/_helpers.tpl
@@ -160,6 +160,20 @@ Is autoscaling using KEDA enabled
 {{- or .Values.autoscaling.enabled .Values.autoscaling.enableWithExistingKEDA | ternary "true" "" -}}
 {{- end -}}
 
+{{/*
+Is ingress enabled
+*/}}
+{{- define "seleniumGrid.ingress.enabled" -}}
+{{- or .Values.ingress.enabled .Values.ingress.enableWithExistingController | ternary "true" "" -}}
+{{- end -}}
+
+{{/*
+Is ingress enabled
+*/}}
+{{- define "seleniumGrid.monitoring.enabled" -}}
+{{- or .Values.monitoring.enabled .Values.monitoring.enabledWithExistingAgent | ternary "true" "" -}}
+{{- end -}}
+
 {{/*
 Is tracing enabled
 */}}
@@ -595,7 +609,7 @@ Graphql Url of the hub or the router
 
 {{- define "seleniumGrid.url.host" -}}
 {{- $host := printf "%s.%s" (include ($.Values.isolateComponents | ternary "seleniumGrid.router.fullname" "seleniumGrid.hub.fullname") $ ) (.Release.Namespace) -}}
-{{- if .Values.ingress.enabled -}}
+{{- if eq (include "seleniumGrid.ingress.enabled" $) "true" -}}
   {{- if and (not .Values.ingress.hostname) .Values.global.K8S_PUBLIC_IP -}}
     {{- $host = .Values.global.K8S_PUBLIC_IP -}}
   {{- else if and .Values.ingress.hostname (ne (tpl .Values.ingress.hostname $) "selenium-grid.local") -}}
@@ -614,7 +628,7 @@ Graphql Url of the hub or the router
 
 {{- define "seleniumGrid.url.port" -}}
 {{- $port := ":4444" -}}
-{{- if .Values.ingress.enabled -}}
+{{- if eq (include "seleniumGrid.ingress.enabled" $) "true" -}}
   {{- if or (ne (.Values.ingress.ports.http | toString) "80") (ne (.Values.ingress.ports.https | toString) "443") -}}
     {{- $port = printf ":%s" (ternary (.Values.ingress.ports.http | toString) (.Values.ingress.ports.https | toString) (eq (include "seleniumGrid.url.schema" .) "http")) -}}
   {{- else if and .Values.ingress.hostname (eq (tpl .Values.ingress.hostname $) "selenium-grid.local") }}

diff --git a/charts/selenium-grid/templates/ingress.yaml b/charts/selenium-grid/templates/ingress.yaml
@@ -1,4 +1,4 @@
-{{- if .Values.ingress.enabled }}
+{{- if eq (include "seleniumGrid.ingress.enabled" $) "true" }}
 {{- if and .Values.ingress.className (not (semverCompare ">=1.18-0" .Capabilities.KubeVersion.GitVersion)) }}
   {{- if not (hasKey .Values.ingress.annotations "kubernetes.io/ingress.class") }}
   {{- $_ := set .Values.ingress.annotations "kubernetes.io/ingress.class" .Values.ingress.className}}

diff --git a/charts/selenium-grid/values.yaml b/charts/selenium-grid/values.yaml
@@ -119,8 +119,10 @@ rbacRoleBinding:
 
 # Configure the ingress resource to access the Grid installation.
 ingress:
-  # Enable or disable ingress resource
-  enabled: true
+  # Enable ingress. Implies installing Ingress NGINX Controller
+  enabled: false
+  # Enable ingress without automatically installing Ingress NGINX Controller
+  enableWithExistingController: true
   # Name of ingress class to select which controller will implement ingress resource
   className: ""
   # Refer to list nginx annotations: https://github.com/kubernetes/ingress-nginx/blob/main/docs/user-guide/nginx-configuration/annotations.md#annotations
@@ -631,8 +633,11 @@ hub:
   priorityClassName: ""
 
 tracing:
+  # Enable tracing. Implies installing Jaeger
   enabled: false
+  # Enable tracing without automatically installing Jaeger
   enabledWithExistingEndpoint: false
+  # Exporter type for tracing. Recommended `otlp` for wide compatibility with observability backends (e.g. Jaeger, Elastic, etc.)
   exporter: otlp
   exporterEndpoint: "http://{{ .Release.Name }}-jaeger-collector.{{ .Release.Namespace }}:4317"
   globalAutoConfigure: true
@@ -650,6 +655,7 @@ tracing:
 
 monitoring:
   enabled: false
+  enabledWithExistingAgent: false
 
 # Keda scaled object configuration
 autoscaling:
@@ -1305,6 +1311,7 @@ customLabels: {}
 
 # Configuration for dependency chart keda
 keda:
+  # enabled: false
   additionalAnnotations:
   http:
     timeout: 60000
@@ -1313,13 +1320,18 @@ keda:
 
 # Configuration for dependency chart ingress-nginx
 ingress-nginx:
-  enabled: false
+  # enabled: false
   controller:
     admissionWebhooks:
       enabled: false
 
+prometheus-stack:
+  # enabled: false
+  cleanPrometheusOperatorObjectNames: true
+
 # Configuration for dependency chart jaeger
 jaeger:
+  # enabled: false
   provisionDataStore:
     cassandra: false
   allInOne:

diff --git a/generate_release_notes.sh b/generate_release_notes.sh
@@ -45,7 +45,7 @@ echo "| rclone | ${RCLONE_VERSION} | ${RCLONE_VERSION} |" >> release_notes.md
 echo "| Java Runtime | ${JRE_VERSION} | ${JRE_VERSION} |" >> release_notes.md
 
 echo "" >> release_notes.md
-echo "### Published Docker images" >> release_notes.md
+echo "### Published Docker images on [Docker Hub](https://hub.docker.com/u/${NAMESPACE})" >> release_notes.md
 echo "<details>" >> release_notes.md
 echo "<summary>Click to see published Docker images</summary>" >> release_notes.md
 echo "" >> release_notes.md
@@ -55,3 +55,6 @@ echo '```' >> release_notes.md
 echo "" >> release_notes.md
 echo "</details>" >> release_notes.md
 
+echo "" >> release_notes.md
+chart_version=$(find . \( -type d -name .git -prune \) -o -type f -wholename '*/selenium-grid/Chart.yaml' -print0 | xargs -0 cat | grep ^version | cut -d ':' -f 2 | tr -d '[:space:]')
+echo "### Published Helm chart version [selenium-grid-${chart_version}](https://github.com/${AUTHORS:-"SeleniumHQ"}/docker-selenium/releases/tag/selenium-grid-${chart_version})" >> release_notes.md
diff --git a/tests/charts/ci/base-auth-ingress-values.yaml b/tests/charts/ci/base-auth-ingress-values.yaml
@@ -8,6 +8,7 @@ serverConfigMap:
     SE_SUPERVISORD_LOG_LEVEL: "error"
 
 ingress:
+  enabled: true
   className: nginx
   hostname: ""
 
@@ -31,7 +32,6 @@ components:
   extraEnvironmentVariables: *extraEnvironmentVariables
 
 ingress-nginx:
-  enabled: true
   controller:
     hostPort:
       enabled: true

diff --git a/tests/charts/make/chart_setup_env.sh b/tests/charts/make/chart_setup_env.sh
@@ -30,7 +30,7 @@ docker version
 docker buildx version
 docker buildx use default
 if [ "$(dpkg --print-architecture)" = "amd64" ]; then
-    docker run --rm --privileged multiarch/qemu-user-static --reset -p yes ;
+    docker run --rm --privileged multiarch/qemu-user-static --reset -p yes --credential yes ;
 else
     docker run --rm --privileged aptman/qus -- -r ;
     docker run --rm --privileged aptman/qus -s -- -p