diff --git a/tests/buildkite/build-cpu-arm64.sh b/tests/buildkite/build-cpu-arm64.sh
index fd00a7971101..3bbc954729ce 100755
--- a/tests/buildkite/build-cpu-arm64.sh
+++ b/tests/buildkite/build-cpu-arm64.sh
@@ -8,7 +8,7 @@ echo "--- Build CPU code targeting ARM64"
 
 source tests/buildkite/conftest.sh
 
-command_wrapper="tests/ci_build/ci_build.sh aarch64 docker"
+command_wrapper="tests/ci_build/ci_build.sh aarch64"
 
 echo "--- Build libxgboost from the source"
 $command_wrapper tests/ci_build/build_via_cmake.sh --conda-env=aarch64_test \
diff --git a/tests/buildkite/build-cpu.sh b/tests/buildkite/build-cpu.sh
index 73e88d8aac02..11679d644de1 100755
--- a/tests/buildkite/build-cpu.sh
+++ b/tests/buildkite/build-cpu.sh
@@ -6,7 +6,7 @@ echo "--- Build CPU code"
 
 source tests/buildkite/conftest.sh
 
-command_wrapper="tests/ci_build/ci_build.sh cpu docker"
+command_wrapper="tests/ci_build/ci_build.sh cpu"
 
 $command_wrapper rm -fv dmlc-core/include/dmlc/build_config_default.h
   # This step is not necessary, but here we include it, to ensure that
diff --git a/tests/buildkite/build-cuda-with-rmm.sh b/tests/buildkite/build-cuda-with-rmm.sh
index 6156082490c0..559bad8a786b 100755
--- a/tests/buildkite/build-cuda-with-rmm.sh
+++ b/tests/buildkite/build-cuda-with-rmm.sh
@@ -15,7 +15,7 @@ else
   arch_flag=""
 fi
 
-command_wrapper="tests/ci_build/ci_build.sh gpu_build_centos7 docker --build-arg "`
+command_wrapper="tests/ci_build/ci_build.sh gpu_build_centos7 --build-arg "`
                 `"CUDA_VERSION_ARG=$CUDA_VERSION --build-arg "`
                 `"NCCL_VERSION_ARG=$NCCL_VERSION --build-arg "`
                 `"RAPIDS_VERSION_ARG=$RAPIDS_VERSION"
@@ -40,13 +40,13 @@ $command_wrapper python tests/ci_build/rename_whl.py python-package/dist/*.whl \
   ${BUILDKITE_COMMIT} ${WHEEL_TAG}
 
 echo "--- Audit binary wheel to ensure it's compliant with manylinux2014 standard"
-tests/ci_build/ci_build.sh auditwheel_x86_64 docker auditwheel repair \
+tests/ci_build/ci_build.sh auditwheel_x86_64 auditwheel repair \
   --plat ${WHEEL_TAG} python-package/dist/*.whl
 $command_wrapper python tests/ci_build/rename_whl.py wheelhouse/*.whl \
   ${BUILDKITE_COMMIT} ${WHEEL_TAG}
 mv -v wheelhouse/*.whl python-package/dist/
 # Make sure that libgomp.so is vendored in the wheel
-tests/ci_build/ci_build.sh auditwheel_x86_64 docker bash -c \
+tests/ci_build/ci_build.sh auditwheel_x86_64 bash -c \
   "unzip -l python-package/dist/*.whl | grep libgomp  || exit -1"
 
 echo "--- Upload Python wheel"
diff --git a/tests/buildkite/build-cuda.sh b/tests/buildkite/build-cuda.sh
index 7bd3492a2d6d..5abc5ca5aebd 100755
--- a/tests/buildkite/build-cuda.sh
+++ b/tests/buildkite/build-cuda.sh
@@ -15,7 +15,7 @@ else
   arch_flag=""
 fi
 
-command_wrapper="tests/ci_build/ci_build.sh gpu_build_centos7 docker --build-arg "`
+command_wrapper="tests/ci_build/ci_build.sh gpu_build_centos7 --build-arg "`
                 `"CUDA_VERSION_ARG=$CUDA_VERSION --build-arg "`
                 `"NCCL_VERSION_ARG=$NCCL_VERSION --build-arg "`
                 `"RAPIDS_VERSION_ARG=$RAPIDS_VERSION"
@@ -39,13 +39,13 @@ $command_wrapper python tests/ci_build/rename_whl.py python-package/dist/*.whl \
   ${BUILDKITE_COMMIT} ${WHEEL_TAG}
 
 echo "--- Audit binary wheel to ensure it's compliant with manylinux2014 standard"
-tests/ci_build/ci_build.sh auditwheel_x86_64 docker auditwheel repair \
+tests/ci_build/ci_build.sh auditwheel_x86_64 auditwheel repair \
   --plat ${WHEEL_TAG} python-package/dist/*.whl
 $command_wrapper python tests/ci_build/rename_whl.py wheelhouse/*.whl \
   ${BUILDKITE_COMMIT} ${WHEEL_TAG}
 mv -v wheelhouse/*.whl python-package/dist/
 # Make sure that libgomp.so is vendored in the wheel
-tests/ci_build/ci_build.sh auditwheel_x86_64 docker bash -c \
+tests/ci_build/ci_build.sh auditwheel_x86_64 bash -c \
   "unzip -l python-package/dist/*.whl | grep libgomp  || exit -1"
 
 echo "--- Upload Python wheel"
diff --git a/tests/buildkite/build-gpu-rpkg.sh b/tests/buildkite/build-gpu-rpkg.sh
index 585dc79ae443..78a534615ac5 100755
--- a/tests/buildkite/build-gpu-rpkg.sh
+++ b/tests/buildkite/build-gpu-rpkg.sh
@@ -6,7 +6,7 @@ source tests/buildkite/conftest.sh
 
 echo "--- Build XGBoost R package with CUDA"
 
-tests/ci_build/ci_build.sh gpu_build_r_centos7 docker \
+tests/ci_build/ci_build.sh gpu_build_r_centos7 \
 			   --build-arg CUDA_VERSION_ARG=${CUDA_VERSION} \
 			   --build-arg R_VERSION_ARG=${R_VERSION} \
 			   tests/ci_build/build_r_pkg_with_cuda.sh \
diff --git a/tests/buildkite/build-jvm-doc.sh b/tests/buildkite/build-jvm-doc.sh
index a2d658e48d2b..d168eb8cc58d 100755
--- a/tests/buildkite/build-jvm-doc.sh
+++ b/tests/buildkite/build-jvm-doc.sh
@@ -5,7 +5,7 @@ set -euo pipefail
 source tests/buildkite/conftest.sh
 
 echo "--- Build JVM packages doc"
-tests/ci_build/ci_build.sh jvm docker tests/ci_build/build_jvm_doc.sh ${BRANCH_NAME}
+tests/ci_build/ci_build.sh jvm tests/ci_build/build_jvm_doc.sh ${BRANCH_NAME}
 if [[ ($is_pull_request == 0) && ($is_release_branch == 1) ]]
 then
   echo "--- Upload JVM packages doc"
diff --git a/tests/buildkite/build-jvm-packages.sh b/tests/buildkite/build-jvm-packages.sh
index 33cfffe713bc..12393c56154d 100755
--- a/tests/buildkite/build-jvm-packages.sh
+++ b/tests/buildkite/build-jvm-packages.sh
@@ -5,13 +5,13 @@ set -euo pipefail
 source tests/buildkite/conftest.sh
 
 echo "--- Build XGBoost JVM packages scala 2.12"
-tests/ci_build/ci_build.sh jvm docker tests/ci_build/build_jvm_packages.sh \
+tests/ci_build/ci_build.sh jvm tests/ci_build/build_jvm_packages.sh \
   ${SPARK_VERSION}
 
 
 echo "--- Build XGBoost JVM packages scala 2.13"
 
-tests/ci_build/ci_build.sh jvm docker tests/ci_build/build_jvm_packages.sh \
+tests/ci_build/ci_build.sh jvm tests/ci_build/build_jvm_packages.sh \
   ${SPARK_VERSION} "" "" "true"
 
 echo "--- Stash XGBoost4J JARs"
diff --git a/tests/buildkite/deploy-jvm-packages.sh b/tests/buildkite/deploy-jvm-packages.sh
index a3410b294bd6..812a6c5cafec 100755
--- a/tests/buildkite/deploy-jvm-packages.sh
+++ b/tests/buildkite/deploy-jvm-packages.sh
@@ -7,7 +7,7 @@ source tests/buildkite/conftest.sh
 if [[ ($is_pull_request == 0) && ($is_release_branch == 1) ]]
 then
   echo "--- Deploy JVM packages to xgboost-maven-repo S3 repo"
-  tests/ci_build/ci_build.sh jvm_gpu_build docker \
+  tests/ci_build/ci_build.sh jvm_gpu_build \
     --build-arg CUDA_VERSION_ARG=${CUDA_VERSION} \
     --build-arg NCCL_VERSION_ARG=${NCCL_VERSION} \
     tests/ci_build/deploy_jvm_packages.sh ${SPARK_VERSION}
diff --git a/tests/buildkite/run-clang-tidy.sh b/tests/buildkite/run-clang-tidy.sh
index 1a664f5687a1..95ff010c20f1 100755
--- a/tests/buildkite/run-clang-tidy.sh
+++ b/tests/buildkite/run-clang-tidy.sh
@@ -6,6 +6,6 @@ echo "--- Run clang-tidy"
 
 source tests/buildkite/conftest.sh
 
-tests/ci_build/ci_build.sh clang_tidy docker \
+tests/ci_build/ci_build.sh clang_tidy \
   --build-arg CUDA_VERSION_ARG=${CUDA_VERSION} \
   python3 tests/ci_build/tidy.py --cuda-archs 75
diff --git a/tests/buildkite/test-integration-jvm-packages.sh b/tests/buildkite/test-integration-jvm-packages.sh
index 9f477720a52a..51f74afe9006 100755
--- a/tests/buildkite/test-integration-jvm-packages.sh
+++ b/tests/buildkite/test-integration-jvm-packages.sh
@@ -9,5 +9,5 @@ buildkite-agent artifact download "jvm-packages/xgboost4j/target/*.jar" . --step
 buildkite-agent artifact download "jvm-packages/xgboost4j-spark/target/*.jar" . --step build-jvm-packages
 buildkite-agent artifact download "jvm-packages/xgboost4j-example/target/*.jar" . --step build-jvm-packages
 export CI_DOCKER_EXTRA_PARAMS_INIT='-e RUN_INTEGRATION_TEST=1'
-tests/ci_build/ci_build.sh jvm_cross docker --build-arg JDK_VERSION=${JDK_VERSION} \
+tests/ci_build/ci_build.sh jvm_cross --build-arg JDK_VERSION=${JDK_VERSION} \
   --build-arg SPARK_VERSION=${SPARK_VERSION} tests/ci_build/test_jvm_cross.sh
diff --git a/tests/buildkite/test-python-cpu-arm64.sh b/tests/buildkite/test-python-cpu-arm64.sh
index ed1ad101b7c9..68a428034073 100755
--- a/tests/buildkite/test-python-cpu-arm64.sh
+++ b/tests/buildkite/test-python-cpu-arm64.sh
@@ -8,4 +8,4 @@ echo "--- Test Python CPU ARM64"
 buildkite-agent artifact download "python-package/dist/*.whl" . --step build-cpu-arm64
 buildkite-agent artifact download "xgboost" . --step build-cpu-arm64
 chmod +x ./xgboost
-tests/ci_build/ci_build.sh aarch64 docker tests/ci_build/test_python.sh cpu-arm64
+tests/ci_build/ci_build.sh aarch64 tests/ci_build/test_python.sh cpu-arm64
diff --git a/tests/buildkite/test-python-cpu.sh b/tests/buildkite/test-python-cpu.sh
index 938e1184eabf..6c53dc2821bc 100755
--- a/tests/buildkite/test-python-cpu.sh
+++ b/tests/buildkite/test-python-cpu.sh
@@ -13,4 +13,4 @@ chmod +x ./xgboost
 
 export BUILDKITE_ANALYTICS_TOKEN=$(get_aws_secret buildkite/test_analytics/cpu)
 set_buildkite_env_vars_in_container
-tests/ci_build/ci_build.sh cpu docker tests/ci_build/test_python.sh cpu
+tests/ci_build/ci_build.sh cpu tests/ci_build/test_python.sh cpu