diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml
index 1a43ffdde3bd2..c4a89c4358ba9 100644
--- a/.github/workflows/docker.yml
+++ b/.github/workflows/docker.yml
@@ -8,27 +8,7 @@ on:
     types: [synchronize, opened, reopened, ready_for_review]
 
 jobs:
-  config:
-    runs-on: "ubuntu-latest"
-    outputs:
-      has-secrets: ${{ steps.check.outputs.has-secrets }}
-    steps:
-      - name: "Check for secrets"
-        id: check
-        env:
-          DOCKERHUB_USER: ${{ secrets.DOCKERHUB_USER }}
-          DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }}
-        run: |
-          if [[ -n "$DOCKERHUB_USER" && -n "$DOCKERHUB_TOKEN" ]]; then
-            echo "has-secrets=true" >> "$GITHUB_ENV"
-            echo "has secrets!"
-          else
-            echo "has-secrets=false" >> "$GITHUB_ENV"
-            echo "no secrets!"
-          fi
   docker-build:
-    needs: config
-    if: needs.config.outputs.has-secrets == 'true'
     name: docker-build
     runs-on: ubuntu-latest
     strategy:
@@ -63,9 +43,8 @@ jobs:
         run: |
           ./scripts/docker_build_push.sh "" ${{ matrix.target }} ${{ matrix.platform }}
 
+
   ephemeral-docker-build:
-    needs: config
-    if: needs.config.outputs.has-secrets
     name: docker-build
     runs-on: ubuntu-latest
     steps:
diff --git a/scripts/docker_build_push.sh b/scripts/docker_build_push.sh
index 5ad7d2d999734..3d0271cb2b4d2 100755
--- a/scripts/docker_build_push.sh
+++ b/scripts/docker_build_push.sh
@@ -130,15 +130,22 @@ if [[ -n "${BUILD_TARGET}" ]]; then
   TARGET_ARGUMENT="--target ${BUILD_TARGET}"
 fi
 
+# Building the cache settings
 CACHE_REF="${REPO_NAME}-cache:${TARGET}-${BUILD_ARG}"
 CACHE_REF=$(echo "${CACHE_REF}" | tr -d '.')
+CACHE_FROM_ARG="--cache-from=type=registry,ref=${CACHE_REF}"
+CACHE_TO_ARG=""
+if [ -n "${DOCKERHUB_TOKEN}" ]; then
+  # need to be logged in to push to the cache
+  CACHE_TO_ARG="--cache-to=type=registry,mode=max,ref=${CACHE_REF}"
+fi
 
 docker buildx build \
   ${TARGET_ARGUMENT} \
   ${DOCKER_ARGS} \
-  --cache-from=type=registry,ref=${CACHE_REF} \
-  --cache-to=type=registry,mode=max,ref=${CACHE_REF} \
   ${DOCKER_TAGS} \
+  ${CACHE_FROM_ARG} \
+  ${CACHE_TO_ARG} \
   --platform ${BUILD_PLATFORM} \
   --label "sha=${SHA}" \
   --label "built_at=$(date)" \