diff --git a/.github/actions/smoke-tests/action.yaml b/.github/actions/smoke-tests/action.yaml
index 288f414389..d80f10e247 100644
--- a/.github/actions/smoke-tests/action.yaml
+++ b/.github/actions/smoke-tests/action.yaml
@@ -40,12 +40,12 @@ runs:
       with:
         file: build/Dockerfile
         context: '.'
-        # Disable cache until https://github.com/docker/buildx/issues/681 is fixed
-        # cache-from: type=gha,scope=${{ inputs.image }}
-        # cache-to: type=gha,scope=${{ inputs.image }},mode=max
+        cache-from: type=gha,scope=${{ inputs.image }}
+        cache-to: type=gha,scope=${{ inputs.image }},mode=max
         target: goreleaser
         tags: 'docker.io/nginx/nginx-ingress:${{ inputs.image }}-${{ github.sha }}'
         load: true
+        pull: true
         build-args: BUILD_OS=${{ inputs.image }}
 
     - name: Build Test-Runner Container
@@ -53,10 +53,10 @@ runs:
       with:
         file: tests/docker/Dockerfile
         context: '.'
-        # Disable cache until https://github.com/docker/buildx/issues/681 is fixed
-        # cache-from: type=gha,scope=test-runner
-        # cache-to: type=gha,scope=test-runner,mode=max
+        cache-from: type=gha,scope=test-runner
+        cache-to: type=gha,scope=test-runner,mode=max
         tags: test-runner:${{ github.sha }}
+        pull: true
         load: true
 
     - name: Deploy Kubernetes
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 7b5d0d8420..ab349f6649 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -178,11 +178,11 @@ jobs:
         with:
           file: build/Dockerfile
           context: '.'
-          # Disable cache until https://github.com/docker/buildx/issues/681 is fixed
-          # cache-from: type=gha
-          # cache-to: type=gha,mode=max
+          cache-from: type=gha,scope=debian
+          cache-to: type=gha,scope=debian,mode=max
           target: goreleaser
           tags: nginx-ingress:${{ github.sha }}
+          pull: true
           load: true
           build-args: BUILD_OS=debian
       - name: Deploy Kubernetes
@@ -322,9 +322,8 @@ jobs:
         with:
           file: build/Dockerfile
           context: '.'
-          # Disable cache until https://github.com/docker/buildx/issues/681 is fixed
-          # cache-from: type=gha
-          # cache-to: type=gha,mode=max
+          cache-from: type=gha,scope=${{ matrix.image }}
+          cache-to: type=gha,scope=${{ matrix.image }},mode=max
           target: goreleaser
           tags: ${{ steps.meta.outputs.tags }}
           labels: ${{ steps.meta.outputs.labels }}
diff --git a/.github/workflows/update-docker-images.yml b/.github/workflows/update-docker-images.yml
index 5e7fa9e16b..a669033479 100644
--- a/.github/workflows/update-docker-images.yml
+++ b/.github/workflows/update-docker-images.yml
@@ -209,13 +209,14 @@ jobs:
         with:
           file: build/Dockerfile
           context: '.'
-          cache-from: type=gha
-          cache-to: type=gha,mode=max
+          cache-from: type=gha,scope=${{ matrix.image }}
+          cache-to: type=gha,scope=${{ matrix.image }},mode=max
           target: goreleaser
           tags: ${{ steps.meta.outputs.tags }}
           labels: ${{ steps.meta.outputs.labels }}
           platforms: ${{ matrix.platforms }}
           push: true
+          pull: true
           build-args: |
             BUILD_OS=${{ matrix.image }}
             IC_VERSION=v${{ needs.variables.outputs.kic-tag }}