diff --git a/.github/workflows/release-artifacts.yml b/.github/workflows/release-artifacts.yml
index 0beb418a07c3..eb294f161939 100644
--- a/.github/workflows/release-artifacts.yml
+++ b/.github/workflows/release-artifacts.yml
@@ -48,12 +48,43 @@ jobs:
         distro: ${{ fromJson(needs.get-distros.outputs.distros) }}
 
     steps:
-      - uses: actions/checkout@v2
+      - name: Checkout
+        uses: actions/checkout@v2
         with:
           path: src
-      - uses: actions/setup-python@v2
-      - run: ./src/scripts-dev/build_debian_packages "${{ matrix.distro }}"
-      - uses: actions/upload-artifact@v2
+
+      - name: Set up Docker Buildx
+        id: buildx
+        uses: docker/setup-buildx-action@v1
+        with:
+          install: true
+
+      - name: Set up docker layer caching
+        uses: actions/cache@v2
+        with:
+          path: /tmp/.buildx-cache
+          key: ${{ runner.os }}-buildx-${{ github.sha }}
+          restore-keys: |
+            ${{ runner.os }}-buildx-
+
+      - name: Set up python
+        uses: actions/setup-python@v2
+
+      - name: Build the packages
+        # see https://github.com/docker/build-push-action/issues/252
+        # for the cache magic here
+        run: |
+          ./src/scripts-dev/build_debian_packages \
+            --docker-build-arg=--cache-from=type=local,src=/tmp/.buildx-cache \
+            --docker-build-arg=--cache-to=type=local,mode=max,dest=/tmp/.buildx-cache-new \
+            --docker-build-arg=--progress=plain \
+            --docker-build-arg=--load \
+            "${{ matrix.distro }}"
+          rm -rf /tmp/.buildx-cache
+          mv /tmp/.buildx-cache-new /tmp/.buildx-cache
+
+      - name: Upload debs as artifacts
+        uses: actions/upload-artifact@v2
         with:
           name: debs
           path: debs/*
diff --git a/changelog.d/10431.misc b/changelog.d/10431.misc
new file mode 100644
index 000000000000..34b9b49da680
--- /dev/null
+++ b/changelog.d/10431.misc
@@ -0,0 +1 @@
+Use a docker image cache for the prerequisites for the debian package build.
diff --git a/scripts-dev/build_debian_packages b/scripts-dev/build_debian_packages
index e25c5bb2650d..0ed1c679fd94 100755
--- a/scripts-dev/build_debian_packages
+++ b/scripts-dev/build_debian_packages
@@ -17,6 +17,7 @@ import subprocess
 import sys
 import threading
 from concurrent.futures import ThreadPoolExecutor
+from typing import Optional, Sequence
 
 DISTS = (
     "debian:buster",
@@ -39,8 +40,11 @@ projdir = os.path.dirname(os.path.dirname(os.path.realpath(__file__)))
 
 
 class Builder(object):
-    def __init__(self, redirect_stdout=False):
+    def __init__(
+        self, redirect_stdout=False, docker_build_args: Optional[Sequence[str]] = None
+    ):
         self.redirect_stdout = redirect_stdout
+        self._docker_build_args = tuple(docker_build_args or ())
         self.active_containers = set()
         self._lock = threading.Lock()
         self._failed = False
@@ -79,8 +83,8 @@ class Builder(object):
             stdout = None
 
         # first build a docker image for the build environment
-        subprocess.check_call(
-            [
+        build_args = (
+            (
                 "docker",
                 "build",
                 "--tag",
@@ -89,8 +93,13 @@ class Builder(object):
                 "distro=" + dist,
                 "-f",
                 "docker/Dockerfile-dhvirtualenv",
-                "docker",
-            ],
+            )
+            + self._docker_build_args
+            + ("docker",)
+        )
+
+        subprocess.check_call(
+            build_args,
             stdout=stdout,
             stderr=subprocess.STDOUT,
             cwd=projdir,
@@ -147,9 +156,7 @@ class Builder(object):
                 self.active_containers.remove(c)
 
 
-def run_builds(dists, jobs=1, skip_tests=False):
-    builder = Builder(redirect_stdout=(jobs > 1))
-
+def run_builds(builder, dists, jobs=1, skip_tests=False):
     def sig(signum, _frame):
         print("Caught SIGINT")
         builder.kill_containers()
@@ -180,6 +187,11 @@ if __name__ == "__main__":
         action="store_true",
         help="skip running tests after building",
     )
+    parser.add_argument(
+        "--docker-build-arg",
+        action="append",
+        help="specify an argument to pass to docker build",
+    )
     parser.add_argument(
         "--show-dists-json",
         action="store_true",
@@ -195,4 +207,12 @@ if __name__ == "__main__":
     if args.show_dists_json:
         print(json.dumps(DISTS))
     else:
-        run_builds(dists=args.dist, jobs=args.jobs, skip_tests=args.no_check)
+        builder = Builder(
+            redirect_stdout=(args.jobs > 1), docker_build_args=args.docker_build_arg
+        )
+        run_builds(
+            builder,
+            dists=args.dist,
+            jobs=args.jobs,
+            skip_tests=args.no_check,
+        )