diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 534447e77c0..767c9a23403 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -74,6 +74,7 @@ jobs:
           done
 
   build:
+    if: ${{ github.event_name != 'pull_request' || matrix.python-version == '3.11' }}
     needs: [fmt] # don't wait for clippy as fails rarely and takes longer
     name: python${{ matrix.python-version }}-${{ matrix.platform.python-architecture }} ${{ matrix.platform.os }} rust-${{ matrix.rust }}
     runs-on: ${{ matrix.platform.os }}