diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
index b8fda765034..8c95631a1f4 100644
--- a/.github/workflows/test.yml
+++ b/.github/workflows/test.yml
@@ -138,6 +138,13 @@ jobs:
           sed -i.bak -e 's/Paths_haskell_language_server/Paths_hls/g' \
                      src/**/*.hs exe/*.hs
 
+      # this is only safe if the test environment is isolated
+      - name: clean ide cache
+        run: |
+          pgrep -af ghcide; true
+          pgrep -af hls; true
+          rm -rf ~/.cache/ghcide
+
       - if: ${{ needs.pre_job.outputs.should_skip != 'true' }}
         name: Build
         # Retry it three times to workaround compiler segfaults in windows