diff --git a/.github/workflows/flags.yml b/.github/workflows/flags.yml
index 8cb859df5b6..3f746adeae1 100644
--- a/.github/workflows/flags.yml
+++ b/.github/workflows/flags.yml
@@ -36,18 +36,6 @@ jobs:
                          , ".circleci/**"
                          , "**/stack*.yaml"
                          ]'
-      # If we only change ghcide downstream packages we have not test ghcide itself
-      - id: skip_ghcide_check
-        uses: fkirc/skip-duplicate-actions@v3.4.1
-        with:
-          cancel_others: false
-          paths_ignore: '[ "hls-test-utils/**"
-                         , "plugins/**"
-                         , "src/**"
-                         , "exe/**"
-                         , "test/**"
-                         , "shake-bench/**"
-                         ]'
 
   flags:
     if: needs.pre_job.outputs.should_skip != 'true'
@@ -131,6 +119,10 @@ jobs:
                 ${{ env.cache-name }}-${{ runner.os }}-${{ matrix.ghc }}-
                 ${{ env.cache-name }}-${{ runner.os }}-
 
+      # To ensure we get the lastest hackage index and not relying on haskell action logic
+      - if: steps.compiled-deps.outputs.cache-hit != 'true'
+        run: cabal update
+
       - name: Build `hls-graph` with flags
         run: cabal v2-build hls-graph --flags="pedantic embed-files stm-stats"
 
@@ -145,7 +137,7 @@ jobs:
 
       # repeating builds to workaround segfaults in windows and ghc-8.8.4
       - name: Build
-        run: cabal v2-build ghcide --flags="ghc-patched-unboxed-bytecode test-exe executable bench-exe" || cabal v2-build ghcide --flags="ghc-patched-unboxed-bytecode test-exe executable bench-exe" || cabal v2-build ghcide --flags="ghc-patched-unboxed-bytecode test-exe executable bench-exe"
+        run: cabal v2-build ghcide --flags="ghc-patched-unboxed-bytecode test-exe executable bench-exe"
 
   flags_post_job:
     if: always()