Try a new approach for caching Maven local repository

We used to cache a repo once a week but only after building the Java 17 build. The new approach is the following: - We use dependency:go-offline to populate the cache, hoping the quality will be better - We drop the cache entirely every month - We have one cache entry per branch and we keep updating it during the month by creating new caches - We use restore-keys to find the best cache possible for a given run - We share this cache with all the workflows (cherry picked from commit b3756d8)
quarkusio · Aug 20, 2024 · 3842a4e · 3842a4e
1 parent 1b39cd8
commit 3842a4e
Show file tree

Hide file tree

Showing 7 changed files with 141 additions and 59 deletions.
diff --git a/.github/workflows/ci-actions-incremental.yml b/.github/workflows/ci-actions-incremental.yml
@@ -105,7 +105,8 @@ jobs:
     outputs:
       gib_args: ${{ steps.get-gib-args.outputs.gib_args }}
       gib_impacted: ${{ steps.get-gib-impacted.outputs.impacted_modules }}
-      m2-cache-key: ${{ steps.cache-key.outputs.m2-cache-key }}
+      m2-monthly-branch-cache-key: ${{ steps.cache-key.outputs.m2-monthly-branch-cache-key }}
+      m2-monthly-cache-key: ${{ steps.cache-key.outputs.m2-monthly-cache-key }}
       quarkus-metadata-cache-key: ${{ steps.cache-key.outputs.quarkus-metadata-cache-key }}
       quarkus-metadata-cache-key-default: ${{ steps.cache-key.outputs.quarkus-metadata-cache-key-default }}
     steps:
@@ -129,17 +130,29 @@ jobs:
       - name: Generate cache key
         id: cache-key
         run: |
+          CURRENT_BRANCH="${{ github.base_ref || github.ref_name }}"
+          CURRENT_MONTH=$(/bin/date -u "+%Y-%m")
+          ROOT_CACHE_KEY="m2-cache"
+          echo "m2-monthly-cache-key=${ROOT_CACHE_KEY}-${CURRENT_MONTH}" >> $GITHUB_OUTPUT
+          echo "m2-monthly-branch-cache-key=${ROOT_CACHE_KEY}-${CURRENT_MONTH}-${CURRENT_BRANCH}" >> $GITHUB_OUTPUT
           CURRENT_WEEK=$(/bin/date -u "+%Y-%U")
-          echo "m2-cache-key=m2-cache-${CURRENT_WEEK}" >> $GITHUB_OUTPUT
           echo "quarkus-metadata-cache-key=quarkus-metadata-cache-${CURRENT_WEEK}-${{ github.ref_name }}" >> $GITHUB_OUTPUT
           echo "quarkus-metadata-cache-key-default=quarkus-metadata-cache-${CURRENT_WEEK}-${{ github.event.repository.default_branch }}" >> $GITHUB_OUTPUT
       - name: Cache Maven Repository
         id: cache-maven
         uses: actions/cache@v4
         with:
           path: ~/.m2/repository
-          # refresh cache every week to avoid unlimited growth
-          key: ${{ steps.cache-key.outputs.m2-cache-key }}
+          # This will never work but it should fallback to the restore key below and get the latest cache for this branch.
+          # The cache is dropped monthly to prevent unlimited growth.
+          # The cache is per branch.
+          key: ${{ steps.cache-key.outputs.m2-monthly-branch-cache-key }}-${{ github.run_id }}
+          restore-keys: |
+            ${{ steps.cache-key.outputs.m2-monthly-branch-cache-key }}-
+            ${{ steps.cache-key.outputs.m2-monthly-cache-key }}-
+      - name: Populate the cache
+        run: |
+          ./mvnw -T2C $COMMON_MAVEN_ARGS dependency:go-offline
       - name: Verify native-tests.json
         run: ./.github/verify-tests-json.sh native-tests.json integration-tests/
       - name: Verify virtual-threads-tests.json
@@ -362,8 +375,10 @@ jobs:
         uses: actions/cache/restore@v4
         with:
           path: ~/.m2/repository
-          # refresh cache every week to avoid unlimited growth
-          key: ${{ needs.build-jdk17.outputs.m2-cache-key }}
+          key: ${{ needs.build-jdk17.outputs.m2-monthly-branch-cache-key }}-${{ github.run_id }}
+          restore-keys: |
+            ${{ needs.build-jdk17.outputs.m2-monthly-branch-cache-key }}-
+            ${{ needs.build-jdk17.outputs.m2-monthly-cache-key }}-
       - name: Download .m2/repository/io/quarkus
         uses: actions/download-artifact@v4
         with:
@@ -467,8 +482,10 @@ jobs:
         uses: actions/cache/restore@v4
         with:
           path: ~/.m2/repository
-          # refresh cache every week to avoid unlimited growth
-          key: ${{ needs.build-jdk17.outputs.m2-cache-key }}
+          key: ${{ needs.build-jdk17.outputs.m2-monthly-branch-cache-key }}-${{ github.run_id }}
+          restore-keys: |
+            ${{ needs.build-jdk17.outputs.m2-monthly-branch-cache-key }}-
+            ${{ needs.build-jdk17.outputs.m2-monthly-cache-key }}-
       - name: Download .m2/repository/io/quarkus
         uses: actions/download-artifact@v4
         with:
@@ -567,8 +584,10 @@ jobs:
         uses: actions/cache/restore@v4
         with:
           path: ~/.m2/repository
-          # refresh cache every week to avoid unlimited growth
-          key: ${{ needs.build-jdk17.outputs.m2-cache-key }}
+          key: ${{ needs.build-jdk17.outputs.m2-monthly-branch-cache-key }}-${{ github.run_id }}
+          restore-keys: |
+            ${{ needs.build-jdk17.outputs.m2-monthly-branch-cache-key }}-
+            ${{ needs.build-jdk17.outputs.m2-monthly-cache-key }}-
       - name: Download .m2/repository/io/quarkus
         uses: actions/download-artifact@v4
         with:
@@ -657,8 +676,10 @@ jobs:
         uses: actions/cache/restore@v4
         with:
           path: ~/.m2/repository
-          # refresh cache every week to avoid unlimited growth
-          key: ${{ needs.build-jdk17.outputs.m2-cache-key }}
+          key: ${{ needs.build-jdk17.outputs.m2-monthly-branch-cache-key }}-${{ github.run_id }}
+          restore-keys: |
+            ${{ needs.build-jdk17.outputs.m2-monthly-branch-cache-key }}-
+            ${{ needs.build-jdk17.outputs.m2-monthly-cache-key }}-
       - name: Download .m2/repository/io/quarkus
         uses: actions/download-artifact@v4
         with:
@@ -751,8 +772,10 @@ jobs:
         uses: actions/cache/restore@v4
         with:
           path: ~/.m2/repository
-          # refresh cache every week to avoid unlimited growth
-          key: ${{ needs.build-jdk17.outputs.m2-cache-key }}
+          key: ${{ needs.build-jdk17.outputs.m2-monthly-branch-cache-key }}-${{ github.run_id }}
+          restore-keys: |
+            ${{ needs.build-jdk17.outputs.m2-monthly-branch-cache-key }}-
+            ${{ needs.build-jdk17.outputs.m2-monthly-cache-key }}-
       - name: Download .m2/repository/io/quarkus
         uses: actions/download-artifact@v4
         with:
@@ -832,8 +855,10 @@ jobs:
         uses: actions/cache/restore@v4
         with:
           path: ~/.m2/repository
-          # refresh cache every week to avoid unlimited growth
-          key: ${{ needs.build-jdk17.outputs.m2-cache-key }}
+          key: ${{ needs.build-jdk17.outputs.m2-monthly-branch-cache-key }}-${{ github.run_id }}
+          restore-keys: |
+            ${{ needs.build-jdk17.outputs.m2-monthly-branch-cache-key }}-
+            ${{ needs.build-jdk17.outputs.m2-monthly-cache-key }}-
       - name: Download .m2/repository/io/quarkus
         uses: actions/download-artifact@v4
         with:
@@ -914,8 +939,10 @@ jobs:
         uses: actions/cache/restore@v4
         with:
           path: ~/.m2/repository
-          # refresh cache every week to avoid unlimited growth
-          key: ${{ needs.build-jdk17.outputs.m2-cache-key }}
+          key: ${{ needs.build-jdk17.outputs.m2-monthly-branch-cache-key }}-${{ github.run_id }}
+          restore-keys: |
+            ${{ needs.build-jdk17.outputs.m2-monthly-branch-cache-key }}-
+            ${{ needs.build-jdk17.outputs.m2-monthly-cache-key }}-
       - name: Download .m2/repository/io/quarkus
         uses: actions/download-artifact@v4
         with:
@@ -996,8 +1023,10 @@ jobs:
         uses: actions/cache/restore@v4
         with:
           path: ~/.m2/repository
-          # refresh cache every week to avoid unlimited growth
-          key: ${{ needs.build-jdk17.outputs.m2-cache-key }}
+          key: ${{ needs.build-jdk17.outputs.m2-monthly-branch-cache-key }}-${{ github.run_id }}
+          restore-keys: |
+            ${{ needs.build-jdk17.outputs.m2-monthly-branch-cache-key }}-
+            ${{ needs.build-jdk17.outputs.m2-monthly-cache-key }}-
       - name: Download .m2/repository/io/quarkus
         uses: actions/download-artifact@v4
         with:
@@ -1107,8 +1136,10 @@ jobs:
         uses: actions/cache/restore@v4
         with:
           path: ~/.m2/repository
-          # refresh cache every week to avoid unlimited growth
-          key: ${{ needs.build-jdk17.outputs.m2-cache-key }}
+          key: ${{ needs.build-jdk17.outputs.m2-monthly-branch-cache-key }}-${{ github.run_id }}
+          restore-keys: |
+            ${{ needs.build-jdk17.outputs.m2-monthly-branch-cache-key }}-
+            ${{ needs.build-jdk17.outputs.m2-monthly-cache-key }}-
       - name: Download .m2/repository/io/quarkus
         uses: actions/download-artifact@v4
         with:

diff --git a/.github/workflows/deploy-snapshots.yml b/.github/workflows/deploy-snapshots.yml
@@ -25,18 +25,23 @@ jobs:
         with:
           distribution: temurin
           java-version: 17
-      - name: Get Date
-        id: get-date
+      - name: Generate cache key
+        id: cache-key
         run: |
-          echo "date=$(/bin/date -u "+%Y-%m")" >> $GITHUB_OUTPUT
-        shell: bash
-      - name: Cache Maven Repository
-        id: cache-maven
-        uses: actions/cache@v4
+          CURRENT_BRANCH="${{ github.base_ref || github.ref_name }}"
+          CURRENT_MONTH=$(/bin/date -u "+%Y-%m")
+          ROOT_CACHE_KEY="m2-cache"
+          echo "m2-monthly-cache-key=${ROOT_CACHE_KEY}-${CURRENT_MONTH}" >> $GITHUB_OUTPUT
+          echo "m2-monthly-branch-cache-key=${ROOT_CACHE_KEY}-${CURRENT_MONTH}-${CURRENT_BRANCH}" >> $GITHUB_OUTPUT
+      - name: Restore Maven Repository
+        uses: actions/cache/restore@v4
         with:
           path: ~/.m2/repository
-          # refresh cache every month to avoid unlimited growth
-          key: deploy-snapshots-${{ steps.get-date.outputs.date }}
+          # This will always fail but we will default to the best matching restore keys
+          key: ${{ steps.cache-key.outputs.m2-monthly-branch-cache-key }}-${{ github.run_id }}
+          restore-keys: |
+            ${{ steps.cache-key.outputs.m2-monthly-branch-cache-key }}-
+            ${{ steps.cache-key.outputs.m2-monthly-cache-key }}-
       - name: Build and Deploy
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_API_TOKEN }}

diff --git a/.github/workflows/doc-build.yml b/.github/workflows/doc-build.yml
@@ -58,17 +58,23 @@ jobs:
         with:
           distribution: temurin
           java-version: 17
-      - name: Get Date
-        id: get-date
+      - name: Generate cache key
+        id: cache-key
         run: |
-          echo "date=$(/bin/date -u "+%Y-%m")" >> $GITHUB_OUTPUT
-      - name: Cache Maven Repository
-        id: cache-maven
-        uses: actions/cache@v4
+          CURRENT_BRANCH="${{ github.base_ref || github.ref_name }}"
+          CURRENT_MONTH=$(/bin/date -u "+%Y-%m")
+          ROOT_CACHE_KEY="m2-cache"
+          echo "m2-monthly-cache-key=${ROOT_CACHE_KEY}-${CURRENT_MONTH}" >> $GITHUB_OUTPUT
+          echo "m2-monthly-branch-cache-key=${ROOT_CACHE_KEY}-${CURRENT_MONTH}-${CURRENT_BRANCH}" >> $GITHUB_OUTPUT
+      - name: Restore Maven Repository
+        uses: actions/cache/restore@v4
         with:
           path: ~/.m2/repository
-          # refresh cache every month to avoid unlimited growth
-          key: q2maven-doc-${{ steps.get-date.outputs.date }}
+          # This will always fail but we will default to the best matching restore keys
+          key: ${{ steps.cache-key.outputs.m2-monthly-branch-cache-key }}-${{ github.run_id }}
+          restore-keys: |
+            ${{ steps.cache-key.outputs.m2-monthly-branch-cache-key }}-
+            ${{ steps.cache-key.outputs.m2-monthly-cache-key }}-
       - name: Build
         run: |
           ./mvnw -DquicklyDocs -B --settings .github/mvn-settings.xml

diff --git a/.github/workflows/native-it-selected-graalvm.yml b/.github/workflows/native-it-selected-graalvm.yml
@@ -77,17 +77,23 @@ jobs:
       - name: Generate cache key
         id: cache-key
         run: |
+          CURRENT_BRANCH="${{ github.base_ref || github.ref_name }}"
+          CURRENT_MONTH=$(/bin/date -u "+%Y-%m")
+          ROOT_CACHE_KEY="m2-cache"
+          echo "m2-monthly-cache-key=${ROOT_CACHE_KEY}-${CURRENT_MONTH}" >> $GITHUB_OUTPUT
+          echo "m2-monthly-branch-cache-key=${ROOT_CACHE_KEY}-${CURRENT_MONTH}-${CURRENT_BRANCH}" >> $GITHUB_OUTPUT
           CURRENT_WEEK=$(/bin/date -u "+%Y-%U")
-          echo "m2-cache-key=m2-cache-${CURRENT_WEEK}" >> $GITHUB_OUTPUT
           echo "quarkus-metadata-cache-key=quarkus-metadata-cache-${CURRENT_WEEK}-${{ github.ref_name }}" >> $GITHUB_OUTPUT
           echo "quarkus-metadata-cache-key-default=quarkus-metadata-cache-${CURRENT_WEEK}-${{ github.event.repository.default_branch }}" >> $GITHUB_OUTPUT
-      - name: Cache Maven Repository
-        id: cache-maven
-        uses: actions/cache@v4
+      - name: Restore Maven Repository
+        uses: actions/cache/restore@v4
         with:
           path: ~/.m2/repository
-          # refresh cache every week to avoid unlimited growth
-          key: ${{ steps.cache-key.outputs.m2-cache-key }}
+          # This will always fail but we will default to the best matching restore keys
+          key: ${{ steps.cache-key.outputs.m2-monthly-branch-cache-key }}-${{ github.run_id }}
+          restore-keys: |
+            ${{ steps.cache-key.outputs.m2-monthly-branch-cache-key }}-
+            ${{ steps.cache-key.outputs.m2-monthly-cache-key }}-
       - name: Verify native-tests.json
         run: ./.github/verify-tests-json.sh native-tests.json integration-tests/
       - name: Verify virtual-threads-tests.json

diff --git a/.github/workflows/owasp-check.yml b/.github/workflows/owasp-check.yml
@@ -26,6 +26,23 @@ jobs:
         distribution: temurin
         java-version: 17
 
+    - name: Generate cache key
+      id: cache-key
+      run: |
+        CURRENT_BRANCH="${{ github.base_ref || github.ref_name }}"
+        CURRENT_MONTH=$(/bin/date -u "+%Y-%m")
+        ROOT_CACHE_KEY="m2-cache"
+        echo "m2-monthly-cache-key=${ROOT_CACHE_KEY}-${CURRENT_MONTH}" >> $GITHUB_OUTPUT
+        echo "m2-monthly-branch-cache-key=${ROOT_CACHE_KEY}-${CURRENT_MONTH}-${CURRENT_BRANCH}" >> $GITHUB_OUTPUT
+    - name: Restore Maven Repository
+      uses: actions/cache/restore@v4
+      with:
+        path: ~/.m2/repository
+        # This will always fail but we will default to the best matching restore keys
+        key: ${{ steps.cache-key.outputs.m2-monthly-branch-cache-key }}-${{ github.run_id }}
+        restore-keys: |
+          ${{ steps.cache-key.outputs.m2-monthly-branch-cache-key }}-
+          ${{ steps.cache-key.outputs.m2-monthly-cache-key }}-
 
     - name: Build Java
       run: ./mvnw -B --settings .github/mvn-settings.xml -Dquickly-ci install

diff --git a/.github/workflows/podman-build.yml b/.github/workflows/podman-build.yml
@@ -81,13 +81,23 @@ jobs:
       - name: Set up testcontainers for podman
         run:  |
           echo ryuk.container.privileged=true > ~/.testcontainers.properties
-      - name: Cache Maven Repository
-        id: cache-maven
-        uses: actions/cache@v4
+      - name: Generate cache key
+        id: cache-key
+        run: |
+          CURRENT_BRANCH="${{ github.base_ref || github.ref_name }}"
+          CURRENT_MONTH=$(/bin/date -u "+%Y-%m")
+          ROOT_CACHE_KEY="m2-cache"
+          echo "m2-monthly-cache-key=${ROOT_CACHE_KEY}-${CURRENT_MONTH}" >> $GITHUB_OUTPUT
+          echo "m2-monthly-branch-cache-key=${ROOT_CACHE_KEY}-${CURRENT_MONTH}-${CURRENT_BRANCH}" >> $GITHUB_OUTPUT
+      - name: Restore Maven Repository
+        uses: actions/cache/restore@v4
         with:
           path: ~/.m2/repository
-          # refresh cache every month to avoid unlimited growth
-          key: q2maven-${{ steps.get-date.outputs.date }}
+          # This will always fail but we will default to the best matching restore keys
+          key: ${{ steps.cache-key.outputs.m2-monthly-branch-cache-key }}-${{ github.run_id }}
+          restore-keys: |
+            ${{ steps.cache-key.outputs.m2-monthly-branch-cache-key }}-
+            ${{ steps.cache-key.outputs.m2-monthly-cache-key }}-
       - name: Initial build
         run: |
           ./mvnw -T1C $COMMON_MAVEN_ARGS -DskipTests -DskipITs -DskipDocs -Dinvoker.skip -Dskip.gradle.tests -Djbang.skip -Dtruststore.skip -Dno-format -Dtcks -Prelocations clean install

diff --git a/.github/workflows/sonarcloud.yml b/.github/workflows/sonarcloud.yml
@@ -16,21 +16,28 @@ jobs:
         with:
           distribution: temurin
           java-version: 17
-      - name: Get Date
-        id: get-date
+      - name: Generate cache key
+        id: cache-key
         run: |
-          echo "date=$(/bin/date -u "+%Y-%m")" >> $GITHUB_OUTPUT
-        shell: bash
+          CURRENT_BRANCH="${{ github.base_ref || github.ref_name }}"
+          CURRENT_MONTH=$(/bin/date -u "+%Y-%m")
+          ROOT_CACHE_KEY="m2-cache"
+          echo "m2-monthly-cache-key=${ROOT_CACHE_KEY}-${CURRENT_MONTH}" >> $GITHUB_OUTPUT
+          echo "m2-monthly-branch-cache-key=${ROOT_CACHE_KEY}-${CURRENT_MONTH}-${CURRENT_BRANCH}" >> $GITHUB_OUTPUT
+      - name: Restore Maven Repository
+        uses: actions/cache/restore@v4
+        with:
+          path: ~/.m2/repository
+          # This will always fail but we will default to the best matching restore keys
+          key: ${{ steps.cache-key.outputs.m2-monthly-branch-cache-key }}-${{ github.run_id }}
+          restore-keys: |
+            ${{ steps.cache-key.outputs.m2-monthly-branch-cache-key }}-
+            ${{ steps.cache-key.outputs.m2-monthly-cache-key }}-
       - name: Cache SonarCloud packages
         uses: actions/cache@v4
         with:
           path: ~/.sonar/cache
           key: ${{ runner.os }}-sonar-${{ steps.get-date.outputs.date }}
-      - name: Cache Maven packages
-        uses: actions/cache@v4
-        with:
-          path: ~/.m2
-          key: ${{ runner.os }}-m2-${{ steps.get-date.outputs.date }}
       - name: Build and analyze
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}