Cache bootstrap step in all CI workflows (#24968)

* Move bootstrap configuration files to separate dir Keeping all files used to bootstrap the project in a dedicated directory will simplify environment cache invalidation. * Compatibility for old location of bootstrap script * Cache bootstrap step in all CI workflows * Update constraints location in config files * Hash all pigweed files when creating cache key
project-chip · Feb 14, 2023 · 3d9bc88 · 3d9bc88
1 parent e7528bc
commit 3d9bc88
Show file tree

Hide file tree

Showing 70 changed files with 522 additions and 157 deletions.
diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml
@@ -66,6 +66,14 @@ jobs:
               run: |
                   mkdir /tmp/cores || true
                   sysctl -w kernel.core_pattern=/tmp/cores/core.%u.%p.%t || true
+
+            - name: Bootstrap cache
+              uses: actions/cache@v3
+              with:
+                  key: ${{ runner.os }}-env-${{ hashFiles('scripts/setup/*', 'third_party/pigweed/**') }}
+                  path: |
+                      .environment
+                      build_overrides/pigweed_environment.gni
             - name: Bootstrap
               timeout-minutes: 10
               run: scripts/build/gn_bootstrap.sh
@@ -77,6 +85,7 @@ jobs:
                   path: |
                       .environment/gn_out/.ninja_log
                       .environment/pigweed-venv/*.log
+
             - name: Setup Build
               run: scripts/build/gn_gen.sh --args="chip_config_memory_debug_checks=true chip_config_memory_debug_dmalloc=false"
             - name: Run Build
@@ -173,6 +182,14 @@ jobs:
               run: |
                   mkdir /tmp/cores || true
                   sysctl -w kernel.core_pattern=/tmp/cores/core.%u.%p.%t || true
+
+            - name: Bootstrap cache
+              uses: actions/cache@v3
+              with:
+                  key: ${{ runner.os }}-env-${{ hashFiles('scripts/setup/*', 'third_party/pigweed/**') }}
+                  path: |
+                      .environment
+                      build_overrides/pigweed_environment.gni
             - name: Bootstrap
               timeout-minutes: 10
               run: scripts/build/gn_bootstrap.sh
@@ -184,6 +201,7 @@ jobs:
                   path: |
                       .environment/gn_out/.ninja_log
                       .environment/pigweed-venv/*.log
+
             - name: Setup and Build Simulated Device
               timeout-minutes: 20
               run: |
@@ -323,9 +341,18 @@ jobs:
                   attempt_delay: 2000
             - name: Checkout submodules
               run: scripts/checkout_submodules.py --shallow --platform linux
+
+            - name: Bootstrap cache
+              uses: actions/cache@v3
+              with:
+                  key: ${{ runner.os }}-env-${{ hashFiles('scripts/setup/*', 'third_party/pigweed/**') }}
+                  path: |
+                      .environment
+                      build_overrides/pigweed_environment.gni
             - name: Bootstrap
               timeout-minutes: 10
               run: scripts/build/gn_bootstrap.sh
+
             - name: Setup Build, Run Build and Run Tests
               timeout-minutes: 50
               run: |
@@ -372,6 +399,14 @@ jobs:
             - name: Try to ensure the directory for diagnostic log collection exists
               run: |
                   mkdir -p ~/Library/Logs/DiagnosticReports || true
+
+            - name: Bootstrap cache
+              uses: actions/cache@v3
+              with:
+                  key: ${{ runner.os }}-env-${{ hashFiles('scripts/setup/*', 'third_party/pigweed/**') }}
+                  path: |
+                      .environment
+                      build_overrides/pigweed_environment.gni
             - name: Bootstrap
               timeout-minutes: 25
               run: scripts/build/gn_bootstrap.sh
@@ -383,6 +418,7 @@ jobs:
                   path: |
                       .environment/gn_out/.ninja_log
                       .environment/pigweed-venv/*.log
+
             - name: Setup and Build Simulated Device
               timeout-minutes: 20
               run: |
@@ -462,6 +498,14 @@ jobs:
                   attempt_delay: 2000
             - name: Checkout submodules
               run: scripts/checkout_submodules.py --shallow --platform linux
+
+            - name: Bootstrap cache
+              uses: actions/cache@v3
+              with:
+                  key: ${{ runner.os }}-env-${{ hashFiles('scripts/setup/*', 'third_party/pigweed/**') }}
+                  path: |
+                      .environment
+                      build_overrides/pigweed_environment.gni
             - name: Bootstrap
               timeout-minutes: 10
               run: scripts/build/gn_bootstrap.sh
@@ -472,8 +516,8 @@ jobs:
                   name: bootstrap-logs
                   path: |
                       .environment/gn_out/.ninja_log
-                      .environment/pigweed-venv/*.log           
+                      .environment/pigweed-venv/*.log
+
             - name: Run Build Coverage
               timeout-minutes: 20
               run: ./scripts/build_coverage.sh
-
diff --git a/.github/workflows/chef.yaml b/.github/workflows/chef.yaml
@@ -43,6 +43,13 @@ jobs:
                   attempt_delay: 2000
             - name: Checkout submodules
               run: scripts/checkout_submodules.py --shallow --platform linux
+            - name: Bootstrap cache
+              uses: actions/cache@v3
+              with:
+                  key: ${{ runner.os }}-env-${{ hashFiles('scripts/setup/*', 'third_party/pigweed/**') }}
+                  path: |
+                      .environment
+                      build_overrides/pigweed_environment.gni
             - name: Bootstrap
               timeout-minutes: 10
               run: scripts/build/gn_bootstrap.sh
@@ -71,6 +78,13 @@ jobs:
                   attempt_delay: 2000
             - name: Checkout submodules
               run: scripts/checkout_submodules.py --shallow --platform esp32
+            - name: Bootstrap cache
+              uses: actions/cache@v3
+              with:
+                  key: ${{ runner.os }}-env-${{ hashFiles('scripts/setup/*', 'third_party/pigweed/**') }}
+                  path: |
+                      .environment
+                      build_overrides/pigweed_environment.gni
             - name: Bootstrap
               timeout-minutes: 10
               run: scripts/build/gn_bootstrap.sh
@@ -99,6 +113,13 @@ jobs:
                   attempt_delay: 2000
             - name: Checkout submodules
               run: scripts/checkout_submodules.py --shallow --platform nrfconnect
+            - name: Bootstrap cache
+              uses: actions/cache@v3
+              with:
+                  key: ${{ runner.os }}-env-${{ hashFiles('scripts/setup/*', 'third_party/pigweed/**') }}
+                  path: |
+                      .environment
+                      build_overrides/pigweed_environment.gni
             - name: Bootstrap
               timeout-minutes: 10
               run: scripts/build/gn_bootstrap.sh

diff --git a/.github/workflows/darwin-tests.yaml b/.github/workflows/darwin-tests.yaml
@@ -64,6 +64,14 @@ jobs:
                   sudo chown ${USER} /cores || true
                   mkdir -p ~/Library/Logs/DiagnosticReports || true
                   mkdir objdir-clone || true
+
+            - name: Bootstrap cache
+              uses: actions/cache@v3
+              with:
+                  key: ${{ runner.os }}-env-${{ hashFiles('scripts/setup/*', 'third_party/pigweed/**') }}
+                  path: |
+                      .environment
+                      build_overrides/pigweed_environment.gni
             - name: Bootstrap
               timeout-minutes: 25
               run: scripts/build/gn_bootstrap.sh
@@ -75,6 +83,7 @@ jobs:
                   path: |
                       .environment/gn_out/.ninja_log
                       .environment/pigweed-venv/*.log
+
             - name: Delete Defaults
               run: defaults delete com.apple.dt.xctest.tool
               continue-on-error: true

diff --git a/.github/workflows/darwin.yaml b/.github/workflows/darwin.yaml
@@ -44,6 +44,14 @@ jobs:
               run: scripts/checkout_submodules.py --shallow --platform darwin
             - name: Setup Environment
               run: brew install [email protected]
+
+            - name: Bootstrap cache
+              uses: actions/cache@v3
+              with:
+                  key: ${{ runner.os }}-env-${{ hashFiles('scripts/setup/*', 'third_party/pigweed/**') }}
+                  path: |
+                      .environment
+                      build_overrides/pigweed_environment.gni
             - name: Bootstrap
               timeout-minutes: 25
               run: scripts/build/gn_bootstrap.sh
@@ -55,6 +63,7 @@ jobs:
                   path: |
                       .environment/gn_out/.ninja_log
                       .environment/pigweed-venv/*.log
+
             - name: Block zap-cli from being used
               # xcodebuild is NOT expected to require zap-cli
               run: scripts/run_in_build_env.sh 'mv $PW_ENVIRONMENT_ROOT/cipd/packages/zap/zap-cli $PW_ENVIRONMENT_ROOT/cipd/packages/zap/zap-cli.moved'

diff --git a/.github/workflows/docbuild.yaml b/.github/workflows/docbuild.yaml
@@ -27,14 +27,11 @@ jobs:
           path: matter
           fetch-depth: 0
       - name: Install Python
-        uses: actions/setup-python@v2
+        uses: actions/setup-python@v4
         with:
           python-version: 3.8
-      - name: cache-pip
-        uses: actions/cache@v1
-        with:
-          path: ~/.cache/pip
-          key: ${{ runner.os }}-doc-pip
+          cache-dependency-path: matter/docs/requirements.txt
+          cache: pip
       - name: Install base dependencies
         working-directory: matter
         run: |

diff --git a/.github/workflows/examples-ameba.yaml b/.github/workflows/examples-ameba.yaml
@@ -49,6 +49,15 @@ jobs:
                   attempt_delay: 2000
             - name: Checkout submodules
               run: scripts/checkout_submodules.py --shallow --platform ameba
+            - name: Bootstrap cache
+              uses: actions/cache@v3
+              with:
+                  key: ${{ runner.os }}-env-${{ hashFiles('scripts/setup/*', 'third_party/pigweed/**') }}
+                  path: |
+                      .environment
+                      build_overrides/pigweed_environment.gni
+            - name: Bootstrap
+              run: scripts/build/gn_bootstrap.sh
             - name: Build example Ameba All Clusters App
               run: |
                   ./scripts/run_in_build_env.sh \

diff --git a/.github/workflows/examples-bouffalolab.yaml b/.github/workflows/examples-bouffalolab.yaml
@@ -55,7 +55,14 @@ jobs:
         env:
           GH_CONTEXT: ${{ toJson(github) }}
         run: scripts/tools/memory/gh_sizes_environment.py "${GH_CONTEXT}"
-
+
+      - name: Bootstrap cache
+        uses: actions/cache@v3
+        with:
+            key: ${{ runner.os }}-env-${{ hashFiles('scripts/setup/*', 'third_party/pigweed/**') }}
+            path: |
+                .environment
+                build_overrides/pigweed_environment.gni
       - name: Bootstrap
         timeout-minutes: 25
         run: scripts/build/gn_bootstrap.sh
@@ -67,6 +74,7 @@ jobs:
           path: |
             .environment/gn_out/.ninja_log
             .environment/pigweed-venv/*.log
+
       - name: Build example BL602 Lighting App
         timeout-minutes: 30
         run: |

diff --git a/.github/workflows/examples-cc13x2x7_26x2x7.yaml b/.github/workflows/examples-cc13x2x7_26x2x7.yaml
@@ -56,6 +56,14 @@ jobs:
               env:
                   GH_CONTEXT: ${{ toJson(github) }}
               run: scripts/tools/memory/gh_sizes_environment.py "${GH_CONTEXT}"
+
+            - name: Bootstrap cache
+              uses: actions/cache@v3
+              with:
+                  key: ${{ runner.os }}-env-${{ hashFiles('scripts/setup/*', 'third_party/pigweed/**') }}
+                  path: |
+                      .environment
+                      build_overrides/pigweed_environment.gni
             - name: Bootstrap
               timeout-minutes: 25
               run: scripts/build/gn_bootstrap.sh
@@ -67,6 +75,7 @@ jobs:
                   path: |
                       .environment/gn_out/.ninja_log
                       .environment/pigweed-venv/*.log
+
             - name: Build examples
               timeout-minutes: 100
               run: |

diff --git a/.github/workflows/examples-cc32xx.yaml b/.github/workflows/examples-cc32xx.yaml
@@ -53,6 +53,14 @@ jobs:
               env:
                   GH_CONTEXT: ${{ toJson(github) }}
               run: scripts/tools/memory/gh_sizes_environment.py "${GH_CONTEXT}"
+
+            - name: Bootstrap cache
+              uses: actions/cache@v3
+              with:
+                  key: ${{ runner.os }}-env-${{ hashFiles('scripts/setup/*', 'third_party/pigweed/**') }}
+                  path: |
+                      .environment
+                      build_overrides/pigweed_environment.gni
             - name: Bootstrap
               timeout-minutes: 25
               run: scripts/build/gn_bootstrap.sh
@@ -64,6 +72,7 @@ jobs:
                   path: |
                       .environment/gn_out/.ninja_log
                       .environment/pigweed-venv/*.log
+
             - name: Build examples
               timeout-minutes: 60
               run: |
@@ -79,7 +88,7 @@ jobs:
                     cc32xx CC3235SF_LAUNCHXL lock \
                     out/artifacts/cc32xx-lock/chip-CC3235SF_LAUNCHXL-lock-example.out \
                     /tmp/bloat_reports/
-            
+
             - name: Uploading Size Reports
               uses: actions/upload-artifact@v2
               if: ${{ !env.ACT }}

diff --git a/.github/workflows/examples-efr32.yaml b/.github/workflows/examples-efr32.yaml
@@ -62,6 +62,13 @@ jobs:
           GH_CONTEXT: ${{ toJson(github) }}
         run: scripts/tools/memory/gh_sizes_environment.py "${GH_CONTEXT}"
 
+      - name: Bootstrap cache
+        uses: actions/cache@v3
+        with:
+            key: ${{ runner.os }}-env-${{ hashFiles('scripts/setup/*', 'third_party/pigweed/**') }}
+            path: |
+                .environment
+                build_overrides/pigweed_environment.gni
       - name: Bootstrap
         timeout-minutes: 25
         run: scripts/build/gn_bootstrap.sh

diff --git a/.github/workflows/examples-esp32.yaml b/.github/workflows/examples-esp32.yaml
@@ -56,6 +56,13 @@ jobs:
                   GH_CONTEXT: ${{ toJson(github) }}
               run: scripts/tools/memory/gh_sizes_environment.py "${GH_CONTEXT}"
 
+            - name: Bootstrap cache
+              uses: actions/cache@v3
+              with:
+                  key: ${{ runner.os }}-env-${{ hashFiles('scripts/setup/*', 'third_party/pigweed/**') }}
+                  path: |
+                      .environment
+                      build_overrides/pigweed_environment.gni
             - name: Bootstrap
               timeout-minutes: 10
               run: scripts/build/gn_bootstrap.sh
@@ -67,6 +74,7 @@ jobs:
                   path: |
                       .environment/gn_out/.ninja_log
                       .environment/pigweed-venv/*.log
+
             - name: Build some M5Stack variations
               timeout-minutes: 60
               run: |
@@ -164,10 +172,16 @@ jobs:
             - name: Checkout submodules
               run: scripts/checkout_submodules.py --shallow --platform esp32
 
+            - name: Bootstrap cache
+              uses: actions/cache@v3
+              with:
+                  key: ${{ runner.os }}-env-${{ hashFiles('scripts/setup/*', 'third_party/pigweed/**') }}
+                  path: |
+                      .environment
+                      build_overrides/pigweed_environment.gni
             - name: Bootstrap
               timeout-minutes: 10
               run: scripts/build/gn_bootstrap.sh
-
             - name: Uploading bootstrap logs
               uses: actions/upload-artifact@v3
               if: ${{ always() && !env.ACT }}

diff --git a/.github/workflows/examples-infineon.yaml b/.github/workflows/examples-infineon.yaml
@@ -56,6 +56,13 @@ jobs:
                   GH_CONTEXT: ${{ toJson(github) }}
               run: scripts/tools/memory/gh_sizes_environment.py "${GH_CONTEXT}"
 
+            - name: Bootstrap cache
+              uses: actions/cache@v3
+              with:
+                  key: ${{ runner.os }}-env-${{ hashFiles('scripts/setup/*', 'third_party/pigweed/**') }}
+                  path: |
+                      .environment
+                      build_overrides/pigweed_environment.gni
             - name: Bootstrap
               timeout-minutes: 25
               run: scripts/build/gn_bootstrap.sh
@@ -67,6 +74,7 @@ jobs:
                   path: |
                       .environment/gn_out/.ninja_log
                       .environment/pigweed-venv/*.log
+
             - name: Build PSoC6 lock-app example
               timeout-minutes: 15
               run: |