Include the work directory path in the bootstrap cache key (to accoun…

…t for containerized/non-containerized builds) (project-chip#32638)

* Silabs requirements are for non-Docker builds

* CI: Don't run bootstrap if we restored from cache

Handle caching and log upload directly within the bootstrap action, and don't
perform the actual bootstrap if we have a cache hit. Include the platform name
in the cache key.

* Include the work directory path in the bootstrap cache key

This is because the bootstrapped Pigweed environment does not like to be
relocated. Also don't touch submodules in bootstrap, other than ensuring that
Pigweed is available.

* Include runner.arch as well

We may start using macos-14 runners in the future which are arm.

.github/actions/bootstrap-cache/action.yaml

@@ -1,5 +1,5 @@
 name: Bootstrap cache
-description: Bootstrap cache
+description: Bootstrap cache (deprecated)
 runs:
   using: "composite"
   steps:

.github/actions/bootstrap/action.yaml

@@ -5,9 +5,56 @@ inputs:
     description: "Platform name"
     required: false
     default: none
+  bootstrap-log-name:
+    description: "Bootstrap log name"
+    required: false
+    default: bootstrap-logs-${{ github.job }}
+outputs:
+  cache-hit:
+    description: "Bootstrap environment was restored from cache"
+    value: ${{ fromJSON(steps.bootstrap-cache.outputs.outputs).cache-hit }} # retry returns all outputs in `outputs`
+
 runs:
   using: "composite"
   steps:
-    - name: Bootstrap
+    - name: Calculate bootstrap cache key
+      id: bootstrap-cache-key
+      shell: bash
+      run: |
+        # Calculate bootstrap cache key
+        # In addition to the various setup files, the work directory matters as well,
+        # because the bootstrapped Pigweed environment contains absolute paths.
+        FILES_HASH="${{ hashFiles('scripts/setup/*', 'third_party/pigweed/**') }}"
+        FINAL_HASH="$(echo "$PWD:$FILES_HASH" | shasum -a 256 | cut -d' ' -f1)"
+        echo "Calculated bootstrap cache key for '$PWD': $FINAL_HASH"
+        echo "hash=$FINAL_HASH" >> "$GITHUB_OUTPUT"
+
+    - uses: Wandalen/[email protected]
+      name: Bootstrap from cache
+      id: bootstrap-cache
+      continue-on-error: true
+      with:
+        action: buildjet/cache@v4
+        attempt_limit: 3
+        attempt_delay: 2000
+        with: |
+          key: ${{ runner.os }}-${{ runner.arch }}-${{ inputs.platform }}-${{ steps.bootstrap-cache-key.outputs.hash}}
+          path: |
+              .environment
+              build_overrides/pigweed_environment.gni
+
+    - name: Run bootstrap
+      if: fromJSON(steps.bootstrap-cache.outputs.outputs).cache-hit != 'true'
+      env:
+        PW_NO_CIPD_CACHE_DIR: Y
       shell: bash
-      run: bash scripts/bootstrap.sh -p all,${{ inputs.platform }}
+      run: source scripts/bootstrap.sh -p all,${{ inputs.platform }}
+
+    - name: Uploading bootstrap logs
+      uses: actions/upload-artifact@v4
+      if: always() && !env.ACT && fromJSON(steps.bootstrap-cache.outputs.outputs).cache-hit != 'true'
+      with:
+        name: ${{ inputs.bootstrap-log-name }}
+        path: |
+          .environment/gn_out/.ninja_log
+          .environment/pigweed-venv/*.log

.github/actions/checkout-submodules-and-bootstrap/action.yaml

@@ -26,21 +26,14 @@ runs:
       with:
         platform: ${{ inputs.platform }}
         extra-parameters: ${{ inputs.extra-submodule-parameters }}
-    - name: Bootstrap Cache
-      uses: ./.github/actions/bootstrap-cache
     - name: Bootstrap
       uses: ./.github/actions/bootstrap
-      env:
-        PW_NO_CIPD_CACHE_DIR: Y
       with:
         platform: ${{ inputs.platform }}
+        bootstrap-log-name: ${{ inputs.bootstrap-log-name }}
     - name: Dump disk info after checkout submodule & Bootstrap
       shell: bash
       run: scripts/dump_diskspace_info.sh
-    - name: Upload Bootstrap Logs
-      uses: ./.github/actions/upload-bootstrap-logs
-      with:
-        bootstrap-log-name: ${{ inputs.bootstrap-log-name }}
     - name: Work around TSAN ASLR issues
       if: runner.os == 'Linux' && !env.ACT
       shell: bash

.github/workflows/cirque.yaml

@@ -57,6 +57,7 @@ jobs:
               with:
                 platform: linux
 
+            # TODO: Is what's being cached here actually compatible with a regular bootstrap?
             - name: Bootstrap Cache
               uses: ./.github/actions/bootstrap-cache
             - name: Bootstrap Cirque

.github/workflows/fuzzing-build.yaml

@@ -46,12 +46,8 @@ jobs:
               run: |
                   mkdir objdir-clone || true
 
-            - name: Bootstrap Cache
-              uses: ./.github/actions/bootstrap-cache
             - name: Bootstrap
               uses: ./.github/actions/bootstrap
-            - name: Upload Bootstrap Logs
-              uses: ./.github/actions/upload-bootstrap-logs
 
             - name: Build all-clusters-app
               run: |
@@ -84,12 +80,8 @@ jobs:
               run: |
                   mkdir objdir-clone || true
 
-            - name: Bootstrap Cache
-              uses: ./.github/actions/bootstrap-cache
             - name: Bootstrap
               uses: ./.github/actions/bootstrap
-            - name: Upload Bootstrap Logs
-              uses: ./.github/actions/upload-bootstrap-logs
 
             - name: Build all-clusters-app
               run: |

.github/workflows/release_artifacts.yaml

@@ -39,12 +39,8 @@ jobs:
               uses: actions/checkout@v4
               with:
                 ref: "${{ github.event.inputs.releaseTag }}"
-            - name: Bootstrap Cache
-              uses: ./.github/actions/bootstrap-cache
             - name: Bootstrap
               uses: ./.github/actions/bootstrap
-            - name: Upload Bootstrap Logs
-              uses: ./.github/actions/upload-bootstrap-logs
 
             - name: Build
               run: scripts/examples/esp_example.sh all-clusters-app
@@ -74,12 +70,8 @@ jobs:
               uses: actions/checkout@v4
               with:
                 ref: "${{ github.event.inputs.releaseTag }}"
-            - name: Bootstrap Cache
-              uses: ./.github/actions/bootstrap-cache
             - name: Bootstrap
               uses: ./.github/actions/bootstrap
-            - name: Upload Bootstrap Logs
-              uses: ./.github/actions/upload-bootstrap-logs
 
             - name: Build example EFR32 Lock App
               run: scripts/examples/gn_silabs_example.sh examples/lock-app/efr32/

scripts/setup/bootstrap.sh

@@ -21,14 +21,14 @@ _install_additional_pip_requirements() {
     # figure out additional pip install items
     while [ $# -gt 0 ]; do
         case $1 in
-            -p | --platform)
-                _SETUP_PLATFORM=$2
-                shift # argument
-                shift # value
-                ;;
-            *)
-                shift
-                ;;
+        -p | --platform)
+            _SETUP_PLATFORM=$2
+            shift # argument
+            shift # value
+            ;;
+        *)
+            shift
+            ;;
         esac
     done
 
@@ -41,7 +41,7 @@ _install_additional_pip_requirements() {
 
         for platform in ${_SETUP_PLATFORM}; do
             # Allow none as an alias of nothing extra installed (like -p none)
-            if [ "$platform" != "none" ]; then
+            if [ "$platform" != "none" -a -e "$_CHIP_ROOT/scripts/setup/requirements.$platform.txt" ]; then
                 echo "Installing pip requirements for $platform..."
                 pip install -q \
                     -r "$_CHIP_ROOT/scripts/setup/requirements.$platform.txt" \
@@ -66,7 +66,7 @@ _bootstrap_or_activate() {
     local _BOOTSTRAP_NAME="${_BOOTSTRAP_PATH##*/}"
     local _BOOTSTRAP_DIR="${_BOOTSTRAP_PATH%/*}"
     # Strip off the 'scripts[/setup]' directory, leaving the root of the repo.
-    _CHIP_ROOT="$(cd "${_BOOTSTRAP_DIR%/setup}/.." > /dev/null && pwd)"
+    _CHIP_ROOT="$(cd "${_BOOTSTRAP_DIR%/setup}/.." >/dev/null && pwd)"
 
     local _CONFIG_FILE="scripts/setup/environment.json"
 
@@ -75,16 +75,6 @@ _bootstrap_or_activate() {
         unset PW_CONFIG_FILE
     fi
 
-    if [ ! -f "$_CHIP_ROOT/third_party/pigweed/repo/pw_env_setup/util.sh" ]; then
-        # Make sure our submodule remotes are correct for this revision.
-        git submodule sync --recursive
-        git submodule update --init
-    elif [ "$_BOOTSTRAP_NAME" = "bootstrap.sh" ]; then
-        # In this case, only update already checked out submodules.
-        git submodule sync --recursive
-        git submodule update
-    fi
-
     PW_BRANDING_BANNER="$_CHIP_ROOT/scripts/setup/banner.txt"
     export PW_BRANDING_BANNER
 
@@ -94,6 +84,10 @@ _bootstrap_or_activate() {
     PW_ROOT="$_CHIP_ROOT/third_party/pigweed/repo"
     export PW_ROOT
 
+    # Update or init the pigweed submodule if necessary. Don't touch any other submodules.
+    if [ "$_BOOTSTRAP_NAME" = "bootstrap.sh" -a ! -f "$_CHIP_ROOT/third_party/pigweed/repo/pw_env_setup/util.sh" ]; then
+        git submodule update --init "$_CHIP_ROOT/third_party/pigweed/repo"
+    fi
     . "$_CHIP_ROOT/third_party/pigweed/repo/pw_env_setup/util.sh"
 
     _chip_bootstrap_banner() {
@@ -105,7 +99,7 @@ _bootstrap_or_activate() {
 
     local _PW_BANNER_FUNC="_chip_bootstrap_banner"
 
-    # Force the Pigweed environment directory to be '.environment'
+    # Default the Pigweed environment directory to be '.environment'
     if [ -z "$PW_ENVIRONMENT_ROOT" ]; then
         export PW_ENVIRONMENT_ROOT="$PW_PROJECT_ROOT/.environment"
     fi