From d26d2e5d0acb5b305d8a3a5a8136fcfd4277a328 Mon Sep 17 00:00:00 2001 From: Karsten Sperling <113487422+ksperling-apple@users.noreply.github.com> Date: Tue, 26 Mar 2024 11:12:33 +1300 Subject: [PATCH] Include the work directory path in the bootstrap cache key (to account for containerized/non-containerized builds) (#32638) * Silabs requirements are for non-Docker builds * CI: Don't run bootstrap if we restored from cache Handle caching and log upload directly within the bootstrap action, and don't perform the actual bootstrap if we have a cache hit. Include the platform name in the cache key. * Include the work directory path in the bootstrap cache key This is because the bootstrapped Pigweed environment does not like to be relocated. Also don't touch submodules in bootstrap, other than ensuring that Pigweed is available. * Include runner.arch as well We may start using macos-14 runners in the future which are arm. --- .github/actions/bootstrap-cache/action.yaml | 2 +- .github/actions/bootstrap/action.yaml | 51 ++++++++++++++++++- .../action.yaml | 9 +--- .../actions/upload-bootstrap-logs/action.yaml | 18 ------- .github/workflows/cirque.yaml | 1 + .github/workflows/fuzzing-build.yaml | 8 --- .github/workflows/release_artifacts.yaml | 8 --- scripts/setup/bootstrap.sh | 36 ++++++------- ...abs_docker.txt => requirements.silabs.txt} | 0 9 files changed, 67 insertions(+), 66 deletions(-) delete mode 100644 .github/actions/upload-bootstrap-logs/action.yaml rename scripts/setup/{requirements.silabs_docker.txt => requirements.silabs.txt} (100%) diff --git a/.github/actions/bootstrap-cache/action.yaml b/.github/actions/bootstrap-cache/action.yaml index b22ed1e0ac60bf..9a883ecf22da79 100644 --- a/.github/actions/bootstrap-cache/action.yaml +++ b/.github/actions/bootstrap-cache/action.yaml @@ -1,5 +1,5 @@ name: Bootstrap cache -description: Bootstrap cache +description: Bootstrap cache (deprecated) runs: using: "composite" steps: diff --git a/.github/actions/bootstrap/action.yaml b/.github/actions/bootstrap/action.yaml index 8f94830b8c0e51..16a2fba998fcc0 100644 --- a/.github/actions/bootstrap/action.yaml +++ b/.github/actions/bootstrap/action.yaml @@ -5,9 +5,56 @@ inputs: description: "Platform name" required: false default: none + bootstrap-log-name: + description: "Bootstrap log name" + required: false + default: bootstrap-logs-${{ github.job }} +outputs: + cache-hit: + description: "Bootstrap environment was restored from cache" + value: ${{ fromJSON(steps.bootstrap-cache.outputs.outputs).cache-hit }} # retry returns all outputs in `outputs` + runs: using: "composite" steps: - - name: Bootstrap + - name: Calculate bootstrap cache key + id: bootstrap-cache-key + shell: bash + run: | + # Calculate bootstrap cache key + # In addition to the various setup files, the work directory matters as well, + # because the bootstrapped Pigweed environment contains absolute paths. + FILES_HASH="${{ hashFiles('scripts/setup/*', 'third_party/pigweed/**') }}" + FINAL_HASH="$(echo "$PWD:$FILES_HASH" | shasum -a 256 | cut -d' ' -f1)" + echo "Calculated bootstrap cache key for '$PWD': $FINAL_HASH" + echo "hash=$FINAL_HASH" >> "$GITHUB_OUTPUT" + + - uses: Wandalen/wretry.action@v1.4.10 + name: Bootstrap from cache + id: bootstrap-cache + continue-on-error: true + with: + action: buildjet/cache@v4 + attempt_limit: 3 + attempt_delay: 2000 + with: | + key: ${{ runner.os }}-${{ runner.arch }}-${{ inputs.platform }}-${{ steps.bootstrap-cache-key.outputs.hash}} + path: | + .environment + build_overrides/pigweed_environment.gni + + - name: Run bootstrap + if: fromJSON(steps.bootstrap-cache.outputs.outputs).cache-hit != 'true' + env: + PW_NO_CIPD_CACHE_DIR: Y shell: bash - run: bash scripts/bootstrap.sh -p all,${{ inputs.platform }} + run: source scripts/bootstrap.sh -p all,${{ inputs.platform }} + + - name: Uploading bootstrap logs + uses: actions/upload-artifact@v4 + if: always() && !env.ACT && fromJSON(steps.bootstrap-cache.outputs.outputs).cache-hit != 'true' + with: + name: ${{ inputs.bootstrap-log-name }} + path: | + .environment/gn_out/.ninja_log + .environment/pigweed-venv/*.log diff --git a/.github/actions/checkout-submodules-and-bootstrap/action.yaml b/.github/actions/checkout-submodules-and-bootstrap/action.yaml index 7424ca529f74df..df3fdff1e02ce4 100644 --- a/.github/actions/checkout-submodules-and-bootstrap/action.yaml +++ b/.github/actions/checkout-submodules-and-bootstrap/action.yaml @@ -26,21 +26,14 @@ runs: with: platform: ${{ inputs.platform }} extra-parameters: ${{ inputs.extra-submodule-parameters }} - - name: Bootstrap Cache - uses: ./.github/actions/bootstrap-cache - name: Bootstrap uses: ./.github/actions/bootstrap - env: - PW_NO_CIPD_CACHE_DIR: Y with: platform: ${{ inputs.platform }} + bootstrap-log-name: ${{ inputs.bootstrap-log-name }} - name: Dump disk info after checkout submodule & Bootstrap shell: bash run: scripts/dump_diskspace_info.sh - - name: Upload Bootstrap Logs - uses: ./.github/actions/upload-bootstrap-logs - with: - bootstrap-log-name: ${{ inputs.bootstrap-log-name }} - name: Work around TSAN ASLR issues if: runner.os == 'Linux' && !env.ACT shell: bash diff --git a/.github/actions/upload-bootstrap-logs/action.yaml b/.github/actions/upload-bootstrap-logs/action.yaml deleted file mode 100644 index 522058c16ff93c..00000000000000 --- a/.github/actions/upload-bootstrap-logs/action.yaml +++ /dev/null @@ -1,18 +0,0 @@ -name: Upload bootstrap logs -description: Upload bootstrap logs -inputs: - bootstrap-log-name: - description: "Bootstrap log name" - required: false - default: bootstrap-logs-${{ github.job }} -runs: - using: "composite" - steps: - - name: Uploading bootstrap logs - uses: actions/upload-artifact@v4 - if: ${{ always() && !env.ACT }} - with: - name: ${{ inputs.bootstrap-log-name }} - path: | - .environment/gn_out/.ninja_log - .environment/pigweed-venv/*.log diff --git a/.github/workflows/cirque.yaml b/.github/workflows/cirque.yaml index 1f0b7c5d327fcf..919085ac77ee0f 100644 --- a/.github/workflows/cirque.yaml +++ b/.github/workflows/cirque.yaml @@ -57,6 +57,7 @@ jobs: with: platform: linux + # TODO: Is what's being cached here actually compatible with a regular bootstrap? - name: Bootstrap Cache uses: ./.github/actions/bootstrap-cache - name: Bootstrap Cirque diff --git a/.github/workflows/fuzzing-build.yaml b/.github/workflows/fuzzing-build.yaml index ee19873cc32f2a..f354ce369c27fd 100644 --- a/.github/workflows/fuzzing-build.yaml +++ b/.github/workflows/fuzzing-build.yaml @@ -46,12 +46,8 @@ jobs: run: | mkdir objdir-clone || true - - name: Bootstrap Cache - uses: ./.github/actions/bootstrap-cache - name: Bootstrap uses: ./.github/actions/bootstrap - - name: Upload Bootstrap Logs - uses: ./.github/actions/upload-bootstrap-logs - name: Build all-clusters-app run: | @@ -84,12 +80,8 @@ jobs: run: | mkdir objdir-clone || true - - name: Bootstrap Cache - uses: ./.github/actions/bootstrap-cache - name: Bootstrap uses: ./.github/actions/bootstrap - - name: Upload Bootstrap Logs - uses: ./.github/actions/upload-bootstrap-logs - name: Build all-clusters-app run: | diff --git a/.github/workflows/release_artifacts.yaml b/.github/workflows/release_artifacts.yaml index 5aa38ecc893d28..57a1ce7b325dc3 100644 --- a/.github/workflows/release_artifacts.yaml +++ b/.github/workflows/release_artifacts.yaml @@ -39,12 +39,8 @@ jobs: uses: actions/checkout@v4 with: ref: "${{ github.event.inputs.releaseTag }}" - - name: Bootstrap Cache - uses: ./.github/actions/bootstrap-cache - name: Bootstrap uses: ./.github/actions/bootstrap - - name: Upload Bootstrap Logs - uses: ./.github/actions/upload-bootstrap-logs - name: Build run: scripts/examples/esp_example.sh all-clusters-app @@ -74,12 +70,8 @@ jobs: uses: actions/checkout@v4 with: ref: "${{ github.event.inputs.releaseTag }}" - - name: Bootstrap Cache - uses: ./.github/actions/bootstrap-cache - name: Bootstrap uses: ./.github/actions/bootstrap - - name: Upload Bootstrap Logs - uses: ./.github/actions/upload-bootstrap-logs - name: Build example EFR32 Lock App run: scripts/examples/gn_silabs_example.sh examples/lock-app/efr32/ diff --git a/scripts/setup/bootstrap.sh b/scripts/setup/bootstrap.sh index 1b813216222efe..0a2a898eaf02c5 100644 --- a/scripts/setup/bootstrap.sh +++ b/scripts/setup/bootstrap.sh @@ -21,14 +21,14 @@ _install_additional_pip_requirements() { # figure out additional pip install items while [ $# -gt 0 ]; do case $1 in - -p | --platform) - _SETUP_PLATFORM=$2 - shift # argument - shift # value - ;; - *) - shift - ;; + -p | --platform) + _SETUP_PLATFORM=$2 + shift # argument + shift # value + ;; + *) + shift + ;; esac done @@ -41,7 +41,7 @@ _install_additional_pip_requirements() { for platform in ${_SETUP_PLATFORM}; do # Allow none as an alias of nothing extra installed (like -p none) - if [ "$platform" != "none" ]; then + if [ "$platform" != "none" -a -e "$_CHIP_ROOT/scripts/setup/requirements.$platform.txt" ]; then echo "Installing pip requirements for $platform..." pip install -q \ -r "$_CHIP_ROOT/scripts/setup/requirements.$platform.txt" \ @@ -66,7 +66,7 @@ _bootstrap_or_activate() { local _BOOTSTRAP_NAME="${_BOOTSTRAP_PATH##*/}" local _BOOTSTRAP_DIR="${_BOOTSTRAP_PATH%/*}" # Strip off the 'scripts[/setup]' directory, leaving the root of the repo. - _CHIP_ROOT="$(cd "${_BOOTSTRAP_DIR%/setup}/.." > /dev/null && pwd)" + _CHIP_ROOT="$(cd "${_BOOTSTRAP_DIR%/setup}/.." >/dev/null && pwd)" local _CONFIG_FILE="scripts/setup/environment.json" @@ -75,16 +75,6 @@ _bootstrap_or_activate() { unset PW_CONFIG_FILE fi - if [ ! -f "$_CHIP_ROOT/third_party/pigweed/repo/pw_env_setup/util.sh" ]; then - # Make sure our submodule remotes are correct for this revision. - git submodule sync --recursive - git submodule update --init - elif [ "$_BOOTSTRAP_NAME" = "bootstrap.sh" ]; then - # In this case, only update already checked out submodules. - git submodule sync --recursive - git submodule update - fi - PW_BRANDING_BANNER="$_CHIP_ROOT/scripts/setup/banner.txt" export PW_BRANDING_BANNER @@ -94,6 +84,10 @@ _bootstrap_or_activate() { PW_ROOT="$_CHIP_ROOT/third_party/pigweed/repo" export PW_ROOT + # Update or init the pigweed submodule if necessary. Don't touch any other submodules. + if [ "$_BOOTSTRAP_NAME" = "bootstrap.sh" -a ! -f "$_CHIP_ROOT/third_party/pigweed/repo/pw_env_setup/util.sh" ]; then + git submodule update --init "$_CHIP_ROOT/third_party/pigweed/repo" + fi . "$_CHIP_ROOT/third_party/pigweed/repo/pw_env_setup/util.sh" _chip_bootstrap_banner() { @@ -105,7 +99,7 @@ _bootstrap_or_activate() { local _PW_BANNER_FUNC="_chip_bootstrap_banner" - # Force the Pigweed environment directory to be '.environment' + # Default the Pigweed environment directory to be '.environment' if [ -z "$PW_ENVIRONMENT_ROOT" ]; then export PW_ENVIRONMENT_ROOT="$PW_PROJECT_ROOT/.environment" fi diff --git a/scripts/setup/requirements.silabs_docker.txt b/scripts/setup/requirements.silabs.txt similarity index 100% rename from scripts/setup/requirements.silabs_docker.txt rename to scripts/setup/requirements.silabs.txt