From 2261d8d1aadf36971f8449b00564f1642ff446ee Mon Sep 17 00:00:00 2001 From: Karsten Sperling Date: Mon, 18 Mar 2024 19:55:38 +1300 Subject: [PATCH 1/4] Silabs requirements are for non-Docker builds --- .../{requirements.silabs_docker.txt => requirements.silabs.txt} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename scripts/setup/{requirements.silabs_docker.txt => requirements.silabs.txt} (100%) diff --git a/scripts/setup/requirements.silabs_docker.txt b/scripts/setup/requirements.silabs.txt similarity index 100% rename from scripts/setup/requirements.silabs_docker.txt rename to scripts/setup/requirements.silabs.txt From 89802bcb802998238a93a2ebaf05df75ecb20cc0 Mon Sep 17 00:00:00 2001 From: Karsten Sperling Date: Mon, 18 Mar 2024 14:32:56 +1300 Subject: [PATCH 2/4] CI: Don't run bootstrap if we restored from cache Handle caching and log upload directly within the bootstrap action, and don't perform the actual bootstrap if we have a cache hit. Include the platform name in the cache key. --- .github/actions/bootstrap-cache/action.yaml | 2 +- .github/actions/bootstrap/action.yaml | 35 +++++++++++++++++-- .../action.yaml | 9 +---- .../actions/upload-bootstrap-logs/action.yaml | 18 ---------- .github/workflows/cirque.yaml | 1 + .github/workflows/fuzzing-build.yaml | 8 ----- .github/workflows/release_artifacts.yaml | 8 ----- scripts/setup/bootstrap.sh | 20 +++++------ 8 files changed, 46 insertions(+), 55 deletions(-) delete mode 100644 .github/actions/upload-bootstrap-logs/action.yaml diff --git a/.github/actions/bootstrap-cache/action.yaml b/.github/actions/bootstrap-cache/action.yaml index b22ed1e0ac60bf..9a883ecf22da79 100644 --- a/.github/actions/bootstrap-cache/action.yaml +++ b/.github/actions/bootstrap-cache/action.yaml @@ -1,5 +1,5 @@ name: Bootstrap cache -description: Bootstrap cache +description: Bootstrap cache (deprecated) runs: using: "composite" steps: diff --git a/.github/actions/bootstrap/action.yaml b/.github/actions/bootstrap/action.yaml index 8f94830b8c0e51..32fe6c9cfedba3 100644 --- a/.github/actions/bootstrap/action.yaml +++ b/.github/actions/bootstrap/action.yaml @@ -5,9 +5,40 @@ inputs: description: "Platform name" required: false default: none + bootstrap-log-name: + description: "Bootstrap log name" + required: false + default: bootstrap-logs-${{ github.job }} + runs: using: "composite" steps: - - name: Bootstrap + - uses: Wandalen/wretry.action@v1.4.10 + name: Bootstrap from cache + id: bootstrap-cache + continue-on-error: true + with: + action: buildjet/cache@v4 + attempt_limit: 3 + attempt_delay: 2000 + with: | + key: ${{ runner.os }}-${{ inputs.platform }}-env-${{ hashFiles('scripts/setup/*', 'third_party/pigweed/**') }} + path: | + .environment + build_overrides/pigweed_environment.gni + + - name: Run bootstrap + if: fromJSON(steps.bootstrap-cache.outputs.outputs).cache-hit != 'true' # retry returns all outputs in `outputs` + env: + PW_NO_CIPD_CACHE_DIR: Y shell: bash - run: bash scripts/bootstrap.sh -p all,${{ inputs.platform }} + run: source scripts/bootstrap.sh -p all,${{ inputs.platform }} + + - name: Uploading bootstrap logs + uses: actions/upload-artifact@v3 + if: always() && !env.ACT && fromJSON(steps.bootstrap-cache.outputs.outputs).cache-hit != 'true' + with: + name: ${{ inputs.bootstrap-log-name }} + path: | + .environment/gn_out/.ninja_log + .environment/pigweed-venv/*.log diff --git a/.github/actions/checkout-submodules-and-bootstrap/action.yaml b/.github/actions/checkout-submodules-and-bootstrap/action.yaml index 7424ca529f74df..df3fdff1e02ce4 100644 --- a/.github/actions/checkout-submodules-and-bootstrap/action.yaml +++ b/.github/actions/checkout-submodules-and-bootstrap/action.yaml @@ -26,21 +26,14 @@ runs: with: platform: ${{ inputs.platform }} extra-parameters: ${{ inputs.extra-submodule-parameters }} - - name: Bootstrap Cache - uses: ./.github/actions/bootstrap-cache - name: Bootstrap uses: ./.github/actions/bootstrap - env: - PW_NO_CIPD_CACHE_DIR: Y with: platform: ${{ inputs.platform }} + bootstrap-log-name: ${{ inputs.bootstrap-log-name }} - name: Dump disk info after checkout submodule & Bootstrap shell: bash run: scripts/dump_diskspace_info.sh - - name: Upload Bootstrap Logs - uses: ./.github/actions/upload-bootstrap-logs - with: - bootstrap-log-name: ${{ inputs.bootstrap-log-name }} - name: Work around TSAN ASLR issues if: runner.os == 'Linux' && !env.ACT shell: bash diff --git a/.github/actions/upload-bootstrap-logs/action.yaml b/.github/actions/upload-bootstrap-logs/action.yaml deleted file mode 100644 index 522058c16ff93c..00000000000000 --- a/.github/actions/upload-bootstrap-logs/action.yaml +++ /dev/null @@ -1,18 +0,0 @@ -name: Upload bootstrap logs -description: Upload bootstrap logs -inputs: - bootstrap-log-name: - description: "Bootstrap log name" - required: false - default: bootstrap-logs-${{ github.job }} -runs: - using: "composite" - steps: - - name: Uploading bootstrap logs - uses: actions/upload-artifact@v4 - if: ${{ always() && !env.ACT }} - with: - name: ${{ inputs.bootstrap-log-name }} - path: | - .environment/gn_out/.ninja_log - .environment/pigweed-venv/*.log diff --git a/.github/workflows/cirque.yaml b/.github/workflows/cirque.yaml index 1f0b7c5d327fcf..919085ac77ee0f 100644 --- a/.github/workflows/cirque.yaml +++ b/.github/workflows/cirque.yaml @@ -57,6 +57,7 @@ jobs: with: platform: linux + # TODO: Is what's being cached here actually compatible with a regular bootstrap? - name: Bootstrap Cache uses: ./.github/actions/bootstrap-cache - name: Bootstrap Cirque diff --git a/.github/workflows/fuzzing-build.yaml b/.github/workflows/fuzzing-build.yaml index ee19873cc32f2a..f354ce369c27fd 100644 --- a/.github/workflows/fuzzing-build.yaml +++ b/.github/workflows/fuzzing-build.yaml @@ -46,12 +46,8 @@ jobs: run: | mkdir objdir-clone || true - - name: Bootstrap Cache - uses: ./.github/actions/bootstrap-cache - name: Bootstrap uses: ./.github/actions/bootstrap - - name: Upload Bootstrap Logs - uses: ./.github/actions/upload-bootstrap-logs - name: Build all-clusters-app run: | @@ -84,12 +80,8 @@ jobs: run: | mkdir objdir-clone || true - - name: Bootstrap Cache - uses: ./.github/actions/bootstrap-cache - name: Bootstrap uses: ./.github/actions/bootstrap - - name: Upload Bootstrap Logs - uses: ./.github/actions/upload-bootstrap-logs - name: Build all-clusters-app run: | diff --git a/.github/workflows/release_artifacts.yaml b/.github/workflows/release_artifacts.yaml index 5aa38ecc893d28..57a1ce7b325dc3 100644 --- a/.github/workflows/release_artifacts.yaml +++ b/.github/workflows/release_artifacts.yaml @@ -39,12 +39,8 @@ jobs: uses: actions/checkout@v4 with: ref: "${{ github.event.inputs.releaseTag }}" - - name: Bootstrap Cache - uses: ./.github/actions/bootstrap-cache - name: Bootstrap uses: ./.github/actions/bootstrap - - name: Upload Bootstrap Logs - uses: ./.github/actions/upload-bootstrap-logs - name: Build run: scripts/examples/esp_example.sh all-clusters-app @@ -74,12 +70,8 @@ jobs: uses: actions/checkout@v4 with: ref: "${{ github.event.inputs.releaseTag }}" - - name: Bootstrap Cache - uses: ./.github/actions/bootstrap-cache - name: Bootstrap uses: ./.github/actions/bootstrap - - name: Upload Bootstrap Logs - uses: ./.github/actions/upload-bootstrap-logs - name: Build example EFR32 Lock App run: scripts/examples/gn_silabs_example.sh examples/lock-app/efr32/ diff --git a/scripts/setup/bootstrap.sh b/scripts/setup/bootstrap.sh index 1b813216222efe..d723e86f70924a 100644 --- a/scripts/setup/bootstrap.sh +++ b/scripts/setup/bootstrap.sh @@ -21,14 +21,14 @@ _install_additional_pip_requirements() { # figure out additional pip install items while [ $# -gt 0 ]; do case $1 in - -p | --platform) - _SETUP_PLATFORM=$2 - shift # argument - shift # value - ;; - *) - shift - ;; + -p | --platform) + _SETUP_PLATFORM=$2 + shift # argument + shift # value + ;; + *) + shift + ;; esac done @@ -41,7 +41,7 @@ _install_additional_pip_requirements() { for platform in ${_SETUP_PLATFORM}; do # Allow none as an alias of nothing extra installed (like -p none) - if [ "$platform" != "none" ]; then + if [ "$platform" != "none" -a -e "$_CHIP_ROOT/scripts/setup/requirements.$platform.txt" ]; then echo "Installing pip requirements for $platform..." pip install -q \ -r "$_CHIP_ROOT/scripts/setup/requirements.$platform.txt" \ @@ -66,7 +66,7 @@ _bootstrap_or_activate() { local _BOOTSTRAP_NAME="${_BOOTSTRAP_PATH##*/}" local _BOOTSTRAP_DIR="${_BOOTSTRAP_PATH%/*}" # Strip off the 'scripts[/setup]' directory, leaving the root of the repo. - _CHIP_ROOT="$(cd "${_BOOTSTRAP_DIR%/setup}/.." > /dev/null && pwd)" + _CHIP_ROOT="$(cd "${_BOOTSTRAP_DIR%/setup}/.." >/dev/null && pwd)" local _CONFIG_FILE="scripts/setup/environment.json" From caf9b40d4bbeb5a40dec2b50e205cd36ae4f46fb Mon Sep 17 00:00:00 2001 From: Karsten Sperling Date: Wed, 20 Mar 2024 15:50:18 +1300 Subject: [PATCH 3/4] Include the work directory path in the bootstrap cache key This is because the bootstrapped Pigweed environment does not like to be relocated. Also don't touch submodules in bootstrap, other than ensuring that Pigweed is available. --- .github/actions/bootstrap/action.yaml | 22 +++++++++++++++++++--- scripts/setup/bootstrap.sh | 16 +++++----------- 2 files changed, 24 insertions(+), 14 deletions(-) diff --git a/.github/actions/bootstrap/action.yaml b/.github/actions/bootstrap/action.yaml index 32fe6c9cfedba3..5e359dae4797d7 100644 --- a/.github/actions/bootstrap/action.yaml +++ b/.github/actions/bootstrap/action.yaml @@ -9,10 +9,26 @@ inputs: description: "Bootstrap log name" required: false default: bootstrap-logs-${{ github.job }} +outputs: + cache-hit: + description: "Bootstrap environment was restored from cache" + value: ${{ fromJSON(steps.bootstrap-cache.outputs.outputs).cache-hit }} # retry returns all outputs in `outputs` runs: using: "composite" steps: + - name: Calculate bootstrap cache key + id: bootstrap-cache-key + shell: bash + run: | + # Calculate bootstrap cache key + # In addition to the various setup files, the work directory matters as well, + # because the bootstrapped Pigweed environment contains absolute paths. + FILES_HASH="${{ hashFiles('scripts/setup/*', 'third_party/pigweed/**') }}" + FINAL_HASH="$(echo "$PWD:$FILES_HASH" | shasum -a 256 | cut -d' ' -f1)" + echo "Calculated bootstrap cache key for '$PWD': $FINAL_HASH" + echo "hash=$FINAL_HASH" >> "$GITHUB_OUTPUT" + - uses: Wandalen/wretry.action@v1.4.10 name: Bootstrap from cache id: bootstrap-cache @@ -22,20 +38,20 @@ runs: attempt_limit: 3 attempt_delay: 2000 with: | - key: ${{ runner.os }}-${{ inputs.platform }}-env-${{ hashFiles('scripts/setup/*', 'third_party/pigweed/**') }} + key: ${{ runner.os }}-${{ inputs.platform }}-${{ steps.bootstrap-cache-key.outputs.hash}} path: | .environment build_overrides/pigweed_environment.gni - name: Run bootstrap - if: fromJSON(steps.bootstrap-cache.outputs.outputs).cache-hit != 'true' # retry returns all outputs in `outputs` + if: fromJSON(steps.bootstrap-cache.outputs.outputs).cache-hit != 'true' env: PW_NO_CIPD_CACHE_DIR: Y shell: bash run: source scripts/bootstrap.sh -p all,${{ inputs.platform }} - name: Uploading bootstrap logs - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v4 if: always() && !env.ACT && fromJSON(steps.bootstrap-cache.outputs.outputs).cache-hit != 'true' with: name: ${{ inputs.bootstrap-log-name }} diff --git a/scripts/setup/bootstrap.sh b/scripts/setup/bootstrap.sh index d723e86f70924a..0a2a898eaf02c5 100644 --- a/scripts/setup/bootstrap.sh +++ b/scripts/setup/bootstrap.sh @@ -75,16 +75,6 @@ _bootstrap_or_activate() { unset PW_CONFIG_FILE fi - if [ ! -f "$_CHIP_ROOT/third_party/pigweed/repo/pw_env_setup/util.sh" ]; then - # Make sure our submodule remotes are correct for this revision. - git submodule sync --recursive - git submodule update --init - elif [ "$_BOOTSTRAP_NAME" = "bootstrap.sh" ]; then - # In this case, only update already checked out submodules. - git submodule sync --recursive - git submodule update - fi - PW_BRANDING_BANNER="$_CHIP_ROOT/scripts/setup/banner.txt" export PW_BRANDING_BANNER @@ -94,6 +84,10 @@ _bootstrap_or_activate() { PW_ROOT="$_CHIP_ROOT/third_party/pigweed/repo" export PW_ROOT + # Update or init the pigweed submodule if necessary. Don't touch any other submodules. + if [ "$_BOOTSTRAP_NAME" = "bootstrap.sh" -a ! -f "$_CHIP_ROOT/third_party/pigweed/repo/pw_env_setup/util.sh" ]; then + git submodule update --init "$_CHIP_ROOT/third_party/pigweed/repo" + fi . "$_CHIP_ROOT/third_party/pigweed/repo/pw_env_setup/util.sh" _chip_bootstrap_banner() { @@ -105,7 +99,7 @@ _bootstrap_or_activate() { local _PW_BANNER_FUNC="_chip_bootstrap_banner" - # Force the Pigweed environment directory to be '.environment' + # Default the Pigweed environment directory to be '.environment' if [ -z "$PW_ENVIRONMENT_ROOT" ]; then export PW_ENVIRONMENT_ROOT="$PW_PROJECT_ROOT/.environment" fi From 8ff269659f5f9d74319e66afaa3dc9aa1d208e28 Mon Sep 17 00:00:00 2001 From: Karsten Sperling Date: Mon, 25 Mar 2024 15:26:35 +1300 Subject: [PATCH 4/4] Include runner.arch as well We may start using macos-14 runners in the future which are arm. --- .github/actions/bootstrap/action.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/actions/bootstrap/action.yaml b/.github/actions/bootstrap/action.yaml index 5e359dae4797d7..16a2fba998fcc0 100644 --- a/.github/actions/bootstrap/action.yaml +++ b/.github/actions/bootstrap/action.yaml @@ -38,7 +38,7 @@ runs: attempt_limit: 3 attempt_delay: 2000 with: | - key: ${{ runner.os }}-${{ inputs.platform }}-${{ steps.bootstrap-cache-key.outputs.hash}} + key: ${{ runner.os }}-${{ runner.arch }}-${{ inputs.platform }}-${{ steps.bootstrap-cache-key.outputs.hash}} path: | .environment build_overrides/pigweed_environment.gni