From 036e7bfaa173b0aef5a66e04854094e8045d9a03 Mon Sep 17 00:00:00 2001 From: Karsten Sperling <113487422+ksperling-apple@users.noreply.github.com> Date: Fri, 29 Mar 2024 02:26:29 +1300 Subject: [PATCH] Darwin CI: use macos-13 runners and other tweaks (#32748) * Darwin CI: use macos-13 runners and other tweaks According to github docs these have 4 CPUs (vs 3 for macos-12 == latest). In practice compile steps seem to be about 7% faster. Also use the pre-installed Python (3.12), remove the "aggregation" step, and run crash log collection after failure (not success), and condense disk space output to large directories only. * CI: Save bootstrap cache right after bootrap This avoids caching a corrupted cache in cases where a job modifies the environment (e.g. the Darwin job hiding zap-cli). It also makes the cache available slightly sooner. Also add plumbing for splitting caches over buildjet and github backends. * Remove zap-cli restore workaround * Update darwin-tests workflow as well --- .github/actions/bootstrap/action.yaml | 59 +++++++++++++++++---------- .github/actions/dynamic/action.yaml | 38 +++++++++++++++++ .github/workflows/darwin-tests.yaml | 2 +- .github/workflows/darwin.yaml | 30 ++++---------- scripts/dump_diskspace_info.sh | 2 +- 5 files changed, 87 insertions(+), 44 deletions(-) create mode 100644 .github/actions/dynamic/action.yaml diff --git a/.github/actions/bootstrap/action.yaml b/.github/actions/bootstrap/action.yaml index 16a2fba998fcc0..749dcb739b9854 100644 --- a/.github/actions/bootstrap/action.yaml +++ b/.github/actions/bootstrap/action.yaml @@ -12,49 +12,66 @@ inputs: outputs: cache-hit: description: "Bootstrap environment was restored from cache" - value: ${{ fromJSON(steps.bootstrap-cache.outputs.outputs).cache-hit }} # retry returns all outputs in `outputs` + value: ${{ fromJSON(steps.restore.outputs.outputs).cache-hit }} # dynamic action returns all outputs in `outputs` runs: using: "composite" steps: - - name: Calculate bootstrap cache key - id: bootstrap-cache-key + - name: Determine bootstrap cache configuration + id: prepare shell: bash run: | - # Calculate bootstrap cache key + # Determine bootstrap cache configuration # In addition to the various setup files, the work directory matters as well, # because the bootstrapped Pigweed environment contains absolute paths. + echo "Calculating bootstrap cache key for '$PWD'" FILES_HASH="${{ hashFiles('scripts/setup/*', 'third_party/pigweed/**') }}" FINAL_HASH="$(echo "$PWD:$FILES_HASH" | shasum -a 256 | cut -d' ' -f1)" - echo "Calculated bootstrap cache key for '$PWD': $FINAL_HASH" - echo "hash=$FINAL_HASH" >> "$GITHUB_OUTPUT" + echo key="${RUNNER_OS}-${RUNNER_ARCH}-${{ inputs.platform }}-${FINAL_HASH}" | tee -a "$GITHUB_OUTPUT" - - uses: Wandalen/wretry.action@v1.4.10 - name: Bootstrap from cache - id: bootstrap-cache + # Split caches across backends + case "$RUNNER_OS" in + macOS) echo backend=actions;; + *) echo backend=buildjet;; + esac | tee -a "$GITHUB_OUTPUT" + + - name: Bootstrap from cache + id: restore + uses: ./.github/actions/dynamic continue-on-error: true with: - action: buildjet/cache@v4 - attempt_limit: 3 - attempt_delay: 2000 + action: ${{ steps.prepare.outputs.backend }}/cache/restore@v4 with: | - key: ${{ runner.os }}-${{ runner.arch }}-${{ inputs.platform }}-${{ steps.bootstrap-cache-key.outputs.hash}} + key: ${{ steps.prepare.outputs.key }} path: | - .environment - build_overrides/pigweed_environment.gni + .environment + build_overrides/pigweed_environment.gni - name: Run bootstrap - if: fromJSON(steps.bootstrap-cache.outputs.outputs).cache-hit != 'true' + if: ${{ fromJSON(steps.restore.outputs.outputs).cache-hit != 'true' }} env: - PW_NO_CIPD_CACHE_DIR: Y + PW_NO_CIPD_CACHE_DIR: 1 + PW_ENVSETUP_NO_BANNER: 1 shell: bash run: source scripts/bootstrap.sh -p all,${{ inputs.platform }} - - name: Uploading bootstrap logs + - name: Save bootstrap cache + uses: ./.github/actions/dynamic + if: ${{ fromJSON(steps.restore.outputs.outputs).cache-hit != 'true' }} + continue-on-error: true + with: + action: ${{ steps.prepare.outputs.backend }}/cache/save@v4 + with: | + key: ${{ steps.prepare.outputs.key }} + path: | + .environment + build_overrides/pigweed_environment.gni + + - name: Upload bootstrap logs uses: actions/upload-artifact@v4 - if: always() && !env.ACT && fromJSON(steps.bootstrap-cache.outputs.outputs).cache-hit != 'true' + if: ${{ always() && !env.ACT && fromJSON(steps.restore.outputs.outputs).cache-hit != 'true' }} with: name: ${{ inputs.bootstrap-log-name }} path: | - .environment/gn_out/.ninja_log - .environment/pigweed-venv/*.log + .environment/gn_out/.ninja_log + .environment/pigweed-venv/*.log diff --git a/.github/actions/dynamic/action.yaml b/.github/actions/dynamic/action.yaml new file mode 100644 index 00000000000000..7cee233c4f4ce2 --- /dev/null +++ b/.github/actions/dynamic/action.yaml @@ -0,0 +1,38 @@ +name: Dynamic +description: Dynamically resolves another GitHub action +inputs: + action: + description: Action reference + required: true + with: + description: Action inputs as multi-line YAML string + required: false + default: "" +outputs: + outputs: + description: JSON object of outputs from the action + value: ${{ steps.run.outputs.outputs }} +runs: + using: composite + steps: + - name: Instantiate + shell: bash + run: | + # Dynamically invoke ${{ inputs.action }} + with='${{ inputs.with }}' + [[ -z "$with" ]] || with="$(echo ' with:'; sed -e 's/^/ /' <<<"$with")" + mkdir -p ./.tmp/dynamic-action-instance + cat <./.tmp/dynamic-action-instance/action.yaml + runs: + using: composite + steps: + - id: run + uses: ${{ inputs.action }} + $with + outputs: + outputs: + value: $(echo '$'){{ toJSON(steps.run.outputs) }} + END + - name: Run + id: run + uses: ./.tmp/dynamic-action-instance diff --git a/.github/workflows/darwin-tests.yaml b/.github/workflows/darwin-tests.yaml index e6e14ec4929a11..56fd5856428e10 100644 --- a/.github/workflows/darwin-tests.yaml +++ b/.github/workflows/darwin-tests.yaml @@ -49,7 +49,7 @@ jobs: LSAN_OPTIONS: detect_leaks=1 malloc_context_size=40 suppressions=scripts/tests/chiptest/lsan-mac-suppressions.txt if: github.actor != 'restyled-io[bot]' - runs-on: macos-latest + runs-on: macos-13 steps: - name: Checkout diff --git a/.github/workflows/darwin.yaml b/.github/workflows/darwin.yaml index bacfb90318c04a..6e12f1df3e11ed 100644 --- a/.github/workflows/darwin.yaml +++ b/.github/workflows/darwin.yaml @@ -31,7 +31,7 @@ jobs: framework: name: Build framework if: github.actor != 'restyled-io[bot]' - runs-on: macos-latest + runs-on: macos-13 strategy: matrix: options: # We don't need a full matrix @@ -48,30 +48,28 @@ jobs: steps: - name: Checkout uses: actions/checkout@v4 - - name: Setup Environment - run: brew install python@3.9 - name: Checkout submodules & Bootstrap uses: ./.github/actions/checkout-submodules-and-bootstrap with: platform: darwin bootstrap-log-name: bootstrap-logs-framework-${{ matrix.options.flavor }} - name: Block zap-cli from being used + env: + PW_ENVSETUP_NO_BANNER: 1 run: | # Framework builds are NOT expected to require zap-cli - scripts/run_in_build_env.sh 'D=$(dirname $(which zap-cli)) && mv $D/zap-cli $D/zap-cli.moved' + scripts/run_in_build_env.sh 'rm -- "$(which zap-cli)"' # run_in_build_env.sh is used to ensure PATH is set to something that would otherwise find zap-cli scripts/run_in_build_env.sh '(zap-cli --version && exit 1) || exit 0' - name: Build working-directory: src/darwin/Framework run: xcodebuild -target "Matter" ${{ matrix.options.arguments }} - # Now restore zap-cli, so that bootstrap caching does not cache the state when it's been renamed. - - name: Make zap-cli work again - run: scripts/run_in_build_env.sh 'D=$(dirname $(which zap-cli.moved)) && mv $D/zap-cli.moved $D/zap-cli' tests: name: Run framework tests if: github.actor != 'restyled-io[bot]' - runs-on: macos-latest + needs: [ framework ] # serialize to avoid running to many parallel macos runners + runs-on: macos-13 strategy: matrix: options: # We don't need a full matrix @@ -85,8 +83,6 @@ jobs: steps: - name: Checkout uses: actions/checkout@v4 - - name: Setup Environment - run: brew install python@3.9 - name: Checkout submodules & Bootstrap uses: ./.github/actions/checkout-submodules-and-bootstrap with: @@ -120,6 +116,7 @@ jobs: OTHER_CFLAGS='${inherited} -Werror -Wconversion' CHIP_IS_BLE=NO GCC_PREPROCESSOR_DEFINITIONS='${inherited} MTR_NO_AVAILABILITY=1 ${{ matrix.options.defines }}' \ > >(tee /tmp/darwin/framework-tests/darwin-tests.log) 2> >(tee /tmp/darwin/framework-tests/darwin-tests-err.log >&2) - name: Collect crash logs + if: failure() && !env.ACT run: | mkdir -p /tmp/darwin/framework-tests find ~/Library/Developer/Xcode/DerivedData /Library/Logs/DiagnosticReports -name '*.ips' -print0 | xargs -0 -J % cp % /tmp/darwin/framework-tests @@ -134,12 +131,11 @@ jobs: tv-casting-bridge: name: Build TV Casting Bridge example if: github.actor != 'restyled-io[bot]' - runs-on: macos-latest + needs: [ framework ] # serialize to avoid running to many parallel macos runners + runs-on: macos-13 steps: - name: Checkout uses: actions/checkout@v4 - - name: Setup Environment - run: brew install python@3.9 - name: Checkout submodules & Bootstrap uses: ./.github/actions/checkout-submodules-and-bootstrap with: @@ -147,11 +143,3 @@ jobs: - name: Build working-directory: examples/tv-casting-app/darwin/MatterTvCastingBridge run: xcodebuild -target "MatterTvCastingBridge" -sdk iphoneos - - darwin: - name: Build Darwin # Matches the previous monolithic build that's marked "required" for PRs - needs: [ framework, tests ] - runs-on: macos-latest - steps: - - name: Done - run: 'true' # nothing to do diff --git a/scripts/dump_diskspace_info.sh b/scripts/dump_diskspace_info.sh index 2e947f444e746a..177e1ba3aad53b 100755 --- a/scripts/dump_diskspace_info.sh +++ b/scripts/dump_diskspace_info.sh @@ -36,7 +36,7 @@ echo echo "Storage Space Used By Key Directories:" for directory in "${listOfDirectories[@]}"; do if [ -d "$directory" ]; then - du -d1 -h "$directory" | sort -h + du -d1 -h "$directory" | grep '^[[:space:]]*[0-9.]*[MG][[:space:]]' | sort -rh echo fi done