From 6cc70c2fe735a55c6718f0f10bfffc88b517c667 Mon Sep 17 00:00:00 2001 From: Andrei Litvin Date: Sat, 28 Jan 2023 14:18:12 -0500 Subject: [PATCH] Remove zap checkout for darwin builds in CI, assume CIPD provides this (#24677) * Remove zap checkout for darwin builds in CI, assume CIPD provides this. If this works, we can work on modifying the dockerfiles for linux too * Make zap-cli NOT available for xcodebuild * Add --version to zap-cli to not try to run more things * The zap test should be under the build environment, to get the path working * Run in build environment when checking for zap --------- Co-authored-by: Andrei Litvin --- .github/workflows/build.yaml | 16 ---------- .github/workflows/darwin-tests.yaml | 14 --------- .github/workflows/darwin.yaml | 44 +++++++--------------------- .github/workflows/fuzzing-build.yaml | 16 ---------- .github/workflows/tests.yaml | 26 ---------------- scripts/tools/zap/version_update.py | 32 +++++++++++--------- 6 files changed, 28 insertions(+), 120 deletions(-) diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index 628abbbf5aba72..8a847a3e034df6 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -352,12 +352,6 @@ jobs: timeout-minutes: 200 runs-on: macos-latest if: github.actor != 'restyled-io[bot]' - - env: - # NOTE: Generally kept in sync within the repo using - # scripts/tools/zap/version_update.py - ZAP_VERSION: v2023.01.19-nightly - ZAP_INSTALL_PATH: /usr/local/zap/zap-v2023.01.19-nightly steps: - uses: Wandalen/wretry.action@v1.0.36 @@ -388,16 +382,6 @@ jobs: ls -la /usr/local/Cellar/openssl@1.1 OPEN_SSL_VERSION=`ls -la /usr/local/Cellar/openssl@1.1 | cat | tail -n1 | awk '{print $NF}'` ln -s /usr/local/Cellar/openssl@1.1/$OPEN_SSL_VERSION/lib/pkgconfig/* . - - name: Install zap - run: | - sudo mkdir -p $ZAP_INSTALL_PATH - sudo chown `whoami` $ZAP_INSTALL_PATH - curl -L https://github.com/project-chip/zap/releases/download/${ZAP_VERSION}/zap-mac.zip \ - --output $ZAP_INSTALL_PATH/zap-mac.zip - cd $ZAP_INSTALL_PATH - unzip zap-mac.zip zap-cli - rm zap-mac.zip - ./zap-cli --version - name: Bootstrap timeout-minutes: 25 run: scripts/build/gn_bootstrap.sh diff --git a/.github/workflows/darwin-tests.yaml b/.github/workflows/darwin-tests.yaml index d9e70b82503984..7d1db5f479dc6c 100644 --- a/.github/workflows/darwin-tests.yaml +++ b/.github/workflows/darwin-tests.yaml @@ -39,10 +39,6 @@ jobs: env: BUILD_VARIANT: ${{matrix.build_variant}} LSAN_OPTIONS: detect_leaks=1 malloc_context_size=40 suppressions=scripts/tests/chiptest/lsan-mac-suppressions.txt - # NOTE: Generally kept in sync within the repo using - # scripts/tools/zap/version_update.py - ZAP_VERSION: v2023.01.19-nightly - ZAP_INSTALL_PATH: /usr/local/zap/zap-v2023.01.19-nightly if: github.actor != 'restyled-io[bot]' runs-on: macos-latest @@ -76,16 +72,6 @@ jobs: ls -la /usr/local/Cellar/openssl@1.1 OPEN_SSL_VERSION=`ls -la /usr/local/Cellar/openssl@1.1 | cat | tail -n1 | awk '{print $NF}'` ln -s /usr/local/Cellar/openssl@1.1/$OPEN_SSL_VERSION/lib/pkgconfig/* . - - name: Install zap - run: | - sudo mkdir -p $ZAP_INSTALL_PATH - sudo chown `whoami` $ZAP_INSTALL_PATH - curl -L https://github.com/project-chip/zap/releases/download/${ZAP_VERSION}/zap-mac.zip \ - --output $ZAP_INSTALL_PATH/zap-mac.zip - cd $ZAP_INSTALL_PATH - unzip zap-mac.zip zap-cli - rm zap-mac.zip - ./zap-cli --version - name: Bootstrap timeout-minutes: 25 run: scripts/build/gn_bootstrap.sh diff --git a/.github/workflows/darwin.yaml b/.github/workflows/darwin.yaml index 447c16fe8eb88a..fb29e67cd0ce98 100644 --- a/.github/workflows/darwin.yaml +++ b/.github/workflows/darwin.yaml @@ -52,22 +52,6 @@ jobs: ls -la /usr/local/Cellar/openssl@1.1 OPEN_SSL_VERSION=`ls -la /usr/local/Cellar/openssl@1.1 | cat | tail -n1 | awk '{print $NF}'` ln -s /usr/local/Cellar/openssl@1.1/$OPEN_SSL_VERSION/lib/pkgconfig/* . - - name: Install zap - run: | - sudo mkdir -p $ZAP_INSTALL_PATH - sudo chown `whoami` $ZAP_INSTALL_PATH - curl -L https://github.com/project-chip/zap/releases/download/${ZAP_VERSION}/zap-mac.zip \ - --output $ZAP_INSTALL_PATH/zap-mac.zip - cd $ZAP_INSTALL_PATH - unzip zap-mac.zip zap-cli - rm zap-mac.zip - ./zap-cli --version - env: - # NOTE: Generally kept in sync within the repo using - # scripts/tools/zap/version_update.py - # This is scoped to only the steps that don't use xcodebuild. - ZAP_VERSION: v2023.01.19-nightly - ZAP_INSTALL_PATH: /usr/local/zap/zap-v2023.01.19-nightly - name: Bootstrap timeout-minutes: 25 run: scripts/build/gn_bootstrap.sh @@ -79,6 +63,12 @@ jobs: path: | .environment/gn_out/.ninja_log .environment/pigweed-venv/*.log + - name: Block zap-cli from being used + # xcodebuild is NOT expected to require zap-cli + run: scripts/run_in_build_env.sh 'mv $PW_ENVIRONMENT_ROOT/cipd/packages/zap/zap-cli $PW_ENVIRONMENT_ROOT/cipd/packages/zap/zap-cli.moved' + - name: Validate zap-cli is NOT available + # run_in_build_env.sh is used to ensure PATH is set to something that would otherwise find zap-cli + run: scripts/run_in_build_env.sh '(zap-cli --version && exit 1) || exit 0' - name: Run iOS Build Debug timeout-minutes: 50 working-directory: src/darwin/Framework @@ -118,36 +108,22 @@ jobs: - name: Clean Build run: xcodebuild clean working-directory: src/darwin/Framework + - name: Make zap-cli work again + run: scripts/run_in_build_env.sh 'mv $PW_ENVIRONMENT_ROOT/cipd/packages/zap/zap-cli.moved $PW_ENVIRONMENT_ROOT/cipd/packages/zap/zap-cli' + - name: Validate zap-cli is again available + run: scripts/run_in_build_env.sh 'zap-cli --version' - name: Build example All Clusters Server timeout-minutes: 15 run: | scripts/examples/gn_build_example.sh examples/all-clusters-app/linux out/debug chip_config_network_layer_ble=false - env: - # NOTE: Generally kept in sync within the repo using - # scripts/tools/zap/version_update.py - # This is scoped to only the steps that don't use xcodebuild. - ZAP_VERSION: v2023.01.19-nightly - ZAP_INSTALL_PATH: /usr/local/zap/zap-v2023.01.19-nightly - name: Build example OTA Provider timeout-minutes: 10 run: | scripts/examples/gn_build_example.sh examples/ota-provider-app/linux out/debug chip_config_network_layer_ble=false - env: - # NOTE: Generally kept in sync within the repo using - # scripts/tools/zap/version_update.py - # This is scoped to only the steps that don't use xcodebuild. - ZAP_VERSION: v2023.01.19-nightly - ZAP_INSTALL_PATH: /usr/local/zap/zap-v2023.01.19-nightly - name: Build example OTA Requestor timeout-minutes: 10 run: | scripts/examples/gn_build_example.sh examples/ota-requestor-app/linux out/debug chip_config_network_layer_ble=false - env: - # NOTE: Generally kept in sync within the repo using - # scripts/tools/zap/version_update.py - # This is scoped to only the steps that don't use xcodebuild. - ZAP_VERSION: v2023.01.19-nightly - ZAP_INSTALL_PATH: /usr/local/zap/zap-v2023.01.19-nightly - name: Delete Defaults run: defaults delete com.apple.dt.xctest.tool continue-on-error: true diff --git a/.github/workflows/fuzzing-build.yaml b/.github/workflows/fuzzing-build.yaml index 0ffb8d982d13dd..3f8e0424010bf5 100644 --- a/.github/workflows/fuzzing-build.yaml +++ b/.github/workflows/fuzzing-build.yaml @@ -88,12 +88,6 @@ jobs: runs-on: macos-latest if: github.actor != 'restyled-io[bot]' - env: - # NOTE: Generally kept in sync within the repo using - # scripts/tools/zap/version_update.py - ZAP_VERSION: v2023.01.19-nightly - ZAP_INSTALL_PATH: /usr/local/zap/zap-v2023.01.19-nightly - steps: - uses: Wandalen/wretry.action@v1.0.36 name: Checkout @@ -117,16 +111,6 @@ jobs: ls -la /usr/local/Cellar/openssl@1.1 OPEN_SSL_VERSION=`ls -la /usr/local/Cellar/openssl@1.1 | cat | tail -n1 | awk '{print $NF}'` ln -s /usr/local/Cellar/openssl@1.1/$OPEN_SSL_VERSION/lib/pkgconfig/* . - - name: Install zap - run: | - sudo mkdir -p $ZAP_INSTALL_PATH - sudo chown `whoami` $ZAP_INSTALL_PATH - curl -L https://github.com/project-chip/zap/releases/download/${ZAP_VERSION}/zap-mac.zip \ - --output $ZAP_INSTALL_PATH/zap-mac.zip - cd $ZAP_INSTALL_PATH - unzip zap-mac.zip zap-cli - rm zap-mac.zip - ./zap-cli --version - name: Bootstrap timeout-minutes: 25 run: scripts/build/gn_bootstrap.sh diff --git a/.github/workflows/tests.yaml b/.github/workflows/tests.yaml index cebb51d5d4115d..8d0365846e86ab 100644 --- a/.github/workflows/tests.yaml +++ b/.github/workflows/tests.yaml @@ -260,10 +260,6 @@ jobs: CHIP_TOOL_VARIANT: ${{matrix.chip_tool}} TSAN_OPTIONS: "halt_on_error=1" LSAN_OPTIONS: detect_leaks=1 suppressions=scripts/tests/chiptest/lsan-mac-suppressions.txt - # NOTE: Generally kept in sync within the repo using - # scripts/tools/zap/version_update.py - ZAP_VERSION: v2023.01.19-nightly - ZAP_INSTALL_PATH: /usr/local/zap/zap-v2023.01.19-nightly if: github.actor != 'restyled-io[bot]' runs-on: macos-latest @@ -297,15 +293,6 @@ jobs: ls -la /usr/local/Cellar/openssl@1.1 OPEN_SSL_VERSION=`ls -la /usr/local/Cellar/openssl@1.1 | cat | tail -n1 | awk '{print $NF}'` ln -s /usr/local/Cellar/openssl@1.1/$OPEN_SSL_VERSION/lib/pkgconfig/* . - - name: Install zap - run: | - sudo mkdir -p $ZAP_INSTALL_PATH - sudo chown `whoami` $ZAP_INSTALL_PATH - curl -L https://github.com/project-chip/zap/releases/download/${ZAP_VERSION}/zap-mac.zip \ - --output $ZAP_INSTALL_PATH/zap-mac.zip - cd $ZAP_INSTALL_PATH - unzip zap-mac.zip zap-cli - rm zap-mac.zip - name: Bootstrap timeout-minutes: 25 run: scripts/build/gn_bootstrap.sh @@ -546,10 +533,6 @@ jobs: env: BUILD_VARIANT: ${{matrix.build_variant}} TSAN_OPTIONS: "halt_on_error=1" - # NOTE: Generally kept in sync within the repo using - # scripts/tools/zap/version_update.py - ZAP_VERSION: v2023.01.19-nightly - ZAP_INSTALL_PATH: /usr/local/zap/zap-v2023.01.19-nightly if: github.actor != 'restyled-io[bot]' && false runs-on: macos-latest @@ -577,15 +560,6 @@ jobs: ls -la /usr/local/Cellar/openssl@1.1 OPEN_SSL_VERSION=`ls -la /usr/local/Cellar/openssl@1.1 | cat | tail -n1 | awk '{print $NF}'` ln -s /usr/local/Cellar/openssl@1.1/$OPEN_SSL_VERSION/lib/pkgconfig/* . - - name: Install zap - run: | - sudo mkdir -p $ZAP_INSTALL_PATH - sudo chown `whoami` $ZAP_INSTALL_PATH - curl -L https://github.com/project-chip/zap/releases/download/${ZAP_VERSION}/zap-mac.zip \ - --output $ZAP_INSTALL_PATH/zap-mac.zip - cd $ZAP_INSTALL_PATH - unzip zap-mac.zip zap-cli - rm zap-mac.zip - name: Bootstrap timeout-minutes: 25 run: scripts/build/gn_bootstrap.sh diff --git a/scripts/tools/zap/version_update.py b/scripts/tools/zap/version_update.py index 9d577913139130..81fd1c20779bcc 100755 --- a/scripts/tools/zap/version_update.py +++ b/scripts/tools/zap/version_update.py @@ -49,11 +49,6 @@ # Set as a separate list to not pay the price of a full grep as the list of # files is not likely to change often USAGE_FILES_DEPENDING_ON_ZAP_VERSION = [ - '.github/workflows/build.yaml', - '.github/workflows/darwin-tests.yaml', - '.github/workflows/darwin.yaml', - '.github/workflows/fuzzing-build.yaml', - '.github/workflows/tests.yaml', 'integrations/docker/images/chip-cert-bins/Dockerfile', 'scripts/zap.json', ] @@ -96,9 +91,11 @@ class UpdateChoice(Flag): # # That line is of the form "MIN_ZAP_VERSION = '2021.1.9'" ZAP_EXECUTION_SCRIPT = 'scripts/tools/zap/zap_execution.py' -ZAP_EXECUTION_MIN_RE = re.compile(r'(MIN_ZAP_VERSION = .)(\d\d\d\d\.\d\d?\.\d\d?)(.)') +ZAP_EXECUTION_MIN_RE = re.compile( + r'(MIN_ZAP_VERSION = .)(\d\d\d\d\.\d\d?\.\d\d?)(.)') -CHIP_ROOT_DIR = os.path.abspath(os.path.join(os.path.dirname(__file__), '..', '..', '..')) +CHIP_ROOT_DIR = os.path.abspath(os.path.join( + os.path.dirname(__file__), '..', '..', '..')) @click.command() @@ -117,14 +114,16 @@ class UpdateChoice(Flag): default=None, help='What version of ZAP to update to (like "v2023.01.09-nightly". If not set, versions will just be printed.') def version_update(log_level, update, new_version): - coloredlogs.install(level=__LOG_LEVELS__[log_level], fmt='%(asctime)s %(levelname)-7s %(message)s') + coloredlogs.install(level=__LOG_LEVELS__[ + log_level], fmt='%(asctime)s %(levelname)-7s %(message)s') update = __UPDATE_CHOICES__[update] if new_version: parsed = ZAP_VERSION_RE.match(new_version) if not parsed: - logging.error(f"Version '{new_version}' does not seem to parse as a ZAP VERSION") + logging.error( + f"Version '{new_version}' does not seem to parse as a ZAP VERSION") sys.exit(1) # get the numeric version for zap_execution @@ -161,15 +160,18 @@ def version_update(log_level, update, new_version): while m: version = file_data[m.start():m.end()] if version == new_version: - logging.warning("Nothing to replace. Version already %s", version) + logging.warning( + "Nothing to replace. Version already %s", version) break - file_data = file_data[:m.start()] + new_version + file_data[m.end():] + file_data = file_data[:m.start()] + \ + new_version + file_data[m.end():] need_replace = True search_pos = m.end() # generally ok since our versions are fixed length m = ZAP_VERSION_RE.search(file_data, search_pos) if need_replace: - logging.info('Replacing with version %s in %s', new_version, name) + logging.info('Replacing with version %s in %s', + new_version, name) with open(os.path.join(CHIP_ROOT_DIR, name), 'wt') as f: f.write(file_data) @@ -183,8 +185,10 @@ def version_update(log_level, update, new_version): logging.info("Min version %s in %s", m.group(2), ZAP_EXECUTION_SCRIPT) if new_version: new_min_version = ("%d.%d.%d" % zap_min_version) - file_data = file_data[:m.start()] + m.group(1) + new_min_version + m.group(3) + file_data[m.end():] - logging.info('Updating min version to %s in %s', new_min_version, ZAP_EXECUTION_SCRIPT) + file_data = file_data[:m.start()] + m.group(1) + \ + new_min_version + m.group(3) + file_data[m.end():] + logging.info('Updating min version to %s in %s', + new_min_version, ZAP_EXECUTION_SCRIPT) with open(os.path.join(CHIP_ROOT_DIR, ZAP_EXECUTION_SCRIPT), 'wt') as f: f.write(file_data)