diff --git a/.github/workflows/bloat_check.yaml b/.github/workflows/bloat_check.yaml index da6bc36782823f..ce9d5b2d0f5968 100644 --- a/.github/workflows/bloat_check.yaml +++ b/.github/workflows/bloat_check.yaml @@ -30,7 +30,7 @@ jobs: runs-on: ubuntu-latest container: - image: connectedhomeip/chip-build:0.6.30 + image: connectedhomeip/chip-build:0.6.31 steps: - uses: Wandalen/wretry.action@v1.0.36 diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index e74d931ef8111f..67f0edc2357bb9 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -36,7 +36,7 @@ jobs: if: github.actor != 'restyled-io[bot]' container: - image: connectedhomeip/chip-build:0.6.30 + image: connectedhomeip/chip-build:0.6.31 volumes: - "/tmp/log_output:/tmp/test_logs" options: --privileged --sysctl "net.ipv6.conf.all.disable_ipv6=0 @@ -138,7 +138,7 @@ jobs: if: github.actor != 'restyled-io[bot]' container: - image: connectedhomeip/chip-build:0.6.30 + image: connectedhomeip/chip-build:0.6.31 volumes: - "/tmp/log_output:/tmp/test_logs" options: --privileged --sysctl "net.ipv6.conf.all.disable_ipv6=0 @@ -251,6 +251,14 @@ jobs: --pregen-dir ./zzz_pregenerated \ build \ " + - name: Check no code generation in output + run: | + CNT=$(find -name "CHIPClusters.h" out | wc -l) + if [ "${CNT}" != "0" ]; then + echo "ERROR: found unexpected generated files:" + find -name "CHIPClusters.h" out + exit 1 + fi - name: Undo code pre-generation changes (make compile time codegen work again) run: | rm -rf ./zzz_pregenerated @@ -290,7 +298,7 @@ jobs: if: github.actor != 'restyled-io[bot]' container: - image: connectedhomeip/chip-build:0.6.30 + image: connectedhomeip/chip-build:0.6.31 volumes: - "/tmp/log_output:/tmp/test_logs" options: --sysctl "net.ipv6.conf.all.disable_ipv6=0 @@ -344,6 +352,12 @@ jobs: timeout-minutes: 200 runs-on: macos-latest if: github.actor != 'restyled-io[bot]' + + env: + # NOTE: Generally kept in sync within the repo using + # scripts/tools/zap/version_update.py + ZAP_VERSION: v2023.01.09-nightly + ZAP_INSTALL_PATH: /usr/local/zap/zap-v2023.01.09-nightly steps: - uses: Wandalen/wretry.action@v1.0.36 @@ -374,6 +388,16 @@ jobs: ls -la /usr/local/Cellar/openssl@1.1 OPEN_SSL_VERSION=`ls -la /usr/local/Cellar/openssl@1.1 | cat | tail -n1 | awk '{print $NF}'` ln -s /usr/local/Cellar/openssl@1.1/$OPEN_SSL_VERSION/lib/pkgconfig/* . + - name: Install zap + run: | + sudo mkdir -p $ZAP_INSTALL_PATH + sudo chown `whoami` $ZAP_INSTALL_PATH + curl -L https://github.com/project-chip/zap/releases/download/${ZAP_VERSION}/zap-mac.zip \ + --output $ZAP_INSTALL_PATH/zap-mac.zip + cd $ZAP_INSTALL_PATH + unzip zap-mac.zip zap-cli + rm zap-mac.zip + ./zap-cli --version - name: Bootstrap timeout-minutes: 25 run: scripts/build/gn_bootstrap.sh @@ -447,7 +471,7 @@ jobs: if: github.actor != 'restyled-io[bot]' container: - image: connectedhomeip/chip-build:0.6.30 + image: connectedhomeip/chip-build:0.6.31 volumes: - "/tmp/log_output:/tmp/test_logs" options: --privileged --sysctl "net.ipv6.conf.all.disable_ipv6=0 @@ -478,4 +502,4 @@ jobs: - name: Run Build Coverage timeout-minutes: 20 run: ./scripts/build_coverage.sh - \ No newline at end of file + diff --git a/.github/workflows/chef.yaml b/.github/workflows/chef.yaml index 2e109de498da9b..bc3f8c7db8c04e 100644 --- a/.github/workflows/chef.yaml +++ b/.github/workflows/chef.yaml @@ -29,7 +29,7 @@ jobs: if: github.actor != 'restyled-io[bot]' container: - image: connectedhomeip/chip-build:0.6.30 + image: connectedhomeip/chip-build:0.6.31 options: --user root steps: @@ -57,7 +57,7 @@ jobs: if: github.actor != 'restyled-io[bot]' container: - image: connectedhomeip/chip-build-esp32:0.6.30 + image: connectedhomeip/chip-build-esp32:0.6.31 options: --user root steps: @@ -85,7 +85,7 @@ jobs: if: github.actor != 'restyled-io[bot]' container: - image: connectedhomeip/chip-build-nrf-platform:0.6.30 + image: connectedhomeip/chip-build-nrf-platform:0.6.31 options: --user root steps: diff --git a/.github/workflows/cirque.yaml b/.github/workflows/cirque.yaml index 47aa37c7fbf43c..742866f1be3061 100644 --- a/.github/workflows/cirque.yaml +++ b/.github/workflows/cirque.yaml @@ -29,7 +29,7 @@ jobs: timeout-minutes: 90 env: - DOCKER_RUN_VERSION: 0.6.30 + DOCKER_RUN_VERSION: 0.6.31 GITHUB_CACHE_PATH: /tmp/cirque-cache/ runs-on: ubuntu-latest @@ -38,7 +38,7 @@ jobs: # need to run with privilege, which isn't supported by job.XXX.contaner # https://github.com/actions/container-action/issues/2 # container: - # image: connectedhomeip/chip-build-cirque:0.6.30 + # image: connectedhomeip/chip-build-cirque:0.6.31 # volumes: # - "/tmp:/tmp" # - "/dev/pts:/dev/pts" diff --git a/.github/workflows/darwin-tests.yaml b/.github/workflows/darwin-tests.yaml index 7d1db5f479dc6c..4050aacefdebb0 100644 --- a/.github/workflows/darwin-tests.yaml +++ b/.github/workflows/darwin-tests.yaml @@ -39,6 +39,10 @@ jobs: env: BUILD_VARIANT: ${{matrix.build_variant}} LSAN_OPTIONS: detect_leaks=1 malloc_context_size=40 suppressions=scripts/tests/chiptest/lsan-mac-suppressions.txt + # NOTE: Generally kept in sync within the repo using + # scripts/tools/zap/version_update.py + ZAP_VERSION: v2023.01.09-nightly + ZAP_INSTALL_PATH: /usr/local/zap/zap-v2023.01.09-nightly if: github.actor != 'restyled-io[bot]' runs-on: macos-latest @@ -72,6 +76,16 @@ jobs: ls -la /usr/local/Cellar/openssl@1.1 OPEN_SSL_VERSION=`ls -la /usr/local/Cellar/openssl@1.1 | cat | tail -n1 | awk '{print $NF}'` ln -s /usr/local/Cellar/openssl@1.1/$OPEN_SSL_VERSION/lib/pkgconfig/* . + - name: Install zap + run: | + sudo mkdir -p $ZAP_INSTALL_PATH + sudo chown `whoami` $ZAP_INSTALL_PATH + curl -L https://github.com/project-chip/zap/releases/download/${ZAP_VERSION}/zap-mac.zip \ + --output $ZAP_INSTALL_PATH/zap-mac.zip + cd $ZAP_INSTALL_PATH + unzip zap-mac.zip zap-cli + rm zap-mac.zip + ./zap-cli --version - name: Bootstrap timeout-minutes: 25 run: scripts/build/gn_bootstrap.sh diff --git a/.github/workflows/darwin.yaml b/.github/workflows/darwin.yaml index a3eda2ef715b0b..0c47b619a69013 100644 --- a/.github/workflows/darwin.yaml +++ b/.github/workflows/darwin.yaml @@ -31,6 +31,12 @@ jobs: if: github.actor != 'restyled-io[bot]' runs-on: macos-latest + env: + # NOTE: Generally kept in sync within the repo using + # scripts/tools/zap/version_update.py + ZAP_VERSION: v2023.01.09-nightly + ZAP_INSTALL_PATH: /usr/local/zap/zap-v2023.01.09-nightly + steps: - uses: Wandalen/wretry.action@v1.0.36 name: Checkout @@ -52,6 +58,16 @@ jobs: ls -la /usr/local/Cellar/openssl@1.1 OPEN_SSL_VERSION=`ls -la /usr/local/Cellar/openssl@1.1 | cat | tail -n1 | awk '{print $NF}'` ln -s /usr/local/Cellar/openssl@1.1/$OPEN_SSL_VERSION/lib/pkgconfig/* . + - name: Install zap + run: | + sudo mkdir -p $ZAP_INSTALL_PATH + sudo chown `whoami` $ZAP_INSTALL_PATH + curl -L https://github.com/project-chip/zap/releases/download/${ZAP_VERSION}/zap-mac.zip \ + --output $ZAP_INSTALL_PATH/zap-mac.zip + cd $ZAP_INSTALL_PATH + unzip zap-mac.zip zap-cli + rm zap-mac.zip + ./zap-cli --version - name: Bootstrap timeout-minutes: 25 run: scripts/build/gn_bootstrap.sh diff --git a/.github/workflows/doxygen.yaml b/.github/workflows/doxygen.yaml index a0d90fb6dd7bf8..6814ed4d8f4823 100644 --- a/.github/workflows/doxygen.yaml +++ b/.github/workflows/doxygen.yaml @@ -82,7 +82,7 @@ jobs: runs-on: ubuntu-20.04 container: - image: connectedhomeip/chip-build-doxygen:0.6.30 + image: connectedhomeip/chip-build-doxygen:0.6.31 if: github.actor != 'restyled-io[bot]' diff --git a/.github/workflows/examples-ameba.yaml b/.github/workflows/examples-ameba.yaml index fe6ec75bc1d053..dc5d598a194da9 100644 --- a/.github/workflows/examples-ameba.yaml +++ b/.github/workflows/examples-ameba.yaml @@ -35,7 +35,7 @@ jobs: if: github.actor != 'restyled-io[bot]' container: - image: connectedhomeip/chip-build-ameba:0.6.30 + image: connectedhomeip/chip-build-ameba:0.6.31 options: --user root steps: diff --git a/.github/workflows/examples-bouffalolab.yaml b/.github/workflows/examples-bouffalolab.yaml index 450dda1c5e19e9..fd7afcc5a8db97 100644 --- a/.github/workflows/examples-bouffalolab.yaml +++ b/.github/workflows/examples-bouffalolab.yaml @@ -35,7 +35,7 @@ jobs: if: github.actor != 'restyled-io[bot]' container: - image: connectedhomeip/chip-build-bouffalolab:0.6.30 + image: connectedhomeip/chip-build-bouffalolab:0.6.31 volumes: - "/tmp/bloat_reports:/tmp/bloat_reports" steps: diff --git a/.github/workflows/examples-cc13x2x7_26x2x7.yaml b/.github/workflows/examples-cc13x2x7_26x2x7.yaml index 4d25db0057f90a..83ca902133eb1a 100644 --- a/.github/workflows/examples-cc13x2x7_26x2x7.yaml +++ b/.github/workflows/examples-cc13x2x7_26x2x7.yaml @@ -37,7 +37,7 @@ jobs: if: github.actor != 'restyled-io[bot]' container: - image: connectedhomeip/chip-build-ti:0.6.30 + image: connectedhomeip/chip-build-ti:0.6.31 volumes: - "/tmp/bloat_reports:/tmp/bloat_reports" steps: diff --git a/.github/workflows/examples-efr32.yaml b/.github/workflows/examples-efr32.yaml index 0e0f47a7190235..c88a2e43d53a7e 100644 --- a/.github/workflows/examples-efr32.yaml +++ b/.github/workflows/examples-efr32.yaml @@ -38,7 +38,7 @@ jobs: if: github.actor != 'restyled-io[bot]' container: - image: connectedhomeip/chip-build-efr32:0.6.30 + image: connectedhomeip/chip-build-efr32:0.6.31 volumes: - "/tmp/bloat_reports:/tmp/bloat_reports" steps: diff --git a/.github/workflows/examples-esp32.yaml b/.github/workflows/examples-esp32.yaml index 0dcf9ab1146b3f..6650825d090343 100644 --- a/.github/workflows/examples-esp32.yaml +++ b/.github/workflows/examples-esp32.yaml @@ -34,7 +34,7 @@ jobs: if: github.actor != 'restyled-io[bot]' container: - image: connectedhomeip/chip-build-esp32:0.6.30 + image: connectedhomeip/chip-build-esp32:0.6.31 volumes: - "/tmp/bloat_reports:/tmp/bloat_reports" @@ -93,6 +93,14 @@ jobs: build \ --copy-artifacts-to out/artifacts \ " + - name: Check no code generation in output + run: | + CNT=$(find -name "CHIPClusters.h" out | wc -l) + if [ "${CNT}" != "0" ]; then + echo "ERROR: found unexpected generated files:" + find -name "CHIPClusters.h" out + exit 1 + fi - name: Undo code pregeneration changes run: | rm -rf ./zzz_pregenerated @@ -140,7 +148,7 @@ jobs: if: github.actor != 'restyled-io[bot]' container: - image: connectedhomeip/chip-build-esp32:0.6.30 + image: connectedhomeip/chip-build-esp32:0.6.31 volumes: - "/tmp/bloat_reports:/tmp/bloat_reports" diff --git a/.github/workflows/examples-infineon.yaml b/.github/workflows/examples-infineon.yaml index 337812324e99df..9845f5f5e451ab 100644 --- a/.github/workflows/examples-infineon.yaml +++ b/.github/workflows/examples-infineon.yaml @@ -35,7 +35,7 @@ jobs: if: github.actor != 'restyled-io[bot]' container: - image: connectedhomeip/chip-build-infineon:0.6.30 + image: connectedhomeip/chip-build-infineon:0.6.31 volumes: - "/tmp/bloat_reports:/tmp/bloat_reports" steps: diff --git a/.github/workflows/examples-k32w.yaml b/.github/workflows/examples-k32w.yaml index f061bd6a3790e3..4d625b9705202a 100644 --- a/.github/workflows/examples-k32w.yaml +++ b/.github/workflows/examples-k32w.yaml @@ -37,7 +37,7 @@ jobs: if: github.actor != 'restyled-io[bot]' container: - image: connectedhomeip/chip-build-k32w:0.6.30 + image: connectedhomeip/chip-build-k32w:0.6.31 volumes: - "/tmp/bloat_reports:/tmp/bloat_reports" steps: diff --git a/.github/workflows/examples-linux-arm.yaml b/.github/workflows/examples-linux-arm.yaml index 070caf359cff5e..3a054918c60f87 100644 --- a/.github/workflows/examples-linux-arm.yaml +++ b/.github/workflows/examples-linux-arm.yaml @@ -34,7 +34,7 @@ jobs: if: github.actor != 'restyled-io[bot]' container: - image: connectedhomeip/chip-build-crosscompile:0.6.30 + image: connectedhomeip/chip-build-crosscompile:0.6.31 volumes: - "/tmp/bloat_reports:/tmp/bloat_reports" diff --git a/.github/workflows/examples-linux-imx.yaml b/.github/workflows/examples-linux-imx.yaml index 40df5727eba7e4..405aeec2845106 100644 --- a/.github/workflows/examples-linux-imx.yaml +++ b/.github/workflows/examples-linux-imx.yaml @@ -34,7 +34,7 @@ jobs: if: github.actor != 'restyled-io[bot]' container: - image: connectedhomeip/chip-build-imx:0.6.30 + image: connectedhomeip/chip-build-imx:0.6.31 steps: - uses: Wandalen/wretry.action@v1.0.36 diff --git a/.github/workflows/examples-linux-standalone.yaml b/.github/workflows/examples-linux-standalone.yaml index 5dc4b9e66f756f..ac40486ce1055c 100644 --- a/.github/workflows/examples-linux-standalone.yaml +++ b/.github/workflows/examples-linux-standalone.yaml @@ -34,7 +34,7 @@ jobs: if: github.actor != 'restyled-io[bot]' container: - image: connectedhomeip/chip-build:0.6.30 + image: connectedhomeip/chip-build:0.6.31 volumes: - "/tmp/bloat_reports:/tmp/bloat_reports" diff --git a/.github/workflows/examples-mbed.yaml b/.github/workflows/examples-mbed.yaml index 72d23e7deab915..313af85b2a4ce7 100644 --- a/.github/workflows/examples-mbed.yaml +++ b/.github/workflows/examples-mbed.yaml @@ -40,7 +40,7 @@ jobs: if: github.actor != 'restyled-io[bot]' container: - image: connectedhomeip/chip-build-mbed-os:0.6.30 + image: connectedhomeip/chip-build-mbed-os:0.6.31 volumes: - "/tmp/bloat_reports:/tmp/bloat_reports" diff --git a/.github/workflows/examples-mw320.yaml b/.github/workflows/examples-mw320.yaml index 8ca61ab37b4a96..4c187ef05ffaa5 100755 --- a/.github/workflows/examples-mw320.yaml +++ b/.github/workflows/examples-mw320.yaml @@ -37,7 +37,7 @@ jobs: if: github.actor != 'restyled-io[bot]' container: - image: connectedhomeip/chip-build:0.6.30 + image: connectedhomeip/chip-build:0.6.31 volumes: - "/tmp/bloat_reports:/tmp/bloat_reports" steps: diff --git a/.github/workflows/examples-nrfconnect.yaml b/.github/workflows/examples-nrfconnect.yaml index 12de9d4424a633..f776e537f7fac8 100644 --- a/.github/workflows/examples-nrfconnect.yaml +++ b/.github/workflows/examples-nrfconnect.yaml @@ -37,7 +37,7 @@ jobs: if: github.actor != 'restyled-io[bot]' container: - image: connectedhomeip/chip-build-nrf-platform:0.6.30 + image: connectedhomeip/chip-build-nrf-platform:0.6.31 volumes: - "/tmp/bloat_reports:/tmp/bloat_reports" diff --git a/.github/workflows/examples-openiotsdk.yaml b/.github/workflows/examples-openiotsdk.yaml index 70ca00b83bcb2c..d755b6c3bf63d1 100644 --- a/.github/workflows/examples-openiotsdk.yaml +++ b/.github/workflows/examples-openiotsdk.yaml @@ -35,7 +35,7 @@ jobs: if: github.actor != 'restyled-io[bot]' container: - image: connectedhomeip/chip-build-openiotsdk:0.6.30 + image: connectedhomeip/chip-build-openiotsdk:0.6.31 volumes: - "/tmp/bloat_reports:/tmp/bloat_reports" options: --privileged diff --git a/.github/workflows/examples-qpg.yaml b/.github/workflows/examples-qpg.yaml index b9d79b20c24d44..c585d124d307c1 100644 --- a/.github/workflows/examples-qpg.yaml +++ b/.github/workflows/examples-qpg.yaml @@ -37,7 +37,7 @@ jobs: if: github.actor != 'restyled-io[bot]' container: - image: connectedhomeip/chip-build:0.6.30 + image: connectedhomeip/chip-build:0.6.31 volumes: - "/tmp/bloat_reports:/tmp/bloat_reports" steps: diff --git a/.github/workflows/examples-telink.yaml b/.github/workflows/examples-telink.yaml index 06c7a149b240fa..4ed8e346b7127d 100644 --- a/.github/workflows/examples-telink.yaml +++ b/.github/workflows/examples-telink.yaml @@ -35,7 +35,7 @@ jobs: if: github.actor != 'restyled-io[bot]' container: - image: connectedhomeip/chip-build-telink:0.6.30 + image: connectedhomeip/chip-build-telink:0.6.31 volumes: - "/tmp/bloat_reports:/tmp/bloat_reports" diff --git a/.github/workflows/examples-tizen.yaml b/.github/workflows/examples-tizen.yaml index bf6ce9029e4e37..8d1fb9dc1b92c4 100644 --- a/.github/workflows/examples-tizen.yaml +++ b/.github/workflows/examples-tizen.yaml @@ -33,7 +33,7 @@ jobs: if: github.actor != 'restyled-io[bot]' container: - image: connectedhomeip/chip-build-tizen:0.6.30 + image: connectedhomeip/chip-build-tizen:0.6.31 options: --user root volumes: - "/tmp/bloat_reports:/tmp/bloat_reports" diff --git a/.github/workflows/full-android.yaml b/.github/workflows/full-android.yaml index ed8532ee765314..9d10c53dd298ac 100644 --- a/.github/workflows/full-android.yaml +++ b/.github/workflows/full-android.yaml @@ -37,7 +37,7 @@ jobs: if: github.actor != 'restyled-io[bot]' container: - image: connectedhomeip/chip-build-android:0.6.30 + image: connectedhomeip/chip-build-android:0.6.31 volumes: - "/tmp/log_output:/tmp/test_logs" diff --git a/.github/workflows/fuzzing-build.yaml b/.github/workflows/fuzzing-build.yaml index a6712c966022b5..57af4e5f358bda 100644 --- a/.github/workflows/fuzzing-build.yaml +++ b/.github/workflows/fuzzing-build.yaml @@ -34,7 +34,7 @@ jobs: if: github.actor != 'restyled-io[bot]' container: - image: connectedhomeip/chip-build:0.6.30 + image: connectedhomeip/chip-build:0.6.31 volumes: - "/tmp/log_output:/tmp/test_logs" @@ -88,6 +88,12 @@ jobs: runs-on: macos-latest if: github.actor != 'restyled-io[bot]' + env: + # NOTE: Generally kept in sync within the repo using + # scripts/tools/zap/version_update.py + ZAP_VERSION: v2023.01.09-nightly + ZAP_INSTALL_PATH: /usr/local/zap/zap-v2023.01.09-nightly + steps: - uses: Wandalen/wretry.action@v1.0.36 name: Checkout @@ -111,6 +117,16 @@ jobs: ls -la /usr/local/Cellar/openssl@1.1 OPEN_SSL_VERSION=`ls -la /usr/local/Cellar/openssl@1.1 | cat | tail -n1 | awk '{print $NF}'` ln -s /usr/local/Cellar/openssl@1.1/$OPEN_SSL_VERSION/lib/pkgconfig/* . + - name: Install zap + run: | + sudo mkdir -p $ZAP_INSTALL_PATH + sudo chown `whoami` $ZAP_INSTALL_PATH + curl -L https://github.com/project-chip/zap/releases/download/${ZAP_VERSION}/zap-mac.zip \ + --output $ZAP_INSTALL_PATH/zap-mac.zip + cd $ZAP_INSTALL_PATH + unzip zap-mac.zip zap-cli + rm zap-mac.zip + ./zap-cli --version - name: Bootstrap timeout-minutes: 25 run: scripts/build/gn_bootstrap.sh diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index 11bccad5b80828..0aba07866a6209 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -28,7 +28,7 @@ jobs: if: github.actor != 'restyled-io[bot]' container: - image: connectedhomeip/chip-build:0.6.27 + image: connectedhomeip/chip-build:0.6.31 steps: - uses: Wandalen/wretry.action@v1.0.36 diff --git a/.github/workflows/qemu.yaml b/.github/workflows/qemu.yaml index 3ad312ac5ea939..73f8937471e9a6 100644 --- a/.github/workflows/qemu.yaml +++ b/.github/workflows/qemu.yaml @@ -37,7 +37,7 @@ jobs: if: github.actor != 'restyled-io[bot]' container: - image: connectedhomeip/chip-build-esp32-qemu:0.6.30 + image: connectedhomeip/chip-build-esp32-qemu:0.6.31 volumes: - "/tmp/log_output:/tmp/test_logs" diff --git a/.github/workflows/release_artifacts.yaml b/.github/workflows/release_artifacts.yaml index b0fedd8c3a07eb..c709bfb23949a9 100644 --- a/.github/workflows/release_artifacts.yaml +++ b/.github/workflows/release_artifacts.yaml @@ -29,7 +29,7 @@ jobs: runs-on: ubuntu-latest container: - image: connectedhomeip/chip-build-esp32:0.6.30 + image: connectedhomeip/chip-build-esp32:0.6.31 steps: - uses: Wandalen/wretry.action@v1.0.36 @@ -75,7 +75,7 @@ jobs: runs-on: ubuntu-latest container: - image: connectedhomeip/chip-build-efr32:0.6.30 + image: connectedhomeip/chip-build-efr32:0.6.31 steps: - uses: Wandalen/wretry.action@v1.0.36 name: Checkout diff --git a/.github/workflows/smoketest-android.yaml b/.github/workflows/smoketest-android.yaml index 749d90e9522863..c7908fb810ff3c 100644 --- a/.github/workflows/smoketest-android.yaml +++ b/.github/workflows/smoketest-android.yaml @@ -37,7 +37,7 @@ jobs: if: github.actor != 'restyled-io[bot]' container: - image: connectedhomeip/chip-build-android:0.6.30 + image: connectedhomeip/chip-build-android:0.6.31 volumes: - "/tmp/log_output:/tmp/test_logs" diff --git a/.github/workflows/tests.yaml b/.github/workflows/tests.yaml index e9c77bf734a3e9..4231e7b0decea0 100644 --- a/.github/workflows/tests.yaml +++ b/.github/workflows/tests.yaml @@ -47,7 +47,7 @@ jobs: runs-on: ubuntu-latest container: - image: connectedhomeip/chip-build:0.6.30 + image: connectedhomeip/chip-build:0.6.31 options: --privileged --sysctl "net.ipv6.conf.all.disable_ipv6=0 net.ipv4.conf.all.forwarding=1 net.ipv6.conf.all.forwarding=1" @@ -260,6 +260,10 @@ jobs: CHIP_TOOL_VARIANT: ${{matrix.chip_tool}} TSAN_OPTIONS: "halt_on_error=1" LSAN_OPTIONS: detect_leaks=1 suppressions=scripts/tests/chiptest/lsan-mac-suppressions.txt + # NOTE: Generally kept in sync within the repo using + # scripts/tools/zap/version_update.py + ZAP_VERSION: v2023.01.09-nightly + ZAP_INSTALL_PATH: /usr/local/zap/zap-v2023.01.09-nightly if: github.actor != 'restyled-io[bot]' runs-on: macos-latest @@ -293,6 +297,15 @@ jobs: ls -la /usr/local/Cellar/openssl@1.1 OPEN_SSL_VERSION=`ls -la /usr/local/Cellar/openssl@1.1 | cat | tail -n1 | awk '{print $NF}'` ln -s /usr/local/Cellar/openssl@1.1/$OPEN_SSL_VERSION/lib/pkgconfig/* . + - name: Install zap + run: | + sudo mkdir -p $ZAP_INSTALL_PATH + sudo chown `whoami` $ZAP_INSTALL_PATH + curl -L https://github.com/project-chip/zap/releases/download/${ZAP_VERSION}/zap-mac.zip \ + --output $ZAP_INSTALL_PATH/zap-mac.zip + cd $ZAP_INSTALL_PATH + unzip zap-mac.zip zap-cli + rm zap-mac.zip - name: Bootstrap timeout-minutes: 25 run: scripts/build/gn_bootstrap.sh @@ -305,7 +318,7 @@ jobs: .environment/gn_out/.ninja_log .environment/pigweed-venv/*.log - name: Build Apps - timeout-minutes: 75 + timeout-minutes: 90 run: | ./scripts/run_in_build_env.sh \ "./scripts/build/build_examples.py \ @@ -370,7 +383,7 @@ jobs: runs-on: ubuntu-latest container: - image: connectedhomeip/chip-build:0.6.30 + image: connectedhomeip/chip-build:0.6.31 options: --privileged --sysctl "net.ipv6.conf.all.disable_ipv6=0 net.ipv4.conf.all.forwarding=0 net.ipv6.conf.all.forwarding=0" @@ -444,7 +457,7 @@ jobs: runs-on: ubuntu-latest container: - image: connectedhomeip/chip-build:0.6.30 + image: connectedhomeip/chip-build:0.6.31 options: --privileged --sysctl "net.ipv6.conf.all.disable_ipv6=0 net.ipv4.conf.all.forwarding=0 net.ipv6.conf.all.forwarding=0" @@ -531,6 +544,10 @@ jobs: env: BUILD_VARIANT: ${{matrix.build_variant}} TSAN_OPTIONS: "halt_on_error=1" + # NOTE: Generally kept in sync within the repo using + # scripts/tools/zap/version_update.py + ZAP_VERSION: v2023.01.09-nightly + ZAP_INSTALL_PATH: /usr/local/zap/zap-v2023.01.09-nightly if: github.actor != 'restyled-io[bot]' && false runs-on: macos-latest @@ -558,6 +575,15 @@ jobs: ls -la /usr/local/Cellar/openssl@1.1 OPEN_SSL_VERSION=`ls -la /usr/local/Cellar/openssl@1.1 | cat | tail -n1 | awk '{print $NF}'` ln -s /usr/local/Cellar/openssl@1.1/$OPEN_SSL_VERSION/lib/pkgconfig/* . + - name: Install zap + run: | + sudo mkdir -p $ZAP_INSTALL_PATH + sudo chown `whoami` $ZAP_INSTALL_PATH + curl -L https://github.com/project-chip/zap/releases/download/${ZAP_VERSION}/zap-mac.zip \ + --output $ZAP_INSTALL_PATH/zap-mac.zip + cd $ZAP_INSTALL_PATH + unzip zap-mac.zip zap-cli + rm zap-mac.zip - name: Bootstrap timeout-minutes: 25 run: scripts/build/gn_bootstrap.sh diff --git a/.github/workflows/unit_integration_test.yaml b/.github/workflows/unit_integration_test.yaml index 04e3ee8578a206..f502b52712b392 100644 --- a/.github/workflows/unit_integration_test.yaml +++ b/.github/workflows/unit_integration_test.yaml @@ -37,7 +37,7 @@ jobs: runs-on: ubuntu-latest container: - image: connectedhomeip/chip-build:0.6.30 + image: connectedhomeip/chip-build:0.6.31 volumes: - "/tmp/log_output:/tmp/test_logs" options: --privileged --sysctl "net.ipv6.conf.all.disable_ipv6=0 net.ipv4.conf.all.forwarding=1 net.ipv6.conf.all.forwarding=1" diff --git a/.github/workflows/zap_regeneration.yaml b/.github/workflows/zap_regeneration.yaml index c77bc0b2ee7b50..8d02e0ae40188f 100644 --- a/.github/workflows/zap_regeneration.yaml +++ b/.github/workflows/zap_regeneration.yaml @@ -44,9 +44,14 @@ jobs: token: ${{ github.token }} attempt_limit: 3 attempt_delay: 2000 + - name: Checkout submodules + run: scripts/checkout_submodules.py --shallow --platform linux + - name: Bootstrap + timeout-minutes: 10 + run: scripts/build/gn_bootstrap.sh - name: Generate all timeout-minutes: 5 - run: scripts/tools/zap_regen_all.py + run: ./scripts/run_in_build_env.sh scripts/tools/zap_regen_all.py - name: Ensure git works in current working directory run: git config --global --add safe.directory `pwd` - name: Add uncommitted changes diff --git a/.github/workflows/zap_templates.yaml b/.github/workflows/zap_templates.yaml index 6d319f73124359..496a3147d86b9e 100644 --- a/.github/workflows/zap_templates.yaml +++ b/.github/workflows/zap_templates.yaml @@ -44,8 +44,13 @@ jobs: token: ${{ github.token }} attempt_limit: 3 attempt_delay: 2000 + - name: Checkout submodules + run: scripts/checkout_submodules.py --shallow --platform linux + - name: Bootstrap + timeout-minutes: 10 + run: scripts/build/gn_bootstrap.sh - name: Generate all - run: scripts/tools/zap_regen_all.py + run: ./scripts/run_in_build_env.sh scripts/tools/zap_regen_all.py - name: Ensure git works in current working directory run: git config --global --add safe.directory `pwd` - name: Check for uncommited changes diff --git a/.restyled.yaml b/.restyled.yaml index 247249683cca2d..4bafb5c29ec2e8 100644 --- a/.restyled.yaml +++ b/.restyled.yaml @@ -70,6 +70,7 @@ exclude: - "src/controller/python/chip/clusters/Objects.py" # generated file, no point to restyle - "src/controller/python/chip/clusters/CHIPClusters.py" # generated file, no point to restyle - "scripts/py_matter_idl/matter_idl/tests/outputs/**/*" # Matches generated output 1:1 + - "scripts/tools/zap/tests/outputs/**/*" # Matches generated output 1:1 - "examples/chef/sample_app_util/test_files/*.yaml" - "examples/chef/zzz_generated/**/*" - "examples/platform/nxp/k32w/k32w0/scripts/demo_generated_certs/**/*" @@ -77,6 +78,7 @@ exclude: - "scripts/run_codegen_targets.sh" # shellharden breaks for loops over command outputs - "src/darwin/Framework/CHIP/zap-generated/MTRBaseClusters.h" # Too big? - "src/darwin/Framework/CHIP/zap-generated/MTRBaseClusters.mm" # Too big? + - "zzz_generated/**/*" # already clang-formatted by our zap tooling changed_paths: diff --git a/BUILD.gn b/BUILD.gn index ce3a154774d272..138a2c35784ec4 100644 --- a/BUILD.gn +++ b/BUILD.gn @@ -267,6 +267,10 @@ if (current_toolchain != "${dir_pw_toolchain}/default:default") { "//scripts/py_matter_yamltests:matter_yamltests.tests", "//src:tests_run", ] + + if (current_os == "linux" || current_os == "mac") { + deps += [ "${chip_root}/scripts/tools/zap:tests" ] + } } } } else { diff --git a/build/chip/chip_codegen.cmake b/build/chip/chip_codegen.cmake index 885d189f43f6de..dd7e4a645d92f9 100644 --- a/build/chip/chip_codegen.cmake +++ b/build/chip/chip_codegen.cmake @@ -92,6 +92,146 @@ function(chip_codegen TARGET_NAME) endforeach() + set(${ARG_OUTPUT_FILES} "${OUT_NAMES}" PARENT_SCOPE) + set(${ARG_OUTPUT_PATH} "${GEN_FOLDER}" PARENT_SCOPE) + + # allow adding dependencies to a phony target since no codegen is done + add_custom_target(${TARGET_NAME}) + endif() +endfunction() + +# Run chip code generation using zap +# +# Example usage: +# chip_codegen("app" +# INPUT "some_file.zap" +# GENERATOR "app-templates" +# OUTPUTS +# "zap-generated/access.h", +# "zap-generated/CHIPClientCallbacks.h", +# "zap-generated/CHIPClusters.h" +# "zap-generated/endpoint_config.h", +# "zap-generated/gen_config.h", +# "zap-generated/IMClusterCommandHandler.cpp" +# OUTPUT_PATH DIR_NAME_VAR +# OUTPUT_FILES FILE_NAMES_VAR +# ) +# +# Arguments: +# INPUT - the name of the ".zap" file to use for generation +# GENERATOR - generator to use, like "app-templates" +# OUTPUTS - EXPECTED output names +# +# OUTPUT_PATH - [OUT] output variable will contain the directory where the +# files will be generated +# +# OUTPUT_FILES - [OUT] output variable will contain the path of generated files. +# suitable to be added within a build target +# +function(chip_zapgen TARGET_NAME) + cmake_parse_arguments(ARG + "" + "INPUT;GENERATOR;OUTPUT_PATH;OUTPUT_FILES" + "OUTPUTS" + ${ARGN} + ) + + set(CHIP_CODEGEN_PREGEN_DIR "" CACHE PATH "Pre-generated directory to use instead of compile-time code generation.") + + if ("${CHIP_CODEGEN_PREGEN_DIR}" STREQUAL "") + set(GEN_FOLDER "${CMAKE_BINARY_DIR}/gen/${TARGET_NAME}/zapgen/${ARG_GENERATOR}") + + string(REPLACE ";" "\n" OUTPUT_AS_NEWLINES "${ARG_OUTPUTS}") + + file(MAKE_DIRECTORY "${GEN_FOLDER}") + file(GENERATE + OUTPUT "${GEN_FOLDER}/expected.outputs" + CONTENT "${OUTPUT_AS_NEWLINES}" + ) + + + set(OUT_NAMES) + foreach(NAME IN LISTS ARG_OUTPUTS) + list(APPEND OUT_NAMES "${GEN_FOLDER}/${NAME}") + endforeach() + + if ("${ARG_GENERATOR}" STREQUAL "app-templates") + SET(TEMPLATE_PATH "${CHIP_ROOT}/src/app/zap-templates/app-templates.json") + + # TODO: unclear how to maintain these: there is no parser that can figure + # out links of template files and zap files and such + SET(EXTRA_DEPENDENCIES + "${CHIP_ROOT}/src/app/zap-templates/partials/header.zapt" + "${CHIP_ROOT}/src/app/zap-templates/partials/idl/command_request_response.zapt" + "${CHIP_ROOT}/src/app/zap-templates/partials/idl/command_request_struct.zapt" + "${CHIP_ROOT}/src/app/zap-templates/partials/idl/command_response_struct.zapt" + "${CHIP_ROOT}/src/app/zap-templates/partials/idl/structure_definition.zapt" + "${CHIP_ROOT}/src/app/zap-templates/partials/idl/structure_member.zapt" + "${CHIP_ROOT}/src/app/zap-templates/partials/im_command_handler_cluster_commands.zapt" + "${CHIP_ROOT}/src/app/zap-templates/templates/app/access.zapt" + "${CHIP_ROOT}/src/app/zap-templates/templates/app/CHIPClientCallbacks.zapt" + "${CHIP_ROOT}/src/app/zap-templates/templates/app/CHIPClusters.zapt" + "${CHIP_ROOT}/src/app/zap-templates/templates/app/endpoint_config.zapt" + "${CHIP_ROOT}/src/app/zap-templates/templates/app/gen_config.zapt" + "${CHIP_ROOT}/src/app/zap-templates/templates/app/im-cluster-command-handler.zapt" + "${CHIP_ROOT}/src/app/zap-templates/templates/app/MatterIDL.zapt" + ) + SET(OUTPUT_SUBDIR "zap-generated") + else() + message(SEND_ERROR "Unsupported zap generator: ${ARG_GENERATOR}") + endif() + + # Python is expected to be in the path + # + # find_package(Python3 REQUIRED) + # + # TODO: lockfile support should be removed as this serializes zap + # (slower), however this is currently done because on Darwin zap startup + # may conflict and error out with: + # Error: EEXIST: file already exists, mkdir '/var/folders/24/8k48jl6d249_n_qfxwsl6xvm0000gn/T/pkg/465fcc8a6282e28dc7a166859d5814d34e2fb94249a72fa9229033b5b32dff1a' + add_custom_command( + OUTPUT ${OUT_NAMES} + COMMAND "${CHIP_ROOT}/scripts/tools/zap/generate.py" + ARGS + "--no-prettify-output" + "--templates" "${TEMPLATE_PATH}" + "--output-dir" "${GEN_FOLDER}/${OUTPUT_SUBDIR}" + "--lock-file" "${CMAKE_BINARY_DIR}/zap_gen.lock" + "--parallel" + "${ARG_INPUT}" + DEPENDS + "${ARG_INPUT}" + ${EXTRA_DEPENDENCIES} + VERBATIM + ) + + add_custom_target(${TARGET_NAME} DEPENDS "${OUT_NAMES}") + + # Forward outputs to the parent + set(${ARG_OUTPUT_FILES} "${OUT_NAMES}" PARENT_SCOPE) + set(${ARG_OUTPUT_PATH} "${GEN_FOLDER}" PARENT_SCOPE) + else() + # Gets a path such as: + # examples/lock-app/lock-common/lock-app.zap + file(RELATIVE_PATH MATTER_FILE_PATH "${CHIP_ROOT}" ${ARG_INPUT}) + + # Removes the trailing file extension to get something like: + # examples/lock-app/lock-common/lock-app + string(REGEX REPLACE "\.zap$" "" CODEGEN_DIR_PATH "${MATTER_FILE_PATH}") + + + # Build the final location within the pregen directory + set(GEN_FOLDER "${CHIP_CODEGEN_PREGEN_DIR}/${CODEGEN_DIR_PATH}/zap/${ARG_GENERATOR}") + + # TODO: build a fake target of ${TARGET_NAME} + + # Here we have ${CHIP_CODEGEN_PREGEN_DIR} + set(OUT_NAMES) + foreach(NAME IN LISTS ARG_OUTPUTS) + list(APPEND OUT_NAMES "${GEN_FOLDER}/${NAME}") + endforeach() + + set(${ARG_OUTPUT_FILES} "${OUT_NAMES}" PARENT_SCOPE) set(${ARG_OUTPUT_PATH} "${GEN_FOLDER}" PARENT_SCOPE) diff --git a/build/chip/chip_codegen.gni b/build/chip/chip_codegen.gni index d59277ae1e484d..49ba661f679e38 100644 --- a/build/chip/chip_codegen.gni +++ b/build/chip/chip_codegen.gni @@ -26,7 +26,7 @@ declare_args() { # Code generation that will happen at build time. # -# +# uses `scripts/codegen.py` for code generation. template("_chip_build_time_codegen") { _name = target_name _generator = invoker.generator @@ -93,6 +93,168 @@ template("_chip_build_time_codegen") { } } +# Code generation that will happen at build time. +# +# variables: +# input +# The ".zap" file to use to start the code generation +# +# generator +# Name of the generator to use. Supported variants: +# - "app-templates" +# +# +# +# deps, public_configs +# Forwarded to the resulting source set +# +# +# uses `zap` for code generation. +template("_chip_build_time_zapgen") { + _name = target_name + _generator = invoker.generator + + config("${_name}_config") { + include_dirs = [ "${target_gen_dir}/zapgen/" ] + } + + assert(_generator == "app-templates") + + if (_generator == "app-templates") { + _template_path = + rebase_path("${chip_root}/src/app/zap-templates/app-templates.json") + + _partials_dir = "${chip_root}/src/app/zap-templates/partials" + _template_dir = "${chip_root}/src/app/zap-templates/templates/app" + + # TODO: unclear how to maintain these: there is no parser that can figure + # out links of template files and zap files and such + _extra_dependencies = [ + "${_partials_dir}/header.zapt", + "${_partials_dir}/im_command_handler_cluster_commands.zapt", + + # Application templates, actually generating files + "${_template_dir}/access.zapt", + "${_template_dir}/CHIPClientCallbacks.zapt", + "${_template_dir}/CHIPClusters.zapt", + "${_template_dir}/endpoint_config.zapt", + "${_template_dir}/gen_config.zapt", + "${_template_dir}/im-cluster-command-handler.zapt", + ] + + _output_subdir = "zap-generated" + } + + pw_python_action("${_name}_zap_pregen") { + script = "${chip_root}/scripts/tools/zap/generate.py" + + # TODO: this seems to touch internals. Is this ok? speeds up builds! + _pw_internal_run_in_venv = false + + _idl_file = invoker.input + + args = [ + "--no-prettify-output", + "--templates", + _template_path, + "--output-dir", + rebase_path(target_gen_dir) + "/zap_pregen/" + _output_subdir, + + # TODO: lock file support should be removed as this serializes zap + # (slower), however this is currently done because on Darwin zap startup + # may conflict and error out with: + # Error: EEXIST: file already exists, mkdir '/var/folders/24/8k48jl6d249_n_qfxwsl6xvm0000gn/T/pkg/465fcc8a6282e28dc7a166859d5814d34e2fb94249a72fa9229033b5b32dff1a' + "--lock-file", + rebase_path("${root_out_dir}/zap_gen.lock"), + "--parallel", + _idl_file, + ] + + inputs = [ + _idl_file, + _template_path, + ] + inputs += _extra_dependencies + + # ensure any change in codegen files will result in a rebuild + inputs += matter_idl_generator_files + + sources = [ _idl_file ] + + outputs = [] + foreach(name, invoker.outputs) { + outputs += [ "${target_gen_dir}/zap_pregen/${name}" ] + } + + forward_variables_from(invoker, [ "prune_outputs" ]) + if (defined(prune_outputs)) { + foreach(name, prune_outputs) { + outputs += [ "${target_gen_dir}/zap_pregen/${name}" ] + } + } + } + + # This action ensures that any "extra" files generated by zap codegen + # are actually deleted. + # + # This is to avoid double-codegen of configurations like endpoint config + # or access credentials being generated for both "controller client" and + # application-specific + pw_python_action("${_name}_files") { + # TODO: this seems to touch internals. Is this ok? speeds up builds! + _pw_internal_run_in_venv = false + + script = "${chip_root}/scripts/tools/zap/prune_outputs.py" + + _keep_file = rebase_path("${target_gen_dir}/${_name}.keep.outputs") + write_file(_keep_file, invoker.outputs, "list lines") + + args = [ + "--keep", + _keep_file, + "--input-dir", + rebase_path("${target_gen_dir}/zap_pregen/"), + "--output-dir", + rebase_path("${target_gen_dir}/zapgen/"), + ] + + inputs = [] + foreach(name, invoker.outputs) { + inputs += [ "${target_gen_dir}/zap_pregen/${name}" ] + } + + outputs = [] + foreach(name, invoker.outputs) { + outputs += [ "${target_gen_dir}/zapgen/${name}" ] + } + + deps = [ ":${_name}_zap_pregen" ] + } + + source_set(_name) { + sources = [] + foreach(name, invoker.outputs) { + sources += [ "${target_gen_dir}/zapgen/${name}" ] + } + + public_configs = [ ":${_name}_config" ] + + if (defined(invoker.public_configs)) { + public_configs += invoker.public_configs + } + + forward_variables_from(invoker, [ "deps" ]) + + if (!defined(public_deps)) { + public_deps = [] + } + public_deps += [ + ":${_name}_files", + ":${_name}_zap_pregen", + ] + } +} + # Defines a target that runs code generation based on # scripts/codegen.py # @@ -158,7 +320,7 @@ template("chip_codegen") { } else { _name = target_name - # This contstructs a path like: + # This constructs a path like: # FROM all-clusters-app.matter (inside examples/all-clusters-app/all-clusters-common/) # USING "cpp-app" for generator: # => ${pregen_dir}/examples/all-clusters-app/all-clusters-common/all-clusters-app/codegen/cpp-app @@ -187,3 +349,143 @@ template("chip_codegen") { } } } + +# Defines a target that runs code generation based on +# scripts/codegen.py +# +# Arguments: +# input +# The ".matter" file to use to start the code generation +# +# generator +# Name of the generator to use (e.g. java, cpp-app) +# +# outputs +# Explicit names of the expected outputs. Enforced to validate that +# expected outputs are generated when processing input files. +# +# deps, public_configs +# Forwarded to the resulting source set +# +# Command line parameters: +# +# chip_code_pre_generated_directory: +# - If this is set, generation will NOT happen at compile time but rather +# the code generation is assumed to have already happened and reside in +# the given location. +# - The TOP LEVEL directory is assumed to be given. Actual location for +# individual generators is expected to be of the form +# // +# +# NOTE: content of "outputs" is verified to match the output of codegen.py +# exactly. It is not inferred on purpose, to make build-rules explicit +# and verifiable (even though codegen.py can at runtime report its outputs) +# +# To find the list of generated files, you can run codegen.py with the +# "--name-only" argument +# +# NOTE: +# the result of the target_name WILL BE a `source_set`. Treat it as such. +# +# Example usage: +# +# chip_codegen("java-jni-generate") { +# input = "controller-clusters.matter" +# generator = "java" +# +# outputs = [ +# "jni/IdentifyClient-ReadImpl.cpp", +# "jni/IdentifyClient-InvokeSubscribeImpl.cpp", +# # ... more to follow +# ] +# } +# +template("chip_zapgen") { + if (chip_code_pre_generated_directory == "") { + _chip_build_time_zapgen(target_name) { + forward_variables_from(invoker, + [ + "deps", + "generator", + "input", + "outputs", + "public_configs", + "prune_outputs", + ]) + } + } else { + _name = target_name + + # This contstructs a path like: + # FROM all-clusters-app.zap (inside examples/all-clusters-app/all-clusters-common/) + # USING "cpp-app" for generator: + # => ${pregen_dir}/examples/all-clusters-app/all-clusters-common/all-clusters-app/codegen/cpp-app + _generation_dir = + chip_code_pre_generated_directory + "/" + + string_replace(rebase_path(invoker.input, chip_root), ".zap", "") + + "/zap/" + invoker.generator + + config("${_name}_config") { + include_dirs = [ "${_generation_dir}" ] + } + + # Pick up only the headers and mark them available to use + # Specifically controller seems to require header files but NOT cpp (does) + # not want to include cpp compilation of IM command handler data + source_set("${_name}_headers") { + sources = [] + foreach(name, invoker.outputs) { + if (get_path_info(name, "extension") == "h") { + sources += [ "${_generation_dir}/${name}" ] + } + } + + # Ugly, but references WILL reference back into main code. + check_includes = false + } + + # need to have consistent naming. Make sure "files" exists + action("${_name}_files") { + script = "${chip_root}/scripts/tools/zap/check_file_existence.py" + + _output_name = "${target_gen_dir}/${_name}_files_checked.stamp" + + args = [ + "--touch", + rebase_path(_output_name), + ] + outputs = [ _output_name ] + + foreach(name, invoker.outputs) { + args += [ + "--exists", + rebase_path("${_generation_dir}/${name}"), + ] + } + + # Depending on the files gets access to the headers + public_deps = [ ":${_name}_headers" ] + } + + source_set(_name) { + forward_variables_from(invoker, + [ + "deps", + "public_configs", + "prune_outputs", + ]) + if (!defined(public_configs)) { + public_configs = [] + } + public_configs += [ ":${_name}_config" ] + + sources = [] + foreach(name, invoker.outputs) { + sources += [ "${_generation_dir}/${name}" ] + } + + # Ugly, but references WILL reference back into main code. + check_includes = false + } + } +} diff --git a/build/chip/esp32/esp32_codegen.cmake b/build/chip/esp32/esp32_codegen.cmake index f9e0479662b1a2..19c481c9cf9a92 100644 --- a/build/chip/esp32/esp32_codegen.cmake +++ b/build/chip/esp32/esp32_codegen.cmake @@ -15,8 +15,6 @@ # limitations under the License. # - - macro(chip_app_component_codegen IDL_NAME) include("${CHIP_ROOT}/build/chip/chip_codegen.cmake") @@ -39,10 +37,40 @@ macro(chip_app_component_codegen IDL_NAME) OUTPUT_FILES APP_GEN_FILES ) + add_dependencies(${COMPONENT_LIB} app-codegen) target_include_directories(${COMPONENT_LIB} PUBLIC "${APP_GEN_DIR}") + target_sources(${COMPONENT_LIB} PRIVATE ${APP_GEN_FILES}) + endif() +endmacro() - add_dependencies(${COMPONENT_LIB} app-codegen) +macro(chip_app_component_zapgen ZAP_NAME) + include("${CHIP_ROOT}/build/chip/chip_codegen.cmake") - target_sources(${COMPONENT_LIB} PRIVATE ${APP_GEN_FILES}) + # The IDF build system performs a two-pass expansion to determine + # component expansion. The first pass runs in script-mode + # to determine idf_component_register REQUIRES and PRIV_REQUIRES. + # + # We can only set up code generation during the 2nd pass + # + # see https://docs.espressif.com/projects/esp-idf/en/latest/esp32/api-guides/build-system.html + if (NOT CMAKE_BUILD_EARLY_EXPANSION) + + chip_zapgen(app-zapgen + INPUT "${ZAP_NAME}" + GENERATOR "app-templates" + OUTPUTS + "zap-generated/access.h" + "zap-generated/CHIPClientCallbacks.h" + "zap-generated/CHIPClusters.h" + "zap-generated/endpoint_config.h" + "zap-generated/gen_config.h" + "zap-generated/IMClusterCommandHandler.cpp" + OUTPUT_PATH APP_TEMPLATE_GEN_DIR + OUTPUT_FILES APP_TEMPLATE_GEN_FILES + ) + + add_dependencies(${COMPONENT_LIB} app-zapgen) + target_include_directories(${COMPONENT_LIB} PUBLIC "${APP_TEMPLATE_GEN_DIR}") + target_sources(${COMPONENT_LIB} PRIVATE ${APP_TEMPLATE_GEN_FILES}) endif() endmacro() diff --git a/config/mbed/CMakeLists.txt b/config/mbed/CMakeLists.txt index ecef938c0a8f07..b50f46e0acc2f3 100644 --- a/config/mbed/CMakeLists.txt +++ b/config/mbed/CMakeLists.txt @@ -492,11 +492,7 @@ if (NOT ${APP_TARGET} MATCHES "shell") ${CHIP_ROOT}/src/app/clusters/ota-requestor/ota-requestor-server.cpp ) else() - target_include_directories(${APP_TARGET} - PRIVATE ${GEN_DIR}/ota-requestor-app) - - target_sources(${APP_TARGET} - PRIVATE ${GEN_DIR}/ota-requestor-app/zap-generated/IMClusterCommandHandler.cpp) + target_include_directories(${APP_TARGET} PRIVATE ${GEN_DIR}/ota-requestor-app) endif() list(APPEND CHIP_DEFINES diff --git a/docs/code_generation.md b/docs/code_generation.md index c676fa887ebd7c..57dc512a1f7055 100644 --- a/docs/code_generation.md +++ b/docs/code_generation.md @@ -176,13 +176,13 @@ Additionally, individual code regeneration can be done using -o zzz_generated/bridge-app/zap-generated ``` -### `*.matter` code generation +### Compile-time code generation / pre-generated code -`*.matter` code generation can be done either at compile time or it can use -pre-generated output. +A subset of code generation (both `codegen.py` and `zap-cli`) is done at compile +time or can use pre-generated output (based on gn/cmake arguments) -Rules for how `codegen.py` is invoked and how includes/sources are set are -defined at: +Rules for how `generate.py`/`codegen.py` is invoked at compile time are defined +at: - `src/app/chip_data_model.cmake` - `src/app/chip_data_model.gni` @@ -201,9 +201,88 @@ Code pre-generation can be used: - To check changes in generated code across versions, beyond the comparisons of golden image tests in `scripts/py_matter_idl/matter_idl/tests` -The script to trigger code pre-generation is `scripts/code_pregenerate.py` and +The script to trigger code pre-generation is `scripts/codepregen.py` and requires the pre-generation output directory as an argument ```bash -scripts/code_pregenerate.py ${OUTPUT_DIRECTORY:-./zzz_pregenerated/} +scripts/codepregen.py ${OUTPUT_DIRECTORY:-./zzz_pregenerated/} + +# To generate a single output you can use `--input-glob`: + +scripts/codepregen.py --input-glob "*all-clusters*" ${OUTPUT_DIRECTORY:-./zzz_pregenerated/} +``` + +### Using pre-generated code + +Instead of generating code at compile time, the chip build system accepts usage +of a pre-generated folder. It assumes the structure that `codepregen.py` +creates. To invoke use: + +- `build_examples.py` builds accept `--pregen-dir` as an argument, such as: + + ```shell + ./scripts/build/build_examples.py --target $TARGET --pregen-dir $PREGEN_DIR build + ``` + +- `gn` builds allow setting `chip_code_pre_generated_directory` as an + argument, such as: + + ```shell + gn gen --check --fail-on-unused-args --args='chip_code_pre_generated_directory="/some/pregen/dir"' + ``` + +- `cmake` builds allow setting `CHIP_CODEGEN_PREGEN_DIR` variable (which will + get propagated to the underlying `gn` builds as needed), such as: + + ```shell + + west build --cmake-only \ + -d /workspace/out/nrf-nrf5340dk-light \ + -b nrf5340dk_nrf5340_cpuapp \ + /workspace/examples/lighting-app/nrfconnect + -- -DCHIP_CODEGEN_PREGEN_DIR=/some/pregen/dir + + idf.py -C examples/all-clusters-app/esp32 \ + -B /workspace/out/esp32-m5stack-all-clusters \ + -DCHIP_CODEGEN_PREGEN_DIR=/some/pregen/dir \ + reconfigure + + cmake -S /workspace/examples/lighting-app/mbed \ + -B /workspace/out/mbed-cy8cproto_062_4343w-light \ + -GNinja \ + -DMBED_OS_PATH=/workspace/third_party/mbed-os/repo \ + -DMBED_OS_PATH=/workspace/third_party/mbed-os/repo \ + -DMBED_OS_POSIX_SOCKET_PATH=/workspace/third_party/mbed-os-posix-socket/repo \ + -DCHIP_CODEGEN_PREGEN_DIR=/some/pregen/dir + ``` + +### Code generation unit testing + +Code generation is assumed stable between builds and the build system aims to +detect changes in code gen using golden image tests. + +#### `codegen.py` tests + +These tests run against golden inputs/outputs from `scripts/idl/tests`. + +`available_tests.yaml` contains the full list of expected generators and outputs +and the test is run via `test_generators.py`. Use the environment variable +`IDL_GOLDEN_REGENERATE` to force golden image replacement during running of +`ninja check`: + +```shell +IDL_GOLDEN_REGENERATE=1 ninja check +``` + +#### `generate.py` tests + +These tests run against golden inputs/outputs from `scripts/tools/zap/tests`. + +`available_tests.yaml` contains the full list of expected generators and outputs +and the test is run via `scripts/tools/zap/test_generate.py`. Use the +environment variable `ZAP_GENERATE_GOLDEN_REGENERATE` to force golden image +replacement during running of `ninja check`. + +```shell +ZAP_GENERATE_GOLDEN_REGENERATE=1 ninja check ``` diff --git a/examples/all-clusters-app/ameba/chip_main.cmake b/examples/all-clusters-app/ameba/chip_main.cmake index 7241e44892de85..df0cb9c4064032 100755 --- a/examples/all-clusters-app/ameba/chip_main.cmake +++ b/examples/all-clusters-app/ameba/chip_main.cmake @@ -149,8 +149,6 @@ endif (matter_enable_ota_requestor) list( APPEND ${list_chip_main_sources} - ${chip_dir}/zzz_generated/all-clusters-app/zap-generated/IMClusterCommandHandler.cpp - ${chip_dir}/examples/all-clusters-app/all-clusters-common/src/bridged-actions-stub.cpp ${chip_dir}/examples/all-clusters-app/all-clusters-common/src/static-supported-modes-manager.cpp diff --git a/examples/all-clusters-app/esp32/main/CMakeLists.txt b/examples/all-clusters-app/esp32/main/CMakeLists.txt index 615e07525e4290..a4003980dedde0 100644 --- a/examples/all-clusters-app/esp32/main/CMakeLists.txt +++ b/examples/all-clusters-app/esp32/main/CMakeLists.txt @@ -141,6 +141,7 @@ get_filename_component(CHIP_ROOT ${CMAKE_SOURCE_DIR}/third_party/connectedhomeip include("${CHIP_ROOT}/build/chip/esp32/esp32_codegen.cmake") chip_app_component_codegen("${CHIP_ROOT}/examples/all-clusters-app/all-clusters-common/all-clusters-app.matter") +chip_app_component_zapgen("${CHIP_ROOT}/examples/all-clusters-app/all-clusters-common/all-clusters-app.zap") set_property(TARGET ${COMPONENT_LIB} PROPERTY CXX_STANDARD 17) target_compile_options(${COMPONENT_LIB} PRIVATE "-DCHIP_HAVE_CONFIG_H") diff --git a/examples/all-clusters-app/mbed/CMakeLists.txt b/examples/all-clusters-app/mbed/CMakeLists.txt index 53ad3823244089..a654994ddea9c6 100644 --- a/examples/all-clusters-app/mbed/CMakeLists.txt +++ b/examples/all-clusters-app/mbed/CMakeLists.txt @@ -57,7 +57,6 @@ target_include_directories(${APP_TARGET} PRIVATE target_sources(${APP_TARGET} PRIVATE main/main.cpp main/AppTask.cpp - ${GEN_DIR}/all-clusters-app/zap-generated/IMClusterCommandHandler.cpp ${MBED_COMMON}/util/LEDWidget.cpp ${MBED_COMMON}/util/DFUManager.cpp ${ALL_CLUSTERS_COMMON}/src/bridged-actions-stub.cpp diff --git a/examples/all-clusters-app/nrfconnect/CMakeLists.txt b/examples/all-clusters-app/nrfconnect/CMakeLists.txt index abcd8fb5060011..fc50ade57e9c96 100644 --- a/examples/all-clusters-app/nrfconnect/CMakeLists.txt +++ b/examples/all-clusters-app/nrfconnect/CMakeLists.txt @@ -60,7 +60,6 @@ target_sources(app PRIVATE ${ALL_CLUSTERS_COMMON_DIR}/src/static-supported-modes-manager.cpp ${ALL_CLUSTERS_COMMON_DIR}/src/bridged-actions-stub.cpp ${ALL_CLUSTERS_COMMON_DIR}/src/binding-handler.cpp - ${GEN_DIR}/all-clusters-app/zap-generated/IMClusterCommandHandler.cpp ${NRFCONNECT_COMMON}/util/LEDWidget.cpp) chip_configure_data_model(app diff --git a/examples/all-clusters-app/telink/CMakeLists.txt b/examples/all-clusters-app/telink/CMakeLists.txt index 6e9dd95f0e84ec..9b981dfc1766f9 100644 --- a/examples/all-clusters-app/telink/CMakeLists.txt +++ b/examples/all-clusters-app/telink/CMakeLists.txt @@ -53,7 +53,6 @@ target_sources(app PRIVATE ${ALL_CLUSTERS_COMMON_DIR}/src/static-supported-modes-manager.cpp ${ALL_CLUSTERS_COMMON_DIR}/src/bridged-actions-stub.cpp ${ALL_CLUSTERS_COMMON_DIR}/src/binding-handler.cpp - ${GEN_DIR}/all-clusters-app/zap-generated/IMClusterCommandHandler.cpp ${TELINK_COMMON}/util/src/LEDWidget.cpp ${TELINK_COMMON}/util/src/ButtonManager.cpp ${TELINK_COMMON}/util/src/ThreadUtil.cpp) diff --git a/examples/all-clusters-app/tizen/BUILD.gn b/examples/all-clusters-app/tizen/BUILD.gn index 9a9957fff11de4..ba1cf3a3ca51bd 100644 --- a/examples/all-clusters-app/tizen/BUILD.gn +++ b/examples/all-clusters-app/tizen/BUILD.gn @@ -45,13 +45,13 @@ executable("chip-all-clusters-app") { deps = [ ":chip-all-clusters-common", + "${chip_root}/examples/all-clusters-app/all-clusters-common", "${chip_root}/examples/platform/tizen:app-main", "${chip_root}/src/lib", ] include_dirs = [ "${chip_root}/examples/all-clusters-app/all-clusters-common/include", - "${chip_root}/zzz_generated/all-clusters-app", "include", ] diff --git a/examples/all-clusters-minimal-app/ameba/chip_main.cmake b/examples/all-clusters-minimal-app/ameba/chip_main.cmake index 29563def1566a3..d78344f713497e 100755 --- a/examples/all-clusters-minimal-app/ameba/chip_main.cmake +++ b/examples/all-clusters-minimal-app/ameba/chip_main.cmake @@ -128,8 +128,6 @@ endif (matter_enable_ota_requestor) list( APPEND ${list_chip_main_sources} - ${chip_dir}/zzz_generated/all-clusters-minimal-app/zap-generated/IMClusterCommandHandler.cpp - ${chip_dir}/examples/all-clusters-minimal-app/all-clusters-common/src/bridged-actions-stub.cpp ${chip_dir}/examples/all-clusters-minimal-app/all-clusters-common/src/static-supported-modes-manager.cpp diff --git a/examples/all-clusters-minimal-app/esp32/main/CMakeLists.txt b/examples/all-clusters-minimal-app/esp32/main/CMakeLists.txt index a51bc9cdf120de..4994beed15155e 100644 --- a/examples/all-clusters-minimal-app/esp32/main/CMakeLists.txt +++ b/examples/all-clusters-minimal-app/esp32/main/CMakeLists.txt @@ -132,6 +132,7 @@ get_filename_component(CHIP_ROOT ${CMAKE_SOURCE_DIR}/third_party/connectedhomeip include("${CHIP_ROOT}/build/chip/esp32/esp32_codegen.cmake") chip_app_component_codegen("${CHIP_ROOT}/examples/all-clusters-minimal-app/all-clusters-common/all-clusters-minimal-app.matter") +chip_app_component_zapgen("${CHIP_ROOT}/examples/all-clusters-minimal-app/all-clusters-common/all-clusters-minimal-app.zap") set_property(TARGET ${COMPONENT_LIB} PROPERTY CXX_STANDARD 17) target_compile_options(${COMPONENT_LIB} PRIVATE "-DCHIP_HAVE_CONFIG_H") diff --git a/examples/all-clusters-minimal-app/mbed/CMakeLists.txt b/examples/all-clusters-minimal-app/mbed/CMakeLists.txt index d4ffe4cc103c15..dc503a4e9b5970 100644 --- a/examples/all-clusters-minimal-app/mbed/CMakeLists.txt +++ b/examples/all-clusters-minimal-app/mbed/CMakeLists.txt @@ -57,7 +57,6 @@ target_include_directories(${APP_TARGET} PRIVATE target_sources(${APP_TARGET} PRIVATE main/main.cpp main/AppTask.cpp - ${GEN_DIR}/all-clusters-minimal-app/zap-generated/IMClusterCommandHandler.cpp ${MBED_COMMON}/util/LEDWidget.cpp ${MBED_COMMON}/util/DFUManager.cpp ${ALL_CLUSTERS_COMMON}/src/bridged-actions-stub.cpp diff --git a/examples/all-clusters-minimal-app/nrfconnect/CMakeLists.txt b/examples/all-clusters-minimal-app/nrfconnect/CMakeLists.txt index 7097d08a4bb375..6fdc49b494018e 100644 --- a/examples/all-clusters-minimal-app/nrfconnect/CMakeLists.txt +++ b/examples/all-clusters-minimal-app/nrfconnect/CMakeLists.txt @@ -58,7 +58,6 @@ target_sources(app PRIVATE ${ALL_CLUSTERS_COMMON_DIR}/src/static-supported-modes-manager.cpp ${ALL_CLUSTERS_COMMON_DIR}/src/bridged-actions-stub.cpp ${ALL_CLUSTERS_COMMON_DIR}/src/binding-handler.cpp - ${GEN_DIR}/all-clusters-minimal-app/zap-generated/IMClusterCommandHandler.cpp ${NRFCONNECT_COMMON}/util/LEDWidget.cpp) chip_configure_data_model(app diff --git a/examples/all-clusters-minimal-app/telink/CMakeLists.txt b/examples/all-clusters-minimal-app/telink/CMakeLists.txt index 9f0c60f81af439..0661e4038e31cd 100644 --- a/examples/all-clusters-minimal-app/telink/CMakeLists.txt +++ b/examples/all-clusters-minimal-app/telink/CMakeLists.txt @@ -52,7 +52,6 @@ target_sources(app PRIVATE ${ALL_CLUSTERS_COMMON_DIR}/src/static-supported-modes-manager.cpp ${ALL_CLUSTERS_COMMON_DIR}/src/bridged-actions-stub.cpp ${ALL_CLUSTERS_COMMON_DIR}/src/binding-handler.cpp - ${GEN_DIR}/all-clusters-minimal-app/zap-generated/IMClusterCommandHandler.cpp ${TELINK_COMMON}/util/src/LEDWidget.cpp ${TELINK_COMMON}/util/src/ButtonManager.cpp ${TELINK_COMMON}/util/src/ThreadUtil.cpp) diff --git a/examples/all-clusters-minimal-app/tizen/BUILD.gn b/examples/all-clusters-minimal-app/tizen/BUILD.gn index 9b081bc565b22d..138c6e9c2e6850 100644 --- a/examples/all-clusters-minimal-app/tizen/BUILD.gn +++ b/examples/all-clusters-minimal-app/tizen/BUILD.gn @@ -45,13 +45,13 @@ executable("chip-all-clusters-minimal-app") { deps = [ ":chip-all-clusters-common", + "${chip_root}/examples/all-clusters-minimal-app/all-clusters-common", "${chip_root}/examples/platform/tizen:app-main", "${chip_root}/src/lib", ] include_dirs = [ "${chip_root}/examples/all-clusters-app/all-clusters-common/include", - "${chip_root}/zzz_generated/all-clusters-minimal-app", "include", ] diff --git a/examples/bridge-app/esp32/main/CMakeLists.txt b/examples/bridge-app/esp32/main/CMakeLists.txt index 31365efaf2f14d..62b57d8d3fb2e7 100644 --- a/examples/bridge-app/esp32/main/CMakeLists.txt +++ b/examples/bridge-app/esp32/main/CMakeLists.txt @@ -58,6 +58,7 @@ get_filename_component(CHIP_ROOT ${CMAKE_SOURCE_DIR}/third_party/connectedhomeip include("${CHIP_ROOT}/build/chip/esp32/esp32_codegen.cmake") chip_app_component_codegen("${CHIP_ROOT}/examples/bridge-app/bridge-common/bridge-app.matter") +chip_app_component_zapgen("${CHIP_ROOT}/examples/bridge-app/bridge-common/bridge-app.zap") set_property(TARGET ${COMPONENT_LIB} PROPERTY CXX_STANDARD 14) target_compile_options(${COMPONENT_LIB} PRIVATE "-DCHIP_HAVE_CONFIG_H") diff --git a/examples/chef/ameba/chip_main.cmake b/examples/chef/ameba/chip_main.cmake index b4f4c0318eba1f..c92f1e0b471eb0 100755 --- a/examples/chef/ameba/chip_main.cmake +++ b/examples/chef/ameba/chip_main.cmake @@ -35,8 +35,6 @@ endif (matter_enable_ota_requestor) list( APPEND ${list_chip_main_sources} - ${chip_dir}/examples/chef/out/${SAMPLE_NAME}/zap-generated/IMClusterCommandHandler.cpp - ${matter_example_path}//main/chipinterface.cpp ${matter_example_path}//main/DeviceCallbacks.cpp ${matter_example_path}//main/CHIPDeviceManager.cpp diff --git a/examples/chef/esp32/main/CMakeLists.txt b/examples/chef/esp32/main/CMakeLists.txt index 00a46722388cc9..cf32321bb54baa 100644 --- a/examples/chef/esp32/main/CMakeLists.txt +++ b/examples/chef/esp32/main/CMakeLists.txt @@ -109,6 +109,7 @@ idf_component_register(PRIV_INCLUDE_DIRS include("${CHIP_ROOT}/build/chip/esp32/esp32_codegen.cmake") chip_app_component_codegen("${CHEF}/devices/${SAMPLE_NAME}.matter") +chip_app_component_zapgen("${CHEF}/devices/${SAMPLE_NAME}.zap") set_property(TARGET ${COMPONENT_LIB} PROPERTY CXX_STANDARD 17) target_compile_options(${COMPONENT_LIB} PRIVATE "-DCHIP_HAVE_CONFIG_H") diff --git a/examples/chef/nrfconnect/CMakeLists.txt b/examples/chef/nrfconnect/CMakeLists.txt index 408b079921e6bf..a5473bed795eca 100644 --- a/examples/chef/nrfconnect/CMakeLists.txt +++ b/examples/chef/nrfconnect/CMakeLists.txt @@ -86,14 +86,12 @@ endif (CONFIG_ENABLE_CHIP_SHELL) target_sources(app PRIVATE ${CHEF}/nrfconnect/main.cpp ${CHEF}/common/stubs.cpp - ${GEN_DIR}/IMClusterCommandHandler.cpp ) message(STATUS ${CHEF}/devices/${SAMPLE_NAME}.zap) chip_configure_data_model(app INCLUDE_SERVER ZAP_FILE ${CHEF}/devices/${SAMPLE_NAME}.zap - GEN_DIR ${GEN_DIR} ) include(${CHIP_ROOT}/config/nrfconnect/app/flashing.cmake) diff --git a/examples/contact-sensor-app/telink/CMakeLists.txt b/examples/contact-sensor-app/telink/CMakeLists.txt index b75ad4764331b4..a2ddd3d332163f 100755 --- a/examples/contact-sensor-app/telink/CMakeLists.txt +++ b/examples/contact-sensor-app/telink/CMakeLists.txt @@ -49,7 +49,6 @@ target_sources(app PRIVATE src/ContactSensorManager.cpp src/main.cpp src/ZclCallbacks.cpp - ${GEN_DIR}/contact-sensor-app/zap-generated/IMClusterCommandHandler.cpp ${TELINK_COMMON}/util/src/LEDWidget.cpp ${TELINK_COMMON}/util/src/ButtonManager.cpp ${TELINK_COMMON}/util/src/ThreadUtil.cpp) diff --git a/examples/light-switch-app/ameba/chip_main.cmake b/examples/light-switch-app/ameba/chip_main.cmake index 19f592d882929f..4a23725ae121e4 100755 --- a/examples/light-switch-app/ameba/chip_main.cmake +++ b/examples/light-switch-app/ameba/chip_main.cmake @@ -149,8 +149,6 @@ endif (matter_enable_ota_requestor) list( APPEND ${list_chip_main_sources} - ${chip_dir}/zzz_generated/light-switch-app/zap-generated/IMClusterCommandHandler.cpp - ${chip_dir}/examples/light-switch-app/ameba/main/chipinterface.cpp ${chip_dir}/examples/light-switch-app/ameba/main/BindingHandler.cpp ${chip_dir}/examples/light-switch-app/ameba/main/DeviceCallbacks.cpp diff --git a/examples/light-switch-app/esp32/main/CMakeLists.txt b/examples/light-switch-app/esp32/main/CMakeLists.txt index 5e52ded45aeed1..9fecd318807163 100644 --- a/examples/light-switch-app/esp32/main/CMakeLists.txt +++ b/examples/light-switch-app/esp32/main/CMakeLists.txt @@ -65,6 +65,7 @@ get_filename_component(CHIP_ROOT ${CMAKE_SOURCE_DIR}/third_party/connectedhomeip include("${CHIP_ROOT}/build/chip/esp32/esp32_codegen.cmake") chip_app_component_codegen("${CHIP_ROOT}/examples/light-switch-app/light-switch-common/light-switch-app.matter") +chip_app_component_zapgen("${CHIP_ROOT}/examples/light-switch-app/light-switch-common/light-switch-app.zap") set_property(TARGET ${COMPONENT_LIB} PROPERTY CXX_STANDARD 17) target_compile_options(${COMPONENT_LIB} PRIVATE "-DCHIP_HAVE_CONFIG_H") diff --git a/examples/light-switch-app/nrfconnect/CMakeLists.txt b/examples/light-switch-app/nrfconnect/CMakeLists.txt index 58c380e4d83c9f..9e0f317f948fab 100644 --- a/examples/light-switch-app/nrfconnect/CMakeLists.txt +++ b/examples/light-switch-app/nrfconnect/CMakeLists.txt @@ -59,7 +59,6 @@ target_sources(app PRIVATE main/LightSwitch.cpp main/ShellCommands.cpp main/BindingHandler.cpp - ${GEN_DIR}/light-switch-app/zap-generated/IMClusterCommandHandler.cpp ${NRFCONNECT_COMMON}/util/LEDWidget.cpp) diff --git a/examples/light-switch-app/telink/CMakeLists.txt b/examples/light-switch-app/telink/CMakeLists.txt index 40b9490510d47a..3f67e533375932 100755 --- a/examples/light-switch-app/telink/CMakeLists.txt +++ b/examples/light-switch-app/telink/CMakeLists.txt @@ -50,7 +50,6 @@ target_sources(app PRIVATE src/main.cpp src/ZclCallbacks.cpp src/binding-handler.cpp - ${GEN_DIR}/light-switch-app/zap-generated/IMClusterCommandHandler.cpp ${TELINK_COMMON}/util/src/LEDWidget.cpp ${TELINK_COMMON}/util/src/ButtonManager.cpp ${TELINK_COMMON}/util/src/ThreadUtil.cpp) diff --git a/examples/lighting-app/ameba/chip_main.cmake b/examples/lighting-app/ameba/chip_main.cmake index db7ec9beec2200..0a71784057ee32 100755 --- a/examples/lighting-app/ameba/chip_main.cmake +++ b/examples/lighting-app/ameba/chip_main.cmake @@ -141,8 +141,6 @@ endif (matter_enable_ota_requestor) list( APPEND ${list_chip_main_sources} - ${chip_dir}/zzz_generated/lighting-app/zap-generated/IMClusterCommandHandler.cpp - ${chip_dir}/examples/lighting-app/lighting-common/src/ColorFormat.cpp ${chip_dir}/examples/lighting-app/ameba/main/chipinterface.cpp diff --git a/examples/lighting-app/esp32/main/CMakeLists.txt b/examples/lighting-app/esp32/main/CMakeLists.txt index bd4914505a7e4d..c1f1e30767e455 100644 --- a/examples/lighting-app/esp32/main/CMakeLists.txt +++ b/examples/lighting-app/esp32/main/CMakeLists.txt @@ -95,6 +95,7 @@ get_filename_component(CHIP_ROOT ${CMAKE_SOURCE_DIR}/third_party/connectedhomeip include("${CHIP_ROOT}/build/chip/esp32/esp32_codegen.cmake") chip_app_component_codegen("${CHIP_ROOT}/examples/lighting-app/lighting-common/lighting-app.matter") +chip_app_component_zapgen("${CHIP_ROOT}/examples/lighting-app/lighting-common/lighting-app.zap") set_property(TARGET ${COMPONENT_LIB} PROPERTY CXX_STANDARD 17) target_compile_options(${COMPONENT_LIB} PRIVATE "-DCHIP_HAVE_CONFIG_H") diff --git a/examples/lighting-app/mbed/CMakeLists.txt b/examples/lighting-app/mbed/CMakeLists.txt index 58c75c9d691efb..abe63789638f23 100644 --- a/examples/lighting-app/mbed/CMakeLists.txt +++ b/examples/lighting-app/mbed/CMakeLists.txt @@ -57,7 +57,6 @@ target_sources(${APP_TARGET} PRIVATE main/LightingManager.cpp main/main.cpp main/ZclCallbacks.cpp - ${GEN_DIR}/lighting-app/zap-generated/IMClusterCommandHandler.cpp ${MBED_COMMON}/util/LEDWidget.cpp ${MBED_COMMON}/util/DFUManager.cpp ) diff --git a/examples/lighting-app/nrfconnect/CMakeLists.txt b/examples/lighting-app/nrfconnect/CMakeLists.txt index 81e2bd7f42c0cc..94996c75ab7255 100644 --- a/examples/lighting-app/nrfconnect/CMakeLists.txt +++ b/examples/lighting-app/nrfconnect/CMakeLists.txt @@ -59,14 +59,12 @@ target_sources(app PRIVATE main/AppTask.cpp main/main.cpp main/ZclCallbacks.cpp - ${GEN_DIR}/lighting-app/zap-generated/IMClusterCommandHandler.cpp ${NRFCONNECT_COMMON}/util/LEDWidget.cpp ${NRFCONNECT_COMMON}/util/PWMDevice.cpp) chip_configure_data_model(app INCLUDE_SERVER ZAP_FILE ${CMAKE_CURRENT_SOURCE_DIR}/../lighting-common/lighting-app.zap - GEN_DIR ${GEN_DIR}/lighting-app/zap-generated ) if(CONFIG_CHIP_OTA_REQUESTOR OR CONFIG_MCUMGR_SMP_BT) diff --git a/examples/lighting-app/telink/CMakeLists.txt b/examples/lighting-app/telink/CMakeLists.txt index 5beea781359c18..a917c8d6ea6b36 100644 --- a/examples/lighting-app/telink/CMakeLists.txt +++ b/examples/lighting-app/telink/CMakeLists.txt @@ -50,7 +50,6 @@ target_sources(app PRIVATE src/LightingManager.cpp src/main.cpp src/ZclCallbacks.cpp - ${GEN_DIR}/lighting-app/zap-generated/IMClusterCommandHandler.cpp ${TELINK_COMMON}/util/src/LEDWidget.cpp ${TELINK_COMMON}/util/src/ButtonManager.cpp ${TELINK_COMMON}/util/src/ThreadUtil.cpp) @@ -58,7 +57,6 @@ target_sources(app PRIVATE chip_configure_data_model(app INCLUDE_SERVER ZAP_FILE ${CMAKE_CURRENT_SOURCE_DIR}/../lighting-common/lighting-app.zap - GEN_DIR ${GEN_DIR}/lighting-app/zap-generated ) if(CONFIG_CHIP_OTA_REQUESTOR) diff --git a/examples/lock-app/esp32/main/CMakeLists.txt b/examples/lock-app/esp32/main/CMakeLists.txt index 85c931431c2d00..8dd4bc1dc0a774 100644 --- a/examples/lock-app/esp32/main/CMakeLists.txt +++ b/examples/lock-app/esp32/main/CMakeLists.txt @@ -202,3 +202,5 @@ get_filename_component(CHIP_ROOT ${CMAKE_SOURCE_DIR}/third_party/connectedhomeip include("${CHIP_ROOT}/build/chip/esp32/esp32_codegen.cmake") chip_app_component_codegen("${CHIP_ROOT}/examples/lock-app/lock-common/lock-app.matter") +chip_app_component_zapgen("${CHIP_ROOT}/examples/lock-app/lock-common/lock-app.zap") + diff --git a/examples/lock-app/mbed/CMakeLists.txt b/examples/lock-app/mbed/CMakeLists.txt index 794eece035c8c2..29cb86631ab749 100644 --- a/examples/lock-app/mbed/CMakeLists.txt +++ b/examples/lock-app/mbed/CMakeLists.txt @@ -57,7 +57,6 @@ target_sources(${APP_TARGET} PRIVATE main/BoltLockManager.cpp main/main.cpp main/ZclCallbacks.cpp - ${GEN_DIR}/lock-app/zap-generated/IMClusterCommandHandler.cpp ${MBED_COMMON}/util/LEDWidget.cpp ${MBED_COMMON}/util/DFUManager.cpp ) diff --git a/examples/lock-app/nrfconnect/CMakeLists.txt b/examples/lock-app/nrfconnect/CMakeLists.txt index 5c9bd36fbc31c6..7e7dc3ce49b4f2 100644 --- a/examples/lock-app/nrfconnect/CMakeLists.txt +++ b/examples/lock-app/nrfconnect/CMakeLists.txt @@ -57,7 +57,6 @@ target_sources(app PRIVATE main/BoltLockManager.cpp main/main.cpp main/ZclCallbacks.cpp - ${GEN_DIR}/lock-app/zap-generated/IMClusterCommandHandler.cpp ${NRFCONNECT_COMMON}/util/LEDWidget.cpp) chip_configure_data_model(app diff --git a/examples/lock-app/openiotsdk/CMakeLists.txt b/examples/lock-app/openiotsdk/CMakeLists.txt index 67308591faa7a0..1e0125b99c0497 100644 --- a/examples/lock-app/openiotsdk/CMakeLists.txt +++ b/examples/lock-app/openiotsdk/CMakeLists.txt @@ -68,7 +68,6 @@ target_sources(${APP_TARGET} main/ZclCallbacks.cpp main/LockManager.cpp main/LockEndpoint.cpp - ${GEN_DIR}/lock-app/zap-generated/IMClusterCommandHandler.cpp ) target_link_libraries(${APP_TARGET} diff --git a/examples/ota-provider-app/esp32/main/CMakeLists.txt b/examples/ota-provider-app/esp32/main/CMakeLists.txt index 10d5eadde50d1c..8c58d0b4b4fc18 100644 --- a/examples/ota-provider-app/esp32/main/CMakeLists.txt +++ b/examples/ota-provider-app/esp32/main/CMakeLists.txt @@ -61,6 +61,7 @@ idf_component_register(PRIV_INCLUDE_DIRS get_filename_component(CHIP_ROOT ${CMAKE_SOURCE_DIR}/third_party/connectedhomeip REALPATH) include("${CHIP_ROOT}/build/chip/esp32/esp32_codegen.cmake") chip_app_component_codegen("${CHIP_ROOT}/examples/ota-provider-app/ota-provider-common/ota-provider-app.matter") +chip_app_component_zapgen("${CHIP_ROOT}/examples/ota-provider-app/ota-provider-common/ota-provider-app.zap") spiffs_create_partition_image(img_storage ${CMAKE_SOURCE_DIR}/spiffs_image FLASH_IN_PROJECT) set_property(TARGET ${COMPONENT_LIB} PROPERTY CXX_STANDARD 14) diff --git a/examples/ota-requestor-app/ameba/chip_main.cmake b/examples/ota-requestor-app/ameba/chip_main.cmake index b977c848c541ea..11db9b502b8001 100644 --- a/examples/ota-requestor-app/ameba/chip_main.cmake +++ b/examples/ota-requestor-app/ameba/chip_main.cmake @@ -13,8 +13,6 @@ include(${prj_root}/GCC-RELEASE/project_hp/asdk/includepath.cmake) list( APPEND ${list_chip_main_sources} - ${chip_dir}/zzz_generated/ota-requestor-app/zap-generated/IMClusterCommandHandler.cpp - ${chip_dir}/examples/ota-requestor-app/ameba/main/chipinterface.cpp ${chip_dir}/examples/ota-requestor-app/ameba/main/Globals.cpp ${chip_dir}/examples/ota-requestor-app/ameba/main/LEDWidget.cpp diff --git a/examples/ota-requestor-app/esp32/main/CMakeLists.txt b/examples/ota-requestor-app/esp32/main/CMakeLists.txt index 306707421ec134..cdd50a0b220c1b 100644 --- a/examples/ota-requestor-app/esp32/main/CMakeLists.txt +++ b/examples/ota-requestor-app/esp32/main/CMakeLists.txt @@ -89,6 +89,7 @@ idf_component_register(PRIV_INCLUDE_DIRS ${PRIV_INCLUDE_DIRS_LIST} include("${CHIP_ROOT}/build/chip/esp32/esp32_codegen.cmake") chip_app_component_codegen("${CHIP_ROOT}/examples/ota-requestor-app/ota-requestor-common/ota-requestor-app.matter") +chip_app_component_zapgen("${CHIP_ROOT}/examples/ota-requestor-app/ota-requestor-common/ota-requestor-app.zap") set_property(TARGET ${COMPONENT_LIB} PROPERTY CXX_STANDARD 17) target_compile_options(${COMPONENT_LIB} PRIVATE "-DCHIP_HAVE_CONFIG_H") diff --git a/examples/ota-requestor-app/mbed/CMakeLists.txt b/examples/ota-requestor-app/mbed/CMakeLists.txt index 742f8415fbc2cc..d377b0ee60b6b3 100644 --- a/examples/ota-requestor-app/mbed/CMakeLists.txt +++ b/examples/ota-requestor-app/mbed/CMakeLists.txt @@ -55,7 +55,6 @@ target_include_directories(${APP_TARGET} PRIVATE target_sources(${APP_TARGET} PRIVATE main/main.cpp main/AppTask.cpp - ${GEN_DIR}/ota-requestor-app/zap-generated/IMClusterCommandHandler.cpp ${MBED_COMMON}/util/LEDWidget.cpp ${MBED_COMMON}/util/DFUManager.cpp ) @@ -63,7 +62,6 @@ target_sources(${APP_TARGET} PRIVATE chip_configure_data_model(${APP_TARGET} INCLUDE_SERVER ZAP_FILE ${CHIP_ROOT}/examples/ota-requestor-app/ota-requestor-common/ota-requestor-app.zap - GEN_DIR ${CHIP_ROOT}/zzz_generated/ota-requestor-app/zap-generated ) target_link_libraries(${APP_TARGET} mbed-os-posix-socket mbed-os mbed-ble mbed-events mbed-netsocket mbed-storage mbed-storage-kv-global-api mbed-mbedtls mbed-emac chip) diff --git a/examples/ota-requestor-app/telink/CMakeLists.txt b/examples/ota-requestor-app/telink/CMakeLists.txt index 6b634ce3eb5dbc..2a44136882b383 100644 --- a/examples/ota-requestor-app/telink/CMakeLists.txt +++ b/examples/ota-requestor-app/telink/CMakeLists.txt @@ -49,7 +49,6 @@ target_sources(app PRIVATE src/AppTask.cpp src/main.cpp src/ZclCallbacks.cpp - ${GEN_DIR}/ota-requestor-app/zap-generated/IMClusterCommandHandler.cpp ${TELINK_COMMON}/util/src/LEDWidget.cpp ${TELINK_COMMON}/util/src/ButtonManager.cpp ${TELINK_COMMON}/util/src/ThreadUtil.cpp) diff --git a/examples/pump-app/nrfconnect/CMakeLists.txt b/examples/pump-app/nrfconnect/CMakeLists.txt index 87f561d54f9242..59ac3654a27d29 100644 --- a/examples/pump-app/nrfconnect/CMakeLists.txt +++ b/examples/pump-app/nrfconnect/CMakeLists.txt @@ -56,7 +56,6 @@ target_sources(app PRIVATE main/PumpManager.cpp main/main.cpp main/ZclCallbacks.cpp - ${GEN_DIR}/pump-app/zap-generated/IMClusterCommandHandler.cpp ${NRFCONNECT_COMMON}/util/LEDWidget.cpp) chip_configure_data_model(app diff --git a/examples/pump-controller-app/nrfconnect/CMakeLists.txt b/examples/pump-controller-app/nrfconnect/CMakeLists.txt index 32da45fd205f85..3534eb8444a4fb 100644 --- a/examples/pump-controller-app/nrfconnect/CMakeLists.txt +++ b/examples/pump-controller-app/nrfconnect/CMakeLists.txt @@ -56,7 +56,6 @@ target_sources(app PRIVATE main/PumpManager.cpp main/main.cpp main/ZclCallbacks.cpp - ${GEN_DIR}/pump-controller-app/zap-generated/IMClusterCommandHandler.cpp ${NRFCONNECT_COMMON}/util/LEDWidget.cpp) chip_configure_data_model(app diff --git a/examples/shell/mbed/CMakeLists.txt b/examples/shell/mbed/CMakeLists.txt index e58007e9c7549a..7a8dc44aa73407 100644 --- a/examples/shell/mbed/CMakeLists.txt +++ b/examples/shell/mbed/CMakeLists.txt @@ -65,7 +65,6 @@ target_sources(${APP_TARGET} PRIVATE chip_configure_data_model(${APP_TARGET} ZAP_FILE ${CHIP_ROOT}/examples/ota-requestor-app/ota-requestor-common/ota-requestor-app.zap - GEN_DIR ${CHIP_ROOT}/zzz_generated/ota-requestor-app/zap-generated ) target_link_libraries(${APP_TARGET} mbed-os-posix-socket mbed-os mbed-ble mbed-events mbed-netsocket mbed-storage mbed-storage-kv-global-api mbed-mbedtls mbed-emac chip) diff --git a/examples/temperature-measurement-app/esp32/main/CMakeLists.txt b/examples/temperature-measurement-app/esp32/main/CMakeLists.txt index cae8e844be7978..43f968137df88f 100644 --- a/examples/temperature-measurement-app/esp32/main/CMakeLists.txt +++ b/examples/temperature-measurement-app/esp32/main/CMakeLists.txt @@ -81,6 +81,7 @@ idf_component_register(PRIV_INCLUDE_DIRS ${PRIV_INCLUDE_DIRS_LIST} include("${CHIP_ROOT}/build/chip/esp32/esp32_codegen.cmake") chip_app_component_codegen("${CHIP_ROOT}/examples/temperature-measurement-app/esp32/main/temperature-measurement.matter") +chip_app_component_zapgen("${CHIP_ROOT}/examples/temperature-measurement-app/esp32/main/temperature-measurement.zap") set_property(TARGET ${COMPONENT_LIB} PROPERTY CXX_STANDARD 17) target_compile_options(${COMPONENT_LIB} PRIVATE "-DCHIP_HAVE_CONFIG_H") diff --git a/examples/thermostat/telink/CMakeLists.txt b/examples/thermostat/telink/CMakeLists.txt index 6d00c892a2ed72..dd1399f54ab19f 100755 --- a/examples/thermostat/telink/CMakeLists.txt +++ b/examples/thermostat/telink/CMakeLists.txt @@ -51,7 +51,6 @@ target_sources(app PRIVATE src/TemperatureManager.cpp src/main.cpp src/ZclCallbacks.cpp - ${GEN_DIR}/thermostat/zap-generated/IMClusterCommandHandler.cpp ${TELINK_COMMON}/util/src/LEDWidget.cpp ${TELINK_COMMON}/util/src/ButtonManager.cpp ${TELINK_COMMON}/util/src/ThreadUtil.cpp) diff --git a/examples/tv-casting-app/tv-casting-common/include/CastingServer.h b/examples/tv-casting-app/tv-casting-common/include/CastingServer.h index 7dfbfc7ce057d4..efa95615dc7c4b 100644 --- a/examples/tv-casting-app/tv-casting-common/include/CastingServer.h +++ b/examples/tv-casting-app/tv-casting-common/include/CastingServer.h @@ -36,7 +36,7 @@ #include #include #include -#include +#include #include constexpr chip::System::Clock::Seconds16 kCommissioningWindowTimeout = chip::System::Clock::Seconds16(3 * 60); diff --git a/examples/window-app/nrfconnect/CMakeLists.txt b/examples/window-app/nrfconnect/CMakeLists.txt index 3f36551a8982c5..92b43dc05fc779 100644 --- a/examples/window-app/nrfconnect/CMakeLists.txt +++ b/examples/window-app/nrfconnect/CMakeLists.txt @@ -58,7 +58,6 @@ target_sources(app PRIVATE main/main.cpp main/ZclCallbacks.cpp main/WindowCovering.cpp - ${GEN_DIR}/window-app/zap-generated/IMClusterCommandHandler.cpp ${NRFCONNECT_COMMON}/util/LEDWidget.cpp ${NRFCONNECT_COMMON}/util/PWMDevice.cpp) diff --git a/examples/window-app/silabs/efr32/include/LcdPainter.h b/examples/window-app/silabs/efr32/include/LcdPainter.h index 4a30bdede2c3ea..d0740167bdd06c 100644 --- a/examples/window-app/silabs/efr32/include/LcdPainter.h +++ b/examples/window-app/silabs/efr32/include/LcdPainter.h @@ -19,8 +19,8 @@ #pragma once // clang-format off -#include "app-common/app-common/zap-generated/enums.h" -#include "app-common/app-common/zap-generated/cluster-enums.h" +#include "app-common/zap-generated/enums.h" +#include "app-common/zap-generated/cluster-enums.h" // clang-format on #include diff --git a/integrations/cloudbuild/build-all.yaml b/integrations/cloudbuild/build-all.yaml index 4c4df463b3b69a..ad228b056e05a0 100644 --- a/integrations/cloudbuild/build-all.yaml +++ b/integrations/cloudbuild/build-all.yaml @@ -6,7 +6,7 @@ steps: - "--init" - "--recursive" id: Submodules - - name: "connectedhomeip/chip-build-vscode:0.6.30" + - name: "connectedhomeip/chip-build-vscode:0.6.31" env: - PW_ENVIRONMENT_ROOT=/pwenv args: @@ -21,7 +21,7 @@ steps: path: /pwenv timeout: 900s - - name: "connectedhomeip/chip-build-vscode:0.6.30" + - name: "connectedhomeip/chip-build-vscode:0.6.31" env: - PW_ENVIRONMENT_ROOT=/pwenv args: @@ -76,7 +76,7 @@ steps: --target k32w-shell build --create-archives /workspace/artifacts/ - - name: "connectedhomeip/chip-build-vscode:0.6.30" + - name: "connectedhomeip/chip-build-vscode:0.6.31" env: - PW_ENVIRONMENT_ROOT=/pwenv args: diff --git a/integrations/cloudbuild/chef.yaml b/integrations/cloudbuild/chef.yaml index fc614000279964..da5edd7dadb6ae 100644 --- a/integrations/cloudbuild/chef.yaml +++ b/integrations/cloudbuild/chef.yaml @@ -1,5 +1,5 @@ steps: - - name: "connectedhomeip/chip-build-vscode:0.6.30" + - name: "connectedhomeip/chip-build-vscode:0.6.31" env: - PW_ENVIRONMENT_ROOT=/pwenv args: @@ -12,7 +12,7 @@ steps: path: /pwenv timeout: 2700s - - name: "connectedhomeip/chip-build-vscode:0.6.30" + - name: "connectedhomeip/chip-build-vscode:0.6.31" env: - PW_ENVIRONMENT_ROOT=/pwenv args: @@ -26,7 +26,7 @@ steps: - name: pwenv path: /pwenv - - name: "connectedhomeip/chip-build-vscode:0.6.30" + - name: "connectedhomeip/chip-build-vscode:0.6.31" env: - PW_ENVIRONMENT_ROOT=/pwenv args: diff --git a/integrations/cloudbuild/smoke-test.yaml b/integrations/cloudbuild/smoke-test.yaml index db2dd9c0269c66..059f29ccb72da8 100644 --- a/integrations/cloudbuild/smoke-test.yaml +++ b/integrations/cloudbuild/smoke-test.yaml @@ -1,5 +1,5 @@ steps: - - name: "connectedhomeip/chip-build-vscode:0.6.30" + - name: "connectedhomeip/chip-build-vscode:0.6.31" entrypoint: "bash" args: - "-c" @@ -7,7 +7,7 @@ steps: git config --global --add safe.directory "*" git submodule update --init --recursive id: Submodules - - name: "connectedhomeip/chip-build-vscode:0.6.30" + - name: "connectedhomeip/chip-build-vscode:0.6.31" env: - PW_ENVIRONMENT_ROOT=/pwenv args: @@ -22,7 +22,7 @@ steps: path: /pwenv timeout: 900s - - name: "connectedhomeip/chip-build-vscode:0.6.30" + - name: "connectedhomeip/chip-build-vscode:0.6.31" id: ESP32 env: - PW_ENVIRONMENT_ROOT=/pwenv @@ -40,7 +40,7 @@ steps: volumes: - name: pwenv path: /pwenv - - name: "connectedhomeip/chip-build-vscode:0.6.30" + - name: "connectedhomeip/chip-build-vscode:0.6.31" id: NRFConnect env: - PW_ENVIRONMENT_ROOT=/pwenv @@ -61,7 +61,7 @@ steps: - name: pwenv path: /pwenv - - name: "connectedhomeip/chip-build-vscode:0.6.30" + - name: "connectedhomeip/chip-build-vscode:0.6.31" id: EFR32 env: - PW_ENVIRONMENT_ROOT=/pwenv @@ -83,7 +83,7 @@ steps: - name: pwenv path: /pwenv - - name: "connectedhomeip/chip-build-vscode:0.6.30" + - name: "connectedhomeip/chip-build-vscode:0.6.31" id: Linux env: - PW_ENVIRONMENT_ROOT=/pwenv @@ -141,7 +141,7 @@ steps: - name: pwenv path: /pwenv - - name: "connectedhomeip/chip-build-vscode:0.6.30" + - name: "connectedhomeip/chip-build-vscode:0.6.31" id: Android env: - PW_ENVIRONMENT_ROOT=/pwenv diff --git a/integrations/docker/images/chip-build/Dockerfile b/integrations/docker/images/chip-build/Dockerfile index fee9a70dbc726a..eb47a434e3c37c 100644 --- a/integrations/docker/images/chip-build/Dockerfile +++ b/integrations/docker/images/chip-build/Dockerfile @@ -178,6 +178,11 @@ RUN set -x \ # Install a known ZAP release # Only keep the cli version, since `zap` is 143MB and not usable (UI) +# +# NOTE: zap version occurs in several files (CI and docker) +# for a uniform update, a separate update script exists: +# +# scripts/tools/zap/version_update.py ENV ZAP_VERSION=v2023.01.09-nightly RUN set -x \ && mkdir -p /opt/zap-${ZAP_VERSION} \ diff --git a/scripts/build/builders/efr32.py b/scripts/build/builders/efr32.py index 412726aa6a708c..eda3c4c348c9f4 100644 --- a/scripts/build/builders/efr32.py +++ b/scripts/build/builders/efr32.py @@ -241,6 +241,13 @@ def generate(self): cmd += ['--dotfile=%s' % self.dotfile] extra_args = self.GnBuildArgs() + + if self.options.pw_command_launcher: + extra_args.append('pw_command_launcher="%s"' % self.options.pw_command_launcher) + + if self.options.pregen_dir: + extra_args.append('chip_code_pre_generated_directory="%s"' % self.options.pregen_dir) + if extra_args: cmd += ['--args=%s' % ' '.join(extra_args)] diff --git a/scripts/codepregen.py b/scripts/codepregen.py index 2464e7c122106a..6344c9a38142a9 100755 --- a/scripts/codepregen.py +++ b/scripts/codepregen.py @@ -24,11 +24,14 @@ import click try: - from pregenerate import FindPregenerationTargets + from pregenerate import FindPregenerationTargets, TargetFilter except: import os sys.path.append(os.path.abspath(os.path.dirname(__file__))) - from pregenerate import FindPregenerationTargets + from pregenerate import FindPregenerationTargets, TargetFilter + +from pregenerate.executors import DryRunner, ShellRunner +from pregenerate.types import IdlFileType try: import coloredlogs @@ -64,12 +67,26 @@ def _ParallelGenerateOne(arg): '--parallel/--no-parallel', default=True, help='Do parallel/multiprocessing codegen.') +@click.option( + '--dry-run/--no-dry-run', + default=False, + help='Do not actually execute commands, just log') +@click.option( + '--generator', + default='all', + type=click.Choice(['all', 'zap', 'codegen']), + help='To what code generator to restrict the generation.') +@click.option( + '--input-glob', + default=None, + multiple=True, + help='Restrict file generation inputs to the specified glob patterns.') @click.option( '--sdk-root', default=None, help='Path to the SDK root (where .zap/.matter files exist).') @click.argument('output_dir') -def main(log_level, parallel, sdk_root, output_dir): +def main(log_level, parallel, dry_run, generator, input_glob, sdk_root, output_dir): if _has_coloredlogs: coloredlogs.install(level=__LOG_LEVELS__[ log_level], fmt='%(asctime)s %(levelname)-7s %(message)s') @@ -93,11 +110,21 @@ def main(log_level, parallel, sdk_root, output_dir): logging.info(f"Pre-generating {sdk_root} data into {output_dir}") - if not os.path.exists(output_dir): - os.makedirs(output_dir) + if not dry_run: + runner = ShellRunner() + else: + runner = DryRunner() + + filter = TargetFilter(path_glob=input_glob) + + if generator == 'zap': + filter.file_type = IdlFileType.ZAP + elif generator == 'codegen': + filter.file_type = IdlFileType.MATTER - targets = FindPregenerationTargets(sdk_root) + targets = FindPregenerationTargets(sdk_root, filter, runner) + runner.ensure_directory_exists(output_dir) if parallel: target_and_dir = zip(targets, itertools.repeat(output_dir)) with multiprocessing.Pool() as pool: diff --git a/scripts/pregenerate/__init__.py b/scripts/pregenerate/__init__.py index eab3205dc9d981..c5033f406fdf7c 100644 --- a/scripts/pregenerate/__init__.py +++ b/scripts/pregenerate/__init__.py @@ -13,17 +13,15 @@ # limitations under the License. +import fnmatch import logging import os +from dataclasses import dataclass, field +from typing import Iterator, List, Optional -from typing import Iterator - -from .types import InputIdlFile, IdlFileType - - -from .pregenerators import CodegenJavaPregenerator -from .pregenerators import CodegenBridgePregenerator -from .pregenerators import CodegenCppAppPregenerator +from .types import IdlFileType, InputIdlFile +from .using_codegen import CodegenBridgePregenerator, CodegenCppAppPregenerator, CodegenJavaPregenerator +from .using_zap import ZapApplicationPregenerator def FindAllIdls(sdk_root: str) -> Iterator[InputIdlFile]: @@ -49,7 +47,26 @@ def FindAllIdls(sdk_root: str) -> Iterator[InputIdlFile]: relative_path=os.path.join(root[sdk_root_length+1:], file)) -def FindPregenerationTargets(sdk_root: str): +@dataclass +class TargetFilter: + # If set, only the specified files are accepted for codegen + file_type: Optional[IdlFileType] = None + + # If non-empty only the given paths will be code-generated + path_glob: List[str] = field(default_factory=list) + + +# TODO: the build GlobMatcher is more complete by supporting `{}` grouping +# For now this limited glob seems sufficient. +class GlobMatcher: + def __init__(self, pattern: str): + self.pattern = pattern + + def matches(self, s: str): + return fnmatch.fnmatch(s, self.pattern) + + +def FindPregenerationTargets(sdk_root: str, filter: TargetFilter, runner): """Finds all relevand pre-generation targets in the given SDK root. @@ -58,12 +75,28 @@ def FindPregenerationTargets(sdk_root: str): """ generators = [ + # Jinja-based codegen CodegenBridgePregenerator(sdk_root), CodegenJavaPregenerator(sdk_root), CodegenCppAppPregenerator(sdk_root), + + # ZAP codegen + ZapApplicationPregenerator(sdk_root), ] + path_matchers = [GlobMatcher(pattern) for pattern in filter.path_glob] + for idl in FindAllIdls(sdk_root): + if filter.file_type is not None: + if idl.file_type != filter.file_type: + logging.debug(f"Will not process file of type {idl.file_type}: {idl.relative_path}") + continue + + if path_matchers: + if all([not matcher.matches(idl.relative_path) for matcher in path_matchers]): + logging.debug(f"Glob not matched for {idl.relative_path}") + continue + for generator in generators: if generator.Accept(idl): - yield generator.CreateTarget(idl) + yield generator.CreateTarget(idl, runner=runner) diff --git a/scripts/pregenerate/executors.py b/scripts/pregenerate/executors.py new file mode 100644 index 00000000000000..2b5e0e03f70084 --- /dev/null +++ b/scripts/pregenerate/executors.py @@ -0,0 +1,38 @@ +# Copyright (c) 2022 Project CHIP Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import logging +import os +import shlex +import subprocess + + +class ShellRunner: + def run(self, cmd, cwd=None): + logging.debug(f"Executing {cmd}") + subprocess.check_call(cmd, cwd=cwd) + + def ensure_directory_exists(self, dir): + os.makedirs(dir, exist_ok=True) + + +class DryRunner: + def run(self, cmd, cwd=None): + if cwd: + logging.info(f"DRY-RUN: {shlex.join(cmd)} in {cwd}") + else: + logging.info(f"DRY-RUN: {shlex.join(cmd)}") + + def ensure_directory_exists(self, dir): + logging.info(f"DRY-RUN: mkdir {dir}") diff --git a/scripts/pregenerate/pregenerators.py b/scripts/pregenerate/using_codegen.py similarity index 78% rename from scripts/pregenerate/pregenerators.py rename to scripts/pregenerate/using_codegen.py index c3b05b3bdda354..0ca159704b6f4a 100644 --- a/scripts/pregenerate/pregenerators.py +++ b/scripts/pregenerate/using_codegen.py @@ -16,29 +16,35 @@ import os import shlex import subprocess +from enum import Enum, auto from .types import IdlFileType, InputIdlFile -CODEGEN_PY_PATH = os.path.join(os.path.dirname(__file__), '..', 'codegen.py') +CODEGEN_PY_PATH = os.path.abspath(os.path.join( + os.path.dirname(__file__), '..', 'codegen.py')) class CodegenTarget: """A target that uses `scripts/codegen.py` to generate files.""" - def __init__(self, idl: InputIdlFile, generator: str, sdk_root: str): + def __init__(self, idl: InputIdlFile, generator: str, sdk_root: str, runner): self.idl = idl self.generator = generator self.sdk_root = sdk_root + self.runner = runner if idl.file_type != IdlFileType.MATTER: - raise Exception(f"Can only code generate for `*.matter` input files, not for {idl}") + raise Exception( + f"Can only code generate for `*.matter` input files, not for {idl}") def Generate(self, output_root: str): '''Runs codegen.py to generate in the specified directory''' - output_dir = os.path.join(output_root, self.idl.pregen_subdir, self.generator) + output_dir = os.path.join( + output_root, self.idl.pregen_subdir, self.generator) - logging.info(f"Generating: {self.generator}:{self.idl.relative_path} into {output_dir}") + logging.info( + f"Generating: {self.generator}:{self.idl.relative_path} into {output_dir}") cmd = [ CODEGEN_PY_PATH, @@ -49,7 +55,7 @@ def Generate(self, output_root: str): ] logging.debug(f"Executing {cmd}") - subprocess.check_call(cmd) + self.runner.run(cmd) class CodegenBridgePregenerator: @@ -63,8 +69,8 @@ def Accept(self, idl: InputIdlFile): # bridge codegen return idl.relative_path == "examples/dynamic-bridge-app/bridge-common/bridge-app.matter" - def CreateTarget(self, idl: InputIdlFile): - return CodegenTarget(sdk_root=self.sdk_root, idl=idl, generator="bridge") + def CreateTarget(self, idl: InputIdlFile, runner): + return CodegenTarget(sdk_root=self.sdk_root, idl=idl, generator="bridge", runner=runner) class CodegenJavaPregenerator: @@ -78,8 +84,8 @@ def Accept(self, idl: InputIdlFile): # bridge codegen return idl.relative_path == "src/controller/data_model/controller-clusters.matter" - def CreateTarget(self, idl: InputIdlFile): - return CodegenTarget(sdk_root=self.sdk_root, idl=idl, generator="java") + def CreateTarget(self, idl: InputIdlFile, runner): + return CodegenTarget(sdk_root=self.sdk_root, idl=idl, generator="java", runner=runner) class CodegenCppAppPregenerator: @@ -98,5 +104,5 @@ def Accept(self, idl: InputIdlFile): return True - def CreateTarget(self, idl: InputIdlFile): - return CodegenTarget(sdk_root=self.sdk_root, idl=idl, generator="cpp-app") + def CreateTarget(self, idl: InputIdlFile, runner): + return CodegenTarget(sdk_root=self.sdk_root, idl=idl, generator="cpp-app", runner=runner) diff --git a/scripts/pregenerate/using_zap.py b/scripts/pregenerate/using_zap.py new file mode 100644 index 00000000000000..4d28f213370655 --- /dev/null +++ b/scripts/pregenerate/using_zap.py @@ -0,0 +1,91 @@ +# Copyright (c) 2022 Project CHIP Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import logging +import os +import shlex +import subprocess +from enum import Enum, auto + +from .types import IdlFileType, InputIdlFile + +ZAP_GENERATE_PATH = os.path.abspath(os.path.join(os.path.dirname(__file__), '..', 'tools', 'zap', 'generate.py')) + + +class ZapGeneratorType(Enum): + APPLICATION_TEMPLATES = auto() + + @property + def generation_template(self): + if self == ZapGeneratorType.APPLICATION_TEMPLATES: + return 'src/app/zap-templates/app-templates.json' + else: + raise Exception("Missing ZAP Generation type implementation") + + @property + def subdir(self): + if self == ZapGeneratorType.APPLICATION_TEMPLATES: + return 'app-templates/zap-generated' + else: + raise Exception("Missing ZAP Generation type implementation") + + +class ZapTarget: + def __init__(self, idl: InputIdlFile, generation_type: ZapGeneratorType, sdk_root: str, runner): + self.idl = idl + self.sdk_root = sdk_root + self.generation_type = generation_type + self.runner = runner + + if idl.file_type != IdlFileType.ZAP: + raise Exception(f"Can only code generate for `*.zap` input files, not for {idl}") + + def Generate(self, output_root: str): + '''Runs generate.py to generate in the specified directory root''' + + output_dir = os.path.join(output_root, self.idl.pregen_subdir, self.generation_type.subdir) + + logging.info(f"Generating: {self.generation_type}:{self.idl.relative_path} into {output_dir}") + + self.runner.ensure_directory_exists(output_dir) + + cmd = [ + ZAP_GENERATE_PATH, + '--templates', self.generation_type.generation_template, + '--output-dir', output_dir, + '--parallel', + self.idl.relative_path + ] + logging.debug(f"Executing {cmd}") + self.runner.run(cmd, cwd=self.sdk_root) + + +class ZapApplicationPregenerator: + """Pregeneration logic for `src/app/zap-templates/app-templates.json` """ + + def __init__(self, sdk_root): + self.sdk_root = sdk_root + + def Accept(self, idl: InputIdlFile): + if idl.file_type != IdlFileType.ZAP: + return False + + # FIXME: implement a proper check + if 'test_files' in idl.relative_path: + return False + return True + + def CreateTarget(self, idl: InputIdlFile, runner): + # TODO: add additional arguments: tell how to invoke zap/codegen + return ZapTarget(sdk_root=self.sdk_root, generation_type=ZapGeneratorType.APPLICATION_TEMPLATES, idl=idl, runner=runner) diff --git a/scripts/tools/zap/BUILD.gn b/scripts/tools/zap/BUILD.gn new file mode 100644 index 00000000000000..67dedad0fcb542 --- /dev/null +++ b/scripts/tools/zap/BUILD.gn @@ -0,0 +1,37 @@ +# Copyright (c) 2020 Project CHIP Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import("//build_overrides/build.gni") +import("//build_overrides/chip.gni") + +action("tests") { + script = "test_generate.py" + + _stamp = "${target_gen_dir}/generate.test.passed" + + inputs = [ + "tests/available_tests.yaml", + "tests/inputs/all-clusters-app.zap", + "tests/inputs/lighting-app.zap", + ] + + args = [ + "--stamp", + rebase_path(_stamp), + "--output", + rebase_path("${target_gen_dir}"), + ] + + outputs = [ _stamp ] +} diff --git a/scripts/tools/zap/check_file_existence.py b/scripts/tools/zap/check_file_existence.py new file mode 100755 index 00000000000000..b1ddf5beab9a66 --- /dev/null +++ b/scripts/tools/zap/check_file_existence.py @@ -0,0 +1,52 @@ +#!/usr/bin/env python +# Copyright (c) 2022 Project CHIP Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import argparse +import os +import sys + + +def main(): + parser = argparse.ArgumentParser( + description='Validate that some specific files exist (or not)') + + parser.add_argument('--touch', help="Create this file on success") + parser.add_argument('--exists', action='append', default=[], help="Validate that these files exist") + parser.add_argument('--missing', action='append', default=[], help="Validate that these files DO NOT exist") + + args = parser.parse_args() + + if args.touch: + if os.path.exists(args.touch): + os.remove(args.touch) + + for name in args.exists: + if not os.path.exists(name): + print(f"File {name} was NOT FOUND.") + sys.exit(1) + + for name in args.missing: + if os.path.exists(name): + print(f"File {name} was FOUND but expected missing.") + sys.exit(1) + + if args.touch: + open(args.touch, "wb").close() + + sys.exit(0) + + +if __name__ == '__main__': + main() diff --git a/scripts/tools/zap/convert.py b/scripts/tools/zap/convert.py index c3fab0ddd880e6..06b2bca8aa9112 100755 --- a/scripts/tools/zap/convert.py +++ b/scripts/tools/zap/convert.py @@ -21,6 +21,8 @@ import subprocess import sys +from zap_execution import ZapTool + CHIP_ROOT_DIR = os.path.realpath( os.path.join(os.path.dirname(__file__), '../../..')) @@ -81,7 +83,8 @@ def detectZclFile(zapFile): # found the right path, try to figure out the actual path if package["pathRelativity"] == "relativeToZap": - path = os.path.abspath(os.path.join(os.path.dirname(zapFile), package["path"])) + path = os.path.abspath(os.path.join( + os.path.dirname(zapFile), package["path"])) else: path = package["path"] @@ -91,40 +94,9 @@ def detectZclFile(zapFile): def runConversion(zap_file): templates_file = getFilePath('src/app/zap-templates/app-templates.json') zcl_file = detectZclFile(zap_file) - - # Accepted environment variables, in order: - # - # ZAP_DEVELOPMENT_PATH - the path to a zap development environment. This is - # a zap checkout, used for local development - # ZAP_INSTALL_PATH - the path where zap-cli exists. This is if zap-cli - # is NOT in the current path - - if 'ZAP_DEVELOPMENT_PATH' in os.environ: - convert_cmd = ['node', 'src-script/zap-start.js', 'convert'] - working_directory = os.environ['ZAP_DEVELOPMENT_PATH'] - # Make sure we don't try to munge the package.json in the ZAP repo. - os.environ['ZAP_SKIP_REAL_VERSION'] = '1' - elif 'ZAP_INSTALL_PATH' in os.environ: - convert_cmd = [os.path.join(os.environ['ZAP_INSTALL_PATH'], 'zap-cli'), 'convert'] - working_directory = None - else: - convert_cmd = ['zap-cli', 'convert'] - working_directory = None - - try: - subprocess.check_call(convert_cmd + ['-z', zcl_file, '-g', templates_file, '-o', zap_file, zap_file], cwd=working_directory) - except FileNotFoundError as e: - print(f'FAILED TO EXECUTE ZAP CONVERSION: {e.strerror} - "{e.filename}"') - print('*'*80) - print('* You may need to install zap. Please ensure one of these applies:') - print('* - `zap-cli` is in $PATH. Install from https://github.com/project-chip/zap/releases') - print('* see docs/guides/BUILDING.md for details') - print('* - `zap-cli` is in $ZAP_INSTALL_PATH. Use this option if you') - print('* installed zap but do not want to update $PATH') - print('* - Point $ZAP_DEVELOPMENT_PATH to your local copy of zap that you') - print('* develop on (to use a developer build of zap)') - print('*'*80) - sys.exit(1) + tool = ZapTool() + tool.run('convert', '-z', zcl_file, '-g', + templates_file, '-o', zap_file, zap_file) def main(): @@ -132,7 +104,8 @@ def main(): zap_file, run_bootstrap = runArgumentsParser() if run_bootstrap: - subprocess.check_call(getFilePath("scripts/tools/zap/zap_bootstrap.sh"), shell=True) + subprocess.check_call(getFilePath( + "scripts/tools/zap/zap_bootstrap.sh"), shell=True) os.chdir(CHIP_ROOT_DIR) diff --git a/scripts/tools/zap/generate.py b/scripts/tools/zap/generate.py index a777bf5865747d..288a2e7f726ee2 100755 --- a/scripts/tools/zap/generate.py +++ b/scripts/tools/zap/generate.py @@ -24,6 +24,9 @@ import urllib.request from dataclasses import dataclass from pathlib import Path +from typing import Optional + +from zap_execution import ZapTool @dataclass @@ -34,6 +37,9 @@ class CmdLineArgs: outputDir: str runBootstrap: bool parallel: bool = True + prettify_output: bool = True + version_check: bool = True + lock_file: Optional[str] = None CHIP_ROOT_DIR = os.path.realpath( @@ -83,7 +89,8 @@ def detectZclFile(zapFile): # found the right path, try to figure out the actual path if package["pathRelativity"] == "relativeToZap": - path = os.path.abspath(os.path.join(os.path.dirname(zapFile), package["path"])) + path = os.path.abspath(os.path.join( + os.path.dirname(zapFile), package["path"])) else: path = package["path"] @@ -107,7 +114,17 @@ def runArgumentsParser() -> CmdLineArgs: help='Automatically run ZAP bootstrap. By default the bootstrap is not triggered') parser.add_argument('--parallel', action='store_true') parser.add_argument('--no-parallel', action='store_false', dest='parallel') + parser.add_argument('--lock-file', help='serialize zap invocations by using the specified lock file.') + parser.add_argument('--prettify-output', action='store_true') + parser.add_argument('--no-prettify-output', + action='store_false', dest='prettify_output') + parser.add_argument('--version-check', action='store_true') + parser.add_argument('--no-version-check', + action='store_false', dest='version_check') parser.set_defaults(parallel=True) + parser.set_defaults(prettify_output=True) + parser.set_defaults(version_check=True) + parser.set_defaults(lock_file=None) args = parser.parse_args() # By default, this script assumes that the global CHIP template is used with @@ -153,41 +170,26 @@ def extractGeneratedIdl(output_dir, zap_config_path): os.rename(idl_path, target_path) -def runGeneration(zap_file, zcl_file, templates_file, output_dir): - # Accepted environment variables, in order: - # - # ZAP_DEVELOPMENT_PATH - the path to a zap development environment. This is - # a zap checkout, used for local development - # ZAP_INSTALL_PATH - the path where zap-cli exists. This is if zap-cli - # is NOT in the current path - - if 'ZAP_DEVELOPMENT_PATH' in os.environ: - generate_cmd = ['node', 'src-script/zap-start.js', 'generate'] - working_directory = os.environ['ZAP_DEVELOPMENT_PATH'] - # Make sure we don't try to munge the package.json in the ZAP repo. - os.environ['ZAP_SKIP_REAL_VERSION'] = '1' - elif 'ZAP_INSTALL_PATH' in os.environ: - generate_cmd = [os.path.join(os.environ['ZAP_INSTALL_PATH'], 'zap-cli'), 'generate'] - working_directory = None - else: - generate_cmd = ['zap-cli', 'generate'] - working_directory = None +def runGeneration(cmdLineArgs): + zap_file = cmdLineArgs.zapFile + zcl_file = cmdLineArgs.zclFile + templates_file = cmdLineArgs.templateFile + output_dir = cmdLineArgs.outputDir + parallel = cmdLineArgs.parallel - try: - subprocess.check_call(generate_cmd + ['-z', zcl_file, '-g', templates_file, - '-i', zap_file, '-o', output_dir], cwd=working_directory) - except FileNotFoundError as e: - print(f'FAILED TO EXECUTE ZAP GENERATION: {e.strerror} - "{e.filename}"') - print('*'*80) - print('* You may need to install zap. Please ensure one of these applies:') - print('* - `zap-cli` is in $PATH. Install from https://github.com/project-chip/zap/releases') - print('* see docs/guides/BUILDING.md for details') - print('* - `zap-cli` is in $ZAP_INSTALL_PATH. Use this option if you') - print('* installed zap but do not want to update $PATH') - print('* - Point $ZAP_DEVELOPMENT_PATH to your local copy of zap that you') - print('* develop on (to use a developer build of zap)') - print('*'*80) - sys.exit(1) + tool = ZapTool() + + if cmdLineArgs.version_check: + tool.version_check() + + args = ['-z', zcl_file, '-g', templates_file, + '-i', zap_file, '-o', output_dir] + + if parallel: + # Parallel-compatible runs will need separate state + args.append('--tempState') + + tool.run('generate', *args) extractGeneratedIdl(output_dir, zap_file) @@ -204,22 +206,22 @@ def runClangPrettifier(templates_file, output_dir): filepath)[1] in listOfSupportedFileExtensions, outputs)) if len(clangOutputs) > 0: - # The "clang-format" pigweed comes with is now version 14, which - # changed behavior from version 13 and earlier regarding some - # whitespace formatting. Unfortunately, all the CI bits run - # clang-format 13 or earlier, so we get styling mismatches. + # NOTE: clang-format may differ in time. Currently pigweed comes + # with clang-format 15. CI may have clang-format-10 installed + # on linux. # - # Try some older clang-format versions just in case they are - # installed. In particular, clang-format-13 is available on various - # Linux distributions and clang-format-11 is available via homebrew - # on Mac. If all else fails, fall back to clang-format. - clang_formats = ['clang-format-13', 'clang-format-12', 'clang-format-11', 'clang-format'] + # We generally want consistent formatting, so + # at this point attempt to use clang-format 15. + clang_formats = ['clang-format-15', 'clang-format'] for clang_format in clang_formats: args = [clang_format, '-i'] args.extend(clangOutputs) try: subprocess.check_call(args) err = None + print('Formatted using %s (%s)' % (clang_format, subprocess.check_output([clang_format, '--version']))) + for outputName in clangOutputs: + print(' - %s' % outputName) break except Exception as thrown: err = thrown @@ -257,40 +259,59 @@ def runJavaPrettifier(templates_file, output_dir): print('google-java-format error:', err) +class LockFileSerializer: + def __init__(self, path): + self.lock_file_path = path + self.lock_file = None + + def __enter__(self): + if not self.lock_file_path: + return + + self.lock_file = open(self.lock_file_path, 'wb') + fcntl.lockf(self.lock_file, fcntl.LOCK_EX) + + def __exit__(self, *args): + if not self.lock_file: + return + + fcntl.lockf(self.lock_file, fcntl.LOCK_UN) + close(self.lock_file) + self.lock_file = None + + def main(): checkPythonVersion() cmdLineArgs = runArgumentsParser() - if cmdLineArgs.runBootstrap: - subprocess.check_call(getFilePath("scripts/tools/zap/zap_bootstrap.sh"), shell=True) - - # The maximum memory usage is over 4GB (#15620) - os.environ["NODE_OPTIONS"] = "--max-old-space-size=8192" + with LockFileSerializer(cmdLineArgs.lock_file) as lock: + if cmdLineArgs.runBootstrap: + subprocess.check_call(getFilePath("scripts/tools/zap/zap_bootstrap.sh"), shell=True) - if cmdLineArgs.parallel: - # Parallel-compatible runs will need separate state - os.environ["ZAP_TEMPSTATE"] = "1" + # The maximum memory usage is over 4GB (#15620) + os.environ["NODE_OPTIONS"] = "--max-old-space-size=8192" - # `zap-cli` may extract things into a temporary directory. ensure extraction - # does not conflict. - with tempfile.TemporaryDirectory(prefix='zap') as temp_dir: - old_temp = os.environ['TEMP'] if 'TEMP' in os.environ else None - os.environ['TEMP'] = temp_dir + # `zap-cli` may extract things into a temporary directory. ensure extraction + # does not conflict. + with tempfile.TemporaryDirectory(prefix='zap') as temp_dir: + old_temp = os.environ['TEMP'] if 'TEMP' in os.environ else None + os.environ['TEMP'] = temp_dir - runGeneration(cmdLineArgs.zapFile, cmdLineArgs.zclFile, cmdLineArgs.templateFile, cmdLineArgs.outputDir) + runGeneration(cmdLineArgs) - if old_temp: - os.environ['TEMP'] = old_temp - else: - del os.environ['TEMP'] + if old_temp: + os.environ['TEMP'] = old_temp + else: + del os.environ['TEMP'] - prettifiers = [ - runClangPrettifier, - runJavaPrettifier, - ] + if cmdLineArgs.prettify_output: + prettifiers = [ + runClangPrettifier, + runJavaPrettifier, + ] - for prettifier in prettifiers: - prettifier(cmdLineArgs.templateFile, cmdLineArgs.outputDir) + for prettifier in prettifiers: + prettifier(cmdLineArgs.templateFile, cmdLineArgs.outputDir) if __name__ == '__main__': diff --git a/scripts/tools/zap/prune_outputs.py b/scripts/tools/zap/prune_outputs.py new file mode 100644 index 00000000000000..bc223fa2a010c6 --- /dev/null +++ b/scripts/tools/zap/prune_outputs.py @@ -0,0 +1,43 @@ +#!/usr/bin/env python +# Copyright (c) 2022 Project CHIP Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import argparse +import os +import shutil +import sys + + +def main(): + parser = argparse.ArgumentParser( + description='Delete files based on an input file listing files to be removed') + + parser.add_argument('--keep', required=True, help="File containing names to keep (copy over)") + parser.add_argument('--output-dir', required=True, help="Output directory to copy files into") + parser.add_argument('--input-dir', required=True, help="Input directory to get the files from") + + args = parser.parse_args() + + with open(args.keep, "rt") as f: + for source in f.readlines(): + source = source.strip() + if not source: + continue + target = os.path.join(args.output_dir, source) + os.makedirs(os.path.dirname(target), exist_ok=True) + shutil.copyfile(os.path.join(args.input_dir, source), target) + + +if __name__ == '__main__': + main() diff --git a/scripts/tools/zap/test_generate.py b/scripts/tools/zap/test_generate.py new file mode 100755 index 00000000000000..73353ee0e51f4e --- /dev/null +++ b/scripts/tools/zap/test_generate.py @@ -0,0 +1,198 @@ +#!/usr/bin/env python +# Copyright (c) 2022 Project CHIP Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import glob +import logging +import os +import shutil +import subprocess +import sys +import unittest +from dataclasses import dataclass, field +from typing import List, Optional + +import yaml + +TESTS_DIR = os.path.join(os.path.dirname(__file__), "tests") +CHIP_ROOT = os.path.abspath(os.path.join( + os.path.dirname(__file__), "../../..")) + + +@dataclass +class ProgramArguments: + stamp_file: Optional[str] = None + regenerate_golden: bool = False + output_directory: str = '' + + +PROGRAM_ARGUMENTS = None + + +@dataclass +class ExpectedOutput: + file_name: str + golden_path: str + + +@dataclass +class GeneratorTestCase: + template: str + outputs: List[ExpectedOutput] = field(default_factory=list) + + def add_outputs(self, yaml_outputs_dict): + for file_name, golden_path in yaml_outputs_dict.items(): + self.outputs.append(ExpectedOutput( + file_name=file_name, golden_path=golden_path)) + + +@dataclass +class GeneratorTest: + zap: str + context: ProgramArguments + test_cases: List[GeneratorTestCase] = field(default_factory=list) + + def add_test_cases(self, yaml_test_case_dict): + for json, outputs in yaml_test_case_dict.items(): + test_case = GeneratorTestCase(template=json) + test_case.add_outputs(outputs) + self.test_cases.append(test_case) + + def run_test_cases(self, checker: unittest.TestCase): + for test in self.test_cases: + with checker.subTest(template=test.template): + output_directory = os.path.join( + self.context.output_directory, + os.path.splitext(os.path.basename(self.zap))[0], + os.path.splitext(os.path.basename(test.template))[0] + ) + + # ensure a clean start as ALL outputs will be compared + if os.path.exists(output_directory): + shutil.rmtree(output_directory) + os.makedirs(output_directory, exist_ok=True) + + subprocess.check_call([ + f"{CHIP_ROOT}/scripts/tools/zap/generate.py", + "--parallel", + "--output-dir", + output_directory, + "--templates", + os.path.join(TESTS_DIR, test.template), + os.path.join(TESTS_DIR, self.zap), + ], cwd=output_directory) + + # Files generated, ready to check: + # - every output file MUST exist in the golden image list + # - every golden image file MUST exist in the output + # - content must match + + # test.outputs contain: + # - file_name + # - golden_path + expected_files = set([o.file_name for o in test.outputs]) + actual_files = set([ + name[len(output_directory)+1:] for name in glob.glob(f"{output_directory}/**/*", recursive=True) + ]) + + checker.assertEqual( + expected_files, actual_files, msg="Expected and actual generated file list MUST be identical.") + + # All files exist, ready to do the compare + for entry in test.outputs: + expected = os.path.join(TESTS_DIR, entry.golden_path) + actual = os.path.join(output_directory, entry.file_name) + + try: + subprocess.check_call(["diff", actual, expected]) + except: + if self.context.regenerate_golden: + print( + f"Copying updated golden image from {actual} to {expected}") + subprocess.check_call(["cp", actual, expected]) + else: + raise + + +def build_tests(yaml_data, context: ProgramArguments) -> List[GeneratorTest]: + """ + Transforms the YAML dictonary (Dict[str, Dict[str, Dict[str,str]]]) into + a generator test structure. + """ + result = [] + + for input_zap, test_cases in yaml_data.items(): + generator = GeneratorTest(zap=input_zap, context=context) + generator.add_test_cases(test_cases) + result.append(generator) + + return result + + +class TestGenerators(unittest.TestCase): + def test_generators(self): + with open(os.path.join(TESTS_DIR, "available_tests.yaml"), 'rt') as stream: + yaml_data = yaml.safe_load(stream) + + global PROGRAM_ARGUMENTS + for test in build_tests(yaml_data, context=PROGRAM_ARGUMENTS): + with self.subTest(zap=test.zap): + test.run_test_cases(self) + + +def process_arguments(): + """Parses sys.argv and extracts arguments that are specific to the script.""" + args = sys.argv[:] + + program_args = ProgramArguments() + + if '--regenerate' in args: + idx = args.index('--regenerate') + program_args.regenerate_golden = True + del args[idx] + elif 'ZAP_GENERATE_GOLDEN_REGENERATE' in os.environ: + # Allow `ZAP_GENERATE_GOLDEN_REGENERATE=1 ninja check` to also + # update golden images + program_args.regenerate_golden = True + + if '--stamp' in args: + idx = args.index('--stamp') + program_args.stamp_file = args[idx + 1] + del args[idx+1] + del args[idx] + + if '--output' in args: + idx = args.index('--output') + program_args.output_directory = args[idx + 1] + del args[idx+1] + del args[idx] + else: + raise Exception("`--output` argument is required") + + return program_args, args + + +if __name__ == '__main__': + process_args, unittest_args = process_arguments() + + if process_args.stamp_file: + if os.path.exists(process_args.stamp_file): + os.remove(process_args.stamp_file) + + PROGRAM_ARGUMENTS = process_args + + unittest.main(argv=unittest_args) + + if process_args.stamp_file: + open(process_args.stamp_file, "wb").close() diff --git a/scripts/tools/zap/tests/available_tests.yaml b/scripts/tools/zap/tests/available_tests.yaml new file mode 100644 index 00000000000000..0a086c7de6b2f2 --- /dev/null +++ b/scripts/tools/zap/tests/available_tests.yaml @@ -0,0 +1,29 @@ +# These are the available tests codegen.py +# +# General format is: +# +#