State Response compressor /decompressor is implemented in zstd, but disabled yet #7116
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| # | |
| # Copyright Quadrivium LLC | |
| # All Rights Reserved | |
| # SPDX-License-Identifier: Apache-2.0 | |
| # | |
| name: Main Build Pipeline | |
| on: | |
| push: | |
| branches: [ master ] | |
| tags: [ '*' ] | |
| pull_request: | |
| workflow_dispatch: | |
| inputs: | |
| run_all_steps: | |
| description: 'Run all steps' | |
| required: true | |
| default: 'false' | |
| type: 'choice' | |
| options: | |
| - 'true' | |
| - 'false' | |
| env: | |
| BUILD_DIR: build | |
| KAGOME_IN_DOCKER: 1 | |
| CTEST_OUTPUT_ON_FAILURE: 1 | |
| GITHUB_HUNTER_USERNAME: ${{ secrets.HUNTER_USERNAME }} | |
| GITHUB_HUNTER_TOKEN: ${{ secrets.HUNTER_TOKEN }} | |
| # Change all container: values to this variable once this feature is available | |
| # https://github.community/t/how-to-set-and-access-a-workflow-variable/17335/6 | |
| CACHE_VERSION: v01 | |
| CACHE_PATHS: | | |
| ~/Library/Caches/pip | |
| ~/.cargo | |
| ~/.ccache | |
| ~/.hunter | |
| ~/.rustup | |
| DOCKER_REGISTRY_PATH: ${{ secrets.GCP_REGISTRY }}/${{ secrets.GCP_PROJECT }}/kagome-dev/ | |
| PROJECT_ID: ${{ secrets.GCP_PROJECT }} | |
| CACHE_PATH: ./build/cache | |
| DEFAULT_BUILD_TYPE: Release | |
| GIT_REF_NAME: ${{ github.ref_name }} | |
| IS_MAIN_OR_TAG: ${{ github.event_name == 'push' && (github.ref == 'refs/heads/master' || startsWith(github.ref, 'refs/tags/')) }} | |
| jobs: | |
| MacOS: | |
| runs-on: macos-14 | |
| timeout-minutes: 120 | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - uses: actions/cache@v4 | |
| with: | |
| path: ${{ env.CACHE_PATHS }} | |
| key: ${{ github.job }}-${{ env.CACHE_VERSION }} | |
| - name: install | |
| run: ./housekeeping/macos/dependency.sh | |
| env: | |
| KAGOME_MAC_CI: 1 | |
| - name: build | |
| run: ./housekeeping/make_build.sh -G Ninja -DCLEAR_OBJS=ON -DCOVERAGE=OFF -DWASM_COMPILER=WasmEdge -DCMAKE_TOOLCHAIN_FILE=cmake/toolchain/cxx20.cmake | |
| env: | |
| KAGOME_IN_DOCKER: 0 | |
| KAGOME_MAC_CI: 1 | |
| Linux: | |
| if: false # Need to fix | |
| # ${{ | |
| # !(github.ref == 'refs/heads/master' || | |
| # startsWith( github.ref, 'refs/tags/' )) }} | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| options: | |
| - name: "Linux: clang-16 External Project" | |
| run: ./housekeeping/make_external_build.sh -DCLEAR_OBJS=ON -DCMAKE_TOOLCHAIN_FILE=../../cmake/toolchain/clang-16_cxx20.cmake | |
| name: "${{ matrix.options.name }}" | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 120 | |
| container: qdrvm/kagome-dev:9-minideb | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - uses: actions/cache@v4 | |
| with: | |
| path: ${{ env.CACHE_PATHS }} | |
| key: ${{ github.job }}-${{ matrix.options.name }}-${{ env.CACHE_VERSION }} | |
| - name: df du | |
| run: | | |
| df -m || true | |
| du -hd1 /__w /github || true | |
| - name: Install mold | |
| run: ./housekeeping/ci_install_mold.sh --make-default | |
| - name: "${{ matrix.options.name }}" | |
| run: "${{ matrix.options.run }}" | |
| - name: df du | |
| if: ${{ always() }} | |
| run: | | |
| df -m || true | |
| du -hd1 /__w /github || true | |
| Linux-self-hosted: | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| options: | |
| - name: "Self-hosted: Linux: gcc-12 ASAN" | |
| run: ./housekeeping/make_build.sh -DCLEAR_OBJS=ON -DCMAKE_TOOLCHAIN_FILE=cmake/toolchain/gcc-12_cxx20.cmake -DASAN=ON | |
| - name: "Self-hosted: Linux: clang-16 TSAN WAVM" | |
| run: ./housekeeping/make_build.sh -DCLEAR_OBJS=ON -DCMAKE_TOOLCHAIN_FILE=cmake/toolchain/clang-16_cxx20.cmake -DTSAN=ON -DWASM_COMPILER=WAVM | |
| - name: "Self-hosted: Linux: clang-16 UBSAN" | |
| run: ./housekeeping/make_build.sh -DCLEAR_OBJS=ON -DCMAKE_TOOLCHAIN_FILE=cmake/toolchain/clang-16_cxx20.cmake -DUBSAN=ON | |
| # Need to fix | |
| # - name: "Self-hosted: Linux: clang-16 External Project" | |
| # run: ./housekeeping/make_external_build.sh -DCLEAR_OBJS=ON -DCMAKE_TOOLCHAIN_FILE=../../cmake/toolchain/clang-16_cxx20.cmake | |
| name: "${{ matrix.options.name }}" | |
| runs-on: [ actions-runner-controller ] | |
| timeout-minutes: 120 | |
| container: qdrvm/kagome-dev:9-minideb | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - uses: actions/cache@v4 | |
| with: | |
| path: ${{ env.CACHE_PATHS }} | |
| key: ${{ github.job }}-${{ matrix.options.name }}-${{ env.CACHE_VERSION }} | |
| - name: Install mold | |
| run: ./housekeeping/ci_install_mold.sh --make-default | |
| - name: "${{ matrix.options.name }}" | |
| run: "${{ matrix.options.run }}" | |
| coverage-self-hosted: | |
| if: false # ${{ github.ref == 'refs/heads/master' || startsWith( github.ref, 'refs/tags/') || contains( github.event.pull_request.labels.*.name, 'Non-master self-hosted') }} | |
| name: "Self-hosted: Linux: gcc-12 coverage/sonar" | |
| runs-on: [ actions-runner-controller ] | |
| timeout-minutes: 120 | |
| container: qdrvm/kagome-dev:9-minideb | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - uses: actions/cache@v4 | |
| with: | |
| path: ${{ env.CACHE_PATHS }} | |
| key: ${{ github.job }}-${{ env.CACHE_VERSION }} | |
| - name: Install mold | |
| run: ./housekeeping/ci_install_mold.sh --make-default | |
| - name: makeBuild | |
| env: | |
| BUILD_FINAL_TARGET: ctest_coverage | |
| run: ./housekeeping/make_build.sh -DCMAKE_TOOLCHAIN_FILE=cmake/toolchain/gcc-12_cxx20.cmake -DCOVERAGE=ON -DCLEAR_OBJS=ON | |
| - if: ${{ | |
| github.event_name != 'pull_request' || | |
| github.event.pull_request.head.repo.full_name == github.repository }} | |
| name: Submit Coverage | |
| env: | |
| CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }} | |
| run: if [ "CODECOV_TOKEN" != "null" ]; then ./housekeeping/codecov.sh; else echo "Some secret undefined. Step passed..."; fi | |
| - if: ${{ | |
| github.event_name != 'pull_request' || | |
| github.event.pull_request.head.repo.full_name == github.repository }} | |
| name: Sonar | |
| env: | |
| SONAR_TOKEN: ${{ secrets.SONAR_TOKEN }} | |
| GITHUB_USERNAME: ${{ secrets.GITHUB_USERNAME }} | |
| GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
| BRANCH_NAME: ${{ github.ref }} | |
| run: if [ "$SONAR_TOKEN" != "null" -a "$GITHUB_USERNAME" != "null" -a "$GITHUB_TOKEN" != "null" ]; then ./housekeeping/sonar.sh; else echo "Some secret undefined. Step passed..."; fi | |
| Minideb: | |
| if: ${{ | |
| !(github.ref == 'refs/heads/master' || | |
| startsWith( github.ref, 'refs/tags/' )) }} | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| options: | |
| - name: "Minideb: Build Debug" | |
| build-type: "Debug" | |
| - name: "Minideb: Build Release" | |
| build-type: "Release" | |
| name: "${{ matrix.options.name }}" | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 120 | |
| container: qdrvm/kagome-dev:9-minideb | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - uses: actions/cache@v4 | |
| with: | |
| path: ${{ env.CACHE_PATHS }} | |
| key: ${{ github.job }}-${{ matrix.options.name }}-${{ env.CACHE_VERSION }} | |
| - name: "${{ matrix.options.name }}" | |
| env: | |
| BUILD_TYPE: "${{ matrix.options.build-type }}" | |
| run: ./housekeeping/docker/kagome-dev/make.sh | |
| kagome_dev_docker_build_tidy: | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 180 | |
| name: "Linux: clang-tidy" | |
| steps: | |
| - name: "Checkout repository" | |
| uses: actions/checkout@v4 | |
| - name: "Authenticate with Google Cloud" | |
| uses: 'google-github-actions/auth@v2' | |
| with: | |
| credentials_json: ${{ secrets.GCP_SERVICE_ACCOUNT_KEY }} | |
| - name: "Set up Cloud SDK" | |
| uses: 'google-github-actions/setup-gcloud@v2' | |
| - name: "Configure Docker for GCR" | |
| run: | | |
| gcloud auth configure-docker --quiet | |
| gcloud auth configure-docker ${{ secrets.GCP_REGISTRY }} --quiet | |
| - name: "Check version" | |
| working-directory: ./housekeeping/docker/kagome-dev | |
| run: | | |
| SHORT_COMMIT_HASH=$(grep 'short_commit_hash:' commit_hash.txt | cut -d ' ' -f 2) | |
| echo "short_commit_hash=${SHORT_COMMIT_HASH}" >> $GITHUB_ENV | |
| - name: "Cache dependencies" | |
| id: cache-restore | |
| uses: actions/cache/restore@v4 | |
| with: | |
| path: ${{ env.CACHE_PATH }} | |
| key: ${{ github.job }}-${{ env.CACHE_VERSION }}-tidy-${{ env.short_commit_hash }} | |
| restore-keys: | | |
| ${{ github.job }}-${{ env.CACHE_VERSION }}-tidy- | |
| - name: "Build target" | |
| working-directory: ./housekeeping/docker/kagome-dev | |
| run: | |
| make kagome_dev_docker_build_tidy \ | |
| DOCKER_REGISTRY_PATH=${DOCKER_REGISTRY_PATH} \ | |
| GITHUB_HUNTER_USERNAME=${{ secrets.HUNTER_USERNAME }} \ | |
| GITHUB_HUNTER_TOKEN=${{ secrets.HUNTER_TOKEN }} \ | |
| CI="true" | |
| - name: "Cleaning cache" | |
| run: | | |
| find ${{ env.CACHE_PATH }} -name '*.pdf' -exec rm {} \; | |
| - name: "Always Save Cache" | |
| id: cache-save | |
| if: always() && (steps.cache-restore.outputs.cache-hit != 'true' || env.package_exist != 'True') | |
| uses: actions/cache/save@v4 | |
| with: | |
| path: ${{ env.CACHE_PATH }} | |
| key: ${{ steps.cache-restore.outputs.cache-primary-key }} | |
| kagome_dev_docker_build: | |
| runs-on: [ actions-runner-controller ] | |
| timeout-minutes: 180 | |
| if: ${{ | |
| github.ref == 'refs/heads/master' || | |
| startsWith( github.ref, 'refs/tags/') || | |
| contains( github.event.pull_request.labels.*.name, 'Non-master self-hosted') }} || | |
| github.event.inputs.run_all_steps == 'true' }} | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| options: | |
| - name: "Self-hosted: Minideb: Build Debug" | |
| build_type: "Debug" | |
| - name: "Self-hosted: Minideb: Build Release" | |
| build_type: "Release" | |
| - name: "Self-hosted: Minideb: Build RelWithDebInfo" | |
| build_type: "RelWithDebInfo" | |
| name: "${{ matrix.options.name }}" | |
| steps: | |
| - name: "Checkout repository" | |
| uses: actions/checkout@v4 | |
| - name: "Authenticate with Google Cloud" | |
| uses: 'google-github-actions/auth@v2' | |
| with: | |
| credentials_json: ${{ secrets.GCP_SERVICE_ACCOUNT_KEY }} | |
| - name: "Set up Cloud SDK" | |
| uses: 'google-github-actions/setup-gcloud@v2' | |
| - name: "Configure Docker for GCR" | |
| run: | | |
| gcloud auth configure-docker --quiet | |
| gcloud auth configure-docker ${{ secrets.GCP_REGISTRY }} --quiet | |
| - name: "Get Polkadot and Zombienet versions" | |
| working-directory: ./housekeeping/docker/kagome-dev | |
| run: make get_versions | |
| - name: "Check package version" | |
| working-directory: ./housekeeping/docker/kagome-dev | |
| run: | | |
| ARTIFACTS_REPO=$(grep '^ARTIFACTS_REPO ?=' Makefile | sed 's/ARTIFACTS_REPO ?= //') | |
| REGION=$(grep '^REGION ?=' Makefile | sed 's/REGION ?= //') | |
| SHORT_COMMIT_HASH=$(grep 'short_commit_hash:' commit_hash.txt | cut -d ' ' -f 2) | |
| PACKAGE_VERSION="$(date +'%y.%m.%d')-${SHORT_COMMIT_HASH}-${{ matrix.options.build_type }}" | |
| echo "PACKAGE_VERSION=${PACKAGE_VERSION}" | |
| echo "package_version=${PACKAGE_VERSION}" >> $GITHUB_ENV | |
| echo "short_commit_hash=${SHORT_COMMIT_HASH}" >> $GITHUB_ENV | |
| gcloud config set artifacts/repository $ARTIFACTS_REPO | |
| gcloud config set artifacts/location $REGION | |
| PACKAGE_EXIST=$(gcloud artifacts versions list --package=kagome-dev --format=json | jq -e ".[] | select(.name | endswith(\"${PACKAGE_VERSION}\"))" > /dev/null && echo "True" || echo "False") | |
| echo "package_exist=${PACKAGE_EXIST}" >> $GITHUB_ENV | |
| - name: "Cache dependencies" | |
| if: env.package_exist == 'False' | |
| id: cache-restore | |
| uses: actions/cache/restore@v4 | |
| with: | |
| path: ${{ env.CACHE_PATH }} | |
| key: ${{ github.job }}-${{ env.CACHE_VERSION }}-${{ matrix.options.build_type }}-${{ env.short_commit_hash }} | |
| restore-keys: | | |
| ${{ github.job }}-${{ env.CACHE_VERSION }}-${{ matrix.options.build_type }}- | |
| - name: "Build target" | |
| if: env.package_exist == 'False' | |
| working-directory: ./housekeeping/docker/kagome-dev | |
| run: make kagome_dev_docker_build \ | |
| DOCKER_REGISTRY_PATH=${DOCKER_REGISTRY_PATH} \ | |
| GITHUB_HUNTER_USERNAME=${{ secrets.HUNTER_USERNAME }} \ | |
| GITHUB_HUNTER_TOKEN=${{ secrets.HUNTER_TOKEN }} \ | |
| BUILD_TYPE=${{ matrix.options.build_type }} | |
| - name: "Push Kagome APT Package" | |
| if: env.package_exist == 'False' | |
| working-directory: ./housekeeping/docker/kagome-dev | |
| run: | | |
| SHORT_COMMIT_HASH=$(grep 'short_commit_hash:' commit_hash.txt | cut -d ' ' -f 2) | |
| make upload_apt_package \ | |
| BUILD_TYPE=${{ matrix.options.build_type }} | |
| - name: Set up Docker Buildx | |
| if: env.package_exist == 'False' | |
| uses: docker/[email protected] | |
| - name: "Build Docker Image" | |
| if: env.package_exist == 'False' | |
| working-directory: ./housekeeping/docker/kagome-dev | |
| run: make kagome_image_build \ | |
| KAGOME_PACKAGE_VERSION=${{ env.package_version }} \ | |
| DOCKER_REGISTRY_PATH=${DOCKER_REGISTRY_PATH} \ | |
| BUILD_TYPE=${{ matrix.options.build_type }} | |
| - name: "Push Docker Image to GCR" | |
| if: env.package_exist == 'False' | |
| working-directory: ./housekeeping/docker/kagome-dev | |
| run: make kagome_image_push \ | |
| DOCKER_REGISTRY_PATH=${DOCKER_REGISTRY_PATH} \ | |
| BUILD_TYPE=${{ matrix.options.build_type }} \ | |
| GIT_REF_NAME=${{ env.GIT_REF_NAME }} \ | |
| IS_MAIN_OR_TAG=${{ env.IS_MAIN_OR_TAG }} | |
| - name: "Login to Docker Hub" | |
| uses: docker/login-action@v3 | |
| if: ${{ | |
| (env.package_exist == 'False') && | |
| (matrix.options.build_type == 'Release') }} | |
| with: | |
| username: ${{ secrets.DOCKER_USERNAME }} | |
| password: ${{ secrets.DOCKER_TOKEN }} | |
| - name: "Push Docker Image to Docker Hub" | |
| if: ${{ | |
| (env.package_exist == 'False') && | |
| (matrix.options.build_type == 'Release') }} | |
| working-directory: ./housekeeping/docker/kagome-dev | |
| run: make kagome_image_push_dockerhub \ | |
| DOCKER_REGISTRY_PATH=${DOCKER_REGISTRY_PATH} \ | |
| BUILD_TYPE=${{ matrix.options.build_type }} \ | |
| GIT_REF_NAME=${{ env.GIT_REF_NAME }} \ | |
| IS_MAIN_OR_TAG=${{ env.IS_MAIN_OR_TAG }} | |
| - name: "Cleaning cache" | |
| if: env.package_exist == 'False' | |
| run: | | |
| find ${{ env.CACHE_PATH }} -name '*.pdf' -exec rm {} \; | |
| - name: "Always Save Cache" | |
| id: cache-save | |
| if: always() && (steps.cache-restore.outputs.cache-hit != 'true' || env.package_exist != 'True') | |
| uses: actions/cache/save@v4 | |
| with: | |
| path: ${{ env.CACHE_PATH }} | |
| key: ${{ steps.cache-restore.outputs.cache-primary-key }} | |
| Zombie-Tests: | |
| runs-on: [ actions-runner-controller ] | |
| timeout-minutes: 60 | |
| needs: [kagome_dev_docker_build] | |
| if: ${{ | |
| github.ref == 'refs/heads/master' || | |
| startsWith( github.ref, 'refs/tags/') || | |
| github.event.inputs.run_all_steps == 'true' }} | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| options: | |
| - name: "PVF preparation & execution time" | |
| test: "test-polkadot-functional-0001-parachains-pvf" | |
| # - name: "Disputes initiation, conclusion and lag" | |
| # test: "test-polkadot-functional-0002-parachains-disputes" | |
| - name: "BEEFY voting and finality, test MMR proofs. Assumes Rococo sessions of 1 minute" | |
| test: "test-polkadot-functional-0003-beefy-and-mmr" | |
| # - name: "Dispute finality lag when 1/3 of parachain validators always attempt to include an invalid block" | |
| # test: "test-polkadot-functional-0004-parachains-garbage-candidate" | |
| # - name: "Past-session dispute slashing" | |
| # test: "test-polkadot-functional-0005-parachains-disputes-past-session" | |
| # - name: "Test if parachains make progress with most of approvals being tranch0" | |
| # test: "test-polkadot-functional-0006-parachains-max-tranche0" | |
| - name: "Test if disputes triggered on finalized blocks within scope always end as valid" | |
| test: "test-polkadot-functional-0007-dispute-freshly-finalized" | |
| - name: "Test if disputes triggered on finalized blocks out of scope never get to be confirmed and concluded" | |
| test: "test-polkadot-functional-0008-dispute-old-finalized" | |
| - name: "Approval voting coalescing does not lag finality" | |
| test: "test-polkadot-functional-0009-approval-voting-coalescing" | |
| # - name: "Test validator disabling effects" | |
| # test: "test-polkadot-functional-0010-validator-disabling" | |
| - name: "Test we are producing blocks at 6 seconds clip" | |
| test: "test-polkadot-functional-0011-async-backing-6-seconds-rate" | |
| - name: "Systematic chunk recovery is used if the chunk mapping feature is enabled." | |
| test: "test-polkadot-functional-0013-systematic-chunk-recovery" | |
| steps: | |
| - name: "Checkout repository" | |
| uses: actions/checkout@v4 | |
| # - name: Create and enable swap | |
| # run: | | |
| # free -h | |
| # df -h | |
| # sudo fallocate -l 4G /swapfile | |
| # sudo chmod 600 /swapfile | |
| # sudo mkswap /swapfile | |
| # sudo swapon /swapfile | |
| # sudo swapon --show | |
| # free -h | |
| # df -h | |
| - name: "Authenticate with Google Cloud" | |
| uses: 'google-github-actions/auth@v2' | |
| with: | |
| credentials_json: ${{ secrets.GCP_SERVICE_ACCOUNT_KEY }} | |
| - name: "Set up Cloud SDK" | |
| uses: 'google-github-actions/setup-gcloud@v2' | |
| - name: "Configure Docker for GCR" | |
| run: | | |
| gcloud auth configure-docker --quiet | |
| gcloud auth configure-docker ${{ secrets.GCP_REGISTRY }} --quiet | |
| - name: "Get Polkadot and Zombienet versions" | |
| working-directory: ./housekeeping/docker/kagome-dev | |
| run: make get_versions | |
| - name: "Get Package Version" | |
| working-directory: ./housekeeping/docker/kagome-dev | |
| run: | | |
| ARTIFACTS_REPO=$(grep '^ARTIFACTS_REPO ?=' Makefile | sed 's/ARTIFACTS_REPO ?= //') | |
| REGION=$(grep '^REGION ?=' Makefile | sed 's/REGION ?= //') | |
| SHORT_COMMIT_HASH=$(grep 'short_commit_hash:' commit_hash.txt | cut -d ' ' -f 2) | |
| PACKAGE_VERSION="$(date +'%y.%m.%d')-${SHORT_COMMIT_HASH}-${{ env.DEFAULT_BUILD_TYPE }}" | |
| echo "PACKAGE_VERSION=${PACKAGE_VERSION}" | |
| echo "package_version=${PACKAGE_VERSION}" >> $GITHUB_ENV | |
| - name: "Get Zombie Tester Version" | |
| run: | | |
| ZOMBIE_TESTER_IMAGE=${DOCKER_REGISTRY_PATH}zombie_tester:latest | |
| echo "zombie_tester_image=${ZOMBIE_TESTER_IMAGE}" | |
| echo "zombie_tester_image=${ZOMBIE_TESTER_IMAGE}" >> $GITHUB_ENV | |
| - name: "Run test" | |
| working-directory: ./zombienet/docker | |
| run: | | |
| make ${{ matrix.options.test }} \ | |
| ZOMBIE_TESTER_IMAGE=${{ env.zombie_tester_image }} \ | |
| KAGOME_PACKAGE_VERSION=${{ env.package_version }} | |
| free -h | |
| df -h | |
| - name: "Upload test logs as artifact" | |
| if: always() | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: logs-${{ matrix.options.test }} | |
| path: /tmp/test_logs | |
| - name: "Show system info" | |
| if: always() | |
| run: | | |
| free -h | |
| df -h |