From 66352a4c420cf9d49aebe15f617c4498b7e87cd6 Mon Sep 17 00:00:00 2001 From: Andrew Nikitin Date: Tue, 12 Apr 2022 20:25:35 +0300 Subject: [PATCH] build(tooling): DEV-1096 Improve CI/CD pipelines for release process (#304) * Only run one instance of linter for specific ref * Bump Checkout action version * Remove broken timeout argument * Made style consistent throughout Lint action * Create dispatcher workflow * Splitting out build * Fix YAML formatting errors * Remove wait on lint * Specify Go version for linting * Make Ubuntu version consistent * Change test.yml to build.yml * Restore Go setup * Combine build tarball and Debian into single job * Added Github CodeQL Action * DEV-1237 Fix issues with build and test workflows (#309) * Deactivate lint and release workflows * Remove set pipefail for workflow * Use script to check for validator status instead of inline GitHub Actions * Fix release tagging * Remove unused script * Fix variable passing * FInal release test Co-authored-by: Ankur Banerjee Co-authored-by: jay-dee7 --- .github/workflows/build.yml | 130 +++++++ .github/workflows/dispatch.yml | 25 +- .github/workflows/lint.yml | 57 +-- .github/workflows/release.yml | 274 +++----------- .github/workflows/test.yml | 337 +++++------------- build-tools/build-deb.sh | 4 +- docker/Dockerfile | 6 + docker/localnet/docker-compose.env | 6 +- docker/persistent-chains/docker-compose.env | 6 +- tests/e2e-complex/deb-install/add-observer.sh | 2 +- .../deb-install/check-promotion.sh | 13 + .../deb-install/promote-validator.sh | 2 + tests/e2e-complex/upgrade/common.sh | 8 +- .../e2e-complex/upgrade/upgrade_and_check.sh | 2 +- tests/e2e-pytest/helpers.py | 12 +- 15 files changed, 359 insertions(+), 525 deletions(-) create mode 100644 .github/workflows/build.yml create mode 100644 tests/e2e-complex/deb-install/check-promotion.sh diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml new file mode 100644 index 000000000..1e6b1ffd4 --- /dev/null +++ b/.github/workflows/build.yml @@ -0,0 +1,130 @@ +name: "Build" +on: + workflow_call: + outputs: + VERSION: + description: "Build version number" + value: ${{ jobs.build-binary.outputs.VERSION }} +defaults: + run: + shell: bash + + +jobs: + go-unit-tests: + name: "Golang unit tests" + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v3 + - uses: actions/setup-go@v3 + with: + go-version: '1.17' + + - name: Run Golang unit tests + run: go test -v ./... + + build-binary: + name: "Node binary" + runs-on: ubuntu-latest + outputs: + VERSION: ${{ steps.set-version.outputs.VERSION }} + + steps: + - uses: actions/setup-go@v3 + with: + go-version: '1.17' + + - name: Fetch Golang protoc compiler plugins + env: + GOLANG_PROTOBUF_VERSION: 1.3.5 + GOGO_PROTOBUF_VERSION: 1.3.2 + GRPC_GATEWAY_VERSION: 1.14.7 + # Taken from: tendermintdev/sdk-proto-gen:v0.2 + run: | + go get \ + github.com/golang/protobuf/protoc-gen-go@v"$GOLANG_PROTOBUF_VERSION" \ + github.com/gogo/protobuf/protoc-gen-gogo@v"$GOGO_PROTOBUF_VERSION" \ + github.com/gogo/protobuf/protoc-gen-gogofast@v"$GOGO_PROTOBUF_VERSION" \ + github.com/gogo/protobuf/protoc-gen-gogofaster@v"$GOGO_PROTOBUF_VERSION" \ + github.com/grpc-ecosystem/grpc-gateway/protoc-gen-grpc-gateway@v"$GRPC_GATEWAY_VERSION" \ + github.com/grpc-ecosystem/grpc-gateway/protoc-gen-swagger@v"$GRPC_GATEWAY_VERSION" \ + github.com/regen-network/cosmos-proto/protoc-gen-gocosmos@latest + - name: Install buf + env: + PREFIX: "/usr/local" + VERSION: "1.0.0-rc8" + run: | + curl -sSL "https://github.com/bufbuild/buf/releases/download/v${VERSION}/buf-$(uname -s)-$(uname -m).tar.gz" | \ + sudo tar -xvzf - -C "${PREFIX}" --strip-components 1 + + - uses: actions/checkout@v3 + with: + fetch-depth: 0 # Required to fetch version + + - name: Build node binary + run: | + make proto-gen build + + - name: Store artifact + uses: actions/upload-artifact@v3 + with: + name: cheqd-noded + path: build/cheqd-noded + + - name: Set version number + id: set-version + run: | + VERSION=$(build/cheqd-noded version 2>&1) + echo ::set-output name=VERSION::"$VERSION" + - name: Install fpm + run: | + sudo apt-get install ruby ruby-dev rubygems build-essential + sudo gem install --no-document fpm + + - name: Build Debian package + working-directory: ./build-tools + run: | + ./build-deb.sh "../build/cheqd-noded" + + - name: Store Debian package artifact + uses: actions/upload-artifact@v3 + env: + VERSION: ${{ steps.set-version.outputs.VERSION }} + with: + name: cheqd-node_${{ env.VERSION }}_amd64.deb + path: build-tools/output/cheqd-node_${{ env.VERSION }}_amd64.deb + + build-docker-images: + name: "Docker images" + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v3 + with: + fetch-depth: 0 # Required to fetch version + + - name: Build cheqd-cli Docker image 'cheqd-noded' as entrypoint + # TODO: Get rid of UID and GID + run: docker build --target base -t cheqd-cli -f docker/Dockerfile --build-arg UID="$(id -u)" --build-arg GID="$(id -g)" . + + - name: Build cheqd-node Docker image with 'node-start' as entrypoint + run: docker build --target node -t cheqd-node -f docker/Dockerfile --build-arg UID="$(id -u)" --build-arg GID="$(id -g)" . + + - name: Save cheqd-cli Docker image + run: docker save -o cheqd-cli-image.tar cheqd-cli + + - name: Store cheqd-cli artifact + uses: actions/upload-artifact@v3 + with: + name: cheqd-cli-image.tar + path: cheqd-cli-image.tar + + - name: Save cheqd-node Docker image + run: docker save -o cheqd-node-image.tar cheqd-node + + - name: Store cheqd-node artifact + uses: actions/upload-artifact@v3 + with: + name: cheqd-node-image.tar + path: cheqd-node-image.tar diff --git a/.github/workflows/dispatch.yml b/.github/workflows/dispatch.yml index 77b7e1c7d..de5529219 100644 --- a/.github/workflows/dispatch.yml +++ b/.github/workflows/dispatch.yml @@ -1,11 +1,32 @@ name: "Workflow Dispatch" +on: push +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true -on: [ push, workflow_dispatch ] jobs: + call-lint: + name: "Lint" uses: ./.github/workflows/lint.yml - call-test: + call-build: + name: "Build" needs: call-lint + uses: ./.github/workflows/build.yml + + call-test: + name: "Test" + needs: call-build uses: ./.github/workflows/test.yml + with: + VERSION: ${{ needs.call-build.outputs.VERSION }} + + call-release: + name: "Release" + needs: [call-test, call-build] + if: startsWith(github.ref, 'refs/tags/v') + uses: ./.github/workflows/release.yml + with: + RELEASE_VERSION: ${{ needs.call-build.outputs.VERSION }} diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index 4377cc28f..25ef01008 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -1,55 +1,64 @@ -name: Lint +name: "Lint" on: workflow_call defaults: run: shell: bash -concurrency: - group: ${{ github.workflow }}-${{ github.ref }} - cancel-in-progress: true + jobs: sh-euox-pipefail-check: - name: "Lint: Check for 'set -euox pipefail' in shell scripts" + name: "Shell pipefail check" runs-on: ubuntu-latest + steps: - uses: actions/checkout@v3 - - name: Check that all shell scripts use 'set -euox pipefail' - run: bash .github/scripts/ensure_set_euox_pipefail.sh - - # We can't use VALIDATE_GO from super linter because of this issue: - # https://github.com/github/super-linter/issues/143 - go-lint: - name: "Lint: Golang" - runs-on: ubuntu-latest - steps: - - uses: actions/setup-go@v2 - - uses: actions/checkout@v3 - - name: golangci-lint - uses: golangci/golangci-lint-action@v2 - with: - version: latest - args: --timeout 5m0s + - name: Run 'set -euox pipefail' check + run: bash ./.github/scripts/ensure_set_euox_pipefail.sh + md-link-check: - name: "Lint: Check for broken Markdown links" + name: "Broken Markdown links" runs-on: ubuntu-latest + steps: - uses: actions/checkout@v3 - - name: Check if Markdown links are valid + + - name: Run Markdown link check uses: gaurav-nelson/github-action-markdown-link-check@v1 with: config-file: '.github/linters/mlc_config.json' use-quiet-mode: 'yes' use-verbose-mode: 'yes' + + go-lint: + # We can't use VALIDATE_GO from super linter because of this issue: + # https://github.com/github/super-linter/issues/143 + name: "Golang" + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v3 + + - uses: actions/setup-go@v3 + with: + go-version: '1.17' + + - name: Run golangci-lint + uses: golangci/golangci-lint-action@v2 + with: + version: latest + args: --timeout 5m0s super-lint: - name: "Lint: Super Linter" + name: "Super Linter" runs-on: ubuntu-latest + steps: - uses: actions/checkout@v3 with: fetch-depth: 0 # Required to fetch version + - name: Run Super Linter uses: github/super-linter/slim@v4 env: diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 2638f7c89..8377ba499 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -1,164 +1,53 @@ -name: Release - +name: "Release" on: - push: - tags: - - "v[0-9]+.[0-9]+.[0-9]+" + workflow_call: + inputs: + RELEASE_VERSION: + description: "Release version number" + type: string +defaults: + run: + shell: bash + jobs: - wait-for-all-previous: - name: Wait for test and lint workflows - runs-on: ubuntu-20.04 - steps: - - name: Wait for all jobs finished on workflow Test and Lint - uses: lewagon/wait-on-check-action@v1.0.0 - with: - ref: ${{ github.sha }} - running-workflow-name: 'Wait for test and lint workflows' - repo-token: ${{ secrets.GITHUB_TOKEN }} - wait-interval: 30 - # ToDo: Get rid of it while moving test images to volumes - # This duplicating is needed because we have to pass GID and UID params - # while building docker images for tests - build-node-images: - name: Build cheqd-node and cheqd-cli images - runs-on: ubuntu-20.04 - needs: wait-for-all-previous + release-packages: + name: "Release tar/Debian packages" + runs-on: ubuntu-latest + permissions: + contents: write + env: + RELEASE_VERSION: ${{ inputs.RELEASE_VERSION }} + steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v3 with: fetch-depth: 0 # Required to fetch version - - name: Build cheqd-cli with 'cheqd-noded' as entrypoint - run: docker build --target base -t cheqd-cli -f docker/Dockerfile . - - - name: Build cheqd-node with 'node-start' as entrypoint - run: docker build --target node -t cheqd-node -f docker/Dockerfile . - - - name: Save cheqd-cli - run: docker save -o cheqd-cli-image.tar cheqd-cli - - - name: Save cheqd-node - run: docker save -o cheqd-node-image.tar cheqd-node - - - name: Store cheqd-cli artifact - uses: actions/upload-artifact@v2 - with: - name: cheqd-cli-image.tar - path: cheqd-cli-image.tar - - - name: Store cheqd-node artifact - uses: actions/upload-artifact@v2 + - name: Download node binary from build stage + uses: actions/download-artifact@v3 with: - name: cheqd-node-image.tar - path: cheqd-node-image.tar - - pre-release: - name: Pre-release - runs-on: ubuntu-20.04 - environment: Pre-Release - needs: build-node-images - outputs: - RCVERSION: ${{ steps.set-version.outputs.RCVERSION }} - steps: - - name: Get artifact from tests - uses: dawidd6/action-download-artifact@v2 - with: - workflow: test.yml name: cheqd-noded - name: Restore binary permissions run: sudo chmod +x cheqd-noded - - name: Set RC version - id: set-version - run: | - VERSION=$(./cheqd-noded version 2>&1) - echo "::set-output name=RCVERSION::$VERSION-rc" - - push-rc-images: - name: Push RC images - runs-on: ubuntu-20.04 - needs: pre-release - env: - RCVERSION: ${{ needs.pre-release.outputs.RCVERSION }} - REGISTRY: ghcr.io - steps: - - name: Get cheqd-cli image from tests - uses: actions/download-artifact@v2 - with: - name: cheqd-cli-image.tar - - - name: Get cheqd-node image from tests - uses: actions/download-artifact@v2 - with: - name: cheqd-node-image.tar - - - name: Load cli image - run: docker load -i cheqd-cli-image.tar - - - name: Load node image - run: docker load -i cheqd-node-image.tar - - - name: Log in to the Container registry - uses: docker/login-action@v1 - with: - registry: ${{ env.REGISTRY }} - username: ${{ github.actor }} - password: ${{ secrets.GITHUB_TOKEN }} - - - name: Push cheqd-node image - run: | - docker tag cheqd-node ghcr.io/${{ github.repository }}:${{ env.RCVERSION }} - docker push ghcr.io/${{ github.repository }}:${{ env.RCVERSION }} - docker tag cheqd-node ghcr.io/${{ github.repository }}:latest - docker push ghcr.io/${{ github.repository }}:latest - - - name: Push cheqd-cli image - run: | - docker tag cheqd-cli ghcr.io/${{ github.repository_owner }}/cheqd-cli:${{ env.RCVERSION }} - docker push ghcr.io/${{ github.repository_owner }}/cheqd-cli:${{ env.RCVERSION }} - docker tag cheqd-cli ghcr.io/${{ github.repository_owner }}/cheqd-cli:latest - docker push ghcr.io/${{ github.repository_owner }}/cheqd-cli:latest - - push-rc-debs: - name: Push RC packages - runs-on: ubuntu-20.04 - needs: pre-release - env: - RCVERSION: ${{ needs.pre-release.outputs.RCVERSION }} - steps: - - uses: actions/checkout@v2 - - - name: Get artifact from tests - uses: dawidd6/action-download-artifact@v2 - with: - workflow: test.yml - name: cheqd-noded - - name: Restore binary permissions - run: sudo chmod +x cheqd-noded - - name: Install fpm run: | sudo apt-get install ruby ruby-dev rubygems build-essential sudo gem install --no-document fpm - - name: Build RC deb + - name: Build Debian package for release working-directory: ./build-tools run: | - ./build-deb.sh "../cheqd-noded" "${{ env.RCVERSION }}" + ./build-deb.sh "../cheqd-noded" - - name: Store RC deb package artifact - uses: actions/upload-artifact@v2 + - name: Store Debian package artifact for release + uses: actions/upload-artifact@v3 with: - name: cheqd-node_${{ env.RCVERSION }}_amd64.deb - path: build-tools/output/cheqd-node_${{ env.RCVERSION }}_amd64.deb - - - name: Build tar - working-directory: ./build-tools - run: | - ./build-tar.sh "../cheqd-noded" "${{ env.RCVERSION }}" + name: cheqd-node_${{ env.RELEASE_VERSION }}_amd64.deb + path: build-tools/output/cheqd-node_${{ env.RELEASE_VERSION }}_amd64.deb - uses: "marvinpinto/action-automatic-releases@latest" with: @@ -166,102 +55,29 @@ jobs: automatic_release_tag: "${{ env.GITHUB_REF_NAME }}" prerelease: true draft: true - title: "v${{ env.RCVERSION }}" - files: | - build-tools/output/cheqd-node_${{ env.RCVERSION }}_amd64.deb - build-tools/output/cheqd-node_${{ env.RCVERSION }}.tar.gz - - release: - name: Release - runs-on: ubuntu-20.04 - environment: Release - needs: [ push-rc-debs, push-rc-images] - outputs: - VERSION: ${{ steps.set-version.outputs.VERSION }} - steps: - - name: Get artifact from tests - uses: dawidd6/action-download-artifact@v2 - with: - workflow: test.yml - name: cheqd-noded - - - name: Restore binary permissions - run: sudo chmod +x cheqd-noded - - - name: Set release version - id: set-version - run: | - VERSION=$(./cheqd-noded version 2>&1) - echo "::set-output name=VERSION::$VERSION" - - push-release-debs: - name: Push release packages - runs-on: ubuntu-20.04 - needs: release - env: - VERSION: ${{ needs.release.outputs.VERSION }} - steps: - - uses: actions/checkout@v2 - - - name: Get artifact from tests - uses: dawidd6/action-download-artifact@v2 - with: - workflow: test.yml - name: cheqd-noded - - - name: Restore binary permissions - run: sudo chmod +x cheqd-noded - - - name: Install fpm - run: | - sudo apt-get install ruby ruby-dev rubygems build-essential - sudo gem install --no-document fpm - - - name: Build release deb - working-directory: ./build-tools - run: | - ./build-deb.sh "../cheqd-noded" - - - name: Build tar - working-directory: ./build-tools - run: | - ./build-tar.sh "../cheqd-noded" - - - uses: "marvinpinto/action-automatic-releases@latest" - with: - repo_token: "${{ secrets.GITHUB_TOKEN }}" - automatic_release_tag: "${{ env.GITHUB_REF_NAME }}" - prerelease: true - title: "v${{ env.VERSION }}" files: | - build-tools/output/cheqd-node_${{ env.VERSION }}_amd64.deb - build-tools/output/cheqd-node_${{ env.VERSION }}.tar.gz + build-tools/output/cheqd-node_${{ env.RELEASE_VERSION }}_amd64.deb + cheqd-noded - push-release-images: - name: Push release images - runs-on: ubuntu-20.04 - needs: release + release-docker-images: + name: "Publish Docker images for new version" + runs-on: ubuntu-latest env: - VERSION: ${{ needs.release.outputs.VERSION }} REGISTRY: ghcr.io + RELEASE_VERSION: ${{ inputs.RELEASE_VERSION }} + steps: - - name: Get cheqd-cli image from tests - uses: actions/download-artifact@v2 - with: - name: cheqd-cli-image.tar - - - name: Get cheqd-node image from tests - uses: actions/download-artifact@v2 + - uses: actions/checkout@v3 with: - name: cheqd-node-image.tar + fetch-depth: 0 # Required to fetch version - - name: Load cli image - run: docker load -i cheqd-cli-image.tar + - name: Build cheqd-cli Docker image 'cheqd-noded' as entrypoint and no build args + run: docker build --target base -t cheqd-cli -f docker/Dockerfile . - - name: Load node image - run: docker load -i cheqd-node-image.tar + - name: Build cheqd-node Docker image with 'node-start' as entrypoint and no build args + run: docker build --target node -t cheqd-node -f docker/Dockerfile . - - name: Log in to the Container registry + - name: Login to Container registry uses: docker/login-action@v1 with: registry: ${{ env.REGISTRY }} @@ -270,14 +86,14 @@ jobs: - name: Push cheqd-node image run: | - docker tag cheqd-node ghcr.io/${{ github.repository }}:${{ env.VERSION }} - docker push ghcr.io/${{ github.repository }}:${{ env.VERSION }} + docker tag cheqd-node ghcr.io/${{ github.repository }}:${{ env.RELEASE_VERSION }} docker tag cheqd-node ghcr.io/${{ github.repository }}:latest + docker push ghcr.io/${{ github.repository }}:${{ env.RELEASE_VERSION }} docker push ghcr.io/${{ github.repository }}:latest - name: Push cheqd-cli image run: | - docker tag cheqd-cli ghcr.io/${{ github.repository_owner }}/cheqd-cli:${{ env.VERSION }} - docker push ghcr.io/${{ github.repository_owner }}/cheqd-cli:${{ env.VERSION }} + docker tag cheqd-cli ghcr.io/${{ github.repository_owner }}/cheqd-cli:${{ env.RELEASE_VERSION }} docker tag cheqd-cli ghcr.io/${{ github.repository_owner }}/cheqd-cli:latest + docker push ghcr.io/${{ github.repository_owner }}/cheqd-cli:${{ env.RELEASE_VERSION }} docker push ghcr.io/${{ github.repository_owner }}/cheqd-cli:latest diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index ecd1b4f8d..e432ad8c3 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -1,248 +1,87 @@ -name: "Build and Test" - +name: "Test" on: workflow_call: - paths-ignore: - - '**.md' - - 'docs/**' - - 'architecture/**' - - '.gitbook/**' - + inputs: + VERSION: + description: "Build version number" + type: string +defaults: + run: + shell: bash env: NODE_CONFIGS_BASE: "/home/runner/work/cheqd-node/cheqd-node/docker/localnet/network-config" -jobs: - - run-unit-test: - name: Run unit tests - runs-on: ubuntu-20.04 - needs: wait-for-lint - steps: - - name: Set up Go 1.17 - uses: actions/setup-go@v2 - with: - go-version: 1.17 - - - uses: actions/checkout@v2 - - - name: Run tests - run: go test -v ./... - - build-binary: - name: "Build: binary" - runs-on: ubuntu-20.04 - needs: wait-for-lint - outputs: - VERSION: ${{ steps.set-version.outputs.VERSION }} - steps: - - name: Set up Go 1.17 - uses: actions/setup-go@v2 - with: - go-version: 1.17 - - - name: Get go protoc compiler plugins - env: - GOLANG_PROTOBUF_VERSION: 1.3.5 - GOGO_PROTOBUF_VERSION: 1.3.2 - GRPC_GATEWAY_VERSION: 1.14.7 - # Taken from: tendermintdev/sdk-proto-gen:v0.2 - run: | - go get \ - github.com/golang/protobuf/protoc-gen-go@v"$GOLANG_PROTOBUF_VERSION" \ - github.com/gogo/protobuf/protoc-gen-gogo@v"$GOGO_PROTOBUF_VERSION" \ - github.com/gogo/protobuf/protoc-gen-gogofast@v"$GOGO_PROTOBUF_VERSION" \ - github.com/gogo/protobuf/protoc-gen-gogofaster@v"$GOGO_PROTOBUF_VERSION" \ - github.com/grpc-ecosystem/grpc-gateway/protoc-gen-grpc-gateway@v"$GRPC_GATEWAY_VERSION" \ - github.com/grpc-ecosystem/grpc-gateway/protoc-gen-swagger@v"$GRPC_GATEWAY_VERSION" \ - github.com/regen-network/cosmos-proto/protoc-gen-gocosmos@latest - - - name: Install buf - env: - PREFIX: "/usr/local" - VERSION: "1.0.0-rc8" - run: | - curl -sSL "https://github.com/bufbuild/buf/releases/download/v${VERSION}/buf-$(uname -s)-$(uname -m).tar.gz" | \ - sudo tar -xvzf - -C "${PREFIX}" --strip-components 1 - - - uses: actions/checkout@v2 - with: - fetch-depth: 0 # Required to fetch version - - - name: Build - run: | - make proto-gen build - - - name: Store artifact - uses: actions/upload-artifact@v2 - with: - name: cheqd-noded - path: build/cheqd-noded - - - name: Set version - id: set-version - run: | - VERSION=$(build/cheqd-noded version 2>&1) - echo "::set-output name=VERSION::$VERSION" - - build-tar-package: - name: "Build: tar package" - runs-on: ubuntu-20.04 - needs: build-binary - env: - VERSION: ${{ needs.build-binary.outputs.VERSION }} - steps: - - uses: actions/checkout@v2 - - - name: Load binary artifact - uses: actions/download-artifact@v2 - with: - name: cheqd-noded - - - name: Restore binary permissions - run: sudo chmod +x cheqd-noded - - - name: Build tar - working-directory: ./build-tools - run: | - ./build-tar.sh "../cheqd-noded" - - - name: Store tar package artifact - uses: actions/upload-artifact@v2 - with: - name: cheqd-node_${{ env.VERSION }}.tar.gz - path: build-tools/output/cheqd-node_${{ env.VERSION }}.tar.gz - - build-deb-package: - name: "Build: deb package" - runs-on: ubuntu-20.04 - needs: build-binary - env: - VERSION: ${{ needs.build-binary.outputs.VERSION }} - steps: - - uses: actions/checkout@v2 - - - name: Load binary artifact - uses: actions/download-artifact@v2 - with: - name: cheqd-noded - - - name: Restore binary permissions - run: sudo chmod +x cheqd-noded - - - name: Install fpm - run: | - sudo apt-get install ruby ruby-dev rubygems build-essential - sudo gem install --no-document fpm - - - name: Build deb - working-directory: ./build-tools - run: | - ./build-deb.sh "../cheqd-noded" - - - name: Store deb package artifact - uses: actions/upload-artifact@v2 - with: - name: cheqd-node_${{ env.VERSION }}_amd64.deb - path: build-tools/output/cheqd-node_${{ env.VERSION }}_amd64.deb - - build-node-images: - name: "Build: cheqd-node and cheqd-cli images" - runs-on: ubuntu-20.04 - needs: wait-for-lint - steps: - - uses: actions/checkout@v2 - with: - fetch-depth: 0 # Required to fetch version - - name: Build cheqd-cli with 'cheqd-noded' as entrypoint - # TODO: Get rid of UID and GID - run: docker build --target base -t cheqd-cli -f docker/Dockerfile --build-arg UID="$(id -u)" --build-arg GID="$(id -g)" . - - - name: Build cheqd-node with 'node-start' as entrypoint - run: docker build --target node -t cheqd-node -f docker/Dockerfile --build-arg UID="$(id -u)" --build-arg GID="$(id -g)" . - - - name: Save cheqd-cli - run: docker save -o cheqd-cli-image.tar cheqd-cli - - - name: Save cheqd-node - run: docker save -o cheqd-node-image.tar cheqd-node - - - name: Store cheqd-cli artifact - uses: actions/upload-artifact@v2 - with: - name: cheqd-cli-image.tar - path: cheqd-cli-image.tar - - - name: Store cheqd-node artifact - uses: actions/upload-artifact@v2 - with: - name: cheqd-node-image.tar - path: cheqd-node-image.tar +jobs: - test-new-node-setup-from-deb: - name: "Test: New node setup from deb" - runs-on: ubuntu-20.04 - needs: [build-binary, build-deb-package, build-node-images] + test-new-node-setup: + name: "New node setup" + runs-on: ubuntu-latest env: - VERSION: ${{ needs.build-binary.outputs.VERSION }} + VERSION: ${{ inputs.VERSION }} + steps: - - name: Download deb - uses: actions/download-artifact@v2 + - uses: actions/checkout@v3 + + - name: Download Debian package + uses: actions/download-artifact@v3 with: name: cheqd-node_${{ env.VERSION }}_amd64.deb - - name: Install deb + - name: Install Debian package run: sudo CHEQD_HOME_DIR=/home/runner/cheqd dpkg -i cheqd-node_${{ env.VERSION }}_amd64.deb - - name: Download node image - uses: actions/download-artifact@v2 + - name: Download node Docker image + uses: actions/download-artifact@v3 with: name: cheqd-node-image.tar - - name: Load node image + - name: Load node Docker image run: docker load -i cheqd-node-image.tar - - name: Check out - uses: actions/checkout@v2 - - - name: Set up 4 validators + 2 observers node docker pool + - name: Setup 4 Validators + 2 Observers Docker localnet working-directory: ./docker/localnet run: | + set -euo pipefail bash gen-network-config.sh CHEQD_NODE_IMAGE=cheqd-node CHEQD_NODE_VERSION=latest docker compose up -d - - name: Wait for chain - run: bash tests/tools/wait-for-chain.sh + - name: Check all Docker localnet nodes are active + run: | + set -euo pipefail + bash ./tests/tools/wait-for-chain.sh - - name: Add observer node + - name: Add an Observer node using Debian package working-directory: ./tests/e2e-complex/deb-install run: | - ./add-observer.sh - cheqd-noded status -n tcp://localhost:26677 2>&1 - ./wait.sh "[[ $(cheqd-noded status -n 'tcp://localhost:26677' 2>&1 | wc -l) == 1 ]] && echo "New node returns status!"" + set -euo pipefail + bash add-observer.sh + bash wait.sh "[[ $(cheqd-noded status -n 'tcp://localhost:26677' 2>&1 | wc -l) == 1 ]] && echo "Observer node is up"" - - name: Promote observer to validator + - name: Promote Observer to Validator working-directory: ./tests/e2e-complex/deb-install run: | - ./promote-validator.sh - cheqd-noded query staking validators --node "http://localhost:26657" | sed -nr 's/.*status: (.*?).*/\1/p' "$1" | while read -r x; do [[ "BOND_STATUS_BONDED" == "$x" ]] && echo "Validator's status is bonded!" || exit 1 ; done + bash promote-validator.sh + bash check-promotion.sh # shellcheck disable=SC2016 - ./wait.sh '[[ $(curl -s localhost:26657/block | sed -nr '"'"'s/.*signature": (.*?).*/\1/p'"'"' | wc -l) == 5 ]] && echo "There are 5 validators signatures in block!"' + bash wait.sh '[[ $(curl -s localhost:26657/block | sed -nr '"'"'s/.*signature": (.*?).*/\1/p'"'"' | wc -l) == 5 ]] && echo "There are 5 validators signatures in block!"' # shellcheck disable=SC2016 - ./wait.sh '[[ $(curl -s localhost:26657/block | sed -nr '"'"'s/.*(signature": null).*/\1/p'"'"' | wc -l) == 0 ]] && echo "There are no null signatures in block!"' + bash wait.sh '[[ $(curl -s localhost:26657/block | sed -nr '"'"'s/.*(signature": null).*/\1/p'"'"' | wc -l) == 0 ]] && echo "There are no null signatures in block!"' - name: Gather logs on failure if: ${{ failure() }} run: | journalctl -ex | grep cheqd - run-python-based-integration-tests: - name: "Test: Run python based cosmos and identity tests" - runs-on: ubuntu-20.04 - needs: [build-node-images, build-binary] + python-integration-tests: + name: "Python based Cosmos and identity tests" + runs-on: ubuntu-latest + steps: - - name: Load binary artifact - uses: actions/download-artifact@v2 + - uses: actions/checkout@v3 + + - name: Load node binary artifact + uses: actions/download-artifact@v3 with: name: cheqd-noded path: /home/runner/.local/bin @@ -250,38 +89,36 @@ jobs: - name: Restore binary permissions run: sudo chmod +x /home/runner/.local/bin/cheqd-noded - - name: Download node image - uses: actions/download-artifact@v2 + - name: Download node Docker image + uses: actions/download-artifact@v3 with: name: cheqd-node-image.tar - - name: Load node image + - name: Load node Docker image run: docker load -i cheqd-node-image.tar - - name: Check out - uses: actions/checkout@v2 - - - name: Set up 4 validators + 2 observers node docker pool + - name: Setup 4 Validators + 2 Observers Docker localnet working-directory: ./docker/localnet run: | bash gen-network-config.sh CHEQD_NODE_IMAGE=cheqd-node CHEQD_NODE_VERSION=latest docker compose up -d - - name: Wait for chain + - name: Check all Docker localnet nodes are active run: bash tests/tools/wait-for-chain.sh - - name: Set up test environment + - name: Setup Python environment working-directory: ./tests/e2e-pytest run: | + set -euo pipefail pip3 install -r requirements.txt >> /dev/null - cp -Rf "${NODE_CONFIGS_BASE}"/validator-0/. /home/runner/.cheqdnode - cp -Rf "${NODE_CONFIGS_BASE}"/validator-1/. /home/runner/.cheqdnode - sudo chmod -R 777 /home/runner + cp -Rf "${NODE_CONFIGS_BASE}"/validator-0/. /home/runner/.cheqdnode/ + cp -Rf "${NODE_CONFIGS_BASE}"/validator-1/. /home/runner/.cheqdnode/ + sudo chmod -R 775 /home/runner/ - - name: Run cosmos-related tests + - name: Run Cosmos-related tests working-directory: ./tests/e2e-pytest run: | - set -euxo pipefail + set -euo pipefail OP0_ADDRESS=$(cheqd-noded keys list --keyring-backend "test" --home "${NODE_CONFIGS_BASE}/validator-0" | sed -nr 's/.*address: (.*?).*/\1/p' | sed 's/\r//g') export OP0_ADDRESS OP1_ADDRESS=$(cheqd-noded keys list --keyring-backend "test" --home "${NODE_CONFIGS_BASE}/validator-1" | sed -nr 's/.*address: (.*?).*/\1/p' | sed 's/\r//g') @@ -291,19 +128,22 @@ jobs: - name: Run identity-related tests # TODO: Move into separate stage? working-directory: ./tests/e2e-pytest run: | + set -euo pipefail OP0_ADDRESS=$(cheqd-noded keys list --keyring-backend "test" | sed -nr 's/.*address: (.*?).*/\1/p' | sed -n 1p | sed 's/\r//g') export OP0_ADDRESS OP1_ADDRESS=$(cheqd-noded keys list --keyring-backend "test" | sed -nr 's/.*address: (.*?).*/\1/p' | sed -n 2p | sed 's/\r//g') export OP1_ADDRESS pytest -v -rP test_identity.py - run-bash-based-integration-tests: - name: "Test: Run bash based identity tests" - runs-on: ubuntu-20.04 - needs: [build-binary, build-node-images] + bash-integration-tests: + name: "Bash-based identity tests" + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + - name: Load binary artifact - uses: actions/download-artifact@v2 + uses: actions/download-artifact@v3 with: name: cheqd-noded path: /home/runner/.local/bin @@ -311,64 +151,61 @@ jobs: - name: Restore binary permissions run: sudo chmod +x /home/runner/.local/bin/cheqd-noded - - name: Download node image - uses: actions/download-artifact@v2 + - name: Download node Docker image + uses: actions/download-artifact@v3 with: name: cheqd-node-image.tar - - name: Load node image + - name: Load node Docker image run: docker load -i cheqd-node-image.tar - - name: Check out - uses: actions/checkout@v2 - - - name: Set up 4 validators + 2 observers node docker pool + - name: Setup 4 Validators + 2 Observers Docker localnet working-directory: ./docker/localnet run: | bash gen-network-config.sh CHEQD_NODE_IMAGE=cheqd-node CHEQD_NODE_VERSION=latest docker compose up -d - - name: Wait for chain - run: bash tests/tools/wait-for-chain.sh + - name: Check all Docker localnet nodes are active + run: bash ./tests/tools/wait-for-chain.sh - name: Import keys - working-directory: docker/localnet + working-directory: ./docker/localnet run: | bash import_keys.sh - - name: Set up and run tests - working-directory: tests/e2e-bash + - name: Run tests + working-directory: ./tests/e2e-bash run: | bash run_all.sh - run-upgrade-test-positive-case: - name: "Test: Run positive case for upgrade" - runs-on: ubuntu-20.04 - needs: build-node-images + node-upgrade-test: + name: "Run positive case for upgrade" + runs-on: ubuntu-latest + steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v3 with: - fetch-depth: 0 # It's needed for getting the full version of package by `git describe` + fetch-depth: 0 - - name: Download cli image - uses: actions/download-artifact@v2 + - name: Download cheqd-cli Docker image + uses: actions/download-artifact@v3 with: name: cheqd-cli-image.tar - - name: Load cli image + - name: Load cheqd-cli Docker image run: docker load -i cheqd-cli-image.tar - - name: Chown for current user + - name: Give current user ownership run: sudo chown "$USER":"$USER" . - - name: Prepare nodes for checking upgrade + - name: Prepare nodes for upgrade working-directory: ./tests/e2e-complex/upgrade run: bash prepare.sh - - name: Initiate the upgrade process + - name: Initiate upgrade working-directory: ./tests/e2e-complex/upgrade run: bash initiate_upgrade.sh - - name: Make the upgrade and check results + - name: Check for successful upgrade working-directory: ./tests/e2e-complex/upgrade run: bash upgrade_and_check.sh diff --git a/build-tools/build-deb.sh b/build-tools/build-deb.sh index 4cc055599..88c7bf84c 100755 --- a/build-tools/build-deb.sh +++ b/build-tools/build-deb.sh @@ -20,8 +20,8 @@ PKG_NAME="cheqd-node" BUILD_DIR="build" OUTPUT_DIR="output" -mkdir "${BUILD_DIR}" -mkdir "${OUTPUT_DIR}" +mkdir -p "${BUILD_DIR}" +mkdir -p "${OUTPUT_DIR}" # Prepare content PACKAGE_CONTENT="${BUILD_DIR}/deb-package-content" diff --git a/docker/Dockerfile b/docker/Dockerfile index 81d6d7bf7..96be75059 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -52,6 +52,9 @@ RUN make proto-gen build ############################################################### FROM ubuntu:focal AS base +LABEL org.opencontainers.image.description "cheqd CLI Docker image" +LABEL org.opencontainers.image.source "https://github.com/cheqd/cheqd-node" +LABEL org.opencontainers.image.documentation "https://docs.cheqd.io/node" # Copy compiled node binary from Stage 1 COPY --from=builder /app/build/cheqd-noded /bin @@ -83,6 +86,9 @@ ENTRYPOINT [ "cheqd-noded" ] ############################################################### FROM base AS node +LABEL org.opencontainers.image.description "cheqd Node Docker image" +LABEL org.opencontainers.image.source "https://github.com/cheqd/cheqd-node" +LABEL org.opencontainers.image.documentation "https://docs.cheqd.io/node" # Set runner script COPY --chown=cheqd:cheqd docker/entrypoint.sh /bin/node-start diff --git a/docker/localnet/docker-compose.env b/docker/localnet/docker-compose.env index 921c9c241..d5f7bae95 100644 --- a/docker/localnet/docker-compose.env +++ b/docker/localnet/docker-compose.env @@ -7,6 +7,6 @@ CHEQD_NODE_IMAGE="ghcr.io/cheqd/cheqd-node" # Define cheqd-noded software release version -# Current MAINNET recommended version: v0.3.1 -# Current TESTNET recommended version: v0.3.1 -CHEQD_NODE_VERSION="0.3.1" +# Current MAINNET recommended version: v0.5.0 +# Current TESTNET recommended version: v0.5.0 +CHEQD_NODE_VERSION="0.5.0" diff --git a/docker/persistent-chains/docker-compose.env b/docker/persistent-chains/docker-compose.env index ac20ceccb..c147e8b63 100644 --- a/docker/persistent-chains/docker-compose.env +++ b/docker/persistent-chains/docker-compose.env @@ -11,9 +11,9 @@ CHEQD_NETWORK="mainnet" # Define cheqd-noded software release version -# Current MAINNET recommended version: v0.3.1 -# Current TESTNET recommended version: v0.3.1 -CHEQD_NODE_VERSION="0.4.1" +# Current MAINNET recommended version: v0.5.0 +# Current TESTNET recommended version: v0.5.0 +CHEQD_NODE_VERSION="0.5.0" ############################################################### diff --git a/tests/e2e-complex/deb-install/add-observer.sh b/tests/e2e-complex/deb-install/add-observer.sh index 345009d69..aee89961c 100755 --- a/tests/e2e-complex/deb-install/add-observer.sh +++ b/tests/e2e-complex/deb-install/add-observer.sh @@ -11,7 +11,7 @@ sudo -u cheqd cheqd-noded configure p2p persistent-peers "${PERSISTENT_PEERS}" sudo cp "${NODE_CONFIGS_BASE}/validator-0/config/genesis.json" "/home/runner/cheqd/.cheqdnode/config" -sudo chmod -R 777 "/home/runner/cheqd/.cheqdnode" +sudo chmod -R 755 "/home/runner/cheqd/.cheqdnode" # Configure ports because they conflict with localnet sudo -u cheqd cheqd-noded configure p2p laddr "tcp://0.0.0.0:26676" diff --git a/tests/e2e-complex/deb-install/check-promotion.sh b/tests/e2e-complex/deb-install/check-promotion.sh new file mode 100644 index 000000000..ba710cd5e --- /dev/null +++ b/tests/e2e-complex/deb-install/check-promotion.sh @@ -0,0 +1,13 @@ +#!/bin/bash + +set -euox pipefail + +all_validators_cmd='cheqd-noded query staking validators --node http://localhost:26657' + +amount_bonded="$(${all_validators_cmd} | grep -c BOND_STATUS_BONDED | xargs)" +amount_all="$(${all_validators_cmd} | grep -c status | xargs)" + +if [ "${amount_all}" != "${amount_bonded}" ]; +then + exit 1 +fi \ No newline at end of file diff --git a/tests/e2e-complex/deb-install/promote-validator.sh b/tests/e2e-complex/deb-install/promote-validator.sh index 6f5bc17df..e2b40e45f 100755 --- a/tests/e2e-complex/deb-install/promote-validator.sh +++ b/tests/e2e-complex/deb-install/promote-validator.sh @@ -13,7 +13,9 @@ sudo -u cheqd cheqd-noded keys add node5-operator --keyring-backend "test" OP5_ADDRESS=$(sudo -u cheqd cheqd-noded keys list --keyring-backend "test"| sed -nr 's/.*address: (.*?).*/\1/p' | sed -n 1p | sed 's/\r//g') NODE5_PUBKEY=$(sudo -u cheqd cheqd-noded tendermint show-validator | sed 's/\r//g') + # Send tokens from operator0 cheqd-noded tx bank send "${OP0_ADDRESS}" "${OP5_ADDRESS}" 1100000000000000ncheq --chain-id cheqd --fees 5000000ncheq --node "http://localhost:26657" -y --keyring-backend "test" --home "${NODE_CONFIGS_BASE}/validator-0" + # Send promote validator from operator5 sudo -u cheqd cheqd-noded tx staking create-validator --amount 1000000000000000ncheq --from node5-operator --chain-id cheqd --min-self-delegation="1" --gas-prices="25ncheq" --pubkey "${NODE5_PUBKEY}" --commission-max-change-rate="0.02" --commission-max-rate="0.02" --commission-rate="0.01" --gas 500000 --node "http://localhost:26657" -y --keyring-backend "test" diff --git a/tests/e2e-complex/upgrade/common.sh b/tests/e2e-complex/upgrade/common.sh index d7de40875..a473505dc 100644 --- a/tests/e2e-complex/upgrade/common.sh +++ b/tests/e2e-complex/upgrade/common.sh @@ -77,8 +77,8 @@ function docker_compose_down () { # Clean environment function clean_env () { - rm -rf node_configs - rm -f $FNAME_TXHASHES + sudo rm -rf node_configs + sudo rm -f $FNAME_TXHASHES } # Run command using local generated keys from node_configs/client @@ -86,8 +86,8 @@ function local_client_tx () { cheqd_noded_docker "$@" --home node_configs/client/.cheqdnode/ --keyring-backend test } -function make_777 () { - sudo chmod -R 777 node_configs +function make_775 () { + sudo chmod -R 775 node_configs } diff --git a/tests/e2e-complex/upgrade/upgrade_and_check.sh b/tests/e2e-complex/upgrade/upgrade_and_check.sh index 8ec5589cc..9585faa1e 100755 --- a/tests/e2e-complex/upgrade/upgrade_and_check.sh +++ b/tests/e2e-complex/upgrade/upgrade_and_check.sh @@ -22,7 +22,7 @@ bash ../../tools/wait-for-chain.sh "$UPGRADE_HEIGHT" $((3 * VOTING_PERIOD)) docker_compose_down # Make all the data accessible -make_777 +make_775 # Start docker-compose with new base image on new version docker_compose_up "$CHEQD_IMAGE_TO" "$(pwd)" diff --git a/tests/e2e-pytest/helpers.py b/tests/e2e-pytest/helpers.py index a0adf7506..d868dc137 100644 --- a/tests/e2e-pytest/helpers.py +++ b/tests/e2e-pytest/helpers.py @@ -12,14 +12,14 @@ ENCODING = "utf-8" READ_BUFFER = 60000 -TEST_NET_NETWORK = "cheqd-testnet-2" +TEST_NET_NETWORK = "cheqd-testnet-4" LOCAL_NET_NETWORK = "cheqd" -TEST_NET_NODE_TCP = "--node 'tcp://seed1.us.testnet.cheqd.network:26657'" -TEST_NET_NODE_HTTP = "--node http://node1.eu.testnet.cheqd.network:26657/" -LOCAL_NET_NODE_TCP = "--node 'tcp://localhost:26657'" +TEST_NET_NODE_TCP = "--node tcp://rpc.cheqd.network:443" +TEST_NET_NODE_HTTP = "--node https://rpc.cheqd.network/" +LOCAL_NET_NODE_TCP = "--node tcp://localhost:26657" LOCAL_NET_NODE_HTTP = "--node http://localhost:26657/" -TEST_NET_DESTINATION = f"{TEST_NET_NODE_TCP} --chain-id 'cheqd-testnet-2'" -TEST_NET_DESTINATION_HTTP = f"{TEST_NET_NODE_HTTP} --chain-id 'cheqd-testnet-2'" +TEST_NET_DESTINATION = f"{TEST_NET_NODE_TCP} --chain-id 'cheqd-testnet-4'" +TEST_NET_DESTINATION_HTTP = f"{TEST_NET_NODE_HTTP} --chain-id 'cheqd-testnet-4'" LOCAL_NET_DESTINATION = f"{LOCAL_NET_NODE_TCP} --chain-id 'cheqd'" LOCAL_NET_DESTINATION_HTTP = f"{LOCAL_NET_NODE_HTTP} --chain-id 'cheqd'" GAS_AMOUNT = 90000 # 70000 throws `out of gas` sometimes