From 3c24fe07dbbc1f0cbec49ce29808ddf473960c62 Mon Sep 17 00:00:00 2001 From: Fraccaroli Gianmarco Date: Thu, 21 Jul 2022 16:17:21 +0200 Subject: [PATCH] ci: move to github actions --- .changelog/unreleased/ci/222-new-ci.md | 2 + .../improvements/1202-expectrl-logging.md | 3 + .../improvements/1239-hide-tm-log.md | 3 + .github/workflows/automation.yml | 45 +++ .github/workflows/build-and-test.yml | 295 ++++++++++++++++++ .github/workflows/build-tendermint.yml | 40 +++ .github/workflows/checks.yml | 79 +++++ .github/workflows/cron.yml | 72 +++++ .github/workflows/docker.yml | 59 ++++ .github/workflows/docs.yml | 113 +++++++ .github/workflows/release.yml | 107 +++++++ .github/workflows/scripts/audit.py | 115 +++++++ .github/workflows/scripts/publish-wasm.py | 73 +++++ .github/workflows/scripts/udeps.py | 132 ++++++++ .github/workflows/scripts/update-wasm.py | 88 ++++++ Cargo.lock | 1 + Makefile | 55 +++- apps/build.rs | 5 +- apps/src/lib/client/tx.rs | 33 +- apps/src/lib/node/ledger/mod.rs | 9 + apps/src/lib/node/ledger/tendermint_node.rs | 61 ++-- apps/src/lib/wasm_loader/mod.rs | 2 +- rust-toolchain.toml | 1 + scripts/ci/README.md | 19 -- scripts/ci/nix-check.sh | 26 -- scripts/ci/poetry.lock | 120 ------- scripts/ci/pyproject.toml | 15 - scripts/ci/update-drone-config.py | 59 ---- scripts/{install => }/get_tendermint.sh | 1 - scripts/make-package.sh | 2 +- shared/build.rs | 5 +- tests/Cargo.toml | 1 + tests/src/e2e/eth_bridge_tests.rs | 2 +- tests/src/e2e/gossip_tests.rs | 8 +- tests/src/e2e/helpers.rs | 137 +++++++- tests/src/e2e/ledger_tests.rs | 65 +--- tests/src/e2e/setup.rs | 264 ++++++++++------ wasm/checksums.json | 34 +- wasm/checksums.py | 9 +- wasm/wasm_source/Makefile | 4 +- wasm_for_tests/wasm_source/Makefile | 4 +- 41 files changed, 1686 insertions(+), 482 deletions(-) create mode 100644 .changelog/unreleased/ci/222-new-ci.md create mode 100644 .changelog/unreleased/improvements/1202-expectrl-logging.md create mode 100644 .changelog/unreleased/improvements/1239-hide-tm-log.md create mode 100644 .github/workflows/automation.yml create mode 100644 .github/workflows/build-and-test.yml create mode 100644 .github/workflows/build-tendermint.yml create mode 100644 .github/workflows/checks.yml create mode 100644 .github/workflows/cron.yml create mode 100644 .github/workflows/docker.yml create mode 100644 .github/workflows/docs.yml create mode 100644 .github/workflows/release.yml create mode 100644 .github/workflows/scripts/audit.py create mode 100644 .github/workflows/scripts/publish-wasm.py create mode 100644 .github/workflows/scripts/udeps.py create mode 100644 .github/workflows/scripts/update-wasm.py delete mode 100644 scripts/ci/README.md delete mode 100644 scripts/ci/nix-check.sh delete mode 100644 scripts/ci/poetry.lock delete mode 100644 scripts/ci/pyproject.toml delete mode 100644 scripts/ci/update-drone-config.py rename scripts/{install => }/get_tendermint.sh (99%) diff --git a/.changelog/unreleased/ci/222-new-ci.md b/.changelog/unreleased/ci/222-new-ci.md new file mode 100644 index 00000000000..50e7fec98ac --- /dev/null +++ b/.changelog/unreleased/ci/222-new-ci.md @@ -0,0 +1,2 @@ +- New CI using Github Actions + ([#222](https://github.com/anoma/namada/pull/222)) diff --git a/.changelog/unreleased/improvements/1202-expectrl-logging.md b/.changelog/unreleased/improvements/1202-expectrl-logging.md new file mode 100644 index 00000000000..1b4e3fdf224 --- /dev/null +++ b/.changelog/unreleased/improvements/1202-expectrl-logging.md @@ -0,0 +1,3 @@ +- Better logging for end-to-end tests, and logs are + stored to disk in the test's temporary working directory + ([#1202](https://github.com/anoma/anoma/pull/1202)) diff --git a/.changelog/unreleased/improvements/1239-hide-tm-log.md b/.changelog/unreleased/improvements/1239-hide-tm-log.md new file mode 100644 index 00000000000..fb4ade85ebf --- /dev/null +++ b/.changelog/unreleased/improvements/1239-hide-tm-log.md @@ -0,0 +1,3 @@ +- Hidden the stdout of Tendermint process by default. To include + it in the node's output, run with `ANOMA_TM_STDOUT=true` + ([#1239](https://github.com/anoma/anoma/pull/1239)) diff --git a/.github/workflows/automation.yml b/.github/workflows/automation.yml new file mode 100644 index 00000000000..1dc63f782a1 --- /dev/null +++ b/.github/workflows/automation.yml @@ -0,0 +1,45 @@ +name: Automation Tasks + +on: + issue_comment: + types: [created] + +permissions: + id-token: write + contents: write + +env: + GIT_LFS_SKIP_SMUDGE: 1 + +jobs: + tasks: + if: ${{ github.event.issue.pull_request }} + runs-on: ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + os: [ubuntu-latest] + command: [publish-wasm.py, update-wasm.py] + + steps: + - name: Configure AWS Credentials + uses: aws-actions/configure-aws-credentials@v1 + with: + role-to-assume: arn:aws:iam::375643557360:role/anoma-github-action-ci-master + aws-region: eu-west-1 + - uses: xt0rted/pull-request-comment-branch@v1 + id: comment-branch + - uses: actions/checkout@v3 + if: success() + with: + ref: ${{ steps.comment-branch.outputs.head_ref }} + - name: Run task + env: + GITHUB_CONTEXT: ${{ toJson(github) }} + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + GITHUB_READ_ORG_TOKEN: ${{ secrets.GT_READ_ORG }} + run: | + git config --global user.name 'github-actions[bot]' + git config --global user.email 'github-actions[bot]@users.noreply.github.com' + pip3 install ghapi >/dev/null 2>&1 + python3 .github/workflows/scripts/${{ matrix.command }} diff --git a/.github/workflows/build-and-test.yml b/.github/workflows/build-and-test.yml new file mode 100644 index 00000000000..ab4b2101544 --- /dev/null +++ b/.github/workflows/build-and-test.yml @@ -0,0 +1,295 @@ +name: Build and Test + +on: + push: + branches: + - main + pull_request: + workflow_dispatch: + +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number }} + cancel-in-progress: true + +permissions: + id-token: write + contents: read + +env: + GIT_LFS_SKIP_SMUDGE: 1 + + +jobs: + build-wasm: + timeout-minutes: 30 + runs-on: ${{ matrix.os }} + container: + image: ghcr.io/fraccaman/namada:wasm-0.6.1 + strategy: + fail-fast: false + matrix: + os: [ubuntu-latest] + wasm_cache_version: ["v1"] + + steps: + - name: Checkout repo + uses: actions/checkout@v3 + - name: Show rust toolchain info + run: rustup show + - name: Build WASM + run: cp wasm/checksums.json wasm/original-checksums.json && make build-wasm-scripts + - name: Upload wasm artifacts + uses: actions/upload-artifact@v3 + with: + name: wasm-${{ github.sha }} + path: | + wasm/tx_*.wasm + wasm/vp_*.wasm + wasm/checksums.json + - name: Test Wasm + run: make test-wasm + - name: Check wasm up-to-date + run: cmp -- wasm/checksums.json wasm/original-checksums.json + - name: Print diff + if: failure() + run: diff -y -W 150 wasm/checksums.json wasm/original-checksums.json --suppress-common-lines + + update-wasm: + runs-on: ${{ matrix.os }} + if: ${{ github.event_name == 'push' && github.ref == 'refs/heads/main' && needs.build-wasm.result == 'success' }} + timeout-minutes: 30 + needs: [build-wasm] + strategy: + fail-fast: false + matrix: + os: [ubuntu-latest] + + steps: + - name: Configure AWS Credentials + uses: aws-actions/configure-aws-credentials@v1 + with: + role-to-assume: arn:aws:iam::375643557360:role/anoma-github-action-ci-master + aws-region: eu-west-1 + - name: Download wasm artifacts + uses: actions/download-artifact@v3 + with: + name: wasm-${{ github.sha }} + path: ./wasm + - name: Update WASM + run: aws s3 sync wasm s3://$BUCKET_NAME --acl public-read --exclude "*" --include "*.wasm" --exclude "*/*" --region $AWS_REGION + env: + BUCKET_NAME: namada-wasm-master + AWS_REGION: eu-west-1 + + anoma: + runs-on: ${{ matrix.os }} + timeout-minutes: 40 + strategy: + fail-fast: false + matrix: + os: [ubuntu-latest] + nightly_version: [nightly-2022-05-20] + make: + - name: ABCI + suffix: '' + cache_key: anoma + cache_version: v1 + wait_for: anoma-release (ubuntu-latest, ABCI Release build, anoma-e2e-release, v1) + tendermint_artifact: tendermint-unreleased-29e5fbcc648510e4763bd0af0b461aed92c21f30 + + env: + CARGO_INCREMENTAL: 0 + RUST_BACKTRACE: full + RUSTC_WRAPPER: sccache + SCCACHE_CACHE_SIZE: 100G + SCCACHE_BUCKET: namada-sccache-master + + steps: + - name: Checkout repo + uses: actions/checkout@v3 + - name: Configure AWS Credentials + uses: aws-actions/configure-aws-credentials@v1 + with: + role-to-assume: arn:aws:iam::375643557360:role/anoma-github-action-ci-master + aws-region: eu-west-1 + - name: Install sccache (ubuntu-latest) + if: matrix.os == 'ubuntu-latest' + env: + LINK: https://github.com/mozilla/sccache/releases/download + SCCACHE_VERSION: v0.3.0 + run: | + SCCACHE_FILE=sccache-$SCCACHE_VERSION-x86_64-unknown-linux-musl + mkdir -p $HOME/.local/bin + curl -L "$LINK/$SCCACHE_VERSION/$SCCACHE_FILE.tar.gz" | tar xz + mv -f $SCCACHE_FILE/sccache $HOME/.local/bin/sccache + chmod +x $HOME/.local/bin/sccache + echo "$HOME/.local/bin" >> $GITHUB_PATH + - name: Install sccache (macos-latest) + if: matrix.os == 'macos-latest' + run: | + brew update + brew install sccache + - name: Setup rust toolchain + uses: oxidecomputer/actions-rs_toolchain@ad3f86084a8a5acf2c09cb691421b31cf8af7a36 + with: + profile: default + override: true + - name: Setup rust nightly + uses: oxidecomputer/actions-rs_toolchain@ad3f86084a8a5acf2c09cb691421b31cf8af7a36 + with: + toolchain: ${{ matrix.nightly_version }} + profile: default + - name: Cache cargo registry + uses: actions/cache@v3 + continue-on-error: false + with: + path: | + ~/.cargo/registry + ~/.cargo/git + key: ${{ runner.os }}-${{ matrix.make.cache_key }}-${{ matrix.make.cache_version }}-cargo-${{ hashFiles('**/Cargo.lock') }} + restore-keys: ${{ runner.os }}-${{ matrix.make.cache_key }}-${{ matrix.make.cache_version }}-cargo- + - name: Start sccache server + run: sccache --start-server + - name: Build + run: make build${{ matrix.make.suffix }} + - name: Build test + run: make build-test${{ matrix.make.suffix }} + - name: Download wasm artifacts + uses: actions/download-artifact@v3 + with: + name: wasm-${{ github.sha }} + path: ./wasm + - name: Run unit test + run: make test-unit${{ matrix.make.suffix }} + - name: Wait for release binaries + uses: lewagon/wait-on-check-action@master + with: + ref: ${{ github.event.pull_request.head.sha || github.ref }} + check-name: ${{ matrix.make.wait_for }} + repo-token: ${{ secrets.GITHUB_TOKEN }} + wait-interval: 30 + allowed-conclusions: success + - name: Download tendermint binaries + uses: dawidd6/action-download-artifact@v2 + with: + github_token: ${{secrets.GITHUB_TOKEN}} + workflow: build-tendermint.yml + workflow_conclusion: success + name: ${{ matrix.make.tendermint_artifact }} + path: /usr/local/bin + - name: Download anoma binaries + uses: actions/download-artifact@v3 + with: + name: binaries${{ matrix.make.suffix }}-${{ github.sha }} + path: ./target/release/ + - name: Change permissions + run: | + chmod +x target/release/namada + chmod +x target/release/namadaw + chmod +x target/release/namadan + chmod +x target/release/namadac + chmod +x /usr/local/bin/tendermint + - name: Set tendermint ENV for abci++ tests + run: echo "TENDERMINT=/usr/local/bin/tendermint" >> $GITHUB_ENV + - name: Run e2e test + run: make test-e2e${{ matrix.make.suffix }} + env: + ANOMA_TENDERMINT_WEBSOCKET_TIMEOUT: 20 + ANOMA_E2E_USE_PREBUILT_BINARIES: "true" + ANOMA_E2E_KEEP_TEMP: "true" + ENV_VAR_TM_STDOUT: "false" + ANOMA_LOG_COLOR: "false" + TM_LOG_LEVEL: "info" + ANOMA_LOG: "info" + - name: Upload e2e logs + if: success() || failure() + uses: actions/upload-artifact@v3 + with: + name: logs-e2e${{ matrix.make.suffix }}-${{ github.sha }} + path: /tmp/.*/logs/ + retention-days: 5 + - name: Print sccache stats + if: always() + run: sccache --show-stats + - name: Stop sccache server + if: always() + run: sccache --stop-server || true + + anoma-release: + runs-on: ${{ matrix.os }} + timeout-minutes: 40 + strategy: + fail-fast: false + matrix: + os: [ubuntu-latest] + make: + - name: ABCI Release build + suffix: '' + cache_key: anoma-e2e-release + cache_version: "v1" + + env: + CARGO_INCREMENTAL: 0 + RUST_BACKTRACE: full + RUSTC_WRAPPER: sccache + SCCACHE_CACHE_SIZE: 100G + SCCACHE_BUCKET: namada-sccache-master + + steps: + - name: Checkout repo + uses: actions/checkout@v3 + - name: Configure AWS Credentials + uses: aws-actions/configure-aws-credentials@v1 + with: + role-to-assume: arn:aws:iam::375643557360:role/anoma-github-action-ci-master + aws-region: eu-west-1 + - name: Install sccache (ubuntu-latest) + if: matrix.os == 'ubuntu-latest' + env: + LINK: https://github.com/mozilla/sccache/releases/download + SCCACHE_VERSION: v0.3.0 + run: | + SCCACHE_FILE=sccache-$SCCACHE_VERSION-x86_64-unknown-linux-musl + mkdir -p $HOME/.local/bin + curl -L "$LINK/$SCCACHE_VERSION/$SCCACHE_FILE.tar.gz" | tar xz + mv -f $SCCACHE_FILE/sccache $HOME/.local/bin/sccache + chmod +x $HOME/.local/bin/sccache + echo "$HOME/.local/bin" >> $GITHUB_PATH + - name: Install sccache (macos-latest) + if: matrix.os == 'macos-latest' + run: | + brew update + brew install sccache + - name: Setup rust toolchain + uses: oxidecomputer/actions-rs_toolchain@ad3f86084a8a5acf2c09cb691421b31cf8af7a36 + with: + profile: default + override: true + - name: Cache cargo registry + uses: actions/cache@v3 + continue-on-error: false + with: + path: | + ~/.cargo/registry + ~/.cargo/git + key: ${{ runner.os }}-${{ matrix.make.cache_key }}-${{ matrix.make.cache_version }}-cargo-${{ hashFiles('**/Cargo.lock') }} + restore-keys: ${{ runner.os }}-${{ matrix.make.cache_key }}-${{ matrix.make.cache_version }}-cargo- + - name: Start sccache server + run: sccache --start-server + - name: Build + run: make build-release${{ matrix.make.suffix }} + - name: Upload target binaries + uses: actions/upload-artifact@v3 + with: + name: binaries${{ matrix.make.suffix }}-${{ github.sha }} + path: | + target/release/namada + target/release/namadac + target/release/namadaw + target/release/namadan + - name: Print sccache stats + if: always() + run: sccache --show-stats + - name: Stop sccache server + if: always() + run: sccache --stop-server || true diff --git a/.github/workflows/build-tendermint.yml b/.github/workflows/build-tendermint.yml new file mode 100644 index 00000000000..44fb028aed1 --- /dev/null +++ b/.github/workflows/build-tendermint.yml @@ -0,0 +1,40 @@ +name: Build tendermint binaries + +on: + schedule: + - cron: "0 0 * * *" + workflow_dispatch: + +permissions: + id-token: write + contents: read + +env: + GIT_LFS_SKIP_SMUDGE: 1 + + +jobs: + tendermint: + runs-on: ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + os: [ubuntu-latest] + make: + - name: tendermint-0.34.19 + tendermint_version: 2f231ceb952a2426cf3c0abaf0b455aadd11e5b2 + - name: tendermint-unreleased + tendermint_version: 29e5fbcc648510e4763bd0af0b461aed92c21f30 + - name: tendermint-0.34.11 + tendermint_version: c2908ef785bb5ca6e015910368e4b7a32821b9ff + + steps: + - name: Build ${{ matrix.make.name }} + run: | + git clone https://github.com/tendermint/tendermint.git && cd tendermint + git checkout ${{ matrix.make.tendermint_version }} && make build + - name: Upload ${{ matrix.make.name }} binary + uses: actions/upload-artifact@v3 + with: + name: ${{ matrix.make.name }}-${{ matrix.make.tendermint_version }} + path: tendermint/build/tendermint diff --git a/.github/workflows/checks.yml b/.github/workflows/checks.yml new file mode 100644 index 00000000000..e3d2c4ce5c4 --- /dev/null +++ b/.github/workflows/checks.yml @@ -0,0 +1,79 @@ +name: Run checks + +on: + push: + branches: + - main + pull_request: + workflow_dispatch: + +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number }} + cancel-in-progress: true + +permissions: + id-token: write + contents: read + +env: + GIT_LFS_SKIP_SMUDGE: 1 + +jobs: + clippy-fmt: + runs-on: ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + os: [ubuntu-latest] + nightly_version: [nightly-2022-05-20] + make: + - name: Clippy + command: clippy + cache_subkey: clippy + cache_version: v1 + - name: Format + command: check && make fmt-check + cache_subkey: fmt + cache_version: v1 + + env: + CARGO_INCREMENTAL: 0 + + steps: + - name: Checkout repo + uses: actions/checkout@v3 + - name: Setup rust toolchain + uses: oxidecomputer/actions-rs_toolchain@ad3f86084a8a5acf2c09cb691421b31cf8af7a36 + with: + target: wasm32-unknown-unknown + profile: default + override: true + - name: Setup rust nightly + uses: oxidecomputer/actions-rs_toolchain@ad3f86084a8a5acf2c09cb691421b31cf8af7a36 + with: + toolchain: ${{ matrix.nightly_version }} + target: wasm32-unknown-unknown + profile: default + - name: Show rust toolchain info + run: rustup show + - name: Cache cargo + uses: actions/cache@v3 + with: + path: | + ~/.cargo/registry/index + ~/.cargo/registry/cache + ~/.cargo/git + ~/.cargo/.crates.toml + ~/.cargo/.crates2.json + ./target + ./wasm/tx_template/target + ./wasm/vp_template/target + ./wasm_for_tests/wasm_source/target + key: ${{ runner.os }}-${{ matrix.make.cache_subkey }}-${{ matrix.make.cache_version }}-cargo-${{ hashFiles('**/Cargo.lock') }} + restore-keys: ${{ runner.os }}-${{ matrix.make.cache_subkey }}-${{ matrix.make.cache_version }}-cargo- + - name: ${{ matrix.make.name }} + run: make ${{ matrix.make.command }} + - name: Clean cache folder + run: | + cargo install cargo-cache --no-default-features --features ci-autoclean cargo-cache || true + cargo-cache diff --git a/.github/workflows/cron.yml b/.github/workflows/cron.yml new file mode 100644 index 00000000000..86f9940a77e --- /dev/null +++ b/.github/workflows/cron.yml @@ -0,0 +1,72 @@ +name: Run period checks + +on: + schedule: + - cron: "0 0 * * *" + workflow_dispatch: + +permissions: + id-token: write + contents: read + issues: write + +env: + GIT_LFS_SKIP_SMUDGE: 1 + +jobs: + cron: + runs-on: ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + os: [ubuntu-latest] + nightly_version: [nightly-2022-05-20] + make: + - name: Audit + command: audit + version: rustsec/rustsec@cargo-audit/v0.17.0 + cache_subkey: audit + cache_version: v1 + - name: Unused dependencies + command: udeps + version: est31/cargo-udeps@v0.1.30 + cache_subkey: udeps + cache_version: v1 + + env: + CARGO_INCREMENTAL: 0 + + steps: + - name: Checkout repo + uses: actions/checkout@v3 + - name: Setup rust nightly + uses: oxidecomputer/actions-rs_toolchain@ad3f86084a8a5acf2c09cb691421b31cf8af7a36 + with: + toolchain: ${{ matrix.nightly_version }} + profile: minimal + - name: Show rust toolchain info + run: rustup show + - name: Cache cargo + uses: actions/cache@v3 + with: + path: | + ~/.cargo/registry/index + ~/.cargo/registry/cache + ~/.cargo/git + ~/.cargo/.crates.toml + ~/.cargo/.crates2.json + ./target + key: ${{ runner.os }}-${{ matrix.make.cache_subkey }}-${{ matrix.make.cache_version }}-cargo-${{ hashFiles('**/Cargo.lock') }} + restore-keys: ${{ runner.os }}-${{ matrix.make.cache_subkey }}-${{ matrix.make.cache_version }}-cargo- + - name: Install cargo ${{ matrix.make.command }} + run: curl https://i.jpillora.com/${{ matrix.make.version }}! | bash + - name: ${{ matrix.make.name }} + working-directory: ./.github/workflows/scripts + run: | + pip3 install github3.py >/dev/null 2>&1 + pip3 install -Iv cryptography==37.0.4 >/dev/null 2>&1 + python3 ${{ matrix.make.command }}.py + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + - name: Clean cache folder + run: cargo install cargo-cache --no-default-features --features ci-autoclean cargo-cache && cargo-cache diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml new file mode 100644 index 00000000000..77318025f78 --- /dev/null +++ b/.github/workflows/docker.yml @@ -0,0 +1,59 @@ +name: Build docker images + +on: + workflow_dispatch: + inputs: + tag: + description: 'The docker image tag' + required: true + +env: + REPOSITORY_OWNER: "fraccaman/namada" + GIT_LFS_SKIP_SMUDGE: 1 + +jobs: + docker: + runs-on: ${{ matrix.os }} + permissions: + packages: write + strategy: + fail-fast: false + matrix: + os: [ubuntu-latest] + make: + - name: Build & Push WASM docker image + image: wasm + path: ./docker/namada-wasm + - name: Build and Push Namada docker image + image: namada + path: ./docker/namada-build + + steps: + - name: Checkout repo + uses: actions/checkout@v3 + - name: Set up QEMU + uses: docker/setup-qemu-action@v2 + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v2 + - name: Docker meta + id: meta + uses: docker/metadata-action@v3 + with: + images: ghcr.io/anoma/namada + github-token: ${{ secrets.GITHUB_TOKEN }} + - name: Login to GHCR + uses: docker/login-action@v2 + with: + registry: ghcr.io + username: ${{ github.repository_owner }} + password: ${{ secrets.GITHUB_TOKEN }} + - name: ${{ matrix.make.name }} + uses: docker/build-push-action@v3 + with: + context: . + file: ${{ matrix.make.path }}/Dockerfile + push: true + tags: ghcr.io/${{ env.REPOSITORY_OWNER }}:${{ matrix.make.image }}-${{ github.event.inputs.tag }} + labels: ${{ steps.meta.outputs.labels }} + cache-from: type=registry,ref=${{ matrix.make.image }}-${{ github.event.inputs.tag }} + cache-to: type=inline diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml new file mode 100644 index 00000000000..de9fde4e51a --- /dev/null +++ b/.github/workflows/docs.yml @@ -0,0 +1,113 @@ +name: Build docs + +on: + push: + branches: + - main + pull_request: + workflow_dispatch: + +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number }} + cancel-in-progress: true + +permissions: + id-token: write + contents: read + +env: + GIT_LFS_SKIP_SMUDGE: 1 + +jobs: + docs: + runs-on: ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + os: [ubuntu-latest] + nightly_version: [nightly-2022-05-20] + mdbook_version: [rust-lang/mdbook@v0.4.18] + mdbook_mermaid: [badboy/mdbook-mermaid@v0.11.1] + mdbook_linkcheck: [Michael-F-Bryan/mdbook-linkcheck@v0.7.6] + mdbook_open_on_gh: [badboy/mdbook-open-on-gh@v2.2.0] + mdbook_admonish: [tommilligan/mdbook-admonish@v1.7.0] + mdbook_katex: [lzanini/mdbook-katex@v0.2.10] + make: + - name: Build specs + folder: documentation/specs + bucket: namada-specs-static-website + command: cd documentation/specs && mdbook build + cache_subkey: specs + cache_version: v1 + - name: Build docs + folder: documentation/docs + bucket: namada-docs-static-website + command: cd documentation/docs && mdbook build + cache_subkey: docs + cache_version: v1 + + env: + CARGO_INCREMENTAL: 0 + RUSTC_WRAPPER: sccache + SCCACHE_CACHE_SIZE: 100G + SCCACHE_BUCKET: namada-sccache-master + + steps: + - name: Checkout repo + uses: actions/checkout@v3 + - name: Configure AWS Credentials + uses: aws-actions/configure-aws-credentials@v1 + with: + role-to-assume: arn:aws:iam::375643557360:role/anoma-github-action-ci-master + aws-region: eu-west-1 + - name: Install sccache (ubuntu-latest) + if: matrix.os == 'ubuntu-latest' + env: + LINK: https://github.com/mozilla/sccache/releases/download + SCCACHE_VERSION: v0.3.0 + run: | + SCCACHE_FILE=sccache-$SCCACHE_VERSION-x86_64-unknown-linux-musl + mkdir -p $HOME/.local/bin + curl -L "$LINK/$SCCACHE_VERSION/$SCCACHE_FILE.tar.gz" | tar xz + mv -f $SCCACHE_FILE/sccache $HOME/.local/bin/sccache + chmod +x $HOME/.local/bin/sccache + echo "$HOME/.local/bin" >> $GITHUB_PATH + - name: Install sccache (macos-latest) + if: matrix.os == 'macos-latest' + run: | + brew update + brew install sccache + - name: Setup rust toolchain + uses: oxidecomputer/actions-rs_toolchain@ad3f86084a8a5acf2c09cb691421b31cf8af7a36 + with: + profile: default + override: true + - name: Setup rust nightly + uses: oxidecomputer/actions-rs_toolchain@ad3f86084a8a5acf2c09cb691421b31cf8af7a36 + with: + toolchain: ${{ matrix.nightly_version }} + profile: default + - name: Show rust toolchain info + run: rustup show + - name: Start sccache server + run: sccache --start-server + - name: Install cargo tool + run: | + curl https://i.jpillora.com/${{ matrix.mdbook_version }}! | bash + curl https://i.jpillora.com/${{ matrix.mdbook_mermaid }}! | bash + curl https://i.jpillora.com/${{ matrix.mdbook_linkcheck }}! | bash + curl https://i.jpillora.com/${{ matrix.mdbook_open_on_gh }}! | bash + curl https://i.jpillora.com/${{ matrix.mdbook_admonish }}! | bash + curl https://i.jpillora.com/${{ matrix.mdbook_katex }}! | bash + cd ${{ matrix.make.folder }} && mdbook-admonish install + - name: ${{ matrix.make.name }} + run: ${{ matrix.make.command }} + - name: Publish docs + if: ${{ github.event_name == 'push' && github.ref == 'refs/heads/main' }} + run: aws s3 sync ${{ matrix.make.folder }}/book/html/ s3://${{ matrix.make.bucket }} --region eu-west-1 --delete + - name: Print sccache stats + if: always() + run: sccache --show-stats + - name: Stop sccache server + if: always() + run: sccache --stop-server || true diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml new file mode 100644 index 00000000000..70e2c3492fa --- /dev/null +++ b/.github/workflows/release.yml @@ -0,0 +1,107 @@ +name: Release + +on: + push: + tags: + - "v[0-9]+.[0-9]+.[0-9]+" + +permissions: + id-token: write + contents: write + +env: + GIT_LFS_SKIP_SMUDGE: 1 + +jobs: + build: + runs-on: ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + os: [ubuntu-latest, macos-latest] + anoma_cache_version: [v1] + make: + - name: Build package + command: package + + env: + RUST_BACKTRACE: full + RUSTC_WRAPPER: sccache + SCCACHE_CACHE_SIZE: 100G + SCCACHE_BUCKET: namada-sccache-master + + steps: + - name: Checkout repo + uses: actions/checkout@v3 + - name: Configure AWS Credentials + uses: aws-actions/configure-aws-credentials@v1 + with: + role-to-assume: arn:aws:iam::375643557360:role/anoma-github-action-ci-master + aws-region: eu-west-1 + - name: Install sccache (ubuntu-latest) + if: matrix.os == 'ubuntu-latest' + env: + LINK: https://github.com/mozilla/sccache/releases/download + SCCACHE_VERSION: v0.3.0 + run: | + SCCACHE_FILE=sccache-$SCCACHE_VERSION-x86_64-unknown-linux-musl + mkdir -p $HOME/.local/bin + curl -L "$LINK/$SCCACHE_VERSION/$SCCACHE_FILE.tar.gz" | tar xz + mv -f $SCCACHE_FILE/sccache $HOME/.local/bin/sccache + chmod +x $HOME/.local/bin/sccache + echo "$HOME/.local/bin" >> $GITHUB_PATH + - name: Install sccache (macos-latest) + if: matrix.os == 'macos-latest' + run: | + brew update + brew install sccache + - name: Setup rust toolchain + uses: oxidecomputer/actions-rs_toolchain@ad3f86084a8a5acf2c09cb691421b31cf8af7a36 + with: + profile: minimal + override: true + - name: Cache cargo registry + uses: actions/cache@v3 + continue-on-error: false + with: + path: | + ~/.cargo/registry + ~/.cargo/git + key: ${{ runner.os }}-anoma-release-${{ matrix.anoma_cache_version }}-${{ hashFiles('**/Cargo.lock') }} + restore-keys: ${{ runner.os }}-anoma-release-${{ matrix.anoma_cache_version }} + - name: Start sccache server + run: sccache --start-server + - name: ${{ matrix.make.name }} + run: make ${{ matrix.make.command }} + - name: Upload binaries package + uses: actions/upload-artifact@v3 + with: + name: release-${{ matrix.os }}-${{ github.sha }} + path: ./*.tar.gz + - name: Print sccache stats + if: always() + run: sccache --show-stats + - name: Stop sccache server + if: always() + run: sccache --stop-server || true + + release: + needs: build + runs-on: ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + os: [ubuntu-latest] + + steps: + - id: get_version + uses: battila7/get-version-action@v2 + - name: Download release artifacts + uses: actions/download-artifact@v3 + - name: Create release + uses: softprops/action-gh-release@v1 + with: + draft: true + files: ./**/*.tar.gz + tag_name: ${{ steps.get_version.outputs.version }} + name: Namada ${{ steps.get_version.outputs.version-without-v }} diff --git a/.github/workflows/scripts/audit.py b/.github/workflows/scripts/audit.py new file mode 100644 index 00000000000..f28c5c94371 --- /dev/null +++ b/.github/workflows/scripts/audit.py @@ -0,0 +1,115 @@ +import os +import json +import subprocess +import urllib +import github3 +import urllib.request +from urllib.request import urlopen + +REPOSITORY_NAME = os.environ['GITHUB_REPOSITORY_OWNER'] + +GH_TOKEN = os.environ['GITHUB_TOKEN'] + +GET_ISSUES_URL = 'https://api.github.com/repos/{}/{}/issues'.format( + REPOSITORY_NAME, 'namada') +CREATE_ISSUE_URL = 'https://api.github.com/repos/{}/{}/issues'.format( + REPOSITORY_NAME, 'namada') +MODIFY_ISSUE = 'https://api.github.com/repos/{}/{}/issues/{}'.format( + REPOSITORY_NAME, 'namada', '{}') +HEADERS = { + 'Content-Type': 'application/json', + 'Accept': 'application/vnd.github.v3+json', + 'Authorization': 'token {}'.format(GH_TOKEN) +} + +ISSUE_TITLE = 'Cargo Audit' +ISSUE_LABEL = 'dependencies' + + +# 0 - not exist,2 already exist, else issue number +def check_issue_status(body: str) -> int: + params = {'creator': 'github-actions[bot]', 'state': 'open'} + params_encoded = urllib.parse.urlencode(params) + req = urllib.request.Request("{}?{}".format( + CREATE_ISSUE_URL, params_encoded), headers=HEADERS) + + with urlopen(req) as response: + issues = json.load(response) + + for issue in issues: + title_check = issue['title'] == ISSUE_TITLE + if title_check: + if issue['body'] == body: + return 2 + return issue['number'] + return 0 + + +def modify_issue(issue_number: int, body: str): + body = {"body": body} + encoded_body = json.dumps(body).encode('ascii') + req = urllib.request.Request(MODIFY_ISSUE.format( + issue_number), data=encoded_body, headers=HEADERS) + req.get_method = lambda: 'PATCH' + + with urlopen(req) as response: + json.load(response) + + +def create_issue(body: str): + body = {"title": ISSUE_TITLE, "body": body, "labels": [ISSUE_LABEL]} + encoded_body = json.dumps(body).encode('ascii') + req = urllib.request.Request( + CREATE_ISSUE_URL, data=encoded_body, headers=HEADERS) + + with urlopen(req) as response: + json.load(response) + + +issue_template = '# Vulnerabilities \n{}' +table_header = '| Id | Package | Title | Date |\n|----:|---------:|-------:|------:|' +table_row = '|[{0}]({advisory_db}{0})|{1}|{2}|{3}|' + +table = [table_header] + +current_dir = os.path.dirname(os.path.abspath(__file__)) +cwd = os.environ['GITHUB_WORKSPACE'] + +command = ['cargo', 'audit', '--json'] +p = subprocess.Popen(command, stdout=subprocess.PIPE, cwd=cwd) +output = p.stdout.read() +retcode = p.wait() + +vulnerabilities = json.loads(output)['vulnerabilities'] +if int(vulnerabilities['count']) == 0: + print("No vulnerabilities found.") + exit(0) + +for vulnerability in vulnerabilities['list']: + vuln_description = vulnerability['advisory'] + vuln_id = vuln_description['id'] + vuln_title = vuln_description['title'] + vuln_package = vuln_description['package'] + vuln_date = vuln_description['date'] + new_table_row = table_row.format(vuln_id, vuln_package, vuln_title, vuln_date, + # link issues by their ID to the advisory DB + advisory_db='https://rustsec.org/advisories/') + table.append(new_table_row) + +table_rendered = '\n'.join(table) +issue_body = issue_template.format(table_rendered) + +if not GH_TOKEN: + print("Invalid github token.") + exit(0) + +issue_status = check_issue_status(issue_body) + +if issue_status == 0: + print("Create new issue.") + create_issue(issue_body) +elif issue_status == 2: + print("Issue already created.") +else: + print("Issue updated.") + modify_issue(issue_status, issue_body) diff --git a/.github/workflows/scripts/publish-wasm.py b/.github/workflows/scripts/publish-wasm.py new file mode 100644 index 00000000000..2e7b66b7d83 --- /dev/null +++ b/.github/workflows/scripts/publish-wasm.py @@ -0,0 +1,73 @@ +from ghapi.all import GhApi +from os import environ +from json import loads, load +from tempfile import gettempdir +import subprocess + + +def download_artifact(link: str, path: str, token: str): + return subprocess.run(["curl", "-H", "Accept: application/vnd.github+json".format(token), "-H", "Authorization: token {}".format(token), link, "-L", "-o", "{}/wasm.zip".format(path)], capture_output=True) + + +def unzip(path: str): + return subprocess.run(["unzip", "-o", "{}/wasm.zip".format(path), "-d", path], capture_output=True) + + +def publish_wasm(path: str, file_name: str, bucket: str): + return subprocess.run(["aws", "s3", "cp", "{}/{}".format(path, file_name), "s3://{}".format(bucket), "--acl", "public-read"], capture_output=True) + + +def log(data: str): + print(data) + + +PR_COMMENT = 'pls publish wasm' +TOKEN = environ["GITHUB_TOKEN"] +READ_ORG_TOKEN = environ['GITHUB_READ_ORG_TOKEN'] +REPOSITORY_NAME = environ['GITHUB_REPOSITORY_OWNER'] +TMP_DIRECTORY = gettempdir() +ARTIFACT_PER_PAGE = 75 +WASM_BUCKET = 'namada-wasm-master' + +read_org_api = GhApi(token=READ_ORG_TOKEN) +api = GhApi(owner=REPOSITORY_NAME, repo="namada", token=TOKEN) + +user_membership = read_org_api.teams.get_membership_for_user_in_org( + 'heliaxdev', 'company', 'fraccaman') +if user_membership['state'] != 'active': + exit(0) + +comment_event = loads(environ['GITHUB_CONTEXT']) +pr_comment = comment_event['event']['comment']['body'] +pr_number = comment_event['event']['issue']['number'] + +if pr_comment == PR_COMMENT: + pr_info = api.pulls.get(pr_number) + head_sha = pr_info['head']['sha'] + + artifacts = api.actions.list_artifacts_for_repo(per_page=ARTIFACT_PER_PAGE) + + for artifact in artifacts['artifacts']: + if 'wasm' in artifact['name'] and artifact['workflow_run']['head_sha'] == head_sha and not artifact['expired']: + artifact_download_url = artifact['archive_download_url'] + + log("Downloading artifacts...") + curl_command_outcome = download_artifact( + artifact_download_url, TMP_DIRECTORY, TOKEN) + if curl_command_outcome.returncode != 0: + exit(1) + + log("Unzipping wasm.zip...") + unzip_command_outcome = unzip(TMP_DIRECTORY) + if unzip_command_outcome.returncode != 0: + exit(1) + + checksums = load(open("{}/checksums.json".format(TMP_DIRECTORY))) + for wasm in checksums.values(): + log("Uploading {}...".format(wasm)) + publish_wasm_command_outcome = publish_wasm( + TMP_DIRECTORY, wasm, WASM_BUCKET) + if publish_wasm_command_outcome.returncode != 0: + print("Error uploading {}!".format(wasm)) + + log("Done!") diff --git a/.github/workflows/scripts/udeps.py b/.github/workflows/scripts/udeps.py new file mode 100644 index 00000000000..9dcbde294b1 --- /dev/null +++ b/.github/workflows/scripts/udeps.py @@ -0,0 +1,132 @@ +import os +import json +import subprocess +import urllib +import github3 +import urllib.request +from urllib.request import urlopen + +REPOSITORY_NAME = os.environ['GITHUB_REPOSITORY_OWNER'] + +GH_TOKEN = os.environ['GITHUB_TOKEN'] + +GET_ISSUES_URL = 'https://api.github.com/repos/{}/{}/issues'.format( + REPOSITORY_NAME, 'namada') +CREATE_ISSUE_URL = 'https://api.github.com/repos/{}/{}/issues'.format( + REPOSITORY_NAME, 'namada') +MODIFY_ISSUE = 'https://api.github.com/repos/{}/{}/issues/{}'.format( + REPOSITORY_NAME, 'namada', '{}') +HEADERS = { + 'Content-Type': 'application/json', + 'Accept': 'application/vnd.github.v3+json', + 'Authorization': 'token {}'.format(GH_TOKEN) +} + +ISSUE_TITLE = 'Cargo Udeps' +ISSUE_LABEL = 'dependencies' + + +def get_nightly_from_file() -> str: + workspace = os.environ['GITHUB_WORKSPACE'] + return open("{}/rust-nightly-version".format(workspace), "r").read().strip() + + +# 0 - not exist,2 already exist, else issue number +def check_issue_status(body: str) -> int: + params = {'creator': 'github-actions[bot]', 'state': 'open'} + params_encoded = urllib.parse.urlencode(params) + req = urllib.request.Request("{}?{}".format( + CREATE_ISSUE_URL, params_encoded), headers=HEADERS) + + with urlopen(req) as response: + issues = json.load(response) + for issue in issues: + title_check = issue['title'] == ISSUE_TITLE + if title_check: + if issue['body'] == body: + return 2 + return issue['number'] + return 0 + + +def modify_issue(issue_number: int, body: str): + body = {"body": body} + encoded_body = json.dumps(body).encode('ascii') + req = urllib.request.Request(MODIFY_ISSUE.format( + issue_number), data=encoded_body, headers=HEADERS) + req.get_method = lambda: 'PATCH' + + with urlopen(req) as response: + json.load(response) + + +def create_issue(body: str): + body = {"title": ISSUE_TITLE, "body": body, "labels": [ISSUE_LABEL]} + encoded_body = json.dumps(body).encode('ascii') + req = urllib.request.Request( + CREATE_ISSUE_URL, data=encoded_body, headers=HEADERS) + + with urlopen(req) as response: + json.load(response) + + +issue_template = '# Unused dependencies \n{}' +table_header = '| Crate | Package | Type |\n|----:|-------:|-------:|' +table_row = '|{}|{}|{}|' + +table = [table_header] + +nightly_version = get_nightly_from_file() +command = ['cargo', '+{}'.format(nightly_version), 'udeps', '--output', 'json'] + +current_dir = os.path.dirname(os.path.abspath(__file__)) +cwd = os.environ['GITHUB_WORKSPACE'] + +p = subprocess.Popen(command, stdout=subprocess.PIPE, cwd=cwd) +output = p.stdout.read() +retcode = p.wait() + +unused_deps = None + +try: + unused_deps = json.loads(output) +except Exception as e: + print(output) + print(e) + exit(0) + +if unused_deps['success']: + print("No unused dependencies found.") + exit(0) + +for crate in unused_deps['unused_deps'].keys(): + info = unused_deps['unused_deps'][crate] + create_name = crate.split(" (")[0] + for normal in info['normal']: + new_table_row = table_row.format(create_name, normal, 'normal') + table.append(new_table_row) + for development in info['development']: + new_table_row = table_row.format( + create_name, development, 'development') + table.append(new_table_row) + for build in info['build']: + new_table_row = table_row.format(create_name, build, 'build') + table.append(new_table_row) + +table_rendered = '\n'.join(table) +issue_body = issue_template.format(table_rendered) + +if not GH_TOKEN: + print("Executed locally.") + exit(0) + +issue_status = check_issue_status(issue_body) + +if issue_status == 0: + print("Create new issue.") + create_issue(issue_body) +elif issue_status == 2: + print("Issue already created.") +else: + print("Issue updated.") + modify_issue(issue_status, issue_body) diff --git a/.github/workflows/scripts/update-wasm.py b/.github/workflows/scripts/update-wasm.py new file mode 100644 index 00000000000..0fd8b138073 --- /dev/null +++ b/.github/workflows/scripts/update-wasm.py @@ -0,0 +1,88 @@ +from ghapi.all import GhApi +from os import environ +from json import loads +from tempfile import gettempdir +import subprocess + + +def log(data: str): + print(data) + + +def download_artifact(link: str, path: str, token: str): + return subprocess.run(["curl", "-H", "Accept: application/vnd.github+json".format(token), "-H", "Authorization: token {}".format(token), link, "-L", "-o", "{}/wasm.zip".format(path)], capture_output=True) + + +def unzip(path: str): + return subprocess.run(["unzip", "-o", "{}/wasm.zip".format(path), "-d", path], capture_output=True) + + +def replace_checksums(path: str): + return subprocess.run(["mv", "{}/checksums.json".format(path), "wasm/"], capture_output=True) + + +def commit_and_push(): + outcome = subprocess.run(["git", "status", "--porcelain"], capture_output=True) + if not len(outcome.stdout): + return outcome + outcome = subprocess.run( + ["git", "add", "wasm/checksums.json"], capture_output=True) + if outcome.returncode != 0: + return outcome + outcome = subprocess.run( + ["git", "commit", "-m", "[ci skip] wasm checksums update"], capture_output=True) + if outcome.returncode != 0: + return outcome + return subprocess.run(["git", "push"], capture_output=True) + + +PR_COMMENT = 'pls update wasm' +TOKEN = environ["GITHUB_TOKEN"] +READ_ORG_TOKEN = environ['GITHUB_READ_ORG_TOKEN'] +REPOSITORY_NAME = environ['GITHUB_REPOSITORY_OWNER'] +TMP_DIRECTORY = gettempdir() +ARTIFACT_PER_PAGE = 75 + +read_org_api = GhApi(token=READ_ORG_TOKEN) +api = GhApi(owner=REPOSITORY_NAME, repo="namada", token=TOKEN) + +user_membership = read_org_api.teams.get_membership_for_user_in_org( + 'heliaxdev', 'company', 'fraccaman') +if user_membership['state'] != 'active': + exit(0) + +comment_event = loads(environ['GITHUB_CONTEXT']) +pr_comment = comment_event['event']['comment']['body'] +pr_number = comment_event['event']['issue']['number'] + +if pr_comment == PR_COMMENT: + pr_info = api.pulls.get(pr_number) + head_sha = pr_info['head']['sha'] + + artifacts = api.actions.list_artifacts_for_repo(per_page=ARTIFACT_PER_PAGE) + + for artifact in artifacts['artifacts']: + if 'wasm' in artifact['name'] and artifact['workflow_run']['head_sha'] == head_sha and not artifact['expired']: + artifact_download_url = artifact['archive_download_url'] + + log("Downloading artifacts...") + curl_command_outcome = download_artifact( + artifact_download_url, TMP_DIRECTORY, TOKEN) + if curl_command_outcome.returncode != 0: + exit(1) + + unzip_command_outcome = unzip(TMP_DIRECTORY) + if unzip_command_outcome.returncode != 0: + exit(1) + + log("Replacing checksums.json...") + replace_command_outcome = replace_checksums(TMP_DIRECTORY) + if replace_command_outcome.returncode != 0: + exit(1) + + log("Pushing new checksums.json...") + commit_and_push_command_outcome = commit_and_push() + if commit_and_push_command_outcome.returncode != 0: + exit(1) + + log("Done!") diff --git a/Cargo.lock b/Cargo.lock index 1b962cedbf9..31d67389e0d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4072,6 +4072,7 @@ dependencies = [ "pretty_assertions", "proptest", "prost 0.9.0", + "rand 0.8.5", "serde_json", "sha2 0.9.9", "tempfile", diff --git a/Makefile b/Makefile index 2e9c5e7fcc5..d1feaa2235c 100644 --- a/Makefile +++ b/Makefile @@ -29,7 +29,10 @@ build-test-abci-plus-plus: $(cargo) build --tests --no-default-features --features "ABCI-plus-plus" build-release: - ANOMA_DEV=false $(cargo) build --release --package namada_apps + ANOMA_DEV=false $(cargo) build --release --package namada_apps --manifest-path Cargo.toml --features "ABCI" + +build-release-abci-plus-plus: + ANOMA_DEV=false $(cargo) build --release --package namada_apps --no-default-features --features "ABCI-plus-plus" check-release: ANOMA_DEV=false $(cargo) check --release --package namada_apps @@ -125,50 +128,75 @@ audit: test: test-unit test-e2e test-wasm test-e2e: - RUST_BACKTRACE=1 $(cargo) test e2e -- --test-threads=1 + RUST_BACKTRACE=1 $(cargo) test e2e \ + -- \ + --test-threads=1 \ + -Z unstable-options --report-time test-e2e-abci-plus-plus: RUST_BACKTRACE=1 $(cargo) test e2e \ --manifest-path ./tests/Cargo.toml \ --no-default-features \ --features "wasm-runtime ABCI-plus-plus namada_apps/ABCI-plus-plus" \ - -- --test-threads=1 + -- \ + --test-threads=1 \ + -Z unstable-options --report-time test-unit-abci-plus-plus: $(cargo) test \ --manifest-path ./apps/Cargo.toml \ --no-default-features \ - --features "testing std ABCI-plus-plus" && \ - $(cargo) test --manifest-path ./proof_of_stake/Cargo.toml \ - --features "testing" && \ + --features "testing std ABCI-plus-plus" \ + -- \ + -Z unstable-options --report-time && \ + $(cargo) test \ + --manifest-path \ + ./proof_of_stake/Cargo.toml \ + --features "testing" \ + -- \ + -Z unstable-options --report-time && \ $(cargo) test \ --manifest-path ./shared/Cargo.toml \ --no-default-features \ - --features "testing wasm-runtime ABCI-plus-plus ibc-mocks" && \ + --features "testing wasm-runtime ABCI-plus-plus ibc-mocks" \ + -- \ + -Z unstable-options --report-time && \ $(cargo) test \ --manifest-path ./tests/Cargo.toml \ --no-default-features \ --features "wasm-runtime ABCI-plus-plus namada_apps/ABCI-plus-plus" \ - -- --skip e2e && \ + -- \ + --skip e2e \ + -Z unstable-options --report-time && \ $(cargo) test \ --manifest-path ./vm_env/Cargo.toml \ --no-default-features \ - --features "ABCI-plus-plus" + --features "ABCI-plus-plus" \ + -- \ + -Z unstable-options --report-time test-unit: $(cargo) test --no-default-features \ --features "wasm-runtime ABCI ibc-mocks-abci" \ - -- --skip e2e + -- \ + --skip e2e \ + -Z unstable-options --report-time test-wasm: make -C $(wasms) test -test-wasm-template = $(cargo) test --manifest-path $(wasm)/Cargo.toml +test-wasm-template = $(cargo) test \ + --manifest-path $(wasm)/Cargo.toml \ + -- \ + -Z unstable-options --report-time test-wasm-templates: $(foreach wasm,$(wasm_templates),$(test-wasm-template) && ) true test-debug: - $(debug-cargo) test -- --nocapture + $(debug-cargo) test \ + -- \ + --nocapture \ + -Z unstable-options --report-time fmt-wasm = $(cargo) +$(nightly) fmt --manifest-path $(wasm)/Cargo.toml fmt: @@ -220,9 +248,6 @@ opt-wasm: clean-wasm-scripts: make -C $(wasms) clean -publish-wasm: - aws s3 sync wasm s3://heliax-anoma-wasm-v1 --acl public-read --exclude "*" --include "*.wasm" --exclude "*/*" - dev-deps: $(rustup) toolchain install $(nightly) $(rustup) target add wasm32-unknown-unknown diff --git a/apps/build.rs b/apps/build.rs index 8a49789c8bd..514785c3e0c 100644 --- a/apps/build.rs +++ b/apps/build.rs @@ -72,6 +72,8 @@ fn main() { } } + let mut use_rustfmt = false; + // The version should match the one we use in the `Makefile` if let Ok(rustfmt_toolchain) = read_to_string(RUSTFMT_TOOLCHAIN_SRC) { // Try to find the path to rustfmt. @@ -91,6 +93,7 @@ fn main() { if !rustfmt.is_empty() { println!("using rustfmt from path \"{}\"", rustfmt); env::set_var("RUSTFMT", rustfmt); + use_rustfmt = true } } } @@ -98,7 +101,7 @@ fn main() { tonic_build::configure() .out_dir("src/lib/proto/generated") - .format(true) + .format(use_rustfmt) .extern_path(".types", "::namada::proto::generated::types") // This warning appears in tonic generated code .server_mod_attribute(".", "#[allow(clippy::unit_arg)]") diff --git a/apps/src/lib/client/tx.rs b/apps/src/lib/client/tx.rs index 1d0fc6c39a2..5335f7d4503 100644 --- a/apps/src/lib/client/tx.rs +++ b/apps/src/lib/client/tx.rs @@ -1,6 +1,8 @@ use std::borrow::Cow; use std::convert::TryFrom; +use std::env; use std::fs::File; +use std::time::Duration; use async_std::io::{self, WriteExt}; use borsh::BorshSerialize; @@ -71,6 +73,9 @@ const TX_UNBOND_WASM: &str = "tx_unbond.wasm"; const TX_WITHDRAW_WASM: &str = "tx_withdraw.wasm"; const VP_NFT: &str = "vp_nft.wasm"; +const ENV_VAR_ANOMA_TENDERMINT_WEBSOCKET_TIMEOUT: &str = + "ANOMA_TENDERMINT_WEBSOCKET_TIMEOUT"; + pub async fn submit_custom(ctx: Context, args: args::TxCustom) { let tx_code = ctx.read_wasm(args.code_path); let data = args.data_path.map(|data_path| { @@ -1116,9 +1121,21 @@ pub async fn broadcast_tx( } => (tx, wrapper_hash, decrypted_hash), _ => panic!("Cannot broadcast a dry-run transaction"), }; + + let websocket_timeout = + if let Ok(val) = env::var(ENV_VAR_ANOMA_TENDERMINT_WEBSOCKET_TIMEOUT) { + if let Ok(timeout) = val.parse::() { + Duration::new(timeout, 0) + } else { + Duration::new(300, 0) + } + } else { + Duration::new(300, 0) + }; + let mut wrapper_tx_subscription = TendermintWebsocketClient::open( WebSocketAddress::try_from(address.clone())?, - None, + Some(websocket_timeout), )?; let response = wrapper_tx_subscription @@ -1236,9 +1253,21 @@ pub async fn submit_tx( } => (tx, wrapper_hash, decrypted_hash), _ => panic!("Cannot broadcast a dry-run transaction"), }; + + let websocket_timeout = + if let Ok(val) = env::var(ENV_VAR_ANOMA_TENDERMINT_WEBSOCKET_TIMEOUT) { + if let Ok(timeout) = val.parse::() { + Duration::new(timeout, 0) + } else { + Duration::new(300, 0) + } + } else { + Duration::new(300, 0) + }; + let mut wrapper_tx_subscription = TendermintWebsocketClient::open( WebSocketAddress::try_from(address.clone())?, - None, + Some(websocket_timeout), )?; // It is better to subscribe to the transaction before it is broadcast diff --git a/apps/src/lib/node/ledger/mod.rs b/apps/src/lib/node/ledger/mod.rs index ca2c01c56c4..03f5aa36c7e 100644 --- a/apps/src/lib/node/ledger/mod.rs +++ b/apps/src/lib/node/ledger/mod.rs @@ -371,6 +371,7 @@ async fn run_aux(config: config::Ledger, wasm_dir: PathBuf) { }; // Construct our ABCI application. + let tendermint_mode = config.tendermint.tendermint_mode.clone(); let ledger_address = config.shell.ledger_address; let (shell, abci_service) = AbcippShim::new( config, @@ -402,6 +403,14 @@ async fn run_aux(config: config::Ledger, wasm_dir: PathBuf) { let shell_handler = thread_builder .spawn(move || { tracing::info!("Anoma ledger node started."); + match tendermint_mode { + TendermintMode::Validator => { + tracing::info!("This node is a validator"); + } + TendermintMode::Full | TendermintMode::Seed => { + tracing::info!("This node is not a validator"); + } + } shell.run() }) .expect("Must be able to start a thread for the shell"); diff --git a/apps/src/lib/node/ledger/tendermint_node.rs b/apps/src/lib/node/ledger/tendermint_node.rs index 0a2a48d7537..c38146cd355 100644 --- a/apps/src/lib/node/ledger/tendermint_node.rs +++ b/apps/src/lib/node/ledger/tendermint_node.rs @@ -1,4 +1,6 @@ +use std::env; use std::path::{Path, PathBuf}; +use std::process::Stdio; use std::str::FromStr; use borsh::BorshSerialize; @@ -30,6 +32,9 @@ use tokio::process::Command; use crate::config; +/// Env. var to output Tendermint log to stdout +pub const ENV_VAR_TM_STDOUT: &str = "ANOMA_TM_STDOUT"; + #[derive(Error, Debug)] pub enum Error { #[error("Failed to initialize Tendermint: {0}")] @@ -141,33 +146,39 @@ pub async fn run( update_tendermint_config(&home_dir, config).await?; - let mut tendermint_node = if !cfg!(feature = "ABCI") { - Command::new(&tendermint_path) - .args(&[ - "start", - "--mode", - &mode, - "--proxy-app", - &ledger_address, - "--home", - &home_dir_string, - ]) - .kill_on_drop(true) - .spawn() - .map_err(Error::StartUp)? + let mut tendermint_node = Command::new(&tendermint_path); + if !cfg!(feature = "ABCI") { + tendermint_node.args(&[ + "start", + "--mode", + &mode, + "--proxy-app", + &ledger_address, + "--home", + &home_dir_string, + ]) } else { - Command::new(&tendermint_path) - .args(&[ - "start", - "--proxy_app", - &ledger_address, - "--home", - &home_dir_string, - ]) - .kill_on_drop(true) - .spawn() - .map_err(Error::StartUp)? + tendermint_node.args(&[ + "start", + "--proxy_app", + &ledger_address, + "--home", + &home_dir_string, + ]) + }; + + let log_stdout = match env::var(ENV_VAR_TM_STDOUT) { + Ok(val) => val.to_ascii_lowercase().trim() == "true", + _ => false, }; + if !log_stdout { + tendermint_node.stdout(Stdio::null()); + } + + let mut tendermint_node = tendermint_node + .kill_on_drop(true) + .spawn() + .map_err(Error::StartUp)?; tracing::info!("Tendermint node started"); tokio::select! { diff --git a/apps/src/lib/wasm_loader/mod.rs b/apps/src/lib/wasm_loader/mod.rs index 6e174b7b5c8..b6cb424457d 100644 --- a/apps/src/lib/wasm_loader/mod.rs +++ b/apps/src/lib/wasm_loader/mod.rs @@ -32,7 +32,7 @@ pub enum Error { #[serde(transparent)] pub struct Checksums(pub HashMap); -const S3_URL: &str = "https://heliax-anoma-wasm-v1.s3.eu-west-1.amazonaws.com"; +const S3_URL: &str = "https://namada-wasm-master.s3.eu-west-1.amazonaws.com"; impl Checksums { /// Read WASM checksums from the given path diff --git a/rust-toolchain.toml b/rust-toolchain.toml index f7e58b63ce2..b12fcffcfec 100644 --- a/rust-toolchain.toml +++ b/rust-toolchain.toml @@ -1,3 +1,4 @@ [toolchain] channel = "1.61.0" components = ["rustc", "cargo", "rust-std", "rust-docs", "rls", "rust-src", "rust-analysis"] +targets = ['wasm32-unknown-unknown'] \ No newline at end of file diff --git a/scripts/ci/README.md b/scripts/ci/README.md deleted file mode 100644 index 7592907aeaa..00000000000 --- a/scripts/ci/README.md +++ /dev/null @@ -1,19 +0,0 @@ -# Drone script - -This folder contains some helper scripts to manage the pipeline. - -## update-drone-config.py - -This script is useful when you have to modify either `.drone.yml` or one of the `Makefile`. It should be modified only when adding a new `Makefile` or script to call within the CI. This is also possible by commenting `update ci` on the PR. - -### How to run: -- Ask for aws credential and [setup aws-cli with a profile](https://docs.aws.amazon.com/cli/latest/userguide/cli-configure-quickstart.html#cli-configure-quickstart-profiles). -- [Install poetry](https://python-poetry.org/docs/). -- [Install drone cli](https://docs.drone.io/cli/install/). -- Run `poetry install`. -- Run `poetry run python update-drone-config.py`. Check options with `--help` flag. - - If you need to use a profile different than `default` use `--aws-profile`. -- Check that `.drone.yml` has changed. -- Commit and push. - - diff --git a/scripts/ci/nix-check.sh b/scripts/ci/nix-check.sh deleted file mode 100644 index 35f7d9ceafc..00000000000 --- a/scripts/ci/nix-check.sh +++ /dev/null @@ -1,26 +0,0 @@ -#!/usr/bin/env bash - -set -euo pipefail - -echo Checking crate2nix... -nix run .#generateCargoNix -nix run .#generateCargoNixABCI-plus-plus -if ! git diff --exit-code --quiet crate2nix; then - echo error: generated crate2nix files are not up to date in git >&2 - exit 1 -fi - -echo Checking flake... -nix flake check -nix flake show - -echo Checking the Anoma program... -nix run 2>&1 | grep "Anoma command line interface" - -echo Checking auxiliary WASM checksums output... -nix build .#wasm -[ "$(find -L result -type f)" = checksums.json ] - -echo Checking docs build... -nix build .#anoma-docs -[ "$(ls result)" = html ] diff --git a/scripts/ci/poetry.lock b/scripts/ci/poetry.lock deleted file mode 100644 index 139a991a68c..00000000000 --- a/scripts/ci/poetry.lock +++ /dev/null @@ -1,120 +0,0 @@ -[[package]] -name = "boto3" -version = "1.20.25" -description = "The AWS SDK for Python" -category = "main" -optional = false -python-versions = ">= 3.6" - -[package.dependencies] -botocore = ">=1.23.25,<1.24.0" -jmespath = ">=0.7.1,<1.0.0" -s3transfer = ">=0.5.0,<0.6.0" - -[package.extras] -crt = ["botocore[crt] (>=1.21.0,<2.0a0)"] - -[[package]] -name = "botocore" -version = "1.23.25" -description = "Low-level, data-driven core of boto 3." -category = "main" -optional = false -python-versions = ">= 3.6" - -[package.dependencies] -jmespath = ">=0.7.1,<1.0.0" -python-dateutil = ">=2.1,<3.0.0" -urllib3 = ">=1.25.4,<1.27" - -[package.extras] -crt = ["awscrt (==0.12.5)"] - -[[package]] -name = "jmespath" -version = "0.10.0" -description = "JSON Matching Expressions" -category = "main" -optional = false -python-versions = ">=2.6, !=3.0.*, !=3.1.*, !=3.2.*" - -[[package]] -name = "python-dateutil" -version = "2.8.2" -description = "Extensions to the standard Python datetime module" -category = "main" -optional = false -python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,>=2.7" - -[package.dependencies] -six = ">=1.5" - -[[package]] -name = "s3transfer" -version = "0.5.0" -description = "An Amazon S3 Transfer Manager" -category = "main" -optional = false -python-versions = ">= 3.6" - -[package.dependencies] -botocore = ">=1.12.36,<2.0a.0" - -[package.extras] -crt = ["botocore[crt] (>=1.20.29,<2.0a.0)"] - -[[package]] -name = "six" -version = "1.16.0" -description = "Python 2 and 3 compatibility utilities" -category = "main" -optional = false -python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*" - -[[package]] -name = "urllib3" -version = "1.26.7" -description = "HTTP library with thread-safe connection pooling, file post, and more." -category = "main" -optional = false -python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, <4" - -[package.extras] -brotli = ["brotlipy (>=0.6.0)"] -secure = ["pyOpenSSL (>=0.14)", "cryptography (>=1.3.4)", "idna (>=2.0.0)", "certifi", "ipaddress"] -socks = ["PySocks (>=1.5.6,!=1.5.7,<2.0)"] - -[metadata] -lock-version = "1.1" -python-versions = "^3.8" -content-hash = "f477cf16d1090982cc7942a40cbc098059d8bc6b14a507a562d5f3c28809efb0" - -[metadata.files] -boto3 = [ - {file = "boto3-1.20.25-py3-none-any.whl", hash = "sha256:a03ab82efcbf457c36dab4afc545d0ddb72b5140c9993cffc78bc23b8d8baa40"}, - {file = "boto3-1.20.25.tar.gz", hash = "sha256:fa686e8e0a0559124aa9f19dcc25e6cc428f18ff11f779bb7346b883b353fdc2"}, -] -botocore = [ - {file = "botocore-1.23.25-py3-none-any.whl", hash = "sha256:98f8a23ca035edfbd8a8028e624eaf08ba22452dd78ad62eeb03fcd0bbd5f58d"}, - {file = "botocore-1.23.25.tar.gz", hash = "sha256:9eb71d5ee1ea335b3968751346601e923c66a7b324b1dc58e360d14e0d2ac136"}, -] -jmespath = [ - {file = "jmespath-0.10.0-py2.py3-none-any.whl", hash = "sha256:cdf6525904cc597730141d61b36f2e4b8ecc257c420fa2f4549bac2c2d0cb72f"}, - {file = "jmespath-0.10.0.tar.gz", hash = "sha256:b85d0567b8666149a93172712e68920734333c0ce7e89b78b3e987f71e5ed4f9"}, -] -python-dateutil = [ - {file = "python-dateutil-2.8.2.tar.gz", hash = "sha256:0123cacc1627ae19ddf3c27a5de5bd67ee4586fbdd6440d9748f8abb483d3e86"}, - {file = "python_dateutil-2.8.2-py2.py3-none-any.whl", hash = "sha256:961d03dc3453ebbc59dbdea9e4e11c5651520a876d0f4db161e8674aae935da9"}, -] -s3transfer = [ - {file = "s3transfer-0.5.0-py3-none-any.whl", hash = "sha256:9c1dc369814391a6bda20ebbf4b70a0f34630592c9aa520856bf384916af2803"}, - {file = "s3transfer-0.5.0.tar.gz", hash = "sha256:50ed823e1dc5868ad40c8dc92072f757aa0e653a192845c94a3b676f4a62da4c"}, -] -six = [ - {file = "six-1.16.0-py2.py3-none-any.whl", hash = "sha256:8abb2f1d86890a2dfb989f9a77cfcfd3e47c2a354b01111771326f8aa26e0254"}, - {file = "six-1.16.0.tar.gz", hash = "sha256:1e61c37477a1626458e36f7b1d82aa5c9b094fa4802892072e49de9c60c4c926"}, -] -urllib3 = [ - {file = "urllib3-1.26.7-py2.py3-none-any.whl", hash = "sha256:c4fdf4019605b6e5423637e01bc9fe4daef873709a7973e195ceba0a62bbc844"}, - {file = "urllib3-1.26.7.tar.gz", hash = "sha256:4987c65554f7a2dbf30c18fd48778ef124af6fab771a377103da0585e2336ece"}, -] diff --git a/scripts/ci/pyproject.toml b/scripts/ci/pyproject.toml deleted file mode 100644 index 8948015cc11..00000000000 --- a/scripts/ci/pyproject.toml +++ /dev/null @@ -1,15 +0,0 @@ -[tool.poetry] -authors = ["Gianmarco Fraccaroli "] -description = "update drone configuration signature" -name = "drone-signature-updater" -version = "0.1.0" - -[tool.poetry.dependencies] -boto3 = "^1.17.84" -python = "^3.8" - -[tool.poetry.dev-dependencies] - -[build-system] -build-backend = "poetry.core.masonry.api" -requires = ["poetry-core>=1.0.0"] diff --git a/scripts/ci/update-drone-config.py b/scripts/ci/update-drone-config.py deleted file mode 100644 index cb7906d499c..00000000000 --- a/scripts/ci/update-drone-config.py +++ /dev/null @@ -1,59 +0,0 @@ -import os -from pathlib import Path -from typing import Dict - - -from botocore.config import Config -import argparse - -import boto3 - - -DRONE_FILE: str = ".drone" -DRONE_FILE_SUFFIX: str = ".yml" -REPOSITORY: str = "anoma/anoma" - -boto_config = Config( - region_name='eu-west-1', - retries={ - 'max_attempts': 5, - 'mode': 'standard' - } -) - -client = boto3.client('ssm', config=boto_config) - - -def get_project_root() -> Path: - return Path(__file__).parent.parent.parent - - -def get_drone_token() -> str: - try: - return client.get_parameter( - Name='drone_machine_secret', - WithDecryption=True - )['Parameter']['Value'] - except Exception as e: - print(e) - exit(1) - - -def sign_drone_config(): - project_root = get_project_root() - token = get_drone_token() - try: - os.system(' '.join(["cd {} &&".format(project_root), "DRONE_TOKEN={}".format(token), "DRONE_SERVER={}".format('https://ci.heliax.dev'), "drone", "sign", "--save", REPOSITORY])) - except Exception as e: - print(e) - exit(1) - - -if __name__ == "__main__": - parser = argparse.ArgumentParser("Update drone configuration.") - parser.add_argument('--aws-profile', help='The name of the AWS profile to use.', type=str, default="default") - args = parser.parse_args() - - boto3.setup_default_session(profile_name=args.aws_profile) - - sign_drone_config() \ No newline at end of file diff --git a/scripts/install/get_tendermint.sh b/scripts/get_tendermint.sh similarity index 99% rename from scripts/install/get_tendermint.sh rename to scripts/get_tendermint.sh index 0e916fbc176..8ad0498304b 100755 --- a/scripts/install/get_tendermint.sh +++ b/scripts/get_tendermint.sh @@ -45,4 +45,3 @@ curl -LsSfo "$TMP_PATH"/tendermint.tar.gz "$RELEASE_URL" || error_exit "tendermi cd $TARGET_PATH sudo tar -xvzf $TMP_PATH/tendermint.tar.gz tendermint || error_exit "tendermint release extraction failed" - diff --git a/scripts/make-package.sh b/scripts/make-package.sh index 8ae06589bb7..a95c3c5ad72 100755 --- a/scripts/make-package.sh +++ b/scripts/make-package.sh @@ -4,7 +4,7 @@ set -e -VERSION="$(git describe --dirty --broken)" +VERSION="$(git describe --tags)" PLATFORM="$(uname -s)-$(uname -m)" PACKAGE_NAME="namada-${VERSION}-${PLATFORM}" BIN="namada namadac namadan namadaw" diff --git a/shared/build.rs b/shared/build.rs index e2f529c6d0b..d87f7a918fd 100644 --- a/shared/build.rs +++ b/shared/build.rs @@ -32,6 +32,8 @@ fn main() { } } + let mut use_rustfmt = false; + // The version should match the one we use in the `Makefile` if let Ok(rustfmt_toolchain) = read_to_string(RUSTFMT_TOOLCHAIN_SRC) { // Try to find the path to rustfmt. @@ -51,6 +53,7 @@ fn main() { if !rustfmt.is_empty() { println!("using rustfmt from path \"{}\"", rustfmt); env::set_var("RUSTFMT", rustfmt); + use_rustfmt = true } } } @@ -58,7 +61,7 @@ fn main() { tonic_build::configure() .out_dir("src/proto/generated") - .format(true) + .format(use_rustfmt) // TODO try to add json encoding to simplify use for user // .type_attribute("types.Intent", "#[derive(serde::Serialize, // serde::Deserialize)]") diff --git a/tests/Cargo.toml b/tests/Cargo.toml index ea4fe1cfcc8..a4ea87e21e6 100644 --- a/tests/Cargo.toml +++ b/tests/Cargo.toml @@ -59,6 +59,7 @@ libp2p = "0.38.0" pretty_assertions = "0.7.2" # A fork with state machine testing proptest = {git = "https://github.com/heliaxdev/proptest", branch = "tomas/sm"} +rand = "0.8" toml = "0.5.9" # This is used to enable logging from tests diff --git a/tests/src/e2e/eth_bridge_tests.rs b/tests/src/e2e/eth_bridge_tests.rs index 4fcac7cc8fa..70e753e8131 100644 --- a/tests/src/e2e/eth_bridge_tests.rs +++ b/tests/src/e2e/eth_bridge_tests.rs @@ -41,7 +41,7 @@ fn everything() { anoman_ledger.exp_string("Committed block hash").unwrap(); let _bg_ledger = anoman_ledger.background(); - let tx_data_path = test.base_dir.path().join("queue_storage_key.txt"); + let tx_data_path = test.test_dir.path().join("queue_storage_key.txt"); std::fs::write(&tx_data_path, &storage_key("queue")[..]).unwrap(); let tx_code_path = wasm_abs_path(TX_WRITE_STORAGE_KEY_WASM); diff --git a/tests/src/e2e/gossip_tests.rs b/tests/src/e2e/gossip_tests.rs index ebf34b948f2..feb0e88324c 100644 --- a/tests/src/e2e/gossip_tests.rs +++ b/tests/src/e2e/gossip_tests.rs @@ -32,6 +32,7 @@ use crate::{run, run_as}; /// 1. Check that a gossip node can start and stop cleanly /// 2. Check that two peers connected to the same seed node discover each other #[test] +#[ignore] // this is not currently being developed, run with `cargo test -- --ignored` fn run_gossip() -> Result<()> { let test = setup::network(|genesis| setup::add_validators(2, genesis), None)?; @@ -97,6 +98,7 @@ fn run_gossip() -> Result<()> { /// and sends them to the matchmaker. The matchmaker should be able to match /// them into a transfer transaction and submit it to the ledger. #[test] +#[ignore] // this is not currently being developed, run with `cargo test -- --ignored` fn match_intents() -> Result<()> { let test = setup::single_node_net()?; @@ -127,9 +129,9 @@ fn match_intents() -> Result<()> { ledger.exp_regex(r"Committed block hash.*, height: [0-9]+")?; let bg_ledger = ledger.background(); - let intent_a_path_input = test.base_dir.path().join("intent.A.data"); - let intent_b_path_input = test.base_dir.path().join("intent.B.data"); - let intent_c_path_input = test.base_dir.path().join("intent.C.data"); + let intent_a_path_input = test.test_dir.path().join("intent.A.data"); + let intent_b_path_input = test.test_dir.path().join("intent.B.data"); + let intent_c_path_input = test.test_dir.path().join("intent.C.data"); let albert = find_address(&test, ALBERT)?; let bertha = find_address(&test, BERTHA)?; diff --git a/tests/src/e2e/helpers.rs b/tests/src/e2e/helpers.rs index 805b4c820f2..f72044b80a9 100644 --- a/tests/src/e2e/helpers.rs +++ b/tests/src/e2e/helpers.rs @@ -1,16 +1,21 @@ //! E2E test helpers +use std::path::Path; +use std::process::Command; use std::str::FromStr; +use std::{env, time}; use color_eyre::eyre::Result; +use color_eyre::owo_colors::OwoColorize; +use escargot::CargoBuild; use eyre::eyre; use namada::types::address::Address; use namada::types::key::*; use namada::types::storage::Epoch; use namada_apps::config::{Config, TendermintMode}; -use super::setup::Test; -use crate::e2e::setup::{Bin, Who}; +use super::setup::{Test, ENV_VAR_DEBUG, ENV_VAR_USE_PREBUILT_BINARIES}; +use crate::e2e::setup::{Bin, Who, APPS_PACKAGE}; use crate::run; /// Find the address of an account by its alias from the wallet @@ -19,10 +24,14 @@ pub fn find_address(test: &Test, alias: impl AsRef) -> Result
{ test, Bin::Wallet, &["address", "find", "--alias", alias.as_ref()], - Some(1) + Some(10) )?; - let (unread, matched) = find.exp_regex("Found address .*\n")?; - let address_str = matched.trim().rsplit_once(' ').unwrap().1; + let (unread, matched) = find.exp_regex("Found address .*")?; + let address_str = strip_trailing_newline(&matched) + .trim() + .rsplit_once(' ') + .unwrap() + .1; let address = Address::from_str(address_str).map_err(|e| { eyre!(format!( "Address: {} parsed from {}, Error: {}\n\nOutput: {}", @@ -73,12 +82,20 @@ pub fn find_keypair( alias.as_ref(), "--unsafe-show-secret" ], - Some(1) + Some(10) )?; - let (_unread, matched) = find.exp_regex("Public key: .*\n")?; - let pk = matched.trim().rsplit_once(' ').unwrap().1; - let (unread, matched) = find.exp_regex("Secret key: .*\n")?; - let sk = matched.trim().rsplit_once(' ').unwrap().1; + let (_unread, matched) = find.exp_regex("Public key: .*")?; + let pk = strip_trailing_newline(&matched) + .trim() + .rsplit_once(' ') + .unwrap() + .1; + let (unread, matched) = find.exp_regex("Secret key: .*")?; + let sk = strip_trailing_newline(&matched) + .trim() + .rsplit_once(' ') + .unwrap() + .1; let key = format!("{}{}", sk, pk); common::SecretKey::from_str(&key).map_err(|e| { eyre!(format!( @@ -104,10 +121,14 @@ pub fn find_voting_power( "--ledger-address", ledger_address ], - Some(1) + Some(10) )?; - let (unread, matched) = find.exp_regex("voting power: .*\n")?; - let voting_power_str = matched.trim().rsplit_once(' ').unwrap().1; + let (unread, matched) = find.exp_regex("voting power: .*")?; + let voting_power_str = strip_trailing_newline(&matched) + .trim() + .rsplit_once(' ') + .unwrap() + .1; u64::from_str(voting_power_str).map_err(|e| { eyre!(format!( "Voting power: {} parsed from {}, Error: {}\n\nOutput: {}", @@ -122,10 +143,14 @@ pub fn get_epoch(test: &Test, ledger_address: &str) -> Result { test, Bin::Client, &["epoch", "--ledger-address", ledger_address], - Some(5) + Some(10) )?; - let (unread, matched) = find.exp_regex("Last committed epoch: .*\n")?; - let epoch_str = matched.trim().rsplit_once(' ').unwrap().1; + let (unread, matched) = find.exp_regex("Last committed epoch: .*")?; + let epoch_str = strip_trailing_newline(&matched) + .trim() + .rsplit_once(' ') + .unwrap() + .1; let epoch = u64::from_str(epoch_str).map_err(|e| { eyre!(format!( "Epoch: {} parsed from {}, Error: {}\n\nOutput: {}", @@ -134,3 +159,83 @@ pub fn get_epoch(test: &Test, ledger_address: &str) -> Result { })?; Ok(Epoch(epoch)) } + +pub fn generate_bin_command(bin_name: &str, manifest_path: &Path) -> Command { + let use_prebuilt_binaries = match env::var(ENV_VAR_USE_PREBUILT_BINARIES) { + Ok(var) => var.to_ascii_lowercase() != "false", + Err(_) => false, + }; + + // Allow to run in debug + let run_debug = match env::var(ENV_VAR_DEBUG) { + Ok(val) => val.to_ascii_lowercase() != "false", + _ => false, + }; + + if !use_prebuilt_binaries { + let build_cmd = if !cfg!(feature = "ABCI") { + CargoBuild::new() + .package(APPS_PACKAGE) + .manifest_path(manifest_path) + .no_default_features() + .features("ABCI-plus-plus") + // Explicitly disable dev, in case it's enabled when a test is + // invoked + .env("ANOMA_DEV", "false") + .bin(bin_name) + } else { + CargoBuild::new() + .package(APPS_PACKAGE) + .manifest_path(manifest_path) + .features("ABCI") + // Explicitly disable dev, in case it's enabled when a test is + // invoked + .env("ANOMA_DEV", "false") + .bin(bin_name) + }; + + let build_cmd = if run_debug { + build_cmd + } else { + // Use the same build settings as `make build-release` + build_cmd.release() + }; + + let now = time::Instant::now(); + // ideally we would print the compile command here, but escargot doesn't + // implement Display or Debug for CargoBuild + println!( + "\n{}: {}", + "`cargo build` starting".underline().bright_blue(), + bin_name + ); + + let command = build_cmd.run().unwrap(); + println!( + "\n{}: {}ms", + "`cargo build` finished after".underline().bright_blue(), + now.elapsed().as_millis() + ); + + command.command() + } else { + let dir = if run_debug { + format!("target/debug/{}", bin_name) + } else { + format!("target/release/{}", bin_name) + }; + println!( + "\n{}: {}", + "Running prebuilt binaries from".underline().green(), + dir + ); + std::process::Command::new(dir) + } +} + +fn strip_trailing_newline(input: &str) -> &str { + input + .strip_suffix("\r\n") + .or_else(|| input.strip_suffix('\n')) + .unwrap_or(input) +} diff --git a/tests/src/e2e/ledger_tests.rs b/tests/src/e2e/ledger_tests.rs index 1bbc74b5d55..f150e6eac0a 100644 --- a/tests/src/e2e/ledger_tests.rs +++ b/tests/src/e2e/ledger_tests.rs @@ -244,12 +244,6 @@ fn ledger_txs_and_queries() -> Result<()> { run_as!(test, Who::Validator(0), Bin::Node, &["ledger"], Some(40))?; ledger.exp_string("Anoma ledger node started")?; - if !cfg!(feature = "ABCI") { - ledger.exp_string("started node")?; - } else { - ledger.exp_string("Started node")?; - } - let _bg_ledger = ledger.background(); let vp_user = wasm_abs_path(VP_USER_WASM); @@ -423,11 +417,6 @@ fn invalid_transactions() -> Result<()> { let mut ledger = run_as!(test, Who::Validator(0), Bin::Node, &["ledger"], Some(40))?; ledger.exp_string("Anoma ledger node started")?; - if !cfg!(feature = "ABCI") { - ledger.exp_string("started node")?; - } else { - ledger.exp_string("Started node")?; - } // Wait to commit a block ledger.exp_regex(r"Committed block hash.*, height: [0-9]+")?; @@ -435,7 +424,7 @@ fn invalid_transactions() -> Result<()> { // 2. Submit a an invalid transaction (trying to mint tokens should fail // in the token's VP) - let tx_data_path = test.base_dir.path().join("tx.data"); + let tx_data_path = test.test_dir.path().join("tx.data"); let transfer = token::Transfer { source: find_address(&test, DAEWON)?, target: find_address(&test, ALBERT)?, @@ -583,11 +572,6 @@ fn pos_bonds() -> Result<()> { run_as!(test, Who::Validator(0), Bin::Node, &["ledger"], Some(40))?; ledger.exp_string("Anoma ledger node started")?; - if !cfg!(feature = "ABCI") { - ledger.exp_string("started node")?; - } else { - ledger.exp_string("Started node")?; - } let _bg_ledger = ledger.background(); let validator_one_rpc = get_actor_rpc(&test, &Who::Validator(0)); @@ -781,11 +765,6 @@ fn pos_init_validator() -> Result<()> { run_as!(test, Who::Validator(0), Bin::Node, &["ledger"], Some(40))?; ledger.exp_string("Anoma ledger node started")?; - if !cfg!(feature = "ABCI") { - ledger.exp_string("started node")?; - } else { - ledger.exp_string("Started node")?; - } let _bg_ledger = ledger.background(); let validator_one_rpc = get_actor_rpc(&test, &Who::Validator(0)); @@ -948,11 +927,6 @@ fn ledger_many_txs_in_a_block() -> Result<()> { run_as!(*test, Who::Validator(0), Bin::Node, &["ledger"], Some(40))?; ledger.exp_string("Anoma ledger node started")?; - if !cfg!(feature = "ABCI") { - ledger.exp_string("started node")?; - } else { - ledger.exp_string("Started node")?; - } // Wait to commit a block ledger.exp_regex(r"Committed block hash.*, height: [0-9]+")?; @@ -1043,11 +1017,6 @@ fn proposal_submission() -> Result<()> { run_as!(test, Who::Validator(0), Bin::Node, &["ledger"], Some(40))?; ledger.exp_string("Anoma ledger node started")?; - if !cfg!(feature = "ABCI") { - ledger.exp_string("started node")?; - } else { - ledger.exp_string("Started node")?; - } let _bg_ledger = ledger.background(); let validator_one_rpc = get_actor_rpc(&test, &Who::Validator(0)); @@ -1076,7 +1045,7 @@ fn proposal_submission() -> Result<()> { // 2. Submit valid proposal let valid_proposal_json_path = - test.base_dir.path().join("valid_proposal.json"); + test.test_dir.path().join("valid_proposal.json"); let proposal_code = wasm_abs_path(TX_PROPOSAL_CODE); let albert = find_address(&test, ALBERT)?; @@ -1166,7 +1135,7 @@ fn proposal_submission() -> Result<()> { // 6. Submit an invalid proposal // proposal is invalid due to voting_end_epoch - voting_start_epoch < 3 let invalid_proposal_json_path = - test.base_dir.path().join("invalid_proposal.json"); + test.test_dir.path().join("invalid_proposal.json"); let albert = find_address(&test, ALBERT)?; let invalid_proposal_json = json!( { @@ -1395,11 +1364,6 @@ fn proposal_offline() -> Result<()> { run_as!(test, Who::Validator(0), Bin::Node, &["ledger"], Some(20))?; ledger.exp_string("Anoma ledger node started")?; - if !cfg!(feature = "ABCI") { - ledger.exp_string("started node")?; - } else { - ledger.exp_string("Started node")?; - } let _bg_ledger = ledger.background(); let validator_one_rpc = get_actor_rpc(&test, &Who::Validator(0)); @@ -1428,7 +1392,7 @@ fn proposal_offline() -> Result<()> { // 2. Create an offline proposal let valid_proposal_json_path = - test.base_dir.path().join("valid_proposal.json"); + test.test_dir.path().join("valid_proposal.json"); let albert = find_address(&test, ALBERT)?; let valid_proposal_json = json!( { @@ -1565,7 +1529,6 @@ fn test_genesis_validators() -> Result<()> { self, ValidatorPreGenesisConfig, }; use namada_apps::config::Config; - use tempfile::tempdir; // This test is not using the `setup::network`, because we're setting up // custom genesis validators @@ -1576,7 +1539,7 @@ fn test_genesis_validators() -> Result<()> { }); let working_dir = setup::working_dir(); - let base_dir = tempdir().unwrap(); + let test_dir = setup::TestDir::new(); let checksums_path = working_dir .join("wasm/checksums.json") .to_string_lossy() @@ -1616,14 +1579,14 @@ fn test_genesis_validators() -> Result<()> { ], Some(5), &working_dir, - &base_dir, + &test_dir, "validator", format!("{}:{}", std::file!(), std::line!()), )?; init_genesis_validator_0.assert_success(); let validator_0_pre_genesis_dir = namada_apps::client::utils::validator_pre_genesis_dir( - base_dir.path(), + test_dir.path(), validator_0_alias, ); let config = std::fs::read_to_string( @@ -1652,14 +1615,14 @@ fn test_genesis_validators() -> Result<()> { ], Some(5), &working_dir, - &base_dir, + &test_dir, "validator", format!("{}:{}", std::file!(), std::line!()), )?; init_genesis_validator_1.assert_success(); let validator_1_pre_genesis_dir = namada_apps::client::utils::validator_pre_genesis_dir( - base_dir.path(), + test_dir.path(), validator_1_alias, ); let config = std::fs::read_to_string( @@ -1705,11 +1668,11 @@ fn test_genesis_validators() -> Result<()> { update_validator_config(1, validator_1_config), ), ]); - let genesis_file = base_dir.path().join("e2e-test-genesis-src.toml"); + let genesis_file = test_dir.path().join("e2e-test-genesis-src.toml"); genesis_config::write_genesis_config(&genesis, &genesis_file); let genesis_path = genesis_file.to_string_lossy(); - let archive_dir = base_dir.path().to_string_lossy().to_string(); + let archive_dir = test_dir.path().to_string_lossy().to_string(); let args = vec![ "utils", "init-network", @@ -1730,7 +1693,7 @@ fn test_genesis_validators() -> Result<()> { args, Some(5), &working_dir, - &base_dir, + &test_dir, "validator", format!("{}:{}", std::file!(), std::line!()), )?; @@ -1747,7 +1710,7 @@ fn test_genesis_validators() -> Result<()> { }; let test = setup::Test { working_dir: working_dir.clone(), - base_dir, + test_dir, net, genesis, }; @@ -1844,7 +1807,7 @@ fn test_genesis_validators() -> Result<()> { .unwrap(); // Copy WASMs to each node's chain dir - let chain_dir = test.base_dir.path().join(chain_id.as_str()); + let chain_dir = test.test_dir.path().join(chain_id.as_str()); setup::copy_wasm_to_chain_dir( &working_dir, &chain_dir, diff --git a/tests/src/e2e/setup.rs b/tests/src/e2e/setup.rs index eaa5f856056..2b0aba06966 100644 --- a/tests/src/e2e/setup.rs +++ b/tests/src/e2e/setup.rs @@ -1,29 +1,37 @@ use std::ffi::OsStr; +use std::fmt::Display; +use std::fs::{File, OpenOptions}; use std::net::SocketAddr; use std::path::{Path, PathBuf}; use std::process::Command; use std::str::FromStr; use std::sync::Once; -use std::{env, fs, mem, thread, time}; +use std::time::{SystemTime, UNIX_EPOCH}; +use std::{env, fs, thread, time}; use assert_cmd::assert::OutputAssertExt; use color_eyre::eyre::Result; use color_eyre::owo_colors::OwoColorize; -use escargot::CargoBuild; +use expectrl::process::unix::{PtyStream, UnixProcess}; use expectrl::session::Session; +use expectrl::stream::log::LoggedStream; use expectrl::{Eof, WaitStatus}; use eyre::eyre; +use itertools::{Either, Itertools}; use namada::types::chain::ChainId; use namada_apps::client::utils; use namada_apps::config::genesis::genesis_config::{self, GenesisConfig}; use namada_apps::{config, wallet}; +use rand::Rng; use tempfile::{tempdir, TempDir}; +use crate::e2e::helpers::generate_bin_command; + /// For `color_eyre::install`, which fails if called more than once in the same /// process pub static INIT: Once = Once::new(); -const APPS_PACKAGE: &str = "namada_apps"; +pub const APPS_PACKAGE: &str = "namada_apps"; /// Env. var for running E2E tests in debug mode pub const ENV_VAR_DEBUG: &str = "ANOMA_E2E_DEBUG"; @@ -31,6 +39,11 @@ pub const ENV_VAR_DEBUG: &str = "ANOMA_E2E_DEBUG"; /// Env. var for keeping temporary files created by the E2E tests const ENV_VAR_KEEP_TEMP: &str = "ANOMA_E2E_KEEP_TEMP"; +/// Env. var to use a set of prebuilt binaries. This variable holds the path to +/// a folder. +pub const ENV_VAR_USE_PREBUILT_BINARIES: &str = + "ANOMA_E2E_USE_PREBUILT_BINARIES"; + /// The E2E tests genesis config source. /// This file must contain a single validator with alias "validator-0". /// To add more validators, use the [`add_validators`] function in the call to @@ -95,7 +108,6 @@ pub fn single_node_net() -> Result { network(|genesis| genesis, None) } -/// Setup a configurable network. pub fn network( update_genesis: impl Fn(GenesisConfig) -> GenesisConfig, consensus_timeout_commit: Option<&'static str>, @@ -105,8 +117,9 @@ pub fn network( eprintln!("Failed setting up colorful error reports {}", err); } }); + let working_dir = working_dir(); - let base_dir = tempdir().unwrap(); + let test_dir = TestDir::new(); // Open the source genesis file let mut genesis = genesis_config::open_genesis_config( @@ -131,7 +144,7 @@ pub fn network( // Run `init-network` to generate the finalized genesis config, keys and // addresses and update WASM checksums - let genesis_file = base_dir.path().join("e2e-test-genesis-src.toml"); + let genesis_file = test_dir.path().join("e2e-test-genesis-src.toml"); genesis_config::write_genesis_config(&genesis, &genesis_file); let genesis_path = genesis_file.to_string_lossy(); let checksums_path = working_dir @@ -161,7 +174,7 @@ pub fn network( args, Some(5), &working_dir, - &base_dir, + &test_dir, "validator", format!("{}:{}", std::file!(), std::line!()), )?; @@ -177,7 +190,7 @@ pub fn network( // Move the "others" accounts wallet in the main base dir, so that we can // use them with `Who::NonValidator` - let chain_dir = base_dir.path().join(net.chain_id.as_str()); + let chain_dir = test_dir.path().join(net.chain_id.as_str()); std::fs::rename( wallet::wallet_file( chain_dir @@ -197,7 +210,7 @@ pub fn network( Ok(Test { working_dir, - base_dir, + test_dir, net, genesis, }) @@ -213,33 +226,64 @@ pub enum Bin { #[derive(Debug)] pub struct Test { + /// The dir where the tests run from, usually the repo root dir pub working_dir: PathBuf, - pub base_dir: TempDir, + /// Temporary test directory is used as the default base-dir for running + /// Anoma cmds + pub test_dir: TestDir, pub net: Network, pub genesis: GenesisConfig, } -impl Drop for Test { - fn drop(&mut self) { +#[derive(Debug)] +pub struct TestDir(Either); + +impl AsRef for TestDir { + fn as_ref(&self) -> &Path { + match &self.0 { + Either::Left(temp_dir) => temp_dir.path(), + Either::Right(path) => path.as_ref(), + } + } +} + +impl TestDir { + /// Setup a `TestDir` in a temporary directory. The directory will be + /// automatically deleted after the test run, unless `ENV_VAR_KEEP_TEMP` + /// is set to `true`. + pub fn new() -> Self { let keep_temp = match env::var(ENV_VAR_KEEP_TEMP) { Ok(val) => val.to_ascii_lowercase() != "false", _ => false, }; + if keep_temp { - if cfg!(any(unix, target_os = "redox", target_os = "wasi")) { - let path = mem::replace(&mut self.base_dir, tempdir().unwrap()); - println!( - "{}: \"{}\"", - "Keeping temporary directory at".underline().yellow(), - path.path().to_string_lossy() - ); - mem::forget(path); - } else { - eprintln!( - "Setting {} is not supported on this platform", - ENV_VAR_KEEP_TEMP - ); - } + let path = tempdir().unwrap().into_path(); + println!( + "{}: \"{}\"", + "Keeping test directory at".underline().yellow(), + path.to_string_lossy() + ); + Self(Either::Right(path)) + } else { + Self(Either::Left(tempdir().unwrap())) + } + } + + /// Get the [`Path`] to the test directory. + pub fn path(&self) -> &Path { + self.as_ref() + } +} + +impl Drop for Test { + fn drop(&mut self) { + if let Either::Right(path) = &self.test_dir.0 { + println!( + "{}: \"{}\"", + "Keeping test directory at".underline().yellow(), + path.to_string_lossy() + ); } } } @@ -369,9 +413,9 @@ impl Test { pub fn get_base_dir(&self, who: &Who) -> PathBuf { match who { - Who::NonValidator => self.base_dir.path().to_owned(), + Who::NonValidator => self.test_dir.path().to_owned(), Who::Validator(index) => self - .base_dir + .test_dir .path() .join(self.net.chain_id.as_str()) .join(utils::NET_ACCOUNTS_DIR) @@ -401,8 +445,20 @@ pub fn working_dir() -> PathBuf { /// A command under test pub struct AnomaCmd { - pub session: Session, + pub session: Session>, pub cmd_str: String, + pub log_path: PathBuf, +} + +impl Display for AnomaCmd { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!( + f, + "{}\nLogs: {}", + self.cmd_str, + self.log_path.to_string_lossy() + ) + } } /// A command under test running on a background thread @@ -465,15 +521,18 @@ impl AnomaCmd { /// Wrapper over the inner `PtySession`'s functions with custom error /// reporting. pub fn exp_string(&mut self, needle: &str) -> Result { - let found = self - .session - .expect_eager(needle) - .map_err(|e| eyre!(format!("{}\n Needle: {}", e, needle)))?; + let found = self.session.expect_eager(needle).map_err(|e| { + eyre!("{}\nCommand: {}\n Needle: {}", e, self, needle) + })?; if found.is_empty() { - Err(eyre!(format!("Expected needle not found: {}", needle))) + Err(eyre!( + "Expected needle not found\nCommand: {}\n Needle: {}", + self, + needle + )) } else { String::from_utf8(found.before().to_vec()) - .map_err(|e| eyre!(format!("{}", e))) + .map_err(|e| eyre!("Error: {}\nCommand: {}", e, self)) } } @@ -488,15 +547,19 @@ impl AnomaCmd { let found = self .session .expect_eager(expectrl::Regex(regex)) - .map_err(|e| eyre!(format!("{}", e)))?; + .map_err(|e| eyre!("Error: {}\nCommand: {}", e, self))?; if found.is_empty() { - Err(eyre!(format!("Expected regex not found: {}", regex))) + Err(eyre!( + "Expected regex not found: {}\nCommand: {}", + regex, + self + )) } else { let unread = String::from_utf8(found.before().to_vec()) - .map_err(|e| eyre!(format!("{}", e)))?; + .map_err(|e| eyre!("Error: {}\nCommand: {}", e, self))?; let matched = String::from_utf8(found.matches().next().unwrap().to_vec()) - .map_err(|e| eyre!(format!("{}", e)))?; + .map_err(|e| eyre!("Error: {}\nCommand: {}", e, self))?; Ok((unread, matched)) } } @@ -508,13 +571,15 @@ impl AnomaCmd { /// reporting. #[allow(dead_code)] pub fn exp_eof(&mut self) -> Result { - let found = - self.session.expect_eager(Eof).map_err(|e| eyre!("{}", e))?; + let found = self + .session + .expect_eager(Eof) + .map_err(|e| eyre!("Error: {}\nCommand: {}", e, self))?; if found.is_empty() { - Err(eyre!("Expected EOF")) + Err(eyre!("Expected EOF\nCommand: {}", self)) } else { String::from_utf8(found.before().to_vec()) - .map_err(|e| eyre!(format!("{}", e))) + .map_err(|e| eyre!(format!("Error: {}\nCommand: {}", e, self))) } } @@ -529,7 +594,7 @@ impl AnomaCmd { pub fn send_control(&mut self, c: char) -> Result<()> { self.session .send_control(c) - .map_err(|e| eyre!(format!("{}", e))) + .map_err(|e| eyre!("Error: {}\nCommand: {}", e, self)) } /// send line to repl (and flush output) and then, if echo_on=true wait for @@ -541,7 +606,7 @@ impl AnomaCmd { pub fn send_line(&mut self, line: &str) -> Result<()> { self.session .send_line(line) - .map_err(|e| eyre!(format!("{}", e))) + .map_err(|e| eyre!("Error: {}\nCommand: {}", e, self)) } } @@ -550,7 +615,7 @@ impl Drop for AnomaCmd { // attempt to clean up the process println!( "{}: {}", - "Waiting for command to finish".underline().yellow(), + "> Sending Ctrl+C to command".underline().yellow(), self.cmd_str, ); let _result = self.send_control('c'); @@ -558,7 +623,7 @@ impl Drop for AnomaCmd { Err(error) => { eprintln!( "\n{}: {}\n{}: {}", - "Error waiting for command to finish".underline().red(), + "> Error ensuring command is finished".underline().red(), self.cmd_str, "Error".underline().red(), error, @@ -567,21 +632,21 @@ impl Drop for AnomaCmd { Ok(output) => { println!( "\n{}: {}", - "Command finished".underline().green(), + "> Command finished".underline().green(), self.cmd_str, ); let output = output.trim(); if !output.is_empty() { println!( "\n{}: {}\n\n{}", - "Unread output for command".underline().yellow(), + "> Unread output for command".underline().yellow(), self.cmd_str, output ); } else { println!( "\n{}: {}", - "No unread output for command".underline().green(), + "> No unread output for command".underline().green(), self.cmd_str ); } @@ -607,60 +672,21 @@ where S: AsRef, { // Root cargo workspace manifest path - let manifest_path = working_dir.as_ref().join("Cargo.toml"); let bin_name = match bin { - Bin::Node => "anoman", - Bin::Client => "anomac", - Bin::Wallet => "anomaw", - }; - // Allow to run in debug - let run_debug = match env::var(ENV_VAR_DEBUG) { - Ok(val) => val.to_ascii_lowercase() != "false", - _ => false, - }; - let build_cmd = if !cfg!(feature = "ABCI") { - CargoBuild::new() - .package(APPS_PACKAGE) - .manifest_path(manifest_path) - .no_default_features() - .features("ABCI-plus-plus") - // Explicitly disable dev, in case it's enabled when a test is - // invoked - .env("ANOMA_DEV", "false") - .bin(bin_name) - } else { - CargoBuild::new() - .package(APPS_PACKAGE) - .manifest_path(manifest_path) - .features("ABCI") - // Explicitly disable dev, in case it's enabled when a test is - // invoked - .env("ANOMA_DEV", "false") - .bin(bin_name) + Bin::Node => "namadan", + Bin::Client => "namadac", + Bin::Wallet => "namadaw", }; - let build_cmd = if run_debug { - build_cmd - } else { - // Use the same build settings as `make build-release` - build_cmd.release() - }; - let now = time::Instant::now(); - // ideally we would print the compile command here, but escargot doesn't - // implement Display or Debug for CargoBuild - println!( - "\n{}: {}", - "`cargo build` starting".underline().bright_blue(), - bin_name - ); - let mut run_cmd = build_cmd.run().unwrap().command(); - println!( - "\n{}: {}ms", - "`cargo build` finished after".underline().bright_blue(), - now.elapsed().as_millis() + + let mut run_cmd = generate_bin_command( + bin_name, + &working_dir.as_ref().join("Cargo.toml"), ); run_cmd - .env("ANOMA_LOG", "anoma=info") + .env("ANOMA_LOG", "info") + .env("TM_LOG_LEVEL", "info") + .env("ANOMA_LOG_COLOR", "false") .current_dir(working_dir) .args(&[ "--base-dir", @@ -669,10 +695,13 @@ where mode, ]) .args(args); - let cmd_str = format!("{:?}", run_cmd); - println!("{}: {}", "Running".underline().green(), cmd_str); - let mut session = Session::spawn(run_cmd).map_err(|e| { + let args: String = + run_cmd.get_args().map(|s| s.to_string_lossy()).join(" "); + let cmd_str = + format!("{} {}", run_cmd.get_program().to_string_lossy(), args); + + let session = Session::spawn(run_cmd).map_err(|e| { eyre!( "\n\n{}: {}\n{}: {}\n{}: {}", "Failed to run".underline().red(), @@ -683,9 +712,37 @@ where e ) })?; + + let log_path = { + let mut rng = rand::thread_rng(); + let log_dir = base_dir.as_ref().join("logs"); + fs::create_dir_all(&log_dir)?; + log_dir.join(&format!( + "{}-{}-{}.log", + SystemTime::now() + .duration_since(UNIX_EPOCH) + .unwrap() + .as_micros(), + bin_name, + rng.gen::() + )) + }; + let logger = OpenOptions::new() + .write(true) + .create_new(true) + .open(&log_path)?; + let mut session = session.with_log(logger).unwrap(); + session.set_expect_timeout(timeout_sec.map(std::time::Duration::from_secs)); - let mut cmd_process = AnomaCmd { session, cmd_str }; + let mut cmd_process = AnomaCmd { + session, + cmd_str, + log_path, + }; + + println!("{}:\n{}", "> Running".underline().green(), &cmd_process); + if let Bin::Node = &bin { // When running a node command, we need to wait a bit before checking // status @@ -710,6 +767,7 @@ where } } } + Ok(cmd_process) } diff --git a/wasm/checksums.json b/wasm/checksums.json index fe8a7c99c58..96cd9d2c14b 100644 --- a/wasm/checksums.json +++ b/wasm/checksums.json @@ -1,19 +1,19 @@ { - "tx_bond.wasm": "tx_bond.653c91f0df7f09333bc57d381a37bc58f1ed1c059d1d2327e5c15b0661ad4b21.wasm", - "tx_from_intent.wasm": "tx_from_intent.80a2901a4b2b1e4ae0f62d8a29585f040a5389bb2133e4cbf502dbb71435e285.wasm", - "tx_ibc.wasm": "tx_ibc.0acf643307e82e85f83dbe59d5420ab8be7b4b543396128087d8c4b8e8eca1c3.wasm", - "tx_init_account.wasm": "tx_init_account.0edede3dc1d86a20e26ebd42d68c8bca3ec94fb8d34eacd6019e06823c46e761.wasm", - "tx_init_nft.wasm": "tx_init_nft.01ad73d15b8e573c2ab9f9b57903cac7a4b0056b37258332ec610862b1ad1bea.wasm", - "tx_init_proposal.wasm": "tx_init_proposal.d907365d182c711af6068cedfcc493d9373e1a556e1c623307751690620803ff.wasm", - "tx_init_validator.wasm": "tx_init_validator.cb1d93bdd90bcd5f00f37667d505094b9cadadadb8eed6f973e31cb3bd7126b1.wasm", - "tx_mint_nft.wasm": "tx_mint_nft.e717d96003ded40c6fea055b637901f184f0a6c557090fd329ff439e79fa692c.wasm", - "tx_transfer.wasm": "tx_transfer.8be48f207a6acb3951f040cce7d67e134f4528a6d56263180878d8486eeb851a.wasm", - "tx_unbond.wasm": "tx_unbond.859cb6c5df93f49163554b07d91e2498e73c84719c03b92b197cddd2680d8b92.wasm", - "tx_update_vp.wasm": "tx_update_vp.66305f5b1cb883576cd27698a87fc6ed07abadd5b585024d8e88f0941543f08d.wasm", - "tx_vote_proposal.wasm": "tx_vote_proposal.8cb051589a349c42245b20bd72b643432055e1c38e135c99379931ea4181f685.wasm", - "tx_withdraw.wasm": "tx_withdraw.731e92766e44b48f88548f07143d7a472f58a8ddda85069db8ada451e1277446.wasm", - "vp_nft.wasm": "vp_nft.a08f5364f11760779e3e3c80551eb2173069b6992a16509923954d136963a1c7.wasm", - "vp_testnet_faucet.wasm": "vp_testnet_faucet.b4ae5ae69051eeba4cba95ec52a4533a96d3399ad1b1f432149f1ef4613cd7d9.wasm", - "vp_token.wasm": "vp_token.29689b38d96dfda4f198b4d63aa27491d55d782d1e8048af5e3251639138ebbd.wasm", - "vp_user.wasm": "vp_user.17f8ce32c1fbf03e8a90caf4df01c880cd4b11a22673595f2441f1a931c07e28.wasm" + "tx_bond.wasm": "tx_bond.26b75c40063bbfa5a78fb8b75214e187b7a5892cecf7d7220d55d14fc4100e56.wasm", + "tx_from_intent.wasm": "tx_from_intent.c225573c7f1bbc0b021b8e49c2ce78c45f3df41a4ba05a104f8109d8e855a5d9.wasm", + "tx_ibc.wasm": "tx_ibc.56f5d416b37b3b31dabcee4cdccbbf44afef052cf90053499bbd28e50d6b8434.wasm", + "tx_init_account.wasm": "tx_init_account.31e5960ad3db5a5f21322e46cf165fb1947e3a271b9d681a1f71dca569773197.wasm", + "tx_init_nft.wasm": "tx_init_nft.012d127e54625e4e548874d54c5d3cc6cf9758c0e6793f5bebb12a73a23c2cb4.wasm", + "tx_init_proposal.wasm": "tx_init_proposal.e977966ecae20346cb1e7b9488231172918724da76c13318f57db17eb709e10f.wasm", + "tx_init_validator.wasm": "tx_init_validator.57b1b172621396af4abc5f077615262984cf2b291710bed23d5826303d24f3d7.wasm", + "tx_mint_nft.wasm": "tx_mint_nft.dbe8b009861807f0b6544f629d26de9ed8527211c2124ed8622f7c83eb4450d9.wasm", + "tx_transfer.wasm": "tx_transfer.46838c747351199bb86202dfb5e74df1f08b323173dca8669e0b54dd924081c4.wasm", + "tx_unbond.wasm": "tx_unbond.db9c32b59850a6c3515ae9ad9d4ea4991fd84d0bc9c41db5cca8ffc4f1b22d89.wasm", + "tx_update_vp.wasm": "tx_update_vp.cee7b0d45a0d8ceaf398293a181dccfa53cecb4fbcbbeed36f6ccb8c98cdf3f6.wasm", + "tx_vote_proposal.wasm": "tx_vote_proposal.f8807efb8339aecaf2635a2f27e757b98d3be52e97c1d8aab4f461e0d79b117d.wasm", + "tx_withdraw.wasm": "tx_withdraw.be5cd54a1de04a91cfb7554b05edc59c9ae4015c44038f1d9ab4ae31a17be183.wasm", + "vp_nft.wasm": "vp_nft.3f8b544f941755eb4f63790a87918b1348b445285f4200d3a8997cbe74656787.wasm", + "vp_testnet_faucet.wasm": "vp_testnet_faucet.a29f50e992b59af5c045f94d4d69c19f6e1445beb1364b798a6cd98473e1a598.wasm", + "vp_token.wasm": "vp_token.f0f6f46fd4c03d92fd14eb9bb1681524778d84bb944a1a7ac9b2a1957e7d14b9.wasm", + "vp_user.wasm": "vp_user.3bec3d40a694e38b7a3d70040e245514e7135e200b94b0ac1fc4067920ad9aff.wasm" } \ No newline at end of file diff --git a/wasm/checksums.py b/wasm/checksums.py index a9097e34b73..4b532b2ee8e 100644 --- a/wasm/checksums.py +++ b/wasm/checksums.py @@ -6,8 +6,10 @@ checksums = {} for wasm in sorted(glob.glob("wasm/*.wasm")): basename = os.path.basename(wasm) - file_name = os.path.splitext(basename)[0] if wasm.count(".") == 1 else os.path.splitext(basename)[0].split('.')[0] - checksums["{}.wasm".format(file_name)] = "{}.{}.wasm".format(file_name, hashlib.sha256(open(wasm, "rb").read()).hexdigest()) + file_name = os.path.splitext(basename)[0] if wasm.count( + ".") == 1 else os.path.splitext(basename)[0].split('.')[0] + checksums["{}.wasm".format(file_name)] = "{}.{}.wasm".format( + file_name, hashlib.sha256(open(wasm, "rb").read()).hexdigest()) os.rename(wasm, 'wasm/{}'.format(checksums["{}.wasm".format(file_name)])) updated_wasms = list(checksums.values()) @@ -17,4 +19,5 @@ if not basename in updated_wasms: os.remove(wasm) -json.dump(checksums, open("wasm/checksums.json", "w"), indent=4, sort_keys=True) \ No newline at end of file +json.dump(checksums, open("wasm/checksums.json", "w"), + indent=4, sort_keys=True) diff --git a/wasm/wasm_source/Makefile b/wasm/wasm_source/Makefile index 43f85f9009a..39562a4a1e9 100644 --- a/wasm/wasm_source/Makefile +++ b/wasm/wasm_source/Makefile @@ -59,7 +59,9 @@ $(patsubst %,check_%,$(wasms)): check_%: # `cargo test` one of the wasms, e.g. `make test_tx_transfer` $(patsubst %,test_%,$(wasms)): test_%: - $(cargo) test --features $* + $(cargo) test --features $* \ + -- \ + -Z unstable-options --report-time # `cargo watch` one of the wasms, e.g. `make watch_tx_transfer` $(patsubst %,watch_%,$(wasms)): watch_%: diff --git a/wasm_for_tests/wasm_source/Makefile b/wasm_for_tests/wasm_source/Makefile index 4ebcf6023f0..142199dbc99 100644 --- a/wasm_for_tests/wasm_source/Makefile +++ b/wasm_for_tests/wasm_source/Makefile @@ -55,7 +55,9 @@ $(patsubst %,check_%,$(wasms)): check_%: # `cargo test` one of the wasms, e.g. `make test_tx_no_op` $(patsubst %,test_%,$(wasms)): test_%: - $(cargo) test --features $* + $(cargo) test --features $* \ + -- \ + -Z unstable-options --report-time # `cargo watch` one of the wasms, e.g. `make watch_tx_no_op` $(patsubst %,watch_%,$(wasms)): watch_%: