From dbe9c4b4311200c4d50388ef85a3880e1c311cd7 Mon Sep 17 00:00:00 2001 From: joao-paulo-parity Date: Tue, 7 Sep 2021 18:12:20 -0300 Subject: [PATCH] isolate check-polkadot-companion-build for faster iteration deletion will be reverted later --- .gitlab-ci.yml | 603 ------------------------------------------------- 1 file changed, 603 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 74ed64315d62e..ddb23773f93a8 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -165,615 +165,12 @@ default: | tee artifacts/benches/$CI_COMMIT_REF_NAME-$CI_COMMIT_SHORT_SHA/::trie::read::small.json' - sccache -s -#### stage: .pre - -skip-if-draft: - image: paritytech/tools:latest - <<: *kubernetes-env - stage: .pre - rules: - - if: $CI_COMMIT_REF_NAME =~ /^[0-9]+$/ # PRs - script: - - echo "Commit message is ${CI_COMMIT_MESSAGE}" - - echo "Ref is ${CI_COMMIT_REF_NAME}" - - echo "pipeline source is ${CI_PIPELINE_SOURCE}" - - ./.maintain/gitlab/skip_if_draft.sh - -#### stage: check - -check-runtime: - stage: check - image: paritytech/tools:latest - <<: *kubernetes-env - rules: - - if: $CI_COMMIT_REF_NAME =~ /^[0-9]+$/ # PRs - variables: - <<: *default-vars - GITLAB_API: "https://gitlab.parity.io/api/v4" - GITHUB_API_PROJECT: "parity%2Finfrastructure%2Fgithub-api" - script: - - ./.maintain/gitlab/check_runtime.sh - allow_failure: true - -check-signed-tag: - stage: check - image: paritytech/tools:latest - <<: *kubernetes-env - rules: - - if: $CI_COMMIT_REF_NAME =~ /^ci-release-.*$/ - - if: $CI_COMMIT_REF_NAME =~ /^v[0-9]+\.[0-9]+.*$/ # i.e. v1.0, v2.1rc1 - script: - - ./.maintain/gitlab/check_signed.sh - -check-line-width: - stage: check - image: paritytech/tools:latest - <<: *kubernetes-env - rules: - - if: $CI_COMMIT_REF_NAME =~ /^[0-9]+$/ # PRs - script: - - ./.maintain/gitlab/check_line_width.sh - allow_failure: true - -test-dependency-rules: - stage: check - image: paritytech/tools:latest - <<: *kubernetes-env - rules: - - if: $CI_COMMIT_REF_NAME =~ /^[0-9]+$/ # PRs - script: - - .maintain/ensure-deps.sh - -test-prometheus-alerting-rules: - stage: check - image: paritytech/tools:latest - <<: *kubernetes-env - rules: - - if: $CI_PIPELINE_SOURCE == "pipeline" - when: never - - if: $CI_COMMIT_BRANCH - changes: - - .gitlab-ci.yml - - .maintain/monitoring/**/* - script: - - promtool check rules .maintain/monitoring/alerting-rules/alerting-rules.yaml - - cat .maintain/monitoring/alerting-rules/alerting-rules.yaml | - promtool test rules .maintain/monitoring/alerting-rules/alerting-rule-tests.yaml - -#### stage: test - -cargo-deny: - stage: test - <<: *docker-env - <<: *nightly-pipeline - script: - - cargo deny check --hide-inclusion-graph -c .maintain/deny.toml - after_script: - - echo "___The complete log is in the artifacts___" - - cargo deny check -c .maintain/deny.toml 2> deny.log - artifacts: - name: $CI_COMMIT_SHORT_SHA - expire_in: 3 days - when: always - paths: - - deny.log - # FIXME: Temorarily allow to fail. - allow_failure: true - -cargo-fmt: - stage: test - <<: *docker-env - <<: *test-refs - script: - - cargo +nightly fmt --all -- --check - -cargo-check-benches: - stage: test - <<: *docker-env - <<: *test-refs - <<: *collect-artifacts - before_script: - # merges in the master branch on PRs - - *merge-ref-into-master-script - - *rust-info-script - script: - - *cargo-check-benches-script - -node-bench-regression-guard: - # it's not belong to `build` semantically, but dag jobs can't depend on each other - # within the single stage - https://gitlab.com/gitlab-org/gitlab/-/issues/30632 - # more: https://github.com/paritytech/substrate/pull/8519#discussion_r608012402 - stage: build - <<: *docker-env - <<: *test-refs-no-trigger-prs-only - needs: - # this is a DAG - - job: cargo-check-benches - artifacts: true - # this does not like a DAG, just polls the artifact - - project: $CI_PROJECT_PATH - job: cargo-check-benches - ref: master - artifacts: true - variables: - CI_IMAGE: "paritytech/node-bench-regression-guard:latest" - before_script: [""] - script: - - 'node-bench-regression-guard --reference artifacts/benches/master-* - --compare-with artifacts/benches/$CI_COMMIT_REF_NAME-$CI_COMMIT_SHORT_SHA' - -cargo-check-subkey: - stage: test - <<: *docker-env - <<: *test-refs - script: - - cd ./bin/utils/subkey - - SKIP_WASM_BUILD=1 time cargo check --release - - sccache -s - -cargo-check-try-runtime: - stage: test - <<: *docker-env - <<: *test-refs - script: - - time cargo check --features try-runtime - - sccache -s - -cargo-check-wasmer-sandbox: - stage: test - <<: *docker-env - <<: *test-refs - script: - - time cargo check --features wasmer-sandbox - - sccache -s - -test-deterministic-wasm: - stage: test - <<: *docker-env - <<: *test-refs - variables: - <<: *default-vars - WASM_BUILD_NO_COLOR: 1 - script: - # build runtime - - cargo build --verbose --release -p node-runtime - # make checksum - - sha256sum target/release/wbuild/node-runtime/target/wasm32-unknown-unknown/release/node_runtime.wasm > checksum.sha256 - # clean up – FIXME: can we reuse some of the artifacts? - - cargo clean - # build again - - cargo build --verbose --release -p node-runtime - # confirm checksum - - sha256sum -c checksum.sha256 - - sccache -s - -test-linux-stable: &test-linux - stage: test - <<: *docker-env - <<: *test-refs - variables: - <<: *default-vars - # Enable debug assertions since we are running optimized builds for testing - # but still want to have debug assertions. - RUSTFLAGS: "-Cdebug-assertions=y -Dwarnings" - RUST_BACKTRACE: 1 - WASM_BUILD_NO_COLOR: 1 - script: - # this job runs all tests in former runtime-benchmarks, frame-staking and wasmtime tests - - time cargo test --workspace --locked --release --verbose --features runtime-benchmarks --manifest-path bin/node/cli/Cargo.toml - - time cargo test -p frame-support-test --features=conditional-storage --manifest-path frame/support/test/Cargo.toml --test pallet # does not reuse cache 1 min 44 sec - - SUBSTRATE_TEST_TIMEOUT=1 time cargo test -p substrate-test-utils --release --verbose --locked -- --ignored timeout - - sccache -s - -#unleash-check: - #stage: test - #<<: *docker-env - #<<: *test-refs-no-trigger - #script: - #- cargo install cargo-unleash ${CARGO_UNLEASH_INSTALL_PARAMS} - #- cargo unleash de-dev-deps - # Reuse build artifacts when running checks (cuts down check time by 3x) - # TODO: Implement this optimization in cargo-unleash rather than here - #- mkdir -p target/unleash - #- export CARGO_TARGET_DIR=target/unleash - #- cargo unleash check ${CARGO_UNLEASH_PKG_DEF} - # FIXME: this job must not fail, or unleash-to-crates-io will publish broken stuff - #allow_failure: true - -test-frame-examples-compile-to-wasm: - # into one job - stage: test - <<: *docker-env - <<: *test-refs - variables: - <<: *default-vars - # Enable debug assertions since we are running optimized builds for testing - # but still want to have debug assertions. - RUSTFLAGS: "-Cdebug-assertions=y" - RUST_BACKTRACE: 1 - script: - - cd frame/example-offchain-worker/ - - cargo +nightly build --target=wasm32-unknown-unknown --no-default-features - - cd ../example - - cargo +nightly build --target=wasm32-unknown-unknown --no-default-features - - sccache -s - -test-linux-stable-int: - <<: *test-linux - stage: test - script: - - echo "___Logs will be partly shown at the end in case of failure.___" - - echo "___Full log will be saved to the job artifacts only in case of failure.___" - - WASM_BUILD_NO_COLOR=1 - RUST_LOG=sync=trace,consensus=trace,client=trace,state-db=trace,db=trace,forks=trace,state_db=trace,storage_cache=trace - time cargo test -p node-cli --release --verbose --locked -- --ignored - &> ${CI_COMMIT_SHORT_SHA}_int_failure.log - - sccache -s - after_script: - - awk '/FAILED|^error\[/,0' ${CI_COMMIT_SHORT_SHA}_int_failure.log - artifacts: - name: $CI_COMMIT_SHORT_SHA - when: on_failure - expire_in: 3 days - paths: - - ${CI_COMMIT_SHORT_SHA}_int_failure.log - -check-tracing: - stage: test - <<: *docker-env - <<: *test-refs - script: - # with-tracing must be explicitly activated, we run a test to ensure this works as expected in both cases - - time cargo +nightly test --manifest-path primitives/tracing/Cargo.toml --no-default-features - - time cargo +nightly test --manifest-path primitives/tracing/Cargo.toml --no-default-features --features=with-tracing - - sccache -s - -test-full-crypto-feature: - stage: test - <<: *docker-env - <<: *test-refs - variables: - <<: *default-vars - # Enable debug assertions since we are running optimized builds for testing - # but still want to have debug assertions. - RUSTFLAGS: "-Cdebug-assertions=y" - RUST_BACKTRACE: 1 - script: - - cd primitives/core/ - - time cargo +nightly build --verbose --no-default-features --features full_crypto - - cd ../application-crypto - - time cargo +nightly build --verbose --no-default-features --features full_crypto - - sccache -s - -test-wasmer-sandbox: - stage: test - <<: *docker-env - <<: *test-refs-wasmer-sandbox - variables: - <<: *default-vars - script: - - time cargo test --release --features runtime-benchmarks,wasmer-sandbox - - sccache -s - -cargo-check-macos: - stage: test - # shell runner on mac ignores the image set in *docker-env - <<: *docker-env - <<: *test-refs-no-trigger - script: - - SKIP_WASM_BUILD=1 time cargo check --release - - sccache -s - tags: - - osx - -#### stage: build - -check-polkadot-companion-status: - stage: build - image: paritytech/tools:latest - <<: *kubernetes-env - rules: - - if: $CI_COMMIT_REF_NAME =~ /^[0-9]+$/ # PRs - script: - - ./.maintain/gitlab/check_polkadot_companion_status.sh - check-polkadot-companion-build: stage: build <<: *docker-env <<: *test-refs-no-trigger - needs: - - job: test-linux-stable-int - artifacts: false script: - ./.maintain/gitlab/check_polkadot_companion_build.sh after_script: - cd polkadot && git rev-parse --abbrev-ref HEAD allow_failure: true - -build-linux-substrate: &build-binary - stage: build - <<: *collect-artifacts - <<: *docker-env - <<: *build-refs - needs: - - job: test-linux-stable - artifacts: false - before_script: - - mkdir -p ./artifacts/substrate/ - script: - - WASM_BUILD_NO_COLOR=1 time cargo build --release --verbose - - mv ./target/release/substrate ./artifacts/substrate/. - - echo -n "Substrate version = " - - if [ "${CI_COMMIT_TAG}" ]; then - echo "${CI_COMMIT_TAG}" | tee ./artifacts/substrate/VERSION; - else - ./artifacts/substrate/substrate --version | - sed -n -E 's/^substrate ([0-9.]+.*-[0-9a-f]{7,13})-.*$/\1/p' | - tee ./artifacts/substrate/VERSION; - fi - - sha256sum ./artifacts/substrate/substrate | tee ./artifacts/substrate/substrate.sha256 - - printf '\n# building node-template\n\n' - - ./.maintain/node-template-release.sh ./artifacts/substrate/substrate-node-template.tar.gz - - cp -r .maintain/docker/substrate.Dockerfile ./artifacts/substrate/ - - sccache -s - -build-linux-subkey: &build-subkey - stage: build - <<: *collect-artifacts - <<: *docker-env - <<: *build-refs - needs: - - job: cargo-check-subkey - artifacts: false - before_script: - - mkdir -p ./artifacts/subkey - script: - - cd ./bin/utils/subkey - - SKIP_WASM_BUILD=1 time cargo build --release --verbose - - cd - - - mv ./target/release/subkey ./artifacts/subkey/. - - echo -n "Subkey version = " - - ./artifacts/subkey/subkey --version | - sed -n -E 's/^subkey ([0-9.]+.*)/\1/p' | - tee ./artifacts/subkey/VERSION; - - sha256sum ./artifacts/subkey/subkey | tee ./artifacts/subkey/subkey.sha256 - - cp -r .maintain/docker/subkey.Dockerfile ./artifacts/subkey/ - - sccache -s - -build-macos-subkey: - <<: *build-subkey - tags: - - osx - -build-rustdoc: - stage: build - <<: *docker-env - <<: *test-refs - variables: - <<: *default-vars - SKIP_WASM_BUILD: 1 - artifacts: - name: "${CI_JOB_NAME}_${CI_COMMIT_REF_NAME}-doc" - when: on_success - expire_in: 7 days - paths: - - ./crate-docs/ - script: - # FIXME: it fails with `RUSTDOCFLAGS="-Dwarnings"` - - RUSTDOCFLAGS="--html-in-header $(pwd)/.maintain/rustdoc-header.html" - time cargo +nightly doc --no-deps --workspace --all-features --verbose - - rm -f ./target/doc/.lock - - mv ./target/doc ./crate-docs - # FIXME: remove me after CI image gets nonroot - - chown -R nonroot:nonroot ./crate-docs - - echo "" > ./crate-docs/index.html - - sccache -s - -#### stage: publish - -.build-push-docker-image: &build-push-docker-image - <<: *build-refs - <<: *kubernetes-env - image: quay.io/buildah/stable - variables: &docker-build-vars - <<: *default-vars - GIT_STRATEGY: none - DOCKERFILE: $PRODUCT.Dockerfile - IMAGE_NAME: docker.io/parity/$PRODUCT - before_script: - - cd ./artifacts/$PRODUCT/ - - VERSION="$(cat ./VERSION)" - - echo "${PRODUCT} version = ${VERSION}" - - test -z "${VERSION}" && exit 1 - script: - - test "$Docker_Hub_User_Parity" -a "$Docker_Hub_Pass_Parity" || - ( echo "no docker credentials provided"; exit 1 ) - - buildah bud - --format=docker - --build-arg VCS_REF="${CI_COMMIT_SHA}" - --build-arg BUILD_DATE="$(date -u '+%Y-%m-%dT%H:%M:%SZ')" - --tag "$IMAGE_NAME:$VERSION" - --tag "$IMAGE_NAME:latest" - --file "$DOCKERFILE" . - - echo "$Docker_Hub_Pass_Parity" | - buildah login --username "$Docker_Hub_User_Parity" --password-stdin docker.io - - buildah info - - buildah push --format=v2s2 "$IMAGE_NAME:$VERSION" - - buildah push --format=v2s2 "$IMAGE_NAME:latest" - after_script: - - buildah logout --all - # pass artifacts to the trigger-simnet job - - echo "SUBSTRATE_IMAGE_NAME=${IMAGE_NAME}" | tee -a ./artifacts/$PRODUCT/build.env - - IMAGE_TAG="$(cat ./artifacts/$PRODUCT/VERSION)" - - echo "SUBSTRATE_IMAGE_TAG=${IMAGE_TAG}" | tee -a ./artifacts/$PRODUCT/build.env - - cat ./artifacts/$PRODUCT/build.env - -publish-docker-substrate: - stage: publish - <<: *build-push-docker-image - <<: *build-refs - needs: - - job: build-linux-substrate - artifacts: true - variables: - <<: *docker-build-vars - PRODUCT: substrate - artifacts: - reports: - # this artifact is used in trigger-simnet job - # https://docs.gitlab.com/ee/ci/multi_project_pipelines.html#with-variable-inheritance - dotenv: ./artifacts/substrate/build.env - -publish-docker-subkey: - stage: publish - <<: *build-push-docker-image - needs: - - job: build-linux-subkey - artifacts: true - variables: - <<: *docker-build-vars - PRODUCT: subkey - -publish-s3-release: - stage: publish - <<: *build-refs - <<: *kubernetes-env - needs: - - job: build-linux-substrate - artifacts: true - - job: build-linux-subkey - artifacts: true - image: paritytech/awscli:latest - variables: - GIT_STRATEGY: none - BUCKET: "releases.parity.io" - PREFIX: "substrate/${ARCH}-${DOCKER_OS}" - script: - - aws s3 sync ./artifacts/ s3://${BUCKET}/${PREFIX}/$(cat ./artifacts/substrate/VERSION)/ - - echo "update objects in latest path" - - aws s3 sync s3://${BUCKET}/${PREFIX}/$(cat ./artifacts/substrate/VERSION)/ s3://${BUCKET}/${PREFIX}/latest/ - after_script: - - aws s3 ls s3://${BUCKET}/${PREFIX}/latest/ - --recursive --human-readable --summarize - -publish-rustdoc: - stage: publish - <<: *kubernetes-env - image: paritytech/tools:latest - variables: - GIT_DEPTH: 100 - rules: - - if: $CI_PIPELINE_SOURCE == "pipeline" - when: never - - if: $CI_PIPELINE_SOURCE == "web" && $CI_COMMIT_REF_NAME == "master" - - if: $CI_COMMIT_REF_NAME == "master" - # `needs:` can be removed after CI image gets nonroot. In this case `needs:` stops other - # artifacts from being dowloaded by this job. - needs: - - job: build-rustdoc - artifacts: true - script: - - rm -rf /tmp/* - # Set git config - - rm -rf .git/config - - git config user.email "devops-team@parity.io" - - git config user.name "${GITHUB_USER}" - - git config remote.origin.url "https://${GITHUB_TOKEN}@github.com/paritytech/substrate.git" - - git config remote.origin.fetch "+refs/heads/*:refs/remotes/origin/*" - - git fetch origin gh-pages - # Save README and docs - - cp -r ./crate-docs/ /tmp/doc/ - - cp README.md /tmp/doc/ - - git checkout gh-pages - # Remove everything and restore generated docs and README - - rm -rf ./* - - mv /tmp/doc/* . - # Upload files - - git add --all --force - # `git commit` has an exit code of > 0 if there is nothing to commit. - # This causes GitLab to exit immediately and marks this job failed. - # We don't want to mark the entire job failed if there's nothing to - # publish though, hence the `|| true`. - - git commit -m "Updated docs for ${CI_COMMIT_REF_NAME}" || - echo "___Nothing to commit___" - - git push origin gh-pages --force - after_script: - - rm -rf .git/ ./* - -publish-draft-release: - stage: publish - image: paritytech/tools:latest - rules: - - if: $CI_COMMIT_REF_NAME =~ /^ci-release-.*$/ - - if: $CI_COMMIT_REF_NAME =~ /^v[0-9]+\.[0-9]+.*$/ # i.e. v1.0, v2.1rc1 - script: - - ./.maintain/gitlab/publish_draft_release.sh - allow_failure: true - -unleash-to-crates-io: - stage: publish - <<: *docker-env - rules: - - if: $CI_COMMIT_REF_NAME =~ /^ci-release-.*$/ - # FIXME: wait until https://github.com/paritytech/cargo-unleash/issues/50 is fixed, also - # remove allow_failure: true on the check job - # - if: $CI_COMMIT_REF_NAME =~ /^v[0-9]+\.[0-9]+.*$/ # i.e. v1.0, v2.1rc1 - script: - - cargo install cargo-unleash ${CARGO_UNLEASH_INSTALL_PARAMS} - - cargo unleash em-dragons --no-check --owner github:paritytech:core-devs ${CARGO_UNLEASH_PKG_DEF} - allow_failure: true - -#### stage: deploy - -deploy-prometheus-alerting-rules: - stage: deploy - needs: - - job: test-prometheus-alerting-rules - artifacts: false - allow_failure: true - trigger: - project: parity/infrastructure/cloud-infra - variables: - SUBSTRATE_CI_COMMIT_NAME: "${CI_COMMIT_REF_NAME}" - SUBSTRATE_CI_COMMIT_REF: "${CI_COMMIT_SHORT_SHA}" - UPSTREAM_TRIGGER_PROJECT: "${CI_PROJECT_PATH}" - rules: - - if: $CI_PIPELINE_SOURCE == "pipeline" - when: never - - if: $CI_COMMIT_REF_NAME == "master" - changes: - - .gitlab-ci.yml - - .maintain/monitoring/**/* - -# Runs "quick" and "long" tests on nightly schedule and on commit / merge to master -# A "quick" test is a smoke test where basic check-expect tests run by -# checking values from metrics exposed by the app. -# A "long" test is the load testing where we send 50K transactions into the -# network and check if all completed successfully -simnet-tests: - stage: deploy - image: docker.io/paritytech/simnet:${SIMNET_REF} - <<: *kubernetes-env - rules: - - if: $CI_PIPELINE_SOURCE == "pipeline" - when: never - - if: $CI_PIPELINE_SOURCE == "web" && $CI_COMMIT_REF_NAME == "master" - - if: $CI_COMMIT_REF_NAME == "master" - needs: - - job: publish-docker-substrate - # variables: - # `build.env` brings here `${SUBSTRATE_IMAGE_NAME}` and `${SUBSTRATE_IMAGE_TAG}` - # (`$VERSION` here, # i.e. `2643-0.8.29-5f689e0a-6b24dc54`). - # ${SIMNET_REF} is a gitlab variable - before_script: - - echo "Simnet Tests Config - docker.io/paritytech/simnet:${SIMNET_REF} - ${SUBSTRATE_IMAGE_NAME} ${SUBSTRATE_IAMGE_TAG}" - script: - - /home/nonroot/simnet/gurke/scripts/run-test-environment-manager.sh - --github-remote-dir="https://github.com/paritytech/substrate/tree/master/simnet_tests" - --config="simnet_tests/configs/default_local_testnet.toml" - --image="${SUBSTRATE_IMAGE_NAME}:${SUBSTRATE_IMAGE_TAG}" - retry: 2 - tags: - - parity-simnet