Skip to content

Commit

Permalink
[ci] Add missing components to CI (paritytech#1201)
Browse files Browse the repository at this point in the history
* [ci] Add missing components to CI

* add timestamp

* add pipeline-stopper-artifacts to jobs

* break clippy

* mv definition

* fix clippy

* comment timestamp, rm verbose

* commen cancel

* rm --verbose from build jobs

* disable rusty-cachier before_script

* enable timestamp back

* rollback .gitignore

* rollback .gitignore

* rollback .gitignore
  • Loading branch information
alvicsam authored Aug 28, 2023
1 parent c168a77 commit 6e394f8
Show file tree
Hide file tree
Showing 4 changed files with 161 additions and 63 deletions.
190 changes: 144 additions & 46 deletions .gitlab-ci.yml
Original file line number Diff line number Diff line change
@@ -1,32 +1,9 @@
# .gitlab-ci.yml
# polkadot-sdk | CI definitions (via GitLab CI)
#
# substrate
# FYI: Pipelines can be triggered manually through the web UI (if you have enough permissions)
#
# pipelines can be triggered manually in the web
#
# Currently the file is divided into subfiles. Each stage has a different file which
# can be found here: gitlab/pipeline/<stage_name>.yml
#
# Instead of YAML anchors "extends" is used.
# Useful links:
# https://docs.gitlab.com/ee/ci/yaml/index.html#extends
# https://docs.gitlab.com/ee/ci/yaml/yaml_optimization.html#reference-tags
#
# SAMPLE JOB TEMPLATE - This is not a complete example but is enough to build a
# simple CI job. For full documentation, visit https://docs.gitlab.com/ee/ci/yaml/
#
# my-example-job:
# stage: test # One of the stages listed below this job (required)
# image: paritytech/tools:latest # Any docker image (required)
# allow_failure: true # Allow the pipeline to continue if this job fails (default: false)
# needs:
# - job: test-linux # Any jobs that are required to run before this job (optional)
# variables:
# MY_ENVIRONMENT_VARIABLE: "some useful value" # Environment variables passed to the job (optional)
# script:
# - echo "List of shell commands to run in your job"
# - echo "You can also just specify a script here, like so:"
# - ./gitlab/my_amazing_script.sh
# Currently, entire CI instructions are split into different subfiles. Each CI stage has a corresponding
# file which can be found here: .gitlab/pipeline/<stage_name>.yml

stages:
- check
Expand Down Expand Up @@ -84,23 +61,26 @@ default:

.prepare-env:
before_script:
# TODO: remove unset invocation when we'll be free from 'ENV RUSTC_WRAPPER=sccache' & sccache
# itself in all images
- unset RUSTC_WRAPPER
# $WASM_BUILD_WORKSPACE_HINT enables wasm-builder to find the Cargo.lock from within generated
# packages
- export WASM_BUILD_WORKSPACE_HINT="$PWD"
# ensure that RUSTFLAGS are set correctly
- echo $RUSTFLAGS

.common-before-script:
before_script:
- !reference [.job-switcher, before_script]
- !reference [.timestamp, before_script]
- !reference [.pipeline-stopper-vars, script]

.job-switcher:
before_script:
- if echo "$CI_DISABLED_JOBS" | grep -xF "$CI_JOB_NAME"; then echo "The job has been cancelled in CI settings"; exit 0; fi

.kubernetes-env:
image: "${CI_IMAGE}"
before_script:
# - !reference [.job-switcher, before_script]
- !reference [.common-before-script, before_script]
- !reference [.prepare-env, before_script]
tags:
- kubernetes-parity-build
Expand All @@ -112,9 +92,10 @@ default:
- rustup +nightly show
- cargo +nightly --version

# collecting vars for pipeline stopper
# they will be used if the job fails
.pipeline-stopper-vars:
script:
- !reference [.job-switcher, before_script]
- echo "Collecting env variables for the cancel-pipeline job"
- echo "FAILED_JOB_URL=${CI_JOB_URL}" > pipeline-stopper.env
- echo "FAILED_JOB_NAME=${CI_JOB_NAME}" >> pipeline-stopper.env
Expand All @@ -128,28 +109,26 @@ default:
.docker-env:
image: "${CI_IMAGE}"
before_script:
- !reference [.job-switcher, before_script]
- !reference [.common-before-script, before_script]
- !reference [.prepare-env, before_script]
- !reference [.rust-info-script, script]
- !reference [.rusty-cachier, before_script]
- !reference [.pipeline-stopper-vars, script]
after_script:
- !reference [.rusty-cachier, after_script]
tags:
- linux-docker-vm-c2

# rusty-cachier's hidden job. Parts of this job are used to instrument the pipeline's other real jobs with rusty-cachier
# Description of the commands is available here - https://gitlab.parity.io/parity/infrastructure/ci_cd/rusty-cachier/client#description
# rusty-cachier's commands are described here: https://gitlab.parity.io/parity/infrastructure/ci_cd/rusty-cachier/client#description
.rusty-cachier:
before_script:
# - curl -s https://gitlab-ci-token:${CI_JOB_TOKEN}@gitlab.parity.io/parity/infrastructure/ci_cd/rusty-cachier/client/-/raw/release/util/install.sh | bash
# - rusty-cachier environment check --gracefully
# - $(rusty-cachier environment inject)
# - rusty-cachier project mtime
# - mkdir -p cargo_home cargo_target_dir
# - export CARGO_HOME=$CI_PROJECT_DIR/cargo_home
# - export CARGO_TARGET_DIR=$CI_PROJECT_DIR/cargo_target_dir
# - find . \( -path ./cargo_target_dir -o -path ./cargo_home \) -prune -o -type f -exec touch -t 202005260100 {} +
# - git restore-mtime
# - rusty-cachier --version
# - rusty-cachier project touch-changed
- echo tbd
after_script:
- echo tbd
# - env RUSTY_CACHIER_SUPRESS_OUTPUT=true rusty-cachier snapshot destroy

.common-refs:
rules:
Expand Down Expand Up @@ -233,6 +212,125 @@ include:
# zombienet jobs
- .gitlab/pipeline/zombienet.yml
# # timestamp handler
# - project: parity/infrastructure/ci_cd/shared
# ref: v0.2
# file: /common/timestamp.yml
- project: parity/infrastructure/ci_cd/shared
ref: v0.2
file: /common/timestamp.yml

# This job cancels the whole pipeline if any of provided jobs fail.
# In a DAG, every jobs chain is executed independently of others. The `fail_fast` principle suggests
# to fail the pipeline as soon as possible to shorten the feedback loop.
.cancel-pipeline-template:
stage: .post
rules:
- if: $CI_COMMIT_REF_NAME =~ /^[0-9]+$/ # PRs
when: on_failure
variables:
PROJECT_ID: "${CI_PROJECT_ID}"
PROJECT_NAME: "${CI_PROJECT_NAME}"
PIPELINE_ID: "${CI_PIPELINE_ID}"
FAILED_JOB_URL: "${FAILED_JOB_URL}"
FAILED_JOB_NAME: "${FAILED_JOB_NAME}"
PR_NUM: "${PR_NUM}"
trigger:
project: "parity/infrastructure/ci_cd/pipeline-stopper"
branch: "as-improve"

remove-cancel-pipeline-message:
stage: .post
rules:
- if: $CI_COMMIT_REF_NAME =~ /^[0-9]+$/ # PRs
variables:
PROJECT_ID: "${CI_PROJECT_ID}"
PROJECT_NAME: "${CI_PROJECT_NAME}"
PIPELINE_ID: "${CI_PIPELINE_ID}"
FAILED_JOB_URL: "https://gitlab.com"
FAILED_JOB_NAME: "nope"
PR_NUM: "${CI_COMMIT_REF_NAME}"
trigger:
project: "parity/infrastructure/ci_cd/pipeline-stopper"
# need to copy jobs this way because otherwise gitlab will wait
# for all 3 jobs to finish instead of cancelling if one fails
cancel-pipeline-test-linux-stable1:
extends: .cancel-pipeline-template
needs:
- job: "test-linux-stable 1/3"

cancel-pipeline-test-linux-stable2:
extends: .cancel-pipeline-template
needs:
- job: "test-linux-stable 2/3"

cancel-pipeline-test-linux-stable3:
extends: .cancel-pipeline-template
needs:
- job: "test-linux-stable 3/3"

cancel-pipeline-test-linux-stable-additional-tests:
extends: .cancel-pipeline-template
needs:
- job: "test-linux-stable-additional-tests"

cancel-pipeline-test-linux-stable-slow:
extends: .cancel-pipeline-template
needs:
- job: "test-linux-stable-slow"

cancel-pipeline-cargo-check-benches1:
extends: .cancel-pipeline-template
needs:
- job: "cargo-check-benches 1/2"

cancel-pipeline-cargo-check-benches2:
extends: .cancel-pipeline-template
needs:
- job: "cargo-check-benches 2/2"

cancel-pipeline-test-linux-stable-int:
extends: .cancel-pipeline-template
needs:
- job: test-linux-stable-int

cancel-pipeline-cargo-check-each-crate-1:
extends: .cancel-pipeline-template
needs:
- job: "cargo-check-each-crate 1/6"

cancel-pipeline-cargo-check-each-crate-2:
extends: .cancel-pipeline-template
needs:
- job: "cargo-check-each-crate 2/6"

cancel-pipeline-cargo-check-each-crate-3:
extends: .cancel-pipeline-template
needs:
- job: "cargo-check-each-crate 3/6"

cancel-pipeline-cargo-check-each-crate-4:
extends: .cancel-pipeline-template
needs:
- job: "cargo-check-each-crate 4/6"

cancel-pipeline-cargo-check-each-crate-5:
extends: .cancel-pipeline-template
needs:
- job: "cargo-check-each-crate 5/6"

cancel-pipeline-cargo-check-each-crate-6:
extends: .cancel-pipeline-template
needs:
- job: "cargo-check-each-crate 6/6"

cancel-pipeline-cargo-check-each-crate-macos:
extends: .cancel-pipeline-template
needs:
- job: cargo-check-each-crate-macos

cancel-pipeline-check-tracing:
extends: .cancel-pipeline-template
needs:
- job: check-tracing

cancel-pipeline-cargo-clippy:
extends: .cancel-pipeline-template
needs:
- job: cargo-clippy
14 changes: 7 additions & 7 deletions .gitlab/pipeline/build.yml
Original file line number Diff line number Diff line change
Expand Up @@ -46,8 +46,8 @@ build-test-collators:
- .run-immediately
- .collect-artifacts
script:
- time cargo build --locked --profile testnet --verbose -p test-parachain-adder-collator
- time cargo build --locked --profile testnet --verbose -p test-parachain-undying-collator
- time cargo build --locked --profile testnet -p test-parachain-adder-collator
- time cargo build --locked --profile testnet -p test-parachain-undying-collator
# pack artifacts
- mkdir -p ./artifacts
- mv ./target/testnet/adder-collator ./artifacts/.
Expand All @@ -66,7 +66,7 @@ build-malus:
- .run-immediately
- .collect-artifacts
script:
- time cargo build --locked --profile testnet --verbose -p polkadot-test-malus --bin malus --bin polkadot-prepare-worker --bin polkadot-execute-worker
- time cargo build --locked --profile testnet -p polkadot-test-malus --bin malus --bin polkadot-prepare-worker --bin polkadot-execute-worker
# pack artifacts
- mkdir -p ./artifacts
- mv ./target/testnet/malus ./artifacts/.
Expand Down Expand Up @@ -111,7 +111,7 @@ build-rustdoc:
script:
# FIXME: it fails with `RUSTDOCFLAGS="-Dwarnings"` and `--all-features`
# FIXME: return to stable when https://github.com/rust-lang/rust/issues/96937 gets into stable
- time cargo doc --workspace --verbose --no-deps
- time cargo doc --workspace --no-deps
- rm -f ./target/doc/.lock
- mv ./target/doc ./crate-docs
# FIXME: remove me after CI image gets nonroot
Expand Down Expand Up @@ -280,7 +280,7 @@ build-linux-substrate:
# see https://github.com/paritytech/ci_cd/issues/682#issuecomment-1340953589
- git checkout -B "$CI_COMMIT_REF_NAME" "$CI_COMMIT_SHA"
script:
- WASM_BUILD_NO_COLOR=1 time cargo build --locked --release --verbose
- WASM_BUILD_NO_COLOR=1 time cargo build --locked --release
- mv $CARGO_TARGET_DIR/release/substrate-node ./artifacts/substrate/substrate
- echo -n "Substrate version = "
- if [ "${CI_COMMIT_TAG}" ]; then
Expand Down Expand Up @@ -308,7 +308,7 @@ build-linux-substrate:
- mkdir -p ./artifacts/subkey
script:
- cd ./substrate/bin/utils/subkey
- SKIP_WASM_BUILD=1 time cargo build --locked --release --verbose
- SKIP_WASM_BUILD=1 time cargo build --locked --release
# - cd -
# - mv $CARGO_TARGET_DIR/release/subkey ./artifacts/subkey/.
# - echo -n "Subkey version = "
Expand All @@ -331,7 +331,7 @@ build-subkey-linux:
# - mkdir -p ./artifacts/subkey
# script:
# - cd ./bin/utils/subkey
# - SKIP_WASM_BUILD=1 time cargo build --locked --release --verbose
# - SKIP_WASM_BUILD=1 time cargo build --locked --release
# - cd -
# - mv ./target/release/subkey ./artifacts/subkey/.
# - echo -n "Subkey version = "
Expand Down
4 changes: 1 addition & 3 deletions .gitlab/pipeline/check.yml
Original file line number Diff line number Diff line change
Expand Up @@ -3,10 +3,9 @@ cargo-clippy:
extends:
- .docker-env
- .common-refs
- .pipeline-stopper-artifacts
script:
- SKIP_WASM_BUILD=1 env -u RUSTFLAGS cargo clippy --all-targets --locked --workspace
# fixme!
allow_failure: true

check-try-runtime:
stage: check
Expand Down Expand Up @@ -43,7 +42,6 @@ cargo-deny-licenses:
script:
- $CARGO_DENY_CMD --hide-inclusion-graph
after_script:
# - !reference [.rusty-cachier, after_script]
- echo "___The complete log is in the artifacts___"
- $CARGO_DENY_CMD 2> deny.log
- if [ $CI_JOB_STATUS != 'success' ]; then
Expand Down
Loading

0 comments on commit 6e394f8

Please sign in to comment.