diff --git a/substrate/scripts/ci/common/lib.sh b/substrate/scripts/ci/common/lib.sh
deleted file mode 100755
index 08c2fe81ada04..0000000000000
--- a/substrate/scripts/ci/common/lib.sh
+++ /dev/null
@@ -1,117 +0,0 @@
-#!/bin/sh
-
-api_base="https://api.github.com/repos"
-
-# Function to take 2 git tags/commits and get any lines from commit messages
-# that contain something that looks like a PR reference: e.g., (#1234)
-sanitised_git_logs(){
- git --no-pager log --pretty=format:"%s" "$1...$2" |
- # Only find messages referencing a PR
- grep -E '\(#[0-9]+\)' |
- # Strip any asterisks
- sed 's/^* //g' |
- # And add them all back
- sed 's/^/* /g'
-}
-
-# Returns the last published release on github
-# Note: we can't just use /latest because that ignores prereleases
-# repo: 'organization/repo'
-# Usage: last_github_release "$repo"
-last_github_release(){
- i=0
- # Iterate over releases until we find the last release that's not just a draft
- while [ $i -lt 29 ]; do
- out=$(curl -H "Authorization: token $GITHUB_RELEASE_TOKEN" -s "$api_base/$1/releases" | jq ".[$i]")
- echo "$out"
- # Ugh when echoing to jq, we need to translate newlines into spaces :/
- if [ "$(echo "$out" | tr '\r\n' ' ' | jq '.draft')" = "false" ]; then
- echo "$out" | tr '\r\n' ' ' | jq '.tag_name'
- return
- else
- i=$((i + 1))
- fi
- done
-}
-
-# Checks whether a tag on github has been verified
-# repo: 'organization/repo'
-# tagver: 'v1.2.3'
-# Usage: check_tag $repo $tagver
-check_tag () {
- repo=$1
- tagver=$2
- tag_out=$(curl -H "Authorization: token $GITHUB_RELEASE_TOKEN" -s "$api_base/$repo/git/refs/tags/$tagver")
- tag_sha=$(echo "$tag_out" | jq -r .object.sha)
- object_url=$(echo "$tag_out" | jq -r .object.url)
- if [ "$tag_sha" = "null" ]; then
- return 2
- fi
- verified_str=$(curl -H "Authorization: token $GITHUB_RELEASE_TOKEN" -s "$object_url" | jq -r .verification.verified)
- if [ "$verified_str" = "true" ]; then
- # Verified, everything is good
- return 0
- else
- # Not verified. Bad juju.
- return 1
- fi
-}
-
-# Checks whether a given PR has a given label.
-# repo: 'organization/repo'
-# pr_id: 12345
-# label: B1-silent
-# Usage: has_label $repo $pr_id $label
-has_label(){
- repo="$1"
- pr_id="$2"
- label="$3"
-
- # These will exist if the function is called in Gitlab.
- # If the function's called in Github, we should have GITHUB_ACCESS_TOKEN set
- # already.
- if [ -n "$GITHUB_RELEASE_TOKEN" ]; then
- GITHUB_TOKEN="$GITHUB_RELEASE_TOKEN"
- elif [ -n "$GITHUB_PR_TOKEN" ]; then
- GITHUB_TOKEN="$GITHUB_PR_TOKEN"
- fi
-
- out=$(curl -H "Authorization: token $GITHUB_TOKEN" -s "$api_base/$repo/pulls/$pr_id")
- [ -n "$(echo "$out" | tr -d '\r\n' | jq ".labels | .[] | select(.name==\"$label\")")" ]
-}
-
-# Formats a message into a JSON string for posting to Matrix
-# message: 'any plaintext message'
-# formatted_message: 'optional message formatted in html'
-# Usage: structure_message $content $formatted_content (optional)
-structure_message() {
- if [ -z "$2" ]; then
- body=$(jq -Rs --arg body "$1" '{"msgtype": "m.text", $body}' < /dev/null)
- else
- body=$(jq -Rs --arg body "$1" --arg formatted_body "$2" '{"msgtype": "m.text", $body, "format": "org.matrix.custom.html", $formatted_body}' < /dev/null)
- fi
- echo "$body"
-}
-
-# Post a message to a matrix room
-# body: '{body: "JSON string produced by structure_message"}'
-# room_id: !fsfSRjgjBWEWffws:matrix.parity.io
-# access_token: see https://matrix.org/docs/guides/client-server-api/
-# Usage: send_message $body (json formatted) $room_id $access_token
-send_message() {
- curl -XPOST -d "$1" "https://m.parity.io/_matrix/client/r0/rooms/$2/send/m.room.message?access_token=$3"
-}
-
-# Check for runtime changes between two commits. This is defined as any changes
-# to bin/node/src/runtime, frame/ and primitives/sr_* trees.
-has_runtime_changes() {
- from=$1
- to=$2
- if git diff --name-only "${from}...${to}" \
- | grep -q -e '^frame/' -e '^primitives/'
- then
- return 0
- else
- return 1
- fi
-}
diff --git a/substrate/scripts/ci/github/check_labels.sh b/substrate/scripts/ci/github/check_labels.sh
deleted file mode 100755
index 7b0aed9fe7345..0000000000000
--- a/substrate/scripts/ci/github/check_labels.sh
+++ /dev/null
@@ -1,68 +0,0 @@
-#!/usr/bin/env bash
-set -e
-
-#shellcheck source=../common/lib.sh
-source "$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )/../common/lib.sh"
-
-repo="$GITHUB_REPOSITORY"
-pr="$GITHUB_PR"
-
-ensure_labels() {
- for label in "$@"; do
- if has_label "$repo" "$pr" "$label"; then
- return 0
- fi
- done
- return 1
-}
-
-# Must have one of the following labels
-releasenotes_labels=(
- 'B0-silent'
- 'B3-apinoteworthy'
- 'B5-clientnoteworthy'
- 'B7-runtimenoteworthy'
-)
-
-criticality_labels=(
- 'C1-low 📌'
- 'C3-medium 📣'
- 'C7-high ❗️'
- 'C9-critical ‼️'
-)
-
-audit_labels=(
- 'D1-audited 👍'
- 'D2-notlive 💤'
- 'D3-trivial 🧸'
- 'D5-nicetohaveaudit ⚠️'
- 'D9-needsaudit 👮'
-)
-
-echo "[+] Checking release notes (B) labels"
-if ensure_labels "${releasenotes_labels[@]}"; then
- echo "[+] Release notes label detected. All is well."
-else
- echo "[!] Release notes label not detected. Please add one of: ${releasenotes_labels[*]}"
- exit 1
-fi
-
-echo "[+] Checking release criticality (C) labels"
-if ensure_labels "${criticality_labels[@]}"; then
- echo "[+] Release criticality label detected. All is well."
-else
- echo "[!] Release criticality label not detected. Please add one of: ${criticality_labels[*]}"
- exit 1
-fi
-
-if has_runtime_changes origin/master "${HEAD_SHA}"; then
- echo "[+] Runtime changes detected. Checking audit (D) labels"
- if ensure_labels "${audit_labels[@]}"; then
- echo "[+] Release audit label detected. All is well."
- else
- echo "[!] Release audit label not detected. Please add one of: ${audit_labels[*]}"
- exit 1
- fi
-fi
-
-exit 0
diff --git a/substrate/scripts/ci/github/generate_changelog.sh b/substrate/scripts/ci/github/generate_changelog.sh
deleted file mode 100755
index 32ac1760a6117..0000000000000
--- a/substrate/scripts/ci/github/generate_changelog.sh
+++ /dev/null
@@ -1,85 +0,0 @@
-#!/usr/bin/env bash
-
-# shellcheck source=../common/lib.sh
-source "$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )/../common/lib.sh"
-
-version="$2"
-last_version="$1"
-
-all_changes="$(sanitised_git_logs "$last_version" "$version")"
-runtime_changes=""
-api_changes=""
-client_changes=""
-changes=""
-migrations=""
-
-while IFS= read -r line; do
- pr_id=$(echo "$line" | sed -E 's/.*#([0-9]+)\)$/\1/')
-
- # Skip if the PR has the silent label - this allows us to skip a few requests
- if has_label 'paritytech/substrate' "$pr_id" 'B0-silent'; then
- continue
- fi
- if has_label 'paritytech/substrate' "$pr_id" 'B3-apinoteworthy' ; then
- api_changes="$api_changes
-$line"
- fi
- if has_label 'paritytech/substrate' "$pr_id" 'B5-clientnoteworthy'; then
- client_changes="$client_changes
-$line"
- fi
- if has_label 'paritytech/substrate' "$pr_id" 'B7-runtimenoteworthy'; then
- runtime_changes="$runtime_changes
-$line"
- fi
- if has_label 'paritytech/substrate' "$pr_id" 'E1-runtime-migration'; then
- migrations="$migrations
-$line"
- fi
-done <<< "$all_changes"
-
-# Make the substrate section if there are any substrate changes
-if [ -n "$runtime_changes" ] ||
- [ -n "$api_changes" ] ||
- [ -n "$client_changes" ] ||
- [ -n "$migrations" ]; then
- changes=$(cat << EOF
-Substrate changes
------------------
-
-EOF
-)
- if [ -n "$runtime_changes" ]; then
- changes="$changes
-
-Runtime
--------
-$runtime_changes"
- fi
- if [ -n "$client_changes" ]; then
- changes="$changes
-
-Client
-------
-$client_changes"
- fi
- if [ -n "$api_changes" ]; then
- changes="$changes
-
-API
----
-$api_changes"
- fi
- release_text="$release_text
-
-$changes"
-fi
-if [ -n "$migrations" ]; then
- changes="$changes
-
-Runtime Migrations
-------------------
-$migrations"
-fi
-
-echo "$changes"
diff --git a/substrate/scripts/ci/gitlab/check-each-crate.py b/substrate/scripts/ci/gitlab/check-each-crate.py
deleted file mode 100755
index adad4f5bd5835..0000000000000
--- a/substrate/scripts/ci/gitlab/check-each-crate.py
+++ /dev/null
@@ -1,57 +0,0 @@
-#!/usr/bin/env python3
-
-# A script that checks each workspace crate individually.
-# It's relevant to check workspace crates individually because otherwise their compilation problems
-# due to feature misconfigurations won't be caught, as exemplified by
-# https://github.com/paritytech/substrate/issues/12705
-#
-# `check-each-crate.py target_group groups_total`
-#
-# - `target_group`: Integer starting from 1, the group this script should execute.
-# - `groups_total`: Integer starting from 1, total number of groups.
-
-import subprocess, sys
-
-# Get all crates
-output = subprocess.check_output(["cargo", "tree", "--locked", "--workspace", "--depth", "0", "--prefix", "none"])
-
-# Convert the output into a proper list
-crates = []
-for line in output.splitlines():
- if line != b"":
- crates.append(line.decode('utf8').split(" ")[0])
-
-# Make the list unique and sorted
-crates = list(set(crates))
-crates.sort()
-
-target_group = int(sys.argv[1]) - 1
-groups_total = int(sys.argv[2])
-
-if len(crates) == 0:
- print("No crates detected!", file=sys.stderr)
- sys.exit(1)
-
-print(f"Total crates: {len(crates)}", file=sys.stderr)
-
-crates_per_group = len(crates) // groups_total
-
-# If this is the last runner, we need to take care of crates
-# after the group that we lost because of the integer division.
-if target_group + 1 == groups_total:
- overflow_crates = len(crates) % groups_total
-else:
- overflow_crates = 0
-
-print(f"Crates per group: {crates_per_group}", file=sys.stderr)
-
-# Check each crate
-for i in range(0, crates_per_group + overflow_crates):
- crate = crates_per_group * target_group + i
-
- print(f"Checking {crates[crate]}", file=sys.stderr)
-
- res = subprocess.run(["cargo", "check", "--locked", "-p", crates[crate]])
-
- if res.returncode != 0:
- sys.exit(1)
diff --git a/substrate/scripts/ci/gitlab/check_runtime.sh b/substrate/scripts/ci/gitlab/check_runtime.sh
deleted file mode 100755
index 71d6965ecf4fb..0000000000000
--- a/substrate/scripts/ci/gitlab/check_runtime.sh
+++ /dev/null
@@ -1,121 +0,0 @@
-#!/bin/sh
-#
-#
-# check for any changes in the node/src/runtime, frame/ and primitives/sr_* trees. if
-# there are any changes found, it should mark the PR breaksconsensus and
-# "auto-fail" the PR if there isn't a change in the runtime/src/lib.rs file
-# that alters the version.
-
-set -e # fail on any error
-
-#shellcheck source=../common/lib.sh
-. "$(dirname "${0}")/../common/lib.sh"
-
-VERSIONS_FILE="bin/node/runtime/src/lib.rs"
-
-boldprint () { printf "|\n| \033[1m%s\033[0m\n|\n" "${@}"; }
-boldcat () { printf "|\n"; while read -r l; do printf "| \033[1m%s\033[0m\n" "${l}"; done; printf "|\n" ; }
-
-github_label () {
- echo
- echo "# run github-api job for labeling it ${1}"
- curl -sS -X POST \
- -F "token=${CI_JOB_TOKEN}" \
- -F "ref=master" \
- -F "variables[LABEL]=${1}" \
- -F "variables[PRNO]=${CI_COMMIT_REF_NAME}" \
- "${GITLAB_API}/projects/${GITHUB_API_PROJECT}/trigger/pipeline"
-}
-
-
-boldprint "latest 10 commits of ${CI_COMMIT_REF_NAME}"
-git log --graph --oneline --decorate=short -n 10
-
-boldprint "make sure the master branch and release tag are available in shallow clones"
-git fetch --depth="${GIT_DEPTH:-100}" origin master
-git fetch --depth="${GIT_DEPTH:-100}" origin release
-git tag -f release FETCH_HEAD
-git log -n1 release
-
-
-boldprint "check if the wasm sources changed"
-if ! has_runtime_changes origin/master "${CI_COMMIT_SHA}"
-then
- boldcat <<-EOT
-
- no changes to the runtime source code detected
-
- EOT
-
- exit 0
-fi
-
-
-
-# check for spec_version updates: if the spec versions changed, then there is
-# consensus-critical logic that has changed. the runtime wasm blobs must be
-# rebuilt.
-
-add_spec_version="$(git diff tags/release ${CI_COMMIT_SHA} -- "${VERSIONS_FILE}" \
- | sed -n -r "s/^\+[[:space:]]+spec_version: +([0-9]+),$/\1/p")"
-sub_spec_version="$(git diff tags/release ${CI_COMMIT_SHA} -- "${VERSIONS_FILE}" \
- | sed -n -r "s/^\-[[:space:]]+spec_version: +([0-9]+),$/\1/p")"
-
-
-
-if [ "${add_spec_version}" != "${sub_spec_version}" ]
-then
-
- boldcat <<-EOT
-
- changes to the runtime sources and changes in the spec version.
-
- spec_version: ${sub_spec_version} -> ${add_spec_version}
-
- EOT
- exit 0
-
-else
- # check for impl_version updates: if only the impl versions changed, we assume
- # there is no consensus-critical logic that has changed.
-
- add_impl_version="$(git diff tags/release ${CI_COMMIT_SHA} -- "${VERSIONS_FILE}" \
- | sed -n -r 's/^\+[[:space:]]+impl_version: +([0-9]+),$/\1/p')"
- sub_impl_version="$(git diff tags/release ${CI_COMMIT_SHA} -- "${VERSIONS_FILE}" \
- | sed -n -r 's/^\-[[:space:]]+impl_version: +([0-9]+),$/\1/p')"
-
-
- # see if the impl version changed
- if [ "${add_impl_version}" != "${sub_impl_version}" ]
- then
- boldcat <<-EOT
-
- changes to the runtime sources and changes in the impl version.
-
- impl_version: ${sub_impl_version} -> ${add_impl_version}
-
- EOT
- exit 0
- fi
-
-
- boldcat <<-EOT
-
- wasm source files changed but not the spec/impl version. If changes made do not alter logic,
- just bump 'impl_version'. If they do change logic, bump 'spec_version'.
-
- source file directories:
- - bin/node/src/runtime
- - frame
- - primitives/sr-*
-
- versions file: ${VERSIONS_FILE}
-
- EOT
-fi
-
-# dropped through. there's something wrong; exit 1.
-
-exit 1
-
-# vim: noexpandtab
diff --git a/substrate/scripts/ci/gitlab/check_signed.sh b/substrate/scripts/ci/gitlab/check_signed.sh
deleted file mode 100755
index 20d47c2304767..0000000000000
--- a/substrate/scripts/ci/gitlab/check_signed.sh
+++ /dev/null
@@ -1,16 +0,0 @@
-#!/usr/bin/env bash
-
-# shellcheck source=../common/lib.sh
-source "$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )/../common/lib.sh"
-
-version="$CI_COMMIT_TAG"
-
-echo '[+] Checking tag has been signed'
-check_tag "paritytech/substrate" "$version"
-case $? in
- 0) echo '[+] Tag found and has been signed'; exit 0
- ;;
- 1) echo '[!] Tag found but has not been signed. Aborting release.'; exit 1
- ;;
- 2) echo '[!] Tag not found. Aborting release.'; exit 1
-esac
diff --git a/substrate/scripts/ci/gitlab/ensure-deps.sh b/substrate/scripts/ci/gitlab/ensure-deps.sh
deleted file mode 100755
index 7087200cef518..0000000000000
--- a/substrate/scripts/ci/gitlab/ensure-deps.sh
+++ /dev/null
@@ -1,80 +0,0 @@
-#!/usr/bin/env bash
-
-# The script is meant to check if the rules regarding packages
-# dependencies are satisfied.
-# The general format is:
-# [top-lvl-dir] MESSAGE/[other-top-dir]
-
-# For instance no crate within `./client` directory
-# is allowed to import any crate with a directory path containing `frame`.
-# Such rule is just: `client crates must not depend on anything in /frame`.
-
-# The script should be run from the main repo directory!
-
-set -u
-
-# HARD FAILING
-MUST_NOT=(
- "client crates must not depend on anything in /frame"
- "client crates must not depend on anything in /node"
- "frame crates must not depend on anything in /node"
- "frame crates must not depend on anything in /client"
- "primitives crates must not depend on anything in /frame"
-)
-
-# ONLY DISPLAYED, script still succeeds
-PLEASE_DONT=(
- "primitives crates should not depend on anything in /client"
-)
-
-VIOLATIONS=()
-PACKAGES=()
-
-function check_rule() {
- rule=$1
- from=$(echo $rule | cut -f1 -d\ )
- to=$(echo $rule | cut -f2 -d\/)
-
- cd $from
- echo "Checking rule '$rule'"
- packages=$(find -name Cargo.toml | xargs grep -wn "path.*\.\.\/$to")
- has_references=$(echo -n $packages | wc -c)
- if [ "$has_references" != "0" ]; then
- VIOLATIONS+=("$rule")
- # Find packages that violate:
- PACKAGES+=("$packages")
- fi
- cd - > /dev/null
-}
-
-for rule in "${MUST_NOT[@]}"
-do
- check_rule "$rule";
-done
-
-# Only the MUST NOT will be counted towards failure
-HARD_VIOLATIONS=${#VIOLATIONS[@]}
-
-
-for rule in "${PLEASE_DONT[@]}"
-do
- check_rule "$rule";
-done
-
-# Display violations and fail
-I=0
-for v in "${VIOLATIONS[@]}"
-do
- cat << EOF
-
-===========================================
-======= Violation of rule: $v
-===========================================
-${PACKAGES[$I]}
-
-
-EOF
- I=$I+1
-done
-
-exit $HARD_VIOLATIONS
diff --git a/substrate/scripts/ci/gitlab/pipeline/build.yml b/substrate/scripts/ci/gitlab/pipeline/build.yml
deleted file mode 100644
index 8f63f6ecc3911..0000000000000
--- a/substrate/scripts/ci/gitlab/pipeline/build.yml
+++ /dev/null
@@ -1,215 +0,0 @@
-# This file is part of .gitlab-ci.yml
-# Here are all jobs that are executed during "build" stage
-
-# PIPELINE_SCRIPTS_TAG can be found in the project variables
-
-.check-dependent-project:
- stage: build
- # DAG: this is artificial dependency
- needs:
- - job: cargo-clippy
- artifacts: false
- extends:
- - .docker-env
- - .test-refs-no-trigger-prs-only
- variables:
- RUSTFLAGS: "-D warnings"
- script:
- - cargo install --locked --git https://github.com/paritytech/try-runtime-cli --rev a93c9b5abe5d31a4cf1936204f7e5c489184b521
- - git clone
- --depth=1
- --branch="$PIPELINE_SCRIPTS_TAG"
- https://github.com/paritytech/pipeline-scripts
- - ./pipeline-scripts/check_dependent_project.sh
- --org paritytech
- --dependent-repo "$DEPENDENT_REPO"
- --github-api-token "$GITHUB_PR_TOKEN"
- --extra-dependencies "$EXTRA_DEPENDENCIES"
- --companion-overrides "$COMPANION_OVERRIDES"
-
-.check-runtime-migration:
- extends:
- - .check-dependent-project
- - .test-refs-no-trigger-prs-only
- variables:
- DEPENDENT_REPO: polkadot
- COMPANION_OVERRIDES: |
- substrate: polkadot-v*
- polkadot: release-v*
- COMPANION_CHECK_COMMAND: >
- time cargo build --release -p "$NETWORK"-runtime --features try-runtime &&
- time try-runtime \
- --runtime ./target/release/wbuild/"$NETWORK"-runtime/target/wasm32-unknown-unknown/release/"$NETWORK"_runtime.wasm \
- on-runtime-upgrade --checks=pre-and-post live --uri wss://${NETWORK}-try-runtime-node.parity-chains.parity.io:443
-
-# Individual jobs are set up for each dependent project so that they can be ran in parallel.
-# Arguably we could generate a job for each companion in the PR's description using Gitlab's
-# parent-child pipelines but that's more complicated.
-
-check-runtime-migration-polkadot:
- extends:
- - .check-runtime-migration
- variables:
- NETWORK: polkadot
-
-check-runtime-migration-kusama:
- extends: .check-runtime-migration
- variables:
- NETWORK: kusama
-
-check-runtime-migration-rococo:
- extends: .check-runtime-migration
- variables:
- NETWORK: rococo
- allow_failure: true
-
-check-runtime-migration-westend:
- extends: .check-runtime-migration
- variables:
- NETWORK: westend
-
-check-dependent-polkadot:
- extends: .check-dependent-project
- variables:
- DEPENDENT_REPO: polkadot
- COMPANION_OVERRIDES: |
- substrate: polkadot-v*
- polkadot: release-v*
- # enable the same feature flags as polkadot's test-linux-stable
- COMPANION_CHECK_COMMAND: >
- cargo check --all-targets --workspace
- --features=runtime-benchmarks,runtime-metrics,try-runtime
- rules:
- - if: $CI_COMMIT_REF_NAME =~ /^[0-9]+$/ #PRs
-
-check-dependent-cumulus:
- extends: .check-dependent-project
- variables:
- DEPENDENT_REPO: cumulus
- EXTRA_DEPENDENCIES: polkadot
- COMPANION_OVERRIDES: |
- substrate: polkadot-v*
- polkadot: release-v*
- rules:
- - if: $CI_COMMIT_REF_NAME =~ /^[0-9]+$/ #PRs
-
-build-linux-substrate:
- stage: build
- extends:
- - .collect-artifacts
- - .docker-env
- - .build-refs
- variables:
- # this variable gets overriden by "rusty-cachier environment inject", use the value as default
- CARGO_TARGET_DIR: "$CI_PROJECT_DIR/target"
- needs:
- - job: test-linux-stable
- artifacts: false
- before_script:
- - !reference [.timestamp, before_script]
- - !reference [.job-switcher, before_script]
- - mkdir -p ./artifacts/substrate/
- - !reference [.rusty-cachier, before_script]
- # tldr: we need to checkout the branch HEAD explicitly because of our dynamic versioning approach while building the substrate binary
- # see https://github.com/paritytech/ci_cd/issues/682#issuecomment-1340953589
- - git checkout -B "$CI_COMMIT_REF_NAME" "$CI_COMMIT_SHA"
- script:
- - rusty-cachier snapshot create
- - WASM_BUILD_NO_COLOR=1 time cargo build --locked --release -p node-cli --verbose
- - mv $CARGO_TARGET_DIR/release/substrate-node ./artifacts/substrate/substrate
- - echo -n "Substrate version = "
- - if [ "${CI_COMMIT_TAG}" ]; then
- echo "${CI_COMMIT_TAG}" | tee ./artifacts/substrate/VERSION;
- else
- ./artifacts/substrate/substrate --version |
- cut -d ' ' -f 2 | tee ./artifacts/substrate/VERSION;
- fi
- - sha256sum ./artifacts/substrate/substrate | tee ./artifacts/substrate/substrate.sha256
- - cp -r ./scripts/ci/docker/substrate.Dockerfile ./artifacts/substrate/
- - printf '\n# building node-template\n\n'
- - ./scripts/ci/node-template-release.sh ./artifacts/substrate/substrate-node-template.tar.gz
- - rusty-cachier cache upload
-
-.build-subkey:
- stage: build
- extends:
- - .collect-artifacts
- - .docker-env
- - .publish-refs
- variables:
- # this variable gets overriden by "rusty-cachier environment inject", use the value as default
- CARGO_TARGET_DIR: "$CI_PROJECT_DIR/target"
- before_script:
- - !reference [.timestamp, before_script]
- - !reference [.job-switcher, before_script]
- - mkdir -p ./artifacts/subkey
- - !reference [.rusty-cachier, before_script]
- script:
- - rusty-cachier snapshot create
- - cd ./bin/utils/subkey
- - SKIP_WASM_BUILD=1 time cargo build --locked --release --verbose
- - cd -
- - mv $CARGO_TARGET_DIR/release/subkey ./artifacts/subkey/.
- - echo -n "Subkey version = "
- - ./artifacts/subkey/subkey --version |
- sed -n -E 's/^subkey ([0-9.]+.*)/\1/p' |
- tee ./artifacts/subkey/VERSION;
- - sha256sum ./artifacts/subkey/subkey | tee ./artifacts/subkey/subkey.sha256
- - cp -r ./scripts/ci/docker/subkey.Dockerfile ./artifacts/subkey/
- - rusty-cachier cache upload
-
-build-subkey-linux:
- extends: .build-subkey
-
-build-subkey-macos:
- extends: .build-subkey
- # duplicating before_script & script sections from .build-subkey hidden job
- # to overwrite rusty-cachier integration as it doesn't work on macos
- before_script:
- # skip timestamp script, the osx bash doesn't support printf %()T
- - !reference [.job-switcher, before_script]
- - mkdir -p ./artifacts/subkey
- script:
- - cd ./bin/utils/subkey
- - SKIP_WASM_BUILD=1 time cargo build --locked --release --verbose
- - cd -
- - mv ./target/release/subkey ./artifacts/subkey/.
- - echo -n "Subkey version = "
- - ./artifacts/subkey/subkey --version |
- sed -n -E 's/^subkey ([0-9.]+.*)/\1/p' |
- tee ./artifacts/subkey/VERSION;
- - sha256sum ./artifacts/subkey/subkey | tee ./artifacts/subkey/subkey.sha256
- - cp -r ./scripts/ci/docker/subkey.Dockerfile ./artifacts/subkey/
- after_script: [""]
- tags:
- - osx
-
-build-rustdoc:
- stage: build
- extends:
- - .docker-env
- - .test-refs
- variables:
- SKIP_WASM_BUILD: 1
- DOC_INDEX_PAGE: "substrate/index.html" # default redirected page
- # this variable gets overriden by "rusty-cachier environment inject", use the value as default
- CARGO_TARGET_DIR: "$CI_PROJECT_DIR/target"
- artifacts:
- name: "${CI_JOB_NAME}_${CI_COMMIT_REF_NAME}-doc"
- when: on_success
- expire_in: 7 days
- paths:
- - ./crate-docs/
- # DAG: this is artificial dependency
- needs:
- - job: cargo-clippy
- artifacts: false
- script:
- - rusty-cachier snapshot create
- - time cargo doc --locked --workspace --all-features --verbose --no-deps
- - rm -f $CARGO_TARGET_DIR/doc/.lock
- - mv $CARGO_TARGET_DIR/doc ./crate-docs
- # FIXME: remove me after CI image gets nonroot
- - chown -R nonroot:nonroot ./crate-docs
- - echo "" > ./crate-docs/index.html
- - rusty-cachier cache upload
diff --git a/substrate/scripts/ci/gitlab/pipeline/check.yml b/substrate/scripts/ci/gitlab/pipeline/check.yml
deleted file mode 100644
index 576daec9b4331..0000000000000
--- a/substrate/scripts/ci/gitlab/pipeline/check.yml
+++ /dev/null
@@ -1,78 +0,0 @@
-# This file is part of .gitlab-ci.yml
-# Here are all jobs that are executed during "check" stage
-
-check-runtime:
- stage: check
- extends:
- - .kubernetes-env
- - .test-refs-no-trigger-prs-only
- variables:
- CI_IMAGE: "paritytech/tools:latest"
- GITLAB_API: "https://gitlab.parity.io/api/v4"
- GITHUB_API_PROJECT: "parity%2Finfrastructure%2Fgithub-api"
- script:
- - ./scripts/ci/gitlab/check_runtime.sh
- allow_failure: true
-
-check-signed-tag:
- stage: check
- extends: .kubernetes-env
- variables:
- CI_IMAGE: "paritytech/tools:latest"
- rules:
- - if: $CI_COMMIT_REF_NAME =~ /^ci-release-.*$/
- - if: $CI_COMMIT_REF_NAME =~ /^v[0-9]+\.[0-9]+.*$/ # i.e. v1.0, v2.1rc1
- script:
- - ./scripts/ci/gitlab/check_signed.sh
-
-test-dependency-rules:
- stage: check
- extends:
- - .kubernetes-env
- - .test-refs-no-trigger-prs-only
- variables:
- CI_IMAGE: "paritytech/tools:latest"
- script:
- - ./scripts/ci/gitlab/ensure-deps.sh
-
-test-rust-features:
- stage: check
- extends:
- - .kubernetes-env
- - .test-refs-no-trigger-prs-only
- script:
- - git clone
- --depth=1
- --branch="$PIPELINE_SCRIPTS_TAG"
- https://github.com/paritytech/pipeline-scripts
- - bash ./pipeline-scripts/rust-features.sh .
-
-test-rust-feature-propagation:
- stage: check
- extends:
- - .kubernetes-env
- - .test-refs-no-trigger-prs-only
- script:
- - cargo install --locked --version 0.7.4 -q -f zepter && zepter --version
- - echo "👉 Hello developer! If you see this CI check failing then it means that one of the crates is missing a feature for one of its dependencies. The output below tells you which feature needs to be added for which dependency to which crate. You can do this by modifying the Cargo.toml file. For more context see the MR where this check was introduced https://github.com/paritytech/substrate/pull/14660"
- - zepter lint propagate-feature --feature try-runtime --left-side-feature-missing=ignore --workspace --feature-enables-dep="try-runtime:frame-try-runtime" --locked
- - zepter lint propagate-feature --feature runtime-benchmarks --left-side-feature-missing=ignore --workspace --feature-enables-dep="runtime-benchmarks:frame-benchmarking" --locked
- - zepter lint propagate-feature --feature std --left-side-feature-missing=ignore --workspace --locked
- allow_failure: true # Experimental
-
-test-prometheus-alerting-rules:
- stage: check
- extends: .kubernetes-env
- variables:
- CI_IMAGE: "paritytech/tools:latest"
- rules:
- - if: $CI_PIPELINE_SOURCE == "pipeline"
- when: never
- - if: $CI_COMMIT_BRANCH
- changes:
- - .gitlab-ci.yml
- - ./scripts/ci/monitoring/**/*
- script:
- - promtool check rules ./scripts/ci/monitoring/alerting-rules/alerting-rules.yaml
- - cat ./scripts/ci/monitoring/alerting-rules/alerting-rules.yaml |
- promtool test rules ./scripts/ci/monitoring/alerting-rules/alerting-rule-tests.yaml
diff --git a/substrate/scripts/ci/gitlab/pipeline/publish.yml b/substrate/scripts/ci/gitlab/pipeline/publish.yml
deleted file mode 100644
index c90af7ba347ba..0000000000000
--- a/substrate/scripts/ci/gitlab/pipeline/publish.yml
+++ /dev/null
@@ -1,270 +0,0 @@
-# This file is part of .gitlab-ci.yml
-# Here are all jobs that are executed during "publish" stage
-
-.build-push-docker-image-common:
- extends:
- - .kubernetes-env
- stage: publish
- variables:
- CI_IMAGE: $BUILDAH_IMAGE
- GIT_STRATEGY: none
- DOCKERFILE: $PRODUCT.Dockerfile
- IMAGE_NAME: docker.io/$IMAGE_PATH
- before_script:
- - !reference [.kubernetes-env, before_script]
- - cd ./artifacts/$PRODUCT/
- - VERSION="$(cat ./VERSION)"
- - echo "${PRODUCT} version = ${VERSION}"
- - test -z "${VERSION}" && exit 1
- script:
- - test "$DOCKER_USER" -a "$DOCKER_PASS" ||
- ( echo "no docker credentials provided"; exit 1 )
- - $BUILDAH_COMMAND build
- --format=docker
- --build-arg VCS_REF="${CI_COMMIT_SHA}"
- --build-arg BUILD_DATE="$(date -u '+%Y-%m-%dT%H:%M:%SZ')"
- --build-arg IMAGE_NAME="${IMAGE_PATH}"
- --tag "$IMAGE_NAME:$VERSION"
- --tag "$IMAGE_NAME:latest"
- --file "$DOCKERFILE" .
- - echo "$DOCKER_PASS" |
- buildah login --username "$DOCKER_USER" --password-stdin docker.io
- - $BUILDAH_COMMAND info
- - $BUILDAH_COMMAND push --format=v2s2 "$IMAGE_NAME:$VERSION"
- - $BUILDAH_COMMAND push --format=v2s2 "$IMAGE_NAME:latest"
- after_script:
- - buildah logout --all
- - echo "SUBSTRATE_IMAGE_NAME=${IMAGE_NAME}" | tee -a ./artifacts/$PRODUCT/build.env
- - IMAGE_TAG="$(cat ./artifacts/$PRODUCT/VERSION)"
- - echo "SUBSTRATE_IMAGE_TAG=${IMAGE_TAG}" | tee -a ./artifacts/$PRODUCT/build.env
- - cat ./artifacts/$PRODUCT/build.env
-
-.build-push-docker-image:
- extends:
- - .publish-refs
- - .build-push-docker-image-common
- variables:
- IMAGE_PATH: parity/$PRODUCT
- DOCKER_USER: $Docker_Hub_User_Parity
- DOCKER_PASS: $Docker_Hub_Pass_Parity
-
-.push-docker-image-description:
- stage: publish
- extends:
- - .kubernetes-env
- variables:
- CI_IMAGE: paritytech/dockerhub-description
- DOCKERHUB_REPOSITORY: parity/$PRODUCT
- DOCKER_USERNAME: $Docker_Hub_User_Parity
- DOCKER_PASSWORD: $Docker_Hub_Pass_Parity
- README_FILEPATH: $CI_PROJECT_DIR/scripts/ci/docker/$PRODUCT.Dockerfile.README.md
- rules:
- - if: $CI_COMMIT_REF_NAME == $CI_DEFAULT_BRANCH && $CI_PIPELINE_SOURCE == "push"
- changes:
- - scripts/ci/docker/$PRODUCT.Dockerfile.README.md
- before_script:
- - echo
- script:
- - cd / && sh entrypoint.sh
-
-# publish image to docker.io/paritypr, (e.g. for later use in zombienet testing)
-.build-push-image-temporary:
- extends:
- - .build-refs
- - .build-push-docker-image-common
- variables:
- IMAGE_PATH: paritypr/$PRODUCT
- DOCKER_USER: $PARITYPR_USER
- DOCKER_PASS: $PARITYPR_PASS
-
-publish-docker-substrate:
- extends: .build-push-docker-image
- needs:
- - job: build-linux-substrate
- artifacts: true
- variables:
- PRODUCT: substrate
-
-publish-docker-description-substrate:
- extends: .push-docker-image-description
- variables:
- PRODUCT: substrate
- SHORT_DESCRIPTION: "Substrate Docker Image."
-
-publish-docker-substrate-temporary:
- extends: .build-push-image-temporary
- needs:
- - job: build-linux-substrate
- artifacts: true
- variables:
- PRODUCT: substrate
- artifacts:
- reports:
- # this artifact is used in zombienet-tests job
- # https://docs.gitlab.com/ee/ci/multi_project_pipelines.html#with-variable-inheritance
- dotenv: ./artifacts/$PRODUCT/build.env
- expire_in: 24h
-
-publish-docker-subkey:
- extends: .build-push-docker-image
- needs:
- - job: build-subkey-linux
- artifacts: true
- variables:
- PRODUCT: subkey
-
-publish-docker-description-subkey:
- extends: .push-docker-image-description
- variables:
- PRODUCT: subkey
- SHORT_DESCRIPTION: "The subkey program is a key management utility for Substrate-based blockchains."
-
-publish-s3-release:
- stage: publish
- extends:
- - .publish-refs
- - .kubernetes-env
- needs:
- - job: build-linux-substrate
- artifacts: true
- - job: build-subkey-linux
- artifacts: true
- image: paritytech/awscli:latest
- variables:
- GIT_STRATEGY: none
- BUCKET: "releases.parity.io"
- PREFIX: "substrate/${ARCH}-${DOCKER_OS}"
- script:
- - aws s3 sync ./artifacts/ s3://${BUCKET}/${PREFIX}/$(cat ./artifacts/substrate/VERSION)/
- - echo "update objects in latest path"
- - aws s3 sync s3://${BUCKET}/${PREFIX}/$(cat ./artifacts/substrate/VERSION)/ s3://${BUCKET}/${PREFIX}/latest/
- after_script:
- - aws s3 ls s3://${BUCKET}/${PREFIX}/latest/
- --recursive --human-readable --summarize
-
-publish-rustdoc:
- stage: publish
- extends: .kubernetes-env
- variables:
- CI_IMAGE: node:16
- GIT_DEPTH: 100
- RUSTDOCS_DEPLOY_REFS: "master"
- rules:
- - if: $CI_PIPELINE_SOURCE == "pipeline"
- when: never
- - if: $CI_PIPELINE_SOURCE == "web" && $CI_COMMIT_REF_NAME == "master"
- - if: $CI_COMMIT_REF_NAME == "master"
- - if: $CI_COMMIT_REF_NAME =~ /^monthly-20[0-9]{2}-[0-9]{2}.*$/ # to support: monthly-2021-09+1
- - if: $CI_COMMIT_REF_NAME =~ /^v[0-9]+\.[0-9]+.*$/ # i.e. v1.0, v2.1rc1
- # `needs:` can be removed after CI image gets nonroot. In this case `needs:` stops other
- # artifacts from being dowloaded by this job.
- needs:
- - job: build-rustdoc
- artifacts: true
- script:
- # If $CI_COMMIT_REF_NAME doesn't match one of $RUSTDOCS_DEPLOY_REFS space-separated values, we
- # exit immediately.
- # Putting spaces at the front and back to ensure we are not matching just any substring, but the
- # whole space-separated value.
- - '[[ " ${RUSTDOCS_DEPLOY_REFS} " =~ " ${CI_COMMIT_REF_NAME} " ]] || exit 0'
- # setup ssh
- - eval $(ssh-agent)
- - ssh-add - <<< ${GITHUB_SSH_PRIV_KEY}
- - mkdir ~/.ssh && touch ~/.ssh/known_hosts
- - ssh-keyscan -t rsa github.com >> ~/.ssh/known_hosts
- # Set git config
- - git config user.email "devops-team@parity.io"
- - git config user.name "${GITHUB_USER}"
- - git config remote.origin.url "git@github.com:/paritytech/${CI_PROJECT_NAME}.git"
- - git config remote.origin.fetch "+refs/heads/*:refs/remotes/origin/*"
- - git fetch origin gh-pages
- # Save README and docs
- - cp -r ./crate-docs/ /tmp/doc/
- - cp README.md /tmp/doc/
- # we don't need to commit changes because we copy docs to /tmp
- - git checkout gh-pages --force
- # Install `index-tpl-crud` and generate index.html based on RUSTDOCS_DEPLOY_REFS
- - which index-tpl-crud &> /dev/null || yarn global add @substrate/index-tpl-crud
- - index-tpl-crud upsert ./index.html ${CI_COMMIT_REF_NAME}
- # Ensure the destination dir doesn't exist.
- - rm -rf ${CI_COMMIT_REF_NAME}
- - mv -f /tmp/doc ${CI_COMMIT_REF_NAME}
- # Upload files
- - git add --all
- # `git commit` has an exit code of > 0 if there is nothing to commit.
- # This causes GitLab to exit immediately and marks this job failed.
- # We don't want to mark the entire job failed if there's nothing to
- # publish though, hence the `|| true`.
- - git commit -m "___Updated docs for ${CI_COMMIT_REF_NAME}___" ||
- echo "___Nothing to commit___"
- - git push origin gh-pages --force
- after_script:
- - rm -rf .git/ ./*
-
-publish-draft-release:
- stage: publish
- image: paritytech/tools:latest
- rules:
- - if: $CI_COMMIT_REF_NAME =~ /^ci-release-.*$/
- - if: $CI_COMMIT_REF_NAME =~ /^v[0-9]+\.[0-9]+.*$/ # i.e. v1.0, v2.1rc1
- script:
- - ./scripts/ci/gitlab/publish_draft_release.sh
- allow_failure: true
-
-.publish-crates-template:
- stage: publish
- extends:
- - .crates-publishing-template
- - .crates-publishing-pipeline
- # We don't want multiple jobs racing to publish crates as it's redundant and they might overwrite
- # the releases of one another. Use resource_group to ensure that at most one instance of this job
- # is running at any given time.
- resource_group: crates-publishing
- # crates.io currently rate limits crate publishing at 1 per minute:
- # https://github.com/paritytech/release-engineering/issues/123#issuecomment-1335509748
- # Taking into account the 202 (as of Dec 07, 2022) publishable Substrate crates, in the worst
- # case, due to the rate limits alone, we'd have to wait through at least 202 minutes of delay.
- # Taking into account also the verification steps and extra synchronization delays after
- # publishing the crate, the job needs to have a much higher timeout than average.
- timeout: 9h
- # A custom publishing environment is used for us to be able to set up protected secrets
- # specifically for it
- environment: publish-crates
- script:
- - rusty-cachier snapshot create
- - git clone
- --depth 1
- --branch "$RELENG_SCRIPTS_BRANCH"
- https://github.com/paritytech/releng-scripts.git
- - CRATESIO_TARGET_INSTANCE=default ./releng-scripts/publish-crates
- - rusty-cachier cache upload
-
-publish-crates:
- extends: .publish-crates-template
- # publish-crates should only be run if publish-crates-locally passes
- needs:
- - job: check-crate-publishing
- artifacts: false
-
-publish-crates-manual:
- extends: .publish-crates-template
- when: manual
- interruptible: false
-
-check-crate-publishing:
- stage: publish
- extends:
- - .crates-publishing-template
- - .crates-publishing-pipeline
- # When lots of crates are taken into account (for example on master where all crates are tested)
- # the job might take a long time, as evidenced by:
- # https://gitlab.parity.io/parity/mirrors/substrate/-/jobs/2269364
- timeout: 4h
- script:
- - rusty-cachier snapshot create
- - git clone
- --depth 1
- --branch "$RELENG_SCRIPTS_BRANCH"
- https://github.com/paritytech/releng-scripts.git
- - CRATESIO_TARGET_INSTANCE=local ./releng-scripts/publish-crates
- - rusty-cachier cache upload
diff --git a/substrate/scripts/ci/gitlab/pipeline/test.yml b/substrate/scripts/ci/gitlab/pipeline/test.yml
deleted file mode 100644
index 9c057e8d91582..0000000000000
--- a/substrate/scripts/ci/gitlab/pipeline/test.yml
+++ /dev/null
@@ -1,494 +0,0 @@
-# This file is part of .gitlab-ci.yml
-# Here are all jobs that are executed during "test" stage
-
-# It's more like a check and it belongs to the previous stage, but we want to run this job with real tests in parallel
-find-fail-ci-phrase:
- stage: test
- variables:
- CI_IMAGE: "paritytech/tools:latest"
- ASSERT_REGEX: "FAIL-CI"
- GIT_DEPTH: 1
- extends:
- - .kubernetes-env
- script:
- - set +e
- - rg --line-number --hidden --type rust --glob '!{.git,target}' "$ASSERT_REGEX" .; exit_status=$?
- - if [ $exit_status -eq 0 ]; then
- echo "$ASSERT_REGEX was found, exiting with 1";
- exit 1;
- else
- echo "No $ASSERT_REGEX was found, exiting with 0";
- exit 0;
- fi
-
-cargo-deny-licenses:
- stage: test
- extends:
- - .docker-env
- - .test-refs
- variables:
- CARGO_DENY_CMD: "cargo deny --all-features check licenses -c ./scripts/ci/deny.toml"
- script:
- - rusty-cachier snapshot create
- - $CARGO_DENY_CMD --hide-inclusion-graph
- - rusty-cachier cache upload
- after_script:
- - !reference [.rusty-cachier, after_script]
- - echo "___The complete log is in the artifacts___"
- - $CARGO_DENY_CMD 2> deny.log
- - if [ $CI_JOB_STATUS != 'success' ]; then
- echo 'Please check license of your crate or add an exception to scripts/ci/deny.toml';
- fi
- artifacts:
- name: $CI_COMMIT_SHORT_SHA
- expire_in: 3 days
- when: always
- paths:
- - deny.log
-
-cargo-fmt:
- stage: test
- variables:
- RUSTY_CACHIER_TOOLCHAIN: nightly
- extends:
- - .docker-env
- - .test-refs
- script:
- - rusty-cachier snapshot create
- - cargo +nightly fmt --all -- --check
- - rusty-cachier cache upload
-
-cargo-fmt-manifest:
- stage: test
- extends:
- - .docker-env
- - .test-refs
- script:
- - cargo install zepter --locked --version 0.11.1 -q -f --no-default-features && zepter --version
- - echo "👉 Hello developer! If you see this CI check failing then it means that one of the your changes in a Cargo.toml file introduced ill-formatted or unsorted features. Please take a look at 'docs/STYLE_GUIDE.md#manifest-formatting' to find out more."
- - zepter format features --check
- allow_failure: true # Experimental
-
-cargo-clippy:
- stage: test
- # this is an artificial job dependency, for pipeline optimization using GitLab's DAGs
- needs:
- - job: cargo-fmt
- artifacts: false
- extends:
- - .docker-env
- - .test-refs
- script:
- - echo $RUSTFLAGS
- - cargo version && cargo clippy --version
- - rusty-cachier snapshot create
- - SKIP_WASM_BUILD=1 env -u RUSTFLAGS cargo clippy --locked --all-targets --workspace
- - rusty-cachier cache upload
-
-cargo-check-benches:
- stage: test
- variables:
- CI_JOB_NAME: "cargo-check-benches"
- extends:
- - .docker-env
- - .test-refs-check-benches
- - .collect-artifacts
- - .pipeline-stopper-artifacts
- before_script:
- - !reference [.timestamp, before_script]
- # perform rusty-cachier operations before any further modifications to the git repo to make cargo feel cheated not so much
- - !reference [.rust-info-script, script]
- - !reference [.job-switcher, before_script]
- - !reference [.rusty-cachier, before_script]
- - !reference [.pipeline-stopper-vars, script]
- # merges in the master branch on PRs. skip if base is not master
- - 'if [ $CI_COMMIT_REF_NAME != "master" ]; then
- BASE=$(curl -s -H "Authorization: Bearer ${GITHUB_PR_TOKEN}" https://api.github.com/repos/paritytech/substrate/pulls/${CI_COMMIT_REF_NAME} | jq -r .base.ref);
- printf "Merging base branch %s\n" "${BASE:=master}";
- if [ $BASE != "master" ]; then
- echo "$BASE is not master, skipping merge";
- else
- git config user.email "ci@gitlab.parity.io";
- git fetch origin "refs/heads/${BASE}";
- git merge --verbose --no-edit FETCH_HEAD;
- fi
- fi'
- parallel: 2
- script:
- - rusty-cachier snapshot create
- - mkdir -p ./artifacts/benches/$CI_COMMIT_REF_NAME-$CI_COMMIT_SHORT_SHA
- # this job is executed in parallel on two runners
- - echo "___Running benchmarks___";
- - case ${CI_NODE_INDEX} in
- 1)
- SKIP_WASM_BUILD=1 time cargo check --locked --benches --all;
- cargo run --locked --release -p node-bench -- ::trie::read::small --json
- | tee ./artifacts/benches/$CI_COMMIT_REF_NAME-$CI_COMMIT_SHORT_SHA/::trie::read::small.json;
- echo "___Uploading cache for rusty-cachier___";
- rusty-cachier cache upload
- ;;
- 2)
- cargo run --locked --release -p node-bench -- ::node::import::sr25519::transfer_keep_alive::paritydb::small --json
- | tee ./artifacts/benches/$CI_COMMIT_REF_NAME-$CI_COMMIT_SHORT_SHA/::node::import::sr25519::transfer_keep_alive::paritydb::small.json
- ;;
- esac
-
-node-bench-regression-guard:
- # it's not belong to `build` semantically, but dag jobs can't depend on each other
- # within the single stage - https://gitlab.com/gitlab-org/gitlab/-/issues/30632
- # more: https://github.com/paritytech/substrate/pull/8519#discussion_r608012402
- stage: build
- extends:
- - .docker-env
- - .test-refs-no-trigger-prs-only
- needs:
- # this is a DAG
- - job: cargo-check-benches
- artifacts: true
- # polls artifact from master to compare with current result
- # need to specify both parallel jobs from master because of the bug
- # https://gitlab.com/gitlab-org/gitlab/-/issues/39063
- - project: $CI_PROJECT_PATH
- job: "cargo-check-benches 1/2"
- ref: master
- artifacts: true
- - project: $CI_PROJECT_PATH
- job: "cargo-check-benches 2/2"
- ref: master
- artifacts: true
- variables:
- CI_IMAGE: "paritytech/node-bench-regression-guard:latest"
- before_script:
- - !reference [.timestamp, before_script]
- script:
- - echo "------- IMPORTANT -------"
- - echo "node-bench-regression-guard depends on the results of a cargo-check-benches job"
- - echo "In case of this job failure, check your pipeline's cargo-check-benches"
- - "node-bench-regression-guard --reference artifacts/benches/master-*
- --compare-with artifacts/benches/$CI_COMMIT_REF_NAME-$CI_COMMIT_SHORT_SHA"
- after_script: [""]
-
-cargo-check-try-runtime-and-experimental:
- stage: test
- extends:
- - .docker-env
- - .test-refs
- script:
- - rusty-cachier snapshot create
- - time cargo check --workspace --locked --features try-runtime,experimental
- - rusty-cachier cache upload
-
-test-deterministic-wasm:
- stage: test
- # this is an artificial job dependency, for pipeline optimization using GitLab's DAGs
- needs:
- - job: cargo-check-try-runtime-and-experimental
- artifacts: false
- extends:
- - .docker-env
- - .test-refs
- variables:
- WASM_BUILD_NO_COLOR: 1
- # this variable gets overriden by "rusty-cachier environment inject", use the value as default
- CARGO_TARGET_DIR: "$CI_PROJECT_DIR/target"
- script:
- - rusty-cachier snapshot create
- # build runtime
- - cargo build --locked --verbose --release -p kitchensink-runtime
- # make checksum
- - sha256sum $CARGO_TARGET_DIR/release/wbuild/kitchensink-runtime/target/wasm32-unknown-unknown/release/kitchensink_runtime.wasm > checksum.sha256
- # clean up
- - rm -rf $CARGO_TARGET_DIR/release/wbuild
- # build again
- - cargo build --locked --verbose --release -p kitchensink-runtime
- # confirm checksum
- - sha256sum -c ./checksum.sha256
- # clean up again, don't put release binaries into the cache
- - rm -rf $CARGO_TARGET_DIR/release/wbuild
- - rusty-cachier cache upload
-
-test-linux-stable:
- stage: test
- extends:
- - .docker-env
- - .test-refs
- - .pipeline-stopper-artifacts
- variables:
- # Enable debug assertions since we are running optimized builds for testing
- # but still want to have debug assertions.
- RUSTFLAGS: "-C debug-assertions -D warnings"
- RUST_BACKTRACE: 1
- WASM_BUILD_NO_COLOR: 1
- WASM_BUILD_RUSTFLAGS: "-C debug-assertions -D warnings"
- # Ensure we run the UI tests.
- RUN_UI_TESTS: 1
- # needed for rusty-cachier to keep cache in test-linux-stable folder and not in test-linux-stable-1/3
- CI_JOB_NAME: "test-linux-stable"
- parallel: 3
- script:
- - rusty-cachier snapshot create
- # this job runs all tests in former runtime-benchmarks, frame-staking and wasmtime tests
- # tests are partitioned by nextest and executed in parallel on $CI_NODE_TOTAL runners
- - echo "Node index - ${CI_NODE_INDEX}. Total amount - ${CI_NODE_TOTAL}"
- - time cargo nextest run --workspace
- --locked
- --release
- --verbose
- --features runtime-benchmarks,try-runtime,experimental
- --manifest-path ./bin/node/cli/Cargo.toml
- --partition count:${CI_NODE_INDEX}/${CI_NODE_TOTAL}
- # run runtime-api tests with `enable-staging-api` feature
- - time cargo nextest run -p sp-api-test --features enable-staging-api
- # we need to update cache only from one job
- - if [ ${CI_NODE_INDEX} == 1 ]; then rusty-cachier cache upload; fi
- # Upload tests results to Elasticsearch
- - echo "Upload test results to Elasticsearch"
- - cat target/nextest/default/junit.xml | xq . > target/nextest/default/junit.json
- - |
- curl -v -XPOST --http1.1 \
- -u ${ELASTIC_USERNAME}:${ELASTIC_PASSWORD} \
- https://elasticsearch.parity-build.parity.io/unit-tests/_doc/${CI_JOB_ID} \
- -H 'Content-Type: application/json' \
- -d @target/nextest/default/junit.json || echo "failed to upload junit report"
- artifacts:
- when: always
- paths:
- - target/nextest/default/junit.xml
- reports:
- junit: target/nextest/default/junit.xml
-
-test-frame-support:
- stage: test
- extends:
- - .docker-env
- - .test-refs
- variables:
- # Enable debug assertions since we are running optimized builds for testing
- # but still want to have debug assertions.
- RUSTFLAGS: "-C debug-assertions -D warnings"
- RUST_BACKTRACE: 1
- WASM_BUILD_NO_COLOR: 1
- WASM_BUILD_RUSTFLAGS: "-C debug-assertions -D warnings"
- # Ensure we run the UI tests.
- RUN_UI_TESTS: 1
- script:
- - rusty-cachier snapshot create
- - cat /cargo_target_dir/debug/.fingerprint/memory_units-759eddf317490d2b/lib-memory_units.json || true
- - time cargo test --verbose --locked -p frame-support-test --features=frame-feature-testing,no-metadata-docs,try-runtime,experimental --manifest-path ./frame/support/test/Cargo.toml
- - time cargo test --verbose --locked -p frame-support-test --features=frame-feature-testing,frame-feature-testing-2,no-metadata-docs,try-runtime,experimental --manifest-path ./frame/support/test/Cargo.toml
- - SUBSTRATE_TEST_TIMEOUT=1 time cargo test -p substrate-test-utils --release --verbose --locked -- --ignored timeout
- - cat /cargo_target_dir/debug/.fingerprint/memory_units-759eddf317490d2b/lib-memory_units.json || true
- - rusty-cachier cache upload
-
-# This job runs tests that don't work with cargo-nextest in test-linux-stable
-test-linux-stable-extra:
- stage: test
- extends:
- - .docker-env
- - .test-refs
- variables:
- # Enable debug assertions since we are running optimized builds for testing
- # but still want to have debug assertions.
- RUSTFLAGS: "-C debug-assertions -D warnings"
- RUST_BACKTRACE: 1
- WASM_BUILD_NO_COLOR: 1
- WASM_BUILD_RUSTFLAGS: "-C debug-assertions -D warnings"
- # Ensure we run the UI tests.
- RUN_UI_TESTS: 1
- script:
- - rusty-cachier snapshot create
- # Run node-cli tests
- # TODO: add to test-linux-stable-nextest after fix https://github.com/paritytech/substrate/issues/11321
- - time cargo test node-cli --workspace --locked --release --verbose --features runtime-benchmarks --manifest-path ./bin/node/cli/Cargo.toml
- # Run doctests
- # TODO: add to test-linux-stable-nextest after fix https://github.com/nextest-rs/nextest/issues/16
- - time cargo test --doc --workspace --locked --release --verbose --features runtime-benchmarks --manifest-path ./bin/node/cli/Cargo.toml
- - rusty-cachier cache upload
-
-# This job runs all benchmarks defined in the `/bin/node/runtime` once to check that there are no errors.
-quick-benchmarks:
- stage: test
- extends:
- - .docker-env
- - .test-refs
- variables:
- # Enable debug assertions since we are running optimized builds for testing
- # but still want to have debug assertions.
- RUSTFLAGS: "-C debug-assertions -D warnings"
- RUST_BACKTRACE: "full"
- WASM_BUILD_NO_COLOR: 1
- WASM_BUILD_RUSTFLAGS: "-C debug-assertions -D warnings"
- script:
- - rusty-cachier snapshot create
- - time cargo run --locked --release -p node-cli --features runtime-benchmarks -- benchmark pallet --wasm-execution compiled --chain dev --pallet "*" --extrinsic "*" --steps 2 --repeat 1
- - rusty-cachier cache upload
-
-test-frame-examples-compile-to-wasm:
- # into one job
- stage: test
- extends:
- - .docker-env
- - .test-refs
- variables:
- # Enable debug assertions since we are running optimized builds for testing
- # but still want to have debug assertions.
- RUSTFLAGS: "-C debug-assertions"
- RUST_BACKTRACE: 1
- script:
- - rusty-cachier snapshot create
- - cd ./frame/examples/offchain-worker/
- - cargo build --locked --target=wasm32-unknown-unknown --no-default-features
- - cd ../basic
- - cargo build --locked --target=wasm32-unknown-unknown --no-default-features
- - rusty-cachier cache upload
-
-test-linux-stable-int:
- stage: test
- extends:
- - .docker-env
- - .test-refs
- - .pipeline-stopper-artifacts
- variables:
- # Enable debug assertions since we are running optimized builds for testing
- # but still want to have debug assertions.
- RUSTFLAGS: "-C debug-assertions -D warnings"
- RUST_BACKTRACE: 1
- WASM_BUILD_NO_COLOR: 1
- WASM_BUILD_RUSTFLAGS: "-C debug-assertions -D warnings"
- # Ensure we run the UI tests.
- RUN_UI_TESTS: 1
- script:
- - rusty-cachier snapshot create
- - WASM_BUILD_NO_COLOR=1
- RUST_LOG=sync=trace,consensus=trace,client=trace,state-db=trace,db=trace,forks=trace,state_db=trace,storage_cache=trace
- time cargo test -p node-cli --release --verbose --locked -- --ignored
- - rusty-cachier cache upload
-
-# more information about this job can be found here:
-# https://github.com/paritytech/substrate/pull/6916
-check-tracing:
- stage: test
- # this is an artificial job dependency, for pipeline optimization using GitLab's DAGs
- needs:
- - job: test-linux-stable-int
- artifacts: false
- extends:
- - .docker-env
- - .test-refs
- - .pipeline-stopper-artifacts
- script:
- - rusty-cachier snapshot create
- # with-tracing must be explicitly activated, we run a test to ensure this works as expected in both cases
- - time cargo test --locked --manifest-path ./primitives/tracing/Cargo.toml --no-default-features
- - time cargo test --locked --manifest-path ./primitives/tracing/Cargo.toml --no-default-features --features=with-tracing
- - rusty-cachier cache upload
-
-# more information about this job can be found here:
-# https://github.com/paritytech/substrate/pull/3778
-test-full-crypto-feature:
- stage: test
- # this is an artificial job dependency, for pipeline optimization using GitLab's DAGs
- needs:
- - job: check-tracing
- artifacts: false
- extends:
- - .docker-env
- - .test-refs
- variables:
- # Enable debug assertions since we are running optimized builds for testing
- # but still want to have debug assertions.
- RUSTFLAGS: "-C debug-assertions"
- RUST_BACKTRACE: 1
- script:
- - rusty-cachier snapshot create
- - cd primitives/core/
- - time cargo build --locked --verbose --no-default-features --features full_crypto
- - cd ../application-crypto
- - time cargo build --locked --verbose --no-default-features --features full_crypto
- - rusty-cachier cache upload
-
-check-rustdoc:
- stage: test
- extends:
- - .docker-env
- - .test-refs
- variables:
- SKIP_WASM_BUILD: 1
- RUSTDOCFLAGS: "-Dwarnings"
- script:
- - rusty-cachier snapshot create
- - time cargo doc --locked --workspace --all-features --verbose --no-deps
- - rusty-cachier cache upload
-
-cargo-check-each-crate:
- stage: test
- extends:
- - .docker-env
- - .test-refs
- - .collect-artifacts
- - .pipeline-stopper-artifacts
- variables:
- # $CI_JOB_NAME is set manually so that rusty-cachier can share the cache for all
- # "cargo-check-each-crate I/N" jobs
- CI_JOB_NAME: cargo-check-each-crate
- script:
- - rusty-cachier snapshot create
- - PYTHONUNBUFFERED=x time ./scripts/ci/gitlab/check-each-crate.py "$CI_NODE_INDEX" "$CI_NODE_TOTAL"
- # need to update cache only from one job
- - if [ "$CI_NODE_INDEX" == 1 ]; then rusty-cachier cache upload; fi
- parallel: 2
-
-cargo-check-each-crate-macos:
- stage: test
- extends:
- - .test-refs
- - .collect-artifacts
- - .pipeline-stopper-artifacts
- before_script:
- # skip timestamp script, the osx bash doesn't support printf %()T
- - !reference [.job-switcher, before_script]
- - !reference [.rust-info-script, script]
- - !reference [.pipeline-stopper-vars, script]
- variables:
- SKIP_WASM_BUILD: 1
- script:
- # TODO: enable rusty-cachier once it supports Mac
- # TODO: use parallel jobs, as per cargo-check-each-crate, once more Mac runners are available
- # - time ./scripts/ci/gitlab/check-each-crate.py 1 1
- - time cargo check --workspace --locked
- tags:
- - osx
-
-cargo-hfuzz:
- stage: test
- extends:
- - .docker-env
- - .test-refs
- - .pipeline-stopper-artifacts
- variables:
- # max 10s per iteration, 60s per file
- HFUZZ_RUN_ARGS: >
- --exit_upon_crash
- --exit_code_upon_crash 1
- --timeout 10
- --run_time 60
- # use git version of honggfuzz-rs until v0.5.56 is out, we need a few recent changes:
- # https://github.com/rust-fuzz/honggfuzz-rs/pull/75 to avoid breakage on debian
- # https://github.com/rust-fuzz/honggfuzz-rs/pull/81 fix to the above pr
- # https://github.com/rust-fuzz/honggfuzz-rs/pull/82 fix for handling rusty-cachier's absolute CARGO_TARGET_DIR
- HFUZZ_BUILD_ARGS: >
- --config=patch.crates-io.honggfuzz.git="https://github.com/altaua/honggfuzz-rs"
- --config=patch.crates-io.honggfuzz.rev="205f7c8c059a0d98fe1cb912cdac84f324cb6981"
- artifacts:
- name: "hfuzz-$CI_COMMIT_SHORT_SHA"
- expire_in: 7 days
- when: on_failure
- paths:
- - primitives/arithmetic/fuzzer/hfuzz_workspace/
- script:
- - cd ./primitives/arithmetic/fuzzer
- - rusty-cachier snapshot create
- - cargo hfuzz build
- - rusty-cachier cache upload
- - for target in $(cargo read-manifest | jq -r '.targets | .[] | .name'); do
- cargo hfuzz run "$target" || { printf "fuzzing failure for %s\n" "$target"; exit 1; }; done
diff --git a/substrate/scripts/ci/gitlab/pipeline/zombienet.yml b/substrate/scripts/ci/gitlab/pipeline/zombienet.yml
deleted file mode 100644
index 31ee510343278..0000000000000
--- a/substrate/scripts/ci/gitlab/pipeline/zombienet.yml
+++ /dev/null
@@ -1,67 +0,0 @@
-# This file is part of .gitlab-ci.yml
-# Here are all jobs that are executed during "zombienet" stage
-
-# common settings for all zombienet jobs
-.zombienet-common:
- before_script:
- - echo "Zombie-net Tests Config"
- - echo "${ZOMBIENET_IMAGE}"
- - echo "${SUBSTRATE_IMAGE_NAME} ${SUBSTRATE_IMAGE_TAG}"
- - echo "${GH_DIR}"
- - export DEBUG=zombie,zombie::network-node
- - export ZOMBIENET_INTEGRATION_TEST_IMAGE=${SUBSTRATE_IMAGE_NAME}:${SUBSTRATE_IMAGE_TAG}
- - echo "${ZOMBIENET_INTEGRATION_TEST_IMAGE}"
- stage: zombienet
- image: "${ZOMBIENET_IMAGE}"
- needs:
- - job: publish-docker-substrate-temporary
- extends:
- - .kubernetes-env
- - .zombienet-refs
- variables:
- GH_DIR: "https://github.com/paritytech/substrate/tree/${CI_COMMIT_SHA}/zombienet"
- FF_DISABLE_UMASK_FOR_DOCKER_EXECUTOR: 1
- artifacts:
- name: "${CI_JOB_NAME}_${CI_COMMIT_REF_NAME}"
- when: always
- expire_in: 2 days
- paths:
- - ./zombienet-logs
- after_script:
- - mkdir -p ./zombienet-logs
- - cp /tmp/zombie*/logs/* ./zombienet-logs/
- retry: 2
- tags:
- - zombienet-polkadot-integration-test
-
-zombienet-0000-block-building:
- extends:
- - .zombienet-common
- script:
- - /home/nonroot/zombie-net/scripts/ci/run-test-env-manager.sh
- --github-remote-dir="${GH_DIR}/0000-block-building"
- --test="block-building.zndsl"
-
-zombienet-0001-basic-warp-sync:
- extends:
- - .zombienet-common
- script:
- - /home/nonroot/zombie-net/scripts/ci/run-test-env-manager.sh
- --github-remote-dir="${GH_DIR}/0001-basic-warp-sync"
- --test="test-warp-sync.zndsl"
-
-zombienet-0002-validators-warp-sync:
- extends:
- - .zombienet-common
- script:
- - /home/nonroot/zombie-net/scripts/ci/run-test-env-manager.sh
- --github-remote-dir="${GH_DIR}/0002-validators-warp-sync"
- --test="test-validators-warp-sync.zndsl"
-
-zombienet-0003-block-building-warp-sync:
- extends:
- - .zombienet-common
- script:
- - /home/nonroot/zombie-net/scripts/ci/run-test-env-manager.sh
- --github-remote-dir="${GH_DIR}/0003-block-building-warp-sync"
- --test="test-block-building-warp-sync.zndsl"
diff --git a/substrate/scripts/ci/gitlab/prettier.sh b/substrate/scripts/ci/gitlab/prettier.sh
deleted file mode 100755
index 299bbee179dca..0000000000000
--- a/substrate/scripts/ci/gitlab/prettier.sh
+++ /dev/null
@@ -1,6 +0,0 @@
-#!/bin/sh
-
-# meant to be installed via
-# git config filter.ci-prettier.clean "scripts/ci/gitlab/prettier.sh"
-
-prettier --parser yaml
diff --git a/substrate/scripts/ci/gitlab/publish_draft_release.sh b/substrate/scripts/ci/gitlab/publish_draft_release.sh
deleted file mode 100755
index 88d1de0e04fe3..0000000000000
--- a/substrate/scripts/ci/gitlab/publish_draft_release.sh
+++ /dev/null
@@ -1,54 +0,0 @@
-#!/usr/bin/env bash
-
-# shellcheck source=../common/lib.sh
-source "$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )/../common/lib.sh"
-
-version="$CI_COMMIT_TAG"
-
-# Note that this is not the last *tagged* version, but the last *published* version
-last_version=$(last_github_release 'paritytech/substrate')
-
-release_text="$(./generate_release_text.sh "$last_version" "$version")"
-
-echo "[+] Pushing release to github"
-# Create release on github
-release_name="Substrate $version"
-data=$(jq -Rs --arg version "$version" \
- --arg release_name "$release_name" \
- --arg release_text "$release_text" \
-'{
- "tag_name": $version,
- "target_commitish": "master",
- "name": $release_name,
- "body": $release_text,
- "draft": true,
- "prerelease": false
-}' < /dev/null)
-
-out=$(curl -s -X POST --data "$data" -H "Authorization: token $GITHUB_RELEASE_TOKEN" "$api_base/paritytech/substrate/releases")
-
-html_url=$(echo "$out" | jq -r .html_url)
-
-if [ "$html_url" == "null" ]
-then
- echo "[!] Something went wrong posting:"
- echo "$out"
-else
- echo "[+] Release draft created: $html_url"
-fi
-
-echo '[+] Sending draft release URL to Matrix'
-
-msg_body=$(cat <Release pipeline for Substrate $version complete.
-Draft release created: $html_url
-EOF
-)
-send_message "$(structure_message "$msg_body" "$formatted_msg_body")" "!aJymqQYtCjjqImFLSb:parity.io" "$RELEASENOTES_MATRIX_V2_ACCESS_TOKEN"
-
-echo "[+] Done! Maybe the release worked..."