Move scripts used in CI to the new location (#11008)

Move scripts used in CI to the new location - **./scripts/ci/**

* Move github scripts

* Move more files

* Move ci scripts and fix dependencies

* Update docs/node-template-release.md

Co-authored-by: João Paulo Silva de Souza <[email protected]>

* Remove Cargo.lock

* Apply suggestions from code review

Co-authored-by: Denis Pisarev <[email protected]>

* Make more paths uniform

Co-authored-by: João Paulo Silva de Souza <[email protected]>
Co-authored-by: Denis Pisarev <[email protected]>

.github/workflows/check-labels.yml

@@ -14,7 +14,7 @@ jobs:
           ref: ${{ github.event.pull_request.head.ref }}
           repository: ${{ github.event.pull_request.head.repo.full_name }}
       - name: Check labels
-        run: bash ${{ github.workspace }}/.maintain/github/check_labels.sh
+        run: bash ${{ github.workspace }}/scripts/ci/github/check_labels.sh
         env:
           GITHUB_PR: ${{ github.event.pull_request.number }}
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

.github/workflows/monthly-tag.yml

@@ -29,7 +29,7 @@ jobs:
           echo "" >> Changelog.md
           echo "## Changes since last snapshot (${{ steps.tags.outputs.old }})" >> Changelog.md
           echo "" >> Changelog.md
-          ./.maintain/gitlab/generate_changelog.sh ${{ steps.tags.outputs.old }} >>  Changelog.md
+          ./scripts/ci/github/generate_changelog.sh ${{ steps.tags.outputs.old }} >>  Changelog.md
       - name: Release snapshot
         id: release-snapshot
         uses: actions/create-release@latest

.gitignore

@@ -24,4 +24,4 @@ rls*.log
 .cargo-remote.toml
 *.bin
 *.iml
-.maintain/node-template-release/Cargo.lock
+scripts/ci/node-template-release/Cargo.lock

.gitlab-ci.yml

@@ -18,7 +18,7 @@
 #   script:
 #     - echo "List of shell commands to run in your job"
 #     - echo "You can also just specify a script here, like so:"
-#     - ./.maintain/gitlab/my_amazing_script.sh
+#     - ./scripts/ci/gitlab/my_amazing_script.sh
 
 stages:
   - check
@@ -165,12 +165,12 @@ default:
     fi
 
 .cargo-check-benches-script:       &cargo-check-benches-script
-  - mkdir -p artifacts/benches/$CI_COMMIT_REF_NAME-$CI_COMMIT_SHORT_SHA
+  - mkdir -p ./artifacts/benches/$CI_COMMIT_REF_NAME-$CI_COMMIT_SHORT_SHA
   - SKIP_WASM_BUILD=1 time cargo +nightly check --benches --all
   - 'cargo run --release -p node-bench -- ::node::import::native::sr25519::transfer_keep_alive::paritydb::small --json
-    | tee artifacts/benches/$CI_COMMIT_REF_NAME-$CI_COMMIT_SHORT_SHA/::node::import::native::sr25519::transfer_keep_alive::paritydb::small.json'
+    | tee ./artifacts/benches/$CI_COMMIT_REF_NAME-$CI_COMMIT_SHORT_SHA/::node::import::native::sr25519::transfer_keep_alive::paritydb::small.json'
   - 'cargo run --release -p node-bench -- ::trie::read::small --json
-    | tee artifacts/benches/$CI_COMMIT_REF_NAME-$CI_COMMIT_SHORT_SHA/::trie::read::small.json'
+    | tee ./artifacts/benches/$CI_COMMIT_REF_NAME-$CI_COMMIT_SHORT_SHA/::trie::read::small.json'
   - sccache -s
 
 .build-linux-substrate-script:     &build-linux-substrate-script
@@ -185,7 +185,7 @@ default:
           tee ./artifacts/substrate/VERSION;
     fi
   - sha256sum ./artifacts/substrate/substrate | tee ./artifacts/substrate/substrate.sha256
-  - cp -r .maintain/docker/substrate.Dockerfile ./artifacts/substrate/
+  - cp -r ./scripts/ci/docker/substrate.Dockerfile ./artifacts/substrate/
   - sccache -s
 
 #### Vault secrets
@@ -241,7 +241,7 @@ skip-if-draft:
      - echo "Commit message is ${CI_COMMIT_MESSAGE}"
      - echo "Ref is ${CI_COMMIT_REF_NAME}"
      - echo "pipeline source is ${CI_PIPELINE_SOURCE}"
-     - ./.maintain/gitlab/skip_if_draft.sh
+     - ./scripts/ci/gitlab/skip_if_draft.sh
 
 #### stage:                        check
 
@@ -256,7 +256,7 @@ check-runtime:
     GITLAB_API:                    "https://gitlab.parity.io/api/v4"
     GITHUB_API_PROJECT:            "parity%2Finfrastructure%2Fgithub-api"
   script:
-    - ./.maintain/gitlab/check_runtime.sh
+    - ./scripts/ci/gitlab/check_runtime.sh
   allow_failure:                   true
 
 check-signed-tag:
@@ -267,7 +267,7 @@ check-signed-tag:
     - if: $CI_COMMIT_REF_NAME =~ /^ci-release-.*$/
     - if: $CI_COMMIT_REF_NAME =~ /^v[0-9]+\.[0-9]+.*$/              # i.e. v1.0, v2.1rc1
   script:
-    - ./.maintain/gitlab/check_signed.sh
+    - ./scripts/ci/gitlab/check_signed.sh
 
 test-dependency-rules:
   stage:                           check
@@ -276,7 +276,7 @@ test-dependency-rules:
   rules:
     - if: $CI_COMMIT_REF_NAME =~ /^[0-9]+$/                         # PRs
   script:
-    - .maintain/ensure-deps.sh
+    - ./scripts/ci/gitlab/ensure-deps.sh
 
 test-prometheus-alerting-rules:
   stage:                           check
@@ -288,11 +288,11 @@ test-prometheus-alerting-rules:
     - if: $CI_COMMIT_BRANCH
       changes:
         - .gitlab-ci.yml
-        - .maintain/monitoring/**/*
+        - ./scripts/ci/monitoring/**/*
   script:
-    - promtool check rules .maintain/monitoring/alerting-rules/alerting-rules.yaml
-    - cat .maintain/monitoring/alerting-rules/alerting-rules.yaml |
-        promtool test rules .maintain/monitoring/alerting-rules/alerting-rule-tests.yaml
+    - promtool check rules ./scripts/ci/monitoring/alerting-rules/alerting-rules.yaml
+    - cat ./scripts/ci/monitoring/alerting-rules/alerting-rules.yaml |
+        promtool test rules ./scripts/ci/monitoring/alerting-rules/alerting-rule-tests.yaml
 
 #### stage:                        test
 
@@ -301,10 +301,10 @@ cargo-deny:
   <<:                              *docker-env
   <<:                              *nightly-pipeline
   script:
-    - cargo deny check --hide-inclusion-graph -c .maintain/deny.toml
+    - cargo deny check --hide-inclusion-graph -c ./scripts/ci/deny.toml
   after_script:
     - echo "___The complete log is in the artifacts___"
-    - cargo deny check -c .maintain/deny.toml 2> deny.log
+    - cargo deny check -c ./scripts/ci/deny.toml 2> deny.log
   artifacts:
     name:                          $CI_COMMIT_SHORT_SHA
     expire_in:                     3 days
@@ -404,13 +404,13 @@ test-deterministic-wasm:
     # build runtime
     - cargo build --verbose --release -p node-runtime
     # make checksum
-    - sha256sum target/release/wbuild/node-runtime/target/wasm32-unknown-unknown/release/node_runtime.wasm > checksum.sha256
+    - sha256sum ./target/release/wbuild/node-runtime/target/wasm32-unknown-unknown/release/node_runtime.wasm > checksum.sha256
     # clean up – FIXME: can we reuse some of the artifacts?
     - cargo clean
     # build again
     - cargo build --verbose --release -p node-runtime
     # confirm checksum
-    - sha256sum -c checksum.sha256
+    - sha256sum -c ./checksum.sha256
     - sccache -s
 
 test-linux-stable:                 &test-linux
@@ -426,8 +426,8 @@ test-linux-stable:                 &test-linux
     WASM_BUILD_NO_COLOR:           1
   script:
     # this job runs all tests in former runtime-benchmarks, frame-staking and wasmtime tests
-    - time cargo test --workspace --locked --release --verbose --features runtime-benchmarks --manifest-path bin/node/cli/Cargo.toml
-    - time cargo test -p frame-support-test --features=conditional-storage,no-metadata-docs --manifest-path frame/support/test/Cargo.toml --test pallet # does not reuse cache 1 min 44 sec
+    - time cargo test --workspace --locked --release --verbose --features runtime-benchmarks --manifest-path ./bin/node/cli/Cargo.toml
+    - time cargo test -p frame-support-test --features=conditional-storage,no-metadata-docs --manifest-path ./frame/support/test/Cargo.toml --test pallet # does not reuse cache 1 min 44 sec
     - SUBSTRATE_TEST_TIMEOUT=1 time cargo test -p substrate-test-utils --release --verbose --locked -- --ignored timeout
     - sccache -s
 
@@ -443,7 +443,7 @@ test-frame-examples-compile-to-wasm:
     RUSTFLAGS:                     "-Cdebug-assertions=y"
     RUST_BACKTRACE: 1
   script:
-    - cd frame/examples/offchain-worker/
+    - cd ./frame/examples/offchain-worker/
     - cargo +nightly build --target=wasm32-unknown-unknown --no-default-features
     - cd ../basic
     - cargo +nightly build --target=wasm32-unknown-unknown --no-default-features
@@ -475,8 +475,8 @@ check-tracing:
   <<:                              *test-refs
   script:
     # with-tracing must be explicitly activated, we run a test to ensure this works as expected in both cases
-    - time cargo +nightly test --manifest-path primitives/tracing/Cargo.toml --no-default-features
-    - time cargo +nightly test --manifest-path primitives/tracing/Cargo.toml --no-default-features --features=with-tracing
+    - time cargo +nightly test --manifest-path ./primitives/tracing/Cargo.toml --no-default-features
+    - time cargo +nightly test --manifest-path ./primitives/tracing/Cargo.toml --no-default-features --features=with-tracing
     - sccache -s
 
 test-full-crypto-feature:
@@ -568,7 +568,7 @@ build-linux-substrate:
   script:
     - *build-linux-substrate-script
     - printf '\n# building node-template\n\n'
-    - ./.maintain/node-template-release.sh ./artifacts/substrate/substrate-node-template.tar.gz
+    - ./scripts/ci/node-template-release.sh ./artifacts/substrate/substrate-node-template.tar.gz
 
 build-linux-subkey:                &build-subkey
   stage:                           build
@@ -590,7 +590,7 @@ build-linux-subkey:                &build-subkey
         sed -n -E 's/^subkey ([0-9.]+.*)/\1/p' |
           tee ./artifacts/subkey/VERSION;
     - sha256sum ./artifacts/subkey/subkey | tee ./artifacts/subkey/subkey.sha256
-    - cp -r .maintain/docker/subkey.Dockerfile ./artifacts/subkey/
+    - cp -r ./scripts/ci/docker/subkey.Dockerfile ./artifacts/subkey/
     - sccache -s
 
 build-macos-subkey:
@@ -784,7 +784,7 @@ publish-draft-release:
     - if: $CI_COMMIT_REF_NAME =~ /^ci-release-.*$/
     - if: $CI_COMMIT_REF_NAME =~ /^v[0-9]+\.[0-9]+.*$/              # i.e. v1.0, v2.1rc1
   script:
-    - ./.maintain/gitlab/publish_draft_release.sh
+    - ./scripts/ci/gitlab/publish_draft_release.sh
   allow_failure:                   true
 
 #### stage:                        deploy
@@ -807,4 +807,4 @@ deploy-prometheus-alerting-rules:
     - if: $CI_COMMIT_REF_NAME == "master"
       changes:
         - .gitlab-ci.yml
-        - .maintain/monitoring/**/*
+        - ./scripts/ci/monitoring/**/*

docs/CODEOWNERS

@@ -19,7 +19,7 @@
 # - The latest matching rule, if multiple, takes precedence.
 
 # CI
-/.maintain/ @paritytech/ci
+/scripts/ci/ @paritytech/ci
 /.github/ @paritytech/ci
 /.gitlab-ci.yml @paritytech/ci
 

docs/node-template-release.md

@@ -7,7 +7,7 @@ the existence of your current git commit ID in the remote repository.
 	Assume you are in root directory of Substrate. Run:
 
 	```bash
-	cd .maintain/
+	cd scripts/ci/
 	./node-template-release.sh <output tar.gz file>
 	```
 
@@ -50,7 +50,7 @@ commit in Substrate remote repository, such as:
 	```
 
 	P.S: This step can be automated if we update `node-template-release` package in
-	`.maintain/node-template-release`.
+	`scripts/ci/node-template-release`.
 
 4. Once the three `Cargo.toml`s are updated, compile and confirm that the Node Template builds. Then
 commit the changes to a new branch in [Substrate Node Template](https://github.com/substrate-developer-hub/substrate-node-template), and make a PR.

.maintain/docker/subkey.Dockerfile → scripts/ci/docker/subkey.Dockerfile

@@ -8,7 +8,7 @@ LABEL io.parity.image.authors="[email protected]" \
 	io.parity.image.vendor="Parity Technologies" \
 	io.parity.image.title="parity/subkey" \
 	io.parity.image.description="Subkey: key generating utility for Substrate." \
-	io.parity.image.source="https://github.com/paritytech/substrate/blob/${VCS_REF}/.maintain/docker/subkey.Dockerfile" \
+	io.parity.image.source="https://github.com/paritytech/substrate/blob/${VCS_REF}/scripts/ci/docker/subkey.Dockerfile" \
 	io.parity.image.revision="${VCS_REF}" \
 	io.parity.image.created="${BUILD_DATE}" \
 	io.parity.image.documentation="https://github.com/paritytech/substrate/tree/${VCS_REF}/subkey"

.maintain/docker/substrate.Dockerfile → scripts/ci/docker/substrate.Dockerfile

@@ -8,7 +8,7 @@ LABEL io.parity.image.authors="[email protected]" \
 	io.parity.image.vendor="Parity Technologies" \
 	io.parity.image.title="parity/substrate" \
 	io.parity.image.description="Substrate: The platform for blockchain innovators." \
-	io.parity.image.source="https://github.com/paritytech/substrate/blob/${VCS_REF}/.maintain/docker/Dockerfile" \
+	io.parity.image.source="https://github.com/paritytech/substrate/blob/${VCS_REF}/scripts/ci/docker/Dockerfile" \
 	io.parity.image.revision="${VCS_REF}" \
 	io.parity.image.created="${BUILD_DATE}" \
 	io.parity.image.documentation="https://wiki.parity.io/Parity-Substrate"

.maintain/node-template-release.sh → scripts/ci/node-template-release.sh

@@ -11,6 +11,6 @@ if [ "$#" -ne 1 ]; then
 fi
 
 PATH_TO_ARCHIVE=$1
-cd $PROJECT_ROOT/.maintain/node-template-release
+cd $PROJECT_ROOT/scripts/ci/node-template-release
 
 cargo run $PROJECT_ROOT/bin/node-template $PROJECT_ROOT/$PATH_TO_ARCHIVE