Merge branch 'rh-async-backing-feature' into rh-async-backing-prediction

.github/CODEOWNERS

@@ -1,7 +1,6 @@
-bridges/ @tomusdrw @svyatonik @adoerr @acatangiu @antonio-dropulic
+bridges/ @tomusdrw @svyatonik @acatangiu
 
 # CI
 /.github/ @paritytech/ci @chevdor
-/scripts/github/ @paritytech/ci @chevdor
-/scripts/gitlab/ @paritytech/ci @chevdor
+/scripts/ci/ @paritytech/ci @chevdor
 /.gitlab-ci.yml @paritytech/ci

.github/workflows/check-labels.yml

@@ -15,7 +15,7 @@ jobs:
           ref: ${{ github.event.pull_request.head.ref }}
           repository: ${{ github.event.pull_request.head.repo.full_name }}
       - name: Check labels
-        run: bash ${{ github.workspace }}/scripts/github/check_labels.sh
+        run: bash ${{ github.workspace }}/scripts/ci/github/check_labels.sh
         env:
           GITHUB_PR: ${{ github.event.pull_request.number }}
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

.github/workflows/extrinsic-ordering-check-from-bin.yml

@@ -62,7 +62,7 @@ jobs:
           $CMD >> output.txt
           sed -z -i 's/\n\n/\n/g' output.txt
           cat output.txt | egrep -n -i ''
-          SUMMARY=$(./scripts/github/extrinsic-ordering-filter.sh output.txt)
+          SUMMARY=$(./scripts/ci/github/extrinsic-ordering-filter.sh output.txt)
           echo -e $SUMMARY
           echo -e $SUMMARY >> output.txt
 

.github/workflows/honggfuzz.yml

@@ -42,7 +42,7 @@ jobs:
 
       - name: Run fuzzer
         working-directory: xcm/xcm-simulator/fuzzer/
-        run: bash $GITHUB_WORKSPACE/scripts/github/run_fuzzer.sh xcm-fuzzer
+        run: bash $GITHUB_WORKSPACE/scripts/ci/github/run_fuzzer.sh xcm-fuzzer
 
   erasure-coding-round-trip:
     runs-on: ubuntu-latest
@@ -88,7 +88,7 @@ jobs:
 
       - name: Run fuzzer
         working-directory: erasure-coding/fuzzer
-        run: bash $GITHUB_WORKSPACE/scripts/github/run_fuzzer.sh round_trip
+        run: bash $GITHUB_WORKSPACE/scripts/ci/github/run_fuzzer.sh round_trip
 
   erasure-coding-reconstruct:
     runs-on: ubuntu-latest
@@ -134,4 +134,4 @@ jobs:
 
       - name: Run fuzzer
         working-directory: erasure-coding/fuzzer
-        run: bash $GITHUB_WORKSPACE/scripts/github/run_fuzzer.sh reconstruct
+        run: bash $GITHUB_WORKSPACE/scripts/ci/github/run_fuzzer.sh reconstruct

.github/workflows/pr-custom-review.yml

@@ -11,12 +11,31 @@ on:
       - synchronize
       - review_requested
       - review_request_removed
+      - ready_for_review
+      - converted_to_draft
   pull_request_review:
 
 jobs:
   pr-custom-review:
     runs-on: ubuntu-latest
     steps:
+      - name: Skip if pull request is in Draft
+        # `if: github.event.pull_request.draft == true` should be kept here, at
+        # the step level, rather than at the job level. The latter is not
+        # recommended because when the PR is moved from "Draft" to "Ready to
+        # review" the workflow will immediately be passing (since it was skipped),
+        # even though it hasn't actually ran, since it takes a few seconds for
+        # the workflow to start. This is also disclosed in:
+        # https://github.community/t/dont-run-actions-on-draft-pull-requests/16817/17
+        # That scenario would open an opportunity for the check to be bypassed:
+        # 1. Get your PR approved
+        # 2. Move it to Draft
+        # 3. Push whatever commits you want
+        # 4. Move it to "Ready for review"; now the workflow is passing (it was
+        #    skipped) and "Check reviews" is also passing (it won't be updated
+        #    until the workflow is finished)
+        if: github.event.pull_request.draft == true
+        run: exit 1
       - name: pr-custom-review
         uses: paritytech/pr-custom-review@v2
         with:

.github/workflows/publish-docker-manual.yml

@@ -37,7 +37,7 @@ jobs:
         uses: docker/build-push-action@v2
         with:
           push: true
-          file: scripts/dockerfiles/polkadot_injected_release.Dockerfile
+          file: scripts/ci/dockerfiles/polkadot_injected_release.Dockerfile
           tags: |
             parity/polkadot:latest
             parity/polkadot:${{ github.event.inputs.version }}

.github/workflows/publish-docker-release.yml

@@ -30,7 +30,7 @@ jobs:
         uses: docker/build-push-action@v2
         with:
           push: true
-          file: scripts/dockerfiles/polkadot_injected_release.Dockerfile
+          file: scripts/ci/dockerfiles/polkadot_injected_release.Dockerfile
           tags: |
             parity/polkadot:latest
             parity/polkadot:${{ github.event.release.tag_name }}

.github/workflows/publish-draft-release.yml

@@ -86,7 +86,7 @@ jobs:
 
       - name: Prepare tooling
         run: |
-          cd polkadot/scripts/changelog
+          cd polkadot/scripts/ci/changelog
           gem install bundler changelogerator:0.9.1
           bundle install
           changelogerator --help
@@ -115,7 +115,7 @@ jobs:
           ls -al $KUSAMA_DIGEST
           ls -al $POLKADOT_DIGEST
 
-          cd polkadot/scripts/changelog
+          cd polkadot/scripts/ci/changelog
 
           ./bin/changelog ${GITHUB_REF}
           ls -al release-notes.md
@@ -126,7 +126,7 @@ jobs:
         with:
           name: release-notes-context
           path: |
-            polkadot/scripts/changelog/context.json
+            polkadot/scripts/ci/changelog/context.json
             **/*_srtool_output.json
 
       - name: Create draft release
@@ -137,7 +137,7 @@ jobs:
         with:
           tag_name: ${{ github.ref }}
           release_name: Polkadot ${{ github.ref }}
-          body_path: ./polkadot/scripts/changelog/release-notes.md
+          body_path: ./polkadot/scripts/ci/changelog/release-notes.md
           draft: true
 
   publish-runtimes:
@@ -160,7 +160,7 @@ jobs:
       - name: Get runtime version
         id: get-runtime-ver
         run: |
-          echo "require './scripts/github/lib.rb'" > script.rb
+          echo "require './scripts/ci/github/lib.rb'" > script.rb
           echo "puts get_runtime(runtime: \"${{ matrix.runtime }}\", runtime_dir: \"$RUNTIME_DIR\")" >> script.rb
 
           echo "Current folder: $PWD"

.gitlab-ci.yml

@@ -204,7 +204,7 @@ check-runtime:
     GITLAB_API:                    "https://gitlab.parity.io/api/v4"
     GITHUB_API_PROJECT:            "parity%2Finfrastructure%2Fgithub-api"
   script:
-    - ./scripts/gitlab/check_runtime.sh
+    - ./scripts/ci/gitlab/check_runtime.sh
   allow_failure:                   true
 
 cargo-fmt:
@@ -273,7 +273,7 @@ spellcheck:
     - git fetch origin +${CI_DEFAULT_BRANCH}:${CI_DEFAULT_BRANCH}
     - echo "___Spellcheck is going to check your diff___"
     - cargo spellcheck list-files -vvv $(git diff --diff-filter=AM --name-only $(git merge-base ${CI_COMMIT_SHA} ${CI_DEFAULT_BRANCH} -- :^bridges))
-    - time cargo spellcheck check -vvv --cfg=scripts/gitlab/spellcheck.toml --checkers hunspell --code 1
+    - time cargo spellcheck check -vvv --cfg=scripts/ci/gitlab/spellcheck.toml --checkers hunspell --code 1
         $(git diff --diff-filter=AM --name-only $(git merge-base ${CI_COMMIT_SHA} ${CI_DEFAULT_BRANCH} -- :^bridges))
   allow_failure:                   true
 
@@ -295,7 +295,7 @@ build-test-collators:
     - echo -n "${CI_COMMIT_REF_NAME}-${CI_COMMIT_SHORT_SHA}" > ./artifacts/EXTRATAG
     - echo "adder-collator version = $(cat ./artifacts/VERSION) (EXTRATAG = $(cat ./artifacts/EXTRATAG))"
     - echo "undying-collator version = $(cat ./artifacts/VERSION) (EXTRATAG = $(cat ./artifacts/EXTRATAG))"
-    - cp -r scripts/* ./artifacts
+    - cp -r ./scripts/* ./artifacts
 
 build-malus:
   stage:                           stage1
@@ -312,7 +312,7 @@ build-malus:
     - echo -n "${CI_COMMIT_REF_NAME}" > ./artifacts/VERSION
     - echo -n "${CI_COMMIT_REF_NAME}-${CI_COMMIT_SHORT_SHA}" > ./artifacts/EXTRATAG
     - echo "polkadot-test-malus = $(cat ./artifacts/VERSION) (EXTRATAG = $(cat ./artifacts/EXTRATAG))"
-    - cp -r scripts/* ./artifacts
+    - cp -r ./scripts/* ./artifacts
 
 #### stage:                        stage2
 
@@ -330,18 +330,16 @@ build-malus:
         "--branch=$PIPELINE_SCRIPTS_TAG"
         https://github.com/paritytech/pipeline-scripts
     - ./pipeline-scripts/check_dependent_project.sh
-        paritytech
-        polkadot
-        --polkadot
-        "$DEPENDENT_REPO"
-        "$GITHUB_PR_TOKEN"
-        "$CARGO_UPDATE_CRATES"
+        --org paritytech
+        --dependent-repo "$DEPENDENT_REPO"
+        --github-api-token "$GITHUB_PR_TOKEN"
+        --extra-dependencies "$EXTRA_DEPENDENCIES"
 
 check-dependent-cumulus:
   <<: *check-dependent-project
   variables:
     DEPENDENT_REPO: cumulus
-    CARGO_UPDATE_CRATES: "sp-io polkadot-runtime-common"
+    EXTRA_DEPENDENCIES: substrate
 
 test-node-metrics:
   stage:                           stage2
@@ -370,7 +368,7 @@ test-deterministic-wasm:
   <<:                              *docker-env
   <<:                              *compiler-info
   script:
-    - ./scripts/gitlab/test_deterministic_wasm.sh
+    - ./scripts/ci/gitlab/test_deterministic_wasm.sh
 
 check-transaction-versions:
   stage:                           stage2
@@ -387,7 +385,7 @@ check-transaction-versions:
     - git config remote.origin.url "https://github.com/paritytech/polkadot.git"
     - git fetch origin release
   script:
-    - scripts/gitlab/check_extrinsics_ordering.sh
+    - ./scripts/ci/gitlab/check_extrinsics_ordering.sh
 
 
 
@@ -408,8 +406,8 @@ publish-polkadot-debug-image:
     - if: $CI_COMMIT_REF_NAME =~ /^v[0-9]+\.[0-9]+.*$/              # i.e. v1.0, v2.1rc1
   variables:
     <<:                            *image-variables
-    # scripts/dockerfiles/polkadot_injected_debug.Dockerfile
-    DOCKERFILE:                    dockerfiles/polkadot_injected_debug.Dockerfile
+    # scripts/ci/dockerfiles/polkadot_injected_debug.Dockerfile
+    DOCKERFILE:                    ci/dockerfiles/polkadot_injected_debug.Dockerfile
     IMAGE_NAME:                    docker.io/paritypr/polkadot-debug
   needs:
     - job:                         build-linux-stable
@@ -432,8 +430,8 @@ publish-test-collators-image:
   <<:                              *zombienet-refs
   variables:
     <<:                            *image-variables
-    # scripts/dockerfiles/collator_injected.Dockerfile
-    DOCKERFILE:                    dockerfiles/collator_injected.Dockerfile
+    # scripts/ci/dockerfiles/collator_injected.Dockerfile
+    DOCKERFILE:                    ci/dockerfiles/collator_injected.Dockerfile
     IMAGE_NAME:                    docker.io/paritypr/colander
   needs:
     - job:                         build-test-collators
@@ -455,8 +453,8 @@ publish-malus-image:
   <<:                              *zombienet-refs
   variables:
     <<:                            *image-variables
-    # scripts/dockerfiles/malus_injected.Dockerfile
-    DOCKERFILE:                    dockerfiles/malus_injected.Dockerfile
+    # scripts/ci/dockerfiles/malus_injected.Dockerfile
+    DOCKERFILE:                    ci/dockerfiles/malus_injected.Dockerfile
     IMAGE_NAME:                    docker.io/paritypr/malus
   needs:
     - job:                         build-malus
@@ -521,7 +519,7 @@ update_polkadot_weights:           &update-weights
     paths:
       - ${RUNTIME}_weights_${CI_COMMIT_SHORT_SHA}.patch
   script:
-    - ./scripts/run_benches_for_runtime.sh $RUNTIME
+    - ./scripts/ci/run_benches_for_runtime.sh $RUNTIME
     - git diff -P > ${RUNTIME}_weights_${CI_COMMIT_SHORT_SHA}.patch
   # uses the "shell" executors
   tags:
@@ -582,7 +580,7 @@ generate-impl-guide:
     name: michaelfbryan/mdbook-docker-image:v0.4.4
     entrypoint: [""]
   script:
-    - mdbook build roadmap/implementers-guide
+    - mdbook build ./roadmap/implementers-guide
 
 check-try-runtime:
   stage:                           stage3
@@ -609,8 +607,8 @@ check-no-default-features:
   <<:                              *compiler-info
   script:
     # Check that polkadot-cli will compile no default features.
-    - pushd node/service && cargo check --no-default-features && popd
-    - pushd cli && cargo check --no-default-features --features "service" && popd
+    - pushd ./node/service && cargo check --no-default-features && popd
+    - pushd ./cli && cargo check --no-default-features --features "service" && popd
     - sccache -s
 
 build-short-benchmark:
@@ -792,7 +790,7 @@ publish-rustdoc:
     - git fetch origin gh-pages
     # Save README and docs
     - cp -r ./crate-docs/ /tmp/doc/
-    - cp README.md /tmp/doc/
+    - cp ./README.md /tmp/doc/
     - git checkout gh-pages
     # Remove everything and restore generated docs and README
     - rm -rf ./*