Zombienet add tests (#1321)

* changes to read json spec in test binary

* add zombienet tests

* fmt

* use {{COL_IMAGE}} and clean config

* add comment and use relay image from env

* use test-parachain image from pr

* fix warns

* fix warns

* fmt

* typo

* fix ci to use zombienet image

* fix spawn nodes for test

* reorg test

* add within to test

* remove check for full node collators is up

* add tests for pov, mirate solo to para, sync blocks

* bump zombienet image

* add job dep with artifacts

* add sleep for test

* fix after merge

* fmt

* bump zombienet version

* changes from clap

* use base/shared params

* fmt

* debug ci

* add upgrade test

* update js test for debug

* less debug in test

* print assertion

* fix upgrade test

* Collator key only needed if we run as collator

* [Fix] Benchmark build artifact folder creation (#1518)

* Trivial networking changes for Substrate PR #11940 (#1486)

* Trivial networking changes for Substrate PR paritytech/substrate#11940

* Apply formatting rules

* update lockfile for {"polkadot", "substrate"}

Co-authored-by: parity-processbot <>

* bump zombienet version

* update network def for test

* typo

Co-authored-by: Sebastian Kunert <[email protected]>
Co-authored-by: Roman Useinov <[email protected]>
Co-authored-by: Nazar Mokrynskyi <[email protected]>

.gitlab-ci.yml

@@ -11,6 +11,7 @@ stages:
   - benchmarks-build
   - benchmarks-run
   - publish
+  - integration-test
 
 default:
   interruptible:                   true
@@ -28,6 +29,7 @@ variables:
   CI_IMAGE:                        "paritytech/ci-linux:production"
   DOCKER_OS:                       "debian:stretch"
   ARCH:                            "x86_64"
+  ZOMBIENET_IMAGE:                 "docker.io/paritytech/zombienet:v1.2.53"
 
 .collect-artifacts:                &collect-artifacts
   artifacts:
@@ -84,6 +86,15 @@ variables:
   rules:
     - if: $CI_COMMIT_REF_NAME =~ /^release-parachains-v[0-9].*$/              # i.e. release-parachains-v1.0, release-parachains-v2.1rc1, release-parachains-v3000
 
+.zombienet-refs:                   &zombienet-refs
+  rules:
+    - if: $CI_PIPELINE_SOURCE == "pipeline"
+      when: never
+    - if: $CI_PIPELINE_SOURCE == "schedule"
+      when: never
+    - if: $CI_COMMIT_REF_NAME == "master"
+    - if: $CI_COMMIT_REF_NAME =~ /^[0-9]+$/                         # PRs
+
 .docker-env:                       &docker-env
   image:                           "${CI_IMAGE}"
   before_script:
@@ -212,6 +223,8 @@ build-test-parachain:
     - echo "___Packing the artifacts___"
     - mkdir -p ./artifacts
     - mv ./target/release/test-parachain ./artifacts/.
+    - mkdir -p ./artifacts/zombienet
+    - mv ./target/release/wbuild/cumulus-test-runtime/wasm_binary_spec_version_incremented.rs.compact.compressed.wasm ./artifacts/zombienet/.
 
 #### stage:                        publish
 
@@ -395,6 +408,161 @@ update-parachain-template:
       --github-api-token "$GITHUB_TOKEN"
       --polkadot-branch "$CI_COMMIT_REF_NAME"
 
+#### stage:                        integration-test
+
+zombienet-0001-sync_blocks_from_tip_without_connected_collator:
+  stage:                           integration-test
+  image:                           "${ZOMBIENET_IMAGE}"
+  <<:                              *zombienet-refs
+  needs:
+    - job:                         build-push-image-test-parachain
+  variables:
+    POLKADOT_IMAGE:                "docker.io/paritypr/polkadot-debug:master"
+    GH_DIR:                        "https://github.com/paritytech/cumulus/tree/${CI_COMMIT_SHORT_SHA}/zombienet_tests"
+    COL_IMAGE:                     "docker.io/paritypr/test-parachain:${CI_COMMIT_REF_NAME}-${CI_COMMIT_SHORT_SHA}"
+  before_script:
+    - echo "Zombie-net Tests Config"
+    - echo "${ZOMBIENET_IMAGE}"
+    - echo "${RELAY_IMAGE}"
+    - echo "${COL_IMAGE}"
+    - echo "${GH_DIR}"
+    - export DEBUG=zombie
+    - export RELAY_IMAGE=${POLKADOT_IMAGE}
+    - export COL_IMAGE=${COL_IMAGE}
+  script:
+    - /home/nonroot/zombie-net/scripts/ci/run-test-env-manager.sh
+        --github-remote-dir="${GH_DIR}"
+        --concurrency=1
+        --test="0001-sync_blocks_from_tip_without_connected_collator.feature"
+  allow_failure:                   true
+  retry: 2
+  tags:
+    - zombienet-polkadot-integration-test
+
+zombienet-0002-pov_recovery:
+  stage:                           integration-test
+  image:                           "${ZOMBIENET_IMAGE}"
+  <<:                              *zombienet-refs
+  needs:
+    - job:                         build-push-image-test-parachain
+  variables:
+    POLKADOT_IMAGE:                "docker.io/paritypr/polkadot-debug:master"
+    GH_DIR:                        "https://github.com/paritytech/cumulus/tree/${CI_COMMIT_SHORT_SHA}/zombienet_tests"
+    COL_IMAGE:                     "docker.io/paritypr/test-parachain:${CI_COMMIT_REF_NAME}-${CI_COMMIT_SHORT_SHA}"
+  before_script:
+    - echo "Zombie-net Tests Config"
+    - echo "${ZOMBIENET_IMAGE}"
+    - echo "${RELAY_IMAGE}"
+    - echo "${COL_IMAGE}"
+    - echo "${GH_DIR}"
+    - export DEBUG=zombie
+    - export RELAY_IMAGE=${POLKADOT_IMAGE}
+    - export COL_IMAGE=${COL_IMAGE}
+  script:
+    - /home/nonroot/zombie-net/scripts/ci/run-test-env-manager.sh
+        --github-remote-dir="${GH_DIR}"
+        --concurrency=1
+        --test="0002-pov_recovery.feature"
+  allow_failure:                   true
+  retry: 2
+  tags:
+    - zombienet-polkadot-integration-test
+
+zombienet-0003-full_node_catching_up:
+  stage:                           integration-test
+  image:                           "${ZOMBIENET_IMAGE}"
+  <<:                              *zombienet-refs
+  needs:
+    - job:                         build-push-image-test-parachain
+  variables:
+    POLKADOT_IMAGE:                "docker.io/paritypr/polkadot-debug:master"
+    GH_DIR:                        "https://github.com/paritytech/cumulus/tree/${CI_COMMIT_SHORT_SHA}/zombienet_tests"
+    COL_IMAGE:                     "docker.io/paritypr/test-parachain:${CI_COMMIT_REF_NAME}-${CI_COMMIT_SHORT_SHA}"
+  before_script:
+    - echo "Zombie-net Tests Config"
+    - echo "${ZOMBIENET_IMAGE}"
+    - echo "${RELAY_IMAGE}"
+    - echo "${COL_IMAGE}"
+    - echo "${GH_DIR}"
+    - export DEBUG=zombie
+    - export RELAY_IMAGE=${POLKADOT_IMAGE}
+    - export COL_IMAGE=${COL_IMAGE}
+  script:
+    - /home/nonroot/zombie-net/scripts/ci/run-test-env-manager.sh
+        --github-remote-dir="${GH_DIR}"
+        --concurrency=1
+        --test="0003-full_node_catching_up.feature"
+  allow_failure:                   true
+  retry: 2
+  tags:
+    - zombienet-polkadot-integration-test
+
+zombienet-0004-runtime_upgrade:
+  stage:                           integration-test
+  image:                           "${ZOMBIENET_IMAGE}"
+  <<:                              *zombienet-refs
+  needs:
+    - job:                         build-push-image-test-parachain
+    - job:                         build-test-parachain
+      artifacts:                   true
+  variables:
+    POLKADOT_IMAGE:                "docker.io/paritypr/polkadot-debug:master"
+    GH_DIR:                        "https://github.com/paritytech/cumulus/tree/${CI_COMMIT_SHORT_SHA}/zombienet_tests"
+    COL_IMAGE:                     "docker.io/paritypr/test-parachain:${CI_COMMIT_REF_NAME}-${CI_COMMIT_SHORT_SHA}"
+  before_script:
+    - ls -ltr *
+    - cp ./artifacts/zombienet/wasm_binary_spec_version_incremented.rs.compact.compressed.wasm /tmp/
+    - ls /tmp
+    - echo "Zombie-net Tests Config"
+    - echo "${ZOMBIENET_IMAGE}"
+    - echo "${RELAY_IMAGE}"
+    - echo "${COL_IMAGE}"
+    - echo "${GH_DIR}"
+    - export DEBUG=zombie
+    - export RELAY_IMAGE=${POLKADOT_IMAGE}
+    - export COL_IMAGE=${COL_IMAGE}
+  script:
+    - /home/nonroot/zombie-net/scripts/ci/run-test-env-manager.sh
+        --github-remote-dir="${GH_DIR}"
+        --concurrency=1
+        --test="0004-runtime_upgrade.feature"
+  allow_failure:                   true
+  retry: 2
+  tags:
+    - zombienet-polkadot-integration-test
+
+zombienet-0005-migrate_solo_to_para:
+  stage:                           integration-test
+  image:                           "${ZOMBIENET_IMAGE}"
+  <<:                              *zombienet-refs
+  needs:
+    - job:                         build-push-image-test-parachain
+    - job:                         build-test-parachain
+      artifacts:                   true
+  variables:
+    POLKADOT_IMAGE:                "docker.io/paritypr/polkadot-debug:master"
+    GH_DIR:                        "https://github.com/paritytech/cumulus/tree/${CI_COMMIT_SHORT_SHA}/zombienet_tests"
+    COL_IMAGE:                     "docker.io/paritypr/test-parachain:${CI_COMMIT_REF_NAME}-${CI_COMMIT_SHORT_SHA}"
+  before_script:
+    - ls -ltr *
+    - echo "Zombie-net Tests Config"
+    - echo "${ZOMBIENET_IMAGE}"
+    - echo "${RELAY_IMAGE}"
+    - echo "${COL_IMAGE}"
+    - echo "${GH_DIR}"
+    - export DEBUG=zombie
+    - export RELAY_IMAGE=${POLKADOT_IMAGE}
+    - export COL_IMAGE=${COL_IMAGE}
+  script:
+    - /home/nonroot/zombie-net/scripts/ci/run-test-env-manager.sh
+        --github-remote-dir="${GH_DIR}"
+        --concurrency=1
+        --test="0005-migrate_solo_to_para.feature"
+  allow_failure:                   true
+  retry: 2
+  tags:
+    - zombienet-polkadot-integration-test
+
 #### stage:                        .post
 
 # This job cancels the whole pipeline if any of provided jobs fail.

test/service/src/cli.rs

@@ -288,9 +288,16 @@ impl SubstrateCli for TestCollatorCli {
 		2017
 	}
 
-	fn load_spec(&self, _: &str) -> std::result::Result<Box<dyn sc_service::ChainSpec>, String> {
-		Ok(Box::new(cumulus_test_service::get_chain_spec(ParaId::from(self.parachain_id)))
-			as Box<_>)
+	fn load_spec(&self, id: &str) -> std::result::Result<Box<dyn sc_service::ChainSpec>, String> {
+		Ok(match id {
+			"" => Box::new(cumulus_test_service::get_chain_spec(ParaId::from(self.parachain_id)))
+				as Box<_>,
+			path => {
+				let chain_spec =
+					cumulus_test_service::chain_spec::ChainSpec::from_json_file(path.into())?;
+				Box::new(chain_spec)
+			},
+		})
 	}
 
 	fn native_runtime_version(_: &Box<dyn ChainSpec>) -> &'static RuntimeVersion {

test/service/src/lib.rs

@@ -18,7 +18,7 @@
 
 #![warn(missing_docs)]
 
-mod chain_spec;
+pub mod chain_spec;
 mod genesis;
 
 use std::{

test/service/src/main.rs

@@ -16,7 +16,7 @@
 
 mod cli;
 
-use std::sync::Arc;
+use std::{io::Write, sync::Arc};
 
 use cli::{RelayChainCli, Subcommand, TestCollatorCli};
 use cumulus_client_cli::generate_genesis_block;
@@ -43,14 +43,31 @@ fn main() -> Result<(), sc_cli::Error> {
 			let runner = cli.create_runner(cmd)?;
 			runner.sync_run(|config| cmd.run(config.chain_spec, config.network))
 		},
-		Some(Subcommand::ExportGenesisState(cmd)) => {
-			let runner = cli.create_runner(cmd)?;
-			runner.sync_run(|_config| {
-				let parachain_id = ParaId::from(cmd.parachain_id);
-				let spec = cumulus_test_service::get_chain_spec(parachain_id);
-				let state_version = cumulus_test_service::runtime::VERSION.state_version();
-				cmd.base.run::<parachains_common::Block>(&spec, state_version)
-			})
+
+		Some(Subcommand::ExportGenesisState(params)) => {
+			let mut builder = sc_cli::LoggerBuilder::new("");
+			builder.with_profiling(sc_tracing::TracingReceiver::Log, "");
+			let _ = builder.init();
+
+			let spec =
+				cli.load_spec(&params.base.shared_params.chain.clone().unwrap_or_default())?;
+			let state_version = cumulus_test_service::runtime::VERSION.state_version();
+
+			let block: parachains_common::Block = generate_genesis_block(&*spec, state_version)?;
+			let raw_header = block.header().encode();
+			let output_buf = if params.base.raw {
+				raw_header
+			} else {
+				format!("0x{:?}", HexDisplay::from(&block.header().encode())).into_bytes()
+			};
+
+			if let Some(output) = &params.base.output {
+				std::fs::write(output, output_buf)?;
+			} else {
+				std::io::stdout().write_all(&output_buf)?;
+			}
+
+			Ok(())
 		},
 		Some(Subcommand::ExportGenesisWasm(cmd)) => {
 			let runner = cli.create_runner(cmd)?;
@@ -106,7 +123,7 @@ fn main() -> Result<(), sc_cli::Error> {
 				if config.role.is_authority() { "yes" } else { "no" }
 			);
 
-			let collator_key = Some(CollatorPair::generate().0);
+			let collator_key = config.role.is_authority().then(|| CollatorPair::generate().0);
 
 			let consensus = cli
 				.use_null_consensus

zombienet_tests/0001-sync_blocks_from_tip_without_connected_collator.feature

@@ -1,4 +1,4 @@
-Description: Small Network test
+Description: Sync blocks from tip without connected collator test
 Network: ./0001-sync_blocks_from_tip_without_connected_collator.toml
 Creds: config
 
@@ -7,11 +7,9 @@ alice: is up
 bob: is up
 charlie: is up
 dave: is up
-ferdie: is up
-eve: is up
 
 alice: parachain 2000 is registered within 225 seconds
 alice: parachain 2000 block height is at least 10 within 250 seconds
 
 ferdie: reports block height is at least 12 within 250 seconds
-eve: reports block height is at least 12 within 250 seconds
+eve: reports block height is at least 12 within 250 seconds

zombienet_tests/0001-sync_blocks_from_tip_without_connected_collator.toml

@@ -1,5 +1,5 @@
 [relaychain]
-default_image = "docker.io/paritypr/polkadot-debug:master"
+default_image = "{{RELAY_IMAGE}}"
 default_command = "polkadot"
 default_args = [ "-lparachain=debug" ]
 
@@ -11,9 +11,7 @@ chain = "rococo-local"
 
   [[relaychain.nodes]]
   name = "bob"
-  image = "docker.io/paritypr/polkadot-debug:5236-0.9.18-c55660e9-be16bd72"
   validator = true
-  args = ["--database=paritydb-experimental"]
 
 [[parachains]]
 id = 2000
@@ -23,30 +21,30 @@ cumulus_based = true
   [[parachains.collators]]
   name = "charlie"
   validator = true
-  image = "docker.io/parity/polkadot-collator:latest"
-  command = "test-collator"
+  image = "{{COL_IMAGE}}"
+  command = "test-parachain"
   args = ["-lparachain=debug"]
 
   # run dave as parachain full node
   [[parachains.collators]]
   name = "dave"
   validator = false
-  image = "docker.io/parity/polkadot-collator:latest"
-  command = "test-collator"
+  image = "{{COL_IMAGE}}"
+  command = "test-parachain"
   args = ["-lparachain=debug"]
 
   # run eve as parachain full node that is only connected to dave
   [[parachains.collators]]
   name = "eve"
   validator = false
-  image = "docker.io/parity/polkadot-collator:latest"
-  command = "test-collator"
-  args = ["--reserved-only", "--reserved-nodes {{'dave'|zombie}}"]
+  image = "{{COL_IMAGE}}"
+  command = "test-parachain"
+  args = ["--reserved-only", "--reserved-nodes {{'dave'|zombie('multiAddress')}}"]
 
   # run ferdie as parachain full node that is only connected to dave
   [[parachains.collators]]
   name = "ferdie"
   validator = false
-  image = "docker.io/parity/polkadot-collator:latest"
-  command = "test-collator"
-  args = ["--reserved-only", "--reserved-nodes {{'dave'|zombie}}"]
+  image = "{{COL_IMAGE}}"
+  command = "test-parachain"
+  args = ["--reserved-only", "--reserved-nodes {{'dave'|zombie('multiAddress')}}", "--relay-chain-rpc-url {{'alice'|zombie('wsUri')}}"]

zombienet_tests/0002-pov_recovery.feature

@@ -1,4 +1,4 @@
-Description: Small Network test
+Description: PoV recovery test
 Network: ./0002-pov_recovery.toml
 Creds: config