455 create integration tests for stellar mainnet with archived proofs (…

…#456)

* Refactor integration test helpers

* Continue refactoring

* Change function signature

* Fix compilation

* Create mainnet_config in chain_spec.rs

* Remove unused specs

* Adjust provider client helper

* Update default genesis of stellar relay pallet

* Amend refactoring

* Add new constant for wrapped currency

* Choose wrapped currency based on network

* Split runtime into `mainnet` and `testnet`

* Amend refactoring

* WIP

* More WIP

* WIP

* More WIP

* Revert changes and try again with included RPC

* WIP still compiles

* Duplicate code to start instant seal node

* Fix merge issues

* Rename genesis and impl name

* Rename `spec_name` and `impl_name`
These have to match what we have in the custom subxt client
otherwise the integration tests will not start properly

* Bump versions

* Update versions in overlay config

* Amend version bump

* Extract `impl_name` to extra variable

* Do integration tests for test and mainnet

* Fix small issue in oracle agent

* Fix issue about wrong runtime being executed

* Small refactoring

* Duplicate test for replace

* Remove redundant helper function

* Update overlay config files

* Create test for issue execution from archive

* Update testnet config files as well

* Change sleep duration

* Fix creation of correct `TransactionSetType` from archived entry

* Small refactoring

* Small refactoring of tests

* Fix potential overflow

* Increase default Stellar fee value from `p90` to `p95`

* Increase `TIMEOUT` for vault integration tests

* Fix clippy warning

* Use linear backoff delay

* Change DEFAULT_MAX_BACKOFF_DELAY_IN_SECS and remove restriction

Cargo.toml

@@ -19,10 +19,10 @@ members = [
     "pallets/stellar-relay",
     "pallets/vault-registry",
     "primitives",
-    "rpc",
     "pallets/*/rpc",
     "testchain/node",
-    "testchain/runtime",
+    "testchain/runtime/mainnet",
+    "testchain/runtime/testnet",
 ]
 [profile.release]
 panic = "unwind"

clients/runtime/Cargo.toml

@@ -57,7 +57,7 @@ oracle = { path = "../../pallets/oracle", optional = true }
 rand = { version = "0.7", optional = true }
 tempdir = { version = "0.3.7", optional = true }
 testchain = { package = "spacewalk-standalone", path = "../../testchain/node", optional = true }
-testchain-runtime = { package = "spacewalk-runtime-standalone", path = "../../testchain/runtime", optional = true }
+testchain-runtime = { package = "spacewalk-runtime-standalone-testnet", path = "../../testchain/runtime/testnet", optional = true }
 
 # Substrate Stellar Dependencies
 substrate-stellar-sdk = { git = "https://github.com/pendulum-chain/substrate-stellar-sdk", branch = "polkadot-v0.9.40" }
@@ -68,7 +68,7 @@ wallet = { path = "../wallet" }
 env_logger = "0.8.3"
 tempdir = "0.3.7"
 runtime = { path = ".", features = ["testing-utils"] }
-testchain-runtime = { package = "spacewalk-runtime-standalone", path = "../../testchain/runtime", features = ["testing-utils"] }
+testchain-runtime = { package = "spacewalk-runtime-standalone-testnet", path = "../../testchain/runtime/testnet", features = ["testing-utils"] }
 
 
 

clients/runtime/src/integration/mod.rs

@@ -57,16 +57,33 @@ impl XCMCurrencyConversion for MockValue {
 /// Start a new instance of the parachain. The second item in the returned tuple must remain in
 /// scope as long as the parachain is active, since dropping it will remove the temporary directory
 /// that the parachain uses
-pub async fn default_provider_client(key: AccountKeyring) -> (SubxtClient, TempDir) {
+pub async fn default_provider_client(
+	key: AccountKeyring,
+	is_public_network: bool,
+) -> (SubxtClient, TempDir) {
+	let chain_spec = if is_public_network {
+		testchain::chain_spec::mainnet_config()
+	} else {
+		testchain::chain_spec::testnet_config()
+	};
+
+	// This does not necessarily have to match the `impl_name` of the runtime. We give it a
+	// descriptive name because it will be shown when starting the testing client.
+	let impl_name = if is_public_network {
+		"spacewalk-standalone-mainnet"
+	} else {
+		"spacewalk-standalone-testnet"
+	};
+
 	let tmp = TempDir::new("spacewalk-parachain-").expect("failed to create tempdir");
 	let config = SubxtClientConfig {
-		impl_name: "spacewalk-parachain-full-client",
+		impl_name,
 		impl_version: "0.0.1",
 		author: "SatoshiPay",
 		copyright_start_year: 2020,
 		db: DatabaseSource::ParityDb { path: tmp.path().join("db") },
 		keystore: KeystoreConfig::Path { path: tmp.path().join("keystore"), password: None },
-		chain_spec: testchain::chain_spec::development_config(),
+		chain_spec,
 		role: Role::Authority(key),
 		telemetry: None,
 		wasm_method: WasmExecutionMethod::Compiled {
@@ -79,8 +96,15 @@ pub async fn default_provider_client(key: AccountKeyring) -> (SubxtClient, TempD
 	let mut service_config = config.into_service_config();
 	service_config.offchain_worker.enabled = true;
 
-	let (task_manager, rpc_handlers) =
-		testchain::service::start_instant(service_config).await.unwrap();
+	let (task_manager, rpc_handlers) = if is_public_network {
+		testchain::service::start_instant_mainnet(service_config)
+			.await
+			.expect("Couldn't start mainnet")
+	} else {
+		testchain::service::start_instant_testnet(service_config)
+			.await
+			.expect("Couldn't start testnet")
+	};
 
 	let client = SubxtClient::new(task_manager, rpc_handlers);
 

clients/runtime/src/rpc.rs

@@ -39,6 +39,7 @@ compile_error!("Tests are only supported for the standalone-metadata");
 cfg_if::cfg_if! {
 	if #[cfg(feature = "standalone-metadata")] {
 		const DEFAULT_SPEC_VERSION: Range<u32> = 1..100;
+		// This has to match the `spec_name` in the runtime, otherwise the runtime will be rejected.
 		pub const DEFAULT_SPEC_NAME: &str = "spacewalk-standalone";
 		// The prefix for the testchain is 42
 		pub const SS58_PREFIX: u16 = 42;

clients/runtime/src/tests.rs

@@ -39,7 +39,9 @@ async fn set_exchange_rate(client: SubxtClient) {
 
 #[tokio::test(flavor = "multi_thread")]
 async fn test_getters() {
-	let (client, _tmp_dir) = default_provider_client(AccountKeyring::Alice).await;
+	let is_public_network = false;
+	let (client, _tmp_dir) =
+		default_provider_client(AccountKeyring::Alice, is_public_network).await;
 	let parachain_rpc = setup_provider(client.clone(), AccountKeyring::Alice).await;
 
 	tokio::join!(
@@ -62,7 +64,9 @@ async fn test_getters() {
 #[ignore]
 #[tokio::test(flavor = "multi_thread")]
 async fn test_invalid_tx_matching() {
-	let (client, _tmp_dir) = default_provider_client(AccountKeyring::Alice).await;
+	let is_public_network = false;
+	let (client, _tmp_dir) =
+		default_provider_client(AccountKeyring::Alice, is_public_network).await;
 	let parachain_rpc = setup_provider(client.clone(), AccountKeyring::Alice).await;
 	let err = parachain_rpc.get_invalid_tx_error(AccountKeyring::Bob.into()).await;
 	assert!(err.is_invalid_transaction().is_some())
@@ -71,15 +75,19 @@ async fn test_invalid_tx_matching() {
 #[ignore]
 #[tokio::test(flavor = "multi_thread")]
 async fn test_too_low_priority_matching() {
-	let (client, _tmp_dir) = default_provider_client(AccountKeyring::Alice).await;
+	let is_public_network = false;
+	let (client, _tmp_dir) =
+		default_provider_client(AccountKeyring::Alice, is_public_network).await;
 	let parachain_rpc = setup_provider(client.clone(), AccountKeyring::Alice).await;
 	let err = parachain_rpc.get_too_low_priority_error(AccountKeyring::Bob.into()).await;
 	assert!(err.is_pool_too_low_priority())
 }
 
 #[tokio::test(flavor = "multi_thread")]
 async fn test_subxt_processing_events_after_dispatch_error() {
-	let (client, _tmp_dir) = default_provider_client(AccountKeyring::Alice).await;
+	let is_public_network = false;
+	let (client, _tmp_dir) =
+		default_provider_client(AccountKeyring::Alice, is_public_network).await;
 
 	let oracle_provider = setup_provider(client.clone(), AccountKeyring::Bob).await;
 	let invalid_oracle = setup_provider(client, AccountKeyring::Dave).await;
@@ -100,7 +108,9 @@ async fn test_subxt_processing_events_after_dispatch_error() {
 
 #[tokio::test(flavor = "multi_thread")]
 async fn test_register_vault() {
-	let (client, _tmp_dir) = default_provider_client(AccountKeyring::Alice).await;
+	let is_public_network = false;
+	let (client, _tmp_dir) =
+		default_provider_client(AccountKeyring::Alice, is_public_network).await;
 	let parachain_rpc = setup_provider(client.clone(), AccountKeyring::Alice).await;
 	set_exchange_rate(client.clone()).await;
 

clients/stellar-relay-lib/resources/config/mainnet/stellar_relay_config_mainnet_iowa.json

@@ -5,14 +5,12 @@
   },
   "node_info": {
     "ledger_version": 20,
-    "overlay_version": 30,
+    "overlay_version": 32,
     "overlay_min_version": 27,
-    "version_str": "stellar-core 20.0.2 (669916b56106a72a7f79ab8b4a7898e77b28b49e)",
+    "version_str": "stellar-core 20.2.0 (34d82fc00426643e16b7ad59c9fde169b778eb4b)",
     "is_pub_net": true
   },
   "stellar_history_archive_urls": [
-    "https://stellar-history-us-iowa.satoshipay.io",
-    "https://stellar-history-de-fra.satoshipay.io",
-    "https://stellar-history-sg-sin.satoshipay.io"
+    "http://history.stellar.org/prd/core-live/core_live_001"
   ]
 }

clients/stellar-relay-lib/src/connection/connector/message_reader.rs

@@ -114,7 +114,7 @@ async fn read_message_from_stellar(connector: &mut Connector) -> Result<Xdr, Err
 			},
 			Ok(size) => {
 				// The next few bytes was read. Add it to the readbuf.
-				lack_bytes_from_prev -= size;
+				lack_bytes_from_prev = lack_bytes_from_prev.saturating_sub(size);
 				readbuf.append(&mut buff_for_reading);
 				// make sure to cleanup the buffer
 				buff_for_reading = vec![0; 4];

clients/stellar-relay-lib/src/tests/mod.rs

@@ -144,6 +144,6 @@ async fn stellar_overlay_disconnect_works() {
 	overlay_connection.stop();
 
 	// let the disconnection call pass for a few seconds, before checking its status.
-	sleep(Duration::from_secs(3));
+	sleep(Duration::from_secs(15));
 	assert!(!overlay_connection.is_alive());
 }

clients/vault/resources/config/mainnet/stellar_relay_config_frankfurt.json

@@ -5,9 +5,9 @@
   },
   "node_info": {
     "ledger_version": 20,
-    "overlay_version": 30,
+    "overlay_version": 32,
     "overlay_min_version": 27,
-    "version_str": "stellar-core 20.0.2 (669916b56106a72a7f79ab8b4a7898e77b28b49e)",
+    "version_str": "stellar-core 20.2.0 (34d82fc00426643e16b7ad59c9fde169b778eb4b)",
     "is_pub_net": true
   },
   "stellar_history_archive_urls": [

clients/vault/resources/config/mainnet/stellar_relay_config_iowa.json

@@ -5,14 +5,12 @@
   },
   "node_info": {
     "ledger_version": 20,
-    "overlay_version": 30,
+    "overlay_version": 32,
     "overlay_min_version": 27,
-    "version_str": "stellar-core 20.0.2 (669916b56106a72a7f79ab8b4a7898e77b28b49e)",
+    "version_str": "stellar-core 20.2.0 (34d82fc00426643e16b7ad59c9fde169b778eb4b)",
     "is_pub_net": true
   },
   "stellar_history_archive_urls": [
-    "https://stellar-history-us-iowa.satoshipay.io",
-    "https://stellar-history-de-fra.satoshipay.io",
-    "https://stellar-history-sg-sin.satoshipay.io"
+    "http://history.stellar.org/prd/core-live/core_live_001"
   ]
 }

clients/vault/resources/config/mainnet/stellar_relay_config_singapore.json

@@ -5,9 +5,9 @@
   },
   "node_info": {
     "ledger_version": 20,
-    "overlay_version": 30,
+    "overlay_version": 32,
     "overlay_min_version": 27,
-    "version_str": "stellar-core 20.0.2 (669916b56106a72a7f79ab8b4a7898e77b28b49e)",
+    "version_str": "stellar-core 20.2.0 (34d82fc00426643e16b7ad59c9fde169b778eb4b)",
     "is_pub_net": true
   },
   "stellar_history_archive_urls": [