Add the code for compiling node-cli for WASM-browser (#3974)

* Extract CLI to separate module in node/cli

* Make node/cli compile for WASM

* More work on node/cli browser

* More work on browser node

* More work

* More work

* Purge a bit the CI script

* More clean up

* Remove substrate-finality-grandpa from the CI

Its tests use tokio, which fails to compile.

* Address review

* Add rocksdb feature to the service

* Fix substrate-service WASM CI

* Apply suggestions from code review

Co-Authored-By: Bastian Köcher <[email protected]>

* Don't WASM-compile substrate-service altogether

.gitlab-ci.yml

@@ -202,25 +202,13 @@ check-web-wasm:
     - time cargo web build -p sr-io
     - time cargo web build -p sr-primitives
     - time cargo web build -p sr-std
-    - time cargo web build -p substrate-chain-spec
     - time cargo web build -p substrate-client
     - time cargo web build -p substrate-consensus-aura
     - time cargo web build -p substrate-consensus-babe
     - time cargo web build -p substrate-consensus-common
-    - time cargo web build -p substrate-keyring
-    - time cargo web build -p substrate-keystore
-    - time cargo web build -p substrate-executor
-    - time cargo web build -p substrate-network
-    - time cargo web build -p substrate-offchain
-    - time cargo web build -p substrate-panic-handler
-    - time cargo web build -p substrate-peerset
-    - time cargo web build -p substrate-primitives
-    - time cargo web build -p substrate-rpc-servers
-    - time cargo web build -p substrate-serializer
-    - time cargo web build -p substrate-state-db
-    - time cargo web build -p substrate-state-machine
     - time cargo web build -p substrate-telemetry
-    - time cargo web build -p substrate-trie
+    # Note: the command below is a bit weird because several Cargo issues prevent us from compiling the node in a more straight-forward way.
+    - time cargo build --manifest-path=node/cli/Cargo.toml --no-default-features --features "browser" --target=wasm32-unknown-unknown
     - sccache -s
 
 node-exits:

core/cli/Cargo.toml

@@ -28,7 +28,7 @@ header-metadata = { package = "substrate-header-metadata", path = "../../core/cl
 network = { package = "substrate-network", path = "../../core/network" }
 sr-primitives = { path = "../../core/sr-primitives" }
 primitives = { package = "substrate-primitives", path = "../../core/primitives" }
-service = { package = "substrate-service", path = "../../core/service" }
+service = { package = "substrate-service", path = "../../core/service", default-features = false }
 state-machine = { package = "substrate-state-machine", path = "../../core/state-machine" }
 substrate-telemetry = { path = "../../core/telemetry" }
 keyring = { package = "substrate-keyring", path = "../keyring" }

core/service/Cargo.toml

@@ -4,6 +4,12 @@ version = "2.0.0"
 authors = ["Parity Technologies <[email protected]>"]
 edition = "2018"
 
+[features]
+default = ["rocksdb"]
+# The RocksDB feature activates the RocksDB database backend. If it is not activated, and you pass
+# a path to a database, an error will be produced at runtime.
+rocksdb = ["client_db/kvdb-rocksdb"]
+
 [dependencies]
 derive_more = "0.15.0"
 futures = "0.1.29"
@@ -29,7 +35,7 @@ consensus_common = { package = "substrate-consensus-common", path = "../../core/
 network = { package = "substrate-network", path = "../../core/network" }
 chain-spec = { package = "substrate-chain-spec", path = "../chain-spec" }
 client = { package = "substrate-client", path = "../../core/client" }
-client_db = { package = "substrate-client-db", path = "../../core/client/db", features = ["kvdb-rocksdb"] }
+client_db = { package = "substrate-client-db", path = "../../core/client/db" }
 codec = { package = "parity-scale-codec", version = "1.0.0" }
 substrate-executor = { path = "../../core/executor" }
 transaction_pool = { package = "substrate-transaction-pool", path = "../../core/transaction-pool" }

core/service/src/lib.rs

@@ -541,15 +541,16 @@ fn start_rpc_servers<C, G, E, H: FnMut() -> rpc_servers::RpcHandler<rpc::Metadat
 
 /// Starts RPC servers that run in their own thread, and returns an opaque object that keeps them alive.
 #[cfg(target_os = "unknown")]
-fn start_rpc_servers<C, G, E, H: FnMut() -> components::RpcHandler>(
+fn start_rpc_servers<C, G, E, H: FnMut() -> rpc_servers::RpcHandler<rpc::Metadata>>(
 	_: &Configuration<C, G, E>,
 	_: H
-) -> Result<Box<std::any::Any + Send + Sync>, error::Error> {
+) -> Result<Box<dyn std::any::Any + Send + Sync>, error::Error> {
 	Ok(Box::new(()))
 }
 
 /// An RPC session. Used to perform in-memory RPC queries (ie. RPC queries that don't go through
 /// the HTTP or WebSockets server).
+#[derive(Clone)]
 pub struct RpcSession {
 	metadata: rpc::Metadata,
 }

core/service/test/Cargo.toml

@@ -12,7 +12,7 @@ log = "0.4.8"
 env_logger = "0.7.0"
 fdlimit = "0.1.1"
 futures03 = { package = "futures-preview", version = "=0.3.0-alpha.19", features = ["compat"] }
-service = { package = "substrate-service", path = "../../../core/service" }
+service = { package = "substrate-service", path = "../../../core/service", default-features = false }
 network = { package = "substrate-network", path = "../../../core/network" }
 consensus = { package = "substrate-consensus-common", path = "../../../core/consensus/common" }
 client = { package = "substrate-client", path = "../../../core/client" }

node/cli/Cargo.toml

@@ -16,14 +16,15 @@ is-it-maintained-open-issues = { repository = "paritytech/substrate" }
 [[bin]]
 name = "substrate"
 path = "bin/main.rs"
+required-features = ["cli"]
+
+[lib]
+crate-type = ["cdylib", "rlib"]
 
 [dependencies]
 log = "0.4.8"
-tokio = "0.1.22"
 futures = "0.1.29"
-exit-future = "0.1.4"
 jsonrpc-core = "13.2.0"
-cli = { package = "substrate-cli", path = "../../core/cli" }
 codec = { package = "parity-scale-codec", version = "1.0.0" }
 sr-io = { path = "../../core/sr-io" }
 client = { package = "substrate-client", path = "../../core/client" }
@@ -35,7 +36,7 @@ node-primitives = { path = "../primitives" }
 hex-literal = "0.2.1"
 substrate-rpc = { package = "substrate-rpc", path = "../../core/rpc" }
 substrate-basic-authorship = { path = "../../core/basic-authorship" }
-substrate-service = { path = "../../core/service" }
+substrate-service = { path = "../../core/service", default-features = false }
 chain-spec = { package = "substrate-chain-spec", path = "../../core/chain-spec" }
 transaction_pool = { package = "substrate-transaction-pool", path = "../../core/transaction-pool" }
 network = { package = "substrate-network", path = "../../core/network" }
@@ -47,7 +48,6 @@ sr-primitives = { path = "../../core/sr-primitives" }
 node-executor = { path = "../executor" }
 substrate-telemetry = { package = "substrate-telemetry", path = "../../core/telemetry" }
 structopt = "0.3.3"
-transaction-factory = { path = "../../test-utils/transaction-factory" }
 keyring = { package = "substrate-keyring", path = "../../core/keyring" }
 indices = { package = "srml-indices", path = "../../srml/indices" }
 timestamp = { package = "srml-timestamp", path = "../../srml/timestamp", default-features = false }
@@ -60,9 +60,26 @@ transaction-payment = { package = "srml-transaction-payment", path = "../../srml
 support = { package = "srml-support", path = "../../srml/support", default-features = false }
 im_online = { package = "srml-im-online", path = "../../srml/im-online", default-features = false }
 serde = { version = "1.0.101", features = [ "derive" ] }
-client_db = { package = "substrate-client-db", path = "../../core/client/db", features = ["kvdb-rocksdb"] }
+client_db = { package = "substrate-client-db", path = "../../core/client/db", default-features = false }
 offchain = { package = "substrate-offchain", path = "../../core/offchain" }
-ctrlc = { version = "3.1.3", features = ["termination"] }
+
+# CLI-specific dependencies
+tokio = { version = "0.1.22", optional = true }
+exit-future = { version = "0.1.4", optional = true }
+substrate-cli = { path = "../../core/cli", optional = true }
+transaction-factory = { path = "../../test-utils/transaction-factory", optional = true }
+ctrlc = { version = "3.1.3", features = ["termination"], optional = true }
+
+# WASM-specific dependencies
+libp2p = { version = "0.12.0", default-features = false, optional = true }
+clear_on_drop = { version = "0.2.3", features = ["no_cc"], optional = true }	# Imported just for the `no_cc` feature
+console_error_panic_hook = { version = "0.1.1", optional = true }
+console_log = { version = "0.1.2", optional = true }
+js-sys = { version = "0.3.22", optional = true }
+wasm-bindgen = { version = "0.2.45", optional = true }
+wasm-bindgen-futures = { version = "0.3.22", optional = true }
+kvdb-memorydb = { git = "https://github.com/paritytech/parity-common", rev="b0317f649ab2c665b7987b8475878fc4d2e1f81d", optional = true }
+rand6 = { package = "rand", version = "0.6", features = ["wasm-bindgen"], optional = true }	# Imported just for the `wasm-bindgen` feature
 
 [dev-dependencies]
 keystore = { package = "substrate-keystore", path = "../../core/keystore" }
@@ -73,6 +90,29 @@ futures03 = { package = "futures-preview", version = "0.3.0-alpha.19" }
 tempfile = "3.1.0"
 
 [build-dependencies]
-cli = { package = "substrate-cli", path = "../../core/cli" }
+substrate-cli = { package = "substrate-cli", path = "../../core/cli" }
 structopt = "0.3.3"
 vergen = "3.0.4"
+
+[features]
+default = ["cli"]
+browser = [
+	"clear_on_drop",
+	"console_error_panic_hook",
+	"console_log",
+	"js-sys",
+	"libp2p",
+	"wasm-bindgen",
+	"wasm-bindgen-futures",
+	"kvdb-memorydb",
+	"rand/wasm-bindgen",
+	"rand6"
+]
+cli = [
+	"substrate-cli",
+	"transaction-factory",
+	"tokio",
+	"exit-future",
+	"ctrlc",
+	"substrate-service/rocksdb"
+]

node/cli/bin/main.rs

@@ -18,15 +18,15 @@
 
 #![warn(missing_docs)]
 
-use cli::VersionInfo;
 use futures::sync::oneshot;
 use futures::{future, Future};
+use substrate_cli::VersionInfo;
 
 use std::cell::RefCell;
 
 // handles ctrl-c
 struct Exit;
-impl cli::IntoExit for Exit {
+impl substrate_cli::IntoExit for Exit {
 	type Exit = future::MapErr<oneshot::Receiver<()>, fn(oneshot::Canceled) -> ()>;
 	fn into_exit(self) -> Self::Exit {
 		// can't use signal directly here because CtrlC takes only `Fn`.
@@ -43,7 +43,7 @@ impl cli::IntoExit for Exit {
 	}
 }
 
-fn main() -> Result<(), cli::error::Error> {
+fn main() -> Result<(), substrate_cli::error::Error> {
 	let version = VersionInfo {
 		name: "Substrate Node",
 		commit: env!("VERGEN_SHA_SHORT"),

node/cli/browser-demo/.gitignore

@@ -0,0 +1 @@
+pkg

node/cli/browser-demo/README.md

@@ -0,0 +1,10 @@
+# How to run this demo
+
+```sh
+cargo install wasm-pack		# If necessary
+
+# From the `node/cli` directory (parent from this README)
+wasm-pack build --target web --out-dir ./demo/pkg --no-typescript --release -- --no-default-features --features "browser"
+
+xdg-open index.html
+```

node/cli/browser-demo/build.sh

@@ -0,0 +1,3 @@
+#!/usr/bin/env sh
+wasm-pack build --target web --out-dir ./browser-demo/pkg --no-typescript --release ./.. -- --no-default-features --features "browser"
+python -m SimpleHTTPServer 8000

node/cli/browser-demo/index.html

@@ -0,0 +1,39 @@
+<!DOCTYPE html>
+<html>
+  <head>
+	<meta http-equiv="Content-type" content="text/html; charset=utf-8"/>
+	<title>Substrate node</title>
+	<link rel="shortcut icon" href="/favicon.png" />
+	<script type="module">
+import { start_client, default as init } from './pkg/node_cli.js';
+import ws from './ws.js';
+
+function log(msg) {
+	document.getElementsByTagName('body')[0].innerHTML += msg + '\n';
+}
+
+async function start() {
+	log('Loading WASM');
+	await init('./pkg/node_cli_bg.wasm');
+	log('Successfully loaded WASM');
+
+	// Build our client.
+	log('Starting client');
+	let client = start_client(ws());
+	log('Client started');
+
+	client.rpcSubscribe('{"method":"chain_subscribeNewHead","params":[],"id":1,"jsonrpc":"2.0"}',
+		(r) => log("New chain head: " + r));
+
+	setInterval(() => {
+		client
+			.rpcSend('{"method":"system_networkState","params":[],"id":1,"jsonrpc":"2.0"}')
+			.then((r) => log("Network state: " + r));
+	}, 1000);
+}
+
+start();
+	</script>
+  </head>
+  <body style="white-space: pre"></body>
+</html>