diff --git a/CHANGELOG.md b/CHANGELOG.md index fe99b6bba2..99d7875be6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,7 +2,8 @@ ## Unreleased -> Nothing yet. +- [scripts] + - Added "full-mesh" scripts to simulate multiple interconnected networks on a local machine ([#897]) ## v0.3.0 *May 7h, 2021* @@ -72,6 +73,7 @@ and periodically refresh IBC clients. The relayer now also supports [ICS 027 (In [#871]: https://github.com/informalsystems/ibc-rs/issues/871 [#873]: https://github.com/informalsystems/ibc-rs/issues/873 [#878]: https://github.com/informalsystems/ibc-rs/issues/878 +[#897]: https://github.com/informalsystems/ibc-rs/issues/897 [#909]: https://github.com/informalsystems/ibc-rs/issues/909 ## v0.2.0 diff --git a/scripts/full-mesh/.gitignore b/scripts/full-mesh/.gitignore new file mode 100644 index 0000000000..89348ef504 --- /dev/null +++ b/scripts/full-mesh/.gitignore @@ -0,0 +1,6 @@ +docker-compose.gaiad.yml +docker-compose.yml +paths +network*/ +.hermes/ +.gaia/ diff --git a/scripts/full-mesh/1-create-network-config.sh b/scripts/full-mesh/1-create-network-config.sh new file mode 100755 index 0000000000..344e962acc --- /dev/null +++ b/scripts/full-mesh/1-create-network-config.sh @@ -0,0 +1,80 @@ +#!/bin/sh + +############################## +## Script setup +############################## + +# Fail if there are any errors +set -eu + +# Run in a subfolder, not in your $HOME (safeguard against accidental .gaia-deletions) +test "$HOME" != "$(pwd)" + +EXPECTED_GAIAD_VERSION=4.2.1 +DOCKER_IMAGE="cephalopodequipment/gaiad:${EXPECTED_GAIAD_VERSION}" +GAIAD="docker run --rm -v $(pwd):/home/gaiad --entrypoint /usr/bin/gaiad ${DOCKER_IMAGE}" +N="${1:-5}" +echo "N=$N" + +############################## +## Create gaiad config +############################## + +for i in $(seq 1 "$N") +do + test -d "network$i" && continue + echo "i=$i" + $GAIAD testnet --chain-id "network$i" --keyring-backend test --node-dir-prefix "network${i}v" -o . --node-daemon-home . --v 1 + mv "network${i}v0" "network$i" + mv "gentxs" "network$i" + $GAIAD keys add "network${i}c0" --keyring-backend test --keyring-dir "network$i" --output json > "network${i}/client_wallet_seed.json" + $GAIAD add-genesis-account "network${i}c0" "10000000stake,10000network${i}coin" --keyring-backend test --home "network$i" +done + +############################## +## Create docker-compose.gaiad.yml +############################## + +cat < docker-compose.gaiad.yml +services: +EOF + +for i in $(seq 1 "$N") +do + RPC="$((26000 + 10 * i))" + LEET="$((RPC + 1))" + GRPC="$((RPC + 2))" + PP="$((RPC + 3))" + + cat <> docker-compose.gaiad.yml + network${i}: + container_name: network${i} + image: "$DOCKER_IMAGE" + entrypoint: "/usr/bin/gaiad" + command: start --x-crisis-skip-assert-invariants + ports: + - "$RPC:26657" + - "$LEET:1317" + - "$GRPC:9090" + - "$PP:26656" + volumes: + - $(pwd)/network${i}:/home/gaiad/.gaia +EOF + +done + +############################## +## Create full-mesh (default) networks connection graph +############################## + +echo "# IBC full-mesh connection graph" > paths + +for i in $(seq 1 "$((N-1))") +do + for j in $(seq "$((i+1))" "$N") + do + test $i -eq $j && continue + echo "$i $j" >> paths + done +done + diff --git a/scripts/full-mesh/2-create-hermes-config.sh b/scripts/full-mesh/2-create-hermes-config.sh new file mode 100755 index 0000000000..f8ade4824e --- /dev/null +++ b/scripts/full-mesh/2-create-hermes-config.sh @@ -0,0 +1,67 @@ +#!/bin/sh + +############################## +## Script setup +############################## + +# Fail if there are any errors +set -eu + +# Run in a subfolder, not in your $HOME (safeguard against accidental .hermes-deletions) +test "$HOME" != "$(pwd)" +# But if hermes was not run locally, we're going to overcome the issue of not having a --home parameter +ln -s $(pwd)/.hermes $HOME/.hermes || echo "$HOME/.hermes is already set, make sure you link it to .hermes here if you run things locally" + +EXPECTED_HERMES_VERSION=0.2.0 +DOCKER_IMAGE="informaldev/hermes:${EXPECTED_HERMES_VERSION}" +HERMES="docker run --rm -v $(pwd):/home/hermes --entrypoint /usr/bin/hermes ${DOCKER_IMAGE}" +N="${1:-5}" +echo "N=$N" + +############################## +## Create config.toml +############################## + +rm -rf .hermes && mkdir .hermes + +cat < .hermes/config.toml +[global] +strategy = 'naive' +log_level = 'info' + +EOF + +for i in $(seq 1 "$N") +do + RPC="$((26000 + 10 * i))" + GRPC="$((RPC + 2))" + + cat <> .hermes/config.toml +[[chains]] +id='network${i}' +rpc_addr='http://localhost:${RPC}' +grpc_addr='https://localhost:${GRPC}' +websocket_addr='ws://localhost:${RPC}/websocket' +rpc_timeout='1s' +account_prefix='cosmos' +key_name='network${i}c0' +store_prefix='ibc' +fee_denom='stake' + +[chains.trust_threshold] +numerator = '1' +denominator = '3' + +EOF + +done + +############################## +## Add keys to hermes +############################## + +for i in $(seq 1 "$N") +do + $HERMES keys add "network$i" -f "network${i}/client_wallet_seed.json" +done + diff --git a/scripts/full-mesh/3-create-onchain-config.sh b/scripts/full-mesh/3-create-onchain-config.sh new file mode 100755 index 0000000000..1d85a47049 --- /dev/null +++ b/scripts/full-mesh/3-create-onchain-config.sh @@ -0,0 +1,32 @@ +#!/bin/sh + +############################## +## Script setup +############################## + +# Fail if there are any errors +set -eu + +# Run in a subfolder, not in your $HOME (safeguard against accidental .hermes-deletions) +test "$HOME" != "$(pwd)" +# But if hermes was not run locally, we're going to overcome the issue of not having a --home parameter +ln -s $(pwd)/.hermes $HOME/.hermes || echo "$HOME/.hermes is already set, make sure you link it to .hermes here if you run things locally" + +EXPECTED_HERMES_VERSION=0.2.0 +DOCKER_IMAGE="informaldev/hermes:${EXPECTED_HERMES_VERSION}" +HERMES="docker run --rm -v $(pwd)/.hermes:/home/hermes/.hermes --net=host --entrypoint /usr/bin/hermes ${DOCKER_IMAGE}" + +############################## +## Create paths on the networks +############################## + +while read -r line; +do + VALID="$(echo "$line" | grep '^[ ]*[0-9]\+[ ]\+[0-9]\+[ ]*$' || echo "")" + test -n "$VALID" || continue + i="$(echo "$line" | grep -o '[0-9]\+' | head -1)" + j="$(echo "$line" | grep -o '[0-9]\+' | tail -1)" + echo "Connecting network$i and network$j" + $HERMES create channel "network$i" "network$j" --port-a transfer --port-b transfer +done < paths + diff --git a/scripts/full-mesh/4-create-relayer-config.sh b/scripts/full-mesh/4-create-relayer-config.sh new file mode 100755 index 0000000000..3006e3b971 --- /dev/null +++ b/scripts/full-mesh/4-create-relayer-config.sh @@ -0,0 +1,43 @@ +#!/bin/sh + +############################## +## Script setup +############################## + +# Fail if there are any errors +set -eu + +# Run in a subfolder, not in your $HOME (safeguard against accidental .hermes-deletions) +test "$HOME" != "$(pwd)" + +EXPECTED_HERMES_VERSION=0.2.0 +DOCKER_IMAGE="informaldev/hermes:${EXPECTED_HERMES_VERSION}" + +############################## +## Create docker-compose.yml (set of hermes relayers) +############################## + +cat < docker-compose.yml +services: +EOF + +while read -r line; +do + VALID="$(echo "$line" | grep '^[ ]*[0-9]\+[ ]\+[0-9]\+[ ]*$' || echo "")" + test -n "$VALID" || continue + i="$(echo "$line" | grep -o '[0-9]\+' | head -1)" + j="$(echo "$line" | grep -o '[0-9]\+' | tail -1)" + + cat <> docker-compose.yml + hermes-${i}-${j}: + container_name: hermes-${i}-${j} + network_mode: "host" + image: "$DOCKER_IMAGE" + entrypoint: "/usr/bin/hermes" + command: start network$i network$j -p transfer -c channel-0 + volumes: + - $(pwd)/.hermes:/home/hermes/.hermes +EOF + +done < paths + diff --git a/scripts/full-mesh/Makefile b/scripts/full-mesh/Makefile new file mode 100644 index 0000000000..f47f8a12ef --- /dev/null +++ b/scripts/full-mesh/Makefile @@ -0,0 +1,41 @@ +# Number of nodes +N?=5 + +all: setup-network setup-relay + +setup-network: config-files network-up +setup-relay: config-onchain relay-up + +config-files: + ./1-create-network-config.sh $(N) + ./2-create-hermes-config.sh $(N) + +network-up: + docker compose -f docker-compose.gaiad.yml -p gaiad up -d + sleep 5 + +network-start: + docker compose -f docker-compose.gaiad.yml -p gaiad start + +network-stop: + docker compose -f docker-compose.gaiad.yml -p gaiad stop + +network-down: + docker compose -f docker-compose.gaiad.yml -p gaiad down || true + +config-onchain: paths + ./3-create-onchain-config.sh + ./4-create-relayer-config.sh + +# If this command fails with aliases, you need to upgrade Docker CLI to 1.0.14 or above. +relay-up: + docker compose up -d + +relay-down: + docker compose down || true + +clean: relay-down network-down + @rm -rf gentxs network[0-9]* docker-compose.gaiad.yml docker-compose.yml paths .gaia .hermes + @echo DONE + +.PHONY: config-files network-up network-down config-onchain relay-up relay-start relay-stop relay-down clean all diff --git a/scripts/full-mesh/README.md b/scripts/full-mesh/README.md new file mode 100644 index 0000000000..1bf1dae5e2 --- /dev/null +++ b/scripts/full-mesh/README.md @@ -0,0 +1,189 @@ +# Multi-chain (full-mesh) tool +This tool will create multiple chains and IBC connections among them with minimal help. + +Tested on MacOS Mojave. + +## Requirements +* Docker (with Docker CLI v1.0.14 or above, released in 2021 April) +* `make` for convenient command execution + +Note: the files are kept simple where possible. If `make` is not available, you can copy-paste the instructions from the +Makefile fairly easily. + +Note: if you get startup issues with "network-scoped alias support" errors, you need to [upgrade your Docker CLI to v1.0.14 or above](https://github.com/docker/compose-cli/issues/1637). + +## Quick start +Run the below to create 5 blockchain networks with a full mesh of IBC connections. +``` +make +``` +The chains will be named network1, network2, network3, network4 and network5 and they are created using `docker compose`. + +Sending all IBC connection messages can take up to 10-12 minutes (todo: make this parallel). + +If you want something quicker, run the same command instead for 3 blockchain networks by running: +``` +make N=3 +``` + +All configuration is locally editable: networks have their own folder with the configuration and the database so +you can easily change anything or reset the network. Networks can be stopped and restarted using regular docker commands. + +The hermes configuration is stored locally under the `.hermes` folder. + +To delete all networks and configuration, run: +``` +make clean +``` +This will stop the chains, the relayers and clean up the configuration files. + +### Less-than full mesh +You can tell the setup process which blockchain networks should have IBC connections. + +For this, you have to run the setup process in two steps and edit a file in-between that describes the network connections: +``` +make setup-network +vi paths +make setup-relay +``` +The first step will run the blockchain networks and provide a file (`paths`) that contains the full-mesh configuration. +Edit this file to your liking by removing any connections among blockchain networks that you don't want to set up. +The third command will resume the setup process and create the hermes configuration and start the relayers based on this file. + +Example paths file for a five-network full mesh: +``` +# IBC full-mesh connection graph +1 2 +1 3 +1 4 +1 5 +2 3 +2 4 +2 5 +3 4 +3 5 +4 5 +``` + +## Command reference + +![MAKE command hierarchy](make.png "MAKE hierarchy") + +### `make all` +* Same as `make`. +* Equals to `make setup-network setup-relay` +* Creates a 5-network full-mesh. Use the `N=` parameter to change the number of networks. + +### `make setup-network` +* Set up the configuration for the gaiad blockchain networks and start them using docker compose. +* Equals to `make config-files network-up` +* Creates a 5-network full-mesh. Use the `N=` parameter to change the number of networks. + +### `make setup-relay` +* Set up the configuration for hermes and the relayer processes and start them using docker compose. +* Equals to `make config-onchain relay-up` +* Uses the `paths` file to figure out which networks have connections. + +### `make config-files` +* Creates the blockchain networks configuration files. (Under `network1`, `network2`, etc...) +* Creates the `.hermes` folder and adds all network keys to hermes. +* Creates the `paths` file with the descriptions of the network connections. +* Equals to running two bash scripts (`1-create-network-config.sh` and `2-create-hermes-config.sh`). +* Creates a 5-network configuration. Use the `N=` parameter to change the number of networks. + +### `make network-up` +* Runs the gaiad blockchain networks using docker. +* Equals to `docker compose -f docker-compose.gaiad.yml -p gaiad up -d` +* Pairs with `make network-down`. + +Convenience function. There is another set of containers (the relayers) that are configured separately with less parameters. +(Hence the `--project` and `--file` parameters here.) + +### `make config-onchain` +* Configure the networks' IBC connections to each other using hermes. +* Configure the `docker-compose.yml` that describes the relayer processes among the networks. +* Equals to running two bash scripts (`3-create-onchain-config.sh` and `4-create-relayer-config.sh`). +* Uses the `paths` file to figure out which networks connect to which networks. + +### `make relay-up` +* Start the hermes relayer services using `docker-compose.yml`. +* Equals to `docker compose up -d` (yay, easy). +* Pairs with `make relay-down`. + +### `make relay-down` +* Stop the hermes relayer services using `docker-compose.yml`. +* This does not delete the hermes configuration. +* Equals to `docker compose down`. +* Pairs with `make relay-up`. + +### `make network-down` +* Stop and delete the gaiad blockchain networks' docker containers. +* This does not delete the blockchain configuration or database. +* Equals to `docker compose -f docker-compose.gaiad.yml -p gaiad down -d` +* Pairs with `make network-up`. + +### `make network-start` +* Start an existing gaiad blockchain networks' docker containers that were previously stopped. +* Equals to `docker compose -f docker-compose.gaiad.yml -p gaiad start` +* Pairs with `make network-stop`. + +### `make network-stop` +* Stop a running set of gaiad blockchain networks' docker containers. +* Equals to `docker compose -f docker-compose.gaiad.yml -p gaiad stop` +* Pairs with `make network-start`. + +### `make clean` +* Stop and delete all docker containers (gaiad and relayers). +* Delete all networks data and configuration as well as hermes configuration. +* Might show error messages on the screen if some of the docker containers are not running. + It will still exit with error code 0. + +## Wallets +Each chain has two wallet addresses preconfigured with tokens: one for the validator and one additional for generic use. + +Wallets can be reached by using the `--keyring-backend test --keyring-dir networkX` parameters where networkX is the +chain's configuration folder. + +## Ports +All ports are forwarded to the host machine on port 26XXY where XX is the (zero-padded) network identifier number and +Y is the port as follows: +``` +0 - RPC (26657) +1 - APP (1317) +2 - GRPC (9090) +3 - P2P (26656) +``` + +This way `network1`'s ports are: +``` +26010 - RPC +26011 - APP +26012 - GRPC +26013 - P2P +``` + +whereas `network5`'s ports are: +``` +26050 - RPC +26051 - APP +26052 - GRPC +26053 - P2P +``` + +## Examples +Query list of keys on network1: +``` +gaiad keys list --keyring-backend test --keyring-dir network1 +``` + +Get the account balance of a key on network1: +``` +gaiad query bank balances cosmos135vqr04clpx0d0nw3xljpuggps7ffvq3u5mww7 --node http://localhost:26010 +``` + +## Statistics +Setup on my machine: +* N=2 takes about 1.5 minutes +* N=3 takes about 4 minutes +* N=4 takes about 8 minutes +* N=5 takes about 11 minutes diff --git a/scripts/full-mesh/make.drawio b/scripts/full-mesh/make.drawio new file mode 100644 index 0000000000..36a0ecfc94 --- /dev/null +++ b/scripts/full-mesh/make.drawio @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/scripts/full-mesh/make.png b/scripts/full-mesh/make.png new file mode 100644 index 0000000000..ed853d7817 Binary files /dev/null and b/scripts/full-mesh/make.png differ