From bf6575fd38842b63973561e2684af4d8f28660ff Mon Sep 17 00:00:00 2001 From: Gyuho Lee Date: Mon, 20 Mar 2023 13:45:58 +0800 Subject: [PATCH 01/67] avalanche-ops.aws.md: initial commit for TokenVM Signed-off-by: Gyuho Lee --- examples/tokenvm/avalanche-ops.aws.md | 228 ++++++++++++++++++++++++++ 1 file changed, 228 insertions(+) create mode 100644 examples/tokenvm/avalanche-ops.aws.md diff --git a/examples/tokenvm/avalanche-ops.aws.md b/examples/tokenvm/avalanche-ops.aws.md new file mode 100644 index 0000000000..aef4ff25a4 --- /dev/null +++ b/examples/tokenvm/avalanche-ops.aws.md @@ -0,0 +1,228 @@ +## Deploy TokenVM in custom network with avalanche-ops + +[avalanche-ops](https://github.com/ava-labs/avalanche-ops) provides a command-line interface to set up nodes and install custom VMs with the support for custom networks. + +### Step 1 + +Install [Rust](https://www.rust-lang.org/tools/install) to compile avalanche-ops, or download from the [latest release page](https://github.com/ava-labs/avalanche-ops/releases/tag/latest): + +```bash +git clone git@github.com:ava-labs/avalanche-ops.git +cd ./avalanche-ops +./scripts/build.release.sh +./target/release/avalancheup-aws --help +``` + +```bash +# to download on Mac +export LINUX_ARCH_TYPE=$(uname -m) +echo ${LINUX_ARCH_TYPE} +rm -f ./avalancheup-aws.${LINUX_ARCH_TYPE}-apple-darwin +wget "https://github.com/ava-labs/avalanche-ops/releases/download/latest/avalancheup-aws.${LINUX_ARCH_TYPE}-apple-darwin" +mv ./avalancheup-aws.${LINUX_ARCH_TYPE}-apple-darwin /tmp/avalancheup-aws +chmod +x /tmp/avalancheup-aws +/tmp/avalancheup-aws --help +``` + +```bash +# to download on Linux +export LINUX_ARCH_TYPE=$(uname -m) +echo ${LINUX_ARCH_TYPE} +rm -f ./avalancheup-aws.${LINUX_ARCH_TYPE}-linux-gnu +wget "https://github.com/ava-labs/avalanche-ops/releases/download/latest/avalancheup-aws.${LINUX_ARCH_TYPE}-linux-gnu" +mv ./avalancheup-aws.${LINUX_ARCH_TYPE}-linux-gnu /tmp/avalancheup-aws +chmod +x /tmp/avalancheup-aws +/tmp/avalancheup-aws --help +``` + +### Step 2 + +Now we can spin up a new network of 6 nodes with some defaults: +- `avalanche-ops` supports [Graviton-based processors](https://aws.amazon.com/ec2/graviton/) (ARM64). Use `--arch-type arm64` to run nodes in ARM64 CPUs. +- `avalanche-ops` supports [EC2 Spot instances](https://aws.amazon.com/ec2/spot/) for cost savings. Use `--instance-mode=spot` to run instances in spot mode. + +```bash +# launch/download all artifacts for x86 platform +avalancheup-aws default-spec \ +--arch-type amd64 \ +--rust-os-type ubuntu20.04 \ +--anchor-nodes 3 \ +--non-anchor-nodes 3 \ +--region us-west-2 \ +--instance-mode=on-demand \ +--instance-size=2xlarge \ +--ip-mode=elastic \ +--metrics-fetch-interval-seconds 60 \ +--network-name custom \ +--keys-to-generate 5 +``` + +The default commands above will download the public release binaries from github. To test your own binaries, use the following flags to upload to S3. These binaries must be built for the target remote machine platform (e.g., build for `aarch64` and Linux to run them in Graviton processors): + +```bash +--upload-artifacts-aws-volume-provisioner-local-bin ${AWS_VOLUME_PROVISIONER_BIN_PATH} \ +--upload-artifacts-aws-ip-provisioner-local-bin ${AWS_IP_PROVISIONER_BIN_PATH} \ +--upload-artifacts-avalanche-telemetry-cloudwatch-local-bin ${AVALANCHE_TELEMETRY_CLOUDWATCH_BIN_PATH} \ +--upload-artifacts-avalanched-aws-local-bin ${AVALANCHED_AWS_BIN_PATH} \ +--upload-artifacts-avalanchego-local-bin ${AVALANCHEGO_BIN_PATH} \ +``` + +It is recommended to specify your own artifacts to avoid flaky github release page downloads. + +### Step 3 + +The `default-spec` command above will output the following `apply` and `delete` commands that you can copy and paste: + +```bash +# run the following to create resources +vi /home/ubuntu/aops-custom-****-***.yaml + +avalancheup-aws apply \ +--spec-file-path /home/ubuntu/aops-custom-****-***.yaml + +# run the following to delete resources +avalancheup-aws delete \ +--delete-cloudwatch-log-group \ +--delete-s3-objects \ +--delete-ebs-volumes \ +--delete-elastic-ips \ +--spec-file-path /home/ubuntu/aops-custom-****-***.yaml +``` + +That is, `apply` creates AWS resources, whereas `delete` destroys after testing is done. + +*SECURITY*: By default, the SSH and HTTP ports are open to public. Once you complete provisioning the nodes, go to EC2 security group to restrict the inbound rules to your IP. + +*NOTE*: In rare cases, you may encounter [aws-sdk-rust#611](https://github.com/awslabs/aws-sdk-rust/issues/611) where AWS SDK call hangs, which blocks node bootstraps. If a node takes too long to start, connect to that instance (e..g, use SSM sesson from your AWS console), and restart the agent with the command `sudo systemctl restart avalanched-aws`. + +### Step 4 + +Now that the network and nodes are up, let's install two subnets, each of which runs its own TokenVM. Note that the above, successful `avalancheup-aws apply` run will output the following commands as an example, which you can customize depending on the VMs: + +```bash +# EXAMPLE: write subnet config +avalancheup-aws subnet-config \ +--log-level=info \ +--proposer-min-block-delay 250000000 \ +--file-path /tmp/subnet-config.json + +# EXAMPLE: install subnet + custom blockchain in all nodes +avalancheup-aws install-subnet-chain \ +--log-level info \ +--region us-west-2 \ +--s3-bucket avalanche-ops-***-us-west-2 \ +--s3-key-prefix aops-custom-***/install-subnet-chain \ +--ssm-doc aops-custom-***-ssm-install-subnet-chain \ +--chain-rpc-url http://52.88.8.107:9650 \ +... +``` + +For instance, use the following to **deploy two TokenVM subnets and create separate TokenVM chains** (similar to [scripts/run.sh](./scripts/run.sh)): + +```bash +avalancheup-aws subnet-config \ +--log-level=info \ +--proposer-min-block-delay 0 \ +--file-path /tmp/subnet-config.json + +cat < /tmp/allocations.json +[{"address":"token1rvzhmceq997zntgvravfagsks6w0ryud3rylh4cdvayry0dl97nsjzf3yp", "balance":1000000000000}] +EOF +rm -f /tmp/tokenvm-genesis.json +/tmp/token-cli genesis generate /tmp/allocations.json \ +--genesis-file /tmp/tokenvm-genesis.json +cat /tmp/tokenvm-genesis.json + +cat < /tmp/tokenvm-chain-config.json +{ + "mempoolSize": 10000000, + "mempoolExemptPayers":["token1rvzhmceq997zntgvravfagsks6w0ryud3rylh4cdvayry0dl97nsjzf3yp"], + "parallelism": 5, + "trackedPairs":["*"], + "logLevel": "info", + "stateSyncServerDelay": 0 +} +EOF +cat /tmp/tokenvm-chain-config.json +``` + +Note that `--chain-name tokenvm*` and `--node-ids-to-instance-ids` values are different to split nodes into two separate subnets. And `avalancheup-aws install-subnet-chain` **will add the specified nodes as primary network validators (regardless of it's for customer or public networks) before adding them as subnet validators**. So, please make sure the `--key` has enough balance: + +```bash +# this will prompt you to confirm the outcome! +# so make sure to check the outputs! +avalancheup-aws install-subnet-chain \ +--log-level info \ +--region us-west-2 \ +--s3-bucket avalanche-ops-***-us-west-2 \ +--s3-key-prefix aops-custom-***/install-subnet-chain \ +--ssm-doc aops-custom-***-ssm-install-subnet-chain \ +--chain-rpc-url http://52.88.8.107:9650 \ +--key 0x56289e99c94b6912bfc12adc093c9b51124f0dc54ac7a766b2bc5ccf558d8027 \ +--staking-perioid-in-days 15 \ +--subnet-config-local-path /tmp/subnet-config.json \ +--subnet-config-remote-dir /data/avalanche-configs/subnets \ +--vm-binary-local-path /tmp/tokenvm \ +--vm-binary-remote-dir /data/avalanche-plugins \ +--chain-name tokenvm1 \ +--chain-genesis-path /tmp/tokenvm-genesis.json \ +--chain-config-local-path /tmp/tokenvm-chain-config.json \ +--chain-config-remote-dir /data/avalanche-configs/chains \ +--avalanchego-config-remote-path /data/avalanche-configs/config.json \ +--node-ids-to-instance-ids '{"NodeID-31HAxw7wYJ2u2HQHBkwwF26bnFuxnh2sa":"i-04f6ea239218440f0","NodeID-PLM2si9LWaqvzid6AeJa9tft7rzDXXKg2":"i-0459af0e4cf31fcfa","NodeID-D1RtuFSbmcTiRVY991vkq5UvsTBUpHHLR":"i-0d44cfe5420370c30"}' +# VM Id 'tHBYNu8ikt25R77fH4znHYC4B5mkaEnXPFmsJnECZjq59dySw', chain name 'tokenvm1' +# SUCCESS: subnet Id 2DLqm2Wk4SFtdmeqkmyfAvTdfvNKmbjwZwztMqRaQCwnDbRHCo, blockchain Id G9CbuiKLoyeYhabA8ph7TBiisKt9LS6Hx1QRgoajnwxd1xFC8 + +# this will prompt you to confirm the outcome! +# so make sure to check the outputs! +avalancheup-aws install-subnet-chain \ +--log-level info \ +--region us-west-2 \ +--s3-bucket avalanche-ops-***-us-west-2 \ +--s3-key-prefix aops-custom-***/install-subnet-chain \ +--ssm-doc aops-custom-***-ssm-install-subnet-chain \ +--chain-rpc-url http://52.88.8.107:9650 \ +--key 0x56289e99c94b6912bfc12adc093c9b51124f0dc54ac7a766b2bc5ccf558d8027 \ +--staking-perioid-in-days 15 \ +--subnet-config-local-path /tmp/subnet-config.json \ +--subnet-config-remote-dir /data/avalanche-configs/subnets \ +--vm-binary-local-path /tmp/tokenvm \ +--vm-binary-remote-dir /data/avalanche-plugins \ +--chain-name tokenvm2 \ +--chain-genesis-path /tmp/tokenvm-genesis.json \ +--chain-config-local-path /tmp/tokenvm-chain-config.json \ +--chain-config-remote-dir /data/avalanche-configs/chains \ +--avalanchego-config-remote-path /data/avalanche-configs/config.json \ +--node-ids-to-instance-ids '{"NodeID-PNc5fwhKLDLGHBF81qnjTwsdjFbdcZxn1":"i-0c5114f6b1e48e671","NodeID-8oqy47xm46RcdTNFxYo8dpBJtPFqQLLeG":"i-083ccebe7b8d40bc9","NodeID-HC9HVTiThxL6d55bNGqV3bkKPwQmLvEs1":"i-0806a42d5bb597dc5"}' +# VM Id 'tHBYNu8ikt4i8cEV4nsSuj7Ldc9sXAHc8L6qKRJR4e5CR7T3t', chain name 'tokenvm2' +# SUCCESS: subnet Id 2km1PAKNK4vSfpyjgK2iFFoBFD3mXHBAWfDBVsPuzKtNYdPRTY, blockchain Id F1c3GayrV5E7NQdxUCbVfnRPsMSxGbTuHvJLS4R48M3Dcazs6 +``` + +To make sure chains are launched successfully, just check the `health` endpoints: + +> curl http://52.88.8.107:9650/ext/health +> +> {"checks":{"C":{"message":{"consensus":{"longestRunningBlock":"0s","outstandingBlocks":0},"vm":null},"timestamp":"2023-03-14T15:55:08.256378577Z","duration":14020},"F1c3GayrV5E7NQdxUCbVfnRPsMSxGbTuHvJLS4R48M3Dcazs6":{"message":{"consensus":{"longestRunningBlock":"0s","outstandingBlocks":0},"vm":{"database":{"v1.4.5":null},"health":200}},"timestamp":"2023-03-14T15:55:08.257445968Z","duration":1061711},"P":{"message":{"consensus":{"longestRunningBlock":"0s","outstandingBlocks":0},"vm":{"2km1PAKNK4vSfpyjgK2iFFoBFD3mXHBAWfDBVsPuzKtNYdPRTY-percentConnected":1,"primary-percentConnected":1}},"timestamp":"2023-03-14T15:55:08.256454548Z","duration":74871},"X":{"message":{"consensus":{"outstandingVertices":0,"snowstorm":{"outstandingTransactions":0}},"vm":null},"timestamp":"2023-03-14T15:55:08.256459848Z","duration":15850},"bootstrapped":{"message":[],"timestamp":"2023-03-14T15:55:08.256438778Z","duration":6020},"database":{"timestamp":"2023-03-14T15:55:08.256464299Z","duration":1420},"diskspace":{"message":{"availableDiskBytes":299770843136},"timestamp":"2023-03-14T15:55:08.256466309Z","duration":4070},"network":{"message":{"connectedPeers":5,"sendFailRate":0,"timeSinceLastMsgReceived":"256.459548ms","timeSinceLastMsgSent":"256.459548ms"},"timestamp":"2023-03-14T15:55:08.256461929Z","duration":5551},"router":{"message":{"longestRunningRequest":"0s","outstandingRequests":0},"timestamp":"2023-03-14T15:55:08.256363507Z","duration":17231}},"healthy":true} + +> curl http://44.224.148.127:9650/ext/health +> +> {"checks":{"C":{"message":{"consensus":{"longestRunningBlock":"0s","outstandingBlocks":0},"vm":null},"timestamp":"2023-03-14T15:55:08.203325635Z","duration":6570},"F1c3GayrV5E7NQdxUCbVfnRPsMSxGbTuHvJLS4R48M3Dcazs6":{"message":{"consensus":{"longestRunningBlock":"0s","outstandingBlocks":0},"vm":{"database":{"v1.4.5":null},"health":200}},"timestamp":"2023-03-14T15:55:08.204161784Z","duration":927270},"P":{"message":{"consensus":{"longestRunningBlock":"0s","outstandingBlocks":0},"vm":{"2km1PAKNK4vSfpyjgK2iFFoBFD3mXHBAWfDBVsPuzKtNYdPRTY-percentConnected":1,"primary-percentConnected":1}},"timestamp":"2023-03-14T15:55:08.203318015Z","duration":63060},"X":{"message":{"consensus":{"outstandingVertices":0,"snowstorm":{"outstandingTransactions":0}},"vm":null},"timestamp":"2023-03-14T15:55:08.203233714Z","duration":18360},"bootstrapped":{"message":[],"timestamp":"2023-03-14T15:55:08.203205494Z","duration":3010},"database":{"timestamp":"2023-03-14T15:55:08.203214634Z","duration":1350},"diskspace":{"message":{"availableDiskBytes":299770839040},"timestamp":"2023-03-14T15:55:08.203252895Z","duration":5361},"network":{"message":{"connectedPeers":5,"sendFailRate":0,"timeSinceLastMsgReceived":"203.209294ms","timeSinceLastMsgSent":"203.209294ms"},"timestamp":"2023-03-14T15:55:08.203212714Z","duration":6550},"router":{"message":{"longestRunningRequest":"0s","outstandingRequests":0},"timestamp":"2023-03-14T15:55:08.203338225Z","duration":21220}},"healthy":true} + +## Deploy TokenVM in Fuji network with avalanche-ops + +Same as above, except you use `--network-name fuji` and do not need anchor nodes: + +```bash +avalancheup-aws default-spec \ +--arch-type amd64 \ +--rust-os-type ubuntu20.04 \ +--non-anchor-nodes 6 \ +--region us-west-2 \ +--instance-mode=on-demand \ +--instance-size=2xlarge \ +--ip-mode=elastic \ +--metrics-fetch-interval-seconds 60 \ +--network-name fuji +``` + +And make sure the nodes are in sync with the chain state before installing subnets/chains with `avalancheup-aws install-subnet-chain`. You can check the status of the nodes either via HTTP `/health` endpoints or CloudWatch logs. From 1bc9568e7f0e36517eae2a60dc4795fe899b969b Mon Sep 17 00:00:00 2001 From: Gyuho Lee Date: Mon, 20 Mar 2023 16:21:54 +0800 Subject: [PATCH 02/67] specify --subnet-validate-period-in-days Signed-off-by: Gyuho Lee --- examples/tokenvm/avalanche-ops.aws.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/tokenvm/avalanche-ops.aws.md b/examples/tokenvm/avalanche-ops.aws.md index aef4ff25a4..8f0ab843a5 100644 --- a/examples/tokenvm/avalanche-ops.aws.md +++ b/examples/tokenvm/avalanche-ops.aws.md @@ -146,7 +146,7 @@ EOF cat /tmp/tokenvm-chain-config.json ``` -Note that `--chain-name tokenvm*` and `--node-ids-to-instance-ids` values are different to split nodes into two separate subnets. And `avalancheup-aws install-subnet-chain` **will add the specified nodes as primary network validators (regardless of it's for customer or public networks) before adding them as subnet validators**. So, please make sure the `--key` has enough balance: +Note that `--chain-name tokenvm*` and `--node-ids-to-instance-ids` values are different to split nodes into two separate subnets. And `avalancheup-aws install-subnet-chain` **will add the specified nodes as primary network validators (regardless of it's for customer or public networks) before adding them as subnet validators**. So, please make sure the `--key` has enough balance and use `--subnet-validate-period-in-days` flag to set custom validate priods (defaults to 15-day for primary network, 14-day for subnet validation): ```bash # this will prompt you to confirm the outcome! From 35bc2d217605c58d6d842adb3892a1bc91983902 Mon Sep 17 00:00:00 2001 From: Gyuho Lee Date: Mon, 20 Mar 2023 16:22:40 +0800 Subject: [PATCH 03/67] fix typo Signed-off-by: Gyuho Lee --- examples/tokenvm/avalanche-ops.aws.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/examples/tokenvm/avalanche-ops.aws.md b/examples/tokenvm/avalanche-ops.aws.md index 8f0ab843a5..f80ff35dfc 100644 --- a/examples/tokenvm/avalanche-ops.aws.md +++ b/examples/tokenvm/avalanche-ops.aws.md @@ -159,7 +159,7 @@ avalancheup-aws install-subnet-chain \ --ssm-doc aops-custom-***-ssm-install-subnet-chain \ --chain-rpc-url http://52.88.8.107:9650 \ --key 0x56289e99c94b6912bfc12adc093c9b51124f0dc54ac7a766b2bc5ccf558d8027 \ ---staking-perioid-in-days 15 \ +--subnet-validate-period-in-days 15 \ --subnet-config-local-path /tmp/subnet-config.json \ --subnet-config-remote-dir /data/avalanche-configs/subnets \ --vm-binary-local-path /tmp/tokenvm \ @@ -183,7 +183,7 @@ avalancheup-aws install-subnet-chain \ --ssm-doc aops-custom-***-ssm-install-subnet-chain \ --chain-rpc-url http://52.88.8.107:9650 \ --key 0x56289e99c94b6912bfc12adc093c9b51124f0dc54ac7a766b2bc5ccf558d8027 \ ---staking-perioid-in-days 15 \ +--subnet-validate-period-in-days 15 \ --subnet-config-local-path /tmp/subnet-config.json \ --subnet-config-remote-dir /data/avalanche-configs/subnets \ --vm-binary-local-path /tmp/tokenvm \ From 88b06f57be3f67dd6f8a51c9e7f1d1078064fcfc Mon Sep 17 00:00:00 2001 From: Gyuho Lee Date: Mon, 20 Mar 2023 16:55:40 +0800 Subject: [PATCH 04/67] default Signed-off-by: Gyuho Lee --- examples/tokenvm/avalanche-ops.aws.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/tokenvm/avalanche-ops.aws.md b/examples/tokenvm/avalanche-ops.aws.md index f80ff35dfc..f0d4d72846 100644 --- a/examples/tokenvm/avalanche-ops.aws.md +++ b/examples/tokenvm/avalanche-ops.aws.md @@ -146,7 +146,7 @@ EOF cat /tmp/tokenvm-chain-config.json ``` -Note that `--chain-name tokenvm*` and `--node-ids-to-instance-ids` values are different to split nodes into two separate subnets. And `avalancheup-aws install-subnet-chain` **will add the specified nodes as primary network validators (regardless of it's for customer or public networks) before adding them as subnet validators**. So, please make sure the `--key` has enough balance and use `--subnet-validate-period-in-days` flag to set custom validate priods (defaults to 15-day for primary network, 14-day for subnet validation): +Note that `--chain-name tokenvm*` and `--node-ids-to-instance-ids` values are different to split nodes into two separate subnets. And `avalancheup-aws install-subnet-chain` **will add the specified nodes as primary network validators (regardless of it's for customer or public networks) before adding them as subnet validators**. So, please make sure the `--key` has enough balance and use `--primary-network-validate-period-in-days` and `--subnet-validate-period-in-days` flags to set custom validate priods (defaults to 15-day for primary network, 14-day for subnet validation): ```bash # this will prompt you to confirm the outcome! From d17e31d2650596c94ff34de15e4241e3bacbe8a4 Mon Sep 17 00:00:00 2001 From: Gyuho Lee Date: Mon, 20 Mar 2023 17:01:05 +0800 Subject: [PATCH 05/67] update Signed-off-by: Gyuho Lee --- examples/tokenvm/avalanche-ops.aws.md | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/examples/tokenvm/avalanche-ops.aws.md b/examples/tokenvm/avalanche-ops.aws.md index f0d4d72846..b7ebd9fc96 100644 --- a/examples/tokenvm/avalanche-ops.aws.md +++ b/examples/tokenvm/avalanche-ops.aws.md @@ -159,7 +159,8 @@ avalancheup-aws install-subnet-chain \ --ssm-doc aops-custom-***-ssm-install-subnet-chain \ --chain-rpc-url http://52.88.8.107:9650 \ --key 0x56289e99c94b6912bfc12adc093c9b51124f0dc54ac7a766b2bc5ccf558d8027 \ ---subnet-validate-period-in-days 15 \ +--primary-network-validate-period-in-days 15 \ +--subnet-validate-period-in-days 14 \ --subnet-config-local-path /tmp/subnet-config.json \ --subnet-config-remote-dir /data/avalanche-configs/subnets \ --vm-binary-local-path /tmp/tokenvm \ @@ -183,7 +184,8 @@ avalancheup-aws install-subnet-chain \ --ssm-doc aops-custom-***-ssm-install-subnet-chain \ --chain-rpc-url http://52.88.8.107:9650 \ --key 0x56289e99c94b6912bfc12adc093c9b51124f0dc54ac7a766b2bc5ccf558d8027 \ ---subnet-validate-period-in-days 15 \ +--primary-network-validate-period-in-days 15 \ +--subnet-validate-period-in-days 14 \ --subnet-config-local-path /tmp/subnet-config.json \ --subnet-config-remote-dir /data/avalanche-configs/subnets \ --vm-binary-local-path /tmp/tokenvm \ From b612818c564096a5135844310466f52a0cdf19ed Mon Sep 17 00:00:00 2001 From: Gyuho Lee Date: Mon, 20 Mar 2023 17:18:20 +0800 Subject: [PATCH 06/67] just use 16 day Signed-off-by: Gyuho Lee --- examples/tokenvm/avalanche-ops.aws.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/examples/tokenvm/avalanche-ops.aws.md b/examples/tokenvm/avalanche-ops.aws.md index b7ebd9fc96..1dad919f6b 100644 --- a/examples/tokenvm/avalanche-ops.aws.md +++ b/examples/tokenvm/avalanche-ops.aws.md @@ -146,7 +146,7 @@ EOF cat /tmp/tokenvm-chain-config.json ``` -Note that `--chain-name tokenvm*` and `--node-ids-to-instance-ids` values are different to split nodes into two separate subnets. And `avalancheup-aws install-subnet-chain` **will add the specified nodes as primary network validators (regardless of it's for customer or public networks) before adding them as subnet validators**. So, please make sure the `--key` has enough balance and use `--primary-network-validate-period-in-days` and `--subnet-validate-period-in-days` flags to set custom validate priods (defaults to 15-day for primary network, 14-day for subnet validation): +Note that `--chain-name tokenvm*` and `--node-ids-to-instance-ids` values are different to split nodes into two separate subnets. And `avalancheup-aws install-subnet-chain` **will add the specified nodes as primary network validators (regardless of it's for customer or public networks) before adding them as subnet validators**. So, please make sure the `--key` has enough balance and use `--primary-network-validate-period-in-days` and `--subnet-validate-period-in-days` flags to set custom validate priods (defaults to 16-day for primary network, 14-day for subnet validation): ```bash # this will prompt you to confirm the outcome! @@ -159,7 +159,7 @@ avalancheup-aws install-subnet-chain \ --ssm-doc aops-custom-***-ssm-install-subnet-chain \ --chain-rpc-url http://52.88.8.107:9650 \ --key 0x56289e99c94b6912bfc12adc093c9b51124f0dc54ac7a766b2bc5ccf558d8027 \ ---primary-network-validate-period-in-days 15 \ +--primary-network-validate-period-in-days 16 \ --subnet-validate-period-in-days 14 \ --subnet-config-local-path /tmp/subnet-config.json \ --subnet-config-remote-dir /data/avalanche-configs/subnets \ @@ -184,7 +184,7 @@ avalancheup-aws install-subnet-chain \ --ssm-doc aops-custom-***-ssm-install-subnet-chain \ --chain-rpc-url http://52.88.8.107:9650 \ --key 0x56289e99c94b6912bfc12adc093c9b51124f0dc54ac7a766b2bc5ccf558d8027 \ ---primary-network-validate-period-in-days 15 \ +--primary-network-validate-period-in-days 16 \ --subnet-validate-period-in-days 14 \ --subnet-config-local-path /tmp/subnet-config.json \ --subnet-config-remote-dir /data/avalanche-configs/subnets \ From 80adbf0732ffb500c69a59c039afbc745f82e26b Mon Sep 17 00:00:00 2001 From: Gyuho Lee Date: Wed, 22 Mar 2023 12:09:15 +0800 Subject: [PATCH 07/67] clarify aws credential requirements + node provision steps Signed-off-by: Gyuho Lee --- examples/tokenvm/avalanche-ops.aws.md | 21 ++++++++++++++++++++- 1 file changed, 20 insertions(+), 1 deletion(-) diff --git a/examples/tokenvm/avalanche-ops.aws.md b/examples/tokenvm/avalanche-ops.aws.md index 1dad919f6b..70800958de 100644 --- a/examples/tokenvm/avalanche-ops.aws.md +++ b/examples/tokenvm/avalanche-ops.aws.md @@ -60,6 +60,8 @@ avalancheup-aws default-spec \ The default commands above will download the public release binaries from github. To test your own binaries, use the following flags to upload to S3. These binaries must be built for the target remote machine platform (e.g., build for `aarch64` and Linux to run them in Graviton processors): ```bash +avalancheup-aws default-spec \ +... --upload-artifacts-aws-volume-provisioner-local-bin ${AWS_VOLUME_PROVISIONER_BIN_PATH} \ --upload-artifacts-aws-ip-provisioner-local-bin ${AWS_IP_PROVISIONER_BIN_PATH} \ --upload-artifacts-avalanche-telemetry-cloudwatch-local-bin ${AVALANCHE_TELEMETRY_CLOUDWATCH_BIN_PATH} \ @@ -69,9 +71,26 @@ The default commands above will download the public release binaries from github It is recommended to specify your own artifacts to avoid flaky github release page downloads. +*NOTE*: The `default-spec` (and `apply`) command only provisions nodes, not custom VMs. The subnet and custom VM installation are done via `install-subnet-chain` sub-commands. + ### Step 3 -The `default-spec` command above will output the following `apply` and `delete` commands that you can copy and paste: +The `default-spec` command itselt does not create any resources, and instead outputs the following `apply` and `delete` commands that you can copy and paste: + +```bash +# first make sure you have access to the AWS credential +aws sts get-caller-identity + +# "avalancheup-aws apply" would not work... +# An error occurred (ExpiredToken) when calling the GetCallerIdentity operation: The security token included in the request is expired + +# "avalancheup-aws apply" will work with the following credential +# { +# "UserId": "...:abc@abc.com", +# "Account": "...", +# "Arn": "arn:aws:sts::....:assumed-role/name/abc@abc.com" +# } +``` ```bash # run the following to create resources From 488221a2f059f470acc2e616e30e48a9c699e6ed Mon Sep 17 00:00:00 2001 From: Patrick O'Grady Date: Thu, 23 Mar 2023 18:56:16 -0700 Subject: [PATCH 08/67] Read URIs from avalanche-ops (#116) * read from avalanche-ops * fix error * linter * change to ops --- examples/tokenvm/cmd/token-cli/cmd/chain.go | 64 +++++++++++++++++++++ examples/tokenvm/cmd/token-cli/cmd/root.go | 18 ++++-- 2 files changed, 77 insertions(+), 5 deletions(-) diff --git a/examples/tokenvm/cmd/token-cli/cmd/chain.go b/examples/tokenvm/cmd/token-cli/cmd/chain.go index 2e63d1ecd7..6c125bfd17 100644 --- a/examples/tokenvm/cmd/token-cli/cmd/chain.go +++ b/examples/tokenvm/cmd/token-cli/cmd/chain.go @@ -7,6 +7,7 @@ package cmd import ( "context" "fmt" + "os" "reflect" "strconv" "strings" @@ -18,6 +19,7 @@ import ( "github.com/ava-labs/hypersdk/utils" "github.com/ava-labs/hypersdk/vm" "github.com/spf13/cobra" + "gopkg.in/yaml.v2" "github.com/ava-labs/hypersdk/examples/tokenvm/actions" "github.com/ava-labs/hypersdk/examples/tokenvm/auth" @@ -126,6 +128,68 @@ var importANRChainCmd = &cobra.Command{ }, } +type AvalancheOpsConfig struct { + Resources struct { + CreatedNodes []struct { + HTTPEndpoint string `yaml:"httpEndpoint"` + } `yaml:"created_nodes"` + } `yaml:"resources"` +} + +var importAvalancheOpsChainCmd = &cobra.Command{ + Use: "import-ops [chainID] [path]", + PreRunE: func(cmd *cobra.Command, args []string) error { + if len(args) != 2 { + return ErrInvalidArgs + } + _, err := ids.FromString(args[0]) + return err + }, + RunE: func(_ *cobra.Command, args []string) error { + // Delete previous items + if deleteOtherChains { + oldChains, err := DeleteChains() + if err != nil { + return err + } + if len(oldChains) > 0 { + utils.Outf("{{yellow}}deleted old chains:{{/}} %+v\n", oldChains) + } + } + + // Load chainID + chainID, err := ids.FromString(args[0]) + if err != nil { + return err + } + + // Load yaml file + var opsConfig AvalancheOpsConfig + yamlFile, err := os.ReadFile(args[1]) + if err != nil { + return err + } + err = yaml.Unmarshal(yamlFile, &opsConfig) + if err != nil { + return err + } + + // Add chains + for _, node := range opsConfig.Resources.CreatedNodes { + uri := fmt.Sprintf("%s/ext/bc/%s", node.HTTPEndpoint, chainID) + if err := StoreChain(chainID, uri); err != nil { + return err + } + utils.Outf( + "{{yellow}}stored chainID:{{/}} %s {{yellow}}uri:{{/}} %s\n", + chainID, + uri, + ) + } + return StoreDefault(defaultChainKey, chainID[:]) + }, +} + var setChainCmd = &cobra.Command{ Use: "set", RunE: func(*cobra.Command, []string) error { diff --git a/examples/tokenvm/cmd/token-cli/cmd/root.go b/examples/tokenvm/cmd/token-cli/cmd/root.go index e3893d392a..f942ab6f37 100644 --- a/examples/tokenvm/cmd/token-cli/cmd/root.go +++ b/examples/tokenvm/cmd/token-cli/cmd/root.go @@ -24,11 +24,12 @@ var ( dbPath string db database.Database - genesisFile string - minUnitPrice int64 - hideTxs bool - randomRecipient bool - maxTxBacklog int + genesisFile string + minUnitPrice int64 + hideTxs bool + randomRecipient bool + maxTxBacklog int + deleteOtherChains bool rootCmd = &cobra.Command{ Use: "token-cli", @@ -95,9 +96,16 @@ func init() { false, "hide txs", ) + importAvalancheOpsChainCmd.PersistentFlags().BoolVar( + &deleteOtherChains, + "delete-other-chains", + true, + "delete other chains", + ) chainCmd.AddCommand( importChainCmd, importANRChainCmd, + importAvalancheOpsChainCmd, setChainCmd, chainInfoCmd, watchChainCmd, From ca8327e0018e30dc2aeb58699f02a3ab09b93c2e Mon Sep 17 00:00:00 2001 From: Patrick O'Grady Date: Tue, 28 Mar 2023 13:34:24 -0400 Subject: [PATCH 09/67] progress --- .../{avalanche-ops.aws.md => DEVNETS.md} | 100 ++++++++++-------- examples/tokenvm/README.md | 6 ++ 2 files changed, 62 insertions(+), 44 deletions(-) rename examples/tokenvm/{avalanche-ops.aws.md => DEVNETS.md} (80%) diff --git a/examples/tokenvm/avalanche-ops.aws.md b/examples/tokenvm/DEVNETS.md similarity index 80% rename from examples/tokenvm/avalanche-ops.aws.md rename to examples/tokenvm/DEVNETS.md index 70800958de..fd0e05a924 100644 --- a/examples/tokenvm/avalanche-ops.aws.md +++ b/examples/tokenvm/DEVNETS.md @@ -1,20 +1,34 @@ -## Deploy TokenVM in custom network with avalanche-ops - -[avalanche-ops](https://github.com/ava-labs/avalanche-ops) provides a command-line interface to set up nodes and install custom VMs with the support for custom networks. - -### Step 1 - -Install [Rust](https://www.rust-lang.org/tools/install) to compile avalanche-ops, or download from the [latest release page](https://github.com/ava-labs/avalanche-ops/releases/tag/latest): +## Deploying Devnets +_In the world of Avalanche, we refer to short-lived, test Subnets as Devnets._ + +### Step 1: Install `avalanche-ops` +[avalanche-ops](https://github.com/ava-labs/avalanche-ops) provides a command-line +interface to setup nodes and install Custom VMs on both custom networks and on +Fuji using [AWS CloudFormation](https://aws.amazon.com/cloudformation/). `avalanche-ops` +is written in Rust, so you must either install Rust and compile `avalanche-ops` or +download the latest from the `avalanche-ops` repo. + +#### Option 1: Install Rust and Compile +Install [Rust](https://www.rust-lang.org/tools/install): +```bash +curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh +``` +Compile `avalanche-ops`: ```bash git clone git@github.com:ava-labs/avalanche-ops.git cd ./avalanche-ops ./scripts/build.release.sh -./target/release/avalancheup-aws --help +mv ./target/release/avalancheup-aws /tmp/avalancheup-aws +chmod +x /tmp/avalancheup-aws +/tmp/avalancheup-aws --help ``` +#### Option 2: Download Pre-Built CLI +You can view a full list of pre-built binaries on the [latest release page](https://github.com/ava-labs/avalanche-ops/releases/tag/latest). + +##### Install `avalanche-ops` on Mac ```bash -# to download on Mac export LINUX_ARCH_TYPE=$(uname -m) echo ${LINUX_ARCH_TYPE} rm -f ./avalancheup-aws.${LINUX_ARCH_TYPE}-apple-darwin @@ -24,8 +38,8 @@ chmod +x /tmp/avalancheup-aws /tmp/avalancheup-aws --help ``` +##### Install `avalanche-ops` on Linux ```bash -# to download on Linux export LINUX_ARCH_TYPE=$(uname -m) echo ${LINUX_ARCH_TYPE} rm -f ./avalancheup-aws.${LINUX_ARCH_TYPE}-linux-gnu @@ -35,7 +49,16 @@ chmod +x /tmp/avalancheup-aws /tmp/avalancheup-aws --help ``` -### Step 2 +### Step 2: Install and Configure `aws-cli` +Next, make sure to install the [AWS CLI](https://docs.aws.amazon.com/cli/latest/userguide/getting-started-install.html). + +Once you've installed the AWS CLI, run `aws configure` to set the access key to +use while deploying your devnet. + +### Step 3: Download `token-cli` and `tokenvm` + + +### Step 3: Plan Local Network Deploy Now we can spin up a new network of 6 nodes with some defaults: - `avalanche-ops` supports [Graviton-based processors](https://aws.amazon.com/ec2/graviton/) (ARM64). Use `--arch-type arm64` to run nodes in ARM64 CPUs. @@ -43,7 +66,7 @@ Now we can spin up a new network of 6 nodes with some defaults: ```bash # launch/download all artifacts for x86 platform -avalancheup-aws default-spec \ +/tmp/avalancheup-aws default-spec \ --arch-type amd64 \ --rust-os-type ubuntu20.04 \ --anchor-nodes 3 \ @@ -57,7 +80,14 @@ avalancheup-aws default-spec \ --keys-to-generate 5 ``` -The default commands above will download the public release binaries from github. To test your own binaries, use the following flags to upload to S3. These binaries must be built for the target remote machine platform (e.g., build for `aarch64` and Linux to run them in Graviton processors): +The `default-spec` (and `apply`) command only provisions nodes, not Custom VMs. +The Subnet and Custom VM installation are done via `install-subnet-chain` sub-commands that follow. + +#### Using Custom `avalanchego` Binaries +The default command above will download the `avalanchego` public release binaries from GitHub. +To test your own binaries, use the following flags to upload to S3. These binaries must be built +for the target remote machine platform (e.g., build for `aarch64` and Linux to run them in +Graviton processors): ```bash avalancheup-aws default-spec \ @@ -71,9 +101,7 @@ avalancheup-aws default-spec \ It is recommended to specify your own artifacts to avoid flaky github release page downloads. -*NOTE*: The `default-spec` (and `apply`) command only provisions nodes, not custom VMs. The subnet and custom VM installation are done via `install-subnet-chain` sub-commands. - -### Step 3 +### Step 4: Apply Local Network Deploy The `default-spec` command itselt does not create any resources, and instead outputs the following `apply` and `delete` commands that you can copy and paste: @@ -110,45 +138,29 @@ avalancheup-aws delete \ That is, `apply` creates AWS resources, whereas `delete` destroys after testing is done. -*SECURITY*: By default, the SSH and HTTP ports are open to public. Once you complete provisioning the nodes, go to EC2 security group to restrict the inbound rules to your IP. +*SECURITY*: By default, the SSH and HTTP ports are open to public. Once you complete provisioning the nodes, go to EC2 security +group to restrict the inbound rules to your IP. -*NOTE*: In rare cases, you may encounter [aws-sdk-rust#611](https://github.com/awslabs/aws-sdk-rust/issues/611) where AWS SDK call hangs, which blocks node bootstraps. If a node takes too long to start, connect to that instance (e..g, use SSM sesson from your AWS console), and restart the agent with the command `sudo systemctl restart avalanched-aws`. +*NOTE*: In rare cases, you may encounter [aws-sdk-rust#611](https://github.com/awslabs/aws-sdk-rust/issues/611) +where AWS SDK call hangs, which blocks node bootstraps. If a node takes too long to start, connect to that +instance (e..g, use SSM sesson from your AWS console), and restart the agent with the command `sudo systemctl restart avalanched-aws`. -### Step 4 - -Now that the network and nodes are up, let's install two subnets, each of which runs its own TokenVM. Note that the above, successful `avalancheup-aws apply` run will output the following commands as an example, which you can customize depending on the VMs: - -```bash -# EXAMPLE: write subnet config -avalancheup-aws subnet-config \ ---log-level=info \ ---proposer-min-block-delay 250000000 \ ---file-path /tmp/subnet-config.json - -# EXAMPLE: install subnet + custom blockchain in all nodes -avalancheup-aws install-subnet-chain \ ---log-level info \ ---region us-west-2 \ ---s3-bucket avalanche-ops-***-us-west-2 \ ---s3-key-prefix aops-custom-***/install-subnet-chain \ ---ssm-doc aops-custom-***-ssm-install-subnet-chain \ ---chain-rpc-url http://52.88.8.107:9650 \ -... -``` +### Step 5: Install Subnets -For instance, use the following to **deploy two TokenVM subnets and create separate TokenVM chains** (similar to [scripts/run.sh](./scripts/run.sh)): +Now that the network and nodes are up, let's install two subnets, each of which runs its own `tokenvm`. Use the following commands to +do so (similar to [scripts/run.sh](./scripts/run.sh)): ```bash avalancheup-aws subnet-config \ --log-level=info \ --proposer-min-block-delay 0 \ ---file-path /tmp/subnet-config.json +--file-path /tmp/avalanche-ops/subnet-config.json -cat < /tmp/allocations.json +cat < /tmp/avalanche-ops/allocations.json [{"address":"token1rvzhmceq997zntgvravfagsks6w0ryud3rylh4cdvayry0dl97nsjzf3yp", "balance":1000000000000}] EOF -rm -f /tmp/tokenvm-genesis.json -/tmp/token-cli genesis generate /tmp/allocations.json \ +rm -f /tmp/avalanche-ops/tokenvm-genesis.json +/tmp/token-cli genesis generate /tmp/avalanche-ops/allocations.json \ --genesis-file /tmp/tokenvm-genesis.json cat /tmp/tokenvm-genesis.json diff --git a/examples/tokenvm/README.md b/examples/tokenvm/README.md index 52e2aee536..c9a4d6604a 100644 --- a/examples/tokenvm/README.md +++ b/examples/tokenvm/README.md @@ -426,6 +426,12 @@ command: docker-compose -f trace/zipkin.yml down ``` +## Deploying to a Devnet +_In the world of Avalanche, we refer to short-lived, test Subnets as Devnets._ + +To programaticaly deploy `tokenvm` to a distributed cluster of nodes running on +your own custom network or on Fuji, check out this [doc](DEVNETS.md). + ## Future Work _If you want to take the lead on any of these items, please [start a discussion](https://github.com/ava-labs/hypersdk/discussions) or reach From 69c0366b6c38fcdf42cb4c74cca8255fa2e4050e Mon Sep 17 00:00:00 2001 From: Patrick O'Grady Date: Tue, 28 Mar 2023 14:12:48 -0400 Subject: [PATCH 10/67] remove install from source option on rust --- examples/tokenvm/DEVNETS.md | 31 +++++++------------------------ 1 file changed, 7 insertions(+), 24 deletions(-) diff --git a/examples/tokenvm/DEVNETS.md b/examples/tokenvm/DEVNETS.md index fd0e05a924..fbb665a866 100644 --- a/examples/tokenvm/DEVNETS.md +++ b/examples/tokenvm/DEVNETS.md @@ -4,30 +4,12 @@ _In the world of Avalanche, we refer to short-lived, test Subnets as Devnets._ ### Step 1: Install `avalanche-ops` [avalanche-ops](https://github.com/ava-labs/avalanche-ops) provides a command-line interface to setup nodes and install Custom VMs on both custom networks and on -Fuji using [AWS CloudFormation](https://aws.amazon.com/cloudformation/). `avalanche-ops` -is written in Rust, so you must either install Rust and compile `avalanche-ops` or -download the latest from the `avalanche-ops` repo. +Fuji using [AWS CloudFormation](https://aws.amazon.com/cloudformation/). -#### Option 1: Install Rust and Compile -Install [Rust](https://www.rust-lang.org/tools/install): -```bash -curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -``` - -Compile `avalanche-ops`: -```bash -git clone git@github.com:ava-labs/avalanche-ops.git -cd ./avalanche-ops -./scripts/build.release.sh -mv ./target/release/avalancheup-aws /tmp/avalancheup-aws -chmod +x /tmp/avalancheup-aws -/tmp/avalancheup-aws --help -``` - -#### Option 2: Download Pre-Built CLI -You can view a full list of pre-built binaries on the [latest release page](https://github.com/ava-labs/avalanche-ops/releases/tag/latest). +You can view a full list of pre-built binaries on the [latest release +page](https://github.com/ava-labs/avalanche-ops/releases/tag/latest). -##### Install `avalanche-ops` on Mac +#### Option 1: Install `avalanche-ops` on Mac ```bash export LINUX_ARCH_TYPE=$(uname -m) echo ${LINUX_ARCH_TYPE} @@ -38,7 +20,7 @@ chmod +x /tmp/avalancheup-aws /tmp/avalancheup-aws --help ``` -##### Install `avalanche-ops` on Linux +#### Option 2: Install `avalanche-ops` on Linux ```bash export LINUX_ARCH_TYPE=$(uname -m) echo ${LINUX_ARCH_TYPE} @@ -50,7 +32,8 @@ chmod +x /tmp/avalancheup-aws ``` ### Step 2: Install and Configure `aws-cli` -Next, make sure to install the [AWS CLI](https://docs.aws.amazon.com/cli/latest/userguide/getting-started-install.html). +Next, install the [AWS CLI](https://docs.aws.amazon.com/cli/latest/userguide/getting-started-install.html). This is used to +manipulate CloudFormation. Once you've installed the AWS CLI, run `aws configure` to set the access key to use while deploying your devnet. From bdf3b961109d64dca4ac56b05a0dfc803c63f063 Mon Sep 17 00:00:00 2001 From: Patrick O'Grady Date: Tue, 28 Mar 2023 14:58:32 -0400 Subject: [PATCH 11/67] save progress --- examples/tokenvm/DEVNETS.md | 32 +++++++++++++++++++++----------- examples/tokenvm/go.mod | 2 +- 2 files changed, 22 insertions(+), 12 deletions(-) diff --git a/examples/tokenvm/DEVNETS.md b/examples/tokenvm/DEVNETS.md index fbb665a866..345ce933d1 100644 --- a/examples/tokenvm/DEVNETS.md +++ b/examples/tokenvm/DEVNETS.md @@ -11,22 +11,22 @@ page](https://github.com/ava-labs/avalanche-ops/releases/tag/latest). #### Option 1: Install `avalanche-ops` on Mac ```bash -export LINUX_ARCH_TYPE=$(uname -m) -echo ${LINUX_ARCH_TYPE} -rm -f ./avalancheup-aws.${LINUX_ARCH_TYPE}-apple-darwin -wget "https://github.com/ava-labs/avalanche-ops/releases/download/latest/avalancheup-aws.${LINUX_ARCH_TYPE}-apple-darwin" -mv ./avalancheup-aws.${LINUX_ARCH_TYPE}-apple-darwin /tmp/avalancheup-aws +export ARCH_TYPE=$(uname -m) +echo ${ARCH_TYPE} +rm -f ./tmp/avalancheup-aws +wget "https://github.com/ava-labs/avalanche-ops/releases/download/latest/avalancheup-aws.${ARCH_TYPE}-apple-darwin" +mv ./avalancheup-aws.${ARCH_TYPE}-apple-darwin /tmp/avalancheup-aws chmod +x /tmp/avalancheup-aws /tmp/avalancheup-aws --help ``` #### Option 2: Install `avalanche-ops` on Linux ```bash -export LINUX_ARCH_TYPE=$(uname -m) -echo ${LINUX_ARCH_TYPE} -rm -f ./avalancheup-aws.${LINUX_ARCH_TYPE}-linux-gnu -wget "https://github.com/ava-labs/avalanche-ops/releases/download/latest/avalancheup-aws.${LINUX_ARCH_TYPE}-linux-gnu" -mv ./avalancheup-aws.${LINUX_ARCH_TYPE}-linux-gnu /tmp/avalancheup-aws +export ARCH_TYPE=$(uname -m) +echo ${ARCH_TYPE} +rm -f ./tmp/avalancheup-aws +wget "https://github.com/ava-labs/avalanche-ops/releases/download/latest/avalancheup-aws.${ARCH_TYPE}-linux-gnu" +mv ./avalancheup-aws.${ARCH_TYPE}-linux-gnu /tmp/avalancheup-aws chmod +x /tmp/avalancheup-aws /tmp/avalancheup-aws --help ``` @@ -38,7 +38,17 @@ manipulate CloudFormation. Once you've installed the AWS CLI, run `aws configure` to set the access key to use while deploying your devnet. -### Step 3: Download `token-cli` and `tokenvm` +### Step 3: Download `token-cli` +```bash +``` + +### Step 4: Download `tokenvm` +`tokenvm` is currently compiled with `GOAMD=v1`. If you want to significantly +improve the performance of cryptographic operations, you should consider +building with [`v3+`](https://github.com/golang/go/wiki/MinimumRequirements#amd64). + +```bash +``` ### Step 3: Plan Local Network Deploy diff --git a/examples/tokenvm/go.mod b/examples/tokenvm/go.mod index 2d99506f7f..3b4336629d 100644 --- a/examples/tokenvm/go.mod +++ b/examples/tokenvm/go.mod @@ -13,6 +13,7 @@ require ( github.com/prometheus/client_golang v1.14.0 github.com/spf13/cobra v1.6.1 go.uber.org/zap v1.24.0 + gopkg.in/yaml.v2 v2.4.0 ) require ( @@ -138,7 +139,6 @@ require ( gopkg.in/ini.v1 v1.66.4 // indirect gopkg.in/natefinch/lumberjack.v2 v2.0.0 // indirect gopkg.in/urfave/cli.v1 v1.20.0 // indirect - gopkg.in/yaml.v2 v2.4.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect ) From 8dfba9c7dde1cac6ca916f616c90cb6ed9f314df Mon Sep 17 00:00:00 2001 From: Patrick O'Grady Date: Tue, 28 Mar 2023 15:25:47 -0400 Subject: [PATCH 12/67] more cleanup --- examples/tokenvm/DEVNETS.md | 95 +++++++++++++++++++++++++++---------- 1 file changed, 69 insertions(+), 26 deletions(-) diff --git a/examples/tokenvm/DEVNETS.md b/examples/tokenvm/DEVNETS.md index 345ce933d1..2a9a5b21a9 100644 --- a/examples/tokenvm/DEVNETS.md +++ b/examples/tokenvm/DEVNETS.md @@ -51,7 +51,7 @@ building with [`v3+`](https://github.com/golang/go/wiki/MinimumRequirements#amd6 ``` -### Step 3: Plan Local Network Deploy +### Step 5: Plan Local Network Deploy Now we can spin up a new network of 6 nodes with some defaults: - `avalanche-ops` supports [Graviton-based processors](https://aws.amazon.com/ec2/graviton/) (ARM64). Use `--arch-type arm64` to run nodes in ARM64 CPUs. @@ -94,9 +94,29 @@ avalancheup-aws default-spec \ It is recommended to specify your own artifacts to avoid flaky github release page downloads. -### Step 4: Apply Local Network Deploy +#### Restricting Instance Types +By default, `avalanche-ops` will attempt to use all available instance types in +a region in your cluster. If you'd like to restrict which instance types are used, +edit the output `aops-custom-****-***.yaml` file before deploying: +```yaml +machine: + anchor_nodes: 3 + non_anchor_nodes: 3 + arch_type: amd64 + rust_os_type: ubuntu20.04 + instance_types: + - t3a.2xlarge + - t3.2xlarge + - t2.2xlarge + - c6a.2xlarge + - m6a.2xlarge + - m5.2xlarge + - c5.2xlarge +``` + +### Step 6: Apply Local Network Deploy -The `default-spec` command itselt does not create any resources, and instead outputs the following `apply` and `delete` commands that you can copy and paste: +The `default-spec` command itself does not create any resources, and instead outputs the following `apply` and `delete` commands that you can copy and paste: ```bash # first make sure you have access to the AWS credential @@ -138,12 +158,14 @@ group to restrict the inbound rules to your IP. where AWS SDK call hangs, which blocks node bootstraps. If a node takes too long to start, connect to that instance (e..g, use SSM sesson from your AWS console), and restart the agent with the command `sudo systemctl restart avalanched-aws`. -### Step 5: Install Subnets +### Step 7: Generate Configs Now that the network and nodes are up, let's install two subnets, each of which runs its own `tokenvm`. Use the following commands to do so (similar to [scripts/run.sh](./scripts/run.sh)): ```bash +rm -rf /tmp/avalanche-ops + avalancheup-aws subnet-config \ --log-level=info \ --proposer-min-block-delay 0 \ @@ -154,28 +176,28 @@ cat < /tmp/avalanche-ops/allocations.json EOF rm -f /tmp/avalanche-ops/tokenvm-genesis.json /tmp/token-cli genesis generate /tmp/avalanche-ops/allocations.json \ ---genesis-file /tmp/tokenvm-genesis.json -cat /tmp/tokenvm-genesis.json +--genesis-file /tmp/avalanche-ops/tokenvm-genesis.json +cat /tmp/avalanche-ops/tokenvm-genesis.json -cat < /tmp/tokenvm-chain-config.json +cat < /tmp/avalanche-ops/tokenvm-chain-config.json { "mempoolSize": 10000000, + "mempoolPayerSize": 10000000, "mempoolExemptPayers":["token1rvzhmceq997zntgvravfagsks6w0ryud3rylh4cdvayry0dl97nsjzf3yp"], - "parallelism": 5, + "streamingBacklogSize": 10000000, "trackedPairs":["*"], - "logLevel": "info", - "stateSyncServerDelay": 0 + "logLevel": "info" } EOF -cat /tmp/tokenvm-chain-config.json +cat /tmp/avalanche-ops/tokenvm-chain-config.json ``` -Note that `--chain-name tokenvm*` and `--node-ids-to-instance-ids` values are different to split nodes into two separate subnets. And `avalancheup-aws install-subnet-chain` **will add the specified nodes as primary network validators (regardless of it's for customer or public networks) before adding them as subnet validators**. So, please make sure the `--key` has enough balance and use `--primary-network-validate-period-in-days` and `--subnet-validate-period-in-days` flags to set custom validate priods (defaults to 16-day for primary network, 14-day for subnet validation): +### Step 8: Install Chains +You can run the following commands to spin up 2 `tokenvm` Devnets. Make sure to +replace the `***` fields, IP addresses, key, and `node-ids-to-instance-ids` with your own data: ```bash -# this will prompt you to confirm the outcome! -# so make sure to check the outputs! -avalancheup-aws install-subnet-chain \ +/tmp/avalancheup-aws install-subnet-chain \ --log-level info \ --region us-west-2 \ --s3-bucket avalanche-ops-***-us-west-2 \ @@ -185,13 +207,13 @@ avalancheup-aws install-subnet-chain \ --key 0x56289e99c94b6912bfc12adc093c9b51124f0dc54ac7a766b2bc5ccf558d8027 \ --primary-network-validate-period-in-days 16 \ --subnet-validate-period-in-days 14 \ ---subnet-config-local-path /tmp/subnet-config.json \ +--subnet-config-local-path /tmp/avalanche-ops/subnet-config.json \ --subnet-config-remote-dir /data/avalanche-configs/subnets \ ---vm-binary-local-path /tmp/tokenvm \ +--vm-binary-local-path /tmp/avalanche-ops/tokenvm \ --vm-binary-remote-dir /data/avalanche-plugins \ --chain-name tokenvm1 \ ---chain-genesis-path /tmp/tokenvm-genesis.json \ ---chain-config-local-path /tmp/tokenvm-chain-config.json \ +--chain-genesis-path /tmp/avalanche-ops/tokenvm-genesis.json \ +--chain-config-local-path /tmp/avalanche-ops/tokenvm-chain-config.json \ --chain-config-remote-dir /data/avalanche-configs/chains \ --avalanchego-config-remote-path /data/avalanche-configs/config.json \ --node-ids-to-instance-ids '{"NodeID-31HAxw7wYJ2u2HQHBkwwF26bnFuxnh2sa":"i-04f6ea239218440f0","NodeID-PLM2si9LWaqvzid6AeJa9tft7rzDXXKg2":"i-0459af0e4cf31fcfa","NodeID-D1RtuFSbmcTiRVY991vkq5UvsTBUpHHLR":"i-0d44cfe5420370c30"}' @@ -200,7 +222,7 @@ avalancheup-aws install-subnet-chain \ # this will prompt you to confirm the outcome! # so make sure to check the outputs! -avalancheup-aws install-subnet-chain \ +/tmp/avalancheup-aws install-subnet-chain \ --log-level info \ --region us-west-2 \ --s3-bucket avalanche-ops-***-us-west-2 \ @@ -210,13 +232,13 @@ avalancheup-aws install-subnet-chain \ --key 0x56289e99c94b6912bfc12adc093c9b51124f0dc54ac7a766b2bc5ccf558d8027 \ --primary-network-validate-period-in-days 16 \ --subnet-validate-period-in-days 14 \ ---subnet-config-local-path /tmp/subnet-config.json \ +--subnet-config-local-path /tmp/avalanche-ops/subnet-config.json \ --subnet-config-remote-dir /data/avalanche-configs/subnets \ ---vm-binary-local-path /tmp/tokenvm \ +--vm-binary-local-path /tmp/avalanche-ops/tokenvm \ --vm-binary-remote-dir /data/avalanche-plugins \ --chain-name tokenvm2 \ ---chain-genesis-path /tmp/tokenvm-genesis.json \ ---chain-config-local-path /tmp/tokenvm-chain-config.json \ +--chain-genesis-path /tmp/avalanche-ops/tokenvm-genesis.json \ +--chain-config-local-path /tmp/avalanche-ops/tokenvm-chain-config.json \ --chain-config-remote-dir /data/avalanche-configs/chains \ --avalanchego-config-remote-path /data/avalanche-configs/config.json \ --node-ids-to-instance-ids '{"NodeID-PNc5fwhKLDLGHBF81qnjTwsdjFbdcZxn1":"i-0c5114f6b1e48e671","NodeID-8oqy47xm46RcdTNFxYo8dpBJtPFqQLLeG":"i-083ccebe7b8d40bc9","NodeID-HC9HVTiThxL6d55bNGqV3bkKPwQmLvEs1":"i-0806a42d5bb597dc5"}' @@ -224,7 +246,15 @@ avalancheup-aws install-subnet-chain \ # SUCCESS: subnet Id 2km1PAKNK4vSfpyjgK2iFFoBFD3mXHBAWfDBVsPuzKtNYdPRTY, blockchain Id F1c3GayrV5E7NQdxUCbVfnRPsMSxGbTuHvJLS4R48M3Dcazs6 ``` -To make sure chains are launched successfully, just check the `health` endpoints: +*Note*: `--chain-name tokenvm*` and `--node-ids-to-instance-ids` values are different to split nodes +into two separate subnets. And `avalancheup-aws install-subnet-chain` **will add the specified nodes +as primary network validators (regardless of it's for customer or public networks) before adding them +as subnet validators**. So, please make sure the `--key` has enough balance and use +`--primary-network-validate-period-in-days` and `--subnet-validate-period-in-days` flags to set custom +validate priods (defaults to 16-day for primary network, 14-day for subnet validation). + +### Step 9: Check Healthiness +To make sure chains are launched successfully, query the `health` endpoints: > curl http://52.88.8.107:9650/ext/health > @@ -234,6 +264,17 @@ To make sure chains are launched successfully, just check the `health` endpoints > > {"checks":{"C":{"message":{"consensus":{"longestRunningBlock":"0s","outstandingBlocks":0},"vm":null},"timestamp":"2023-03-14T15:55:08.203325635Z","duration":6570},"F1c3GayrV5E7NQdxUCbVfnRPsMSxGbTuHvJLS4R48M3Dcazs6":{"message":{"consensus":{"longestRunningBlock":"0s","outstandingBlocks":0},"vm":{"database":{"v1.4.5":null},"health":200}},"timestamp":"2023-03-14T15:55:08.204161784Z","duration":927270},"P":{"message":{"consensus":{"longestRunningBlock":"0s","outstandingBlocks":0},"vm":{"2km1PAKNK4vSfpyjgK2iFFoBFD3mXHBAWfDBVsPuzKtNYdPRTY-percentConnected":1,"primary-percentConnected":1}},"timestamp":"2023-03-14T15:55:08.203318015Z","duration":63060},"X":{"message":{"consensus":{"outstandingVertices":0,"snowstorm":{"outstandingTransactions":0}},"vm":null},"timestamp":"2023-03-14T15:55:08.203233714Z","duration":18360},"bootstrapped":{"message":[],"timestamp":"2023-03-14T15:55:08.203205494Z","duration":3010},"database":{"timestamp":"2023-03-14T15:55:08.203214634Z","duration":1350},"diskspace":{"message":{"availableDiskBytes":299770839040},"timestamp":"2023-03-14T15:55:08.203252895Z","duration":5361},"network":{"message":{"connectedPeers":5,"sendFailRate":0,"timeSinceLastMsgReceived":"203.209294ms","timeSinceLastMsgSent":"203.209294ms"},"timestamp":"2023-03-14T15:55:08.203212714Z","duration":6550},"router":{"message":{"longestRunningRequest":"0s","outstandingRequests":0},"timestamp":"2023-03-14T15:55:08.203338225Z","duration":21220}},"healthy":true} +### Step 10: Run Spam +You can import the demo key and the network configuration from `avalanche-ops` +using the following commands: +```bash +/tmp/token-cli key import demo.pk +/tmp/token-cli chain import-ops +``` + +Once the network information is imported, you can then run `/tmp/token-cli spam +run` to push network activity on the devnet. + ## Deploy TokenVM in Fuji network with avalanche-ops Same as above, except you use `--network-name fuji` and do not need anchor nodes: @@ -251,4 +292,6 @@ avalancheup-aws default-spec \ --network-name fuji ``` -And make sure the nodes are in sync with the chain state before installing subnets/chains with `avalancheup-aws install-subnet-chain`. You can check the status of the nodes either via HTTP `/health` endpoints or CloudWatch logs. +Make sure the nodes are in sync with the chain state before installing +subnets/chains with `avalancheup-aws install-subnet-chain`. You can check the status +of the nodes either via HTTP `/health` endpoints or CloudWatch logs. From 1f140df9dfd8d3168de99af4d2d213004ce1edad Mon Sep 17 00:00:00 2001 From: Patrick O'Grady Date: Tue, 28 Mar 2023 15:42:24 -0400 Subject: [PATCH 13/67] first pass at downloads --- examples/tokenvm/DEVNETS.md | 19 ++++++++++++++++++- 1 file changed, 18 insertions(+), 1 deletion(-) diff --git a/examples/tokenvm/DEVNETS.md b/examples/tokenvm/DEVNETS.md index 2a9a5b21a9..a29d192957 100644 --- a/examples/tokenvm/DEVNETS.md +++ b/examples/tokenvm/DEVNETS.md @@ -38,8 +38,17 @@ manipulate CloudFormation. Once you've installed the AWS CLI, run `aws configure` to set the access key to use while deploying your devnet. -### Step 3: Download `token-cli` +### Step 3: Install `token-cli` ```bash +export ARCH_TYPE=$(uname -m) +echo ${ARCH_TYPE} +export OS_TYPE=$(uname | tr '[:upper:]' '[:lower:]') +echo ${OS_TYPE} +export HYPERSDK_VERSION="0.0.4" +rm -f ./tmp/token-cli +wget "https://github.com/ava-labs/hypersdk/releases/download/v${HYPERSDK_VERSION}/hypersdk_${HYPERSDK_VERSION}_${OS_TYPE}_${ARCH_TYPE}.tar.gz +tar -xvf hypersdk_${HYPERSDK_VERSION}_${OS_TYPE}_${ARCH_TYPE}.tar.gz +mv hypersdk_${HYPERSDK_VERSION}_${OS_TYPE}_${ARCH_TYPE}/token-cli /tmp/token-cli ``` ### Step 4: Download `tokenvm` @@ -48,8 +57,16 @@ improve the performance of cryptographic operations, you should consider building with [`v3+`](https://github.com/golang/go/wiki/MinimumRequirements#amd64). ```bash +export HYPERSDK_VERSION="0.0.4" +rm -f ./tmp/tokenvm +wget "https://github.com/ava-labs/hypersdk/releases/download/v${HYPERSDK_VERSION}/hypersdk_${HYPERSDK_VERSION}_linux_amd64.tar.gz +tar -xvf hypersdk_${HYPERSDK_VERSION}_linux_amd64.tar.gz +mv hypersdk_${HYPERSDK_VERSION}_linux_amd64/tokenvm /tmp/tokenvm ``` +*Note*: We must install the linux build because that is compatible with the AWS +deployment target. If you use Graivtron CPUs, make sure to use `arm64` here. + ### Step 5: Plan Local Network Deploy From 59a04c4dba894cb9b69435a6548742fa16faade9 Mon Sep 17 00:00:00 2001 From: Patrick O'Grady Date: Tue, 28 Mar 2023 15:43:57 -0400 Subject: [PATCH 14/67] one more tidbit --- examples/tokenvm/DEVNETS.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/examples/tokenvm/DEVNETS.md b/examples/tokenvm/DEVNETS.md index a29d192957..e09dc5b76b 100644 --- a/examples/tokenvm/DEVNETS.md +++ b/examples/tokenvm/DEVNETS.md @@ -168,6 +168,12 @@ avalancheup-aws delete \ That is, `apply` creates AWS resources, whereas `delete` destroys after testing is done. +If you see the following command, that is expected (it means `avalanche-ops` is +reusing a S3 bucket it previously created): +``` +[2023-03-23T23:49:01Z WARN aws_manager::s3] bucket already exists so returning early (original error 'service error') +``` + *SECURITY*: By default, the SSH and HTTP ports are open to public. Once you complete provisioning the nodes, go to EC2 security group to restrict the inbound rules to your IP. From b3c06b960fef0f15d96b7d085c04c0c42ce71026 Mon Sep 17 00:00:00 2001 From: Patrick O'Grady Date: Tue, 28 Mar 2023 15:45:31 -0400 Subject: [PATCH 15/67] tmp nit --- examples/tokenvm/DEVNETS.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/examples/tokenvm/DEVNETS.md b/examples/tokenvm/DEVNETS.md index e09dc5b76b..e4770913c2 100644 --- a/examples/tokenvm/DEVNETS.md +++ b/examples/tokenvm/DEVNETS.md @@ -13,7 +13,7 @@ page](https://github.com/ava-labs/avalanche-ops/releases/tag/latest). ```bash export ARCH_TYPE=$(uname -m) echo ${ARCH_TYPE} -rm -f ./tmp/avalancheup-aws +rm -f /tmp/avalancheup-aws wget "https://github.com/ava-labs/avalanche-ops/releases/download/latest/avalancheup-aws.${ARCH_TYPE}-apple-darwin" mv ./avalancheup-aws.${ARCH_TYPE}-apple-darwin /tmp/avalancheup-aws chmod +x /tmp/avalancheup-aws @@ -24,7 +24,7 @@ chmod +x /tmp/avalancheup-aws ```bash export ARCH_TYPE=$(uname -m) echo ${ARCH_TYPE} -rm -f ./tmp/avalancheup-aws +rm -f /tmp/avalancheup-aws wget "https://github.com/ava-labs/avalanche-ops/releases/download/latest/avalancheup-aws.${ARCH_TYPE}-linux-gnu" mv ./avalancheup-aws.${ARCH_TYPE}-linux-gnu /tmp/avalancheup-aws chmod +x /tmp/avalancheup-aws From ace6bf321b6f62709cbe6ecfc0216ca6c7567a32 Mon Sep 17 00:00:00 2001 From: Patrick O'Grady Date: Tue, 28 Mar 2023 15:48:23 -0400 Subject: [PATCH 16/67] cleanup commands --- examples/tokenvm/DEVNETS.md | 12 ++++-------- 1 file changed, 4 insertions(+), 8 deletions(-) diff --git a/examples/tokenvm/DEVNETS.md b/examples/tokenvm/DEVNETS.md index e4770913c2..20f6c66392 100644 --- a/examples/tokenvm/DEVNETS.md +++ b/examples/tokenvm/DEVNETS.md @@ -11,22 +11,18 @@ page](https://github.com/ava-labs/avalanche-ops/releases/tag/latest). #### Option 1: Install `avalanche-ops` on Mac ```bash -export ARCH_TYPE=$(uname -m) -echo ${ARCH_TYPE} rm -f /tmp/avalancheup-aws -wget "https://github.com/ava-labs/avalanche-ops/releases/download/latest/avalancheup-aws.${ARCH_TYPE}-apple-darwin" -mv ./avalancheup-aws.${ARCH_TYPE}-apple-darwin /tmp/avalancheup-aws +wget "https://github.com/ava-labs/avalanche-ops/releases/download/latest/avalancheup-aws.aarch64-apple-darwin" +mv ./avalancheup-aws.aarch64-apple-darwin /tmp/avalancheup-aws chmod +x /tmp/avalancheup-aws /tmp/avalancheup-aws --help ``` #### Option 2: Install `avalanche-ops` on Linux ```bash -export ARCH_TYPE=$(uname -m) -echo ${ARCH_TYPE} rm -f /tmp/avalancheup-aws -wget "https://github.com/ava-labs/avalanche-ops/releases/download/latest/avalancheup-aws.${ARCH_TYPE}-linux-gnu" -mv ./avalancheup-aws.${ARCH_TYPE}-linux-gnu /tmp/avalancheup-aws +wget "https://github.com/ava-labs/avalanche-ops/releases/download/latest/avalancheup-aws.x86_64-linux-gnu" +mv ./avalancheup-aws.x86_64-linux-gnu /tmp/avalancheup-aws chmod +x /tmp/avalancheup-aws /tmp/avalancheup-aws --help ``` From 71dff034eb11be12aedd18bbb84e19a2f9da83d4 Mon Sep 17 00:00:00 2001 From: Patrick O'Grady Date: Tue, 28 Mar 2023 15:49:30 -0400 Subject: [PATCH 17/67] missing quote --- examples/tokenvm/DEVNETS.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/examples/tokenvm/DEVNETS.md b/examples/tokenvm/DEVNETS.md index 20f6c66392..0bcf463392 100644 --- a/examples/tokenvm/DEVNETS.md +++ b/examples/tokenvm/DEVNETS.md @@ -42,7 +42,7 @@ export OS_TYPE=$(uname | tr '[:upper:]' '[:lower:]') echo ${OS_TYPE} export HYPERSDK_VERSION="0.0.4" rm -f ./tmp/token-cli -wget "https://github.com/ava-labs/hypersdk/releases/download/v${HYPERSDK_VERSION}/hypersdk_${HYPERSDK_VERSION}_${OS_TYPE}_${ARCH_TYPE}.tar.gz +wget "https://github.com/ava-labs/hypersdk/releases/download/v${HYPERSDK_VERSION}/hypersdk_${HYPERSDK_VERSION}_${OS_TYPE}_${ARCH_TYPE}.tar.gz" tar -xvf hypersdk_${HYPERSDK_VERSION}_${OS_TYPE}_${ARCH_TYPE}.tar.gz mv hypersdk_${HYPERSDK_VERSION}_${OS_TYPE}_${ARCH_TYPE}/token-cli /tmp/token-cli ``` @@ -55,7 +55,7 @@ building with [`v3+`](https://github.com/golang/go/wiki/MinimumRequirements#amd6 ```bash export HYPERSDK_VERSION="0.0.4" rm -f ./tmp/tokenvm -wget "https://github.com/ava-labs/hypersdk/releases/download/v${HYPERSDK_VERSION}/hypersdk_${HYPERSDK_VERSION}_linux_amd64.tar.gz +wget "https://github.com/ava-labs/hypersdk/releases/download/v${HYPERSDK_VERSION}/hypersdk_${HYPERSDK_VERSION}_linux_amd64.tar.gz" tar -xvf hypersdk_${HYPERSDK_VERSION}_linux_amd64.tar.gz mv hypersdk_${HYPERSDK_VERSION}_linux_amd64/tokenvm /tmp/tokenvm ``` From 6a9e06d11a99d4d5b7a938ef0af36277867fee75 Mon Sep 17 00:00:00 2001 From: Patrick O'Grady Date: Tue, 28 Mar 2023 15:52:32 -0400 Subject: [PATCH 18/67] more command cleanup --- examples/tokenvm/DEVNETS.md | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/examples/tokenvm/DEVNETS.md b/examples/tokenvm/DEVNETS.md index 0bcf463392..aab63a099f 100644 --- a/examples/tokenvm/DEVNETS.md +++ b/examples/tokenvm/DEVNETS.md @@ -43,8 +43,10 @@ echo ${OS_TYPE} export HYPERSDK_VERSION="0.0.4" rm -f ./tmp/token-cli wget "https://github.com/ava-labs/hypersdk/releases/download/v${HYPERSDK_VERSION}/hypersdk_${HYPERSDK_VERSION}_${OS_TYPE}_${ARCH_TYPE}.tar.gz" -tar -xvf hypersdk_${HYPERSDK_VERSION}_${OS_TYPE}_${ARCH_TYPE}.tar.gz +tar -xvf hypersdk_${HYPERSDK_VERSION}_${OS_TYPE}_${ARCH_TYPE}.tar.gz -C hypersdk_${HYPERSDK_VERSION}_${OS_TYPE}_${ARCH_TYPE} mv hypersdk_${HYPERSDK_VERSION}_${OS_TYPE}_${ARCH_TYPE}/token-cli /tmp/token-cli +rm -rf hypersdk_${HYPERSDK_VERSION}_${OS_TYPE}_${ARCH_TYPE} +rm -rf hypersdk_${HYPERSDK_VERSION}_${OS_TYPE}_${ARCH_TYPE}.tar.gz ``` ### Step 4: Download `tokenvm` @@ -56,8 +58,10 @@ building with [`v3+`](https://github.com/golang/go/wiki/MinimumRequirements#amd6 export HYPERSDK_VERSION="0.0.4" rm -f ./tmp/tokenvm wget "https://github.com/ava-labs/hypersdk/releases/download/v${HYPERSDK_VERSION}/hypersdk_${HYPERSDK_VERSION}_linux_amd64.tar.gz" -tar -xvf hypersdk_${HYPERSDK_VERSION}_linux_amd64.tar.gz +tar -xvf hypersdk_${HYPERSDK_VERSION}_linux_amd64.tar.gz -C hypersdk_${HYPERSDK_VERSION}_linux_amd64/tokenvm mv hypersdk_${HYPERSDK_VERSION}_linux_amd64/tokenvm /tmp/tokenvm +rm -rf hypersdk_${HYPERSDK_VERSION}_linux_amd64/tokenvm +rm -rf hypersdk_${HYPERSDK_VERSION}_linux_amd64.tar.gz ``` *Note*: We must install the linux build because that is compatible with the AWS From d1ac0b1330c0c0b12a22e66638041c31ef4859a4 Mon Sep 17 00:00:00 2001 From: Patrick O'Grady Date: Tue, 28 Mar 2023 15:55:17 -0400 Subject: [PATCH 19/67] more path cleanup --- examples/tokenvm/DEVNETS.md | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/examples/tokenvm/DEVNETS.md b/examples/tokenvm/DEVNETS.md index aab63a099f..01c5f6b6df 100644 --- a/examples/tokenvm/DEVNETS.md +++ b/examples/tokenvm/DEVNETS.md @@ -43,10 +43,11 @@ echo ${OS_TYPE} export HYPERSDK_VERSION="0.0.4" rm -f ./tmp/token-cli wget "https://github.com/ava-labs/hypersdk/releases/download/v${HYPERSDK_VERSION}/hypersdk_${HYPERSDK_VERSION}_${OS_TYPE}_${ARCH_TYPE}.tar.gz" -tar -xvf hypersdk_${HYPERSDK_VERSION}_${OS_TYPE}_${ARCH_TYPE}.tar.gz -C hypersdk_${HYPERSDK_VERSION}_${OS_TYPE}_${ARCH_TYPE} -mv hypersdk_${HYPERSDK_VERSION}_${OS_TYPE}_${ARCH_TYPE}/token-cli /tmp/token-cli -rm -rf hypersdk_${HYPERSDK_VERSION}_${OS_TYPE}_${ARCH_TYPE} +mkdir tmp-hypersdk +tar -xvf hypersdk_${HYPERSDK_VERSION}_${OS_TYPE}_${ARCH_TYPE}.tar.gz -C tmp-hypersdk rm -rf hypersdk_${HYPERSDK_VERSION}_${OS_TYPE}_${ARCH_TYPE}.tar.gz +mv tmp-hypersdk/token-cli /tmp/token-cli +rm -rf tmp-hypersdk ``` ### Step 4: Download `tokenvm` @@ -58,10 +59,11 @@ building with [`v3+`](https://github.com/golang/go/wiki/MinimumRequirements#amd6 export HYPERSDK_VERSION="0.0.4" rm -f ./tmp/tokenvm wget "https://github.com/ava-labs/hypersdk/releases/download/v${HYPERSDK_VERSION}/hypersdk_${HYPERSDK_VERSION}_linux_amd64.tar.gz" -tar -xvf hypersdk_${HYPERSDK_VERSION}_linux_amd64.tar.gz -C hypersdk_${HYPERSDK_VERSION}_linux_amd64/tokenvm -mv hypersdk_${HYPERSDK_VERSION}_linux_amd64/tokenvm /tmp/tokenvm -rm -rf hypersdk_${HYPERSDK_VERSION}_linux_amd64/tokenvm +mkdir tmp-hypersdk +tar -xvf hypersdk_${HYPERSDK_VERSION}_linux_amd64.tar.gz -C tmp-hypersdk rm -rf hypersdk_${HYPERSDK_VERSION}_linux_amd64.tar.gz +mv tmp-hypersdk/tokenvm /tmp/tokenvm +rm -rf tmp-hypersdk ``` *Note*: We must install the linux build because that is compatible with the AWS From 0486c439c85057c55222353f0983eaaa25ca888c Mon Sep 17 00:00:00 2001 From: Patrick O'Grady Date: Tue, 28 Mar 2023 16:10:27 -0400 Subject: [PATCH 20/67] update instructions --- examples/tokenvm/DEVNETS.md | 23 ++++++++++++++++++++--- 1 file changed, 20 insertions(+), 3 deletions(-) diff --git a/examples/tokenvm/DEVNETS.md b/examples/tokenvm/DEVNETS.md index 01c5f6b6df..e9e0e4dd8b 100644 --- a/examples/tokenvm/DEVNETS.md +++ b/examples/tokenvm/DEVNETS.md @@ -289,7 +289,7 @@ To make sure chains are launched successfully, query the `health` endpoints: > > {"checks":{"C":{"message":{"consensus":{"longestRunningBlock":"0s","outstandingBlocks":0},"vm":null},"timestamp":"2023-03-14T15:55:08.203325635Z","duration":6570},"F1c3GayrV5E7NQdxUCbVfnRPsMSxGbTuHvJLS4R48M3Dcazs6":{"message":{"consensus":{"longestRunningBlock":"0s","outstandingBlocks":0},"vm":{"database":{"v1.4.5":null},"health":200}},"timestamp":"2023-03-14T15:55:08.204161784Z","duration":927270},"P":{"message":{"consensus":{"longestRunningBlock":"0s","outstandingBlocks":0},"vm":{"2km1PAKNK4vSfpyjgK2iFFoBFD3mXHBAWfDBVsPuzKtNYdPRTY-percentConnected":1,"primary-percentConnected":1}},"timestamp":"2023-03-14T15:55:08.203318015Z","duration":63060},"X":{"message":{"consensus":{"outstandingVertices":0,"snowstorm":{"outstandingTransactions":0}},"vm":null},"timestamp":"2023-03-14T15:55:08.203233714Z","duration":18360},"bootstrapped":{"message":[],"timestamp":"2023-03-14T15:55:08.203205494Z","duration":3010},"database":{"timestamp":"2023-03-14T15:55:08.203214634Z","duration":1350},"diskspace":{"message":{"availableDiskBytes":299770839040},"timestamp":"2023-03-14T15:55:08.203252895Z","duration":5361},"network":{"message":{"connectedPeers":5,"sendFailRate":0,"timeSinceLastMsgReceived":"203.209294ms","timeSinceLastMsgSent":"203.209294ms"},"timestamp":"2023-03-14T15:55:08.203212714Z","duration":6550},"router":{"message":{"longestRunningRequest":"0s","outstandingRequests":0},"timestamp":"2023-03-14T15:55:08.203338225Z","duration":21220}},"healthy":true} -### Step 10: Run Spam +### Step 10: Initialize `token-cli` You can import the demo key and the network configuration from `avalanche-ops` using the following commands: ```bash @@ -297,8 +297,25 @@ using the following commands: /tmp/token-cli chain import-ops ``` -Once the network information is imported, you can then run `/tmp/token-cli spam -run` to push network activity on the devnet. +### Step 11: Start Integrated Block Explorer +To view activity on each Subnet you created, you can run the `token-cli`'s +integrated block explorer. To do this, run the following command: +```bash +/tmp/token-cli chain watch --hide-txs +``` + +If you don't plan to load test the devnet, you may wush to just run the +following command to get additional transaction details: +```bash +/tmp/token-cli chain watch +``` + +### Step 12: Run Load +Once the network information is imported, you can then run the following +command to drive an arbitrary amount of load: +```bash +/tmp/token-cli spam run +``` ## Deploy TokenVM in Fuji network with avalanche-ops From fbc55d1dd6cebf71277f7f94c531db1e73c31c2e Mon Sep 17 00:00:00 2001 From: Patrick O'Grady Date: Tue, 28 Mar 2023 16:11:07 -0400 Subject: [PATCH 21/67] add correct input for import-ops command --- examples/tokenvm/DEVNETS.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/tokenvm/DEVNETS.md b/examples/tokenvm/DEVNETS.md index e9e0e4dd8b..f85bc93cc1 100644 --- a/examples/tokenvm/DEVNETS.md +++ b/examples/tokenvm/DEVNETS.md @@ -294,7 +294,7 @@ You can import the demo key and the network configuration from `avalanche-ops` using the following commands: ```bash /tmp/token-cli key import demo.pk -/tmp/token-cli chain import-ops +/tmp/token-cli chain import-ops ``` ### Step 11: Start Integrated Block Explorer From 1a85afd3f7a4ab231b4d77406df3c5b8a904e0ca Mon Sep 17 00:00:00 2001 From: Patrick O'Grady Date: Tue, 28 Mar 2023 16:22:45 -0400 Subject: [PATCH 22/67] more tweaks to simplify instructions --- examples/tokenvm/DEVNETS.md | 59 +++++++++++-------------------------- 1 file changed, 17 insertions(+), 42 deletions(-) diff --git a/examples/tokenvm/DEVNETS.md b/examples/tokenvm/DEVNETS.md index f85bc93cc1..f88004d92b 100644 --- a/examples/tokenvm/DEVNETS.md +++ b/examples/tokenvm/DEVNETS.md @@ -191,7 +191,7 @@ do so (similar to [scripts/run.sh](./scripts/run.sh)): ```bash rm -rf /tmp/avalanche-ops -avalancheup-aws subnet-config \ +/tmp/avalancheup-aws subnet-config \ --log-level=info \ --proposer-min-block-delay 0 \ --file-path /tmp/avalanche-ops/subnet-config.json @@ -225,36 +225,32 @@ replace the `***` fields, IP addresses, key, and `node-ids-to-instance-ids` with /tmp/avalancheup-aws install-subnet-chain \ --log-level info \ --region us-west-2 \ ---s3-bucket avalanche-ops-***-us-west-2 \ ---s3-key-prefix aops-custom-***/install-subnet-chain \ ---ssm-doc aops-custom-***-ssm-install-subnet-chain \ ---chain-rpc-url http://52.88.8.107:9650 \ ---key 0x56289e99c94b6912bfc12adc093c9b51124f0dc54ac7a766b2bc5ccf558d8027 \ +--s3-bucket \ +--s3-key-prefix \ +--ssm-doc \ +--chain-rpc-url \ +--key \ --primary-network-validate-period-in-days 16 \ --subnet-validate-period-in-days 14 \ --subnet-config-local-path /tmp/avalanche-ops/subnet-config.json \ --subnet-config-remote-dir /data/avalanche-configs/subnets \ ---vm-binary-local-path /tmp/avalanche-ops/tokenvm \ +--vm-binary-local-path /tmp/tokenvm \ --vm-binary-remote-dir /data/avalanche-plugins \ --chain-name tokenvm1 \ --chain-genesis-path /tmp/avalanche-ops/tokenvm-genesis.json \ --chain-config-local-path /tmp/avalanche-ops/tokenvm-chain-config.json \ --chain-config-remote-dir /data/avalanche-configs/chains \ --avalanchego-config-remote-path /data/avalanche-configs/config.json \ ---node-ids-to-instance-ids '{"NodeID-31HAxw7wYJ2u2HQHBkwwF26bnFuxnh2sa":"i-04f6ea239218440f0","NodeID-PLM2si9LWaqvzid6AeJa9tft7rzDXXKg2":"i-0459af0e4cf31fcfa","NodeID-D1RtuFSbmcTiRVY991vkq5UvsTBUpHHLR":"i-0d44cfe5420370c30"}' -# VM Id 'tHBYNu8ikt25R77fH4znHYC4B5mkaEnXPFmsJnECZjq59dySw', chain name 'tokenvm1' -# SUCCESS: subnet Id 2DLqm2Wk4SFtdmeqkmyfAvTdfvNKmbjwZwztMqRaQCwnDbRHCo, blockchain Id G9CbuiKLoyeYhabA8ph7TBiisKt9LS6Hx1QRgoajnwxd1xFC8 +--node-ids-to-instance-ids -# this will prompt you to confirm the outcome! -# so make sure to check the outputs! /tmp/avalancheup-aws install-subnet-chain \ --log-level info \ --region us-west-2 \ ---s3-bucket avalanche-ops-***-us-west-2 \ ---s3-key-prefix aops-custom-***/install-subnet-chain \ ---ssm-doc aops-custom-***-ssm-install-subnet-chain \ ---chain-rpc-url http://52.88.8.107:9650 \ ---key 0x56289e99c94b6912bfc12adc093c9b51124f0dc54ac7a766b2bc5ccf558d8027 \ +--s3-bucket \ +--s3-key-prefix \ +--ssm-doc \ +--chain-rpc-url \ +--key \ --primary-network-validate-period-in-days 16 \ --subnet-validate-period-in-days 14 \ --subnet-config-local-path /tmp/avalanche-ops/subnet-config.json \ @@ -266,30 +262,10 @@ replace the `***` fields, IP addresses, key, and `node-ids-to-instance-ids` with --chain-config-local-path /tmp/avalanche-ops/tokenvm-chain-config.json \ --chain-config-remote-dir /data/avalanche-configs/chains \ --avalanchego-config-remote-path /data/avalanche-configs/config.json \ ---node-ids-to-instance-ids '{"NodeID-PNc5fwhKLDLGHBF81qnjTwsdjFbdcZxn1":"i-0c5114f6b1e48e671","NodeID-8oqy47xm46RcdTNFxYo8dpBJtPFqQLLeG":"i-083ccebe7b8d40bc9","NodeID-HC9HVTiThxL6d55bNGqV3bkKPwQmLvEs1":"i-0806a42d5bb597dc5"}' -# VM Id 'tHBYNu8ikt4i8cEV4nsSuj7Ldc9sXAHc8L6qKRJR4e5CR7T3t', chain name 'tokenvm2' -# SUCCESS: subnet Id 2km1PAKNK4vSfpyjgK2iFFoBFD3mXHBAWfDBVsPuzKtNYdPRTY, blockchain Id F1c3GayrV5E7NQdxUCbVfnRPsMSxGbTuHvJLS4R48M3Dcazs6 +--node-ids-to-instance-ids ``` -*Note*: `--chain-name tokenvm*` and `--node-ids-to-instance-ids` values are different to split nodes -into two separate subnets. And `avalancheup-aws install-subnet-chain` **will add the specified nodes -as primary network validators (regardless of it's for customer or public networks) before adding them -as subnet validators**. So, please make sure the `--key` has enough balance and use -`--primary-network-validate-period-in-days` and `--subnet-validate-period-in-days` flags to set custom -validate priods (defaults to 16-day for primary network, 14-day for subnet validation). - -### Step 9: Check Healthiness -To make sure chains are launched successfully, query the `health` endpoints: - -> curl http://52.88.8.107:9650/ext/health -> -> {"checks":{"C":{"message":{"consensus":{"longestRunningBlock":"0s","outstandingBlocks":0},"vm":null},"timestamp":"2023-03-14T15:55:08.256378577Z","duration":14020},"F1c3GayrV5E7NQdxUCbVfnRPsMSxGbTuHvJLS4R48M3Dcazs6":{"message":{"consensus":{"longestRunningBlock":"0s","outstandingBlocks":0},"vm":{"database":{"v1.4.5":null},"health":200}},"timestamp":"2023-03-14T15:55:08.257445968Z","duration":1061711},"P":{"message":{"consensus":{"longestRunningBlock":"0s","outstandingBlocks":0},"vm":{"2km1PAKNK4vSfpyjgK2iFFoBFD3mXHBAWfDBVsPuzKtNYdPRTY-percentConnected":1,"primary-percentConnected":1}},"timestamp":"2023-03-14T15:55:08.256454548Z","duration":74871},"X":{"message":{"consensus":{"outstandingVertices":0,"snowstorm":{"outstandingTransactions":0}},"vm":null},"timestamp":"2023-03-14T15:55:08.256459848Z","duration":15850},"bootstrapped":{"message":[],"timestamp":"2023-03-14T15:55:08.256438778Z","duration":6020},"database":{"timestamp":"2023-03-14T15:55:08.256464299Z","duration":1420},"diskspace":{"message":{"availableDiskBytes":299770843136},"timestamp":"2023-03-14T15:55:08.256466309Z","duration":4070},"network":{"message":{"connectedPeers":5,"sendFailRate":0,"timeSinceLastMsgReceived":"256.459548ms","timeSinceLastMsgSent":"256.459548ms"},"timestamp":"2023-03-14T15:55:08.256461929Z","duration":5551},"router":{"message":{"longestRunningRequest":"0s","outstandingRequests":0},"timestamp":"2023-03-14T15:55:08.256363507Z","duration":17231}},"healthy":true} - -> curl http://44.224.148.127:9650/ext/health -> -> {"checks":{"C":{"message":{"consensus":{"longestRunningBlock":"0s","outstandingBlocks":0},"vm":null},"timestamp":"2023-03-14T15:55:08.203325635Z","duration":6570},"F1c3GayrV5E7NQdxUCbVfnRPsMSxGbTuHvJLS4R48M3Dcazs6":{"message":{"consensus":{"longestRunningBlock":"0s","outstandingBlocks":0},"vm":{"database":{"v1.4.5":null},"health":200}},"timestamp":"2023-03-14T15:55:08.204161784Z","duration":927270},"P":{"message":{"consensus":{"longestRunningBlock":"0s","outstandingBlocks":0},"vm":{"2km1PAKNK4vSfpyjgK2iFFoBFD3mXHBAWfDBVsPuzKtNYdPRTY-percentConnected":1,"primary-percentConnected":1}},"timestamp":"2023-03-14T15:55:08.203318015Z","duration":63060},"X":{"message":{"consensus":{"outstandingVertices":0,"snowstorm":{"outstandingTransactions":0}},"vm":null},"timestamp":"2023-03-14T15:55:08.203233714Z","duration":18360},"bootstrapped":{"message":[],"timestamp":"2023-03-14T15:55:08.203205494Z","duration":3010},"database":{"timestamp":"2023-03-14T15:55:08.203214634Z","duration":1350},"diskspace":{"message":{"availableDiskBytes":299770839040},"timestamp":"2023-03-14T15:55:08.203252895Z","duration":5361},"network":{"message":{"connectedPeers":5,"sendFailRate":0,"timeSinceLastMsgReceived":"203.209294ms","timeSinceLastMsgSent":"203.209294ms"},"timestamp":"2023-03-14T15:55:08.203212714Z","duration":6550},"router":{"message":{"longestRunningRequest":"0s","outstandingRequests":0},"timestamp":"2023-03-14T15:55:08.203338225Z","duration":21220}},"healthy":true} - -### Step 10: Initialize `token-cli` +### Step 9: Initialize `token-cli` You can import the demo key and the network configuration from `avalanche-ops` using the following commands: ```bash @@ -297,7 +273,7 @@ using the following commands: /tmp/token-cli chain import-ops ``` -### Step 11: Start Integrated Block Explorer +### Step 10: Start Integrated Block Explorer To view activity on each Subnet you created, you can run the `token-cli`'s integrated block explorer. To do this, run the following command: ```bash @@ -310,7 +286,7 @@ following command to get additional transaction details: /tmp/token-cli chain watch ``` -### Step 12: Run Load +### Step 11: Run Load Once the network information is imported, you can then run the following command to drive an arbitrary amount of load: ```bash @@ -318,7 +294,6 @@ command to drive an arbitrary amount of load: ``` ## Deploy TokenVM in Fuji network with avalanche-ops - Same as above, except you use `--network-name fuji` and do not need anchor nodes: ```bash From 801b6fc3695cc1222b6b958d83440f23b47c3a6e Mon Sep 17 00:00:00 2001 From: Patrick O'Grady Date: Tue, 28 Mar 2023 16:27:02 -0400 Subject: [PATCH 23/67] make second deploy optional --- examples/tokenvm/DEVNETS.md | 49 ++++++++++++++++++++----------------- 1 file changed, 27 insertions(+), 22 deletions(-) diff --git a/examples/tokenvm/DEVNETS.md b/examples/tokenvm/DEVNETS.md index f88004d92b..bec173ab18 100644 --- a/examples/tokenvm/DEVNETS.md +++ b/examples/tokenvm/DEVNETS.md @@ -220,7 +220,6 @@ cat /tmp/avalanche-ops/tokenvm-chain-config.json ### Step 8: Install Chains You can run the following commands to spin up 2 `tokenvm` Devnets. Make sure to replace the `***` fields, IP addresses, key, and `node-ids-to-instance-ids` with your own data: - ```bash /tmp/avalancheup-aws install-subnet-chain \ --log-level info \ @@ -242,27 +241,6 @@ replace the `***` fields, IP addresses, key, and `node-ids-to-instance-ids` with --chain-config-remote-dir /data/avalanche-configs/chains \ --avalanchego-config-remote-path /data/avalanche-configs/config.json \ --node-ids-to-instance-ids - -/tmp/avalancheup-aws install-subnet-chain \ ---log-level info \ ---region us-west-2 \ ---s3-bucket \ ---s3-key-prefix \ ---ssm-doc \ ---chain-rpc-url \ ---key \ ---primary-network-validate-period-in-days 16 \ ---subnet-validate-period-in-days 14 \ ---subnet-config-local-path /tmp/avalanche-ops/subnet-config.json \ ---subnet-config-remote-dir /data/avalanche-configs/subnets \ ---vm-binary-local-path /tmp/avalanche-ops/tokenvm \ ---vm-binary-remote-dir /data/avalanche-plugins \ ---chain-name tokenvm2 \ ---chain-genesis-path /tmp/avalanche-ops/tokenvm-genesis.json \ ---chain-config-local-path /tmp/avalanche-ops/tokenvm-chain-config.json \ ---chain-config-remote-dir /data/avalanche-configs/chains \ ---avalanchego-config-remote-path /data/avalanche-configs/config.json \ ---node-ids-to-instance-ids ``` ### Step 9: Initialize `token-cli` @@ -293,6 +271,33 @@ command to drive an arbitrary amount of load: /tmp/token-cli spam run ``` +### [OPTIONAL] Step 12: Deploy Another Subnet +To test Avalanche Warp Messaging, you must be running at least 2 Subnets. To do +so, just replicate the command you ran above with a different `--chain-name` (and +a different set of validators): +```bash +/tmp/avalancheup-aws install-subnet-chain \ +--log-level info \ +--region us-west-2 \ +--s3-bucket \ +--s3-key-prefix \ +--ssm-doc \ +--chain-rpc-url \ +--key \ +--primary-network-validate-period-in-days 16 \ +--subnet-validate-period-in-days 14 \ +--subnet-config-local-path /tmp/avalanche-ops/subnet-config.json \ +--subnet-config-remote-dir /data/avalanche-configs/subnets \ +--vm-binary-local-path /tmp/tokenvm \ +--vm-binary-remote-dir /data/avalanche-plugins \ +--chain-name tokenvm2 \ +--chain-genesis-path /tmp/avalanche-ops/tokenvm-genesis.json \ +--chain-config-local-path /tmp/avalanche-ops/tokenvm-chain-config.json \ +--chain-config-remote-dir /data/avalanche-configs/chains \ +--avalanchego-config-remote-path /data/avalanche-configs/config.json \ +--node-ids-to-instance-ids +``` + ## Deploy TokenVM in Fuji network with avalanche-ops Same as above, except you use `--network-name fuji` and do not need anchor nodes: From 85bb73de8ffd762cad82055e0a039daa3ac1edd5 Mon Sep 17 00:00:00 2001 From: Patrick O'Grady Date: Tue, 28 Mar 2023 16:41:44 -0400 Subject: [PATCH 24/67] add more clarify to balance check --- examples/tokenvm/DEVNETS.md | 3 ++- examples/tokenvm/cmd/token-cli/cmd/key.go | 21 ++++++++++++++++++--- examples/tokenvm/cmd/token-cli/cmd/root.go | 7 +++++++ 3 files changed, 27 insertions(+), 4 deletions(-) diff --git a/examples/tokenvm/DEVNETS.md b/examples/tokenvm/DEVNETS.md index bec173ab18..64ebf6e056 100644 --- a/examples/tokenvm/DEVNETS.md +++ b/examples/tokenvm/DEVNETS.md @@ -248,7 +248,8 @@ You can import the demo key and the network configuration from `avalanche-ops` using the following commands: ```bash /tmp/token-cli key import demo.pk -/tmp/token-cli chain import-ops +/tmp/token-cli chain import-ops +/tmp/token-cli key balance --check-all-chains ``` ### Step 10: Start Integrated Block Explorer diff --git a/examples/tokenvm/cmd/token-cli/cmd/key.go b/examples/tokenvm/cmd/token-cli/cmd/key.go index 55ba1bb7d8..a449872408 100644 --- a/examples/tokenvm/cmd/token-cli/cmd/key.go +++ b/examples/tokenvm/cmd/token-cli/cmd/key.go @@ -124,7 +124,12 @@ var balanceKeyCmd = &cobra.Command{ Use: "balance", RunE: func(*cobra.Command, []string) error { ctx := context.Background() - _, priv, _, cli, err := defaultActor() + + priv, err := GetDefaultKey() + if err != nil { + return err + } + _, uris, err := GetDefaultChain() if err != nil { return err } @@ -133,7 +138,17 @@ var balanceKeyCmd = &cobra.Command{ if err != nil { return err } - _, _, err = getAssetInfo(ctx, cli, priv.PublicKey(), assetID, true) - return err + + max := len(uris) + if !checkAllChains { + max = 1 + } + for _, uri := range uris[:max] { + hutils.Outf("{{yellow}}uri:{{/}} %s\n", uri) + if _, _, err = getAssetInfo(ctx, client.New(uri), priv.PublicKey(), assetID, true); err != nil { + return err + } + } + return nil }, } diff --git a/examples/tokenvm/cmd/token-cli/cmd/root.go b/examples/tokenvm/cmd/token-cli/cmd/root.go index f942ab6f37..485f60f7ad 100644 --- a/examples/tokenvm/cmd/token-cli/cmd/root.go +++ b/examples/tokenvm/cmd/token-cli/cmd/root.go @@ -30,6 +30,7 @@ var ( randomRecipient bool maxTxBacklog int deleteOtherChains bool + checkAllChains bool rootCmd = &cobra.Command{ Use: "token-cli", @@ -82,6 +83,12 @@ func init() { ) // key + balanceKeyCmd.PersistentFlags().BoolVar( + &checkAllChains, + "check-all-chains", + false, + "check all chains", + ) keyCmd.AddCommand( genKeyCmd, importKeyCmd, From 9a78a12907387c4d996bf2ba5015cd2b91e48d0b Mon Sep 17 00:00:00 2001 From: Patrick O'Grady Date: Tue, 28 Mar 2023 16:45:23 -0400 Subject: [PATCH 25/67] add improved default listner port --- examples/tokenvm/DEVNETS.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/examples/tokenvm/DEVNETS.md b/examples/tokenvm/DEVNETS.md index 64ebf6e056..fc2867a432 100644 --- a/examples/tokenvm/DEVNETS.md +++ b/examples/tokenvm/DEVNETS.md @@ -211,7 +211,9 @@ cat < /tmp/avalanche-ops/tokenvm-chain-config.json "mempoolExemptPayers":["token1rvzhmceq997zntgvravfagsks6w0ryud3rylh4cdvayry0dl97nsjzf3yp"], "streamingBacklogSize": 10000000, "trackedPairs":["*"], - "logLevel": "info" + "logLevel": "info", + "decisionsPort": 9652, + "blocksPort": 9653, } EOF cat /tmp/avalanche-ops/tokenvm-chain-config.json From 99afb5c17038efe8bd1b2bdef8caaff0a07e7336 Mon Sep 17 00:00:00 2001 From: Patrick O'Grady Date: Tue, 28 Mar 2023 16:48:04 -0400 Subject: [PATCH 26/67] more logging --- examples/tokenvm/cmd/token-cli/cmd/chain.go | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/examples/tokenvm/cmd/token-cli/cmd/chain.go b/examples/tokenvm/cmd/token-cli/cmd/chain.go index 6c125bfd17..398fe4dc34 100644 --- a/examples/tokenvm/cmd/token-cli/cmd/chain.go +++ b/examples/tokenvm/cmd/token-cli/cmd/chain.go @@ -17,6 +17,7 @@ import ( "github.com/ava-labs/avalanchego/ids" "github.com/ava-labs/avalanchego/utils/logging" "github.com/ava-labs/hypersdk/utils" + hutils "github.com/ava-labs/hypersdk/utils" "github.com/ava-labs/hypersdk/vm" "github.com/spf13/cobra" "gopkg.in/yaml.v2" @@ -243,7 +244,9 @@ var watchChainCmd = &cobra.Command{ if err != nil { return err } - scli, err := vm.NewBlockRPCClient(fmt.Sprintf("%s:%d", host, port)) + uri := fmt.Sprintf("%s:%d", host, port) + hutils.Outf("{{yellow}}uri:{{/}} %s\n", uri) + scli, err := vm.NewBlockRPCClient(uri) if err != nil { return err } From 80f9f208f56da488e8caea64b721255114036f89 Mon Sep 17 00:00:00 2001 From: Patrick O'Grady Date: Wed, 29 Mar 2023 11:14:46 -0400 Subject: [PATCH 27/67] fix static lint --- examples/tokenvm/cmd/token-cli/cmd/chain.go | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/examples/tokenvm/cmd/token-cli/cmd/chain.go b/examples/tokenvm/cmd/token-cli/cmd/chain.go index 398fe4dc34..309e83e51b 100644 --- a/examples/tokenvm/cmd/token-cli/cmd/chain.go +++ b/examples/tokenvm/cmd/token-cli/cmd/chain.go @@ -17,7 +17,6 @@ import ( "github.com/ava-labs/avalanchego/ids" "github.com/ava-labs/avalanchego/utils/logging" "github.com/ava-labs/hypersdk/utils" - hutils "github.com/ava-labs/hypersdk/utils" "github.com/ava-labs/hypersdk/vm" "github.com/spf13/cobra" "gopkg.in/yaml.v2" @@ -245,7 +244,7 @@ var watchChainCmd = &cobra.Command{ return err } uri := fmt.Sprintf("%s:%d", host, port) - hutils.Outf("{{yellow}}uri:{{/}} %s\n", uri) + utils.Outf("{{yellow}}uri:{{/}} %s\n", uri) scli, err := vm.NewBlockRPCClient(uri) if err != nil { return err From 701663d884ee5225704ca5eec4670e10266acc3a Mon Sep 17 00:00:00 2001 From: Patrick O'Grady Date: Wed, 29 Mar 2023 12:14:21 -0400 Subject: [PATCH 28/67] add log viewing to DEVNETS --- examples/tokenvm/DEVNETS.md | 25 ++++++++++++++++++++++++- 1 file changed, 24 insertions(+), 1 deletion(-) diff --git a/examples/tokenvm/DEVNETS.md b/examples/tokenvm/DEVNETS.md index fc2867a432..02e7012ad9 100644 --- a/examples/tokenvm/DEVNETS.md +++ b/examples/tokenvm/DEVNETS.md @@ -213,7 +213,7 @@ cat < /tmp/avalanche-ops/tokenvm-chain-config.json "trackedPairs":["*"], "logLevel": "info", "decisionsPort": 9652, - "blocksPort": 9653, + "blocksPort": 9653 } EOF cat /tmp/avalanche-ops/tokenvm-chain-config.json @@ -245,6 +245,23 @@ replace the `***` fields, IP addresses, key, and `node-ids-to-instance-ids` with --node-ids-to-instance-ids ``` +#### Viewing Logs +1) Open the [AWS CloudWatch](https://aws.amazon.com/cloudwatch) product on your +AWS Console +2) Click "Logs Insights" +3) Use the following query to view all logs (in reverse-chronological order) +for all nodes in your Devnet: +``` +fields @timestamp, @message, @logStream, @log +| filter(@logStream not like "avalanche-telemetry-cloudwatch.log") +| filter(@logStream not like "syslog") +| sort @timestamp desc +| limit 20 +``` + +*Note*: The "Log Group" you are asked to select should have a similar name as +the spec file that was output earlier. + ### Step 9: Initialize `token-cli` You can import the demo key and the network configuration from `avalanche-ops` using the following commands: @@ -254,6 +271,12 @@ using the following commands: /tmp/token-cli key balance --check-all-chains ``` +If you previously used `token-cli`, you may want to delete data you previously +ingested into it. You can do this with the following command: +```bash +rm -rf .token-cli +``` + ### Step 10: Start Integrated Block Explorer To view activity on each Subnet you created, you can run the `token-cli`'s integrated block explorer. To do this, run the following command: From 4ece0c4c6c7b72b5b677c3af4d4ca92f7dd812cb Mon Sep 17 00:00:00 2001 From: Patrick O'Grady Date: Wed, 29 Mar 2023 14:14:15 -0400 Subject: [PATCH 29/67] add TODO to optimize unmarshaling --- chain/block.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/chain/block.go b/chain/block.go index 008452c121..454a423555 100644 --- a/chain/block.go +++ b/chain/block.go @@ -135,6 +135,8 @@ func ParseBlock( ctx, span := vm.Tracer().Start(ctx, "chain.ParseBlock") defer span.End() + // TODO: avoid un-marshaling again if we already have ID + // (https://github.com/ava-labs/hypersdk/issues/125) blk, err := UnmarshalBlock(source, vm) if err != nil { return nil, err From 3c4428f5bd2a456c5a7208dc1fe541e0e1301143 Mon Sep 17 00:00:00 2001 From: Patrick O'Grady Date: Wed, 29 Mar 2023 22:00:17 -0400 Subject: [PATCH 30/67] remove unnecessary todo --- chain/block.go | 2 -- 1 file changed, 2 deletions(-) diff --git a/chain/block.go b/chain/block.go index 454a423555..008452c121 100644 --- a/chain/block.go +++ b/chain/block.go @@ -135,8 +135,6 @@ func ParseBlock( ctx, span := vm.Tracer().Start(ctx, "chain.ParseBlock") defer span.End() - // TODO: avoid un-marshaling again if we already have ID - // (https://github.com/ava-labs/hypersdk/issues/125) blk, err := UnmarshalBlock(source, vm) if err != nil { return nil, err From 8c44511ff08d44fd046d03fd9004883419485249 Mon Sep 17 00:00:00 2001 From: Patrick O'Grady Date: Wed, 29 Mar 2023 22:08:40 -0400 Subject: [PATCH 31/67] add prometheus instructions to devnets file --- examples/tokenvm/DEVNETS.md | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/examples/tokenvm/DEVNETS.md b/examples/tokenvm/DEVNETS.md index 02e7012ad9..077e64abd0 100644 --- a/examples/tokenvm/DEVNETS.md +++ b/examples/tokenvm/DEVNETS.md @@ -262,6 +262,29 @@ fields @timestamp, @message, @logStream, @log *Note*: The "Log Group" you are asked to select should have a similar name as the spec file that was output earlier. +#### Viewing Metrics +To view metrics, first download and install [Prometheus](https://prometheus.io/download/) +using the following commands: +```bash +wget https://github.com/prometheus/prometheus/releases/download/v2.43.0/prometheus-2.43.0.darwin-amd64.tar.gz +tar -xvf prometheus-2.43.0.darwin-amd64.tar.gz +rm prometheus-2.43.0.darwin-amd64.tar.gz +mv prometheus-2.43.0.darwin-amd64/prometheus /tmp/prometheus +rm -rf prometheus-2.43.0.darwin-amd64 +``` + +Once you have Prometheus installed, run the following command to auto-generate +a configuration file: +```bash +/tmp/token-cli metrics prometheus /tmp/prometheus.yml +``` + +In a separate terminal, then run the following command to view collected +metrics: +```bash +/tmp/prometheus --config.file=/tmp/prometheus.yml +``` + ### Step 9: Initialize `token-cli` You can import the demo key and the network configuration from `avalanche-ops` using the following commands: From 6f11b9c7907473c120f48cc3367edd75f959122b Mon Sep 17 00:00:00 2001 From: Patrick O'Grady Date: Wed, 29 Mar 2023 22:15:20 -0400 Subject: [PATCH 32/67] wait for 10 runs to start issuing txs --- examples/tokenvm/cmd/token-cli/cmd/spam.go | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/examples/tokenvm/cmd/token-cli/cmd/spam.go b/examples/tokenvm/cmd/token-cli/cmd/spam.go index 008c0b67d0..972a6069eb 100644 --- a/examples/tokenvm/cmd/token-cli/cmd/spam.go +++ b/examples/tokenvm/cmd/token-cli/cmd/spam.go @@ -252,6 +252,7 @@ var runSpamCmd = &cobra.Command{ // broadcast txs t := time.NewTimer(0) // ensure no duplicates created defer t.Stop() + var runs int for !exiting { select { case <-t.C: @@ -308,6 +309,14 @@ var runSpamCmd = &cobra.Command{ issuer.l.Lock() issuer.outstandingTxs++ issuer.l.Unlock() + + // Only send 1 transaction per second until we are sure Snowman++ is + // activated. + if runs < 10 { + hutils.Outf("{{yellow}}only sending 1 tx in %d run{{/}}\n", runs) + runs++ + break + } } l.Lock() infl.Lock() From 983a86f17979680a67b96e3bf68895ca1a6d2738 Mon Sep 17 00:00:00 2001 From: Patrick O'Grady Date: Wed, 29 Mar 2023 22:22:38 -0400 Subject: [PATCH 33/67] add more configs to custom genesis --- examples/tokenvm/DEVNETS.md | 4 ++- examples/tokenvm/cmd/token-cli/cmd/genesis.go | 6 ++++ examples/tokenvm/cmd/token-cli/cmd/root.go | 34 +++++++++++++------ 3 files changed, 33 insertions(+), 11 deletions(-) diff --git a/examples/tokenvm/DEVNETS.md b/examples/tokenvm/DEVNETS.md index 077e64abd0..1bd1611e1b 100644 --- a/examples/tokenvm/DEVNETS.md +++ b/examples/tokenvm/DEVNETS.md @@ -201,7 +201,9 @@ cat < /tmp/avalanche-ops/allocations.json EOF rm -f /tmp/avalanche-ops/tokenvm-genesis.json /tmp/token-cli genesis generate /tmp/avalanche-ops/allocations.json \ ---genesis-file /tmp/avalanche-ops/tokenvm-genesis.json +--genesis-file /tmp/avalanche-ops/tokenvm-genesis.json \ +--window-target-units 100000000000 \ +--window-target-blocks 20 cat /tmp/avalanche-ops/tokenvm-genesis.json cat < /tmp/avalanche-ops/tokenvm-chain-config.json diff --git a/examples/tokenvm/cmd/token-cli/cmd/genesis.go b/examples/tokenvm/cmd/token-cli/cmd/genesis.go index dc92e414fa..f86fdf2f4b 100644 --- a/examples/tokenvm/cmd/token-cli/cmd/genesis.go +++ b/examples/tokenvm/cmd/token-cli/cmd/genesis.go @@ -34,6 +34,12 @@ var genGenesisCmd = &cobra.Command{ if minUnitPrice >= 0 { g.MinUnitPrice = uint64(minUnitPrice) } + if windowTargetUnits >= 0 { + g.WindowTargetUnits = uint64(windowTargetUnits) + } + if windowTargetBlocks >= 0 { + g.WindowTargetBlocks = uint64(windowTargetBlocks) + } a, err := os.ReadFile(args[0]) if err != nil { diff --git a/examples/tokenvm/cmd/token-cli/cmd/root.go b/examples/tokenvm/cmd/token-cli/cmd/root.go index 485f60f7ad..cfeea3d5d8 100644 --- a/examples/tokenvm/cmd/token-cli/cmd/root.go +++ b/examples/tokenvm/cmd/token-cli/cmd/root.go @@ -24,13 +24,15 @@ var ( dbPath string db database.Database - genesisFile string - minUnitPrice int64 - hideTxs bool - randomRecipient bool - maxTxBacklog int - deleteOtherChains bool - checkAllChains bool + genesisFile string + minUnitPrice int64 + windowTargetUnits int64 + windowTargetBlocks int64 + hideTxs bool + randomRecipient bool + maxTxBacklog int + deleteOtherChains bool + checkAllChains bool rootCmd = &cobra.Command{ Use: "token-cli", @@ -66,9 +68,6 @@ func init() { rootCmd.SilenceErrors = true // genesis - genesisCmd.AddCommand( - genGenesisCmd, - ) genGenesisCmd.PersistentFlags().StringVar( &genesisFile, "genesis-file", @@ -81,6 +80,21 @@ func init() { -1, "minimum price", ) + genGenesisCmd.PersistentFlags().Int64Var( + &windowTargetUnits, + "window-target-units", + -1, + "window target units", + ) + genGenesisCmd.PersistentFlags().Int64Var( + &windowTargetBlocks, + "window-target-blocks", + -1, + "window target blocks", + ) + genesisCmd.AddCommand( + genGenesisCmd, + ) // key balanceKeyCmd.PersistentFlags().BoolVar( From 325958d698e6ef237189a9ef18d01ffd490cecec Mon Sep 17 00:00:00 2001 From: Patrick O'Grady Date: Wed, 29 Mar 2023 22:43:45 -0400 Subject: [PATCH 34/67] add metrics file auto-generation --- examples/tokenvm/DEVNETS.md | 7 +- examples/tokenvm/cmd/token-cli/cmd/metrics.go | 84 +++++++++++++++++++ examples/tokenvm/cmd/token-cli/cmd/root.go | 6 ++ 3 files changed, 95 insertions(+), 2 deletions(-) create mode 100644 examples/tokenvm/cmd/token-cli/cmd/metrics.go diff --git a/examples/tokenvm/DEVNETS.md b/examples/tokenvm/DEVNETS.md index 1bd1611e1b..d730092ffd 100644 --- a/examples/tokenvm/DEVNETS.md +++ b/examples/tokenvm/DEVNETS.md @@ -278,15 +278,18 @@ rm -rf prometheus-2.43.0.darwin-amd64 Once you have Prometheus installed, run the following command to auto-generate a configuration file: ```bash -/tmp/token-cli metrics prometheus /tmp/prometheus.yml +/tmp/token-cli metrics prometheus /tmp/prometheus.yaml ``` In a separate terminal, then run the following command to view collected metrics: ```bash -/tmp/prometheus --config.file=/tmp/prometheus.yml +/tmp/prometheus --config.file=/tmp/prometheus.yaml ``` +To remove previously ingested data, delete for a folder called `data` in the +directory where you last ran Prometheus. + ### Step 9: Initialize `token-cli` You can import the demo key and the network configuration from `avalanche-ops` using the following commands: diff --git a/examples/tokenvm/cmd/token-cli/cmd/metrics.go b/examples/tokenvm/cmd/token-cli/cmd/metrics.go new file mode 100644 index 0000000000..ab5eea9430 --- /dev/null +++ b/examples/tokenvm/cmd/token-cli/cmd/metrics.go @@ -0,0 +1,84 @@ +// Copyright (C) 2023, Ava Labs, Inc. All rights reserved. +// See the file LICENSE for licensing terms. + +package cmd + +import ( + "os" + "strings" + + "github.com/spf13/cobra" + "gopkg.in/yaml.v2" +) + +var metricsCmd = &cobra.Command{ + Use: "metrics", + RunE: func(*cobra.Command, []string) error { + return ErrMissingSubcommand + }, +} + +type PrometheusStaticConfig struct { + Targets []string `yaml:"targets"` +} + +type PrometheusScrapeConfig struct { + JobName string `yaml:"job_name"` + StaticConfigs []*PrometheusStaticConfig `yaml:"static_configs"` + MetricsPath string `yaml:"metrics_path"` +} + +type PrometheusConfig struct { + Global struct { + ScrapeInterval string `yaml:"scrape_interval"` + EvaluationInterval string `yaml:"evaluation_interval"` + } `yaml:"global"` + ScrapeConfigs []*PrometheusScrapeConfig `yaml:"scrape_configs"` +} + +var prometheusCmd = &cobra.Command{ + Use: "prometheus [path] [output]", + PreRunE: func(cmd *cobra.Command, args []string) error { + if len(args) != 2 { + return ErrInvalidArgs + } + return nil + }, + RunE: func(_ *cobra.Command, args []string) error { + // Load HTTP Endpoints + var opsConfig AvalancheOpsConfig + yamlFile, err := os.ReadFile(args[0]) + if err != nil { + return err + } + err = yaml.Unmarshal(yamlFile, &opsConfig) + if err != nil { + return err + } + endpoints := make([]string, len(opsConfig.Resources.CreatedNodes)) + for i, node := range opsConfig.Resources.CreatedNodes { + endpoints[i] = strings.TrimPrefix(node.HTTPEndpoint, "http://") + } + + // Create Prometheus YAML + var prometheusConfig PrometheusConfig + prometheusConfig.Global.ScrapeInterval = "15s" + prometheusConfig.Global.EvaluationInterval = "15s" + prometheusConfig.ScrapeConfigs = []*PrometheusScrapeConfig{ + { + JobName: "prometheus", + StaticConfigs: []*PrometheusStaticConfig{ + { + Targets: endpoints, + }, + }, + MetricsPath: "/ext/metrics", + }, + } + yamlData, err := yaml.Marshal(&prometheusConfig) + if err != nil { + return err + } + return os.WriteFile(args[1], yamlData, fsModeWrite) + }, +} diff --git a/examples/tokenvm/cmd/token-cli/cmd/root.go b/examples/tokenvm/cmd/token-cli/cmd/root.go index cfeea3d5d8..377de3f516 100644 --- a/examples/tokenvm/cmd/token-cli/cmd/root.go +++ b/examples/tokenvm/cmd/token-cli/cmd/root.go @@ -49,6 +49,7 @@ func init() { chainCmd, actionCmd, spamCmd, + metricsCmd, ) rootCmd.PersistentFlags().StringVar( &dbPath, @@ -165,6 +166,11 @@ func init() { spamCmd.AddCommand( runSpamCmd, ) + + // metrics + metricsCmd.AddCommand( + prometheusCmd, + ) } func Execute() error { From 019261b952ada0432edd6a8921eb02b48ef4fe38 Mon Sep 17 00:00:00 2001 From: Patrick O'Grady Date: Wed, 29 Mar 2023 23:05:17 -0400 Subject: [PATCH 35/67] add useful queries --- examples/tokenvm/DEVNETS.md | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/examples/tokenvm/DEVNETS.md b/examples/tokenvm/DEVNETS.md index d730092ffd..7fa5982f82 100644 --- a/examples/tokenvm/DEVNETS.md +++ b/examples/tokenvm/DEVNETS.md @@ -287,6 +287,20 @@ metrics: /tmp/prometheus --config.file=/tmp/prometheus.yaml ``` +Here are some useful queries (on an example chainID `3rihqpXh6ZJqxL2dsrVysKkEKroiD9tvQWLS6iWVnd8K4HST2`): +* `avalanche_3rihqpXh6ZJqxL2dsrVysKkEKroiD9tvQWLS6iWVnd8K4HST2_blks_processing` +* `avalanche_3rihqpXh6ZJqxL2dsrVysKkEKroiD9tvQWLS6iWVnd8K4HST2_blks_rejected_count` +* `avalanche_3rihqpXh6ZJqxL2dsrVysKkEKroiD9tvQWLS6iWVnd8K4HST2_blks_accepted_count` +* [verify latency in ms] `increase(avalanche_3rihqpXh6ZJqxL2dsrVysKkEKroiD9tvQWLS6iWVnd8K4HST2_vm_metervm_verify_sum[30s:1s])/increase(avalanche_3rihqpXh6ZJqxL2dsrVysKkEKroiD9tvQWLS6iWVnd8K4HST2_vm_metervm_verify_count[30s:1s])/1000000` +* [accept latency in ms] `increase(avalanche_3rihqpXh6ZJqxL2dsrVysKkEKroiD9tvQWLS6iWVnd8K4HST2_vm_metervm_accept_sum[30s:1s])/increase(avalanche_3rihqpXh6ZJqxL2dsrVysKkEKroiD9tvQWLS6iWVnd8K4HST2_vm_metervm_accept_count[30s:1s])/1000000` +* [transactions per second] `deriv(avalanche_3rihqpXh6ZJqxL2dsrVysKkEKroiD9tvQWLS6iWVnd8K4HST2_vm_hyper_sdk_vm_txs_accepted[30s:1s])` +* `avalanche_3rihqpXh6ZJqxL2dsrVysKkEKroiD9tvQWLS6iWVnd8K4HST2_vm_hyper_sdk_vm_txs_accepted` +* [disk used] `300000000000-avalanche_resource_tracker_disk_available_space` +* `avalanche_resource_tracker_cpu_usage` +* `avalanche_3rihqpXh6ZJqxL2dsrVysKkEKroiD9tvQWLS6iWVnd8K4HST2_vm_go_goroutines` +* `avalanche_3rihqpXh6ZJqxL2dsrVysKkEKroiD9tvQWLS6iWVnd8K4HST2_vm_go_memstats_alloc_bytes` +* `avalanche_network_inbound_conn_throttler_rate_limited` + To remove previously ingested data, delete for a folder called `data` in the directory where you last ran Prometheus. From 8175b49e7bf6b4cbf3c5a75cecab0644c97456c9 Mon Sep 17 00:00:00 2001 From: Patrick O'Grady Date: Thu, 30 Mar 2023 00:15:28 -0400 Subject: [PATCH 36/67] make distribution faster --- examples/tokenvm/DEVNETS.md | 3 ++ examples/tokenvm/cmd/token-cli/cmd/spam.go | 51 ++++++++++++++-------- 2 files changed, 36 insertions(+), 18 deletions(-) diff --git a/examples/tokenvm/DEVNETS.md b/examples/tokenvm/DEVNETS.md index 7fa5982f82..4005598219 100644 --- a/examples/tokenvm/DEVNETS.md +++ b/examples/tokenvm/DEVNETS.md @@ -294,9 +294,12 @@ Here are some useful queries (on an example chainID `3rihqpXh6ZJqxL2dsrVysKkEKro * [verify latency in ms] `increase(avalanche_3rihqpXh6ZJqxL2dsrVysKkEKroiD9tvQWLS6iWVnd8K4HST2_vm_metervm_verify_sum[30s:1s])/increase(avalanche_3rihqpXh6ZJqxL2dsrVysKkEKroiD9tvQWLS6iWVnd8K4HST2_vm_metervm_verify_count[30s:1s])/1000000` * [accept latency in ms] `increase(avalanche_3rihqpXh6ZJqxL2dsrVysKkEKroiD9tvQWLS6iWVnd8K4HST2_vm_metervm_accept_sum[30s:1s])/increase(avalanche_3rihqpXh6ZJqxL2dsrVysKkEKroiD9tvQWLS6iWVnd8K4HST2_vm_metervm_accept_count[30s:1s])/1000000` * [transactions per second] `deriv(avalanche_3rihqpXh6ZJqxL2dsrVysKkEKroiD9tvQWLS6iWVnd8K4HST2_vm_hyper_sdk_vm_txs_accepted[30s:1s])` +* [accepted block size] `increase(avalanche_3rihqpXh6ZJqxL2dsrVysKkEKroiD9tvQWLS6iWVnd8K4HST2_blks_accepted_container_size_sum[30s:1s])/increase(avalanche_3rihqpXh6ZJqxL2dsrVysKkEKroiD9tvQWLS6iWVnd8K4HST2_blks_accepted_count[30s:1s])` * `avalanche_3rihqpXh6ZJqxL2dsrVysKkEKroiD9tvQWLS6iWVnd8K4HST2_vm_hyper_sdk_vm_txs_accepted` * [disk used] `300000000000-avalanche_resource_tracker_disk_available_space` * `avalanche_resource_tracker_cpu_usage` +* `avalanche_resource_tracker_disk_reads` +* `avalanche_resource_tracker_disk_writes` * `avalanche_3rihqpXh6ZJqxL2dsrVysKkEKroiD9tvQWLS6iWVnd8K4HST2_vm_go_goroutines` * `avalanche_3rihqpXh6ZJqxL2dsrVysKkEKroiD9tvQWLS6iWVnd8K4HST2_vm_go_memstats_alloc_bytes` * `avalanche_network_inbound_conn_throttler_rate_limited` diff --git a/examples/tokenvm/cmd/token-cli/cmd/spam.go b/examples/tokenvm/cmd/token-cli/cmd/spam.go index 972a6069eb..6a1079bb37 100644 --- a/examples/tokenvm/cmd/token-cli/cmd/spam.go +++ b/examples/tokenvm/cmd/token-cli/cmd/spam.go @@ -130,7 +130,19 @@ var runSpamCmd = &cobra.Command{ assetString(ids.Empty), ) accounts := make([]crypto.PrivateKey, numAccounts) - txs := make([]ids.ID, numAccounts) + port, err := cli.DecisionsPort(ctx) + if err != nil { + return err + } + u, err := url.Parse(uris[0]) + if err != nil { + return err + } + tcpURI := fmt.Sprintf("%s:%d", u.Hostname(), port) + dcli, err := vm.NewDecisionRPCClient(tcpURI) + if err != nil { + return err + } for i := 0; i < numAccounts; i++ { // Create account pk, err := crypto.GeneratePrivateKey() @@ -140,7 +152,7 @@ var runSpamCmd = &cobra.Command{ accounts[i] = pk // Send funds - submit, tx, _, err := cli.GenerateTransaction(ctx, nil, &actions.Transfer{ + _, tx, _, err := cli.GenerateTransaction(ctx, nil, &actions.Transfer{ To: pk.PublicKey(), Asset: ids.Empty, Value: distAmount, @@ -148,17 +160,19 @@ var runSpamCmd = &cobra.Command{ if err != nil { return err } - if err := submit(ctx); err != nil { + if err := dcli.IssueTx(tx); err != nil { return err } - txs[i] = tx.ID() } for i := 0; i < numAccounts; i++ { - success, err := cli.WaitForTransaction(ctx, txs[i]) + _, dErr, result, err := dcli.Listen() if err != nil { return err } - if !success { + if dErr != nil { + return dErr + } + if !result.Success { // Should never happen return ErrTxFailed } @@ -364,8 +378,10 @@ var runSpamCmd = &cobra.Command{ // Return funds hutils.Outf("{{yellow}}returning funds to %s{{/}}\n", utils.Address(key.PublicKey())) - var returnedBalance uint64 - txs = make([]ids.ID, numAccounts) + var ( + returnedBalance uint64 + sent int + ) for i := 0; i < numAccounts; i++ { address := utils.Address(accounts[i].PublicKey()) balance, err := cli.Balance(ctx, address, ids.Empty) @@ -375,9 +391,10 @@ var runSpamCmd = &cobra.Command{ if transferFee > balance { continue } + sent++ // Send funds returnAmt := balance - transferFee - submit, tx, _, err := cli.GenerateTransaction(ctx, nil, &actions.Transfer{ + _, tx, _, err := cli.GenerateTransaction(ctx, nil, &actions.Transfer{ To: key.PublicKey(), Asset: ids.Empty, Value: returnAmt, @@ -385,22 +402,20 @@ var runSpamCmd = &cobra.Command{ if err != nil { return err } - if err := submit(ctx); err != nil { + if err := dcli.IssueTx(tx); err != nil { return err } - txs[i] = tx.ID() returnedBalance += returnAmt } - for i := 0; i < numAccounts; i++ { - if txs[i] == ids.Empty { - // No balance to return - continue - } - success, err := cli.WaitForTransaction(ctx, txs[i]) + for i := 0; i < sent; i++ { + _, dErr, result, err := dcli.Listen() if err != nil { return err } - if !success { + if dErr != nil { + return dErr + } + if !result.Success { // Should never happen return ErrTxFailed } From a4792e0d2429182b4e4c827bf91c427fb633cf7e Mon Sep 17 00:00:00 2001 From: Patrick O'Grady Date: Thu, 30 Mar 2023 00:30:58 -0400 Subject: [PATCH 37/67] make returns much faster --- examples/tokenvm/DEVNETS.md | 7 +++++++ examples/tokenvm/cmd/token-cli/cmd/spam.go | 20 ++++++++++++++------ 2 files changed, 21 insertions(+), 6 deletions(-) diff --git a/examples/tokenvm/DEVNETS.md b/examples/tokenvm/DEVNETS.md index 4005598219..99d4903e05 100644 --- a/examples/tokenvm/DEVNETS.md +++ b/examples/tokenvm/DEVNETS.md @@ -342,6 +342,13 @@ command to drive an arbitrary amount of load: /tmp/token-cli spam run ``` +#### Limiting Inflight Txs +To ensure you don't create too large of a backlog on the network, you can add +run the following command (defaults to `72000`): +```bash +/tmp/token-cli spam run --max-tx-backlog 5000 +``` + ### [OPTIONAL] Step 12: Deploy Another Subnet To test Avalanche Warp Messaging, you must be running at least 2 Subnets. To do so, just replicate the command you ran above with a different `--chain-name` (and diff --git a/examples/tokenvm/cmd/token-cli/cmd/spam.go b/examples/tokenvm/cmd/token-cli/cmd/spam.go index 6a1079bb37..121f67e410 100644 --- a/examples/tokenvm/cmd/token-cli/cmd/spam.go +++ b/examples/tokenvm/cmd/token-cli/cmd/spam.go @@ -143,6 +143,7 @@ var runSpamCmd = &cobra.Command{ if err != nil { return err } + funds := map[crypto.PublicKey]uint64{} for i := 0; i < numAccounts; i++ { // Create account pk, err := crypto.GeneratePrivateKey() @@ -163,6 +164,12 @@ var runSpamCmd = &cobra.Command{ if err := dcli.IssueTx(tx); err != nil { return err } + funds[pk.PublicKey()] = distAmount + + // Ensure Snowman++ is activated + if i < 10 { + time.Sleep(500 * time.Millisecond) + } } for i := 0; i < numAccounts; i++ { _, dErr, result, err := dcli.Listen() @@ -320,6 +327,7 @@ var runSpamCmd = &cobra.Command{ hutils.Outf("{{orange}}failed to issue:{{/}} %v\n", err) continue } + funds[accounts[i].PublicKey()] -= (fees + 1) issuer.l.Lock() issuer.outstandingTxs++ issuer.l.Unlock() @@ -327,7 +335,6 @@ var runSpamCmd = &cobra.Command{ // Only send 1 transaction per second until we are sure Snowman++ is // activated. if runs < 10 { - hutils.Outf("{{yellow}}only sending 1 tx in %d run{{/}}\n", runs) runs++ break } @@ -383,11 +390,7 @@ var runSpamCmd = &cobra.Command{ sent int ) for i := 0; i < numAccounts; i++ { - address := utils.Address(accounts[i].PublicKey()) - balance, err := cli.Balance(ctx, address, ids.Empty) - if err != nil { - return err - } + balance := funds[accounts[i].PublicKey()] if transferFee > balance { continue } @@ -406,6 +409,11 @@ var runSpamCmd = &cobra.Command{ return err } returnedBalance += returnAmt + + // Ensure Snowman++ is activated + if i < 10 { + time.Sleep(500 * time.Millisecond) + } } for i := 0; i < sent; i++ { _, dErr, result, err := dcli.Listen() From ec027ec62454b68e9ea6b5cbe47f2e5e0482fa0a Mon Sep 17 00:00:00 2001 From: Patrick O'Grady Date: Thu, 30 Mar 2023 00:46:17 -0400 Subject: [PATCH 38/67] add directions for rate limit --- examples/tokenvm/DEVNETS.md | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/examples/tokenvm/DEVNETS.md b/examples/tokenvm/DEVNETS.md index 99d4903e05..a6753b9c0c 100644 --- a/examples/tokenvm/DEVNETS.md +++ b/examples/tokenvm/DEVNETS.md @@ -133,6 +133,22 @@ machine: - c5.2xlarge ``` +#### Increasing Rate Limits +If you are attempting to stress test the Devnet, you should disable CPU, Disk, +and Bandwidth rate limiting. You can do this by adding the following lines to +`avalanchego_config` in the spec file: +```yaml +avalanchego_config: + proposervm-use-current-height: true + throttler-inbound-validator-alloc-size: 107374182 + throttler-inbound-node-max-processing-msgs: 100000 + throttler-inbound-bandwidth-refill-rate: 1073741824 + throttler-inbound-bandwidth-max-burst-size: 1073741824 + throttler-inbound-cpu-validator-alloc: 100000 + throttler-inbound-disk-validator-alloc: 10737418240000 + throttler-outbound-validator-alloc-size: 107374182 +``` + ### Step 6: Apply Local Network Deploy The `default-spec` command itself does not create any resources, and instead outputs the following `apply` and `delete` commands that you can copy and paste: @@ -221,6 +237,10 @@ EOF cat /tmp/avalanche-ops/tokenvm-chain-config.json ``` +*Note*: Make sure that port `9652` and `9653` are open on the AWS Security Group applied to all +nodes otherwise the `token-cli` will not work properly. This requirement will +be removed when the HyperSDK migrates to using proper WebSockets. + ### Step 8: Install Chains You can run the following commands to spin up 2 `tokenvm` Devnets. Make sure to replace the `***` fields, IP addresses, key, and `node-ids-to-instance-ids` with your own data: From b24e91b83eb1dd4301bc30e09ee2dc610ecda287 Mon Sep 17 00:00:00 2001 From: Patrick O'Grady Date: Thu, 30 Mar 2023 00:47:43 -0400 Subject: [PATCH 39/67] add directions for RL override --- examples/tokenvm/DEVNETS.md | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/examples/tokenvm/DEVNETS.md b/examples/tokenvm/DEVNETS.md index a6753b9c0c..6f9e3b48f7 100644 --- a/examples/tokenvm/DEVNETS.md +++ b/examples/tokenvm/DEVNETS.md @@ -77,7 +77,6 @@ Now we can spin up a new network of 6 nodes with some defaults: - `avalanche-ops` supports [EC2 Spot instances](https://aws.amazon.com/ec2/spot/) for cost savings. Use `--instance-mode=spot` to run instances in spot mode. ```bash -# launch/download all artifacts for x86 platform /tmp/avalancheup-aws default-spec \ --arch-type amd64 \ --rust-os-type ubuntu20.04 \ @@ -149,6 +148,10 @@ avalanchego_config: throttler-outbound-validator-alloc-size: 107374182 ``` +Make sure to remove `throttler-inbound-at-large-alloc-size` and +`throttler-inbound-node-max-at-large-bytes` from the default YAML, otherwise +you will have duplicate keys. + ### Step 6: Apply Local Network Deploy The `default-spec` command itself does not create any resources, and instead outputs the following `apply` and `delete` commands that you can copy and paste: From 249d7abb11735a1e22bcadd93444810e9357d830 Mon Sep 17 00:00:00 2001 From: Patrick O'Grady Date: Thu, 30 Mar 2023 12:11:27 -0400 Subject: [PATCH 40/67] add instructions for CloudWatch metrics --- examples/tokenvm/DEVNETS.md | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/examples/tokenvm/DEVNETS.md b/examples/tokenvm/DEVNETS.md index 6f9e3b48f7..a5e94b65a6 100644 --- a/examples/tokenvm/DEVNETS.md +++ b/examples/tokenvm/DEVNETS.md @@ -152,6 +152,15 @@ Make sure to remove `throttler-inbound-at-large-alloc-size` and `throttler-inbound-node-max-at-large-bytes` from the default YAML, otherwise you will have duplicate keys. +#### Supporting All Metrics +By default, `avalanche-ops` does not support ingest all metrics into AWS +CloudWatch. To ingest all metrics, change the configured filters in +`upload_artifacts.prometheus_metrics_rules_file_path` to: +```yaml +filters: + - regex: ^*$ +``` + ### Step 6: Apply Local Network Deploy The `default-spec` command itself does not create any resources, and instead outputs the following `apply` and `delete` commands that you can copy and paste: @@ -273,7 +282,7 @@ replace the `***` fields, IP addresses, key, and `node-ids-to-instance-ids` with #### Viewing Logs 1) Open the [AWS CloudWatch](https://aws.amazon.com/cloudwatch) product on your AWS Console -2) Click "Logs Insights" +2) Click "Logs Insights" on the left pane 3) Use the following query to view all logs (in reverse-chronological order) for all nodes in your Devnet: ``` @@ -288,6 +297,13 @@ fields @timestamp, @message, @logStream, @log the spec file that was output earlier. #### Viewing Metrics +#### Option 1: AWS CloudWatch +1) Open the [AWS CloudWatch](https://aws.amazon.com/cloudwatch) product on your +AWS Console +2) Click "All Metrics" on the left pane +3) Click the "Custom Namespace" that matches the name of the spec file + +#### Option 2: Prometheus To view metrics, first download and install [Prometheus](https://prometheus.io/download/) using the following commands: ```bash From d6b9a15b16043f2cbd9a53a07a064dc8b0e70e78 Mon Sep 17 00:00:00 2001 From: Patrick O'Grady Date: Thu, 30 Mar 2023 13:53:38 -0400 Subject: [PATCH 41/67] nits on README --- examples/tokenvm/DEVNETS.md | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/examples/tokenvm/DEVNETS.md b/examples/tokenvm/DEVNETS.md index a5e94b65a6..a837bfbe15 100644 --- a/examples/tokenvm/DEVNETS.md +++ b/examples/tokenvm/DEVNETS.md @@ -41,7 +41,7 @@ echo ${ARCH_TYPE} export OS_TYPE=$(uname | tr '[:upper:]' '[:lower:]') echo ${OS_TYPE} export HYPERSDK_VERSION="0.0.4" -rm -f ./tmp/token-cli +rm -f /tmp/token-cli wget "https://github.com/ava-labs/hypersdk/releases/download/v${HYPERSDK_VERSION}/hypersdk_${HYPERSDK_VERSION}_${OS_TYPE}_${ARCH_TYPE}.tar.gz" mkdir tmp-hypersdk tar -xvf hypersdk_${HYPERSDK_VERSION}_${OS_TYPE}_${ARCH_TYPE}.tar.gz -C tmp-hypersdk @@ -57,7 +57,7 @@ building with [`v3+`](https://github.com/golang/go/wiki/MinimumRequirements#amd6 ```bash export HYPERSDK_VERSION="0.0.4" -rm -f ./tmp/tokenvm +rm -f /tmp/tokenvm wget "https://github.com/ava-labs/hypersdk/releases/download/v${HYPERSDK_VERSION}/hypersdk_${HYPERSDK_VERSION}_linux_amd64.tar.gz" mkdir tmp-hypersdk tar -xvf hypersdk_${HYPERSDK_VERSION}_linux_amd64.tar.gz -C tmp-hypersdk @@ -188,7 +188,7 @@ avalancheup-aws apply \ --spec-file-path /home/ubuntu/aops-custom-****-***.yaml # run the following to delete resources -avalancheup-aws delete \ +/tmp/avalancheup-aws delete \ --delete-cloudwatch-log-group \ --delete-s3-objects \ --delete-ebs-volumes \ @@ -307,6 +307,7 @@ AWS Console To view metrics, first download and install [Prometheus](https://prometheus.io/download/) using the following commands: ```bash +rm -f /tmp/prometheus wget https://github.com/prometheus/prometheus/releases/download/v2.43.0/prometheus-2.43.0.darwin-amd64.tar.gz tar -xvf prometheus-2.43.0.darwin-amd64.tar.gz rm prometheus-2.43.0.darwin-amd64.tar.gz From b13d80e2a5c7901751df99e6bf075feefe5063ce Mon Sep 17 00:00:00 2001 From: Patrick O'Grady Date: Thu, 30 Mar 2023 14:04:59 -0400 Subject: [PATCH 42/67] add data folder exemption --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index f491a759c2..195dc0bee8 100644 --- a/.gitignore +++ b/.gitignore @@ -58,3 +58,4 @@ dist/ tmp-storage-testing .token-cli* *.html +data/ From 81c5cc4c0ccb269270064a5386a40e10d3fe2633 Mon Sep 17 00:00:00 2001 From: Patrick O'Grady Date: Thu, 30 Mar 2023 14:34:09 -0400 Subject: [PATCH 43/67] add instruction for SSH in nodes --- examples/tokenvm/DEVNETS.md | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/examples/tokenvm/DEVNETS.md b/examples/tokenvm/DEVNETS.md index a837bfbe15..1a41d65878 100644 --- a/examples/tokenvm/DEVNETS.md +++ b/examples/tokenvm/DEVNETS.md @@ -389,7 +389,15 @@ run the following command (defaults to `72000`): /tmp/token-cli spam run --max-tx-backlog 5000 ``` -### [OPTIONAL] Step 12: Deploy Another Subnet +### [OPTIONAL] Step 12: SSH Into Nodes +You can SSH into any machine created by `avalanche-ops` using the SSH key +automatically generated during the `apply` command. The commands for doing so +are emitted during `apply` and look something like this: +```bash +ssh -o "StrictHostKeyChecking no" -i aops-custom-202303-21qJUU-ec2-access.key ubuntu@34.209.76.123 +``` + +### [OPTIONAL] Step 13: Deploy Another Subnet To test Avalanche Warp Messaging, you must be running at least 2 Subnets. To do so, just replicate the command you ran above with a different `--chain-name` (and a different set of validators): From d82c80a81e8c907960feafd5fa3e06ec29173574 Mon Sep 17 00:00:00 2001 From: Patrick O'Grady Date: Thu, 30 Mar 2023 23:10:50 -0400 Subject: [PATCH 44/67] stop running fio --- .github/workflows/tokenvm-load-tests.yml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/tokenvm-load-tests.yml b/.github/workflows/tokenvm-load-tests.yml index dfd4f27ece..dae0dd19da 100644 --- a/.github/workflows/tokenvm-load-tests.yml +++ b/.github/workflows/tokenvm-load-tests.yml @@ -17,12 +17,12 @@ jobs: steps: - name: Checkout uses: actions/checkout@v3 - - name: Install fio - run: sudo apt-get -y install fio - - name: Run disk tests - working-directory: ./examples/tokenvm - shell: bash - run: scripts/tests.disk.sh + # - name: Install fio + # run: sudo apt-get -y install fio + # - name: Run disk tests + # working-directory: ./examples/tokenvm + # shell: bash + # run: scripts/tests.disk.sh - name: Set up Go uses: actions/setup-go@v3 with: From b2685dad30c483ca3bbeab204c240b124a7e4bbb Mon Sep 17 00:00:00 2001 From: Patrick O'Grady Date: Thu, 30 Mar 2023 23:53:30 -0400 Subject: [PATCH 45/67] store full query --- examples/tokenvm/DEVNETS.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/examples/tokenvm/DEVNETS.md b/examples/tokenvm/DEVNETS.md index 1a41d65878..1aa864a501 100644 --- a/examples/tokenvm/DEVNETS.md +++ b/examples/tokenvm/DEVNETS.md @@ -344,6 +344,12 @@ Here are some useful queries (on an example chainID `3rihqpXh6ZJqxL2dsrVysKkEKro * `avalanche_3rihqpXh6ZJqxL2dsrVysKkEKroiD9tvQWLS6iWVnd8K4HST2_vm_go_memstats_alloc_bytes` * `avalanche_network_inbound_conn_throttler_rate_limited` +To estimate how many `ms/s` the consensus engine is spending busy, use the +following query (on an example chainID `k3i5Cxk7UyLWWaEZKC1XSPJbQHSdSMJeZP2awecWxwyuqFDk7`): +``` +increase(avalanche_k3i5Cxk7UyLWWaEZKC1XSPJbQHSdSMJeZP2awecWxwyuqFDk7_handler_chits_sum[30s])/1000000/30 + increase(avalanche_k3i5Cxk7UyLWWaEZKC1XSPJbQHSdSMJeZP2awecWxwyuqFDk7_handler_notify_sum[30s])/1000000/30 + increase(avalanche_k3i5Cxk7UyLWWaEZKC1XSPJbQHSdSMJeZP2awecWxwyuqFDk7_handler_get_sum[30s])/1000000/30 + increase(avalanche_k3i5Cxk7UyLWWaEZKC1XSPJbQHSdSMJeZP2awecWxwyuqFDk7_handler_push_query_sum[30s])/1000000/30 + increase(avalanche_k3i5Cxk7UyLWWaEZKC1XSPJbQHSdSMJeZP2awecWxwyuqFDk7_handler_put_sum[30s])/1000000/30 + increase(avalanche_k3i5Cxk7UyLWWaEZKC1XSPJbQHSdSMJeZP2awecWxwyuqFDk7_handler_pull_query_sum[30s])/1000000/30 + increase(avalanche_k3i5Cxk7UyLWWaEZKC1XSPJbQHSdSMJeZP2awecWxwyuqFDk7_handler_query_failed_sum[30s])/1000000/30 +``` + To remove previously ingested data, delete for a folder called `data` in the directory where you last ran Prometheus. From b28acc908c0070e89b9735dee06b8e14037d4368 Mon Sep 17 00:00:00 2001 From: Patrick O'Grady Date: Fri, 31 Mar 2023 00:03:55 -0400 Subject: [PATCH 46/67] add consensus isolation --- examples/tokenvm/DEVNETS.md | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/examples/tokenvm/DEVNETS.md b/examples/tokenvm/DEVNETS.md index 1aa864a501..3a2470ed8b 100644 --- a/examples/tokenvm/DEVNETS.md +++ b/examples/tokenvm/DEVNETS.md @@ -344,12 +344,19 @@ Here are some useful queries (on an example chainID `3rihqpXh6ZJqxL2dsrVysKkEKro * `avalanche_3rihqpXh6ZJqxL2dsrVysKkEKroiD9tvQWLS6iWVnd8K4HST2_vm_go_memstats_alloc_bytes` * `avalanche_network_inbound_conn_throttler_rate_limited` -To estimate how many `ms/s` the consensus engine is spending busy, use the +To estimate how many `ms/s` the consensus engine and `tokenvm` are spending busy, use the following query (on an example chainID `k3i5Cxk7UyLWWaEZKC1XSPJbQHSdSMJeZP2awecWxwyuqFDk7`): ``` increase(avalanche_k3i5Cxk7UyLWWaEZKC1XSPJbQHSdSMJeZP2awecWxwyuqFDk7_handler_chits_sum[30s])/1000000/30 + increase(avalanche_k3i5Cxk7UyLWWaEZKC1XSPJbQHSdSMJeZP2awecWxwyuqFDk7_handler_notify_sum[30s])/1000000/30 + increase(avalanche_k3i5Cxk7UyLWWaEZKC1XSPJbQHSdSMJeZP2awecWxwyuqFDk7_handler_get_sum[30s])/1000000/30 + increase(avalanche_k3i5Cxk7UyLWWaEZKC1XSPJbQHSdSMJeZP2awecWxwyuqFDk7_handler_push_query_sum[30s])/1000000/30 + increase(avalanche_k3i5Cxk7UyLWWaEZKC1XSPJbQHSdSMJeZP2awecWxwyuqFDk7_handler_put_sum[30s])/1000000/30 + increase(avalanche_k3i5Cxk7UyLWWaEZKC1XSPJbQHSdSMJeZP2awecWxwyuqFDk7_handler_pull_query_sum[30s])/1000000/30 + increase(avalanche_k3i5Cxk7UyLWWaEZKC1XSPJbQHSdSMJeZP2awecWxwyuqFDk7_handler_query_failed_sum[30s])/1000000/30 ``` +To isolate just how many `ms/s` the consensus engine is spending busy (removing +"build", "verify", and "accept" time spent in `tokenvm`), use the following +command: +``` +increase(avalanche_k3i5Cxk7UyLWWaEZKC1XSPJbQHSdSMJeZP2awecWxwyuqFDk7_handler_chits_sum[30s])/1000000/30 + increase(avalanche_k3i5Cxk7UyLWWaEZKC1XSPJbQHSdSMJeZP2awecWxwyuqFDk7_handler_notify_sum[30s])/1000000/30 + increase(avalanche_k3i5Cxk7UyLWWaEZKC1XSPJbQHSdSMJeZP2awecWxwyuqFDk7_handler_get_sum[30s])/1000000/30 + increase(avalanche_k3i5Cxk7UyLWWaEZKC1XSPJbQHSdSMJeZP2awecWxwyuqFDk7_handler_push_query_sum[30s])/1000000/30 + increase(avalanche_k3i5Cxk7UyLWWaEZKC1XSPJbQHSdSMJeZP2awecWxwyuqFDk7_handler_put_sum[30s])/1000000/30 + increase(avalanche_k3i5Cxk7UyLWWaEZKC1XSPJbQHSdSMJeZP2awecWxwyuqFDk7_handler_pull_query_sum[30s])/1000000/30 + increase(avalanche_k3i5Cxk7UyLWWaEZKC1XSPJbQHSdSMJeZP2awecWxwyuqFDk7_handler_query_failed_sum[30s])/1000000/30 - increase(avalanche_k3i5Cxk7UyLWWaEZKC1XSPJbQHSdSMJeZP2awecWxwyuqFDk7_vm_metervm_build_block_sum[30s])/1000000/30 - increase(avalanche_k3i5Cxk7UyLWWaEZKC1XSPJbQHSdSMJeZP2awecWxwyuqFDk7_vm_metervm_verify_sum[30s])/1000000/30 - increase(avalanche_k3i5Cxk7UyLWWaEZKC1XSPJbQHSdSMJeZP2awecWxwyuqFDk7_vm_metervm_accept_sum[30s])/1000000/30 +``` + To remove previously ingested data, delete for a folder called `data` in the directory where you last ran Prometheus. From 91847a1854fe5c8cf6b00fcb4a957b85c68b6f4d Mon Sep 17 00:00:00 2001 From: Patrick O'Grady Date: Fri, 31 Mar 2023 00:21:21 -0400 Subject: [PATCH 47/67] add additional tuning to config --- examples/tokenvm/DEVNETS.md | 2 ++ examples/tokenvm/cmd/token-cli/cmd/genesis.go | 3 +++ examples/tokenvm/cmd/token-cli/cmd/root.go | 7 +++++ examples/tokenvm/config/config.go | 27 +++++++++++-------- examples/tokenvm/controller/controller.go | 4 ++- 5 files changed, 31 insertions(+), 12 deletions(-) diff --git a/examples/tokenvm/DEVNETS.md b/examples/tokenvm/DEVNETS.md index 3a2470ed8b..a287a25206 100644 --- a/examples/tokenvm/DEVNETS.md +++ b/examples/tokenvm/DEVNETS.md @@ -230,6 +230,7 @@ EOF rm -f /tmp/avalanche-ops/tokenvm-genesis.json /tmp/token-cli genesis generate /tmp/avalanche-ops/allocations.json \ --genesis-file /tmp/avalanche-ops/tokenvm-genesis.json \ +--max-block-unts 4000000 \ --window-target-units 100000000000 \ --window-target-blocks 20 cat /tmp/avalanche-ops/tokenvm-genesis.json @@ -242,6 +243,7 @@ cat < /tmp/avalanche-ops/tokenvm-chain-config.json "streamingBacklogSize": 10000000, "trackedPairs":["*"], "logLevel": "info", + "preferredBlocksPerSecond": 3, "decisionsPort": 9652, "blocksPort": 9653 } diff --git a/examples/tokenvm/cmd/token-cli/cmd/genesis.go b/examples/tokenvm/cmd/token-cli/cmd/genesis.go index f86fdf2f4b..b2090f110b 100644 --- a/examples/tokenvm/cmd/token-cli/cmd/genesis.go +++ b/examples/tokenvm/cmd/token-cli/cmd/genesis.go @@ -34,6 +34,9 @@ var genGenesisCmd = &cobra.Command{ if minUnitPrice >= 0 { g.MinUnitPrice = uint64(minUnitPrice) } + if maxBlockUnits >= 0 { + g.MaxBlockUnits = uint64(maxBlockUnits) + } if windowTargetUnits >= 0 { g.WindowTargetUnits = uint64(windowTargetUnits) } diff --git a/examples/tokenvm/cmd/token-cli/cmd/root.go b/examples/tokenvm/cmd/token-cli/cmd/root.go index 377de3f516..f2ea62d36a 100644 --- a/examples/tokenvm/cmd/token-cli/cmd/root.go +++ b/examples/tokenvm/cmd/token-cli/cmd/root.go @@ -26,6 +26,7 @@ var ( genesisFile string minUnitPrice int64 + maxBlockUnits int64 windowTargetUnits int64 windowTargetBlocks int64 hideTxs bool @@ -81,6 +82,12 @@ func init() { -1, "minimum price", ) + genGenesisCmd.PersistentFlags().Int64Var( + &maxBlockUnits, + "max-block-units", + -1, + "max block units", + ) genGenesisCmd.PersistentFlags().Int64Var( &windowTargetUnits, "window-target-units", diff --git a/examples/tokenvm/config/config.go b/examples/tokenvm/config/config.go index bc4cf1ceb2..899444e826 100644 --- a/examples/tokenvm/config/config.go +++ b/examples/tokenvm/config/config.go @@ -20,6 +20,8 @@ import ( var _ vm.Config = (*Config)(nil) +const DefaultPreferredBlocksPerSecond = 2 + type Config struct { *config.Config @@ -45,9 +47,10 @@ type Config struct { TrackedPairs []string `json:"trackedPairs"` // which asset ID pairs we care about // Misc - TestMode bool `json:"testMode"` // makes gossip/building manual - LogLevel logging.Level `json:"logLevel"` - Parallelism int `json:"parallelism"` + TestMode bool `json:"testMode"` // makes gossip/building manual + LogLevel logging.Level `json:"logLevel"` + Parallelism int `json:"parallelism"` + PreferredBlocksPerSecond uint64 `json:"preferredBlocksPerSecond"` // State Sync StateSyncServerDelay time.Duration `json:"stateSyncServerDelay"` // for testing @@ -81,20 +84,22 @@ func New(nodeID ids.NodeID, b []byte) (*Config, error) { func (c *Config) setDefault() { c.LogLevel = c.Config.GetLogLevel() c.Parallelism = c.Config.GetParallelism() + c.PreferredBlocksPerSecond = DefaultPreferredBlocksPerSecond c.MempoolSize = c.Config.GetMempoolSize() c.MempoolPayerSize = c.Config.GetMempoolPayerSize() c.StateSyncServerDelay = c.Config.GetStateSyncServerDelay() c.StreamingBacklogSize = c.Config.GetStreamingBacklogSize() } -func (c *Config) GetLogLevel() logging.Level { return c.LogLevel } -func (c *Config) GetTestMode() bool { return c.TestMode } -func (c *Config) GetParallelism() int { return c.Parallelism } -func (c *Config) GetMempoolSize() int { return c.MempoolSize } -func (c *Config) GetMempoolPayerSize() int { return c.MempoolPayerSize } -func (c *Config) GetMempoolExemptPayers() [][]byte { return c.parsedExemptPayers } -func (c *Config) GetDecisionsPort() uint16 { return c.DecisionsPort } -func (c *Config) GetBlocksPort() uint16 { return c.BlocksPort } +func (c *Config) GetLogLevel() logging.Level { return c.LogLevel } +func (c *Config) GetTestMode() bool { return c.TestMode } +func (c *Config) GetParallelism() int { return c.Parallelism } +func (c *Config) GetPreferredBlocksPerSecond() uint64 { return c.PreferredBlocksPerSecond } +func (c *Config) GetMempoolSize() int { return c.MempoolSize } +func (c *Config) GetMempoolPayerSize() int { return c.MempoolPayerSize } +func (c *Config) GetMempoolExemptPayers() [][]byte { return c.parsedExemptPayers } +func (c *Config) GetDecisionsPort() uint16 { return c.DecisionsPort } +func (c *Config) GetBlocksPort() uint16 { return c.BlocksPort } func (c *Config) GetTraceConfig() *trace.Config { return &trace.Config{ Enabled: c.TraceEnabled, diff --git a/examples/tokenvm/controller/controller.go b/examples/tokenvm/controller/controller.go index 76a7d29155..f2a098c2ee 100644 --- a/examples/tokenvm/controller/controller.go +++ b/examples/tokenvm/controller/controller.go @@ -143,7 +143,9 @@ func (c *Controller) Initialize( build = builder.NewManual(inner) gossip = gossiper.NewManual(inner) } else { - build = builder.NewTime(inner, builder.DefaultTimeConfig()) + bcfg := builder.DefaultTimeConfig() + bcfg.PreferredBlocksPerSecond = c.config.GetPreferredBlocksPerSecond() + build = builder.NewTime(inner, bcfg) gcfg := gossiper.DefaultProposerConfig() gcfg.BuildProposerDiff = 1 // don't gossip if producing the next block gossip = gossiper.NewProposer(inner, gcfg) From 13f8c3acc8f2978d8fefbba4c69bedae14faa9dd Mon Sep 17 00:00:00 2001 From: Patrick O'Grady Date: Fri, 31 Mar 2023 00:36:27 -0400 Subject: [PATCH 48/67] typo --- examples/tokenvm/DEVNETS.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/tokenvm/DEVNETS.md b/examples/tokenvm/DEVNETS.md index a287a25206..295fa26986 100644 --- a/examples/tokenvm/DEVNETS.md +++ b/examples/tokenvm/DEVNETS.md @@ -230,7 +230,7 @@ EOF rm -f /tmp/avalanche-ops/tokenvm-genesis.json /tmp/token-cli genesis generate /tmp/avalanche-ops/allocations.json \ --genesis-file /tmp/avalanche-ops/tokenvm-genesis.json \ ---max-block-unts 4000000 \ +--max-block-units 4000000 \ --window-target-units 100000000000 \ --window-target-blocks 20 cat /tmp/avalanche-ops/tokenvm-genesis.json From 857c473ef88cb59e40dc8f67e072fc063f51a1fa Mon Sep 17 00:00:00 2001 From: Patrick O'Grady Date: Fri, 31 Mar 2023 00:54:20 -0400 Subject: [PATCH 49/67] add support for continuous profiling --- config/config.go | 4 ++++ examples/tokenvm/config/config.go | 23 +++++++++++++++++++++-- vm/dependencies.go | 2 ++ vm/vm.go | 13 ++++++++++++- 4 files changed, 39 insertions(+), 3 deletions(-) diff --git a/config/config.go b/config/config.go index fe17ee88ba..3a100141bc 100644 --- a/config/config.go +++ b/config/config.go @@ -9,6 +9,7 @@ import ( "time" "github.com/ava-labs/avalanchego/utils/logging" + "github.com/ava-labs/avalanchego/utils/profiler" "github.com/ava-labs/avalanchego/utils/units" "github.com/ava-labs/hypersdk/trace" "github.com/ava-labs/hypersdk/vm" @@ -34,3 +35,6 @@ func (c *Config) GetStateSyncParallelism() int { return 4 } func (c *Config) GetStateSyncMinBlocks() uint64 { return 256 } func (c *Config) GetStateSyncServerDelay() time.Duration { return 0 } // used for testing func (c *Config) GetBlockLRUSize() int { return 128 } +func (c *Config) GetContinuousProfilerConfig() *profiler.Config { + return &profiler.Config{Enabled: false} +} diff --git a/examples/tokenvm/config/config.go b/examples/tokenvm/config/config.go index 899444e826..7af3a940e6 100644 --- a/examples/tokenvm/config/config.go +++ b/examples/tokenvm/config/config.go @@ -9,6 +9,7 @@ import ( "github.com/ava-labs/avalanchego/ids" "github.com/ava-labs/avalanchego/utils/logging" + "github.com/ava-labs/avalanchego/utils/profiler" "github.com/ava-labs/hypersdk/config" "github.com/ava-labs/hypersdk/trace" "github.com/ava-labs/hypersdk/vm" @@ -20,7 +21,11 @@ import ( var _ vm.Config = (*Config)(nil) -const DefaultPreferredBlocksPerSecond = 2 +const ( + defaultPreferredBlocksPerSecond = 2 + defaultContinuousProfilerFrequency = 1 * time.Minute + defaultContinuousProfilerMaxFiles = 10 +) type Config struct { *config.Config @@ -29,6 +34,9 @@ type Config struct { TraceEnabled bool `json:"traceEnabled"` TraceSampleRate float64 `json:"traceSampleRate"` + // Profiling + ContinuousProfileDir string `json:"continuousProfileDir"` + // Streaming Ports DecisionsPort uint16 `json:"decisionsPort"` BlocksPort uint16 `json:"blocksPort"` @@ -84,7 +92,7 @@ func New(nodeID ids.NodeID, b []byte) (*Config, error) { func (c *Config) setDefault() { c.LogLevel = c.Config.GetLogLevel() c.Parallelism = c.Config.GetParallelism() - c.PreferredBlocksPerSecond = DefaultPreferredBlocksPerSecond + c.PreferredBlocksPerSecond = defaultPreferredBlocksPerSecond c.MempoolSize = c.Config.GetMempoolSize() c.MempoolPayerSize = c.Config.GetMempoolPayerSize() c.StateSyncServerDelay = c.Config.GetStateSyncServerDelay() @@ -111,3 +119,14 @@ func (c *Config) GetTraceConfig() *trace.Config { } func (c *Config) GetStateSyncServerDelay() time.Duration { return c.StateSyncServerDelay } func (c *Config) GetStreamingBacklogSize() int { return c.StreamingBacklogSize } +func (c *Config) GetContinuousProfilerConfig() *profiler.Config { + if len(c.ContinuousProfileDir) == 0 { + return &profiler.Config{Enabled: false} + } + return &profiler.Config{ + Enabled: true, + Dir: c.ContinuousProfileDir, + Freq: defaultContinuousProfilerFrequency, + MaxNumFiles: defaultContinuousProfilerMaxFiles, + } +} diff --git a/vm/dependencies.go b/vm/dependencies.go index 65a88af568..ca507a065c 100644 --- a/vm/dependencies.go +++ b/vm/dependencies.go @@ -12,6 +12,7 @@ import ( "github.com/ava-labs/avalanchego/snow" "github.com/ava-labs/avalanchego/snow/engine/common" atrace "github.com/ava-labs/avalanchego/trace" + "github.com/ava-labs/avalanchego/utils/profiler" "github.com/ava-labs/hypersdk/builder" "github.com/ava-labs/hypersdk/chain" @@ -37,6 +38,7 @@ type Config interface { GetStateSyncMinBlocks() uint64 GetStateSyncServerDelay() time.Duration GetBlockLRUSize() int + GetContinuousProfilerConfig() *profiler.Config } type Genesis interface { diff --git a/vm/vm.go b/vm/vm.go index ef20c1d9d6..a627984136 100644 --- a/vm/vm.go +++ b/vm/vm.go @@ -23,6 +23,7 @@ import ( "github.com/ava-labs/avalanchego/trace" "github.com/ava-labs/avalanchego/utils" "github.com/ava-labs/avalanchego/utils/crypto/bls" + "github.com/ava-labs/avalanchego/utils/profiler" "github.com/ava-labs/avalanchego/version" "github.com/ava-labs/avalanchego/x/merkledb" syncEng "github.com/ava-labs/avalanchego/x/sync" @@ -113,7 +114,8 @@ type VM struct { // Network manager routes p2p messages to pre-registered handlers networkManager *NetworkManager - metrics *Metrics + metrics *Metrics + profiler profiler.ContinuousProfiler ready chan struct{} stop chan struct{} @@ -180,6 +182,12 @@ func (vm *VM) Initialize( ctx, span := vm.tracer.Start(ctx, "VM.Initialize") defer span.End() + // Setup profiler + if cfg := vm.config.GetContinuousProfilerConfig(); cfg.Enabled { + vm.profiler = profiler.NewContinuous(cfg.Dir, cfg.Freq, cfg.MaxNumFiles) + go vm.profiler.Dispatch() + } + // Instantiate DBs vm.stateDB, err = merkledb.New(ctx, vm.rawStateDB, merkledb.Config{ HistoryLength: vm.config.GetStateHistoryLength(), @@ -456,6 +464,9 @@ func (vm *VM) Shutdown(ctx context.Context) error { vm.builder.Done() vm.gossiper.Done() vm.workers.Stop() + if vm.profiler != nil { + vm.profiler.Shutdown() + } // Shutdown controller once all mechanisms that could invoke it have // shutdown. From 00e26acdc0915e5848ce50edcd8380101059d192 Mon Sep 17 00:00:00 2001 From: Patrick O'Grady Date: Fri, 31 Mar 2023 00:57:41 -0400 Subject: [PATCH 50/67] add continuous profiler to config --- examples/tokenvm/DEVNETS.md | 1 + examples/tokenvm/config/config.go | 6 +++--- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/examples/tokenvm/DEVNETS.md b/examples/tokenvm/DEVNETS.md index 295fa26986..8437c5d495 100644 --- a/examples/tokenvm/DEVNETS.md +++ b/examples/tokenvm/DEVNETS.md @@ -244,6 +244,7 @@ cat < /tmp/avalanche-ops/tokenvm-chain-config.json "trackedPairs":["*"], "logLevel": "info", "preferredBlocksPerSecond": 3, + "continuousProfileDir": "/var/log/tokenvm-profile", "decisionsPort": 9652, "blocksPort": 9653 } diff --git a/examples/tokenvm/config/config.go b/examples/tokenvm/config/config.go index 7af3a940e6..61ca055d11 100644 --- a/examples/tokenvm/config/config.go +++ b/examples/tokenvm/config/config.go @@ -35,7 +35,7 @@ type Config struct { TraceSampleRate float64 `json:"traceSampleRate"` // Profiling - ContinuousProfileDir string `json:"continuousProfileDir"` + ContinuousProfilerDir string `json:"continuousProfilerDir"` // Streaming Ports DecisionsPort uint16 `json:"decisionsPort"` @@ -120,12 +120,12 @@ func (c *Config) GetTraceConfig() *trace.Config { func (c *Config) GetStateSyncServerDelay() time.Duration { return c.StateSyncServerDelay } func (c *Config) GetStreamingBacklogSize() int { return c.StreamingBacklogSize } func (c *Config) GetContinuousProfilerConfig() *profiler.Config { - if len(c.ContinuousProfileDir) == 0 { + if len(c.ContinuousProfilerDir) == 0 { return &profiler.Config{Enabled: false} } return &profiler.Config{ Enabled: true, - Dir: c.ContinuousProfileDir, + Dir: c.ContinuousProfilerDir, Freq: defaultContinuousProfilerFrequency, MaxNumFiles: defaultContinuousProfilerMaxFiles, } From 8234f75c8607e182e3decb48ba220924ba7c818d Mon Sep 17 00:00:00 2001 From: Patrick O'Grady Date: Fri, 31 Mar 2023 00:58:24 -0400 Subject: [PATCH 51/67] fix profiler dir --- examples/tokenvm/DEVNETS.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/tokenvm/DEVNETS.md b/examples/tokenvm/DEVNETS.md index 8437c5d495..b2a3f92403 100644 --- a/examples/tokenvm/DEVNETS.md +++ b/examples/tokenvm/DEVNETS.md @@ -244,7 +244,7 @@ cat < /tmp/avalanche-ops/tokenvm-chain-config.json "trackedPairs":["*"], "logLevel": "info", "preferredBlocksPerSecond": 3, - "continuousProfileDir": "/var/log/tokenvm-profile", + "continuousProfilerDir": "/var/log/tokenvm-profile", "decisionsPort": 9652, "blocksPort": 9653 } From f64e601b0071c2d7f178fe9defd284269ba0d4ef Mon Sep 17 00:00:00 2001 From: Patrick O'Grady Date: Fri, 31 Mar 2023 01:10:31 -0400 Subject: [PATCH 52/67] add nolint --- config/config.go | 3 ++- vm/vm.go | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/config/config.go b/config/config.go index 3a100141bc..e8550cbd6f 100644 --- a/config/config.go +++ b/config/config.go @@ -34,7 +34,8 @@ func (c *Config) GetTraceConfig() *trace.Config { return &trace.Config{ func (c *Config) GetStateSyncParallelism() int { return 4 } func (c *Config) GetStateSyncMinBlocks() uint64 { return 256 } func (c *Config) GetStateSyncServerDelay() time.Duration { return 0 } // used for testing -func (c *Config) GetBlockLRUSize() int { return 128 } +func (c *Config) GetBlockLRUSize() int { return 128 } + func (c *Config) GetContinuousProfilerConfig() *profiler.Config { return &profiler.Config{Enabled: false} } diff --git a/vm/vm.go b/vm/vm.go index a627984136..009e57bffe 100644 --- a/vm/vm.go +++ b/vm/vm.go @@ -185,7 +185,7 @@ func (vm *VM) Initialize( // Setup profiler if cfg := vm.config.GetContinuousProfilerConfig(); cfg.Enabled { vm.profiler = profiler.NewContinuous(cfg.Dir, cfg.Freq, cfg.MaxNumFiles) - go vm.profiler.Dispatch() + go vm.profiler.Dispatch() //nolint:errcheck } // Instantiate DBs From 480e4742062352f605ce2ea14af9ae7e358c3318 Mon Sep 17 00:00:00 2001 From: Gyuho Lee Date: Mon, 3 Apr 2023 19:08:43 +0900 Subject: [PATCH 53/67] DEVNETS: comment "instance-types" flags Signed-off-by: Gyuho Lee --- examples/tokenvm/DEVNETS.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/examples/tokenvm/DEVNETS.md b/examples/tokenvm/DEVNETS.md index b2a3f92403..96daad2b02 100644 --- a/examples/tokenvm/DEVNETS.md +++ b/examples/tokenvm/DEVNETS.md @@ -77,6 +77,8 @@ Now we can spin up a new network of 6 nodes with some defaults: - `avalanche-ops` supports [EC2 Spot instances](https://aws.amazon.com/ec2/spot/) for cost savings. Use `--instance-mode=spot` to run instances in spot mode. ```bash +# use "--instance-types=c5.2xlarge,m5.2xlarge" +# to overwrite "--instance-size" flag value /tmp/avalancheup-aws default-spec \ --arch-type amd64 \ --rust-os-type ubuntu20.04 \ From 52f5a9cd0fa67a67c5a21b094da929505439170c Mon Sep 17 00:00:00 2001 From: Patrick O'Grady Date: Mon, 3 Apr 2023 08:57:30 -0700 Subject: [PATCH 54/67] Skip verify on build (#126) * insert height into state * add merkledb metrics * cleanup more metrics handling * add additional arguments for config * add HeightKey * log lock wait * use NodeID as random * fix profiles * gofumpt * gofmt * add more logs * perform proper verify * cache built block * reduce memory usage of tstate * improve memory efficiency of storage * align devnets and scripts * re-add proposer delay * use default peer gossip size --- chain/block.go | 78 ++++++++++++-------- chain/builder.go | 21 +++++- chain/dependencies.go | 1 + chain/transaction.go | 2 - config/config.go | 5 +- examples/tokenvm/DEVNETS.md | 5 +- examples/tokenvm/config/config.go | 6 +- examples/tokenvm/controller/state_manager.go | 4 + examples/tokenvm/scripts/run.sh | 9 ++- examples/tokenvm/storage/storage.go | 63 ++++++++++++---- examples/tokenvm/tests/e2e/e2e_test.go | 31 +++++--- mempool/mempool.go | 7 +- tstate/tstate.go | 14 ++-- vm/metrics.go | 6 +- vm/proposer_monitor.go | 2 +- vm/streaming.go | 6 +- vm/vm.go | 23 ++++-- vm/vm_test.go | 7 +- 18 files changed, 197 insertions(+), 93 deletions(-) diff --git a/chain/block.go b/chain/block.go index 008452c121..dce91963b7 100644 --- a/chain/block.go +++ b/chain/block.go @@ -5,6 +5,7 @@ package chain import ( "context" + "encoding/binary" "fmt" "time" @@ -97,6 +98,7 @@ type StatelessBlock struct { txsSet set.Set[ids.ID] warpMessages map[ids.ID]*warpJob + containsWarp bool // this allows us to avoid allocating a map when we build bctx *block.Context vdrState validators.State @@ -143,7 +145,8 @@ func ParseBlock( return ParseStatefulBlock(ctx, blk, source, status, vm) } -func (b *StatelessBlock) populateTxs(ctx context.Context, verifySigs bool) error { +// populateTxs is only called on blocks we did not build +func (b *StatelessBlock) populateTxs(ctx context.Context) error { ctx, span := b.vm.Tracer().Start(ctx, "StatelessBlock.populateTxs") defer span.End() @@ -159,10 +162,7 @@ func (b *StatelessBlock) populateTxs(ctx context.Context, verifySigs bool) error b.txsSet = set.NewSet[ids.ID](len(b.Txs)) b.warpMessages = map[ids.ID]*warpJob{} for _, tx := range b.Txs { - sigTask := tx.AuthAsyncVerify() - if verifySigs { - b.sigJob.Go(sigTask) - } + b.sigJob.Go(tx.AuthAsyncVerify()) if b.txsSet.Contains(tx.ID()) { return ErrDuplicateTx } @@ -188,6 +188,7 @@ func (b *StatelessBlock) populateTxs(ctx context.Context, verifySigs bool) error verifiedChan: make(chan bool, 1), warpNum: len(b.warpMessages), } + b.containsWarp = true } } b.sigJob.Done(func() { sspan.End() }) @@ -243,13 +244,12 @@ func ParseStatefulBlock( } // Populate hashes and tx set - return b, b.populateTxs(ctx, true) + return b, b.populateTxs(ctx) } -// [init] is used during block building and testing -// TODO: remove init -func (b *StatelessBlock) init(ctx context.Context, results []*Result, validateSigs bool) error { - ctx, span := b.vm.Tracer().Start(ctx, "StatelessBlock.init") +// [initializeBuilt] is invoked after a block is built +func (b *StatelessBlock) initializeBuilt(ctx context.Context, state merkledb.TrieView, results []*Result) error { + _, span := b.vm.Tracer().Start(ctx, "StatelessBlock.initializeBuilt") defer span.End() blk, err := b.StatefulBlock.Marshal(b.vm.Registry()) @@ -258,11 +258,17 @@ func (b *StatelessBlock) init(ctx context.Context, results []*Result, validateSi } b.bytes = blk b.id = utils.ToID(b.bytes) + b.state = state b.t = time.Unix(b.StatefulBlock.Tmstmp, 0) b.results = results - - // Populate hashes and tx set - return b.populateTxs(ctx, validateSigs) + b.txsSet = set.NewSet[ids.ID](len(b.Txs)) + for _, tx := range b.Txs { + b.txsSet.Add(tx.ID()) + if tx.WarpMessage != nil { + b.containsWarp = true + } + } + return nil } // implements "snowman.Block.choices.Decidable" @@ -270,7 +276,7 @@ func (b *StatelessBlock) ID() ids.ID { return b.id } // implements "block.WithVerifyContext" func (b *StatelessBlock) ShouldVerifyWithContext(context.Context) (bool, error) { - return len(b.warpMessages) > 0, nil + return b.containsWarp, nil } // implements "block.WithVerifyContext" @@ -283,6 +289,7 @@ func (b *StatelessBlock) VerifyWithContext(ctx context.Context, bctx *block.Cont attribute.Int64("height", int64(b.Hght)), attribute.Bool("stateReady", stateReady), attribute.Int64("pchainHeight", int64(bctx.PChainHeight)), + attribute.Bool("built", b.Processed()), ), ) defer span.End() @@ -303,6 +310,7 @@ func (b *StatelessBlock) Verify(ctx context.Context) error { attribute.Int("txs", len(b.Txs)), attribute.Int64("height", int64(b.Hght)), attribute.Bool("stateReady", stateReady), + attribute.Bool("built", b.Processed()), ), ) defer span.End() @@ -311,9 +319,18 @@ func (b *StatelessBlock) Verify(ctx context.Context) error { } func (b *StatelessBlock) verify(ctx context.Context, stateReady bool) error { - // If the state of the accepted tip has been fully fetched, it is safe to - // verify any block. - if stateReady { + log := b.vm.Logger() + switch { + case !stateReady: + // If the state of the accepted tip has not been fully fetched, it is not safe to + // verify any block. + log.Info("skipping verification, state not ready", zap.Uint64("height", b.Hght), zap.Stringer("blkID", b.ID())) + case b.Processed(): + // If we built the block, the state will already be populated and we don't + // need to compute it (we assume that we built a correct block and it isn't + // necessary to re-verify anything). + log.Info("skipping verification, already processed", zap.Uint64("height", b.Hght), zap.Stringer("blkID", b.ID())) + default: // Parent may not be processed when we verify this block so [verify] may // recursively compute missing state. state, err := b.innerVerify(ctx) @@ -325,7 +342,7 @@ func (b *StatelessBlock) verify(ctx context.Context, stateReady bool) error { // At any point after this, we may attempt to verify the block. We should be // sure we are prepared to do so. - + // // NOTE: mempool is modified by VM handler b.vm.Verified(ctx, b) return nil @@ -387,9 +404,8 @@ func (b *StatelessBlock) verifyWarpMessage(ctx context.Context, r Rules, msg *wa // state sync) func (b *StatelessBlock) innerVerify(ctx context.Context) (merkledb.TrieView, error) { var ( - log = b.vm.Logger() - built = len(b.results) > 0 - r = b.vm.Rules(b.Tmstmp) + log = b.vm.Logger() + r = b.vm.Rules(b.Tmstmp) ) // Perform basic correctness checks before doing any expensive work @@ -452,7 +468,7 @@ func (b *StatelessBlock) innerVerify(ctx context.Context) (merkledb.TrieView, er // Start validating warp messages, if they exist var invalidWarpResult bool - if len(b.warpMessages) > 0 { + if b.containsWarp { if b.bctx == nil { log.Error( "missing verify block context", @@ -567,9 +583,12 @@ func (b *StatelessBlock) innerVerify(ctx context.Context) (merkledb.TrieView, er return nil, ErrWarpResultMismatch } + // Store height in state to prevent duplicate roots + if err := state.Insert(ctx, b.vm.StateManager().HeightKey(), binary.BigEndian.AppendUint64(nil, b.Hght)); err != nil { + return nil, err + } + // Compute state root - // TODO: consider adding the parent root or height here to ensure state roots - // are never repeated computedRoot, err := state.GetMerkleRoot(ctx) if err != nil { return nil, err @@ -584,13 +603,10 @@ func (b *StatelessBlock) innerVerify(ctx context.Context) (merkledb.TrieView, er } // Ensure signatures are verified - if !built { // don't need to verify sigs if built - _, sspan := b.vm.Tracer().Start(ctx, "StatelessBlock.Verify.WaitSignatures") - if err := b.sigJob.Wait(); err != nil { - sspan.End() - return nil, err - } - sspan.End() + _, sspan := b.vm.Tracer().Start(ctx, "StatelessBlock.Verify.WaitSignatures") + defer sspan.End() + if err := b.sigJob.Wait(); err != nil { + return nil, err } return state, nil } diff --git a/chain/builder.go b/chain/builder.go index b58ecee53d..02b13f4dfa 100644 --- a/chain/builder.go +++ b/chain/builder.go @@ -5,12 +5,12 @@ package chain import ( "context" + "encoding/binary" "errors" "fmt" "time" "github.com/ava-labs/avalanchego/ids" - "github.com/ava-labs/avalanchego/snow/consensus/snowman" smblock "github.com/ava-labs/avalanchego/snow/engine/snowman/block" "go.opentelemetry.io/otel/attribute" "go.uber.org/zap" @@ -47,7 +47,7 @@ func BuildBlock( vm VM, preferred ids.ID, blockContext *smblock.Context, -) (snowman.Block, error) { +) (*StatelessBlock, error) { ctx, span := vm.Tracer().Start(ctx, "chain.BuildBlock") defer span.End() log := vm.Logger() @@ -88,10 +88,16 @@ func BuildBlock( vdrState = vm.ValidatorState() sm = vm.StateManager() + + start = time.Now() + lockWait time.Duration ) mempoolErr := mempool.Build( ctx, func(fctx context.Context, next *Transaction) (cont bool, restore bool, removeAcct bool, err error) { + if txsAttempted == 0 { + lockWait = time.Since(start) + } txsAttempted++ // Ensure we can process if transaction includes a warp message @@ -249,6 +255,13 @@ func BuildBlock( if err := ts.WriteChanges(ctx, state, vm.Tracer()); err != nil { return nil, err } + + // Store height in state to prevent duplicate roots + if err := state.Insert(ctx, sm.HeightKey(), binary.BigEndian.AppendUint64(nil, b.Hght)); err != nil { + return nil, err + } + + // Compute state root after all data has been written to trie root, err := state.GetMerkleRoot(ctx) if err != nil { return nil, err @@ -256,7 +269,7 @@ func BuildBlock( b.StateRoot = root // Compute block hash and marshaled representation - if err := b.init(ctx, results, false); err != nil { + if err := b.initializeBuilt(ctx, state, results); err != nil { return nil, err } log.Info( @@ -265,6 +278,8 @@ func BuildBlock( zap.Int("attempted", txsAttempted), zap.Int("added", len(b.Txs)), zap.Int("mempool size", b.vm.Mempool().Len(ctx)), + zap.Duration("mempool lock wait", lockWait), + zap.Bool("context", blockContext != nil), ) return b, nil } diff --git a/chain/dependencies.go b/chain/dependencies.go index ca1f469c58..690649ddf8 100644 --- a/chain/dependencies.go +++ b/chain/dependencies.go @@ -105,6 +105,7 @@ type Rules interface { // in a structured manner. If we did not use [StateManager], we may overwrite // state written by actions or auth. type StateManager interface { + HeightKey() []byte IncomingWarpKey(sourceChainID ids.ID, msgID ids.ID) []byte OutgoingWarpKey(txID ids.ID) []byte } diff --git a/chain/transaction.go b/chain/transaction.go index a2932f6eb3..8d98218924 100644 --- a/chain/transaction.go +++ b/chain/transaction.go @@ -125,8 +125,6 @@ func (t *Transaction) Expiry() int64 { return t.Base.Timestamp } func (t *Transaction) UnitPrice() uint64 { return t.Base.UnitPrice } // It is ok to have duplicate ReadKeys...the processor will skip them -// -// TODO: verify the invariant that [t.id] is set by this point func (t *Transaction) StateKeys(stateMapping StateManager) [][]byte { keys := append(t.Action.StateKeys(t.Auth, t.ID()), t.Auth.StateKeys()...) if t.WarpMessage != nil { diff --git a/config/config.go b/config/config.go index e8550cbd6f..220a840392 100644 --- a/config/config.go +++ b/config/config.go @@ -10,7 +10,6 @@ import ( "github.com/ava-labs/avalanchego/utils/logging" "github.com/ava-labs/avalanchego/utils/profiler" - "github.com/ava-labs/avalanchego/utils/units" "github.com/ava-labs/hypersdk/trace" "github.com/ava-labs/hypersdk/vm" ) @@ -28,13 +27,13 @@ func (c *Config) GetDecisionsPort() uint16 { return 0 } // auto-as func (c *Config) GetBlocksPort() uint16 { return 0 } // auto-assigned func (c *Config) GetStreamingBacklogSize() int { return 1024 } func (c *Config) GetStateHistoryLength() int { return 256 } -func (c *Config) GetStateCacheSize() int { return 1 * units.GiB } +func (c *Config) GetStateCacheSize() int { return 65_536 } // nodes func (c *Config) GetAcceptorSize() int { return 1024 } func (c *Config) GetTraceConfig() *trace.Config { return &trace.Config{Enabled: false} } func (c *Config) GetStateSyncParallelism() int { return 4 } func (c *Config) GetStateSyncMinBlocks() uint64 { return 256 } func (c *Config) GetStateSyncServerDelay() time.Duration { return 0 } // used for testing -func (c *Config) GetBlockLRUSize() int { return 128 } +func (c *Config) GetBlockLRUSize() int { return 128 } func (c *Config) GetContinuousProfilerConfig() *profiler.Config { return &profiler.Config{Enabled: false} diff --git a/examples/tokenvm/DEVNETS.md b/examples/tokenvm/DEVNETS.md index 96daad2b02..5b8bd00484 100644 --- a/examples/tokenvm/DEVNETS.md +++ b/examples/tokenvm/DEVNETS.md @@ -148,6 +148,9 @@ avalanchego_config: throttler-inbound-cpu-validator-alloc: 100000 throttler-inbound-disk-validator-alloc: 10737418240000 throttler-outbound-validator-alloc-size: 107374182 + snow-mixed-query-num-push-vdr-uint: 10 + consensus-on-accept-gossip-peer-size: 0 + consensus-accepted-frontier-gossip-peer-size: 0 ``` Make sure to remove `throttler-inbound-at-large-alloc-size` and @@ -234,7 +237,7 @@ rm -f /tmp/avalanche-ops/tokenvm-genesis.json --genesis-file /tmp/avalanche-ops/tokenvm-genesis.json \ --max-block-units 4000000 \ --window-target-units 100000000000 \ ---window-target-blocks 20 +--window-target-blocks 30 cat /tmp/avalanche-ops/tokenvm-genesis.json cat < /tmp/avalanche-ops/tokenvm-chain-config.json diff --git a/examples/tokenvm/config/config.go b/examples/tokenvm/config/config.go index 61ca055d11..c92e0b7067 100644 --- a/examples/tokenvm/config/config.go +++ b/examples/tokenvm/config/config.go @@ -5,6 +5,7 @@ package config import ( "encoding/json" "fmt" + "strings" "time" "github.com/ava-labs/avalanchego/ids" @@ -35,7 +36,7 @@ type Config struct { TraceSampleRate float64 `json:"traceSampleRate"` // Profiling - ContinuousProfilerDir string `json:"continuousProfilerDir"` + ContinuousProfilerDir string `json:"continuousProfilerDir"` // "*" is replaced with rand int // Streaming Ports DecisionsPort uint16 `json:"decisionsPort"` @@ -123,6 +124,9 @@ func (c *Config) GetContinuousProfilerConfig() *profiler.Config { if len(c.ContinuousProfilerDir) == 0 { return &profiler.Config{Enabled: false} } + // Replace all instances of "*" with nodeID. This is useful when + // running multiple instances of tokenvm on the same machine. + c.ContinuousProfilerDir = strings.ReplaceAll(c.ContinuousProfilerDir, "*", c.nodeID.String()) return &profiler.Config{ Enabled: true, Dir: c.ContinuousProfilerDir, diff --git a/examples/tokenvm/controller/state_manager.go b/examples/tokenvm/controller/state_manager.go index 2dc702dcb0..a1054a8f64 100644 --- a/examples/tokenvm/controller/state_manager.go +++ b/examples/tokenvm/controller/state_manager.go @@ -10,6 +10,10 @@ import ( type StateManager struct{} +func (*StateManager) HeightKey() []byte { + return storage.HeightKey() +} + func (*StateManager) IncomingWarpKey(sourceChainID ids.ID, msgID ids.ID) []byte { return storage.IncomingWarpKeyPrefix(sourceChainID, msgID) } diff --git a/examples/tokenvm/scripts/run.sh b/examples/tokenvm/scripts/run.sh index 2ba0789dc9..d3d3bf2682 100755 --- a/examples/tokenvm/scripts/run.sh +++ b/examples/tokenvm/scripts/run.sh @@ -98,6 +98,9 @@ if [[ -z "${GENESIS_PATH}" ]]; then echo "creating VM genesis file with allocations" rm -f /tmp/tokenvm.genesis /tmp/token-cli genesis generate /tmp/allocations.json \ + --max-block-units 4000000 \ + --window-target-units 100000000000 \ + --window-target-blocks 30 \ --genesis-file /tmp/tokenvm.genesis else echo "copying custom genesis file" @@ -111,6 +114,7 @@ fi echo "creating vm config" rm -f /tmp/tokenvm.config +rm -rf /tmp/tokenvm-e2e-profiles cat < /tmp/tokenvm.config { "mempoolSize": 10000000, @@ -119,10 +123,13 @@ cat < /tmp/tokenvm.config "parallelism": 5, "streamingBacklogSize": 10000000, "trackedPairs":["*"], + "preferredBlocksPerSecond": 3, + "continuousProfilerDir":"/tmp/tokenvm-e2e-profiles/*", "logLevel": "${LOGLEVEL}", "stateSyncServerDelay": ${STATESYNC_DELAY} } EOF +mkdir -p /tmp/tokenvm-e2e-profiles ############################ @@ -132,7 +139,7 @@ echo "creating subnet config" rm -f /tmp/tokenvm.subnet cat < /tmp/tokenvm.subnet { - "proposerMinBlockDelay":100000000 + "proposerMinBlockDelay": 100000000 } EOF diff --git a/examples/tokenvm/storage/storage.go b/examples/tokenvm/storage/storage.go index 8527322db7..bc30c8e291 100644 --- a/examples/tokenvm/storage/storage.go +++ b/examples/tokenvm/storage/storage.go @@ -8,6 +8,7 @@ import ( "encoding/binary" "errors" "fmt" + "sync" "github.com/ava-labs/avalanchego/database" "github.com/ava-labs/avalanchego/ids" @@ -44,13 +45,22 @@ const ( assetPrefix = 0x1 orderPrefix = 0x2 loanPrefix = 0x3 - incomingWarpPrefix = 0x4 - outgoingWarpPrefix = 0x5 + heightPrefix = 0x4 + incomingWarpPrefix = 0x5 + outgoingWarpPrefix = 0x6 ) var ( failureByte = byte(0x0) successByte = byte(0x1) + heightKey = []byte{heightPrefix} + + // TODO: extend to other types + balancePrefixPool = sync.Pool{ + New: func() any { + return make([]byte, 1+crypto.PublicKeyLen+consts.IDLen) + }, + } ) // [txPrefix] + [txID] @@ -106,7 +116,7 @@ func GetTransaction( // [accountPrefix] + [address] + [asset] func PrefixBalanceKey(pk crypto.PublicKey, asset ids.ID) (k []byte) { - k = make([]byte, 1+crypto.PublicKeyLen+consts.IDLen) + k = balancePrefixPool.Get().([]byte) k[0] = balancePrefix copy(k[1:], pk[:]) copy(k[1+crypto.PublicKeyLen:], asset[:]) @@ -120,8 +130,20 @@ func GetBalance( pk crypto.PublicKey, asset ids.ID, ) (uint64, error) { + dbKey, bal, err := getBalance(ctx, db, pk, asset) + balancePrefixPool.Put(dbKey) + return bal, err +} + +func getBalance( + ctx context.Context, + db chain.Database, + pk crypto.PublicKey, + asset ids.ID, +) ([]byte, uint64, error) { k := PrefixBalanceKey(pk, asset) - return innerGetBalance(db.GetValue(ctx, k)) + bal, err := innerGetBalance(db.GetValue(ctx, k)) + return k, bal, err } // Used to serve RPC queries @@ -131,8 +153,11 @@ func GetBalanceFromState( pk crypto.PublicKey, asset ids.ID, ) (uint64, error) { - values, errs := f(ctx, [][]byte{PrefixBalanceKey(pk, asset)}) - return innerGetBalance(values[0], errs[0]) + k := PrefixBalanceKey(pk, asset) + values, errs := f(ctx, [][]byte{k}) + bal, err := innerGetBalance(values[0], errs[0]) + balancePrefixPool.Put(k) + return bal, err } func innerGetBalance( @@ -156,8 +181,16 @@ func SetBalance( balance uint64, ) error { k := PrefixBalanceKey(pk, asset) - b := binary.BigEndian.AppendUint64(nil, balance) - return db.Insert(ctx, k, b) + return setBalance(ctx, db, k, balance) +} + +func setBalance( + ctx context.Context, + db chain.Database, + dbKey []byte, + balance uint64, +) error { + return db.Insert(ctx, dbKey, binary.BigEndian.AppendUint64(nil, balance)) } func DeleteBalance( @@ -176,7 +209,7 @@ func AddBalance( asset ids.ID, amount uint64, ) error { - bal, err := GetBalance(ctx, db, pk, asset) + dbKey, bal, err := getBalance(ctx, db, pk, asset) if err != nil { return err } @@ -191,7 +224,7 @@ func AddBalance( amount, ) } - return SetBalance(ctx, db, pk, asset, nbal) + return setBalance(ctx, db, dbKey, nbal) } func SubBalance( @@ -201,7 +234,7 @@ func SubBalance( asset ids.ID, amount uint64, ) error { - bal, err := GetBalance(ctx, db, pk, asset) + dbKey, bal, err := getBalance(ctx, db, pk, asset) if err != nil { return err } @@ -219,9 +252,9 @@ func SubBalance( if nbal == 0 { // If there is no balance left, we should delete the record instead of // setting it to 0. - return db.Remove(ctx, PrefixBalanceKey(pk, asset)) + return db.Remove(ctx, dbKey) } - return SetBalance(ctx, db, pk, asset, nbal) + return setBalance(ctx, db, dbKey, nbal) } // [assetPrefix] + [address] @@ -473,6 +506,10 @@ func SubLoan( return SetLoan(ctx, db, asset, destination, nloan) } +func HeightKey() (k []byte) { + return heightKey +} + func IncomingWarpKeyPrefix(sourceChainID ids.ID, msgID ids.ID) (k []byte) { k = make([]byte, 1+consts.IDLen*2) k[0] = incomingWarpPrefix diff --git a/examples/tokenvm/tests/e2e/e2e_test.go b/examples/tokenvm/tests/e2e/e2e_test.go index 04a4035fa4..1ead6c7c06 100644 --- a/examples/tokenvm/tests/e2e/e2e_test.go +++ b/examples/tokenvm/tests/e2e/e2e_test.go @@ -179,6 +179,8 @@ var _ = ginkgo.BeforeSuite(func() { ctx, execPath, runner_sdk.WithPluginDir(pluginDir), + // We don't diable PUT gossip here because the E2E test adds multiple + // non-validating nodes (which will fall behind). runner_sdk.WithGlobalNodeConfig(`{ "log-display-level":"info", "proposervm-use-current-height":true, @@ -280,18 +282,26 @@ var _ = ginkgo.BeforeSuite(func() { instancesA = []instance{} for _, nodeName := range subnetA { - u := fmt.Sprintf("%s/ext/bc/%s", nodeInfos[nodeName].Uri, blockchainIDA) + info := nodeInfos[nodeName] + u := fmt.Sprintf("%s/ext/bc/%s", info.Uri, blockchainIDA) + nodeID, err := ids.NodeIDFromString(info.GetId()) + gomega.Expect(err).Should(gomega.BeNil()) instancesA = append(instancesA, instance{ - uri: u, - cli: client.New(u), + nodeID: nodeID, + uri: u, + cli: client.New(u), }) } instancesB = []instance{} for _, nodeName := range subnetB { - u := fmt.Sprintf("%s/ext/bc/%s", nodeInfos[nodeName].Uri, blockchainIDB) + info := nodeInfos[nodeName] + u := fmt.Sprintf("%s/ext/bc/%s", info.Uri, blockchainIDB) + nodeID, err := ids.NodeIDFromString(info.GetId()) + gomega.Expect(err).Should(gomega.BeNil()) instancesB = append(instancesB, instance{ - uri: u, - cli: client.New(u), + nodeID: nodeID, + uri: u, + cli: client.New(u), }) } @@ -321,8 +331,9 @@ var ( ) type instance struct { - uri string - cli *client.Client + nodeID ids.NodeID + uri string + cli *client.Client } var _ = ginkgo.AfterSuite(func() { @@ -338,11 +349,11 @@ var _ = ginkgo.AfterSuite(func() { hutils.Outf("{{yellow}}skipping cluster shutdown{{/}}\n\n") hutils.Outf("{{cyan}}Blockchain:{{/}} %s\n", blockchainIDA) for _, member := range instancesA { - hutils.Outf("URI: %s\n", member.uri) + hutils.Outf("%s URI: %s\n", member.nodeID, member.uri) } hutils.Outf("\n{{cyan}}Blockchain:{{/}} %s\n", blockchainIDB) for _, member := range instancesB { - hutils.Outf("URI: %s\n", member.uri) + hutils.Outf("%s URI: %s\n", member.nodeID, member.uri) } } gomega.Expect(anrCli.Close()).Should(gomega.BeNil()) diff --git a/mempool/mempool.go b/mempool/mempool.go index e6c074bf05..9a54e83f9f 100644 --- a/mempool/mempool.go +++ b/mempool/mempool.go @@ -9,9 +9,12 @@ import ( "github.com/ava-labs/avalanchego/ids" "github.com/ava-labs/avalanchego/trace" + "github.com/ava-labs/avalanchego/utils/math" "github.com/ava-labs/avalanchego/utils/set" ) +const maxPrealloc = 4_096 + type Mempool[T Item] struct { tracer trace.Tracer @@ -46,11 +49,11 @@ func New[T Item]( maxPayerSize: maxPayerSize, pm: NewSortedMempool( - maxSize, /* pre-allocate total size */ + math.Min(maxSize, maxPrealloc), func(item T) uint64 { return item.UnitPrice() }, ), tm: NewSortedMempool( - maxSize, /* pre-allocate total size */ + math.Min(maxSize, maxPrealloc), func(item T) uint64 { return uint64(item.Expiry()) }, ), owned: map[string]set.Set[ids.ID]{}, diff --git a/tstate/tstate.go b/tstate/tstate.go index 752f5ee0e8..f839a0e3fb 100644 --- a/tstate/tstate.go +++ b/tstate/tstate.go @@ -54,9 +54,9 @@ type TState struct { // there. scope [][]byte // Stores a list of managed keys in the TState struct. - // We don't use pointers here because these objects very rarely live long. - // They will popped off the stack more easily this way. - ops []op + // Ops is a record of all operations performed on [TState]. Tracking + // operations allows for reverting state to a certain point-in-time. + ops []*op } // New returns a new instance of TState. Initializes the storage and changedKeys @@ -68,7 +68,7 @@ func New(storageSize int, changedSize int) *TState { fetchCache: map[string]*cacheItem{}, - ops: []op{}, + ops: make([]*op, 0, changedSize), } } @@ -129,7 +129,7 @@ func (ts *TState) SetStorage(_ context.Context, key []byte, value []byte) { // Populate rollback (note, we only care if an item was placed in storage // initially) - ts.ops = append(ts.ops, op{ + ts.ops = append(ts.ops, &op{ action: read, k: key, }) @@ -160,7 +160,7 @@ func (ts *TState) Insert(ctx context.Context, key []byte, value []byte) error { k := string(key) // Populate rollback - ts.ops = append(ts.ops, op{ + ts.ops = append(ts.ops, &op{ action: insert, k: key, v: value, @@ -180,7 +180,7 @@ func (ts *TState) Remove(ctx context.Context, key []byte) error { k := string(key) // Populate rollback - ts.ops = append(ts.ops, op{ + ts.ops = append(ts.ops, &op{ action: remove, k: key, pastV: ts.storage[k], diff --git a/vm/metrics.go b/vm/metrics.go index 7a210b2081..370a2cd52c 100644 --- a/vm/metrics.go +++ b/vm/metrics.go @@ -4,7 +4,6 @@ package vm import ( - ametrics "github.com/ava-labs/avalanchego/api/metrics" "github.com/ava-labs/avalanchego/utils/wrappers" "github.com/prometheus/client_golang/prometheus" ) @@ -19,7 +18,7 @@ type Metrics struct { blocksRPCConnections prometheus.Gauge } -func newMetrics(gatherer ametrics.MultiGatherer) (*Metrics, error) { +func newMetrics() (*prometheus.Registry, *Metrics, error) { m := &Metrics{ unitsVerified: prometheus.NewCounter(prometheus.CounterOpts{ Namespace: "chain", @@ -67,7 +66,6 @@ func newMetrics(gatherer ametrics.MultiGatherer) (*Metrics, error) { r.Register(m.txsAccepted), r.Register(m.decisionsRPCConnections), r.Register(m.blocksRPCConnections), - gatherer.Register("hyper_sdk", r), ) - return m, errs.Err + return r, m, errs.Err } diff --git a/vm/proposer_monitor.go b/vm/proposer_monitor.go index 629ec65474..720468b375 100644 --- a/vm/proposer_monitor.go +++ b/vm/proposer_monitor.go @@ -78,7 +78,7 @@ func (p *ProposerMonitor) refresh(ctx context.Context) error { pks[string(bls.PublicKeyToBytes(v.PublicKey))] = struct{}{} } p.validatorPublicKeys = pks - p.vm.snowCtx.Log.Info( + p.vm.snowCtx.Log.Debug( "refreshed proposer monitor", zap.Uint64("previous", p.currentPHeight), zap.Uint64("new", pHeight), diff --git a/vm/streaming.go b/vm/streaming.go index 41bef1e03e..a7e0526ccf 100644 --- a/vm/streaming.go +++ b/vm/streaming.go @@ -142,7 +142,7 @@ func (c *decisionRPCConnection) handleReads() { p := codec.NewReader(msgBytes, chain.NetworkSizeLimit) // will likely be much smaller tx, err := chain.UnmarshalTx(p, c.vm.actionRegistry, c.vm.authRegistry) if err != nil { - c.vm.snowCtx.Log.Error("failed to unmarshal tx", + c.vm.snowCtx.Log.Debug("failed to unmarshal tx", zap.Int("len", len(msgBytes)), zap.Error(err), ) @@ -151,7 +151,7 @@ func (c *decisionRPCConnection) handleReads() { sigVerify := tx.AuthAsyncVerify() if err := sigVerify(); err != nil { - c.vm.snowCtx.Log.Error("failed to verify sig", + c.vm.snowCtx.Log.Debug("failed to verify sig", zap.Error(err), ) return @@ -161,7 +161,7 @@ func (c *decisionRPCConnection) handleReads() { // Submit will remove from [txWaiters] if it is not added txID := tx.ID() if err := c.vm.Submit(ctx, false, []*chain.Transaction{tx})[0]; err != nil { - c.vm.snowCtx.Log.Error("failed to submit tx", + c.vm.snowCtx.Log.Debug("failed to submit tx", zap.Stringer("txID", txID), zap.Error(err), ) diff --git a/vm/vm.go b/vm/vm.go index 009e57bffe..4325682f17 100644 --- a/vm/vm.go +++ b/vm/vm.go @@ -27,6 +27,7 @@ import ( "github.com/ava-labs/avalanchego/version" "github.com/ava-labs/avalanchego/x/merkledb" syncEng "github.com/ava-labs/avalanchego/x/sync" + "github.com/prometheus/client_golang/prometheus" "go.uber.org/zap" @@ -145,10 +146,16 @@ func (vm *VM) Initialize( vm.ready = make(chan struct{}) vm.stop = make(chan struct{}) gatherer := ametrics.NewMultiGatherer() - metrics, err := newMetrics(gatherer) + if err := vm.snowCtx.Metrics.Register(gatherer); err != nil { + return err + } + defaultRegistry, metrics, err := newMetrics() if err != nil { return err } + if err := gatherer.Register("hyper_sdk", defaultRegistry); err != nil { + return err + } vm.metrics = metrics vm.proposerMonitor = NewProposerMonitor(vm) vm.networkManager = NewNetworkManager(vm, appSender) @@ -171,9 +178,6 @@ func (vm *VM) Initialize( if err != nil { return fmt.Errorf("implementation initialization failed: %w", err) } - if err := vm.snowCtx.Metrics.Register(gatherer); err != nil { - return err - } // Setup tracer vm.tracer, err = htrace.New(vm.config.GetTraceConfig()) if err != nil { @@ -189,15 +193,19 @@ func (vm *VM) Initialize( } // Instantiate DBs + merkleRegistry := prometheus.NewRegistry() vm.stateDB, err = merkledb.New(ctx, vm.rawStateDB, merkledb.Config{ HistoryLength: vm.config.GetStateHistoryLength(), NodeCacheSize: vm.config.GetStateCacheSize(), - // TODO: add metrics - Tracer: vm.tracer, + Reg: merkleRegistry, + Tracer: vm.tracer, }) if err != nil { return err } + if err := gatherer.Register("state", merkleRegistry); err != nil { + return err + } // Setup worker cluster vm.workers = workers.New(vm.config.GetParallelism(), 100) @@ -585,7 +593,7 @@ func (vm *VM) ParseBlock(ctx context.Context, source []byte) (snowman.Block, err return nil, err } vm.parsedBlocks.Put(id, newBlk) - vm.snowCtx.Log.Debug( + vm.snowCtx.Log.Info( "parsed block", zap.Stringer("id", newBlk.ID()), zap.Uint64("height", newBlk.Hght), @@ -608,6 +616,7 @@ func (vm *VM) buildBlock( vm.snowCtx.Log.Warn("BuildBlock failed", zap.Error(err)) return nil, err } + vm.parsedBlocks.Put(blk.ID(), blk) return blk, nil } diff --git a/vm/vm_test.go b/vm/vm_test.go index 76d49b55f8..df3e34a6f2 100644 --- a/vm/vm_test.go +++ b/vm/vm_test.go @@ -54,11 +54,10 @@ func TestBlockCache(t *testing.T) { // Init metrics (called in [Accepted]) gatherer := ametrics.NewMultiGatherer() - m, err := newMetrics(gatherer) - if err != nil { - t.Fatal(err) - } + reg, m, err := newMetrics() + require.NoError(err) vm.metrics = m + require.NoError(gatherer.Register("hyper_sdk", reg)) require.NoError(vm.snowCtx.Metrics.Register(gatherer)) // put the block into the cache "vm.blocks" From b93f8aaeb519a775b737a4e8cb37bf67ba534551 Mon Sep 17 00:00:00 2001 From: Patrick O'Grady Date: Mon, 3 Apr 2023 09:07:57 -0700 Subject: [PATCH 55/67] add note about custom instance types --- examples/tokenvm/DEVNETS.md | 27 ++++++--------------------- 1 file changed, 6 insertions(+), 21 deletions(-) diff --git a/examples/tokenvm/DEVNETS.md b/examples/tokenvm/DEVNETS.md index 5b8bd00484..a99ae5d96b 100644 --- a/examples/tokenvm/DEVNETS.md +++ b/examples/tokenvm/DEVNETS.md @@ -77,8 +77,6 @@ Now we can spin up a new network of 6 nodes with some defaults: - `avalanche-ops` supports [EC2 Spot instances](https://aws.amazon.com/ec2/spot/) for cost savings. Use `--instance-mode=spot` to run instances in spot mode. ```bash -# use "--instance-types=c5.2xlarge,m5.2xlarge" -# to overwrite "--instance-size" flag value /tmp/avalancheup-aws default-spec \ --arch-type amd64 \ --rust-os-type ubuntu20.04 \ @@ -86,7 +84,7 @@ Now we can spin up a new network of 6 nodes with some defaults: --non-anchor-nodes 3 \ --region us-west-2 \ --instance-mode=on-demand \ ---instance-size=2xlarge \ +--instance-types=c5.2xlarge \ --ip-mode=elastic \ --metrics-fetch-interval-seconds 60 \ --network-name custom \ @@ -115,24 +113,11 @@ avalancheup-aws default-spec \ It is recommended to specify your own artifacts to avoid flaky github release page downloads. #### Restricting Instance Types -By default, `avalanche-ops` will attempt to use all available instance types in -a region in your cluster. If you'd like to restrict which instance types are used, -edit the output `aops-custom-****-***.yaml` file before deploying: -```yaml -machine: - anchor_nodes: 3 - non_anchor_nodes: 3 - arch_type: amd64 - rust_os_type: ubuntu20.04 - instance_types: - - t3a.2xlarge - - t3.2xlarge - - t2.2xlarge - - c6a.2xlarge - - m6a.2xlarge - - m5.2xlarge - - c5.2xlarge -``` +By default, `avalanche-ops` will attempt to use all available instance types of +a given size (scoped by `--instance-size`) in a region in your cluster. If you'd like to restrict which +instance types are used (and override `--instance-size`), you can populate the `--instance-types` flag. +You can specify a single instance type (`--instance-types=c5.2xlarge`) or a comma-separated list +(`--instance-types=c5.2xlarge,m5.2xlarge`). #### Increasing Rate Limits If you are attempting to stress test the Devnet, you should disable CPU, Disk, From 5e141a92ee30e9dddf4ed112336c34c204dcc918 Mon Sep 17 00:00:00 2001 From: Patrick O'Grady Date: Mon, 3 Apr 2023 12:12:39 -0700 Subject: [PATCH 56/67] revert mainnet gossip changes --- examples/tokenvm/DEVNETS.md | 22 +++++++++++++--------- 1 file changed, 13 insertions(+), 9 deletions(-) diff --git a/examples/tokenvm/DEVNETS.md b/examples/tokenvm/DEVNETS.md index a99ae5d96b..1ac6154d83 100644 --- a/examples/tokenvm/DEVNETS.md +++ b/examples/tokenvm/DEVNETS.md @@ -134,8 +134,8 @@ avalanchego_config: throttler-inbound-disk-validator-alloc: 10737418240000 throttler-outbound-validator-alloc-size: 107374182 snow-mixed-query-num-push-vdr-uint: 10 - consensus-on-accept-gossip-peer-size: 0 - consensus-accepted-frontier-gossip-peer-size: 0 + consensus-on-accept-gossip-peer-size: 5 + consensus-accepted-frontier-gossip-peer-size: 5 ``` Make sure to remove `throttler-inbound-at-large-alloc-size` and @@ -209,15 +209,19 @@ do so (similar to [scripts/run.sh](./scripts/run.sh)): ```bash rm -rf /tmp/avalanche-ops -/tmp/avalancheup-aws subnet-config \ ---log-level=info \ ---proposer-min-block-delay 0 \ ---file-path /tmp/avalanche-ops/subnet-config.json +cat < /tmp/avalanche-ops/tokenvm-subnet-config.json +{ + "proposerMinBlockDelay": 0, + "gossipOnAcceptPeerSize": 0, + "gossipAcceptedFrontierPeerSize": 0 +} +EOF +cat /tmp/avalanche-ops/tokenvm-subnet-config.json cat < /tmp/avalanche-ops/allocations.json -[{"address":"token1rvzhmceq997zntgvravfagsks6w0ryud3rylh4cdvayry0dl97nsjzf3yp", "balance":1000000000000}] +[{"address":"token1rvzhmceq997zntgvravfagsks6w0ryud3rylh4cdvayry0dl97nsjzf3yp", "balance":1000000000000000}] EOF -rm -f /tmp/avalanche-ops/tokenvm-genesis.json + /tmp/token-cli genesis generate /tmp/avalanche-ops/allocations.json \ --genesis-file /tmp/avalanche-ops/tokenvm-genesis.json \ --max-block-units 4000000 \ @@ -418,7 +422,7 @@ a different set of validators): --key \ --primary-network-validate-period-in-days 16 \ --subnet-validate-period-in-days 14 \ ---subnet-config-local-path /tmp/avalanche-ops/subnet-config.json \ +--subnet-config-local-path /tmp/avalanche-ops/tokenvm-subnet-config.json \ --subnet-config-remote-dir /data/avalanche-configs/subnets \ --vm-binary-local-path /tmp/tokenvm \ --vm-binary-remote-dir /data/avalanche-plugins \ From 46393db10307721c8b04e284c5a26e7921b8b30b Mon Sep 17 00:00:00 2001 From: Patrick O'Grady Date: Mon, 3 Apr 2023 12:15:44 -0700 Subject: [PATCH 57/67] more doc tweaks --- examples/tokenvm/DEVNETS.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/examples/tokenvm/DEVNETS.md b/examples/tokenvm/DEVNETS.md index 1ac6154d83..b86cf9fea9 100644 --- a/examples/tokenvm/DEVNETS.md +++ b/examples/tokenvm/DEVNETS.md @@ -125,6 +125,7 @@ and Bandwidth rate limiting. You can do this by adding the following lines to `avalanchego_config` in the spec file: ```yaml avalanchego_config: + ... proposervm-use-current-height: true throttler-inbound-validator-alloc-size: 107374182 throttler-inbound-node-max-processing-msgs: 100000 @@ -208,6 +209,7 @@ do so (similar to [scripts/run.sh](./scripts/run.sh)): ```bash rm -rf /tmp/avalanche-ops +mkdir /tmp/avalanche-ops cat < /tmp/avalanche-ops/tokenvm-subnet-config.json { From 70e8cd0a5f60bcde836cba80bc77256ae3ba8597 Mon Sep 17 00:00:00 2001 From: Patrick O'Grady Date: Mon, 3 Apr 2023 12:35:49 -0700 Subject: [PATCH 58/67] fixing incorrect filepath --- examples/tokenvm/DEVNETS.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/tokenvm/DEVNETS.md b/examples/tokenvm/DEVNETS.md index b86cf9fea9..35bc255b9e 100644 --- a/examples/tokenvm/DEVNETS.md +++ b/examples/tokenvm/DEVNETS.md @@ -266,7 +266,7 @@ replace the `***` fields, IP addresses, key, and `node-ids-to-instance-ids` with --key \ --primary-network-validate-period-in-days 16 \ --subnet-validate-period-in-days 14 \ ---subnet-config-local-path /tmp/avalanche-ops/subnet-config.json \ +--subnet-config-local-path /tmp/avalanche-ops/tokenvm-subnet-config.json \ --subnet-config-remote-dir /data/avalanche-configs/subnets \ --vm-binary-local-path /tmp/tokenvm \ --vm-binary-remote-dir /data/avalanche-plugins \ From f4c4c8b1bb3ddd014be36c97f5653d3ff78d6400 Mon Sep 17 00:00:00 2001 From: Patrick O'Grady Date: Mon, 3 Apr 2023 13:40:22 -0700 Subject: [PATCH 59/67] add avalanchego profiling --- examples/tokenvm/DEVNETS.md | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/examples/tokenvm/DEVNETS.md b/examples/tokenvm/DEVNETS.md index 35bc255b9e..c87a3d4a93 100644 --- a/examples/tokenvm/DEVNETS.md +++ b/examples/tokenvm/DEVNETS.md @@ -88,7 +88,10 @@ Now we can spin up a new network of 6 nodes with some defaults: --ip-mode=elastic \ --metrics-fetch-interval-seconds 60 \ --network-name custom \ ---keys-to-generate 5 +--keys-to-generate 5 \ +--avalanchego-profile-continuous-enabled \ +--avalanchego-profile-continuous-freq 1m \ +--avalanchego-profile-continuous-max-files 10 ``` The `default-spec` (and `apply`) command only provisions nodes, not Custom VMs. From b4745248e15890ba92f8c1ace47c87de5a7342a8 Mon Sep 17 00:00:00 2001 From: Patrick O'Grady Date: Mon, 3 Apr 2023 14:02:15 -0700 Subject: [PATCH 60/67] add star history to readme --- README.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/README.md b/README.md index 693e2754fd..1ec9370433 100644 --- a/README.md +++ b/README.md @@ -580,6 +580,12 @@ that it can understand (so that it can parse inbound AWM messages). In the future, we expect that there will be common message definitions that will be compatible with most `hypervms` (and maintained in the `hypersdk`)._ +## Star History +

+ hypersdk + star history +

+ ## Future Work _If you want to take the lead on any of these items, please [start a discussion](https://github.com/ava-labs/hypersdk/discussions) or reach From 1caf086f3f025d4ff96a930b6904a982078ce034 Mon Sep 17 00:00:00 2001 From: Patrick O'Grady Date: Mon, 3 Apr 2023 14:03:02 -0700 Subject: [PATCH 61/67] fix duplicate image --- README.md | 1 - 1 file changed, 1 deletion(-) diff --git a/README.md b/README.md index 1ec9370433..fca4d5fc4b 100644 --- a/README.md +++ b/README.md @@ -582,7 +582,6 @@ compatible with most `hypervms` (and maintained in the `hypersdk`)._ ## Star History

- hypersdk star history

From 29f552ec4ab59d2078ee75b1f8966398734cb82c Mon Sep 17 00:00:00 2001 From: Patrick O'Grady Date: Mon, 3 Apr 2023 15:41:03 -0700 Subject: [PATCH 62/67] add warning about memory size --- examples/tokenvm/DEVNETS.md | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/examples/tokenvm/DEVNETS.md b/examples/tokenvm/DEVNETS.md index c87a3d4a93..88b55f139d 100644 --- a/examples/tokenvm/DEVNETS.md +++ b/examples/tokenvm/DEVNETS.md @@ -84,7 +84,7 @@ Now we can spin up a new network of 6 nodes with some defaults: --non-anchor-nodes 3 \ --region us-west-2 \ --instance-mode=on-demand \ ---instance-types=c5.2xlarge \ +--instance-types=c5.4xlarge \ --ip-mode=elastic \ --metrics-fetch-interval-seconds 60 \ --network-name custom \ @@ -122,6 +122,10 @@ instance types are used (and override `--instance-size`), you can populate the ` You can specify a single instance type (`--instance-types=c5.2xlarge`) or a comma-separated list (`--instance-types=c5.2xlarge,m5.2xlarge`). +`avalanchego` will use at least ` * (2048[bytesToIDCache] + 2048[decidedBlocksCache])` +for caching blocks. This overhead will be significantly reduced when +[this issue is resolved](https://github.com/ava-labs/hypersdk/issues/129). + #### Increasing Rate Limits If you are attempting to stress test the Devnet, you should disable CPU, Disk, and Bandwidth rate limiting. You can do this by adding the following lines to @@ -142,10 +146,6 @@ avalanchego_config: consensus-accepted-frontier-gossip-peer-size: 5 ``` -Make sure to remove `throttler-inbound-at-large-alloc-size` and -`throttler-inbound-node-max-at-large-bytes` from the default YAML, otherwise -you will have duplicate keys. - #### Supporting All Metrics By default, `avalanche-ops` does not support ingest all metrics into AWS CloudWatch. To ingest all metrics, change the configured filters in @@ -253,7 +253,8 @@ cat /tmp/avalanche-ops/tokenvm-chain-config.json *Note*: Make sure that port `9652` and `9653` are open on the AWS Security Group applied to all nodes otherwise the `token-cli` will not work properly. This requirement will -be removed when the HyperSDK migrates to using proper WebSockets. +be removed when the [HyperSDK migrates to using proper +WebSockets](https://github.com/ava-labs/hypersdk/issues/64). ### Step 8: Install Chains You can run the following commands to spin up 2 `tokenvm` Devnets. Make sure to From c206c4da7471b4f5eac0fb716ff3ff8d0ba398c4 Mon Sep 17 00:00:00 2001 From: Patrick O'Grady Date: Tue, 4 Apr 2023 09:19:48 -0700 Subject: [PATCH 63/67] ensure avalanchego has allocated cores --- config/config.go | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/config/config.go b/config/config.go index 220a840392..24e3bae524 100644 --- a/config/config.go +++ b/config/config.go @@ -14,12 +14,20 @@ import ( "github.com/ava-labs/hypersdk/vm" ) +const avalancheGoMinCPU = 4 + var _ vm.Config = (*Config)(nil) type Config struct{} -func (c *Config) GetLogLevel() logging.Level { return logging.Info } -func (c *Config) GetParallelism() int { return runtime.NumCPU() } +func (c *Config) GetLogLevel() logging.Level { return logging.Info } +func (c *Config) GetParallelism() int { + numCPUs := runtime.NumCPU() + if numCPUs > avalancheGoMinCPU { + return numCPUs - avalancheGoMinCPU + } + return 1 +} func (c *Config) GetMempoolSize() int { return 2_048 } func (c *Config) GetMempoolPayerSize() int { return 32 } func (c *Config) GetMempoolExemptPayers() [][]byte { return nil } From c86620ca4bca0a5a4d0bebf0bff72ada8cd99462 Mon Sep 17 00:00:00 2001 From: Patrick O'Grady Date: Tue, 4 Apr 2023 09:37:38 -0700 Subject: [PATCH 64/67] track received txs --- gossiper/proposer.go | 55 ++++++++++++++++++++++++++++++-------------- 1 file changed, 38 insertions(+), 17 deletions(-) diff --git a/gossiper/proposer.go b/gossiper/proposer.go index 3d6300a0a2..949c215723 100644 --- a/gossiper/proposer.go +++ b/gossiper/proposer.go @@ -30,25 +30,28 @@ type Proposer struct { // bounded by validator count (may be slightly out of date as composition changes) gossipedTxs map[ids.NodeID]*cache.LRU[ids.ID, struct{}] + receivedTxs *cache.LRU[ids.ID, struct{}] } type ProposerConfig struct { - GossipProposerDiff int - GossipProposerDepth int - GossipInterval time.Duration - GossipPeerCacheSize int - GossipMinLife int64 // seconds - BuildProposerDiff int + GossipProposerDiff int + GossipProposerDepth int + GossipInterval time.Duration + GossipPeerCacheSize int + GossipReceivedCacheSize int + GossipMinLife int64 // seconds + BuildProposerDiff int } func DefaultProposerConfig() *ProposerConfig { return &ProposerConfig{ - GossipProposerDiff: 2, - GossipProposerDepth: 3, - GossipInterval: 1 * time.Second, - GossipPeerCacheSize: 10_240, - GossipMinLife: 5, - BuildProposerDiff: 2, + GossipProposerDiff: 2, + GossipProposerDepth: 3, + GossipInterval: 1 * time.Second, + GossipPeerCacheSize: 10_240, + GossipReceivedCacheSize: 65_536, + GossipMinLife: 5, + BuildProposerDiff: 2, } } @@ -60,6 +63,7 @@ func NewProposer(vm VM, cfg *ProposerConfig) *Proposer { doneGossip: make(chan struct{}), gossipedTxs: map[ids.NodeID]*cache.LRU[ids.ID, struct{}]{}, + receivedTxs: &cache.LRU[ids.ID, struct{}]{Size: cfg.GossipReceivedCacheSize}, } } @@ -186,6 +190,16 @@ func (g *Proposer) TriggerGossip(ctx context.Context) error { return true, true, false, nil } + // Don't gossip txs we received from other nodes (original gossiper will + // gossip again if the transaction is still important to them, so our + // gossip will just be useless bytes). + // + // We still keep these transactions in our mempool as they may still be + // the highest-paying transaction to execute at a given time. + if _, has := g.receivedTxs.Get(next.ID()); has { + return true, true, false, nil + } + // PreExecute does not make any changes to state if err := next.PreExecute(ctx, ectx, r, state, now); err != nil { // Do not gossip invalid txs (may become invalid during normal block @@ -236,7 +250,7 @@ func (g *Proposer) HandleAppGossip(ctx context.Context, nodeID ids.NodeID, msg [ } // Mark incoming gossip as held by [nodeID], if it is a validator - ok, err := g.vm.IsValidator(ctx, nodeID) + isValidator, err := g.vm.IsValidator(ctx, nodeID) if err != nil { g.vm.Logger().Warn( "unable to determine if nodeID is validator", @@ -244,18 +258,25 @@ func (g *Proposer) HandleAppGossip(ctx context.Context, nodeID ids.NodeID, msg [ zap.Error(err), ) } - if ok { // will never happen if err - c, ok := g.gossipedTxs[nodeID] + var c *cache.LRU[ids.ID, struct{}] + if isValidator { + var ok bool + c, ok = g.gossipedTxs[nodeID] if !ok { g.gossipedTxs[nodeID] = &cache.LRU[ids.ID, struct{}]{Size: g.cfg.GossipPeerCacheSize} c = g.gossipedTxs[nodeID] } - for _, tx := range txs { + } + + // Add incoming transactions to our caches to prevent useless gossip + for _, tx := range txs { + if c != nil { c.Put(tx.ID(), struct{}{}) } + g.receivedTxs.Put(tx.ID(), struct{}{}) } - // submit incoming gossip + // Submit incoming gossip to mempool start := time.Now() for _, err := range g.vm.Submit(ctx, true, txs) { if err == nil || errors.Is(err, chain.ErrDuplicateTx) { From e4f9ed758cee64bb460e48e55302f8061fd59fe7 Mon Sep 17 00:00:00 2001 From: Patrick O'Grady Date: Tue, 4 Apr 2023 11:40:12 -0700 Subject: [PATCH 65/67] disable compression an profiling --- examples/tokenvm/DEVNETS.md | 38 +++++++++++++++++++++++++++++-------- 1 file changed, 30 insertions(+), 8 deletions(-) diff --git a/examples/tokenvm/DEVNETS.md b/examples/tokenvm/DEVNETS.md index 88b55f139d..e144eba201 100644 --- a/examples/tokenvm/DEVNETS.md +++ b/examples/tokenvm/DEVNETS.md @@ -88,15 +88,24 @@ Now we can spin up a new network of 6 nodes with some defaults: --ip-mode=elastic \ --metrics-fetch-interval-seconds 60 \ --network-name custom \ ---keys-to-generate 5 \ ---avalanchego-profile-continuous-enabled \ ---avalanchego-profile-continuous-freq 1m \ ---avalanchego-profile-continuous-max-files 10 +--keys-to-generate 5 ``` The `default-spec` (and `apply`) command only provisions nodes, not Custom VMs. The Subnet and Custom VM installation are done via `install-subnet-chain` sub-commands that follow. +#### Profiling `avalanchego` +If you'd like to profile `avalanchego`'s CPU and RAM, provide the following +flags to the command above: + +```bash +avalancheup-aws default-spec \ +... +--avalanchego-profile-continuous-enabled \ +--avalanchego-profile-continuous-freq 1m \ +--avalanchego-profile-continuous-max-files 10 +``` + #### Using Custom `avalanchego` Binaries The default command above will download the `avalanchego` public release binaries from GitHub. To test your own binaries, use the following flags to upload to S3. These binaries must be built @@ -142,8 +151,11 @@ avalanchego_config: throttler-inbound-disk-validator-alloc: 10737418240000 throttler-outbound-validator-alloc-size: 107374182 snow-mixed-query-num-push-vdr-uint: 10 + consensus-on-accept-gossip-validator-size: 0 + consensus-on-accept-gossip-non-validator-size: 0 consensus-on-accept-gossip-peer-size: 5 consensus-accepted-frontier-gossip-peer-size: 5 + network-compression-enabled: false ``` #### Supporting All Metrics @@ -243,7 +255,6 @@ cat < /tmp/avalanche-ops/tokenvm-chain-config.json "trackedPairs":["*"], "logLevel": "info", "preferredBlocksPerSecond": 3, - "continuousProfilerDir": "/var/log/tokenvm-profile", "decisionsPort": 9652, "blocksPort": 9653 } @@ -256,6 +267,17 @@ nodes otherwise the `token-cli` will not work properly. This requirement will be removed when the [HyperSDK migrates to using proper WebSockets](https://github.com/ava-labs/hypersdk/issues/64). +#### Profiling `tokenvm` +If you'd like to profile `tokenvm`'s CPU and RAM, add the following line to the +`tokenvm-chain-config.json` file above: + +```json +{ + ... + "continuousProfilerDir": "/var/log/tokenvm-profile" +} +``` + ### Step 8: Install Chains You can run the following commands to spin up 2 `tokenvm` Devnets. Make sure to replace the `***` fields, IP addresses, key, and `node-ids-to-instance-ids` with your own data: @@ -348,16 +370,16 @@ Here are some useful queries (on an example chainID `3rihqpXh6ZJqxL2dsrVysKkEKro * `avalanche_network_inbound_conn_throttler_rate_limited` To estimate how many `ms/s` the consensus engine and `tokenvm` are spending busy, use the -following query (on an example chainID `k3i5Cxk7UyLWWaEZKC1XSPJbQHSdSMJeZP2awecWxwyuqFDk7`): +following query (on an example chainID `2CziMHmCB6obvfaXjstLQAzWy2HJZ2HGj3c8jiaDG16teaeL2X`): ``` -increase(avalanche_k3i5Cxk7UyLWWaEZKC1XSPJbQHSdSMJeZP2awecWxwyuqFDk7_handler_chits_sum[30s])/1000000/30 + increase(avalanche_k3i5Cxk7UyLWWaEZKC1XSPJbQHSdSMJeZP2awecWxwyuqFDk7_handler_notify_sum[30s])/1000000/30 + increase(avalanche_k3i5Cxk7UyLWWaEZKC1XSPJbQHSdSMJeZP2awecWxwyuqFDk7_handler_get_sum[30s])/1000000/30 + increase(avalanche_k3i5Cxk7UyLWWaEZKC1XSPJbQHSdSMJeZP2awecWxwyuqFDk7_handler_push_query_sum[30s])/1000000/30 + increase(avalanche_k3i5Cxk7UyLWWaEZKC1XSPJbQHSdSMJeZP2awecWxwyuqFDk7_handler_put_sum[30s])/1000000/30 + increase(avalanche_k3i5Cxk7UyLWWaEZKC1XSPJbQHSdSMJeZP2awecWxwyuqFDk7_handler_pull_query_sum[30s])/1000000/30 + increase(avalanche_k3i5Cxk7UyLWWaEZKC1XSPJbQHSdSMJeZP2awecWxwyuqFDk7_handler_query_failed_sum[30s])/1000000/30 +increase(avalanche_2CziMHmCB6obvfaXjstLQAzWy2HJZ2HGj3c8jiaDG16teaeL2X_handler_chits_sum[30s])/1000000/30 + increase(avalanche_2CziMHmCB6obvfaXjstLQAzWy2HJZ2HGj3c8jiaDG16teaeL2X_handler_notify_sum[30s])/1000000/30 + increase(avalanche_2CziMHmCB6obvfaXjstLQAzWy2HJZ2HGj3c8jiaDG16teaeL2X_handler_get_sum[30s])/1000000/30 + increase(avalanche_2CziMHmCB6obvfaXjstLQAzWy2HJZ2HGj3c8jiaDG16teaeL2X_handler_push_query_sum[30s])/1000000/30 + increase(avalanche_2CziMHmCB6obvfaXjstLQAzWy2HJZ2HGj3c8jiaDG16teaeL2X_handler_put_sum[30s])/1000000/30 + increase(avalanche_2CziMHmCB6obvfaXjstLQAzWy2HJZ2HGj3c8jiaDG16teaeL2X_handler_pull_query_sum[30s])/1000000/30 + increase(avalanche_2CziMHmCB6obvfaXjstLQAzWy2HJZ2HGj3c8jiaDG16teaeL2X_handler_query_failed_sum[30s])/1000000/30 ``` To isolate just how many `ms/s` the consensus engine is spending busy (removing "build", "verify", and "accept" time spent in `tokenvm`), use the following command: ``` -increase(avalanche_k3i5Cxk7UyLWWaEZKC1XSPJbQHSdSMJeZP2awecWxwyuqFDk7_handler_chits_sum[30s])/1000000/30 + increase(avalanche_k3i5Cxk7UyLWWaEZKC1XSPJbQHSdSMJeZP2awecWxwyuqFDk7_handler_notify_sum[30s])/1000000/30 + increase(avalanche_k3i5Cxk7UyLWWaEZKC1XSPJbQHSdSMJeZP2awecWxwyuqFDk7_handler_get_sum[30s])/1000000/30 + increase(avalanche_k3i5Cxk7UyLWWaEZKC1XSPJbQHSdSMJeZP2awecWxwyuqFDk7_handler_push_query_sum[30s])/1000000/30 + increase(avalanche_k3i5Cxk7UyLWWaEZKC1XSPJbQHSdSMJeZP2awecWxwyuqFDk7_handler_put_sum[30s])/1000000/30 + increase(avalanche_k3i5Cxk7UyLWWaEZKC1XSPJbQHSdSMJeZP2awecWxwyuqFDk7_handler_pull_query_sum[30s])/1000000/30 + increase(avalanche_k3i5Cxk7UyLWWaEZKC1XSPJbQHSdSMJeZP2awecWxwyuqFDk7_handler_query_failed_sum[30s])/1000000/30 - increase(avalanche_k3i5Cxk7UyLWWaEZKC1XSPJbQHSdSMJeZP2awecWxwyuqFDk7_vm_metervm_build_block_sum[30s])/1000000/30 - increase(avalanche_k3i5Cxk7UyLWWaEZKC1XSPJbQHSdSMJeZP2awecWxwyuqFDk7_vm_metervm_verify_sum[30s])/1000000/30 - increase(avalanche_k3i5Cxk7UyLWWaEZKC1XSPJbQHSdSMJeZP2awecWxwyuqFDk7_vm_metervm_accept_sum[30s])/1000000/30 +increase(avalanche_2CziMHmCB6obvfaXjstLQAzWy2HJZ2HGj3c8jiaDG16teaeL2X_handler_chits_sum[30s])/1000000/30 + increase(avalanche_2CziMHmCB6obvfaXjstLQAzWy2HJZ2HGj3c8jiaDG16teaeL2X_handler_notify_sum[30s])/1000000/30 + increase(avalanche_2CziMHmCB6obvfaXjstLQAzWy2HJZ2HGj3c8jiaDG16teaeL2X_handler_get_sum[30s])/1000000/30 + increase(avalanche_2CziMHmCB6obvfaXjstLQAzWy2HJZ2HGj3c8jiaDG16teaeL2X_handler_push_query_sum[30s])/1000000/30 + increase(avalanche_2CziMHmCB6obvfaXjstLQAzWy2HJZ2HGj3c8jiaDG16teaeL2X_handler_put_sum[30s])/1000000/30 + increase(avalanche_2CziMHmCB6obvfaXjstLQAzWy2HJZ2HGj3c8jiaDG16teaeL2X_handler_pull_query_sum[30s])/1000000/30 + increase(avalanche_2CziMHmCB6obvfaXjstLQAzWy2HJZ2HGj3c8jiaDG16teaeL2X_handler_query_failed_sum[30s])/1000000/30 - increase(avalanche_2CziMHmCB6obvfaXjstLQAzWy2HJZ2HGj3c8jiaDG16teaeL2X_vm_metervm_build_block_sum[30s])/1000000/30 - increase(avalanche_2CziMHmCB6obvfaXjstLQAzWy2HJZ2HGj3c8jiaDG16teaeL2X_vm_metervm_verify_sum[30s])/1000000/30 - increase(avalanche_2CziMHmCB6obvfaXjstLQAzWy2HJZ2HGj3c8jiaDG16teaeL2X_vm_metervm_accept_sum[30s])/1000000/30 ``` To remove previously ingested data, delete for a folder called `data` in the From 0b58a3236d0488a1b0c8f18430849b4846bed353 Mon Sep 17 00:00:00 2001 From: Patrick O'Grady Date: Tue, 4 Apr 2023 23:48:21 -0700 Subject: [PATCH 66/67] Add metrics for root calculation and signature wait (#133) * add verify metrics * cleanup wording * add more comments * update resolution --- chain/block.go | 4 ++++ chain/dependencies.go | 8 ++++++++ vm/metrics.go | 27 ++++++++++++++++++++++++++- vm/resolutions.go | 8 ++++++++ 4 files changed, 46 insertions(+), 1 deletion(-) diff --git a/chain/block.go b/chain/block.go index dce91963b7..8e1bdd08cd 100644 --- a/chain/block.go +++ b/chain/block.go @@ -589,10 +589,12 @@ func (b *StatelessBlock) innerVerify(ctx context.Context) (merkledb.TrieView, er } // Compute state root + start := time.Now() computedRoot, err := state.GetMerkleRoot(ctx) if err != nil { return nil, err } + b.vm.RecordRootCalculated(time.Since(start)) if b.StateRoot != computedRoot { return nil, fmt.Errorf( "%w: expected=%s found=%s", @@ -605,9 +607,11 @@ func (b *StatelessBlock) innerVerify(ctx context.Context) (merkledb.TrieView, er // Ensure signatures are verified _, sspan := b.vm.Tracer().Start(ctx, "StatelessBlock.Verify.WaitSignatures") defer sspan.End() + start = time.Now() if err := b.sigJob.Wait(); err != nil { return nil, err } + b.vm.RecordWaitSignatures(time.Since(start)) return state, nil } diff --git a/chain/dependencies.go b/chain/dependencies.go index 690649ddf8..24b9d30356 100644 --- a/chain/dependencies.go +++ b/chain/dependencies.go @@ -5,6 +5,7 @@ package chain import ( "context" + "time" "github.com/ava-labs/avalanchego/ids" "github.com/ava-labs/avalanchego/snow/engine/snowman/block" @@ -62,6 +63,13 @@ type VM interface { // and false if the sync completed with the previous root. UpdateSyncTarget(*StatelessBlock) (bool, error) StateReady() bool + + // Record the duration of various operations to populate chain metrics + // + // If there was a long-lived [Chain] struct, we would store metrics for chain + // there. + RecordRootCalculated(t time.Duration) // only called in Verify + RecordWaitSignatures(t time.Duration) // only called in Verify } type Mempool interface { diff --git a/vm/metrics.go b/vm/metrics.go index 370a2cd52c..a8e02abad2 100644 --- a/vm/metrics.go +++ b/vm/metrics.go @@ -4,6 +4,7 @@ package vm import ( + "github.com/ava-labs/avalanchego/utils/metric" "github.com/ava-labs/avalanchego/utils/wrappers" "github.com/prometheus/client_golang/prometheus" ) @@ -16,9 +17,32 @@ type Metrics struct { txsAccepted prometheus.Counter decisionsRPCConnections prometheus.Gauge blocksRPCConnections prometheus.Gauge + rootCalculated metric.Averager + waitSignatures metric.Averager } func newMetrics() (*prometheus.Registry, *Metrics, error) { + r := prometheus.NewRegistry() + + rootCalculated, err := metric.NewAverager( + "chain", + "root_calculated", + "time spent calculating the state root in verify", + r, + ) + if err != nil { + return nil, nil, err + } + waitSignatures, err := metric.NewAverager( + "chain", + "wait_signatures", + "time spent waiting for signature verification in verify", + r, + ) + if err != nil { + return nil, nil, err + } + m := &Metrics{ unitsVerified: prometheus.NewCounter(prometheus.CounterOpts{ Namespace: "chain", @@ -55,8 +79,9 @@ func newMetrics() (*prometheus.Registry, *Metrics, error) { Name: "blocks_rpc_connections", Help: "number of open blocks connections", }), + rootCalculated: rootCalculated, + waitSignatures: waitSignatures, } - r := prometheus.NewRegistry() errs := wrappers.Errs{} errs.Add( r.Register(m.unitsVerified), diff --git a/vm/resolutions.go b/vm/resolutions.go index 4f06cf558d..d1e46c1387 100644 --- a/vm/resolutions.go +++ b/vm/resolutions.go @@ -321,3 +321,11 @@ func (vm *VM) StateSyncEnabled(ctx context.Context) (bool, error) { func (vm *VM) StateManager() chain.StateManager { return vm.c.StateManager() } + +func (vm *VM) RecordRootCalculated(t time.Duration) { + vm.metrics.rootCalculated.Observe(float64(t)) +} + +func (vm *VM) RecordWaitSignatures(t time.Duration) { + vm.metrics.waitSignatures.Observe(float64(t)) +} From 367a7dd6b948faec31172d7ff451dd99e171001d Mon Sep 17 00:00:00 2001 From: Patrick O'Grady Date: Wed, 5 Apr 2023 18:32:19 -0700 Subject: [PATCH 67/67] [mempool] make balance verification optional (#134) * make balance checking optional * config nit --- config/config.go | 1 + examples/tokenvm/config/config.go | 10 +++++++--- vm/dependencies.go | 1 + vm/vm.go | 31 ++++++++++++++++++++++++++++--- 4 files changed, 37 insertions(+), 6 deletions(-) diff --git a/config/config.go b/config/config.go index 24e3bae524..bd647f776c 100644 --- a/config/config.go +++ b/config/config.go @@ -31,6 +31,7 @@ func (c *Config) GetParallelism() int { func (c *Config) GetMempoolSize() int { return 2_048 } func (c *Config) GetMempoolPayerSize() int { return 32 } func (c *Config) GetMempoolExemptPayers() [][]byte { return nil } +func (c *Config) GetMempoolVerifyBalances() bool { return true } func (c *Config) GetDecisionsPort() uint16 { return 0 } // auto-assigned func (c *Config) GetBlocksPort() uint16 { return 0 } // auto-assigned func (c *Config) GetStreamingBacklogSize() int { return 1024 } diff --git a/examples/tokenvm/config/config.go b/examples/tokenvm/config/config.go index c92e0b7067..4cae30b992 100644 --- a/examples/tokenvm/config/config.go +++ b/examples/tokenvm/config/config.go @@ -26,6 +26,7 @@ const ( defaultPreferredBlocksPerSecond = 2 defaultContinuousProfilerFrequency = 1 * time.Minute defaultContinuousProfilerMaxFiles = 10 + defaultMempoolVerifyBalances = true ) type Config struct { @@ -44,9 +45,10 @@ type Config struct { StreamingBacklogSize int `json:"streamingBacklogSize"` // Mempool - MempoolSize int `json:"mempoolSize"` - MempoolPayerSize int `json:"mempoolPayerSize"` - MempoolExemptPayers []string `json:"mempoolExemptPayers"` + MempoolSize int `json:"mempoolSize"` + MempoolPayerSize int `json:"mempoolPayerSize"` + MempoolExemptPayers []string `json:"mempoolExemptPayers"` + MempoolVerifyBalances bool `json:"mempoolVerifyBalances"` // Order Book // @@ -96,6 +98,7 @@ func (c *Config) setDefault() { c.PreferredBlocksPerSecond = defaultPreferredBlocksPerSecond c.MempoolSize = c.Config.GetMempoolSize() c.MempoolPayerSize = c.Config.GetMempoolPayerSize() + c.MempoolVerifyBalances = defaultMempoolVerifyBalances c.StateSyncServerDelay = c.Config.GetStateSyncServerDelay() c.StreamingBacklogSize = c.Config.GetStreamingBacklogSize() } @@ -107,6 +110,7 @@ func (c *Config) GetPreferredBlocksPerSecond() uint64 { return c.PreferredBlocks func (c *Config) GetMempoolSize() int { return c.MempoolSize } func (c *Config) GetMempoolPayerSize() int { return c.MempoolPayerSize } func (c *Config) GetMempoolExemptPayers() [][]byte { return c.parsedExemptPayers } +func (c *Config) GetMempoolVerifyBalances() bool { return c.MempoolVerifyBalances } func (c *Config) GetDecisionsPort() uint16 { return c.DecisionsPort } func (c *Config) GetBlocksPort() uint16 { return c.BlocksPort } func (c *Config) GetTraceConfig() *trace.Config { diff --git a/vm/dependencies.go b/vm/dependencies.go index ca507a065c..b88be9e9f4 100644 --- a/vm/dependencies.go +++ b/vm/dependencies.go @@ -28,6 +28,7 @@ type Config interface { GetMempoolSize() int GetMempoolPayerSize() int GetMempoolExemptPayers() [][]byte + GetMempoolVerifyBalances() bool GetDecisionsPort() uint16 GetBlocksPort() uint16 GetStreamingBacklogSize() int diff --git a/vm/vm.go b/vm/vm.go index 4325682f17..2d4a7e0541 100644 --- a/vm/vm.go +++ b/vm/vm.go @@ -639,6 +639,28 @@ func (vm *VM) BuildBlockWithContext( return vm.buildBlock(ctx, blockContext) } +func (vm *VM) submitStateless(ctx context.Context, verifySig bool, txs []*chain.Transaction) (errs []error) { + validTxs := []*chain.Transaction{} + for _, tx := range txs { + txID := tx.ID() + // We already verify in streamer, let's avoid re-verification + if verifySig { + sigVerify := tx.AuthAsyncVerify() + if err := sigVerify(); err != nil { + // Failed signature verification is the only safe place to remove + // a transaction in listeners. Every other case may still end up with + // the transaction in a block. + vm.listeners.RemoveTx(txID, err) + errs = append(errs, err) + continue + } + } + validTxs = append(validTxs, tx) + } + vm.mempool.Add(ctx, validTxs) + return errs +} + func (vm *VM) Submit( ctx context.Context, verifySig bool, @@ -655,12 +677,15 @@ func (vm *VM) Submit( return []error{ErrNotReady} } + if !vm.config.GetMempoolVerifyBalances() { + return vm.submitStateless(ctx, verifySig, txs) + } + // Create temporary execution context blk, err := vm.GetStatelessBlock(ctx, vm.preferred) if err != nil { return []error{err} } - // TODO: under load this slows down block verify/accept state, err := blk.State() if err != nil { // This will error if a block does not yet have processed state. @@ -673,8 +698,6 @@ func (vm *VM) Submit( return []error{err} } oldestAllowed := now - r.GetValidityWindow() - - // Process txs to see if should add validTxs := []*chain.Transaction{} for _, tx := range txs { txID := tx.ID() @@ -697,6 +720,8 @@ func (vm *VM) Submit( errs = append(errs, ErrNotAdded) continue } + // TODO: do we need this? (just ensures people can't spam mempool with + // txs from already verified blocks) repeat, err := blk.IsRepeat(ctx, oldestAllowed, []*chain.Transaction{tx}) if err != nil { errs = append(errs, err)