tests: add mainnet state to localosmosis (backport #1627) (#1652)

* tests: add mainnet state to localosmosis (#1627)

* dont change operator address

* state export changes

* additions

* changes

* apt get update

* changes

* extra notes

* adding flags

* add ID

* linter fixes

* move files

(cherry picked from commit 8043e65)

# Conflicts:
#	Makefile
#	cmd/osmosisd/cmd/testnetify/testnetify.py
#	tests/localosmosis/README.md

* Update testnetify.py

Co-authored-by: Adam Tucker <[email protected]>

Makefile

@@ -272,16 +272,25 @@ localnet-keys:
 localnet-build:
 	@docker build -t local:osmosis -f tests/localosmosis/Dockerfile .
 
+localnet-build-state-export:
+	@docker build -t local:osmosis-se --build-arg ID=$(ID) -f tests/localosmosis/mainnet_state/Dockerfile-stateExport .
+
 localnet-start:
 	@docker-compose -f tests/localosmosis/docker-compose.yml up
 
-localnet-remove:
+localnet-start-state-export:
+	@docker-compose -f tests/localosmosis/mainnet_state/docker-compose-state-export.yml up
+
+localnet-stop:
 	@docker-compose -f tests/localosmosis/docker-compose.yml down
 
+localnet-remove: localnet-stop
+	PWD=$(shell pwd)
+	@docker run --user root -v ${PWD}/tests/localosmosis/.osmosisd:/root/osmosis ubuntu /bin/sh -c "rm -rf /root/osmosis/*"
+
+localnet-remove-state-export:
+	@docker-compose -f tests/localosmosis/mainnet_state/docker-compose-state-export.yml down
+
 .PHONY: all build-linux install format lint \
-	go-mod-cache draw-deps clean build \
-	setup-transactions setup-contract-tests-data start-osmosis run-lcd-contract-tests contract-tests \
-	test test-all test-build test-cover test-unit test-race \
-	benchmark \
-	build-docker-osmosisdnode localnet-start localnet-stop \
-	docker-single-node
+	go-mod-cache draw-deps clean build build-contract-tests-hooks \
+	test test-all test-build test-cover test-unit test-race benchmark

cmd/osmosisd/cmd/testnetify/testnetify.py

@@ -6,45 +6,38 @@
 
 #get values from your priv_validator_key.json to later switch with high power validator
 
+daemon_name = "osmosisd"
+
 #get bas64
-result = subprocess.run(["osmosisd","tendermint","show-validator"], stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True)
+result = subprocess.run([daemon_name,"tendermint","show-validator"], stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True)
 base64 = result.stdout.strip()
 ##base64 = '{"@type":"/cosmos.crypto.ed25519.PubKey","key":"3QVAkiUIkKR3B6kkbd+QqzWDdcExoggbZV5fwH4jKDs="}'
-##bash64_wal = '{"@type":"/cosmos.crypto.secp256k1.PubKey","key":"A2MR6q+pOpLtdxh0tHHe2JrEY2KOcvRogtLxHDHzJvOh"}'
 
 #get validator cons pubkey
 val_pubkey = base64[base64.find('key":') +6 :-2]
 ##val_pubkey = "3QVAkiUIkKR3B6kkbd+QqzWDdcExoggbZV5fwH4jKDs="
-#CAN MODIFY
-val_pubkey_wal = "A2MR6q+pOpLtdxh0tHHe2JrEY2KOcvRogtLxHDHzJvOh"
 
 #osmosisd debug pubkey {base64} to get address
-debug_pubkey = subprocess.run(["osmosisd","debug", "pubkey", base64], stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True)
+debug_pubkey = subprocess.run([daemon_name,"debug", "pubkey", base64], stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True)
 
 #address
 address = debug_pubkey.stderr[9: debug_pubkey.stderr.find("\n")]
 ##based on show-valdiator
 ##address = "214D831D6F49A75F9104BDC3F2E12A6CC1FC5669"
-##address_wal = "54366539BF22D6C20982452A4532607014097579"
 
 #feed address into osmosisd debug addr {address} to get bech32 validator address (osmovaloper)
-bech32 = subprocess.run(["osmosisd","debug", "addr", address], stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True)
+bech32 = subprocess.run([daemon_name,"debug", "addr", address], stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True)
 #osmovaloper
 bech32_val = bech32.stderr[bech32.stderr.find("Val: ") + 5: -1]
 ##operator address
 ##bech32_val = "osmovaloper1y9xcx8t0fxn4lygyhhpl9cf2dnqlc4nf4pymm4"
-#CAN MODIFY
-bech32_val_wal = "osmovaloper12smx2wdlyttvyzvzg54y2vnqwq2qjatex7kgq4"
 
 #pass osmovaloper address into osmosisd debug bech32-convert -p osmovalcons
-bech32_convert = subprocess.run(["osmosisd","debug", "bech32-convert", bech32_val, "-p", "osmovalcons"], stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True)
+bech32_convert = subprocess.run([daemon_name,"debug", "bech32-convert", bech32_val, "-p", "osmovalcons"], stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True)
 #osmovalcons
-#CAN MODIFY
 final_address = bech32_convert.stderr[:bech32_convert.stderr.find("\n")]
 ##osmovalcons is taken from show-validator
 ##final_address = "osmovalcons1y9xcx8t0fxn4lygyhhpl9cf2dnqlc4nfpjh8h5"
-##final_address_wal = "osmovalcons12smx2wdlyttvyzvzg54y2vnqwq2qjatejd95v5"
-
 
 #own opp address
 #exchange the op_address and op_pubkey with own address and pubkey or use above mnemonic for following address
@@ -58,6 +51,12 @@
 #CAN MODIFY
 op_pubkey = "A2MR6q+pOpLtdxh0tHHe2JrEY2KOcvRogtLxHDHzJvOh"
 
+#feed address into osmosisd debug addr {address} to get bech32 validator op address (osmovaloper)
+# bech32_op = subprocess.run([daemon_name,"debug", "addr", op_address], stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True)
+#osmovaloper
+# bech32_valoper = bech32_op.stderr[bech32_op.stderr.find("Val: ") + 5: -1]
+# osmovaloper12smx2wdlyttvyzvzg54y2vnqwq2qjatex7kgq4
+
 def sed_inplace(filename, pattern, repl):
     '''
     Perform the pure-Python equivalent of in-place `sed` substitution: e.g.,
@@ -82,7 +81,7 @@ def sed_inplace(filename, pattern, repl):
 #validator cons pubkey:     val_pubkey
 #osmovalcons:               final_address
 #validator hex address:     address
-#osmovaloper:               bech32_val
+#osmovaloper:               bech32_valoper
 #actual account:            op_address
 #accounts pubkey:           op_pubkey
 
@@ -100,8 +99,8 @@ def sed_inplace(filename, pattern, repl):
 sed_inplace("testnet_genesis.json", "16A169951A878247DBE258FDDC71638F6606D156", address)
 
 #replace validator osmovaloper
-print("Replacing osmovaloper1cyw4vw20el8e7ez8080md0r8psg25n0cq98a9n with " + bech32_val_wal)
-sed_inplace("testnet_genesis.json", "osmovaloper1cyw4vw20el8e7ez8080md0r8psg25n0cq98a9n", bech32_val_wal)
+#print("Replacing osmovaloper1cyw4vw20el8e7ez8080md0r8psg25n0cq98a9n with " + bech32_valoper)
+#sed_inplace("testnet_genesis.json", "osmovaloper1cyw4vw20el8e7ez8080md0r8psg25n0cq98a9n", bech32_valoper)
 
 #replace actual account
 print("Replacing osmo1cyw4vw20el8e7ez8080md0r8psg25n0c6j07j5 with " + op_address)
@@ -201,9 +200,10 @@ def sed_inplace(filename, pattern, repl):
 new_power = str(current_power + 1000000000)
 print("New validator power is " + new_power)
 val_power_list[val_power_index]['power'] = new_power
-#get index of val power in app state (osmovaloper) (bech32_val_wal)
+#get index of val power in app state (osmovaloper) (bech32_valoper)
 last_val_power_list = read_test_gen['app_state']['staking']['last_validator_powers']
-last_val_power_index = [i for i, elem in enumerate(last_val_power_list) if bech32_val_wal in elem['address']][0]
+#last_val_power_index = [i for i, elem in enumerate(last_val_power_list) if bech32_valoper in elem['address']][0]
+last_val_power_index = [i for i, elem in enumerate(last_val_power_list) if 'osmovaloper1cyw4vw20el8e7ez8080md0r8psg25n0cq98a9n' in elem['address']][0]
 val_power = int(read_test_gen['app_state']['staking']['last_validator_powers'][last_val_power_index]['power'])
 print("Current validator power in second location is " + str(val_power))
 new_val_power = str(val_power + 1000000000)

tests/localosmosis/README.md

@@ -1,5 +1,7 @@
 # LocalOsmosis
 
+## Use LocalOsmosis to Test Local Changes
+
 You can now quickly test your changes to Osmosis with just a few commands:
 
 1. Make any change to the osmosis code that you want to test
@@ -9,6 +11,9 @@ You can now quickly test your changes to Osmosis with just a few commands:
 
 3. Once complete, run `make localnet-start`
     - You will now be running a local network with your changes!
+    - The files in `tests/e2e/localosmosis/.osmosisd` that are produced
+    by this command can only be removed by running `make localnet-remove`
+    - That will reset the chain to genesis
 
 4. To add your validator wallet and 9 other preloaded wallets automatically, run `make localnet-keys`
     - These keys are added to your --keyring-backend test
@@ -18,6 +23,98 @@ You can now quickly test your changes to Osmosis with just a few commands:
 
 5. To remove all block history and start from scratch, run `make localnet-remove`
 
+6. To stop the chain but keep the state, run `make localnet-stop`
+
+## LocalOsmosis with Mainnet State
+
+Running LocalOsmosis with mainnet state is resource intensive and can take a bit of time. It is recommended to only use this method if you are testing a new feature that must be thoroughly tested before pushing to production.
+
+A few things to note before getting started. The below method will only work if you are using the same version as mainnet. In other words, if mainnet is on v8.0.0 and you try to do this on a v9.0.0 tag or on main, you will run into an error when initializing the genesis. (yes, it is possible to create a state exported testnet on a upcoming release, but that is out of the scope of this tutorial)
+
+Additionally, this process requires 64GB of RAM. If you do not have 64GB of RAM, you will get an OOM error.
+
+1. Set up a node on mainnet (easiest to use the [get.osmosis.zone](https://get.osmosis.zone) tool). This will be the node you use to run the state exported testnet, so ensure it has at least 64GB of RAM.
+```sh
+curl -sL https://get.osmosis.zone/install > i.py && python3 i.py
+```
+
+2. Once the installer is done, ensure your node is hitting blocks.
+```sh
+source ~/.profile
+journalctl -u osmosisd.service -f
+```
+
+3. Stop your Osmosis daemon
+```sh
+systemctl stop osmosisd.service
+```
+
+4. Take a state export snapshot with the following command:
+```sh
+cd $HOME
+osmosisd export 2> testnet_genesis.json
+```
+After a while (~15 minutes), this will create a file called `testnet_genesis.json` which is a snapshot of the current mainnet state.
+
+
+5. Copy the `testnet_genesis.json` to the localosmosis folder within the osmosis repo
+```sh
+cp -r $HOME/testnet_genesis.json $HOME/osmosis/tests/localosmosis
+```
+
+6. Ensure you have docker and docker compose installed/running:
+Docker
+```sh
+sudo apt-get remove docker docker-engine docker.io
+sudo apt-get update
+sudo apt install docker.io -y
+```
+
+Docker Compose
+```sh
+sudo apt install docker-compose -y
+```
+
+7. Compile the local:osmosis-se docker image (~15 minutes, since this process modifies the testnet genesis you provided above). You may change the exported ID to whatever you want the chain-id to be. In this example, we will use the chain-id of localosmosis.
+```sh
+cd $HOME/osmosis
+export ID=local
+make localnet-build-state-export
+```
+
+
+8. Start the local:osmosis-se docker image
+```sh
+make localnet-start-state-export
+```
+
+You will then go through the genesis intialization process. This will take ~15 minutes. You will then hit the first block (not block 1, but the block number after your snapshot was taken), and then you will just see a bunch of p2p error logs with some KV store logs. **This will happen for about 1 hour**, and then you will finally hit blocks at a normal pace.
+
+9. On your host machine, add this specific wallet which holds a large amount of osmo funds
+```sh
+echo "bottom loan skill merry east cradle onion journey palm apology verb edit desert impose absurd oil bubble sweet glove shallow size build burst effort" | osmosisd keys add wallet --recover --keyring-backend test
+```
+
+You now are running a validator with a majority of the voting power with the same mainnet state as when you took the snapshot.
+
+10. On your host machine, you can now query the state export testnet like so:
+```sh
+osmosisd status
+```
+
+11. Here is an example command to ensure complete understanding:
+```sh
+osmosisd tx bank send wallet osmo1nyphwl8p5yx6fxzevjwqunsfqpcxukmtk8t60m 10000000uosmo --chain-id testing1 --keyring-backend test
+```
+
+12. To stop the container and remove its data:
+```sh
+make localnet-remove-state-export
+```
+
+Note: At some point, all the validators (except yours) will get jailed at the same block due to them being offline. When this happens, it make take a little bit of time to process. Once all validators are jailed, you will continue to hit blocks as you did before. If you are only running the validator for a short period of time (< 24 hours) you will not experience this.
+
+
 ## Accounts
 
 LocalOsmosis is pre-configured with one validator and 9 accounts with ION and OSMO balances.

tests/localosmosis/mainnet_state/Dockerfile-stateExport

@@ -0,0 +1,62 @@
+# syntax=docker/dockerfile:1
+
+
+# --------------------------------------------------------
+# Build
+# --------------------------------------------------------
+
+FROM golang:1.18.2-alpine3.15 as build
+
+RUN set -eux; apk add --no-cache ca-certificates build-base;
+RUN apk add git
+# Needed by github.com/zondax/hid
+RUN apk add linux-headers
+
+WORKDIR /osmosis
+COPY . /osmosis
+
+
+# CosmWasm: see https://github.com/CosmWasm/wasmvm/releases
+ADD https://github.com/CosmWasm/wasmvm/releases/download/v1.0.0/libwasmvm_muslc.aarch64.a /lib/libwasmvm_muslc.aarch64.a
+ADD https://github.com/CosmWasm/wasmvm/releases/download/v1.0.0/libwasmvm_muslc.x86_64.a /lib/libwasmvm_muslc.x86_64.a
+RUN sha256sum /lib/libwasmvm_muslc.aarch64.a | grep 7d2239e9f25e96d0d4daba982ce92367aacf0cbd95d2facb8442268f2b1cc1fc
+RUN sha256sum /lib/libwasmvm_muslc.x86_64.a | grep f6282df732a13dec836cda1f399dd874b1e3163504dbd9607c6af915b2740479
+
+# CosmWasm: copy the right library according to architecture. The final location will be found by the linker flag `-lwasmvm_muslc`
+RUN cp /lib/libwasmvm_muslc.$(uname -m).a /lib/libwasmvm_muslc.a
+
+RUN BUILD_TAGS=muslc LINK_STATICALLY=true make build
+
+# --------------------------------------------------------
+# Runner
+# --------------------------------------------------------
+
+FROM ubuntu
+
+COPY --from=build /osmosis/build/osmosisd /bin/osmosisd
+COPY /tests/localosmosis/mainnet_state/statesync.sh /osmosis/statesync.sh
+COPY /tests/localosmosis/mainnet_state/testnetify.py /osmosis/testnetify.py
+COPY /tests/localosmosis/testnet_genesis.json /osmosis/testnet_genesis.json
+
+ENV HOME /osmosis
+WORKDIR $HOME
+# RUN apk update
+# RUN apk add jq
+# RUN apk add moreutils
+# RUN rm -rf /var/cache/apk/*
+# RUN apk add --no-cache python3 py3-pip
+# RUN apt-get update && apt-get install -y software-properties-common gcc && \
+#     add-apt-repository -y ppa:deadsnakes/ppa
+# RUN apt-get update && apt-get install -y python3.6 python3-distutils python3-pip python3-apt
+RUN apt-get update && apt-get install -y python3
+RUN chmod +x /osmosis/statesync.sh
+RUN /osmosis/statesync.sh
+ARG ID=localosmosis
+RUN python3 /osmosis/testnetify.py --chain-id=$ID
+RUN cp testnet_genesis.json .osmosisd/config/genesis.json
+EXPOSE 26656
+EXPOSE 26657
+EXPOSE 1317
+
+ENTRYPOINT ["osmosisd"]
+CMD ["start", "--x-crisis-skip-assert-invariants"]

tests/localosmosis/mainnet_state/docker-compose-state-export.yml

@@ -0,0 +1,15 @@
+version: "3"
+
+services:
+  osmosisd:
+    image: local:osmosis-se
+    user: "root:root"
+    command:
+      - start
+      - --x-crisis-skip-assert-invariants
+      - --rpc.laddr=tcp://0.0.0.0:26657
+    ports:
+      - "26657:26657"
+      - "1317:1317"
+      - "9090:9090"
+      - "9091:9091"

tests/localosmosis/mainnet_state/statesync.sh

@@ -0,0 +1,7 @@
+#!/bin/bash
+
+# initialize osmosis
+osmosisd init --chain-id=localosmosis val
+# remove seeds
+sed -i.bak -E 's#^(seeds[[:space:]]+=[[:space:]]+).*$#\1""#' ~/.osmosisd/config/config.toml
+sed -i.bak -E 's#^(fast_sync[[:space:]]+=[[:space:]]+).*$#\1false#' ~/.osmosisd/config/config.toml