DEV 841: Further docker, tests improvements

.github/workflows/release.yml

@@ -155,38 +155,10 @@ jobs:
           name: cheqd-node-image.tar
           path: cheqd-node-image.tar
 
-  build-testnet-image:
-    name: Build cheqd-testnet image
-    runs-on: ubuntu-20.04
-    needs: build-node-images
-    steps:
-      - name: Check out
-        uses: actions/checkout@v2
-
-      - name: Download node image
-        uses: actions/download-artifact@v2
-        with:
-          name: cheqd-cli-image.tar
-
-      - name: Load node image
-        run: docker load -i cheqd-cli-image.tar
-
-      - name: Build
-        run: docker build -f tests/networks/docker-localnet/Dockerfile --no-cache -t cheqd-testnet .
-
-      - name: Save
-        run: docker save -o cheqd-testnet-image.tar cheqd-testnet
-
-      - name: Store artifact
-        uses: actions/upload-artifact@v2
-        with:
-          name: cheqd-testnet-image.tar
-          path: cheqd-testnet-image.tar
-
   publish:
     name: "Publish images and release"
     # Reference to workflow-setup job is required to access its various outputs.
-    needs: [ build-binary, build-node-images, build-testnet-image, build-deb-package, build-tar-package ]
+    needs: [ build-binary, build-node-images, build-deb-package, build-tar-package ]
     runs-on: ubuntu-latest
     env:
       VERSION: ${{ needs.build-binary.outputs.VERSION }}
@@ -210,14 +182,6 @@ jobs:
       - name: Load cheqd-node image
         run: docker load -i cheqd-node-image.tar
 
-      - name: Download testnet image
-        uses: actions/download-artifact@v2
-        with:
-          name: cheqd-testnet-image.tar
-
-      - name: Load testnet image
-        run: docker load -i cheqd-testnet-image.tar
-
       - name: Login to docker
         run: |
           echo ${{ secrets.GH_PAT }} | docker login ghcr.io --username ${{ secrets.GH_USER }} --password-stdin
@@ -229,20 +193,12 @@ jobs:
           docker push ghcr.io/${{ github.repository }}:${{ env.VERSION }}
           docker push ghcr.io/${{ github.repository }}:latest
 
-
       - name: Push cheqd-cli image
         run: |
-          docker tag cheqd-cli ghcr.io/${{ env.GITHUB_ORG_NAME }}/cheqd-cli:${{ env.VERSION }}
-          docker tag cheqd-cli ghcr.io/${{ env.GITHUB_ORG_NAME }}/cheqd-cli:latest
-          docker push ghcr.io/${{ env.GITHUB_ORG_NAME }}:${{ env.VERSION }}
-          docker push ghcr.io/${{ env.GITHUB_ORG_NAME }}:latest
-
-      - name: Push testnet image
-        run: |
-          docker tag cheqd-testnet ghcr.io/${{ github.repository_owner }}/cheqd-testnet:${{ env.VERSION }}
-          docker tag cheqd-testnet ghcr.io/${{ github.repository_owner }}/cheqd-testnet:latest
-          docker push ghcr.io/${{ github.repository_owner }}/cheqd-testnet:${{ env.VERSION }}
-          docker push ghcr.io/${{ github.repository_owner }}/cheqd-testnet:latest
+          docker tag cheqd-cli ghcr.io/${{ github.repository_owner }}/cheqd-cli:${{ env.VERSION }}
+          docker tag cheqd-cli ghcr.io/${{ github.repository_owner }}/cheqd-cli:latest
+          docker push ghcr.io/${{ github.repository_owner }}/cheqd-cli:${{ env.VERSION }}
+          docker push ghcr.io/${{ github.repository_owner }}/cheqd-cli:latest
 
       - name: Download deb
         uses: actions/download-artifact@v2

.github/workflows/test.yml

@@ -9,10 +9,9 @@ on:
     - '.gitbook/**'
 
 env:
-  NODE_CONFIGS_BASE: "/home/runner/work/cheqd-node/cheqd-node/tests/networks/docker-compose-localnet/network-config"
+  NODE_CONFIGS_BASE: "/home/runner/work/cheqd-node/cheqd-node/docker/localnet/network-config"
 
 jobs:
-
   run-unit-test:
     name: Run unit tests
     runs-on: ubuntu-20.04
@@ -175,34 +174,6 @@ jobs:
           name: cheqd-node-image.tar
           path: cheqd-node-image.tar
 
-  build-testnet-image:
-    name: Build cheqd-testnet image
-    runs-on: ubuntu-20.04
-    needs: build-node-images
-    steps:
-      - name: Check out
-        uses: actions/checkout@v2
-
-      - name: Download node image
-        uses: actions/download-artifact@v2
-        with:
-          name: cheqd-cli-image.tar
-
-      - name: Load node image
-        run: docker load -i cheqd-cli-image.tar
-
-      - name: Build
-        run: docker build -f tests/networks/docker-localnet/Dockerfile --no-cache -t cheqd-testnet .
-
-      - name: Save
-        run: docker save -o cheqd-testnet-image.tar cheqd-testnet
-
-      - name: Store artifact
-        uses: actions/upload-artifact@v2
-        with:
-          name: cheqd-testnet-image.tar
-          path: cheqd-testnet-image.tar
-
   test-new-node-setup-from-deb:
     name: Test new node setup from deb
     runs-on: ubuntu-20.04
@@ -221,34 +192,34 @@ jobs:
       - name: Download node image
         uses: actions/download-artifact@v2
         with:
-          name: cheqd-cli-image.tar
+          name: cheqd-node-image.tar
 
       - name: Load node image
-        run: docker load -i cheqd-cli-image.tar
+        run: docker load -i cheqd-node-image.tar
 
       - name: Check out
         uses: actions/checkout@v2
 
-      - name: Set up 4 validators + 2 observers node docker pool  # TODO: Replace with testnet image
-        working-directory: ./tests/networks/docker-compose-localnet
+      - name: Set up 4 validators + 2 observers node docker pool
+        working-directory: ./docker/localnet
         run: |
           bash gen-network-config.sh
-          docker-compose up -d
+          CHEQD_NODE_IMAGE=cheqd-node CHEQD_NODE_VERSION=latest docker compose up -d
 
       - name: Wait for chain
-        run: bash tests/networks/tools/wait-for-chain.sh
+        run: bash tests/tools/wait-for-chain.sh
 
       - name: Add observer node
-        working-directory: ./tests/networks/docker-compose-localnet
+        working-directory: ./tests/e2e-complex/deb-install
         run: |
           ./add-observer.sh
           cheqd-noded status -n tcp://localhost:26677 2>&1
           ./wait.sh '[[ $(cheqd-noded status -n '"'"'tcp://localhost:26677'"'"' 2>&1 | wc -l) == 1 ]] && echo "New node returns status!"'
 
       - name: Promote observer to validator
-        working-directory: ./tests/networks/docker-compose-localnet
+        working-directory: ./tests/e2e-complex/deb-install
         run: |
-          bash -x promote-validator.sh
+          bash promote-validator.sh
           cheqd-noded query staking validators --node "http://localhost:26657" | sed -nr 's/.*status: (.*?).*/\1/p' $1 | while read x; do [[ "BOND_STATUS_BONDED" == $x ]] && echo "Validator's status is bonded!" || exit 1 ; done
           ./wait.sh '[[ $(curl -s localhost:26657/block | sed -nr '"'"'s/.*signature": (.*?).*/\1/p'"'"' | wc -l) == 5 ]] && echo "There are 5 validators signatures in block!"'
           ./wait.sh '[[ $(curl -s localhost:26657/block | sed -nr '"'"'s/.*(signature": null).*/\1/p'"'"' | wc -l) == 0 ]] && echo "There are no null signatures in block!"'
@@ -258,11 +229,6 @@ jobs:
         run: |
           journalctl -ex | grep cheqd
 
-      - name: Gather logs on failure 2
-        if: ${{ failure() }}
-        run: |
-          journalctl -u service-name.service --no-pager
-
   run-python-based-integration-tests:
     name: Run python based cosmos and identity tests
     runs-on: ubuntu-20.04
@@ -280,50 +246,51 @@ jobs:
       - name: Download node image
         uses: actions/download-artifact@v2
         with:
-          name: cheqd-cli-image.tar
+          name: cheqd-node-image.tar
 
       - name: Load node image
-        run: docker load -i cheqd-cli-image.tar
+        run: docker load -i cheqd-node-image.tar
 
-      - uses: actions/checkout@v2
+      - name: Check out
+        uses: actions/checkout@v2
 
-      - name: Set up 4 validators + 2 observers node docker pool for local test run # Replace with testnet image
-        working-directory: ./tests/networks/docker-compose-localnet
+      - name: Set up 4 validators + 2 observers node docker pool
+        working-directory: ./docker/localnet
         run: |
           bash gen-network-config.sh
-          docker-compose up -d
+          CHEQD_NODE_IMAGE=cheqd-node CHEQD_NODE_VERSION=latest docker compose up -d
 
       - name: Wait for chain
-        run: bash tests/networks/tools/wait-for-chain.sh
+        run: bash tests/tools/wait-for-chain.sh
 
       - name: Set up test environment
-        working-directory: ./tests/e2e
+        working-directory: ./tests/e2e-pytest
         run: |
           pip3 install -r requirements.txt >> /dev/null
           cp -Rf ${NODE_CONFIGS_BASE}/validator-0/. /home/runner/.cheqdnode
           cp -Rf ${NODE_CONFIGS_BASE}/validator-1/. /home/runner/.cheqdnode
           sudo chmod -R 777 /home/runner
 
       - name: Run cosmos-related tests
+        working-directory: ./tests/e2e-pytest
         run: |
           set -euxo pipefail
           export OP0_ADDRESS=$(cheqd-noded keys list --keyring-backend "test" --home "${NODE_CONFIGS_BASE}/validator-0" | sed -nr 's/.*address: (.*?).*/\1/p' | sed 's/\r//g')
           export OP1_ADDRESS=$(cheqd-noded keys list --keyring-backend "test" --home "${NODE_CONFIGS_BASE}/validator-1" | sed -nr 's/.*address: (.*?).*/\1/p' | sed 's/\r//g')
-          pytest -v -rP tests/e2e/test_cosmos.py
+          pytest -v -rP test_cosmos.py
 
       - name: Run identity-related tests # TODO: Move into separate stage?
+        working-directory: ./tests/e2e-pytest
         run: |
           export OP0_ADDRESS=$(cheqd-noded keys list --keyring-backend "test" | sed -nr 's/.*address: (.*?).*/\1/p' | sed -n 1p | sed 's/\r//g')
           export OP1_ADDRESS=$(cheqd-noded keys list --keyring-backend "test" | sed -nr 's/.*address: (.*?).*/\1/p' | sed -n 2p | sed 's/\r//g')
-          pytest -v -rP tests/e2e/test_identity.py
+          pytest -v -rP test_identity.py
 
   run-bash-based-integration-tests:
     name: Run bash based identity tests
     runs-on: ubuntu-20.04
-    needs: [build-binary, build-testnet-image]
+    needs: [build-binary, build-node-images]
     steps:
-      - uses: actions/checkout@v2
-
       - name: Load binary artifact
         uses: actions/download-artifact@v2
         with:
@@ -333,21 +300,28 @@ jobs:
       - name: Restore binary permissions
         run: sudo chmod +x /home/runner/.local/bin/cheqd-noded
 
-      - name: Download testnet image
+      - name: Download node image
         uses: actions/download-artifact@v2
         with:
-          name: cheqd-testnet-image.tar
+          name: cheqd-node-image.tar
+
+      - name: Load node image
+        run: docker load -i cheqd-node-image.tar
 
-      - name: Load testnet image
-        run: docker load -i cheqd-testnet-image.tar
+      - name: Check out
+        uses: actions/checkout@v2
 
-      - name: Run test net container
+      - name: Set up 4 validators + 2 observers node docker pool
+        working-directory: ./docker/localnet
         run: |
-          docker run -d --rm -p "26657:26657" -p "1317:1317" cheqd-testnet
-          bash tests/networks/tools/wait-for-chain.sh
+          bash gen-network-config.sh
+          CHEQD_NODE_IMAGE=cheqd-node CHEQD_NODE_VERSION=latest docker compose up -d
+
+      - name: Wait for chain
+        run: bash tests/tools/wait-for-chain.sh
 
       - name: Set up and run tests
-        working-directory: tests/cli
+        working-directory: tests/e2e-bash
         run: |
           bash import_keys.sh
           bash run_all.sh
@@ -361,25 +335,25 @@ jobs:
         with:
           fetch-depth: 0 # It's needed for getting the full version of package by `git describe`
 
-      - name: Download node image
+      - name: Download cli image
         uses: actions/download-artifact@v2
         with:
           name: cheqd-cli-image.tar
 
-      - name: Load node image
+      - name: Load cli image
         run: docker load -i cheqd-cli-image.tar
 
       - name: Chown for current user
         run: sudo chown $USER:$USER .
 
       - name: Prepare nodes for checking upgrade
-        working-directory: ./tests/upgrade
+        working-directory: ./tests/e2e-complex/upgrade
         run: bash prepare.sh
 
       - name: Initiate the upgrade process
-        working-directory: ./tests/upgrade
+        working-directory: ./tests/e2e-complex/upgrade
         run: bash initiate_upgrade.sh
 
       - name: Make the upgrade and check results
-        working-directory: ./tests/upgrade
+        working-directory: ./tests/e2e-complex/upgrade
         run: bash upgrade_and_check.sh

docker/Dockerfile

@@ -85,7 +85,7 @@ ENTRYPOINT [ "cheqd-noded" ]
 FROM base AS node
 
 # Set runner script
-COPY --chown=cheqd:cheqd docker/persistent-chains/entrypoint.sh /bin/node-start
+COPY --chown=cheqd:cheqd docker/entrypoint.sh /bin/node-start
 RUN chmod +x /bin/node-start
 
 # Default entrypoint for cheqd-noded CLI usage

docker/entrypoint.sh

@@ -0,0 +1,58 @@
+#!/bin/bash
+
+# Inits node configuration and runs the node.
+# e -> exit immediately, u -> treat unset variables as errors and immediately, o -> sets the exit code to the rightmost command 
+set -euo pipefail
+
+# within the container, $HOME=/home/cheqd
+CHEQD_ROOT_DIR="$HOME/.cheqdnode"
+
+# Init node config directory
+if [ ! -d "${CHEQD_ROOT_DIR}/config" ]
+then
+    echo "Node config not found. Initializing."
+    cheqd-noded init "moniker-placeholder"
+else
+    echo "Node config exists. Skipping initialization."
+fi
+
+# Check if a genesis file has been passed in config
+if [ -f "/genesis" ]
+then
+    echo "Genesis file passed. Adding/replacing current genesis file."
+    cp /genesis "${CHEQD_ROOT_DIR}/config/genesis.json"
+else
+    echo "No genesis file passed. Skipping and retaining existing genesis."
+fi
+
+# Check if a seeds file has been passed in config
+if [ -f "/seeds" ]
+then
+    echo "Seeds file passed. Overriding current seeds."
+    cp /seeds "${CHEQD_ROOT_DIR}/config/seeds.txt"
+    CHEQD_NODED_P2P_SEEDS="$(cat "${CHEQD_ROOT_DIR}/config/seeds.txt")"
+    export CHEQD_NODED_P2P_SEEDS
+else
+    echo "No seeds file passed. Skipping and retaining existing seeds."
+fi
+
+# Check if a node_key file has been passed in config
+if [ -f "/node_key" ]
+then
+    echo "Node key file passed. Overriding current key."
+    cp /node_key "${CHEQD_ROOT_DIR}/config/node_key.json"
+else
+    echo "No node key file passed. Skipping and retaining existing node key."
+fi
+
+# Check if a priv_validator_key file has been passed in config
+if [ -f "/priv_validator_key" ]
+then
+    echo "Private validator key file passed. Replacing current key."
+    cp /priv_validator_key "${CHEQD_ROOT_DIR}/config/priv_validator_key.json"
+else
+    echo "No private validator key file passed. Skipping and retaining existing key."
+fi
+
+# Run node
+cheqd-noded start

docker/localnet/container-env/observer-0.env

@@ -0,0 +1,11 @@
+# Environment variables used by the container
+
+# General
+CHEQD_NODED_MONIKER="observer-0"
+CHEQD_NODED_RPC_LADDR="tcp://0.0.0.0:26657"
+CHEQD_NODED_API_ENABLE="true"
+# Consensus
+CHEQD_NODED_CONSENSUS_TIMEOUT_PROPOSE="500ms"
+CHEQD_NODED_CONSENSUS_TIMEOUT_PREVOTE="500ms"
+CHEQD_NODED_CONSENSUS_TIMEOUT_PRECOMMIT="500ms"
+CHEQD_NODED_CONSENSUS_TIMEOUT_COMMIT="500ms"