Unified network configuration generation

.Dockerignore

@@ -1,8 +1,25 @@
-ci
-.devcontainer
+### APP-SPECIFIC
 .github
+*.md
+build-tools/*
+docs/*
+architecture/*
+persistent_chains/*
+
+
+### GENERAL EXCLUSIONS ###
+
+### macOS ###
+# General
+.DS_Store
+.AppleDouble
+.LSOverride
+
+# IDE Exclusions
+.devcontainer
+.vscode
+.vscode-upload.json
 .idea
-vue/node_modules
-vue/src/store/generated
-vue/dist
-secret.yml
+
+# Python
+.pytest_cache

.github/linters/mlc_config.json

@@ -1,3 +1,3 @@
 {
-    "aliveStatusCodes": [200, 206, 999]
-}
+    "aliveStatusCodes": [0, 200, 206, 403, 999]
+}

.github/workflows/lint.yml

@@ -32,14 +32,14 @@ jobs:
           folder-path: "."
 
   sh-euox-pipefail-check:
-    name: Check for 'set -euox pipelail' in shell scripts
+    name: Check for 'set -euox pipefail' in shell scripts
     runs-on: ubuntu-latest
     steps:
       - name: Checkout code
         uses: actions/checkout@v2
 
-      - name: Check that all shell scripts use 'set -euox pipelail'
-        run: bash .github/scripts/enshure_set_euox_pipefail.sh
+      - name: Check that all shell scripts use 'set -euox pipefail'
+        run: bash .github/scripts/ensure_set_euox_pipefail.sh
 
   super-lint:
     name: Run super linter

.github/workflows/release.yml

@@ -20,10 +20,10 @@ jobs:
       - name: Set outputs
         id: cache
         run: |
-          # Exposes org_name/repository_name in lower case. Required by docker.
+          # Exposes org_name/repository_name in lower case. Required by Docker.
           echo "::set-output name=GITHUB_REPOSITORY_NAME::$(echo ${GITHUB_REPOSITORY,,})"
 
-          # Version is tag without `v` prefix. Reuired by deb package.
+          # Version is tag without `v` prefix. Required by Debian package.
           TAG=${GITHUB_REF/refs\/tags\//}
           echo "::set-output name=TAG::$TAG"
 
@@ -39,7 +39,7 @@ jobs:
         with:
           go-version: 1.17
 
-      - name: Get go protoc compiler plugins
+      - name: Get Go protoc compiler plugins
         env:
           GOLANG_PROTOBUF_VERSION: 1.3.5
           GOGO_PROTOBUF_VERSION: 1.3.2
@@ -78,7 +78,7 @@ jobs:
           path: build/cheqd-noded
 
   build-deb-package:
-    name: Build deb package
+    name: Build Debian package
     runs-on: ubuntu-20.04
     needs: [ build-binary, setup-workflow ]
     env:
@@ -97,36 +97,36 @@ jobs:
         run: sudo chmod +x /home/runner/go/bin/cheqd-noded
 
       - name: Make intermediate archive
-        working-directory: ./build_tools
+        working-directory: ./build-tools
         run: |
-          ./build_tar.sh ${{ env.PACKAGE_NAME }} ${{ env.VERSION }}
+          ./build-tar.sh ${{ env.PACKAGE_NAME }} ${{ env.VERSION }}
 
       - name: Install fpm
         run: |
           sudo apt-get install ruby ruby-dev rubygems build-essential
           sudo gem install --no-document fpm
 
-      - name: Build deb # Depends on tar archive
+      - name: Build Debian package # Depends on tar archive
+        working-directory: ./build-tools
         run: |
-          ./build_deb.sh ${{ env.PACKAGE_NAME }} ${{ env.VERSION }}
-        working-directory: ./build_tools
+          ./build-deb.sh ${{ env.PACKAGE_NAME }} ${{ env.VERSION }}
 
-      - name: Store deb package artifact
+      - name: Store Debian package artifact
         uses: actions/upload-artifact@v2
         with:
           name: ${{ env.PACKAGE_NAME }}_${{ env.VERSION }}_amd64.deb
-          path: build_tools/output/${{ env.PACKAGE_NAME }}_${{ env.VERSION }}_amd64.deb
+          path: build-tools/output/${{ env.PACKAGE_NAME }}_${{ env.VERSION }}_amd64.deb
 
   build-node-image:
-    name: Build cheqd-node image
+    name: Build cheqd-node Docker image
     runs-on: ubuntu-20.04
     steps:
       - uses: actions/checkout@v2
         with:
           fetch-depth: 0 # FIXME: Starport requires full repository
 
       - name: Build
-        run: docker build -f docker/cheqd_node/Dockerfile --no-cache -t cheqd-node  --build-arg UID=$(id -u) --build-arg GID=$(id -g) . # TODO: Get rid of UID and GID
+        run: docker build -f docker/Dockerfile --no-cache -t cheqd-node
 
       - name: Save
         run: docker save -o cheqd-node-image.tar cheqd-node
@@ -154,7 +154,7 @@ jobs:
         run: docker load -i cheqd-node-image.tar
 
       - name: Build
-        run: docker build -f docker/single_image_testnet/Dockerfile --no-cache -t cheqd-testnet .
+        run: docker build -f tests/networks/docker-localnet/Dockerfile --no-cache -t cheqd-testnet .
 
       - name: Save
         run: docker save -o cheqd-testnet-image.tar cheqd-testnet

.github/workflows/test.yml

@@ -9,7 +9,7 @@ on:
     - '.gitbook/**'
 
 env:
-  NODE_CONFIGS_BASE: "/home/runner/work/cheqd-node/cheqd-node/tests/networks/docker_compose/node_configs"
+  NODE_CONFIGS_BASE: "/home/runner/work/cheqd-node/cheqd-node/tests/networks/docker-compose-localnet/network-config"
   PACKAGE_NAME: "cheqd-node"
   VERSION: "1.0.0" # Package build scripts require version to be provided
 
@@ -93,25 +93,25 @@ jobs:
         run: sudo chmod +x /home/runner/go/bin/cheqd-noded
 
       - name: Make intermediate archive
-        working-directory: ./build_tools
+        working-directory: ./build-tools
         run: |
-          ./build_tar.sh ${{ env.PACKAGE_NAME }} ${{ env.VERSION }} 
+          ./build-tar.sh ${{ env.PACKAGE_NAME }} ${{ env.VERSION }} 
 
       - name: Install fpm
         run: |
           sudo apt-get install ruby ruby-dev rubygems build-essential
           sudo gem install --no-document fpm
 
       - name: Build deb # Depends on tar archive
+        working-directory: ./build-tools
         run: |
-          ./build_deb.sh ${{ env.PACKAGE_NAME }} ${{ env.VERSION }}
-        working-directory: ./build_tools
+          ./build-deb.sh ${{ env.PACKAGE_NAME }} ${{ env.VERSION }}
 
       - name: Store deb package artifact
         uses: actions/upload-artifact@v2
         with:
           name: ${{ env.PACKAGE_NAME }}_${{ env.VERSION }}_amd64.deb
-          path: build_tools/output/${{ env.PACKAGE_NAME }}_${{ env.VERSION }}_amd64.deb
+          path: build-tools/output/${{ env.PACKAGE_NAME }}_${{ env.VERSION }}_amd64.deb
 
   build-node-image:
     name: Build cheqd-node image
@@ -122,7 +122,7 @@ jobs:
           fetch-depth: 0 # FIXME: Starport requires full repository
 
       - name: Build
-        run: docker build -f docker/cheqd_node/Dockerfile --no-cache -t cheqd-node  --build-arg UID=$(id -u) --build-arg GID=$(id -g) . # TODO: Get rid of UID and GID
+        run: docker build -f docker/Dockerfile --no-cache -t cheqd-node  --build-arg UID=$(id -u) --build-arg GID=$(id -g) . # TODO: Get rid of UID and GID
 
       - name: Save
         run: docker save -o cheqd-node-image.tar cheqd-node
@@ -150,7 +150,7 @@ jobs:
         run: docker load -i cheqd-node-image.tar
 
       - name: Build
-        run: docker build -f docker/single_image_testnet/Dockerfile --no-cache -t cheqd-testnet .
+        run: docker build -f tests/networks/docker-localnet/Dockerfile --no-cache -t cheqd-testnet .
 
       - name: Save
         run: docker save -o cheqd-testnet-image.tar cheqd-testnet
@@ -186,23 +186,25 @@ jobs:
         uses: actions/checkout@v2
 
       - name: Set up 4 validators + 2 observers node docker pool  # TODO: Replace with testnet image
-        working-directory: ./tests/networks/docker_compose
+        working-directory: ./tests/networks/docker-compose-localnet
         run: |
-          ./gen_node_configs.sh
-          ./run_docker.sh
-          ./wait.sh '[[ $(curl -s -N localhost:26657/block | jq -cr '"'"'.result.block.last_commit.height'"'"') -gt 1 ]] && echo "Height is more than 1"'
+          bash gen-network-config.sh
+          docker-compose up -d
+
+      - name: Wait for chain
+        run: bash tests/networks/tools/wait-for-chain.sh
 
       - name: Add observer node
-        working-directory: ./tests/networks/docker_compose
+        working-directory: ./tests/networks/docker-compose-localnet
         run: |
-          ./add_observer.sh
+          ./add-observer.sh
           cheqd-noded status -n tcp://localhost:26677 2>&1
           ./wait.sh '[[ $(cheqd-noded status -n '"'"'tcp://localhost:26677'"'"' 2>&1 | wc -l) == 1 ]] && echo "New node returns status!"'
 
       - name: Promote observer to validator
-        working-directory: ./tests/networks/docker_compose
+        working-directory: ./tests/networks/docker-compose-localnet
         run: |
-          bash -x promote_validator.sh
+          bash -x promote-validator.sh
           cheqd-noded query staking validators --node "http://localhost:26657" | sed -nr 's/.*status: (.*?).*/\1/p' $1 | while read x; do [[ "BOND_STATUS_BONDED" == $x ]] && echo "Validator's status is bonded!" || exit 1 ; done
           ./wait.sh '[[ $(curl -s localhost:26657/block | sed -nr '"'"'s/.*signature": (.*?).*/\1/p'"'"' | wc -l) == 5 ]] && echo "There are 5 validators signatures in block!"'
           ./wait.sh '[[ $(curl -s localhost:26657/block | sed -nr '"'"'s/.*(signature": null).*/\1/p'"'"' | wc -l) == 0 ]] && echo "There are no null signatures in block!"'
@@ -242,24 +244,28 @@ jobs:
       - uses: actions/checkout@v2
 
       - name: Set up 4 validators + 2 observers node docker pool for local test run # Replace with testnet image
-        working-directory: ./tests/networks/docker_compose
+        working-directory: ./tests/networks/docker-compose-localnet
         run: |
-          ./gen_node_configs.sh
-          ./run_docker.sh
-          ./wait.sh '[[ $(curl -s -N localhost:26657/block | jq -cr '"'"'.result.block.last_commit.height'"'"') -gt 1 ]] && echo "Height is more than 1"'
+          bash gen-network-config.sh
+          docker-compose up -d
+
+      - name: Wait for chain
+        run: bash tests/networks/tools/wait-for-chain.sh
 
       - name: Set up test environment
         working-directory: ./tests/e2e
         run: |
           pip3 install -r requirements.txt >> /dev/null
           pip3 install --force-reinstall 'git+https://gitlab.com/evernym/verity/[email protected]#egg=vdr-tools&subdirectory=wrappers/python' -U
-          cp -R ${NODE_CONFIGS_BASE}/client/.cheqdnode /home/runner/
+          cp -Rf ${NODE_CONFIGS_BASE}/validator-0/. /home/runner/.cheqdnode
+          cp -Rf ${NODE_CONFIGS_BASE}/validator-1/. /home/runner/.cheqdnode
           sudo chmod -R 777 /home/runner
 
       - name: Run tests
         run: |
-          export OP0_ADDRESS=$(cheqd-noded keys list --keyring-backend "test" | sed -nr 's/.*address: (.*?).*/\1/p' | sed -n 1p | sed 's/\r//g')
-          export OP1_ADDRESS=$(cheqd-noded keys list --keyring-backend "test" | sed -nr 's/.*address: (.*?).*/\1/p' | sed -n 2p | sed 's/\r//g')
+          set -euxo pipefail
+          export OP0_ADDRESS=$(cheqd-noded keys list --keyring-backend "test" --home "${NODE_CONFIGS_BASE}/validator-0" | sed -nr 's/.*address: (.*?).*/\1/p' | sed 's/\r//g')
+          export OP1_ADDRESS=$(cheqd-noded keys list --keyring-backend "test" --home "${NODE_CONFIGS_BASE}/validator-1" | sed -nr 's/.*address: (.*?).*/\1/p' | sed 's/\r//g')
           pytest -v -rP tests/e2e/test_cosmos.py
 
   run-identity-tests-using-vdr:
@@ -280,7 +286,7 @@ jobs:
       - name: Run test net container
         run: |
           docker run -d --rm -p "26657:26657" cheqd-testnet
-          bash tests/networks/wait_for_chain.sh
+          bash tests/networks/tools/wait-for-chain.sh
 
       - name: Set up test environment
         working-directory: ./tests/e2e
@@ -321,8 +327,8 @@ jobs:
 
       - name: Run test net container
         run: |
-          docker run -d --rm -p "26657:26657"  -p "1317:1317" cheqd-testnet
-          bash tests/networks/wait_for_chain.sh
+          docker run -d --rm -p "26657:26657" -p "1317:1317" cheqd-testnet
+          bash tests/networks/tools/wait-for-chain.sh
 
       - name: Set up and run tests
         working-directory: tests/cli

.gitignore

@@ -1,3 +1,9 @@
+### APP-SPECIFIC EXCLUSIONS ###
+**/network-config
+.idea
+
+### GENERAL EXCLUSIONS ###
+
 ### macOS ###
 # General
 .DS_Store
@@ -30,14 +36,6 @@ Temporary Items
 .devcontainer
 .vscode
 
-# Application specific
-vue/node_modules
-vue/src/store/generated
-vue/dist
-secret.yml
-**/node_configs
-.idea
-
 # Python
 .pytest_cache
 **/__pycache__

SUMMARY.md

@@ -21,9 +21,9 @@
   * [Using cheqd Cosmos CLI for token transactions](docs/cheqd-cli/cheqd-cli-token-transactions.md)
 * [Building from source](docs/build-and-networks/README.md)
   * [Building and testing](docs/build-and-networks/README.md)
-  * [cheqd-testsnet docker image](docs/build-and-networks/local-docker-network.md)
-  * [Docker Compose Based Localnet](docs/build-and-networks/docker-compose.md)
-  * [Setting up a new network](docs/build-and-networks/build-and-networks.md)
+  * [Docker Based Localnet](docs/build-and-networks/local-docker-network.md)
+  * [Docker Compose Based Localnet](docs/build-and-networks/local-docker-compose-network.md)
+  * [Setting up a new network](docs/build-and-networks/manual-network-setup.md)
 * [Client-app Identity APIs](docs/identity-api/README.md)
   * [Error messages](docs/identity-api/identity-api-error-messages.md)
 

build_tools/postinst → build-tools/postinst

@@ -1,4 +1,4 @@
-#!/bin/sh
+#!/bin/bash
 
 CHEQD_USER_NAME=cheqd