ci: Execute interactive installer test [DEV-1944] (#460)

* remove configureCmd

* remove post_install method from installer.py

* remove add-observer.sh from upgrade/cosmovisor

* remove add-observer.sh from interactive-install

* remove post_install method usage

* keep post_install method empty

* Update config.toml and app.toml defaults

* Bump alpine image version to latest

* Update default persistent flags

* Set --gas flag correctly

* Delete legacy interactive test folder

* Add installer test to workflow

* Update package-lock.json

* Fix workflow syntax error in interactive-installer job

* add questionary to bump cosmovisor

* move bump logic to setup_cosmovisor

* Check for success

* Change existing install check

* Update existing installation check

* Update installer check

* Fix linter issue

* Update test.yml

* Fix linter error

* Update test.yml

* Debugging installer flow

* Update build.yml

* Set up tmate on failure

* Try Andrew's technique

* Change home path

* Fix home path

* Update installer.py

* auto detect os arch for cosmovisor binary

* set default bump cosmovisor to yes

* set correct OS arch

* keep first install steps only in setup_cosmovisor

* set cosmovisor ENV vars

* remove sudo from export

* write to .profile using python

* Move filesx

* fix path to .profile

* log cosmovisor version when upgrade

* update .profile for current session

* exec source ~/.profile without stdout

* add execute permission

* Update installer.py

* add try/except

* get version anyway

* suppress err in bash

* add new line

* Delete outdated Cosmovisor check

* Create answers.txt

* Disable tmate

* use v1.2.0 instead

* try appending to .bashrc

* try with os.environ

* compare version and ask for bump

* rename function

* suppress error

* wrap std output in triple quotes

* add logs

* fix getting current version

* handle edge case

* add more info

* fix typo

* remove logs

* semantics

* add bump logic

* avoid fresh install during upgrade

* Debug only

* Update build.yml

* move new binary to installation directory

* try cosmovisor init

* extract to methods

* set ENV vars right after cosmovisor install

* edit upgrade steps

* add logs

* keep only interview logic at ask

* add fixes

* move info log

* minor fix

* stop and reload cosmovisor systemd during bump

* fix fresh install

* add logs

* more logs

* more log

* logs

* more logs

* add try except

* logs

* fix

* logs

* logs

* logs

* logs

* logs

* fix

* fix

* fix env var check

* remove unused function

* move getting cosmovisor download url to its own function

* check if .bashrc is writable or not

* fix syntax issues

* change ~ ownership to cheqd

* fix quotes syntax

* temporarily disable editing current user .bashrc

* remove is_file_writable

* write to file and read version for it

* remove using os

* add logs

* use tee

* remove debug logs

* write to bashrc python way

* add fix

* update installer.py

* Check default shell

* extract download and unzip to its own method

* reuse download_and_unzip

* add logs

* revert changes

* fix wrap in directory issue for versions lower than 1.0.1

* Update build.yml

* First we should check all existing different OS release,
then we should report about failure

* Add cosmovisor path to default PATH

* Remove previous commit code

* Reverse code

* Change runner from ubuntu-latest to ubuntu-20.04

* Fix linter mistakes

* Reset to version in develop

* Reset more correctly to develop branch

* Fix merge errors

* Delete Dockerfile approach

* Move installer test to test pipeline

* Rename to install from scratch

* Create upgrade-existing.txt

* Add upgrade path test

Co-authored-by: Benyam Seifu <[email protected]>
Co-authored-by: Alexander Kolesov <[email protected]>
Co-authored-by: Benyam Seifu <[email protected]>
Co-authored-by: Filip Djokic <[email protected]>
Co-authored-by: abdulla-ashurov <[email protected]>
Co-authored-by: abdulla-ashurov <[email protected]>

.github/workflows/test.yml

@@ -13,6 +13,29 @@ permissions:
 
 jobs:
 
+  interactive-installer:
+    name: "Interactive Installer"
+    runs-on: ubuntu-20.04
+
+    steps:
+      - uses: actions/checkout@v3
+
+      - name: "Fresh install"
+        working-directory: ./installer
+        run: |
+          sudo bash -c "python3 installer.py < install-from-scratch.txt"
+
+      - name: "Upgrade existing installation"
+        working-directory: ./installer
+        run: |
+          sudo bash -c "python3 installer.py < upgrade-existing.txt"
+
+      - name: Debugging with tmate
+        if: ${{ failure() }}
+        uses: mxschmitt/[email protected]
+        with:
+          timeout-minutes: 15
+
   unit-tests:
     name: "Unit Tests"
     runs-on: ubuntu-20.04

installer/install-from-scratch.txt

@@ -0,0 +1,14 @@
+2
+/home/runner/.local/cheqd
+yes
+mainnet
+yes
+no
+test-install
+1.1.1.1
+27657
+27656
+100ncheq
+[email protected]:26656,[email protected]:26656
+debug
+plain

installer/installer.py

@@ -19,6 +19,9 @@
 import signal
 import platform
 import copy
+import re
+import shutil
+
 
 ###############################################################
 ###     				Installer defaults    				###
@@ -77,8 +80,9 @@
 DEFAULT_LOG_LEVEL = "error"
 DEFAULT_LOG_FORMAT = "json"
 
-def sigint_handler(signal, frame):
-    print ('Exiting from cheqd-node installer')
+
+def sigint_handler (signal, frame):
+    print('Exiting from cheqd-node installer')
     sys.exit(0)
 
 signal.signal(signal.SIGINT, sigint_handler)
@@ -92,8 +96,7 @@ def search_and_replace(search_text, replace_text, file_path):
                 data = file.read()
                 data = data.replace(line, replace_text)
             with open(file_path, 'w') as file:
-                file.write(data)    
-
+                file.write(data)
     file.close()
 
 class Release:
@@ -109,10 +112,11 @@ def get_release_url(self):
             os_name = platform.system()
             for _url_item in self.assets:
                 _url = _url_item["browser_download_url"]
-                version_without_v_prefix = self.version.replace('v','',1)
+                version_without_v_prefix = self.version.replace('v', '' ,1)
                 if os.path.basename(_url) == f"cheqd-noded-{version_without_v_prefix}-{os_name}-{os_arch}.tar.gz" or \
                     os.path.basename(_url) == "cheqd-noded":
-                    return _url          
+                    return _url      
+
             else:
                 failure_exit(f"No asset found to download for release: {self.version}")
         except:
@@ -182,7 +186,7 @@ def cosmovisor_service_cfg(self):
                 f.read()
             )
         self.remove_safe(fname)
-        return s 
+        return s
 
     @property
     def logrotate_cfg(self):
@@ -203,9 +207,9 @@ def rsyslog_cfg(self):
         fname = os.path.basename(RSYSLOG_TEMPLATE)
         self.exec(f"wget -c {RSYSLOG_TEMPLATE}")
         with open(fname) as f:
-            s =re.sub(
+            s = re.sub(
                 r'({BINARY_FOR_LOGGING}|{CHEQD_LOG_DIR})',
-                lambda m:{'{BINARY_FOR_LOGGING}': binary_name,
+                lambda m: {'{BINARY_FOR_LOGGING}': binary_name,
                         '{CHEQD_LOG_DIR}': self.cheqd_log_dir}[m.group()],
                 f.read()
             )
@@ -263,7 +267,10 @@ def get_binary(self):
         try:
             self.exec(f"wget -c {binary_url}")
             if fname.find(".tar.gz") != -1:
-                self.exec(f"tar -xzf {fname} -C . --strip-components=1")
+                if self.version.replace('v', '') >= '1.0.1':
+                    self.exec(f"tar -xzf {fname} -C .")
+                else:
+                    self.exec(f"tar -xzf {fname} -C . --strip-components=1")
                 self.remove_safe(fname)
             self.exec(f"chmod +x {DEFAULT_BINARY_NAME}")
         except:
@@ -277,15 +284,13 @@ def is_user_exists(self, username) -> bool:
         except KeyError:
             self.log(f"User {username} does not exist")
             return False
-
 
     def remove_safe(self, path, is_dir=False):
         if is_dir and os.path.exists(path):
             shutil.rmtree(path)
         if os.path.exists(path):
             os.remove(path)
 
-
     def pre_install(self):
         if self.interviewer.is_from_scratch:
             self.log("Removing user's data and configs")
@@ -432,23 +437,23 @@ def install(self):
     def post_install(self):
         # Init the node with provided moniker
         if not os.path.exists(os.path.join(self.cheqd_config_dir, 'genesis.json')):
-            self.exec(f"""sudo su -c 'cheqd-noded init "{self.interviewer.moniker}"' {DEFAULT_CHEQD_USER}""")
-    
+            self.exec(f"""sudo su -c 'cheqd-noded init {self.interviewer.moniker}' {DEFAULT_CHEQD_USER}""")
+
             # Downloading genesis file
             self.exec(f"curl {GENESIS_FILE.format(self.interviewer.chain)} > {os.path.join(self.cheqd_config_dir, 'genesis.json')}")
             shutil.chown(os.path.join(self.cheqd_config_dir, 'genesis.json'),
                 DEFAULT_CHEQD_USER,
                 DEFAULT_CHEQD_USER)
-                         
+
         # Replace the default RCP port to listen to anyone
         rpc_default_value= 'laddr = "tcp://127.0.0.1:{}"'.format(DEFAULT_RPC_PORT)
         new_rpc_default_value = 'laddr = "tcp://0.0.0.0:{}"'.format(DEFAULT_RPC_PORT)
-        search_and_replace(rpc_default_value,new_rpc_default_value, os.path.join(self.cheqd_config_dir, "config.toml"))
+        search_and_replace(rpc_default_value, new_rpc_default_value, os.path.join(self.cheqd_config_dir, "config.toml"))
 
         # Set create empty blocks to false by default
         create_empty_blocks_search_text = 'create_empty_blocks = true'
         create_empty_blocks_replace_text = 'create_empty_blocks = false'
-        search_and_replace(create_empty_blocks_search_text,create_empty_blocks_replace_text, os.path.join(self.cheqd_config_dir, "config.toml"))
+        search_and_replace(create_empty_blocks_search_text, create_empty_blocks_replace_text, os.path.join(self.cheqd_config_dir, "config.toml"))
 
         # Setting up the external_address
         if self.interviewer.external_address:
@@ -461,7 +466,7 @@ def post_install(self):
         seeds_search_text = 'seeds = ""'
         seeds_replace_text= 'seeds = "{}"'.format(seeds)
         search_and_replace(seeds_search_text, seeds_replace_text, os.path.join(self.cheqd_config_dir, "config.toml"))
-        
+
         # Setting up the RPC port
         if self.interviewer.rpc_port:
             rpc_laddr_search_text= 'laddr = "tcp://0.0.0.0:{}"'.format(DEFAULT_RPC_PORT)
@@ -471,20 +476,20 @@ def post_install(self):
         if self.interviewer.p2p_port:
             p2p_laddr_search_text='laddr = "tcp://0.0.0.0:{}"'.format(DEFAULT_P2P_PORT)
             p2p_laddr_replace_text='laddr = "tcp://0.0.0.0:{}"'.format(self.interviewer.p2p_port)
-            search_and_replace(p2p_laddr_search_text,p2p_laddr_replace_text, os.path.join(self.cheqd_config_dir, "config.toml"))
-        
+            search_and_replace(p2p_laddr_search_text, p2p_laddr_replace_text, os.path.join(self.cheqd_config_dir, "config.toml"))
+
         # Setting up min gas-price
         if self.interviewer.gas_price:
             min_gas_price_search_text='minimum-gas-prices = '
             min_gas_price_replace_text = 'minimum-gas-prices = "{}"'.format(self.interviewer.gas_price)
             search_and_replace(min_gas_price_search_text, min_gas_price_replace_text, os.path.join(self.cheqd_config_dir, "app.toml"))
-        
+
         # Setting up persistent peers
         if self.interviewer.persistent_peers:
             persistent_peers_search_text='persistent_peers = ""'
             persistent_peers_replace_text='persistent_peers = "{}"'.format(self.interviewer.persistent_peers)
-            search_and_replace(persistent_peers_search_text,persistent_peers_replace_text, os.path.join(self.cheqd_config_dir, "config.toml"))
-        
+            search_and_replace(persistent_peers_search_text, persistent_peers_replace_text, os.path.join(self.cheqd_config_dir, "config.toml"))
+
         # Setting up log level
         if self.interviewer.log_level:
             log_level_search_text = 'log_level'
@@ -542,10 +547,12 @@ def setup_cosmovisor(self):
             self.remove_safe("CHANGELOG.md")
             self.remove_safe("README.md")
             self.remove_safe("LICENSE")
+
             self.mkdir_p(self.cosmovisor_root_dir)
             self.mkdir_p(os.path.join(self.cosmovisor_root_dir, "genesis"))
             self.mkdir_p(os.path.join(self.cosmovisor_root_dir, "genesis/bin"))
             self.mkdir_p(os.path.join(self.cosmovisor_root_dir, "upgrades"))
+
             if not os.path.exists(os.path.join(DEFAULT_INSTALL_PATH, DEFAULT_COSMOVISOR_BINARY_NAME)):
                 self.log(f"Moving Cosmovisor binary to installation directory")
                 shutil.move("./cosmovisor", DEFAULT_INSTALL_PATH)
@@ -573,7 +580,7 @@ def setup_cosmovisor(self):
                             os.path.join(self.cosmovisor_root_dir, "current"))
                 self.log(f"Changing owner to {DEFAULT_CHEQD_USER} user")
                 self.exec(f"chown -R {DEFAULT_CHEQD_USER}:{DEFAULT_CHEQD_USER} {self.cosmovisor_root_dir}")
-        
+
             self.log(f"Changing directory ownership for Cosmovisor to {DEFAULT_CHEQD_USER} user")
             self.exec(f"chown -R {DEFAULT_CHEQD_USER}:{DEFAULT_CHEQD_USER} {self.cosmovisor_root_dir}")
         except:
@@ -623,12 +630,12 @@ def download_snapshot(self):
                     self.log(f"Snapshot download was successful but checksums do not match.")
                     failure_exit(f"Snapshot download was successful but checksums do not match.")
             elif int(archive_size) > int(free_disk_space):
-                failure_exit (f"Snapshot archive is too large to fit in free disk space. Please free up some space and try again.")
+                failure_exit(f"Snapshot archive is too large to fit in free disk space. Please free up some space and try again.")
             else:
-                failure_exit (f"Error encountered when downloading snapshot archive.")
+                failure_exit(f"Error encountered when downloading snapshot archive.")
         except:
             failure_exit(f"Failed to download snapshot")
-        
+
     def untar_from_snapshot(self):
         try:
             archive_path = os.path.join(self.cheqd_root_dir, os.path.basename(self.interviewer.snapshot_url))
@@ -638,7 +645,7 @@ def untar_from_snapshot(self):
 
             # Extract to cheqd node data directory EXCEPT for validator state
             self.exec(f"sudo su -c 'pv {archive_path} | tar --use-compress-program=lz4 -xf - -C {self.cheqd_root_dir} --exclude priv_validator_state.json' {DEFAULT_CHEQD_USER}")
-            
+
             # Delete snapshot archive file
             self.log(f"Snapshot extraction was successful. Deleting snapshot archive.")
             self.remove_safe(archive_path)
@@ -655,8 +662,8 @@ def untar_from_snapshot(self):
             failure_exit(f"Failed to extract snapshot")
 
     def print_success(self):
-        self.log("The cheqd-noded has been successfully installed")
-        
+        self.log("The cheqd-noded binary has been successfully installed")
+
 
 class Interviewer:
     def __init__(self,
@@ -688,7 +695,6 @@ def __init__(self,
     def cheqd_root_dir(self):
         return os.path.join(self.home_dir, ".cheqdnode")
 
-
     @property
     def cheqd_config_dir(self):
         return os.path.join(self.cheqd_root_dir, "config")
@@ -764,15 +770,15 @@ def p2p_port(self) -> str:
     @property
     def gas_price(self) -> str:
         return self._gas_price
-    
+
     @property
     def persistent_peers(self) -> str:
         return self._persistent_peers
-    
+
     @property
     def log_level(self) -> str:
         return self._log_level
-    
+
     @property
     def log_format(self) -> str:
         return self._log_format
@@ -844,11 +850,11 @@ def p2p_port(self, p2p_port):
     @gas_price.setter
     def gas_price(self, gas_price):
         self._gas_price = gas_price
-    
+
     @persistent_peers.setter
     def persistent_peers(self, persistent_peers):
         self._persistent_peers = persistent_peers
-    
+
     @log_level.setter
     def log_level(self, log_level):
         self._log_level = log_level
@@ -918,7 +924,7 @@ def ask_for_version(self):
         all_releases.insert(0, default)
         for i, release in enumerate(all_releases[0: LAST_N_RELEASES]):
             print(f"{i + 1}) {release.version}")
-        release_num = int(self.ask("Choose list option number above to select version of cheqd-node to install", 
+        release_num = int(self.ask("Choose list option number above to select version of cheqd-node to install",
             default=1))
         if release_num >= 1 and release_num <= len(all_releases):
             self.release = all_releases[release_num - 1]
@@ -1061,14 +1067,14 @@ def ask_for_persistent_peers(self):
             f"INFO: Persistent peers are nodes that you want to always keep connected to. "
             f"Values for persistent peers should be specified in format: <nodeID>@<IP>:<port>,<nodeID>@<IP>:<port>... "
             f"Specify persistent peers [default: none]: {os.linesep}")
-    
+
     def ask_for_log_level(self):
         self.log_level = self.ask(
-            f"Specify log level (error | info | debug)", default=DEFAULT_LOG_LEVEL)
-    
+            f"Specify log level (trace|debug|info|warn|error|fatal|panic)", default=DEFAULT_LOG_LEVEL)
+
     def ask_for_log_format(self):
         self.log_format = self.ask(
-            f"Specify log format (plain | json)", default=DEFAULT_LOG_FORMAT)
+            f"Specify log format (json|plain)", default=DEFAULT_LOG_FORMAT)
 
     def prepare_url_for_latest(self) -> str:
         template = TESTNET_SNAPSHOT if self.chain == "testnet" else MAINNET_SNAPSHOT
@@ -1094,8 +1100,9 @@ def is_url_exists(self, url):
         self.verbose = curr_verbose
         return False
 
+
 if __name__ == '__main__':
-    
+
     # Steps to execute if installing from scratch
     def install_steps():
         interviewer.ask_for_setup()
@@ -1121,15 +1128,15 @@ def upgrade_steps():
             interviewer.ask_for_rewrite_rsyslog()
         if os.path.exists(DEFAULT_LOGROTATE_FILE):
             interviewer.ask_for_rewrite_logrotate()
-    
+
     # Ask user for information
     interviewer = Interviewer()
     interviewer.ask_for_version()
     interviewer.ask_for_home_directory(default=DEFAULT_HOME)
 
     # Check if cheqd configuration directory exists
     is_installed = interviewer.is_already_installed()
-    
+
     # First-time new node setup
     if is_installed is False:
         install_steps()

installer/upgrade-existing.txt

@@ -0,0 +1,7 @@
+2
+/home/runner/.local/cheqd
+yes
+yes
+yes
+yes
+yes