Rolling upgrade test automation

multinode-demo/validator.sh

@@ -268,6 +268,10 @@ wallet() {
 setup_validator_accounts() {
   declare node_sol=$1
 
+  if [[ -n "$SKIP_ACCOUNTS_CREATION" ]]; then
+    return 0
+  fi
+
   if ! wallet vote-account "$vote_account"; then
     if ((airdrops_enabled)); then
       echo "Adding $node_sol to validator identity account:"

net/net.sh

@@ -874,12 +874,6 @@ while [[ -n $1 ]]; do
       shift 1
     elif [[ $1 == --extra-primordial-stakes ]]; then
       extraPrimordialStakes=$2
-      # Extra primoridial stakes require that all of the validators start at
-      # the same time. Force async init and wait for supermajority here.
-      waitForNodeInit=false
-      if [[ -z "$maybeWaitForSupermajority" ]]; then
-        maybeWaitForSupermajority="--wait-for-supermajority 1"
-      fi
       shift 2
     else
       usage "Unknown long option: $1"
@@ -1009,6 +1003,21 @@ if [[ -n "$maybeWaitForSupermajority" && -n "$maybeWarpSlot" ]]; then
   fi
 fi
 
+if [[ -n "$extraPrimordialStakes" ]]; then
+  # Extra primoridial stakes require that all of the validators start at
+  # the same time. Force async init and wait for supermajority here.
+  waitForNodeInit=false
+  if [[ -z "$maybeWaitForSupermajority" ]]; then
+    waitSlot=
+    if [[ -n "$maybeWarpSlot" ]]; then
+      read -r _ waitSlot <<<"$maybeWarpSlot"
+    else
+      waitSlot=1
+    fi
+    maybeWaitForSupermajority="--wait-for-supermajority $waitSlot"
+  fi
+fi
+
 checkPremptibleInstances
 
 case $command in

net/remote/remote-node.sh

@@ -385,7 +385,14 @@ EOF
     args+=(--init-complete-file "$initCompleteFile")
     # shellcheck disable=SC2206 # Don't want to double quote $extraNodeArgs
     args+=($extraNodeArgs)
+
+    maybeSkipAccountsCreation=
+    if [[ $nodeIndex -le $extraPrimordialStakes ]]; then
+      maybeSkipAccountsCreation="export SKIP_ACCOUNTS_CREATION=1"
+    fi
+
 cat >> ~/solana/on-reboot <<EOF
+    $maybeSkipAccountsCreation
     nohup multinode-demo/validator.sh ${args[@]} > validator.log.\$now 2>&1 &
     pid=\$!
     oom_score_adj "\$pid" 1000
@@ -397,7 +404,7 @@ EOF
       net/remote/remote-node-wait-init.sh 600
     fi
 
-    if [[ $skipSetup != true && $nodeType != blockstreamer ]]; then
+    if [[ $skipSetup != true && $nodeType != blockstreamer && -z $maybeSkipAccountsCreation ]]; then
       # Wait for the validator to catch up to the bootstrap validator before
       # delegating stake to it
       solana --url http://"$entrypointIp":8899 catchup config/validator-identity.json

scripts/wallet-sanity.sh

@@ -20,8 +20,11 @@ args+=(--keypair "$SOLANA_CONFIG_DIR"/faucet.json)
 node_readiness=false
 timeout=60
 while [[ $timeout -gt 0 ]]; do
-  output=$($solana_cli "${args[@]}" transaction-count)
-  if [[ -n $output ]]; then
+  set +e
+  output=$($solana_cli "${args[@]}" transaction-count --commitment max)
+  rc=$?
+  set -e
+  if [[ $rc -eq 0 && -n $output ]]; then
     node_readiness=true
     break
   fi

system-test/rolling-upgrade/rolling_upgrade.sh

@@ -0,0 +1,62 @@
+#!/usr/bin/env bash
+
+DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
+REPO_ROOT=${DIR}/../..
+
+# shellcheck source=/dev/null # Ignore generated source target
+source "${REPO_ROOT}/net/config/config"
+# shellcheck source=system-test/automation_utils.sh
+source "${REPO_ROOT}/system-test/automation_utils.sh"
+
+NET_SH="${REPO_ROOT}/net/net.sh"
+
+set -x
+
+: "${UPGRADE_INITIAL_DELAY:=0}" # Time to wait before starting rolling upgrade
+: "${UPGRADE_INTERVALIDATOR_DELAY:?}" # Time to wait between validators during upgrade
+: "${UPGRADE_POST_TEST_DELAY:=0}" #Time to wait after upgrade
+
+sleep_if_positive() {
+  declare delay=$1
+  if [[ "$delay" -gt 0 ]]; then
+    sleep "$delay"
+  fi
+}
+
+# Fetch new software and upload it to the bootstrap validator
+declare -g sw_version_args
+get_net_launch_software_version_launch_args "$UPGRADE_CHANNEL" "upgrade-release" sw_version_args
+# shellcheck disable=2086 # $sw_version_args holds two args. Don't quote!
+"$NET_SH" upgrade $sw_version_args
+
+# Wait initial delay, if any
+sleep_if_positive "$UPGRADE_INITIAL_DELAY"
+
+if [[ "$UPGRADE_INITIAL_DELAY" -gt 0 ]]; then
+  "$NET_SH" sanity
+fi
+
+# Restart validators one by one
+# shellcheck disable=SC2154 # sourced from config above
+for i in "${!validatorIpList[@]}"; do
+  if [[ "$i" -eq 0 ]]; then
+    # net.sh doesn't support restarting the bootstrap validator yet
+    continue
+  fi
+
+  declare ipAddress="${validatorIpList[$i]}"
+
+  "$NET_SH" stopnode -i "$ipAddress"
+  "$NET_SH" startnode -r -i "$ipAddress"
+
+  # This could be replaced with something based on `solana catchup`
+  sleep_if_positive "$UPGRADE_INTERVALIDATOR_DELAY"
+
+  "$NET_SH" sanity
+done
+
+sleep_if_positive "$UPGRADE_POST_TEST_DELAY"
+
+if [[ "$UPGRADE_POST_TEST_DELAY" -gt 0 ]]; then
+  "$NET_SH" sanity
+fi

system-test/testnet-automation.sh

@@ -134,11 +134,16 @@ function launch_testnet() {
     maybeAsyncNodeInit="--async-node-init"
   fi
 
+  declare maybeExtraPrimordialStakes
+  if [[ -n "$EXTRA_PRIMORDIAL_STAKES" ]]; then
+    maybeExtraPrimordialStakes="--extra-primordial-stakes $EXTRA_PRIMORDIAL_STAKES"
+  fi
+
   # shellcheck disable=SC2068
   # shellcheck disable=SC2086
   "${REPO_ROOT}"/net/net.sh start $version_args \
     -c idle=$NUMBER_OF_CLIENT_NODES $maybeStartAllowBootFailures \
-    --gpu-mode $startGpuMode $maybeWarpSlot $maybeAsyncNodeInit
+    --gpu-mode $startGpuMode $maybeWarpSlot $maybeAsyncNodeInit $maybeExtraPrimordialStakes
 
   execution_step "Waiting for bootstrap validator's stake to fall below ${BOOTSTRAP_VALIDATOR_MAX_STAKE_THRESHOLD}%"
   wait_for_bootstrap_validator_stake_drop "$BOOTSTRAP_VALIDATOR_MAX_STAKE_THRESHOLD"