salt,script: Move upgrade logic from orchestrate to script

Since we use a script and we rely on salt-master running in a static pod
on the bootstrap node we need to move some logic outside of the salt
orchestrate to the script so that salt-master restart can be handled
properly (and not brutaly interupt a salt orchestrate execution.
- Etcd cluster upgrade is now part of the upgrade script
- All APIServers upgrade is now part of the uppgrade script
- Upgrade bootstrap engines (kubelet + containerd) locally
- Then call the orchestrate to upgrade all nodes one by one

Fixes: #2908

salt/metalk8s/orchestrate/upgrade/init.sls

@@ -10,30 +10,6 @@ Execute the upgrade prechecks:
         orchestrate:
           dest_version: {{ dest_version }}
 
-Upgrade etcd cluster:
-  salt.runner:
-    - name: state.orchestrate
-    - mods:
-      - metalk8s.orchestrate.etcd
-    - saltenv: {{ saltenv }}
-    - pillar:
-        orchestrate:
-          dest_version: {{ dest_version }}
-    - require:
-      - salt: Execute the upgrade prechecks
-
-Upgrade apiserver instances:
-  salt.runner:
-    - name: state.orchestrate
-    - mods:
-      - metalk8s.orchestrate.apiserver
-    - saltenv: {{ saltenv }}
-    - pillar:
-        orchestrate:
-          dest_version: {{ dest_version }}
-    - require:
-      - salt: Upgrade etcd cluster
-
 {%- set cp_nodes = salt.metalk8s.minions_by_role('master') | sort %}
 {%- set other_nodes = pillar.metalk8s.nodes.keys() | difference(cp_nodes) | sort %}
 
@@ -66,7 +42,7 @@ Check pillar on {{ node }} before installing apiserver-proxy:
     - retry:
         attempts: 5
     - require:
-      - salt: Upgrade apiserver instances
+      - salt: Execute the upgrade prechecks
 
 Install apiserver-proxy on {{ node }}:
   salt.state:
@@ -85,7 +61,6 @@ Wait for API server to be available on {{ node }}:
   - verify_ssl: false
   - require:
     - salt: Install apiserver-proxy on {{ node }}
-    - salt: Upgrade etcd cluster
   {%- if previous_node is defined %}
     - salt: Deploy node {{ previous_node }}
   {%- endif %}
@@ -149,5 +124,4 @@ Deploy Kubernetes objects:
     - saltenv: metalk8s-{{ dest_version }}
     - require:
       - salt: Sync module on salt-master
-      - salt: Upgrade etcd cluster
       - salt: Deploy Kubernetes service config objects

scripts/upgrade.sh.in

@@ -82,8 +82,7 @@ upgrade_bootstrap () {
         saltenv="$SALTENV"
 
     "${SALT_CALL}" --local --retcode-passthrough state.sls sync_mods="all" \
-        "['metalk8s.roles.bootstrap.components', 'metalk8s.container-engine']" \
-        saltenv="$SALTENV" \
+        metalk8s.roles.bootstrap.components saltenv="$SALTENV" \
         pillar="{'metalk8s': {'endpoints': {'salt-master': $saltmaster_endpoint, \
         'repositories': $repo_endpoint}}}"
 }
@@ -105,7 +104,7 @@ launch_post_upgrade () {
         saltenv="$SALTENV"
 }
 
-launch_upgrade () {
+upgrade_etcd () {
     SALT_MASTER_CALL=(crictl exec -i "$(get_salt_container)")
     "${SALT_MASTER_CALL[@]}" salt-run saltutil.sync_all \
         saltenv="$SALTENV"
@@ -116,6 +115,34 @@ launch_upgrade () {
     "${SALT_MASTER_CALL[@]}" salt-run saltutil.sync_roster  \
         saltenv="$SALTENV"
 
+    "${SALT_MASTER_CALL[@]}" salt-run state.orchestrate \
+        metalk8s.orchestrate.etcd saltenv="$SALTENV"
+}
+
+upgrade_apiservers () {
+    SALT_MASTER_CALL=(crictl exec -i "$(get_salt_container)")
+    "${SALT_MASTER_CALL[@]}" salt-run state.orchestrate \
+        metalk8s.orchestrate.apiserver saltenv="$SALTENV"
+}
+
+# NOTE: We need to upgrade local engine (kubelet + containerd) locally
+#       before starting the node upgrade as we rely on salt-master running
+#       in a container managed by kubelet and containerd
+upgrade_local_engines () {
+    local saltmaster_endpoint repo_endpoint
+    saltmaster_endpoint="$($SALT_CALL pillar.get \
+        metalk8s:endpoints:salt-master --out txt | cut -d' ' -f2- )"
+    repo_endpoint="$($SALT_CALL pillar.get \
+        metalk8s:endpoints:repositories --out txt | cut -d' ' -f2- )"
+
+    "${SALT_CALL}" --local --retcode-passthrough state.sls sync_mods="all" \
+        metalk8s.kubernetes.kubelet.standalone saltenv="$SALTENV" \
+        pillar="{'metalk8s': {'endpoints': {'salt-master': $saltmaster_endpoint, \
+        'repositories': $repo_endpoint}}}"
+}
+
+upgrade_nodes () {
+    SALT_MASTER_CALL=(crictl exec -i "$(get_salt_container)")
     "${SALT_MASTER_CALL[@]}" salt-run state.orchestrate \
         metalk8s.orchestrate.upgrade saltenv="$SALTENV"
 }
@@ -152,7 +179,10 @@ run "Performing Pre-Upgrade checks" precheck_upgrade
 run "Upgrading bootstrap" upgrade_bootstrap
 run "Setting cluster version to $DESTINATION_VERSION" patch_kubesystem_namespace
 run "Launching the pre-upgrade" launch_pre_upgrade
-run "Launching the upgrade" launch_upgrade
+run "Upgrading ETCD cluster" upgrade_etcd
+run "Upgrading all APIServers" upgrade_apiservers
+run "Upgrading bootstrap engines" upgrade_local_engines
+run "Upgrading all nodes one by one" upgrade_nodes
 run "Launching the post-upgrade" launch_post_upgrade
 
 "$BASE_DIR"/backup.sh