Merge branch 'bugfix/GH-2908-move-logic-from-orchestrate-to-upgrade-s…

…cript' into q/2.7
scality · Nov 17, 2020 · d9eb77f · d9eb77f
2 parents a7046af + fa5c8e1
commit d9eb77f
Show file tree

Hide file tree

Showing 3 changed files with 44 additions and 36 deletions.
diff --git a/salt/metalk8s/orchestrate/apiserver.sls b/salt/metalk8s/orchestrate/apiserver.sls
@@ -1,4 +1,4 @@
-{%- set dest_version = pillar.orchestrate.dest_version %}
+{%- set dest_version = pillar.metalk8s.cluster_version %}
 {%- set master_nodes = salt.metalk8s.minions_by_role('master') %}
 
 {%- for node in master_nodes | sort %}

diff --git a/salt/metalk8s/orchestrate/upgrade/init.sls b/salt/metalk8s/orchestrate/upgrade/init.sls
@@ -1,3 +1,8 @@
+# NOTE: This orchestrate does not follow the Kubernetes upgrade process, and
+#       instead upgrades nodes fully (highstate), one by one.
+#       This orchestrate should only be called after several other upgrade
+#       steps, refer to the upgrade script.
+
 {%- set dest_version = pillar.metalk8s.cluster_version %}
 
 Execute the upgrade prechecks:
@@ -6,33 +11,6 @@ Execute the upgrade prechecks:
     - mods:
       - metalk8s.orchestrate.upgrade.precheck
     - saltenv: {{ saltenv }}
-    - pillar:
-        orchestrate:
-          dest_version: {{ dest_version }}
-
-Upgrade etcd cluster:
-  salt.runner:
-    - name: state.orchestrate
-    - mods:
-      - metalk8s.orchestrate.etcd
-    - saltenv: {{ saltenv }}
-    - pillar:
-        orchestrate:
-          dest_version: {{ dest_version }}
-    - require:
-      - salt: Execute the upgrade prechecks
-
-Upgrade apiserver instances:
-  salt.runner:
-    - name: state.orchestrate
-    - mods:
-      - metalk8s.orchestrate.apiserver
-    - saltenv: {{ saltenv }}
-    - pillar:
-        orchestrate:
-          dest_version: {{ dest_version }}
-    - require:
-      - salt: Upgrade etcd cluster
 
 {%- set cp_nodes = salt.metalk8s.minions_by_role('master') | sort %}
 {%- set other_nodes = pillar.metalk8s.nodes.keys() | difference(cp_nodes) | sort %}
@@ -66,7 +44,7 @@ Check pillar on {{ node }} before installing apiserver-proxy:
     - retry:
         attempts: 5
     - require:
-      - salt: Upgrade apiserver instances
+      - salt: Execute the upgrade prechecks
 
 Install apiserver-proxy on {{ node }}:
   salt.state:
@@ -85,7 +63,6 @@ Wait for API server to be available on {{ node }}:
   - verify_ssl: false
   - require:
     - salt: Install apiserver-proxy on {{ node }}
-    - salt: Upgrade etcd cluster
   {%- if previous_node is defined %}
     - salt: Deploy node {{ previous_node }}
   {%- endif %}
@@ -118,7 +95,7 @@ Deploy node {{ node }}:
     - require:
       - metalk8s_kubernetes: Set node {{ node }} version to {{ dest_version }}
     - require_in:
-      - salt: Deploy Kubernetes objects
+      - salt: Deploy Kubernetes service config objects
 
     {#- Ugly but needed since we have jinja2.7 (`loop.previtem` added in 2.10) #}
     {%- set previous_node = node %}
@@ -149,5 +126,4 @@ Deploy Kubernetes objects:
     - saltenv: metalk8s-{{ dest_version }}
     - require:
       - salt: Sync module on salt-master
-      - salt: Upgrade etcd cluster
       - salt: Deploy Kubernetes service config objects
diff --git a/scripts/upgrade.sh.in b/scripts/upgrade.sh.in
@@ -82,8 +82,7 @@ upgrade_bootstrap () {
         saltenv="$SALTENV"
 
     "${SALT_CALL}" --local --retcode-passthrough state.sls sync_mods="all" \
-        "['metalk8s.roles.bootstrap.components', 'metalk8s.container-engine']" \
-        saltenv="$SALTENV" \
+        metalk8s.roles.bootstrap.components saltenv="$SALTENV" \
         pillar="{'metalk8s': {'endpoints': {'salt-master': $saltmaster_endpoint, \
         'repositories': $repo_endpoint}}}"
 }
@@ -105,7 +104,7 @@ launch_post_upgrade () {
         saltenv="$SALTENV"
 }
 
-launch_upgrade () {
+upgrade_etcd () {
     SALT_MASTER_CALL=(crictl exec -i "$(get_salt_container)")
     "${SALT_MASTER_CALL[@]}" salt-run saltutil.sync_all \
         saltenv="$SALTENV"
@@ -116,6 +115,36 @@ launch_upgrade () {
     "${SALT_MASTER_CALL[@]}" salt-run saltutil.sync_roster  \
         saltenv="$SALTENV"
 
+    "${SALT_MASTER_CALL[@]}" salt-run state.orchestrate \
+        metalk8s.orchestrate.etcd saltenv="$SALTENV"
+}
+
+upgrade_apiservers () {
+    SALT_MASTER_CALL=(crictl exec -i "$(get_salt_container)")
+    "${SALT_MASTER_CALL[@]}" salt-run state.orchestrate \
+        metalk8s.orchestrate.apiserver saltenv="$SALTENV"
+}
+
+# NOTE: We need to upgrade local engine (kubelet + containerd) locally
+#       before starting the node upgrade as we rely on salt-master running
+#       in a container managed by kubelet and containerd
+upgrade_local_engines () {
+    local saltmaster_endpoint repo_endpoint
+    saltmaster_endpoint="$($SALT_CALL pillar.get \
+        metalk8s:endpoints:salt-master --out txt | cut -d' ' -f2- )"
+    repo_endpoint="$($SALT_CALL pillar.get \
+        metalk8s:endpoints:repositories --out txt | cut -d' ' -f2- )"
+
+    # NOTE: Sleep a bit at the end so that salt-master container properly stop
+    #       before going to the next step
+    "${SALT_CALL}" --local --retcode-passthrough state.sls sync_mods="all" \
+        metalk8s.kubernetes.kubelet.standalone saltenv="$SALTENV" \
+        pillar="{'metalk8s': {'endpoints': {'salt-master': $saltmaster_endpoint, \
+        'repositories': $repo_endpoint}}}" && sleep 20
+}
+
+upgrade_nodes () {
+    SALT_MASTER_CALL=(crictl exec -i "$(get_salt_container)")
     "${SALT_MASTER_CALL[@]}" salt-run state.orchestrate \
         metalk8s.orchestrate.upgrade saltenv="$SALTENV"
 }
@@ -152,7 +181,10 @@ run "Performing Pre-Upgrade checks" precheck_upgrade
 run "Upgrading bootstrap" upgrade_bootstrap
 run "Setting cluster version to $DESTINATION_VERSION" patch_kubesystem_namespace
 run "Launching the pre-upgrade" launch_pre_upgrade
-run "Launching the upgrade" launch_upgrade
+run "Upgrading etcd cluster" upgrade_etcd
+run "Upgrading all kube-api-server instances" upgrade_apiservers
+run "Upgrading local containerd and kubelet" upgrade_local_engines
+run "Upgrading all nodes one by one" upgrade_nodes
 run "Launching the post-upgrade" launch_post_upgrade
 
 "$BASE_DIR"/backup.sh