salt: Handle calico-cni-plugin package removal for upgrade/downgrade

NOTE: This commit can be reverted in `development/124.0`

buildchain/buildchain/salt_tree.py

@@ -524,6 +524,7 @@ def task(self) -> types.TaskDict:
     Path("salt/metalk8s/kubernetes/ca/kubernetes/init.sls"),
     Path("salt/metalk8s/kubernetes/ca/kubernetes/installed.sls"),
     Path("salt/metalk8s/kubernetes/cni/calico/deployed.sls"),
+    Path("salt/metalk8s/kubernetes/cni/calico/post-upgrade.sls"),
     Path("salt/metalk8s/kubernetes/controller-manager/init.sls"),
     Path("salt/metalk8s/kubernetes/controller-manager/installed.sls"),
     Path("salt/metalk8s/kubernetes/controller-manager/kubeconfig.sls"),

salt/metalk8s/kubernetes/cni/calico/post-upgrade.sls

@@ -0,0 +1,21 @@
+# The calico-cni-plugin package get removed in 123.0
+# This state can be removed in `development/124.0`
+
+Ensure calico-cni-plugin is removed:
+  pkg.removed:
+    - name: calico-cni-plugin
+
+Restart calico-node:
+  module.run:
+    - cri.stop_pod:
+      - labels:
+          k8s-app: calico-node
+    - onchanges:
+      - pkg: Ensure calico-cni-plugin is removed
+
+Make sure calico-node is up and ready:
+  test.configurable_test_state:
+    - changes: False
+    - result: __slot__:salt:cri.wait_container(name="calico-node", state="running")
+    - require:
+      - module: Restart calico-node

salt/metalk8s/orchestrate/downgrade/pre.sls

@@ -2,5 +2,33 @@
 # NOTE: This state should be called by salt-master using the saltenv of
 # the current version (salt-master should not have been downgraded yet)
 
+{%- set dest_version = pillar.metalk8s.cluster_version %}
+
+{%- if salt.pkg.version_cmp(dest_version, '123.0.0') == -1 %}
+
+# The calico-cni-plugin package get removed in 123.0
+# in order to work properly we need to downgrade calico at the very begining
+# This logic can be removed in `development/124.0`
+
+Downgrade calico:
+  salt.runner:
+    - name: state.orchestrate
+    - mods:
+      - metalk8s.kubernetes.cni.calico.deployed
+    - saltenv: metalk8s-{{ dest_version }}
+
+Install calico-cni-plugin on every nodes:
+  salt.state:
+    - tgt: '*'
+    - sls:
+      - metalk8s.kubernetes.cni.calico
+    - saltenv: metalk8s-{{ dest_version }}
+    - require:
+      - salt: Downgrade calico
+
+{%- else %}
+
 Nothing to do before downgrading:
   test.nop: []
+
+{%- endif %}

salt/metalk8s/orchestrate/upgrade/post.sls

@@ -5,3 +5,19 @@ include:
   - metalk8s.addons.ui.post-upgrade
   - metalk8s.addons.logging.fluent-bit.deployed.post-upgrade
   - metalk8s.addons.dex.deployed.post-upgrade
+
+
+{%- for node in pillar.metalk8s.nodes.keys() %}
+
+Run post-upgrade on node {{ node }}:
+  salt.state:
+    - tgt: {{ node }}
+    - sls:
+      - metalk8s.kubernetes.cni.calico.post-upgrade
+    - saltenv: {{ saltenv }}
+    {%- if loop.previtem is defined %}
+    - require:
+      - salt: Run post-upgrade on node {{ loop.previtem }}
+    {%- endif %}
+
+{%- endfor %}