[2.23] Fix calico-node in etcd mode (#10768)

* CI: Document the 'all-in-one' layout + small refactoring (#10725) * Rename aio to all-in-one and document it ADTM. Acronyms don't tell much. * Refactor vm_count in tests provisioning * Add test case for calico using etcd datastore (#10722) * Add multinode ci layout * Add test case for calico using etcd datastore * Fix calico-node in etcd mode (#10438) * Calico : add ETCD endpoints to install-cni container * Calico : remove nodename from configmap in etcd mode --------- Co-authored-by: Olivier Levitt <[email protected]>
kubernetes-sigs · Jan 12, 2024 · 3d76c30 · 3d76c30
1 parent 20a9e20
commit 3d76c30
Show file tree

Hide file tree

Showing 19 changed files with 88 additions and 30 deletions.
diff --git a/.gitlab-ci/packet.yml b/.gitlab-ci/packet.yml
@@ -31,8 +31,8 @@ packet_cleanup_old:
     - make cleanup-packet
   after_script: []
 
-# The ubuntu20-calico-aio jobs are meant as early stages to prevent running the full CI if something is horribly broken
-packet_ubuntu20-calico-aio:
+# The ubuntu20-calico-all-in-one jobs are meant as early stages to prevent running the full CI if something is horribly broken
+packet_ubuntu20-calico-all-in-one:
   stage: deploy-part1
   extends: .packet_pr
   when: on_success
@@ -41,22 +41,27 @@ packet_ubuntu20-calico-aio:
 
 # ### PR JOBS PART2
 
-packet_ubuntu20-aio-docker:
+packet_ubuntu20-all-in-one-docker:
   stage: deploy-part2
   extends: .packet_pr
   when: on_success
 
-packet_ubuntu20-calico-aio-hardening:
+packet_ubuntu20-calico-all-in-one-hardening:
   stage: deploy-part2
   extends: .packet_pr
   when: on_success
 
-packet_ubuntu22-aio-docker:
+packet_ubuntu22-all-in-one-docker:
   stage: deploy-part2
   extends: .packet_pr
   when: on_success
 
-packet_ubuntu22-calico-aio:
+packet_ubuntu22-calico-all-in-one:
+  stage: deploy-part2
+  extends: .packet_pr
+  when: on_success
+
+packet_ubuntu22-calico-etcd-datastore:
   stage: deploy-part2
   extends: .packet_pr
   when: on_success
@@ -235,7 +240,7 @@ packet_fedora37-calico-swap-selinux:
   extends: .packet_pr
   when: manual
 
-packet_amazon-linux-2-aio:
+packet_amazon-linux-2-all-in-one:
   stage: deploy-part2
   extends: .packet_pr
   when: manual

diff --git a/docs/test_cases.md b/docs/test_cases.md
@@ -1,6 +1,6 @@
 # Node Layouts
 
-There are four node layout types: `default`, `separate`, `ha`, and `scale`.
+There are six node layout types: `default`, `separate`, `ha`, `scale`, `all-in-one`, and `multinode`.
 
 `default` is a non-HA two nodes setup with one separate `kube_node`
 and the `etcd` group merged with the `kube_control_plane`.
@@ -16,6 +16,10 @@ in the Ansible inventory. This helps test TLS certificate generation at scale
 to prevent regressions and profile certain long-running tasks. These nodes are
 never actually deployed, but certificates are generated for them.
 
+`all-in-one` layout use a single node for with `kube_control_plane`, `etcd` and `kube_node` merged.
+
+`multinode` layout consists of two separate `kube_node` and a merged single `etcd+kube_control_plane` node.
+
 Note, the canal network plugin deploys flannel as well plus calico policy controller.
 
 ## Test cases

diff --git a/roles/network_plugin/calico/templates/calico-config.yml.j2 b/roles/network_plugin/calico/templates/calico-config.yml.j2
@@ -36,12 +36,6 @@ data:
           {% if calico_datastore == "kdd" %}
             "datastore_type": "kubernetes",
             "nodename": "__KUBERNETES_NODE_NAME__",
-          {% else %}
-          {% if cloud_provider is defined %}
-            "nodename": "{{ calico_kubelet_name.stdout }}",
-            {% else %}
-            "nodename": "{{ calico_baremetal_nodename }}",
-          {% endif %}
           {% endif %}
             "type": "calico",
             "log_level": "info",

diff --git a/roles/network_plugin/calico/templates/calico-node.yml.j2 b/roles/network_plugin/calico/templates/calico-node.yml.j2
@@ -95,6 +95,13 @@ spec:
             # Prevents the container from sleeping forever.
             - name: SLEEP
               value: "false"
+{% if calico_datastore == "etcd" %}
+            - name: ETCD_ENDPOINTS
+              valueFrom:
+                configMapKeyRef:
+                  name: calico-config
+                  key: etcd_endpoints
+{% endif %}
 {% if calico_datastore == "kdd" %}
             # Set the hostname based on the k8s node name.
             - name: KUBERNETES_NODE_NAME

diff --git a/tests/cloud_playbooks/create-gce.yml b/tests/cloud_playbooks/create-gce.yml
@@ -23,7 +23,7 @@
         instance_names: >-
           {%- if mode in ['separate', 'separate-scale', 'ha', 'ha-scale'] -%}
           k8s-{{ test_name }}-1,k8s-{{ test_name }}-2,k8s-{{ test_name }}-3
-          {%- elif mode == 'aio' -%}
+          {%- elif mode == 'all-in-one' -%}
           k8s-{{ test_name }}-1
           {%- else -%}
           k8s-{{ test_name }}-1,k8s-{{ test_name }}-2

diff --git a/tests/cloud_playbooks/roles/packet-ci/defaults/main.yml b/tests/cloud_playbooks/roles/packet-ci/defaults/main.yml
@@ -18,7 +18,7 @@ memory_allocation_ratio: 1
 inventory_path: "/tmp/{{ test_name }}/inventory"
 
 # Deployment mode
-mode: aio
+mode: all-in-one
 
 # Cloud init config for each os type
 # distro: fedora -> I2Nsb3VkLWNvbmZpZwpzeXN0ZW1faW5mbzoKICBkaXN0cm86IGZlZG9yYQp1c2VyczoKIC0gbmFtZToga3ViZXNwcmF5CiAgIGdyb3Vwczogd2hlZWwKICAgc3VkbzogJ0FMTD0oQUxMKSBOT1BBU1NXRDpBTEwnCiAgIHNoZWxsOiAvYmluL2Jhc2gKICAgbG9ja19wYXNzd2Q6IEZhbHNlCiAgIGhvbWU6IC9ob21lL2t1YmVzcHJheQogICBzc2hfYXV0aG9yaXplZF9rZXlzOgogICAgIC0gc3NoLXJzYSBBQUFBQjNOemFDMXljMkVBQUFBREFRQUJBQUFCQVFDYW5UaS9lS3gwK3RIWUpBZURocStzRlMyT2JVUDEvSTY5ZjdpVjNVdGtLbFQyMEpmVzFmNkZlWHQvMDRWZjI3V1FxK05xczZ2R0JxRDlRWFNZdWYrdDAvczdFUExqVGVpOW1lMW1wcXIrdVRlK0tEdFRQMzlwZkQzL2VWQ2FlQjcyNkdQMkZrYUQwRnpwbUViNjZPM05xaHhPUTk2R3gvOVhUdXcvSzNsbGo0T1ZENkdyalIzQjdjNFh0RUJzWmNacHBNSi9vSDFtR3lHWGRoMzFtV1FTcUFSTy9QOFU4R3d0MCtIR3BVd2gvaGR5M3QrU1lvVEIyR3dWYjB6b3lWd3RWdmZEUXpzbThmcTNhdjRLdmV6OGtZdU5ESnYwNXg0bHZVWmdSMTVaRFJYc0FuZGhReXFvWGRDTEFlMCtlYUtYcTlCa1d4S0ZiOWhQZTBBVWpqYTU=

diff --git a/tests/cloud_playbooks/roles/packet-ci/tasks/main.yml b/tests/cloud_playbooks/roles/packet-ci/tasks/main.yml
@@ -3,10 +3,6 @@
 - name: "Include custom vars for ci job: {{ ci_job_name }}"
   include_vars: "../files/{{ ci_job_name }}.yml"
 
-- name: Set VM count needed for CI test_id
-  set_fact:
-    vm_count: "{%- if mode in ['separate', 'separate-scale', 'ha', 'ha-scale', 'ha-recover', 'ha-recover-noquorum'] -%}{{ 3 | int }}{%- elif mode == 'aio' -%}{{ 1 | int }}{%- else -%}{{ 2 | int }}{%- endif -%}"
-
 - name: Cleamup old VMs
   import_tasks: cleanup-old-vms.yml
 

diff --git a/tests/cloud_playbooks/roles/packet-ci/templates/inventory.j2 b/tests/cloud_playbooks/roles/packet-ci/templates/inventory.j2
@@ -33,7 +33,7 @@ instance-2
 
 [etcd]
 instance-1
-{% elif mode == "aio" %}
+{% elif mode == "all-in-one" %}
 [kube_control_plane]
 instance-1
 
@@ -81,6 +81,16 @@ instance-2
 [broken_etcd]
 instance-1 etcd_member_name=etcd2
 instance-2 etcd_member_name=etcd3
+{% elif mode == "multinode" %}
+[kube_control_plane]
+instance-1
+
+[etcd]
+instance-1
+
+[kube_node]
+instance-2
+instance-3
 {% endif %}
 
 [k8s_cluster:children]

diff --git a/tests/cloud_playbooks/roles/packet-ci/vars/main.yml b/tests/cloud_playbooks/roles/packet-ci/vars/main.yml
@@ -0,0 +1,13 @@
+---
+_vm_count_dict:
+  separate: 3
+  separate-scale: 3
+  ha: 3
+  ha-scale: 3
+  ha-recover: 3
+  ha-recover-noquorum: 3
+  all-in-one: 1
+  multinode: 3
+  default: 2
+
+vm_count: "{{ _vm_count_dict[mode | d('default')] }}"
diff --git a/tests/files/packet_amazon-linux-2-aio.yml → ...iles/packet_amazon-linux-2-all-in-one.yml b/tests/files/packet_amazon-linux-2-aio.yml → ...iles/packet_amazon-linux-2-all-in-one.yml
@@ -1,4 +1,4 @@
 ---
 # Instance settings
 cloud_image: amazon-linux-2
-mode: aio
+mode: all-in-one
diff --git a/tests/files/packet_ubuntu20-aio-docker.yml → ...les/packet_ubuntu20-all-in-one-docker.yml b/tests/files/packet_ubuntu20-aio-docker.yml → ...les/packet_ubuntu20-all-in-one-docker.yml
@@ -1,7 +1,7 @@
 ---
 # Instance settings
 cloud_image: ubuntu-2004
-mode: aio
+mode: all-in-one
 
 # Kubespray settings
 auto_renew_certificates: true

diff --git a/tests/files/packet_ubuntu20-calico-aio-ansible-2_11.yml b/tests/files/packet_ubuntu20-calico-aio-ansible-2_11.yml
diff --git a/tests/files/packet_ubuntu20-calico-all-in-one-ansible-2_11.yml b/tests/files/packet_ubuntu20-calico-all-in-one-ansible-2_11.yml
@@ -0,0 +1 @@
+packet_ubuntu20-calico-all-in-one.yml
diff --git a/.../packet_ubuntu20-calico-aio-hardening.yml → ..._ubuntu20-calico-all-in-one-hardening.yml b/.../packet_ubuntu20-calico-aio-hardening.yml → ..._ubuntu20-calico-all-in-one-hardening.yml
@@ -1,7 +1,7 @@
 ---
 # Instance settings
 cloud_image: ubuntu-2004
-mode: aio
+mode: all-in-one
 
 # Kubespray settings
 auto_renew_certificates: true

diff --git a/tests/files/packet_ubuntu20-calico-aio.yml → ...les/packet_ubuntu20-calico-all-in-one.yml b/tests/files/packet_ubuntu20-calico-aio.yml → ...les/packet_ubuntu20-calico-all-in-one.yml
@@ -1,7 +1,7 @@
 ---
 # Instance settings
 cloud_image: ubuntu-2004
-mode: aio
+mode: all-in-one
 
 # Kubespray settings
 auto_renew_certificates: true

diff --git a/tests/files/packet_ubuntu22-aio-docker.yml → ...les/packet_ubuntu22-all-in-one-docker.yml b/tests/files/packet_ubuntu22-aio-docker.yml → ...les/packet_ubuntu22-all-in-one-docker.yml
@@ -1,7 +1,7 @@
 ---
 # Instance settings
 cloud_image: ubuntu-2204
-mode: aio
+mode: all-in-one
 vm_memory: 1600Mi
 
 # Kubespray settings

diff --git a/tests/files/packet_ubuntu22-calico-aio.yml → ...les/packet_ubuntu22-calico-all-in-one.yml b/tests/files/packet_ubuntu22-calico-aio.yml → ...les/packet_ubuntu22-calico-all-in-one.yml
@@ -1,7 +1,7 @@
 ---
 # Instance settings
 cloud_image: ubuntu-2204
-mode: aio
+mode: all-in-one
 vm_memory: 1600Mi
 
 # Kubespray settings

diff --git a/tests/files/packet_ubuntu22-calico-etcd-datastore.yml b/tests/files/packet_ubuntu22-calico-etcd-datastore.yml
@@ -0,0 +1,29 @@
+---
+# Instance settings
+cloud_image: ubuntu-2204
+mode: multinode
+vm_memory: 1600Mi
+
+# Kubespray settings
+auto_renew_certificates: true
+
+# Currently ipvs not available on KVM: https://packages.ubuntu.com/search?suite=focal&arch=amd64&mode=exactfilename&searchon=contents&keywords=ip_vs_sh.ko
+kube_proxy_mode: iptables
+enable_nodelocaldns: False
+
+containerd_registries:
+  "docker.io": "https://mirror.gcr.io"
+
+containerd_registries_mirrors:
+  - prefix: docker.io
+    mirrors:
+      - host: https://mirror.gcr.io
+        capabilities: ["pull", "resolve"]
+        skip_verify: false
+  - prefix: 172.19.16.11:5000
+    mirrors:
+      - host: http://172.19.16.11:5000
+        capabilities: ["pull", "resolve", "push"]
+        skip_verify: true
+
+calico_datastore: "etcd"
diff --git a/tests/templates/inventory-gce.j2 b/tests/templates/inventory-gce.j2
@@ -1,6 +1,6 @@
 {% set node1 = gce.instance_data[0].name %}
 {{node1}} ansible_ssh_host={{gce.instance_data[0].public_ip}}
-{% if mode != "aio" %}
+{% if mode != "all-in-one" %}
 {% set node2 = gce.instance_data[1].name %}
 {{node2}} ansible_ssh_host={{gce.instance_data[1].public_ip}}
 {% endif %}
@@ -46,7 +46,7 @@
 
 [etcd]
 {{node1}}
-{% elif mode == "aio" %}
+{% elif mode == "all-in-one" %}
 [kube_control_plane]
 {{node1}}