From 65c67c5c51c2c606ec0d9baed4f71f7eef2069e7 Mon Sep 17 00:00:00 2001 From: Max Gautier Date: Fri, 13 Sep 2024 11:25:08 +0200 Subject: [PATCH 01/10] CI: use Kubernetes GC to delete kubevirt vms This leverage the Kubernetes GC to delete kubevirt VMs, by using ownerReferences, with the CI pod running the playbook as the owner. This concretely means that the control plane in our CI cluster will delete the kubevirt VMs associated with a particular ci job as soon as that pod job is deleted, which usually happens when the job terminates, (barring errors, which will be addressed in the cluster directly) Upgrade to kubevirt.io/v1 for the VirtualMachine manifests, since the alpha version is deprecated. --- .../roles/packet-ci/templates/vm.yml.j2 | 14 +++++++++++--- .../cloud_playbooks/roles/packet-ci/vars/main.yml | 8 ++++++++ 2 files changed, 19 insertions(+), 3 deletions(-) diff --git a/tests/cloud_playbooks/roles/packet-ci/templates/vm.yml.j2 b/tests/cloud_playbooks/roles/packet-ci/templates/vm.yml.j2 index 371656b7387..f5f7cc02a86 100644 --- a/tests/cloud_playbooks/roles/packet-ci/templates/vm.yml.j2 +++ b/tests/cloud_playbooks/roles/packet-ci/templates/vm.yml.j2 @@ -1,13 +1,21 @@ --- -apiVersion: kubevirt.io/v1alpha3 +apiVersion: kubevirt.io/v1 kind: VirtualMachine metadata: - name: "instance-{{ vm_id }}" - namespace: "{{ test_name }}" + generateName: test-vm- + namespace: {{ pod_namespace }} annotations: kubespray.com/ci.template-path: "tests/cloud_playbooks/roles/packet-ci/templates/vm.yml.j2" labels: kubevirt.io/os: {{ cloud_image }} + ci_job_id: "{{ ci_job_id }}" + ci_job_name: "{{ ci_job_name }}" + # leverage the Kubernetes GC for resources cleanup + ownerReferences: + - apiVersion: v1 + kind: Pod + name: "{{ pod_name }}" + uid: "{{ pod_uid }}" spec: running: true template: diff --git a/tests/cloud_playbooks/roles/packet-ci/vars/main.yml b/tests/cloud_playbooks/roles/packet-ci/vars/main.yml index ba448587d48..103da858b0d 100644 --- a/tests/cloud_playbooks/roles/packet-ci/vars/main.yml +++ b/tests/cloud_playbooks/roles/packet-ci/vars/main.yml @@ -9,3 +9,11 @@ _vm_count_dict: default: 2 vm_count: "{{ _vm_count_dict[mode | d('default')] }}" + +# Get pod metadata / CI vars from environment + +ci_job_id: "{{ lookup('ansible.builtin.env', 'CI_JOB_ID', default=undefined) }}" +ci_job_name: "{{ lookup('ansible.builtin.env', 'CI_JOB_NAME', default=undefined) }}" +pod_name: "{{ lookup('ansible.builtin.env', 'POD_NAME', default=undefined) }}" +pod_uid: "{{ lookup('ansible.builtin.env', 'POD_UID', default=undefined) }}" +pod_namespace: "{{ lookup('ansible.builtin.env', 'POD_NAMESPACE', default=undefined) }}" From e8ee422808a2f7813f2282ab8cde8c9475bb1bc5 Mon Sep 17 00:00:00 2001 From: Max Gautier Date: Fri, 13 Sep 2024 12:02:11 +0200 Subject: [PATCH 02/10] CI: remove deletion tasks of 'packet' VMs Kubevirt VMs deletion will be handled by the Kubernetes GC (see previous commit), remove all the codes handling that. --- .gitlab-ci/packet.yml | 8 ----- tests/Makefile | 13 +------- tests/cloud_playbooks/cleanup-packet.yml | 8 ----- tests/cloud_playbooks/delete-packet.yml | 11 ------- .../roles/cleanup-packet-ci/tasks/main.yml | 16 ---------- .../roles/packet-ci/tasks/cleanup-old-vms.yml | 17 ----------- .../roles/packet-ci/tasks/create-vms.yml | 7 ----- .../roles/packet-ci/tasks/delete-vms.yml | 30 ------------------- .../roles/packet-ci/tasks/main.yml | 11 ------- 9 files changed, 1 insertion(+), 120 deletions(-) delete mode 100644 tests/cloud_playbooks/cleanup-packet.yml delete mode 100644 tests/cloud_playbooks/delete-packet.yml delete mode 100644 tests/cloud_playbooks/roles/cleanup-packet-ci/tasks/main.yml delete mode 100644 tests/cloud_playbooks/roles/packet-ci/tasks/cleanup-old-vms.yml delete mode 100644 tests/cloud_playbooks/roles/packet-ci/tasks/delete-vms.yml diff --git a/.gitlab-ci/packet.yml b/.gitlab-ci/packet.yml index 1d909da8394..d7b3a34e902 100644 --- a/.gitlab-ci/packet.yml +++ b/.gitlab-ci/packet.yml @@ -65,14 +65,6 @@ allow_failure: true extends: .packet -packet_cleanup_old: - stage: deploy-part1 - extends: .packet_periodic - script: - - cd tests - - make cleanup-packet - after_script: [] - # The ubuntu20-calico-all-in-one jobs are meant as early stages to prevent running the full CI if something is horribly broken packet_ubuntu20-calico-all-in-one: stage: deploy-part1 diff --git a/tests/Makefile b/tests/Makefile index f052978b15e..18e87a2ddd1 100644 --- a/tests/Makefile +++ b/tests/Makefile @@ -20,18 +20,7 @@ create-packet: init-packet -e pipeline_id="$(CI_PIPELINE_ID)" \ -e inventory_path=$(INVENTORY) -delete-packet: - ansible-playbook cloud_playbooks/delete-packet.yml -c local \ - $(ANSIBLE_LOG_LEVEL) \ - -e @"files/${CI_JOB_NAME}.yml" \ - -e test_id=$(TEST_ID) \ - -e branch="$(CI_COMMIT_BRANCH)" \ - -e pipeline_id="$(CI_PIPELINE_ID)" \ - -e inventory_path=$(INVENTORY) - -cleanup-packet: - ansible-playbook cloud_playbooks/cleanup-packet.yml -c local \ - $(ANSIBLE_LOG_LEVEL) +delete-packet: ; create-vagrant: vagrant up diff --git a/tests/cloud_playbooks/cleanup-packet.yml b/tests/cloud_playbooks/cleanup-packet.yml deleted file mode 100644 index 2ba5e30210a..00000000000 --- a/tests/cloud_playbooks/cleanup-packet.yml +++ /dev/null @@ -1,8 +0,0 @@ ---- - -- name: Cleanup packet vms - hosts: localhost - gather_facts: false - become: true - roles: - - { role: cleanup-packet-ci } diff --git a/tests/cloud_playbooks/delete-packet.yml b/tests/cloud_playbooks/delete-packet.yml deleted file mode 100644 index 7320da62238..00000000000 --- a/tests/cloud_playbooks/delete-packet.yml +++ /dev/null @@ -1,11 +0,0 @@ ---- - -- name: Terminate Packet VMs - hosts: localhost - gather_facts: false - become: true - vars: - ci_job_name: "{{ lookup('env', 'CI_JOB_NAME') }}" - test_name: "{{ test_id | regex_replace('\\.', '-') }}" - roles: - - { role: packet-ci, vm_cleanup: true } diff --git a/tests/cloud_playbooks/roles/cleanup-packet-ci/tasks/main.yml b/tests/cloud_playbooks/roles/cleanup-packet-ci/tasks/main.yml deleted file mode 100644 index 9256b2d54f5..00000000000 --- a/tests/cloud_playbooks/roles/cleanup-packet-ci/tasks/main.yml +++ /dev/null @@ -1,16 +0,0 @@ ---- - -- name: Fetch a list of namespaces - kubernetes.core.k8s_info: - api_version: v1 - kind: Namespace - label_selectors: - - cijobs = true - register: namespaces - -- name: Delete stale namespaces for more than 2 hours - command: "kubectl delete namespace {{ item.metadata.name }}" - failed_when: false - loop: "{{ namespaces.resources }}" - when: - - (now() - (item.metadata.creationTimestamp | to_datetime("%Y-%m-%dT%H:%M:%SZ"))).total_seconds() >= 7200 diff --git a/tests/cloud_playbooks/roles/packet-ci/tasks/cleanup-old-vms.yml b/tests/cloud_playbooks/roles/packet-ci/tasks/cleanup-old-vms.yml deleted file mode 100644 index 052a44fd170..00000000000 --- a/tests/cloud_playbooks/roles/packet-ci/tasks/cleanup-old-vms.yml +++ /dev/null @@ -1,17 +0,0 @@ ---- - -- name: Fetch a list of namespaces - kubernetes.core.k8s_info: - api_version: v1 - kind: Namespace - label_selectors: - - cijobs = true - - branch = {{ branch_name_sane }} - register: namespaces - -- name: Delete older namespaces - command: "kubectl delete namespace {{ item.metadata.name }}" - failed_when: false - loop: "{{ namespaces.resources }}" - when: - - (item.metadata.labels.pipeline_id | int) < (pipeline_id | int) diff --git a/tests/cloud_playbooks/roles/packet-ci/tasks/create-vms.yml b/tests/cloud_playbooks/roles/packet-ci/tasks/create-vms.yml index 2a73b674d3f..28ef298ab9e 100644 --- a/tests/cloud_playbooks/roles/packet-ci/tasks/create-vms.yml +++ b/tests/cloud_playbooks/roles/packet-ci/tasks/create-vms.yml @@ -1,11 +1,4 @@ --- - -- name: "Create CI namespace {{ test_name }} for test vms" - shell: |- - kubectl create namespace {{ test_name }} && - kubectl label namespace {{ test_name }} cijobs=true branch="{{ branch_name_sane }}" pipeline_id="{{ pipeline_id }}" - changed_when: false - - name: "Create temp dir /tmp/{{ test_name }} for CI files" file: path: "/tmp/{{ test_name }}" diff --git a/tests/cloud_playbooks/roles/packet-ci/tasks/delete-vms.yml b/tests/cloud_playbooks/roles/packet-ci/tasks/delete-vms.yml deleted file mode 100644 index 75156584a89..00000000000 --- a/tests/cloud_playbooks/roles/packet-ci/tasks/delete-vms.yml +++ /dev/null @@ -1,30 +0,0 @@ ---- - -- name: Check if temp directory for {{ test_name }} exists - stat: - path: "/tmp/{{ test_name }}" - get_attributes: false - get_checksum: false - get_mime: false - register: temp_dir_details - -- name: "Cleanup temp directory for {{ test_name }}" - file: - path: "/tmp/{{ test_name }}" - state: absent - -- name: "Cleanup namespace for {{ test_name }}" - command: "kubectl delete namespace {{ test_name }}" - changed_when: false - -- name: Wait for namespace {{ test_name }} to be fully deleted - command: kubectl get ns {{ test_name }} - register: delete_namespace - failed_when: - - delete_namespace.rc == 0 - changed_when: - - delete_namespace.rc == 0 - retries: 12 - delay: 10 - until: - - delete_namespace.rc != 0 diff --git a/tests/cloud_playbooks/roles/packet-ci/tasks/main.yml b/tests/cloud_playbooks/roles/packet-ci/tasks/main.yml index f2dea7f6bef..3ab879742d9 100644 --- a/tests/cloud_playbooks/roles/packet-ci/tasks/main.yml +++ b/tests/cloud_playbooks/roles/packet-ci/tasks/main.yml @@ -2,16 +2,5 @@ - name: "Include custom vars for ci job: {{ ci_job_name }}" include_vars: "../files/{{ ci_job_name }}.yml" - -- name: Cleamup old VMs - import_tasks: cleanup-old-vms.yml - - name: Create VMs import_tasks: create-vms.yml - when: - - not vm_cleanup - -- name: Delete VMs - import_tasks: delete-vms.yml - when: - - vm_cleanup | default(false) From c46e5dc33a890dde4d2c137fa247c0319eeaecb0 Mon Sep 17 00:00:00 2001 From: Max Gautier Date: Sat, 14 Sep 2024 14:51:27 +0200 Subject: [PATCH 03/10] CI: use VirtualMachineInstance for VMs VMI in Kubevirt are the abstraction below VirtualMachine. - We don't really need the extra abstraction of VirtualMachine objects - Convert the waiting for VMs ip address to use kubernetes.core.k8s_info and no shell pipeline --- .../roles/packet-ci/tasks/create-vms.yml | 26 +++--- .../roles/packet-ci/templates/vm.yml.j2 | 81 +++++++++---------- 2 files changed, 51 insertions(+), 56 deletions(-) diff --git a/tests/cloud_playbooks/roles/packet-ci/tasks/create-vms.yml b/tests/cloud_playbooks/roles/packet-ci/tasks/create-vms.yml index 28ef298ab9e..c4deca53058 100644 --- a/tests/cloud_playbooks/roles/packet-ci/tasks/create-vms.yml +++ b/tests/cloud_playbooks/roles/packet-ci/tasks/create-vms.yml @@ -20,19 +20,19 @@ changed_when: false loop: "{{ vms_files }}" -- name: Wait for vms to have ipaddress assigned - shell: "set -o pipefail && kubectl get vmis -n {{ test_name }} instance-{{ vm_id }} -o json | jq '.status.interfaces[].ipAddress' | tr -d '\"'" - args: - executable: /bin/bash - changed_when: false - register: vm_ips - loop: "{{ range(1, vm_count | int + 1, 1) | list }}" - loop_control: - index_var: vm_id - retries: 20 - delay: 15 - until: - - vm_ips.stdout | ansible.utils.ipaddr +- name: Wait for vms to have IP addresses + kubernetes.core.k8s_info: + api_version: kubevirt.io/v1 + kind: VirtualMachineInstance + label_selectors: + - "ci_job_id={{ ci_job_id }}" + namespace: "{{ pod_namespace }}" + register: vmis + until: vmis.resources + | map(attribute='status.interfaces.0') + | rejectattr('ipAddress', 'defined') == [] + retries: 30 + delay: 10 - name: "Create inventory for CI test in file /tmp/{{ test_name }}/inventory" template: diff --git a/tests/cloud_playbooks/roles/packet-ci/templates/vm.yml.j2 b/tests/cloud_playbooks/roles/packet-ci/templates/vm.yml.j2 index f5f7cc02a86..98c64a8ce89 100644 --- a/tests/cloud_playbooks/roles/packet-ci/templates/vm.yml.j2 +++ b/tests/cloud_playbooks/roles/packet-ci/templates/vm.yml.j2 @@ -1,6 +1,6 @@ --- apiVersion: kubevirt.io/v1 -kind: VirtualMachine +kind: VirtualMachineInstance metadata: generateName: test-vm- namespace: {{ pod_namespace }} @@ -8,6 +8,8 @@ metadata: kubespray.com/ci.template-path: "tests/cloud_playbooks/roles/packet-ci/templates/vm.yml.j2" labels: kubevirt.io/os: {{ cloud_image }} + kubevirt.io/size: small + kubevirt.io/domain: "{{ test_name }}" ci_job_id: "{{ ci_job_id }}" ci_job_name: "{{ ci_job_name }}" # leverage the Kubernetes GC for resources cleanup @@ -17,46 +19,39 @@ metadata: name: "{{ pod_name }}" uid: "{{ pod_uid }}" spec: - running: true - template: - metadata: - labels: - kubevirt.io/size: small - kubevirt.io/domain: "{{ test_name }}" - spec: - domain: - devices: - blockMultiQueue: true - disks: - - disk: - bus: virtio - name: containervolume - cache: writethrough - - disk: - bus: virtio - name: cloudinitvolume - interfaces: - - name: default - bridge: {} - cpu: - cores: {{ vm_cpu_cores }} - sockets: {{ vm_cpu_sockets }} - threads: {{ vm_cpu_threads }} - resources: - requests: - memory: "{{ vm_memory * memory_allocation_ratio }}Mi" - cpu: {{ vm_cpu_cores * cpu_allocation_ratio }} - limits: - memory: "{{ vm_memory }}Mi" - cpu: {{ vm_cpu_cores }} - networks: + domain: + devices: + blockMultiQueue: true + disks: + - disk: + bus: virtio + name: containervolume + cache: writethrough + - disk: + bus: virtio + name: cloudinitvolume + interfaces: - name: default - pod: {} - terminationGracePeriodSeconds: 0 - volumes: - - name: containervolume - containerDisk: - image: quay.io/kubespray/vm-{{ cloud_image }} - - name: cloudinitvolume - cloudInitNoCloud: - userDataBase64: {{ cloud_init[cloud_image] }} + bridge: {} + cpu: + cores: {{ vm_cpu_cores }} + sockets: {{ vm_cpu_sockets }} + threads: {{ vm_cpu_threads }} + resources: + requests: + memory: "{{ vm_memory * memory_allocation_ratio }}Mi" + cpu: {{ vm_cpu_cores * cpu_allocation_ratio }} + limits: + memory: "{{ vm_memory }}Mi" + cpu: {{ vm_cpu_cores }} + networks: + - name: default + pod: {} + terminationGracePeriodSeconds: 0 + volumes: + - name: containervolume + containerDisk: + image: quay.io/kubespray/vm-{{ cloud_image }} + - name: cloudinitvolume + cloudInitNoCloud: + userDataBase64: {{ cloud_init[cloud_image] }} From a8e7238c9f9dbf3abb76cfcc8e6276b95156d403 Mon Sep 17 00:00:00 2001 From: Max Gautier Date: Fri, 20 Sep 2024 14:31:47 +0200 Subject: [PATCH 04/10] CI: make the ansible inventory a directory Not constraining the inventory to .ini allows us to use dynamic inventory, which is needed for simplifying kubevirt jobs inventory. Also reduces the scope of the ANSIBLE_INVENTORY variable. --- .gitlab-ci.yml | 1 - tests/Makefile | 7 ++----- tests/scripts/testcases_cleanup.sh | 2 +- tests/scripts/testcases_run.sh | 3 ++- 4 files changed, 5 insertions(+), 8 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 5bd8c1b2211..c1cd83bb53f 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -22,7 +22,6 @@ variables: GCE_PREEMPTIBLE: "false" ANSIBLE_KEEP_REMOTE_FILES: "1" ANSIBLE_CONFIG: ./tests/ansible.cfg - ANSIBLE_INVENTORY: ./inventory/sample/${CI_JOB_NAME}-${BUILD_NUMBER}.ini IDEMPOT_CHECK: "false" RESET_CHECK: "false" REMOVE_NODE_CHECK: "false" diff --git a/tests/Makefile b/tests/Makefile index 18e87a2ddd1..50256b197d6 100644 --- a/tests/Makefile +++ b/tests/Makefile @@ -1,5 +1,3 @@ -INVENTORY=$(PWD)/../inventory/sample/${CI_JOB_NAME}-${BUILD_NUMBER}.ini - init-packet: mkdir -p $(HOME)/.ssh echo $(PACKET_VM_SSH_PRIVATE_KEY) | base64 -d > $(HOME)/.ssh/id_rsa @@ -18,14 +16,13 @@ create-packet: init-packet -e test_id=$(TEST_ID) \ -e branch="$(CI_COMMIT_BRANCH)" \ -e pipeline_id="$(CI_PIPELINE_ID)" \ - -e inventory_path=$(INVENTORY) + -e inventory_path=$(INVENTORY_DIR) delete-packet: ; create-vagrant: vagrant up - cp $(CI_PROJECT_DIR)/.vagrant/provisioners/ansible/inventory/vagrant_ansible_inventory $(INVENTORY) - + cp $(CI_PROJECT_DIR)/.vagrant/provisioners/ansible/inventory/vagrant_ansible_inventory $(INVENTORY_DIR) delete-vagrant: vagrant destroy -f diff --git a/tests/scripts/testcases_cleanup.sh b/tests/scripts/testcases_cleanup.sh index 71b7fdc5b16..19978449043 100755 --- a/tests/scripts/testcases_cleanup.sh +++ b/tests/scripts/testcases_cleanup.sh @@ -1,7 +1,7 @@ #!/bin/bash set -euxo pipefail -cd tests && make delete-${CI_PLATFORM} -s ; cd - +make -C tests delete-${CI_PLATFORM} -s if [ -d ~/.ara ] ; then tar czvf ${CI_PROJECT_DIR}/cluster-dump/ara.tgz ~/.ara diff --git a/tests/scripts/testcases_run.sh b/tests/scripts/testcases_run.sh index db689b8c05e..6235d2ed348 100755 --- a/tests/scripts/testcases_run.sh +++ b/tests/scripts/testcases_run.sh @@ -31,8 +31,9 @@ export ANSIBLE_REMOTE_USER=$SSH_USER export ANSIBLE_BECOME=true export ANSIBLE_BECOME_USER=root export ANSIBLE_CALLBACK_PLUGINS="$(python -m ara.setup.callback_plugins)" +export ANSIBLE_INVENTORY=${CI_PROJECT_DIR}/inventory/sample/ -cd tests && make create-${CI_PLATFORM} -s ; cd - +make -C tests INVENTORY_DIR=${ANSIBLE_INVENTORY} create-${CI_PLATFORM} -s ansible-playbook tests/cloud_playbooks/wait-for-ssh.yml # Flatcar Container Linux needs auto update disabled From 329ffd45f0e607dfdac81690d7687960d59d512a Mon Sep 17 00:00:00 2001 From: Max Gautier Date: Fri, 20 Sep 2024 14:39:24 +0200 Subject: [PATCH 05/10] CI: use kubevirt.core dynamic inventory This allows a single source of truth for the virtual machines in a kubevirt ci-run. `etcd_member_name` should be correctly handled in kubespray-defaults for testing the recover cases. --- pipeline.Dockerfile | 2 +- .../roles/packet-ci/tasks/create-vms.yml | 31 ++---- .../packet-ci/templates/inv.kubevirt.yml.j2 | 16 +++ .../roles/packet-ci/templates/inventory.j2 | 98 ------------------- .../roles/packet-ci/templates/vm.yml.j2 | 3 + .../roles/packet-ci/vars/main.yml | 38 +++++-- 6 files changed, 56 insertions(+), 132 deletions(-) create mode 100644 tests/cloud_playbooks/roles/packet-ci/templates/inv.kubevirt.yml.j2 delete mode 100644 tests/cloud_playbooks/roles/packet-ci/templates/inventory.j2 diff --git a/pipeline.Dockerfile b/pipeline.Dockerfile index 49d00ae4f6a..a8af1897b4b 100644 --- a/pipeline.Dockerfile +++ b/pipeline.Dockerfile @@ -60,4 +60,4 @@ RUN update-alternatives --install /usr/bin/python python /usr/bin/python3 1 \ && vagrant plugin install vagrant-libvirt \ # Install Kubernetes collections && pip install --no-compile --no-cache-dir kubernetes \ - && ansible-galaxy collection install kubernetes.core + && ansible-galaxy collection install kubernetes.core kubevirt.core diff --git a/tests/cloud_playbooks/roles/packet-ci/tasks/create-vms.yml b/tests/cloud_playbooks/roles/packet-ci/tasks/create-vms.yml index c4deca53058..30e27c76fb1 100644 --- a/tests/cloud_playbooks/roles/packet-ci/tasks/create-vms.yml +++ b/tests/cloud_playbooks/roles/packet-ci/tasks/create-vms.yml @@ -1,24 +1,11 @@ --- -- name: "Create temp dir /tmp/{{ test_name }} for CI files" - file: - path: "/tmp/{{ test_name }}" - state: directory - mode: "0755" - -- name: Template vm files for CI job - set_fact: - vms_files: "{{ vms_files + [lookup('ansible.builtin.template', 'vm.yml.j2') | from_yaml] }}" - vars: - vms_files: [] - loop: "{{ range(1, vm_count | int + 1, 1) | list }}" - loop_control: - index_var: vm_id - - name: Start vms for CI job + vars: + tvars: + kubespray_groups: "{{ item }}" kubernetes.core.k8s: - definition: "{{ item }}" - changed_when: false - loop: "{{ vms_files }}" + definition: "{{ lookup('template', 'vm.yml.j2', template_vars=tvars) }}" + loop: "{{ scenarios[mode | d('default')] }}" - name: Wait for vms to have IP addresses kubernetes.core.k8s_info: @@ -34,10 +21,8 @@ retries: 30 delay: 10 -- name: "Create inventory for CI test in file /tmp/{{ test_name }}/inventory" +- name: "Create inventory for CI tests" template: - src: "inventory.j2" - dest: "{{ inventory_path }}" + src: "inv.kubevirt.yml.j2" + dest: "{{ inventory_path }}/inv.kubevirt.yml" mode: "0644" - vars: - vms: "{{ vm_ips }}" diff --git a/tests/cloud_playbooks/roles/packet-ci/templates/inv.kubevirt.yml.j2 b/tests/cloud_playbooks/roles/packet-ci/templates/inv.kubevirt.yml.j2 new file mode 100644 index 00000000000..435a5f5c345 --- /dev/null +++ b/tests/cloud_playbooks/roles/packet-ci/templates/inv.kubevirt.yml.j2 @@ -0,0 +1,16 @@ +plugin: kubevirt.core.kubevirt +namespaces: +- {{ pod_namespace }} +label_selector: ci_job_id={{ ci_job_id }} +create_groups: true +compose: + ci_groups: | + group_names | + select('ansible.builtin.match', 'label_kubespray_io*') | + map('regex_replace', 'label_kubespray_io_(.*)_true', '\1') +use_service: false +host_format: "{name}" +keyed_groups: +- key: ci_groups + prefix: "" + separator: "" diff --git a/tests/cloud_playbooks/roles/packet-ci/templates/inventory.j2 b/tests/cloud_playbooks/roles/packet-ci/templates/inventory.j2 deleted file mode 100644 index 1801a740a3b..00000000000 --- a/tests/cloud_playbooks/roles/packet-ci/templates/inventory.j2 +++ /dev/null @@ -1,98 +0,0 @@ -[all] -{% for instance in vms.results %} -instance-{{ loop.index }} ansible_host={{instance.stdout}} -{% endfor %} - -{% if mode == "separate" %} -[kube_control_plane] -instance-1 - -[kube_node] -instance-2 - -[etcd] -instance-3 -{% elif mode == "ha" %} -[kube_control_plane] -instance-1 -instance-2 - -[kube_node] -instance-3 - -[etcd] -instance-1 -instance-2 -instance-3 -{% elif mode == "default" %} -[kube_control_plane] -instance-1 - -[kube_node] -instance-2 - -[etcd] -instance-1 -{% elif mode == "all-in-one" %} -[kube_control_plane] -instance-1 - -[kube_node] -instance-1 - -[etcd] -instance-1 -{% elif mode == "ha-recover" %} -[kube_control_plane] -instance-1 -instance-2 - -[kube_node] -instance-3 - -[etcd] -instance-3 -instance-1 -instance-2 - -[broken_kube_control_plane] -instance-2 - -[broken_etcd] -instance-2 etcd_member_name=etcd3 -{% elif mode == "ha-recover-noquorum" %} -[kube_control_plane] -instance-3 -instance-1 -instance-2 - -[kube_node] -instance-3 - -[etcd] -instance-3 -instance-1 -instance-2 - -[broken_kube_control_plane] -instance-1 -instance-2 - -[broken_etcd] -instance-1 etcd_member_name=etcd2 -instance-2 etcd_member_name=etcd3 -{% elif mode == "node-etcd-client" %} -[kube_control_plane] -instance-1 - -[etcd] -instance-1 -instance-2 -instance-3 - -[kube_node] -instance-1 -instance-2 -instance-3 -instance-4 -{% endif %} diff --git a/tests/cloud_playbooks/roles/packet-ci/templates/vm.yml.j2 b/tests/cloud_playbooks/roles/packet-ci/templates/vm.yml.j2 index 98c64a8ce89..139103c7243 100644 --- a/tests/cloud_playbooks/roles/packet-ci/templates/vm.yml.j2 +++ b/tests/cloud_playbooks/roles/packet-ci/templates/vm.yml.j2 @@ -12,6 +12,9 @@ metadata: kubevirt.io/domain: "{{ test_name }}" ci_job_id: "{{ ci_job_id }}" ci_job_name: "{{ ci_job_name }}" + {% for group in kubespray_groups -%} + kubespray.io/{{ group }}: "true" + {% endfor -%} # leverage the Kubernetes GC for resources cleanup ownerReferences: - apiVersion: v1 diff --git a/tests/cloud_playbooks/roles/packet-ci/vars/main.yml b/tests/cloud_playbooks/roles/packet-ci/vars/main.yml index 103da858b0d..8eb9c7dbed8 100644 --- a/tests/cloud_playbooks/roles/packet-ci/vars/main.yml +++ b/tests/cloud_playbooks/roles/packet-ci/vars/main.yml @@ -1,14 +1,32 @@ --- -_vm_count_dict: - separate: 3 - ha: 3 - ha-recover: 3 - ha-recover-noquorum: 3 - all-in-one: 1 - node-etcd-client: 4 - default: 2 - -vm_count: "{{ _vm_count_dict[mode | d('default')] }}" +# This is a list of nodes with groups for each scenario/cluster layouts +scenarios: + separate: + - ['kube_control_plane'] + - ['kube_node'] + - ['etcd'] + ha: + - ['kube_control_plane', 'etcd'] + - ['kube_control_plane', 'etcd'] + - ['kube_node', 'etcd'] + default: + - ['kube_control_plane', 'etcd'] + - ['kube_node'] + all-in-one: + - ['kube_control_plane', 'etcd', 'kube_node'] + ha-recover: + - ['kube_control_plane', 'etcd'] + - ['kube_control_plane', 'etcd', 'broken_kube_control_plane', 'broken_etcd'] + - ['kube_node', 'etcd'] + ha-recover-noquorum: + - ['kube_control_plane', 'etcd', 'broken_kube_control_plane', 'broken_etcd'] + - ['kube_control_plane', 'etcd', 'broken_kube_control_plane', 'broken_etcd'] + - ['kube_node', 'etcd'] + node-etcd-client: + - ['kube_node', 'kube_control_plane', 'etcd'] + - ['kube_node', 'etcd'] + - ['kube_node', 'etcd'] + - ['kube_node'] # Get pod metadata / CI vars from environment From ff4de880ae54c36120e7ac562df9e692b050e06f Mon Sep 17 00:00:00 2001 From: Max Gautier Date: Wed, 13 Nov 2024 16:01:17 +0100 Subject: [PATCH 06/10] CI: Replace kubevirt dynamic inventory with generated yaml VirtualMachineInstance resources sometimes temporarily loose their IP (at least as far as the kubevirt controllers can see). See https://github.com/kubevirt/kubevirt/issues/12698 for the upstream bug. This does not seems to affect actual connection (if it did, our current CI would not work). However, our CI execute multiple playbooks, and in particular: 1. The provisioning playbook (which checks that the IPs have been provisioned by querying the K8S API) 2. Kubespray itself If any of the VirtualMachineInstance looses its IP between after 1 checked for it, and before 2 starts, the dynamic inventory (which is invoked when the playbook is launched by ansible-playbook) will not have an ip for that host, and will try to use the name for ssh, which of course will not work. Instead, when we have a valid state during provisioning (all IPs presents), use it to construct a static inventory which will be used for the rest of the CI run. --- pipeline.Dockerfile | 2 +- .../roles/packet-ci/tasks/create-vms.yml | 26 ++++++++++++++++--- .../packet-ci/templates/inv.kubevirt.yml.j2 | 16 ------------ .../roles/packet-ci/templates/vm.yml.j2 | 5 ++-- 4 files changed, 25 insertions(+), 24 deletions(-) delete mode 100644 tests/cloud_playbooks/roles/packet-ci/templates/inv.kubevirt.yml.j2 diff --git a/pipeline.Dockerfile b/pipeline.Dockerfile index a8af1897b4b..49d00ae4f6a 100644 --- a/pipeline.Dockerfile +++ b/pipeline.Dockerfile @@ -60,4 +60,4 @@ RUN update-alternatives --install /usr/bin/python python /usr/bin/python3 1 \ && vagrant plugin install vagrant-libvirt \ # Install Kubernetes collections && pip install --no-compile --no-cache-dir kubernetes \ - && ansible-galaxy collection install kubernetes.core kubevirt.core + && ansible-galaxy collection install kubernetes.core diff --git a/tests/cloud_playbooks/roles/packet-ci/tasks/create-vms.yml b/tests/cloud_playbooks/roles/packet-ci/tasks/create-vms.yml index 30e27c76fb1..21206e8881d 100644 --- a/tests/cloud_playbooks/roles/packet-ci/tasks/create-vms.yml +++ b/tests/cloud_playbooks/roles/packet-ci/tasks/create-vms.yml @@ -21,8 +21,26 @@ retries: 30 delay: 10 -- name: "Create inventory for CI tests" - template: - src: "inv.kubevirt.yml.j2" - dest: "{{ inventory_path }}/inv.kubevirt.yml" +- name: Massage VirtualMachineInstance data into an Ansible inventory structure + vars: + ips: "{{ vmis.resources | map(attribute='status.interfaces.0.ipAddress') }}" + names: "{{ vmis.resources | map(attribute='metadata.name') }}" + _groups: "{{ vmis.resources | map(attribute='metadata.annotations.ansible_groups') | map('split', ',') }}" + hosts: "{{ ips | zip(_groups, names) + | map('zip', ['ansible_host', 'ansible_groups', 'k8s_vmi_name']) + | map('map', 'reverse') | map('community.general.dict') }}" + loop: "{{ hosts | map(attribute='ansible_groups') | flatten | unique }}" + set_fact: + ci_inventory: "{{ ci_inventory|d({}) | combine({ + item: { + 'hosts': hosts | selectattr('ansible_groups', 'contains', item) + | rekey_on_member('k8s_vmi_name') + } + }) + }}" + +- name: Create inventory for CI tests + copy: + content: "{{ ci_inventory | to_yaml }}" + dest: "{{ inventory_path }}/ci_inventory.yml" mode: "0644" diff --git a/tests/cloud_playbooks/roles/packet-ci/templates/inv.kubevirt.yml.j2 b/tests/cloud_playbooks/roles/packet-ci/templates/inv.kubevirt.yml.j2 deleted file mode 100644 index 435a5f5c345..00000000000 --- a/tests/cloud_playbooks/roles/packet-ci/templates/inv.kubevirt.yml.j2 +++ /dev/null @@ -1,16 +0,0 @@ -plugin: kubevirt.core.kubevirt -namespaces: -- {{ pod_namespace }} -label_selector: ci_job_id={{ ci_job_id }} -create_groups: true -compose: - ci_groups: | - group_names | - select('ansible.builtin.match', 'label_kubespray_io*') | - map('regex_replace', 'label_kubespray_io_(.*)_true', '\1') -use_service: false -host_format: "{name}" -keyed_groups: -- key: ci_groups - prefix: "" - separator: "" diff --git a/tests/cloud_playbooks/roles/packet-ci/templates/vm.yml.j2 b/tests/cloud_playbooks/roles/packet-ci/templates/vm.yml.j2 index 139103c7243..9209449636f 100644 --- a/tests/cloud_playbooks/roles/packet-ci/templates/vm.yml.j2 +++ b/tests/cloud_playbooks/roles/packet-ci/templates/vm.yml.j2 @@ -6,15 +6,14 @@ metadata: namespace: {{ pod_namespace }} annotations: kubespray.com/ci.template-path: "tests/cloud_playbooks/roles/packet-ci/templates/vm.yml.j2" + ansible_groups: "{{ kubespray_groups | join(',') }}" + # This does not use a dns prefix because dots are hard to escape with map(attribute=) in Jinja labels: kubevirt.io/os: {{ cloud_image }} kubevirt.io/size: small kubevirt.io/domain: "{{ test_name }}" ci_job_id: "{{ ci_job_id }}" ci_job_name: "{{ ci_job_name }}" - {% for group in kubespray_groups -%} - kubespray.io/{{ group }}: "true" - {% endfor -%} # leverage the Kubernetes GC for resources cleanup ownerReferences: - apiVersion: v1 From ceb4b2fa7def5ecfb9cc5c7e16c663ba6641cb74 Mon Sep 17 00:00:00 2001 From: Max Gautier Date: Fri, 18 Oct 2024 12:11:41 +0200 Subject: [PATCH 07/10] CI: use current tests/ directory for upgrade testing We should not rollback our test setup during upgrade test. The only reason to do that would be for incompatible changes in the test inventory, and we already checkout master for those (${CI_JOB_NAME}.yml) Also do some cleanup by removing unnecessary intermediary variables --- .gitlab-ci.yml | 4 ---- tests/Makefile | 2 +- tests/cloud_playbooks/create-packet.yml | 3 --- tests/common/_kubespray_test_settings.yml | 5 ----- ...ocker_hub_registry_mirror.yml => common_vars.yml} | 10 ++++++---- tests/scripts/testcases_run.sh | 12 +++++------- 6 files changed, 12 insertions(+), 24 deletions(-) delete mode 100644 tests/common/_kubespray_test_settings.yml rename tests/{common/_docker_hub_registry_mirror.yml => common_vars.yml} (95%) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index c1cd83bb53f..2eaa8f04660 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -11,10 +11,6 @@ variables: GITLAB_REPOSITORY: 'kargo-ci/kubernetes-sigs-kubespray' ANSIBLE_FORCE_COLOR: "true" MAGIC: "ci check this" - TEST_ID: "$CI_PIPELINE_ID-$CI_JOB_ID" - CI_TEST_VARS: "./tests/files/${CI_JOB_NAME}.yml" - CI_TEST_REGISTRY_MIRROR: "./tests/common/_docker_hub_registry_mirror.yml" - CI_TEST_SETTING: "./tests/common/_kubespray_test_settings.yml" GS_ACCESS_KEY_ID: $GS_KEY GS_SECRET_ACCESS_KEY: $GS_SECRET CONTAINER_ENGINE: docker diff --git a/tests/Makefile b/tests/Makefile index 50256b197d6..33e0e9dcb63 100644 --- a/tests/Makefile +++ b/tests/Makefile @@ -13,7 +13,7 @@ create-packet: init-packet ansible-playbook cloud_playbooks/create-packet.yml -c local \ $(ANSIBLE_LOG_LEVEL) \ -e @"files/${CI_JOB_NAME}.yml" \ - -e test_id=$(TEST_ID) \ + -e test_name="$(subst .,-,$(CI_PIPELINE_ID)-$(CI_JOB_ID))" \ -e branch="$(CI_COMMIT_BRANCH)" \ -e pipeline_id="$(CI_PIPELINE_ID)" \ -e inventory_path=$(INVENTORY_DIR) diff --git a/tests/cloud_playbooks/create-packet.yml b/tests/cloud_playbooks/create-packet.yml index 2cd08b54d18..26c39217f1e 100644 --- a/tests/cloud_playbooks/create-packet.yml +++ b/tests/cloud_playbooks/create-packet.yml @@ -4,8 +4,5 @@ hosts: localhost gather_facts: false become: true - vars: - ci_job_name: "{{ lookup('env', 'CI_JOB_NAME') }}" - test_name: "{{ test_id | regex_replace('\\.', '-') }}" roles: - { role: packet-ci, vm_cleanup: false } diff --git a/tests/common/_kubespray_test_settings.yml b/tests/common/_kubespray_test_settings.yml deleted file mode 100644 index 67da05c50e2..00000000000 --- a/tests/common/_kubespray_test_settings.yml +++ /dev/null @@ -1,5 +0,0 @@ ---- -# Kubespray settings for tests -deploy_netchecker: true -dns_min_replicas: 1 -unsafe_show_logs: true diff --git a/tests/common/_docker_hub_registry_mirror.yml b/tests/common_vars.yml similarity index 95% rename from tests/common/_docker_hub_registry_mirror.yml rename to tests/common_vars.yml index 8875d217338..a8645ba26bf 100644 --- a/tests/common/_docker_hub_registry_mirror.yml +++ b/tests/common_vars.yml @@ -1,4 +1,10 @@ --- +# Kubespray settings for tests +deploy_netchecker: true +dns_min_replicas: 1 +unsafe_show_logs: true + +# Registry mirrors settings docker_registry_mirrors: - "https://mirror.gcr.io" @@ -34,7 +40,3 @@ nginx_image_repo: "{{ quay_image_repo }}/kubespray/nginx" flannel_image_repo: "{{ quay_image_repo }}/kubespray/flannel" flannel_init_image_repo: "{{ quay_image_repo }}/kubespray/flannel-cni-plugin" - -# Kubespray settings for tests -deploy_netchecker: true -dns_min_replicas: 1 diff --git a/tests/scripts/testcases_run.sh b/tests/scripts/testcases_run.sh index 6235d2ed348..81632eeee1d 100755 --- a/tests/scripts/testcases_run.sh +++ b/tests/scripts/testcases_run.sh @@ -18,10 +18,9 @@ fi # Check out latest tag if testing upgrade if [ "${UPGRADE_TEST}" != "false" ]; then git fetch --all && git checkout "$KUBESPRAY_VERSION" - # Checkout the CI vars file so it is available - git checkout "${CI_COMMIT_SHA}" tests/files/${CI_JOB_NAME}.yml - git checkout "${CI_COMMIT_SHA}" ${CI_TEST_REGISTRY_MIRROR} - git checkout "${CI_COMMIT_SHA}" ${CI_TEST_SETTING} + # Checkout the current tests/ directory ; even when testing old version, + # we want the up-to-date test setup/provisionning + git checkout "${CI_COMMIT_SHA}" -- tests/ fi # needed for ara not to complain @@ -57,9 +56,8 @@ shift # We can set --limit here and still pass it as supplemental args because `--limit` is a 'last one wins' option ansible-playbook \ $ANSIBLE_LOG_LEVEL \ - -e @${CI_TEST_SETTING} \ - -e @${CI_TEST_REGISTRY_MIRROR} \ - -e @${CI_TEST_VARS} \ + -e @tests/common_vars.yml \ + -e @tests/files/${CI_JOB_NAME}.yml \ -e local_release_dir=${PWD}/downloads \ "$@" \ ${playbook} From b5464afa55a3ea5af8b74fe0dfe5e9e025691918 Mon Sep 17 00:00:00 2001 From: Max Gautier Date: Fri, 18 Oct 2024 13:25:39 +0200 Subject: [PATCH 08/10] CI: workaround for upgrade test backward compatibility The new CI does not define k8s_cluster group, so it relies on kubernetes-sigs/kubespray#11559. This does not work for upgrade testing (which use the previous release). We can revert this commit after 2.27.0 --- tests/cloud_playbooks/roles/packet-ci/tasks/create-vms.yml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/tests/cloud_playbooks/roles/packet-ci/tasks/create-vms.yml b/tests/cloud_playbooks/roles/packet-ci/tasks/create-vms.yml index 21206e8881d..beed40c4540 100644 --- a/tests/cloud_playbooks/roles/packet-ci/tasks/create-vms.yml +++ b/tests/cloud_playbooks/roles/packet-ci/tasks/create-vms.yml @@ -1,8 +1,11 @@ --- - name: Start vms for CI job vars: + # Workaround for compatibility when testing upgrades with old == before e9d406ed088d4291ef1d9018c170a4deed2bf928 + # TODO: drop after 2.27.0 + legacy_groups: "{{ (['kube_control_plane', 'kube_node', 'calico_rr'] | intersect(item) | length > 0) | ternary(['k8s_cluster'], []) }}" tvars: - kubespray_groups: "{{ item }}" + kubespray_groups: "{{ item + legacy_groups }}" kubernetes.core.k8s: definition: "{{ lookup('template', 'vm.yml.j2', template_vars=tvars) }}" loop: "{{ scenarios[mode | d('default')] }}" From 236a7486f40e1bcc7d0fb143d0d8ca0945d3039f Mon Sep 17 00:00:00 2001 From: Max Gautier Date: Mon, 4 Nov 2024 13:18:04 +0100 Subject: [PATCH 09/10] CI: directly use ANSIBLE_VERBOSITY instead of tweaking command line https://docs.ansible.com/ansible/latest/reference_appendices/config.html#envvar-ANSIBLE_VERBOSITY --- .gitlab-ci.yml | 2 +- tests/Makefile | 1 - tests/scripts/testcases_run.sh | 1 - 3 files changed, 1 insertion(+), 3 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 2eaa8f04660..e2b8864f422 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -23,7 +23,7 @@ variables: REMOVE_NODE_CHECK: "false" UPGRADE_TEST: "false" MITOGEN_ENABLE: "false" - ANSIBLE_LOG_LEVEL: "-vv" + ANSIBLE_VERBOSITY: 2 RECOVER_CONTROL_PLANE_TEST: "false" RECOVER_CONTROL_PLANE_TEST_GROUPS: "etcd[2:]:kube_control_plane[1:]" TERRAFORM_VERSION: 1.3.7 diff --git a/tests/Makefile b/tests/Makefile index 33e0e9dcb63..8a275f1dc9e 100644 --- a/tests/Makefile +++ b/tests/Makefile @@ -11,7 +11,6 @@ delete-tf: create-packet: init-packet ansible-playbook cloud_playbooks/create-packet.yml -c local \ - $(ANSIBLE_LOG_LEVEL) \ -e @"files/${CI_JOB_NAME}.yml" \ -e test_name="$(subst .,-,$(CI_PIPELINE_ID)-$(CI_JOB_ID))" \ -e branch="$(CI_COMMIT_BRANCH)" \ diff --git a/tests/scripts/testcases_run.sh b/tests/scripts/testcases_run.sh index 81632eeee1d..66eabcc8f4e 100755 --- a/tests/scripts/testcases_run.sh +++ b/tests/scripts/testcases_run.sh @@ -55,7 +55,6 @@ playbook=$1 shift # We can set --limit here and still pass it as supplemental args because `--limit` is a 'last one wins' option ansible-playbook \ - $ANSIBLE_LOG_LEVEL \ -e @tests/common_vars.yml \ -e @tests/files/${CI_JOB_NAME}.yml \ -e local_release_dir=${PWD}/downloads \ From 47f67818b60c38a54e1caef2d0f2f7f02e349d48 Mon Sep 17 00:00:00 2001 From: Max Gautier Date: Wed, 13 Nov 2024 17:25:59 +0100 Subject: [PATCH 10/10] CI: (packet-ci) simplify role layout --- .../roles/packet-ci/tasks/create-vms.yml | 49 ----------------- .../roles/packet-ci/tasks/main.yml | 54 +++++++++++++++++-- 2 files changed, 50 insertions(+), 53 deletions(-) delete mode 100644 tests/cloud_playbooks/roles/packet-ci/tasks/create-vms.yml diff --git a/tests/cloud_playbooks/roles/packet-ci/tasks/create-vms.yml b/tests/cloud_playbooks/roles/packet-ci/tasks/create-vms.yml deleted file mode 100644 index beed40c4540..00000000000 --- a/tests/cloud_playbooks/roles/packet-ci/tasks/create-vms.yml +++ /dev/null @@ -1,49 +0,0 @@ ---- -- name: Start vms for CI job - vars: - # Workaround for compatibility when testing upgrades with old == before e9d406ed088d4291ef1d9018c170a4deed2bf928 - # TODO: drop after 2.27.0 - legacy_groups: "{{ (['kube_control_plane', 'kube_node', 'calico_rr'] | intersect(item) | length > 0) | ternary(['k8s_cluster'], []) }}" - tvars: - kubespray_groups: "{{ item + legacy_groups }}" - kubernetes.core.k8s: - definition: "{{ lookup('template', 'vm.yml.j2', template_vars=tvars) }}" - loop: "{{ scenarios[mode | d('default')] }}" - -- name: Wait for vms to have IP addresses - kubernetes.core.k8s_info: - api_version: kubevirt.io/v1 - kind: VirtualMachineInstance - label_selectors: - - "ci_job_id={{ ci_job_id }}" - namespace: "{{ pod_namespace }}" - register: vmis - until: vmis.resources - | map(attribute='status.interfaces.0') - | rejectattr('ipAddress', 'defined') == [] - retries: 30 - delay: 10 - -- name: Massage VirtualMachineInstance data into an Ansible inventory structure - vars: - ips: "{{ vmis.resources | map(attribute='status.interfaces.0.ipAddress') }}" - names: "{{ vmis.resources | map(attribute='metadata.name') }}" - _groups: "{{ vmis.resources | map(attribute='metadata.annotations.ansible_groups') | map('split', ',') }}" - hosts: "{{ ips | zip(_groups, names) - | map('zip', ['ansible_host', 'ansible_groups', 'k8s_vmi_name']) - | map('map', 'reverse') | map('community.general.dict') }}" - loop: "{{ hosts | map(attribute='ansible_groups') | flatten | unique }}" - set_fact: - ci_inventory: "{{ ci_inventory|d({}) | combine({ - item: { - 'hosts': hosts | selectattr('ansible_groups', 'contains', item) - | rekey_on_member('k8s_vmi_name') - } - }) - }}" - -- name: Create inventory for CI tests - copy: - content: "{{ ci_inventory | to_yaml }}" - dest: "{{ inventory_path }}/ci_inventory.yml" - mode: "0644" diff --git a/tests/cloud_playbooks/roles/packet-ci/tasks/main.yml b/tests/cloud_playbooks/roles/packet-ci/tasks/main.yml index 3ab879742d9..de6e48821ec 100644 --- a/tests/cloud_playbooks/roles/packet-ci/tasks/main.yml +++ b/tests/cloud_playbooks/roles/packet-ci/tasks/main.yml @@ -1,6 +1,52 @@ --- - -- name: "Include custom vars for ci job: {{ ci_job_name }}" +- name: Include custom vars for ci job include_vars: "../files/{{ ci_job_name }}.yml" -- name: Create VMs - import_tasks: create-vms.yml + +- name: Start vms for CI job + vars: + # Workaround for compatibility when testing upgrades with old == before e9d406ed088d4291ef1d9018c170a4deed2bf928 + # TODO: drop after 2.27.0 + legacy_groups: "{{ (['kube_control_plane', 'kube_node', 'calico_rr'] | intersect(item) | length > 0) | ternary(['k8s_cluster'], []) }}" + tvars: + kubespray_groups: "{{ item + legacy_groups }}" + kubernetes.core.k8s: + definition: "{{ lookup('template', 'vm.yml.j2', template_vars=tvars) }}" + loop: "{{ scenarios[mode | d('default')] }}" + +- name: Wait for vms to have IP addresses + kubernetes.core.k8s_info: + api_version: kubevirt.io/v1 + kind: VirtualMachineInstance + label_selectors: + - "ci_job_id={{ ci_job_id }}" + namespace: "{{ pod_namespace }}" + register: vmis + until: vmis.resources + | map(attribute='status.interfaces.0') + | rejectattr('ipAddress', 'defined') == [] + retries: 30 + delay: 10 + +- name: Massage VirtualMachineInstance data into an Ansible inventory structure + vars: + ips: "{{ vmis.resources | map(attribute='status.interfaces.0.ipAddress') }}" + names: "{{ vmis.resources | map(attribute='metadata.name') }}" + _groups: "{{ vmis.resources | map(attribute='metadata.annotations.ansible_groups') | map('split', ',') }}" + hosts: "{{ ips | zip(_groups, names) + | map('zip', ['ansible_host', 'ansible_groups', 'k8s_vmi_name']) + | map('map', 'reverse') | map('community.general.dict') }}" + loop: "{{ hosts | map(attribute='ansible_groups') | flatten | unique }}" + set_fact: + ci_inventory: "{{ ci_inventory|d({}) | combine({ + item: { + 'hosts': hosts | selectattr('ansible_groups', 'contains', item) + | rekey_on_member('k8s_vmi_name') + } + }) + }}" + +- name: Create inventory for CI tests + copy: + content: "{{ ci_inventory | to_yaml }}" + dest: "{{ inventory_path }}/ci_inventory.yml" + mode: "0644"