From 829be897dd239c9bb8383c02289f079c12499734 Mon Sep 17 00:00:00 2001 From: Jonathan Rosser Date: Thu, 9 Jan 2025 15:00:16 +0000 Subject: [PATCH 1/4] Use calico images from quay.io for workload clusters Instead of getting these from docker.io and running into rate limits use images from quay.io --- magnum_cluster_api/cmd/image_loader.py | 6 +++--- magnum_cluster_api/image_utils.py | 2 ++ magnum_cluster_api/manifests/calico/v3.24.2.yaml | 10 +++++----- magnum_cluster_api/manifests/calico/v3.25.2.yaml | 10 +++++----- magnum_cluster_api/manifests/calico/v3.26.5.yaml | 10 +++++----- magnum_cluster_api/manifests/calico/v3.27.4.yaml | 10 +++++----- magnum_cluster_api/manifests/calico/v3.28.2.yaml | 10 +++++----- magnum_cluster_api/manifests/calico/v3.29.0.yaml | 10 +++++----- tools/sync-calico | 4 ++++ 9 files changed, 39 insertions(+), 33 deletions(-) diff --git a/magnum_cluster_api/cmd/image_loader.py b/magnum_cluster_api/cmd/image_loader.py index c1702678..4d2116d3 100644 --- a/magnum_cluster_api/cmd/image_loader.py +++ b/magnum_cluster_api/cmd/image_loader.py @@ -177,9 +177,9 @@ def _get_kubeadm_images(version: str): def _get_calico_images(tag="v3.24.2"): return [ - f"docker.io/calico/cni:{tag}", - f"docker.io/calico/kube-controllers:{tag}", - f"docker.io/calico/node:{tag}", + f"quay.io/calico/cni:{tag}", + f"quay.io/calico/kube-controllers:{tag}", + f"quay.io/calico/node:{tag}", ] diff --git a/magnum_cluster_api/image_utils.py b/magnum_cluster_api/image_utils.py index 03cabc26..35952191 100644 --- a/magnum_cluster_api/image_utils.py +++ b/magnum_cluster_api/image_utils.py @@ -57,6 +57,8 @@ def get_image(name: str, repository: str = None): return name new_image_name = name + if name.startswith("quay.io/calico"): + new_image_name = name.replace("quay.io/calico/", f"{repository}/calico/") if name.startswith("docker.io/calico"): new_image_name = name.replace("docker.io/calico/", f"{repository}/calico/") if name.startswith("quay.io/cilium"): diff --git a/magnum_cluster_api/manifests/calico/v3.24.2.yaml b/magnum_cluster_api/manifests/calico/v3.24.2.yaml index b718a097..9f92f1a4 100644 --- a/magnum_cluster_api/manifests/calico/v3.24.2.yaml +++ b/magnum_cluster_api/manifests/calico/v3.24.2.yaml @@ -4390,7 +4390,7 @@ spec: # It can be deleted if this is a fresh installation, or if you have already # upgraded to use calico-ipam. - name: upgrade-ipam - image: docker.io/calico/cni:v3.24.2 + image: quay.io/calico/cni:v3.24.2 imagePullPolicy: IfNotPresent command: ["/opt/cni/bin/calico-ipam", "-upgrade"] envFrom: @@ -4418,7 +4418,7 @@ spec: # This container installs the CNI binaries # and CNI network config file on each node. - name: install-cni - image: docker.io/calico/cni:v3.24.2 + image: quay.io/calico/cni:v3.24.2 imagePullPolicy: IfNotPresent command: ["/opt/cni/bin/install"] envFrom: @@ -4461,7 +4461,7 @@ spec: # i.e. bpf at /sys/fs/bpf and cgroup2 at /run/calico/cgroup. Calico-node initialisation is executed # in best effort fashion, i.e. no failure for errors, to not disrupt pod creation in iptable mode. - name: "mount-bpffs" - image: docker.io/calico/node:v3.24.2 + image: quay.io/calico/node:v3.24.2 imagePullPolicy: IfNotPresent command: ["calico-node", "-init", "-best-effort"] volumeMounts: @@ -4487,7 +4487,7 @@ spec: # container programs network policy and routes on each # host. - name: calico-node - image: docker.io/calico/node:v3.24.2 + image: quay.io/calico/node:v3.24.2 imagePullPolicy: IfNotPresent envFrom: - configMapRef: @@ -4704,7 +4704,7 @@ spec: priorityClassName: system-cluster-critical containers: - name: calico-kube-controllers - image: docker.io/calico/kube-controllers:v3.24.2 + image: quay.io/calico/kube-controllers:v3.24.2 imagePullPolicy: IfNotPresent env: # Choose which controllers to run. diff --git a/magnum_cluster_api/manifests/calico/v3.25.2.yaml b/magnum_cluster_api/manifests/calico/v3.25.2.yaml index 17294809..8b1ec8d7 100644 --- a/magnum_cluster_api/manifests/calico/v3.25.2.yaml +++ b/magnum_cluster_api/manifests/calico/v3.25.2.yaml @@ -4440,7 +4440,7 @@ spec: # It can be deleted if this is a fresh installation, or if you have already # upgraded to use calico-ipam. - name: upgrade-ipam - image: docker.io/calico/cni:v3.25.2 + image: quay.io/calico/cni:v3.25.2 imagePullPolicy: IfNotPresent command: ["/opt/cni/bin/calico-ipam", "-upgrade"] envFrom: @@ -4468,7 +4468,7 @@ spec: # This container installs the CNI binaries # and CNI network config file on each node. - name: install-cni - image: docker.io/calico/cni:v3.25.2 + image: quay.io/calico/cni:v3.25.2 imagePullPolicy: IfNotPresent command: ["/opt/cni/bin/install"] envFrom: @@ -4511,7 +4511,7 @@ spec: # i.e. bpf at /sys/fs/bpf and cgroup2 at /run/calico/cgroup. Calico-node initialisation is executed # in best effort fashion, i.e. no failure for errors, to not disrupt pod creation in iptable mode. - name: "mount-bpffs" - image: docker.io/calico/node:v3.25.2 + image: quay.io/calico/node:v3.25.2 imagePullPolicy: IfNotPresent command: ["calico-node", "-init", "-best-effort"] volumeMounts: @@ -4537,7 +4537,7 @@ spec: # container programs network policy and routes on each # host. - name: calico-node - image: docker.io/calico/node:v3.25.2 + image: quay.io/calico/node:v3.25.2 imagePullPolicy: IfNotPresent envFrom: - configMapRef: @@ -4754,7 +4754,7 @@ spec: priorityClassName: system-cluster-critical containers: - name: calico-kube-controllers - image: docker.io/calico/kube-controllers:v3.25.2 + image: quay.io/calico/kube-controllers:v3.25.2 imagePullPolicy: IfNotPresent env: # Choose which controllers to run. diff --git a/magnum_cluster_api/manifests/calico/v3.26.5.yaml b/magnum_cluster_api/manifests/calico/v3.26.5.yaml index 35034a74..9cbf4293 100644 --- a/magnum_cluster_api/manifests/calico/v3.26.5.yaml +++ b/magnum_cluster_api/manifests/calico/v3.26.5.yaml @@ -4639,7 +4639,7 @@ spec: # It can be deleted if this is a fresh installation, or if you have already # upgraded to use calico-ipam. - name: upgrade-ipam - image: docker.io/calico/cni:v3.26.5 + image: quay.io/calico/cni:v3.26.5 imagePullPolicy: IfNotPresent command: ["/opt/cni/bin/calico-ipam", "-upgrade"] envFrom: @@ -4667,7 +4667,7 @@ spec: # This container installs the CNI binaries # and CNI network config file on each node. - name: install-cni - image: docker.io/calico/cni:v3.26.5 + image: quay.io/calico/cni:v3.26.5 imagePullPolicy: IfNotPresent command: ["/opt/cni/bin/install"] envFrom: @@ -4710,7 +4710,7 @@ spec: # i.e. bpf at /sys/fs/bpf and cgroup2 at /run/calico/cgroup. Calico-node initialisation is executed # in best effort fashion, i.e. no failure for errors, to not disrupt pod creation in iptable mode. - name: "mount-bpffs" - image: docker.io/calico/node:v3.26.5 + image: quay.io/calico/node:v3.26.5 imagePullPolicy: IfNotPresent command: ["calico-node", "-init", "-best-effort"] volumeMounts: @@ -4736,7 +4736,7 @@ spec: # container programs network policy and routes on each # host. - name: calico-node - image: docker.io/calico/node:v3.26.5 + image: quay.io/calico/node:v3.26.5 imagePullPolicy: IfNotPresent envFrom: - configMapRef: @@ -4953,7 +4953,7 @@ spec: priorityClassName: system-cluster-critical containers: - name: calico-kube-controllers - image: docker.io/calico/kube-controllers:v3.26.5 + image: quay.io/calico/kube-controllers:v3.26.5 imagePullPolicy: IfNotPresent env: # Choose which controllers to run. diff --git a/magnum_cluster_api/manifests/calico/v3.27.4.yaml b/magnum_cluster_api/manifests/calico/v3.27.4.yaml index d480a79e..9913a0ea 100644 --- a/magnum_cluster_api/manifests/calico/v3.27.4.yaml +++ b/magnum_cluster_api/manifests/calico/v3.27.4.yaml @@ -4777,7 +4777,7 @@ spec: # It can be deleted if this is a fresh installation, or if you have already # upgraded to use calico-ipam. - name: upgrade-ipam - image: docker.io/calico/cni:v3.27.4 + image: quay.io/calico/cni:v3.27.4 imagePullPolicy: IfNotPresent command: ["/opt/cni/bin/calico-ipam", "-upgrade"] envFrom: @@ -4805,7 +4805,7 @@ spec: # This container installs the CNI binaries # and CNI network config file on each node. - name: install-cni - image: docker.io/calico/cni:v3.27.4 + image: quay.io/calico/cni:v3.27.4 imagePullPolicy: IfNotPresent command: ["/opt/cni/bin/install"] envFrom: @@ -4848,7 +4848,7 @@ spec: # i.e. bpf at /sys/fs/bpf and cgroup2 at /run/calico/cgroup. Calico-node initialisation is executed # in best effort fashion, i.e. no failure for errors, to not disrupt pod creation in iptable mode. - name: "mount-bpffs" - image: docker.io/calico/node:v3.27.4 + image: quay.io/calico/node:v3.27.4 imagePullPolicy: IfNotPresent command: ["calico-node", "-init", "-best-effort"] volumeMounts: @@ -4874,7 +4874,7 @@ spec: # container programs network policy and routes on each # host. - name: calico-node - image: docker.io/calico/node:v3.27.4 + image: quay.io/calico/node:v3.27.4 imagePullPolicy: IfNotPresent envFrom: - configMapRef: @@ -5094,7 +5094,7 @@ spec: priorityClassName: system-cluster-critical containers: - name: calico-kube-controllers - image: docker.io/calico/kube-controllers:v3.27.4 + image: quay.io/calico/kube-controllers:v3.27.4 imagePullPolicy: IfNotPresent env: # Choose which controllers to run. diff --git a/magnum_cluster_api/manifests/calico/v3.28.2.yaml b/magnum_cluster_api/manifests/calico/v3.28.2.yaml index f5ed59ca..c543cc04 100644 --- a/magnum_cluster_api/manifests/calico/v3.28.2.yaml +++ b/magnum_cluster_api/manifests/calico/v3.28.2.yaml @@ -4796,7 +4796,7 @@ spec: # It can be deleted if this is a fresh installation, or if you have already # upgraded to use calico-ipam. - name: upgrade-ipam - image: docker.io/calico/cni:v3.28.2 + image: quay.io/calico/cni:v3.28.2 imagePullPolicy: IfNotPresent command: ["/opt/cni/bin/calico-ipam", "-upgrade"] envFrom: @@ -4824,7 +4824,7 @@ spec: # This container installs the CNI binaries # and CNI network config file on each node. - name: install-cni - image: docker.io/calico/cni:v3.28.2 + image: quay.io/calico/cni:v3.28.2 imagePullPolicy: IfNotPresent command: ["/opt/cni/bin/install"] envFrom: @@ -4867,7 +4867,7 @@ spec: # i.e. bpf at /sys/fs/bpf and cgroup2 at /run/calico/cgroup. Calico-node initialisation is executed # in best effort fashion, i.e. no failure for errors, to not disrupt pod creation in iptable mode. - name: "mount-bpffs" - image: docker.io/calico/node:v3.28.2 + image: quay.io/calico/node:v3.28.2 imagePullPolicy: IfNotPresent command: ["calico-node", "-init", "-best-effort"] volumeMounts: @@ -4893,7 +4893,7 @@ spec: # container programs network policy and routes on each # host. - name: calico-node - image: docker.io/calico/node:v3.28.2 + image: quay.io/calico/node:v3.28.2 imagePullPolicy: IfNotPresent envFrom: - configMapRef: @@ -5113,7 +5113,7 @@ spec: priorityClassName: system-cluster-critical containers: - name: calico-kube-controllers - image: docker.io/calico/kube-controllers:v3.28.2 + image: quay.io/calico/kube-controllers:v3.28.2 imagePullPolicy: IfNotPresent env: # Choose which controllers to run. diff --git a/magnum_cluster_api/manifests/calico/v3.29.0.yaml b/magnum_cluster_api/manifests/calico/v3.29.0.yaml index 81e294d5..8de900f4 100644 --- a/magnum_cluster_api/manifests/calico/v3.29.0.yaml +++ b/magnum_cluster_api/manifests/calico/v3.29.0.yaml @@ -6107,7 +6107,7 @@ spec: # It can be deleted if this is a fresh installation, or if you have already # upgraded to use calico-ipam. - name: upgrade-ipam - image: docker.io/calico/cni:v3.29.0 + image: quay.io/calico/cni:v3.29.0 imagePullPolicy: IfNotPresent command: ["/opt/cni/bin/calico-ipam", "-upgrade"] envFrom: @@ -6135,7 +6135,7 @@ spec: # This container installs the CNI binaries # and CNI network config file on each node. - name: install-cni - image: docker.io/calico/cni:v3.29.0 + image: quay.io/calico/cni:v3.29.0 imagePullPolicy: IfNotPresent command: ["/opt/cni/bin/install"] envFrom: @@ -6178,7 +6178,7 @@ spec: # i.e. bpf at /sys/fs/bpf and cgroup2 at /run/calico/cgroup. Calico-node initialisation is executed # in best effort fashion, i.e. no failure for errors, to not disrupt pod creation in iptable mode. - name: "mount-bpffs" - image: docker.io/calico/node:v3.29.0 + image: quay.io/calico/node:v3.29.0 imagePullPolicy: IfNotPresent command: ["calico-node", "-init", "-best-effort"] volumeMounts: @@ -6204,7 +6204,7 @@ spec: # container programs network policy and routes on each # host. - name: calico-node - image: docker.io/calico/node:v3.29.0 + image: quay.io/calico/node:v3.29.0 imagePullPolicy: IfNotPresent envFrom: - configMapRef: @@ -6427,7 +6427,7 @@ spec: priorityClassName: system-cluster-critical containers: - name: calico-kube-controllers - image: docker.io/calico/kube-controllers:v3.29.0 + image: quay.io/calico/kube-controllers:v3.29.0 imagePullPolicy: IfNotPresent env: # Choose which controllers to run. diff --git a/tools/sync-calico b/tools/sync-calico index 92fd8256..70c8a8fb 100755 --- a/tools/sync-calico +++ b/tools/sync-calico @@ -24,3 +24,7 @@ declare -a VERSIONS=( for i in "${VERSIONS[@]}"; do curl -q https://raw.githubusercontent.com/projectcalico/calico/${i}/manifests/calico.yaml > magnum_cluster_api/manifests/calico/${i}.yaml done + +for i in magnum_cluster_api/manifests/calico/* ; do + sed -i '' -e 's/docker\.io/quay\.io/g' "${i}" +done From 6f90614c2665a75c2863ea542a19c2876f7f0b6e Mon Sep 17 00:00:00 2001 From: Mohammed Naser Date: Fri, 10 Jan 2025 01:24:25 -0500 Subject: [PATCH 2/4] Add unit tests + fix Calico regression --- magnum_cluster_api/image_utils.py | 6 ++-- .../tests/unit/test_image_utils.py | 29 +++++++++++++++++++ 2 files changed, 32 insertions(+), 3 deletions(-) diff --git a/magnum_cluster_api/image_utils.py b/magnum_cluster_api/image_utils.py index 35952191..d4ded9f6 100644 --- a/magnum_cluster_api/image_utils.py +++ b/magnum_cluster_api/image_utils.py @@ -58,11 +58,11 @@ def get_image(name: str, repository: str = None): new_image_name = name if name.startswith("quay.io/calico"): - new_image_name = name.replace("quay.io/calico/", f"{repository}/calico/") + new_image_name = name.replace("quay.io/calico/", f"{repository}/calico-") if name.startswith("docker.io/calico"): - new_image_name = name.replace("docker.io/calico/", f"{repository}/calico/") + new_image_name = name.replace("docker.io/calico/", f"{repository}/calico-") if name.startswith("quay.io/cilium"): - new_image_name = name.replace("quay.io/cilium/", f"{repository}/cilium/") + new_image_name = name.replace("quay.io/cilium/", f"{repository}/cilium-") if name.startswith("docker.io/k8scloudprovider"): new_image_name = name.replace("docker.io/k8scloudprovider", repository) if name.startswith("registry.k8s.io/sig-storage"): diff --git a/magnum_cluster_api/tests/unit/test_image_utils.py b/magnum_cluster_api/tests/unit/test_image_utils.py index 5b37e307..cb8b9e33 100644 --- a/magnum_cluster_api/tests/unit/test_image_utils.py +++ b/magnum_cluster_api/tests/unit/test_image_utils.py @@ -19,6 +19,7 @@ import pkg_resources import pytest import yaml +from oslotest import base from magnum_cluster_api import image_utils @@ -52,3 +53,31 @@ def test_update_manifest_images(glob_path): assert init_container["image"].startswith(repository) for container in doc["spec"]["template"]["spec"]["containers"]: assert container["image"].startswith(repository) + + +class ImageUtilsTestCase(base.BaseTestCase): + """Test cases for image_utils""" + + def test_get_image_without_repository(self): + image_name = "docker.io/calico/cni:v3.24.2" + new_image_name = image_utils.get_image(image_name, None) + + self.assertEqual(image_name, new_image_name) + + def test_get_image_for_calico_with_docker(self): + image_name = "docker.io/calico/cni:v3.24.2" + new_image_name = image_utils.get_image(image_name, "registry.atmosphere.dev") + + self.assertEqual("registry.atmosphere.dev/calico-cni:v3.24.2", new_image_name) + + def test_get_image_for_calico_with_quay(self): + image_name = "quay.io/calico/cni:v3.24.2" + new_image_name = image_utils.get_image(image_name, "registry.atmosphere.dev") + + self.assertEqual("registry.atmosphere.dev/calico-cni:v3.24.2", new_image_name) + + def test_get_image_for_cilium(self): + image_name = "quay.io/cilium/cilium:v1.15.3" + new_image_name = image_utils.get_image(image_name, "registry.atmosphere.dev") + + self.assertEqual("registry.atmosphere.dev/cilium-cilium:v1.15.3", new_image_name) From cb89ac72d252a9ddc60581045d606702d0c4c904 Mon Sep 17 00:00:00 2001 From: Mohammed Naser Date: Fri, 10 Jan 2025 01:31:19 -0500 Subject: [PATCH 3/4] Increase retries on image downloads --- zuul.d/playbooks/sonobuoy/run.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/zuul.d/playbooks/sonobuoy/run.yml b/zuul.d/playbooks/sonobuoy/run.yml index e8321477..0e5089f2 100644 --- a/zuul.d/playbooks/sonobuoy/run.yml +++ b/zuul.d/playbooks/sonobuoy/run.yml @@ -15,8 +15,9 @@ url: "{{ image_url }}" dest: "{{ zuul.project.src_dir }}/{{ image_operating_system }}-kube-{{ kube_tag }}.qcow2" register: fetch_artifact - retries: 5 + retries: 60 delay: 10 + until: fetch_artifact is succeeded when: - image_url is defined From 0b409c48f8806292558f2b993d76d0c36c1c1398 Mon Sep 17 00:00:00 2001 From: Mohammed Naser Date: Fri, 10 Jan 2025 01:38:21 -0500 Subject: [PATCH 4/4] Fix unit test for ensuring server groups --- magnum_cluster_api/tests/unit/test_resources.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/magnum_cluster_api/tests/unit/test_resources.py b/magnum_cluster_api/tests/unit/test_resources.py index a5a5b7b0..31a092ad 100644 --- a/magnum_cluster_api/tests/unit/test_resources.py +++ b/magnum_cluster_api/tests/unit/test_resources.py @@ -62,6 +62,9 @@ def test_generate_machine_deployments_for_cluster_with_deleting_node_group( mock_get_image_uuid = mocker.patch("magnum_cluster_api.utils.get_image_uuid") mock_get_image_uuid.return_value = "foo" + mock_ensure_worker_server_group = mocker.patch("magnum_cluster_api.utils.ensure_worker_server_group") + mock_ensure_worker_server_group.return_value = "bar" + mds = resources.generate_machine_deployments_for_cluster( context, cluster,