From b979d9b40c04342185f1bd5dddffe9b636eb98ec Mon Sep 17 00:00:00 2001 From: Mathieu Tortuyaux Date: Wed, 11 Oct 2023 11:30:24 +0200 Subject: [PATCH 1/3] templates: add flatcar-sysext template MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This template allows to use a plain Flatcar image from the Flatcar release server without building with the image-builder. This template will consume Kubernetes systemd-sysext image from the flatcar/sysext-bakery release artifacts. This template will emit a /run/reboot-required file flag when a new Kubernetes release has been downloaded and that we need to coordinate the reboot of the node using Kured for example. Signed-off-by: Mathieu Tortuyaux Co-authored-by: Kai Lüke --- Makefile | 6 +- .../flatcar-sysext/kustomization.yaml | 7 + .../flatcar-sysext/patch-flatcar.yaml | 185 ++++++++++++ .../cluster-template-flatcar-sysext.yaml | 265 ++++++++++++++++++ 4 files changed, 461 insertions(+), 2 deletions(-) create mode 100644 kustomize/v1alpha7/flatcar-sysext/kustomization.yaml create mode 100644 kustomize/v1alpha7/flatcar-sysext/patch-flatcar.yaml create mode 100644 templates/cluster-template-flatcar-sysext.yaml diff --git a/Makefile b/Makefile index 4ecf6aaccf..c1968fe83a 100644 --- a/Makefile +++ b/Makefile @@ -158,7 +158,8 @@ e2e-templates: $(addprefix $(E2E_NO_ARTIFACT_TEMPLATES_DIR)/, \ cluster-template-without-lb.yaml \ cluster-template.yaml \ cluster-template-flatcar.yaml \ - cluster-template-k8s-upgrade.yaml) + cluster-template-k8s-upgrade.yaml \ + cluster-template-flatcar-sysext.yaml) # Currently no templates that require CI artifacts # $(addprefix $(E2E_TEMPLATES_DIR)/, add-templates-here.yaml) \ @@ -406,7 +407,8 @@ release-notes: $(RELEASE_NOTES) ## Generate release notes templates: ## Generate cluster templates templates: templates/cluster-template.yaml \ templates/cluster-template-without-lb.yaml \ - templates/cluster-template-flatcar.yaml + templates/cluster-template-flatcar.yaml \ + templates/cluster-template-flatcar-sysext.yaml templates/cluster-template.yaml: kustomize/v1alpha8/default $(KUSTOMIZE) FORCE $(KUSTOMIZE) build "$<" > "$@" diff --git a/kustomize/v1alpha7/flatcar-sysext/kustomization.yaml b/kustomize/v1alpha7/flatcar-sysext/kustomization.yaml new file mode 100644 index 0000000000..b5cb7540a4 --- /dev/null +++ b/kustomize/v1alpha7/flatcar-sysext/kustomization.yaml @@ -0,0 +1,7 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +resources: +- ../default + +patches: +- path: patch-flatcar.yaml diff --git a/kustomize/v1alpha7/flatcar-sysext/patch-flatcar.yaml b/kustomize/v1alpha7/flatcar-sysext/patch-flatcar.yaml new file mode 100644 index 0000000000..9f8b858c08 --- /dev/null +++ b/kustomize/v1alpha7/flatcar-sysext/patch-flatcar.yaml @@ -0,0 +1,185 @@ +--- +apiVersion: infrastructure.cluster.x-k8s.io/v1alpha7 +kind: OpenStackCluster +metadata: + name: ${CLUSTER_NAME} +spec: + apiServerLoadBalancer: + $patch: delete +--- +apiVersion: controlplane.cluster.x-k8s.io/v1beta1 +kind: KubeadmControlPlane +metadata: + name: "${CLUSTER_NAME}-control-plane" +spec: + replicas: ${CONTROL_PLANE_MACHINE_COUNT} + kubeadmConfigSpec: + joinConfiguration: + nodeRegistration: + name: $${COREOS_OPENSTACK_HOSTNAME} + kubeletExtraArgs: + provider-id: null + initConfiguration: + nodeRegistration: + name: $${COREOS_OPENSTACK_HOSTNAME} + kubeletExtraArgs: + # Fixme(lentzi90): This is here just to override the value set in the default + # kustomization. It will be replaced with a value that works for flatcar in + # https://github.com/kubernetes-sigs/cluster-api-provider-openstack/pull/1564 + provider-id: null + format: ignition + ignition: + containerLinuxConfig: + additionalConfig: | + storage: + links: + - path: /etc/extensions/kubernetes.raw + hard: false + target: /opt/extensions/kubernetes/kubernetes-${KUBERNETES_VERSION}-x86-64.raw + files: + - path: /etc/sysupdate.kubernetes.d/kubernetes-${KUBERNETES_VERSION%.*}.conf + mode: 0644 + contents: + remote: + url: https://github.com/flatcar/sysext-bakery/releases/download/latest/kubernetes-${KUBERNETES_VERSION%.*}.conf + - path: /etc/sysupdate.d/noop.conf + mode: 0644 + contents: + remote: + url: https://github.com/flatcar/sysext-bakery/releases/download/latest/noop.conf + - path: /opt/extensions/kubernetes/kubernetes-${KUBERNETES_VERSION}-x86-64.raw + contents: + remote: + url: https://github.com/flatcar/sysext-bakery/releases/download/latest/kubernetes-${KUBERNETES_VERSION}-x86-64.raw + systemd: + units: + - name: systemd-sysupdate.service + dropins: + - name: kubernetes.conf + contents: | + [Service] + ExecStartPre=/usr/bin/sh -c "readlink --canonicalize /etc/extensions/kubernetes.raw > /tmp/kubernetes" + ExecStartPre=/usr/lib/systemd/systemd-sysupdate -C kubernetes update + ExecStartPost=/usr/bin/sh -c "readlink --canonicalize /etc/extensions/kubernetes.raw > /tmp/kubernetes-new" + ExecStartPost=/usr/bin/sh -c "if ! cmp --silent /tmp/kubernetes /tmp/kubernetes-new; then touch /run/reboot-required; fi" + - name: update-engine.service + # Set this to 'false' if you want to enable Flatcar auto-update + mask: ${FLATCAR_DISABLE_AUTO_UPDATE:=true} + - name: locksmithd.service + # NOTE: To coordinate the node reboot in this context, we recommend to use Kured. + mask: true + - name: systemd-sysupdate.timer + # Set this to 'true' if you want to enable the Kubernetes auto-update. + # NOTE: Only patches version will be pulled. + enabled: false + - name: coreos-metadata-sshkeys@.service + enabled: true + - name: kubeadm.service + enabled: true + dropins: + - name: 10-flatcar.conf + contents: | + [Unit] + Requires=containerd.service coreos-metadata.service + After=containerd.service coreos-metadata.service + [Service] + EnvironmentFile=/run/metadata/flatcar + preKubeadmCommands: + - export COREOS_OPENSTACK_HOSTNAME=$${COREOS_OPENSTACK_HOSTNAME%.*} + - envsubst < /etc/kubeadm.yml > /etc/kubeadm.yml.tmp + - mv /etc/kubeadm.yml.tmp /etc/kubeadm.yml +--- +apiVersion: bootstrap.cluster.x-k8s.io/v1beta1 +kind: KubeadmConfigTemplate +metadata: + name: ${CLUSTER_NAME}-md-0 +spec: + template: + spec: + joinConfiguration: + nodeRegistration: + name: $${COREOS_OPENSTACK_HOSTNAME} + kubeletExtraArgs: + # Fixme(lentzi90): This is here just to override the value set in the default + # kustomization. It will be replaced with a value that works for flatcar in + # https://github.com/kubernetes-sigs/cluster-api-provider-openstack/pull/1564 + provider-id: null + preKubeadmCommands: + - export COREOS_OPENSTACK_HOSTNAME=$${COREOS_OPENSTACK_HOSTNAME%.*} + - envsubst < /etc/kubeadm.yml > /etc/kubeadm.yml.tmp + - mv /etc/kubeadm.yml.tmp /etc/kubeadm.yml + format: ignition + ignition: + containerLinuxConfig: + additionalConfig: | + storage: + links: + - path: /etc/extensions/kubernetes.raw + hard: false + target: /opt/extensions/kubernetes/kubernetes-${KUBERNETES_VERSION}-x86-64.raw + files: + - path: /etc/sysupdate.kubernetes.d/kubernetes-${KUBERNETES_VERSION%.*}.conf + mode: 0644 + contents: + remote: + url: https://github.com/flatcar/sysext-bakery/releases/download/latest/kubernetes-${KUBERNETES_VERSION%.*}.conf + - path: /etc/sysupdate.d/noop.conf + mode: 0644 + contents: + remote: + url: https://github.com/flatcar/sysext-bakery/releases/download/latest/noop.conf + - path: /opt/extensions/kubernetes/kubernetes-${KUBERNETES_VERSION}-x86-64.raw + contents: + remote: + url: https://github.com/flatcar/sysext-bakery/releases/download/latest/kubernetes-${KUBERNETES_VERSION}-x86-64.raw + systemd: + units: + - name: systemd-sysupdate.service + dropins: + - name: kubernetes.conf + contents: | + [Service] + ExecStartPre=/usr/bin/sh -c "readlink --canonicalize /etc/extensions/kubernetes.raw > /tmp/kubernetes" + ExecStartPre=/usr/lib/systemd/systemd-sysupdate -C kubernetes update + ExecStartPost=/usr/bin/sh -c "readlink --canonicalize /etc/extensions/kubernetes.raw > /tmp/kubernetes-new" + ExecStartPost=/usr/bin/sh -c "if ! cmp --silent /tmp/kubernetes /tmp/kubernetes-new; then touch /run/reboot-required; fi" + - name: update-engine.service + # Set this to 'false' if you want to enable Flatcar auto-update + mask: ${FLATCAR_DISABLE_AUTO_UPDATE:=true} + - name: locksmithd.service + # NOTE: To coordinate the node reboot in this context, we recommend to use Kured. + mask: true + - name: systemd-sysupdate.timer + # Set this to 'true' if you want to enable the Kubernetes auto-update. + # NOTE: Only patches version will be pulled. + enabled: false + - name: coreos-metadata-sshkeys@.service + enabled: true + - name: kubeadm.service + enabled: true + dropins: + - name: 10-flatcar.conf + contents: | + [Unit] + Requires=containerd.service coreos-metadata.service + After=containerd.service coreos-metadata.service + [Service] + EnvironmentFile=/run/metadata/flatcar +--- +apiVersion: infrastructure.cluster.x-k8s.io/v1alpha7 +kind: OpenStackMachineTemplate +metadata: + name: ${CLUSTER_NAME}-md-0 +spec: + template: + spec: + image: ${FLATCAR_IMAGE_NAME} +--- +apiVersion: infrastructure.cluster.x-k8s.io/v1alpha7 +kind: OpenStackMachineTemplate +metadata: + name: ${CLUSTER_NAME}-control-plane +spec: + template: + spec: + image: ${FLATCAR_IMAGE_NAME} diff --git a/templates/cluster-template-flatcar-sysext.yaml b/templates/cluster-template-flatcar-sysext.yaml new file mode 100644 index 0000000000..fa0ef1bf02 --- /dev/null +++ b/templates/cluster-template-flatcar-sysext.yaml @@ -0,0 +1,265 @@ +apiVersion: v1 +data: + cacert: ${OPENSTACK_CLOUD_CACERT_B64} + clouds.yaml: ${OPENSTACK_CLOUD_YAML_B64} +kind: Secret +metadata: + labels: + clusterctl.cluster.x-k8s.io/move: "true" + name: ${CLUSTER_NAME}-cloud-config +--- +apiVersion: bootstrap.cluster.x-k8s.io/v1beta1 +kind: KubeadmConfigTemplate +metadata: + name: ${CLUSTER_NAME}-md-0 +spec: + template: + spec: + files: [] + format: ignition + ignition: + containerLinuxConfig: + additionalConfig: | + storage: + links: + - path: /etc/extensions/kubernetes.raw + hard: false + target: /opt/extensions/kubernetes/kubernetes-${KUBERNETES_VERSION}-x86-64.raw + files: + - path: /etc/sysupdate.kubernetes.d/kubernetes-${KUBERNETES_VERSION%.*}.conf + mode: 0644 + contents: + remote: + url: https://github.com/flatcar/sysext-bakery/releases/download/latest/kubernetes-${KUBERNETES_VERSION%.*}.conf + - path: /etc/sysupdate.d/noop.conf + mode: 0644 + contents: + remote: + url: https://github.com/flatcar/sysext-bakery/releases/download/latest/noop.conf + - path: /opt/extensions/kubernetes/kubernetes-${KUBERNETES_VERSION}-x86-64.raw + contents: + remote: + url: https://github.com/flatcar/sysext-bakery/releases/download/latest/kubernetes-${KUBERNETES_VERSION}-x86-64.raw + systemd: + units: + - name: systemd-sysupdate.service + dropins: + - name: kubernetes.conf + contents: | + [Service] + ExecStartPre=/usr/bin/sh -c "readlink --canonicalize /etc/extensions/kubernetes.raw > /tmp/kubernetes" + ExecStartPre=/usr/lib/systemd/systemd-sysupdate -C kubernetes update + ExecStartPost=/usr/bin/sh -c "readlink --canonicalize /etc/extensions/kubernetes.raw > /tmp/kubernetes-new" + ExecStartPost=/usr/bin/sh -c "if ! cmp --silent /tmp/kubernetes /tmp/kubernetes-new; then touch /run/reboot-required; fi" + - name: update-engine.service + # Set this to 'false' if you want to enable Flatcar auto-update + mask: ${FLATCAR_DISABLE_AUTO_UPDATE:=true} + - name: locksmithd.service + # NOTE: To coordinate the node reboot in this context, we recommend to use Kured. + mask: true + - name: systemd-sysupdate.timer + # Set this to 'true' if you want to enable the Kubernetes auto-update. + # NOTE: Only patches version will be pulled. + enabled: false + - name: coreos-metadata-sshkeys@.service + enabled: true + - name: kubeadm.service + enabled: true + dropins: + - name: 10-flatcar.conf + contents: | + [Unit] + Requires=containerd.service coreos-metadata.service + After=containerd.service coreos-metadata.service + [Service] + EnvironmentFile=/run/metadata/flatcar + joinConfiguration: + nodeRegistration: + kubeletExtraArgs: + cloud-provider: external + name: $${COREOS_OPENSTACK_HOSTNAME} + preKubeadmCommands: + - export COREOS_OPENSTACK_HOSTNAME=$${COREOS_OPENSTACK_HOSTNAME%.*} + - envsubst < /etc/kubeadm.yml > /etc/kubeadm.yml.tmp + - mv /etc/kubeadm.yml.tmp /etc/kubeadm.yml +--- +apiVersion: cluster.x-k8s.io/v1beta1 +kind: Cluster +metadata: + name: ${CLUSTER_NAME} +spec: + clusterNetwork: + pods: + cidrBlocks: + - 192.168.0.0/16 + serviceDomain: cluster.local + controlPlaneRef: + apiVersion: controlplane.cluster.x-k8s.io/v1beta1 + kind: KubeadmControlPlane + name: ${CLUSTER_NAME}-control-plane + infrastructureRef: + apiVersion: infrastructure.cluster.x-k8s.io/v1alpha7 + kind: OpenStackCluster + name: ${CLUSTER_NAME} +--- +apiVersion: cluster.x-k8s.io/v1beta1 +kind: MachineDeployment +metadata: + name: ${CLUSTER_NAME}-md-0 +spec: + clusterName: ${CLUSTER_NAME} + replicas: ${WORKER_MACHINE_COUNT} + selector: + matchLabels: null + template: + spec: + bootstrap: + configRef: + apiVersion: bootstrap.cluster.x-k8s.io/v1beta1 + kind: KubeadmConfigTemplate + name: ${CLUSTER_NAME}-md-0 + clusterName: ${CLUSTER_NAME} + failureDomain: ${OPENSTACK_FAILURE_DOMAIN} + infrastructureRef: + apiVersion: infrastructure.cluster.x-k8s.io/v1alpha7 + kind: OpenStackMachineTemplate + name: ${CLUSTER_NAME}-md-0 + version: ${KUBERNETES_VERSION} +--- +apiVersion: controlplane.cluster.x-k8s.io/v1beta1 +kind: KubeadmControlPlane +metadata: + name: ${CLUSTER_NAME}-control-plane +spec: + kubeadmConfigSpec: + clusterConfiguration: + apiServer: + extraArgs: + cloud-provider: external + controllerManager: + extraArgs: + cloud-provider: external + files: [] + format: ignition + ignition: + containerLinuxConfig: + additionalConfig: | + storage: + links: + - path: /etc/extensions/kubernetes.raw + hard: false + target: /opt/extensions/kubernetes/kubernetes-${KUBERNETES_VERSION}-x86-64.raw + files: + - path: /etc/sysupdate.kubernetes.d/kubernetes-${KUBERNETES_VERSION%.*}.conf + mode: 0644 + contents: + remote: + url: https://github.com/flatcar/sysext-bakery/releases/download/latest/kubernetes-${KUBERNETES_VERSION%.*}.conf + - path: /etc/sysupdate.d/noop.conf + mode: 0644 + contents: + remote: + url: https://github.com/flatcar/sysext-bakery/releases/download/latest/noop.conf + - path: /opt/extensions/kubernetes/kubernetes-${KUBERNETES_VERSION}-x86-64.raw + contents: + remote: + url: https://github.com/flatcar/sysext-bakery/releases/download/latest/kubernetes-${KUBERNETES_VERSION}-x86-64.raw + systemd: + units: + - name: systemd-sysupdate.service + dropins: + - name: kubernetes.conf + contents: | + [Service] + ExecStartPre=/usr/bin/sh -c "readlink --canonicalize /etc/extensions/kubernetes.raw > /tmp/kubernetes" + ExecStartPre=/usr/lib/systemd/systemd-sysupdate -C kubernetes update + ExecStartPost=/usr/bin/sh -c "readlink --canonicalize /etc/extensions/kubernetes.raw > /tmp/kubernetes-new" + ExecStartPost=/usr/bin/sh -c "if ! cmp --silent /tmp/kubernetes /tmp/kubernetes-new; then touch /run/reboot-required; fi" + - name: update-engine.service + # Set this to 'false' if you want to enable Flatcar auto-update + mask: ${FLATCAR_DISABLE_AUTO_UPDATE:=true} + - name: locksmithd.service + # NOTE: To coordinate the node reboot in this context, we recommend to use Kured. + mask: true + - name: systemd-sysupdate.timer + # Set this to 'true' if you want to enable the Kubernetes auto-update. + # NOTE: Only patches version will be pulled. + enabled: false + - name: coreos-metadata-sshkeys@.service + enabled: true + - name: kubeadm.service + enabled: true + dropins: + - name: 10-flatcar.conf + contents: | + [Unit] + Requires=containerd.service coreos-metadata.service + After=containerd.service coreos-metadata.service + [Service] + EnvironmentFile=/run/metadata/flatcar + initConfiguration: + nodeRegistration: + kubeletExtraArgs: + cloud-provider: external + name: $${COREOS_OPENSTACK_HOSTNAME} + joinConfiguration: + nodeRegistration: + kubeletExtraArgs: + cloud-provider: external + name: $${COREOS_OPENSTACK_HOSTNAME} + preKubeadmCommands: + - export COREOS_OPENSTACK_HOSTNAME=$${COREOS_OPENSTACK_HOSTNAME%.*} + - envsubst < /etc/kubeadm.yml > /etc/kubeadm.yml.tmp + - mv /etc/kubeadm.yml.tmp /etc/kubeadm.yml + machineTemplate: + infrastructureRef: + apiVersion: infrastructure.cluster.x-k8s.io/v1alpha7 + kind: OpenStackMachineTemplate + name: ${CLUSTER_NAME}-control-plane + replicas: ${CONTROL_PLANE_MACHINE_COUNT} + version: ${KUBERNETES_VERSION} +--- +apiVersion: infrastructure.cluster.x-k8s.io/v1alpha7 +kind: OpenStackCluster +metadata: + name: ${CLUSTER_NAME} +spec: + cloudName: ${OPENSTACK_CLOUD} + dnsNameservers: + - ${OPENSTACK_DNS_NAMESERVERS} + externalNetworkId: ${OPENSTACK_EXTERNAL_NETWORK_ID} + identityRef: + kind: Secret + name: ${CLUSTER_NAME}-cloud-config + managedSecurityGroups: true + nodeCidr: 10.6.0.0/24 +--- +apiVersion: infrastructure.cluster.x-k8s.io/v1alpha7 +kind: OpenStackMachineTemplate +metadata: + name: ${CLUSTER_NAME}-control-plane +spec: + template: + spec: + cloudName: ${OPENSTACK_CLOUD} + flavor: ${OPENSTACK_CONTROL_PLANE_MACHINE_FLAVOR} + identityRef: + kind: Secret + name: ${CLUSTER_NAME}-cloud-config + image: ${FLATCAR_IMAGE_NAME} + sshKeyName: ${OPENSTACK_SSH_KEY_NAME} +--- +apiVersion: infrastructure.cluster.x-k8s.io/v1alpha7 +kind: OpenStackMachineTemplate +metadata: + name: ${CLUSTER_NAME}-md-0 +spec: + template: + spec: + cloudName: ${OPENSTACK_CLOUD} + flavor: ${OPENSTACK_NODE_MACHINE_FLAVOR} + identityRef: + kind: Secret + name: ${CLUSTER_NAME}-cloud-config + image: ${FLATCAR_IMAGE_NAME} + sshKeyName: ${OPENSTACK_SSH_KEY_NAME} From d85b5fc973ac089e05573135b07dc7cf4bf82a0e Mon Sep 17 00:00:00 2001 From: Mathieu Tortuyaux Date: Wed, 11 Oct 2023 11:37:02 +0200 Subject: [PATCH 2/3] test: add flatcar-sysext test This test consumes the flatcar-sysext template to deploy a workload cluster. It uses a plain Flatcar OpenStack image from the release servers as it does not need to rely on the image-builder built image. Signed-off-by: Mathieu Tortuyaux --- hack/ci/cloud-init/controller.yaml.tpl | 3 +- test/e2e/data/e2e_conf.yaml | 3 ++ .../flatcar-sysext/kustomization.yaml | 7 +++ test/e2e/shared/defaults.go | 1 + test/e2e/suites/e2e/e2e_test.go | 45 +++++++++++++++++++ 5 files changed, 58 insertions(+), 1 deletion(-) create mode 100644 test/e2e/data/kustomize/flatcar-sysext/kustomization.yaml diff --git a/hack/ci/cloud-init/controller.yaml.tpl b/hack/ci/cloud-init/controller.yaml.tpl index f83ddda1b6..4b872d512b 100644 --- a/hack/ci/cloud-init/controller.yaml.tpl +++ b/hack/ci/cloud-init/controller.yaml.tpl @@ -64,7 +64,8 @@ IMAGE_URLS+="https://storage.googleapis.com/artifacts.k8s-staging-capi-openstack.appspot.com/test/cirros/2022-12-05/cirros-0.6.1-x86_64-disk.img," IMAGE_URLS+="https://storage.googleapis.com/artifacts.k8s-staging-capi-openstack.appspot.com/test/ubuntu/2023-09-29/ubuntu-2204-kube-v1.27.2.img," IMAGE_URLS+="https://storage.googleapis.com/artifacts.k8s-staging-capi-openstack.appspot.com/test/ubuntu/2023-09-29/ubuntu-2204-kube-v1.28.2.img," - IMAGE_URLS+="https://storage.googleapis.com/artifacts.k8s-staging-capi-openstack.appspot.com/test/flatcar/flatcar-stable-3602.2.0-kube-v1.28.2.img" + IMAGE_URLS+="https://storage.googleapis.com/artifacts.k8s-staging-capi-openstack.appspot.com/test/flatcar/flatcar-stable-3602.2.0-kube-v1.28.2.img," + IMAGE_URLS+="https://stable.release.flatcar-linux.net/amd64-usr/current/flatcar_production_openstack_image.img" [[post-config|$NOVA_CONF]] [DEFAULT] diff --git a/test/e2e/data/e2e_conf.yaml b/test/e2e/data/e2e_conf.yaml index f17655b66f..c4c54f9956 100644 --- a/test/e2e/data/e2e_conf.yaml +++ b/test/e2e/data/e2e_conf.yaml @@ -211,7 +211,10 @@ variables: # The default user for SSH connections from bastion to machines SSH_USER_MACHINE: "ubuntu" EXP_KUBEADM_BOOTSTRAP_FORMAT_IGNITION: "true" + # The Flatcar image produced by the image-builder OPENSTACK_FLATCAR_IMAGE_NAME: "flatcar-stable-3602.2.0-kube-v1.28.2" + # A plain Flatcar from the Flatcar releases server + FLATCAR_IMAGE_NAME: "flatcar_production_openstack_image" intervals: conformance/wait-control-plane: ["30m", "10s"] diff --git a/test/e2e/data/kustomize/flatcar-sysext/kustomization.yaml b/test/e2e/data/kustomize/flatcar-sysext/kustomization.yaml new file mode 100644 index 0000000000..f8d0816c32 --- /dev/null +++ b/test/e2e/data/kustomize/flatcar-sysext/kustomization.yaml @@ -0,0 +1,7 @@ +--- +resources: +- ../../../../../kustomize/v1alpha7/flatcar-sysext + +components: +- ../common-patches/cni +- ../common-patches/ccm diff --git a/test/e2e/shared/defaults.go b/test/e2e/shared/defaults.go index 9c82de4f04..4ab3e9a7db 100644 --- a/test/e2e/shared/defaults.go +++ b/test/e2e/shared/defaults.go @@ -55,6 +55,7 @@ const ( FlavorKCPRemediation = "kcp-remediation" FlavorFlatcar = "flatcar" FlavorKubernetesUpgrade = "k8s-upgrade" + FlavorFlatcarSysext = "flatcar-sysext" ) // DefaultScheme returns the default scheme to use for testing. diff --git a/test/e2e/suites/e2e/e2e_test.go b/test/e2e/suites/e2e/e2e_test.go index 90930a9fd3..fa27ec588c 100644 --- a/test/e2e/suites/e2e/e2e_test.go +++ b/test/e2e/suites/e2e/e2e_test.go @@ -205,6 +205,51 @@ var _ = Describe("e2e tests [PR-Blocking]", func() { }) }) + Describe("Workload cluster (flatcar-sysext)", func() { + It("should be creatable and deletable", func() { + // Flatcar default user is "core" + shared.SetEnvVar(shared.SSHUserMachine, "core", false) + + shared.Logf("Creating a cluster") + clusterName := fmt.Sprintf("cluster-%s", namespace.Name) + configCluster := defaultConfigCluster(clusterName, namespace.Name) + configCluster.ControlPlaneMachineCount = pointer.Int64(3) + configCluster.WorkerMachineCount = pointer.Int64(1) + configCluster.Flavor = shared.FlavorFlatcarSysext + createCluster(ctx, configCluster, clusterResources) + md := clusterResources.MachineDeployments + + workerMachines := framework.GetMachinesByMachineDeployments(ctx, framework.GetMachinesByMachineDeploymentsInput{ + Lister: e2eCtx.Environment.BootstrapClusterProxy.GetClient(), + ClusterName: clusterName, + Namespace: namespace.Name, + MachineDeployment: *md[0], + }) + controlPlaneMachines := framework.GetControlPlaneMachinesByCluster(ctx, framework.GetControlPlaneMachinesByClusterInput{ + Lister: e2eCtx.Environment.BootstrapClusterProxy.GetClient(), + ClusterName: clusterName, + Namespace: namespace.Name, + }) + Expect(workerMachines).To(HaveLen(1)) + Expect(controlPlaneMachines).To(HaveLen(3)) + + shared.Logf("Waiting for worker nodes to be in Running phase") + statusChecks := []framework.MachineStatusCheck{framework.MachinePhaseCheck(string(clusterv1.MachinePhaseRunning))} + machineStatusInput := framework.WaitForMachineStatusCheckInput{ + Getter: e2eCtx.Environment.BootstrapClusterProxy.GetClient(), + Machine: &workerMachines[0], + StatusChecks: statusChecks, + } + framework.WaitForMachineStatusCheck(ctx, machineStatusInput, e2eCtx.E2EConfig.GetIntervals(specName, "wait-machine-status")...) + + workloadCluster := e2eCtx.Environment.BootstrapClusterProxy.GetWorkloadCluster(ctx, namespace.Name, clusterName) + + waitForDaemonSetRunning(ctx, workloadCluster.GetClient(), "kube-system", "openstack-cloud-controller-manager") + + waitForNodesReadyWithoutCCMTaint(ctx, workloadCluster.GetClient(), 4) + }) + }) + Describe("Workload cluster (without lb)", func() { It("Should create port(s) with custom options", func() { shared.Logf("Creating a cluster") From 5a39a20eb377425cb238eb63ded29f9e5847c25b Mon Sep 17 00:00:00 2001 From: Mathieu Tortuyaux Date: Thu, 12 Oct 2023 09:21:53 +0200 Subject: [PATCH 3/3] doc: mention Flatcar sysext template MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Mathieu Tortuyaux Co-authored-by: Kai Lüke --- .../src/clusteropenstack/configuration.md | 38 +++++++++++++++++-- 1 file changed, 34 insertions(+), 4 deletions(-) diff --git a/docs/book/src/clusteropenstack/configuration.md b/docs/book/src/clusteropenstack/configuration.md index a767139134..7802107f02 100644 --- a/docs/book/src/clusteropenstack/configuration.md +++ b/docs/book/src/clusteropenstack/configuration.md @@ -74,10 +74,33 @@ The image can be referenced by exposing it as an environment variable `OPENSTACK Some OS like [Fedora CoreOS](https://getfedora.org/en/coreos) or [Flatcar](https://www.flatcar.org/) do not use cloud-init but [Ignition](https://coreos.github.io/ignition/) to provision the instance. You need to enable the [Ignition experimental feature](https://cluster-api.sigs.k8s.io/tasks/experimental-features/ignition.html): `export EXP_KUBEADM_BOOTSTRAP_FORMAT_IGNITION=true` -To use Flatcar image: -* Build the image with the [image-builder](https://image-builder.sigs.k8s.io/capi/providers/openstack.html): `make OEM_ID=openstack build-qemu-flatcar` -* Export the name of the uploaded image: `export OPENSTACK_FLATCAR_IMAGE_NAME=flatcar-stable-3374.2.5-kube-v1.25.6` -* When generating the cluster configuration, use the following Cluster API [flavor](https://cluster-api.sigs.k8s.io/clusterctl/commands/generate-cluster.html?#flavors): `--flavor flatcar` (_NOTE_: Don't forget to refer to the [external-cloud-provider](https://cluster-api-openstack.sigs.k8s.io/topics/external-cloud-provider.html) section) +Flatcar comes in two [flavor][flavor] variants: +* `flatcar` + + This variant relies on a Flatcar image built using the image-builder project: the Kubernetes version is bound to the Flatcar version and a rebuild of the image is required for each Kubernetes or Flatcar upgrade. + + To build and use Flatcar image: + * Build the image with the [image-builder][image-builder]: `make OEM_ID=openstack build-qemu-flatcar` + * Upload the image + * Export the name of the uploaded image: `export OPENSTACK_FLATCAR_IMAGE_NAME=flatcar-stable-3374.2.5-kube-v1.25.6` + * When generating the cluster configuration, use the following Cluster API [flavor][flavor]: `--flavor flatcar` (_NOTE_: Don't forget to refer to the [external-cloud-provider][external-cloud-provider] section) + +* `flatcar-sysext` + + This variant relies on a plain Flatcar image and it leverages [systemd-sysext][systemd-sysext] feature to install and update Kubernetes components: the Kubernetes version is not bound to the Flatcar version (i.e Flatcar can be independently upgraded from Kubernetes and vice versa). + + The template comes with a [systemd-sysupdate][systemd-sysupdate] configuration file that will download each new patch version of Kubernetes (i.e if you start with Kubernetes 1.x.y, systemd-sysupdate will automatically pull 1.x.y+1 but not 1.x+1.y), please note that this behavior is disabled by default. To enable the Kubernetes auto-update you can: + * Update the template to enable the `systemd-sysupdate.timer` + * Or run the following command on the nodes: `sudo systemctl enable --now systemd-sysupdate.timer` + + When the Kubernetes release reaches end-of-life it will not receive updates anymore. To switch to a new major version, do a `sudo rm /etc/sysupdate.kubernetes.d/kubernetes-*.conf` and download the new update config into the folder with `cd /etc/sysupdate.kubernetes.d && sudo wget https://github.com/flatcar/sysext-bakery/releases/download/latest/kubernetes-${KUBERNETES_VERSION%.*}.conf`. + + To coordinate the node reboot, we recommend to use [Kured][kured]. Note that running `kubeadm upgrade apply` on the first controller and `kubeadm upgrade node` on all other nodes is not automated (yet), see the [docs](https://kubernetes.io/docs/tasks/administer-cluster/kubeadm/kubeadm-upgrade/). + + To use Flatcar image: + * Upload an image on OpenStack from the Flatcar release servers (e.g for Stable, you might use this image: https://stable.release.flatcar-linux.net/amd64-usr/current/flatcar_production_openstack_image.img) + * Export the name of the uploaded image: `export FLATCAR_IMAGE_NAME=flatcar_production_openstack_image` + * When generating the cluster configuration, use the following Cluster API [flavor][flavor]: `--flavor flatcar-sysext` (_NOTE_: Don't forget to refer to the [external-cloud-provider][external-cloud-provider] section) ## SSH key pair @@ -640,3 +663,10 @@ $ kubectl get openstackcluster NAME CLUSTER READY NETWORK SUBNET BASTION nonha nonha true 2e2a2fad-28c0-4159-8898-c0a2241a86a7 53cb77ab-86a6-4f2c-8d87-24f8411f15de 10.0.0.213 ``` + +[external-cloud-provider]: https://cluster-api-openstack.sigs.k8s.io/topics/external-cloud-provider.html +[flavor]: https://cluster-api.sigs.k8s.io/clusterctl/commands/generate-cluster.html?#flavors +[image-builder]: https://image-builder.sigs.k8s.io/capi/providers/openstack.html +[kured]: https://github.com/kubereboot/kured +[systemd-sysext]: https://www.flatcar.org/docs/latest/provisioning/sysext/ +[systemd-sysupdate]: https://www.freedesktop.org/software/systemd/man/latest/sysupdate.d.html