diff --git a/docs/book/src/developer/providers/v1.0-to-v1.1.md b/docs/book/src/developer/providers/v1.0-to-v1.1.md index bdcddbec36f7..4b83af787289 100644 --- a/docs/book/src/developer/providers/v1.0-to-v1.1.md +++ b/docs/book/src/developer/providers/v1.0-to-v1.1.md @@ -28,6 +28,10 @@ are kept in sync with the versions used by `sigs.k8s.io/controller-runtime`. * The `third_party/kubernetes-drain` package is deprecated, as we're now using `k8s.io/kubectl/pkg/drain` instead ([PR](https://github.com/kubernetes-sigs/cluster-api/pull/5440)). * `util/version.CompareWithBuildIdentifiers` has been deprecated, please use `util/version.Compare(a, b, WithBuildTags())` instead. +### Removals + +* `KCPUpgradeSpec` has been removed. Please use `ClusterUpgradeConformanceSpec` instead. + ### API Change * Some controllers have been moved to internal to reduce their API surface. We now only diff --git a/test/e2e/cluster_upgrade.go b/test/e2e/cluster_upgrade.go index 34d0586a094d..70ba0194295c 100644 --- a/test/e2e/cluster_upgrade.go +++ b/test/e2e/cluster_upgrade.go @@ -41,6 +41,15 @@ type ClusterUpgradeConformanceSpecInput struct { SkipCleanup bool SkipConformanceTests bool + // ControlPlaneMachineCount is used in `config cluster` to configure the count of the control plane machines used in the test. + // Default is 1. + ControlPlaneMachineCount *int64 + // WorkerMachineCount is used in `config cluster` to configure the count of the worker machines used in the test. + // NOTE: If the WORKER_MACHINE_COUNT var is used multiple times in the cluster template, the absolute count of + // worker machines is a multiple of WorkerMachineCount. + // Default is 2. + WorkerMachineCount *int64 + // Flavor to use when creating the cluster for testing, "upgrades" is used if not specified. Flavor *string } @@ -57,9 +66,13 @@ func ClusterUpgradeConformanceSpec(ctx context.Context, inputGetter func() Clust specName = "k8s-upgrade-and-conformance" ) var ( - input ClusterUpgradeConformanceSpecInput - namespace *corev1.Namespace - cancelWatches context.CancelFunc + input ClusterUpgradeConformanceSpecInput + namespace *corev1.Namespace + cancelWatches context.CancelFunc + + controlPlaneMachineCount int64 + workerMachineCount int64 + clusterResources *clusterctl.ApplyClusterTemplateAndWaitResult kubetestConfigFilePath string ) @@ -81,6 +94,18 @@ func ClusterUpgradeConformanceSpec(ctx context.Context, inputGetter func() Clust kubetestConfigFilePath = input.E2EConfig.GetVariable(kubetestConfigurationVariable) Expect(kubetestConfigFilePath).To(BeAnExistingFile(), "%s should be a valid kubetest config file") + if input.ControlPlaneMachineCount == nil { + controlPlaneMachineCount = 1 + } else { + controlPlaneMachineCount = *input.ControlPlaneMachineCount + } + + if input.WorkerMachineCount == nil { + workerMachineCount = 2 + } else { + workerMachineCount = *input.WorkerMachineCount + } + // Setup a Namespace where to host objects for this spec and create a watcher for the Namespace events. namespace, cancelWatches = setupSpecNamespace(ctx, specName, input.BootstrapClusterProxy, input.ArtifactFolder) clusterResources = new(clusterctl.ApplyClusterTemplateAndWaitResult) @@ -89,11 +114,6 @@ func ClusterUpgradeConformanceSpec(ctx context.Context, inputGetter func() Clust It("Should create and upgrade a workload cluster and run kubetest", func() { By("Creating a workload cluster") - var controlPlaneMachineCount int64 = 1 - // clusterTemplateWorkerMachineCount is used for ConfigCluster, as it is used for MachineDeployments and - // MachinePools - var clusterTemplateWorkerMachineCount int64 = 2 - clusterctl.ApplyClusterTemplateAndWait(ctx, clusterctl.ApplyClusterTemplateAndWaitInput{ ClusterProxy: input.BootstrapClusterProxy, ConfigCluster: clusterctl.ConfigClusterInput{ @@ -106,7 +126,7 @@ func ClusterUpgradeConformanceSpec(ctx context.Context, inputGetter func() Clust ClusterName: fmt.Sprintf("%s-%s", specName, util.RandomString(6)), KubernetesVersion: input.E2EConfig.GetVariable(KubernetesVersionUpgradeFrom), ControlPlaneMachineCount: pointer.Int64Ptr(controlPlaneMachineCount), - WorkerMachineCount: pointer.Int64Ptr(clusterTemplateWorkerMachineCount), + WorkerMachineCount: pointer.Int64Ptr(workerMachineCount), }, WaitForClusterIntervals: input.E2EConfig.GetIntervals(specName, "wait-cluster"), WaitForControlPlaneIntervals: input.E2EConfig.GetIntervals(specName, "wait-control-plane"), diff --git a/test/e2e/cluster_upgrade_test.go b/test/e2e/cluster_upgrade_test.go index 94531ca15d60..c575358586f9 100644 --- a/test/e2e/cluster_upgrade_test.go +++ b/test/e2e/cluster_upgrade_test.go @@ -22,6 +22,7 @@ package e2e import ( . "github.com/onsi/ginkgo" "k8s.io/utils/pointer" + "sigs.k8s.io/cluster-api/test/framework/clusterctl" ) var _ = Describe("When upgrading a workload cluster and testing K8S conformance [Conformance] [K8s-Upgrade]", func() { @@ -51,3 +52,57 @@ var _ = Describe("When upgrading a workload cluster using ClusterClass", func() } }) }) + +var _ = Describe("When upgrading a workload cluster with a single control plane machine", func() { + ClusterUpgradeConformanceSpec(ctx, func() ClusterUpgradeConformanceSpecInput { + return ClusterUpgradeConformanceSpecInput{ + E2EConfig: e2eConfig, + ClusterctlConfigPath: clusterctlConfigPath, + BootstrapClusterProxy: bootstrapClusterProxy, + ArtifactFolder: artifactFolder, + SkipCleanup: skipCleanup, + // This test is run in CI in parallel with other tests. To keep the test duration reasonable + // the conformance tests are skipped. + SkipConformanceTests: true, + ControlPlaneMachineCount: pointer.Int64(1), + WorkerMachineCount: pointer.Int64(1), + Flavor: pointer.String(clusterctl.DefaultFlavor), + } + }) +}) + +var _ = Describe("When upgrading a workload cluster with a HA control plane", func() { + ClusterUpgradeConformanceSpec(ctx, func() ClusterUpgradeConformanceSpecInput { + return ClusterUpgradeConformanceSpecInput{ + E2EConfig: e2eConfig, + ClusterctlConfigPath: clusterctlConfigPath, + BootstrapClusterProxy: bootstrapClusterProxy, + ArtifactFolder: artifactFolder, + SkipCleanup: skipCleanup, + // This test is run in CI in parallel with other tests. To keep the test duration reasonable + // the conformance tests are skipped. + SkipConformanceTests: true, + ControlPlaneMachineCount: pointer.Int64(3), + WorkerMachineCount: pointer.Int64(1), + Flavor: pointer.String(clusterctl.DefaultFlavor), + } + }) +}) + +var _ = Describe("When upgrading a workload cluster with a HA control plane using scale-in rollout", func() { + ClusterUpgradeConformanceSpec(ctx, func() ClusterUpgradeConformanceSpecInput { + return ClusterUpgradeConformanceSpecInput{ + E2EConfig: e2eConfig, + ClusterctlConfigPath: clusterctlConfigPath, + BootstrapClusterProxy: bootstrapClusterProxy, + ArtifactFolder: artifactFolder, + SkipCleanup: skipCleanup, + // This test is run in CI in parallel with other tests. To keep the test duration reasonable + // the conformance tests are skipped. + SkipConformanceTests: true, + ControlPlaneMachineCount: pointer.Int64(3), + WorkerMachineCount: pointer.Int64(1), + Flavor: pointer.String("kcp-scale-in"), + } + }) +}) diff --git a/test/e2e/kcp_upgrade.go b/test/e2e/kcp_upgrade.go deleted file mode 100644 index 68aa5d3e1863..000000000000 --- a/test/e2e/kcp_upgrade.go +++ /dev/null @@ -1,115 +0,0 @@ -/* -Copyright 2020 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package e2e - -import ( - "context" - "fmt" - "os" - "path/filepath" - - . "github.com/onsi/ginkgo" - . "github.com/onsi/gomega" - corev1 "k8s.io/api/core/v1" - "k8s.io/utils/pointer" - "sigs.k8s.io/cluster-api/test/framework" - "sigs.k8s.io/cluster-api/test/framework/clusterctl" - "sigs.k8s.io/cluster-api/util" -) - -// KCPUpgradeSpecInput is the input for KCPUpgradeSpec. -type KCPUpgradeSpecInput struct { - E2EConfig *clusterctl.E2EConfig - ClusterctlConfigPath string - BootstrapClusterProxy framework.ClusterProxy - ArtifactFolder string - SkipCleanup bool - ControlPlaneMachineCount int64 - Flavor string -} - -// KCPUpgradeSpec implements a test that verifies KCP to properly upgrade a control plane. -func KCPUpgradeSpec(ctx context.Context, inputGetter func() KCPUpgradeSpecInput) { - var ( - specName = "kcp-upgrade" - input KCPUpgradeSpecInput - namespace *corev1.Namespace - cancelWatches context.CancelFunc - clusterResources *clusterctl.ApplyClusterTemplateAndWaitResult - ) - - BeforeEach(func() { - Expect(ctx).NotTo(BeNil(), "ctx is required for %s spec", specName) - input = inputGetter() - Expect(input.E2EConfig).ToNot(BeNil(), "Invalid argument. input.E2EConfig can't be nil when calling %s spec", specName) - Expect(input.ClusterctlConfigPath).To(BeAnExistingFile(), "Invalid argument. input.ClusterctlConfigPath must be an existing file when calling %s spec", specName) - Expect(input.BootstrapClusterProxy).ToNot(BeNil(), "Invalid argument. input.BootstrapClusterProxy can't be nil when calling %s spec", specName) - Expect(os.MkdirAll(input.ArtifactFolder, 0750)).To(Succeed(), "Invalid argument. input.ArtifactFolder can't be created for %s spec", specName) - Expect(input.ControlPlaneMachineCount).ToNot(BeZero()) - Expect(input.E2EConfig.Variables).To(HaveKey(KubernetesVersionUpgradeTo)) - Expect(input.E2EConfig.Variables).To(HaveKey(KubernetesVersionUpgradeFrom)) - Expect(input.E2EConfig.Variables).To(HaveKey(EtcdVersionUpgradeTo)) - Expect(input.E2EConfig.Variables).To(HaveKey(CoreDNSVersionUpgradeTo)) - - // Setup a Namespace where to host objects for this spec and create a watcher for the namespace events. - namespace, cancelWatches = setupSpecNamespace(ctx, specName, input.BootstrapClusterProxy, input.ArtifactFolder) - clusterResources = new(clusterctl.ApplyClusterTemplateAndWaitResult) - }) - - It("Should successfully upgrade Kubernetes, DNS, kube-proxy, and etcd", func() { - By("Creating a workload cluster") - clusterctl.ApplyClusterTemplateAndWait(ctx, clusterctl.ApplyClusterTemplateAndWaitInput{ - ClusterProxy: input.BootstrapClusterProxy, - ConfigCluster: clusterctl.ConfigClusterInput{ - LogFolder: filepath.Join(input.ArtifactFolder, "clusters", input.BootstrapClusterProxy.GetName()), - ClusterctlConfigPath: input.ClusterctlConfigPath, - KubeconfigPath: input.BootstrapClusterProxy.GetKubeconfigPath(), - InfrastructureProvider: clusterctl.DefaultInfrastructureProvider, - Flavor: input.Flavor, - Namespace: namespace.Name, - ClusterName: fmt.Sprintf("%s-%s", specName, util.RandomString(6)), - KubernetesVersion: input.E2EConfig.GetVariable(KubernetesVersionUpgradeFrom), - ControlPlaneMachineCount: pointer.Int64Ptr(input.ControlPlaneMachineCount), - WorkerMachineCount: pointer.Int64Ptr(1), - }, - WaitForClusterIntervals: input.E2EConfig.GetIntervals(specName, "wait-cluster"), - WaitForControlPlaneIntervals: input.E2EConfig.GetIntervals(specName, "wait-control-plane"), - WaitForMachineDeployments: input.E2EConfig.GetIntervals(specName, "wait-worker-nodes"), - }, clusterResources) - - By("Upgrading Kubernetes, DNS, kube-proxy, and etcd versions") - framework.UpgradeControlPlaneAndWaitForUpgrade(ctx, framework.UpgradeControlPlaneAndWaitForUpgradeInput{ - ClusterProxy: input.BootstrapClusterProxy, - Cluster: clusterResources.Cluster, - ControlPlane: clusterResources.ControlPlane, - EtcdImageTag: input.E2EConfig.GetVariable(EtcdVersionUpgradeTo), - DNSImageTag: input.E2EConfig.GetVariable(CoreDNSVersionUpgradeTo), - KubernetesUpgradeVersion: input.E2EConfig.GetVariable(KubernetesVersionUpgradeTo), - WaitForMachinesToBeUpgraded: input.E2EConfig.GetIntervals(specName, "wait-machine-upgrade"), - WaitForKubeProxyUpgrade: input.E2EConfig.GetIntervals(specName, "wait-machine-upgrade"), - WaitForDNSUpgrade: input.E2EConfig.GetIntervals(specName, "wait-machine-upgrade"), - WaitForEtcdUpgrade: input.E2EConfig.GetIntervals(specName, "wait-machine-upgrade"), - }) - - By("PASSED!") - }) - - AfterEach(func() { - // Dumps all the resources in the spec namespace, then cleanups the cluster object and the spec namespace itself. - dumpSpecResourcesAndCleanup(ctx, specName, input.BootstrapClusterProxy, input.ArtifactFolder, namespace, cancelWatches, clusterResources.Cluster, input.E2EConfig.GetIntervals, input.SkipCleanup) - }) -} diff --git a/test/e2e/kcp_upgrade_test.go b/test/e2e/kcp_upgrade_test.go deleted file mode 100644 index dd455e9939ca..000000000000 --- a/test/e2e/kcp_upgrade_test.go +++ /dev/null @@ -1,67 +0,0 @@ -//go:build e2e -// +build e2e - -/* -Copyright 2020 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package e2e - -import ( - . "github.com/onsi/ginkgo" - "sigs.k8s.io/cluster-api/test/framework/clusterctl" -) - -var _ = Describe("When testing KCP upgrade in a single control plane cluster", func() { - KCPUpgradeSpec(ctx, func() KCPUpgradeSpecInput { - return KCPUpgradeSpecInput{ - E2EConfig: e2eConfig, - ClusterctlConfigPath: clusterctlConfigPath, - BootstrapClusterProxy: bootstrapClusterProxy, - ArtifactFolder: artifactFolder, - SkipCleanup: skipCleanup, - ControlPlaneMachineCount: 1, - Flavor: clusterctl.DefaultFlavor, - } - }) -}) - -var _ = Describe("When testing KCP upgrade in a HA cluster", func() { - KCPUpgradeSpec(ctx, func() KCPUpgradeSpecInput { - return KCPUpgradeSpecInput{ - E2EConfig: e2eConfig, - ClusterctlConfigPath: clusterctlConfigPath, - BootstrapClusterProxy: bootstrapClusterProxy, - ArtifactFolder: artifactFolder, - SkipCleanup: skipCleanup, - ControlPlaneMachineCount: 3, - Flavor: clusterctl.DefaultFlavor, - } - }) -}) - -var _ = Describe("When testing KCP upgrade in a HA cluster using scale in rollout", func() { - KCPUpgradeSpec(ctx, func() KCPUpgradeSpecInput { - return KCPUpgradeSpecInput{ - E2EConfig: e2eConfig, - ClusterctlConfigPath: clusterctlConfigPath, - BootstrapClusterProxy: bootstrapClusterProxy, - ArtifactFolder: artifactFolder, - SkipCleanup: skipCleanup, - ControlPlaneMachineCount: 3, - Flavor: "kcp-scale-in", - } - }) -})