Merge pull request #2236 from Nordix/clusterctl-e2e

✨ clusterctl e2e tests

cmd/clusterctl/test/README.md

@@ -0,0 +1,26 @@
+# Running the tests
+
+NOTE: the e2e tests may not work on a Mac; they should however work just fine on any Linux distro. Mac support will be added in a follow-up PR.
+
+Currently, overrides are needed for the cluster-api, kubeadm-bootstrap and kubeadm-control-plane providers. The override for the infra provider docker must be removed as it is generated locally by the e2e test script:
+
+	cmd/clusterctl/hack/local-overrides.py
+	rm -rf $HOME/.cluster-api/overrides/docker
+
+The entire test suite can be run using the script:
+
+	./run-e2e.sh
+
+To run specific tests, use the `GINKGO_FOCUS`
+
+	GINKGO_FOCUS="clusterctl create cluster" ./run-e2e.sh
+
+## Skip local build of CAPD
+
+By default, the a local capd image will be built and loaded into kind. This can be skipped as so:
+
+	SKIP_DOCKER_BUILD=1 ./run-e2e.sh
+
+You can also specifiy a pre-build image and skip the build:
+
+	SKIP_DOCKER_BUILD=1 MANAGER_IMAGE=gcr.io/my-project-name/docker-provider-manager-amd64:dev ./run-e2e.sh

cmd/clusterctl/test/e2e/config_cluster_test.go

@@ -0,0 +1,97 @@
+// +build e2e
+
+/*
+Copyright 2020 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package e2e
+
+import (
+	"fmt"
+	"os"
+	"time"
+
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+	"github.com/pkg/errors"
+
+	v1 "k8s.io/api/core/v1"
+	"sigs.k8s.io/kind/pkg/cluster"
+)
+
+var _ = Describe("clusterctl create cluster", func() {
+	var (
+		mgmtInfo     testMgmtClusterInfo
+		workloadInfo testWorkloadClusterInfo
+	)
+	BeforeEach(func() {
+		var err error
+		// Mgmt cluster info object
+		mgmtInfo = testMgmtClusterInfo{
+			clusterctlConfigFile:    clusterctlConfigFile,
+			coreProvider:            "cluster-api:v0.3.0",
+			bootstrapProviders:      []string{"kubeadm-bootstrap:v0.3.0"},
+			controlPlaneProviders:   []string{"kubeadm-control-plane:v0.3.0"},
+			infrastructureProviders: []string{"docker:v0.3.0"},
+		}
+		// Create the mgmt cluster and client
+		mgmtInfo.mgmtCluster, err = CreateKindCluster(kindConfigFile)
+		Expect(err).ToNot(HaveOccurred())
+		mgmtInfo.mgmtClient, err = mgmtInfo.mgmtCluster.GetClient()
+		Expect(err).NotTo(HaveOccurred())
+
+		initTestMgmtCluster(ctx, mgmtInfo)
+
+		// Workload cluster info object
+		workloadInfo = testWorkloadClusterInfo{
+			workloadClusterName:      "e2e-workload-cluster",
+			kubernetesVersion:        "1.14.2",
+			controlPlaneMachineCount: 1,
+			workerMachineCount:       0,
+		}
+		// Let's setup some varibles for the workload cluster template
+		Expect(os.Setenv("DOCKER_SERVICE_CIDRS", "\"10.96.0.0/12\"")).To(Succeed())
+		Expect(os.Setenv("DOCKER_POD_CIDRS", "\"192.168.0.0/16\"")).To(Succeed())
+		createTestWorkloadCluster(ctx, mgmtInfo, workloadInfo)
+	})
+
+	AfterEach(func() {
+		fmt.Fprintf(GinkgoWriter, "Tearing down kind mgmt cluster\n")
+		mgmtInfo.mgmtCluster.Teardown(ctx)
+		fmt.Fprintf(GinkgoWriter, "Tearing down kind workload cluster\n")
+		if err := cluster.NewProvider().Delete(workloadInfo.workloadClusterName, ""); err != nil {
+			// Treat this as a non critical error
+			fmt.Fprintf(GinkgoWriter, "Deleting the kind cluster %q failed. You may need to remove this by hand.\n", workloadInfo.workloadClusterName)
+		}
+	})
+
+	Context("using specific core, control-plane, bootstrap, and capd provider version", func() {
+		It("should create a workload cluster", func() {
+			Eventually(func() ([]v1.Node, error) {
+				workloadClient, err := mgmtInfo.mgmtCluster.GetWorkloadClient(ctx, "default", workloadInfo.workloadClusterName)
+				if err != nil {
+					return nil, errors.Wrap(err, "failed to get workload client")
+				}
+				nodeList := v1.NodeList{}
+				if err := workloadClient.List(ctx, &nodeList); err != nil {
+					return nil, err
+				}
+				return nodeList.Items, nil
+			}, 5*time.Minute, 10*time.Second).Should(HaveLen(2))
+
+			// TODO: Check config file and env variables defined above.
+		})
+	})
+})

cmd/clusterctl/test/e2e/delete_test.go

@@ -0,0 +1,125 @@
+// +build e2e
+
+/*
+Copyright 2020 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package e2e
+
+import (
+	"fmt"
+	"time"
+
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+
+	appsv1 "k8s.io/api/apps/v1"
+	corev1 "k8s.io/api/core/v1"
+	apierrors "k8s.io/apimachinery/pkg/api/errors"
+	"k8s.io/apimachinery/pkg/api/meta"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+
+	clusterctlclient "sigs.k8s.io/cluster-api/cmd/clusterctl/pkg/client"
+	infrav1 "sigs.k8s.io/cluster-api/test/infrastructure/docker/api/v1alpha3"
+)
+
+var _ = Describe("clusterctl delete", func() {
+	var (
+		mgmtInfo      testMgmtClusterInfo
+		deleteOptions clusterctlclient.DeleteOptions
+	)
+
+	BeforeEach(func() {
+		var err error
+		// Mgmt cluster info object
+		mgmtInfo = testMgmtClusterInfo{
+			clusterctlConfigFile:    clusterctlConfigFile,
+			coreProvider:            "cluster-api:v0.3.0",
+			bootstrapProviders:      []string{"kubeadm-bootstrap:v0.3.0"},
+			controlPlaneProviders:   []string{"kubeadm-control-plane:v0.3.0"},
+			infrastructureProviders: []string{"docker:v0.3.0"},
+		}
+		// Create the mgmt cluster and client
+		mgmtInfo.mgmtCluster, err = CreateKindCluster(kindConfigFile)
+		Expect(err).ToNot(HaveOccurred())
+		mgmtInfo.mgmtClient, err = mgmtInfo.mgmtCluster.GetClient()
+		Expect(err).NotTo(HaveOccurred())
+
+		initTestMgmtCluster(ctx, mgmtInfo)
+
+	}, setupTimeout)
+
+	JustBeforeEach(func() {
+		c, err := clusterctlclient.New(mgmtInfo.clusterctlConfigFile)
+		Expect(err).ToNot(HaveOccurred())
+		err = c.Delete(deleteOptions)
+		Expect(err).ToNot(HaveOccurred())
+	})
+
+	AfterEach(func() {
+		fmt.Fprintf(GinkgoWriter, "Tearing down kind clusters\n")
+		mgmtInfo.mgmtCluster.Teardown(ctx)
+	})
+
+	Context("deletes the infra provider", func() {
+		BeforeEach(func() {
+			deleteOptions = clusterctlclient.DeleteOptions{
+				Kubeconfig: mgmtInfo.mgmtCluster.KubeconfigPath,
+				Providers:  []string{"docker"},
+			}
+		})
+		It("should delete of all infra provider components except the hosting namespace and the CRDs.", func() {
+			Eventually(
+				func() bool {
+					if !apierrors.IsNotFound(mgmtInfo.mgmtClient.Get(ctx, client.ObjectKey{Namespace: "capd-system", Name: "capd-controller-manager"}, &appsv1.Deployment{})) {
+						return false
+					}
+					// TODO: check that namespace and CRD are still present.
+					return true
+				}, 3*time.Minute, 5*time.Second,
+			).Should(BeTrue())
+		})
+	})
+	Context("deletes everything", func() {
+		BeforeEach(func() {
+			deleteOptions = clusterctlclient.DeleteOptions{
+				Kubeconfig:           mgmtInfo.mgmtCluster.KubeconfigPath,
+				ForceDeleteNamespace: true,
+				ForceDeleteCRD:       true,
+				Providers:            []string{},
+			}
+		})
+		It("should reset the management cluster to its original state", func() {
+			Eventually(
+				func() bool {
+					// TODO: check all components are deleted.
+					if !apierrors.IsNotFound(mgmtInfo.mgmtClient.Get(ctx, client.ObjectKey{Namespace: "capd-system", Name: "capd-controller-manager"}, &appsv1.Deployment{})) {
+						return false
+					}
+					// TODO: check namespace of all components are deleted.
+					if !apierrors.IsNotFound(mgmtInfo.mgmtClient.Get(ctx, client.ObjectKey{Name: "capd-system"}, &corev1.Namespace{})) {
+						return false
+					}
+					// TODO: check that all CRDs are deleted.
+					err := mgmtInfo.mgmtClient.Get(ctx, client.ObjectKey{Namespace: "default", Name: "foo-cluster"}, &infrav1.DockerCluster{})
+					if _, ok := err.(*meta.NoResourceMatchError); !ok {
+						return false
+					}
+					return true
+				}, 3*time.Minute, 5*time.Second,
+			).Should(BeTrue())
+		})
+	})
+})

cmd/clusterctl/test/e2e/e2e_suite_test.go

@@ -0,0 +1,72 @@
+// +build e2e
+
+/*
+Copyright 2020 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package e2e
+
+import (
+	"context"
+	"fmt"
+	"os"
+	"testing"
+
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+)
+
+func TestE2E(t *testing.T) {
+	RegisterFailHandler(Fail)
+	RunSpecs(t, "ClusterCtl E2E Suite")
+}
+
+const (
+	setupTimeout = 10 * 60
+)
+
+var (
+	ctx                  context.Context
+	managerImage         string
+	kindConfigFile       string
+	clusterctlConfigFile string
+)
+
+var _ = BeforeSuite(func() {
+	ctx = context.Background()
+	// Docker image to load into the kind cluster for testing
+	managerImage = os.Getenv("MANAGER_IMAGE")
+	if managerImage == "" {
+		fmt.Fprintf(GinkgoWriter, "MANAGER_IMAGE not specified, using default %v\n", "gcr.io/k8s-staging-capi-docker/capd-manager-amd64:dev")
+		managerImage = "gcr.io/k8s-staging-capi-docker/capd-manager-amd64:dev"
+	} else {
+		fmt.Fprintf(GinkgoWriter, "Using MANAGER_IMAGE %v\n", managerImage)
+	}
+	kindConfigFile = os.Getenv("KIND_CONFIG_FILE")
+	if kindConfigFile == "" {
+		fmt.Fprintf(GinkgoWriter, "KIND_CONFIG_FILE not found, capd mgmt cluster wil be created without any special kind configuration.\n")
+	} else {
+		fmt.Fprintf(GinkgoWriter, "Using KIND_CONFIG_FILE: %v\n", kindConfigFile)
+	}
+	clusterctlConfigFile = os.Getenv("CLUSTERCTL_CONFIG")
+	if clusterctlConfigFile == "" {
+		fmt.Fprintf(GinkgoWriter, "CLUSTERCTL_CONFIG not found.\n")
+	} else {
+		fmt.Fprintf(GinkgoWriter, "Using CLUSTERCTL_CONFIG: %v\n", clusterctlConfigFile)
+	}
+}, setupTimeout)
+
+var _ = AfterSuite(func() {
+})

cmd/clusterctl/test/e2e/go.mod

@@ -0,0 +1,23 @@
+module sigs.k8s.io/cluster-api/cmd/clusterctl/test/e2e
+
+go 1.13
+
+require (
+	github.com/onsi/ginkgo v1.11.0
+	github.com/onsi/gomega v1.8.1
+	github.com/pkg/errors v0.9.0
+	k8s.io/api v0.0.0-20191121015604-11707872ac1c
+	k8s.io/apimachinery v0.17.0
+	k8s.io/client-go v11.0.0+incompatible
+	sigs.k8s.io/cluster-api v0.2.9
+	sigs.k8s.io/cluster-api/test/framework v0.0.0-20200125173702-54f26d7fd2b5
+	sigs.k8s.io/cluster-api/test/infrastructure/docker v0.0.0-20200125173702-54f26d7fd2b5
+	sigs.k8s.io/controller-runtime v0.4.0
+	sigs.k8s.io/kind v0.7.0
+)
+
+replace (
+	k8s.io/client-go => k8s.io/client-go v0.0.0-20190918160344-1fbdaa4c8d90
+	sigs.k8s.io/cluster-api => ../../../..
+	sigs.k8s.io/cluster-api/test/framework => ../../../../test/framework
+)