Convert etcd docker test

Signed-off-by: Derek Nola <[email protected]>

.github/workflows/e2e.yaml

@@ -171,7 +171,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        dtest: [basics, bootstraptoken, cacerts, lazypull, skew, upgrade]
+        dtest: [basics, bootstraptoken, cacerts, etcd, lazypull, skew, upgrade]
     env:
       BRANCH_NAME: ${{ needs.build-go-tests.outputs.branch_name }}
     steps:

tests/docker/etcd/etcd_test.go

@@ -0,0 +1,79 @@
+package main
+
+import (
+	"flag"
+	"os"
+	"testing"
+
+	tester "github.com/k3s-io/k3s/tests/docker"
+	. "github.com/onsi/ginkgo/v2"
+	. "github.com/onsi/gomega"
+)
+
+var k3sImage = flag.String("k3sImage", "", "The k3s image used to provision containers")
+var config *tester.TestConfig
+
+func Test_DockerEtcd(t *testing.T) {
+	flag.Parse()
+	RegisterFailHandler(Fail)
+	RunSpecs(t, "Etcd Docker Test Suite")
+}
+
+var _ = Describe("Etcd Tests", Ordered, func() {
+
+	Context("Test a 3 server cluster", func() {
+		It("should setup the cluster configuration", func() {
+			var err error
+			config, err = tester.NewTestConfig(*k3sImage)
+			Expect(err).NotTo(HaveOccurred())
+		})
+		It("should provision servers", func() {
+			Expect(config.ProvisionServers(3)).To(Succeed())
+			Eventually(func() error {
+				return tester.DeploymentsReady([]string{"coredns", "local-path-provisioner", "metrics-server", "traefik"}, config.KubeconfigFile)
+			}, "60s", "5s").Should(Succeed())
+			Eventually(func(g Gomega) {
+				g.Expect(tester.ParseNodes(config.KubeconfigFile)).To(HaveLen(3))
+				g.Expect(tester.NodesReady(config.KubeconfigFile)).To(Succeed())
+			}, "60s", "5s").Should(Succeed())
+		})
+		It("should destroy the cluster", func() {
+			Expect(config.Cleanup()).To(Succeed())
+		})
+	})
+
+	Context("Test a Split Role cluster with 3 etcd, 2 control-plane, 1 agents", func() {
+		It("should setup the cluster configuration", func() {
+			var err error
+			config, err = tester.NewTestConfig(*k3sImage)
+			Expect(err).NotTo(HaveOccurred())
+			Expect(os.Setenv("SERVER_0_ARGS", "--disable-apiserver --disable-controller-manager --disable-scheduler --cluster-init")).To(Succeed())
+			Expect(os.Setenv("SERVER_1_ARGS", "--disable-apiserver --disable-controller-manager --disable-scheduler")).To(Succeed())
+			Expect(os.Setenv("SERVER_2_ARGS", "--disable-apiserver --disable-controller-manager --disable-scheduler")).To(Succeed())
+			Expect(os.Setenv("SERVER_3_ARGS", "--disable-etcd")).To(Succeed())
+			Expect(os.Setenv("SERVER_4_ARGS", "--disable-etcd")).To(Succeed())
+		})
+		It("should provision servers and agents", func() {
+			Expect(config.ProvisionServers(5)).To(Succeed())
+			Expect(config.ProvisionAgents(1)).To(Succeed())
+			Eventually(func() error {
+				return tester.DeploymentsReady([]string{"coredns", "local-path-provisioner", "metrics-server", "traefik"}, config.KubeconfigFile)
+			}, "90s", "5s").Should(Succeed())
+			Eventually(func(g Gomega) {
+				g.Expect(tester.ParseNodes(config.KubeconfigFile)).To(HaveLen(6))
+				g.Expect(tester.NodesReady(config.KubeconfigFile)).To(Succeed())
+			}, "60s", "5s").Should(Succeed())
+		})
+	})
+})
+
+var failed bool
+var _ = AfterEach(func() {
+	failed = failed || CurrentSpecReport().Failed()
+})
+
+var _ = AfterSuite(func() {
+	if config != nil && !failed {
+		config.Cleanup()
+	}
+})

tests/docker/test-helpers.go

@@ -96,11 +96,17 @@ func getPort() int {
 }
 
 // ProvisionServers starts the required number of k3s servers
-// and updates the kubeconfig file with the first server details
+// and updates the kubeconfig file with the first cp server details
 func (config *TestConfig) ProvisionServers(numOfServers int) error {
 	config.NumServers = numOfServers
 	for i := 0; i < config.NumServers; i++ {
 
+		// If a server i already exists, skip. This is useful for scenarios where
+		// the first server is started seperate from the rest of the servers
+		if config.Servers != nil && i < len(config.Servers) {
+			continue
+		}
+
 		testID := filepath.Base(config.TestDir)
 		name := fmt.Sprintf("k3s-server-%d-%s", i, strings.ToLower(testID))
 
@@ -111,6 +117,18 @@ func (config *TestConfig) ProvisionServers(numOfServers int) error {
 
 		serverImage := getEnvOrDefault("K3S_IMAGE_SERVER", config.K3sImage)
 
+		var joinOrStart string
+		if numOfServers > 0 {
+			if i == 0 {
+				joinOrStart = "--cluster-init"
+			} else {
+				if config.Servers[0].URL == "" {
+					return fmt.Errorf("first server URL is empty")
+				}
+				joinOrStart = fmt.Sprintf("--server %s", config.Servers[0].URL)
+			}
+		}
+
 		// Assemble all the Docker args
 		dRun := strings.Join([]string{"docker run -d",
 			"--name", name,
@@ -124,7 +142,7 @@ func (config *TestConfig) ProvisionServers(numOfServers int) error {
 			os.Getenv(fmt.Sprintf("SERVER_%d_DOCKER_ARGS", i)),
 			os.Getenv("REGISTRY_CLUSTER_ARGS"),
 			serverImage,
-			"server", os.Getenv("ARGS"), os.Getenv("SERVER_ARGS"), os.Getenv(fmt.Sprintf("SERVER_%d_ARGS", i))}, " ")
+			"server", joinOrStart, os.Getenv("SERVER_ARGS"), os.Getenv(fmt.Sprintf("SERVER_%d_ARGS", i))}, " ")
 		if out, err := RunCommand(dRun); err != nil {
 			return fmt.Errorf("failed to run server container: %s: %v", out, err)
 		}
@@ -147,11 +165,14 @@ func (config *TestConfig) ProvisionServers(numOfServers int) error {
 
 		fmt.Printf("Started %s @ %s\n", name, url)
 
+		// Sleep for a bit to allow the first server to start
+		if i == 0 && numOfServers > 1 {
+			time.Sleep(10 * time.Second)
+		}
 	}
 
 	// Wait for kubeconfig to be available
 	time.Sleep(5 * time.Second)
-	// Write kubeconfig from first serRver
 	return copyAndModifyKubeconfig(config)
 }
 
@@ -211,30 +232,32 @@ func (config *TestConfig) ProvisionAgents(numOfAgents int) error {
 	return nil
 }
 
+func (config *TestConfig) RemoveNode(nodeName string) error {
+	cmd := fmt.Sprintf("docker stop %s", nodeName)
+	if _, err := RunCommand(cmd); err != nil {
+		return fmt.Errorf("failed to stop node %s: %v", nodeName, err)
+	}
+	cmd = fmt.Sprintf("docker rm %s", nodeName)
+	if _, err := RunCommand(cmd); err != nil {
+		return fmt.Errorf("failed to remove node %s: %v", nodeName, err)
+	}
+	return nil
+}
+
 func (config *TestConfig) Cleanup() error {
 
 	errs := make([]error, 0)
 	// Stop and remove all servers
 	for _, server := range config.Servers {
-		cmd := fmt.Sprintf("docker stop %s", server.Name)
-		if _, err := RunCommand(cmd); err != nil {
-			errs = append(errs, fmt.Errorf("failed to stop server %s: %v", server.Name, err))
-		}
-		cmd = fmt.Sprintf("docker rm %s", server.Name)
-		if _, err := RunCommand(cmd); err != nil {
-			errs = append(errs, fmt.Errorf("failed to remove server %s: %v", server.Name, err))
+		if err := config.RemoveNode(server.Name); err != nil {
+			errs = append(errs, err)
 		}
 	}
 
 	// Stop and remove all agents
 	for _, agent := range config.Agents {
-		cmd := fmt.Sprintf("docker stop %s", agent.Name)
-		if _, err := RunCommand(cmd); err != nil {
-			errs = append(errs, fmt.Errorf("failed to stop agent %s: %v", agent.Name, err))
-		}
-		cmd = fmt.Sprintf("docker rm %s", agent.Name)
-		if _, err := RunCommand(cmd); err != nil {
-			errs = append(errs, fmt.Errorf("failed to remove agent %s: %v", agent.Name, err))
+		if err := config.RemoveNode(agent.Name); err != nil {
+			errs = append(errs, err)
 		}
 	}
 
@@ -246,19 +269,33 @@ func (config *TestConfig) Cleanup() error {
 	if config.TestDir != "" {
 		return os.RemoveAll(config.TestDir)
 	}
+	config.Agents = nil
+	config.Servers = nil
 	return nil
 }
 
+// copyAndModifyKubeconfig copies out kubeconfig from first control-plane server
+// and updates the port to match the external port
 func copyAndModifyKubeconfig(config *TestConfig) error {
 	if len(config.Servers) == 0 {
 		return fmt.Errorf("no servers available to copy kubeconfig")
 	}
-	cmd := fmt.Sprintf("docker cp %s:/etc/rancher/k3s/k3s.yaml %s/kubeconfig.yaml", config.Servers[0].Name, config.TestDir)
+
+	serverID := 0
+	for i := range config.Servers {
+		server_args := os.Getenv(fmt.Sprintf("SERVER_%d_ARGS", i))
+		if !strings.Contains(server_args, "--disable-apiserver") {
+			serverID = i
+			break
+		}
+	}
+
+	cmd := fmt.Sprintf("docker cp %s:/etc/rancher/k3s/k3s.yaml %s/kubeconfig.yaml", config.Servers[serverID].Name, config.TestDir)
 	if _, err := RunCommand(cmd); err != nil {
 		return fmt.Errorf("failed to copy kubeconfig: %v", err)
 	}
 
-	cmd = fmt.Sprintf("sed -i -e \"s/:6443/:%d/g\" %s/kubeconfig.yaml", config.Servers[0].Port, config.TestDir)
+	cmd = fmt.Sprintf("sed -i -e \"s/:6443/:%d/g\" %s/kubeconfig.yaml", config.Servers[serverID].Port, config.TestDir)
 	if _, err := RunCommand(cmd); err != nil {
 		return fmt.Errorf("failed to update kubeconfig: %v", err)
 	}