Add integration tests for etc-snapshot server flags and refactor /tes…

…ts/integration/integration.go/K3sStartServer This adds integration tests for the following flags: "--etcd-snapshot-name","--etcd-snapshot-dir","--etcd-snapshot-retention","--etcd-snapshot-schedule-cron" and "--etcd-snapshot-compress". It also refactors K3sStartServer to receive as argument a string slice instead of a variadic arg and stop applying strings.Fields() into inputArgs, so it can accept arguments that have space in their definition. Signed-off-by: Ian Cardoso <[email protected]>
k3s-io · Apr 18, 2023 · 1ac4c84 · 1ac4c84
1 parent d9f40d4
commit 1ac4c84
Show file tree

Hide file tree

Showing 9 changed files with 89 additions and 30 deletions.
diff --git a/tests/integration/certrotation/certrotation_int_test.go b/tests/integration/certrotation/certrotation_int_test.go
@@ -21,7 +21,7 @@ var _ = BeforeSuite(func() {
 		var err error
 		testLock, err = testutil.K3sTestLock()
 		Expect(err).ToNot(HaveOccurred())
-		server, err = testutil.K3sStartServer(serverArgs...)
+		server, err = testutil.K3sStartServer(serverArgs)
 		Expect(err).ToNot(HaveOccurred())
 	}
 })
@@ -56,7 +56,7 @@ var _ = Describe("certificate rotation", Ordered, func() {
 		})
 		It("start k3s server", func() {
 			var err error
-			server2, err = testutil.K3sStartServer(serverArgs...)
+			server2, err = testutil.K3sStartServer(serverArgs)
 			Expect(err).ToNot(HaveOccurred())
 		})
 		It("starts up with no problems", func() {

diff --git a/tests/integration/custometcdargs/custometcdargs_int_test.go b/tests/integration/custometcdargs/custometcdargs_int_test.go
@@ -13,16 +13,17 @@ import (
 var customEtcdArgsServer *testutil.K3sServer
 var customEtcdArgsServerArgs = []string{
 	"--cluster-init",
-	"--etcd-arg quota-backend-bytes=858993459",
+	"--etcd-arg=quota-backend-bytes=858993459",
 }
+
 var testLock int
 
 var _ = BeforeSuite(func() {
 	if !testutil.IsExistingServer() {
 		var err error
 		testLock, err = testutil.K3sTestLock()
 		Expect(err).ToNot(HaveOccurred())
-		customEtcdArgsServer, err = testutil.K3sStartServer(customEtcdArgsServerArgs...)
+		customEtcdArgsServer, err = testutil.K3sStartServer(customEtcdArgsServerArgs)
 		Expect(err).ToNot(HaveOccurred())
 	}
 })

diff --git a/tests/integration/dualstack/dualstack_int_test.go b/tests/integration/dualstack/dualstack_int_test.go
@@ -13,8 +13,8 @@ import (
 var dualStackServer *testutil.K3sServer
 var dualStackServerArgs = []string{
 	"--cluster-init",
-	"--cluster-cidr 10.42.0.0/16,2001:cafe:42:0::/56",
-	"--service-cidr 10.43.0.0/16,2001:cafe:42:1::/112",
+	"--cluster-cidr", "10.42.0.0/16,2001:cafe:42:0::/56",
+	"--service-cidr", "10.43.0.0/16,2001:cafe:42:1::/112",
 	"--disable-network-policy",
 }
 var testLock int
@@ -24,7 +24,7 @@ var _ = BeforeSuite(func() {
 		var err error
 		testLock, err = testutil.K3sTestLock()
 		Expect(err).ToNot(HaveOccurred())
-		dualStackServer, err = testutil.K3sStartServer(dualStackServerArgs...)
+		dualStackServer, err = testutil.K3sStartServer(dualStackServerArgs)
 		Expect(err).ToNot(HaveOccurred())
 	}
 })

diff --git a/tests/integration/etcdrestore/etcd_restore_int_test.go b/tests/integration/etcdrestore/etcd_restore_int_test.go
@@ -19,7 +19,7 @@ var _ = BeforeSuite(func() {
 		var err error
 		testLock, err = testutil.K3sTestLock()
 		Expect(err).ToNot(HaveOccurred())
-		server1, err = testutil.K3sStartServer(restoreServerArgs...)
+		server1, err = testutil.K3sStartServer(restoreServerArgs)
 		Expect(err).ToNot(HaveOccurred())
 	}
 })
@@ -75,7 +75,7 @@ var _ = Describe("etcd snapshot restore", Ordered, func() {
 		})
 		It("start k3s server", func() {
 			var err error
-			server2, err = testutil.K3sStartServer(restoreServerArgs...)
+			server2, err = testutil.K3sStartServer(restoreServerArgs)
 			Expect(err).ToNot(HaveOccurred())
 		})
 		It("starts up with no problems", func() {

diff --git a/tests/integration/etcdsnapshot/etcdsnapshot_int_test.go b/tests/integration/etcdsnapshot/etcdsnapshot_int_test.go
@@ -1,6 +1,8 @@
 package snapshot_test
 
 import (
+	"fmt"
+	"path/filepath"
 	"regexp"
 	"strings"
 	"testing"
@@ -14,13 +16,17 @@ import (
 var server *testutil.K3sServer
 var serverArgs = []string{"--cluster-init"}
 var testLock int
+var populatedTestSnapshotDir string
+var emptyTestSnapshotDir string
+var etcdSnapshotFilePattern = "test-snapshot"
+var etcdSnapshotRetention = 1
 
 var _ = BeforeSuite(func() {
 	if !testutil.IsExistingServer() {
 		var err error
 		testLock, err = testutil.K3sTestLock()
 		Expect(err).ToNot(HaveOccurred())
-		server, err = testutil.K3sStartServer(serverArgs...)
+		server, err = testutil.K3sStartServer(serverArgs)
 		Expect(err).ToNot(HaveOccurred())
 	}
 })
@@ -111,6 +117,60 @@ var _ = Describe("etcd snapshots", Ordered, func() {
 			}
 		})
 	})
+	When("a new etcd is created with server start flags", func() {
+		It("kills previous server and start up with no problems", func() {
+			var err error
+			Expect(testutil.K3sKillServer(server)).To(Succeed())
+			localServerArgs := []string{"--cluster-init",
+				"--etcd-snapshot-name", etcdSnapshotFilePattern,
+				"--etcd-snapshot-dir", populatedTestSnapshotDir,
+				"--etcd-snapshot-retention", fmt.Sprint(etcdSnapshotRetention),
+				"--etcd-snapshot-schedule-cron", `* * * * *`,
+				"--etcd-snapshot-compress"}
+			server, err = testutil.K3sStartServer(localServerArgs)
+			Expect(err).ToNot(HaveOccurred())
+			Eventually(func() error {
+				return testutil.K3sDefaultDeployments()
+			}, "180s", "5s").Should(Succeed())
+
+		})
+		It("saves an etcd snapshot with specified name and it should be no more than 1 compressed file", func() {
+
+			Eventually(func() (int, error) {
+				matches, err := filepath.Glob(filepath.Join(populatedTestSnapshotDir, fmt.Sprintf("%s%s%s", "*", etcdSnapshotFilePattern, "*.zip")))
+				return len(matches), err
+			}, "180s", "30s").Should(Equal(etcdSnapshotRetention))
+			Consistently(func() (int, error) {
+				matches, err := filepath.Glob(filepath.Join(populatedTestSnapshotDir, fmt.Sprintf("%s%s%s", "*", etcdSnapshotFilePattern, "*.zip")))
+				return len(matches), err
+			}, "120s", "30s").Should(Equal(etcdSnapshotRetention))
+		})
+		It("kills previous server and start up with no problems and disabled snapshots", func() {
+
+			var err error
+			Expect(testutil.K3sKillServer(server)).To(Succeed())
+			localServerArgs := []string{"--cluster-init",
+				"--etcd-snapshot-dir", emptyTestSnapshotDir,
+				"--etcd-snapshot-schedule-cron", `* * * * *`,
+				"--etcd-disable-snapshots"}
+			server, err = testutil.K3sStartServer(localServerArgs)
+			Expect(err).ToNot(HaveOccurred())
+			Eventually(func() error {
+				return testutil.K3sDefaultDeployments()
+			}, "180s", "5s").Should(Succeed())
+
+		})
+		It("should not save any snapshot", func() {
+			Consistently(func() error {
+				matches, err := filepath.Glob(filepath.Join(emptyTestSnapshotDir, "*"))
+				if matches != nil || err != nil {
+					return fmt.Errorf("something went wrong: err != nil (%v) or matches != nil (%v)", err, matches)
+				}
+				return nil
+			}, "180s", "60s").Should(Succeed())
+		})
+	})
+
 })
 
 var failed bool
@@ -130,5 +190,7 @@ var _ = AfterSuite(func() {
 
 func Test_IntegrationEtcdSnapshot(t *testing.T) {
 	RegisterFailHandler(Fail)
+	populatedTestSnapshotDir = t.TempDir()
+	emptyTestSnapshotDir = t.TempDir()
 	RunSpecs(t, "Etcd Snapshot Suite")
 }
diff --git a/tests/integration/integration.go b/tests/integration/integration.go
@@ -205,17 +205,13 @@ func K3sTestLock() (int, error) {
 // K3sStartServer acquires an exclusive lock on a temporary file, then launches a k3s cluster
 // with the provided arguments. Subsequent/parallel calls to this function will block until
 // the original lock is cleared using K3sKillServer
-func K3sStartServer(inputArgs ...string) (*K3sServer, error) {
+func K3sStartServer(inputArgs []string) (*K3sServer, error) {
 	if !IsRoot() {
 		return nil, errors.New("integration tests must be run as sudo/root")
 	}
 
-	var cmdArgs []string
-	for _, arg := range inputArgs {
-		cmdArgs = append(cmdArgs, strings.Fields(arg)...)
-	}
 	k3sBin := findK3sExecutable()
-	k3sCmd := append([]string{"server"}, cmdArgs...)
+	k3sCmd := append([]string{"server"}, inputArgs...)
 	cmd := exec.Command(k3sBin, k3sCmd...)
 	// Give the server a new group id so we can kill it and its children later
 	cmd.SysProcAttr = &syscall.SysProcAttr{Setpgid: true}

diff --git a/tests/integration/localstorage/localstorage_int_test.go b/tests/integration/localstorage/localstorage_int_test.go
@@ -21,7 +21,7 @@ var _ = BeforeSuite(func() {
 		var err error
 		testLock, err = testutil.K3sTestLock()
 		Expect(err).ToNot(HaveOccurred())
-		localStorageServer, err = testutil.K3sStartServer(localStorageServerArgs...)
+		localStorageServer, err = testutil.K3sStartServer(localStorageServerArgs)
 		Expect(err).ToNot(HaveOccurred())
 	}
 })

diff --git a/tests/integration/secretsencryption/secretsencryption_int_test.go b/tests/integration/secretsencryption/secretsencryption_int_test.go
@@ -21,7 +21,7 @@ var _ = BeforeSuite(func() {
 		var err error
 		testLock, err = testutil.K3sTestLock()
 		Expect(err).ToNot(HaveOccurred())
-		secretsEncryptionServer, err = testutil.K3sStartServer(secretsEncryptionServerArgs...)
+		secretsEncryptionServer, err = testutil.K3sStartServer(secretsEncryptionServerArgs)
 		Expect(err).ToNot(HaveOccurred())
 	}
 })
@@ -63,7 +63,7 @@ var _ = Describe("secrets encryption rotation", Ordered, func() {
 		It("restarts the server", func() {
 			var err error
 			Expect(testutil.K3sKillServer(secretsEncryptionServer)).To(Succeed())
-			secretsEncryptionServer, err = testutil.K3sStartServer(secretsEncryptionServerArgs...)
+			secretsEncryptionServer, err = testutil.K3sStartServer(secretsEncryptionServerArgs)
 			Expect(err).ToNot(HaveOccurred())
 			Eventually(func() error {
 				return testutil.K3sDefaultDeployments()
@@ -88,7 +88,7 @@ var _ = Describe("secrets encryption rotation", Ordered, func() {
 		It("restarts the server", func() {
 			var err error
 			Expect(testutil.K3sKillServer(secretsEncryptionServer)).To(Succeed())
-			secretsEncryptionServer, err = testutil.K3sStartServer(secretsEncryptionServerArgs...)
+			secretsEncryptionServer, err = testutil.K3sStartServer(secretsEncryptionServerArgs)
 			Expect(err).ToNot(HaveOccurred())
 			Eventually(func() error {
 				return testutil.K3sDefaultDeployments()
@@ -122,7 +122,7 @@ var _ = Describe("secrets encryption rotation", Ordered, func() {
 		It("restarts the server", func() {
 			var err error
 			Expect(testutil.K3sKillServer(secretsEncryptionServer)).To(Succeed())
-			secretsEncryptionServer, err = testutil.K3sStartServer(secretsEncryptionServerArgs...)
+			secretsEncryptionServer, err = testutil.K3sStartServer(secretsEncryptionServerArgs)
 			Expect(err).ToNot(HaveOccurred())
 			Eventually(func() error {
 				return testutil.K3sDefaultDeployments()

diff --git a/tests/integration/startup/startup_int_test.go b/tests/integration/startup/startup_int_test.go
@@ -31,7 +31,7 @@ var _ = Describe("startup tests", Ordered, func() {
 	When("a default server is created", func() {
 		It("is created with no arguments", func() {
 			var err error
-			startupServer, err = testutil.K3sStartServer(startupServerArgs...)
+			startupServer, err = testutil.K3sStartServer(startupServerArgs)
 			Expect(err).ToNot(HaveOccurred())
 		})
 		It("has the default pods deployed", func() {
@@ -48,7 +48,7 @@ var _ = Describe("startup tests", Ordered, func() {
 		It("is created with cluster-init arguments", func() {
 			var err error
 			startupServerArgs = []string{"--cluster-init"}
-			startupServer, err = testutil.K3sStartServer(startupServerArgs...)
+			startupServer, err = testutil.K3sStartServer(startupServerArgs)
 			Expect(err).ToNot(HaveOccurred())
 		})
 		It("has the default pods deployed", func() {
@@ -65,7 +65,7 @@ var _ = Describe("startup tests", Ordered, func() {
 		It("is created with disable arguments", func() {
 			var err error
 			startupServerArgs = []string{"--disable", "traefik"}
-			startupServer, err = testutil.K3sStartServer(startupServerArgs...)
+			startupServer, err = testutil.K3sStartServer(startupServerArgs)
 			Expect(err).ToNot(HaveOccurred())
 		})
 		It("has the default pods without traefik deployed", func() {
@@ -91,7 +91,7 @@ var _ = Describe("startup tests", Ordered, func() {
 		It("is created with node-ip arguments", func() {
 			var err error
 			startupServerArgs = []string{"--node-ip", "11.22.33.44", "--node-external-ip", "55.66.77.88"}
-			startupServer, err = testutil.K3sStartServer(startupServerArgs...)
+			startupServer, err = testutil.K3sStartServer(startupServerArgs)
 			Expect(err).ToNot(HaveOccurred())
 		})
 		It("has the node deployed with correct IPs", func() {
@@ -129,7 +129,7 @@ var _ = Describe("startup tests", Ordered, func() {
 		It("is created with data-dir flag", func() {
 			var err error
 			startupServerArgs = []string{"--data-dir", tempDir}
-			startupServer, err = testutil.K3sStartServer(startupServerArgs...)
+			startupServer, err = testutil.K3sStartServer(startupServerArgs)
 			Expect(err).ToNot(HaveOccurred())
 		})
 		It("has the default pods deployed", func() {
@@ -157,7 +157,7 @@ var _ = Describe("startup tests", Ordered, func() {
 		It("is created with node-name with-node-id, node-label and node-taint flags", func() {
 			var err error
 			startupServerArgs = []string{"--node-name", "customnoder", "--with-node-id", "--node-label", "foo=bar", "--node-taint", "alice=bob:PreferNoSchedule"}
-			startupServer, err = testutil.K3sStartServer(startupServerArgs...)
+			startupServer, err = testutil.K3sStartServer(startupServerArgs)
 			Expect(err).ToNot(HaveOccurred())
 		})
 		It("has the default pods deployed", func() {
@@ -192,7 +192,7 @@ var _ = Describe("startup tests", Ordered, func() {
 		It("is created with prefer-bundled-bin flag", func() {
 			var err error
 			startupServerArgs = []string{"--prefer-bundled-bin"}
-			startupServer, err = testutil.K3sStartServer(startupServerArgs...)
+			startupServer, err = testutil.K3sStartServer(startupServerArgs)
 			Expect(err).ToNot(HaveOccurred())
 		})
 		It("has the default pods deployed", func() {
@@ -213,7 +213,7 @@ var _ = Describe("startup tests", Ordered, func() {
 	When("a server with a dummy pod", func() {
 		It("is created with no arguments", func() {
 			var err error
-			startupServer, err = testutil.K3sStartServer(startupServerArgs...)
+			startupServer, err = testutil.K3sStartServer(startupServerArgs)
 			Expect(err).ToNot(HaveOccurred())
 		})
 		It("has the default pods deployed", func() {
@@ -231,7 +231,7 @@ var _ = Describe("startup tests", Ordered, func() {
 		It("restarts the server", func() {
 			var err error
 			Expect(testutil.K3sStopServer(startupServer)).To(Succeed())
-			startupServer, err = testutil.K3sStartServer(startupServerArgs...)
+			startupServer, err = testutil.K3sStartServer(startupServerArgs)
 			Expect(err).ToNot(HaveOccurred())
 			Eventually(func() error {
 				return testutil.K3sDefaultDeployments()