diff --git a/tests/integration/certrotation/certrotation_int_test.go b/tests/integration/certrotation/certrotation_int_test.go index ffd7fad53a4f..21c58072083e 100644 --- a/tests/integration/certrotation/certrotation_int_test.go +++ b/tests/integration/certrotation/certrotation_int_test.go @@ -21,7 +21,7 @@ var _ = BeforeSuite(func() { var err error testLock, err = testutil.K3sTestLock() Expect(err).ToNot(HaveOccurred()) - server, err = testutil.K3sStartServer(serverArgs...) + server, err = testutil.K3sStartServer(serverArgs) Expect(err).ToNot(HaveOccurred()) } }) @@ -56,7 +56,7 @@ var _ = Describe("certificate rotation", Ordered, func() { }) It("start k3s server", func() { var err error - server2, err = testutil.K3sStartServer(serverArgs...) + server2, err = testutil.K3sStartServer(serverArgs) Expect(err).ToNot(HaveOccurred()) }) It("starts up with no problems", func() { diff --git a/tests/integration/custometcdargs/custometcdargs_int_test.go b/tests/integration/custometcdargs/custometcdargs_int_test.go index 20ea9803d7e5..0d0cc628eb1f 100644 --- a/tests/integration/custometcdargs/custometcdargs_int_test.go +++ b/tests/integration/custometcdargs/custometcdargs_int_test.go @@ -13,8 +13,9 @@ import ( var customEtcdArgsServer *testutil.K3sServer var customEtcdArgsServerArgs = []string{ "--cluster-init", - "--etcd-arg quota-backend-bytes=858993459", + "--etcd-arg=quota-backend-bytes=858993459", } + var testLock int var _ = BeforeSuite(func() { @@ -22,7 +23,7 @@ var _ = BeforeSuite(func() { var err error testLock, err = testutil.K3sTestLock() Expect(err).ToNot(HaveOccurred()) - customEtcdArgsServer, err = testutil.K3sStartServer(customEtcdArgsServerArgs...) + customEtcdArgsServer, err = testutil.K3sStartServer(customEtcdArgsServerArgs) Expect(err).ToNot(HaveOccurred()) } }) diff --git a/tests/integration/dualstack/dualstack_int_test.go b/tests/integration/dualstack/dualstack_int_test.go index 81bbc208e9e7..fae52502fe50 100644 --- a/tests/integration/dualstack/dualstack_int_test.go +++ b/tests/integration/dualstack/dualstack_int_test.go @@ -13,8 +13,8 @@ import ( var dualStackServer *testutil.K3sServer var dualStackServerArgs = []string{ "--cluster-init", - "--cluster-cidr 10.42.0.0/16,2001:cafe:42:0::/56", - "--service-cidr 10.43.0.0/16,2001:cafe:42:1::/112", + "--cluster-cidr", "10.42.0.0/16,2001:cafe:42:0::/56", + "--service-cidr", "10.43.0.0/16,2001:cafe:42:1::/112", "--disable-network-policy", } var testLock int @@ -24,7 +24,7 @@ var _ = BeforeSuite(func() { var err error testLock, err = testutil.K3sTestLock() Expect(err).ToNot(HaveOccurred()) - dualStackServer, err = testutil.K3sStartServer(dualStackServerArgs...) + dualStackServer, err = testutil.K3sStartServer(dualStackServerArgs) Expect(err).ToNot(HaveOccurred()) } }) diff --git a/tests/integration/etcdrestore/etcd_restore_int_test.go b/tests/integration/etcdrestore/etcd_restore_int_test.go index 22bb0f2b6ee5..6579e6d044a1 100644 --- a/tests/integration/etcdrestore/etcd_restore_int_test.go +++ b/tests/integration/etcdrestore/etcd_restore_int_test.go @@ -19,7 +19,7 @@ var _ = BeforeSuite(func() { var err error testLock, err = testutil.K3sTestLock() Expect(err).ToNot(HaveOccurred()) - server1, err = testutil.K3sStartServer(restoreServerArgs...) + server1, err = testutil.K3sStartServer(restoreServerArgs) Expect(err).ToNot(HaveOccurred()) } }) @@ -75,7 +75,7 @@ var _ = Describe("etcd snapshot restore", Ordered, func() { }) It("start k3s server", func() { var err error - server2, err = testutil.K3sStartServer(restoreServerArgs...) + server2, err = testutil.K3sStartServer(restoreServerArgs) Expect(err).ToNot(HaveOccurred()) }) It("starts up with no problems", func() { diff --git a/tests/integration/etcdsnapshot/etcdsnapshot_int_test.go b/tests/integration/etcdsnapshot/etcdsnapshot_int_test.go index 1df2a208bebb..23bbe24cd19b 100644 --- a/tests/integration/etcdsnapshot/etcdsnapshot_int_test.go +++ b/tests/integration/etcdsnapshot/etcdsnapshot_int_test.go @@ -1,6 +1,8 @@ package snapshot_test import ( + "fmt" + "path/filepath" "regexp" "strings" "testing" @@ -14,13 +16,17 @@ import ( var server *testutil.K3sServer var serverArgs = []string{"--cluster-init"} var testLock int +var populatedTestSnapshotDir string +var emptyTestSnapshotDir string +var etcdSnapshotFilePattern = "test-snapshot" +var etcdSnapshotRetention = 1 var _ = BeforeSuite(func() { if !testutil.IsExistingServer() { var err error testLock, err = testutil.K3sTestLock() Expect(err).ToNot(HaveOccurred()) - server, err = testutil.K3sStartServer(serverArgs...) + server, err = testutil.K3sStartServer(serverArgs) Expect(err).ToNot(HaveOccurred()) } }) @@ -111,6 +117,60 @@ var _ = Describe("etcd snapshots", Ordered, func() { } }) }) + When("a new etcd is created with server start flags", func() { + It("kills previous server and start up with no problems", func() { + var err error + Expect(testutil.K3sKillServer(server)).To(Succeed()) + localServerArgs := []string{"--cluster-init", + "--etcd-snapshot-name", etcdSnapshotFilePattern, + "--etcd-snapshot-dir", populatedTestSnapshotDir, + "--etcd-snapshot-retention", fmt.Sprint(etcdSnapshotRetention), + "--etcd-snapshot-schedule-cron", `* * * * *`, + "--etcd-snapshot-compress"} + server, err = testutil.K3sStartServer(localServerArgs) + Expect(err).ToNot(HaveOccurred()) + Eventually(func() error { + return testutil.K3sDefaultDeployments() + }, "180s", "5s").Should(Succeed()) + + }) + It("saves an etcd snapshot with specified name and it should be no more than 1 compressed file", func() { + + Eventually(func() (int, error) { + matches, err := filepath.Glob(filepath.Join(populatedTestSnapshotDir, fmt.Sprintf("%s%s%s", "*", etcdSnapshotFilePattern, "*.zip"))) + return len(matches), err + }, "180s", "30s").Should(Equal(etcdSnapshotRetention)) + Consistently(func() (int, error) { + matches, err := filepath.Glob(filepath.Join(populatedTestSnapshotDir, fmt.Sprintf("%s%s%s", "*", etcdSnapshotFilePattern, "*.zip"))) + return len(matches), err + }, "120s", "30s").Should(Equal(etcdSnapshotRetention)) + }) + It("kills previous server and start up with no problems and disabled snapshots", func() { + + var err error + Expect(testutil.K3sKillServer(server)).To(Succeed()) + localServerArgs := []string{"--cluster-init", + "--etcd-snapshot-dir", emptyTestSnapshotDir, + "--etcd-snapshot-schedule-cron", `* * * * *`, + "--etcd-disable-snapshots"} + server, err = testutil.K3sStartServer(localServerArgs) + Expect(err).ToNot(HaveOccurred()) + Eventually(func() error { + return testutil.K3sDefaultDeployments() + }, "180s", "5s").Should(Succeed()) + + }) + It("should not save any snapshot", func() { + Consistently(func() error { + matches, err := filepath.Glob(filepath.Join(emptyTestSnapshotDir, "*")) + if matches != nil || err != nil { + return fmt.Errorf("something went wrong: err != nil (%v) or matches != nil (%v)", err, matches) + } + return nil + }, "180s", "60s").Should(Succeed()) + }) + }) + }) var failed bool @@ -130,5 +190,7 @@ var _ = AfterSuite(func() { func Test_IntegrationEtcdSnapshot(t *testing.T) { RegisterFailHandler(Fail) + populatedTestSnapshotDir = t.TempDir() + emptyTestSnapshotDir = t.TempDir() RunSpecs(t, "Etcd Snapshot Suite") } diff --git a/tests/integration/integration.go b/tests/integration/integration.go index be60e7d8cf93..78a559773dcd 100644 --- a/tests/integration/integration.go +++ b/tests/integration/integration.go @@ -205,17 +205,13 @@ func K3sTestLock() (int, error) { // K3sStartServer acquires an exclusive lock on a temporary file, then launches a k3s cluster // with the provided arguments. Subsequent/parallel calls to this function will block until // the original lock is cleared using K3sKillServer -func K3sStartServer(inputArgs ...string) (*K3sServer, error) { +func K3sStartServer(inputArgs []string) (*K3sServer, error) { if !IsRoot() { return nil, errors.New("integration tests must be run as sudo/root") } - var cmdArgs []string - for _, arg := range inputArgs { - cmdArgs = append(cmdArgs, strings.Fields(arg)...) - } k3sBin := findK3sExecutable() - k3sCmd := append([]string{"server"}, cmdArgs...) + k3sCmd := append([]string{"server"}, inputArgs...) cmd := exec.Command(k3sBin, k3sCmd...) // Give the server a new group id so we can kill it and its children later cmd.SysProcAttr = &syscall.SysProcAttr{Setpgid: true} diff --git a/tests/integration/localstorage/localstorage_int_test.go b/tests/integration/localstorage/localstorage_int_test.go index be1ebd6e4081..434bd8f344bc 100644 --- a/tests/integration/localstorage/localstorage_int_test.go +++ b/tests/integration/localstorage/localstorage_int_test.go @@ -21,7 +21,7 @@ var _ = BeforeSuite(func() { var err error testLock, err = testutil.K3sTestLock() Expect(err).ToNot(HaveOccurred()) - localStorageServer, err = testutil.K3sStartServer(localStorageServerArgs...) + localStorageServer, err = testutil.K3sStartServer(localStorageServerArgs) Expect(err).ToNot(HaveOccurred()) } }) diff --git a/tests/integration/secretsencryption/secretsencryption_int_test.go b/tests/integration/secretsencryption/secretsencryption_int_test.go index 8bab882cdb66..7be70df79d6c 100644 --- a/tests/integration/secretsencryption/secretsencryption_int_test.go +++ b/tests/integration/secretsencryption/secretsencryption_int_test.go @@ -21,7 +21,7 @@ var _ = BeforeSuite(func() { var err error testLock, err = testutil.K3sTestLock() Expect(err).ToNot(HaveOccurred()) - secretsEncryptionServer, err = testutil.K3sStartServer(secretsEncryptionServerArgs...) + secretsEncryptionServer, err = testutil.K3sStartServer(secretsEncryptionServerArgs) Expect(err).ToNot(HaveOccurred()) } }) @@ -63,7 +63,7 @@ var _ = Describe("secrets encryption rotation", Ordered, func() { It("restarts the server", func() { var err error Expect(testutil.K3sKillServer(secretsEncryptionServer)).To(Succeed()) - secretsEncryptionServer, err = testutil.K3sStartServer(secretsEncryptionServerArgs...) + secretsEncryptionServer, err = testutil.K3sStartServer(secretsEncryptionServerArgs) Expect(err).ToNot(HaveOccurred()) Eventually(func() error { return testutil.K3sDefaultDeployments() @@ -88,7 +88,7 @@ var _ = Describe("secrets encryption rotation", Ordered, func() { It("restarts the server", func() { var err error Expect(testutil.K3sKillServer(secretsEncryptionServer)).To(Succeed()) - secretsEncryptionServer, err = testutil.K3sStartServer(secretsEncryptionServerArgs...) + secretsEncryptionServer, err = testutil.K3sStartServer(secretsEncryptionServerArgs) Expect(err).ToNot(HaveOccurred()) Eventually(func() error { return testutil.K3sDefaultDeployments() @@ -122,7 +122,7 @@ var _ = Describe("secrets encryption rotation", Ordered, func() { It("restarts the server", func() { var err error Expect(testutil.K3sKillServer(secretsEncryptionServer)).To(Succeed()) - secretsEncryptionServer, err = testutil.K3sStartServer(secretsEncryptionServerArgs...) + secretsEncryptionServer, err = testutil.K3sStartServer(secretsEncryptionServerArgs) Expect(err).ToNot(HaveOccurred()) Eventually(func() error { return testutil.K3sDefaultDeployments() diff --git a/tests/integration/startup/startup_int_test.go b/tests/integration/startup/startup_int_test.go index e161ba0280c5..2b227d0411b5 100644 --- a/tests/integration/startup/startup_int_test.go +++ b/tests/integration/startup/startup_int_test.go @@ -31,7 +31,7 @@ var _ = Describe("startup tests", Ordered, func() { When("a default server is created", func() { It("is created with no arguments", func() { var err error - startupServer, err = testutil.K3sStartServer(startupServerArgs...) + startupServer, err = testutil.K3sStartServer(startupServerArgs) Expect(err).ToNot(HaveOccurred()) }) It("has the default pods deployed", func() { @@ -48,7 +48,7 @@ var _ = Describe("startup tests", Ordered, func() { It("is created with cluster-init arguments", func() { var err error startupServerArgs = []string{"--cluster-init"} - startupServer, err = testutil.K3sStartServer(startupServerArgs...) + startupServer, err = testutil.K3sStartServer(startupServerArgs) Expect(err).ToNot(HaveOccurred()) }) It("has the default pods deployed", func() { @@ -65,7 +65,7 @@ var _ = Describe("startup tests", Ordered, func() { It("is created with disable arguments", func() { var err error startupServerArgs = []string{"--disable", "traefik"} - startupServer, err = testutil.K3sStartServer(startupServerArgs...) + startupServer, err = testutil.K3sStartServer(startupServerArgs) Expect(err).ToNot(HaveOccurred()) }) It("has the default pods without traefik deployed", func() { @@ -91,7 +91,7 @@ var _ = Describe("startup tests", Ordered, func() { It("is created with node-ip arguments", func() { var err error startupServerArgs = []string{"--node-ip", "11.22.33.44", "--node-external-ip", "55.66.77.88"} - startupServer, err = testutil.K3sStartServer(startupServerArgs...) + startupServer, err = testutil.K3sStartServer(startupServerArgs) Expect(err).ToNot(HaveOccurred()) }) It("has the node deployed with correct IPs", func() { @@ -129,7 +129,7 @@ var _ = Describe("startup tests", Ordered, func() { It("is created with data-dir flag", func() { var err error startupServerArgs = []string{"--data-dir", tempDir} - startupServer, err = testutil.K3sStartServer(startupServerArgs...) + startupServer, err = testutil.K3sStartServer(startupServerArgs) Expect(err).ToNot(HaveOccurred()) }) It("has the default pods deployed", func() { @@ -157,7 +157,7 @@ var _ = Describe("startup tests", Ordered, func() { It("is created with node-name with-node-id, node-label and node-taint flags", func() { var err error startupServerArgs = []string{"--node-name", "customnoder", "--with-node-id", "--node-label", "foo=bar", "--node-taint", "alice=bob:PreferNoSchedule"} - startupServer, err = testutil.K3sStartServer(startupServerArgs...) + startupServer, err = testutil.K3sStartServer(startupServerArgs) Expect(err).ToNot(HaveOccurred()) }) It("has the default pods deployed", func() { @@ -192,7 +192,7 @@ var _ = Describe("startup tests", Ordered, func() { It("is created with prefer-bundled-bin flag", func() { var err error startupServerArgs = []string{"--prefer-bundled-bin"} - startupServer, err = testutil.K3sStartServer(startupServerArgs...) + startupServer, err = testutil.K3sStartServer(startupServerArgs) Expect(err).ToNot(HaveOccurred()) }) It("has the default pods deployed", func() { @@ -213,7 +213,7 @@ var _ = Describe("startup tests", Ordered, func() { When("a server with a dummy pod", func() { It("is created with no arguments", func() { var err error - startupServer, err = testutil.K3sStartServer(startupServerArgs...) + startupServer, err = testutil.K3sStartServer(startupServerArgs) Expect(err).ToNot(HaveOccurred()) }) It("has the default pods deployed", func() { @@ -231,7 +231,7 @@ var _ = Describe("startup tests", Ordered, func() { It("restarts the server", func() { var err error Expect(testutil.K3sStopServer(startupServer)).To(Succeed()) - startupServer, err = testutil.K3sStartServer(startupServerArgs...) + startupServer, err = testutil.K3sStartServer(startupServerArgs) Expect(err).ToNot(HaveOccurred()) Eventually(func() error { return testutil.K3sDefaultDeployments()