Merge pull request #3959 from hashicorp/ginkgo-e2e

End to end tests using ginkgo

e2e/README.md

@@ -0,0 +1,19 @@
+End to End Tests
+================
+
+This package contains integration tests that are not run by default. To run them use the `-integration` flag. Example:
+
+```
+$ cd e2e/rescheduling/
+$ go test -integration
+Running Suite: Server Side Restart Tests
+========================================
+Random Seed: 1520633027
+Will run 7 of 7 specs
+
+•••••••
+Ran 7 of 7 Specs in 4.231 seconds
+SUCCESS! -- 7 Passed | 0 Failed | 0 Pending | 0 Skipped PASS
+ok  	github.com/hashicorp/nomad/e2e/rescheduling	4.239s
+
+```

e2e/rescheduling/input/norescheduling.hcl

@@ -0,0 +1,29 @@
+job "test1" {
+  datacenters = ["dc1"]
+  type = "service"
+
+  group "t1" {
+    count = 3
+    task "t1" {
+      driver = "raw_exec"
+      config {
+         command = "bash"
+         args    = ["-c", "lol 5000"]
+      }
+    }
+    update {
+       max_parallel = 1
+       min_healthy_time = "10s"
+       auto_revert = false
+    }
+    restart {
+        attempts = 0
+        delay    = "0s"
+        mode     = "fail"
+    }
+    reschedule {
+      attempts = 0
+      interval = "5m"
+   }
+  }
+}

e2e/rescheduling/input/reschedule_success.hcl

@@ -0,0 +1,24 @@
+job "test3" {
+  datacenters = ["dc1"]
+  type = "service"
+
+  group "t3" {
+    count = 3
+    task "t3" {
+      driver = "raw_exec"
+      config {
+         command = "bash"
+         args    = ["-c", "a=`cksum <<< \"${NOMAD_ALLOC_ID}\"| cut -d ' ' -f1`; if ! (( a % 2 )); then sleep 5000; else exit -1; fi"]
+      }
+    }
+    restart {
+        attempts = 0
+        delay    = "0s"
+        mode     = "fail"
+    }
+    reschedule {
+      attempts = 2
+      interval = "5m"
+   }
+  }
+}

e2e/rescheduling/input/rescheduling_canary.hcl

@@ -0,0 +1,30 @@
+job "test5" {
+  datacenters = ["dc1"]
+  type = "service"
+
+  group "t5" {
+    count = 3
+    task "t5" {
+      driver = "raw_exec"
+      config {
+         command = "bash"
+         args    = ["-c", "sleep 5000"]
+      }
+    }
+    update {
+       max_parallel = 1
+       canary = 1
+       min_healthy_time = "1s"
+       auto_revert = false
+    }
+    restart {
+        attempts = 0
+        delay    = "0s"
+        mode     = "fail"
+    }
+    reschedule {
+      attempts = 3
+      interval = "5m"
+   }
+  }
+}

e2e/rescheduling/input/rescheduling_fail.hcl

@@ -0,0 +1,24 @@
+job "test2" {
+  datacenters = ["dc1"]
+  type = "service"
+
+  group "t2" {
+    count = 3
+    task "t2" {
+      driver = "raw_exec"
+      config {
+         command = "bash"
+         args    = ["-c", "lol 5000"]
+      }
+    }
+    restart {
+        attempts = 0
+        delay    = "0s"
+        mode     = "fail"
+    }
+    reschedule {
+      attempts = 2
+      interval = "5m"
+   }
+  }
+}

e2e/rescheduling/input/rescheduling_update.hcl

@@ -0,0 +1,29 @@
+job "test4" {
+  datacenters = ["dc1"]
+  type = "service"
+
+  group "t4" {
+    count = 3
+    task "t4" {
+      driver = "raw_exec"
+      config {
+         command = "bash"
+         args    = ["-c", "sleep 5000"]
+      }
+    }
+    update {
+       max_parallel = 1
+       min_healthy_time = "10s"
+       auto_revert = false
+    }
+    restart {
+        attempts = 0
+        delay    = "0s"
+        mode     = "fail"
+    }
+    reschedule {
+      attempts = 3
+      interval = "5m"
+   }
+  }
+}

e2e/rescheduling/server_side_restarts_suite_test.go

@@ -0,0 +1,19 @@
+package rescheduling
+
+import (
+	"flag"
+	"testing"
+
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+)
+
+var integration = flag.Bool("integration", false, "run integration tests")
+
+func TestServerSideRestarts(t *testing.T) {
+	if !*integration {
+		t.Skip("skipping test in non-integration mode.")
+	}
+	RegisterFailHandler(Fail)
+	RunSpecs(t, "Server Side Restart Tests")
+}

e2e/rescheduling/server_side_restarts_test.go

@@ -0,0 +1,154 @@
+package rescheduling
+
+import (
+	"time"
+
+	"github.com/hashicorp/nomad/api"
+	"github.com/hashicorp/nomad/jobspec"
+	_ "github.com/hashicorp/nomad/jobspec"
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+)
+
+var _ = Describe("Server Side Restart Tests", func() {
+
+	var (
+		jobs     *api.Jobs
+		system   *api.System
+		job      *api.Job
+		err      error
+		specFile string
+
+		// allocStatuses is a helper function that pulls
+		// out client statuses from a slice of allocs
+		allocStatuses = func() []string {
+			allocs, _, err := jobs.Allocations(*job.ID, false, nil)
+			Expect(err).ShouldNot(HaveOccurred())
+			var ret []string
+			for _, a := range allocs {
+				ret = append(ret, a.ClientStatus)
+			}
+			return ret
+		}
+
+		// allocStatusesRescheduled is a helper function that pulls
+		// out client statuses only from rescheduled allocs
+		allocStatusesRescheduled = func() []string {
+			allocs, _, err := jobs.Allocations(*job.ID, false, nil)
+			Expect(err).ShouldNot(HaveOccurred())
+			var ret []string
+			for _, a := range allocs {
+				if a.RescheduleTracker != nil && len(a.RescheduleTracker.Events) > 0 {
+					ret = append(ret, a.ClientStatus)
+				}
+			}
+			return ret
+		}
+	)
+
+	BeforeSuite(func() {
+		conf := api.DefaultConfig()
+		conf.Address = "http://localhost:4646"
+
+		// Create client
+		client, err := api.NewClient(conf)
+		Expect(err).ShouldNot(HaveOccurred())
+		jobs = client.Jobs()
+		system = client.System()
+	})
+
+	JustBeforeEach(func() {
+		job, err = jobspec.ParseFile(specFile)
+		Expect(err).ShouldNot(HaveOccurred())
+
+		resp, _, err := jobs.Register(job, nil)
+		Expect(err).ShouldNot(HaveOccurred())
+		Expect(resp.EvalID).ShouldNot(BeEmpty())
+
+	})
+
+	AfterEach(func() {
+		//Deregister job
+		jobs.Deregister(*job.ID, true, nil)
+		system.GarbageCollect()
+	})
+
+	Describe("Reschedule Stanza Tests", func() {
+
+		Context("No reschedule attempts", func() {
+			BeforeEach(func() {
+				specFile = "input/norescheduling.hcl"
+			})
+
+			It("Should have exactly three allocs and all failed", func() {
+				Eventually(allocStatuses, 5*time.Second, time.Second).Should(ConsistOf([]string{"failed", "failed", "failed"}))
+			})
+		})
+
+		Context("Reschedule attempts maxed out", func() {
+			BeforeEach(func() {
+				specFile = "input/rescheduling_fail.hcl"
+			})
+			// Expect 3 original plus 6 rescheduled allocs from 2 attempts
+			var expected []string
+			for i := 0; i < 9; i++ {
+				expected = append(expected, "failed")
+			}
+			It("Should have all failed", func() {
+				Eventually(allocStatuses, 5*time.Second, time.Second).ShouldNot(
+					SatisfyAll(ContainElement("pending"),
+						ContainElement("running")))
+			})
+		})
+
+		Context("Reschedule attempts succeeded", func() {
+			BeforeEach(func() {
+				specFile = "input/reschedule_success.hcl"
+			})
+			It("Should have some running allocs", func() {
+				Eventually(allocStatuses, 5*time.Second, time.Second).Should(
+					ContainElement("running"))
+			})
+		})
+
+		Context("Reschedule with update stanza", func() {
+			BeforeEach(func() {
+				specFile = "input/rescheduling_update.hcl"
+			})
+			It("Should have all running allocs", func() {
+				Eventually(allocStatuses, 3*time.Second, time.Second).Should(
+					ConsistOf([]string{"running", "running", "running"}))
+			})
+			Context("Updating job to make allocs fail", func() {
+				It("Should have no rescheduled allocs", func() {
+					job.TaskGroups[0].Tasks[0].Config["args"] = []string{"-c", "lol"}
+					_, _, err := jobs.Register(job, nil)
+					Expect(err).ShouldNot(HaveOccurred())
+					Eventually(allocStatusesRescheduled, 2*time.Second, time.Second).Should(BeEmpty())
+				})
+			})
+
+		})
+
+		Context("Reschedule with canary", func() {
+			BeforeEach(func() {
+				specFile = "input/rescheduling_canary.hcl"
+			})
+			It("Should have all running allocs", func() {
+				Eventually(allocStatuses, 3*time.Second, time.Second).Should(
+					ConsistOf([]string{"running", "running", "running"}))
+			})
+			Context("Updating job to make allocs fail", func() {
+				It("Should have no rescheduled allocs", func() {
+					job.TaskGroups[0].Tasks[0].Config["args"] = []string{"-c", "lol"}
+					_, _, err := jobs.Register(job, nil)
+					Expect(err).ShouldNot(HaveOccurred())
+					Eventually(allocStatusesRescheduled, 2*time.Second, time.Second).Should(BeEmpty())
+				})
+			})
+
+		})
+
+	})
+
+})