e2e: add job scaling test suite.

e2e/e2e_test.go

@@ -27,9 +27,10 @@ import (
 	_ "github.com/hashicorp/nomad/e2e/podman"
 	_ "github.com/hashicorp/nomad/e2e/quotas"
 	_ "github.com/hashicorp/nomad/e2e/rescheduling"
+	_ "github.com/hashicorp/nomad/e2e/scaling"
+	_ "github.com/hashicorp/nomad/e2e/scalingpolicies"
 	_ "github.com/hashicorp/nomad/e2e/spread"
 	_ "github.com/hashicorp/nomad/e2e/systemsched"
-	_ "github.com/hashicorp/nomad/e2e/scalingpolicies"
 	_ "github.com/hashicorp/nomad/e2e/taskevents"
 	_ "github.com/hashicorp/nomad/e2e/vaultsecrets"
 	_ "github.com/hashicorp/nomad/e2e/volumes"

e2e/scaling/input/namespace_a_1.nomad

@@ -0,0 +1,34 @@
+job "horizontally_scalable" {
+  datacenters = ["dc1"]
+  type        = "service"
+  namespace   = "NamespaceA"
+
+  update {
+    health_check = "task_states"
+  }
+
+  constraint {
+    attribute = "${attr.kernel.name}"
+    value     = "linux"
+  }
+
+  group "horizontally_scalable" {
+
+    scaling {
+      min     = 2
+      max     = 3
+      enabled = true
+
+      policy {}
+    }
+
+    task "test" {
+      driver = "raw_exec"
+
+      config {
+        command = "bash"
+        args    = ["-c", "sleep 15000"]
+      }
+    }
+  }
+}

e2e/scaling/input/namespace_default_1.nomad

@@ -0,0 +1,33 @@
+job "horizontally_scalable" {
+  datacenters = ["dc1"]
+  type        = "service"
+
+  update {
+    health_check = "task_states"
+  }
+
+  constraint {
+    attribute = "${attr.kernel.name}"
+    value     = "linux"
+  }
+
+  group "horizontally_scalable" {
+
+    scaling {
+      min     = 2
+      max     = 3
+      enabled = true
+
+      policy {}
+    }
+
+    task "test" {
+      driver = "raw_exec"
+
+      config {
+        command = "bash"
+        args    = ["-c", "sleep 15000"]
+      }
+    }
+  }
+}

e2e/scaling/input/namespace_default_2.nomad

@@ -0,0 +1,35 @@
+job "horizontally_scalable" {
+  datacenters = ["dc1"]
+  type        = "service"
+
+  update {
+    health_check = "task_states"
+  }
+
+  constraint {
+    attribute = "${attr.kernel.name}"
+    value     = "linux"
+  }
+
+  group "horizontally_scalable" {
+
+    count = 4
+
+    scaling {
+      min     = 2
+      max     = 3
+      enabled = true
+
+      policy {}
+    }
+
+    task "test" {
+      driver = "raw_exec"
+
+      config {
+        command = "bash"
+        args    = ["-c", "sleep 15000"]
+      }
+    }
+  }
+}

e2e/scaling/input/namespace_default_3.nomad

@@ -0,0 +1,35 @@
+job "horizontally_scalable" {
+  datacenters = ["dc1"]
+  type        = "service"
+
+  update {
+    health_check = "task_states"
+  }
+
+  constraint {
+    attribute = "${attr.kernel.name}"
+    value     = "linux"
+  }
+
+  group "horizontally_scalable" {
+
+    count = 4
+
+    scaling {
+      min     = 5
+      max     = 6
+      enabled = true
+
+      policy {}
+    }
+
+    task "test" {
+      driver = "raw_exec"
+
+      config {
+        command = "bash"
+        args    = ["-c", "sleep 15000"]
+      }
+    }
+  }
+}

e2e/scaling/scaling.go

@@ -0,0 +1,164 @@
+package scaling
+
+import (
+	"os"
+
+	"github.com/hashicorp/nomad/api"
+	"github.com/hashicorp/nomad/e2e/e2eutil"
+	"github.com/hashicorp/nomad/e2e/framework"
+	"github.com/hashicorp/nomad/helper"
+	"github.com/hashicorp/nomad/helper/uuid"
+)
+
+type ScalingE2ETest struct {
+	framework.TC
+	namespaceIDs     []string
+	namespacedJobIDs [][2]string
+}
+
+func init() {
+	framework.AddSuites(&framework.TestSuite{
+		Component:   "Scaling",
+		CanRunLocal: true,
+		Cases: []framework.TestCase{
+			new(ScalingE2ETest),
+		},
+	})
+
+}
+
+func (tc *ScalingE2ETest) BeforeAll(f *framework.F) {
+	e2eutil.WaitForLeader(f.T(), tc.Nomad())
+	e2eutil.WaitForNodesReady(f.T(), tc.Nomad(), 1)
+}
+
+func (tc *ScalingE2ETest) AfterEach(f *framework.F) {
+	if os.Getenv("NOMAD_TEST_SKIPCLEANUP") == "1" {
+		return
+	}
+
+	for _, namespacedJob := range tc.namespacedJobIDs {
+		_, err := e2eutil.Command("nomad", "job", "stop", "-purge", "-namespace",
+			namespacedJob[0], namespacedJob[1])
+		f.NoError(err)
+	}
+	tc.namespacedJobIDs = [][2]string{}
+
+	for _, ns := range tc.namespaceIDs {
+		_, err := e2eutil.Command("nomad", "namespace", "delete", ns)
+		f.NoError(err)
+	}
+	tc.namespaceIDs = []string{}
+
+	_, err := e2eutil.Command("nomad", "system", "gc")
+	f.NoError(err)
+}
+
+// TestScalingBasic performs basic scaling e2e tests within a single namespace.
+func (tc *ScalingE2ETest) TestScalingBasic(f *framework.F) {
+	defaultNS := "default"
+
+	// Register a job with a scaling policy. The group doesn't include the
+	// count parameter, therefore Nomad should dynamically set this value to
+	// the policy min.
+	jobID := "test-scaling-" + uuid.Generate()[0:8]
+	f.NoError(e2eutil.Register(jobID, "scaling/input/namespace_default_1.nomad"))
+	tc.namespacedJobIDs = append(tc.namespacedJobIDs, [2]string{defaultNS, jobID})
+	f.NoError(e2eutil.WaitForAllocStatusExpected(jobID, defaultNS, []string{"running", "running"}),
+		"job should be running with 2 allocs")
+
+	// Ensure we wait for the deployment to finish, otherwise scaling will
+	// fail.
+	f.NoError(e2eutil.WaitForLastDeploymentStatus(jobID, defaultNS, "successful", nil))
+
+	// Simple scaling action.
+	testMeta := map[string]interface{}{"scaling-e2e-test": "value"}
+	scaleResp, _, err := tc.Nomad().Jobs().Scale(
+		jobID, "horizontally_scalable", helper.IntToPtr(3),
+		"Nomad e2e testing", false, testMeta, nil)
+	f.NoError(err)
+	f.NotEmpty(scaleResp.EvalID)
+	f.NoError(e2eutil.WaitForAllocStatusExpected(jobID, defaultNS, []string{"running", "running", "running"}),
+		"job should be running with 3 allocs")
+
+	// Ensure we wait for the deployment to finish, otherwise scaling will
+	// fail for this reason.
+	f.NoError(e2eutil.WaitForLastDeploymentStatus(jobID, defaultNS, "successful", nil))
+
+	// Attempt break break the policy min/max parameters.
+	_, _, err = tc.Nomad().Jobs().Scale(
+		jobID, "horizontally_scalable", helper.IntToPtr(4),
+		"Nomad e2e testing", false, nil, nil)
+	f.Error(err)
+	_, _, err = tc.Nomad().Jobs().Scale(
+		jobID, "horizontally_scalable", helper.IntToPtr(1),
+		"Nomad e2e testing", false, nil, nil)
+	f.Error(err)
+
+	// Check the scaling events.
+	statusResp, _, err := tc.Nomad().Jobs().ScaleStatus(jobID, nil)
+	f.NoError(err)
+	f.Len(statusResp.TaskGroups["horizontally_scalable"].Events, 1)
+	f.Equal(testMeta, statusResp.TaskGroups["horizontally_scalable"].Events[0].Meta)
+
+	// Remove the job.
+	_, _, err = tc.Nomad().Jobs().Deregister(jobID, true, nil)
+	f.NoError(err)
+	f.NoError(tc.Nomad().System().GarbageCollect())
+	tc.namespacedJobIDs = [][2]string{}
+
+	// Attempt job registrations where the group count violates the policy
+	// min/max parameters.
+	f.Error(e2eutil.Register(jobID, "scaling/input/namespace_default_2.nomad"))
+	f.Error(e2eutil.Register(jobID, "scaling/input/namespace_default_3.nomad"))
+}
+
+// TestScalingNamespaces runs tests to ensure the job scaling endpoint adheres
+// to Nomad's basic namespace principles.
+func (tc *ScalingE2ETest) TestScalingNamespaces(f *framework.F) {
+
+	defaultNS := "default"
+	ANS := "NamespaceA"
+
+	// Create our non-default namespace.
+	_, err := e2eutil.Command("nomad", "namespace", "apply", ANS)
+	f.NoError(err, "could not create namespace")
+	tc.namespaceIDs = append(tc.namespaceIDs, ANS)
+
+	defaultJobID := "test-scaling-default-" + uuid.Generate()[0:8]
+	aJobID := "test-scaling-a-" + uuid.Generate()[0:8]
+
+	// Register and wait for the job deployments to succeed.
+	f.NoError(e2eutil.Register(defaultJobID, "scaling/input/namespace_default_1.nomad"))
+	f.NoError(e2eutil.Register(aJobID, "scaling/input/namespace_a_1.nomad"))
+	f.NoError(e2eutil.WaitForLastDeploymentStatus(defaultJobID, defaultNS, "successful", nil))
+	f.NoError(e2eutil.WaitForLastDeploymentStatus(aJobID, ANS, "successful", nil))
+
+	tc.namespacedJobIDs = append(tc.namespacedJobIDs, [2]string{defaultNS, defaultJobID})
+	tc.namespacedJobIDs = append(tc.namespacedJobIDs, [2]string{ANS, aJobID})
+
+	// Setup the WriteOptions for each namespace.
+	defaultWriteOpts := api.WriteOptions{Namespace: defaultNS}
+	aWriteOpts := api.WriteOptions{Namespace: ANS}
+
+	// We shouldn't be able to trigger scaling across the namespace boundary.
+	_, _, err = tc.Nomad().Jobs().Scale(
+		defaultJobID, "horizontally_scalable", helper.IntToPtr(3),
+		"Nomad e2e testing", false, nil, &aWriteOpts)
+	f.Error(err)
+	_, _, err = tc.Nomad().Jobs().Scale(
+		aJobID, "horizontally_scalable", helper.IntToPtr(3),
+		"Nomad e2e testing", false, nil, &defaultWriteOpts)
+	f.Error(err)
+
+	// We should be able to trigger scaling when using the correct namespace,
+	// duh.
+	_, _, err = tc.Nomad().Jobs().Scale(
+		defaultJobID, "horizontally_scalable", helper.IntToPtr(3),
+		"Nomad e2e testing", false, nil, &defaultWriteOpts)
+	f.NoError(err)
+	_, _, err = tc.Nomad().Jobs().Scale(
+		aJobID, "horizontally_scalable", helper.IntToPtr(3),
+		"Nomad e2e testing", false, nil, &aWriteOpts)
+	f.NoError(err)
+}