-
Notifications
You must be signed in to change notification settings - Fork 2k
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Merge pull request #9767 from hashicorp/f-e2e-job-scaling-suite
e2e: add job scaling test suite.
- Loading branch information
Showing
6 changed files
with
303 additions
and
1 deletion.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,34 @@ | ||
| job "horizontally_scalable" { | ||
| datacenters = ["dc1"] | ||
| type = "service" | ||
| namespace = "NamespaceA" | ||
|
|
||
| update { | ||
| health_check = "task_states" | ||
| } | ||
|
|
||
| constraint { | ||
| attribute = "${attr.kernel.name}" | ||
| value = "linux" | ||
| } | ||
|
|
||
| group "horizontally_scalable" { | ||
|
|
||
| scaling { | ||
| min = 2 | ||
| max = 3 | ||
| enabled = true | ||
|
|
||
| policy {} | ||
| } | ||
|
|
||
| task "test" { | ||
| driver = "raw_exec" | ||
|
|
||
| config { | ||
| command = "bash" | ||
| args = ["-c", "sleep 15000"] | ||
| } | ||
| } | ||
| } | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,33 @@ | ||
| job "horizontally_scalable" { | ||
| datacenters = ["dc1"] | ||
| type = "service" | ||
|
|
||
| update { | ||
| health_check = "task_states" | ||
| } | ||
|
|
||
| constraint { | ||
| attribute = "${attr.kernel.name}" | ||
| value = "linux" | ||
| } | ||
|
|
||
| group "horizontally_scalable" { | ||
|
|
||
| scaling { | ||
| min = 2 | ||
| max = 3 | ||
| enabled = true | ||
|
|
||
| policy {} | ||
| } | ||
|
|
||
| task "test" { | ||
| driver = "raw_exec" | ||
|
|
||
| config { | ||
| command = "bash" | ||
| args = ["-c", "sleep 15000"] | ||
| } | ||
| } | ||
| } | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,35 @@ | ||
| job "horizontally_scalable" { | ||
| datacenters = ["dc1"] | ||
| type = "service" | ||
|
|
||
| update { | ||
| health_check = "task_states" | ||
| } | ||
|
|
||
| constraint { | ||
| attribute = "${attr.kernel.name}" | ||
| value = "linux" | ||
| } | ||
|
|
||
| group "horizontally_scalable" { | ||
|
|
||
| count = 4 | ||
|
|
||
| scaling { | ||
| min = 2 | ||
| max = 3 | ||
| enabled = true | ||
|
|
||
| policy {} | ||
| } | ||
|
|
||
| task "test" { | ||
| driver = "raw_exec" | ||
|
|
||
| config { | ||
| command = "bash" | ||
| args = ["-c", "sleep 15000"] | ||
| } | ||
| } | ||
| } | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,35 @@ | ||
| job "horizontally_scalable" { | ||
| datacenters = ["dc1"] | ||
| type = "service" | ||
|
|
||
| update { | ||
| health_check = "task_states" | ||
| } | ||
|
|
||
| constraint { | ||
| attribute = "${attr.kernel.name}" | ||
| value = "linux" | ||
| } | ||
|
|
||
| group "horizontally_scalable" { | ||
|
|
||
| count = 4 | ||
|
|
||
| scaling { | ||
| min = 5 | ||
| max = 6 | ||
| enabled = true | ||
|
|
||
| policy {} | ||
| } | ||
|
|
||
| task "test" { | ||
| driver = "raw_exec" | ||
|
|
||
| config { | ||
| command = "bash" | ||
| args = ["-c", "sleep 15000"] | ||
| } | ||
| } | ||
| } | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,164 @@ | ||
| package scaling | ||
|
|
||
| import ( | ||
| "os" | ||
|
|
||
| "github.com/hashicorp/nomad/api" | ||
| "github.com/hashicorp/nomad/e2e/e2eutil" | ||
| "github.com/hashicorp/nomad/e2e/framework" | ||
| "github.com/hashicorp/nomad/helper" | ||
| "github.com/hashicorp/nomad/helper/uuid" | ||
| ) | ||
|
|
||
| type ScalingE2ETest struct { | ||
| framework.TC | ||
| namespaceIDs []string | ||
| namespacedJobIDs [][2]string | ||
| } | ||
|
|
||
| func init() { | ||
| framework.AddSuites(&framework.TestSuite{ | ||
| Component: "Scaling", | ||
| CanRunLocal: true, | ||
| Cases: []framework.TestCase{ | ||
| new(ScalingE2ETest), | ||
| }, | ||
| }) | ||
|
|
||
| } | ||
|
|
||
| func (tc *ScalingE2ETest) BeforeAll(f *framework.F) { | ||
| e2eutil.WaitForLeader(f.T(), tc.Nomad()) | ||
| e2eutil.WaitForNodesReady(f.T(), tc.Nomad(), 1) | ||
| } | ||
|
|
||
| func (tc *ScalingE2ETest) AfterEach(f *framework.F) { | ||
| if os.Getenv("NOMAD_TEST_SKIPCLEANUP") == "1" { | ||
| return | ||
| } | ||
|
|
||
| for _, namespacedJob := range tc.namespacedJobIDs { | ||
| _, err := e2eutil.Command("nomad", "job", "stop", "-purge", "-namespace", | ||
| namespacedJob[0], namespacedJob[1]) | ||
| f.NoError(err) | ||
| } | ||
| tc.namespacedJobIDs = [][2]string{} | ||
|
|
||
| for _, ns := range tc.namespaceIDs { | ||
| _, err := e2eutil.Command("nomad", "namespace", "delete", ns) | ||
| f.NoError(err) | ||
| } | ||
| tc.namespaceIDs = []string{} | ||
|
|
||
| _, err := e2eutil.Command("nomad", "system", "gc") | ||
| f.NoError(err) | ||
| } | ||
|
|
||
| // TestScalingBasic performs basic scaling e2e tests within a single namespace. | ||
| func (tc *ScalingE2ETest) TestScalingBasic(f *framework.F) { | ||
| defaultNS := "default" | ||
|
|
||
| // Register a job with a scaling policy. The group doesn't include the | ||
| // count parameter, therefore Nomad should dynamically set this value to | ||
| // the policy min. | ||
| jobID := "test-scaling-" + uuid.Generate()[0:8] | ||
| f.NoError(e2eutil.Register(jobID, "scaling/input/namespace_default_1.nomad")) | ||
| tc.namespacedJobIDs = append(tc.namespacedJobIDs, [2]string{defaultNS, jobID}) | ||
| f.NoError(e2eutil.WaitForAllocStatusExpected(jobID, defaultNS, []string{"running", "running"}), | ||
| "job should be running with 2 allocs") | ||
|
|
||
| // Ensure we wait for the deployment to finish, otherwise scaling will | ||
| // fail. | ||
| f.NoError(e2eutil.WaitForLastDeploymentStatus(jobID, defaultNS, "successful", nil)) | ||
|
|
||
| // Simple scaling action. | ||
| testMeta := map[string]interface{}{"scaling-e2e-test": "value"} | ||
| scaleResp, _, err := tc.Nomad().Jobs().Scale( | ||
| jobID, "horizontally_scalable", helper.IntToPtr(3), | ||
| "Nomad e2e testing", false, testMeta, nil) | ||
| f.NoError(err) | ||
| f.NotEmpty(scaleResp.EvalID) | ||
| f.NoError(e2eutil.WaitForAllocStatusExpected(jobID, defaultNS, []string{"running", "running", "running"}), | ||
| "job should be running with 3 allocs") | ||
|
|
||
| // Ensure we wait for the deployment to finish, otherwise scaling will | ||
| // fail for this reason. | ||
| f.NoError(e2eutil.WaitForLastDeploymentStatus(jobID, defaultNS, "successful", nil)) | ||
|
|
||
| // Attempt break break the policy min/max parameters. | ||
| _, _, err = tc.Nomad().Jobs().Scale( | ||
| jobID, "horizontally_scalable", helper.IntToPtr(4), | ||
| "Nomad e2e testing", false, nil, nil) | ||
| f.Error(err) | ||
| _, _, err = tc.Nomad().Jobs().Scale( | ||
| jobID, "horizontally_scalable", helper.IntToPtr(1), | ||
| "Nomad e2e testing", false, nil, nil) | ||
| f.Error(err) | ||
|
|
||
| // Check the scaling events. | ||
| statusResp, _, err := tc.Nomad().Jobs().ScaleStatus(jobID, nil) | ||
| f.NoError(err) | ||
| f.Len(statusResp.TaskGroups["horizontally_scalable"].Events, 1) | ||
| f.Equal(testMeta, statusResp.TaskGroups["horizontally_scalable"].Events[0].Meta) | ||
|
|
||
| // Remove the job. | ||
| _, _, err = tc.Nomad().Jobs().Deregister(jobID, true, nil) | ||
| f.NoError(err) | ||
| f.NoError(tc.Nomad().System().GarbageCollect()) | ||
| tc.namespacedJobIDs = [][2]string{} | ||
|
|
||
| // Attempt job registrations where the group count violates the policy | ||
| // min/max parameters. | ||
| f.Error(e2eutil.Register(jobID, "scaling/input/namespace_default_2.nomad")) | ||
| f.Error(e2eutil.Register(jobID, "scaling/input/namespace_default_3.nomad")) | ||
| } | ||
|
|
||
| // TestScalingNamespaces runs tests to ensure the job scaling endpoint adheres | ||
| // to Nomad's basic namespace principles. | ||
| func (tc *ScalingE2ETest) TestScalingNamespaces(f *framework.F) { | ||
|
|
||
| defaultNS := "default" | ||
| ANS := "NamespaceA" | ||
|
|
||
| // Create our non-default namespace. | ||
| _, err := e2eutil.Command("nomad", "namespace", "apply", ANS) | ||
| f.NoError(err, "could not create namespace") | ||
| tc.namespaceIDs = append(tc.namespaceIDs, ANS) | ||
|
|
||
| defaultJobID := "test-scaling-default-" + uuid.Generate()[0:8] | ||
| aJobID := "test-scaling-a-" + uuid.Generate()[0:8] | ||
|
|
||
| // Register and wait for the job deployments to succeed. | ||
| f.NoError(e2eutil.Register(defaultJobID, "scaling/input/namespace_default_1.nomad")) | ||
| f.NoError(e2eutil.Register(aJobID, "scaling/input/namespace_a_1.nomad")) | ||
| f.NoError(e2eutil.WaitForLastDeploymentStatus(defaultJobID, defaultNS, "successful", nil)) | ||
| f.NoError(e2eutil.WaitForLastDeploymentStatus(aJobID, ANS, "successful", nil)) | ||
|
|
||
| tc.namespacedJobIDs = append(tc.namespacedJobIDs, [2]string{defaultNS, defaultJobID}) | ||
| tc.namespacedJobIDs = append(tc.namespacedJobIDs, [2]string{ANS, aJobID}) | ||
|
|
||
| // Setup the WriteOptions for each namespace. | ||
| defaultWriteOpts := api.WriteOptions{Namespace: defaultNS} | ||
| aWriteOpts := api.WriteOptions{Namespace: ANS} | ||
|
|
||
| // We shouldn't be able to trigger scaling across the namespace boundary. | ||
| _, _, err = tc.Nomad().Jobs().Scale( | ||
| defaultJobID, "horizontally_scalable", helper.IntToPtr(3), | ||
| "Nomad e2e testing", false, nil, &aWriteOpts) | ||
| f.Error(err) | ||
| _, _, err = tc.Nomad().Jobs().Scale( | ||
| aJobID, "horizontally_scalable", helper.IntToPtr(3), | ||
| "Nomad e2e testing", false, nil, &defaultWriteOpts) | ||
| f.Error(err) | ||
|
|
||
| // We should be able to trigger scaling when using the correct namespace, | ||
| // duh. | ||
| _, _, err = tc.Nomad().Jobs().Scale( | ||
| defaultJobID, "horizontally_scalable", helper.IntToPtr(3), | ||
| "Nomad e2e testing", false, nil, &defaultWriteOpts) | ||
| f.NoError(err) | ||
| _, _, err = tc.Nomad().Jobs().Scale( | ||
| aJobID, "horizontally_scalable", helper.IntToPtr(3), | ||
| "Nomad e2e testing", false, nil, &aWriteOpts) | ||
| f.NoError(err) | ||
| } |