Merge pull request #4461 from hashicorp/f-e2e-framework

E2E: Initial Framework

e2e/e2e.go

@@ -0,0 +1,11 @@
+package e2e
+
+import (
+	"testing"
+
+	"github.com/hashicorp/nomad/e2e/framework"
+)
+
+func RunE2ETests(t *testing.T) {
+	framework.Run(t)
+}

e2e/e2e_test.go

@@ -0,0 +1,11 @@
+package e2e
+
+import (
+	"testing"
+
+	_ "github.com/hashicorp/nomad/e2e/example"
+)
+
+func TestE2E(t *testing.T) {
+	RunE2ETests(t)
+}

e2e/example/e2e_test.go

@@ -0,0 +1,17 @@
+package example
+
+import (
+	"testing"
+
+	"github.com/hashicorp/nomad/e2e/framework"
+)
+
+func TestE2E(t *testing.T) {
+	framework.New().AddSuites(&framework.TestSuite{
+		Component:   "simple",
+		CanRunLocal: true,
+		Cases: []framework.TestCase{
+			new(SimpleExampleTestCase),
+		},
+	}).Run(t)
+}

e2e/example/example.go

@@ -0,0 +1,25 @@
+package example
+
+import (
+	"github.com/hashicorp/nomad/e2e/framework"
+)
+
+func init() {
+	framework.AddSuites(&framework.TestSuite{
+		Component:   "simple",
+		CanRunLocal: true,
+		Cases: []framework.TestCase{
+			new(SimpleExampleTestCase),
+		},
+	})
+}
+
+type SimpleExampleTestCase struct {
+	framework.TC
+}
+
+func (tc *SimpleExampleTestCase) TestExample(f *framework.F) {
+	jobs, _, err := tc.Nomad().Jobs().List(nil)
+	f.NoError(err)
+	f.Empty(jobs)
+}

e2e/framework/case.go

@@ -0,0 +1,79 @@
+package framework
+
+import (
+	"fmt"
+	"testing"
+
+	"github.com/hashicorp/nomad/api"
+)
+
+// TestSuite defines a set of test cases and under what conditions to run them
+type TestSuite struct {
+	Component string // Name of the component/system/feature tested
+
+	CanRunLocal bool        // Flags if the cases are safe to run on a local nomad cluster
+	Cases       []TestCase  // Cases to run
+	Constraints Constraints // Environment constraints to follow
+	Parallel    bool        // If true, will run test cases in parallel
+	Slow        bool        // Slow test suites don't run by default
+
+	// API Clients
+	Consul bool
+	Vault  bool
+}
+
+// Constraints that must be satisfied for a TestSuite to run
+type Constraints struct {
+	Provider    string   // Cloud provider ex. 'aws', 'azure', 'gcp'
+	OS          string   // Operating system ex. 'windows', 'linux'
+	Arch        string   // CPU architecture ex. 'amd64', 'arm64'
+	Environment string   // Environment name ex. 'simple'
+	Tags        []string // Generic tags that must all exist in the environment
+}
+
+func (c Constraints) matches(env Environment) error {
+	if len(c.Provider) != 0 && c.Provider != env.Provider {
+		return fmt.Errorf("provider constraint does not match environment")
+	}
+
+	if len(c.OS) != 0 && c.OS != env.OS {
+		return fmt.Errorf("os constraint does not match environment")
+	}
+
+	if len(c.Arch) != 0 && c.Arch != env.Arch {
+		return fmt.Errorf("arch constraint does not match environment")
+	}
+
+	if len(c.Environment) != 0 && c.Environment != env.Name {
+		return fmt.Errorf("environment constraint does not match environment name")
+	}
+
+	for _, t := range c.Tags {
+		if _, ok := env.Tags[t]; !ok {
+			return fmt.Errorf("tags constraint failed, tag '%s' is not included in environment", t)
+		}
+	}
+	return nil
+}
+
+// TC is the base test case which should be embedded in TestCase implementations.
+type TC struct {
+	t *testing.T
+
+	cluster *ClusterInfo
+}
+
+// Nomad returns a configured nomad api client
+func (tc *TC) Nomad() *api.Client {
+	return tc.cluster.NomadClient
+}
+
+// Name returns the name of the test case which is set to the name of the
+// implementing type.
+func (tc *TC) Name() string {
+	return tc.cluster.Name
+}
+
+func (tc *TC) setClusterInfo(info *ClusterInfo) {
+	tc.cluster = info
+}

e2e/framework/context.go

@@ -0,0 +1,69 @@
+package framework
+
+import (
+	"testing"
+
+	"github.com/hashicorp/nomad/helper/uuid"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+// F is the framework context that is passed to each test.
+// It is used to access the *testing.T context as well as testify helpers
+type F struct {
+	id string
+	*require.Assertions
+	assert *assert.Assertions
+	t      *testing.T
+
+	data map[interface{}]interface{}
+}
+
+func newF(t *testing.T) *F {
+	return newFWithID(uuid.Generate()[:8], t)
+}
+
+func newFFromParent(f *F, t *testing.T) *F {
+	child := newF(t)
+	for k, v := range f.data {
+		child.Set(k, v)
+	}
+	return child
+}
+
+func newFWithID(id string, t *testing.T) *F {
+	ft := &F{
+		id:         id,
+		t:          t,
+		Assertions: require.New(t),
+		assert:     assert.New(t),
+	}
+
+	return ft
+}
+
+// Assert fetches an assert flavor of testify assertions
+// https://godoc.org/github.com/stretchr/testify/assert
+func (f *F) Assert() *assert.Assertions {
+	return f.assert
+}
+
+// T returns the *testing.T context
+func (f *F) T() *testing.T {
+	return f.t
+}
+
+// ID returns the current context ID
+func (f *F) ID() string {
+	return f.id
+}
+
+// Set is used to set arbitrary key/values to pass between before/after and test methods
+func (f *F) Set(key, val interface{}) {
+	f.data[key] = val
+}
+
+// Value retrives values set by the F.Set method
+func (f *F) Value(key interface{}) interface{} {
+	return f.data[key]
+}

e2e/framework/doc.go

@@ -0,0 +1,124 @@
+/*
+Package framework implements a model for developing end-to-end test suites. The
+model includes a top level Framework which TestSuites can be added to. TestSuites
+include conditions under which the suite will run and a list of TestCase
+implementations to run. TestCases can be implemented with methods that run
+before/after each and all tests.
+
+Writing Tests
+
+Tests follow a similar patterns as go tests. They are functions that must start
+with 'Test' and instead of a *testing.T argument, a *framework.F is passed and
+they must have a receiver that implements the TestCase interface.
+A crude example as follows:
+
+	// foo_test.go
+	type MyTestCase struct {
+		framework.TC
+	}
+
+	func (tc *MyTestCase) TestMyFoo(f *framework.F) {
+		f.T().Log("bar")
+	}
+
+	func TestCalledFromGoTest(t *testing.T){
+		framework.New().AddSuites(&framework.TestSuite{
+			Component:   "foo",
+			Cases: []framework.TestCase{
+				new(MyTestCase),
+			},
+		}).Run(t)
+	}
+
+Test cases should embed the TC struct which satisfies the TestCase interface.
+Optionally a TestCase can also implement the Name() function which returns
+a string to name the test case. By default the name is the name of the struct
+type, which in the above example would be "MyTestCase"
+
+Test cases may also optionally implement additional interfaces to define setup
+and teardown logic:
+
+	BeforeEachTest
+	AfterEachTest
+	BeforeAllTests
+	AfterAllTests
+
+The test case struct allows you to setup and teardown state in the struct that
+can be consumed by the tests. For example:
+
+	type ComplexNomadTC struct {
+		framework.TC
+		jobID string
+	}
+
+	func (tc *ComplexNomadTC) BeforeEach(f *framework.F){
+		// Do some complex job setup with a unique prefix string
+		jobID, err := doSomeComplexSetup(tc.Nomad(), f.ID())
+		f.NoError(err)
+		f.Set("jobID", jobID)
+	}
+
+	func (tc *ComplexNomadTC) TestSomeScenario(f *framework.F){
+		jobID := f.Value("jobID").(string)
+		doTestThingWithJob(f, tc.Nomad(), jobID)
+	}
+
+	func (tc *ComplexNomadTC) TestOtherScenario(f *framework.F){
+		jobID := f.Value("jobID").(string)
+		doOtherTestThingWithJob(f, tc.Nomad(), jobID)
+	}
+
+	func (tc *ComplexNomadTC) AfterEach(f *framework.F){
+		jobID := f.Value("jobID").(string)
+		_, _, err := tc.Nomad().Jobs().Deregister(jobID, true, nil)
+		f.NoError(err)
+	}
+
+As demonstrated in the previous example, TC also exposes functions that return
+configured api clients including Nomad, Consul and Vault. If Consul or Vault
+are not provisioned their respective getter functions will return nil.
+
+Testify Integration
+
+Test cases expose a T() function to fetch the current *testing.T context.
+While this means the author is able to most other testing libraries,
+github.com/stretch/testify is recommended and integrated into the framework.
+The TC struct also embeds testify assertions that are preconfigured with the
+current testing context. Additionally TC comes with a Require() method that
+yields a testify Require if that flavor is desired.
+
+	func (tc *MyTestCase) TestWithTestify() {
+		err := someErrFunc()
+		tc.NoError(err)
+		// Or tc.Require().NoError(err)
+	}
+
+Parallelism
+
+The test framework honors go test's parallel feature under certain conditions.
+A TestSuite can be created with the Parallel field set to true to enable
+parallel execution of the test cases of the suite. Tests within a test case
+will be executed sequentially unless f.T().Parallel() is called. Note that if
+multiple tests are to be executed in parallel, access to TC is not syncronized.
+The *framework.F offers a way to store state between before/after each method if
+desired.
+
+	func (tc *MyTestCase) BeforeEach(f *framework.F){
+		jobID, _ := doSomeComplexSetup(tc.Nomad(), f.ID())
+		f.Set("jobID", jobID)
+	}
+
+	func (tc *MyTestCase) TestParallel(f *framework.F){
+		f.T().Parallel()
+		jobID := f.Value("jobID").(string)
+	}
+
+Since test cases have the potential to work with a shared Nomad cluster in parallel
+any resources created or destroyed must be prefixed with a unique identifier for
+each test case. The framework.F struct exposes an ID() function that will return a
+string that is unique with in a test. Therefore, multiple tests with in the case
+can reliably create unique IDs between tests and setup/teardown. The string
+returned is 8 alpha numeric characters.
+
+*/
+package framework