From 0ee63c43eac97d30fdb05b36ed7070f2337e104d Mon Sep 17 00:00:00 2001 From: Alex Dadgar Date: Tue, 18 Sep 2018 18:36:19 -0700 Subject: [PATCH 1/7] add a vault test matrix --- e2e/vault/consts.go | 74 +++++++++++ e2e/vault/matrix.go | 33 +++++ e2e/vault/vault_test.go | 274 ++++++++++++++++++++++++++++++++++++++++ testutil/vault.go | 17 ++- 4 files changed, 393 insertions(+), 5 deletions(-) create mode 100644 e2e/vault/consts.go create mode 100644 e2e/vault/matrix.go create mode 100644 e2e/vault/vault_test.go diff --git a/e2e/vault/consts.go b/e2e/vault/consts.go new file mode 100644 index 00000000000..f62eae073ef --- /dev/null +++ b/e2e/vault/consts.go @@ -0,0 +1,74 @@ +package vault + +import ( + "github.com/hashicorp/nomad/api" + "github.com/hashicorp/nomad/helper" +) + +const ( + // policy is the recommended Nomad Vault policy + policy = `path "auth/token/create/nomad-cluster" { + capabilities = ["update"] +} +path "auth/token/roles/nomad-cluster" { + capabilities = ["read"] +} +path "auth/token/lookup-self" { + capabilities = ["read"] +} + +path "auth/token/lookup" { + capabilities = ["update"] +} +path "auth/token/revoke-accessor" { + capabilities = ["update"] +} +path "sys/capabilities-self" { + capabilities = ["update"] +} +path "auth/token/renew-self" { + capabilities = ["update"] +}` +) + +var ( + // role is the recommended nomad cluster role + role = map[string]interface{}{ + "disallowed_policies": "nomad-server", + "explicit_max_ttl": 0, + "name": "nomad-cluster", + "orphan": false, + "period": 259200, + "renewable": true, + } + + // job is a test job that is used to request a Vault token and cat the token + // out before exiting. + job = &api.Job{ + ID: helper.StringToPtr("test"), + Type: helper.StringToPtr("batch"), + Datacenters: []string{"dc1"}, + TaskGroups: []*api.TaskGroup{ + { + Name: helper.StringToPtr("test"), + Tasks: []*api.Task{ + { + Name: "test", + Driver: "raw_exec", + Config: map[string]interface{}{ + "command": "cat", + "args": []string{"${NOMAD_SECRETS_DIR}/vault_token"}, + }, + Vault: &api.Vault{ + Policies: []string{"default"}, + }, + }, + }, + RestartPolicy: &api.RestartPolicy{ + Attempts: helper.IntToPtr(0), + Mode: helper.StringToPtr("fail"), + }, + }, + }, + } +) diff --git a/e2e/vault/matrix.go b/e2e/vault/matrix.go new file mode 100644 index 00000000000..87325bb250c --- /dev/null +++ b/e2e/vault/matrix.go @@ -0,0 +1,33 @@ +package vault + +var ( + // versions is the set of Vault versions we test for backwards compatibility + versions = []string{ + "0.11.1", + "0.11.0", + "0.10.4", + "0.10.3", + "0.10.2", + "0.10.1", + "0.10.0", + "0.9.6", + "0.9.5", + "0.9.4", + "0.9.3", + "0.9.2", + "0.9.1", + "0.9.0", + "0.8.3", + "0.8.2", + "0.8.1", + "0.8.0", + "0.7.3", + "0.7.2", + "0.7.1", + "0.7.0", + "0.6.5", + "0.6.4", + "0.6.3", + "0.6.2", + } +) diff --git a/e2e/vault/vault_test.go b/e2e/vault/vault_test.go new file mode 100644 index 00000000000..a52f2f66b9d --- /dev/null +++ b/e2e/vault/vault_test.go @@ -0,0 +1,274 @@ +package vault + +import ( + "archive/zip" + "bytes" + "context" + "fmt" + "io" + "io/ioutil" + "net/http" + "os" + "path/filepath" + "runtime" + "testing" + "time" + + "github.com/hashicorp/nomad/command/agent" + "github.com/hashicorp/nomad/helper" + "github.com/hashicorp/nomad/nomad/structs/config" + "github.com/hashicorp/nomad/testutil" + "github.com/stretchr/testify/require" + "golang.org/x/sync/errgroup" + + vapi "github.com/hashicorp/vault/api" +) + +// harness is used to retrieve the required Vault test binaries +type harness struct { + t *testing.T + binDir string + os string + arch string +} + +// newHarness returns a new Vault test harness. +func newHarness(t *testing.T) *harness { + return &harness{ + t: t, + binDir: filepath.Join(os.TempDir(), "vault-bins/"), + os: runtime.GOOS, + arch: runtime.GOARCH, + } +} + +// reconcile retrieves the desired binaries, returning a map of version to +// binary path +func (h *harness) reconcile() map[string]string { + // Get the binaries we need to download + missing := h.diff() + + // Create the directory for the binaries + h.createBinDir() + + ctx, _ := context.WithTimeout(context.Background(), 5*time.Minute) + g, _ := errgroup.WithContext(ctx) + for _, v := range missing { + version := v + g.Go(func() error { + return h.get(version) + }) + } + if err := g.Wait(); err != nil { + h.t.Fatalf("failed getting versions: %v", err) + } + + binaries := make(map[string]string, len(versions)) + for _, v := range versions { + binaries[v] = filepath.Join(h.binDir, v) + } + return binaries +} + +// createBinDir creates the binary directory +func (h *harness) createBinDir() { + // Check if the directory exists, otherwise create it + f, err := os.Stat(h.binDir) + if err != nil && !os.IsNotExist(err) { + h.t.Fatalf("failed to stat directory: %v", err) + } + + if f != nil && f.IsDir() { + return + } else if f != nil { + if err := os.RemoveAll(h.binDir); err != nil { + h.t.Fatalf("failed to remove file at directory path: %v", err) + } + } + + // Create the directory + if err := os.Mkdir(h.binDir, 0700); err != nil { + h.t.Fatalf("failed to make directory: %v", err) + } + if err := os.Chmod(h.binDir, 0700); err != nil { + h.t.Fatalf("failed to chmod: %v", err) + } +} + +// diff returns the binaries that must be downloaded +func (h *harness) diff() (missing []string) { + files, err := ioutil.ReadDir(h.binDir) + if err != nil { + if os.IsNotExist(err) { + return versions + } + + h.t.Fatalf("failed to stat directory: %v", err) + } + + // Build the set we need + missingSet := make(map[string]struct{}, len(versions)) + for _, v := range versions { + missingSet[v] = struct{}{} + } + + for _, f := range files { + delete(missingSet, f.Name()) + } + + for k := range missingSet { + missing = append(missing, k) + } + + return missing +} + +// get retrieves the given Vault binary +func (h *harness) get(version string) error { + resp, err := http.Get( + fmt.Sprintf("https://releases.hashicorp.com/vault/%s/vault_%s_%s_%s.zip", + version, version, h.os, h.arch)) + if err != nil { + return err + } + defer resp.Body.Close() + + // Wrap in an in-mem buffer + b := bytes.NewBuffer(nil) + io.Copy(b, resp.Body) + resp.Body.Close() + + zreader, err := zip.NewReader(bytes.NewReader(b.Bytes()), resp.ContentLength) + if err != nil { + return err + } + + if l := len(zreader.File); l != 1 { + return fmt.Errorf("unexpected number of files in zip: %v", l) + } + + // Copy the file to its destination + file := filepath.Join(h.binDir, version) + out, err := os.OpenFile(file, os.O_RDWR|os.O_CREATE|os.O_EXCL, 0777) + if err != nil { + return err + } + defer out.Close() + + zfile, err := zreader.File[0].Open() + if err != nil { + return fmt.Errorf("failed to open zip file: %v", err) + } + + if _, err := io.Copy(out, zfile); err != nil { + return fmt.Errorf("failed to decompress file to destination: %v", err) + } + + return nil +} + +// TestVaultCompatibility tests compatability across Vault versions +func TestVaultCompatibility(t *testing.T) { + h := newHarness(t) + vaultBinaries := h.reconcile() + + for version, vaultBin := range vaultBinaries { + t.Run(version, func(t *testing.T) { + vbin := vaultBin + testVaultCompatibility(t, vbin) + }) + } +} + +// testVaultCompatibility tests compatability with the given vault binary +func testVaultCompatibility(t *testing.T, vault string) { + require := require.New(t) + + // Create a Vault server + v := testutil.NewTestVaultFromPath(t, vault) + defer v.Stop() + + token := setupVault(t, v.Client) + + // Create a Nomad agent using the created vault + nomad := agent.NewTestAgent(t, t.Name(), func(c *agent.Config) { + if c.Vault == nil { + c.Vault = &config.VaultConfig{} + } + c.Vault.Enabled = helper.BoolToPtr(true) + c.Vault.Token = token + c.Vault.Role = "nomad-cluster" + c.Vault.AllowUnauthenticated = helper.BoolToPtr(true) + c.Vault.Addr = v.HTTPAddr + }) + defer nomad.Shutdown() + + // Submit the Nomad job that requests a Vault token and cats that the Vault + // token is there + c := nomad.Client() + j := c.Jobs() + _, _, err := j.Register(job, nil) + require.NoError(err) + + // Wait for there to be an allocation terminated successfully + //var allocID string + testutil.WaitForResult(func() (bool, error) { + // Get the allocations for the job + allocs, _, err := j.Allocations(*job.ID, false, nil) + if err != nil { + return false, err + } + l := len(allocs) + switch l { + case 0: + return false, fmt.Errorf("want one alloc; got zero") + case 1: + default: + // exit early + t.Fatalf("too many allocations; something failed") + } + alloc := allocs[0] + //allocID = alloc.ID + if alloc.ClientStatus == "complete" { + return true, nil + } + + return false, fmt.Errorf("client status %q", alloc.ClientStatus) + }, func(err error) { + t.Fatalf("allocation did not finish: %v", err) + }) + +} + +// setupVault takes the Vault client and creates the required policies and +// roles. It returns the token that should be used by Nomad +func setupVault(t *testing.T, client *vapi.Client) string { + // Write the policy + sys := client.Sys() + if err := sys.PutPolicy("nomad-server", policy); err != nil { + t.Fatalf("failed to create policy: %v", err) + } + + // Build the role + l := client.Logical() + l.Write("auth/token/roles/nomad-cluster", role) + + // Create a new token with the role + a := client.Auth().Token() + req := vapi.TokenCreateRequest{ + Policies: []string{"nomad-server"}, + Period: "72h", + NoParent: true, + } + s, err := a.Create(&req) + if err != nil { + t.Fatalf("failed to create child token: %v", err) + } + + // Get the client token + if s == nil || s.Auth == nil { + t.Fatalf("bad secret response: %+v", s) + } + + return s.Auth.ClientToken +} diff --git a/testutil/vault.go b/testutil/vault.go index 6ed78d58776..aa7f69bd555 100644 --- a/testutil/vault.go +++ b/testutil/vault.go @@ -8,6 +8,7 @@ import ( "time" "github.com/hashicorp/consul/lib/freeport" + "github.com/hashicorp/nomad/helper/testlog" "github.com/hashicorp/nomad/helper/uuid" "github.com/hashicorp/nomad/nomad/structs/config" vapi "github.com/hashicorp/vault/api" @@ -34,8 +35,7 @@ type TestVault struct { Client *vapi.Client } -// NewTestVault returns a new TestVault instance that has yet to be started -func NewTestVault(t testing.T) *TestVault { +func NewTestVaultFromPath(t testing.T, binary string) *TestVault { for i := 10; i >= 0; i-- { port := freeport.GetT(t, 1)[0] token := uuid.Generate() @@ -43,9 +43,9 @@ func NewTestVault(t testing.T) *TestVault { http := fmt.Sprintf("http://127.0.0.1:%d", port) root := fmt.Sprintf("-dev-root-token-id=%s", token) - cmd := exec.Command("vault", "server", "-dev", bind, root) - cmd.Stdout = os.Stdout - cmd.Stderr = os.Stderr + cmd := exec.Command(binary, "server", "-dev", bind, root) + cmd.Stdout = testlog.NewWriter(t) + cmd.Stderr = testlog.NewWriter(t) // Build the config conf := vapi.DefaultConfig() @@ -112,6 +112,13 @@ func NewTestVault(t testing.T) *TestVault { } return nil + +} + +// NewTestVault returns a new TestVault instance that has yet to be started +func NewTestVault(t testing.T) *TestVault { + // Lookup vault from the path + return NewTestVaultFromPath(t, "vault") } // NewTestVaultDelayed returns a test Vault server that has not been started. From 67ab8eff071e7d1d4313fbdc25bc430f45068acb Mon Sep 17 00:00:00 2001 From: Alex Dadgar Date: Tue, 18 Sep 2018 18:46:33 -0700 Subject: [PATCH 2/7] test automation --- GNUmakefile | 9 +++++++++ scripts/travis.sh | 5 ++++- 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/GNUmakefile b/GNUmakefile index 53b7b6a7e00..147413ea167 100644 --- a/GNUmakefile +++ b/GNUmakefile @@ -253,6 +253,15 @@ test-nomad: dev ## Run Nomad test suites bash -C "$(PROJECT_ROOT)/scripts/test_check.sh" ; \ fi +.PHONY: e2e-test +e2e-test: ## Run the Nomad e2e test suite + @echo "==> Running Nomad E2E test suites:" + go test \ + $(if $(ENABLE_RACE),-race) $(if $(VERBOSE),-v) \ + -cover \ + -timeout=900s \ + github.com/hashicorp/nomad/e2e/vault/ + .PHONY: clean clean: GOPATH=$(shell go env GOPATH) clean: ## Remove build artifacts diff --git a/scripts/travis.sh b/scripts/travis.sh index 541bbb587e6..12998d00638 100755 --- a/scripts/travis.sh +++ b/scripts/travis.sh @@ -17,5 +17,8 @@ fi NOMAD_SLOW_TEST=1 make test TEST_OUTPUT=$? +NOMAD_SLOW_TEST=1 make e2e-test +E2E_TEST_OUTPUT=$? + kill $PING_LOOP_PID -exit $TEST_OUTPUT +exit $(( $((TEST_OUTPUT + E2E_TEST_OUTPUT)) != 0 )) From 7acb3ca2ee9713c82416574f62356623c6984719 Mon Sep 17 00:00:00 2001 From: Alex Dadgar Date: Wed, 19 Sep 2018 10:16:39 -0700 Subject: [PATCH 3/7] vet --- e2e/vault/{consts.go => consts_test.go} | 0 e2e/vault/{matrix.go => matrix_test.go} | 0 e2e/vault/vault_test.go | 8 +++++--- 3 files changed, 5 insertions(+), 3 deletions(-) rename e2e/vault/{consts.go => consts_test.go} (100%) rename e2e/vault/{matrix.go => matrix_test.go} (100%) diff --git a/e2e/vault/consts.go b/e2e/vault/consts_test.go similarity index 100% rename from e2e/vault/consts.go rename to e2e/vault/consts_test.go diff --git a/e2e/vault/matrix.go b/e2e/vault/matrix_test.go similarity index 100% rename from e2e/vault/matrix.go rename to e2e/vault/matrix_test.go diff --git a/e2e/vault/vault_test.go b/e2e/vault/vault_test.go index a52f2f66b9d..46d9f130512 100644 --- a/e2e/vault/vault_test.go +++ b/e2e/vault/vault_test.go @@ -51,7 +51,9 @@ func (h *harness) reconcile() map[string]string { // Create the directory for the binaries h.createBinDir() - ctx, _ := context.WithTimeout(context.Background(), 5*time.Minute) + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Minute) + defer cancel() + g, _ := errgroup.WithContext(ctx) for _, v := range missing { version := v @@ -167,7 +169,7 @@ func (h *harness) get(version string) error { return nil } -// TestVaultCompatibility tests compatability across Vault versions +// TestVaultCompatibility tests compatibility across Vault versions func TestVaultCompatibility(t *testing.T) { h := newHarness(t) vaultBinaries := h.reconcile() @@ -180,7 +182,7 @@ func TestVaultCompatibility(t *testing.T) { } } -// testVaultCompatibility tests compatability with the given vault binary +// testVaultCompatibility tests compatibility with the given vault binary func testVaultCompatibility(t *testing.T, vault string) { require := require.New(t) From ebe6fe208e82d1008457f6218b16d49cae4b2511 Mon Sep 17 00:00:00 2001 From: Alex Dadgar Date: Wed, 19 Sep 2018 10:21:57 -0700 Subject: [PATCH 4/7] run in matrix --- .travis.yml | 4 ++++ GNUmakefile | 3 +++ scripts/travis.sh | 5 +---- 3 files changed, 8 insertions(+), 4 deletions(-) diff --git a/.travis.yml b/.travis.yml index 7edbcfea504..7b2855581d0 100644 --- a/.travis.yml +++ b/.travis.yml @@ -31,6 +31,10 @@ matrix: env: RUN_STATIC_CHECKS=1 SKIP_NOMAD_TESTS=1 - os: osx osx_image: xcode9.1 + - os: linux + dist: trusty + sudo: required + env: RUN_E2E_TESTS=1 SKIP_NOMAD_TESTS=1 allow_failures: # Allow osx to fail as its flaky - os: osx diff --git a/GNUmakefile b/GNUmakefile index 147413ea167..17f99f1fa13 100644 --- a/GNUmakefile +++ b/GNUmakefile @@ -240,6 +240,9 @@ test: ## Run the Nomad test suite and/or the Nomad UI test suite @if [ $(RUN_UI_TESTS) ]; then \ make test-ui; \ fi + @if [ $(RUN_E2E_TESTS) ]; then \ + make e2e-test; \ + fi .PHONY: test-nomad test-nomad: dev ## Run Nomad test suites diff --git a/scripts/travis.sh b/scripts/travis.sh index 12998d00638..541bbb587e6 100755 --- a/scripts/travis.sh +++ b/scripts/travis.sh @@ -17,8 +17,5 @@ fi NOMAD_SLOW_TEST=1 make test TEST_OUTPUT=$? -NOMAD_SLOW_TEST=1 make e2e-test -E2E_TEST_OUTPUT=$? - kill $PING_LOOP_PID -exit $(( $((TEST_OUTPUT + E2E_TEST_OUTPUT)) != 0 )) +exit $TEST_OUTPUT From 575e193332cba12069c2f516640a222c8efe0126 Mon Sep 17 00:00:00 2001 From: Alex Dadgar Date: Wed, 19 Sep 2018 10:23:18 -0700 Subject: [PATCH 5/7] vendor vault api for backwards compatibility --- .../github.com/hashicorp/vault/api/sys_capabilities.go | 10 ++++++++++ vendor/vendor.json | 2 +- 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/vendor/github.com/hashicorp/vault/api/sys_capabilities.go b/vendor/github.com/hashicorp/vault/api/sys_capabilities.go index 242acf96e7a..64b3951dd10 100644 --- a/vendor/github.com/hashicorp/vault/api/sys_capabilities.go +++ b/vendor/github.com/hashicorp/vault/api/sys_capabilities.go @@ -50,5 +50,15 @@ func (c *Sys) Capabilities(token, path string) ([]string, error) { return nil, err } + if len(res) == 0 { + _, ok := secret.Data["capabilities"] + if ok { + err = mapstructure.Decode(secret.Data["capabilities"], &res) + if err != nil { + return nil, err + } + } + } + return res, nil } diff --git a/vendor/vendor.json b/vendor/vendor.json index e9796f46ced..941f55cfa16 100644 --- a/vendor/vendor.json +++ b/vendor/vendor.json @@ -195,7 +195,7 @@ {"path":"github.com/hashicorp/raft-boltdb","checksumSHA1":"QAxukkv54/iIvLfsUP6IK4R0m/A=","revision":"d1e82c1ec3f15ee991f7cc7ffd5b67ff6f5bbaee","revisionTime":"2015-02-01T20:08:39Z"}, {"path":"github.com/hashicorp/serf/coordinate","checksumSHA1":"0PeWsO2aI+2PgVYlYlDPKfzCLEQ=","revision":"80ab48778deee28e4ea2dc4ef1ebb2c5f4063996","revisionTime":"2018-05-07T23:19:28Z"}, {"path":"github.com/hashicorp/serf/serf","checksumSHA1":"QrT+nzyXsD/MmhTjjhcPdnALZ1I=","revision":"80ab48778deee28e4ea2dc4ef1ebb2c5f4063996","revisionTime":"2018-05-07T23:19:28Z"}, - {"path":"github.com/hashicorp/vault/api","checksumSHA1":"+B4wuJNerIUKNAVzld7CmMaNW5A=","revision":"8575f8fedcf8f5a6eb2b4701cb527b99574b5286","revisionTime":"2018-09-06T17:45:45Z"}, + {"path":"github.com/hashicorp/vault/api","checksumSHA1":"DP7dd8OErZVF0q+XfPo0RGkDcLk=","revision":"6e8d91a59c34bd9f323397c30be9651422295c65","revisionTime":"2018-09-19T17:09:49Z"}, {"path":"github.com/hashicorp/vault/helper/compressutil","checksumSHA1":"bSdPFOHaTwEvM4PIvn0PZfn75jM=","revision":"8575f8fedcf8f5a6eb2b4701cb527b99574b5286","revisionTime":"2018-09-06T17:45:45Z"}, {"path":"github.com/hashicorp/vault/helper/consts","checksumSHA1":"QNGGvSYtwk6VCkj4laZPjM2301E=","revision":"8575f8fedcf8f5a6eb2b4701cb527b99574b5286","revisionTime":"2018-09-06T17:45:45Z"}, {"path":"github.com/hashicorp/vault/helper/hclutil","checksumSHA1":"RlqPBLOexQ0jj6jomhiompWKaUg=","revision":"8575f8fedcf8f5a6eb2b4701cb527b99574b5286","revisionTime":"2018-09-06T17:45:45Z"}, From f7f5da204d251d1b161cbf6ecddc5bbeaad6896f Mon Sep 17 00:00:00 2001 From: Alex Dadgar Date: Wed, 19 Sep 2018 10:38:20 -0700 Subject: [PATCH 6/7] build nomad in e2e tests --- GNUmakefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/GNUmakefile b/GNUmakefile index 17f99f1fa13..f1b71a50769 100644 --- a/GNUmakefile +++ b/GNUmakefile @@ -257,7 +257,7 @@ test-nomad: dev ## Run Nomad test suites fi .PHONY: e2e-test -e2e-test: ## Run the Nomad e2e test suite +e2e-test: dev ## Run the Nomad e2e test suite @echo "==> Running Nomad E2E test suites:" go test \ $(if $(ENABLE_RACE),-race) $(if $(VERBOSE),-v) \ From 668a90102f34b4b3340b632573cabb11c4f529ef Mon Sep 17 00:00:00 2001 From: Alex Dadgar Date: Mon, 24 Sep 2018 16:34:15 -0700 Subject: [PATCH 7/7] proper variable capture --- e2e/vault/vault_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/e2e/vault/vault_test.go b/e2e/vault/vault_test.go index 46d9f130512..dce6402514f 100644 --- a/e2e/vault/vault_test.go +++ b/e2e/vault/vault_test.go @@ -175,8 +175,8 @@ func TestVaultCompatibility(t *testing.T) { vaultBinaries := h.reconcile() for version, vaultBin := range vaultBinaries { + vbin := vaultBin t.Run(version, func(t *testing.T) { - vbin := vaultBin testVaultCompatibility(t, vbin) }) }