e2e: add ACL test suite with ACL Role test. (#14398)

This adds a new ACL test suite to the e2e framework which includes
an initial test for ACL roles. The ACL test includes a helper to
track and clean created Nomad resources which keeps the test
cluster clean no matter if the test fails early or not.

e2e/acl/acl_role_test.go

@@ -0,0 +1,153 @@
+package acl
+
+import (
+	"fmt"
+	"testing"
+
+	"github.com/hashicorp/nomad/api"
+	"github.com/hashicorp/nomad/e2e/e2eutil"
+	"github.com/hashicorp/nomad/helper/uuid"
+	"github.com/stretchr/testify/require"
+)
+
+// testACLRole tests basic functionality of ACL roles when used for
+// authorization. It also performs some basic token and policy tests due to the
+// coupling between the ACL objects.
+func testACLRole(t *testing.T) {
+
+	nomadClient := e2eutil.NomadClient(t)
+
+	// Create and defer the cleanup process. This is used to remove all
+	// resources created by this test and covers situations where the test
+	// fails or during normal running.
+	cleanUpProcess := newCleanup()
+	defer cleanUpProcess.run(t, nomadClient)
+
+	// An ACL role must reference an ACL policy that is stored in state. Ensure
+	// this behaviour by attempting to create a role that links to a policy
+	// that does not exist.
+	invalidRole := api.ACLRole{
+		Name:        "e2e-acl-" + uuid.Short(),
+		Description: "E2E ACL Role Testing",
+		Policies:    []*api.ACLRolePolicyLink{{Name: "404-not-found"}},
+	}
+	aclRoleCreateResp, _, err := nomadClient.ACLRoles().Create(&invalidRole, nil)
+	require.ErrorContains(t, err, "cannot find policy 404-not-found")
+	require.Nil(t, aclRoleCreateResp)
+
+	// Create a custom namespace to test along with the default.
+	ns := api.Namespace{
+		Name:        "e2e-acl-" + uuid.Short(),
+		Description: "E2E ACL Role Testing",
+	}
+	_, err = nomadClient.Namespaces().Register(&ns, nil)
+	require.NoError(t, err)
+
+	cleanUpProcess.add(ns.Name, namespaceTestResourceType)
+
+	// Create an ACL policy which will be used to link from the role. This
+	// policy grants read access to our custom namespace.
+	customNamespacePolicy := api.ACLPolicy{
+		Name:        "e2e-acl-" + uuid.Short(),
+		Description: "E2E ACL Role Testing",
+		Rules:       fmt.Sprintf(`namespace %q {policy = "read"}`, ns.Name),
+	}
+	_, err = nomadClient.ACLPolicies().Upsert(&customNamespacePolicy, nil)
+	require.NoError(t, err)
+
+	cleanUpProcess.add(customNamespacePolicy.Name, aclPolicyTestResourceType)
+
+	// Create a valid role with a link to the previously created policy.
+	validRole := api.ACLRole{
+		Name:        "e2e-acl-" + uuid.Short(),
+		Description: "E2E ACL Role Testing",
+		Policies:    []*api.ACLRolePolicyLink{{Name: customNamespacePolicy.Name}},
+	}
+	aclRoleCreateResp, _, err = nomadClient.ACLRoles().Create(&validRole, nil)
+	require.NoError(t, err)
+	require.NotNil(t, aclRoleCreateResp)
+	require.NotEmpty(t, aclRoleCreateResp.ID)
+	require.Equal(t, validRole.Name, aclRoleCreateResp.Name)
+
+	cleanUpProcess.add(aclRoleCreateResp.ID, aclRoleTestResourceType)
+
+	// Perform a role listing and check we have the expected entries.
+	aclRoleListResp, _, err := nomadClient.ACLRoles().List(nil)
+	require.NoError(t, err)
+	require.Len(t, aclRoleListResp, 1)
+	require.Equal(t, aclRoleCreateResp.ID, aclRoleListResp[0].ID)
+
+	// Create our ACL token which is linked to the created ACL role.
+	token := api.ACLToken{
+		Name:  "e2e-acl-" + uuid.Short(),
+		Type:  "client",
+		Roles: []*api.ACLTokenRoleLink{{ID: aclRoleCreateResp.ID}},
+	}
+	aclTokenCreateResp, _, err := nomadClient.ACLTokens().Create(&token, nil)
+	require.NoError(t, err)
+	require.NotNil(t, aclTokenCreateResp)
+
+	cleanUpProcess.add(aclTokenCreateResp.AccessorID, aclTokenTestResourceType)
+
+	// Attempt two job listings against the two available namespaces. The token
+	// only has access to the custom namespace, so the default should return an
+	// error.
+	customNSQueryMeta := api.QueryOptions{Namespace: ns.Name, AuthToken: aclTokenCreateResp.SecretID}
+	defaultNSQueryMeta := api.QueryOptions{Namespace: "default", AuthToken: aclTokenCreateResp.SecretID}
+
+	jobListResp, _, err := nomadClient.Jobs().List(&customNSQueryMeta)
+	require.NoError(t, err)
+	require.Empty(t, jobListResp)
+
+	jobListResp, _, err = nomadClient.Jobs().List(&defaultNSQueryMeta)
+	require.ErrorContains(t, err, "Permission denied")
+
+	// Create an ACL policy which grants read access to the default namespace.
+	defaultNamespacePolicy := api.ACLPolicy{
+		Name:        "e2e-acl-" + uuid.Short(),
+		Description: "E2E ACL Role Testing",
+		Rules:       `namespace "default" {policy = "read"}`,
+	}
+	_, err = nomadClient.ACLPolicies().Upsert(&defaultNamespacePolicy, nil)
+	require.NoError(t, err)
+
+	cleanUpProcess.add(defaultNamespacePolicy.Name, aclPolicyTestResourceType)
+
+	// Update the ACL role to include the new ACL policy that allows read
+	// access to the default namespace.
+	aclRoleCreateResp.Policies = append(aclRoleCreateResp.Policies, &api.ACLRolePolicyLink{
+		Name: defaultNamespacePolicy.Name,
+	})
+	aclRoleUpdateResp, _, err := nomadClient.ACLRoles().Update(aclRoleCreateResp, nil)
+	require.NoError(t, err)
+	require.Equal(t, aclRoleCreateResp.ID, aclRoleUpdateResp.ID)
+	require.Len(t, aclRoleUpdateResp.Policies, 2)
+
+	// Try listing the jobs in the default namespace again to ensure we now
+	// have permission due to the updated role.
+	jobListResp, _, err = nomadClient.Jobs().List(&defaultNSQueryMeta)
+	require.NoError(t, err)
+	require.Empty(t, jobListResp)
+
+	// Delete a policy from under the role.
+	_, err = nomadClient.ACLPolicies().Delete(defaultNamespacePolicy.Name, nil)
+	require.NoError(t, err)
+
+	cleanUpProcess.remove(defaultNamespacePolicy.Name, aclPolicyTestResourceType)
+
+	// The permission to list the job in the default namespace should now be
+	// revoked.
+	jobListResp, _, err = nomadClient.Jobs().List(&defaultNSQueryMeta)
+	require.ErrorContains(t, err, "Permission denied")
+
+	// Delete the ACL role.
+	_, err = nomadClient.ACLRoles().Delete(aclRoleUpdateResp.ID, nil)
+	require.NoError(t, err)
+
+	cleanUpProcess.remove(aclRoleUpdateResp.ID, aclRoleTestResourceType)
+
+	// We should now not be able to list jobs in the custom namespace either as
+	// the token does not have any permissions.
+	jobListResp, _, err = nomadClient.Jobs().List(&customNSQueryMeta)
+	require.ErrorContains(t, err, "Permission denied")
+}

e2e/acl/acl_test.go

@@ -0,0 +1,109 @@
+package acl
+
+import (
+	"testing"
+
+	"github.com/hashicorp/go-set"
+	"github.com/hashicorp/nomad/api"
+	"github.com/hashicorp/nomad/e2e/e2eutil"
+	"github.com/stretchr/testify/assert"
+)
+
+func TestACL(t *testing.T) {
+
+	// Wait until we have a usable cluster before running the tests. The tests
+	// do not currently need a running client.
+	nomadClient := e2eutil.NomadClient(t)
+	e2eutil.WaitForLeader(t, nomadClient)
+
+	// Run our test cases.
+	t.Run("TestACL_Role", testACLRole)
+}
+
+// testResourceType indicates what the resource is so the cleanup process can
+// use the correct API.
+type testResourceType int
+
+const (
+	namespaceTestResourceType testResourceType = iota
+	aclPolicyTestResourceType
+	aclRoleTestResourceType
+	aclTokenTestResourceType
+)
+
+// cleanup stores Nomad resources that have been created by a test which will
+// need to be deleted once the test exits. This ensures other tests can run in
+// a clean environment and reduces the potential for conflicts.
+type cleanup struct {
+	namespaces  *set.Set[string]
+	aclPolicies *set.Set[string]
+	aclRoles    *set.Set[string]
+	aclTokens   *set.Set[string]
+}
+
+// newCleanup generates an initialized cleanup object for immediate use.
+func newCleanup() *cleanup {
+	return &cleanup{
+		namespaces:  set.New[string](0),
+		aclPolicies: set.New[string](0),
+		aclRoles:    set.New[string](0),
+		aclTokens:   set.New[string](0),
+	}
+}
+
+// run triggers a cleanup of all the stored resources. This should typically be
+// called via defer, so it will always run no matter if the test fails or not.
+// Any failure will ultimately fail the test, but will not stop the attempts to
+// delete all the resources.
+func (c *cleanup) run(t *testing.T, nomadClient *api.Client) {
+
+	for _, namespace := range c.namespaces.List() {
+		_, err := nomadClient.Namespaces().Delete(namespace, nil)
+		assert.NoError(t, err)
+	}
+
+	for _, policy := range c.aclPolicies.List() {
+		_, err := nomadClient.ACLPolicies().Delete(policy, nil)
+		assert.NoError(t, err)
+	}
+
+	for _, role := range c.aclRoles.List() {
+		_, err := nomadClient.ACLRoles().Delete(role, nil)
+		assert.NoError(t, err)
+	}
+
+	for _, token := range c.aclTokens.List() {
+		_, err := nomadClient.ACLTokens().Delete(token, nil)
+		assert.NoError(t, err)
+	}
+}
+
+// add the resource identifier to the resource tracker. It will be removed by
+// the cleanup function once it is triggered.
+func (c *cleanup) add(id string, resourceType testResourceType) {
+	switch resourceType {
+	case namespaceTestResourceType:
+		c.namespaces.Insert(id)
+	case aclPolicyTestResourceType:
+		c.aclPolicies.Insert(id)
+	case aclRoleTestResourceType:
+		c.aclRoles.Insert(id)
+	case aclTokenTestResourceType:
+		c.aclTokens.Insert(id)
+	}
+}
+
+// remove the resource identifier from the resource tracker, indicating it is
+// no longer existing on the cluster and does not need to be cleaned.
+func (c *cleanup) remove(id string, resourceType testResourceType) {
+	switch resourceType {
+	case namespaceTestResourceType:
+		c.namespaces.Remove(id)
+	case aclPolicyTestResourceType:
+		c.aclPolicies.Remove(id)
+	case aclRoleTestResourceType:
+		c.aclRoles.Remove(id)
+	case aclTokenTestResourceType:
+		c.aclTokens.Remove(id)
+	}
+}

e2e/acl/doc.go

@@ -0,0 +1,5 @@
+// Package acl provides end-to-end tests for Nomads ACL system.
+//
+// In order to run this test suite only, from the e2e directory you can trigger
+// go test -v -run '^TestACL$' ./acl
+package acl