Merge pull request #10439 from hashicorp/pick-ent-acls-changes

e2e: add e2e tests for consul namespaces on ent with acls

command/agent/consul/acl_testing.go

@@ -119,8 +119,8 @@ func (m *MockACLsAPI) RoleRead(roleID string, _ *api.QueryOptions) (*api.ACLRole
 	}
 }
 
-// Example Consul "operator" tokens for use in tests.
-
+// Example Consul ACL tokens for use in tests. These tokens belong to the
+// default Consul namespace.
 const (
 	ExampleOperatorTokenID0 = "de591604-86eb-1e6f-8b44-d4db752921ae"
 	ExampleOperatorTokenID1 = "59c219c2-47e4-43f3-bb45-258fd13f59d5"
@@ -130,7 +130,20 @@ const (
 	ExampleOperatorTokenID5 = "097cbb45-506b-c79c-ec38-82eb0dc0794a"
 )
 
+// Example Consul ACL tokens for use in tests that match the policies as the
+// tokens above, but these belong to the "banana' Consul namespace.
+const (
+	ExampleOperatorTokenID10 = "ddfe688f-655f-e8dd-1db5-5650eed00aeb"
+	ExampleOperatorTokenID11 = "46d09394-598c-1e55-b7fd-64cd2f409707"
+	ExampleOperatorTokenID12 = "a041cb88-0f4b-0314-89f6-10e1e093d2e5"
+	ExampleOperatorTokenID13 = "cc22a583-243f-3258-14ad-db0e56749657"
+	ExampleOperatorTokenID14 = "5b6d0508-13a6-4bc3-33a1-ba1941e1175b"
+	ExampleOperatorTokenID15 = "e9db1754-c075-d0fc-0a7e-de1e9e7bff98"
+)
+
 var (
+	// In Consul namespace "default"
+
 	ExampleOperatorToken0 = &api.ACLToken{
 		SecretID:    ExampleOperatorTokenID0,
 		AccessorID:  "228865c6-3bf6-6683-df03-06dea2779088 ",
@@ -189,6 +202,67 @@ var (
 		}},
 		Namespace: "default",
 	}
+
+	// In Consul namespace "banana"
+
+	ExampleOperatorToken10 = &api.ACLToken{
+		SecretID:    ExampleOperatorTokenID0,
+		AccessorID:  "76a2c3b5-5d64-9089-f701-660eec2d3554",
+		Description: "Operator Token 0",
+		Namespace:   "banana",
+	}
+
+	ExampleOperatorToken11 = &api.ACLToken{
+		SecretID:    ExampleOperatorTokenID1,
+		AccessorID:  "40f2a36a-0a65-1972-106c-b2e5dd46d6e8",
+		Description: "Operator Token 1",
+		Policies: []*api.ACLTokenPolicyLink{{
+			ID: ExamplePolicyID1,
+		}},
+		Namespace: "banana",
+	}
+
+	ExampleOperatorToken12 = &api.ACLToken{
+		SecretID:    ExampleOperatorTokenID2,
+		AccessorID:  "894f2c5c-b285-71bf-4acb-6344cecf71f3",
+		Description: "Operator Token 2",
+		Policies: []*api.ACLTokenPolicyLink{{
+			ID: ExamplePolicyID2,
+		}},
+		Namespace: "banana",
+	}
+
+	ExampleOperatorToken13 = &api.ACLToken{
+		SecretID:    ExampleOperatorTokenID3,
+		AccessorID:  "2a81ec0b-692e-845e-f5b8-c33c05e5af22",
+		Description: "Operator Token 3",
+		Policies: []*api.ACLTokenPolicyLink{{
+			ID: ExamplePolicyID3,
+		}},
+		Namespace: "banana",
+	}
+
+	ExampleOperatorToken14 = &api.ACLToken{
+		SecretID:    ExampleOperatorTokenID4,
+		AccessorID:  "4273f1cc-5626-7a77-dc65-1f24af035ed5d",
+		Description: "Operator Token 4",
+		Policies:    nil, // no direct policy, only roles
+		Roles: []*api.ACLTokenRoleLink{{
+			ID:   ExampleRoleID1,
+			Name: "example-role-1",
+		}},
+		Namespace: "banana",
+	}
+
+	ExampleOperatorToken15 = &api.ACLToken{
+		SecretID:    ExampleOperatorTokenID5,
+		AccessorID:  "5b78e186-87d8-c1ad-966f-f5fa87b05c9a",
+		Description: "Operator Token 5",
+		Policies: []*api.ACLTokenPolicyLink{{
+			ID: ExamplePolicyID4,
+		}},
+		Namespace: "banana",
+	}
 )
 
 func (m *MockACLsAPI) TokenReadSelf(q *api.QueryOptions) (*api.ACLToken, *api.QueryMeta, error) {
@@ -209,6 +283,21 @@ func (m *MockACLsAPI) TokenReadSelf(q *api.QueryOptions) (*api.ACLToken, *api.Qu
 	case ExampleOperatorTokenID5:
 		return ExampleOperatorToken5, nil, nil
 
+	case ExampleOperatorTokenID11:
+		return ExampleOperatorToken11, nil, nil
+
+	case ExampleOperatorTokenID12:
+		return ExampleOperatorToken12, nil, nil
+
+	case ExampleOperatorTokenID13:
+		return ExampleOperatorToken13, nil, nil
+
+	case ExampleOperatorTokenID14:
+		return ExampleOperatorToken14, nil, nil
+
+	case ExampleOperatorTokenID15:
+		return ExampleOperatorToken15, nil, nil
+
 	default:
 		return nil, nil, errors.New("no such token")
 	}

e2e/connect/acls.go

@@ -22,9 +22,10 @@ type ConnectACLsE2ETest struct {
 	// manageConsulACLs is used to 'enable' and 'disable' Consul ACLs in the
 	// Consul Cluster that has been setup for e2e testing.
 	manageConsulACLs consulacls.Manager
-	// consulMasterToken is set to the generated Consul ACL token after using
+
+	// consulManagementToken is set to the generated Consul ACL token after using
 	// the consul-acls-manage.sh script to enable ACLs.
-	consulMasterToken string
+	consulManagementToken string
 
 	// things to cleanup after each test case
 	jobIDs          []string
@@ -46,7 +47,7 @@ func (tc *ConnectACLsE2ETest) BeforeAll(f *framework.F) {
 
 	// Validate the consul master token exists, otherwise tests are just
 	// going to be a train wreck.
-	tokenLength := len(tc.consulMasterToken)
+	tokenLength := len(tc.consulManagementToken)
 	require.Equal(f.T(), 36, tokenLength, "consul master token wrong length")
 
 	// Validate the CONSUL_HTTP_TOKEN is NOT set, because that will cause
@@ -63,7 +64,7 @@ func (tc *ConnectACLsE2ETest) BeforeAll(f *framework.F) {
 // enableConsulACLs effectively executes `consul-acls-manage.sh enable`, which
 // will activate Consul ACLs, going through the bootstrap process if necessary.
 func (tc *ConnectACLsE2ETest) enableConsulACLs(f *framework.F) {
-	tc.consulMasterToken = tc.manageConsulACLs.Enable(f.T())
+	tc.consulManagementToken = tc.manageConsulACLs.Enable(f.T())
 }
 
 // AfterAll runs after all tests are complete.
@@ -100,14 +101,14 @@ func (tc *ConnectACLsE2ETest) AfterEach(f *framework.F) {
 	// cleanup consul tokens
 	for _, id := range tc.consulTokenIDs {
 		t.Log("cleanup: delete consul token id:", id)
-		_, err := tc.Consul().ACL().TokenDelete(id, &consulapi.WriteOptions{Token: tc.consulMasterToken})
+		_, err := tc.Consul().ACL().TokenDelete(id, &consulapi.WriteOptions{Token: tc.consulManagementToken})
 		f.NoError(err)
 	}
 
 	// cleanup consul policies
 	for _, id := range tc.consulPolicyIDs {
 		t.Log("cleanup: delete consul policy id:", id)
-		_, err := tc.Consul().ACL().PolicyDelete(id, &consulapi.WriteOptions{Token: tc.consulMasterToken})
+		_, err := tc.Consul().ACL().PolicyDelete(id, &consulapi.WriteOptions{Token: tc.consulManagementToken})
 		f.NoError(err)
 	}
 
@@ -128,27 +129,30 @@ func (tc *ConnectACLsE2ETest) AfterEach(f *framework.F) {
 	tc.consulPolicyIDs = []string{}
 }
 
+// todo(shoenig): follow up refactor with e2eutil.ConsulPolicy
 type consulPolicy struct {
 	Name  string // e.g. nomad-operator
 	Rules string // e.g. service "" { policy="write" }
 }
 
+// todo(shoenig): follow up refactor with e2eutil.ConsulPolicy
 func (tc *ConnectACLsE2ETest) createConsulPolicy(p consulPolicy, f *framework.F) string {
 	result, _, err := tc.Consul().ACL().PolicyCreate(&consulapi.ACLPolicy{
 		Name:        p.Name,
 		Description: "test policy " + p.Name,
 		Rules:       p.Rules,
-	}, &consulapi.WriteOptions{Token: tc.consulMasterToken})
+	}, &consulapi.WriteOptions{Token: tc.consulManagementToken})
 	f.NoError(err, "failed to create consul policy")
 	tc.consulPolicyIDs = append(tc.consulPolicyIDs, result.ID)
 	return result.ID
 }
 
+// todo(shoenig): follow up refactor with e2eutil.ConsulPolicy
 func (tc *ConnectACLsE2ETest) createOperatorToken(policyID string, f *framework.F) string {
 	token, _, err := tc.Consul().ACL().TokenCreate(&consulapi.ACLToken{
 		Description: "operator token",
 		Policies:    []*consulapi.ACLTokenPolicyLink{{ID: policyID}},
-	}, &consulapi.WriteOptions{Token: tc.consulMasterToken})
+	}, &consulapi.WriteOptions{Token: tc.consulManagementToken})
 	f.NoError(err, "failed to create operator token")
 	tc.consulTokenIDs = append(tc.consulTokenIDs, token.AccessorID)
 	return token.SecretID
@@ -170,7 +174,7 @@ func (tc *ConnectACLsE2ETest) TestConnectACLsRegisterMasterToken(f *framework.F)
 	// Set the job file to use the consul master token.
 	// One should never do this in practice, but, it should work.
 	// https://www.consul.io/docs/acl/acl-system.html#builtin-tokens
-	job.ConsulToken = &tc.consulMasterToken
+	job.ConsulToken = &tc.consulManagementToken
 
 	// Avoid using Register here, because that would actually create and run the
 	// Job which runs the task, creates the SI token, which all needs to be
@@ -368,7 +372,7 @@ func (tc *ConnectACLsE2ETest) serviceofSIToken(description string) string {
 func (tc *ConnectACLsE2ETest) countSITokens(t *testing.T) map[string]int {
 	aclAPI := tc.Consul().ACL()
 	tokens, _, err := aclAPI.TokenList(&consulapi.QueryOptions{
-		Token: tc.consulMasterToken,
+		Token: tc.consulManagementToken,
 	})
 	require.NoError(t, err)
 

e2e/connect/connect.go

@@ -50,11 +50,11 @@ func init() {
 	})
 
 	// Connect tests with Consul ACLs enabled. These are now gated behind the
-	// NOMAD_TEST_CONNECT_ACLS environment variable, because they cause lots of
+	// NOMAD_TEST_CONSUL_ACLS environment variable, because they cause lots of
 	// problems for e2e test flakiness (due to restarting consul, nomad, etc.).
 	//
 	// Run these tests locally when working on Connect.
-	if os.Getenv("NOMAD_TEST_CONNECT_ACLS") == "1" {
+	if os.Getenv("NOMAD_TEST_CONSUL_ACLS") == "1" {
 		framework.AddSuites(&framework.TestSuite{
 			Component:   "ConnectACLs",
 			CanRunLocal: false,

e2e/consul/namespaces.go

@@ -39,7 +39,12 @@ var (
 
 type ConsulNamespacesE2ETest struct {
 	framework.TC
+
 	jobIDs []string
+
+	// cToken contains the Consul global-management token during ACL enabled
+	// tests (i.e. ConsulNamespacesE2ETestACLs which embeds ConsulNamespacesE2ETest).
+	cToken string
 }
 
 func (tc *ConsulNamespacesE2ETest) BeforeAll(f *framework.F) {
@@ -56,6 +61,9 @@ func (tc *ConsulNamespacesE2ETest) BeforeAll(f *framework.F) {
 		value := fmt.Sprintf("ns_%s", namespace)
 		e2eutil.PutConsulKey(f.T(), tc.Consul(), namespace, "ns-kv-example", value)
 	}
+
+	// make the unused variable linter happy in oss
+	f.T().Log("Consul global-management token:", tc.cToken)
 }
 
 func (tc *ConsulNamespacesE2ETest) AfterAll(f *framework.F) {
@@ -68,13 +76,13 @@ func (tc *ConsulNamespacesE2ETest) TestNamespacesExist(f *framework.F) {
 	require.True(f.T(), helper.CompareSliceSetString(namespaces, append(consulNamespaces, "default")))
 }
 
-func (tc *ConsulNamespacesE2ETest) testConsulRegisterGroupServices(f *framework.F, nsA, nsB, nsC, nsZ string) {
+func (tc *ConsulNamespacesE2ETest) testConsulRegisterGroupServices(f *framework.F, token, nsA, nsB, nsC, nsZ string) {
 	nomadClient := tc.Nomad()
 	jobID := "cns-group-services"
 	tc.jobIDs = append(tc.jobIDs, jobID)
 
 	// Run job and wait for allocs
-	allocations := e2eutil.RegisterAndWaitForAllocs(f.T(), nomadClient, cnsJobGroupServices, jobID, "")
+	allocations := e2eutil.RegisterAndWaitForAllocs(f.T(), nomadClient, cnsJobGroupServices, jobID, token)
 	require.Len(f.T(), allocations, 3)
 	allocIDs := e2eutil.AllocIDsFromAllocationListStubs(allocations)
 	e2eutil.WaitForAllocsRunning(f.T(), tc.Nomad(), allocIDs)
@@ -112,13 +120,13 @@ func (tc *ConsulNamespacesE2ETest) testConsulRegisterGroupServices(f *framework.
 	e2eutil.RequireConsulDeregistered(r, c, nsZ, "z2")
 }
 
-func (tc *ConsulNamespacesE2ETest) testConsulRegisterTaskServices(f *framework.F, nsA, nsB, nsC, nsZ string) {
+func (tc *ConsulNamespacesE2ETest) testConsulRegisterTaskServices(f *framework.F, token, nsA, nsB, nsC, nsZ string) {
 	nomadClient := tc.Nomad()
 	jobID := "cns-task-services"
 	tc.jobIDs = append(tc.jobIDs, jobID)
 
 	// Run job and wait for allocs
-	allocations := e2eutil.RegisterAndWaitForAllocs(f.T(), nomadClient, cnsJobTaskServices, jobID, "")
+	allocations := e2eutil.RegisterAndWaitForAllocs(f.T(), nomadClient, cnsJobTaskServices, jobID, token)
 	require.Len(f.T(), allocations, 3)
 	allocIDs := e2eutil.AllocIDsFromAllocationListStubs(allocations)
 	e2eutil.WaitForAllocsRunning(f.T(), tc.Nomad(), allocIDs)
@@ -154,14 +162,14 @@ func (tc *ConsulNamespacesE2ETest) testConsulRegisterTaskServices(f *framework.F
 	e2eutil.RequireConsulDeregistered(r, c, nsZ, "z2")
 }
 
-func (tc *ConsulNamespacesE2ETest) testConsulTemplateKV(f *framework.F, expB, expZ string) {
+func (tc *ConsulNamespacesE2ETest) testConsulTemplateKV(f *framework.F, token, expB, expZ string) {
 	t := f.T()
 	nomadClient := tc.Nomad()
 	jobID := "cns-template-kv"
 	tc.jobIDs = append(tc.jobIDs, jobID)
 
 	// Run job and wait for allocs to complete
-	allocations := e2eutil.RegisterAndWaitForAllocs(t, nomadClient, cnsJobTemplateKV, jobID, "")
+	allocations := e2eutil.RegisterAndWaitForAllocs(t, nomadClient, cnsJobTemplateKV, jobID, token)
 	require.Len(t, allocations, 2)
 	allocIDs := e2eutil.AllocIDsFromAllocationListStubs(allocations)
 	e2eutil.WaitForAllocsStopped(f.T(), tc.Nomad(), allocIDs)
@@ -183,13 +191,13 @@ func (tc *ConsulNamespacesE2ETest) testConsulTemplateKV(f *framework.F, expB, ex
 	e2eutil.WaitForJobStopped(t, nomadClient, jobID)
 }
 
-func (tc *ConsulNamespacesE2ETest) testConsulConnectSidecars(f *framework.F, nsA, nsZ string) {
+func (tc *ConsulNamespacesE2ETest) testConsulConnectSidecars(f *framework.F, token, nsA, nsZ string) {
 	nomadClient := tc.Nomad()
 	jobID := "cns-connect-sidecars"
 	tc.jobIDs = append(tc.jobIDs, jobID)
 
 	// Run job and wait for allocs
-	allocations := e2eutil.RegisterAndWaitForAllocs(f.T(), nomadClient, cnsJobConnectSidecars, jobID, "")
+	allocations := e2eutil.RegisterAndWaitForAllocs(f.T(), nomadClient, cnsJobConnectSidecars, jobID, token)
 	require.Len(f.T(), allocations, 4)
 	allocIDs := e2eutil.AllocIDsFromAllocationListStubs(allocations)
 	e2eutil.WaitForAllocsRunning(f.T(), tc.Nomad(), allocIDs)
@@ -223,13 +231,13 @@ func (tc *ConsulNamespacesE2ETest) testConsulConnectSidecars(f *framework.F, nsA
 	e2eutil.RequireConsulDeregistered(r, c, nsZ, "count-dashboard-z-sidecar-proxy")
 }
 
-func (tc *ConsulNamespacesE2ETest) testConsulConnectIngressGateway(f *framework.F, nsA, nsZ string) {
+func (tc *ConsulNamespacesE2ETest) testConsulConnectIngressGateway(f *framework.F, token, nsA, nsZ string) {
 	nomadClient := tc.Nomad()
 	jobID := "cns-connect-ingress"
 	tc.jobIDs = append(tc.jobIDs, jobID)
 
 	// Run job and wait for allocs
-	allocations := e2eutil.RegisterAndWaitForAllocs(f.T(), nomadClient, cnsJobConnectIngress, jobID, "")
+	allocations := e2eutil.RegisterAndWaitForAllocs(f.T(), nomadClient, cnsJobConnectIngress, jobID, token)
 	require.Len(f.T(), allocations, 4) // 2 x (1 service + 1 gateway)
 	allocIDs := e2eutil.AllocIDsFromAllocationListStubs(allocations)
 	e2eutil.WaitForAllocsRunning(f.T(), tc.Nomad(), allocIDs)
@@ -261,13 +269,13 @@ func (tc *ConsulNamespacesE2ETest) testConsulConnectIngressGateway(f *framework.
 	e2eutil.DeleteConsulConfigEntry(f.T(), c, nsZ, "ingress-gateway", "my-ingress-service-z")
 }
 
-func (tc *ConsulNamespacesE2ETest) testConsulConnectTerminatingGateway(f *framework.F, nsA, nsZ string) {
+func (tc *ConsulNamespacesE2ETest) testConsulConnectTerminatingGateway(f *framework.F, token, nsA, nsZ string) {
 	nomadClient := tc.Nomad()
 	jobID := "cns-connect-terminating"
 	tc.jobIDs = append(tc.jobIDs, jobID)
 
 	// Run job and wait for allocs
-	allocations := e2eutil.RegisterAndWaitForAllocs(f.T(), nomadClient, cnsJobConnectTerminating, jobID, "")
+	allocations := e2eutil.RegisterAndWaitForAllocs(f.T(), nomadClient, cnsJobConnectTerminating, jobID, token)
 	require.Len(f.T(), allocations, 6) // 2 x (2 services + 1 gateway)
 	allocIDs := e2eutil.AllocIDsFromAllocationListStubs(allocations)
 	e2eutil.WaitForAllocsRunning(f.T(), tc.Nomad(), allocIDs)
@@ -301,13 +309,13 @@ func (tc *ConsulNamespacesE2ETest) testConsulConnectTerminatingGateway(f *framew
 	e2eutil.DeleteConsulConfigEntry(f.T(), c, nsZ, "terminating-gateway", "api-gateway-z")
 }
 
-func (tc *ConsulNamespacesE2ETest) testConsulScriptChecksTask(f *framework.F, nsA, nsZ string) {
+func (tc *ConsulNamespacesE2ETest) testConsulScriptChecksTask(f *framework.F, token, nsA, nsZ string) {
 	nomadClient := tc.Nomad()
 	jobID := "cns-script-checks-task"
 	tc.jobIDs = append(tc.jobIDs, jobID)
 
 	// Run job and wait for allocs
-	allocations := e2eutil.RegisterAndWaitForAllocs(f.T(), nomadClient, cnsJobScriptChecksTask, jobID, "")
+	allocations := e2eutil.RegisterAndWaitForAllocs(f.T(), nomadClient, cnsJobScriptChecksTask, jobID, token)
 	require.Len(f.T(), allocations, 2)
 	allocIDs := e2eutil.AllocIDsFromAllocationListStubs(allocations)
 	e2eutil.WaitForAllocsRunning(f.T(), tc.Nomad(), allocIDs)
@@ -351,13 +359,13 @@ func (tc *ConsulNamespacesE2ETest) testConsulScriptChecksTask(f *framework.F, ns
 	e2eutil.WaitForJobStopped(f.T(), nomadClient, jobID)
 }
 
-func (tc *ConsulNamespacesE2ETest) testConsulScriptChecksGroup(f *framework.F, nsA, nsZ string) {
+func (tc *ConsulNamespacesE2ETest) testConsulScriptChecksGroup(f *framework.F, token, nsA, nsZ string) {
 	nomadClient := tc.Nomad()
 	jobID := "cns-script-checks-group"
 	tc.jobIDs = append(tc.jobIDs, jobID)
 
 	// Run job and wait for allocs
-	allocations := e2eutil.RegisterAndWaitForAllocs(f.T(), nomadClient, cnsJobScriptChecksGroup, jobID, "")
+	allocations := e2eutil.RegisterAndWaitForAllocs(f.T(), nomadClient, cnsJobScriptChecksGroup, jobID, token)
 	require.Len(f.T(), allocations, 2)
 	allocIDs := e2eutil.AllocIDsFromAllocationListStubs(allocations)
 	e2eutil.WaitForAllocsRunning(f.T(), tc.Nomad(), allocIDs)