NE-1324: E2E tests for Assume Role in Shared VPC Cluster

`go.mod`: Bumped openshift/api for DNS PrivateZoneIAMRole field
`go.sum`: Generated
`test/e2e/operator_test.go`: Add TestExternalDNSAssumeRole E2E test
`test/e2e/util.go`: Add getDNSRecordValuesWithAssumeRole to interface
`test/e2e/aws.go`: Add getDNSRecordValuesWithAssumeRole implementation
`test/e2e/azure.go`: Stub out getDNSRecordValuesWithAssumeRole
`test/e2e/gcp.go`: Stub out getDNSRecordValuesWithAssumeRole
`test/e2e/infoblox.go`: Stub out getDNSRecordValuesWithAssumeRole

go.mod

@@ -13,7 +13,7 @@ require (
 	github.com/miekg/dns v1.0.14
 	github.com/onsi/ginkgo v1.16.5
 	github.com/onsi/gomega v1.27.7
-	github.com/openshift/api v0.0.0-20220906163444-2df055c101a3
+	github.com/openshift/api v0.0.0-20230712163317-e19a88e10d9c
 	github.com/openshift/cloud-credential-operator v0.0.0-20211118210017-9066dcc747fa
 	github.com/operator-framework/api v0.11.0
 	google.golang.org/api v0.58.0

go.sum

@@ -517,8 +517,8 @@ github.com/onsi/gomega v1.14.0/go.mod h1:cIuvLEne0aoVhAgh/O6ac0Op8WWw9H6eYCriF+t
 github.com/onsi/gomega v1.15.0/go.mod h1:cIuvLEne0aoVhAgh/O6ac0Op8WWw9H6eYCriF+tEHG0=
 github.com/onsi/gomega v1.27.7 h1:fVih9JD6ogIiHUN6ePK7HJidyEDpWGVB5mzM7cWNXoU=
 github.com/onsi/gomega v1.27.7/go.mod h1:1p8OOlwo2iUUDsHnOrjE5UKYJ+e3W8eQ3qSlRahPmr4=
-github.com/openshift/api v0.0.0-20220906163444-2df055c101a3 h1:JEFTPLulnOSzBIsZZWitpm0SMJ+TFr7kUumAx7LvvGI=
-github.com/openshift/api v0.0.0-20220906163444-2df055c101a3/go.mod h1:9JWn+H7X8wEPPc9D63krigXl8r3F1Mt6/lC98brUyhQ=
+github.com/openshift/api v0.0.0-20230712163317-e19a88e10d9c h1:CrYt+EyqxuL8EulM5w37YSJ2jAxxChbyA0e4rQWosuI=
+github.com/openshift/api v0.0.0-20230712163317-e19a88e10d9c/go.mod h1:yimSGmjsI+XF1mr+AKBs2//fSXIOhhetHGbMlBEfXbs=
 github.com/openshift/cloud-credential-operator v0.0.0-20211118210017-9066dcc747fa h1:q2NffXPZIu0OfddEsV/6SqSpQAwol1VD9lOIc5JEmvE=
 github.com/openshift/cloud-credential-operator v0.0.0-20211118210017-9066dcc747fa/go.mod h1:2yIM8jdbNwuuHOIWwE8wzE+bxu6XvyAPpNzQPK2azgc=
 github.com/opentracing/opentracing-go v1.1.0/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o=

test/e2e/aws.go

@@ -5,6 +5,7 @@ package e2e
 
 import (
 	"fmt"
+	"github.com/aws/aws-sdk-go/aws/credentials/stscreds"
 	"time"
 
 	"github.com/aws/aws-sdk-go/aws"
@@ -21,9 +22,11 @@ import (
 )
 
 type awsTestHelper struct {
-	r53Client *route53.Route53
-	keyID     string
-	secretKey string
+	awsSession          *session.Session
+	r53Client           *route53.Route53
+	r53AssumeRoleClient *route53.Route53
+	keyID               string
+	secretKey           string
 }
 
 func newAWSHelper(isOpenShiftCI bool, kubeClient client.Client) (providerTestHelper, error) {
@@ -32,11 +35,12 @@ func newAWSHelper(isOpenShiftCI bool, kubeClient client.Client) (providerTestHel
 		return nil, err
 	}
 
-	awsSession := session.Must(session.NewSession(&aws.Config{
+	provider.awsSession = session.Must(session.NewSession(&aws.Config{
 		Credentials: credentials.NewStaticCredentials(provider.keyID, provider.secretKey, ""),
 	}))
 
-	provider.r53Client = route53.New(awsSession)
+	provider.r53Client = route53.New(provider.awsSession)
+
 	return provider, nil
 }
 
@@ -184,3 +188,37 @@ func (a *awsTestHelper) prepareConfigurations(isOpenShiftCI bool, kubeClient cli
 	}
 	return nil
 }
+
+func (a *awsTestHelper) createAssumeRoleRoute53Client(assumeRoleARN string) *route53.Route53 {
+	sessRoute53 := a.awsSession.Copy()
+	sessRoute53.Config.WithCredentials(stscreds.NewCredentials(sessRoute53, assumeRoleARN))
+	r53AssumeRoleClient := route53.New(sessRoute53)
+	return r53AssumeRoleClient
+}
+
+func (a *awsTestHelper) getDNSRecordValuesWithAssumeRole(assumeRoleARN, zoneId, recordName, recordType string) (map[string]struct{}, error) {
+	r53AssumeRoleClient := a.createAssumeRoleRoute53Client(assumeRoleARN)
+	records, err := r53AssumeRoleClient.ListResourceRecordSets(&route53.ListResourceRecordSetsInput{
+		HostedZoneId:    &zoneId,
+		StartRecordName: &recordName,
+		StartRecordType: &recordType,
+	})
+	if err != nil {
+		return nil, fmt.Errorf("failed to list resource record sets: %w", err)
+	}
+	//*route53.Route53.ListRe
+	if len(records.ResourceRecordSets) == 0 {
+		return nil, nil
+	}
+
+	recordList := make(map[string]struct{})
+	if records.ResourceRecordSets[0].AliasTarget != nil {
+		recordList[*records.ResourceRecordSets[0].AliasTarget.DNSName] = struct{}{}
+	} else {
+		for _, record := range records.ResourceRecordSets[0].ResourceRecords {
+			recordList[*record.Value] = struct{}{}
+		}
+	}
+
+	return recordList, nil
+}

test/e2e/azure.go

@@ -183,3 +183,7 @@ func getAccessToken(cfg *clusterConfig) (*adal.ServicePrincipalToken, error) {
 	}
 	return nil, fmt.Errorf("no credentials provided for Azure API")
 }
+
+func (a *azureTestHelper) getDNSRecordValuesWithAssumeRole(assumeRoleARN, zoneId, recordName, recordType string) (map[string]struct{}, error) {
+	panic("not implemented")
+}

test/e2e/gcp.go

@@ -194,3 +194,7 @@ func getGCPProjectId(kubeClient client.Client) (string, error) {
 	}
 	return infraConfig.Status.PlatformStatus.GCP.ProjectID, nil
 }
+
+func (g *gcpTestHelper) getDNSRecordValuesWithAssumeRole(assumeRoleARN, zoneId, recordName, recordType string) (map[string]struct{}, error) {
+	panic("not implemented")
+}

test/e2e/infoblox.go

@@ -330,3 +330,7 @@ func (h *infobloxTestHelper) trustGridTLSCert(kubeClient client.Client) error {
 
 	return nil
 }
+
+func (h *infobloxTestHelper) getDNSRecordValuesWithAssumeRole(assumeRoleARN, zoneId, recordName, recordType string) (map[string]struct{}, error) {
+	panic("not implemented")
+}

test/e2e/operator_test.go

@@ -6,12 +6,6 @@ package e2e
 import (
 	"context"
 	"fmt"
-	"os"
-	"strconv"
-	"strings"
-	"testing"
-	"time"
-
 	operatorv1 "github.com/openshift/api/operator/v1"
 	routev1 "github.com/openshift/api/route/v1"
 	appsv1 "k8s.io/api/apps/v1"
@@ -21,7 +15,13 @@ import (
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/types"
 	"k8s.io/apimachinery/pkg/util/wait"
+	"k8s.io/client-go/kubernetes"
 	kscheme "k8s.io/client-go/kubernetes/scheme"
+	"os"
+	"strconv"
+	"strings"
+	"testing"
+	"time"
 
 	configv1 "github.com/openshift/api/config/v1"
 	olmv1alpha1 "github.com/operator-framework/api/pkg/operators/v1alpha1"
@@ -55,6 +55,7 @@ const (
 
 var (
 	kubeClient       client.Client
+	kubeClientSet    *kubernetes.Clientset
 	scheme           *runtime.Scheme
 	nameServers      []string
 	hostedZoneID     string
@@ -94,6 +95,13 @@ func initKubeClient() error {
 	if err != nil {
 		return fmt.Errorf("failed to create kube client: %w", err)
 	}
+
+	kubeClientSet, err = kubernetes.NewForConfig(kubeConfig)
+	if err != nil {
+		fmt.Printf("failed to create kube clientset: %s\n", err)
+		os.Exit(1)
+	}
+
 	return nil
 }
 
@@ -311,46 +319,9 @@ func TestExternalDNSRecordLifecycle(t *testing.T) {
 		_ = kubeClient.Delete(context.TODO(), service)
 	}()
 
-	serviceIPs := make(map[string]struct{})
-	// get the IPs of the loadbalancer which is created for the service
-	if err := wait.PollUntilContextTimeout(context.TODO(), dnsPollingInterval, dnsPollingTimeout, true, func(ctx context.Context) (done bool, err error) {
-		t.Log("Getting IPs of service's load balancer")
-		var service corev1.Service
-		err = kubeClient.Get(ctx, types.NamespacedName{
-			Namespace: testNamespace,
-			Name:      testServiceName,
-		}, &service)
-		if err != nil {
-			return false, err
-		}
-
-		// if there is no associated loadbalancer then retry later
-		if len(service.Status.LoadBalancer.Ingress) < 1 {
-			return false, nil
-		}
-
-		// get the IPs of the loadbalancer
-		if service.Status.LoadBalancer.Ingress[0].IP != "" {
-			serviceIPs[service.Status.LoadBalancer.Ingress[0].IP] = struct{}{}
-		} else if service.Status.LoadBalancer.Ingress[0].Hostname != "" {
-			lbHostname := service.Status.LoadBalancer.Ingress[0].Hostname
-			ips, err := lookupARecord(lbHostname, googleDNSServer)
-			if err != nil {
-				t.Logf("Waiting for IP of loadbalancer %s", lbHostname)
-				// if the hostname cannot be resolved currently then retry later
-				return false, nil
-			}
-			for _, ip := range ips {
-				serviceIPs[ip] = struct{}{}
-			}
-		} else {
-			t.Logf("Waiting for loadbalancer details for service %s", testServiceName)
-			return false, nil
-		}
-		t.Logf("Loadbalancer's IP(s): %v", serviceIPs)
-		return true, nil
-	}); err != nil {
-		t.Fatalf("Failed to get loadbalancer IPs for service %s/%s: %v", testNamespace, testServiceName, err)
+	serviceIPs, err := getServiceIP(t)
+	if err != nil {
+		t.Fatalf("Failed to get service IPs: %v", err)
 	}
 
 	// try all nameservers and fail only if all failed
@@ -678,8 +649,168 @@ func TestExternalDNSSecretCredentialUpdate(t *testing.T) {
 	}
 }
 
+func TestExternalDNSAssumeRoleInSharedRoute53(t *testing.T) {
+	// This only works if the dns config sets the privateZoneIAMRole which indicates it's a "Shared VPC" cluster
+	// Note: Only AWS supports privateZoneIAMRole
+	dnsConfig := configv1.DNS{}
+	err := kubeClient.Get(context.TODO(), types.NamespacedName{Name: "cluster"}, &dnsConfig)
+	if err != nil {
+		t.Errorf("failed to get dns 'cluster': %v\n", err)
+	}
+	if dnsConfig.Spec.Platform.AWS == nil && dnsConfig.Spec.Platform.AWS.PrivateZoneIAMRole == "" {
+		t.Skipf("test skipped on non-shared-VPC cluster")
+	}
+
+	t.Log("Ensuring test namespace")
+	err = kubeClient.Create(context.TODO(), &corev1.Namespace{ObjectMeta: metav1.ObjectMeta{Name: testNamespace}})
+	if err != nil && !errors.IsAlreadyExists(err) {
+		t.Fatalf("Failed to ensure namespace %s: %v", testNamespace, err)
+	}
+
+	// secret is needed only for DNS providers which cannot get their credentials from CCO
+	// namely Infobox, BlueCat
+	t.Log("Creating credentials secret")
+	credSecret := helper.makeCredentialsSecret(operatorNamespace)
+	err = kubeClient.Create(context.TODO(), credSecret)
+	if err != nil {
+		t.Fatalf("Failed to create credentials secret %s/%s: %v", credSecret.Namespace, credSecret.Name, err)
+	}
+
+	// Create an External object that uses the role ARN in the dns config to create DNS records in the private DNS
+	// zone in another AWS account route 53
+	t.Log("Creating ExternalDNS object that assumes role our of private zone in another account's route 53")
+	extDNS := helper.buildExternalDNS(testExtDNSName, dnsConfig.Spec.PrivateZone.ID, dnsConfig.Spec.BaseDomain, credSecret)
+	extDNS.Spec.Provider.AWS.AssumeRole = &operatorv1beta1.ExternalDNSAWSAssumeRoleOptions{
+		ARN: &dnsConfig.Spec.Platform.AWS.PrivateZoneIAMRole,
+	}
+	if err := kubeClient.Create(context.TODO(), &extDNS); err != nil {
+		t.Fatalf("Failed to create external DNS %q: %v", testExtDNSName, err)
+	}
+	defer func() {
+		_ = kubeClient.Delete(context.TODO(), &extDNS)
+	}()
+
+	// create a service of type LoadBalancer with the annotation targeted by the ExternalDNS resource
+	t.Log("Creating source service")
+	service := defaultService(testServiceName, testNamespace)
+	if err := kubeClient.Create(context.TODO(), service); err != nil {
+		t.Fatalf("Failed to create test service %s/%s: %v", testNamespace, testServiceName, err)
+	}
+	defer func() {
+		_ = kubeClient.Delete(context.TODO(), service)
+	}()
+
+	serviceIPs := make(map[string]struct{})
+	if err := wait.PollUntilContextTimeout(context.TODO(), dnsPollingInterval, dnsPollingTimeout, true, func(ctx context.Context) (done bool, err error) {
+		t.Log("Getting IPs of service's load balancer")
+		var service corev1.Service
+		err = kubeClient.Get(ctx, types.NamespacedName{
+			Namespace: testNamespace,
+			Name:      testServiceName,
+		}, &service)
+		if err != nil {
+			return false, err
+		}
+
+		// if there is no associated loadbalancer then retry later
+		if len(service.Status.LoadBalancer.Ingress) < 1 {
+			return false, nil
+		}
+
+		// get the IPs of the loadbalancer
+		if service.Status.LoadBalancer.Ingress[0].IP != "" {
+			serviceIPs[service.Status.LoadBalancer.Ingress[0].IP] = struct{}{}
+		} else if service.Status.LoadBalancer.Ingress[0].Hostname != "" {
+			serviceIPs[service.Status.LoadBalancer.Ingress[0].Hostname] = struct{}{}
+		} else {
+			t.Logf("Waiting for loadbalancer details for service %s", testServiceName)
+			return false, nil
+		}
+		t.Logf("Loadbalancer's IP(s): %v", serviceIPs)
+		return true, nil
+	}); err != nil {
+		t.Errorf("Failed to get loadbalancer IPs for service %s/%s: %v", testNamespace, testServiceName, err)
+	}
+
+	// Query Route 53 API with assume role ARN from the dns config, then compare the results to ensure it matches
+	// the service hostname
+	expectedHost := fmt.Sprintf("%s.%s", testServiceName, dnsConfig.Spec.BaseDomain)
+	if err := wait.PollUntilContextTimeout(context.TODO(), dnsPollingInterval, dnsPollingTimeout, true, func(ctx context.Context) (done bool, err error) {
+		recordValues, err := helper.getDNSRecordValuesWithAssumeRole(dnsConfig.Spec.Platform.AWS.PrivateZoneIAMRole, dnsConfig.Spec.PrivateZone.ID, expectedHost, "A")
+		if err != nil {
+			t.Logf("failed to get DNS record for shared VPC zone: %v", err)
+			return false, nil
+		} else if len(recordValues) == 0 {
+			t.Logf("no DNS records with name %q", expectedHost)
+			return false, nil
+		}
+
+		// all expected IPs should be in the received IPs
+		// but these 2 sets are not necessary equal
+		for ip := range serviceIPs {
+			if _, found := recordValues[ip]; !found {
+				if _, foundWithDot := recordValues[ip+"."]; !foundWithDot {
+					t.Logf("DNS record with name %q didn't contain expected service IP %q", expectedHost, ip)
+					return false, nil
+				}
+			}
+		}
+		t.Logf("DNS record with name %q found in shared Route 53 private zone %q and matched service IPs", expectedHost, dnsConfig.Spec.PrivateZone.ID)
+		return true, nil
+	}); err != nil {
+		t.Fatalf("Failed to verify that DNS has been correctly set.")
+	}
+}
+
 // HELPER FUNCTIONS
 
+func getServiceIP(t *testing.T) (map[string]struct{}, error) {
+	t.Helper()
+	serviceIPs := make(map[string]struct{})
+	// get the IPs of the loadbalancer which is created for the service
+	if err := wait.PollUntilContextTimeout(context.TODO(), dnsPollingInterval, dnsPollingTimeout, true, func(ctx context.Context) (done bool, err error) {
+		t.Log("Getting IPs of service's load balancer")
+		var service corev1.Service
+		err = kubeClient.Get(ctx, types.NamespacedName{
+			Namespace: testNamespace,
+			Name:      testServiceName,
+		}, &service)
+		if err != nil {
+			return false, err
+		}
+
+		// if there is no associated loadbalancer then retry later
+		if len(service.Status.LoadBalancer.Ingress) < 1 {
+			return false, nil
+		}
+
+		// get the IPs of the loadbalancer
+		if service.Status.LoadBalancer.Ingress[0].IP != "" {
+			serviceIPs[service.Status.LoadBalancer.Ingress[0].IP] = struct{}{}
+		} else if service.Status.LoadBalancer.Ingress[0].Hostname != "" {
+			lbHostname := service.Status.LoadBalancer.Ingress[0].Hostname
+			ips, err := lookupARecord(lbHostname, googleDNSServer)
+			if err != nil {
+				t.Logf("Waiting for IP of loadbalancer %s", lbHostname)
+				// if the hostname cannot be resolved currently then retry later
+				return false, nil
+			}
+			for _, ip := range ips {
+				serviceIPs[ip] = struct{}{}
+			}
+		} else {
+			t.Logf("Waiting for loadbalancer details for service %s", testServiceName)
+			return false, nil
+		}
+		t.Logf("Loadbalancer's IP(s): %v", serviceIPs)
+		return true, nil
+	}); err != nil {
+		return nil, fmt.Errorf("Failed to get loadbalancer IPs for service %s/%s: %v", testNamespace, testServiceName, err)
+	}
+
+	return serviceIPs, nil
+}
+
 func ensureOperandResources() error {
 	if os.Getenv(e2eSeparateOperandNsEnvVar) != "true" {
 		return nil

test/e2e/util.go

@@ -46,6 +46,7 @@ type providerTestHelper interface {
 	buildExternalDNS(name, zoneID, zoneDomain string, credsSecret *corev1.Secret) operatorv1beta1.ExternalDNS
 	buildOpenShiftExternalDNS(name, zoneID, zoneDomain, routeName string, credsSecret *corev1.Secret) operatorv1beta1.ExternalDNS
 	buildOpenShiftExternalDNSV1Alpha1(name, zoneID, zoneDomain, routeName string, credsSecret *corev1.Secret) operatorv1alpha1.ExternalDNS
+	getDNSRecordValuesWithAssumeRole(assumeRoleARN, zoneId, recordName, recordType string) (map[string]struct{}, error)
 }
 
 func randomString(n int) string {