feat: add support for non-anonymous login for Network Isolated Clusters

e2e/aks_model.go

@@ -2,9 +2,12 @@ package e2e
 
 import (
 	"context"
+	"encoding/base64"
 	"errors"
 	"fmt"
 	"net"
+	"os"
+	"path/filepath"
 	"strings"
 	"testing"
 
@@ -15,6 +18,11 @@ import (
 	"github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/containerservice/armcontainerservice/v6"
 	"github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/network/armnetwork/v6"
 	"github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/privatedns/armprivatedns"
+	v1 "k8s.io/api/core/v1"
+	errorsk8s "k8s.io/apimachinery/pkg/api/errors"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/client-go/kubernetes"
+	"k8s.io/client-go/tools/clientcmd"
 )
 
 func getKubenetClusterModel(name string) *armcontainerservice.ManagedCluster {
@@ -66,7 +74,7 @@ func getBaseClusterModel(clusterName string) *armcontainerservice.ManagedCluster
 	}
 }
 
-func addAirgapNetworkSettings(ctx context.Context, t *testing.T, clusterModel *armcontainerservice.ManagedCluster) error {
+func addAirgapNetworkSettings(ctx context.Context, t *testing.T, clusterModel *armcontainerservice.ManagedCluster, privateACRName string) error {
 	t.Logf("Adding network settings for airgap cluster %s in rg %s", *clusterModel.Name, *clusterModel.Properties.NodeResourceGroup)
 
 	vnet, err := getClusterVNet(ctx, *clusterModel.Properties.NodeResourceGroup)
@@ -98,7 +106,7 @@ func addAirgapNetworkSettings(ctx context.Context, t *testing.T, clusterModel *a
 		return err
 	}
 
-	err = addPrivateEndpointForACR(ctx, t, *clusterModel.Properties.NodeResourceGroup, vnet)
+	err = addPrivateEndpointForACR(ctx, t, *clusterModel.Properties.NodeResourceGroup, privateACRName, vnet)
 	if err != nil {
 		return err
 	}
@@ -150,7 +158,7 @@ func airGapSecurityGroup(location, clusterFQDN string) (armnetwork.SecurityGroup
 	}, nil
 }
 
-func addPrivateEndpointForACR(ctx context.Context, t *testing.T, nodeResourceGroup string, vnet VNet) error {
+func addPrivateEndpointForACR(ctx context.Context, t *testing.T, nodeResourceGroup, privateACRName string, vnet VNet) error {
 	t.Logf("Checking if private endpoint for private container registry is in rg %s", nodeResourceGroup)
 
 	var err error
@@ -165,7 +173,7 @@ func addPrivateEndpointForACR(ctx context.Context, t *testing.T, nodeResourceGro
 	}
 
 	var peResp armnetwork.PrivateEndpointsClientCreateOrUpdateResponse
-	if peResp, err = createPrivateEndpoint(ctx, t, nodeResourceGroup, privateEndpointName, config.PrivateACRName, vnet); err != nil {
+	if peResp, err = createPrivateEndpoint(ctx, t, nodeResourceGroup, privateEndpointName, privateACRName, vnet); err != nil {
 		return err
 	}
 
@@ -201,8 +209,8 @@ func privateEndpointExists(ctx context.Context, t *testing.T, nodeResourceGroup,
 	return false, nil
 }
 
-func createPrivateAzureContainerRegistry(ctx context.Context, t *testing.T, cluster *armcontainerservice.ManagedCluster, resourceGroup, privateACRName string) error {
-	t.Logf("Creating private Azure Container Registry in rg %s", resourceGroup)
+func createPrivateAzureContainerRegistry(ctx context.Context, t *testing.T, cluster *armcontainerservice.ManagedCluster, resourceGroup, privateACRName string, isNonAnonymousPull bool) error {
+	t.Logf("Creating private Azure Container Registry %s in rg %s", privateACRName, resourceGroup)
 
 	acr, err := config.Azure.RegistriesClient.Get(ctx, resourceGroup, privateACRName, nil)
 	if err == nil {
@@ -238,8 +246,8 @@ func createPrivateAzureContainerRegistry(ctx context.Context, t *testing.T, clus
 			Name: to.Ptr(armcontainerregistry.SKUNamePremium),
 		},
 		Properties: &armcontainerregistry.RegistryProperties{
-			AdminUserEnabled:     to.Ptr(false),
-			AnonymousPullEnabled: to.Ptr(true), // required to pull images from the private ACR without authentication
+			AdminUserEnabled:     to.Ptr(isNonAnonymousPull),  // if non-anonymous pull is enabled, admin user must be enabled to be able to set credentials for the debug pods
+			AnonymousPullEnabled: to.Ptr(!isNonAnonymousPull), // required to pull images from the private ACR without authentication
 		},
 	}
 	pollerResp, err := config.Azure.RegistriesClient.BeginCreate(
@@ -258,9 +266,106 @@ func createPrivateAzureContainerRegistry(ctx context.Context, t *testing.T, clus
 	}
 
 	t.Logf("Private Azure Container Registry created")
-	if err := addCacheRuelsToPrivateAzureContainerRegistry(ctx, t, config.ResourceGroupName, config.PrivateACRName); err != nil {
+
+	if isNonAnonymousPull {
+		t.Logf("Creating the secret for non-anonymous pull ACR for the e2e debug pods")
+		kubeconfigPath := os.Getenv("HOME") + "/.kube/config"
+		if err := fetchAndSaveKubeconfig(ctx, t, resourceGroup, *cluster.Name, kubeconfigPath); err != nil {
+			t.Logf("failed to fetch kubeconfig: %v", err)
+			return err
+		}
+		username, password, err := getAzureContainerRegistryCredentials(ctx, t, resourceGroup, privateACRName)
+		if err != nil {
+			t.Logf("failed to get private ACR credentials: %v", err)
+			return err
+		}
+		if err := createKubernetesSecret(ctx, t, "default", kubeconfigPath, config.Config.ACRSecretName, privateACRName, username, password); err != nil {
+			t.Logf("failed to create Kubernetes secret: %v", err)
+			return err
+		}
+	}
+
+	if err := addCacheRulesToPrivateAzureContainerRegistry(ctx, t, config.ResourceGroupName, privateACRName); err != nil {
 		return fmt.Errorf("failed to add cache rules to private acr: %w", err)
 	}
+
+	return nil
+}
+
+func createKubernetesSecret(ctx context.Context, t *testing.T, namespace, kubeconfigPath, secretName, registryName, username, password string) error {
+	t.Logf("Creating Kubernetes secret %s in namespace %s", secretName, namespace)
+	config, err := clientcmd.BuildConfigFromFlags("", kubeconfigPath)
+	if err != nil {
+		t.Logf("failed to build Kubernetes config: %v", err)
+		return err
+	}
+
+	clientset, err := kubernetes.NewForConfig(config)
+	if err != nil {
+		t.Logf("failed to create Kubernetes client: %v", err)
+		return err
+	}
+
+	auth := base64.StdEncoding.EncodeToString([]byte(fmt.Sprintf("%s:%s", username, password)))
+	dockerConfigJSON := fmt.Sprintf(`{
+		"auths": {
+			"%s.azurecr.io": {
+				"username": "%s",
+				"password": "%s",
+				"auth": "%s"
+			}
+		}
+	}`, registryName, username, password, auth)
+
+	secret := &v1.Secret{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      secretName,
+			Namespace: namespace,
+		},
+		Type: v1.SecretTypeDockerConfigJson,
+		Data: map[string][]byte{
+			v1.DockerConfigJsonKey: []byte(dockerConfigJSON),
+		},
+	}
+	_, err = clientset.CoreV1().Secrets(namespace).Create(ctx, secret, metav1.CreateOptions{})
+	if err != nil {
+		if !errorsk8s.IsAlreadyExists(err) {
+			t.Logf("failed to create Kubernetes secret: %v", err)
+			return err
+		}
+	}
+	t.Logf("Kubernetes secret created")
+	return nil
+}
+
+func getAzureContainerRegistryCredentials(ctx context.Context, t *testing.T, resourceGroup, privateACRName string) (string, string, error) {
+	t.Logf("Getting credentials for private Azure Container Registry in rg %s", resourceGroup)
+	acrCreds, err := config.Azure.RegistriesClient.ListCredentials(ctx, resourceGroup, privateACRName, nil)
+	if err != nil {
+		return "", "", fmt.Errorf("failed to get private ACR credentials: %w", err)
+	}
+	username := *acrCreds.Username
+	password := *acrCreds.Passwords[0].Value
+	t.Logf("Private Azure Container Registry credentials retrieved")
+	return username, password, nil
+}
+
+func fetchAndSaveKubeconfig(ctx context.Context, t *testing.T, resourceGroup, clusterName, kubeconfigPath string) error {
+	adminCredentials, err := config.Azure.AKS.ListClusterAdminCredentials(ctx, resourceGroup, clusterName, nil)
+	if err != nil {
+		return fmt.Errorf("failed to get cluster admin credentials: %w", err)
+	}
+	if len(adminCredentials.Kubeconfigs) == 0 {
+		return fmt.Errorf("no kubeconfig returned for cluster %s", clusterName)
+	}
+
+	if err := os.MkdirAll(filepath.Dir(kubeconfigPath), 0700); err != nil {
+		return fmt.Errorf("failed to create kubeconfig directory: %w", err)
+	}
+	if err := os.WriteFile(kubeconfigPath, adminCredentials.Kubeconfigs[0].Value, 0600); err != nil {
+		return fmt.Errorf("failed to save kubeconfig to %s: %w", kubeconfigPath, err)
+	}
+	t.Logf("Kubeconfig successfully saved to %s", kubeconfigPath)
 	return nil
 }
 
@@ -285,6 +390,11 @@ func shouldRecreateACR(ctx context.Context, t *testing.T, resourceGroup, private
 
 	cacheRules, err := config.Azure.CacheRulesClient.Get(ctx, resourceGroup, privateACRName, "aks-managed-rule", nil)
 	if err != nil {
+		var azErr *azcore.ResponseError
+		if errors.As(err, &azErr) && azErr.StatusCode == 404 {
+			t.Logf("Private ACR cache not found, need to recreate")
+			return nil, true
+		}
 		return fmt.Errorf("failed to get cache rules: %w", err), false
 	}
 	if cacheRules.Properties != nil && cacheRules.Properties.TargetRepository != nil && *cacheRules.Properties.TargetRepository != config.Config.AzureContainerRegistrytargetRepository {
@@ -295,7 +405,7 @@ func shouldRecreateACR(ctx context.Context, t *testing.T, resourceGroup, private
 	return nil, false
 }
 
-func addCacheRuelsToPrivateAzureContainerRegistry(ctx context.Context, t *testing.T, resourceGroup, privateACRName string) error {
+func addCacheRulesToPrivateAzureContainerRegistry(ctx context.Context, t *testing.T, resourceGroup, privateACRName string) error {
 	t.Logf("Adding cache rules to private Azure Container Registry in rg %s", resourceGroup)
 
 	cacheParams := armcontainerregistry.CacheRule{
@@ -324,9 +434,9 @@ func addCacheRuelsToPrivateAzureContainerRegistry(ctx context.Context, t *testin
 	return nil
 }
 
-func createPrivateEndpoint(ctx context.Context, t *testing.T, nodeResourceGroup, privateEndpointName, acrName string, vnet VNet) (armnetwork.PrivateEndpointsClientCreateOrUpdateResponse, error) {
+func createPrivateEndpoint(ctx context.Context, t *testing.T, nodeResourceGroup, privateEndpointName, privateACRName string, vnet VNet) (armnetwork.PrivateEndpointsClientCreateOrUpdateResponse, error) {
 	t.Logf("Creating Private Endpoint in rg %s", nodeResourceGroup)
-	acrID := fmt.Sprintf("/subscriptions/%s/resourceGroups/%s/providers/Microsoft.ContainerRegistry/registries/%s", config.Config.SubscriptionID, config.ResourceGroupName, acrName)
+	acrID := fmt.Sprintf("/subscriptions/%s/resourceGroups/%s/providers/Microsoft.ContainerRegistry/registries/%s", config.Config.SubscriptionID, config.ResourceGroupName, privateACRName)
 
 	peParams := armnetwork.PrivateEndpoint{
 		Location: to.Ptr(config.Config.Location),

e2e/cluster.go

@@ -7,31 +7,37 @@ import (
 	"encoding/json"
 	"errors"
 	"fmt"
+	"net/http"
 	"sync"
 	"testing"
 	"time"
 
 	"github.com/Azure/agentbaker/e2e/config"
 	"github.com/Azure/azure-sdk-for-go/sdk/azcore"
 	"github.com/Azure/azure-sdk-for-go/sdk/azcore/to"
+	"github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/authorization/armauthorization/v2"
 	"github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/compute/armcompute/v6"
 	"github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/containerservice/armcontainerservice/v6"
 	"github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/resources/armresources"
+	"github.com/google/uuid"
 	"k8s.io/apimachinery/pkg/util/wait"
 )
 
 var (
-	clusterKubenet       *Cluster
-	clusterKubenetAirgap *Cluster
-	clusterAzureNetwork  *Cluster
-
-	clusterKubenetError       error
-	clusterKubenetAirgapError error
-	clusterAzureNetworkError  error
-
-	clusterKubenetOnce       sync.Once
-	clusterKubenetAirgapOnce sync.Once
-	clusterAzureNetworkOnce  sync.Once
+	clusterKubenet              *Cluster
+	clusterKubenetAirgap        *Cluster
+	clusterKubenetNonAnonAirgap *Cluster
+	clusterAzureNetwork         *Cluster
+
+	clusterKubenetError              error
+	clusterKubenetAirgapError        error
+	clusterKubenetNonAnonAirgapError error
+	clusterAzureNetworkError         error
+
+	clusterKubenetOnce              sync.Once
+	clusterKubenetAirgapOnce        sync.Once
+	clusterKubenetNonAnonAirgapOnce sync.Once
+	clusterAzureNetworkOnce         sync.Once
 )
 
 type Cluster struct {
@@ -62,26 +68,33 @@ func (c *Cluster) MaxPodsPerNode() (int, error) {
 // sync.Once is used to ensure that only one cluster for the set of tests is created
 func ClusterKubenet(ctx context.Context, t *testing.T) (*Cluster, error) {
 	clusterKubenetOnce.Do(func() {
-		clusterKubenet, clusterKubenetError = prepareCluster(ctx, t, getKubenetClusterModel("abe2e-kubenet"), false)
+		clusterKubenet, clusterKubenetError = prepareCluster(ctx, t, getKubenetClusterModel("abe2e-kubenet"), false, false)
 	})
 	return clusterKubenet, clusterKubenetError
 }
 
 func ClusterKubenetAirgap(ctx context.Context, t *testing.T) (*Cluster, error) {
 	clusterKubenetAirgapOnce.Do(func() {
-		clusterKubenetAirgap, clusterKubenetAirgapError = prepareCluster(ctx, t, getKubenetClusterModel("abe2e-kubenet-airgap-dev"), true) // TODO (alburgess): remove -dev once CCOA is over
+		clusterKubenetAirgap, clusterKubenetAirgapError = prepareCluster(ctx, t, getKubenetClusterModel("abe2e-kubenet-airgap"), true, false)
 	})
 	return clusterKubenetAirgap, clusterKubenetAirgapError
 }
 
+func ClusterKubenetAirgapNonAnon(ctx context.Context, t *testing.T) (*Cluster, error) {
+	clusterKubenetNonAnonAirgapOnce.Do(func() {
+		clusterKubenetNonAnonAirgap, clusterKubenetNonAnonAirgapError = prepareCluster(ctx, t, getKubenetClusterModel("abe2e-kubenet-nonanonpull-airgap"), true, true)
+	})
+	return clusterKubenetNonAnonAirgap, clusterKubenetNonAnonAirgapError
+}
+
 func ClusterAzureNetwork(ctx context.Context, t *testing.T) (*Cluster, error) {
 	clusterAzureNetworkOnce.Do(func() {
-		clusterAzureNetwork, clusterAzureNetworkError = prepareCluster(ctx, t, getAzureNetworkClusterModel("abe2e-azure-network"), false)
+		clusterAzureNetwork, clusterAzureNetworkError = prepareCluster(ctx, t, getAzureNetworkClusterModel("abe2e-azure-network"), false, false)
 	})
 	return clusterAzureNetwork, clusterAzureNetworkError
 }
 
-func prepareCluster(ctx context.Context, t *testing.T, cluster *armcontainerservice.ManagedCluster, isAirgap bool) (*Cluster, error) {
+func prepareCluster(ctx context.Context, t *testing.T, cluster *armcontainerservice.ManagedCluster, isAirgap, isNonAnonymousPull bool) (*Cluster, error) {
 	ctx, cancel := context.WithTimeout(ctx, config.Config.TestTimeoutCluster)
 	defer cancel()
 	cluster.Name = to.Ptr(fmt.Sprintf("%s-%s", *cluster.Name, hash(cluster)))
@@ -101,13 +114,18 @@ func prepareCluster(ctx context.Context, t *testing.T, cluster *armcontainerserv
 		return nil, fmt.Errorf("get cluster subnet: %w", err)
 	}
 
+	privateACRName := config.PrivateACRName
+	if isNonAnonymousPull {
+		privateACRName = config.PrivateACRNameNotAnon
+	}
+	t.Logf("using private acr %q isAnonyomusPull %v", privateACRName, isNonAnonymousPull)
 	if isAirgap {
 		// private acr must be created before we add the debug daemonsets
-		if err := createPrivateAzureContainerRegistry(ctx, t, cluster, config.ResourceGroupName, config.PrivateACRName); err != nil {
+		if err := createPrivateAzureContainerRegistry(ctx, t, cluster, config.ResourceGroupName, privateACRName, isNonAnonymousPull); err != nil {
 			return nil, fmt.Errorf("failed to create private acr: %w", err)
 		}
 
-		if err := addAirgapNetworkSettings(ctx, t, cluster); err != nil {
+		if err := addAirgapNetworkSettings(ctx, t, cluster, privateACRName); err != nil {
 			return nil, fmt.Errorf("add airgap network settings: %w", err)
 		}
 	}
@@ -117,7 +135,17 @@ func prepareCluster(ctx context.Context, t *testing.T, cluster *armcontainerserv
 		return nil, fmt.Errorf("get kube client using cluster %q: %w", *cluster.Name, err)
 	}
 
-	if err := kube.EnsureDebugDaemonsets(ctx, t, isAirgap); err != nil {
+	if isNonAnonymousPull {
+		identity, err := config.Azure.UserAssignedIdentities.Get(ctx, config.ResourceGroupName, config.VMIdentityName, nil)
+		if err != nil {
+			t.Fatalf("failed to get VM identity: %v", err)
+		}
+		if err := assignACRPullToIdentity(ctx, t, privateACRName, *identity.Properties.PrincipalID); err != nil {
+			return nil, fmt.Errorf("assign acr pull to the managed identity: %w", err)
+		}
+	}
+
+	if err := kube.EnsureDebugDaemonsets(ctx, t, isAirgap, privateACRName); err != nil {
 		return nil, fmt.Errorf("ensure debug daemonsets for %q: %w", *cluster.Name, err)
 	}
 
@@ -136,6 +164,31 @@ func prepareCluster(ctx context.Context, t *testing.T, cluster *armcontainerserv
 	}, nil
 }
 
+func assignACRPullToIdentity(ctx context.Context, t *testing.T, privateACRName, principalID string) error {
+	t.Logf("assigning ACR-Pull role to %s", principalID)
+	scope := fmt.Sprintf("/subscriptions/%s/resourceGroups/%s/providers/Microsoft.ContainerRegistry/registries/%s", config.Config.SubscriptionID, config.ResourceGroupName, privateACRName)
+
+	uid := uuid.New().String()
+	_, err := config.Azure.RoleAssignments.Create(ctx, scope, uid, armauthorization.RoleAssignmentCreateParameters{
+		Properties: &armauthorization.RoleAssignmentProperties{
+			PrincipalID: to.Ptr(principalID),
+			// ACR-Pull role definition ID
+			RoleDefinitionID: to.Ptr("/providers/Microsoft.Authorization/roleDefinitions/7f951dda-4ed3-4680-a7ca-43fe172d538d"),
+			PrincipalType:    to.Ptr(armauthorization.PrincipalTypeServicePrincipal),
+		},
+	}, nil)
+	var respError *azcore.ResponseError
+	if err != nil {
+		// if the role assignment already exists, ignore the error
+		if errors.As(err, &respError) && respError.StatusCode == http.StatusConflict {
+			return nil
+		}
+		t.Logf("failed to assign ACR-Pull role to identity %s, error: %v", config.VMIdentityName, err)
+		return err
+	}
+	return nil
+}
+
 func hash(cluster *armcontainerservice.ManagedCluster) string {
 	jsonData, err := json.Marshal(cluster)
 	if err != nil {