Merge pull request #5390 from stijndehaes/feature/add-support-for-wor…

…kload-identity

Added support for azure workload identity

charts/cluster-autoscaler/Chart.yaml

@@ -11,4 +11,4 @@ name: cluster-autoscaler
 sources:
   - https://github.com/kubernetes/autoscaler/tree/master/cluster-autoscaler
 type: application
-version: 9.22.0
+version: 9.23.0

charts/cluster-autoscaler/README.md

@@ -250,6 +250,34 @@ In order to accomplish this, you will first need to create a new IAM role with t
 
 Once you have the IAM role configured, you would then need to `--set rbac.serviceAccount.annotations."eks\.amazonaws\.com/role-arn"=arn:aws:iam::123456789012:role/MyRoleName` when installing.
 
+### Azure - Using azure workload identity
+
+You can use the project [Azure workload identity](https://github.com/Azure/azure-workload-identity), to automatically configure the correct setup for your pods to used federated identity with Azure.
+You can also set the correct settings yourself instead of relying on this project.
+For example the following configuration will configure the Autoscaler to use your federated identity:
+
+```yaml
+azureUseWorkloadIdentityExtension: true
+extraEnv:
+  AZURE_CLIENT_ID: USER ASSIGNED IDENTITY CLIENT ID
+  AZURE_TENANT_ID: USER ASSIGNED IDENTITY TENANT ID
+  AZURE_FEDERATED_TOKEN_FILE: /var/run/secrets/tokens/azure-identity-token
+  AZURE_AUTHORITY_HOST: https://login.microsoftonline.com/
+extraVolumes:
+- name: azure-identity-token
+  projected:
+    defaultMode: 420
+    sources:
+    - serviceAccountToken:
+        audience: api://AzureADTokenExchange
+        expirationSeconds: 3600
+        path: azure-identity-token
+extraVolumeMounts:
+- mountPath: /var/run/secrets/tokens
+  name: azure-identity-token
+  readOnly: true
+```
+
 ## Troubleshooting
 
 The chart will succeed even if the container arguments are incorrect. A few minutes after starting
@@ -303,7 +331,8 @@ Though enough for the majority of installations, the default PodSecurityPolicy _
 | azureResourceGroup | string | `""` | Azure resource group that the cluster is located. Required if `cloudProvider=azure` |
 | azureSubscriptionID | string | `""` | Azure subscription where the resources are located. Required if `cloudProvider=azure` |
 | azureTenantID | string | `""` | Azure tenant where the resources are located. Required if `cloudProvider=azure` |
-| azureUseManagedIdentityExtension | bool | `false` | Whether to use Azure's managed identity extension for credentials. If using MSI, ensure subscription ID, resource group, and azure AKS cluster name are set. |
+| azureUseManagedIdentityExtension | bool | `false` | Whether to use Azure's managed identity extension for credentials. If using MSI, ensure subscription ID, resource group, and azure AKS cluster name are set. You can only use one authentication method at a time, either azureUseWorkloadIdentityExtension or azureUseManagedIdentityExtension should be set. |
+| azureUseWorkloadIdentityExtension | bool | `false` | Whether to use Azure's workload identity extension for credentials. See the project here: https://github.com/Azure/azure-workload-identity for more details. You can only use one authentication method at a time, either azureUseWorkloadIdentityExtension or azureUseManagedIdentityExtension should be set. |
 | azureVMType | string | `"AKS"` | Azure VM type. |
 | cloudConfigPath | string | `""` | Configuration file for cloud provider. |
 | cloudProvider | string | `"aws"` | The cloud provider where the autoscaler runs. Currently only `gce`, `aws`, `azure`, `magnum` and `clusterapi` are supported. `aws` supported for AWS. `gce` for GCE. `azure` for Azure AKS. `magnum` for OpenStack Magnum, `clusterapi` for Cluster API. |

charts/cluster-autoscaler/README.md.gotmpl

@@ -251,6 +251,34 @@ In order to accomplish this, you will first need to create a new IAM role with t
 
 Once you have the IAM role configured, you would then need to `--set rbac.serviceAccount.annotations."eks\.amazonaws\.com/role-arn"=arn:aws:iam::123456789012:role/MyRoleName` when installing.
 
+### Azure - Using azure workload identity
+
+You can use the project [Azure workload identity](https://github.com/Azure/azure-workload-identity), to automatically configure the correct setup for your pods to used federated identity with Azure.
+You can also set the correct settings yourself instead of relying on this project.
+For example the following configuration will configure the Autoscaler to use your federated identity:
+
+```yaml
+azureUseWorkloadIdentityExtension: true
+extraEnv:
+  AZURE_CLIENT_ID: USER ASSIGNED IDENTITY CLIENT ID
+  AZURE_TENANT_ID: USER ASSIGNED IDENTITY TENANT ID
+  AZURE_FEDERATED_TOKEN_FILE: /var/run/secrets/tokens/azure-identity-token
+  AZURE_AUTHORITY_HOST: https://login.microsoftonline.com/
+extraVolumes:
+- name: azure-identity-token
+  projected:
+    defaultMode: 420
+    sources:
+    - serviceAccountToken:
+        audience: api://AzureADTokenExchange
+        expirationSeconds: 3600
+        path: azure-identity-token
+extraVolumeMounts:
+- mountPath: /var/run/secrets/tokens
+  name: azure-identity-token
+  readOnly: true
+```
+
 ## Troubleshooting
 
 The chart will succeed even if the container arguments are incorrect. A few minutes after starting

charts/cluster-autoscaler/templates/deployment.yaml

@@ -159,7 +159,10 @@ spec:
                 secretKeyRef:
                   key: ClusterName
                   name: {{ template "cluster-autoscaler.fullname" . }}
-            {{- if .Values.azureUseManagedIdentityExtension }}
+            {{- if .Values.azureUseWorkloadIdentityExtension }}
+            - name: ARM_USE_WORKLOAD_IDENTITY_EXTENSION
+              value: "true"
+            {{- else if .Values.azureUseManagedIdentityExtension }}
             - name: ARM_USE_MANAGED_IDENTITY_EXTENSION
               value: "true"
             {{- else }}

charts/cluster-autoscaler/values.yaml

@@ -95,7 +95,10 @@ azureClusterName: ""
 # Required if `cloudProvider=azure`
 azureNodeResourceGroup: ""
 
-# azureUseManagedIdentityExtension -- Whether to use Azure's managed identity extension for credentials. If using MSI, ensure subscription ID, resource group, and azure AKS cluster name are set.
+# azureUseWorkloadIdentityExtension -- Whether to use Azure's workload identity extension for credentials. See the project here: https://github.com/Azure/azure-workload-identity for more details. You can only use one authentication method at a time, either azureUseWorkloadIdentityExtension or azureUseManagedIdentityExtension should be set.
+azureUseWorkloadIdentityExtension: false
+
+# azureUseManagedIdentityExtension -- Whether to use Azure's managed identity extension for credentials. If using MSI, ensure subscription ID, resource group, and azure AKS cluster name are set. You can only use one authentication method at a time, either azureUseWorkloadIdentityExtension or azureUseManagedIdentityExtension should be set.
 azureUseManagedIdentityExtension: false
 
 # magnumClusterName -- Cluster name or ID in Magnum.

cluster-autoscaler/cloudprovider/azure/azure_client.go

@@ -21,6 +21,7 @@ import (
 	"fmt"
 	"io/ioutil"
 	"net/http"
+	"os"
 	"time"
 
 	"github.com/Azure/azure-sdk-for-go/services/compute/mgmt/2019-07-01/compute"
@@ -162,6 +163,18 @@ func newServicePrincipalTokenFromCredentials(config *Config, env *azure.Environm
 		return nil, fmt.Errorf("creating the OAuth config: %v", err)
 	}
 
+	if config.UseWorkloadIdentityExtension {
+		klog.V(2).Infoln("azure: using workload identity extension to retrieve access token")
+		jwt, err := os.ReadFile(config.AADFederatedTokenFile)
+		if err != nil {
+			return nil, fmt.Errorf("failed to read a file with a federated token: %v", err)
+		}
+		token, err := adal.NewServicePrincipalTokenFromFederatedToken(*oauthConfig, config.AADClientID, string(jwt), env.ResourceManagerEndpoint)
+		if err != nil {
+			return nil, fmt.Errorf("failed to create a workload identity token: %v", err)
+		}
+		return token, nil
+	}
 	if config.UseManagedIdentityExtension {
 		klog.V(2).Infoln("azure: using managed identity extension to retrieve access token")
 		msiEndpoint, err := adal.GetMSIVMEndpoint()

cluster-autoscaler/cloudprovider/azure/azure_config.go

@@ -97,12 +97,14 @@ type Config struct {
 
 	// Settings for a service principal.
 
-	AADClientID                 string `json:"aadClientId" yaml:"aadClientId"`
-	AADClientSecret             string `json:"aadClientSecret" yaml:"aadClientSecret"`
-	AADClientCertPath           string `json:"aadClientCertPath" yaml:"aadClientCertPath"`
-	AADClientCertPassword       string `json:"aadClientCertPassword" yaml:"aadClientCertPassword"`
-	UseManagedIdentityExtension bool   `json:"useManagedIdentityExtension" yaml:"useManagedIdentityExtension"`
-	UserAssignedIdentityID      string `json:"userAssignedIdentityID" yaml:"userAssignedIdentityID"`
+	AADClientID                  string `json:"aadClientId" yaml:"aadClientId"`
+	AADClientSecret              string `json:"aadClientSecret" yaml:"aadClientSecret"`
+	AADClientCertPath            string `json:"aadClientCertPath" yaml:"aadClientCertPath"`
+	AADClientCertPassword        string `json:"aadClientCertPassword" yaml:"aadClientCertPassword"`
+	AADFederatedTokenFile        string `json:"aadFederatedTokenFile" yaml:"aadFederatedTokenFile"`
+	UseManagedIdentityExtension  bool   `json:"useManagedIdentityExtension" yaml:"useManagedIdentityExtension"`
+	UseWorkloadIdentityExtension bool   `json:"useWorkloadIdentityExtension" yaml:"useWorkloadIdentityExtension"`
+	UserAssignedIdentityID       string `json:"userAssignedIdentityID" yaml:"userAssignedIdentityID"`
 
 	// Configs only for standard vmType (agent pools).
 	Deployment           string                 `json:"deployment" yaml:"deployment"`
@@ -155,7 +157,14 @@ func BuildAzureConfig(configReader io.Reader) (*Config, error) {
 		cfg.Location = os.Getenv("LOCATION")
 		cfg.ResourceGroup = os.Getenv("ARM_RESOURCE_GROUP")
 		cfg.TenantID = os.Getenv("ARM_TENANT_ID")
+		if tenantId := os.Getenv("AZURE_TENANT_ID"); tenantId != "" {
+			cfg.TenantID = tenantId
+		}
 		cfg.AADClientID = os.Getenv("ARM_CLIENT_ID")
+		if clientId := os.Getenv("AZURE_CLIENT_ID"); clientId != "" {
+			cfg.AADClientID = clientId
+		}
+		cfg.AADFederatedTokenFile = os.Getenv("AZURE_FEDERATED_TOKEN_FILE")
 		cfg.AADClientSecret = os.Getenv("ARM_CLIENT_SECRET")
 		cfg.VMType = strings.ToLower(os.Getenv("ARM_VM_TYPE"))
 		cfg.AADClientCertPath = os.Getenv("ARM_CLIENT_CERT_PATH")
@@ -178,6 +187,18 @@ func BuildAzureConfig(configReader io.Reader) (*Config, error) {
 			}
 		}
 
+		useWorkloadIdentityExtensionFromEnv := os.Getenv("ARM_USE_WORKLOAD_IDENTITY_EXTENSION")
+		if len(useWorkloadIdentityExtensionFromEnv) > 0 {
+			cfg.UseWorkloadIdentityExtension, err = strconv.ParseBool(useWorkloadIdentityExtensionFromEnv)
+			if err != nil {
+				return nil, err
+			}
+		}
+
+		if cfg.UseManagedIdentityExtension && cfg.UseWorkloadIdentityExtension {
+			return nil, errors.New("you can not combine both managed identity and workload identity as an authentication mechanism")
+		}
+
 		userAssignedIdentityIDFromEnv := os.Getenv("ARM_USER_ASSIGNED_IDENTITY_ID")
 		if userAssignedIdentityIDFromEnv != "" {
 			cfg.UserAssignedIdentityID = userAssignedIdentityIDFromEnv