From ef295a3bf0fed9bff55016dba5b4459fc6154289 Mon Sep 17 00:00:00 2001 From: sdehaes Date: Wed, 28 Dec 2022 10:38:08 +0100 Subject: [PATCH] Added support for azure workload identity --- charts/cluster-autoscaler/Chart.yaml | 2 +- charts/cluster-autoscaler/README.md | 31 ++++++++++++++++- charts/cluster-autoscaler/README.md.gotmpl | 28 ++++++++++++++++ .../templates/deployment.yaml | 5 ++- charts/cluster-autoscaler/values.yaml | 5 ++- .../cloudprovider/azure/azure_client.go | 13 ++++++++ .../cloudprovider/azure/azure_config.go | 33 +++++++++++++++---- 7 files changed, 107 insertions(+), 10 deletions(-) diff --git a/charts/cluster-autoscaler/Chart.yaml b/charts/cluster-autoscaler/Chart.yaml index ef2c15586fc2..2e231ceb5c73 100644 --- a/charts/cluster-autoscaler/Chart.yaml +++ b/charts/cluster-autoscaler/Chart.yaml @@ -11,4 +11,4 @@ name: cluster-autoscaler sources: - https://github.com/kubernetes/autoscaler/tree/master/cluster-autoscaler type: application -version: 9.22.0 +version: 9.23.0 diff --git a/charts/cluster-autoscaler/README.md b/charts/cluster-autoscaler/README.md index 86a4b6963258..578e570c8410 100644 --- a/charts/cluster-autoscaler/README.md +++ b/charts/cluster-autoscaler/README.md @@ -250,6 +250,34 @@ In order to accomplish this, you will first need to create a new IAM role with t Once you have the IAM role configured, you would then need to `--set rbac.serviceAccount.annotations."eks\.amazonaws\.com/role-arn"=arn:aws:iam::123456789012:role/MyRoleName` when installing. +### Azure - Using azure workload identity + +You can use the project [Azure workload identity](https://github.com/Azure/azure-workload-identity), to automatically configure the correct setup for your pods to used federated identity with Azure. +You can also set the correct settings yourself instead of relying on this project. +For example the following configuration will configure the Autoscaler to use your federated identity: + +```yaml +azureUseWorkloadIdentityExtension: true +extraEnv: + AZURE_CLIENT_ID: USER ASSIGNED IDENTITY CLIENT ID + AZURE_TENANT_ID: USER ASSIGNED IDENTITY TENANT ID + AZURE_FEDERATED_TOKEN_FILE: /var/run/secrets/tokens/azure-identity-token + AZURE_AUTHORITY_HOST: https://login.microsoftonline.com/ +extraVolumes: +- name: azure-identity-token + projected: + defaultMode: 420 + sources: + - serviceAccountToken: + audience: api://AzureADTokenExchange + expirationSeconds: 3600 + path: azure-identity-token +extraVolumeMounts: +- mountPath: /var/run/secrets/tokens + name: azure-identity-token + readOnly: true +``` + ## Troubleshooting The chart will succeed even if the container arguments are incorrect. A few minutes after starting @@ -303,7 +331,8 @@ Though enough for the majority of installations, the default PodSecurityPolicy _ | azureResourceGroup | string | `""` | Azure resource group that the cluster is located. Required if `cloudProvider=azure` | | azureSubscriptionID | string | `""` | Azure subscription where the resources are located. Required if `cloudProvider=azure` | | azureTenantID | string | `""` | Azure tenant where the resources are located. Required if `cloudProvider=azure` | -| azureUseManagedIdentityExtension | bool | `false` | Whether to use Azure's managed identity extension for credentials. If using MSI, ensure subscription ID, resource group, and azure AKS cluster name are set. | +| azureUseManagedIdentityExtension | bool | `false` | Whether to use Azure's managed identity extension for credentials. If using MSI, ensure subscription ID, resource group, and azure AKS cluster name are set. You can only use one authentication method at a time, either azureUseWorkloadIdentityExtension or azureUseManagedIdentityExtension should be set. | +| azureUseWorkloadIdentityExtension | bool | `false` | Whether to use Azure's workload identity extension for credentials. See the project here: https://github.com/Azure/azure-workload-identity for more details. You can only use one authentication method at a time, either azureUseWorkloadIdentityExtension or azureUseManagedIdentityExtension should be set. | | azureVMType | string | `"AKS"` | Azure VM type. | | cloudConfigPath | string | `""` | Configuration file for cloud provider. | | cloudProvider | string | `"aws"` | The cloud provider where the autoscaler runs. Currently only `gce`, `aws`, `azure`, `magnum` and `clusterapi` are supported. `aws` supported for AWS. `gce` for GCE. `azure` for Azure AKS. `magnum` for OpenStack Magnum, `clusterapi` for Cluster API. | diff --git a/charts/cluster-autoscaler/README.md.gotmpl b/charts/cluster-autoscaler/README.md.gotmpl index 4f109c6dccac..80dce493f4fb 100644 --- a/charts/cluster-autoscaler/README.md.gotmpl +++ b/charts/cluster-autoscaler/README.md.gotmpl @@ -251,6 +251,34 @@ In order to accomplish this, you will first need to create a new IAM role with t Once you have the IAM role configured, you would then need to `--set rbac.serviceAccount.annotations."eks\.amazonaws\.com/role-arn"=arn:aws:iam::123456789012:role/MyRoleName` when installing. +### Azure - Using azure workload identity + +You can use the project [Azure workload identity](https://github.com/Azure/azure-workload-identity), to automatically configure the correct setup for your pods to used federated identity with Azure. +You can also set the correct settings yourself instead of relying on this project. +For example the following configuration will configure the Autoscaler to use your federated identity: + +```yaml +azureUseWorkloadIdentityExtension: true +extraEnv: + AZURE_CLIENT_ID: USER ASSIGNED IDENTITY CLIENT ID + AZURE_TENANT_ID: USER ASSIGNED IDENTITY TENANT ID + AZURE_FEDERATED_TOKEN_FILE: /var/run/secrets/tokens/azure-identity-token + AZURE_AUTHORITY_HOST: https://login.microsoftonline.com/ +extraVolumes: +- name: azure-identity-token + projected: + defaultMode: 420 + sources: + - serviceAccountToken: + audience: api://AzureADTokenExchange + expirationSeconds: 3600 + path: azure-identity-token +extraVolumeMounts: +- mountPath: /var/run/secrets/tokens + name: azure-identity-token + readOnly: true +``` + ## Troubleshooting The chart will succeed even if the container arguments are incorrect. A few minutes after starting diff --git a/charts/cluster-autoscaler/templates/deployment.yaml b/charts/cluster-autoscaler/templates/deployment.yaml index 4854ba6eeaa1..a8d98d9cbdcb 100644 --- a/charts/cluster-autoscaler/templates/deployment.yaml +++ b/charts/cluster-autoscaler/templates/deployment.yaml @@ -159,7 +159,10 @@ spec: secretKeyRef: key: ClusterName name: {{ template "cluster-autoscaler.fullname" . }} - {{- if .Values.azureUseManagedIdentityExtension }} + {{- if .Values.azureUseWorkloadIdentityExtension }} + - name: ARM_USE_WORKLOAD_IDENTITY_EXTENSION + value: "true" + {{- else if .Values.azureUseManagedIdentityExtension }} - name: ARM_USE_MANAGED_IDENTITY_EXTENSION value: "true" {{- else }} diff --git a/charts/cluster-autoscaler/values.yaml b/charts/cluster-autoscaler/values.yaml index 36f3455dc44b..44d16c83691f 100644 --- a/charts/cluster-autoscaler/values.yaml +++ b/charts/cluster-autoscaler/values.yaml @@ -95,7 +95,10 @@ azureClusterName: "" # Required if `cloudProvider=azure` azureNodeResourceGroup: "" -# azureUseManagedIdentityExtension -- Whether to use Azure's managed identity extension for credentials. If using MSI, ensure subscription ID, resource group, and azure AKS cluster name are set. +# azureUseWorkloadIdentityExtension -- Whether to use Azure's workload identity extension for credentials. See the project here: https://github.com/Azure/azure-workload-identity for more details. You can only use one authentication method at a time, either azureUseWorkloadIdentityExtension or azureUseManagedIdentityExtension should be set. +azureUseWorkloadIdentityExtension: false + +# azureUseManagedIdentityExtension -- Whether to use Azure's managed identity extension for credentials. If using MSI, ensure subscription ID, resource group, and azure AKS cluster name are set. You can only use one authentication method at a time, either azureUseWorkloadIdentityExtension or azureUseManagedIdentityExtension should be set. azureUseManagedIdentityExtension: false # magnumClusterName -- Cluster name or ID in Magnum. diff --git a/cluster-autoscaler/cloudprovider/azure/azure_client.go b/cluster-autoscaler/cloudprovider/azure/azure_client.go index 8690e1114f4e..f344e79e155f 100644 --- a/cluster-autoscaler/cloudprovider/azure/azure_client.go +++ b/cluster-autoscaler/cloudprovider/azure/azure_client.go @@ -21,6 +21,7 @@ import ( "fmt" "io/ioutil" "net/http" + "os" "time" "github.com/Azure/azure-sdk-for-go/services/compute/mgmt/2019-07-01/compute" @@ -162,6 +163,18 @@ func newServicePrincipalTokenFromCredentials(config *Config, env *azure.Environm return nil, fmt.Errorf("creating the OAuth config: %v", err) } + if config.UseWorkloadIdentityExtension { + klog.V(2).Infoln("azure: using workload identity extension to retrieve access token") + jwt, err := os.ReadFile(config.AADFederatedTokenFile) + if err != nil { + return nil, fmt.Errorf("failed to read a file with a federated token: %v", err) + } + token, err := adal.NewServicePrincipalTokenFromFederatedToken(*oauthConfig, config.AADClientID, string(jwt), env.ResourceManagerEndpoint) + if err != nil { + return nil, fmt.Errorf("failed to create a workload identity token: %v", err) + } + return token, nil + } if config.UseManagedIdentityExtension { klog.V(2).Infoln("azure: using managed identity extension to retrieve access token") msiEndpoint, err := adal.GetMSIVMEndpoint() diff --git a/cluster-autoscaler/cloudprovider/azure/azure_config.go b/cluster-autoscaler/cloudprovider/azure/azure_config.go index a1ef62d6cd9c..72eea5d137d9 100644 --- a/cluster-autoscaler/cloudprovider/azure/azure_config.go +++ b/cluster-autoscaler/cloudprovider/azure/azure_config.go @@ -97,12 +97,14 @@ type Config struct { // Settings for a service principal. - AADClientID string `json:"aadClientId" yaml:"aadClientId"` - AADClientSecret string `json:"aadClientSecret" yaml:"aadClientSecret"` - AADClientCertPath string `json:"aadClientCertPath" yaml:"aadClientCertPath"` - AADClientCertPassword string `json:"aadClientCertPassword" yaml:"aadClientCertPassword"` - UseManagedIdentityExtension bool `json:"useManagedIdentityExtension" yaml:"useManagedIdentityExtension"` - UserAssignedIdentityID string `json:"userAssignedIdentityID" yaml:"userAssignedIdentityID"` + AADClientID string `json:"aadClientId" yaml:"aadClientId"` + AADClientSecret string `json:"aadClientSecret" yaml:"aadClientSecret"` + AADClientCertPath string `json:"aadClientCertPath" yaml:"aadClientCertPath"` + AADClientCertPassword string `json:"aadClientCertPassword" yaml:"aadClientCertPassword"` + AADFederatedTokenFile string `json:"aadFederatedTokenFile" yaml:"aadFederatedTokenFile"` + UseManagedIdentityExtension bool `json:"useManagedIdentityExtension" yaml:"useManagedIdentityExtension"` + UseWorkloadIdentityExtension bool `json:"useWorkloadIdentityExtension" yaml:"useWorkloadIdentityExtension"` + UserAssignedIdentityID string `json:"userAssignedIdentityID" yaml:"userAssignedIdentityID"` // Configs only for standard vmType (agent pools). Deployment string `json:"deployment" yaml:"deployment"` @@ -155,7 +157,14 @@ func BuildAzureConfig(configReader io.Reader) (*Config, error) { cfg.Location = os.Getenv("LOCATION") cfg.ResourceGroup = os.Getenv("ARM_RESOURCE_GROUP") cfg.TenantID = os.Getenv("ARM_TENANT_ID") + if tenantId := os.Getenv("AZURE_TENANT_ID"); tenantId != "" { + cfg.TenantID = tenantId + } cfg.AADClientID = os.Getenv("ARM_CLIENT_ID") + if clientId := os.Getenv("AZURE_CLIENT_ID"); clientId != "" { + cfg.AADClientID = clientId + } + cfg.AADFederatedTokenFile = os.Getenv("AZURE_FEDERATED_TOKEN_FILE") cfg.AADClientSecret = os.Getenv("ARM_CLIENT_SECRET") cfg.VMType = strings.ToLower(os.Getenv("ARM_VM_TYPE")) cfg.AADClientCertPath = os.Getenv("ARM_CLIENT_CERT_PATH") @@ -178,6 +187,18 @@ func BuildAzureConfig(configReader io.Reader) (*Config, error) { } } + useWorkloadIdentityExtensionFromEnv := os.Getenv("ARM_USE_WORKLOAD_IDENTITY_EXTENSION") + if len(useWorkloadIdentityExtensionFromEnv) > 0 { + cfg.UseWorkloadIdentityExtension, err = strconv.ParseBool(useWorkloadIdentityExtensionFromEnv) + if err != nil { + return nil, err + } + } + + if cfg.UseManagedIdentityExtension && cfg.UseWorkloadIdentityExtension { + return nil, errors.New("you can not combine both managed identity and workload identity as an authentication mechanism") + } + userAssignedIdentityIDFromEnv := os.Getenv("ARM_USER_ASSIGNED_IDENTITY_ID") if userAssignedIdentityIDFromEnv != "" { cfg.UserAssignedIdentityID = userAssignedIdentityIDFromEnv