diff --git a/internal/services/containers/kubernetes_cluster_data_source.go b/internal/services/containers/kubernetes_cluster_data_source.go index 88487f654159..742fb432f92c 100644 --- a/internal/services/containers/kubernetes_cluster_data_source.go +++ b/internal/services/containers/kubernetes_cluster_data_source.go @@ -380,6 +380,23 @@ func dataSourceKubernetesCluster() *pluginsdk.Resource { "identity": commonschema.SystemOrUserAssignedIdentityComputed(), + "key_management_service": { + Type: pluginsdk.TypeList, + Computed: true, + Elem: &pluginsdk.Resource{ + Schema: map[string]*pluginsdk.Schema{ + "key_vault_key_id": { + Type: pluginsdk.TypeString, + Computed: true, + }, + "key_vault_network_access": { + Type: pluginsdk.TypeString, + Computed: true, + }, + }, + }, + }, + "kubernetes_version": { Type: pluginsdk.TypeString, Computed: true, @@ -718,6 +735,11 @@ func dataSourceKubernetesClusterRead(d *pluginsdk.ResourceData, meta interface{} return fmt.Errorf("setting `agent_pool_profile`: %+v", err) } + azureKeyVaultKms := flattenKubernetesClusterDataSourceKeyVaultKms(props.SecurityProfile.AzureKeyVaultKms) + if err := d.Set("key_management_service", azureKeyVaultKms); err != nil { + return fmt.Errorf("setting `key_management_service`: %+v", err) + } + kubeletIdentity, err := flattenKubernetesClusterDataSourceIdentityProfile(props.IdentityProfile) if err != nil { return err @@ -827,6 +849,29 @@ func dataSourceKubernetesClusterRead(d *pluginsdk.ResourceData, meta interface{} return nil } +func flattenKubernetesClusterDataSourceKeyVaultKms(input *managedclusters.AzureKeyVaultKms) []interface{} { + azureKeyVaultKms := make([]interface{}, 0) + + if input != nil && input.Enabled != nil && *input.Enabled { + keyId := "" + if v := input.KeyId; v != nil { + keyId = *v + } + + networkAccess := "" + if v := input.KeyVaultNetworkAccess; v != nil { + networkAccess = string(*v) + } + + azureKeyVaultKms = append(azureKeyVaultKms, map[string]interface{}{ + "key_vault_key_id": keyId, + "key_vault_network_access": networkAccess, + }) + } + + return azureKeyVaultKms +} + func flattenKubernetesClusterDataSourceStorageProfile(input *managedclusters.ManagedClusterStorageProfile) []interface{} { storageProfile := make([]interface{}, 0) diff --git a/internal/services/containers/kubernetes_cluster_resource.go b/internal/services/containers/kubernetes_cluster_resource.go index e03e421f0a08..4da532109cbc 100644 --- a/internal/services/containers/kubernetes_cluster_resource.go +++ b/internal/services/containers/kubernetes_cluster_resource.go @@ -31,7 +31,11 @@ import ( "github.com/hashicorp/terraform-provider-azurerm/internal/services/containers/kubernetes" "github.com/hashicorp/terraform-provider-azurerm/internal/services/containers/migration" containerValidate "github.com/hashicorp/terraform-provider-azurerm/internal/services/containers/validate" + keyVaultClient "github.com/hashicorp/terraform-provider-azurerm/internal/services/keyvault/client" + keyVaultParse "github.com/hashicorp/terraform-provider-azurerm/internal/services/keyvault/parse" + keyVaultValidate "github.com/hashicorp/terraform-provider-azurerm/internal/services/keyvault/validate" networkValidate "github.com/hashicorp/terraform-provider-azurerm/internal/services/network/validate" + resourcesClient "github.com/hashicorp/terraform-provider-azurerm/internal/services/resource/client" "github.com/hashicorp/terraform-provider-azurerm/internal/tf/pluginsdk" "github.com/hashicorp/terraform-provider-azurerm/internal/tf/suppress" "github.com/hashicorp/terraform-provider-azurerm/internal/tf/validation" @@ -709,6 +713,28 @@ func resourceKubernetesCluster() *pluginsdk.Resource { }, }, + "key_management_service": { + Type: pluginsdk.TypeList, + Optional: true, + ForceNew: false, + MaxItems: 1, + Elem: &pluginsdk.Resource{ + Schema: map[string]*pluginsdk.Schema{ + "key_vault_key_id": { + Type: pluginsdk.TypeString, + Required: true, + ValidateFunc: keyVaultValidate.NestedItemId, + }, + "key_vault_network_access": { + Type: pluginsdk.TypeString, + Default: string(managedclusters.KeyVaultNetworkAccessTypesPublic), + Optional: true, + ValidateFunc: validation.StringInSlice(managedclusters.PossibleValuesForKeyVaultNetworkAccessTypes(), false), + }, + }, + }, + }, + "microsoft_defender": { Type: pluginsdk.TypeList, Optional: true, @@ -1225,6 +1251,8 @@ func resourceKubernetesClusterCreate(d *pluginsdk.ResourceData, meta interface{} subscriptionId := meta.(*clients.Client).Account.SubscriptionId tenantId := meta.(*clients.Client).Account.TenantId client := meta.(*clients.Client).Containers.KubernetesClustersClient + keyVaultsClient := meta.(*clients.Client).KeyVault + resourcesClient := meta.(*clients.Client).Resource env := meta.(*clients.Client).Containers.Environment ctx, cancel := timeouts.ForCreate(meta.(*clients.Client).StopContext, d) defer cancel() @@ -1332,12 +1360,15 @@ func resourceKubernetesClusterCreate(d *pluginsdk.ResourceData, meta interface{} publicNetworkAccess = managedclusters.PublicNetworkAccessDisabled } - microsoftDefenderRaw := d.Get("microsoft_defender").([]interface{}) - securityProfile := expandKubernetesClusterMicrosoftDefender(d, microsoftDefenderRaw) - storageProfileRaw := d.Get("storage_profile").([]interface{}) storageProfile := expandStorageProfile(storageProfileRaw) + // assemble securityProfile (Defender, WorkloadIdentity, ImageCleaner, AzureKeyVaultKms) + securityProfile := &managedclusters.ManagedClusterSecurityProfile{} + + microsoftDefenderRaw := d.Get("microsoft_defender").([]interface{}) + securityProfile.Defender = expandKubernetesClusterMicrosoftDefender(d, microsoftDefenderRaw) + workloadIdentity := false if v, ok := d.GetOk("workload_identity_enabled"); ok { workloadIdentity = v.(bool) @@ -1346,23 +1377,22 @@ func resourceKubernetesClusterCreate(d *pluginsdk.ResourceData, meta interface{} return fmt.Errorf("`oidc_issuer_enabled` must be set to `true` to enable Azure AD Workload Identity") } - if securityProfile == nil { - securityProfile = &managedclusters.ManagedClusterSecurityProfile{} - } - securityProfile.WorkloadIdentity = &managedclusters.ManagedClusterSecurityProfileWorkloadIdentity{ Enabled: &workloadIdentity, } } - if securityProfile == nil { - securityProfile = &managedclusters.ManagedClusterSecurityProfile{} - } securityProfile.ImageCleaner = &managedclusters.ManagedClusterSecurityProfileImageCleaner{ Enabled: utils.Bool(d.Get("image_cleaner_enabled").(bool)), IntervalHours: utils.Int64(int64(d.Get("image_cleaner_interval_hours").(int))), } + azureKeyVaultKmsRaw := d.Get("key_management_service").([]interface{}) + securityProfile.AzureKeyVaultKms, err = expandKubernetesClusterAzureKeyVaultKms(ctx, keyVaultsClient, resourcesClient, d, azureKeyVaultKmsRaw) + if err != nil { + return err + } + parameters := managedclusters.ManagedCluster{ Name: utils.String(id.ResourceName), ExtendedLocation: expandEdgeZone(d.Get("edge_zone").(string)), @@ -1488,6 +1518,8 @@ func resourceKubernetesClusterUpdate(d *pluginsdk.ResourceData, meta interface{} containersClient := meta.(*clients.Client).Containers nodePoolsClient := containersClient.AgentPoolsClient clusterClient := containersClient.KubernetesClustersClient + keyVaultsClient := meta.(*clients.Client).KeyVault + resourcesClient := meta.(*clients.Client).Resource env := containersClient.Environment ctx, cancel := timeouts.ForUpdate(meta.(*clients.Client).StopContext, d) defer cancel() @@ -1834,11 +1866,18 @@ func resourceKubernetesClusterUpdate(d *pluginsdk.ResourceData, meta interface{} existing.Model.Properties.OidcIssuerProfile = oidcIssuerProfile } + if d.HasChanges("key_management_service") { + updateCluster = true + azureKeyVaultKmsRaw := d.Get("key_management_service").([]interface{}) + azureKeyVaultKms, _ := expandKubernetesClusterAzureKeyVaultKms(ctx, keyVaultsClient, resourcesClient, d, azureKeyVaultKmsRaw) + existing.Model.Properties.SecurityProfile.AzureKeyVaultKms = azureKeyVaultKms + } + if d.HasChanges("microsoft_defender") { updateCluster = true microsoftDefenderRaw := d.Get("microsoft_defender").([]interface{}) microsoftDefender := expandKubernetesClusterMicrosoftDefender(d, microsoftDefenderRaw) - existing.Model.Properties.SecurityProfile = microsoftDefender + existing.Model.Properties.SecurityProfile.Defender = microsoftDefender } if d.HasChanges("storage_profile") { @@ -2207,7 +2246,7 @@ func resourceKubernetesClusterRead(d *pluginsdk.ResourceData, meta interface{}) d.Set("oidc_issuer_enabled", oidcIssuerEnabled) d.Set("oidc_issuer_url", oidcIssuerUrl) - microsoftDefender := flattenKubernetesClusterMicrosoftDefender(props.SecurityProfile) + microsoftDefender := flattenKubernetesClusterMicrosoftDefender(props.SecurityProfile.Defender) if err := d.Set("microsoft_defender", microsoftDefender); err != nil { return fmt.Errorf("setting `microsoft_defender`: %+v", err) } @@ -2223,6 +2262,11 @@ func resourceKubernetesClusterRead(d *pluginsdk.ResourceData, meta interface{}) } d.Set("workload_identity_enabled", workloadIdentity) + azureKeyVaultKms := flattenKubernetesClusterDataSourceKeyVaultKms(props.SecurityProfile.AzureKeyVaultKms) + if err := d.Set("key_management_service", azureKeyVaultKms); err != nil { + return fmt.Errorf("setting `key_management_service`: %+v", err) + } + // adminProfile is only available for RBAC enabled clusters with AAD and local account is not disabled if props.AadProfile != nil && (props.DisableLocalAccounts == nil || !*props.DisableLocalAccounts) { accessProfileId := managedclusters.NewAccessProfileID(id.SubscriptionId, id.ResourceGroupName, id.ResourceName, "clusterAdmin") @@ -3376,6 +3420,41 @@ func expandKubernetesClusterAutoScalerProfile(input []interface{}) *managedclust } } +func expandKubernetesClusterAzureKeyVaultKms(ctx context.Context, keyVaultsClient *keyVaultClient.Client, resourcesClient *resourcesClient.Client, d *pluginsdk.ResourceData, input []interface{}) (*managedclusters.AzureKeyVaultKms, error) { + if ((input == nil) || len(input) == 0) && d.HasChanges("key_management_service") { + return &managedclusters.AzureKeyVaultKms{ + Enabled: utils.Bool(false), + }, nil + } else if (input == nil) || len(input) == 0 { + return nil, nil + } + + raw := input[0].(map[string]interface{}) + kvAccess := managedclusters.KeyVaultNetworkAccessTypes(raw["key_vault_network_access"].(string)) + + azureKeyVaultKms := &managedclusters.AzureKeyVaultKms{ + Enabled: utils.Bool(true), + KeyId: utils.String(raw["key_vault_key_id"].(string)), + KeyVaultNetworkAccess: &kvAccess, + } + + // Set Key vault Resource ID in case public access is disabled + if kvAccess == managedclusters.KeyVaultNetworkAccessTypesPrivate { + keyVaultKeyId, err := keyVaultParse.ParseNestedItemID(*azureKeyVaultKms.KeyId) + if err != nil { + return nil, err + } + keyVaultID, err := keyVaultsClient.KeyVaultIDFromBaseUrl(ctx, resourcesClient, keyVaultKeyId.KeyVaultBaseUrl) + if err != nil { + return nil, fmt.Errorf("retrieving the Resource ID the Key Vault at URL %q: %s", keyVaultKeyId.KeyVaultBaseUrl, err) + } + + azureKeyVaultKms.KeyVaultResourceId = keyVaultID + } + + return azureKeyVaultKms, nil +} + func expandKubernetesClusterMaintenanceConfiguration(input []interface{}) *maintenanceconfigurations.MaintenanceConfigurationProperties { if len(input) == 0 { return nil @@ -3530,13 +3609,11 @@ func flattenKubernetesClusterHttpProxyConfig(props *managedclusters.ManagedClust }) } -func expandKubernetesClusterMicrosoftDefender(d *pluginsdk.ResourceData, input []interface{}) *managedclusters.ManagedClusterSecurityProfile { +func expandKubernetesClusterMicrosoftDefender(d *pluginsdk.ResourceData, input []interface{}) *managedclusters.ManagedClusterSecurityProfileDefender { if (len(input) == 0 || input[0] == nil) && d.HasChange("microsoft_defender") { - return &managedclusters.ManagedClusterSecurityProfile{ - Defender: &managedclusters.ManagedClusterSecurityProfileDefender{ - SecurityMonitoring: &managedclusters.ManagedClusterSecurityProfileDefenderSecurityMonitoring{ - Enabled: utils.Bool(false), - }, + return &managedclusters.ManagedClusterSecurityProfileDefender{ + SecurityMonitoring: &managedclusters.ManagedClusterSecurityProfileDefenderSecurityMonitoring{ + Enabled: utils.Bool(false), }, } } else if len(input) == 0 || input[0] == nil { @@ -3544,23 +3621,21 @@ func expandKubernetesClusterMicrosoftDefender(d *pluginsdk.ResourceData, input [ } config := input[0].(map[string]interface{}) - return &managedclusters.ManagedClusterSecurityProfile{ - Defender: &managedclusters.ManagedClusterSecurityProfileDefender{ - SecurityMonitoring: &managedclusters.ManagedClusterSecurityProfileDefenderSecurityMonitoring{ - Enabled: utils.Bool(true), - }, - LogAnalyticsWorkspaceResourceId: utils.String(config["log_analytics_workspace_id"].(string)), + return &managedclusters.ManagedClusterSecurityProfileDefender{ + SecurityMonitoring: &managedclusters.ManagedClusterSecurityProfileDefenderSecurityMonitoring{ + Enabled: utils.Bool(true), }, + LogAnalyticsWorkspaceResourceId: utils.String(config["log_analytics_workspace_id"].(string)), } } -func flattenKubernetesClusterMicrosoftDefender(input *managedclusters.ManagedClusterSecurityProfile) []interface{} { - if input == nil || input.Defender == nil || (input.Defender.SecurityMonitoring != nil && input.Defender.SecurityMonitoring.Enabled != nil && !*input.Defender.SecurityMonitoring.Enabled) { +func flattenKubernetesClusterMicrosoftDefender(input *managedclusters.ManagedClusterSecurityProfileDefender) []interface{} { + if input == nil || (input.SecurityMonitoring != nil && input.SecurityMonitoring.Enabled != nil && !*input.SecurityMonitoring.Enabled) { return []interface{}{} } logAnalyticsWorkspace := "" - if v := input.Defender.LogAnalyticsWorkspaceResourceId; v != nil { + if v := input.LogAnalyticsWorkspaceResourceId; v != nil { logAnalyticsWorkspace = *v } diff --git a/internal/services/containers/kubernetes_cluster_resource_test.go b/internal/services/containers/kubernetes_cluster_resource_test.go index 527894cb4d76..ce48eb8bdac2 100644 --- a/internal/services/containers/kubernetes_cluster_resource_test.go +++ b/internal/services/containers/kubernetes_cluster_resource_test.go @@ -75,6 +75,26 @@ func TestAccKubernetesCluster_runCommand(t *testing.T) { }) } +func TestAccKubernetesCluster_keyVaultKms(t *testing.T) { + data := acceptance.BuildTestData(t, "azurerm_kubernetes_cluster", "test") + r := KubernetesClusterResource{} + + data.ResourceTest(t, r, []acceptance.TestStep{ + { + Config: r.azureKeyVaultKms(data, currentKubernetesVersion, true), + Check: acceptance.ComposeTestCheckFunc( + check.That(data.ResourceName).ExistsInAzure(r), + ), + }, + { + Config: r.azureKeyVaultKms(data, currentKubernetesVersion, false), + Check: acceptance.ComposeTestCheckFunc( + check.That(data.ResourceName).ExistsInAzure(r), + ), + }, + }) +} + func TestAccKubernetesCluster_storageProfile(t *testing.T) { data := acceptance.BuildTestData(t, "azurerm_kubernetes_cluster", "test") r := KubernetesClusterResource{} @@ -513,6 +533,87 @@ resource "azurerm_kubernetes_cluster" "test" { `, data.RandomInteger, data.Locations.Primary, data.RandomInteger, data.RandomInteger, controlPlaneVersion, tag) } +func (KubernetesClusterResource) azureKeyVaultKms(data acceptance.TestData, controlPlaneVersion string, enabled bool) string { + kmsBlock := "" + if enabled { + kmsBlock = ` + key_management_service { + key_vault_key_id = azurerm_key_vault_key.test.id + }` + } + + return fmt.Sprintf(` +provider "azurerm" { + features {} +} + +data "azurerm_client_config" "current" {} + +resource "azurerm_resource_group" "test" { + name = "acctestRG-aks-%[1]d" + location = "%[2]s" +} + +resource "azurerm_key_vault" "test" { + name = substr("acctest%[1]d", 0, 24) + location = azurerm_resource_group.test.location + resource_group_name = azurerm_resource_group.test.name + tenant_id = data.azurerm_client_config.current.tenant_id + enable_rbac_authorization = true + sku_name = "standard" +} + +resource "azurerm_role_assignment" "test_admin" { + scope = azurerm_key_vault.test.id + role_definition_name = "Key Vault Administrator" + principal_id = data.azurerm_client_config.current.object_id +} + +resource "azurerm_role_assignment" "test" { + scope = azurerm_key_vault.test.id + role_definition_name = "Key Vault Crypto User" + principal_id = azurerm_user_assigned_identity.test.principal_id +} + +resource "azurerm_key_vault_key" "test" { + name = "etcd-encryption" + key_vault_id = azurerm_key_vault.test.id + key_type = "RSA" + key_size = 2048 + key_opts = ["decrypt", "encrypt", "sign", "unwrapKey", "verify", "wrapKey"] + + depends_on = [azurerm_role_assignment.test_admin] +} + +resource "azurerm_user_assigned_identity" "test" { + name = "acctest%[1]d" + resource_group_name = azurerm_resource_group.test.name + location = azurerm_resource_group.test.location +} + +resource "azurerm_kubernetes_cluster" "test" { + name = "acctestaks%[1]d" + location = azurerm_resource_group.test.location + resource_group_name = azurerm_resource_group.test.name + node_resource_group = "${azurerm_resource_group.test.name}-infra" + dns_prefix = "acctestaks%[1]d" + kubernetes_version = %[3]q + + default_node_pool { + name = "default" + node_count = 1 + vm_size = "Standard_DS2_v2" + } + + identity { + type = "UserAssigned" + identity_ids = [azurerm_user_assigned_identity.test.id] + } + %[4]s +} +`, data.RandomInteger, data.Locations.Primary, controlPlaneVersion, kmsBlock) +} + func (KubernetesClusterResource) storageProfile(data acceptance.TestData, controlPlaneVersion string) string { return fmt.Sprintf(` provider "azurerm" { diff --git a/website/docs/d/kubernetes_cluster.html.markdown b/website/docs/d/kubernetes_cluster.html.markdown index 5f56663cb3ab..97031973b07e 100644 --- a/website/docs/d/kubernetes_cluster.html.markdown +++ b/website/docs/d/kubernetes_cluster.html.markdown @@ -56,6 +56,8 @@ The following attributes are exported: * `ingress_application_gateway` - An `ingress_application_gateway` block as documented below. +* `key_management_service` - A `key_management_service` block as documented below. + * `key_vault_secrets_provider` - A `key_vault_secrets_provider` block as documented below. * `private_fqdn` - The FQDN of this Kubernetes Cluster when private link has been enabled. This name is only resolvable inside the Virtual Network where the Azure Kubernetes Service is located @@ -178,6 +180,14 @@ A `upgrade_settings` block exports the following: --- +A `key_management_service` block supports the following: + +* `key_vault_key_id` - Identifier of Azure Key Vault key. See [key identifier format](https://learn.microsoft.com/en-us/azure/key-vault/general/about-keys-secrets-certificates#vault-name-and-object-name) for more details. + +* `key_vault_network_access` - Network access of the key vault. The possible values are `Public` and `Private`. `Public` means the key vault allows public access from all networks. `Private` means the key vault disables public access and enables private link. + +--- + A `key_vault_secrets_provider` block exports the following: * `secret_rotation_enabled` - Is secret rotation enabled? diff --git a/website/docs/r/kubernetes_cluster.html.markdown b/website/docs/r/kubernetes_cluster.html.markdown index 30e552c88929..9b4f3ac820c4 100644 --- a/website/docs/r/kubernetes_cluster.html.markdown +++ b/website/docs/r/kubernetes_cluster.html.markdown @@ -127,6 +127,8 @@ In addition, one of either `identity` or `service_principal` blocks must be spec * `ingress_application_gateway` - (Optional) A `ingress_application_gateway` block as defined below. +* `key_management_service` - (Optional) A `key_management_service` block as defined below. For more details, please visit [Key Management Service (KMS) etcd encryption to an AKS cluster](https://learn.microsoft.com/en-us/azure/aks/use-kms-etcd-encryption). + * `key_vault_secrets_provider` - (Optional) A `key_vault_secrets_provider` block as defined below. For more details, please visit [Azure Keyvault Secrets Provider for AKS](https://docs.microsoft.com/azure/aks/csi-secrets-store-driver). * `kubelet_identity` - (Optional) A `kubelet_identity` block as defined below. @@ -460,6 +462,15 @@ An `identity` block supports the following: --- +A `key_management_service` block supports the following: + +* `key_vault_key_id` - (Optional) Identifier of Azure Key Vault key. See [key identifier format](https://learn.microsoft.com/en-us/azure/key-vault/general/about-keys-secrets-certificates#vault-name-and-object-name) for more details. When Azure Key Vault key management service is enabled, this field is required and must be a valid key identifier. When `enabled` is `false`, leave the field empty. + +* `key_vault_network_access` - (Optional) Network access of the key vault +Network access of key vault. The possible values are `Public` and `Private`. `Public` means the key vault allows public access from all networks. `Private` means the key vault disables public access and enables private link. The default value is `Public`. + +--- + A `key_vault_secrets_provider` block supports the following: * `secret_rotation_enabled` - (Optional) Is secret rotation enabled?