Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

azurerm_kubernetes_cluster - vnet_integration_subnet_id #19517

Closed
wants to merge 1 commit into from
+175 −0
Closed

azurerm_kubernetes_cluster - vnet_integration_subnet_id #19517

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
144 changes: 144 additions & 0 deletions internal/services/containers/kubernetes_cluster_network_resource_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -734,6 +734,43 @@ func TestAccKubernetesCluster_publicNetworkAccess(t *testing.T) {
})
}

func TestAccKubernetesCluster_vnetIntegrationUpdate(t *testing.T) {
data := acceptance.BuildTestData(t, "azurerm_kubernetes_cluster", "test")
r := KubernetesClusterResource{}

data.ResourceTest(t, r, []acceptance.TestStep{
{
Config: r.vnetIntegrationDisabled(data),
Check: acceptance.ComposeTestCheckFunc(
check.That(data.ResourceName).ExistsInAzure(r),
),
},
data.ImportStep(),
{
Config: r.vnetIntegrationEnabled(data),
Check: acceptance.ComposeTestCheckFunc(
check.That(data.ResourceName).ExistsInAzure(r),
),
},
data.ImportStep(),
})
}

func TestAccKubernetesCluster_vnetIntegration(t *testing.T) {
data := acceptance.BuildTestData(t, "azurerm_kubernetes_cluster", "test")
r := KubernetesClusterResource{}

data.ResourceTest(t, r, []acceptance.TestStep{
{
Config: r.vnetIntegrationEnabled(data),
Check: acceptance.ComposeTestCheckFunc(
check.That(data.ResourceName).ExistsInAzure(r),
),
},
data.ImportStep(),
})
}

func (KubernetesClusterResource) advancedNetworkingConfig(data acceptance.TestData, networkPlugin string) string {
return fmt.Sprintf(`
provider "azurerm" {
Expand Down Expand Up @@ -3087,3 +3124,110 @@ resource "azurerm_kubernetes_cluster" "test" {
}
`, data.RandomInteger, data.Locations.Primary, data.RandomInteger, data.RandomInteger, enabled, authorizedIPConfig)
}

func (KubernetesClusterResource) vnetIntegrationDisabled(data acceptance.TestData) string {
return fmt.Sprintf(`
provider "azurerm" {
features {}
}
resource "azurerm_resource_group" "test" {
name = "acctestRG-aks-%[2]d"
location = "%[1]s"
}
resource "azurerm_kubernetes_cluster" "test" {
name = "acctestaks%[2]d"
location = azurerm_resource_group.test.location
resource_group_name = azurerm_resource_group.test.name
dns_prefix = "acctestaks%[2]d"
default_node_pool {
name = "default"
node_count = 1
vm_size = "Standard_D2s_v3"
}
identity {
type = "SystemAssigned"
}
}
`, data.Locations.Primary, data.RandomInteger)
}

func (KubernetesClusterResource) vnetIntegrationEnabled(data acceptance.TestData) string {
return fmt.Sprintf(`
provider "azurerm" {
features {}
}
resource "azurerm_resource_group" "test" {
name = "acctestRG-aks-%[2]d"
location = "%[1]s"
}

resource "azurerm_virtual_network" "test" {
name = "acctest-vnet-%[2]d"
address_space = ["172.16.0.0/20"]
location = azurerm_resource_group.test.location
resource_group_name = azurerm_resource_group.test.name
}

resource "azurerm_subnet" "test" {
name = "subnet1"
resource_group_name = azurerm_resource_group.test.name
virtual_network_name = azurerm_virtual_network.test.name
address_prefixes = ["172.16.0.0/22"]
}

resource "azurerm_subnet" "test1" {
name = "subnet2"
resource_group_name = azurerm_resource_group.test.name
virtual_network_name = azurerm_virtual_network.test.name
address_prefixes = ["172.16.4.0/24"]

delegation {
name = "aks-delegation"
service_delegation {
name = "Microsoft.ContainerService/managedClusters"
actions = ["Microsoft.Network/virtualNetworks/subnets/join/action"]
}
}
}

resource "azurerm_user_assigned_identity" "test" {
name = "acctest-identity-%[2]d"
resource_group_name = azurerm_resource_group.test.name
location = azurerm_resource_group.test.location
}

resource "azurerm_role_assignment" "test" {
scope = azurerm_subnet.test1.id
role_definition_name = "Network Contributor"
principal_id = azurerm_user_assigned_identity.test.principal_id
}

resource "azurerm_kubernetes_cluster" "test" {
name = "acctestaks%[2]d"
location = azurerm_resource_group.test.location
resource_group_name = azurerm_resource_group.test.name
dns_prefix = "acctestaks%[2]d"
default_node_pool {
name = "default"
node_count = 1
vm_size = "Standard_D2s_v3"
}
identity {
type = "SystemAssigned"
}
vnet_integration_subnet_id = azurerm_subnet.test1.id

network_profile {
network_plugin = "kubenet"
pod_cidr = "10.244.0.0/16"
dns_service_ip = "10.10.0.10"
docker_bridge_cidr = "172.18.0.1/16"
service_cidr = "10.10.0.0/16"
}

depends_on = [
azurerm_role_assignment.test
]
}
`, data.Locations.Primary, data.RandomInteger)
}
27 changes: 27 additions & 0 deletions internal/services/containers/kubernetes_cluster_resource.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -30,6 +30,7 @@ import (
"github.com/hashicorp/terraform-provider-azurerm/internal/services/containers/kubernetes"
"github.com/hashicorp/terraform-provider-azurerm/internal/services/containers/migration"
containerValidate "github.com/hashicorp/terraform-provider-azurerm/internal/services/containers/validate"
networkValidate "github.com/hashicorp/terraform-provider-azurerm/internal/services/network/validate"
"github.com/hashicorp/terraform-provider-azurerm/internal/tf/pluginsdk"
"github.com/hashicorp/terraform-provider-azurerm/internal/tf/suppress"
"github.com/hashicorp/terraform-provider-azurerm/internal/tf/validation"
Expand Down Expand Up @@ -63,6 +64,9 @@ func resourceKubernetesCluster() *pluginsdk.Resource {
pluginsdk.ForceNewIfChange("windows_profile.0.gmsa.0.root_domain", func(ctx context.Context, old, new, meta interface{}) bool {
return old != "" && new == ""
}),
pluginsdk.ForceNewIfChange("vnet_integration_subnet_id", func(ctx context.Context, old, new, meta interface{}) bool {
return old != "" && new == ""
}),
),

Timeouts: &pluginsdk.ResourceTimeout{
Expand Down Expand Up @@ -1006,6 +1010,12 @@ func resourceKubernetesCluster() *pluginsdk.Resource {

"tags": commonschema.Tags(),

"vnet_integration_subnet_id": {
Type: pluginsdk.TypeString,
Optional: true,
ValidateFunc: networkValidate.SubnetID,
},

"windows_profile": {
Type: pluginsdk.TypeList,
Optional: true,
Expand Down Expand Up @@ -1181,6 +1191,11 @@ func resourceKubernetesClusterCreate(d *pluginsdk.ResourceData, meta interface{}
DisableRunCommand: utils.Bool(!d.Get("run_command_enabled").(bool)),
}

if subnetId := d.Get("vnet_integration_subnet_id").(string); subnetId != "" {
apiAccessProfile.EnableVnetIntegration = utils.Bool(true)
apiAccessProfile.SubnetId = utils.String(subnetId)
}

nodeResourceGroup := d.Get("node_resource_group").(string)

if d.Get("enable_pod_security_policy").(bool) {
Expand Down Expand Up @@ -1492,6 +1507,15 @@ func resourceKubernetesClusterUpdate(d *pluginsdk.ResourceData, meta interface{}
existing.Model.Properties.ApiServerAccessProfile.EnablePrivateClusterPublicFQDN = utils.Bool(d.Get("private_cluster_public_fqdn_enabled").(bool))
}

if d.HasChange("vnet_integration_subnet_id") {
updateCluster = true
if existing.Model.Properties.ApiServerAccessProfile == nil {
existing.Model.Properties.ApiServerAccessProfile = &managedclusters.ManagedClusterAPIServerAccessProfile{}
}
existing.Model.Properties.ApiServerAccessProfile.EnableVnetIntegration = utils.Bool(true)
existing.Model.Properties.ApiServerAccessProfile.SubnetId = utils.String(d.Get("vnet_integration_subnet_id").(string))
}

if d.HasChange("run_command_enabled") {
updateCluster = true
if existing.Model.Properties.ApiServerAccessProfile == nil {
Expand Down Expand Up @@ -1937,6 +1961,9 @@ func resourceKubernetesClusterRead(d *pluginsdk.ResourceData, meta interface{})
if accessProfile.DisableRunCommand != nil {
runCommandEnabled = !*accessProfile.DisableRunCommand
}
if accessProfile.SubnetId != nil && len(*accessProfile.SubnetId) != 0 {
d.Set("vnet_integration_subnet_id", *accessProfile.SubnetId)
}
switch {
case accessProfile.PrivateDNSZone != nil && strings.EqualFold("System", *accessProfile.PrivateDNSZone):
d.Set("private_dns_zone_id", "System")
Expand Down
4 changes: 4 additions & 0 deletions website/docs/r/kubernetes_cluster.html.markdown
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -161,6 +161,10 @@ In addition, one of either `identity` or `service_principal` blocks must be spec

-> **Note:** If you use BYO DNS Zone, AKS cluster should either use a User Assigned Identity or a service principal (which is deprecated) with the `Private DNS Zone Contributor` role and access to this Private DNS Zone. If `UserAssigned` identity is used - to prevent improper resource order destruction - cluster should depend on the role assignment, like in this example:

* `vnet_integration_subnet_id` - (Optional) The ID of the subnet used in API server vnet integration. Removing this forces a new resource to be created.

-> **Note:** This requires that the Preview Feature `Microsoft.ContainerService/EnableAPIServerVnetIntegrationPreview` is enabled and the Resource Provider is re-registered, see [the documentation](https://learn.microsoft.com/en-us/azure/aks/api-server-vnet-integration) for more information.

* `workload_autoscaler_profile` - (Optional) A `workload_autoscaler_profile` block defined below.

* `workload_identity_enabled` - (Optional) Specifies whether Azure AD Workload Identity should be enabled for the Cluster. Defaults to `false`.
Expand Down