From 467558d88fce869ae84671305d4fdd8c7e0ee545 Mon Sep 17 00:00:00 2001 From: Chad Kittel Date: Thu, 19 May 2022 20:08:57 +0000 Subject: [PATCH 01/16] Apply ACR policies around auth --- acr-stamp.bicep | 69 +++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 69 insertions(+) diff --git a/acr-stamp.bicep b/acr-stamp.bicep index 64a041f8..98a9451e 100644 --- a/acr-stamp.bicep +++ b/acr-stamp.bicep @@ -72,6 +72,16 @@ resource spokeVirtualNetwork 'Microsoft.Network/virtualNetworks@2021-05-01' exis } } +// Built-in policy: Container registries should have anonymous authentication disabled. +var pdAnonymousContainerRegistryAccessDisallowedId = tenantResourceId('Microsoft.Authorization/policyDefinitions', '9f2dea28-e834-476c-99c5-3507b4728395') + +// Built-in policy: Container registries should have repository scoped access token disabled. +var pdAccessTokenContainerRegistryAccessDisallowedId = tenantResourceId('Microsoft.Authorization/policyDefinitions', 'ff05e24e-195c-447e-b322-5e90c9f9f366') + +// Built-in policy: Container registries should have local admin account disabled. +var pdAdminAccountContainerRegistryAccessDisallowedId = tenantResourceId('Microsoft.Authorization/policyDefinitions', 'dc921057-6b28-4fbe-9b83-f7bec05db6c2') + + /*** RESOURCES ***/ // This Log Analytics workspace will be the log sink for all resources in the cluster resource group. This includes ACR, the AKS cluster, Key Vault, etc. It also is the Container Insights log sink for the AKS cluster. @@ -86,6 +96,60 @@ resource laAks 'Microsoft.OperationalInsights/workspaces@2021-06-01' = { } } +// Ensure the container registry does not allow anon access (Azure RBAC only is allowed) +resource paAnonymousContainerRegistryAccessDisallowed 'Microsoft.Authorization/policyAssignments@2021-06-01' = { + name: guid(resourceGroup().id, pdAnonymousContainerRegistryAccessDisallowedId) + location: 'global' + scope: resourceGroup() + properties: { + displayName: take('[acraks${subRgUniqueString}] ${reference(pdAnonymousContainerRegistryAccessDisallowedId, '2021-06-01').displayName}', 120) + description: reference(pdAnonymousContainerRegistryAccessDisallowedId, '2021-06-01').description + enforcementMode: 'Default' + policyDefinitionId: pdAnonymousContainerRegistryAccessDisallowedId + parameters: { + effect: { + value: 'Deny' + } + } + } +} + +// Ensure the container registry does not allow sas token access (Azure RBAC only is allowed) +resource paAccessTokenContainerRegistryAccessDisallowed 'Microsoft.Authorization/policyAssignments@2021-06-01' = { + name: guid(resourceGroup().id, pdAccessTokenContainerRegistryAccessDisallowedId) + location: 'global' + scope: resourceGroup() + properties: { + displayName: take('[acraks${subRgUniqueString}] ${reference(pdAccessTokenContainerRegistryAccessDisallowedId, '2021-06-01').displayName}', 120) + description: reference(pdAccessTokenContainerRegistryAccessDisallowedId, '2021-06-01').description + enforcementMode: 'Default' + policyDefinitionId: pdAccessTokenContainerRegistryAccessDisallowedId + parameters: { + effect: { + value: 'Deny' + } + } + } +} + +// Ensure the container registry does not allow admin account access (Azure RBAC only is allowed) +resource paAdminAccountContainerRegistryAccessDisallowed 'Microsoft.Authorization/policyAssignments@2021-06-01' = { + name: guid(resourceGroup().id, pdAdminAccountContainerRegistryAccessDisallowedId) + location: 'global' + scope: resourceGroup() + properties: { + displayName: take('[acraks${subRgUniqueString}] ${reference(pdAdminAccountContainerRegistryAccessDisallowedId, '2021-06-01').displayName}', 120) + description: reference(pdAdminAccountContainerRegistryAccessDisallowedId, '2021-06-01').description + enforcementMode: 'Default' + policyDefinitionId: pdAdminAccountContainerRegistryAccessDisallowedId + parameters: { + effect: { + value: 'Deny' + } + } + } +} + // Azure Container Registry will be exposed via Private Link, set up the related Private DNS zone and virtual network link to the spoke. resource dnsPrivateZoneAcr 'Microsoft.Network/privateDnsZones@2020-06-01' = { name: 'privatelink.azurecr.io' @@ -108,6 +172,11 @@ resource dnsPrivateZoneAcr 'Microsoft.Network/privateDnsZones@2020-06-01' = { resource acrAks 'Microsoft.ContainerRegistry/registries@2021-09-01' = { name: 'acraks${subRgUniqueString}' location: location + dependsOn: [ + paAccessTokenContainerRegistryAccessDisallowed // These policy assignments are not true dependencies, but we want them in place before we deploy our ACR instance. + paAdminAccountContainerRegistryAccessDisallowed + paAnonymousContainerRegistryAccessDisallowed + ] sku: { name: 'Premium' } From 7072da85c0c18756b19f7a7c79d918acc43f7323 Mon Sep 17 00:00:00 2001 From: Chad Kittel Date: Thu, 19 May 2022 21:52:50 +0000 Subject: [PATCH 02/16] Move some declarations around before doing policy refresh. --- cluster-stamp.bicep | 118 +++++++++++++++++++++++++------------------- 1 file changed, 66 insertions(+), 52 deletions(-) diff --git a/cluster-stamp.bicep b/cluster-stamp.bicep index 5d39aa61..c6bb72bd 100644 --- a/cluster-stamp.bicep +++ b/cluster-stamp.bicep @@ -59,34 +59,20 @@ param gitOpsBootstrappingRepoBranch string = 'main' /*** VARIABLES ***/ var subRgUniqueString = uniqueString('aks', subscription().subscriptionId, resourceGroup().id) - var clusterName = 'aks-${subRgUniqueString}' -var nodeResourceGroupName = 'rg-${clusterName}-nodepools' -var defaultAcrName = 'acraks${subRgUniqueString}' - var agwName = 'apw-${clusterName}' -var wafPolicyName = 'waf-${clusterName}' var aksIngressDomainName = 'aks-ingress.${domainName}' var aksBackendDomainName = 'bu0001a0008-00.${aksIngressDomainName}' -var policyResourceIdAKSLinuxRestrictive = '/providers/Microsoft.Authorization/policySetDefinitions/42b8ef37-b724-4e24-bbc8-7a7708edfe00' -var policyResourceIdEnforceHttpsIngress = '/providers/Microsoft.Authorization/policyDefinitions/1a5b4dca-0b6f-4cf5-907c-56316bc1bf3d' -var policyResourceIdEnforceInternalLoadBalancers = '/providers/Microsoft.Authorization/policyDefinitions/3fc4dc25-5baf-40d8-9b05-7fe74c1bc64e' -var policyResourceIdRoRootFilesystem = '/providers/Microsoft.Authorization/policyDefinitions/df49d893-a74c-421d-bc95-c663042e5b80' -var policyResourceIdEnforceResourceLimits = '/providers/Microsoft.Authorization/policyDefinitions/e345eecc-fa47-480f-9e88-67dcc122b164' -var policyResourceIdEnforceImageSource = '/providers/Microsoft.Authorization/policyDefinitions/febd0533-8e55-448f-b837-bd0e06f16469' -var policyResourceIdEnforceDefenderInCluster = '/providers/Microsoft.Authorization/policyDefinitions/a1840de2-8088-4ea8-b153-b4c723e9cb01' -var policyAssignmentNameAKSLinuxRestrictive = guid(policyResourceIdAKSLinuxRestrictive, resourceGroup().name, clusterName) -var policyAssignmentNameEnforceHttpsIngress = guid(policyResourceIdEnforceHttpsIngress, resourceGroup().name, clusterName) -var policyAssignmentNameEnforceInternalLoadBalancers = guid(policyResourceIdEnforceInternalLoadBalancers, resourceGroup().name, clusterName) -var policyAssignmentNameRoRootFilesystem = guid(policyResourceIdRoRootFilesystem, resourceGroup().name, clusterName) -var policyAssignmentNameEnforceResourceLimits = guid(policyResourceIdEnforceResourceLimits, resourceGroup().name, clusterName) -var policyAssignmentNameEnforceImageSource = guid(policyResourceIdEnforceImageSource, resourceGroup().name, clusterName) -var policyAssignmentNameEnforceDefenderInCluster = guid(policyResourceIdEnforceDefenderInCluster, resourceGroup().name, clusterName) var isUsingAzureRBACasKubernetesRBAC = (subscription().tenantId == k8sControlPlaneAuthorizationTenantId) /*** EXISTING SUBSCRIPTION RESOURCES ***/ +resource nodeResourceGroup 'Microsoft.Resources/resourceGroups@2021-04-01' existing = { + name: 'rg-${clusterName}-nodepools' + scope: subscription() +} + // Built-in Azure RBAC role that is applied to a cluster to indicate they can be considered a user/group of the cluster, subject to additional RBAC permissions resource serviceClusterUserRole 'Microsoft.Authorization/roleDefinitions@2018-01-01-preview' existing = { name: '4abbcc35-e782-43d8-92c5-2d3f1bd2253f' @@ -135,15 +121,32 @@ resource keyVaultSecretsUserRole 'Microsoft.Authorization/roleDefinitions@2018-0 scope: subscription() } -/*** EXISTING HUB RESOURCES ***/ +/*** EXISTING RESOURCE GROUP RESOURCES ***/ +// Azure Container Registry resource acr 'Microsoft.ContainerRegistry/registries@2021-12-01-preview' existing = { - name: defaultAcrName + name: 'acraks${subRgUniqueString}' + scope: resourceGroup() +} + +// Log Analytics Workspace +resource la 'Microsoft.OperationalInsights/workspaces@2021-12-01-preview' existing = { + name: 'la-${clusterName}' + scope: resourceGroup() +} + +// Kubernetes namespace: a0008 +#disable-next-line BCP081 // this namespaces child type doesn't have a defined bicep type yet. +resource nsA0008 'Microsoft.ContainerService/managedClusters/namespaces@2022-01-02-preview' existing = { + parent: mc + name: 'a0008' } +/*** EXISTING HUB RESOURCES ***/ + resource targetResourceGroup 'Microsoft.Resources/resourceGroups@2021-04-01' existing = { - scope: subscription() name: '${split(targetVnetResourceId,'/')[4]}' + scope: subscription() } resource targetVirtualNetwork 'Microsoft.Network/virtualNetworks@2021-05-01' existing = { @@ -161,14 +164,26 @@ resource snetPrivatelinkendpoints 'Microsoft.Network/virtualNetworks/subnets@202 name: 'snet-privatelinkendpoints' } -resource la 'Microsoft.OperationalInsights/workspaces@2021-12-01-preview' existing = { - name: 'la-${clusterName}' -} +// Built-in policy set: AKS Linux Restrictive +var policyResourceIdAKSLinuxRestrictive = tenantResourceId('Microsoft.Authorization/policySetDefinitions', '42b8ef37-b724-4e24-bbc8-7a7708edfe00') -resource nsA0008 'Microsoft.ContainerService/managedClusters/namespaces@2022-01-02-preview' existing = { - parent: mc - name: 'a0008' -} +// Built-in policy: Enforce HTTP Ingress +var policyResourceIdEnforceHttpsIngress = tenantResourceId('Microsoft.Authorization/policyDefinitions', '1a5b4dca-0b6f-4cf5-907c-56316bc1bf3d') + +// Built-in policy: Internal Load Balacner must be used +var policyResourceIdEnforceInternalLoadBalancers = tenantResourceId('Microsoft.Authorization/policyDefinitions', '3fc4dc25-5baf-40d8-9b05-7fe74c1bc64e') + +// Built-in policy: Readonly Root Filesystem must be defined on pods +var policyResourceIdRoRootFilesystem = tenantResourceId('Microsoft.Authorization/policyDefinitions', 'df49d893-a74c-421d-bc95-c663042e5b80') + +// Built-in policy: Resource limits must be defined and not exceeded +var policyResourceIdEnforceResourceLimits = tenantResourceId('Microsoft.Authorization/policyDefinitions', 'e345eecc-fa47-480f-9e88-67dcc122b164') + +// Built-in policy: Images must come from +var policyResourceIdEnforceImageSource = tenantResourceId('Microsoft.Authorization/policyDefinitions', 'febd0533-8e55-448f-b837-bd0e06f16469') + +// Built-in policy: Defender must be enabled. +var policyResourceIdEnforceDefenderInCluster = tenantResourceId('Microsoft.Authorization/policyDefinitions', 'a1840de2-8088-4ea8-b153-b4c723e9cb01') /*** RESOURCES ***/ @@ -891,7 +906,7 @@ resource sqrPodFailed 'Microsoft.Insights/scheduledQueryRules@2018-04-16' = { // Applying the 'AKS Linux Restrictive' policy to the resource group resource paAKSLinuxRestrictive 'Microsoft.Authorization/policyAssignments@2021-06-01' = { - name: policyAssignmentNameAKSLinuxRestrictive + name: guid(policyResourceIdAKSLinuxRestrictive, resourceGroup().name, clusterName) properties: { displayName: '[${clusterName}] ${reference(policyResourceIdAKSLinuxRestrictive, '2020-09-01').displayName}' scope: subscriptionResourceId('Microsoft.Resources/resourceGroups', resourceGroup().name) @@ -915,11 +930,11 @@ resource paAKSLinuxRestrictive 'Microsoft.Authorization/policyAssignments@2021-0 // Applying the 'Enforce HTTPS ingress in Kubernetes cluster' policy to the resource group. resource paEnforceHttpsIngress 'Microsoft.Authorization/policyAssignments@2021-06-01' = { - name: policyAssignmentNameEnforceHttpsIngress + name: guid(policyResourceIdEnforceHttpsIngress, resourceGroup().name, clusterName) location: 'global' + scope: resourceGroup() properties: { displayName: '[${clusterName}] ${reference(policyResourceIdEnforceHttpsIngress, '2020-09-01').displayName}' - scope: subscriptionResourceId('Microsoft.Resources/resourceGroups', resourceGroup().name) policyDefinitionId: policyResourceIdEnforceHttpsIngress parameters: { excludedNamespaces: { @@ -930,15 +945,15 @@ resource paEnforceHttpsIngress 'Microsoft.Authorization/policyAssignments@2021-0 } } } - dependsOn: [] } // Applying the 'Enforce internal load balancers in Kubernetes cluster' policy to the resource group. resource paEnforceInternalLoadBalancers 'Microsoft.Authorization/policyAssignments@2021-06-01' = { - name: policyAssignmentNameEnforceInternalLoadBalancers + name: guid(policyResourceIdEnforceInternalLoadBalancers, resourceGroup().name, clusterName) + location: 'global' + scope: resourceGroup() properties: { displayName: '[${clusterName}] ${reference(policyResourceIdEnforceInternalLoadBalancers, '2020-09-01').displayName}' - scope: subscriptionResourceId('Microsoft.Resources/resourceGroups', resourceGroup().name) policyDefinitionId: policyResourceIdEnforceInternalLoadBalancers parameters: { excludedNamespaces: { @@ -949,15 +964,15 @@ resource paEnforceInternalLoadBalancers 'Microsoft.Authorization/policyAssignmen } } } - dependsOn: [] } // Applying the 'Kubernetes cluster containers should run with a read only root file system' policy to the resource group. resource paRoRootFilesystem 'Microsoft.Authorization/policyAssignments@2021-06-01' = { - name: policyAssignmentNameRoRootFilesystem + name: guid(policyResourceIdRoRootFilesystem, resourceGroup().name, clusterName) + location: 'global' + scope: resourceGroup() properties: { displayName: '[${clusterName}] ${reference(policyResourceIdRoRootFilesystem, '2020-09-01').displayName}' - scope: subscriptionResourceId('Microsoft.Resources/resourceGroups', resourceGroup().name) policyDefinitionId: policyResourceIdRoRootFilesystem parameters: { excludedNamespaces: { @@ -972,16 +987,15 @@ resource paRoRootFilesystem 'Microsoft.Authorization/policyAssignments@2021-06-0 } } } - dependsOn: [] } // Applying the 'Container Images Resource Limits' policy at the resource group level. resource paEnforceResourceLimits 'Microsoft.Authorization/policyAssignments@2021-06-01' = { - name: policyAssignmentNameEnforceResourceLimits - dependsOn: [] + name: guid(policyResourceIdEnforceResourceLimits, resourceGroup().name, clusterName) + location: 'global' + scope: resourceGroup() properties: { displayName: '[${clusterName}] ${reference(policyResourceIdEnforceResourceLimits, '2020-09-01').displayName}' - scope: subscriptionResourceId('Microsoft.Resources/resourceGroups', resourceGroup().name) policyDefinitionId: policyResourceIdEnforceResourceLimits parameters: { cpuLimit: { @@ -1008,14 +1022,15 @@ resource paEnforceResourceLimits 'Microsoft.Authorization/policyAssignments@2021 // Applying the 'Allowed Container Images' regex policy at the resource group level. If all images are pull into your ARC instance as described in these instructions you can remove the docker.io entries. resource paEnforceImageSource 'Microsoft.Authorization/policyAssignments@2021-06-01' = { - name: policyAssignmentNameEnforceImageSource + name: guid(policyResourceIdEnforceImageSource, resourceGroup().name, clusterName) + location: 'global' + scope: resourceGroup() properties: { displayName: '[${clusterName}] ${reference(policyResourceIdEnforceImageSource, '2020-09-01').displayName}' - scope: subscriptionResourceId('Microsoft.Resources/resourceGroups', resourceGroup().name) policyDefinitionId: policyResourceIdEnforceImageSource parameters: { allowedContainerImagesRegex: { - value: '${defaultAcrName}.azurecr.io/.+$|mcr.microsoft.com/.+$|azurearcfork8s.azurecr.io/azurearcflux/images/stable/.+$|docker.io/weaveworks/kured.+$|docker.io/library/.+$' + value: '${acr.name}.azurecr.io/.+$|mcr.microsoft.com/.+$|azurearcfork8s.azurecr.io/azurearcflux/images/stable/.+$|docker.io/weaveworks/kured.+$|docker.io/library/.+$' } excludedNamespaces: { value: [ @@ -1029,16 +1044,16 @@ resource paEnforceImageSource 'Microsoft.Authorization/policyAssignments@2021-06 } } } - dependsOn: [] } // Applying the 'Azure Kubernetes Service clusters should have Defender profile enabled' policy at the resource group level. resource paEnforceDefenderInCluster 'Microsoft.Authorization/policyAssignments@2021-06-01' = { - name: policyAssignmentNameEnforceDefenderInCluster + name: guid(policyResourceIdEnforceDefenderInCluster, resourceGroup().name, clusterName) + location: 'global' + scope: resourceGroup() properties: { displayName: '[${clusterName}] ${reference(policyResourceIdEnforceDefenderInCluster, '2020-09-01').displayName}' description: 'Microsoft Defender for Containers should be enabled in the cluster.' - scope: subscriptionResourceId('Microsoft.Resources/resourceGroups', resourceGroup().name) policyDefinitionId: policyResourceIdEnforceDefenderInCluster parameters: { effect: { @@ -1046,7 +1061,6 @@ resource paEnforceDefenderInCluster 'Microsoft.Authorization/policyAssignments@2 } } } - dependsOn: [] } // The control plane identity used by the cluster. Used for networking access (VNET joining and DNS updating) @@ -1372,7 +1386,7 @@ resource mc 'Microsoft.ContainerService/managedClusters@2022-01-02-preview' = { } } } - nodeResourceGroup: nodeResourceGroupName + nodeResourceGroup: nodeResourceGroup.name enableRBAC: true enablePodSecurityPolicy: false maxAgentPools: 2 @@ -1643,7 +1657,7 @@ resource mc_fluxConfiguration 'Microsoft.KubernetesConfiguration/fluxConfigurati module ndEnsureClusterUserAssignedHasRbacToManageVMSS 'nested_EnsureClusterUserAssignedHasRbacToManageVMSS.bicep' = { name: 'EnsureClusterUserAssignedHasRbacToManageVMSS' - scope: resourceGroup(nodeResourceGroupName) + scope: nodeResourceGroup params: { kubeletidentityObjectId: mc.properties.identityProfile.kubeletidentity.objectId } @@ -1682,7 +1696,7 @@ resource st_diagnosticSettings 'Microsoft.Insights/diagnosticSettings@2021-05-0 } resource wafPolicy 'Microsoft.Network/ApplicationGatewayWebApplicationFirewallPolicies@2021-05-01' = { - name: wafPolicyName + name: 'waf-${clusterName}' location: location properties: { policySettings: { From bfa6ad7df522e4a550a513a5e7a7a5fd03cf0282 Mon Sep 17 00:00:00 2001 From: Chad Kittel Date: Fri, 20 May 2022 21:01:57 +0000 Subject: [PATCH 03/16] stage (and organize) the potential policies --- cluster-stamp.bicep | 178 ++++++++++++++++++++++++++++++++------------ 1 file changed, 132 insertions(+), 46 deletions(-) diff --git a/cluster-stamp.bicep b/cluster-stamp.bicep index c6bb72bd..80852166 100644 --- a/cluster-stamp.bicep +++ b/cluster-stamp.bicep @@ -121,6 +121,102 @@ resource keyVaultSecretsUserRole 'Microsoft.Authorization/roleDefinitions@2018-0 scope: subscription() } +// Azure Resource Provider Policies for Kubernetes + +// Built-in policy: AKS clusters should have Defender profile enabled. +var pdDefenderInClusterEnabledId = tenantResourceId('Microsoft.Authorization/policyDefinitions', 'a1840de2-8088-4ea8-b153-b4c723e9cb01') + +// Built-in policy: AKS clusters should disable Command Invoke. +var pdCommandInvokeDisabledId = tenantResourceId('Microsoft.Authorization/policyDefinitions', '89f2d532-c53c-4f8f-9afa-4927b1114a0d') + +// Built-in policy: AKS should enable Azure Active Directory integration +var pdAadIntegrationEnabledId = tenantResourceId('Microsoft.Authorization/policyDefinitions', '450d2877-ebea-41e8-b00c-e286317d21bf') + +// Built-in policy: AKS should have local authentication methods disabled +var pdLocalAuthDisabledId = tenantResourceId('Microsoft.Authorization/policyDefinitions', '993c2fcd-2b29-49d2-9eb0-df2c3a730c32') + +// Built-in policy: Azure Policy Add-on for AKS should be installed and enabled on your clusters +var pdAzurePolicyEnabledId = tenantResourceId('Microsoft.Authorization/policyDefinitions', '0a15ec92-a229-4763-bb14-0ea34a568f8d') + +// Built-in policy: Authorized IP ranges should be defined on Kubernetes Services +var pdAuthorizedIpRangesDefinedId = tenantResourceId('Microsoft.Authorization/policyDefinitions', '0e246bcf-5f6f-4f87-bc6f-775d4712c7ea') + +// Built-in policy: Kubernetes Services should be upgraded to a non-vulnerable Kubernetes version +var pdOldKuberentesDisabledId = tenantResourceId('Microsoft.Authorization/policyDefinitions', 'fb893a29-21bb-418c-a157-e99480ec364c') + +// Built-in policy: Role-Based Access Control (RBAC) should be used on Kubernetes Services +var pdRbacEnabledId = tenantResourceId('Microsoft.Authorization/policyDefinitions', 'ac4a19c2-fa67-49b4-8ae5-0b2e78c49457') + +// Built-in policy: Azure Kubernetes Service Clusters should use managed identities +var pdManagedIdentitiesEnabledId = tenantResourceId('Microsoft.Authorization/policyDefinitions', 'da6e2401-19da-4532-9141-fb8fbde08431') + +// Running container images should have vulnerability findings resolved +var pdVulnerableImagesResolvedId = tenantResourceId('Microsoft.Authorization/policyDefinitions', '0fc39691-5a3f-4e3e-94ee-2e6447309ad9') + + +// Azure Policy for Kubernetes + +// Built-in initiative: Kubernetes cluster pod security restricted standards for Linux-based workloads +var psdAKSLinuxRestrictiveId = tenantResourceId('Microsoft.Authorization/policySetDefinitions', '42b8ef37-b724-4e24-bbc8-7a7708edfe00') + +// Built-in policy: Kubernetes clusters should be accessible only over HTTPS +var pdEnforceHttpsIngressId = tenantResourceId('Microsoft.Authorization/policyDefinitions', '1a5b4dca-0b6f-4cf5-907c-56316bc1bf3d') + +// Built-in policy: Kubernetes clusters should use internal load balancers +var pdEnforceInternalLoadBalancersId = tenantResourceId('Microsoft.Authorization/policyDefinitions', '3fc4dc25-5baf-40d8-9b05-7fe74c1bc64e') + +// Built-in policy: Kubernetes cluster containers should run with a read only root file system +var pdRoRootFilesystemId = tenantResourceId('Microsoft.Authorization/policyDefinitions', 'df49d893-a74c-421d-bc95-c663042e5b80') + +// Built-in policy: AKS container CPU and memory resource limits should not exceed the specified limits +var pdEnforceResourceLimitsId = tenantResourceId('Microsoft.Authorization/policyDefinitions', 'e345eecc-fa47-480f-9e88-67dcc122b164') + +// Built-in policy: AKS containers should only use allowed images +var pdEnforceImageSourceId = tenantResourceId('Microsoft.Authorization/policyDefinitions', 'febd0533-8e55-448f-b837-bd0e06f16469') + +// Built-in policy: AKS clusters should gate deployment of vulnerable images. +var pdNoVulnerableImagesInClusterId = tenantResourceId('Microsoft.Authorization/policyDefinitions', '13cd7ae3-5bc0-4ac4-a62d-4f7c120b9759') + +// Built-in policy: Ensure cluster containers have readiness or liveness probes configured +var pdContainersMustHaveProbesId = tenantResourceId('Microsoft.Authorization/policyDefinitions', 'b1a9997f-2883-4f12-bdff-2280f99b5915') + +// Built-in policy: Kubernetes cluster pod hostPath volumes should only use allowed host paths +var pdAllowedHostPathsId = tenantResourceId('Microsoft.Authorization/policyDefinitions', '098fc59e-46c7-4d99-9b16-64990e543d75') + +// Built-in policy: Kubernetes cluster containers should not use forbidden sysctl interfaces +var pdForbiddenSysctlId = tenantResourceId('Microsoft.Authorization/policyDefinitions', '56d0a13f-712f-466b-8416-56fb354fb823') + +// Built-in policy: Kubernetes cluster containers should only use allowed AppArmor profiles +var pdAppArmorProfilesId = tenantResourceId('Microsoft.Authorization/policyDefinitions', '511f5417-5d12-434d-ab2e-816901e72a5e') + +// Built-in policy: Kubernetes cluster containers should only use allowed ProcMountType +var pdProcMountTypeId = tenantResourceId('Microsoft.Authorization/policyDefinitions', 'f85eb0dd-92ee-40e9-8a76-db25a507d6d3') + +// Built-in policy: Kubernetes cluster pod FlexVolume volumes should only use allowed drivers +var pdFlexVolumeDriversId = tenantResourceId('Microsoft.Authorization/policyDefinitions', 'f4a8fce0-2dd5-4c21-9a36-8f0ec809d663') + +// Built-in policy: Kubernetes cluster pods and containers should only use allowed SELinux options +var pdSeLinuxOptionsId = tenantResourceId('Microsoft.Authorization/policyDefinitions', 'e1e6c427-07d9-46ab-9689-bfa85431e636') + +// Built-in policy: Kubernetes cluster services should listen only on allowed ports +var pdAllowedPortsId = tenantResourceId('Microsoft.Authorization/policyDefinitions', '233a2a17-77ca-4fb1-9b6b-69223d272a44') + +// Built-in policy: Kubernetes cluster services should only use allowed external IPs +var pdAllowedExternalIPsId = tenantResourceId('Microsoft.Authorization/policyDefinitions', 'd46c275d-1680-448d-b2ec-e495a3b6cc89') + +// Kubernetes clusters should disable automounting API credentials +var pdDisableAutomountApiCredsId = tenantResourceId('Microsoft.Authorization/policyDefinitions', '423dd1ba-798e-40e4-9c4d-b6902674b423') + +// Kubernetes clusters should not allow endpoint edit permissions of ClusterRole/system:aggregate-to-edit +var pdDisallowEndpointEdPermissionsId = tenantResourceId('Microsoft.Authorization/policyDefinitions', '1ddac26b-ed48-4c30-8cc5-3a68c79b8001') + +// Kubernetes clusters should not use specific security capabilities +var pdSecCapId = tenantResourceId('Microsoft.Authorization/policyDefinitions', 'a27c700f-8a22-44ec-961c-41625264370b') + +// Kubernetes clusters should not use the default namespace +var pdDefaultNamespaceId = tenantResourceId('Microsoft.Authorization/policyDefinitions', '9f061a12-e40d-4183-a00e-171812443373') + + /*** EXISTING RESOURCE GROUP RESOURCES ***/ // Azure Container Registry @@ -135,56 +231,40 @@ resource la 'Microsoft.OperationalInsights/workspaces@2021-12-01-preview' existi scope: resourceGroup() } -// Kubernetes namespace: a0008 +// Kubernetes namespace: a0008 -- this doesn't technically exist prior to deployment, but is required as a resource reference later in the template +// to support Azure RBAC-managed API Server access, scoped to the namespace level. #disable-next-line BCP081 // this namespaces child type doesn't have a defined bicep type yet. resource nsA0008 'Microsoft.ContainerService/managedClusters/namespaces@2022-01-02-preview' existing = { parent: mc name: 'a0008' } -/*** EXISTING HUB RESOURCES ***/ +/*** EXISTING SPOKE RESOURCES ***/ +// Spoke resource group resource targetResourceGroup 'Microsoft.Resources/resourceGroups@2021-04-01' existing = { name: '${split(targetVnetResourceId,'/')[4]}' scope: subscription() } +// Spoke virtual network resource targetVirtualNetwork 'Microsoft.Network/virtualNetworks@2021-05-01' existing = { scope: targetResourceGroup name: '${last(split(targetVnetResourceId,'/'))}' } +// Spoke virutual network's subnet for the cluster nodes resource snetClusterNodes 'Microsoft.Network/virtualNetworks/subnets@2021-05-01' existing = { parent: targetVirtualNetwork name: 'snet-clusternodes' } +// Spoke virutual network's subnet for all private endpoints resource snetPrivatelinkendpoints 'Microsoft.Network/virtualNetworks/subnets@2021-05-01' existing = { parent: targetVirtualNetwork name: 'snet-privatelinkendpoints' } -// Built-in policy set: AKS Linux Restrictive -var policyResourceIdAKSLinuxRestrictive = tenantResourceId('Microsoft.Authorization/policySetDefinitions', '42b8ef37-b724-4e24-bbc8-7a7708edfe00') - -// Built-in policy: Enforce HTTP Ingress -var policyResourceIdEnforceHttpsIngress = tenantResourceId('Microsoft.Authorization/policyDefinitions', '1a5b4dca-0b6f-4cf5-907c-56316bc1bf3d') - -// Built-in policy: Internal Load Balacner must be used -var policyResourceIdEnforceInternalLoadBalancers = tenantResourceId('Microsoft.Authorization/policyDefinitions', '3fc4dc25-5baf-40d8-9b05-7fe74c1bc64e') - -// Built-in policy: Readonly Root Filesystem must be defined on pods -var policyResourceIdRoRootFilesystem = tenantResourceId('Microsoft.Authorization/policyDefinitions', 'df49d893-a74c-421d-bc95-c663042e5b80') - -// Built-in policy: Resource limits must be defined and not exceeded -var policyResourceIdEnforceResourceLimits = tenantResourceId('Microsoft.Authorization/policyDefinitions', 'e345eecc-fa47-480f-9e88-67dcc122b164') - -// Built-in policy: Images must come from -var policyResourceIdEnforceImageSource = tenantResourceId('Microsoft.Authorization/policyDefinitions', 'febd0533-8e55-448f-b837-bd0e06f16469') - -// Built-in policy: Defender must be enabled. -var policyResourceIdEnforceDefenderInCluster = tenantResourceId('Microsoft.Authorization/policyDefinitions', 'a1840de2-8088-4ea8-b153-b4c723e9cb01') - /*** RESOURCES ***/ resource alaRgRecommendations 'Microsoft.Insights/activityLogAlerts@2020-10-01' = { @@ -906,11 +986,12 @@ resource sqrPodFailed 'Microsoft.Insights/scheduledQueryRules@2018-04-16' = { // Applying the 'AKS Linux Restrictive' policy to the resource group resource paAKSLinuxRestrictive 'Microsoft.Authorization/policyAssignments@2021-06-01' = { - name: guid(policyResourceIdAKSLinuxRestrictive, resourceGroup().name, clusterName) + name: guid(psdAKSLinuxRestrictiveId, resourceGroup().id, clusterName) + scope: resourceGroup() properties: { - displayName: '[${clusterName}] ${reference(policyResourceIdAKSLinuxRestrictive, '2020-09-01').displayName}' - scope: subscriptionResourceId('Microsoft.Resources/resourceGroups', resourceGroup().name) - policyDefinitionId: policyResourceIdAKSLinuxRestrictive + displayName: take('[${clusterName}] ${reference(psdAKSLinuxRestrictiveId, '2021-06-01').displayName}', 120) + description: reference(psdAKSLinuxRestrictiveId, '2021-06-01').description + policyDefinitionId: psdAKSLinuxRestrictiveId parameters: { excludedNamespaces: { value: [ @@ -930,12 +1011,13 @@ resource paAKSLinuxRestrictive 'Microsoft.Authorization/policyAssignments@2021-0 // Applying the 'Enforce HTTPS ingress in Kubernetes cluster' policy to the resource group. resource paEnforceHttpsIngress 'Microsoft.Authorization/policyAssignments@2021-06-01' = { - name: guid(policyResourceIdEnforceHttpsIngress, resourceGroup().name, clusterName) + name: guid(pdEnforceHttpsIngressId, resourceGroup().id, clusterName) location: 'global' scope: resourceGroup() properties: { - displayName: '[${clusterName}] ${reference(policyResourceIdEnforceHttpsIngress, '2020-09-01').displayName}' - policyDefinitionId: policyResourceIdEnforceHttpsIngress + displayName: take('[${clusterName}] ${reference(pdEnforceHttpsIngressId, '2021-06-01').displayName}', 120) + description: reference(pdEnforceHttpsIngressId, '2021-06-01').description + policyDefinitionId: pdEnforceHttpsIngressId parameters: { excludedNamespaces: { value: [] @@ -949,12 +1031,13 @@ resource paEnforceHttpsIngress 'Microsoft.Authorization/policyAssignments@2021-0 // Applying the 'Enforce internal load balancers in Kubernetes cluster' policy to the resource group. resource paEnforceInternalLoadBalancers 'Microsoft.Authorization/policyAssignments@2021-06-01' = { - name: guid(policyResourceIdEnforceInternalLoadBalancers, resourceGroup().name, clusterName) + name: guid(pdEnforceInternalLoadBalancersId, resourceGroup().id, clusterName) location: 'global' scope: resourceGroup() properties: { - displayName: '[${clusterName}] ${reference(policyResourceIdEnforceInternalLoadBalancers, '2020-09-01').displayName}' - policyDefinitionId: policyResourceIdEnforceInternalLoadBalancers + displayName: take('[${clusterName}] ${reference(pdEnforceInternalLoadBalancersId, '2021-06-01').displayName}', 120) + description: reference(pdEnforceInternalLoadBalancersId, '2021-06-01').description + policyDefinitionId: pdEnforceInternalLoadBalancersId parameters: { excludedNamespaces: { value: [] @@ -968,12 +1051,13 @@ resource paEnforceInternalLoadBalancers 'Microsoft.Authorization/policyAssignmen // Applying the 'Kubernetes cluster containers should run with a read only root file system' policy to the resource group. resource paRoRootFilesystem 'Microsoft.Authorization/policyAssignments@2021-06-01' = { - name: guid(policyResourceIdRoRootFilesystem, resourceGroup().name, clusterName) + name: guid(pdRoRootFilesystemId, resourceGroup().id, clusterName) location: 'global' scope: resourceGroup() properties: { - displayName: '[${clusterName}] ${reference(policyResourceIdRoRootFilesystem, '2020-09-01').displayName}' - policyDefinitionId: policyResourceIdRoRootFilesystem + displayName: take('[${clusterName}] ${reference(pdRoRootFilesystemId, '2021-06-01').displayName}', 120) + description: reference(pdRoRootFilesystemId, '2021-06-01').description + policyDefinitionId: pdRoRootFilesystemId parameters: { excludedNamespaces: { value: [ @@ -991,12 +1075,13 @@ resource paRoRootFilesystem 'Microsoft.Authorization/policyAssignments@2021-06-0 // Applying the 'Container Images Resource Limits' policy at the resource group level. resource paEnforceResourceLimits 'Microsoft.Authorization/policyAssignments@2021-06-01' = { - name: guid(policyResourceIdEnforceResourceLimits, resourceGroup().name, clusterName) + name: guid(pdEnforceResourceLimitsId, resourceGroup().id, clusterName) location: 'global' scope: resourceGroup() properties: { - displayName: '[${clusterName}] ${reference(policyResourceIdEnforceResourceLimits, '2020-09-01').displayName}' - policyDefinitionId: policyResourceIdEnforceResourceLimits + displayName: take('[${clusterName}] ${reference(pdEnforceResourceLimitsId, '2021-06-01').displayName}', 120) + description: reference(pdEnforceResourceLimitsId, '2021-06-01').description + policyDefinitionId: pdEnforceResourceLimitsId parameters: { cpuLimit: { value: '1000m' @@ -1022,12 +1107,13 @@ resource paEnforceResourceLimits 'Microsoft.Authorization/policyAssignments@2021 // Applying the 'Allowed Container Images' regex policy at the resource group level. If all images are pull into your ARC instance as described in these instructions you can remove the docker.io entries. resource paEnforceImageSource 'Microsoft.Authorization/policyAssignments@2021-06-01' = { - name: guid(policyResourceIdEnforceImageSource, resourceGroup().name, clusterName) + name: guid(pdEnforceImageSourceId, resourceGroup().id, clusterName) location: 'global' scope: resourceGroup() properties: { - displayName: '[${clusterName}] ${reference(policyResourceIdEnforceImageSource, '2020-09-01').displayName}' - policyDefinitionId: policyResourceIdEnforceImageSource + displayName: take('[${clusterName}] ${reference(pdEnforceImageSourceId, '2021-06-01').displayName}', 120) + description: reference(pdEnforceImageSourceId, '2021-06-01').description + policyDefinitionId: pdEnforceImageSourceId parameters: { allowedContainerImagesRegex: { value: '${acr.name}.azurecr.io/.+$|mcr.microsoft.com/.+$|azurearcfork8s.azurecr.io/azurearcflux/images/stable/.+$|docker.io/weaveworks/kured.+$|docker.io/library/.+$' @@ -1048,13 +1134,13 @@ resource paEnforceImageSource 'Microsoft.Authorization/policyAssignments@2021-06 // Applying the 'Azure Kubernetes Service clusters should have Defender profile enabled' policy at the resource group level. resource paEnforceDefenderInCluster 'Microsoft.Authorization/policyAssignments@2021-06-01' = { - name: guid(policyResourceIdEnforceDefenderInCluster, resourceGroup().name, clusterName) + name: guid(pdDefenderInClusterEnabledId, resourceGroup().id, clusterName) location: 'global' scope: resourceGroup() properties: { - displayName: '[${clusterName}] ${reference(policyResourceIdEnforceDefenderInCluster, '2020-09-01').displayName}' - description: 'Microsoft Defender for Containers should be enabled in the cluster.' - policyDefinitionId: policyResourceIdEnforceDefenderInCluster + displayName: take('[${clusterName}] ${reference(pdDefenderInClusterEnabledId, '2021-06-01').displayName}', 120) + description: reference(pdDefenderInClusterEnabledId, '2021-06-01').description + policyDefinitionId: pdDefenderInClusterEnabledId parameters: { effect: { value: 'Audit' From 6ce4dd59306451792204b7cd6288b19e6c45e921 Mon Sep 17 00:00:00 2001 From: Chad Kittel Date: Fri, 20 May 2022 21:37:05 +0000 Subject: [PATCH 04/16] Refresh and apply new resource policies --- cluster-stamp.bicep | 208 ++++++++++++++++++++++++++++++++++++-------- 1 file changed, 171 insertions(+), 37 deletions(-) diff --git a/cluster-stamp.bicep b/cluster-stamp.bicep index 80852166..95c12917 100644 --- a/cluster-stamp.bicep +++ b/cluster-stamp.bicep @@ -121,39 +121,6 @@ resource keyVaultSecretsUserRole 'Microsoft.Authorization/roleDefinitions@2018-0 scope: subscription() } -// Azure Resource Provider Policies for Kubernetes - -// Built-in policy: AKS clusters should have Defender profile enabled. -var pdDefenderInClusterEnabledId = tenantResourceId('Microsoft.Authorization/policyDefinitions', 'a1840de2-8088-4ea8-b153-b4c723e9cb01') - -// Built-in policy: AKS clusters should disable Command Invoke. -var pdCommandInvokeDisabledId = tenantResourceId('Microsoft.Authorization/policyDefinitions', '89f2d532-c53c-4f8f-9afa-4927b1114a0d') - -// Built-in policy: AKS should enable Azure Active Directory integration -var pdAadIntegrationEnabledId = tenantResourceId('Microsoft.Authorization/policyDefinitions', '450d2877-ebea-41e8-b00c-e286317d21bf') - -// Built-in policy: AKS should have local authentication methods disabled -var pdLocalAuthDisabledId = tenantResourceId('Microsoft.Authorization/policyDefinitions', '993c2fcd-2b29-49d2-9eb0-df2c3a730c32') - -// Built-in policy: Azure Policy Add-on for AKS should be installed and enabled on your clusters -var pdAzurePolicyEnabledId = tenantResourceId('Microsoft.Authorization/policyDefinitions', '0a15ec92-a229-4763-bb14-0ea34a568f8d') - -// Built-in policy: Authorized IP ranges should be defined on Kubernetes Services -var pdAuthorizedIpRangesDefinedId = tenantResourceId('Microsoft.Authorization/policyDefinitions', '0e246bcf-5f6f-4f87-bc6f-775d4712c7ea') - -// Built-in policy: Kubernetes Services should be upgraded to a non-vulnerable Kubernetes version -var pdOldKuberentesDisabledId = tenantResourceId('Microsoft.Authorization/policyDefinitions', 'fb893a29-21bb-418c-a157-e99480ec364c') - -// Built-in policy: Role-Based Access Control (RBAC) should be used on Kubernetes Services -var pdRbacEnabledId = tenantResourceId('Microsoft.Authorization/policyDefinitions', 'ac4a19c2-fa67-49b4-8ae5-0b2e78c49457') - -// Built-in policy: Azure Kubernetes Service Clusters should use managed identities -var pdManagedIdentitiesEnabledId = tenantResourceId('Microsoft.Authorization/policyDefinitions', 'da6e2401-19da-4532-9141-fb8fbde08431') - -// Running container images should have vulnerability findings resolved -var pdVulnerableImagesResolvedId = tenantResourceId('Microsoft.Authorization/policyDefinitions', '0fc39691-5a3f-4e3e-94ee-2e6447309ad9') - - // Azure Policy for Kubernetes // Built-in initiative: Kubernetes cluster pod security restricted standards for Linux-based workloads @@ -984,9 +951,14 @@ resource sqrPodFailed 'Microsoft.Insights/scheduledQueryRules@2018-04-16' = { dependsOn: [] } +// Resource Group Azure Policy Assignments - Azure Policy for Kubernetes Policies + +// TODO None of these have been refreshed yet, nor are the new ones added. + // Applying the 'AKS Linux Restrictive' policy to the resource group resource paAKSLinuxRestrictive 'Microsoft.Authorization/policyAssignments@2021-06-01' = { name: guid(psdAKSLinuxRestrictiveId, resourceGroup().id, clusterName) + location: 'global' scope: resourceGroup() properties: { displayName: take('[${clusterName}] ${reference(psdAKSLinuxRestrictiveId, '2021-06-01').displayName}', 120) @@ -1132,8 +1104,11 @@ resource paEnforceImageSource 'Microsoft.Authorization/policyAssignments@2021-06 } } -// Applying the 'Azure Kubernetes Service clusters should have Defender profile enabled' policy at the resource group level. -resource paEnforceDefenderInCluster 'Microsoft.Authorization/policyAssignments@2021-06-01' = { +// Resource Group Azure Policy Assignments - Resource Provider Policies + +// Applying the built-in 'Azure Kubernetes Service clusters should have Defender profile enabled' policy at the resource group level. +var pdDefenderInClusterEnabledId = tenantResourceId('Microsoft.Authorization/policyDefinitions', 'a1840de2-8088-4ea8-b153-b4c723e9cb01') +resource paDefenderInClusterEnabled 'Microsoft.Authorization/policyAssignments@2021-06-01' = { name: guid(pdDefenderInClusterEnabledId, resourceGroup().id, clusterName) location: 'global' scope: resourceGroup() @@ -1143,7 +1118,151 @@ resource paEnforceDefenderInCluster 'Microsoft.Authorization/policyAssignments@2 policyDefinitionId: pdDefenderInClusterEnabledId parameters: { effect: { - value: 'Audit' + value: 'Audit' // This policy (as of 1.0.2-preview) does not have a Deny option, otherwise that would be set here. + } + } + } +} + +// Applying the built-in 'Azure Kubernetes Service Clusters should enable Azure Active Directory integration' policy at the resource group level. +var pdAadIntegrationEnabledId = tenantResourceId('Microsoft.Authorization/policyDefinitions', '450d2877-ebea-41e8-b00c-e286317d21bf') +resource paAadIntegrationEnabled 'Microsoft.Authorization/policyAssignments@2021-06-01' = { + name: guid(pdAadIntegrationEnabledId, resourceGroup().id, clusterName) + location: 'global' + scope: resourceGroup() + properties: { + displayName: take('[${clusterName}] ${reference(pdAadIntegrationEnabledId, '2021-06-01').displayName}', 120) + description: reference(pdAadIntegrationEnabledId, '2021-06-01').description + policyDefinitionId: pdAadIntegrationEnabledId + parameters: { + effect: { + value: 'Audit' // This policy (as of 1.0.0) does not have a Deny option, otherwise that would be set here. + } + } + } +} + +// Applying the built-in 'Azure Kubernetes Service Clusters should have local authentication methods disabled' policy at the resource group level. +var pdLocalAuthDisabledId = tenantResourceId('Microsoft.Authorization/policyDefinitions', '993c2fcd-2b29-49d2-9eb0-df2c3a730c32') +resource paLocalAuthDisabled 'Microsoft.Authorization/policyAssignments@2021-06-01' = { + name: guid(pdLocalAuthDisabledId, resourceGroup().id, clusterName) + location: 'global' + scope: resourceGroup() + properties: { + displayName: take('[${clusterName}] ${reference(pdLocalAuthDisabledId, '2021-06-01').displayName}', 120) + description: reference(pdLocalAuthDisabledId, '2021-06-01').description + policyDefinitionId: pdLocalAuthDisabledId + parameters: { + effect: { + value: 'Deny' + } + } + } +} + +// Applying the built-in 'Azure Policy Add-on for Kubernetes service (AKS) should be installed and enabled on your clusters' policy at the resource group level. +var pdAzurePolicyEnabledId = tenantResourceId('Microsoft.Authorization/policyDefinitions', '0a15ec92-a229-4763-bb14-0ea34a568f8d') +resource paAzurePolicyEnabled 'Microsoft.Authorization/policyAssignments@2021-06-01' = { + name: guid(pdAzurePolicyEnabledId, resourceGroup().id, clusterName) + location: 'global' + scope: resourceGroup() + properties: { + displayName: take('[${clusterName}] ${reference(pdAzurePolicyEnabledId, '2021-06-01').displayName}', 120) + description: reference(pdAzurePolicyEnabledId, '2021-06-01').description + policyDefinitionId: pdAzurePolicyEnabledId + parameters: { + effect: { + value: 'Audit' // This policy (as of 1.0.2) does not have a Deny option, otherwise that would be set here. + } + } + } +} + +// Applying the built-in 'Authorized IP ranges should be defined on Kubernetes Services' policy at the resource group level. +var pdAuthorizedIpRangesDefinedId = tenantResourceId('Microsoft.Authorization/policyDefinitions', '0e246bcf-5f6f-4f87-bc6f-775d4712c7ea') +resource paAuthorizedIpRangesDefined 'Microsoft.Authorization/policyAssignments@2021-06-01' = { + name: guid(pdAuthorizedIpRangesDefinedId, resourceGroup().id, clusterName) + location: 'global' + scope: resourceGroup() + properties: { + displayName: take('[${clusterName}] ${reference(pdAuthorizedIpRangesDefinedId, '2021-06-01').displayName}', 120) + description: reference(pdAuthorizedIpRangesDefinedId, '2021-06-01').description + policyDefinitionId: pdAuthorizedIpRangesDefinedId + parameters: { + effect: { + value: 'Audit' // This policy (as of 2.0.1) does not have a Deny option, otherwise that would be set here. + } + } + } +} + +// Applying the built-in 'Kubernetes Services should be upgraded to a non-vulnerable Kubernetes version' policy at the resource group level. +var pdOldKuberentesDisabledId = tenantResourceId('Microsoft.Authorization/policyDefinitions', 'fb893a29-21bb-418c-a157-e99480ec364c') +resource paOldKuberentesDisabled 'Microsoft.Authorization/policyAssignments@2021-06-01' = { + name: guid(pdOldKuberentesDisabledId, resourceGroup().id, clusterName) + location: 'global' + scope: resourceGroup() + properties: { + displayName: take('[${clusterName}] ${reference(pdOldKuberentesDisabledId, '2021-06-01').displayName}', 120) + description: reference(pdOldKuberentesDisabledId, '2021-06-01').description + policyDefinitionId: pdOldKuberentesDisabledId + parameters: { + effect: { + value: 'Audit' // This policy (as of 1.0.2) does not have a Deny option, otherwise that would be set here. + } + } + } +} + +// Applying the built-in 'Role-Based Access Control (RBAC) should be used on Kubernetes Services' policy at the resource group level. +var pdRbacEnabledId = tenantResourceId('Microsoft.Authorization/policyDefinitions', 'ac4a19c2-fa67-49b4-8ae5-0b2e78c49457') +resource paRbacEnabled 'Microsoft.Authorization/policyAssignments@2021-06-01' = { + name: guid(pdRbacEnabledId, resourceGroup().id, clusterName) + location: 'global' + scope: resourceGroup() + properties: { + displayName: take('[${clusterName}] ${reference(pdRbacEnabledId, '2021-06-01').displayName}', 120) + description: reference(pdRbacEnabledId, '2021-06-01').description + policyDefinitionId: pdRbacEnabledId + parameters: { + effect: { + value: 'Audit' // This policy (as of 1.0.2) does not have a Deny option, otherwise that would be set here. + } + } + } +} + +// Applying the built-in 'Azure Kubernetes Service Clusters should use managed identities' policy at the resource group level. +var pdManagedIdentitiesEnabledId = tenantResourceId('Microsoft.Authorization/policyDefinitions', 'da6e2401-19da-4532-9141-fb8fbde08431') +resource paManagedIdentitiesEnabled 'Microsoft.Authorization/policyAssignments@2021-06-01' = { + name: guid(pdManagedIdentitiesEnabledId, resourceGroup().id, clusterName) + location: 'global' + scope: resourceGroup() + properties: { + displayName: take('[${clusterName}] ${reference(pdManagedIdentitiesEnabledId, '2021-06-01').displayName}', 120) + description: reference(pdManagedIdentitiesEnabledId, '2021-06-01').description + policyDefinitionId: pdManagedIdentitiesEnabledId + parameters: { + effect: { + value: 'Audit' // This policy (as of 1.0.0) does not have a Deny option, otherwise that would be set here. + } + } + } +} + +// Applying the built-in 'Running container images should have vulnerability findings resolved' policy at the resource group level. +var pdVulnerableImagesResolvedId = tenantResourceId('Microsoft.Authorization/policyDefinitions', '0fc39691-5a3f-4e3e-94ee-2e6447309ad9') +resource paVulnerableImagesResolved 'Microsoft.Authorization/policyAssignments@2021-06-01' = { + name: guid(pdVulnerableImagesResolvedId, resourceGroup().id, clusterName) + location: 'global' + scope: resourceGroup() + properties: { + displayName: take('[${clusterName}] ${reference(pdVulnerableImagesResolvedId, '2021-06-01').displayName}', 120) + description: reference(pdVulnerableImagesResolvedId, '2021-06-01').description + policyDefinitionId: pdVulnerableImagesResolvedId + parameters: { + effect: { + value: 'AuditIfNotExists' } } } @@ -1547,13 +1666,28 @@ resource mc 'Microsoft.ContainerService/managedClusters@2022-01-02-preview' = { ndEnsureClusterIdentityHasRbacToSelfManagedResources + // Azure Policy for Kubernetes policies that we'd want in place before pods start showing up + // in the cluster. The are not technically a depedency from the resource provider perspective, + // but logically they need to be in place before workloads are, so focing that here. This also + // ensures that the policies are applied to the cluster at bootstrapping time. paAKSLinuxRestrictive paEnforceHttpsIngress paEnforceInternalLoadBalancers paEnforceResourceLimits paRoRootFilesystem paEnforceImageSource - paEnforceDefenderInCluster + + // Azure Resource Provider policies that we'd like to see in place before the cluster is deployed + // They are not technically a dependency, but logically they would have existed on the resource group + // prior to the existence of the cluster, so forcing that here. + paDefenderInClusterEnabled + paAadIntegrationEnabled + paLocalAuthDisabled + paAzurePolicyEnabled + paOldKuberentesDisabled + paRbacEnabled + paManagedIdentitiesEnabled + paVulnerableImagesResolved peKv kvPodMiIngressControllerKeyVaultReader_roleAssignment From 2eb54aaa32bbaa2588f5d894d5586b6bb3ee8438 Mon Sep 17 00:00:00 2001 From: Chad Kittel Date: Mon, 23 May 2022 16:26:27 +0000 Subject: [PATCH 05/16] Selected policies, pre-test --- cluster-stamp.bicep | 323 ++++++++++++++++++++++++++++++++------------ 1 file changed, 237 insertions(+), 86 deletions(-) diff --git a/cluster-stamp.bicep b/cluster-stamp.bicep index 95c12917..3de151da 100644 --- a/cluster-stamp.bicep +++ b/cluster-stamp.bicep @@ -121,69 +121,6 @@ resource keyVaultSecretsUserRole 'Microsoft.Authorization/roleDefinitions@2018-0 scope: subscription() } -// Azure Policy for Kubernetes - -// Built-in initiative: Kubernetes cluster pod security restricted standards for Linux-based workloads -var psdAKSLinuxRestrictiveId = tenantResourceId('Microsoft.Authorization/policySetDefinitions', '42b8ef37-b724-4e24-bbc8-7a7708edfe00') - -// Built-in policy: Kubernetes clusters should be accessible only over HTTPS -var pdEnforceHttpsIngressId = tenantResourceId('Microsoft.Authorization/policyDefinitions', '1a5b4dca-0b6f-4cf5-907c-56316bc1bf3d') - -// Built-in policy: Kubernetes clusters should use internal load balancers -var pdEnforceInternalLoadBalancersId = tenantResourceId('Microsoft.Authorization/policyDefinitions', '3fc4dc25-5baf-40d8-9b05-7fe74c1bc64e') - -// Built-in policy: Kubernetes cluster containers should run with a read only root file system -var pdRoRootFilesystemId = tenantResourceId('Microsoft.Authorization/policyDefinitions', 'df49d893-a74c-421d-bc95-c663042e5b80') - -// Built-in policy: AKS container CPU and memory resource limits should not exceed the specified limits -var pdEnforceResourceLimitsId = tenantResourceId('Microsoft.Authorization/policyDefinitions', 'e345eecc-fa47-480f-9e88-67dcc122b164') - -// Built-in policy: AKS containers should only use allowed images -var pdEnforceImageSourceId = tenantResourceId('Microsoft.Authorization/policyDefinitions', 'febd0533-8e55-448f-b837-bd0e06f16469') - -// Built-in policy: AKS clusters should gate deployment of vulnerable images. -var pdNoVulnerableImagesInClusterId = tenantResourceId('Microsoft.Authorization/policyDefinitions', '13cd7ae3-5bc0-4ac4-a62d-4f7c120b9759') - -// Built-in policy: Ensure cluster containers have readiness or liveness probes configured -var pdContainersMustHaveProbesId = tenantResourceId('Microsoft.Authorization/policyDefinitions', 'b1a9997f-2883-4f12-bdff-2280f99b5915') - -// Built-in policy: Kubernetes cluster pod hostPath volumes should only use allowed host paths -var pdAllowedHostPathsId = tenantResourceId('Microsoft.Authorization/policyDefinitions', '098fc59e-46c7-4d99-9b16-64990e543d75') - -// Built-in policy: Kubernetes cluster containers should not use forbidden sysctl interfaces -var pdForbiddenSysctlId = tenantResourceId('Microsoft.Authorization/policyDefinitions', '56d0a13f-712f-466b-8416-56fb354fb823') - -// Built-in policy: Kubernetes cluster containers should only use allowed AppArmor profiles -var pdAppArmorProfilesId = tenantResourceId('Microsoft.Authorization/policyDefinitions', '511f5417-5d12-434d-ab2e-816901e72a5e') - -// Built-in policy: Kubernetes cluster containers should only use allowed ProcMountType -var pdProcMountTypeId = tenantResourceId('Microsoft.Authorization/policyDefinitions', 'f85eb0dd-92ee-40e9-8a76-db25a507d6d3') - -// Built-in policy: Kubernetes cluster pod FlexVolume volumes should only use allowed drivers -var pdFlexVolumeDriversId = tenantResourceId('Microsoft.Authorization/policyDefinitions', 'f4a8fce0-2dd5-4c21-9a36-8f0ec809d663') - -// Built-in policy: Kubernetes cluster pods and containers should only use allowed SELinux options -var pdSeLinuxOptionsId = tenantResourceId('Microsoft.Authorization/policyDefinitions', 'e1e6c427-07d9-46ab-9689-bfa85431e636') - -// Built-in policy: Kubernetes cluster services should listen only on allowed ports -var pdAllowedPortsId = tenantResourceId('Microsoft.Authorization/policyDefinitions', '233a2a17-77ca-4fb1-9b6b-69223d272a44') - -// Built-in policy: Kubernetes cluster services should only use allowed external IPs -var pdAllowedExternalIPsId = tenantResourceId('Microsoft.Authorization/policyDefinitions', 'd46c275d-1680-448d-b2ec-e495a3b6cc89') - -// Kubernetes clusters should disable automounting API credentials -var pdDisableAutomountApiCredsId = tenantResourceId('Microsoft.Authorization/policyDefinitions', '423dd1ba-798e-40e4-9c4d-b6902674b423') - -// Kubernetes clusters should not allow endpoint edit permissions of ClusterRole/system:aggregate-to-edit -var pdDisallowEndpointEdPermissionsId = tenantResourceId('Microsoft.Authorization/policyDefinitions', '1ddac26b-ed48-4c30-8cc5-3a68c79b8001') - -// Kubernetes clusters should not use specific security capabilities -var pdSecCapId = tenantResourceId('Microsoft.Authorization/policyDefinitions', 'a27c700f-8a22-44ec-961c-41625264370b') - -// Kubernetes clusters should not use the default namespace -var pdDefaultNamespaceId = tenantResourceId('Microsoft.Authorization/policyDefinitions', '9f061a12-e40d-4183-a00e-171812443373') - - /*** EXISTING RESOURCE GROUP RESOURCES ***/ // Azure Container Registry @@ -953,9 +890,8 @@ resource sqrPodFailed 'Microsoft.Insights/scheduledQueryRules@2018-04-16' = { // Resource Group Azure Policy Assignments - Azure Policy for Kubernetes Policies -// TODO None of these have been refreshed yet, nor are the new ones added. - -// Applying the 'AKS Linux Restrictive' policy to the resource group +// Applying the built-in 'Kubernetes cluster pod security restricted standards for Linux-based workloads' initiative at the resource group level. +var psdAKSLinuxRestrictiveId = tenantResourceId('Microsoft.Authorization/policySetDefinitions', '42b8ef37-b724-4e24-bbc8-7a7708edfe00') resource paAKSLinuxRestrictive 'Microsoft.Authorization/policyAssignments@2021-06-01' = { name: guid(psdAKSLinuxRestrictiveId, resourceGroup().id, clusterName) location: 'global' @@ -970,18 +906,40 @@ resource paAKSLinuxRestrictive 'Microsoft.Authorization/policyAssignments@2021-0 'kube-system' 'gatekeeper-system' 'azure-arc' - 'cluster-baseline-settings' + // 'cluster-baseline-settings' TODO: Test + ] + } + allowedCapabilities: { + value: [ + 'CHOWN' + 'DAC_OVERRIDE' + 'FSETID' + 'FOWNER' + 'MKNOD' + 'NET_RAW' + 'SETGID' + 'SETUID' + 'SETFCAP' + 'SETPCAP' + 'NET_BIND_SERVICE' + 'SYS_CHROOT' + 'KILL' + 'AUDIT_WRITE' ] } + requiredDropCapabilities: { + value: [] // None + } effect: { - value: 'audit' + value: 'Audit' } } } dependsOn: [] } -// Applying the 'Enforce HTTPS ingress in Kubernetes cluster' policy to the resource group. +// Applying the built-in 'Kubernetes clusters should be accessible only over HTTPS' policy at the resource group level. +var pdEnforceHttpsIngressId = tenantResourceId('Microsoft.Authorization/policyDefinitions', '1a5b4dca-0b6f-4cf5-907c-56316bc1bf3d') resource paEnforceHttpsIngress 'Microsoft.Authorization/policyAssignments@2021-06-01' = { name: guid(pdEnforceHttpsIngressId, resourceGroup().id, clusterName) location: 'global' @@ -992,16 +950,17 @@ resource paEnforceHttpsIngress 'Microsoft.Authorization/policyAssignments@2021-0 policyDefinitionId: pdEnforceHttpsIngressId parameters: { excludedNamespaces: { - value: [] + value: [] // None } effect: { - value: 'deny' + value: 'Deny' } } } } -// Applying the 'Enforce internal load balancers in Kubernetes cluster' policy to the resource group. +// Applying the built-in 'Kubernetes clusters should use internal load balancers' policy at the resource group level. +var pdEnforceInternalLoadBalancersId = tenantResourceId('Microsoft.Authorization/policyDefinitions', '3fc4dc25-5baf-40d8-9b05-7fe74c1bc64e') resource paEnforceInternalLoadBalancers 'Microsoft.Authorization/policyAssignments@2021-06-01' = { name: guid(pdEnforceInternalLoadBalancersId, resourceGroup().id, clusterName) location: 'global' @@ -1012,16 +971,17 @@ resource paEnforceInternalLoadBalancers 'Microsoft.Authorization/policyAssignmen policyDefinitionId: pdEnforceInternalLoadBalancersId parameters: { excludedNamespaces: { - value: [] + value: [] // None } effect: { - value: 'deny' + value: 'Deny' } } } } -// Applying the 'Kubernetes cluster containers should run with a read only root file system' policy to the resource group. +// Applying the built-in 'Kubernetes cluster containers should run with a read only root file system' policy at the resource group level. +var pdRoRootFilesystemId = tenantResourceId('Microsoft.Authorization/policyDefinitions', 'df49d893-a74c-421d-bc95-c663042e5b80') resource paRoRootFilesystem 'Microsoft.Authorization/policyAssignments@2021-06-01' = { name: guid(pdRoRootFilesystemId, resourceGroup().id, clusterName) location: 'global' @@ -1039,13 +999,14 @@ resource paRoRootFilesystem 'Microsoft.Authorization/policyAssignments@2021-06-0 ] } effect: { - value: 'audit' + value: 'Audit' } } } } -// Applying the 'Container Images Resource Limits' policy at the resource group level. +// Applying the built-in 'AKS container CPU and memory resource limits should not exceed the specified limits' policy at the resource group level. +var pdEnforceResourceLimitsId = tenantResourceId('Microsoft.Authorization/policyDefinitions', 'e345eecc-fa47-480f-9e88-67dcc122b164') resource paEnforceResourceLimits 'Microsoft.Authorization/policyAssignments@2021-06-01' = { name: guid(pdEnforceResourceLimitsId, resourceGroup().id, clusterName) location: 'global' @@ -1056,28 +1017,29 @@ resource paEnforceResourceLimits 'Microsoft.Authorization/policyAssignments@2021 policyDefinitionId: pdEnforceResourceLimitsId parameters: { cpuLimit: { - value: '1000m' + value: '100m' // Was 1000m TODO } memoryLimit: { - value: '512Mi' + value: '92Mi' // Was 512m TODO } excludedNamespaces: { value: [ 'kube-system' 'gatekeeper-system' 'azure-arc' - 'cluster-baseline-settings' - 'flux-system' + // 'cluster-baseline-settings' -- TODO: Set as excluded images + // 'flux-system' -- TODO: Set as excluded images ] } effect: { - value: 'deny' + value: 'Audit' // TODO: Back to Deny } } } } -// Applying the 'Allowed Container Images' regex policy at the resource group level. If all images are pull into your ARC instance as described in these instructions you can remove the docker.io entries. +// Applying the built-in 'AKS containers should only use allowed images' policy at the resource group level. +var pdEnforceImageSourceId = tenantResourceId('Microsoft.Authorization/policyDefinitions', 'febd0533-8e55-448f-b837-bd0e06f16469') resource paEnforceImageSource 'Microsoft.Authorization/policyAssignments@2021-06-01' = { name: guid(pdEnforceImageSourceId, resourceGroup().id, clusterName) location: 'global' @@ -1088,8 +1050,119 @@ resource paEnforceImageSource 'Microsoft.Authorization/policyAssignments@2021-06 policyDefinitionId: pdEnforceImageSourceId parameters: { allowedContainerImagesRegex: { - value: '${acr.name}.azurecr.io/.+$|mcr.microsoft.com/.+$|azurearcfork8s.azurecr.io/azurearcflux/images/stable/.+$|docker.io/weaveworks/kured.+$|docker.io/library/.+$' + // If all images are pull into your ARC instance as described in these instructions you can remove the docker.io entries. + value: '${acr.name}\\.azurecr\\.io/.+$|mcr\\.microsoft\\.com/.+$|azurearcfork8s\\.azurecr\\.io/azurearcflux/images/stable/.+$|docker\\.io/weaveworks/kured.+$|docker\\.io/library/.+$' + } + excludedNamespaces: { + value: [ + 'kube-system' + 'gatekeeper-system' + 'azure-arc' + // TODO: Check on azurearcfork8s + ] + } + effect: { + value: 'Deny' + } + } + } +} + +// Built-in policy: Kubernetes cluster pod hostPath volumes should only use allowed host paths' policy at the resource group level. +var pdAllowedHostPathsId = tenantResourceId('Microsoft.Authorization/policyDefinitions', '098fc59e-46c7-4d99-9b16-64990e543d75') +resource paAllowedHostPaths 'Microsoft.Authorization/policyAssignments@2021-06-01' = { + name: guid(pdAllowedHostPathsId, resourceGroup().id, clusterName) + location: 'global' + scope: resourceGroup() + properties: { + displayName: take('[${clusterName}] ${reference(pdAllowedHostPathsId, '2021-06-01').displayName}', 120) + description: reference(pdAllowedHostPathsId, '2021-06-01').description + policyDefinitionId: pdAllowedHostPathsId + parameters: { + excludedNamespaces: { + value: [ + 'kube-system' + 'gatekeeper-system' + 'azure-arc' + ] + } + allowedHostPaths: { + value: { + paths: [] // None (blocks all host paths) + } + } + effect: { + value: 'Audit' // TODO: Move to Deny + } + } + } +} + +// Built-in policy: Kubernetes cluster services should only use allowed external IPs' policy at the resource group level. +var pdAllowedExternalIPsId = tenantResourceId('Microsoft.Authorization/policyDefinitions', 'd46c275d-1680-448d-b2ec-e495a3b6cc89') +resource paAllowedExternalIPs 'Microsoft.Authorization/policyAssignments@2021-06-01' = { + name: guid(pdAllowedExternalIPsId, resourceGroup().id, clusterName) + location: 'global' + scope: resourceGroup() + properties: { + displayName: take('[${clusterName}] ${reference(pdAllowedExternalIPsId, '2021-06-01').displayName}', 120) + description: reference(pdAllowedExternalIPsId, '2021-06-01').description + policyDefinitionId: pdAllowedExternalIPsId + parameters: { + excludedNamespaces: { + value: [ + 'kube-system' + 'gatekeeper-system' + 'azure-arc' + ] + } + allowedExternalIPs: { + value: [] // None allowed. (Load balancer IP only supported) TODO: that LB IP might show up as this, verify + } + effect: { + value: 'Audit' // TODO: Move to Deny + } + } + } +} + +// Kubernetes clusters should disable automounting API credentials' policy at the resource group level. +var pdDisableAutomountApiCredsId = tenantResourceId('Microsoft.Authorization/policyDefinitions', '423dd1ba-798e-40e4-9c4d-b6902674b423') +resource paDisableAutomountApiCreds 'Microsoft.Authorization/policyAssignments@2021-06-01' = { + name: guid(pdDisableAutomountApiCredsId, resourceGroup().id, clusterName) + location: 'global' + scope: resourceGroup() + properties: { + displayName: take('[${clusterName}] ${reference(pdDisableAutomountApiCredsId, '2021-06-01').displayName}', 120) + description: reference(pdDisableAutomountApiCredsId, '2021-06-01').description + policyDefinitionId: pdDisableAutomountApiCredsId + parameters: { + excludedNamespaces: { + value: [ + 'kube-system' + 'gatekeeper-system' + 'azure-arc' + ] } + effect: { + value: 'Audit' // TODO: Move to Deny + } + } + } +} + +// Kubernetes clusters should not allow endpoint edit permissions of ClusterRole/system:aggregate-to-edit' policy at the resource group level. +// See: CVE-2021-25740 & https://github.com/kubernetes/kubernetes/issues/103675 +var pdDisallowEndpointEditPermissionsId = tenantResourceId('Microsoft.Authorization/policyDefinitions', '1ddac26b-ed48-4c30-8cc5-3a68c79b8001') +resource paDisallowEndpointEditPermissions 'Microsoft.Authorization/policyAssignments@2021-06-01' = { + name: guid(pdDisallowEndpointEditPermissionsId, resourceGroup().id, clusterName) + location: 'global' + scope: resourceGroup() + properties: { + displayName: take('[${clusterName}] ${reference(pdDisallowEndpointEditPermissionsId, '2021-06-01').displayName}', 120) + description: reference(pdDisallowEndpointEditPermissionsId, '2021-06-01').description + policyDefinitionId: pdDisallowEndpointEditPermissionsId + parameters: { excludedNamespaces: { value: [ 'kube-system' @@ -1098,7 +1171,76 @@ resource paEnforceImageSource 'Microsoft.Authorization/policyAssignments@2021-06 ] } effect: { - value: 'deny' + value: 'Audit' // As of 1.0.1, there is no Deny. + } + } + } +} + +// Kubernetes clusters should not use the default namespace' policy at the resource group level. +var pdDisallowNamespaceUsageId = tenantResourceId('Microsoft.Authorization/policyDefinitions', '9f061a12-e40d-4183-a00e-171812443373') +resource paDisallowNamespaceUsage 'Microsoft.Authorization/policyAssignments@2021-06-01' = { + name: guid(pdDisallowNamespaceUsageId, resourceGroup().id, clusterName) + location: 'global' + scope: resourceGroup() + properties: { + displayName: take('[${clusterName}] ${reference(pdDisallowNamespaceUsageId, '2021-06-01').displayName}', 120) + description: reference(pdDisallowNamespaceUsageId, '2021-06-01').description + policyDefinitionId: pdDisallowNamespaceUsageId + parameters: { + excludedNamespaces: { + value: [ + 'kube-system' + 'gatekeeper-system' + 'azure-arc' + ] + } + namespaces: { + value: [ + 'default' // List namespaces you'd like to disallow the usage of (typically 'default') + ] + } + effect: { + value: 'Audit' // Consider moving to Deny, this walkthrough does temporarly deploy a curl image in default, so leaving as Audit + } + } + } +} + +// Built-in policy: AKS clusters should gate deployment of vulnerable images' policy at the resource group level. +// This requires that ALL container images have been scanned by Defender for Cloud. +var pdNoVulnerableImagesInClusterId = tenantResourceId('Microsoft.Authorization/policyDefinitions', '13cd7ae3-5bc0-4ac4-a62d-4f7c120b9759') +resource paNoVulnerableImagesInCluster 'Microsoft.Authorization/policyAssignments@2021-06-01' = { + name: guid(pdNoVulnerableImagesInClusterId, resourceGroup().id, clusterName) + location: 'global' + scope: resourceGroup() + properties: { + displayName: take('[${clusterName}] ${reference(pdNoVulnerableImagesInClusterId, '2021-06-01').displayName}', 120) + description: reference(pdNoVulnerableImagesInClusterId, '2021-06-01').description + policyDefinitionId: pdNoVulnerableImagesInClusterId + parameters: { + excludedNamespaces: { + value: [ + 'kube-system' + 'gatekeeper-system' + 'azure-arc' + ] + } + severityThresholdForExcludingNotPatchableFindings: { + value: 'Medium' // Ignores 'Low' and 'Medium' vulnerabilities without a patch + } + excludeFindingIDs: { + value: [] // Exclude none + } + severity: { + value: { + High: 0 // Allow 0 High + Medium: 1 // Allow 1 Medium + Low: 2 // Allow 2 Low + } + } + effect: { + value: 'Audit' // Or Deny to completely block non-compliant images, but all images must have been scanned by Defender for Cloud } } } @@ -1671,11 +1813,20 @@ resource mc 'Microsoft.ContainerService/managedClusters@2022-01-02-preview' = { // but logically they need to be in place before workloads are, so focing that here. This also // ensures that the policies are applied to the cluster at bootstrapping time. paAKSLinuxRestrictive + paAadIntegrationEnabled + paAllowedExternalIPs + paAllowedHostPaths + paAuthorizedIpRangesDefined + paAzurePolicyEnabled + paDisableAutomountApiCreds + paDisallowEndpointEditPermissions + paDisallowNamespaceUsage paEnforceHttpsIngress + paEnforceImageSource paEnforceInternalLoadBalancers paEnforceResourceLimits + paNoVulnerableImagesInCluster paRoRootFilesystem - paEnforceImageSource // Azure Resource Provider policies that we'd like to see in place before the cluster is deployed // They are not technically a dependency, but logically they would have existed on the resource group From d413893d966a6768dd62b742d2dc22614b8115dc Mon Sep 17 00:00:00 2001 From: Chad Kittel Date: Mon, 23 May 2022 17:15:40 +0000 Subject: [PATCH 06/16] Remove unused policy in ACR --- acr-stamp.bicep | 34 ++++------------------------------ 1 file changed, 4 insertions(+), 30 deletions(-) diff --git a/acr-stamp.bicep b/acr-stamp.bicep index 98a9451e..86adf6be 100644 --- a/acr-stamp.bicep +++ b/acr-stamp.bicep @@ -72,16 +72,6 @@ resource spokeVirtualNetwork 'Microsoft.Network/virtualNetworks@2021-05-01' exis } } -// Built-in policy: Container registries should have anonymous authentication disabled. -var pdAnonymousContainerRegistryAccessDisallowedId = tenantResourceId('Microsoft.Authorization/policyDefinitions', '9f2dea28-e834-476c-99c5-3507b4728395') - -// Built-in policy: Container registries should have repository scoped access token disabled. -var pdAccessTokenContainerRegistryAccessDisallowedId = tenantResourceId('Microsoft.Authorization/policyDefinitions', 'ff05e24e-195c-447e-b322-5e90c9f9f366') - -// Built-in policy: Container registries should have local admin account disabled. -var pdAdminAccountContainerRegistryAccessDisallowedId = tenantResourceId('Microsoft.Authorization/policyDefinitions', 'dc921057-6b28-4fbe-9b83-f7bec05db6c2') - - /*** RESOURCES ***/ // This Log Analytics workspace will be the log sink for all resources in the cluster resource group. This includes ACR, the AKS cluster, Key Vault, etc. It also is the Container Insights log sink for the AKS cluster. @@ -96,7 +86,8 @@ resource laAks 'Microsoft.OperationalInsights/workspaces@2021-06-01' = { } } -// Ensure the container registry does not allow anon access (Azure RBAC only is allowed) +// Apply the built-in 'Container registries should have anonymous authentication disabled' policy. Azure RBAC only is allowed. +var pdAnonymousContainerRegistryAccessDisallowedId = tenantResourceId('Microsoft.Authorization/policyDefinitions', '9f2dea28-e834-476c-99c5-3507b4728395') resource paAnonymousContainerRegistryAccessDisallowed 'Microsoft.Authorization/policyAssignments@2021-06-01' = { name: guid(resourceGroup().id, pdAnonymousContainerRegistryAccessDisallowedId) location: 'global' @@ -114,25 +105,8 @@ resource paAnonymousContainerRegistryAccessDisallowed 'Microsoft.Authorization/p } } -// Ensure the container registry does not allow sas token access (Azure RBAC only is allowed) -resource paAccessTokenContainerRegistryAccessDisallowed 'Microsoft.Authorization/policyAssignments@2021-06-01' = { - name: guid(resourceGroup().id, pdAccessTokenContainerRegistryAccessDisallowedId) - location: 'global' - scope: resourceGroup() - properties: { - displayName: take('[acraks${subRgUniqueString}] ${reference(pdAccessTokenContainerRegistryAccessDisallowedId, '2021-06-01').displayName}', 120) - description: reference(pdAccessTokenContainerRegistryAccessDisallowedId, '2021-06-01').description - enforcementMode: 'Default' - policyDefinitionId: pdAccessTokenContainerRegistryAccessDisallowedId - parameters: { - effect: { - value: 'Deny' - } - } - } -} - -// Ensure the container registry does not allow admin account access (Azure RBAC only is allowed) +// Apply the built-in 'Container registries should have local admin account disabled' policy. Azure RBAC only is allowed. +var pdAdminAccountContainerRegistryAccessDisallowedId = tenantResourceId('Microsoft.Authorization/policyDefinitions', 'dc921057-6b28-4fbe-9b83-f7bec05db6c2') resource paAdminAccountContainerRegistryAccessDisallowed 'Microsoft.Authorization/policyAssignments@2021-06-01' = { name: guid(resourceGroup().id, pdAdminAccountContainerRegistryAccessDisallowedId) location: 'global' From 819eec2fef7f0b2addea2e28847a96ace01b580a Mon Sep 17 00:00:00 2001 From: Chad Kittel Date: Mon, 23 May 2022 17:17:48 +0000 Subject: [PATCH 07/16] missed depends on --- acr-stamp.bicep | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/acr-stamp.bicep b/acr-stamp.bicep index 86adf6be..dac318b3 100644 --- a/acr-stamp.bicep +++ b/acr-stamp.bicep @@ -147,8 +147,7 @@ resource acrAks 'Microsoft.ContainerRegistry/registries@2021-09-01' = { name: 'acraks${subRgUniqueString}' location: location dependsOn: [ - paAccessTokenContainerRegistryAccessDisallowed // These policy assignments are not true dependencies, but we want them in place before we deploy our ACR instance. - paAdminAccountContainerRegistryAccessDisallowed + paAdminAccountContainerRegistryAccessDisallowed // These policy assignments are not true dependencies, but we want them in place before we deploy our ACR instance. paAnonymousContainerRegistryAccessDisallowed ] sku: { From 50407a5d5473d3c151fc5b320c17c0a24f8154de Mon Sep 17 00:00:00 2001 From: Chad Kittel Date: Mon, 23 May 2022 20:15:53 +0000 Subject: [PATCH 08/16] update k8s policies --- cluster-stamp.bicep | 117 ++++++++++++++++++++++++++++++-------------- 1 file changed, 79 insertions(+), 38 deletions(-) diff --git a/cluster-stamp.bicep b/cluster-stamp.bicep index 3de151da..cd79a31d 100644 --- a/cluster-stamp.bicep +++ b/cluster-stamp.bicep @@ -891,6 +891,7 @@ resource sqrPodFailed 'Microsoft.Insights/scheduledQueryRules@2018-04-16' = { // Resource Group Azure Policy Assignments - Azure Policy for Kubernetes Policies // Applying the built-in 'Kubernetes cluster pod security restricted standards for Linux-based workloads' initiative at the resource group level. +// Constraint Names: K8sAzureAllowedSeccomp, K8sAzureAllowedCapabilities, K8sAzureContainerNoPrivilege, K8sAzureHostNetworkingPorts, K8sAzureVolumeTypes, K8sAzureBlockHostNamespaceV2, K8sAzureAllowedUsersGroups, K8sAzureContainerNoPrivilegeEscalation var psdAKSLinuxRestrictiveId = tenantResourceId('Microsoft.Authorization/policySetDefinitions', '42b8ef37-b724-4e24-bbc8-7a7708edfe00') resource paAKSLinuxRestrictive 'Microsoft.Authorization/policyAssignments@2021-06-01' = { name: guid(psdAKSLinuxRestrictiveId, resourceGroup().id, clusterName) @@ -906,30 +907,39 @@ resource paAKSLinuxRestrictive 'Microsoft.Authorization/policyAssignments@2021-0 'kube-system' 'gatekeeper-system' 'azure-arc' - // 'cluster-baseline-settings' TODO: Test + 'flux-system' + + // Known violations + // K8sAzureAllowedSeccomp + // - Kured, no profile defined + // - AAD Pod Identity (nmi & mic), no profile defined + // K8sAzureAllowedCapabilities + // - AAD Pod Identity (nmi), requests default unsupported DAC_READ_SEARCH, NET_ADMIN + // K8sAzureContainerNoPrivilege + // - Kured, requires privileged to perform reboot + // K8sAzureHostNetworkingPorts + // - AAD Pod Identity (nmi), hostNetwork and hostPort usage + // K8sAzureVolumeTypes + // - AAD Pod Identity (nmi & mic), uses hostPath + // K8sAzureBlockHostNamespaceV2 + // - Kured, shared host namespace + // K8sAzureAllowedUsersGroups + // - Kured, no runAsNonRoot, no runAsGroup, no supplementalGroups, no fsGroup + // - AAD Pod Identity (nmi & mic), runAsUser=0, no runAsGroup, no supplementalGroups, no fsGroup + 'cluster-baseline-settings' + + // Known violations + // K8sAzureAllowedSeccomp + // - Traefik, no profile defined + // - aspnetapp-deployment, no profile defined + // K8sAzureVolumeTypes + // - Traefik, uses csi + // K8sAzureAllowedUsersGroups + // - Traefik, no supplementalGroups, no fsGroup + // = aspnetapp-deployment, no supplementalGroups, no fsGroup + 'a0008' ] } - allowedCapabilities: { - value: [ - 'CHOWN' - 'DAC_OVERRIDE' - 'FSETID' - 'FOWNER' - 'MKNOD' - 'NET_RAW' - 'SETGID' - 'SETUID' - 'SETFCAP' - 'SETPCAP' - 'NET_BIND_SERVICE' - 'SYS_CHROOT' - 'KILL' - 'AUDIT_WRITE' - ] - } - requiredDropCapabilities: { - value: [] // None - } effect: { value: 'Audit' } @@ -939,6 +949,7 @@ resource paAKSLinuxRestrictive 'Microsoft.Authorization/policyAssignments@2021-0 } // Applying the built-in 'Kubernetes clusters should be accessible only over HTTPS' policy at the resource group level. +// Constraint Name: K8sAzureIngressHttpsOnly var pdEnforceHttpsIngressId = tenantResourceId('Microsoft.Authorization/policyDefinitions', '1a5b4dca-0b6f-4cf5-907c-56316bc1bf3d') resource paEnforceHttpsIngress 'Microsoft.Authorization/policyAssignments@2021-06-01' = { name: guid(pdEnforceHttpsIngressId, resourceGroup().id, clusterName) @@ -960,6 +971,7 @@ resource paEnforceHttpsIngress 'Microsoft.Authorization/policyAssignments@2021-0 } // Applying the built-in 'Kubernetes clusters should use internal load balancers' policy at the resource group level. +// Constraint Name: K8sAzureLoadBalancerNoPublicIPs var pdEnforceInternalLoadBalancersId = tenantResourceId('Microsoft.Authorization/policyDefinitions', '3fc4dc25-5baf-40d8-9b05-7fe74c1bc64e') resource paEnforceInternalLoadBalancers 'Microsoft.Authorization/policyAssignments@2021-06-01' = { name: guid(pdEnforceInternalLoadBalancersId, resourceGroup().id, clusterName) @@ -981,6 +993,7 @@ resource paEnforceInternalLoadBalancers 'Microsoft.Authorization/policyAssignmen } // Applying the built-in 'Kubernetes cluster containers should run with a read only root file system' policy at the resource group level. +// Constraint Name: K8sAzureReadOnlyRootFilesystem var pdRoRootFilesystemId = tenantResourceId('Microsoft.Authorization/policyDefinitions', 'df49d893-a74c-421d-bc95-c663042e5b80') resource paRoRootFilesystem 'Microsoft.Authorization/policyAssignments@2021-06-01' = { name: guid(pdRoRootFilesystemId, resourceGroup().id, clusterName) @@ -996,16 +1009,27 @@ resource paRoRootFilesystem 'Microsoft.Authorization/policyAssignments@2021-06-0 'kube-system' 'gatekeeper-system' 'azure-arc' + 'flux-system' + ] + } + excludedContainers: { + value: [ + 'kured' // Kured + 'nmi' // AAD Pod Identity + 'mic' // AAD Pod Identity + 'aspnet-webapp-sample' // ASP.NET Core does not support read-only root ] } effect: { - value: 'Audit' + value: 'Deny' } } } } + // Applying the built-in 'AKS container CPU and memory resource limits should not exceed the specified limits' policy at the resource group level. +// Constraint Name: K8sAzureContainerLimits var pdEnforceResourceLimitsId = tenantResourceId('Microsoft.Authorization/policyDefinitions', 'e345eecc-fa47-480f-9e88-67dcc122b164') resource paEnforceResourceLimits 'Microsoft.Authorization/policyAssignments@2021-06-01' = { name: guid(pdEnforceResourceLimitsId, resourceGroup().id, clusterName) @@ -1017,28 +1041,28 @@ resource paEnforceResourceLimits 'Microsoft.Authorization/policyAssignments@2021 policyDefinitionId: pdEnforceResourceLimitsId parameters: { cpuLimit: { - value: '100m' // Was 1000m TODO + value: '500m' // Kured = 500m, AAD Pod Identity = 200m, traefik-ingress-controller = 200m, aspnet-webapp-sample = 100m } memoryLimit: { - value: '92Mi' // Was 512m TODO + value: '1Gi' // AAD Pod Identity = 1Gi, aspnet-webapp-sample = 256Mi, traefik-ingress-controller = 128Mi, Kured = 48Mi } excludedNamespaces: { value: [ 'kube-system' 'gatekeeper-system' 'azure-arc' - // 'cluster-baseline-settings' -- TODO: Set as excluded images - // 'flux-system' -- TODO: Set as excluded images + 'flux-system' ] } effect: { - value: 'Audit' // TODO: Back to Deny + value: 'Deny' } } } } // Applying the built-in 'AKS containers should only use allowed images' policy at the resource group level. +// Constraint Name: K8sAzureContainerAllowedImages var pdEnforceImageSourceId = tenantResourceId('Microsoft.Authorization/policyDefinitions', 'febd0533-8e55-448f-b837-bd0e06f16469') resource paEnforceImageSource 'Microsoft.Authorization/policyAssignments@2021-06-01' = { name: guid(pdEnforceImageSourceId, resourceGroup().id, clusterName) @@ -1051,14 +1075,13 @@ resource paEnforceImageSource 'Microsoft.Authorization/policyAssignments@2021-06 parameters: { allowedContainerImagesRegex: { // If all images are pull into your ARC instance as described in these instructions you can remove the docker.io entries. - value: '${acr.name}\\.azurecr\\.io/.+$|mcr\\.microsoft\\.com/.+$|azurearcfork8s\\.azurecr\\.io/azurearcflux/images/stable/.+$|docker\\.io/weaveworks/kured.+$|docker\\.io/library/.+$' + value: '${acr.name}\\.azurecr\\.io/.+$|mcr\\.microsoft\\.com/.+$|docker\\.io/weaveworks/kured.+$|docker\\.io/library/.+$' } excludedNamespaces: { value: [ 'kube-system' 'gatekeeper-system' 'azure-arc' - // TODO: Check on azurearcfork8s ] } effect: { @@ -1069,6 +1092,7 @@ resource paEnforceImageSource 'Microsoft.Authorization/policyAssignments@2021-06 } // Built-in policy: Kubernetes cluster pod hostPath volumes should only use allowed host paths' policy at the resource group level. +// Constraint Name: K8sAzureHostFilesystem var pdAllowedHostPathsId = tenantResourceId('Microsoft.Authorization/policyDefinitions', '098fc59e-46c7-4d99-9b16-64990e543d75') resource paAllowedHostPaths 'Microsoft.Authorization/policyAssignments@2021-06-01' = { name: guid(pdAllowedHostPathsId, resourceGroup().id, clusterName) @@ -1084,21 +1108,31 @@ resource paAllowedHostPaths 'Microsoft.Authorization/policyAssignments@2021-06-0 'kube-system' 'gatekeeper-system' 'azure-arc' + 'flux-system' ] } allowedHostPaths: { value: { - paths: [] // None (blocks all host paths) + paths: [] // Setting to empty blocks all host paths } } + // AAD Pod Identity mounts: /run/xtables.lock, /etc/default/kubelet, and /etc/kubernetes/azure.json + // If not using AAD Pod Identity (OSS version), then you can remove this exclusion + excludedContainers: { + value: [ + 'nmi' + 'mic' + ] + } effect: { - value: 'Audit' // TODO: Move to Deny + value: 'Deny' } } } } // Built-in policy: Kubernetes cluster services should only use allowed external IPs' policy at the resource group level. +// Constraint Name: K8sAzureExternalIPs var pdAllowedExternalIPsId = tenantResourceId('Microsoft.Authorization/policyDefinitions', 'd46c275d-1680-448d-b2ec-e495a3b6cc89') resource paAllowedExternalIPs 'Microsoft.Authorization/policyAssignments@2021-06-01' = { name: guid(pdAllowedExternalIPsId, resourceGroup().id, clusterName) @@ -1117,16 +1151,17 @@ resource paAllowedExternalIPs 'Microsoft.Authorization/policyAssignments@2021-06 ] } allowedExternalIPs: { - value: [] // None allowed. (Load balancer IP only supported) TODO: that LB IP might show up as this, verify + value: [] // None allowed. (Load balancer IP only supported) } effect: { - value: 'Audit' // TODO: Move to Deny + value: 'Deny' } } } } // Kubernetes clusters should disable automounting API credentials' policy at the resource group level. +// Constraint Name: K8sAzureBlockAutomountToken var pdDisableAutomountApiCredsId = tenantResourceId('Microsoft.Authorization/policyDefinitions', '423dd1ba-798e-40e4-9c4d-b6902674b423') resource paDisableAutomountApiCreds 'Microsoft.Authorization/policyAssignments@2021-06-01' = { name: guid(pdDisableAutomountApiCredsId, resourceGroup().id, clusterName) @@ -1142,10 +1177,13 @@ resource paDisableAutomountApiCreds 'Microsoft.Authorization/policyAssignments@2 'kube-system' 'gatekeeper-system' 'azure-arc' + 'flux-system' + 'cluster-baseline-settings' // Kured, AAD Pod Identity + 'a0008' // Traefik ] } effect: { - value: 'Audit' // TODO: Move to Deny + value: 'Deny' } } } @@ -1153,6 +1191,7 @@ resource paDisableAutomountApiCreds 'Microsoft.Authorization/policyAssignments@2 // Kubernetes clusters should not allow endpoint edit permissions of ClusterRole/system:aggregate-to-edit' policy at the resource group level. // See: CVE-2021-25740 & https://github.com/kubernetes/kubernetes/issues/103675 +// Constraint Name: K8sAzureBlockEndpointEditDefaultRole var pdDisallowEndpointEditPermissionsId = tenantResourceId('Microsoft.Authorization/policyDefinitions', '1ddac26b-ed48-4c30-8cc5-3a68c79b8001') resource paDisallowEndpointEditPermissions 'Microsoft.Authorization/policyAssignments@2021-06-01' = { name: guid(pdDisallowEndpointEditPermissionsId, resourceGroup().id, clusterName) @@ -1178,6 +1217,7 @@ resource paDisallowEndpointEditPermissions 'Microsoft.Authorization/policyAssign } // Kubernetes clusters should not use the default namespace' policy at the resource group level. +// Constraint Name: K8sAzureBlockDefault var pdDisallowNamespaceUsageId = tenantResourceId('Microsoft.Authorization/policyDefinitions', '9f061a12-e40d-4183-a00e-171812443373') resource paDisallowNamespaceUsage 'Microsoft.Authorization/policyAssignments@2021-06-01' = { name: guid(pdDisallowNamespaceUsageId, resourceGroup().id, clusterName) @@ -1209,6 +1249,7 @@ resource paDisallowNamespaceUsage 'Microsoft.Authorization/policyAssignments@202 // Built-in policy: AKS clusters should gate deployment of vulnerable images' policy at the resource group level. // This requires that ALL container images have been scanned by Defender for Cloud. +// Constraint Name: K8sAzureDefenderBlockVulnerableImages var pdNoVulnerableImagesInClusterId = tenantResourceId('Microsoft.Authorization/policyDefinitions', '13cd7ae3-5bc0-4ac4-a62d-4f7c120b9759') resource paNoVulnerableImagesInCluster 'Microsoft.Authorization/policyAssignments@2021-06-01' = { name: guid(pdNoVulnerableImagesInClusterId, resourceGroup().id, clusterName) @@ -1224,10 +1265,11 @@ resource paNoVulnerableImagesInCluster 'Microsoft.Authorization/policyAssignment 'kube-system' 'gatekeeper-system' 'azure-arc' + 'flux-system' ] } severityThresholdForExcludingNotPatchableFindings: { - value: 'Medium' // Ignores 'Low' and 'Medium' vulnerabilities without a patch + value: 'Medium' // Ignores 'Low' and 'Medium' vulnerabilities without a patch. Use 'None' to not allow exclusions for unpatchable findings. } excludeFindingIDs: { value: [] // Exclude none @@ -1635,7 +1677,7 @@ resource pdzAksIngress 'Microsoft.Network/privateDnsZones@2020-06-01' = { } } -resource mc 'Microsoft.ContainerService/managedClusters@2022-01-02-preview' = { +resource mc 'Microsoft.ContainerService/managedClusters@2022-03-02-preview' = { name: clusterName location: location tags: { @@ -1736,7 +1778,6 @@ resource mc 'Microsoft.ContainerService/managedClusters@2022-01-02-preview' = { nodeResourceGroup: nodeResourceGroup.name enableRBAC: true enablePodSecurityPolicy: false - maxAgentPools: 2 networkProfile: { networkPlugin: 'azure' networkPolicy: 'azure' From 19f23a5787a7b44230af0fb4103a85a43638c977 Mon Sep 17 00:00:00 2001 From: Chad Kittel Date: Mon, 23 May 2022 20:53:50 +0000 Subject: [PATCH 09/16] silence last warning --- cluster-stamp.bicep | 2 ++ 1 file changed, 2 insertions(+) diff --git a/cluster-stamp.bicep b/cluster-stamp.bicep index cd79a31d..5a77cd56 100644 --- a/cluster-stamp.bicep +++ b/cluster-stamp.bicep @@ -203,6 +203,7 @@ resource ssPrometheusAll 'Microsoft.OperationalInsights/workspaces/savedSearches parent: la name: 'AllPrometheus' properties: { +#disable-next-line BCP037 etag: '*' category: 'Prometheus' displayName: 'All collected Prometheus information' @@ -216,6 +217,7 @@ resource ssPrometheusKuredRequestedReeboot 'Microsoft.OperationalInsights/worksp parent: la name: 'NodeRebootRequested' properties: { +#disable-next-line BCP037 etag: '*' category: 'Prometheus' displayName: 'Nodes reboot required by kured' From 5873961254f3e5ddac087ce7225940680b7f2e17 Mon Sep 17 00:00:00 2001 From: Chad Kittel Date: Mon, 23 May 2022 20:59:37 +0000 Subject: [PATCH 10/16] consistent ordering of scope --- cluster-stamp.bicep | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/cluster-stamp.bicep b/cluster-stamp.bicep index 5a77cd56..6ef4f871 100644 --- a/cluster-stamp.bicep +++ b/cluster-stamp.bicep @@ -125,14 +125,14 @@ resource keyVaultSecretsUserRole 'Microsoft.Authorization/roleDefinitions@2018-0 // Azure Container Registry resource acr 'Microsoft.ContainerRegistry/registries@2021-12-01-preview' existing = { - name: 'acraks${subRgUniqueString}' scope: resourceGroup() + name: 'acraks${subRgUniqueString}' } // Log Analytics Workspace resource la 'Microsoft.OperationalInsights/workspaces@2021-12-01-preview' existing = { - name: 'la-${clusterName}' scope: resourceGroup() + name: 'la-${clusterName}' } // Kubernetes namespace: a0008 -- this doesn't technically exist prior to deployment, but is required as a resource reference later in the template @@ -147,8 +147,8 @@ resource nsA0008 'Microsoft.ContainerService/managedClusters/namespaces@2022-01- // Spoke resource group resource targetResourceGroup 'Microsoft.Resources/resourceGroups@2021-04-01' existing = { - name: '${split(targetVnetResourceId,'/')[4]}' scope: subscription() + name: '${split(targetVnetResourceId,'/')[4]}' } // Spoke virtual network From 15eeada44d65701571aa2b490ee0f8ce4fe6f243 Mon Sep 17 00:00:00 2001 From: Chad Kittel Date: Mon, 23 May 2022 21:08:35 +0000 Subject: [PATCH 11/16] removing a policy we added exceptions to every namespace for --- cluster-stamp.bicep | 29 ----------------------------- 1 file changed, 29 deletions(-) diff --git a/cluster-stamp.bicep b/cluster-stamp.bicep index 6ef4f871..91959d89 100644 --- a/cluster-stamp.bicep +++ b/cluster-stamp.bicep @@ -1162,35 +1162,6 @@ resource paAllowedExternalIPs 'Microsoft.Authorization/policyAssignments@2021-06 } } -// Kubernetes clusters should disable automounting API credentials' policy at the resource group level. -// Constraint Name: K8sAzureBlockAutomountToken -var pdDisableAutomountApiCredsId = tenantResourceId('Microsoft.Authorization/policyDefinitions', '423dd1ba-798e-40e4-9c4d-b6902674b423') -resource paDisableAutomountApiCreds 'Microsoft.Authorization/policyAssignments@2021-06-01' = { - name: guid(pdDisableAutomountApiCredsId, resourceGroup().id, clusterName) - location: 'global' - scope: resourceGroup() - properties: { - displayName: take('[${clusterName}] ${reference(pdDisableAutomountApiCredsId, '2021-06-01').displayName}', 120) - description: reference(pdDisableAutomountApiCredsId, '2021-06-01').description - policyDefinitionId: pdDisableAutomountApiCredsId - parameters: { - excludedNamespaces: { - value: [ - 'kube-system' - 'gatekeeper-system' - 'azure-arc' - 'flux-system' - 'cluster-baseline-settings' // Kured, AAD Pod Identity - 'a0008' // Traefik - ] - } - effect: { - value: 'Deny' - } - } - } -} - // Kubernetes clusters should not allow endpoint edit permissions of ClusterRole/system:aggregate-to-edit' policy at the resource group level. // See: CVE-2021-25740 & https://github.com/kubernetes/kubernetes/issues/103675 // Constraint Name: K8sAzureBlockEndpointEditDefaultRole From 41ce65cc1adde26e5432d1beb5c5062575f81436 Mon Sep 17 00:00:00 2001 From: Chad Kittel Date: Mon, 23 May 2022 21:23:59 +0000 Subject: [PATCH 12/16] remove empty dependson --- cluster-stamp.bicep | 26 -------------------------- 1 file changed, 26 deletions(-) diff --git a/cluster-stamp.bicep b/cluster-stamp.bicep index 91959d89..ef7beca1 100644 --- a/cluster-stamp.bicep +++ b/cluster-stamp.bicep @@ -196,7 +196,6 @@ resource alaRgRecommendations 'Microsoft.Insights/activityLogAlerts@2020-10-01' enabled: true description: 'All azure advisor alerts' } - dependsOn: [] } resource ssPrometheusAll 'Microsoft.OperationalInsights/workspaces/savedSearches@2020-08-01' = { @@ -210,7 +209,6 @@ resource ssPrometheusAll 'Microsoft.OperationalInsights/workspaces/savedSearches query: 'InsightsMetrics | where Namespace == "prometheus"' version: 1 } - dependsOn: [] } resource ssPrometheusKuredRequestedReeboot 'Microsoft.OperationalInsights/workspaces/savedSearches@2020-08-01' = { @@ -240,7 +238,6 @@ resource sci 'Microsoft.OperationsManagement/solutions@2015-11-01-preview' = { promotionCode: '' publisher: 'Microsoft' } - dependsOn: [] } resource maHighNodeCPUUtilization 'Microsoft.Insights/metricAlerts@2018-03-01' = { @@ -853,7 +850,6 @@ resource skva 'Microsoft.OperationsManagement/solutions@2015-11-01-preview' = { promotionCode: '' publisher: 'Microsoft' } - dependsOn: [] } resource sqrPodFailed 'Microsoft.Insights/scheduledQueryRules@2018-04-16' = { @@ -887,7 +883,6 @@ resource sqrPodFailed 'Microsoft.Insights/scheduledQueryRules@2018-04-16' = { } } } - dependsOn: [] } // Resource Group Azure Policy Assignments - Azure Policy for Kubernetes Policies @@ -947,7 +942,6 @@ resource paAKSLinuxRestrictive 'Microsoft.Authorization/policyAssignments@2021-0 } } } - dependsOn: [] } // Applying the built-in 'Kubernetes clusters should be accessible only over HTTPS' policy at the resource group level. @@ -1505,7 +1499,6 @@ resource kv_diagnosticSettings 'Microsoft.Insights/diagnosticSettings@2021-05-0 } ] } - dependsOn: [] } // Grant the Azure Application Gateway managed identity with key vault reader role permissions; this allows pulling frontend and backend certificates. @@ -1517,7 +1510,6 @@ resource kvMiAppGatewayFrontendSecretsUserRole_roleAssignment 'Microsoft.Authori principalId: miAppGatewayFrontend.properties.principalId principalType: 'ServicePrincipal' } - dependsOn: [] } // Grant the Azure Application Gateway managed identity with key vault reader role permissions; this allows pulling frontend and backend certificates. @@ -1529,7 +1521,6 @@ resource kvMiAppGatewayFrontendKeyVaultReader_roleAssignment 'Microsoft.Authoriz principalId: miAppGatewayFrontend.properties.principalId principalType: 'ServicePrincipal' } - dependsOn: [] } // Grant the AKS cluster ingress controller pod managed identity with key vault reader role permissions; this allows our ingress controller to pull certificates. @@ -1541,7 +1532,6 @@ resource kvPodMiIngressControllerSecretsUserRole_roleAssignment 'Microsoft.Autho principalId: podmiIngressController.properties.principalId principalType: 'ServicePrincipal' } - dependsOn: [] } // Grant the AKS cluster ingress controller pod managed identity with key vault reader role permissions; this allows our ingress controller to pull certificates @@ -1553,7 +1543,6 @@ resource kvPodMiIngressControllerKeyVaultReader_roleAssignment 'Microsoft.Author principalId: podmiIngressController.properties.principalId principalType: 'ServicePrincipal' } - dependsOn: [] } module ndEnsureClusterIdentityHasRbacToSelfManagedResources 'nested_EnsureClusterIdentityHasRbacToSelfManagedResources.bicep' = { @@ -1582,7 +1571,6 @@ resource pdzKv 'Microsoft.Network/privateDnsZones@2020-06-01' = { registrationEnabled: false } } - dependsOn: [] } resource peKv 'Microsoft.Network/privateEndpoints@2021-05-01' = { @@ -1604,7 +1592,6 @@ resource peKv 'Microsoft.Network/privateEndpoints@2021-05-01' = { } ] } - dependsOn: [] resource pdnszg 'privateDnsZoneGroups' = { name: 'default' @@ -1624,7 +1611,6 @@ resource peKv 'Microsoft.Network/privateEndpoints@2021-05-01' = { resource pdzAksIngress 'Microsoft.Network/privateDnsZones@2020-06-01' = { name: aksIngressDomainName location: 'global' - dependsOn: [] resource aksIngressDomainName_bu0001a0008_00 'A' = { name: 'bu0001a0008-00' @@ -1869,7 +1855,6 @@ resource acrKubeletAcrPullRole_roleAssignment 'Microsoft.Authorization/roleAssig principalId: mc.properties.identityProfile.kubeletidentity.objectId principalType: 'ServicePrincipal' } - dependsOn: [] } // Grant the OMS Agent's Managed Identity the metrics publisher role to push alerts @@ -1881,7 +1866,6 @@ resource mcOmsAgentMonitoringMetricsPublisherRole_roleAssignment 'Microsoft.Auth principalId: mc.properties.addonProfiles.omsagent.identity.objectId principalType: 'ServicePrincipal' } - dependsOn: [] } // Grant the AKS cluster with Managed Identity Operator role permissions over the managed identity used for the ingress controller. Allows it to be assigned to the underlying VMSS. @@ -1893,7 +1877,6 @@ resource miKubeletManagedIdentityOperatorRole_roleAssignment 'Microsoft.Authoriz principalId: mc.properties.identityProfile.kubeletidentity.objectId principalType: 'ServicePrincipal' } - dependsOn: [] } resource mcAadAdminGroupClusterAdminRole_roleAssignment 'Microsoft.Authorization/roleAssignments@2020-10-01-preview' = if (isUsingAzureRBACasKubernetesRBAC) { @@ -1905,7 +1888,6 @@ resource mcAadAdminGroupClusterAdminRole_roleAssignment 'Microsoft.Authorization principalId: clusterAdminAadGroupObjectId principalType: 'Group' } - dependsOn: [] } resource mcAadAdminGroupServiceClusterUserRole_roleAssignment 'Microsoft.Authorization/roleAssignments@2020-10-01-preview' = if (isUsingAzureRBACasKubernetesRBAC) { @@ -1917,7 +1899,6 @@ resource mcAadAdminGroupServiceClusterUserRole_roleAssignment 'Microsoft.Authori principalId: clusterAdminAadGroupObjectId principalType: 'Group' } - dependsOn: [] } resource maAadA0008ReaderGroupClusterReaderRole_roleAssignment 'Microsoft.Authorization/roleAssignments@2020-10-01-preview' = if (isUsingAzureRBACasKubernetesRBAC && !(empty(a0008NamespaceReaderAadGroupObjectId)) && (!(a0008NamespaceReaderAadGroupObjectId == clusterAdminAadGroupObjectId))) { @@ -1929,7 +1910,6 @@ resource maAadA0008ReaderGroupClusterReaderRole_roleAssignment 'Microsoft.Author principalId: a0008NamespaceReaderAadGroupObjectId principalType: 'Group' } - dependsOn: [] } resource maAadA0008ReaderGroupServiceClusterUserRole_roleAssignment 'Microsoft.Authorization/roleAssignments@2020-10-01-preview' = if (isUsingAzureRBACasKubernetesRBAC && !(empty(a0008NamespaceReaderAadGroupObjectId)) && (!(a0008NamespaceReaderAadGroupObjectId == clusterAdminAadGroupObjectId))) { @@ -1941,7 +1921,6 @@ resource maAadA0008ReaderGroupServiceClusterUserRole_roleAssignment 'Microsoft.A principalId: a0008NamespaceReaderAadGroupObjectId principalType: 'Group' } - dependsOn: [] } resource mc_diagnosticSettings 'Microsoft.Insights/diagnosticSettings@2021-05-01-preview' = { @@ -1968,7 +1947,6 @@ resource mc_diagnosticSettings 'Microsoft.Insights/diagnosticSettings@2021-05-0 } ] } - dependsOn: [] } // Ensures that flux add-on (extension) is installed. @@ -2046,7 +2024,6 @@ module ndEnsureClusterUserAssignedHasRbacToManageVMSS 'nested_EnsureClusterUserA params: { kubeletidentityObjectId: mc.properties.identityProfile.kubeletidentity.objectId } - dependsOn: [] } resource st 'Microsoft.EventGrid/systemTopics@2021-12-01' = { @@ -2056,7 +2033,6 @@ resource st 'Microsoft.EventGrid/systemTopics@2021-12-01' = { source: mc.id topicType: 'Microsoft.ContainerService.ManagedClusters' } - dependsOn: [] } resource st_diagnosticSettings 'Microsoft.Insights/diagnosticSettings@2021-05-01-preview' = { @@ -2077,7 +2053,6 @@ resource st_diagnosticSettings 'Microsoft.Insights/diagnosticSettings@2021-05-0 } ] } - dependsOn: [] } resource wafPolicy 'Microsoft.Network/ApplicationGatewayWebApplicationFirewallPolicies@2021-05-01' = { @@ -2288,7 +2263,6 @@ resource agwdiagnosticSettings 'Microsoft.Insights/diagnosticSettings@2021-05-01 } ] } - dependsOn: [] } /*** OUTPUTS ***/ From d77c6b4d0887cb089de9d40141ba7f4855bd9be3 Mon Sep 17 00:00:00 2001 From: Chad Kittel Date: Mon, 23 May 2022 21:25:16 +0000 Subject: [PATCH 13/16] Remove undefined policy --- cluster-stamp.bicep | 1 - 1 file changed, 1 deletion(-) diff --git a/cluster-stamp.bicep b/cluster-stamp.bicep index ef7beca1..2e4bed3f 100644 --- a/cluster-stamp.bicep +++ b/cluster-stamp.bicep @@ -1818,7 +1818,6 @@ resource mc 'Microsoft.ContainerService/managedClusters@2022-03-02-preview' = { paAllowedHostPaths paAuthorizedIpRangesDefined paAzurePolicyEnabled - paDisableAutomountApiCreds paDisallowEndpointEditPermissions paDisallowNamespaceUsage paEnforceHttpsIngress From 3aeef38be6689d4f334212feb91876cf8947c406 Mon Sep 17 00:00:00 2001 From: Chad Kittel Date: Mon, 23 May 2022 21:28:55 +0000 Subject: [PATCH 14/16] Remove the two preview policies for Azure Defender for Kubernetes --- cluster-stamp.bicep | 61 --------------------------------------------- 1 file changed, 61 deletions(-) diff --git a/cluster-stamp.bicep b/cluster-stamp.bicep index 2e4bed3f..6db41260 100644 --- a/cluster-stamp.bicep +++ b/cluster-stamp.bicep @@ -1214,47 +1214,6 @@ resource paDisallowNamespaceUsage 'Microsoft.Authorization/policyAssignments@202 } } -// Built-in policy: AKS clusters should gate deployment of vulnerable images' policy at the resource group level. -// This requires that ALL container images have been scanned by Defender for Cloud. -// Constraint Name: K8sAzureDefenderBlockVulnerableImages -var pdNoVulnerableImagesInClusterId = tenantResourceId('Microsoft.Authorization/policyDefinitions', '13cd7ae3-5bc0-4ac4-a62d-4f7c120b9759') -resource paNoVulnerableImagesInCluster 'Microsoft.Authorization/policyAssignments@2021-06-01' = { - name: guid(pdNoVulnerableImagesInClusterId, resourceGroup().id, clusterName) - location: 'global' - scope: resourceGroup() - properties: { - displayName: take('[${clusterName}] ${reference(pdNoVulnerableImagesInClusterId, '2021-06-01').displayName}', 120) - description: reference(pdNoVulnerableImagesInClusterId, '2021-06-01').description - policyDefinitionId: pdNoVulnerableImagesInClusterId - parameters: { - excludedNamespaces: { - value: [ - 'kube-system' - 'gatekeeper-system' - 'azure-arc' - 'flux-system' - ] - } - severityThresholdForExcludingNotPatchableFindings: { - value: 'Medium' // Ignores 'Low' and 'Medium' vulnerabilities without a patch. Use 'None' to not allow exclusions for unpatchable findings. - } - excludeFindingIDs: { - value: [] // Exclude none - } - severity: { - value: { - High: 0 // Allow 0 High - Medium: 1 // Allow 1 Medium - Low: 2 // Allow 2 Low - } - } - effect: { - value: 'Audit' // Or Deny to completely block non-compliant images, but all images must have been scanned by Defender for Cloud - } - } - } -} - // Resource Group Azure Policy Assignments - Resource Provider Policies // Applying the built-in 'Azure Kubernetes Service clusters should have Defender profile enabled' policy at the resource group level. @@ -1401,24 +1360,6 @@ resource paManagedIdentitiesEnabled 'Microsoft.Authorization/policyAssignments@2 } } -// Applying the built-in 'Running container images should have vulnerability findings resolved' policy at the resource group level. -var pdVulnerableImagesResolvedId = tenantResourceId('Microsoft.Authorization/policyDefinitions', '0fc39691-5a3f-4e3e-94ee-2e6447309ad9') -resource paVulnerableImagesResolved 'Microsoft.Authorization/policyAssignments@2021-06-01' = { - name: guid(pdVulnerableImagesResolvedId, resourceGroup().id, clusterName) - location: 'global' - scope: resourceGroup() - properties: { - displayName: take('[${clusterName}] ${reference(pdVulnerableImagesResolvedId, '2021-06-01').displayName}', 120) - description: reference(pdVulnerableImagesResolvedId, '2021-06-01').description - policyDefinitionId: pdVulnerableImagesResolvedId - parameters: { - effect: { - value: 'AuditIfNotExists' - } - } - } -} - // The control plane identity used by the cluster. Used for networking access (VNET joining and DNS updating) resource miClusterControlPlane 'Microsoft.ManagedIdentity/userAssignedIdentities@2018-11-30' = { name: 'mi-${clusterName}-controlplane' @@ -1824,7 +1765,6 @@ resource mc 'Microsoft.ContainerService/managedClusters@2022-03-02-preview' = { paEnforceImageSource paEnforceInternalLoadBalancers paEnforceResourceLimits - paNoVulnerableImagesInCluster paRoRootFilesystem // Azure Resource Provider policies that we'd like to see in place before the cluster is deployed @@ -1837,7 +1777,6 @@ resource mc 'Microsoft.ContainerService/managedClusters@2022-03-02-preview' = { paOldKuberentesDisabled paRbacEnabled paManagedIdentitiesEnabled - paVulnerableImagesResolved peKv kvPodMiIngressControllerKeyVaultReader_roleAssignment From 8e7a4705e063e0ef9bfc177d7c60e55c83b43c52 Mon Sep 17 00:00:00 2001 From: Chad Kittel Date: Mon, 23 May 2022 21:33:16 +0000 Subject: [PATCH 15/16] remove extra comments --- cluster-stamp.bicep | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/cluster-stamp.bicep b/cluster-stamp.bicep index 6db41260..6af0ee5d 100644 --- a/cluster-stamp.bicep +++ b/cluster-stamp.bicep @@ -957,7 +957,7 @@ resource paEnforceHttpsIngress 'Microsoft.Authorization/policyAssignments@2021-0 policyDefinitionId: pdEnforceHttpsIngressId parameters: { excludedNamespaces: { - value: [] // None + value: [] } effect: { value: 'Deny' @@ -979,7 +979,7 @@ resource paEnforceInternalLoadBalancers 'Microsoft.Authorization/policyAssignmen policyDefinitionId: pdEnforceInternalLoadBalancersId parameters: { excludedNamespaces: { - value: [] // None + value: [] } effect: { value: 'Deny' @@ -1147,7 +1147,7 @@ resource paAllowedExternalIPs 'Microsoft.Authorization/policyAssignments@2021-06 ] } allowedExternalIPs: { - value: [] // None allowed. (Load balancer IP only supported) + value: [] // None allowed, internal load balancer IP only supported. } effect: { value: 'Deny' From dc5e3d530ded4ed2ef32f9e56be2a87cdaa62238 Mon Sep 17 00:00:00 2001 From: Chad Kittel Date: Tue, 24 May 2022 14:57:01 +0000 Subject: [PATCH 16/16] Revert inlining policy definition id references --- cluster-stamp.bicep | 83 ++++++++++++++++++++++++++++++++------------- 1 file changed, 60 insertions(+), 23 deletions(-) diff --git a/cluster-stamp.bicep b/cluster-stamp.bicep index 6af0ee5d..eebcf5cb 100644 --- a/cluster-stamp.bicep +++ b/cluster-stamp.bicep @@ -66,6 +66,62 @@ var aksIngressDomainName = 'aks-ingress.${domainName}' var aksBackendDomainName = 'bu0001a0008-00.${aksIngressDomainName}' var isUsingAzureRBACasKubernetesRBAC = (subscription().tenantId == k8sControlPlaneAuthorizationTenantId) +/*** EXISTING TENANT RESOURCES ***/ + +// Built-in 'Kubernetes cluster pod security restricted standards for Linux-based workloads' Azure Policy for Kubernetes initiative definition +var psdAKSLinuxRestrictiveId = tenantResourceId('Microsoft.Authorization/policySetDefinitions', '42b8ef37-b724-4e24-bbc8-7a7708edfe00') + +// Built-in 'Kubernetes clusters should be accessible only over HTTPS' Azure Policy for Kubernetes policy definition +var pdEnforceHttpsIngressId = tenantResourceId('Microsoft.Authorization/policyDefinitions', '1a5b4dca-0b6f-4cf5-907c-56316bc1bf3d') + +// Built-in 'Kubernetes clusters should use internal load balancers' Azure Policy for Kubernetes policy definition +var pdEnforceInternalLoadBalancersId = tenantResourceId('Microsoft.Authorization/policyDefinitions', '3fc4dc25-5baf-40d8-9b05-7fe74c1bc64e') + +// Built-in 'Kubernetes cluster containers should run with a read only root file system' Azure Policy for Kubernetes policy definition +var pdRoRootFilesystemId = tenantResourceId('Microsoft.Authorization/policyDefinitions', 'df49d893-a74c-421d-bc95-c663042e5b80') + +// Built-in 'AKS container CPU and memory resource limits should not exceed the specified limits' Azure Policy for Kubernetes policy definition +var pdEnforceResourceLimitsId = tenantResourceId('Microsoft.Authorization/policyDefinitions', 'e345eecc-fa47-480f-9e88-67dcc122b164') + +// Built-in 'AKS containers should only use allowed images' Azure Policy for Kubernetes policy definition +var pdEnforceImageSourceId = tenantResourceId('Microsoft.Authorization/policyDefinitions', 'febd0533-8e55-448f-b837-bd0e06f16469') + +// Built-in 'Kubernetes cluster pod hostPath volumes should only use allowed host paths' Azure Policy for Kubernetes policy definition +var pdAllowedHostPathsId = tenantResourceId('Microsoft.Authorization/policyDefinitions', '098fc59e-46c7-4d99-9b16-64990e543d75') + +// Built-in 'Kubernetes cluster services should only use allowed external IPs' Azure Policy for Kubernetes policy definition +var pdAllowedExternalIPsId = tenantResourceId('Microsoft.Authorization/policyDefinitions', 'd46c275d-1680-448d-b2ec-e495a3b6cc89') + +// Built-in 'Kubernetes clusters should not allow endpoint edit permissions of ClusterRole/system:aggregate-to-edit' Azure Policy for Kubernetes policy definition +var pdDisallowEndpointEditPermissionsId = tenantResourceId('Microsoft.Authorization/policyDefinitions', '1ddac26b-ed48-4c30-8cc5-3a68c79b8001') + +// Built-in 'Kubernetes clusters should not use the default namespace' Azure Policy for Kubernetes policy definition +var pdDisallowNamespaceUsageId = tenantResourceId('Microsoft.Authorization/policyDefinitions', '9f061a12-e40d-4183-a00e-171812443373') + +// Built-in 'Azure Kubernetes Service clusters should have Defender profile enabled' Azure Policy policy definition +var pdDefenderInClusterEnabledId = tenantResourceId('Microsoft.Authorization/policyDefinitions', 'a1840de2-8088-4ea8-b153-b4c723e9cb01') + +// Built-in 'Azure Kubernetes Service Clusters should enable Azure Active Directory integration' Azure Policy policy definition +var pdAadIntegrationEnabledId = tenantResourceId('Microsoft.Authorization/policyDefinitions', '450d2877-ebea-41e8-b00c-e286317d21bf') + +// Built-in 'Azure Kubernetes Service Clusters should have local authentication methods disabled' Azure Policy policy definition +var pdLocalAuthDisabledId = tenantResourceId('Microsoft.Authorization/policyDefinitions', '993c2fcd-2b29-49d2-9eb0-df2c3a730c32') + +// Built-in 'Azure Policy Add-on for Kubernetes service (AKS) should be installed and enabled on your clusters' Azure Policy policy definition +var pdAzurePolicyEnabledId = tenantResourceId('Microsoft.Authorization/policyDefinitions', '0a15ec92-a229-4763-bb14-0ea34a568f8d') + +// Built-in 'Authorized IP ranges should be defined on Kubernetes Services' Azure Policy policy definition +var pdAuthorizedIpRangesDefinedId = tenantResourceId('Microsoft.Authorization/policyDefinitions', '0e246bcf-5f6f-4f87-bc6f-775d4712c7ea') + +// Built-in 'Kubernetes Services should be upgraded to a non-vulnerable Kubernetes version' Azure Policy policy definition +var pdOldKuberentesDisabledId = tenantResourceId('Microsoft.Authorization/policyDefinitions', 'fb893a29-21bb-418c-a157-e99480ec364c') + +// Built-in 'Role-Based Access Control (RBAC) should be used on Kubernetes Services' Azure Policy policy definition +var pdRbacEnabledId = tenantResourceId('Microsoft.Authorization/policyDefinitions', 'ac4a19c2-fa67-49b4-8ae5-0b2e78c49457') + +// Built-in 'Azure Kubernetes Service Clusters should use managed identities' Azure Policy policy definition +var pdManagedIdentitiesEnabledId = tenantResourceId('Microsoft.Authorization/policyDefinitions', 'da6e2401-19da-4532-9141-fb8fbde08431') + /*** EXISTING SUBSCRIPTION RESOURCES ***/ resource nodeResourceGroup 'Microsoft.Resources/resourceGroups@2021-04-01' existing = { @@ -889,7 +945,6 @@ resource sqrPodFailed 'Microsoft.Insights/scheduledQueryRules@2018-04-16' = { // Applying the built-in 'Kubernetes cluster pod security restricted standards for Linux-based workloads' initiative at the resource group level. // Constraint Names: K8sAzureAllowedSeccomp, K8sAzureAllowedCapabilities, K8sAzureContainerNoPrivilege, K8sAzureHostNetworkingPorts, K8sAzureVolumeTypes, K8sAzureBlockHostNamespaceV2, K8sAzureAllowedUsersGroups, K8sAzureContainerNoPrivilegeEscalation -var psdAKSLinuxRestrictiveId = tenantResourceId('Microsoft.Authorization/policySetDefinitions', '42b8ef37-b724-4e24-bbc8-7a7708edfe00') resource paAKSLinuxRestrictive 'Microsoft.Authorization/policyAssignments@2021-06-01' = { name: guid(psdAKSLinuxRestrictiveId, resourceGroup().id, clusterName) location: 'global' @@ -946,7 +1001,6 @@ resource paAKSLinuxRestrictive 'Microsoft.Authorization/policyAssignments@2021-0 // Applying the built-in 'Kubernetes clusters should be accessible only over HTTPS' policy at the resource group level. // Constraint Name: K8sAzureIngressHttpsOnly -var pdEnforceHttpsIngressId = tenantResourceId('Microsoft.Authorization/policyDefinitions', '1a5b4dca-0b6f-4cf5-907c-56316bc1bf3d') resource paEnforceHttpsIngress 'Microsoft.Authorization/policyAssignments@2021-06-01' = { name: guid(pdEnforceHttpsIngressId, resourceGroup().id, clusterName) location: 'global' @@ -968,7 +1022,6 @@ resource paEnforceHttpsIngress 'Microsoft.Authorization/policyAssignments@2021-0 // Applying the built-in 'Kubernetes clusters should use internal load balancers' policy at the resource group level. // Constraint Name: K8sAzureLoadBalancerNoPublicIPs -var pdEnforceInternalLoadBalancersId = tenantResourceId('Microsoft.Authorization/policyDefinitions', '3fc4dc25-5baf-40d8-9b05-7fe74c1bc64e') resource paEnforceInternalLoadBalancers 'Microsoft.Authorization/policyAssignments@2021-06-01' = { name: guid(pdEnforceInternalLoadBalancersId, resourceGroup().id, clusterName) location: 'global' @@ -990,7 +1043,6 @@ resource paEnforceInternalLoadBalancers 'Microsoft.Authorization/policyAssignmen // Applying the built-in 'Kubernetes cluster containers should run with a read only root file system' policy at the resource group level. // Constraint Name: K8sAzureReadOnlyRootFilesystem -var pdRoRootFilesystemId = tenantResourceId('Microsoft.Authorization/policyDefinitions', 'df49d893-a74c-421d-bc95-c663042e5b80') resource paRoRootFilesystem 'Microsoft.Authorization/policyAssignments@2021-06-01' = { name: guid(pdRoRootFilesystemId, resourceGroup().id, clusterName) location: 'global' @@ -1026,7 +1078,6 @@ resource paRoRootFilesystem 'Microsoft.Authorization/policyAssignments@2021-06-0 // Applying the built-in 'AKS container CPU and memory resource limits should not exceed the specified limits' policy at the resource group level. // Constraint Name: K8sAzureContainerLimits -var pdEnforceResourceLimitsId = tenantResourceId('Microsoft.Authorization/policyDefinitions', 'e345eecc-fa47-480f-9e88-67dcc122b164') resource paEnforceResourceLimits 'Microsoft.Authorization/policyAssignments@2021-06-01' = { name: guid(pdEnforceResourceLimitsId, resourceGroup().id, clusterName) location: 'global' @@ -1059,7 +1110,6 @@ resource paEnforceResourceLimits 'Microsoft.Authorization/policyAssignments@2021 // Applying the built-in 'AKS containers should only use allowed images' policy at the resource group level. // Constraint Name: K8sAzureContainerAllowedImages -var pdEnforceImageSourceId = tenantResourceId('Microsoft.Authorization/policyDefinitions', 'febd0533-8e55-448f-b837-bd0e06f16469') resource paEnforceImageSource 'Microsoft.Authorization/policyAssignments@2021-06-01' = { name: guid(pdEnforceImageSourceId, resourceGroup().id, clusterName) location: 'global' @@ -1087,9 +1137,7 @@ resource paEnforceImageSource 'Microsoft.Authorization/policyAssignments@2021-06 } } -// Built-in policy: Kubernetes cluster pod hostPath volumes should only use allowed host paths' policy at the resource group level. -// Constraint Name: K8sAzureHostFilesystem -var pdAllowedHostPathsId = tenantResourceId('Microsoft.Authorization/policyDefinitions', '098fc59e-46c7-4d99-9b16-64990e543d75') +// Applying the built-in 'Kubernetes cluster pod hostPath volumes should only use allowed host paths' policy at the resource group level. resource paAllowedHostPaths 'Microsoft.Authorization/policyAssignments@2021-06-01' = { name: guid(pdAllowedHostPathsId, resourceGroup().id, clusterName) location: 'global' @@ -1127,9 +1175,8 @@ resource paAllowedHostPaths 'Microsoft.Authorization/policyAssignments@2021-06-0 } } -// Built-in policy: Kubernetes cluster services should only use allowed external IPs' policy at the resource group level. +// Applying the built-in 'Kubernetes cluster services should only use allowed external IPs' policy at the resource group level. // Constraint Name: K8sAzureExternalIPs -var pdAllowedExternalIPsId = tenantResourceId('Microsoft.Authorization/policyDefinitions', 'd46c275d-1680-448d-b2ec-e495a3b6cc89') resource paAllowedExternalIPs 'Microsoft.Authorization/policyAssignments@2021-06-01' = { name: guid(pdAllowedExternalIPsId, resourceGroup().id, clusterName) location: 'global' @@ -1156,10 +1203,9 @@ resource paAllowedExternalIPs 'Microsoft.Authorization/policyAssignments@2021-06 } } -// Kubernetes clusters should not allow endpoint edit permissions of ClusterRole/system:aggregate-to-edit' policy at the resource group level. +// Applying the built-in 'Kubernetes clusters should not allow endpoint edit permissions of ClusterRole/system:aggregate-to-edit' policy at the resource group level. // See: CVE-2021-25740 & https://github.com/kubernetes/kubernetes/issues/103675 // Constraint Name: K8sAzureBlockEndpointEditDefaultRole -var pdDisallowEndpointEditPermissionsId = tenantResourceId('Microsoft.Authorization/policyDefinitions', '1ddac26b-ed48-4c30-8cc5-3a68c79b8001') resource paDisallowEndpointEditPermissions 'Microsoft.Authorization/policyAssignments@2021-06-01' = { name: guid(pdDisallowEndpointEditPermissionsId, resourceGroup().id, clusterName) location: 'global' @@ -1183,9 +1229,8 @@ resource paDisallowEndpointEditPermissions 'Microsoft.Authorization/policyAssign } } -// Kubernetes clusters should not use the default namespace' policy at the resource group level. +// Applying the built-in 'Kubernetes clusters should not use the default namespace' policy at the resource group level. // Constraint Name: K8sAzureBlockDefault -var pdDisallowNamespaceUsageId = tenantResourceId('Microsoft.Authorization/policyDefinitions', '9f061a12-e40d-4183-a00e-171812443373') resource paDisallowNamespaceUsage 'Microsoft.Authorization/policyAssignments@2021-06-01' = { name: guid(pdDisallowNamespaceUsageId, resourceGroup().id, clusterName) location: 'global' @@ -1217,7 +1262,6 @@ resource paDisallowNamespaceUsage 'Microsoft.Authorization/policyAssignments@202 // Resource Group Azure Policy Assignments - Resource Provider Policies // Applying the built-in 'Azure Kubernetes Service clusters should have Defender profile enabled' policy at the resource group level. -var pdDefenderInClusterEnabledId = tenantResourceId('Microsoft.Authorization/policyDefinitions', 'a1840de2-8088-4ea8-b153-b4c723e9cb01') resource paDefenderInClusterEnabled 'Microsoft.Authorization/policyAssignments@2021-06-01' = { name: guid(pdDefenderInClusterEnabledId, resourceGroup().id, clusterName) location: 'global' @@ -1235,7 +1279,6 @@ resource paDefenderInClusterEnabled 'Microsoft.Authorization/policyAssignments@2 } // Applying the built-in 'Azure Kubernetes Service Clusters should enable Azure Active Directory integration' policy at the resource group level. -var pdAadIntegrationEnabledId = tenantResourceId('Microsoft.Authorization/policyDefinitions', '450d2877-ebea-41e8-b00c-e286317d21bf') resource paAadIntegrationEnabled 'Microsoft.Authorization/policyAssignments@2021-06-01' = { name: guid(pdAadIntegrationEnabledId, resourceGroup().id, clusterName) location: 'global' @@ -1253,7 +1296,6 @@ resource paAadIntegrationEnabled 'Microsoft.Authorization/policyAssignments@2021 } // Applying the built-in 'Azure Kubernetes Service Clusters should have local authentication methods disabled' policy at the resource group level. -var pdLocalAuthDisabledId = tenantResourceId('Microsoft.Authorization/policyDefinitions', '993c2fcd-2b29-49d2-9eb0-df2c3a730c32') resource paLocalAuthDisabled 'Microsoft.Authorization/policyAssignments@2021-06-01' = { name: guid(pdLocalAuthDisabledId, resourceGroup().id, clusterName) location: 'global' @@ -1271,7 +1313,6 @@ resource paLocalAuthDisabled 'Microsoft.Authorization/policyAssignments@2021-06- } // Applying the built-in 'Azure Policy Add-on for Kubernetes service (AKS) should be installed and enabled on your clusters' policy at the resource group level. -var pdAzurePolicyEnabledId = tenantResourceId('Microsoft.Authorization/policyDefinitions', '0a15ec92-a229-4763-bb14-0ea34a568f8d') resource paAzurePolicyEnabled 'Microsoft.Authorization/policyAssignments@2021-06-01' = { name: guid(pdAzurePolicyEnabledId, resourceGroup().id, clusterName) location: 'global' @@ -1289,7 +1330,6 @@ resource paAzurePolicyEnabled 'Microsoft.Authorization/policyAssignments@2021-06 } // Applying the built-in 'Authorized IP ranges should be defined on Kubernetes Services' policy at the resource group level. -var pdAuthorizedIpRangesDefinedId = tenantResourceId('Microsoft.Authorization/policyDefinitions', '0e246bcf-5f6f-4f87-bc6f-775d4712c7ea') resource paAuthorizedIpRangesDefined 'Microsoft.Authorization/policyAssignments@2021-06-01' = { name: guid(pdAuthorizedIpRangesDefinedId, resourceGroup().id, clusterName) location: 'global' @@ -1307,7 +1347,6 @@ resource paAuthorizedIpRangesDefined 'Microsoft.Authorization/policyAssignments@ } // Applying the built-in 'Kubernetes Services should be upgraded to a non-vulnerable Kubernetes version' policy at the resource group level. -var pdOldKuberentesDisabledId = tenantResourceId('Microsoft.Authorization/policyDefinitions', 'fb893a29-21bb-418c-a157-e99480ec364c') resource paOldKuberentesDisabled 'Microsoft.Authorization/policyAssignments@2021-06-01' = { name: guid(pdOldKuberentesDisabledId, resourceGroup().id, clusterName) location: 'global' @@ -1325,7 +1364,6 @@ resource paOldKuberentesDisabled 'Microsoft.Authorization/policyAssignments@2021 } // Applying the built-in 'Role-Based Access Control (RBAC) should be used on Kubernetes Services' policy at the resource group level. -var pdRbacEnabledId = tenantResourceId('Microsoft.Authorization/policyDefinitions', 'ac4a19c2-fa67-49b4-8ae5-0b2e78c49457') resource paRbacEnabled 'Microsoft.Authorization/policyAssignments@2021-06-01' = { name: guid(pdRbacEnabledId, resourceGroup().id, clusterName) location: 'global' @@ -1343,7 +1381,6 @@ resource paRbacEnabled 'Microsoft.Authorization/policyAssignments@2021-06-01' = } // Applying the built-in 'Azure Kubernetes Service Clusters should use managed identities' policy at the resource group level. -var pdManagedIdentitiesEnabledId = tenantResourceId('Microsoft.Authorization/policyDefinitions', 'da6e2401-19da-4532-9141-fb8fbde08431') resource paManagedIdentitiesEnabled 'Microsoft.Authorization/policyAssignments@2021-06-01' = { name: guid(pdManagedIdentitiesEnabledId, resourceGroup().id, clusterName) location: 'global'