From a07cb1d463ac1d6847e4e56d0e986212f3097aea Mon Sep 17 00:00:00 2001 From: Fernando Antivero Date: Tue, 4 Oct 2022 19:10:05 +0000 Subject: [PATCH 001/115] add params --- cluster-stamp.bicep | 77 +++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 77 insertions(+) create mode 100644 cluster-stamp.bicep diff --git a/cluster-stamp.bicep b/cluster-stamp.bicep new file mode 100644 index 00000000..3e877850 --- /dev/null +++ b/cluster-stamp.bicep @@ -0,0 +1,77 @@ +targetScope = 'resourceGroup' + +/*** PARAMETERS ***/ + +@description('The regional network spoke VNet Resource ID that the cluster will be joined to') +@minLength(79) +param targetVnetResourceId string + +@description('Azure AD Group in the identified tenant that will be granted the highly privileged cluster-admin role.') +param clusterAdminAadGroupObjectId string + +@description('Your AKS control plane Cluster API authentication tenant') +param k8sControlPlaneAuthorizationTenantId string + +@description('The certificate data for app gateway TLS termination. It is base64') +param appGatewayListenerCertificate string + +@description('The base 64 encoded AKS Ingress Controller public certificate (as .crt or .cer) to be stored in Azure Key Vault as secret and referenced by Azure Application Gateway as a trusted root certificate.') +param aksIngressControllerCertificate string + +@allowed([ + 'australiaeast' + 'canadacentral' + 'centralus' + 'eastus' + 'eastus2' + 'westus2' + 'francecentral' + 'germanywestcentral' + 'northeurope' + 'southafricanorth' + 'southcentralus' + 'uksouth' + 'westeurope' + 'japaneast' + 'southeastasia' +]) +@description('AKS Service, Node Pools, and supporting services (KeyVault, App Gateway, etc) region. This needs to be the same region as the vnet provided in these parameters.') +@minLength(4) +param location string + +@allowed([ + 'australiasoutheast' + 'canadaeast' + 'eastus2' + 'westus' + 'centralus' + 'westcentralus' + 'francesouth' + 'germanynorth' + 'westeurope' + 'ukwest' + 'northeurope' + 'japanwest' + 'southafricawest' + 'northcentralus' + 'eastasia' + 'eastus' + 'westus2' + 'francecentral' + 'uksouth' + 'japaneast' + 'southeastasia' +]) +@description('For Azure resources that support native geo-redunancy, provide the location the redundant service will have its secondary. Should be different than the location parameter and ideally should be a paired region - https://learn.microsoft.com/azure/best-practices-availability-paired-regions. This region does not need to support availability zones.') +@minLength(4) +param geoRedundancyLocation string + +@description('The Azure resource ID of a VM image that will be used for the jump box.') +@minLength(70) +param jumpBoxImageResourceId string + +@description('A cloud init file (starting with #cloud-config) as a base 64 encoded string used to perform image customization on the jump box VMs. Used for user-management in this context.') +@minLength(100) +param jumpBoxCloudInitAsBase64 string = '10.200.0.0/26' + +/*** RESOURCES ***/ From c78f5e00abe65bffdcb034755b775a40982824b8 Mon Sep 17 00:00:00 2001 From: Fernando Antivero Date: Tue, 4 Oct 2022 19:16:11 +0000 Subject: [PATCH 002/115] add cluster control plane user assigned identity --- cluster-stamp.bicep | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/cluster-stamp.bicep b/cluster-stamp.bicep index 3e877850..4fd6b39d 100644 --- a/cluster-stamp.bicep +++ b/cluster-stamp.bicep @@ -74,4 +74,14 @@ param jumpBoxImageResourceId string @minLength(100) param jumpBoxCloudInitAsBase64 string = '10.200.0.0/26' +/*** VARIABLES ***/ + +var subRgUniqueString = uniqueString('aks', subscription().subscriptionId, resourceGroup().id) +var clusterName = 'aks-${subRgUniqueString}' + /*** RESOURCES ***/ + +resource miClusterControlPlane 'Microsoft.ManagedIdentity/userAssignedIdentities@2022-01-31-preview' = { + name: 'mi-${clusterName}-controlplane' + location: location +} From 0e73b01678ef6533275d7b3294f2d2bae3eea4f5 Mon Sep 17 00:00:00 2001 From: Fernando Antivero Date: Tue, 4 Oct 2022 19:25:14 +0000 Subject: [PATCH 003/115] add cluster ingress controller user assigned identity --- cluster-stamp.bicep | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/cluster-stamp.bicep b/cluster-stamp.bicep index 4fd6b39d..eacfda2c 100644 --- a/cluster-stamp.bicep +++ b/cluster-stamp.bicep @@ -85,3 +85,8 @@ resource miClusterControlPlane 'Microsoft.ManagedIdentity/userAssignedIdentities name: 'mi-${clusterName}-controlplane' location: location } + +resource miIngressController 'Microsoft.ManagedIdentity/userAssignedIdentities@2022-01-31-preview' = { + name: 'mi-ingresscontroller' + location: location +} From 3f60cf96c9c60cbbede7b6250691346255d31b0a Mon Sep 17 00:00:00 2001 From: Fernando Antivero Date: Tue, 4 Oct 2022 19:48:39 +0000 Subject: [PATCH 004/115] grant cluster control plane with Azure RBAC operator role permissions --- cluster-stamp.bicep | 23 ++++++++++++++++++++++- 1 file changed, 22 insertions(+), 1 deletion(-) diff --git a/cluster-stamp.bicep b/cluster-stamp.bicep index eacfda2c..4a39a7eb 100644 --- a/cluster-stamp.bicep +++ b/cluster-stamp.bicep @@ -79,14 +79,35 @@ param jumpBoxCloudInitAsBase64 string = '10.200.0.0/26' var subRgUniqueString = uniqueString('aks', subscription().subscriptionId, resourceGroup().id) var clusterName = 'aks-${subRgUniqueString}' +/*** EXISTING RESOURCES ***/ + +@description('Built-in Azure RBAC role that must be applied to the kublet Managed Identity allowing it to further assign adding managed identities to the cluster\'s underlying VMSS.') +resource managedIdentityOperatorRole 'Microsoft.Authorization/roleDefinitions@2022-04-01' existing = { + name: 'f1a07417-d97a-45cb-824c-7a7467783830' + scope: subscription() +} + /*** RESOURCES ***/ +@description('The control plane identity used by the cluster.') resource miClusterControlPlane 'Microsoft.ManagedIdentity/userAssignedIdentities@2022-01-31-preview' = { name: 'mi-${clusterName}-controlplane' location: location } +@description('The in-cluster ingress controller identity used by pod identity agent to acquire access tokens to read ssl certs from Azure KeyVault.') resource miIngressController 'Microsoft.ManagedIdentity/userAssignedIdentities@2022-01-31-preview' = { - name: 'mi-ingresscontroller' + name: 'mi-${clusterName}-ingresscontroller' location: location } + +@description('Grant the cluster control plane managed identity with managed identity operator role permissions; this allows to assign compute with the ingress controller managed identity; this is required for Azure Pod Idenity.') +resource icMiClusterControlPlaneManagedIdentityOperatorRole_roleAssignment 'Microsoft.Authorization/roleAssignments@2022-04-01' = { + scope: miIngressController + name: guid(resourceGroup().id, miClusterControlPlane.name, managedIdentityOperatorRole.id) + properties: { + roleDefinitionId: managedIdentityOperatorRole.id + principalId: miClusterControlPlane.properties.principalId + principalType: 'ServicePrincipal' + } +} From e18c446ef1f06b4e850e24810882a64f84cb5ebe Mon Sep 17 00:00:00 2001 From: Fernando Antivero Date: Mon, 10 Oct 2022 17:58:16 +0000 Subject: [PATCH 005/115] add regional load balancer user assigned indentity --- cluster-stamp.bicep | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/cluster-stamp.bicep b/cluster-stamp.bicep index 4a39a7eb..c0befd08 100644 --- a/cluster-stamp.bicep +++ b/cluster-stamp.bicep @@ -101,6 +101,12 @@ resource miIngressController 'Microsoft.ManagedIdentity/userAssignedIdentities@2 location: location } +@description('The regional load balancer identity used by your Application Gateway instance to acquire access tokens to read ssl certs and secrets from Azure KeyVault.') +resource miAppGateway 'Microsoft.ManagedIdentity/userAssignedIdentities@2022-01-31-preview' = { + name: 'mi-appgateway' + location: location +} + @description('Grant the cluster control plane managed identity with managed identity operator role permissions; this allows to assign compute with the ingress controller managed identity; this is required for Azure Pod Idenity.') resource icMiClusterControlPlaneManagedIdentityOperatorRole_roleAssignment 'Microsoft.Authorization/roleAssignments@2022-04-01' = { scope: miIngressController From ba64949663557e4266dc045590471f62c7079ad1 Mon Sep 17 00:00:00 2001 From: Fernando Antivero Date: Mon, 10 Oct 2022 18:00:46 +0000 Subject: [PATCH 006/115] add the cluster kv instance --- cluster-stamp.bicep | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/cluster-stamp.bicep b/cluster-stamp.bicep index c0befd08..756c92db 100644 --- a/cluster-stamp.bicep +++ b/cluster-stamp.bicep @@ -117,3 +117,28 @@ resource icMiClusterControlPlaneManagedIdentityOperatorRole_roleAssignment 'Micr principalType: 'ServicePrincipal' } } + +@description('The secret storage management resource for the aks regulated cluster.') +resource kv 'Microsoft.KeyVault/vaults@2022-07-01' = { + name: 'kv-${clusterName}' + location: location + properties: { + accessPolicies: [ + ] + sku: { + family: 'A' + name: 'standard' + } + tenantId: subscription().tenantId + networkAcls: { + bypass: 'AzureServices' + defaultAction: 'Allow' + ipRules: [] + virtualNetworkRules: [] + } + enabledForDeployment: false + enabledForDiskEncryption: false + enabledForTemplateDeployment: false + enableSoftDelete: true + } +} From f21a8589ee801e4dd0dacffd7d7e9499d744db33 Mon Sep 17 00:00:00 2001 From: Fernando Antivero Date: Mon, 10 Oct 2022 18:06:06 +0000 Subject: [PATCH 007/115] add get secrets and certs access policy for the in-cluster ingress controller identity --- cluster-stamp.bicep | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/cluster-stamp.bicep b/cluster-stamp.bicep index 756c92db..9d042788 100644 --- a/cluster-stamp.bicep +++ b/cluster-stamp.bicep @@ -124,6 +124,19 @@ resource kv 'Microsoft.KeyVault/vaults@2022-07-01' = { location: location properties: { accessPolicies: [ + { + tenantId: miIngressController.properties.tenantId + objectId: miIngressController.properties.principalId + permissions: { + secrets: [ + 'get' + ] + certificates: [ + 'get' + ] + keys: [] + } + } ] sku: { family: 'A' From 191d983dc39e1f000af17892df0a4d2943e87d1b Mon Sep 17 00:00:00 2001 From: Fernando Antivero Date: Mon, 10 Oct 2022 18:06:22 +0000 Subject: [PATCH 008/115] add get secrets and certs access policy for the regional load balancer identity --- cluster-stamp.bicep | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/cluster-stamp.bicep b/cluster-stamp.bicep index 9d042788..ec8cf2b8 100644 --- a/cluster-stamp.bicep +++ b/cluster-stamp.bicep @@ -124,6 +124,19 @@ resource kv 'Microsoft.KeyVault/vaults@2022-07-01' = { location: location properties: { accessPolicies: [ + { + tenantId: miAppGateway.properties.tenantId + objectId: miAppGateway.properties.principalId + permissions: { + secrets: [ + 'get' + ] + certificates: [ + 'get' + ] + keys: [] + } + } { tenantId: miIngressController.properties.tenantId objectId: miIngressController.properties.principalId From 7adbd8fa8dc34216c49e6b0b6610bd92b1913cd0 Mon Sep 17 00:00:00 2001 From: Fernando Antivero Date: Mon, 10 Oct 2022 19:39:07 +0000 Subject: [PATCH 009/115] add kv secret for app gw ssl cert --- cluster-stamp.bicep | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/cluster-stamp.bicep b/cluster-stamp.bicep index ec8cf2b8..bdec3e4c 100644 --- a/cluster-stamp.bicep +++ b/cluster-stamp.bicep @@ -167,4 +167,12 @@ resource kv 'Microsoft.KeyVault/vaults@2022-07-01' = { enabledForTemplateDeployment: false enableSoftDelete: true } + + // The internet facing Tls certificate to establish https connections between your clients and your regional load balancer + resource kvsGatewaySslCert 'secrets' = { + name: 'sslcert' + properties: { + value: appGatewayListenerCertificate + } + } } From 13b1b5703b93af797dbffef67f17d51935777b45 Mon Sep 17 00:00:00 2001 From: Fernando Antivero Date: Mon, 10 Oct 2022 19:39:38 +0000 Subject: [PATCH 010/115] add kv secret for aks regulated internal ingress controller ssl cert --- cluster-stamp.bicep | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/cluster-stamp.bicep b/cluster-stamp.bicep index bdec3e4c..a26c9c12 100644 --- a/cluster-stamp.bicep +++ b/cluster-stamp.bicep @@ -175,4 +175,12 @@ resource kv 'Microsoft.KeyVault/vaults@2022-07-01' = { value: appGatewayListenerCertificate } } + + // The aks regulated in-cluster Tls certificate to establish https connections between your regional load balancer and your ingress controller enabling e2e tls connections + resource kvsAppGwIngressInternalAksIngressTls 'secrets' = { + name: 'agw-ingress-incluster-aks-ingress-contoso-com-tls' + properties: { + value: aksIngressControllerCertificate + } + } } From fe3a63b98340696cbef02b35ce1c58880a51826b Mon Sep 17 00:00:00 2001 From: Fernando Antivero Date: Mon, 10 Oct 2022 18:56:25 +0000 Subject: [PATCH 011/115] add existing spoke vnet and app gw snet resources --- cluster-stamp.bicep | 19 ++++++++++++++++++- 1 file changed, 18 insertions(+), 1 deletion(-) diff --git a/cluster-stamp.bicep b/cluster-stamp.bicep index a26c9c12..e3f40dc9 100644 --- a/cluster-stamp.bicep +++ b/cluster-stamp.bicep @@ -79,7 +79,24 @@ param jumpBoxCloudInitAsBase64 string = '10.200.0.0/26' var subRgUniqueString = uniqueString('aks', subscription().subscriptionId, resourceGroup().id) var clusterName = 'aks-${subRgUniqueString}' -/*** EXISTING RESOURCES ***/ +/*** EXISTING RESOURCE GROUP RESOURCES ***/ + +@description('Spoke resource group') +resource targetResourceGroup 'Microsoft.Resources/resourceGroups@2021-04-01' existing = { + scope: subscription() + name: '${split(targetVnetResourceId,'/')[4]}' +} + +@description('The Spoke virtual network') +resource targetVirtualNetwork 'Microsoft.Network/virtualNetworks@2022-01-01' existing = { + scope: targetResourceGroup + name: '${last(split(targetVnetResourceId,'/'))}' + + // Spoke virutual network's subnet for application gateway + resource snetApplicationGateway 'subnets' existing = { + name: 'snet-applicationgateway' + } +} @description('Built-in Azure RBAC role that must be applied to the kublet Managed Identity allowing it to further assign adding managed identities to the cluster\'s underlying VMSS.') resource managedIdentityOperatorRole 'Microsoft.Authorization/roleDefinitions@2022-04-01' existing = { From 442ba0ee7bf6f0b7fad3d3306945aa4e79e036ea Mon Sep 17 00:00:00 2001 From: Fernando Antivero Date: Mon, 10 Oct 2022 19:20:55 +0000 Subject: [PATCH 012/115] add log analytics workspace resource --- cluster-stamp.bicep | 232 ++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 232 insertions(+) diff --git a/cluster-stamp.bicep b/cluster-stamp.bicep index e3f40dc9..715504bf 100644 --- a/cluster-stamp.bicep +++ b/cluster-stamp.bicep @@ -201,3 +201,235 @@ resource kv 'Microsoft.KeyVault/vaults@2022-07-01' = { } } } + +@description('The aks regulated cluster log analytics workspace.') +resource law 'Microsoft.OperationalInsights/workspaces@2021-12-01-preview' = { + name: 'law-${clusterName}' + location: location + properties: { + sku: { + name: 'PerGB2018' + } + retentionInDays: 90 + publicNetworkAccessForIngestion: 'Enabled' + publicNetworkAccessForQuery: 'Enabled' + } +} + +@description('The regional load balancer resource that ingests all the client requests and forward them back to the aks regulated cluster after passing the configured WAF rules.') +resource agw 'Microsoft.Network/applicationGateways@2020-11-01' = { + name: 'agw-${clusterName}' + location: location + identity: { + type: 'UserAssigned' + userAssignedIdentities: { + '${miAppGateway.id}': { + } + } + } + zones: [ + '1' + '2' + '3' + ] + properties: { + sku: { + name: 'WAF_v2' + tier: 'WAF_v2' + } + sslPolicy: { + policyType: 'Custom' + cipherSuites: [ + 'TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384' + 'TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256' + ] + minProtocolVersion: 'TLSv1_2' + } + trustedRootCertificates: [ + { + name: 'root-cert-wildcard-aks-ingress-contoso' + properties: { + keyVaultSecretId: kv::kvsAppGwIngressInternalAksIngressTls.properties.secretUri + } + } + ] + gatewayIPConfigurations: [ + { + name: 'agw-ip-configuration' + properties: { + subnet: { + id: targetVirtualNetwork::snetApplicationGateway.id + } + } + } + ] + frontendIPConfigurations: [ + { + name: 'agw-frontend-ip-configuration' + properties: { + publicIPAddress: { + id: resourceId(subscription().subscriptionId, targetResourceGroup.name, 'Microsoft.Network/publicIpAddresses', 'pip-BU0001A0005-00') + } + } + } + ] + frontendPorts: [ + { + name: 'agw-frontend-ports' + properties: { + port: 443 + } + } + ] + autoscaleConfiguration: { + minCapacity: 0 + maxCapacity: 10 + } + webApplicationFirewallConfiguration: { + enabled: true + firewallMode: 'Prevention' + ruleSetType: 'OWASP' + ruleSetVersion: '3.2' + disabledRuleGroups: [] + requestBodyCheck: true + maxRequestBodySizeInKb: 128 + fileUploadLimitInMb: 100 + } + enableHttp2: false + sslCertificates: [ + { + name: 'agw-${clusterName}-ssl-certificate' + properties: { + keyVaultSecretId: kv::kvsGatewaySslCert.properties.secretUri + } + } + ] + probes: [ + { + name: 'probe-bu0001a0005-00.aks-ingress.contoso.com' + properties: { + protocol: 'Https' + path: '/favicon.ico' + interval: 30 + timeout: 30 + unhealthyThreshold: 3 + pickHostNameFromBackendHttpSettings: true + minServers: 0 + match: { + } + } + } + { + name: 'ingress-controller' + properties: { + protocol: 'Https' + path: '/healthz' + interval: 30 + timeout: 30 + unhealthyThreshold: 3 + pickHostNameFromBackendHttpSettings: true + minServers: 0 + match: { + } + } + } + ] + backendAddressPools: [ + { + name: 'bu0001a0005-00.aks-ingress.contoso.com' + properties: { + backendAddresses: [ + { + ipAddress: '10.240.4.4' + } + ] + } + } + ] + backendHttpSettingsCollection: [ + { + name: 'aks-ingress-contoso-backendpool-httpsettings' + properties: { + port: 443 + protocol: 'Https' + cookieBasedAffinity: 'Disabled' + hostName: 'bu0001a0005-00.aks-ingress.contoso.com' + pickHostNameFromBackendAddress: false + requestTimeout: 20 + probe: { + id: resourceId('Microsoft.Network/applicationGateways/probes', 'agw-${clusterName}','probe-bu0001a0005-00.aks-ingress.contoso.com') + } + trustedRootCertificates: [ + { + id: resourceId('Microsoft.Network/applicationGateways/trustedRootCertificates', 'agw-${clusterName}','root-cert-wildcard-aks-ingress-contoso') + } + ] + } + } + ] + httpListeners: [ + { + name: 'listener-https' + properties: { + frontendIPConfiguration: { + id: resourceId('Microsoft.Network/applicationGateways/frontendIPConfigurations', 'agw-${clusterName}','agw-frontend-ip-configuration') + } + frontendPort: { + id: resourceId('Microsoft.Network/applicationGateways/frontendPorts', 'agw-${clusterName}','agw-frontend-ports') + } + protocol: 'Https' + sslCertificate: { + id: resourceId('Microsoft.Network/applicationGateways/sslCertificates', 'agw-${clusterName}','agw-${clusterName}-ssl-certificate') + } + hostName: 'bicycle.contoso.com' + hostNames: [] + requireServerNameIndication: true + } + } + ] + requestRoutingRules: [ + { + name: 'agw-routing-rules' + properties: { + ruleType: 'Basic' + httpListener: { + id: resourceId('Microsoft.Network/applicationGateways/httpListeners', 'agw-${clusterName}','listener-https') + } + backendAddressPool: { + id: resourceId('Microsoft.Network/applicationGateways/backendAddressPools', 'agw-${clusterName}','bu0001a0005-00.aks-ingress.contoso.com') + } + backendHttpSettings: { + id: resourceId('Microsoft.Network/applicationGateways/backendHttpSettingsCollection', 'agw-${clusterName}','aks-ingress-contoso-backendpool-httpsettings') + } + } + } + ] + } +} + +@description('The diagnostic settings configuration for the aks regulated cluster regional load balancer.') +resource agw_diagnosticSettings 'Microsoft.Insights/diagnosticSettings@2021-05-01-preview' = { + scope: agw + name: 'default' + properties: { + workspaceId: law.id + logs: [ + { + category: 'ApplicationGatewayAccessLog' + enabled: true + } + { + category: 'ApplicationGatewayPerformanceLog' + enabled: true + } + { + category: 'ApplicationGatewayFirewallLog' + enabled: true + } + ] + } +} + +/*** OUTPUTS ***/ + +output agwName string = agw.name From 1bde208ff083d8d25224852ce36d186dfe1001da Mon Sep 17 00:00:00 2001 From: Fernando Antivero Date: Mon, 10 Oct 2022 19:52:05 +0000 Subject: [PATCH 013/115] add diagnostic settings cfg for kv instance --- cluster-stamp.bicep | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/cluster-stamp.bicep b/cluster-stamp.bicep index 715504bf..2cad3d02 100644 --- a/cluster-stamp.bicep +++ b/cluster-stamp.bicep @@ -216,6 +216,26 @@ resource law 'Microsoft.OperationalInsights/workspaces@2021-12-01-preview' = { } } +resource kv_diagnosticSettings 'Microsoft.Insights/diagnosticSettings@2021-05-01-preview' = { + scope: kv + name: 'default' + properties: { + workspaceId: law.id + logs: [ + { + category: 'AuditEvent' + enabled: true + } + ] + metrics: [ + { + category: 'AllMetrics' + enabled: true + } + ] + } +} + @description('The regional load balancer resource that ingests all the client requests and forward them back to the aks regulated cluster after passing the configured WAF rules.') resource agw 'Microsoft.Network/applicationGateways@2020-11-01' = { name: 'agw-${clusterName}' From c8a5fdcf03ca1098c4bfd5b94722cadf3b12ffa4 Mon Sep 17 00:00:00 2001 From: Fernando Antivero Date: Mon, 10 Oct 2022 18:38:06 +0000 Subject: [PATCH 014/115] add the cluster app gw instance --- cluster-stamp.bicep | 25 +------------------------ 1 file changed, 1 insertion(+), 24 deletions(-) diff --git a/cluster-stamp.bicep b/cluster-stamp.bicep index 2cad3d02..1a5b9fd5 100644 --- a/cluster-stamp.bicep +++ b/cluster-stamp.bicep @@ -237,7 +237,7 @@ resource kv_diagnosticSettings 'Microsoft.Insights/diagnosticSettings@2021-05-01 } @description('The regional load balancer resource that ingests all the client requests and forward them back to the aks regulated cluster after passing the configured WAF rules.') -resource agw 'Microsoft.Network/applicationGateways@2020-11-01' = { +resource agw 'Microsoft.Network/applicationGateways@2022-01-01' = { name: 'agw-${clusterName}' location: location identity: { @@ -427,29 +427,6 @@ resource agw 'Microsoft.Network/applicationGateways@2020-11-01' = { } } -@description('The diagnostic settings configuration for the aks regulated cluster regional load balancer.') -resource agw_diagnosticSettings 'Microsoft.Insights/diagnosticSettings@2021-05-01-preview' = { - scope: agw - name: 'default' - properties: { - workspaceId: law.id - logs: [ - { - category: 'ApplicationGatewayAccessLog' - enabled: true - } - { - category: 'ApplicationGatewayPerformanceLog' - enabled: true - } - { - category: 'ApplicationGatewayFirewallLog' - enabled: true - } - ] - } -} - /*** OUTPUTS ***/ output agwName string = agw.name From ffa9030a4bb925f4deb5ac2fe30da4019af33a00 Mon Sep 17 00:00:00 2001 From: Fernando Antivero Date: Mon, 10 Oct 2022 20:05:36 +0000 Subject: [PATCH 015/115] add diagnostic settings cfg for app gw instance --- cluster-stamp.bicep | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/cluster-stamp.bicep b/cluster-stamp.bicep index 1a5b9fd5..2a40ec31 100644 --- a/cluster-stamp.bicep +++ b/cluster-stamp.bicep @@ -427,6 +427,29 @@ resource agw 'Microsoft.Network/applicationGateways@2022-01-01' = { } } +@description('The diagnostic settings configuration for the aks regulated cluster regional load balancer.') +resource agw_diagnosticSettings 'Microsoft.Insights/diagnosticSettings@2021-05-01-preview' = { + scope: agw + name: 'default' + properties: { + workspaceId: law.id + logs: [ + { + category: 'ApplicationGatewayAccessLog' + enabled: true + } + { + category: 'ApplicationGatewayPerformanceLog' + enabled: true + } + { + category: 'ApplicationGatewayFirewallLog' + enabled: true + } + ] + } +} + /*** OUTPUTS ***/ output agwName string = agw.name From cac2df2368cf4a75d8d414dc971d4372d5665d0c Mon Sep 17 00:00:00 2001 From: Fernando Antivero Date: Mon, 10 Oct 2022 20:22:33 +0000 Subject: [PATCH 016/115] add spoke subnet containing private ips for private endpoints enabling the aks regulated cluster connect with azure services privately --- cluster-stamp.bicep | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/cluster-stamp.bicep b/cluster-stamp.bicep index 2a40ec31..aae73a5e 100644 --- a/cluster-stamp.bicep +++ b/cluster-stamp.bicep @@ -96,6 +96,11 @@ resource targetVirtualNetwork 'Microsoft.Network/virtualNetworks@2022-01-01' exi resource snetApplicationGateway 'subnets' existing = { name: 'snet-applicationgateway' } + + // Spoke virutual network's subnet for all private endpoints + resource snetPrivatelinkendpoints 'subnets' existing = { + name: 'snet-privatelinkendpoints' + } } @description('Built-in Azure RBAC role that must be applied to the kublet Managed Identity allowing it to further assign adding managed identities to the cluster\'s underlying VMSS.') From cf4fdd9bf617bac5b8a24f3fb7ec022c497eb0f6 Mon Sep 17 00:00:00 2001 From: Fernando Antivero Date: Mon, 10 Oct 2022 20:24:34 +0000 Subject: [PATCH 017/115] add private endpoint for kv enabling aks regulated cluster to connect privately with it --- cluster-stamp.bicep | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/cluster-stamp.bicep b/cluster-stamp.bicep index aae73a5e..bb382121 100644 --- a/cluster-stamp.bicep +++ b/cluster-stamp.bicep @@ -241,6 +241,28 @@ resource kv_diagnosticSettings 'Microsoft.Insights/diagnosticSettings@2021-05-01 } } +@description('The network interface in the spoke vnet that enables connecting privately the aks regulated cluster with kv.') +resource peKv 'Microsoft.Network/privateEndpoints@2022-01-01' = { + name: 'pe-${kv.name}' + location: location + properties: { + subnet: { + id: targetVirtualNetwork::snetPrivatelinkendpoints.id + } + privateLinkServiceConnections: [ + { + name: 'to-${targetVirtualNetwork.name}' + properties: { + privateLinkServiceId: kv.id + groupIds: [ + 'vault' + ] + } + } + ] + } +} + @description('The regional load balancer resource that ingests all the client requests and forward them back to the aks regulated cluster after passing the configured WAF rules.') resource agw 'Microsoft.Network/applicationGateways@2022-01-01' = { name: 'agw-${clusterName}' From 63d206c6280616e2a8755efbb4b09ac169e08bbf Mon Sep 17 00:00:00 2001 From: Fernando Antivero Date: Mon, 10 Oct 2022 20:31:13 +0000 Subject: [PATCH 018/115] add private dns zone for kv private Link support --- cluster-stamp.bicep | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/cluster-stamp.bicep b/cluster-stamp.bicep index bb382121..e72d6e77 100644 --- a/cluster-stamp.bicep +++ b/cluster-stamp.bicep @@ -241,6 +241,24 @@ resource kv_diagnosticSettings 'Microsoft.Insights/diagnosticSettings@2021-05-01 } } +@description('The kv private dns zone required by Private Link support.') +resource pdzKv 'Microsoft.Network/privateDnsZones@2020-06-01' = { + name: 'privatelink.vaultcore.azure.net' + location: 'global' + + // Enabling Azure Key Vault Private Link on the spoke vnet. + resource vnetlnk 'virtualNetworkLinks' = { + name: 'to_${targetVirtualNetwork.name}' + location: 'global' + properties: { + virtualNetwork: { + id: targetVirtualNetwork.id + } + registrationEnabled: false + } + } +} + @description('The network interface in the spoke vnet that enables connecting privately the aks regulated cluster with kv.') resource peKv 'Microsoft.Network/privateEndpoints@2022-01-01' = { name: 'pe-${kv.name}' From fb52a17e3bf6c6a7cc00dea0e372f240b3695361 Mon Sep 17 00:00:00 2001 From: Fernando Antivero Date: Mon, 10 Oct 2022 20:35:22 +0000 Subject: [PATCH 019/115] group the kv private endpoint into the private dns zone for private link support --- cluster-stamp.bicep | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/cluster-stamp.bicep b/cluster-stamp.bicep index e72d6e77..e1f84ab8 100644 --- a/cluster-stamp.bicep +++ b/cluster-stamp.bicep @@ -279,6 +279,20 @@ resource peKv 'Microsoft.Network/privateEndpoints@2022-01-01' = { } ] } + + resource pdzg 'privateDnsZoneGroups' = { + name: 'default' + properties: { + privateDnsZoneConfigs: [ + { + name: 'privatelink-akv-net' + properties: { + privateDnsZoneId: pdzKv.id + } + } + ] + } + } } @description('The regional load balancer resource that ingests all the client requests and forward them back to the aks regulated cluster after passing the configured WAF rules.') From 396c86f6b1fffa18fd70908665bb42ddaf2a98dc Mon Sep 17 00:00:00 2001 From: Fernando Antivero Date: Mon, 10 Oct 2022 20:52:40 +0000 Subject: [PATCH 020/115] add code region for subscription existing resources --- cluster-stamp.bicep | 2 ++ 1 file changed, 2 insertions(+) diff --git a/cluster-stamp.bicep b/cluster-stamp.bicep index e1f84ab8..6f57d384 100644 --- a/cluster-stamp.bicep +++ b/cluster-stamp.bicep @@ -103,6 +103,8 @@ resource targetVirtualNetwork 'Microsoft.Network/virtualNetworks@2022-01-01' exi } } +/*** EXISTING SUBSCRIPTION RESOURCES ***/ + @description('Built-in Azure RBAC role that must be applied to the kublet Managed Identity allowing it to further assign adding managed identities to the cluster\'s underlying VMSS.') resource managedIdentityOperatorRole 'Microsoft.Authorization/roleDefinitions@2022-04-01' existing = { name: 'f1a07417-d97a-45cb-824c-7a7467783830' From cb5bb744ceaf244f564b85f5a4978944a065504c Mon Sep 17 00:00:00 2001 From: Fernando Antivero Date: Mon, 10 Oct 2022 21:00:56 +0000 Subject: [PATCH 021/115] role assign app gw mi with reader role over kv --- cluster-stamp.bicep | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/cluster-stamp.bicep b/cluster-stamp.bicep index 6f57d384..2e0ede1d 100644 --- a/cluster-stamp.bicep +++ b/cluster-stamp.bicep @@ -111,6 +111,12 @@ resource managedIdentityOperatorRole 'Microsoft.Authorization/roleDefinitions@20 scope: subscription() } +@description('Built-in Azure RBAC role that is applied a Key Vault to grant with metadata, certificates, keys and secrets read privileges. Granted to App Gateway\'s managed identity.') +resource keyVaultReaderRole 'Microsoft.Authorization/roleDefinitions@2022-04-01' existing = { + name: '21090545-7ca7-4776-b22c-e363652d74d2' + scope: subscription() +} + /*** RESOURCES ***/ @description('The control plane identity used by the cluster.') @@ -209,6 +215,17 @@ resource kv 'Microsoft.KeyVault/vaults@2022-07-01' = { } } +@description('Grant the Azure Application Gateway managed identity with key vault reader role permissions; this allows pulling frontend and backend certificates.') +resource kvMiAppGatewayKeyVaultReader_roleAssignment 'Microsoft.Authorization/roleAssignments@2020-10-01-preview' = { + scope: kv + name: guid(resourceGroup().id, 'mi-appgateway', keyVaultReaderRole.id) + properties: { + roleDefinitionId: keyVaultReaderRole.id + principalId: miAppGateway.properties.principalId + principalType: 'ServicePrincipal' + } +} + @description('The aks regulated cluster log analytics workspace.') resource law 'Microsoft.OperationalInsights/workspaces@2021-12-01-preview' = { name: 'law-${clusterName}' @@ -486,6 +503,9 @@ resource agw 'Microsoft.Network/applicationGateways@2022-01-01' = { } ] } + dependsOn: [ + kvMiAppGatewayKeyVaultReader_roleAssignment + ] } @description('The diagnostic settings configuration for the aks regulated cluster regional load balancer.') From d28f632f79c3002a64fd6d0acd78e4219f973dcc Mon Sep 17 00:00:00 2001 From: Fernando Antivero Date: Mon, 10 Oct 2022 21:01:40 +0000 Subject: [PATCH 022/115] role assign app gw mi with user secret role over kv --- cluster-stamp.bicep | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/cluster-stamp.bicep b/cluster-stamp.bicep index 2e0ede1d..d688b299 100644 --- a/cluster-stamp.bicep +++ b/cluster-stamp.bicep @@ -117,6 +117,12 @@ resource keyVaultReaderRole 'Microsoft.Authorization/roleDefinitions@2022-04-01' scope: subscription() } +@description('Built-in Azure RBAC role that is applied to a Key Vault to grant with secrets content read privileges. Granted to both Key Vault and our workload\'s identity.') +resource keyVaultSecretsUserRole 'Microsoft.Authorization/roleDefinitions@2022-04-01' existing = { + name: '4633458b-17de-408a-b874-0445c86b69e6' + scope: subscription() +} + /*** RESOURCES ***/ @description('The control plane identity used by the cluster.') @@ -215,6 +221,17 @@ resource kv 'Microsoft.KeyVault/vaults@2022-07-01' = { } } +@description('Grant the Azure Application Gateway managed identity with key vault secrets user role permissions; this allows pulling secrets from key vault.') +resource kvMiAppGatewaySecretsUserRole_roleAssignment 'Microsoft.Authorization/roleAssignments@2020-10-01-preview' = { + scope: kv + name: guid(resourceGroup().id, 'mi-appgateway', keyVaultSecretsUserRole.id) + properties: { + roleDefinitionId: keyVaultSecretsUserRole.id + principalId: miAppGateway.properties.principalId + principalType: 'ServicePrincipal' + } +} + @description('Grant the Azure Application Gateway managed identity with key vault reader role permissions; this allows pulling frontend and backend certificates.') resource kvMiAppGatewayKeyVaultReader_roleAssignment 'Microsoft.Authorization/roleAssignments@2020-10-01-preview' = { scope: kv @@ -505,6 +522,7 @@ resource agw 'Microsoft.Network/applicationGateways@2022-01-01' = { } dependsOn: [ kvMiAppGatewayKeyVaultReader_roleAssignment + kvMiAppGatewaySecretsUserRole_roleAssignment ] } From 02b5490e0997d116586f87a78a6bd4bfd3e3845a Mon Sep 17 00:00:00 2001 From: Fernando Antivero Date: Mon, 10 Oct 2022 21:02:48 +0000 Subject: [PATCH 023/115] ensure private endpoint is created so the app gw access kv privately --- cluster-stamp.bicep | 1 + 1 file changed, 1 insertion(+) diff --git a/cluster-stamp.bicep b/cluster-stamp.bicep index d688b299..21290017 100644 --- a/cluster-stamp.bicep +++ b/cluster-stamp.bicep @@ -521,6 +521,7 @@ resource agw 'Microsoft.Network/applicationGateways@2022-01-01' = { ] } dependsOn: [ + peKv kvMiAppGatewayKeyVaultReader_roleAssignment kvMiAppGatewaySecretsUserRole_roleAssignment ] From 49d5c308492b7c53f931ab824f20a199bc375b83 Mon Sep 17 00:00:00 2001 From: Fernando Antivero Date: Mon, 10 Oct 2022 21:03:26 +0000 Subject: [PATCH 024/115] replace hardcoded zones by pickZones safer function --- cluster-stamp.bicep | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/cluster-stamp.bicep b/cluster-stamp.bicep index 21290017..11132e17 100644 --- a/cluster-stamp.bicep +++ b/cluster-stamp.bicep @@ -342,11 +342,7 @@ resource agw 'Microsoft.Network/applicationGateways@2022-01-01' = { } } } - zones: [ - '1' - '2' - '3' - ] + zones: pickZones('Microsoft.Network', 'applicationGateways', location, 3) properties: { sku: { name: 'WAF_v2' From 5f0966c758688be957ea65c8191267ce70a3bb55 Mon Sep 17 00:00:00 2001 From: Fernando Antivero Date: Tue, 11 Oct 2022 20:49:04 +0000 Subject: [PATCH 025/115] add keyvault name output --- cluster-stamp.bicep | 1 + 1 file changed, 1 insertion(+) diff --git a/cluster-stamp.bicep b/cluster-stamp.bicep index 11132e17..2cd5c874 100644 --- a/cluster-stamp.bicep +++ b/cluster-stamp.bicep @@ -549,3 +549,4 @@ resource agw_diagnosticSettings 'Microsoft.Insights/diagnosticSettings@2021-05-0 /*** OUTPUTS ***/ output agwName string = agw.name +output keyVaultName string = kv.name From 45ce2007d4707a7a841b40aed6d8ae61f4076d4b Mon Sep 17 00:00:00 2001 From: Fernando Antivero Date: Tue, 11 Oct 2022 20:51:44 +0000 Subject: [PATCH 026/115] add log analytics worspaces saved search for prometheus info --- cluster-stamp.bicep | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/cluster-stamp.bicep b/cluster-stamp.bicep index 2cd5c874..405f331d 100644 --- a/cluster-stamp.bicep +++ b/cluster-stamp.bicep @@ -257,6 +257,18 @@ resource law 'Microsoft.OperationalInsights/workspaces@2021-12-01-preview' = { } } +resource lawAllPrometheus 'Microsoft.OperationalInsights/workspaces/savedSearches@2020-08-01' = { + parent: law + name: 'AllPrometheus' + properties: { + eTag: '*' + category: 'Prometheus' + displayName: 'All collected Prometheus information' + query: 'InsightsMetrics | where Namespace == "prometheus"' + version: 1 + } +} + resource kv_diagnosticSettings 'Microsoft.Insights/diagnosticSettings@2021-05-01-preview' = { scope: kv name: 'default' From 4c47cb1c86893528e5d68061fe9b9247dbd6f02d Mon Sep 17 00:00:00 2001 From: Fernando Antivero Date: Tue, 11 Oct 2022 20:53:08 +0000 Subject: [PATCH 027/115] add log analytics worspaces saved search for total number of 403 responses --- cluster-stamp.bicep | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/cluster-stamp.bicep b/cluster-stamp.bicep index 405f331d..05438c26 100644 --- a/cluster-stamp.bicep +++ b/cluster-stamp.bicep @@ -269,6 +269,18 @@ resource lawAllPrometheus 'Microsoft.OperationalInsights/workspaces/savedSearche } } +resource lawForbiddenReponsesOnIngress 'Microsoft.OperationalInsights/workspaces/savedSearches@2020-08-01' = { + parent: law + name: 'ForbiddenReponsesOnIngress' + properties: { + eTag: '*' + category: 'Prometheus' + displayName: 'Increase number of forbidden response on the Ingress Controller' + query: 'let value = toscalar(InsightsMetrics | where Namespace == "prometheus" and Name == "nginx_ingress_controller_requests" | where parse_json(Tags).status == 403 | summarize Value = avg(Val) by bin(TimeGenerated, 5m) | summarize min = min(Value)); InsightsMetrics | where Namespace == "prometheus" and Name == "nginx_ingress_controller_requests" | where parse_json(Tags).status == 403 | summarize AggregatedValue = avg(Val)-value by bin(TimeGenerated, 5m) | order by TimeGenerated | render barchart' + version: 1 + } +} + resource kv_diagnosticSettings 'Microsoft.Insights/diagnosticSettings@2021-05-01-preview' = { scope: kv name: 'default' From 0b51887350b7c375ee148cd281859c0cb540a862 Mon Sep 17 00:00:00 2001 From: Fernando Antivero Date: Tue, 11 Oct 2022 20:54:43 +0000 Subject: [PATCH 028/115] add log analytics worspaces saved search for requested node reboot metric --- cluster-stamp.bicep | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/cluster-stamp.bicep b/cluster-stamp.bicep index 05438c26..dd39868c 100644 --- a/cluster-stamp.bicep +++ b/cluster-stamp.bicep @@ -281,6 +281,18 @@ resource lawForbiddenReponsesOnIngress 'Microsoft.OperationalInsights/workspaces } } +resource lawNodeRebootRequested 'Microsoft.OperationalInsights/workspaces/savedSearches@2020-08-01' = { + parent: law + name: 'NodeRebootRequested' + properties: { + eTag: '*' + category: 'Prometheus' + displayName: 'Nodes reboot required by kured' + query: 'InsightsMetrics | where Namespace == "prometheus" and Name == "kured_reboot_required" | where Val > 0' + version: 1 + } +} + resource kv_diagnosticSettings 'Microsoft.Insights/diagnosticSettings@2021-05-01-preview' = { scope: kv name: 'default' From 9aba6ff9a514b397e87f62f9617185e533205f94 Mon Sep 17 00:00:00 2001 From: Fernando Antivero Date: Tue, 11 Oct 2022 21:03:35 +0000 Subject: [PATCH 029/115] add operation management solution for container insights --- cluster-stamp.bicep | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/cluster-stamp.bicep b/cluster-stamp.bicep index dd39868c..21fe27f1 100644 --- a/cluster-stamp.bicep +++ b/cluster-stamp.bicep @@ -293,6 +293,20 @@ resource lawNodeRebootRequested 'Microsoft.OperationalInsights/workspaces/savedS } } +resource omsContainerInsights 'Microsoft.OperationsManagement/solutions@2015-11-01-preview' = { + name: 'ContainerInsights(${law.name})' + location: location + properties: { + workspaceResourceId: law.id + } + plan: { + name: 'ContainerInsights(${law.name})' + product: 'OMSGallery/ContainerInsights' + promotionCode: '' + publisher: 'Microsoft' + } +} + resource kv_diagnosticSettings 'Microsoft.Insights/diagnosticSettings@2021-05-01-preview' = { scope: kv name: 'default' From c63027eb4f6aaa6a045da6b5a9288eda2e008c74 Mon Sep 17 00:00:00 2001 From: Fernando Antivero Date: Tue, 11 Oct 2022 21:04:43 +0000 Subject: [PATCH 030/115] add operation management solution for vm insights --- cluster-stamp.bicep | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/cluster-stamp.bicep b/cluster-stamp.bicep index 21fe27f1..6a46dc84 100644 --- a/cluster-stamp.bicep +++ b/cluster-stamp.bicep @@ -307,6 +307,20 @@ resource omsContainerInsights 'Microsoft.OperationsManagement/solutions@2015-11- } } +resource omsVmInsights 'Microsoft.OperationsManagement/solutions@2015-11-01-preview' = { + name: 'VMInsights(${law.name})' + location: location + properties: { + workspaceResourceId: law.id + } + plan: { + name: 'VMInsights(${law.name})' + product: 'OMSGallery/VMInsights' + promotionCode: '' + publisher: 'Microsoft' + } +} + resource kv_diagnosticSettings 'Microsoft.Insights/diagnosticSettings@2021-05-01-preview' = { scope: kv name: 'default' From cc5db88cfb1a50cdcad339d75e903b86e98433af Mon Sep 17 00:00:00 2001 From: Fernando Antivero Date: Tue, 11 Oct 2022 21:06:26 +0000 Subject: [PATCH 031/115] add operation management solution for sec insights --- cluster-stamp.bicep | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/cluster-stamp.bicep b/cluster-stamp.bicep index 6a46dc84..f267f939 100644 --- a/cluster-stamp.bicep +++ b/cluster-stamp.bicep @@ -321,6 +321,20 @@ resource omsVmInsights 'Microsoft.OperationsManagement/solutions@2015-11-01-prev } } +resource omsSecurityInsights 'Microsoft.OperationsManagement/solutions@2015-11-01-preview' = { + name: 'SecurityInsights(${law.name})' + location: location + properties: { + workspaceResourceId: law.id + } + plan: { + name: 'SecurityInsights(${law.name})' + product: 'OMSGallery/SecurityInsights' + promotionCode: '' + publisher: 'Microsoft' + } +} + resource kv_diagnosticSettings 'Microsoft.Insights/diagnosticSettings@2021-05-01-preview' = { scope: kv name: 'default' From b920f9e9c368400a3354b09cf8232d431e85aadf Mon Sep 17 00:00:00 2001 From: Fernando Antivero Date: Tue, 11 Oct 2022 21:15:35 +0000 Subject: [PATCH 032/115] add workbook for sec insights --- cluster-stamp.bicep | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/cluster-stamp.bicep b/cluster-stamp.bicep index f267f939..89da3297 100644 --- a/cluster-stamp.bicep +++ b/cluster-stamp.bicep @@ -335,6 +335,26 @@ resource omsSecurityInsights 'Microsoft.OperationsManagement/solutions@2015-11-0 } } +resource miwSecurityInsights 'Microsoft.Insights/workbooks@2022-04-01' = { + name: guid(omsSecurityInsights.name) + location: location + tags: { + 'hidden-title': 'Azure Kubernetes Service (AKS) Security - ${law.name}' + } + kind: 'shared' + properties: { + displayName: 'Azure Kubernetes Service (AKS) Security - ${law.name}' + serializedData: '{"version":"Notebook/1.0","items":[{"type":1,"content":{"json":"## AKS Security\\n"},"name":"text - 2"},{"type":9,"content":{"version":"KqlParameterItem/1.0","crossComponentResources":["{workspaces}"],"parameters":[{"id":"311d3728-7f8a-4b16-8a34-097d099323d5","version":"KqlParameterItem/1.0","name":"subscription","label":"Subscription","type":6,"isRequired":true,"multiSelect":true,"quote":"\'","delimiter":",","value":[],"typeSettings":{"additionalResourceOptions":[],"includeAll":false,"showDefault":false}},{"id":"3a56d260-4fb9-46d6-b121-cea854104c91","version":"KqlParameterItem/1.0","name":"workspaces","label":"Workspaces","type":5,"isRequired":true,"multiSelect":true,"quote":"\'","delimiter":",","query":"where type =~ \'microsoft.operationalinsights/workspaces\'\\r\\n| where strcat(\'/subscriptions/\',subscriptionId) in ({subscription})\\r\\n| project id","crossComponentResources":["{subscription}"],"typeSettings":{"additionalResourceOptions":["value::all"]},"queryType":1,"resourceType":"microsoft.resourcegraph/resources","value":["value::all"]},{"id":"9615cea6-c661-470a-b4ae-1aab8ae6f448","version":"KqlParameterItem/1.0","name":"clustername","label":"Cluster name","type":5,"isRequired":true,"multiSelect":true,"quote":"\'","delimiter":",","query":"where type == \\"microsoft.containerservice/managedclusters\\"\\r\\n| where strcat(\'/subscriptions/\',subscriptionId) in ({subscription})\\r\\n| distinct tolower(id)","crossComponentResources":["{subscription}"],"value":["value::all"],"typeSettings":{"resourceTypeFilter":{"microsoft.containerservice/managedclusters":true},"additionalResourceOptions":["value::all"],"showDefault":false},"timeContext":{"durationMs":86400000},"queryType":1,"resourceType":"microsoft.resourcegraph/resources"},{"id":"236c00ec-1493-4e60-927a-a18b8b120cd5","version":"KqlParameterItem/1.0","name":"timeframe","label":"Time range","type":4,"description":"Time","isRequired":true,"value":{"durationMs":172800000},"typeSettings":{"selectableValues":[{"durationMs":300000},{"durationMs":900000},{"durationMs":1800000},{"durationMs":3600000},{"durationMs":14400000},{"durationMs":43200000},{"durationMs":86400000},{"durationMs":172800000},{"durationMs":259200000},{"durationMs":604800000},{"durationMs":1209600000},{"durationMs":2419200000},{"durationMs":2592000000},{"durationMs":5184000000},{"durationMs":7776000000}],"allowCustom":true},"timeContext":{"durationMs":86400000}},{"id":"bf0a3e4f-fff9-450c-b9d3-c8c1dded9787","version":"KqlParameterItem/1.0","name":"nodeRgDetails","type":1,"query":"where type == \\"microsoft.containerservice/managedclusters\\"\\r\\n| where tolower(id) in ({clustername})\\r\\n| project nodeRG = properties.nodeResourceGroup, subscriptionId, id = toupper(id)\\r\\n| project nodeRgDetails = strcat(\'\\"\', nodeRG, \\";\\", subscriptionId, \\";\\", id, \'\\"\')","crossComponentResources":["value::all"],"isHiddenWhenLocked":true,"timeContext":{"durationMs":86400000},"queryType":1,"resourceType":"microsoft.resourcegraph/resources"},{"id":"df53126c-c40f-43d5-b99f-97ee3785c086","version":"KqlParameterItem/1.0","name":"diagnosticClusters","type":1,"query":"union withsource=_TableName *\\r\\n| where _TableName == \\"AzureDiagnostics\\" and Category == \\"kube-audit\\"\\r\\n| summarize diagnosticClusters = dcount(ResourceId)\\r\\n| project isDiagnosticCluster = iff(diagnosticClusters > 0, \\"yes\\", \\"no\\")","crossComponentResources":["{workspaces}"],"isHiddenWhenLocked":true,"timeContext":{"durationMs":172800000},"timeContextFromParameter":"timeframe","queryType":0,"resourceType":"microsoft.operationalinsights/workspaces"}],"style":"pills","queryType":0,"resourceType":"microsoft.operationalinsights/workspaces"},"name":"parameters - 3"},{"type":11,"content":{"version":"LinkItem/1.0","style":"tabs","links":[{"id":"07cf87dc-8234-47db-850d-ec41b2687b2a","cellValue":"mainTab","linkTarget":"parameter","linkLabel":"Microsoft Defender for Kubernetes","subTarget":"alerts","preText":"","style":"link"},{"id":"44033ee6-d83e-4253-a732-c258ef1da545","cellValue":"mainTab","linkTarget":"parameter","linkLabel":"Analytics over Diagnostic logs","subTarget":"diagnostics","style":"link"}]},"name":"links - 22"},{"type":12,"content":{"version":"NotebookGroup/1.0","groupType":"editable","items":[{"type":1,"content":{"json":"## Microsoft Defender for AKS coverage"},"name":"text - 10"},{"type":3,"content":{"version":"KqlItem/1.0","query":"datatable (Event:string)\\r\\n [\\"AKS Workbook\\"]\\r\\n| extend cluster = (strcat(\\"[\\", \\"{clustername}\\", \\"]\\"))\\r\\n| extend cluster = todynamic(replace(\\"\'\\", \'\\"\', cluster))\\r\\n| mvexpand cluster\\r\\n| extend subscriptionId = extract(@\\"/subscriptions/([^/]+)\\", 1, tostring(cluster))\\r\\n| summarize AksClusters = count() by subscriptionId, DefenderForAks = 0\\r\\n| union\\r\\n(\\r\\nsecurityresources\\r\\n| where type =~ \\"microsoft.security/pricings\\"\\r\\n| where name == \\"KubernetesService\\"\\r\\n| project DefenderForAks = iif(properties.pricingTier == \'Standard\', 1, 0), AksClusters = 0, subscriptionId\\r\\n)\\r\\n| summarize AksClusters = sum(AksClusters), DefenderForAks = sum(DefenderForAks) by subscriptionId\\r\\n| project Subscription = strcat(\'/subscriptions/\', subscriptionId), [\\"AKS clusters\\"] = AksClusters, [\'Defender for AKS\'] = iif(DefenderForAks > 0,\'yes\',\'no\'), [\'Onboard Microsoft Defender\'] = iif(DefenderForAks > 0, \'\', \'https://ms.portal.azure.com/#blade/Microsoft_Azure_Security/SecurityMenuBlade/26\')\\r\\n| order by [\'Defender for AKS\'] asc","size":0,"queryType":1,"resourceType":"microsoft.resourcegraph/resources","crossComponentResources":["{subscription}"],"gridSettings":{"formatters":[{"columnMatch":"Defender for AKS","formatter":18,"formatOptions":{"thresholdsOptions":"icons","thresholdsGrid":[{"operator":"==","thresholdValue":"no","representation":"4","text":""},{"operator":"Default","thresholdValue":null,"representation":"success","text":""}]}},{"columnMatch":"Onboard Microsoft Defender","formatter":7,"formatOptions":{"linkTarget":"Url","linkLabel":""}}]}},"customWidth":"66","name":"query - 9"},{"type":3,"content":{"version":"KqlItem/1.0","query":"datatable (Event:string)\\r\\n [\\"AKS Workbook\\"]\\r\\n| extend cluster = (strcat(\\"[\\", \\"{clustername}\\", \\"]\\"))\\r\\n| extend cluster = todynamic(replace(\\"\'\\", \'\\"\', cluster))\\r\\n| mvexpand cluster\\r\\n| extend subscriptionId = extract(@\\"/subscriptions/([^/]+)\\", 1, tostring(cluster))\\r\\n| summarize AksClusters = count() by subscriptionId, DefenderForAks = 0\\r\\n| union\\r\\n(\\r\\nsecurityresources\\r\\n| where type =~ \\"microsoft.security/pricings\\"\\r\\n| where name == \\"KubernetesService\\"\\r\\n| project DefenderForAks = iif(properties.pricingTier == \'Standard\', 1, 0), AksClusters = 0, subscriptionId\\r\\n)\\r\\n| summarize AksClusters = sum(AksClusters), DefenderForAks = sum(DefenderForAks) by subscriptionId\\r\\n| project Subscription = 1, [\'Defender for AKS\'] = iif(DefenderForAks > 0,\'Protected by Microsoft Defender\',\'Not protected by Microsoft Defender\')","size":0,"queryType":1,"resourceType":"microsoft.resourcegraph/resources","crossComponentResources":["{subscription}"],"visualization":"piechart"},"customWidth":"33","name":"query - 11"},{"type":1,"content":{"json":"### AKS alerts overview"},"name":"text - 21"},{"type":3,"content":{"version":"KqlItem/1.0","query":"SecurityAlert \\r\\n| where tolower(ResourceId) in ({clustername})\\r\\n| where TimeGenerated {timeframe}\\r\\n| where AlertType startswith \\"AKS_\\"\\r\\n| project image = tostring(todynamic(ExtendedProperties)[\\"Container image\\"]), AlertType\\r\\n| where image != \\"\\"\\r\\n| summarize AlertTypes = dcount(AlertType) by image\\r\\n| where AlertTypes > 1\\r\\n//| render piechart \\r\\n","size":4,"title":"Images with multiple alerts","queryType":0,"resourceType":"microsoft.operationalinsights/workspaces","crossComponentResources":["{workspaces}"],"visualization":"tiles","tileSettings":{"showBorder":false,"titleContent":{"columnMatch":"image","formatter":1},"leftContent":{"columnMatch":"AlertTypes","formatter":12,"formatOptions":{"palette":"auto"},"numberFormat":{"unit":17,"options":{"maximumSignificantDigits":3,"maximumFractionDigits":2}}}}},"customWidth":"33","name":"query - 12"},{"type":3,"content":{"version":"KqlItem/1.0","query":"SecurityAlert \\r\\n| where tolower(ResourceId) in ({clustername})\\r\\n| where TimeGenerated {timeframe}\\r\\n| where AlertType startswith \\"AKS_\\"\\r\\n| project AlertType, name = extract(@\\"/MICROSOFT.CONTAINERSERVICE/MANAGEDCLUSTERS/(.+)\\", 1, ResourceId)\\r\\n| summarize AlertTypes = dcount(AlertType) by name\\r\\n| where AlertTypes > 1\\r\\n","size":4,"title":"Clusters with multiple alert types","queryType":0,"resourceType":"microsoft.operationalinsights/workspaces","crossComponentResources":["{workspaces}"],"visualization":"tiles","tileSettings":{"showBorder":false,"titleContent":{"columnMatch":"name","formatter":1},"leftContent":{"columnMatch":"AlertTypes","formatter":12,"formatOptions":{"palette":"auto"},"numberFormat":{"unit":17,"options":{"maximumSignificantDigits":3,"maximumFractionDigits":2}}}}},"customWidth":"33","name":"query - 12 - Copy"},{"type":3,"content":{"version":"KqlItem/1.0","query":"SecurityAlert \\r\\n| where tolower(ResourceId) in ({clustername})\\r\\n| where TimeGenerated {timeframe}\\r\\n| where AlertType startswith \\"AKS_\\"\\r\\n| project AlertType, name = extract(@\\"/MICROSOFT.CONTAINERSERVICE/MANAGEDCLUSTERS/(.+)\\", 1, ResourceId)\\r\\n| summarize count() by name\\r\\n\\r\\n","size":4,"title":"Alerts triggered by cluster","queryType":0,"resourceType":"microsoft.operationalinsights/workspaces","crossComponentResources":["{workspaces}"],"visualization":"tiles","tileSettings":{"showBorder":false,"titleContent":{"columnMatch":"name","formatter":1},"leftContent":{"columnMatch":"count_","formatter":12,"formatOptions":{"palette":"auto"},"numberFormat":{"unit":17,"options":{"maximumSignificantDigits":3,"maximumFractionDigits":2}}}}},"customWidth":"33","name":"query - 12 - Copy - Copy"},{"type":1,"content":{"json":"### Seucirty alerts details\\r\\n\\r\\nTo filter, press on the severities below.\\r\\nYou can also filter based on a specific resource."},"name":"text - 18"},{"type":3,"content":{"version":"KqlItem/1.0","query":"SecurityAlert\\r\\n| where TimeGenerated {timeframe}\\r\\n| extend rg = extract(@\\"/resourcegroups/([^/]+)\\", 1, tolower(ResourceId))\\r\\n| extend sub = extract(@\\"/subscriptions/([^/]+)\\", 1, tolower(ResourceId))\\r\\n| extend nodesDetailsArr = todynamic(\'{nodeRgDetails}\')\\r\\n| mv-expand singleNodeDetails = nodesDetailsArr\\r\\n| extend singleNodeArr = split(singleNodeDetails, \\";\\")\\r\\n| where tolower(singleNodeArr[0]) == rg and tolower(singleNodeArr[1]) == sub\\r\\n| project AlertSeverity\\r\\n| union\\r\\n(\\r\\nSecurityAlert \\r\\n| where tolower(ResourceId) in ({clustername})\\r\\n| where TimeGenerated {timeframe}\\r\\n| where AlertType startswith \\"AKS_\\"\\r\\n| project AlertSeverity\\r\\n)\\r\\n| summarize count() by AlertSeverity","size":0,"title":"Alerts by severity","exportMultipleValues":true,"exportedParameters":[{"fieldName":"AlertSeverity","parameterName":"severity","parameterType":1,"quote":""}],"queryType":0,"resourceType":"microsoft.operationalinsights/workspaces","crossComponentResources":["{workspaces}"],"visualization":"tiles","tileSettings":{"showBorder":false,"titleContent":{"columnMatch":"AlertSeverity","formatter":1},"leftContent":{"columnMatch":"count_","formatter":12,"formatOptions":{"palette":"auto"},"numberFormat":{"unit":17,"options":{"maximumSignificantDigits":3,"maximumFractionDigits":2}}}}},"customWidth":"11","name":"Alerts by severity"},{"type":3,"content":{"version":"KqlItem/1.0","query":"SecurityAlert\\r\\n| where TimeGenerated {timeframe}\\r\\n| where \\"{severity}\\" has AlertSeverity or isempty(\\"{severity}\\")\\r\\n| extend rg = extract(@\\"/resourcegroups/([^/]+)\\", 1, tolower(ResourceId))\\r\\n| extend sub = extract(@\\"/subscriptions/([^/]+)\\", 1, tolower(ResourceId))\\r\\n| extend nodesDetailsArr = todynamic(\'{nodeRgDetails}\')\\r\\n| mv-expand singleNodeDetails = nodesDetailsArr\\r\\n| extend singleNodeArr = split(singleNodeDetails, \\";\\")\\r\\n| where tolower(singleNodeArr[0]) == rg and tolower(singleNodeArr[1]) == sub\\r\\n| project ResourceId\\r\\n| union\\r\\n(\\r\\nSecurityAlert \\r\\n| where tolower(ResourceId) in ({clustername})\\r\\n| where \\"{severity}\\" has AlertSeverity or isempty(\\"{severity}\\")\\r\\n| where TimeGenerated {timeframe}\\r\\n| where AlertType startswith \\"AKS_\\"\\r\\n| project ResourceId\\r\\n)\\r\\n| summarize Alerts = count() by ResourceId\\r\\n| order by Alerts desc\\r\\n| limit 10","size":0,"title":"Resources with most alerts","exportFieldName":"ResourceId","exportParameterName":"selectedResource","exportDefaultValue":"not_selected","queryType":0,"resourceType":"microsoft.operationalinsights/workspaces","crossComponentResources":["{workspaces}"],"gridSettings":{"formatters":[{"columnMatch":"Alerts","formatter":4,"formatOptions":{"palette":"red"}}]}},"customWidth":"22","name":"Resources with most alerts"},{"type":3,"content":{"version":"KqlItem/1.0","query":"\\r\\nSecurityAlert\\r\\n| where TimeGenerated {timeframe}\\r\\n| where \\"{severity}\\" has AlertSeverity or isempty(\\"{severity}\\")\\r\\n| extend rg = extract(@\\"/resourcegroups/([^/]+)\\", 1, tolower(ResourceId))\\r\\n| extend sub = extract(@\\"/subscriptions/([^/]+)\\", 1, tolower(ResourceId))\\r\\n| extend nodesDetailsArr = todynamic(\'{nodeRgDetails}\')\\r\\n| mv-expand singleNodeDetails = nodesDetailsArr\\r\\n| extend singleNodeArr = split(singleNodeDetails, \\";\\")\\r\\n| where tolower(singleNodeArr[0]) == rg and tolower(singleNodeArr[1]) == sub\\r\\n| extend AlertResourceType = \\"VM alerts\\"\\r\\n| union\\r\\n(\\r\\nSecurityAlert \\r\\n| where tolower(ResourceId) in ({clustername})\\r\\n| where \\"{severity}\\" has AlertSeverity or isempty(\\"{severity}\\")\\r\\n| where TimeGenerated {timeframe}\\r\\n| where AlertType startswith \\"AKS_\\"\\r\\n| extend AlertResourceType = \\"Cluster alerts\\"\\r\\n)\\r\\n| summarize Alerts = count() by bin(TimeGenerated, {timeframe:grain}), AlertResourceType","size":0,"title":"Alerts over time","queryType":0,"resourceType":"microsoft.operationalinsights/workspaces","crossComponentResources":["{workspaces}"],"visualization":"timechart"},"customWidth":"66","name":"Alerts over time"},{"type":3,"content":{"version":"KqlItem/1.0","query":"SecurityAlert\\r\\n| where TimeGenerated {timeframe}\\r\\n| where \\"{severity}\\" has AlertSeverity or isempty(\\"{severity}\\")\\r\\n| extend rg = extract(@\\"/resourcegroups/([^/]+)\\", 1, tolower(ResourceId))\\r\\n| extend sub = extract(@\\"/subscriptions/([^/]+)\\", 1, tolower(ResourceId))\\r\\n| extend nodesDetailsArr = todynamic(\'{nodeRgDetails}\')\\r\\n| mv-expand singleNodeDetails = nodesDetailsArr\\r\\n| extend singleNodeArr = split(singleNodeDetails, \\";\\")\\r\\n| where tolower(singleNodeArr[0]) == rg and tolower(singleNodeArr[1]) == sub\\r\\n| where tolower(ResourceId) == tolower(\\"{selectedResource}\\") or \\"{selectedResource}\\" == \\"not_selected\\"\\r\\n| project [\\"Resource name\\"] = ResourceId, TimeGenerated, AlertSeverity, [\\"AKS cluster\\"] = toupper(singleNodeArr[2]), DisplayName, AlertLink\\r\\n| union\\r\\n(\\r\\nSecurityAlert\\r\\n| where TimeGenerated {timeframe}\\r\\n| where tolower(ResourceId) in ({clustername})\\r\\n| where \\"{severity}\\" has AlertSeverity or isempty(\\"{severity}\\")\\r\\n| where AlertType startswith \\"AKS_\\"\\r\\n| where tolower(ResourceId) == tolower(\\"{selectedResource}\\") or \\"{selectedResource}\\" == \\"not_selected\\"\\r\\n| project [\\"Resource name\\"] = ResourceId, TimeGenerated, AlertSeverity, [\\"AKS cluster\\"] = ResourceId, DisplayName, AlertLink\\r\\n)\\r\\n| order by TimeGenerated asc","size":0,"title":"Microsoft Defender alerts","queryType":0,"resourceType":"microsoft.operationalinsights/workspaces","crossComponentResources":["{workspaces}"],"gridSettings":{"formatters":[{"columnMatch":"AlertLink","formatter":7,"formatOptions":{"linkTarget":"Url","linkLabel":"Go to alert "}}],"filter":true},"sortBy":[]},"name":"Microsoft Defender alerts","styleSettings":{"showBorder":true}}]},"conditionalVisibility":{"parameterName":"mainTab","comparison":"isEqualTo","value":"alerts"},"name":"Defender Alerts"},{"type":12,"content":{"version":"NotebookGroup/1.0","groupType":"editable","items":[{"type":1,"content":{"json":"## Diagnostic logs coverage"},"name":"text - 15"},{"type":3,"content":{"version":"KqlItem/1.0","query":"union withsource=_TableName *\\r\\n| where _TableName == \\"AzureDiagnostics\\" and Category == \\"kube-audit\\"\\r\\n| summarize count() by ResourceId = tolower(ResourceId)\\r\\n| summarize logsClusters = make_set(ResourceId)\\r\\n| extend selectedClusters = \\"[{clustername}]\\"\\r\\n| extend selectedClusters = replace(\\"\'\\", \'\\"\', selectedClusters)\\r\\n| extend selectedClusters = todynamic(selectedClusters)\\r\\n| mv-expand clusterId = selectedClusters\\r\\n| project clusterId = toupper(tostring(clusterId)), [\\"Diagnostic logs\\"] = (logsClusters has tostring(clusterId))\\r\\n| extend [\\"Diagnostic settings\\"] = iff([\\"Diagnostic logs\\"] == false, strcat(\\"https://ms.portal.azure.com/#@microsoft.onmicrosoft.com/resource\\", clusterId, \\"/diagnostics\\"), \\"\\")\\r\\n","size":0,"timeContext":{"durationMs":172800000},"timeContextFromParameter":"timeframe","queryType":0,"resourceType":"microsoft.operationalinsights/workspaces","crossComponentResources":["{workspaces}"],"gridSettings":{"formatters":[{"columnMatch":"Diagnostic logs","formatter":18,"formatOptions":{"thresholdsOptions":"icons","thresholdsGrid":[{"operator":"==","thresholdValue":"false","representation":"critical","text":""},{"operator":"Default","thresholdValue":null,"representation":"success","text":""}]}},{"columnMatch":"Diagnostic settings","formatter":7,"formatOptions":{"linkTarget":"Url"}}],"filter":true,"sortBy":[{"itemKey":"$gen_thresholds_Diagnostic logs_1","sortOrder":2}]},"sortBy":[{"itemKey":"$gen_thresholds_Diagnostic logs_1","sortOrder":2}]},"customWidth":"66","name":"query - 14"},{"type":3,"content":{"version":"KqlItem/1.0","query":"union withsource=_TableName *\\r\\n| where _TableName == \\"AzureDiagnostics\\" and Category == \\"kube-audit\\"\\r\\n| summarize count() by ResourceId = tolower(ResourceId)\\r\\n| summarize logsClusters = make_set(ResourceId)\\r\\n| extend selectedClusters = \\"[{clustername}]\\"\\r\\n| extend selectedClusters = replace(\\"\'\\", \'\\"\', selectedClusters)\\r\\n| extend selectedClusters = todynamic(selectedClusters)\\r\\n| mv-expand clusterId = selectedClusters\\r\\n| project clusterId = toupper(tostring(clusterId)), hasDiagnosticLogs = (logsClusters has tostring(clusterId))\\r\\n| summarize [\\"number of clusters\\"] = count() by hasDiagnosticLogs\\r\\n| extend hasDiagnosticLogs = iff(hasDiagnosticLogs == true, \\"Clusters with Diagnostic logs\\", \\"Clusters without Diagnostic logs\\")\\r\\n","size":0,"timeContext":{"durationMs":172800000},"timeContextFromParameter":"timeframe","queryType":0,"resourceType":"microsoft.operationalinsights/workspaces","crossComponentResources":["{workspaces}"],"visualization":"piechart"},"customWidth":"33","name":"query - 17"},{"type":12,"content":{"version":"NotebookGroup/1.0","groupType":"editable","items":[{"type":1,"content":{"json":"## Cluster operations"},"name":"text - 16"},{"type":11,"content":{"version":"LinkItem/1.0","style":"tabs","links":[{"id":"3f616701-fd4b-482c-aff1-a85414daa05c","cellValue":"dispalyedGraph","linkTarget":"parameter","linkLabel":"Masterclient operations","subTarget":"masterclient","preText":"","style":"link"},{"id":"e6fa55f1-7d57-4f5e-8e83-429740853731","cellValue":"dispalyedGraph","linkTarget":"parameter","linkLabel":"Pod creation operations","subTarget":"podCreation","style":"link"},{"id":"f4c46251-0090-4ca3-a81c-0686bff3ff35","cellValue":"dispalyedGraph","linkTarget":"parameter","linkLabel":"Secret get\\\\list operations","subTarget":"secretOperation","style":"link"}]},"name":"links - 11"},{"type":3,"content":{"version":"KqlItem/1.0","query":"AzureDiagnostics \\r\\n| where tolower(ResourceId) in ({clustername})\\r\\n| where TimeGenerated {timeframe}\\r\\n| where Category == \\"kube-audit\\"\\r\\n| where log_s has \\"masterclient\\"\\r\\n| project TimeGenerated, parse_json(log_s), ResourceId\\r\\n| project TimeGenerated, ResourceId, username = tostring(log_s[\\"user\\"].username)\\r\\n| where username == \\"masterclient\\"\\r\\n| extend name = extract(@\\"/MICROSOFT.CONTAINERSERVICE/MANAGEDCLUSTERS/(.+)\\", 1, ResourceId)\\r\\n| summarize count() by name, bin(TimeGenerated, 1h)","size":0,"queryType":0,"resourceType":"microsoft.operationalinsights/workspaces","crossComponentResources":["{workspaces}"],"visualization":"timechart","tileSettings":{"showBorder":false,"titleContent":{"columnMatch":"name","formatter":1},"leftContent":{"columnMatch":"count_","formatter":12,"formatOptions":{"palette":"auto"},"numberFormat":{"unit":17,"options":{"maximumSignificantDigits":3,"maximumFractionDigits":2}}}},"chartSettings":{"yAxis":["count_"]}},"conditionalVisibility":{"parameterName":"dispalyedGraph","comparison":"isEqualTo","value":"masterclient"},"name":"Masterclient operations","styleSettings":{"showBorder":true}},{"type":3,"content":{"version":"KqlItem/1.0","query":"AzureDiagnostics\\r\\n| where Category == \\"kube-audit\\"\\r\\n| where tolower(ResourceId) in ({clustername})\\r\\n| where TimeGenerated {timeframe}\\r\\n| where log_s has \\"pods\\"\\r\\n| project TimeGenerated, parse_json(log_s), ResourceId\\r\\n| project AzureResourceId = ResourceId, TimeGenerated,\\r\\n RequestURI = tostring(log_s[\\"requestURI\\"]),\\r\\n Verb = tostring(log_s[\\"verb\\"]),\\r\\n ObjectRef = log_s[\\"objectRef\\"],\\r\\n ResponseStatus = log_s[\\"responseStatus\\"]\\r\\n//Main query\\r\\n| where ObjectRef.resource == \\"pods\\" and Verb == \\"create\\" and ResponseStatus.code startswith \\"20\\"\\r\\n and RequestURI endswith \\"/pods\\"\\r\\n| extend name = extract(@\\"/MICROSOFT.CONTAINERSERVICE/MANAGEDCLUSTERS/(.+)\\", 1, AzureResourceId)\\r\\n| summarize count() by name, bin(TimeGenerated, 1h)","size":0,"queryType":0,"resourceType":"microsoft.operationalinsights/workspaces","crossComponentResources":["{workspaces}"],"visualization":"timechart"},"conditionalVisibility":{"parameterName":"dispalyedGraph","comparison":"isEqualTo","value":"podCreation"},"name":"pods creation","styleSettings":{"showBorder":true}},{"type":3,"content":{"version":"KqlItem/1.0","query":"AzureDiagnostics\\r\\n| where Category == \\"kube-audit\\"\\r\\n| where tolower(ResourceId) in ({clustername})\\r\\n| where TimeGenerated {timeframe}\\r\\n| where log_s has \\"secrets\\"\\r\\n| project TimeGenerated, parse_json(log_s), ResourceId\\r\\n| project AzureResourceId = ResourceId, TimeGenerated,\\r\\n RequestURI = tostring(log_s[\\"requestURI\\"]),\\r\\n Verb = tostring(log_s[\\"verb\\"]),\\r\\n ObjectRef = log_s[\\"objectRef\\"],\\r\\n ResponseStatus = log_s[\\"responseStatus\\"]\\r\\n//Main query\\r\\n| where ObjectRef.resource == \\"secrets\\" and (Verb == \\"list\\" or Verb == \\"get\\") and ResponseStatus.code startswith \\"20\\"\\r\\n| where ObjectRef.name != \\"tunnelfront\\" and ObjectRef.name != \\"tunnelend\\" and ObjectRef.name != \\"kubernetes-dashboard-key-holder\\"\\r\\n| extend name = extract(@\\"/MICROSOFT.CONTAINERSERVICE/MANAGEDCLUSTERS/(.+)\\", 1, AzureResourceId)\\r\\n| summarize count() by name, bin(TimeGenerated, 1h)","size":0,"timeContext":{"durationMs":172800000},"timeContextFromParameter":"timeframe","queryType":0,"resourceType":"microsoft.operationalinsights/workspaces","crossComponentResources":["{workspaces}"],"visualization":"timechart","gridSettings":{"sortBy":[{"itemKey":"count_","sortOrder":2}]},"sortBy":[{"itemKey":"count_","sortOrder":2}]},"conditionalVisibility":{"parameterName":"dispalyedGraph","comparison":"isEqualTo","value":"secretOperation"},"name":"secrets operation","styleSettings":{"showBorder":true}},{"type":3,"content":{"version":"KqlItem/1.0","query":"let ascAlerts = \\nunion withsource=_TableName *\\n| where _TableName == \\"SecurityAlert\\"\\n| where tolower(ResourceId) in ({clustername})\\n| where TimeGenerated {timeframe}\\n| extend AlertType = column_ifexists(\\"AlertType\\", \\"\\")\\n| where AlertType == \\"AKS_PrivilegedContainer\\"\\n| extend ExtendedProperties = column_ifexists(\\"ExtendedProperties\\", todynamic(\\"\\"))\\n| extend ExtendedProperties = parse_json(ExtendedProperties)\\n| extend AlertLink = column_ifexists(\\"AlertLink\\", \\"\\")\\n| summarize arg_min(TimeGenerated, AlertLink) by AzureResourceId = ResourceId, name = tostring(ExtendedProperties[\\"Pod name\\"]), podNamespace = tostring(ExtendedProperties[\\"Namespace\\"])\\n;\\nlet podOperations = AzureDiagnostics\\n| where Category == \\"kube-audit\\"\\n| where tolower(ResourceId) in ({clustername})\\n| where TimeGenerated {timeframe}\\n| where log_s has \\"privileged\\"\\n| project TimeGenerated, parse_json(log_s), ResourceId\\n| project AzureResourceId = ResourceId, TimeGenerated,\\n RequestURI = tostring(log_s[\\"requestURI\\"]),\\n Verb = tostring(log_s[\\"verb\\"]),\\n ObjectRef = log_s[\\"objectRef\\"],\\n RequestObject = log_s[\\"requestObject\\"],\\n ResponseStatus = log_s[\\"responseStatus\\"],\\n ResponseObject = log_s[\\"responseObject\\"]\\n//Main query\\n| where ObjectRef.resource == \\"pods\\" and Verb == \\"create\\" and ResponseStatus.code startswith \\"20\\" and RequestObject has \\"privileged\\"\\n and RequestURI endswith \\"/pods\\"\\n| extend containers = RequestObject.spec.containers\\n| mvexpand containers\\n| where containers.securityContext.privileged == true\\n| summarize TimeGenerated = min(TimeGenerated) by\\n name = tostring(ResponseObject.metadata.name),\\n podNamespace = tostring(ResponseObject.metadata.namespace),\\n imageName = tostring(containers.image),\\n containerName = tostring(containers.name),\\n AzureResourceId\\n| extend id = strcat(name,\\";\\", AzureResourceId)\\n| extend parent = AzureResourceId\\n| join kind=leftouter (ascAlerts) on AzureResourceId, name, podNamespace\\n;\\nlet cached = materialize(podOperations)\\n;\\nlet clusters = cached | distinct AzureResourceId\\n;\\n// Main query\\ncached\\n| union\\n(\\nclusters\\n| project \\n name = AzureResourceId,\\n id = AzureResourceId,\\n parent = \\"\\" \\n)\\n| project-away name1, podNamespace1, TimeGenerated1","size":1,"title":"Privileged containers creation","queryType":0,"resourceType":"microsoft.operationalinsights/workspaces","crossComponentResources":["{workspaces}"],"visualization":"table","gridSettings":{"formatters":[{"columnMatch":"name","formatter":13,"formatOptions":{"linkTarget":null,"showIcon":true}},{"columnMatch":"AzureResourceId","formatter":5},{"columnMatch":"id","formatter":5},{"columnMatch":"parent","formatter":5},{"columnMatch":"AlertLink","formatter":7,"formatOptions":{"linkTarget":"Url","linkLabel":""}}],"hierarchySettings":{"idColumn":"id","parentColumn":"parent","treeType":0,"expanderColumn":"name"}},"sortBy":[]},"customWidth":"66","name":"Privileged container","styleSettings":{"showBorder":true}},{"type":3,"content":{"version":"KqlItem/1.0","query":"SecurityAlert \\r\\n| where tolower(ResourceId) in ({clustername})\\r\\n| where TimeGenerated {timeframe}\\r\\n| where AlertType == \\"AKS_PrivilegedContainer\\"\\r\\n| project name = extract(@\\"/MICROSOFT.CONTAINERSERVICE/MANAGEDCLUSTERS/(.+)\\", 1, ResourceId)\\r\\n| summarize alert = count() by name","size":1,"title":"AKS clusters with related Microsoft Defender alerts","queryType":0,"resourceType":"microsoft.operationalinsights/workspaces","crossComponentResources":["{workspaces}"],"visualization":"piechart","tileSettings":{"showBorder":false,"titleContent":{"columnMatch":"name","formatter":1},"leftContent":{"columnMatch":"alert","formatter":12,"formatOptions":{"palette":"auto"},"numberFormat":{"unit":17,"options":{"maximumSignificantDigits":3,"maximumFractionDigits":2}}}},"graphSettings":{"type":0,"topContent":{"columnMatch":"name","formatter":1},"centerContent":{"columnMatch":"alert","formatter":1,"numberFormat":{"unit":17,"options":{"maximumSignificantDigits":3,"maximumFractionDigits":2}}}}},"customWidth":"33","name":"query - 7","styleSettings":{"showBorder":true}},{"type":3,"content":{"version":"KqlItem/1.0","query":"let baseQuery = AzureDiagnostics \\r\\n| where Category == \\"kube-audit\\"\\r\\n| where tolower(ResourceId) in ({clustername})\\r\\n| where TimeGenerated {timeframe}\\r\\n| where log_s has \\"exec\\"\\r\\n| project TimeGenerated, parse_json(log_s), ResourceId\\r\\n| project TimeGenerated,\\r\\n AzureResourceId = ResourceId,\\r\\n User = log_s[\\"user\\"],\\r\\n StageTimestamp = todatetime(log_s[\\"stageTimestamp\\"]),\\r\\n Timestamp = todatetime(log_s[\\"timestamp\\"]),\\r\\n Stage = tostring(log_s[\\"stage\\"]),\\r\\n RequestURI = tostring(log_s[\\"requestURI\\"]),\\r\\n UserAgent = tostring(log_s[\\"userAgent\\"]),\\r\\n Verb = tostring(log_s[\\"verb\\"]),\\r\\n ObjectRef = log_s[\\"objectRef\\"],\\r\\n ResponseStatus = log_s[\\"responseStatus\\"]\\r\\n| where ObjectRef.resource == \\"pods\\" and Verb == \\"create\\" and ResponseStatus.code == 101 and ObjectRef.subresource == \\"exec\\"\\r\\n| project operationTime = TimeGenerated,\\r\\n RequestURI,\\r\\n podName = tostring(ObjectRef.name),\\r\\n podNamespace = tostring(ObjectRef.namespace),\\r\\n username = tostring(User.username),\\r\\n AzureResourceId\\r\\n// Parse the exec command\\r\\n| extend commands = extractall(@\\"command=([^\\\\&]*)\\", RequestURI)\\r\\n| extend commandsStr = url_decode(strcat_array(commands, \\" \\"))\\r\\n| project-away [\'commands\'], RequestURI\\r\\n| where username != \\"aksProblemDetector\\"\\r\\n;\\r\\nlet cached = materialize(baseQuery);\\r\\nlet execOperations = \\r\\ncached\\r\\n| summarize operationTime = min(operationTime), numberOfPerations = count() by name = commandsStr, username, podNamespace, podName, AzureResourceId\\r\\n| extend id = name\\r\\n| extend parentId = podName\\r\\n| project id, parentId, name, operationTime, numberOfPerations, podNamespace, username, AzureResourceId\\r\\n;\\r\\nlet podOperations = \\r\\ncached\\r\\n| summarize operationTime = min(operationTime), numberOfPerations = count() by name = podName, podNamespace, AzureResourceId\\r\\n| extend id = name\\r\\n| extend parentId = AzureResourceId\\r\\n| project id, parentId, name, operationTime, numberOfPerations, podNamespace, username = \\"\\", AzureResourceId\\r\\n;\\r\\nlet clusterOperations = \\r\\ncached\\r\\n| summarize operationTime = min(operationTime), numberOfPerations = count() by name = AzureResourceId\\r\\n| extend id = name\\r\\n| extend parentId = \\"\\"\\r\\n| project id, parentId, name, operationTime, numberOfPerations, username = \\"\\", podNamespace = \\"\\", AzureResourceId = name\\r\\n;\\r\\nunion clusterOperations, podOperations, execOperations","size":1,"title":"exec commands","queryType":0,"resourceType":"microsoft.operationalinsights/workspaces","crossComponentResources":["{workspaces}"],"visualization":"table","gridSettings":{"formatters":[{"columnMatch":"id","formatter":5},{"columnMatch":"parentId","formatter":5},{"columnMatch":"numberOfPerations","formatter":4,"formatOptions":{"palette":"blue","compositeBarSettings":{"labelText":"","columnSettings":[]}}},{"columnMatch":"AzureResourceId","formatter":5}],"hierarchySettings":{"idColumn":"id","parentColumn":"parentId","treeType":0,"expanderColumn":"name","expandTopLevel":false}}},"customWidth":"33","name":"exec commands","styleSettings":{"showBorder":true}},{"type":3,"content":{"version":"KqlItem/1.0","query":"SecurityAlert \\r\\n| where AlertType == \\"AKS_MaliciousContainerExec\\"\\r\\n| where tolower(ResourceId) in ({clustername})\\r\\n| where TimeGenerated {timeframe}\\r\\n| project TimeGenerated, ResourceId, ExtendedProperties = todynamic(ExtendedProperties)\\r\\n| project TimeGenerated, ResourceId, [\\"Pod name\\"] = ExtendedProperties[\\"Pod name\\"], Command = ExtendedProperties[\\"Command\\"]","size":1,"title":"Related Microsoft Defender alerts details","queryType":0,"resourceType":"microsoft.operationalinsights/workspaces","crossComponentResources":["{workspaces}"],"gridSettings":{"sortBy":[{"itemKey":"TimeGenerated","sortOrder":1}]},"sortBy":[{"itemKey":"TimeGenerated","sortOrder":1}]},"customWidth":"33","name":"query - 9","styleSettings":{"showBorder":true}},{"type":3,"content":{"version":"KqlItem/1.0","query":"SecurityAlert \\r\\n| where AlertType == \\"AKS_MaliciousContainerExec\\"\\r\\n| where tolower(ResourceId) in ({clustername})\\r\\n| where TimeGenerated {timeframe}\\r\\n| project name = extract(@\\"/MICROSOFT.CONTAINERSERVICE/MANAGEDCLUSTERS/(.+)\\", 1, ResourceId)\\r\\n| summarize alert = count() by name","size":1,"title":"AKS clusters with related Microsoft Defender alerts","queryType":0,"resourceType":"microsoft.operationalinsights/workspaces","crossComponentResources":["{workspaces}"],"visualization":"piechart"},"customWidth":"33","name":"query - 8","styleSettings":{"showBorder":true}},{"type":3,"content":{"version":"KqlItem/1.0","query":"let ascAlerts = \\r\\nunion withsource=_TableName *\\r\\n| where _TableName == \\"SecurityAlert\\"\\r\\n| where tolower(ResourceId) in ({clustername})\\r\\n| where TimeGenerated {timeframe}\\r\\n| extend AlertType = column_ifexists(\\"AlertType\\", \\"\\")\\r\\n| where AlertType == \\"AKS_SensitiveMount\\"\\r\\n| extend ExtendedProperties = column_ifexists(\\"ExtendedProperties\\", todynamic(\\"\\"))\\r\\n| extend ExtendedProperties = parse_json(ExtendedProperties)\\r\\n| extend AlertLink = column_ifexists(\\"AlertLink\\", \\"\\")\\r\\n| summarize arg_min(TimeGenerated, AlertLink) by AzureResourceId = ResourceId, containerName = tostring(ExtendedProperties[\\"Container name\\"]), mountPath = tostring(ExtendedProperties[\\"Sensitive mount path\\"])\\r\\n;\\r\\nlet podOperations = \\r\\nAzureDiagnostics \\r\\n| where tolower(ResourceId) in ({clustername})\\r\\n| where TimeGenerated {timeframe}\\r\\n| where Category == \\"kube-audit\\"\\r\\n| where log_s has \\"hostPath\\"\\r\\n| project TimeGenerated, parse_json(log_s), ResourceId\\r\\n//Parsing\\r\\n| project AzureResourceId = ResourceId, TimeGenerated,\\r\\n Verb = tostring(log_s[\\"verb\\"]),\\r\\n ObjectRef = log_s[\\"objectRef\\"],\\r\\n RequestObject = log_s[\\"requestObject\\"],\\r\\n ResponseStatus = log_s[\\"responseStatus\\"],\\r\\n ResponseObject = log_s[\\"responseObject\\"]\\r\\n//\\r\\n//Main query\\r\\n//\\r\\n| where ObjectRef.resource == \\"pods\\" and Verb == \\"create\\" and ResponseStatus.code startswith \\"20\\" and RequestObject has \\"hostPath\\"\\r\\n| extend volumes = RequestObject.spec.volumes\\r\\n| mvexpand volumes\\r\\n| extend mountPath = volumes.hostPath.path\\r\\n| where mountPath != \\"\\" \\r\\n| extend container = RequestObject.spec.containers\\r\\n| mvexpand container\\r\\n| extend detectionTime = TimeGenerated\\r\\n| project detectionTime,\\r\\n podName = ResponseObject.metadata.name,\\r\\n podNamespace = ResponseObject.metadata.namespace,\\r\\n containerName = container.name,\\r\\n containerImage = container.image,\\r\\n mountPath,\\r\\n mountName = volumes.name,\\r\\n AzureResourceId,\\r\\n container\\r\\n| extend volumeMounts = container.volumeMounts\\r\\n| mv-expand volumeMounts\\r\\n| where tostring(volumeMounts.name) == tostring(mountName)\\r\\n| summarize operationTime = min(detectionTime) by AzureResourceId, name = tostring(podName),tostring(podNamespace), tostring(containerName), tostring(containerImage), tostring(mountPath), tostring(mountName)\\r\\n| extend id = strcat(name, \\";\\", AzureResourceId)\\r\\n| extend parent = AzureResourceId\\r\\n| join kind=leftouter (ascAlerts) on AzureResourceId, containerName, mountPath\\r\\n;\\r\\nlet cached = materialize(podOperations)\\r\\n;\\r\\nlet clusters = cached | distinct AzureResourceId\\r\\n;\\r\\n// Main query\\r\\ncached\\r\\n| union\\r\\n(\\r\\nclusters\\r\\n| project \\r\\n name = toupper(AzureResourceId),\\r\\n id = AzureResourceId,\\r\\n parent = \\"\\" \\r\\n)\\r\\n| project-away containerName1, mountPath1, TimeGenerated\\r\\n","size":1,"title":"hostPath mount","queryType":0,"resourceType":"microsoft.operationalinsights/workspaces","crossComponentResources":["{workspaces}"],"gridSettings":{"formatters":[{"columnMatch":"AzureResourceId","formatter":5},{"columnMatch":"name","formatter":13,"formatOptions":{"linkTarget":null,"showIcon":true}},{"columnMatch":"id","formatter":5},{"columnMatch":"parent","formatter":5},{"columnMatch":"AzureResourceId1","formatter":5},{"columnMatch":"AlertLink","formatter":7,"formatOptions":{"linkTarget":"Url"}}],"hierarchySettings":{"idColumn":"id","parentColumn":"parent","treeType":0,"expanderColumn":"name"}},"sortBy":[]},"customWidth":"66","name":"query - 10","styleSettings":{"showBorder":true}},{"type":3,"content":{"version":"KqlItem/1.0","query":"SecurityAlert \\r\\n| where AlertType == \\"AKS_SensitiveMount\\"\\r\\n| where tolower(ResourceId) in ({clustername})\\r\\n| where TimeGenerated {timeframe}\\r\\n| project name = extract(@\\"/MICROSOFT.CONTAINERSERVICE/MANAGEDCLUSTERS/(.+)\\", 1, ResourceId)\\r\\n| summarize alert = count() by name","size":1,"title":"AKS clusters with related Microsoft Defender alerts","queryType":0,"resourceType":"microsoft.operationalinsights/workspaces","crossComponentResources":["{workspaces}"],"visualization":"piechart","sortBy":[]},"customWidth":"33","name":"query - 10","styleSettings":{"showBorder":true}},{"type":3,"content":{"version":"KqlItem/1.0","query":"let bindingOper = AzureDiagnostics\\r\\n| where Category == \\"kube-audit\\"\\r\\n| where tolower(ResourceId) in ({clustername})\\r\\n| where TimeGenerated {timeframe}\\r\\n| where log_s has \\"clusterrolebindings\\"\\r\\n| project TimeGenerated, parse_json(log_s), ResourceId\\r\\n//Parsing\\r\\n| project AzureResourceId = ResourceId, TimeGenerated,\\r\\n RequestURI = tostring(log_s[\\"requestURI\\"]),\\r\\n User = log_s[\\"user\\"],\\r\\n Verb = tostring(log_s[\\"verb\\"]),\\r\\n ObjectRef = log_s[\\"objectRef\\"],\\r\\n RequestObject = log_s[\\"requestObject\\"],\\r\\n ResponseStatus = log_s[\\"responseStatus\\"]\\r\\n| where ObjectRef.resource == \\"clusterrolebindings\\" and Verb == \\"create\\" and ResponseStatus.code startswith \\"20\\" and RequestObject.roleRef.name == \\"cluster-admin\\" \\r\\n| extend subjects = RequestObject.subjects\\r\\n| mv-expand subjects\\r\\n| project AzureResourceId, TimeGenerated, subjectName = tostring(subjects.name), subjectKind = tostring(subjects[\\"kind\\"]), bindingName = tostring(ObjectRef.name)\\r\\n| summarize operationTime = min(TimeGenerated) by AzureResourceId, subjectName, subjectKind, bindingName\\r\\n| extend id = strcat(subjectName, \\";\\", AzureResourceId)\\r\\n| extend parent = AzureResourceId\\r\\n;\\r\\nlet cached = materialize(bindingOper)\\r\\n;\\r\\nlet clusters = cached | distinct AzureResourceId\\r\\n;\\r\\n// Main query\\r\\ncached\\r\\n| union\\r\\n(\\r\\nclusters\\r\\n| project \\r\\n name = AzureResourceId,\\r\\n id = AzureResourceId,\\r\\n parent = \\"\\" \\r\\n)","size":1,"title":"Cluster-admin binding","queryType":0,"resourceType":"microsoft.operationalinsights/workspaces","crossComponentResources":["{workspaces}"],"visualization":"table","gridSettings":{"formatters":[{"columnMatch":"AzureResourceId","formatter":5},{"columnMatch":"id","formatter":5},{"columnMatch":"parent","formatter":5},{"columnMatch":"name","formatter":13,"formatOptions":{"linkTarget":null,"showIcon":true}}],"hierarchySettings":{"idColumn":"id","parentColumn":"parent","treeType":0,"expanderColumn":"name"}}},"customWidth":"66","name":"query - 5","styleSettings":{"showBorder":true}},{"type":3,"content":{"version":"KqlItem/1.0","query":"SecurityAlert \\r\\n| where tolower(ResourceId) in ({clustername})\\r\\n| where TimeGenerated {timeframe}\\r\\n| where AlertType == \\"AKS_ClusterAdminBinding\\"\\r\\n| project name = extract(@\\"/MICROSOFT.CONTAINERSERVICE/MANAGEDCLUSTERS/(.+)\\", 1, ResourceId)\\r\\n| summarize count() by name","size":1,"title":"AKS clusters with related Microsoft Defender alerts","queryType":0,"resourceType":"microsoft.operationalinsights/workspaces","crossComponentResources":["{workspaces}"],"visualization":"piechart"},"customWidth":"33","name":"query - 11","styleSettings":{"showBorder":true}},{"type":3,"content":{"version":"KqlItem/1.0","query":"AzureDiagnostics\\r\\n| where Category == \\"kube-audit\\"\\r\\n| where tolower(ResourceId) in ({clustername})\\r\\n| where TimeGenerated {timeframe}\\r\\n| where log_s has \\"events\\"\\r\\n| project TimeGenerated, parse_json(log_s), ResourceId\\r\\n//Parsing\\r\\n| project AzureResourceId = ResourceId, \\r\\n TimeGenerated,\\r\\n SourceIPs = tostring(log_s[\\"sourceIPs\\"][0]),\\r\\n User = log_s[\\"user\\"],\\r\\n Verb = tostring(log_s[\\"verb\\"]),\\r\\n ObjectRef = log_s[\\"objectRef\\"],\\r\\n ResponseStatus = log_s[\\"responseStatus\\"]\\r\\n| where ObjectRef.resource == \\"events\\" and Verb == \\"delete\\" and ResponseStatus.code == 200\\r\\n| project TimeGenerated, AzureResourceId, username = tostring(User.username), ipAddr = tostring(SourceIPs), \\r\\n eventName = tostring(ObjectRef.name), eventNamespace = tostring(ObjectRef.namespace), status = tostring(ResponseStatus.code)\\r\\n| summarize operationTime = min(TimeGenerated), eventNames = make_set(eventName, 10) by\\r\\n AzureResourceId, \\r\\n eventNamespace,\\r\\n username,\\r\\n ipAddr\\r\\n// Format the list of the event names\\r\\n| extend eventNames = substring(eventNames, 1 , strlen(eventNames) - 2)\\r\\n| extend eventNames = replace(\'\\"\', \\"\\", eventNames)\\r\\n| extend eventNames = replace(\\",\\", \\", \\", eventNames)","size":1,"title":"Delete events","queryType":0,"resourceType":"microsoft.operationalinsights/workspaces","crossComponentResources":["{workspaces}"]},"name":"query - 6","styleSettings":{"showBorder":true}}]},"conditionalVisibility":{"parameterName":"diagnosticClusters","comparison":"isEqualTo","value":"yes"},"name":"diagnosticData"},{"type":1,"content":{"json":"No Diagnostic Logs data in the selected workspaces. \\r\\nTo enable Diagnostic Logs for your AKS cluster: Go to your AKS cluster --> Diagnostic settings --> Add diagnostic setting --> Select \\"kube-audit\\" and send the data to your workspace.\\r\\n\\r\\nGet more details here: https://learn.microsoft.com/azure/aks/view-master-logs","style":"info"},"conditionalVisibility":{"parameterName":"diagnosticClusters","comparison":"isEqualTo","value":"no"},"name":"text - 4"}]},"conditionalVisibility":{"parameterName":"mainTab","comparison":"isEqualTo","value":"diagnostics"},"name":"diagnostics"}],"fromTemplateId":"sentinel-AksWorkbook","$schema":"https://github.com/Microsoft/Application-Insights-Workbooks/blob/master/schema/workbook.json"}' + version: '1.0' + category: 'sentinel' + sourceId: law.id + tags: [ + 'AksSecurityWorkbook' + '1.2' + ] + } +} + resource kv_diagnosticSettings 'Microsoft.Insights/diagnosticSettings@2021-05-01-preview' = { scope: kv name: 'default' From 904feaad2730620ee89ca7fd73146bda4448c0ca Mon Sep 17 00:00:00 2001 From: Fernando Antivero Date: Tue, 11 Oct 2022 21:17:54 +0000 Subject: [PATCH 033/115] add operation management solution for key vault analytics --- cluster-stamp.bicep | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/cluster-stamp.bicep b/cluster-stamp.bicep index 89da3297..45248008 100644 --- a/cluster-stamp.bicep +++ b/cluster-stamp.bicep @@ -355,6 +355,20 @@ resource miwSecurityInsights 'Microsoft.Insights/workbooks@2022-04-01' = { } } +resource omsKeyVaultAnalytics 'Microsoft.OperationsManagement/solutions@2015-11-01-preview' = { + name: 'KeyVaultAnalytics(${law.name})' + location: location + properties: { + workspaceResourceId: law.id + } + plan: { + name: 'KeyVaultAnalytics(${law.name})' + product: 'OMSGallery/KeyVaultAnalytics' + promotionCode: '' + publisher: 'Microsoft' + } +} + resource kv_diagnosticSettings 'Microsoft.Insights/diagnosticSettings@2021-05-01-preview' = { scope: kv name: 'default' From aa56d6d5d9619b315f9086fd5ffd582cc8e43744 Mon Sep 17 00:00:00 2001 From: Fernando Antivero Date: Tue, 11 Oct 2022 21:20:31 +0000 Subject: [PATCH 034/115] add activity log alerts for the spoke rg --- cluster-stamp.bicep | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/cluster-stamp.bicep b/cluster-stamp.bicep index 45248008..0905f673 100644 --- a/cluster-stamp.bicep +++ b/cluster-stamp.bicep @@ -658,6 +658,33 @@ resource agw_diagnosticSettings 'Microsoft.Insights/diagnosticSettings@2021-05-0 } } +resource alaAllAzureAdvisorAlert 'Microsoft.Insights/activityLogAlerts@2020-10-01' = { + name: 'AllAzureAdvisorAlert' + location: 'Global' + properties: { + scopes: [ + resourceGroup().id + ] + condition: { + allOf: [ + { + field: 'category' + equals: 'Recommendation' + } + { + field: 'operationName' + equals: 'Microsoft.Advisor/recommendations/available/action' + } + ] + } + actions: { + actionGroups: [] + } + enabled: true + description: 'All azure advisor alerts' + } +} + /*** OUTPUTS ***/ output agwName string = agw.name From 3ec82d5efc8fb34002063664efe97c9107db567e Mon Sep 17 00:00:00 2001 From: Fernando Antivero Date: Tue, 11 Oct 2022 21:39:06 +0000 Subject: [PATCH 035/115] add existing spoke subnet for ops management --- cluster-stamp.bicep | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/cluster-stamp.bicep b/cluster-stamp.bicep index 0905f673..274c1ce6 100644 --- a/cluster-stamp.bicep +++ b/cluster-stamp.bicep @@ -101,6 +101,11 @@ resource targetVirtualNetwork 'Microsoft.Network/virtualNetworks@2022-01-01' exi resource snetPrivatelinkendpoints 'subnets' existing = { name: 'snet-privatelinkendpoints' } + + // spoke virtual network's subnet for managment ops + resource snetManagmentOps 'subnets' existing = { + name: 'snet-management-ops' + } } /*** EXISTING SUBSCRIPTION RESOURCES ***/ From 3f7ac3c4e17f878e49a0639d7651ab11e033c209 Mon Sep 17 00:00:00 2001 From: Fernando Antivero Date: Tue, 11 Oct 2022 21:39:55 +0000 Subject: [PATCH 036/115] add vmss for jumpboxes --- cluster-stamp.bicep | 91 +++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 91 insertions(+) diff --git a/cluster-stamp.bicep b/cluster-stamp.bicep index 274c1ce6..0284513a 100644 --- a/cluster-stamp.bicep +++ b/cluster-stamp.bicep @@ -78,6 +78,7 @@ param jumpBoxCloudInitAsBase64 string = '10.200.0.0/26' var subRgUniqueString = uniqueString('aks', subscription().subscriptionId, resourceGroup().id) var clusterName = 'aks-${subRgUniqueString}' +var jumpBoxDefaultAdminUserName = uniqueString(clusterName, resourceGroup().id) /*** EXISTING RESOURCE GROUP RESOURCES ***/ @@ -663,6 +664,96 @@ resource agw_diagnosticSettings 'Microsoft.Insights/diagnosticSettings@2021-05-0 } } +@description('The compute for operations jumpboxes; these machines are assigned to cluster operator users') +resource vmssJumpboxes 'Microsoft.Compute/virtualMachineScaleSets@2020-12-01' = { + name: 'vmss-jumpboxes' + location: location + zones: pickZones('Microsoft.Compute', 'virtualMachineScaleSets', location, 3) + sku: { + name: 'Standard_DS1_v2' + tier: 'Standard' + capacity: 2 + } + properties: { + additionalCapabilities: { + ultraSSDEnabled: false + } + overprovision: false + singlePlacementGroup: true + upgradePolicy: { + mode: 'Automatic' + } + zoneBalance: false + virtualMachineProfile: { + diagnosticsProfile: { + bootDiagnostics: { + enabled: true + } + } + osProfile: { + computerNamePrefix: 'aksjmp' + linuxConfiguration: { + disablePasswordAuthentication: true + provisionVMAgent: true + ssh: { + publicKeys: [ + { + path: '/home/${jumpBoxDefaultAdminUserName}/.ssh/authorized_keys' + keyData: 'ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQCcFvQl2lYPcK1tMB3Tx2R9n8a7w5MJCSef14x0ePRFr9XISWfCVCNKRLM3Al/JSlIoOVKoMsdw5farEgXkPDK5F+SKLss7whg2tohnQNQwQdXit1ZjgOXkis/uft98Cv8jDWPbhwYj+VH/Aif9rx8abfjbvwVWBGeA/OnvfVvXnr1EQfdLJgMTTh+hX/FCXCqsRkQcD91MbMCxpqk8nP6jmsxJBeLrgfOxjH8RHEdSp4fF76YsRFHCi7QOwTE/6U+DpssgQ8MTWRFRat97uTfcgzKe5MOfuZHZ++5WFBgaTr1vhmSbXteGiK7dQXOk2cLxSvKkzeaiju9Jy6hoSl5oMygUVd5fNPQ94QcqTkMxZ9tQ9vPWOHwbdLRD31Ses3IBtDV+S6ehraiXf/L/e0jRUYk8IL/J543gvhOZ0hj2sQqTj9XS2hZkstZtrB2ywrJzV5ByETUU/oF9OsysyFgnaQdyduVqEPHaqXqnJvBngqqas91plyT3tSLMez3iT0s= unused-generated-by-azure' + } + ] + } + } + customData: jumpBoxCloudInitAsBase64 + adminUsername: jumpBoxDefaultAdminUserName + } + storageProfile: { + osDisk: { + createOption: 'FromImage' + caching: 'ReadOnly' + diffDiskSettings: { + option: 'Local' + } + osType: 'Linux' + } + imageReference: { + id: jumpBoxImageResourceId + } + } + networkProfile: { + networkInterfaceConfigurations: [ + { + name: 'vnet-spoke-BU0001A0005-01-nic01' + properties: { + primary: true + enableIPForwarding: false + enableAcceleratedNetworking: false + networkSecurityGroup: null + ipConfigurations: [ + { + name: 'default' + properties: { + primary: true + privateIPAddressVersion: 'IPv4' + publicIPAddressConfiguration: null + subnet: { + id: targetVirtualNetwork::snetManagmentOps.id + } + } + } + ] + } + } + ] + } + } + } + dependsOn: [ + law + omsVmInsights + ] +} + resource alaAllAzureAdvisorAlert 'Microsoft.Insights/activityLogAlerts@2020-10-01' = { name: 'AllAzureAdvisorAlert' location: 'Global' From e8f703a390774d01077021d99cf2517c91ff682d Mon Sep 17 00:00:00 2001 From: Fernando Antivero Date: Tue, 11 Oct 2022 21:45:29 +0000 Subject: [PATCH 037/115] add oms agent vmss extension for linux machines --- cluster-stamp.bicep | 19 ++++++++++++++++++- 1 file changed, 18 insertions(+), 1 deletion(-) diff --git a/cluster-stamp.bicep b/cluster-stamp.bicep index 0284513a..1e8ab961 100644 --- a/cluster-stamp.bicep +++ b/cluster-stamp.bicep @@ -749,9 +749,26 @@ resource vmssJumpboxes 'Microsoft.Compute/virtualMachineScaleSets@2020-12-01' = } } dependsOn: [ - law omsVmInsights ] + + resource extOmsAgentForLinux 'extensions' = { + name: 'OMSExtension' + properties: { + publisher: 'Microsoft.EnterpriseCloud.Monitoring' + type: 'OmsAgentForLinux' + typeHandlerVersion: '1.13' + autoUpgradeMinorVersion: true + settings: { + stopOnMultipleConnections: true + azureResourceId: vmssJumpboxes.id + workspaceId: reference(law.id, '2020-10-01').customerId + } + protectedSettings: { + workspaceKey: listKeys(law.id, '2020-10-01').primarySharedKey + } + } + } } resource alaAllAzureAdvisorAlert 'Microsoft.Insights/activityLogAlerts@2020-10-01' = { From fdfdcca771886c8c1480044ba19d62144db92736 Mon Sep 17 00:00:00 2001 From: Fernando Antivero Date: Tue, 11 Oct 2022 21:48:38 +0000 Subject: [PATCH 038/115] add dependency agent vmss extension for linux machines --- cluster-stamp.bicep | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/cluster-stamp.bicep b/cluster-stamp.bicep index 1e8ab961..eba7b6ed 100644 --- a/cluster-stamp.bicep +++ b/cluster-stamp.bicep @@ -769,6 +769,19 @@ resource vmssJumpboxes 'Microsoft.Compute/virtualMachineScaleSets@2020-12-01' = } } } + + resource extDependencyAgentLinux 'extensions' = { + name: 'DependencyAgentLinux' + properties: { + publisher: 'Microsoft.Azure.Monitoring.DependencyAgent' + type: 'DependencyAgentLinux' + typeHandlerVersion: '9.10' + autoUpgradeMinorVersion: true + } + dependsOn: [ + extOmsAgentForLinux + ] + } } resource alaAllAzureAdvisorAlert 'Microsoft.Insights/activityLogAlerts@2020-10-01' = { From d37ac05ef0c189dc59ff8b87f73a33cb559c9c01 Mon Sep 17 00:00:00 2001 From: Fernando Antivero Date: Tue, 11 Oct 2022 22:00:27 +0000 Subject: [PATCH 039/115] add the pvt container registry for the regulated cluster --- cluster-stamp.bicep | 38 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 38 insertions(+) diff --git a/cluster-stamp.bicep b/cluster-stamp.bicep index eba7b6ed..375e18b4 100644 --- a/cluster-stamp.bicep +++ b/cluster-stamp.bicep @@ -784,6 +784,43 @@ resource vmssJumpboxes 'Microsoft.Compute/virtualMachineScaleSets@2020-12-01' = } } +@description('The private container registry for the aks regulated cluster.') +resource cr 'Microsoft.ContainerRegistry/registries@2022-02-01-preview' = { + name: 'craks${subRgUniqueString}' + location: location + sku: { + name: 'Premium' + } + properties: { + adminUserEnabled: false + networkRuleSet: { + defaultAction: 'Deny' + virtualNetworkRules: [] + ipRules: [] + } + policies: { + quarantinePolicy: { + status: 'disabled' + } + trustPolicy: { + type: 'Notary' + status: 'enabled' + } + retentionPolicy: { + days: 15 + status: 'enabled' + } + } + publicNetworkAccess: 'Disabled' + encryption: { + status: 'disabled' + } + dataEndpointEnabled: true + networkRuleBypassOptions: 'AzureServices' + zoneRedundancy: 'Disabled' + } +} + resource alaAllAzureAdvisorAlert 'Microsoft.Insights/activityLogAlerts@2020-10-01' = { name: 'AllAzureAdvisorAlert' location: 'Global' @@ -815,3 +852,4 @@ resource alaAllAzureAdvisorAlert 'Microsoft.Insights/activityLogAlerts@2020-10-0 output agwName string = agw.name output keyVaultName string = kv.name +output quarantineContainerRegistryName string = cr.name From 884dd72e0c779145076aaf167f972b6cc5a70d2c Mon Sep 17 00:00:00 2001 From: Fernando Antivero Date: Tue, 11 Oct 2022 22:02:54 +0000 Subject: [PATCH 040/115] add cr geo redundancy --- cluster-stamp.bicep | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/cluster-stamp.bicep b/cluster-stamp.bicep index 375e18b4..10bebd1b 100644 --- a/cluster-stamp.bicep +++ b/cluster-stamp.bicep @@ -819,6 +819,11 @@ resource cr 'Microsoft.ContainerRegistry/registries@2022-02-01-preview' = { networkRuleBypassOptions: 'AzureServices' zoneRedundancy: 'Disabled' } + + resource grl 'replications' = { + name: geoRedundancyLocation + location: geoRedundancyLocation + } } resource alaAllAzureAdvisorAlert 'Microsoft.Insights/activityLogAlerts@2020-10-01' = { From db36e28444c31cbaf0f22c962606aaee2ee85f56 Mon Sep 17 00:00:00 2001 From: Fernando Antivero Date: Tue, 11 Oct 2022 22:09:20 +0000 Subject: [PATCH 041/115] add private endpoint for cr enabling aks regulated cluster to connect privately with it --- cluster-stamp.bicep | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/cluster-stamp.bicep b/cluster-stamp.bicep index 10bebd1b..3e73eaad 100644 --- a/cluster-stamp.bicep +++ b/cluster-stamp.bicep @@ -826,6 +826,31 @@ resource cr 'Microsoft.ContainerRegistry/registries@2022-02-01-preview' = { } } +@description('The network interface in the spoke vnet that enables connecting privately the aks regulated cluster with cr.') +resource peCr 'Microsoft.Network/privateEndpoints@2022-05-01' = { + name: 'pe-${cr.name}' + location: location + properties: { + subnet: { + id: targetVirtualNetwork::snetPrivatelinkendpoints.id + } + privateLinkServiceConnections: [ + { + name: 'to-${targetVirtualNetwork.name}' + properties: { + privateLinkServiceId: cr.id + groupIds: [ + 'registry' + ] + } + } + ] + } + dependsOn: [ + cr::grl + ] +} + resource alaAllAzureAdvisorAlert 'Microsoft.Insights/activityLogAlerts@2020-10-01' = { name: 'AllAzureAdvisorAlert' location: 'Global' From 48a4d561667814ef0007f8ab31cdf9d4c10f08a8 Mon Sep 17 00:00:00 2001 From: Fernando Antivero Date: Wed, 12 Oct 2022 13:08:43 +0000 Subject: [PATCH 042/115] bug fix: change kv pdz to existing --- cluster-stamp.bicep | 22 ++++------------------ 1 file changed, 4 insertions(+), 18 deletions(-) diff --git a/cluster-stamp.bicep b/cluster-stamp.bicep index 3e73eaad..c7f79679 100644 --- a/cluster-stamp.bicep +++ b/cluster-stamp.bicep @@ -109,6 +109,10 @@ resource targetVirtualNetwork 'Microsoft.Network/virtualNetworks@2022-01-01' exi } } +resource pdzKv 'Microsoft.Network/privateDnsZones@2020-06-01' existing = { + name: 'privatelink.vaultcore.azure.net' +} + /*** EXISTING SUBSCRIPTION RESOURCES ***/ @description('Built-in Azure RBAC role that must be applied to the kublet Managed Identity allowing it to further assign adding managed identities to the cluster\'s underlying VMSS.') @@ -395,24 +399,6 @@ resource kv_diagnosticSettings 'Microsoft.Insights/diagnosticSettings@2021-05-01 } } -@description('The kv private dns zone required by Private Link support.') -resource pdzKv 'Microsoft.Network/privateDnsZones@2020-06-01' = { - name: 'privatelink.vaultcore.azure.net' - location: 'global' - - // Enabling Azure Key Vault Private Link on the spoke vnet. - resource vnetlnk 'virtualNetworkLinks' = { - name: 'to_${targetVirtualNetwork.name}' - location: 'global' - properties: { - virtualNetwork: { - id: targetVirtualNetwork.id - } - registrationEnabled: false - } - } -} - @description('The network interface in the spoke vnet that enables connecting privately the aks regulated cluster with kv.') resource peKv 'Microsoft.Network/privateEndpoints@2022-01-01' = { name: 'pe-${kv.name}' From 8b829ba70e536745e6f761374a200168c9e07d1c Mon Sep 17 00:00:00 2001 From: Fernando Antivero Date: Wed, 12 Oct 2022 13:19:30 +0000 Subject: [PATCH 043/115] bug fix: use proper naming convention for kv pdzg --- cluster-stamp.bicep | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cluster-stamp.bicep b/cluster-stamp.bicep index c7f79679..70d9d3aa 100644 --- a/cluster-stamp.bicep +++ b/cluster-stamp.bicep @@ -421,7 +421,7 @@ resource peKv 'Microsoft.Network/privateEndpoints@2022-01-01' = { } resource pdzg 'privateDnsZoneGroups' = { - name: 'default' + name: 'for-${kv.name}' properties: { privateDnsZoneConfigs: [ { From c2e8fd5bddf0ab6f5eb13bbc0d91b660d8042de8 Mon Sep 17 00:00:00 2001 From: Fernando Antivero Date: Wed, 12 Oct 2022 13:33:58 +0000 Subject: [PATCH 044/115] add existing pdz for cr --- cluster-stamp.bicep | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/cluster-stamp.bicep b/cluster-stamp.bicep index 70d9d3aa..ef37f67a 100644 --- a/cluster-stamp.bicep +++ b/cluster-stamp.bicep @@ -113,6 +113,10 @@ resource pdzKv 'Microsoft.Network/privateDnsZones@2020-06-01' existing = { name: 'privatelink.vaultcore.azure.net' } +resource pdzCr 'Microsoft.Network/privateDnsZones@2020-06-01' existing = { + name: 'privatelink.azurecr.io' +} + /*** EXISTING SUBSCRIPTION RESOURCES ***/ @description('Built-in Azure RBAC role that must be applied to the kublet Managed Identity allowing it to further assign adding managed identities to the cluster\'s underlying VMSS.') From 9b8ec387054394a38d37e611ae5a8cd8a8ff4bd8 Mon Sep 17 00:00:00 2001 From: Fernando Antivero Date: Wed, 12 Oct 2022 14:24:36 +0000 Subject: [PATCH 045/115] group the cr private endpoint into the private dns zone for private link support --- cluster-stamp.bicep | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/cluster-stamp.bicep b/cluster-stamp.bicep index ef37f67a..8d235c62 100644 --- a/cluster-stamp.bicep +++ b/cluster-stamp.bicep @@ -839,6 +839,20 @@ resource peCr 'Microsoft.Network/privateEndpoints@2022-05-01' = { dependsOn: [ cr::grl ] + + resource pdzg 'privateDnsZoneGroups' = { + name: 'for-${cr.name}' + properties: { + privateDnsZoneConfigs: [ + { + name: 'privatelink-azurecr-io' + properties: { + privateDnsZoneId: pdzCr.id + } + } + ] + } + } } resource alaAllAzureAdvisorAlert 'Microsoft.Insights/activityLogAlerts@2020-10-01' = { From 191f7ade4967ad70677b466c8a90cd47834e64f8 Mon Sep 17 00:00:00 2001 From: Fernando Antivero Date: Wed, 12 Oct 2022 14:30:59 +0000 Subject: [PATCH 046/115] add scheduled query for images being imported from cr repositories different than quarantine/ --- cluster-stamp.bicep | 37 +++++++++++++++++++++++++++++++++++++ 1 file changed, 37 insertions(+) diff --git a/cluster-stamp.bicep b/cluster-stamp.bicep index 8d235c62..dc9294f9 100644 --- a/cluster-stamp.bicep +++ b/cluster-stamp.bicep @@ -855,6 +855,43 @@ resource peCr 'Microsoft.Network/privateEndpoints@2022-05-01' = { } } +@description('The scheduled query that returns images being imported from repositories different than quarantine/') +resource sqrNonQuarantineImportedImgesToCr 'Microsoft.Insights/scheduledQueryRules@2022-06-15' = { + name: 'Image Imported into ACR from ${cr.name} source other than approved Quarantine' + location: location + properties: { + description: 'The only images we want in live/ are those that came from this ACR instance, but from the quarantine/ repository.' + actions: { + actionGroups: [] + } + criteria: { + allOf: [ + { + operator: 'GreaterThan' + query: 'ContainerRegistryRepositoryEvents\r\n| where OperationName == "importImage" and Repository startswith "live/" and MediaType !startswith strcat(_ResourceId, "/quarantine")' + threshold: 0 + timeAggregation: 'Count' + dimensions: [] + failingPeriods: { + minFailingPeriodsToAlert: 1 + numberOfEvaluationPeriods: 1 + } + resourceIdColumn: '' + } + ] + } + enabled: true + evaluationFrequency: 'PT10M' + scopes: [ + cr.id + ] + severity: 3 + windowSize: 'PT10M' + muteActionsDuration: null + overrideQueryTimeRange: null + } +} + resource alaAllAzureAdvisorAlert 'Microsoft.Insights/activityLogAlerts@2020-10-01' = { name: 'AllAzureAdvisorAlert' location: 'Global' From 58f806a84afe37dade511608d77caf2311e5a861 Mon Sep 17 00:00:00 2001 From: Fernando Antivero Date: Wed, 12 Oct 2022 14:37:04 +0000 Subject: [PATCH 047/115] add existing spoke management cr agent pools subnet --- cluster-stamp.bicep | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/cluster-stamp.bicep b/cluster-stamp.bicep index dc9294f9..0524e224 100644 --- a/cluster-stamp.bicep +++ b/cluster-stamp.bicep @@ -107,6 +107,11 @@ resource targetVirtualNetwork 'Microsoft.Network/virtualNetworks@2022-01-01' exi resource snetManagmentOps 'subnets' existing = { name: 'snet-management-ops' } + + // spoke virtual network's subnet for managment cr agent pools + resource snetManagmentCrAgents 'subnets' existing = { + name: 'snet-management-acragents' + } } resource pdzKv 'Microsoft.Network/privateDnsZones@2020-06-01' existing = { From 1db23e61178c0a0837d3fe596900a6942a479552 Mon Sep 17 00:00:00 2001 From: Fernando Antivero Date: Wed, 12 Oct 2022 14:37:29 +0000 Subject: [PATCH 048/115] add cr agent pool --- cluster-stamp.bicep | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/cluster-stamp.bicep b/cluster-stamp.bicep index 0524e224..e2c218ca 100644 --- a/cluster-stamp.bicep +++ b/cluster-stamp.bicep @@ -819,6 +819,17 @@ resource cr 'Microsoft.ContainerRegistry/registries@2022-02-01-preview' = { name: geoRedundancyLocation location: geoRedundancyLocation } + + resource ap 'agentPools@2019-06-01-preview' = { + name: 'acragent' + location: location + properties: { + count: 1 + os: 'Linux' + tier: 'S1' + virtualNetworkSubnetResourceId: targetVirtualNetwork::snetManagmentCrAgents.id + } + } } @description('The network interface in the spoke vnet that enables connecting privately the aks regulated cluster with cr.') From 139993313623faeefcfc26b4e79db3d211aad58c Mon Sep 17 00:00:00 2001 From: Fernando Antivero Date: Wed, 12 Oct 2022 14:40:29 +0000 Subject: [PATCH 049/115] add diagnostic settings for cr --- cluster-stamp.bicep | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/cluster-stamp.bicep b/cluster-stamp.bicep index e2c218ca..cf3b5b1a 100644 --- a/cluster-stamp.bicep +++ b/cluster-stamp.bicep @@ -832,6 +832,31 @@ resource cr 'Microsoft.ContainerRegistry/registries@2022-02-01-preview' = { } } +resource cr_diagnosticSettings 'Microsoft.Insights/diagnosticSettings@2021-05-01-preview' = { + scope: cr + name: 'default' + properties: { + workspaceId: law.id + metrics: [ + { + timeGrain: 'PT1M' + category: 'AllMetrics' + enabled: true + } + ] + logs: [ + { + category: 'ContainerRegistryRepositoryEvents' + enabled: true + } + { + category: 'ContainerRegistryLoginEvents' + enabled: true + } + ] + } +} + @description('The network interface in the spoke vnet that enables connecting privately the aks regulated cluster with cr.') resource peCr 'Microsoft.Network/privateEndpoints@2022-05-01' = { name: 'pe-${cr.name}' From 60c9d506b13e20e916bc2db2f13ba5552e18b7db Mon Sep 17 00:00:00 2001 From: Fernando Antivero Date: Wed, 12 Oct 2022 14:42:09 +0000 Subject: [PATCH 050/115] ensure ContainerRegistryRepositoryEvents is enabled --- cluster-stamp.bicep | 3 +++ 1 file changed, 3 insertions(+) diff --git a/cluster-stamp.bicep b/cluster-stamp.bicep index cf3b5b1a..66babcc7 100644 --- a/cluster-stamp.bicep +++ b/cluster-stamp.bicep @@ -931,6 +931,9 @@ resource sqrNonQuarantineImportedImgesToCr 'Microsoft.Insights/scheduledQueryRul muteActionsDuration: null overrideQueryTimeRange: null } + dependsOn: [ + cr_diagnosticSettings + ] } resource alaAllAzureAdvisorAlert 'Microsoft.Insights/activityLogAlerts@2020-10-01' = { From cb52215cdf6f20bfeca8372be21e3b2d31dfb04b Mon Sep 17 00:00:00 2001 From: Fernando Antivero Date: Wed, 12 Oct 2022 17:46:38 +0000 Subject: [PATCH 051/115] add policy assignments --- cluster-stamp.bicep | 307 +++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 306 insertions(+), 1 deletion(-) diff --git a/cluster-stamp.bicep b/cluster-stamp.bicep index 66babcc7..15b384fd 100644 --- a/cluster-stamp.bicep +++ b/cluster-stamp.bicep @@ -79,6 +79,45 @@ param jumpBoxCloudInitAsBase64 string = '10.200.0.0/26' var subRgUniqueString = uniqueString('aks', subscription().subscriptionId, resourceGroup().id) var clusterName = 'aks-${subRgUniqueString}' var jumpBoxDefaultAdminUserName = uniqueString(clusterName, resourceGroup().id) +var crName = 'craks${subRgUniqueString}' + +/*** EXISTING TENANT RESOURCES ***/ + +@description('Built-in \'Kubernetes cluster pod security restricted standards for Linux-based workloads\' Azure Policy for Kubernetes initiative definition') +var psdAKSLinuxRestrictiveId = tenantResourceId('Microsoft.Authorization/policySetDefinitions', '42b8ef37-b724-4e24-bbc8-7a7708edfe00') + +@description('Built-in \'Kubernetes clusters should be accessible only over HTTPS\' Azure Policy for Kubernetes policy definition') +var pdEnforceHttpsIngressId = tenantResourceId('Microsoft.Authorization/policyDefinitions', '1a5b4dca-0b6f-4cf5-907c-56316bc1bf3d') + +@description('Built-in \'Kubernetes clusters should use internal load balancers\' Azure Policy for Kubernetes policy definition') +var pdEnforceInternalLoadBalancersId = tenantResourceId('Microsoft.Authorization/policyDefinitions', '3fc4dc25-5baf-40d8-9b05-7fe74c1bc64e') + +@description('Built-in \'Kubernetes cluster services should only use allowed external IPs\' Azure Policy for Kubernetes policy definition') +var pdAllowedExternalIPsId = tenantResourceId('Microsoft.Authorization/policyDefinitions', 'd46c275d-1680-448d-b2ec-e495a3b6cc89') + +@description('Built-in \'[Deprecated]: Kubernetes cluster containers should only listen on allowed ports\' Azure Policy policy definition') +var pdApprovedContainerPortsOnly = tenantResourceId('Microsoft.Authorization/policyDefinitions', '440b515e-a580-421e-abeb-b159a61ddcbc') + +@description('Built-in \'Kubernetes cluster services should listen only on allowed ports\' Azure Policy policy definition') +var pdApprovedServicePortsOnly = tenantResourceId('Microsoft.Authorization/policyDefinitions', '233a2a17-77ca-4fb1-9b6b-69223d272a44') + +@description('Built-in \'Kubernetes cluster pods should use specified labels\' Azure Policy policy definition') +var pdMustUseSpecifiedLabels = tenantResourceId('Microsoft.Authorization/policyDefinitions', '46592696-4c7b-4bf3-9e45-6c2763bdc0a6') + +@description('Built-in \'Kubernetes clusters should disable automounting API credentials\' Azure Policy policy definition') +var pdMustNotAutomountApiCreds = tenantResourceId('Microsoft.Authorization/policyDefinitions','423dd1ba-798e-40e4-9c4d-b6902674b423') + +@description('Built-in \'Kubernetes cluster containers should run with a read only root file systemv\' Azure Policy for Kubernetes policy definition') +var pdRoRootFilesystemId = tenantResourceId('Microsoft.Authorization/policyDefinitions', 'df49d893-a74c-421d-bc95-c663042e5b80') + +@description('Built-in \'Kubernetes clusters should not use the default namespace\' Azure Policy for Kubernetes policy definition') +var pdDisallowNamespaceUsageId = tenantResourceId('Microsoft.Authorization/policyDefinitions', '9f061a12-e40d-4183-a00e-171812443373') + +@description('Built-in \'AKS container CPU and memory resource limits should not exceed the specified limits\' Azure Policy for Kubernetes policy definition') +var pdEnforceResourceLimitsId = tenantResourceId('Microsoft.Authorization/policyDefinitions', 'e345eecc-fa47-480f-9e88-67dcc122b164') + +@description('Built-in \'AKS containers should only use allowed images\' Azure Policy for Kubernetes policy definition') +var pdEnforceImageSourceId = tenantResourceId('Microsoft.Authorization/policyDefinitions', 'febd0533-8e55-448f-b837-bd0e06f16469') /*** EXISTING RESOURCE GROUP RESOURCES ***/ @@ -781,7 +820,7 @@ resource vmssJumpboxes 'Microsoft.Compute/virtualMachineScaleSets@2020-12-01' = @description('The private container registry for the aks regulated cluster.') resource cr 'Microsoft.ContainerRegistry/registries@2022-02-01-preview' = { - name: 'craks${subRgUniqueString}' + name: crName location: location sku: { name: 'Premium' @@ -936,6 +975,272 @@ resource sqrNonQuarantineImportedImgesToCr 'Microsoft.Insights/scheduledQueryRul ] } +resource paAksLinuxRestrictive 'Microsoft.Authorization/policyAssignments@2021-06-01' = { + name: guid(psdAKSLinuxRestrictiveId, resourceGroup().name, clusterName) + properties: { + displayName: trim(take('[${clusterName}] ${reference(psdAKSLinuxRestrictiveId, '2020-09-01').displayName}', 125)) + scope: subscriptionResourceId('Microsoft.Resources/resourceGroups', resourceGroup().name) + policyDefinitionId: psdAKSLinuxRestrictiveId + parameters: { + effect: { + value: 'audit' + } + excludedNamespaces: { + value: [ + 'kube-system' + 'gatekeeper-system' + ] + } + } + } +} + +resource paEnforceHttpsIngress 'Microsoft.Authorization/policyAssignments@2020-09-01' = { + name: guid(pdEnforceHttpsIngressId, resourceGroup().name, clusterName) + properties: { + displayName: trim(take('[${clusterName}] ${reference(pdEnforceHttpsIngressId, '2020-09-01').displayName}', 125)) + scope: subscriptionResourceId('Microsoft.Resources/resourceGroups', resourceGroup().name) + policyDefinitionId: pdEnforceHttpsIngressId + parameters: { + effect: { + value: 'deny' + } + excludedNamespaces: { + value: [] + } + } + } +} + +resource paEnforceInternalLoadBalancers 'Microsoft.Authorization/policyAssignments@2020-09-01' = { + name: guid(pdEnforceInternalLoadBalancersId, resourceGroup().name, clusterName) + properties: { + displayName: trim(take('[${clusterName}] ${reference(pdEnforceInternalLoadBalancersId, '2020-09-01').displayName}', 125)) + scope: subscriptionResourceId('Microsoft.Resources/resourceGroups', resourceGroup().name) + policyDefinitionId: pdEnforceInternalLoadBalancersId + parameters: { + effect: { + value: 'deny' + } + excludedNamespaces: { + value: [] + } + } + } +} + +resource paMustNotAutomountApiCreds 'Microsoft.Authorization/policyAssignments@2020-09-01' = { + name: guid(pdMustNotAutomountApiCreds, resourceGroup().name, clusterName) + properties: { + displayName: trim(take('[${clusterName}] ${reference(pdMustNotAutomountApiCreds, '2020-09-01').displayName}', 125)) + scope: subscriptionResourceId('Microsoft.Resources/resourceGroups', resourceGroup().name) + policyDefinitionId: pdMustNotAutomountApiCreds + parameters: { + effect: { + value: 'deny' + } + excludedNamespaces: { + value: [ + 'kube-system' + 'gatekeeper-system' + 'flux-system' + 'falco-system' + 'osm-system' + 'ingress-nginx' + 'cluster-baseline-settings' + ] + } + } + } +} + +resource paMustUseSpecifiedLabels 'Microsoft.Authorization/policyAssignments@2020-09-01' = { + name: guid(pdMustUseSpecifiedLabels, resourceGroup().name, clusterName) + properties: { + displayName: trim(take('[${clusterName}] ${reference(pdMustUseSpecifiedLabels, '2020-09-01').displayName}', 125)) + scope: subscriptionResourceId('Microsoft.Resources/resourceGroups', resourceGroup().name) + policyDefinitionId: pdMustUseSpecifiedLabels + parameters: { + effect: { + value: 'audit' + } + labelsList: { + value: [ + 'pci-scope' + ] + } + } + } +} + +resource paMustUseTheseExternalIps 'Microsoft.Authorization/policyAssignments@2020-09-01' = { + name: guid(pdAllowedExternalIPsId, resourceGroup().name, clusterName) + properties: { + displayName: trim(take('[${clusterName}] ${reference(pdAllowedExternalIPsId, '2020-09-01').displayName}', 125)) + scope: subscriptionResourceId('Microsoft.Resources/resourceGroups', resourceGroup().name) + policyDefinitionId: pdAllowedExternalIPsId + parameters: { + effect: { + value: 'deny' + } + excludedNamespaces: { + value: [ + 'kube-system' + 'gatekeeper-system' + ] + } + allowedExternalIPs: { + value: [] + } + } + } +} + +resource paApprovedContainerPortsOnly 'Microsoft.Authorization/policyAssignments@2020-09-01' = { + name: guid(pdApprovedContainerPortsOnly, resourceGroup().name, clusterName) + properties: { + displayName: trim(take('[${clusterName}-a0005] ${reference(pdApprovedContainerPortsOnly, '2020-09-01').displayName}', 125)) + scope: subscriptionResourceId('Microsoft.Resources/resourceGroups', resourceGroup().name) + policyDefinitionId: pdApprovedContainerPortsOnly + parameters: { + effect: { + value: 'audit' + } + excludedNamespaces: { + value: [] + } + namespaces: { + value: [ + 'a0005-i' + 'a0005-o' + ] + } + allowedContainerPortsList: { + value: [ + '8080' + '15000' + '15003' + '15010' + ] + } + } + } +} + +resource paApprovedServicePortsOnly 'Microsoft.Authorization/policyAssignments@2020-09-01' = { + name: guid(pdApprovedServicePortsOnly, resourceGroup().name, clusterName) + properties: { + displayName: trim(take('[${clusterName}] ${reference(pdApprovedServicePortsOnly, '2020-09-01').displayName}', 125)) + scope: subscriptionResourceId('Microsoft.Resources/resourceGroups', resourceGroup().name) + policyDefinitionId: pdApprovedServicePortsOnly + parameters: { + effect: { + value: 'audit' + } + excludedNamespaces: { + value: [ + 'kube-system' + 'gatekeeper-system' + 'osm-system' + ] + } + allowedServicePortsList: { + value: [ + '443' + '80' + '8080' + ] + } + } + } +} + +resource paRoRootFilesystem 'Microsoft.Authorization/policyAssignments@2020-09-01' = { + name: guid(pdRoRootFilesystemId, resourceGroup().name, clusterName) + properties: { + displayName: trim(take('[${clusterName}] ${reference(pdRoRootFilesystemId, '2020-09-01').displayName}', 125)) + scope: subscriptionResourceId('Microsoft.Resources/resourceGroups', resourceGroup().name) + policyDefinitionId: pdRoRootFilesystemId + parameters: { + effect: { + value: 'audit' + } + excludedNamespaces: { + value: [ + 'kube-system' + 'gatekeeper-system' + ] + } + } + } +} + +resource paBlockDefaultNamespace 'Microsoft.Authorization/policyAssignments@2020-09-01' = { + name: guid(pdDisallowNamespaceUsageId, resourceGroup().name, clusterName) + properties: { + displayName: trim(take('[${clusterName}] ${reference(pdDisallowNamespaceUsageId, '2020-09-01').displayName}', 125)) + scope: subscriptionResourceId('Microsoft.Resources/resourceGroups', resourceGroup().name) + policyDefinitionId: pdDisallowNamespaceUsageId + parameters: { + effect: { + value: 'deny' + } + excludedNamespaces: { + value: [] + } + } + } +} + +resource paEnforceResourceLimits 'Microsoft.Authorization/policyAssignments@2020-09-01' = { + name: guid(pdEnforceResourceLimitsId, resourceGroup().name, clusterName) + properties: { + displayName: trim(take('[${clusterName}] ${reference(pdEnforceResourceLimitsId, '2020-09-01').displayName}', 125)) + scope: subscriptionResourceId('Microsoft.Resources/resourceGroups', resourceGroup().name) + policyDefinitionId: pdEnforceResourceLimitsId + parameters: { + effect: { + value: 'audit' + } + cpuLimit: { + value: '1500m' + } + memoryLimit: { + value: '2Gi' + } + excludedNamespaces: { + value: [ + 'kube-system' + 'gatekeeper-system' + ] + } + } + } +} + +resource paEnforceImageSource 'Microsoft.Authorization/policyAssignments@2020-03-01' = { + name: guid(pdEnforceImageSourceId, resourceGroup().name, clusterName) + properties: { + displayName: trim(take('[${clusterName}] ${reference(pdEnforceImageSourceId, '2020-09-01').displayName}', 125)) + scope: subscriptionResourceId('Microsoft.Resources/resourceGroups', resourceGroup().name) + policyDefinitionId: pdEnforceImageSourceId + parameters: { + allowedContainerImagesRegex: { + value: '${crName}\\.azurecr\\.io\\/live\\/.+$|mcr\\.microsoft\\.com\\/oss\\/(openservicemesh\\/init:|envoyproxy\\/envoy:).+$' + } + effect: { + value: 'deny' + } + excludedNamespaces: { + value: [ + 'kube-system' + 'gatekeeper-system' + ] + } + } + } +} + resource alaAllAzureAdvisorAlert 'Microsoft.Insights/activityLogAlerts@2020-10-01' = { name: 'AllAzureAdvisorAlert' location: 'Global' From c8a17be5dce0664c2ac9bf3a7792b4ffb911361e Mon Sep 17 00:00:00 2001 From: Fernando Antivero Date: Wed, 12 Oct 2022 19:16:57 +0000 Subject: [PATCH 052/115] ensure cluster control plane user assigned identity has all rbacs --- cluster-stamp.bicep | 10 ++ ...dentityHasRbacToSelfManagedResources.bicep | 123 ++++++++++++++++++ 2 files changed, 133 insertions(+) create mode 100644 modules/ensureClusterIdentityHasRbacToSelfManagedResources.bicep diff --git a/cluster-stamp.bicep b/cluster-stamp.bicep index 15b384fd..1532f477 100644 --- a/cluster-stamp.bicep +++ b/cluster-stamp.bicep @@ -1268,6 +1268,16 @@ resource alaAllAzureAdvisorAlert 'Microsoft.Insights/activityLogAlerts@2020-10-0 } } +module ensureClusterIdentityHasRbacToSelfManagedResources 'modules/ensureClusterIdentityHasRbacToSelfManagedResources.bicep' = { + name: 'ensureClusterIdentityHasRbacToSelfManagedResources' + scope: targetResourceGroup + params: { + miClusterControlPlanePrincipalId: miClusterControlPlane.properties.principalId + clusterControlPlaneIdentityName: miClusterControlPlane.name + targetVirtualNetworkName: targetVirtualNetwork.name + } +} + /*** OUTPUTS ***/ output agwName string = agw.name diff --git a/modules/ensureClusterIdentityHasRbacToSelfManagedResources.bicep b/modules/ensureClusterIdentityHasRbacToSelfManagedResources.bicep new file mode 100644 index 00000000..07f02824 --- /dev/null +++ b/modules/ensureClusterIdentityHasRbacToSelfManagedResources.bicep @@ -0,0 +1,123 @@ +targetScope = 'resourceGroup' + +/*** PARAMETERS ***/ + +@description('The AKS Control Plane Principal Id to be given with Network Contributor Role in different spoke subnets, so it can join VMSS and load balancers resources to them.') +@minLength(36) +@maxLength(36) +param miClusterControlPlanePrincipalId string + +@description('The AKS Control Plane Principal Name to be used to create unique role assignments names.') +@minLength(3) +@maxLength(128) +param clusterControlPlaneIdentityName string + +@description('The regional network spoke VNet Resource name that the cluster is being joined to, so it can be used to discover subnets during role assignments.') +@minLength(1) +param targetVirtualNetworkName string + +/*** EXISTING SUBSCRIPTION RESOURCES ***/ + +resource networkContributorRole 'Microsoft.Authorization/roleDefinitions@2022-04-01' existing = { + name: '4d97b98b-1d4f-4787-a291-c67834d212e7' + scope: subscription() +} + +resource dnsZoneContributorRole 'Microsoft.Authorization/roleDefinitions@2022-04-01' existing = { + name: 'b12aa53e-6015-4669-85d0-8515ebb3ae7f' + scope: subscription() +} + +/*** EXISTING SPOKE RESOURCES ***/ + +resource pdzCr 'Microsoft.Network/privateDnsZones@2020-06-01' existing = { + name: 'privatelink.azurecr.io' +} + +resource targetVirtualNetwork 'Microsoft.Network/virtualNetworks@2022-01-01' existing = { + name: targetVirtualNetworkName + + resource snetClusterSystemNodePools 'subnets' existing = { + name: 'snet-cluster-systemnodepool' + } + + resource snetClusterInScopeNodePools 'subnets' existing = { + name: 'snet-cluster-inscopenodepools' + } + + resource snetClusterOutofScopeNodePools 'subnets' existing = { + name: 'snet-cluster-outofscopenodepools' + } + + resource snetClusterIngressServices 'subnets' existing = { + name: 'snet-cluster-ingressservices' + } +} + +/*** RESOURCES ***/ + +resource vnetMiClusterControlPlaneDnsZoneContributorRole_roleAssignment 'Microsoft.Authorization/roleAssignments@2020-04-01-preview' = { + scope: targetVirtualNetwork + name: guid(targetVirtualNetwork.id, dnsZoneContributorRole.id, clusterControlPlaneIdentityName) + properties: { + roleDefinitionId: dnsZoneContributorRole.id + description: 'Allows cluster identity to attach custom DNS zone with Private Link information to this virtual network.' + principalId: miClusterControlPlanePrincipalId + principalType: 'ServicePrincipal' + } +} + +resource snetSystemNodePoolSubnetMiClusterControlPlaneNetworkContributorRole_roleAssignment 'Microsoft.Authorization/roleAssignments@2020-04-01-preview' = { + scope: targetVirtualNetwork::snetClusterSystemNodePools + name: guid(targetVirtualNetwork::snetClusterSystemNodePools.id, networkContributorRole.id, clusterControlPlaneIdentityName) + properties: { + roleDefinitionId: networkContributorRole.id + description: 'Allows cluster identity to join the nodepool vmss resources to this subnet.' + principalId: miClusterControlPlanePrincipalId + principalType: 'ServicePrincipal' + } +} + +resource snetInScopeNodePoolSubnetsnetSystemNodePoolSubnetMiClusterControlPlaneNetworkContributorRole_roleAssignment 'Microsoft.Authorization/roleAssignments@2020-04-01-preview' = { + scope: targetVirtualNetwork::snetClusterInScopeNodePools + name: guid(targetVirtualNetwork::snetClusterInScopeNodePools.id, networkContributorRole.id, clusterControlPlaneIdentityName) + properties: { + roleDefinitionId: networkContributorRole.id + description: 'Allows cluster identity to join the nodepool vmss resources to this subnet.' + principalId: miClusterControlPlanePrincipalId + principalType: 'ServicePrincipal' + } +} + +resource snetOutOfScopeNodePoolSubnetMiClusterControlPlaneNetworkContributorRole_roleAssignment 'Microsoft.Authorization/roleAssignments@2020-04-01-preview' = { + scope: targetVirtualNetwork::snetClusterOutofScopeNodePools + name: guid(targetVirtualNetwork::snetClusterOutofScopeNodePools.id, networkContributorRole.id, clusterControlPlaneIdentityName) + properties: { + roleDefinitionId: networkContributorRole.id + description: 'Allows cluster identity to join the nodepool vmss resources to this subnet.' + principalId: miClusterControlPlanePrincipalId + principalType: 'ServicePrincipal' + } +} + +resource snetIngressServicesSubnetMiClusterControlPlaneNetworkContributorRole_roleAssignment 'Microsoft.Authorization/roleAssignments@2020-04-01-preview' = { + scope: targetVirtualNetwork::snetClusterIngressServices + name: guid(targetVirtualNetwork::snetClusterIngressServices.id, networkContributorRole.id, clusterControlPlaneIdentityName) + properties: { + roleDefinitionId: networkContributorRole.id + description: 'Allows cluster identity to join load balancers (ingress resources) to this subnet.' + principalId: miClusterControlPlanePrincipalId + principalType: 'ServicePrincipal' + } +} + +resource pdzCrPrivatelinkAzmk8sIoMiClusterControlPlaneDnsZoneContributorRole_roleAssignment 'Microsoft.Authorization/roleAssignments@2020-04-01-preview' = { + scope: pdzCr + name: guid(pdzCr.id, dnsZoneContributorRole.id, clusterControlPlaneIdentityName) + properties: { + roleDefinitionId: dnsZoneContributorRole.id + description: 'Allows cluster identity to manage zone Entries for cluster\'s Private Link configuration.' + principalId: miClusterControlPlanePrincipalId + principalType: 'ServicePrincipal' + } +} From 9d52a2114766a1a5b7c9e9d4dcab3638b6e3c194 Mon Sep 17 00:00:00 2001 From: Fernando Antivero Date: Wed, 12 Oct 2022 19:19:55 +0000 Subject: [PATCH 053/115] add existing spoke cluster system node pools subnet --- cluster-stamp.bicep | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/cluster-stamp.bicep b/cluster-stamp.bicep index 1532f477..8fe46e25 100644 --- a/cluster-stamp.bicep +++ b/cluster-stamp.bicep @@ -151,6 +151,11 @@ resource targetVirtualNetwork 'Microsoft.Network/virtualNetworks@2022-01-01' exi resource snetManagmentCrAgents 'subnets' existing = { name: 'snet-management-acragents' } + + // spoke virtual network's subnet for cluster system node pools + resource snetClusterSystemNodePools 'subnets' existing = { + name: 'snet-cluster-systemnodepool' + } } resource pdzKv 'Microsoft.Network/privateDnsZones@2020-06-01' existing = { From c88b30853c68a95e6dd0e2c7be038a927f4222c0 Mon Sep 17 00:00:00 2001 From: Fernando Antivero Date: Wed, 12 Oct 2022 19:20:59 +0000 Subject: [PATCH 054/115] add existing virtual network's subnet for cluster in-scope node pools --- cluster-stamp.bicep | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/cluster-stamp.bicep b/cluster-stamp.bicep index 8fe46e25..9b938169 100644 --- a/cluster-stamp.bicep +++ b/cluster-stamp.bicep @@ -156,6 +156,12 @@ resource targetVirtualNetwork 'Microsoft.Network/virtualNetworks@2022-01-01' exi resource snetClusterSystemNodePools 'subnets' existing = { name: 'snet-cluster-systemnodepool' } + + // spoke virtual network's subnet for cluster in-scope node pools + resource snetClusterInScopeNodePools 'subnets' existing = { + name: 'snet-cluster-inscopenodepools' + } + } resource pdzKv 'Microsoft.Network/privateDnsZones@2020-06-01' existing = { From 76c0ea7c2a6af6a1f178c6b27e6e939e63bb918e Mon Sep 17 00:00:00 2001 From: Fernando Antivero Date: Thu, 13 Oct 2022 14:40:14 +0000 Subject: [PATCH 055/115] add k8s version --- cluster-stamp.bicep | 2 ++ 1 file changed, 2 insertions(+) diff --git a/cluster-stamp.bicep b/cluster-stamp.bicep index 9b938169..bf1b1a11 100644 --- a/cluster-stamp.bicep +++ b/cluster-stamp.bicep @@ -76,6 +76,8 @@ param jumpBoxCloudInitAsBase64 string = '10.200.0.0/26' /*** VARIABLES ***/ +var kubernetesVersion = '1.23.3' + var subRgUniqueString = uniqueString('aks', subscription().subscriptionId, resourceGroup().id) var clusterName = 'aks-${subRgUniqueString}' var jumpBoxDefaultAdminUserName = uniqueString(clusterName, resourceGroup().id) From ecb39b85e04fdf7301804eb9e14fe21a5a972d06 Mon Sep 17 00:00:00 2001 From: Fernando Antivero Date: Thu, 13 Oct 2022 14:40:54 +0000 Subject: [PATCH 056/115] add existing virtual network's subnet for cluster out-scope node pools --- cluster-stamp.bicep | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/cluster-stamp.bicep b/cluster-stamp.bicep index bf1b1a11..7e1e0a8b 100644 --- a/cluster-stamp.bicep +++ b/cluster-stamp.bicep @@ -164,6 +164,11 @@ resource targetVirtualNetwork 'Microsoft.Network/virtualNetworks@2022-01-01' exi name: 'snet-cluster-inscopenodepools' } + // spoke virtual network's subnet for cluster out-scope node pools + resource snetClusterOutScopeNodePools 'subnets' existing = { + name: 'snet-cluster-outofscopenodepools' + } + } resource pdzKv 'Microsoft.Network/privateDnsZones@2020-06-01' existing = { From b9f278a9763022546d4641ba7f063f8f6ae39041 Mon Sep 17 00:00:00 2001 From: Fernando Antivero Date: Thu, 13 Oct 2022 14:42:17 +0000 Subject: [PATCH 057/115] add existing private dns zone for aks regulated cluster --- cluster-stamp.bicep | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/cluster-stamp.bicep b/cluster-stamp.bicep index 7e1e0a8b..4a5ba03d 100644 --- a/cluster-stamp.bicep +++ b/cluster-stamp.bicep @@ -179,6 +179,10 @@ resource pdzCr 'Microsoft.Network/privateDnsZones@2020-06-01' existing = { name: 'privatelink.azurecr.io' } +resource pdzMc 'Microsoft.Network/privateDnsZones@2020-06-01' existing = { + name: 'privatelink.${location}.azmk8s.io' +} + /*** EXISTING SUBSCRIPTION RESOURCES ***/ @description('Built-in Azure RBAC role that must be applied to the kublet Managed Identity allowing it to further assign adding managed identities to the cluster\'s underlying VMSS.') From 1b7b3e3fec06beb0d11ca8ba570738680394a1ab Mon Sep 17 00:00:00 2001 From: Fernando Antivero Date: Thu, 13 Oct 2022 14:42:36 +0000 Subject: [PATCH 058/115] add aks regulated cluster --- cluster-stamp.bicep | 241 ++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 241 insertions(+) diff --git a/cluster-stamp.bicep b/cluster-stamp.bicep index 4a5ba03d..b53d6adf 100644 --- a/cluster-stamp.bicep +++ b/cluster-stamp.bicep @@ -1300,6 +1300,247 @@ module ensureClusterIdentityHasRbacToSelfManagedResources 'modules/ensureCluster } } +resource mc 'Microsoft.ContainerService/managedClusters@2021-05-01' = { + name: clusterName + location: location + tags: { + 'Data classification': 'Confidential' + 'Business unit': 'BU0001' + 'Business criticality': 'Business unit-critical' + } + properties: { + kubernetesVersion: kubernetesVersion + dnsPrefix: uniqueString(subscription().subscriptionId, resourceGroup().id, clusterName) + agentPoolProfiles: [ + { + name: 'npsystem' + count: 3 + vmSize: 'Standard_DS2_v2' + osDiskSizeGB: 80 + osDiskType: 'Ephemeral' + osType: 'Linux' + minCount: 3 + maxCount: 4 + vnetSubnetID: targetVirtualNetwork::snetClusterSystemNodePools.id + enableAutoScaling: true + type: 'VirtualMachineScaleSets' + mode: 'System' + scaleSetPriority: 'Regular' + scaleSetEvictionPolicy: 'Delete' + orchestratorVersion: kubernetesVersion + enableNodePublicIP: false + maxPods: 30 + availabilityZones: [ + '1' + '2' + '3' + ] + upgradeSettings: { + maxSurge: '33%' + } + tags: { + 'pci-scope': 'out-of-scope' + 'Data classification': 'Confidential' + 'Business unit': 'BU0001' + 'Business criticality': 'Business unit-critical' + } + } + { + name: 'npinscope01' + count: 2 + vmSize: 'Standard_DS3_v2' + osDiskSizeGB: 120 + osDiskType: 'Ephemeral' + osType: 'Linux' + minCount: 2 + maxCount: 5 + vnetSubnetID: targetVirtualNetwork::snetClusterInScopeNodePools.id + enableAutoScaling: true + type: 'VirtualMachineScaleSets' + mode: 'User' + scaleSetPriority: 'Regular' + scaleSetEvictionPolicy: 'Delete' + orchestratorVersion: kubernetesVersion + enableNodePublicIP: false + maxPods: 30 + availabilityZones: [ + '1' + '2' + '3' + ] + upgradeSettings: { + maxSurge: '33%' + } + nodeLabels: { + 'pci-scope': 'in-scope' + } + tags: { + 'pci-scope': 'in-scope' + 'Data classification': 'Confidential' + 'Business unit': 'BU0001' + 'Business criticality': 'Business unit-critical' + } + } + { + name: 'npooscope01' + count: 2 + vmSize: 'Standard_DS3_v2' + osDiskSizeGB: 120 + osDiskType: 'Ephemeral' + osType: 'Linux' + minCount: 2 + maxCount: 5 + vnetSubnetID: targetVirtualNetwork::snetClusterOutScopeNodePools.id + enableAutoScaling: true + type: 'VirtualMachineScaleSets' + mode: 'User' + scaleSetPriority: 'Regular' + scaleSetEvictionPolicy: 'Delete' + orchestratorVersion: kubernetesVersion + enableNodePublicIP: false + maxPods: 30 + availabilityZones: [ + '1' + '2' + '3' + ] + upgradeSettings: { + maxSurge: '33%' + } + nodeLabels: { + 'pci-scope': 'out-of-scope' + } + tags: { + 'pci-scope': 'out-of-scope' + 'Data classification': 'Confidential' + 'Business unit': 'BU0001' + 'Business criticality': 'Business unit-critical' + } + } + ] + servicePrincipalProfile: { + clientId: 'msi' + } + addonProfiles: { + httpApplicationRouting: { + enabled: false + } + omsagent: { + enabled: true + config: { + logAnalyticsWorkspaceResourceId: law.id + } + } + aciConnectorLinux: { + enabled: false + } + azurepolicy: { + enabled: true + config: { + version: 'v2' + } + } + openServiceMesh: { + enabled: true + config: { + } + } + azureKeyvaultSecretsProvider: { + enabled: true + config: { + enableSecretRotation: 'false' + } + } + } + nodeResourceGroup: 'rg-${clusterName}-nodepools' + enableRBAC: true + enablePodSecurityPolicy: false + maxAgentPools: 3 + networkProfile: { + networkPlugin: 'azure' + networkPolicy: 'azure' + outboundType: 'userDefinedRouting' + loadBalancerSku: 'standard' + loadBalancerProfile: json('null') + serviceCidr: '172.16.0.0/16' + dnsServiceIP: '172.16.0.10' + dockerBridgeCidr: '172.18.0.1/16' + } + aadProfile: { + managed: true + enableAzureRBAC: false + adminGroupObjectIDs: [ + clusterAdminAadGroupObjectId + ] + tenantID: k8sControlPlaneAuthorizationTenantId + } + autoScalerProfile: { + 'balance-similar-node-groups': 'false' + expander: 'random' + 'max-empty-bulk-delete': '10' + 'max-graceful-termination-sec': '600' + 'max-node-provision-time': '15m' + 'max-total-unready-percentage': '45' + 'new-pod-scale-up-delay': '0s' + 'ok-total-unready-count': '3' + 'scale-down-delay-after-add': '10m' + 'scale-down-delay-after-delete': '20s' + 'scale-down-delay-after-failure': '3m' + 'scale-down-unneeded-time': '10m' + 'scale-down-unready-time': '20m' + 'scale-down-utilization-threshold': '0.5' + 'scan-interval': '10s' + 'skip-nodes-with-local-storage': 'true' + 'skip-nodes-with-system-pods': 'true' + } + apiServerAccessProfile: { + enablePrivateCluster: true + privateDNSZone: pdzMc.id + enablePrivateClusterPublicFQDN: false + } + podIdentityProfile: { + enabled: true + } + disableLocalAccounts: true + securityProfile: { + azureDefender: { + enabled: true + logAnalyticsWorkspaceResourceId: law.id + } + } + } + identity: { + type: 'UserAssigned' + userAssignedIdentities: { + '${miClusterControlPlane.id}': { + } + } + } + sku: { + name: 'Basic' + tier: 'Paid' + } + dependsOn: [ + omsContainerInsights + ensureClusterIdentityHasRbacToSelfManagedResources + + paAksLinuxRestrictive + paEnforceHttpsIngress + paEnforceInternalLoadBalancers + paMustNotAutomountApiCreds + paMustUseSpecifiedLabels + paMustUseTheseExternalIps + paApprovedContainerPortsOnly + paApprovedServicePortsOnly + paRoRootFilesystem + paBlockDefaultNamespace + paEnforceResourceLimits + paEnforceImageSource + vmssJumpboxes + miIngressController + ] +} + /*** OUTPUTS ***/ output agwName string = agw.name From 7c015cd09adff9c3fe5364acecc592141cccc0da Mon Sep 17 00:00:00 2001 From: Fernando Antivero Date: Thu, 13 Oct 2022 14:51:47 +0000 Subject: [PATCH 059/115] add existing role definition for cr pull permission --- cluster-stamp.bicep | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/cluster-stamp.bicep b/cluster-stamp.bicep index b53d6adf..ad32e101 100644 --- a/cluster-stamp.bicep +++ b/cluster-stamp.bicep @@ -203,6 +203,12 @@ resource keyVaultSecretsUserRole 'Microsoft.Authorization/roleDefinitions@2022-0 scope: subscription() } +@description('Built-in Azure RBAC role that is applied to a Container Registry to grant with pull privileges. Granted to AKS kubelet cluster\'s identity.') +resource containerRegistryPullRole 'Microsoft.Authorization/roleDefinitions@2022-04-01' existing = { + name: '7f951dda-4ed3-4680-a7ca-43fe172d538d' + scope: subscription() +} + /*** RESOURCES ***/ @description('The control plane identity used by the cluster.') From 6e0bc57888daba35778cba1a5db395c68f420896 Mon Sep 17 00:00:00 2001 From: Fernando Antivero Date: Thu, 13 Oct 2022 14:52:30 +0000 Subject: [PATCH 060/115] rbac kubelet cluster manage indentity to be able to pull cr images --- cluster-stamp.bicep | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/cluster-stamp.bicep b/cluster-stamp.bicep index ad32e101..8cdf4c72 100644 --- a/cluster-stamp.bicep +++ b/cluster-stamp.bicep @@ -1547,6 +1547,17 @@ resource mc 'Microsoft.ContainerService/managedClusters@2021-05-01' = { ] } +@description('Grant kubelet managed identity with container registry pull role permissions; this allows the AKS Cluster\'s kubelet managed identity to pull images from this container registry.') +resource crMiKubeletContainerRegistryPullRole_roleAssignment 'Microsoft.Authorization/roleAssignments@2022-04-01' = { + scope: cr + name: guid(resourceGroup().id, mc.id, containerRegistryPullRole.id) + properties: { + roleDefinitionId: containerRegistryPullRole.id + principalId: mc.properties.identityProfile.kubeletidentity.objectId + principalType: 'ServicePrincipal' + } +} + /*** OUTPUTS ***/ output agwName string = agw.name From 9d1e7bbf93bc37b3cebf886d58476d411c2d7499 Mon Sep 17 00:00:00 2001 From: Fernando Antivero Date: Thu, 13 Oct 2022 14:56:12 +0000 Subject: [PATCH 061/115] add scheduled query rule for getting the avg of pod failures --- cluster-stamp.bicep | 40 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 40 insertions(+) diff --git a/cluster-stamp.bicep b/cluster-stamp.bicep index 8cdf4c72..2f202c60 100644 --- a/cluster-stamp.bicep +++ b/cluster-stamp.bicep @@ -1558,6 +1558,46 @@ resource crMiKubeletContainerRegistryPullRole_roleAssignment 'Microsoft.Authoriz } } +resource sqrPodFailed 'Microsoft.Insights/scheduledQueryRules@2021-08-01' = { + name: 'PodFailedScheduledQuery' + location: location + properties: { + description: 'Example from: https://learn.microsoft.com/azure/azure-monitor/insights/container-insights-alerts' + actions: { + actionGroups: [] + } + criteria: { + allOf: [ + { + metricMeasureColumn: 'FailedCount' + operator: 'GreaterThan' + query: 'let trendBinSize = 1m;\r\nKubePodInventory\r\n| distinct ClusterName, TimeGenerated\r\n| summarize ClusterSnapshotCount = count() by bin(TimeGenerated, trendBinSize), ClusterName\r\n| join hint.strategy=broadcast (\r\n KubePodInventory\r\n | distinct ClusterName, Computer, PodUid, TimeGenerated, PodStatus\r\n | summarize TotalCount = count(),\r\n PendingCount = sumif(1, PodStatus =~ "Pending"),\r\n RunningCount = sumif(1, PodStatus =~ "Running"),\r\n SucceededCount = sumif(1, PodStatus =~ "Succeeded"),\r\n FailedCount = sumif(1, PodStatus =~ "Failed")\r\n by ClusterName, bin(TimeGenerated, trendBinSize)\r\n )\r\n on ClusterName, TimeGenerated \r\n| extend UnknownCount = TotalCount - PendingCount - RunningCount - SucceededCount - FailedCount\r\n| project TimeGenerated,\r\n ClusterName,\r\n TotalCount = todouble(TotalCount) / ClusterSnapshotCount,\r\n PendingCount = todouble(PendingCount) / ClusterSnapshotCount,\r\n RunningCount = todouble(RunningCount) / ClusterSnapshotCount,\r\n SucceededCount = todouble(SucceededCount) / ClusterSnapshotCount,\r\n FailedCount = todouble(FailedCount) / ClusterSnapshotCount,\r\n UnknownCount = todouble(UnknownCount) / ClusterSnapshotCount\r\n' + threshold: 3 + timeAggregation: 'Average' + dimensions: [] + failingPeriods: { + minFailingPeriodsToAlert: 1 + numberOfEvaluationPeriods: 1 + } + resourceIdColumn: '' + } + ] + } + enabled: true + evaluationFrequency: 'PT5M' + scopes: [ + mc.id + ] + severity: 3 + windowSize: 'PT5M' + muteActionsDuration: null + overrideQueryTimeRange: 'P2D' + } + dependsOn: [ + law + ] +} + /*** OUTPUTS ***/ output agwName string = agw.name From d8e1a9949fbd1ebc58561bd6b1cfd114e22f985c Mon Sep 17 00:00:00 2001 From: Fernando Antivero Date: Thu, 13 Oct 2022 15:07:15 +0000 Subject: [PATCH 062/115] add rbac existing role definition for monitoring metric publisher role permissions --- cluster-stamp.bicep | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/cluster-stamp.bicep b/cluster-stamp.bicep index 2f202c60..f2d81e03 100644 --- a/cluster-stamp.bicep +++ b/cluster-stamp.bicep @@ -209,6 +209,12 @@ resource containerRegistryPullRole 'Microsoft.Authorization/roleDefinitions@2022 scope: subscription() } +@description('Built-in Azure RBAC role that is applied to a Subscription to grant with publishing metrics. Granted to in-cluster agent\'s identity.') +resource monitoringMetricsPublisherRole 'Microsoft.Authorization/roleDefinitions@2022-04-01' existing = { + name: '3913510d-42f4-4e42-8a64-420c390055eb' + scope: subscription() +} + /*** RESOURCES ***/ @description('The control plane identity used by the cluster.') From e661ec75c8f555e4ad3ca041f85d2482b77cd6a1 Mon Sep 17 00:00:00 2001 From: Fernando Antivero Date: Thu, 13 Oct 2022 15:08:02 +0000 Subject: [PATCH 063/115] rbac cluster oms agent manage indentity to be able to push metrics to container insights --- cluster-stamp.bicep | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/cluster-stamp.bicep b/cluster-stamp.bicep index f2d81e03..4cd865e6 100644 --- a/cluster-stamp.bicep +++ b/cluster-stamp.bicep @@ -1564,6 +1564,16 @@ resource crMiKubeletContainerRegistryPullRole_roleAssignment 'Microsoft.Authoriz } } +@description('Grant Oms agent managed identity with publisher metrics role permissions; this allows the Oms agent\'s identity to publish metrics in container insights.') +resource sMiOmsMonitoringMetricPublisherRoleRole_roleAssignment 'Microsoft.Authorization/roleAssignments@2022-04-01' = { + name: guid(mc.id, monitoringMetricsPublisherRole.id) + properties: { + roleDefinitionId: monitoringMetricsPublisherRole.id + principalId: mc.properties.addonProfiles.omsagent.identity.objectId + principalType: 'ServicePrincipal' + } +} + resource sqrPodFailed 'Microsoft.Insights/scheduledQueryRules@2021-08-01' = { name: 'PodFailedScheduledQuery' location: location From 7fa26ad0f57575a47f99d685d42d08809e79e6cb Mon Sep 17 00:00:00 2001 From: Fernando Antivero Date: Thu, 13 Oct 2022 15:11:08 +0000 Subject: [PATCH 064/115] add diagnostic settings for regulated cluster --- cluster-stamp.bicep | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/cluster-stamp.bicep b/cluster-stamp.bicep index 4cd865e6..6cbf4c6e 100644 --- a/cluster-stamp.bicep +++ b/cluster-stamp.bicep @@ -1553,6 +1553,34 @@ resource mc 'Microsoft.ContainerService/managedClusters@2021-05-01' = { ] } +resource mc_diagnosticSettings 'Microsoft.Insights/diagnosticSettings@2021-05-01-preview' = { + name: 'default' + properties: { + workspaceId: law.id + logs: [ + { + category: 'cluster-autoscaler' + enabled: true + } + { + category: 'kube-controller-manager' + enabled: true + } + { + category: 'kube-audit-admin' + enabled: true + } + { + category: 'guard' + enabled: true + } + ] + } + dependsOn: [ + mc + ] +} + @description('Grant kubelet managed identity with container registry pull role permissions; this allows the AKS Cluster\'s kubelet managed identity to pull images from this container registry.') resource crMiKubeletContainerRegistryPullRole_roleAssignment 'Microsoft.Authorization/roleAssignments@2022-04-01' = { scope: cr From 7664f29ccec66417f52db43c9a1af7d95c926e99 Mon Sep 17 00:00:00 2001 From: Fernando Antivero Date: Thu, 13 Oct 2022 15:25:57 +0000 Subject: [PATCH 065/115] add metric alert for high cpu usage in CI-1 --- cluster-stamp.bicep | 44 ++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 44 insertions(+) diff --git a/cluster-stamp.bicep b/cluster-stamp.bicep index 6cbf4c6e..fb26323d 100644 --- a/cluster-stamp.bicep +++ b/cluster-stamp.bicep @@ -1642,6 +1642,50 @@ resource sqrPodFailed 'Microsoft.Insights/scheduledQueryRules@2021-08-01' = { ] } +resource maNodeCpuUtilizationHighCI1 'Microsoft.Insights/metricAlerts@2018-03-01' = { + name: 'Node CPU utilization high for ${clusterName} CI-1' + location: 'global' + properties: { + actions: [] + criteria: { + allOf: [ + { + criterionType: 'StaticThresholdCriterion' + dimensions: [ + { + name: 'host' + operator: 'Include' + values: [ + '*' + ] + } + ] + metricName: 'cpuUsagePercentage' + metricNamespace: 'Insights.Container/nodes' + name: 'Metric1' + operator: 'GreaterThan' + threshold: '80' + timeAggregation: 'Average' + skipMetricValidation: true + } + ] + 'odata.type': 'Microsoft.Azure.Monitor.SingleResourceMultipleMetricCriteria' + } + description: 'Node CPU utilization across the cluster.' + enabled: true + evaluationFrequency: 'PT1M' + scopes: [ + mc.id + ] + severity: 3 + targetResourceType: 'microsoft.containerservice/managedclusters' + windowSize: 'PT5M' + } + dependsOn: [ + omsContainerInsights + ] +} + /*** OUTPUTS ***/ output agwName string = agw.name From dd0ca4f14770b0907db8589ae466a8e17767da20 Mon Sep 17 00:00:00 2001 From: Fernando Antivero Date: Thu, 13 Oct 2022 15:28:20 +0000 Subject: [PATCH 066/115] add metric alert for high cpu usage in CI-2 --- cluster-stamp.bicep | 44 ++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 44 insertions(+) diff --git a/cluster-stamp.bicep b/cluster-stamp.bicep index fb26323d..ba79609e 100644 --- a/cluster-stamp.bicep +++ b/cluster-stamp.bicep @@ -1686,6 +1686,50 @@ resource maNodeCpuUtilizationHighCI1 'Microsoft.Insights/metricAlerts@2018-03-01 ] } +resource maNodeCpuUtilizationHighCI2 'Microsoft.Insights/metricAlerts@2018-03-01' = { + name: 'Node working set memory utilization high for ${clusterName} CI-2' + location: 'global' + properties: { + actions: [] + criteria: { + allOf: [ + { + criterionType: 'StaticThresholdCriterion' + dimensions: [ + { + name: 'host' + operator: 'Include' + values: [ + '*' + ] + } + ] + metricName: 'memoryWorkingSetPercentage' + metricNamespace: 'Insights.Container/nodes' + name: 'Metric1' + operator: 'GreaterThan' + threshold: '80' + timeAggregation: 'Average' + skipMetricValidation: true + } + ] + 'odata.type': 'Microsoft.Azure.Monitor.SingleResourceMultipleMetricCriteria' + } + description: 'Node working set memory utilization across the cluster.' + enabled: true + evaluationFrequency: 'PT1M' + scopes: [ + mc.id + ] + severity: 3 + targetResourceType: 'microsoft.containerservice/managedclusters' + windowSize: 'PT5M' + } + dependsOn: [ + omsContainerInsights + ] +} + /*** OUTPUTS ***/ output agwName string = agw.name From a28b926c8324977558f7f44c6be275323cc17a3d Mon Sep 17 00:00:00 2001 From: Fernando Antivero Date: Thu, 13 Oct 2022 15:30:08 +0000 Subject: [PATCH 067/115] add metric alert for jobs completed more than 6h ago in CI-11 --- cluster-stamp.bicep | 51 +++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 51 insertions(+) diff --git a/cluster-stamp.bicep b/cluster-stamp.bicep index ba79609e..ff23d5ca 100644 --- a/cluster-stamp.bicep +++ b/cluster-stamp.bicep @@ -1730,6 +1730,57 @@ resource maNodeCpuUtilizationHighCI2 'Microsoft.Insights/metricAlerts@2018-03-01 ] } +resource maJobsCompletedMoreThan6hAgoCI11 'Microsoft.Insights/metricAlerts@2018-03-01' = { + name: 'Jobs completed more than 6 hours ago for ${clusterName} CI-11' + location: 'global' + properties: { + actions: [] + criteria: { + allOf: [ + { + criterionType: 'StaticThresholdCriterion' + dimensions: [ + { + name: 'controllerName' + operator: 'Include' + values: [ + '*' + ] + } + { + name: 'kubernetes namespace' + operator: 'Include' + values: [ + '*' + ] + } + ] + metricName: 'completedJobsCount' + metricNamespace: 'Insights.Container/pods' + name: 'Metric1' + operator: 'GreaterThan' + threshold: '0' + timeAggregation: 'Average' + skipMetricValidation: true + } + ] + 'odata.type': 'Microsoft.Azure.Monitor.SingleResourceMultipleMetricCriteria' + } + description: 'This alert monitors completed jobs (more than 6 hours ago).' + enabled: true + evaluationFrequency: 'PT1M' + scopes: [ + mc.id + ] + severity: 3 + targetResourceType: 'microsoft.containerservice/managedclusters' + windowSize: 'PT1M' + } + dependsOn: [ + omsContainerInsights + ] +} + /*** OUTPUTS ***/ output agwName string = agw.name From 4f01473f4a54147d7629b625ebb4248de3c0d706 Mon Sep 17 00:00:00 2001 From: Fernando Antivero Date: Thu, 13 Oct 2022 15:31:26 +0000 Subject: [PATCH 068/115] add metric alert for container high cpu usage in CI-9 --- cluster-stamp.bicep | 51 +++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 51 insertions(+) diff --git a/cluster-stamp.bicep b/cluster-stamp.bicep index ff23d5ca..25d3ecbb 100644 --- a/cluster-stamp.bicep +++ b/cluster-stamp.bicep @@ -1781,6 +1781,57 @@ resource maJobsCompletedMoreThan6hAgoCI11 'Microsoft.Insights/metricAlerts@2018- ] } +resource maContainerCpuUtilizationHighCI9 'Microsoft.Insights/metricAlerts@2018-03-01' = { + name: 'Container CPU usage high for ${clusterName} CI-9' + location: 'global' + properties: { + actions: [] + criteria: { + allOf: [ + { + criterionType: 'StaticThresholdCriterion' + dimensions: [ + { + name: 'controllerName' + operator: 'Include' + values: [ + '*' + ] + } + { + name: 'kubernetes namespace' + operator: 'Include' + values: [ + '*' + ] + } + ] + metricName: 'cpuExceededPercentage' + metricNamespace: 'Insights.Container/containers' + name: 'Metric1' + operator: 'GreaterThan' + threshold: '90' + timeAggregation: 'Average' + skipMetricValidation: true + } + ] + 'odata.type': 'Microsoft.Azure.Monitor.SingleResourceMultipleMetricCriteria' + } + description: 'This alert monitors container CPU utilization.' + enabled: true + evaluationFrequency: 'PT1M' + scopes: [ + mc.id + ] + severity: 3 + targetResourceType: 'microsoft.containerservice/managedclusters' + windowSize: 'PT5M' + } + dependsOn: [ + omsContainerInsights + ] +} + /*** OUTPUTS ***/ output agwName string = agw.name From a987d226ba4a0a036fdc42d0170fc6752ff103b2 Mon Sep 17 00:00:00 2001 From: Fernando Antivero Date: Thu, 13 Oct 2022 15:33:01 +0000 Subject: [PATCH 069/115] add metric alert for container high container working set memory usage in CI-10 --- cluster-stamp.bicep | 52 +++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 52 insertions(+) diff --git a/cluster-stamp.bicep b/cluster-stamp.bicep index 25d3ecbb..1c511465 100644 --- a/cluster-stamp.bicep +++ b/cluster-stamp.bicep @@ -1832,6 +1832,58 @@ resource maContainerCpuUtilizationHighCI9 'Microsoft.Insights/metricAlerts@2018- ] } +resource maContainerWorkingSetMemoryUsageHighCI10 'Microsoft.Insights/metricAlerts@2018-03-01' = { + name: 'Container working set memory usage high for ${clusterName} CI-10' + location: 'global' + properties: { + actions: [] + criteria: { + allOf: [ + { + criterionType: 'StaticThresholdCriterion' + dimensions: [ + { + name: 'controllerName' + operator: 'Include' + values: [ + '*' + ] + } + { + name: 'kubernetes namespace' + operator: 'Include' + values: [ + '*' + ] + } + ] + metricName: 'memoryWorkingSetExceededPercentage' + metricNamespace: 'Insights.Container/containers' + name: 'Metric1' + operator: 'GreaterThan' + threshold: '90' + timeAggregation: 'Average' + skipMetricValidation: true + } + ] + 'odata.type': 'Microsoft.Azure.Monitor.SingleResourceMultipleMetricCriteria' + } + description: 'This alert monitors container working set memory utilization.' + enabled: true + evaluationFrequency: 'PT1M' + scopes: [ + mc.id + ] + severity: 3 + targetResourceType: 'microsoft.containerservice/managedclusters' + windowSize: 'PT5M' + } + dependsOn: [ + omsContainerInsights + ] +} + + /*** OUTPUTS ***/ output agwName string = agw.name From a2cbb3ee8bf50404914ca756faeb7886445cd753 Mon Sep 17 00:00:00 2001 From: Fernando Antivero Date: Thu, 13 Oct 2022 15:34:13 +0000 Subject: [PATCH 070/115] add metric alert for pods in failed state in CI-4 --- cluster-stamp.bicep | 43 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 43 insertions(+) diff --git a/cluster-stamp.bicep b/cluster-stamp.bicep index 1c511465..2ab521e3 100644 --- a/cluster-stamp.bicep +++ b/cluster-stamp.bicep @@ -1883,6 +1883,49 @@ resource maContainerWorkingSetMemoryUsageHighCI10 'Microsoft.Insights/metricAler ] } +resource maPodsInFailedStateCI4 'Microsoft.Insights/metricAlerts@2018-03-01' = { + name: 'Pods in failed state for ${clusterName} CI-4' + location: 'global' + properties: { + actions: [] + criteria: { + allOf: [ + { + criterionType: 'StaticThresholdCriterion' + dimensions: [ + { + name: 'phase' + operator: 'Include' + values: [ + 'Failed' + ] + } + ] + metricName: 'podCount' + metricNamespace: 'Insights.Container/pods' + name: 'Metric1' + operator: 'GreaterThan' + threshold: '0' + timeAggregation: 'Average' + skipMetricValidation: true + } + ] + 'odata.type': 'Microsoft.Azure.Monitor.SingleResourceMultipleMetricCriteria' + } + description: 'Pod status monitoring.' + enabled: true + evaluationFrequency: 'PT1M' + scopes: [ + mc.id + ] + severity: 3 + targetResourceType: 'microsoft.containerservice/managedclusters' + windowSize: 'PT5M' + } + dependsOn: [ + omsContainerInsights + ] +} /*** OUTPUTS ***/ From 32711cca6744c670478fe1d426ff7139c2b1a345 Mon Sep 17 00:00:00 2001 From: Fernando Antivero Date: Thu, 13 Oct 2022 15:35:07 +0000 Subject: [PATCH 071/115] add metric alert for high disk usage in CI-5 --- cluster-stamp.bicep | 52 +++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 52 insertions(+) diff --git a/cluster-stamp.bicep b/cluster-stamp.bicep index 2ab521e3..77fdcd35 100644 --- a/cluster-stamp.bicep +++ b/cluster-stamp.bicep @@ -1927,6 +1927,58 @@ resource maPodsInFailedStateCI4 'Microsoft.Insights/metricAlerts@2018-03-01' = { ] } +resource maDiskUsageHighCI5 'Microsoft.Insights/metricAlerts@2018-03-01' = { + name: 'Disk usage high for ${clusterName} CI-5' + location: 'global' + properties: { + actions: [] + criteria: { + allOf: [ + { + criterionType: 'StaticThresholdCriterion' + dimensions: [ + { + name: 'host' + operator: 'Include' + values: [ + '*' + ] + } + { + name: 'device' + operator: 'Include' + values: [ + '*' + ] + } + ] + metricName: 'DiskUsedPercentage' + metricNamespace: 'Insights.Container/nodes' + name: 'Metric1' + operator: 'GreaterThan' + threshold: '80' + timeAggregation: 'Average' + skipMetricValidation: true + } + ] + 'odata.type': 'Microsoft.Azure.Monitor.SingleResourceMultipleMetricCriteria' + } + description: 'This alert monitors disk usage for all nodes and storage devices.' + enabled: true + evaluationFrequency: 'PT1M' + scopes: [ + mc.id + ] + severity: 3 + targetResourceType: 'microsoft.containerservice/managedclusters' + windowSize: 'PT5M' + } + dependsOn: [ + omsContainerInsights + ] +} + + /*** OUTPUTS ***/ output agwName string = agw.name From a3814e0a4e64022b79718f16f3f69b71b78efa7c Mon Sep 17 00:00:00 2001 From: Fernando Antivero Date: Thu, 13 Oct 2022 15:36:28 +0000 Subject: [PATCH 072/115] add metric alert for nodes not in ready status in CI-3 --- cluster-stamp.bicep | 43 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 43 insertions(+) diff --git a/cluster-stamp.bicep b/cluster-stamp.bicep index 77fdcd35..6e0b0940 100644 --- a/cluster-stamp.bicep +++ b/cluster-stamp.bicep @@ -1978,6 +1978,49 @@ resource maDiskUsageHighCI5 'Microsoft.Insights/metricAlerts@2018-03-01' = { ] } +resource maNodesInNotReadyStatusCI3 'Microsoft.Insights/metricAlerts@2018-03-01' = { + name: 'Nodes in not ready status for ${clusterName} CI-3' + location: 'global' + properties: { + actions: [] + criteria: { + allOf: [ + { + criterionType: 'StaticThresholdCriterion' + dimensions: [ + { + name: 'status' + operator: 'Include' + values: [ + 'NotReady' + ] + } + ] + metricName: 'nodesCount' + metricNamespace: 'Insights.Container/nodes' + name: 'Metric1' + operator: 'GreaterThan' + threshold: '0' + timeAggregation: 'Average' + skipMetricValidation: true + } + ] + 'odata.type': 'Microsoft.Azure.Monitor.SingleResourceMultipleMetricCriteria' + } + description: 'Node status monitoring.' + enabled: true + evaluationFrequency: 'PT1M' + scopes: [ + mc.id + ] + severity: 3 + targetResourceType: 'microsoft.containerservice/managedclusters' + windowSize: 'PT5M' + } + dependsOn: [ + omsContainerInsights + ] +} /*** OUTPUTS ***/ From de66abc0c0c4d1d6f981e240a1a73a903f5fc549 Mon Sep 17 00:00:00 2001 From: Fernando Antivero Date: Thu, 13 Oct 2022 15:37:32 +0000 Subject: [PATCH 073/115] add metric alert for omm killed pods in CI-6 --- cluster-stamp.bicep | 51 +++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 51 insertions(+) diff --git a/cluster-stamp.bicep b/cluster-stamp.bicep index 6e0b0940..7f30f450 100644 --- a/cluster-stamp.bicep +++ b/cluster-stamp.bicep @@ -2022,6 +2022,57 @@ resource maNodesInNotReadyStatusCI3 'Microsoft.Insights/metricAlerts@2018-03-01' ] } +resource maContainersGettingOomKilledCI6 'Microsoft.Insights/metricAlerts@2018-03-01' = { + name: 'Containers getting OOM killed for ${clusterName} CI-6' + location: 'global' + properties: { + actions: [] + criteria: { + allOf: [ + { + criterionType: 'StaticThresholdCriterion' + dimensions: [ + { + name: 'kubernetes namespace' + operator: 'Include' + values: [ + '*' + ] + } + { + name: 'controllerName' + operator: 'Include' + values: [ + '*' + ] + } + ] + metricName: 'oomKilledContainerCount' + metricNamespace: 'Insights.Container/pods' + name: 'Metric1' + operator: 'GreaterThan' + threshold: '0' + timeAggregation: 'Average' + skipMetricValidation: true + } + ] + 'odata.type': 'Microsoft.Azure.Monitor.SingleResourceMultipleMetricCriteria' + } + description: 'This alert monitors number of containers killed due to out of memory (OOM) error.' + enabled: true + evaluationFrequency: 'PT1M' + scopes: [ + mc.id + ] + severity: 3 + targetResourceType: 'microsoft.containerservice/managedclusters' + windowSize: 'PT1M' + } + dependsOn: [ + omsContainerInsights + ] +} + /*** OUTPUTS ***/ output agwName string = agw.name From bb57d23ce4a8371fd2438d78530cad5ca7124982 Mon Sep 17 00:00:00 2001 From: Fernando Antivero Date: Thu, 13 Oct 2022 15:38:30 +0000 Subject: [PATCH 074/115] add metric alert for persisten vol high usage CI-18 --- cluster-stamp.bicep | 51 +++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 51 insertions(+) diff --git a/cluster-stamp.bicep b/cluster-stamp.bicep index 7f30f450..0854084f 100644 --- a/cluster-stamp.bicep +++ b/cluster-stamp.bicep @@ -2073,6 +2073,57 @@ resource maContainersGettingOomKilledCI6 'Microsoft.Insights/metricAlerts@2018-0 ] } +resource maPersistentVolumeUsageHighCI18 'Microsoft.Insights/metricAlerts@2018-03-01' = { + name: 'Persistent volume usage high for ${clusterName} CI-18' + location: 'global' + properties: { + actions: [] + criteria: { + allOf: [ + { + criterionType: 'StaticThresholdCriterion' + dimensions: [ + { + name: 'podName' + operator: 'Include' + values: [ + '*' + ] + } + { + name: 'kubernetesNamespace' + operator: 'Include' + values: [ + '*' + ] + } + ] + metricName: 'pvUsageExceededPercentage' + metricNamespace: 'Insights.Container/persistentvolumes' + name: 'Metric1' + operator: 'GreaterThan' + threshold: '80' + timeAggregation: 'Average' + skipMetricValidation: true + } + ] + 'odata.type': 'Microsoft.Azure.Monitor.SingleResourceMultipleMetricCriteria' + } + description: 'This alert monitors persistent volume utilization.' + enabled: false + evaluationFrequency: 'PT1M' + scopes: [ + mc.id + ] + severity: 3 + targetResourceType: 'microsoft.containerservice/managedclusters' + windowSize: 'PT5M' + } + dependsOn: [ + omsContainerInsights + ] +} + /*** OUTPUTS ***/ output agwName string = agw.name From c76c001fcfeae64f289d174adae8993174f15864 Mon Sep 17 00:00:00 2001 From: Fernando Antivero Date: Thu, 13 Oct 2022 15:39:22 +0000 Subject: [PATCH 075/115] add metric alert for pods not in ready status in CI-8 --- cluster-stamp.bicep | 52 +++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 52 insertions(+) diff --git a/cluster-stamp.bicep b/cluster-stamp.bicep index 0854084f..d5bebdca 100644 --- a/cluster-stamp.bicep +++ b/cluster-stamp.bicep @@ -2124,6 +2124,58 @@ resource maPersistentVolumeUsageHighCI18 'Microsoft.Insights/metricAlerts@2018-0 ] } +resource maPodsNotInReadyStateCI8 'Microsoft.Insights/metricAlerts@2018-03-01' = { + name: 'Pods not in ready state for ${clusterName} CI-8' + location: 'global' + properties: { + actions: [] + criteria: { + allOf: [ + { + criterionType: 'StaticThresholdCriterion' + dimensions: [ + { + name: 'controllerName' + operator: 'Include' + values: [ + '*' + ] + } + { + name: 'kubernetes namespace' + operator: 'Include' + values: [ + '*' + ] + } + ] + metricName: 'PodReadyPercentage' + metricNamespace: 'Insights.Container/pods' + name: 'Metric1' + operator: 'LessThan' + threshold: '80' + timeAggregation: 'Average' + skipMetricValidation: true + } + ] + 'odata.type': 'Microsoft.Azure.Monitor.SingleResourceMultipleMetricCriteria' + } + description: 'This alert monitors for excessive pods not in the ready state.' + enabled: true + evaluationFrequency: 'PT1M' + scopes: [ + mc.id + ] + severity: 3 + targetResourceType: 'microsoft.containerservice/managedclusters' + windowSize: 'PT5M' + } + dependsOn: [ + omsContainerInsights + ] +} + + /*** OUTPUTS ***/ output agwName string = agw.name From e7f2ec5b723170b290f08a6010541c989504b588 Mon Sep 17 00:00:00 2001 From: Fernando Antivero Date: Thu, 13 Oct 2022 15:40:46 +0000 Subject: [PATCH 076/115] add metric alert for restarting continers count in CI-7 --- cluster-stamp.bicep | 50 +++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 50 insertions(+) diff --git a/cluster-stamp.bicep b/cluster-stamp.bicep index d5bebdca..7c971eb9 100644 --- a/cluster-stamp.bicep +++ b/cluster-stamp.bicep @@ -2175,6 +2175,56 @@ resource maPodsNotInReadyStateCI8 'Microsoft.Insights/metricAlerts@2018-03-01' = ] } +resource maRestartingContainerCountCI7 'Microsoft.Insights/metricAlerts@2018-03-01' = { + name: 'Restarting container count for ${clusterName} CI-7' + location: 'global' + properties: { + actions: [] + criteria: { + allOf: [ + { + criterionType: 'StaticThresholdCriterion' + dimensions: [ + { + name: 'kubernetes namespace' + operator: 'Include' + values: [ + '*' + ] + } + { + name: 'controllerName' + operator: 'Include' + values: [ + '*' + ] + } + ] + metricName: 'restartingContainerCount' + metricNamespace: 'Insights.Container/pods' + name: 'Metric1' + operator: 'GreaterThan' + threshold: '0' + timeAggregation: 'Average' + skipMetricValidation: true + } + ] + 'odata.type': 'Microsoft.Azure.Monitor.SingleResourceMultipleMetricCriteria' + } + description: 'This alert monitors number of containers restarting across the cluster.' + enabled: true + evaluationFrequency: 'PT1M' + scopes: [ + mc.id + ] + severity: 3 + targetResourceType: 'Microsoft.ContainerService/managedClusters' + windowSize: 'PT1M' + } + dependsOn: [ + omsContainerInsights + ] +} /*** OUTPUTS ***/ From e8794cd51bd515a8ea3b69ccccffb94440d3c22e Mon Sep 17 00:00:00 2001 From: Fernando Antivero Date: Thu, 13 Oct 2022 15:44:17 +0000 Subject: [PATCH 077/115] bug fix: threshold value type --- cluster-stamp.bicep | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/cluster-stamp.bicep b/cluster-stamp.bicep index 7c971eb9..624237b9 100644 --- a/cluster-stamp.bicep +++ b/cluster-stamp.bicep @@ -1664,7 +1664,7 @@ resource maNodeCpuUtilizationHighCI1 'Microsoft.Insights/metricAlerts@2018-03-01 metricNamespace: 'Insights.Container/nodes' name: 'Metric1' operator: 'GreaterThan' - threshold: '80' + threshold: 80 timeAggregation: 'Average' skipMetricValidation: true } @@ -1708,7 +1708,7 @@ resource maNodeCpuUtilizationHighCI2 'Microsoft.Insights/metricAlerts@2018-03-01 metricNamespace: 'Insights.Container/nodes' name: 'Metric1' operator: 'GreaterThan' - threshold: '80' + threshold: 80 timeAggregation: 'Average' skipMetricValidation: true } @@ -1759,7 +1759,7 @@ resource maJobsCompletedMoreThan6hAgoCI11 'Microsoft.Insights/metricAlerts@2018- metricNamespace: 'Insights.Container/pods' name: 'Metric1' operator: 'GreaterThan' - threshold: '0' + threshold: 0 timeAggregation: 'Average' skipMetricValidation: true } @@ -1810,7 +1810,7 @@ resource maContainerCpuUtilizationHighCI9 'Microsoft.Insights/metricAlerts@2018- metricNamespace: 'Insights.Container/containers' name: 'Metric1' operator: 'GreaterThan' - threshold: '90' + threshold: 90 timeAggregation: 'Average' skipMetricValidation: true } @@ -1861,7 +1861,7 @@ resource maContainerWorkingSetMemoryUsageHighCI10 'Microsoft.Insights/metricAler metricNamespace: 'Insights.Container/containers' name: 'Metric1' operator: 'GreaterThan' - threshold: '90' + threshold: 90 timeAggregation: 'Average' skipMetricValidation: true } @@ -1905,7 +1905,7 @@ resource maPodsInFailedStateCI4 'Microsoft.Insights/metricAlerts@2018-03-01' = { metricNamespace: 'Insights.Container/pods' name: 'Metric1' operator: 'GreaterThan' - threshold: '0' + threshold: 0 timeAggregation: 'Average' skipMetricValidation: true } @@ -1956,7 +1956,7 @@ resource maDiskUsageHighCI5 'Microsoft.Insights/metricAlerts@2018-03-01' = { metricNamespace: 'Insights.Container/nodes' name: 'Metric1' operator: 'GreaterThan' - threshold: '80' + threshold: 80 timeAggregation: 'Average' skipMetricValidation: true } @@ -2000,7 +2000,7 @@ resource maNodesInNotReadyStatusCI3 'Microsoft.Insights/metricAlerts@2018-03-01' metricNamespace: 'Insights.Container/nodes' name: 'Metric1' operator: 'GreaterThan' - threshold: '0' + threshold: 0 timeAggregation: 'Average' skipMetricValidation: true } @@ -2051,7 +2051,7 @@ resource maContainersGettingOomKilledCI6 'Microsoft.Insights/metricAlerts@2018-0 metricNamespace: 'Insights.Container/pods' name: 'Metric1' operator: 'GreaterThan' - threshold: '0' + threshold: 0 timeAggregation: 'Average' skipMetricValidation: true } @@ -2102,7 +2102,7 @@ resource maPersistentVolumeUsageHighCI18 'Microsoft.Insights/metricAlerts@2018-0 metricNamespace: 'Insights.Container/persistentvolumes' name: 'Metric1' operator: 'GreaterThan' - threshold: '80' + threshold: 80 timeAggregation: 'Average' skipMetricValidation: true } @@ -2153,7 +2153,7 @@ resource maPodsNotInReadyStateCI8 'Microsoft.Insights/metricAlerts@2018-03-01' = metricNamespace: 'Insights.Container/pods' name: 'Metric1' operator: 'LessThan' - threshold: '80' + threshold: 80 timeAggregation: 'Average' skipMetricValidation: true } @@ -2204,7 +2204,7 @@ resource maRestartingContainerCountCI7 'Microsoft.Insights/metricAlerts@2018-03- metricNamespace: 'Insights.Container/pods' name: 'Metric1' operator: 'GreaterThan' - threshold: '0' + threshold: 0 timeAggregation: 'Average' skipMetricValidation: true } From 4e56ec7367bd64cf4d878156b812e4380c485ac8 Mon Sep 17 00:00:00 2001 From: Fernando Antivero Date: Thu, 13 Oct 2022 15:51:22 +0000 Subject: [PATCH 078/115] delete the json cluster stamp v0 --- cluster-stamp.json | 2447 -------------------------------------------- 1 file changed, 2447 deletions(-) delete mode 100644 cluster-stamp.json diff --git a/cluster-stamp.json b/cluster-stamp.json deleted file mode 100644 index 626efd94..00000000 --- a/cluster-stamp.json +++ /dev/null @@ -1,2447 +0,0 @@ -{ - "$schema": "https://schema.management.azure.com/schemas/2019-04-01/deploymentTemplate.json#", - "contentVersion": "1.0.1.0", - "parameters": { - "targetVnetResourceId": { - "type": "string", - "minLength": 79, - "metadata": { - "description": "The regional network spoke VNet Resource ID that the cluster will be joined to" - } - }, - "clusterAdminAadGroupObjectId": { - "type": "string", - "metadata": { - "description": "Azure AD Group in the identified tenant that will be granted the highly privileged cluster-admin role." - } - }, - "k8sControlPlaneAuthorizationTenantId": { - "type": "string", - "metadata": { - "description": "Your AKS control plane Cluster API authentication tenant" - } - }, - "appGatewayListenerCertificate": { - "type": "string", - "metadata": { - "description": "The certificate data for app gateway TLS termination. It is base64" - } - }, - "aksIngressControllerCertificate": { - "type": "string", - "metadata": { - "description": "The base 64 encoded AKS Ingress Controller public certificate (as .crt or .cer) to be stored in Azure Key Vault as secret and referenced by Azure Application Gateway as a trusted root certificate." - } - }, - "location": { - "defaultValue": "eastus2", - "type": "string", - "allowedValues": [ - "australiaeast", - "canadacentral", - "centralus", - "eastus", - "eastus2", - "westus2", - "francecentral", - "germanywestcentral", - "northeurope", - "southafricanorth", - "southcentralus", - "uksouth", - "westeurope", - "japaneast", - "southeastasia" - ], - "metadata": { - "description": "AKS Service, Node Pools, and supporting services (KeyVault, App Gateway, etc) region. This needs to be the same region as the vnet provided in these parameters." - } - }, - "geoRedundancyLocation": { - "defaultValue": "centralus", - "type": "string", - "allowedValues": [ - "australiasoutheast", - "canadaeast", - "eastus2", - "westus", - "centralus", - "westcentralus", - "francesouth", - "germanynorth", - "westeurope", - "ukwest", - "northeurope", - "japanwest", - "southafricawest", - "northcentralus", - "eastasia", - "eastus", - "westus2", - "francecentral", - "uksouth", - "japaneast", - "southeastasia" - ], - "metadata": { - "description": "For Azure resources that support native geo-redunancy, provide the location the redundant service will have its secondary. Should be different than the location parameter and ideally should be a paired region - https://learn.microsoft.com/azure/best-practices-availability-paired-regions. This region does not need to support availability zones." - } - }, - "jumpBoxImageResourceId": { - "type": "string", - "minLength": 70, - "metadata": { - "description": "The Azure resource ID of a VM image that will be used for the jump box." - } - }, - "jumpBoxCloudInitAsBase64": { - "type": "string", - "minLength": 100, - "metadata": { - "description": "A cloud init file (starting with #cloud-config) as a base 64 encoded string used to perform image customization on the jump box VMs. Used for user-management in this context." - } - } - }, - "variables": { - "kubernetesVersion": "1.23.3", - - "networkContributorRole": "[concat(subscription().Id, '/providers/Microsoft.Authorization/roleDefinitions/4d97b98b-1d4f-4787-a291-c67834d212e7')]", - "monitoringMetricsPublisherRole": "[concat(subscription().Id, '/providers/Microsoft.Authorization/roleDefinitions/3913510d-42f4-4e42-8a64-420c390055eb')]", - "acrPullRole": "[concat(subscription().Id, '/providers/Microsoft.Authorization/roleDefinitions/7f951dda-4ed3-4680-a7ca-43fe172d538d')]", - "dnsZoneContributorRole": "[concat(subscription().id, '/providers/Microsoft.Authorization/roleDefinitions/b12aa53e-6015-4669-85d0-8515ebb3ae7f')]", - "managedIdentityOperatorRole": "[concat(subscription().Id, '/providers/Microsoft.Authorization/roleDefinitions/f1a07417-d97a-45cb-824c-7a7467783830')]", - - "subRgUniqueString": "[uniqueString('aks', subscription().subscriptionId, resourceGroup().id)]", - - "nodeResourceGroupName": "[concat('rg-', variables('clusterName'), '-nodepools')]", - "clusterName": "[concat('aks-', variables('subRgUniqueString'))]", - "logAnalyticsWorkspaceName": "[concat('la-', variables('clusterName'))]", - "containerInsightsSolutionName": "[concat('ContainerInsights(', variables('logAnalyticsWorkspaceName'),')')]", - "vmInsightsSolutionName": "[concat('VMInsights(', variables('logAnalyticsWorkspaceName'),')')]", - "securityCenterSolutionName": "[concat('SecurityInsights(', variables('logAnalyticsWorkspaceName'),')')]", - "defaultAcrName": "[concat('acraks', variables('subRgUniqueString'))]", - - "vNetResourceGroup": "[split(parameters('targetVnetResourceId'),'/')[4]]", - "vnetName": "[split(parameters('targetVnetResourceId'),'/')[8]]", - "vnetSystemNodePoolSubnetResourceId": "[concat(parameters('targetVnetResourceId'), '/subnets/snet-cluster-systemnodepool')]", - "vnetInScopeNodePoolSubnetResourceId": "[concat(parameters('targetVnetResourceId'), '/subnets/snet-cluster-inscopenodepools')]", - "vnetPrivateLinkSubnetResourceId": "[concat(parameters('targetVnetResourceId'), '/subnets/snet-privatelinkendpoints')]", - "vnetAcrBuildSubnetResourceId": "[concat(parameters('targetVnetResourceId'), '/subnets/snet-management-acragents')]", - "vnetOutOfScopeNodePoolSubnetResourceId": "[concat(parameters('targetVnetResourceId'), '/subnets/snet-cluster-outofscopenodepools')]", - "vnetIngressServicesSubnetResourceId": "[concat(parameters('targetVnetResourceId'), '/subnets/snet-cluster-ingressservices')]", - "vnetManagementOpsSubnetResourceId": "[concat(parameters('targetVnetResourceId'), '/subnets/snet-management-ops')]", - - "agwName": "[concat('apw-', variables('clusterName'))]", - "apwResourceId": "[resourceId('Microsoft.Network/applicationGateways', variables('agwName'))]", - - "jumpBoxDefaultAdminUserName": "[uniqueString(variables('clusterName'), resourceGroup().id)]", - - "clusterControlPlaneIdentityName": "[concat('mi-', variables('clusterName'), '-controlplane')]", - - "keyVaultName": "[concat('kv-', variables('clusterName'))]", - - "policyResourceIdAKSLinuxRestrictive": "/providers/Microsoft.Authorization/policySetDefinitions/42b8ef37-b724-4e24-bbc8-7a7708edfe00", - "policyResourceIdEnforceHttpsIngress": "/providers/Microsoft.Authorization/policyDefinitions/1a5b4dca-0b6f-4cf5-907c-56316bc1bf3d", - "policyResourceIdEnforceInternalLoadBalancers": "/providers/Microsoft.Authorization/policyDefinitions/3fc4dc25-5baf-40d8-9b05-7fe74c1bc64e", - "policyResourceIdRoRootFilesystem": "/providers/Microsoft.Authorization/policyDefinitions/df49d893-a74c-421d-bc95-c663042e5b80", - "policyResourceIdEnforceResourceLimits": "/providers/Microsoft.Authorization/policyDefinitions/e345eecc-fa47-480f-9e88-67dcc122b164", - "policyResourceIdEnforceImageSource": "/providers/Microsoft.Authorization/policyDefinitions/febd0533-8e55-448f-b837-bd0e06f16469", - "policyResourceIdBlockDefaultNamespace": "/providers/Microsoft.Authorization/policyDefinitions/9f061a12-e40d-4183-a00e-171812443373", - "policyResourceIdApprovedContainerPortsOnly": "/providers/Microsoft.Authorization/policyDefinitions/440b515e-a580-421e-abeb-b159a61ddcbc", - "policyResourceIdApprovedServicePortsOnly": "/providers/Microsoft.Authorization/policyDefinitions/233a2a17-77ca-4fb1-9b6b-69223d272a44", - "policyResourceIdMustUseSpecifiedLabels": "/providers/Microsoft.Authorization/policyDefinitions/46592696-4c7b-4bf3-9e45-6c2763bdc0a6", - "policyResourceIdMustUseTheseExternalIps": "/providers/Microsoft.Authorization/policyDefinitions/d46c275d-1680-448d-b2ec-e495a3b6cc89", - "policyResourceIdMustNotAutomountApiCreds": "/providers/Microsoft.Authorization/policyDefinitions/423dd1ba-798e-40e4-9c4d-b6902674b423", - "policyAssignmentNameAKSLinuxRestrictive": "[guid(variables('policyResourceIdAKSLinuxRestrictive'), resourceGroup().name, variables('clusterName'))]", - "policyAssignmentNameEnforceHttpsIngress": "[guid(variables('policyResourceIdEnforceHttpsIngress'), resourceGroup().name, variables('clusterName'))]", - "policyAssignmentNameEnforceInternalLoadBalancers": "[guid(variables('policyResourceIdEnforceInternalLoadBalancers'), resourceGroup().name, variables('clusterName'))]", - "policyAssignmentNameRoRootFilesystem": "[guid(variables('policyResourceIdRoRootFilesystem'), resourceGroup().name, variables('clusterName'))]", - "policyAssignmentNameEnforceResourceLimits": "[guid(variables('policyResourceIdEnforceResourceLimits'), resourceGroup().name, variables('clusterName'))]", - "policyAssignmentNameEnforceImageSource": "[guid(variables('policyResourceIdEnforceImageSource'), resourceGroup().name, variables('clusterName'))]", - "policyAssignmentNameBlockDefaultNamespace": "[guid(variables('policyResourceIdBlockDefaultNamespace'), resourceGroup().name, variables('clusterName'))]", - "policyAssignmentNameApprovedContainerPortsOnly": "[guid(variables('policyResourceIdApprovedContainerPortsOnly'), resourceGroup().name, variables('clusterName'))]", - "policyAssignmentNameApprovedServicePortsOnly": "[guid(variables('policyResourceIdApprovedServicePortsOnly'), resourceGroup().name, variables('clusterName'))]", - "policyAssignmentNameMustUseSpecifiedLabels": "[guid(variables('policyResourceIdMustUseSpecifiedLabels'), resourceGroup().name, variables('clusterName'))]", - "policyAssignmentNameMustUseTheseExternalIps": "[guid(variables('policyResourceIdMustUseTheseExternalIps'), resourceGroup().name, variables('clusterName'))]", - "policyAssignmentNameMustNotAutomountApiCreds": "[guid(variables('policyResourceIdMustNotAutomountApiCreds'), resourceGroup().name, variables('clusterName'))]", - - "aksSecurityCenterWorkbookData": "{\"version\":\"Notebook/1.0\",\"items\":[{\"type\":1,\"content\":{\"json\":\"## AKS Security\\n\"},\"name\":\"text - 2\"},{\"type\":9,\"content\":{\"version\":\"KqlParameterItem/1.0\",\"crossComponentResources\":[\"{workspaces}\"],\"parameters\":[{\"id\":\"311d3728-7f8a-4b16-8a34-097d099323d5\",\"version\":\"KqlParameterItem/1.0\",\"name\":\"subscription\",\"label\":\"Subscription\",\"type\":6,\"isRequired\":true,\"multiSelect\":true,\"quote\":\"'\",\"delimiter\":\",\",\"value\":[],\"typeSettings\":{\"additionalResourceOptions\":[],\"includeAll\":false,\"showDefault\":false}},{\"id\":\"3a56d260-4fb9-46d6-b121-cea854104c91\",\"version\":\"KqlParameterItem/1.0\",\"name\":\"workspaces\",\"label\":\"Workspaces\",\"type\":5,\"isRequired\":true,\"multiSelect\":true,\"quote\":\"'\",\"delimiter\":\",\",\"query\":\"where type =~ 'microsoft.operationalinsights/workspaces'\\r\\n| where strcat('/subscriptions/',subscriptionId) in ({subscription})\\r\\n| project id\",\"crossComponentResources\":[\"{subscription}\"],\"typeSettings\":{\"additionalResourceOptions\":[\"value::all\"]},\"queryType\":1,\"resourceType\":\"microsoft.resourcegraph/resources\",\"value\":[\"value::all\"]},{\"id\":\"9615cea6-c661-470a-b4ae-1aab8ae6f448\",\"version\":\"KqlParameterItem/1.0\",\"name\":\"clustername\",\"label\":\"Cluster name\",\"type\":5,\"isRequired\":true,\"multiSelect\":true,\"quote\":\"'\",\"delimiter\":\",\",\"query\":\"where type == \\\"microsoft.containerservice/managedclusters\\\"\\r\\n| where strcat('/subscriptions/',subscriptionId) in ({subscription})\\r\\n| distinct tolower(id)\",\"crossComponentResources\":[\"{subscription}\"],\"value\":[\"value::all\"],\"typeSettings\":{\"resourceTypeFilter\":{\"microsoft.containerservice/managedclusters\":true},\"additionalResourceOptions\":[\"value::all\"],\"showDefault\":false},\"timeContext\":{\"durationMs\":86400000},\"queryType\":1,\"resourceType\":\"microsoft.resourcegraph/resources\"},{\"id\":\"236c00ec-1493-4e60-927a-a18b8b120cd5\",\"version\":\"KqlParameterItem/1.0\",\"name\":\"timeframe\",\"label\":\"Time range\",\"type\":4,\"description\":\"Time\",\"isRequired\":true,\"value\":{\"durationMs\":172800000},\"typeSettings\":{\"selectableValues\":[{\"durationMs\":300000},{\"durationMs\":900000},{\"durationMs\":1800000},{\"durationMs\":3600000},{\"durationMs\":14400000},{\"durationMs\":43200000},{\"durationMs\":86400000},{\"durationMs\":172800000},{\"durationMs\":259200000},{\"durationMs\":604800000},{\"durationMs\":1209600000},{\"durationMs\":2419200000},{\"durationMs\":2592000000},{\"durationMs\":5184000000},{\"durationMs\":7776000000}],\"allowCustom\":true},\"timeContext\":{\"durationMs\":86400000}},{\"id\":\"bf0a3e4f-fff9-450c-b9d3-c8c1dded9787\",\"version\":\"KqlParameterItem/1.0\",\"name\":\"nodeRgDetails\",\"type\":1,\"query\":\"where type == \\\"microsoft.containerservice/managedclusters\\\"\\r\\n| where tolower(id) in ({clustername})\\r\\n| project nodeRG = properties.nodeResourceGroup, subscriptionId, id = toupper(id)\\r\\n| project nodeRgDetails = strcat('\\\"', nodeRG, \\\";\\\", subscriptionId, \\\";\\\", id, '\\\"')\",\"crossComponentResources\":[\"value::all\"],\"isHiddenWhenLocked\":true,\"timeContext\":{\"durationMs\":86400000},\"queryType\":1,\"resourceType\":\"microsoft.resourcegraph/resources\"},{\"id\":\"df53126c-c40f-43d5-b99f-97ee3785c086\",\"version\":\"KqlParameterItem/1.0\",\"name\":\"diagnosticClusters\",\"type\":1,\"query\":\"union withsource=_TableName *\\r\\n| where _TableName == \\\"AzureDiagnostics\\\" and Category == \\\"kube-audit\\\"\\r\\n| summarize diagnosticClusters = dcount(ResourceId)\\r\\n| project isDiagnosticCluster = iff(diagnosticClusters > 0, \\\"yes\\\", \\\"no\\\")\",\"crossComponentResources\":[\"{workspaces}\"],\"isHiddenWhenLocked\":true,\"timeContext\":{\"durationMs\":172800000},\"timeContextFromParameter\":\"timeframe\",\"queryType\":0,\"resourceType\":\"microsoft.operationalinsights/workspaces\"}],\"style\":\"pills\",\"queryType\":0,\"resourceType\":\"microsoft.operationalinsights/workspaces\"},\"name\":\"parameters - 3\"},{\"type\":11,\"content\":{\"version\":\"LinkItem/1.0\",\"style\":\"tabs\",\"links\":[{\"id\":\"07cf87dc-8234-47db-850d-ec41b2687b2a\",\"cellValue\":\"mainTab\",\"linkTarget\":\"parameter\",\"linkLabel\":\"Microsoft Defender for Kubernetes\",\"subTarget\":\"alerts\",\"preText\":\"\",\"style\":\"link\"},{\"id\":\"44033ee6-d83e-4253-a732-c258ef1da545\",\"cellValue\":\"mainTab\",\"linkTarget\":\"parameter\",\"linkLabel\":\"Analytics over Diagnostic logs\",\"subTarget\":\"diagnostics\",\"style\":\"link\"}]},\"name\":\"links - 22\"},{\"type\":12,\"content\":{\"version\":\"NotebookGroup/1.0\",\"groupType\":\"editable\",\"items\":[{\"type\":1,\"content\":{\"json\":\"## Microsoft Defender for AKS coverage\"},\"name\":\"text - 10\"},{\"type\":3,\"content\":{\"version\":\"KqlItem/1.0\",\"query\":\"datatable (Event:string)\\r\\n [\\\"AKS Workbook\\\"]\\r\\n| extend cluster = (strcat(\\\"[\\\", \\\"{clustername}\\\", \\\"]\\\"))\\r\\n| extend cluster = todynamic(replace(\\\"'\\\", '\\\"', cluster))\\r\\n| mvexpand cluster\\r\\n| extend subscriptionId = extract(@\\\"/subscriptions/([^/]+)\\\", 1, tostring(cluster))\\r\\n| summarize AksClusters = count() by subscriptionId, DefenderForAks = 0\\r\\n| union\\r\\n(\\r\\nsecurityresources\\r\\n| where type =~ \\\"microsoft.security/pricings\\\"\\r\\n| where name == \\\"KubernetesService\\\"\\r\\n| project DefenderForAks = iif(properties.pricingTier == 'Standard', 1, 0), AksClusters = 0, subscriptionId\\r\\n)\\r\\n| summarize AksClusters = sum(AksClusters), DefenderForAks = sum(DefenderForAks) by subscriptionId\\r\\n| project Subscription = strcat('/subscriptions/', subscriptionId), [\\\"AKS clusters\\\"] = AksClusters, ['Defender for AKS'] = iif(DefenderForAks > 0,'yes','no'), ['Onboard Microsoft Defender'] = iif(DefenderForAks > 0, '', 'https://ms.portal.azure.com/#blade/Microsoft_Azure_Security/SecurityMenuBlade/26')\\r\\n| order by ['Defender for AKS'] asc\",\"size\":0,\"queryType\":1,\"resourceType\":\"microsoft.resourcegraph/resources\",\"crossComponentResources\":[\"{subscription}\"],\"gridSettings\":{\"formatters\":[{\"columnMatch\":\"Defender for AKS\",\"formatter\":18,\"formatOptions\":{\"thresholdsOptions\":\"icons\",\"thresholdsGrid\":[{\"operator\":\"==\",\"thresholdValue\":\"no\",\"representation\":\"4\",\"text\":\"\"},{\"operator\":\"Default\",\"thresholdValue\":null,\"representation\":\"success\",\"text\":\"\"}]}},{\"columnMatch\":\"Onboard Microsoft Defender\",\"formatter\":7,\"formatOptions\":{\"linkTarget\":\"Url\",\"linkLabel\":\"\"}}]}},\"customWidth\":\"66\",\"name\":\"query - 9\"},{\"type\":3,\"content\":{\"version\":\"KqlItem/1.0\",\"query\":\"datatable (Event:string)\\r\\n [\\\"AKS Workbook\\\"]\\r\\n| extend cluster = (strcat(\\\"[\\\", \\\"{clustername}\\\", \\\"]\\\"))\\r\\n| extend cluster = todynamic(replace(\\\"'\\\", '\\\"', cluster))\\r\\n| mvexpand cluster\\r\\n| extend subscriptionId = extract(@\\\"/subscriptions/([^/]+)\\\", 1, tostring(cluster))\\r\\n| summarize AksClusters = count() by subscriptionId, DefenderForAks = 0\\r\\n| union\\r\\n(\\r\\nsecurityresources\\r\\n| where type =~ \\\"microsoft.security/pricings\\\"\\r\\n| where name == \\\"KubernetesService\\\"\\r\\n| project DefenderForAks = iif(properties.pricingTier == 'Standard', 1, 0), AksClusters = 0, subscriptionId\\r\\n)\\r\\n| summarize AksClusters = sum(AksClusters), DefenderForAks = sum(DefenderForAks) by subscriptionId\\r\\n| project Subscription = 1, ['Defender for AKS'] = iif(DefenderForAks > 0,'Protected by Microsoft Defender','Not protected by Microsoft Defender')\",\"size\":0,\"queryType\":1,\"resourceType\":\"microsoft.resourcegraph/resources\",\"crossComponentResources\":[\"{subscription}\"],\"visualization\":\"piechart\"},\"customWidth\":\"33\",\"name\":\"query - 11\"},{\"type\":1,\"content\":{\"json\":\"### AKS alerts overview\"},\"name\":\"text - 21\"},{\"type\":3,\"content\":{\"version\":\"KqlItem/1.0\",\"query\":\"SecurityAlert \\r\\n| where tolower(ResourceId) in ({clustername})\\r\\n| where TimeGenerated {timeframe}\\r\\n| where AlertType startswith \\\"AKS_\\\"\\r\\n| project image = tostring(todynamic(ExtendedProperties)[\\\"Container image\\\"]), AlertType\\r\\n| where image != \\\"\\\"\\r\\n| summarize AlertTypes = dcount(AlertType) by image\\r\\n| where AlertTypes > 1\\r\\n//| render piechart \\r\\n\",\"size\":4,\"title\":\"Images with multiple alerts\",\"queryType\":0,\"resourceType\":\"microsoft.operationalinsights/workspaces\",\"crossComponentResources\":[\"{workspaces}\"],\"visualization\":\"tiles\",\"tileSettings\":{\"showBorder\":false,\"titleContent\":{\"columnMatch\":\"image\",\"formatter\":1},\"leftContent\":{\"columnMatch\":\"AlertTypes\",\"formatter\":12,\"formatOptions\":{\"palette\":\"auto\"},\"numberFormat\":{\"unit\":17,\"options\":{\"maximumSignificantDigits\":3,\"maximumFractionDigits\":2}}}}},\"customWidth\":\"33\",\"name\":\"query - 12\"},{\"type\":3,\"content\":{\"version\":\"KqlItem/1.0\",\"query\":\"SecurityAlert \\r\\n| where tolower(ResourceId) in ({clustername})\\r\\n| where TimeGenerated {timeframe}\\r\\n| where AlertType startswith \\\"AKS_\\\"\\r\\n| project AlertType, name = extract(@\\\"/MICROSOFT.CONTAINERSERVICE/MANAGEDCLUSTERS/(.+)\\\", 1, ResourceId)\\r\\n| summarize AlertTypes = dcount(AlertType) by name\\r\\n| where AlertTypes > 1\\r\\n\",\"size\":4,\"title\":\"Clusters with multiple alert types\",\"queryType\":0,\"resourceType\":\"microsoft.operationalinsights/workspaces\",\"crossComponentResources\":[\"{workspaces}\"],\"visualization\":\"tiles\",\"tileSettings\":{\"showBorder\":false,\"titleContent\":{\"columnMatch\":\"name\",\"formatter\":1},\"leftContent\":{\"columnMatch\":\"AlertTypes\",\"formatter\":12,\"formatOptions\":{\"palette\":\"auto\"},\"numberFormat\":{\"unit\":17,\"options\":{\"maximumSignificantDigits\":3,\"maximumFractionDigits\":2}}}}},\"customWidth\":\"33\",\"name\":\"query - 12 - Copy\"},{\"type\":3,\"content\":{\"version\":\"KqlItem/1.0\",\"query\":\"SecurityAlert \\r\\n| where tolower(ResourceId) in ({clustername})\\r\\n| where TimeGenerated {timeframe}\\r\\n| where AlertType startswith \\\"AKS_\\\"\\r\\n| project AlertType, name = extract(@\\\"/MICROSOFT.CONTAINERSERVICE/MANAGEDCLUSTERS/(.+)\\\", 1, ResourceId)\\r\\n| summarize count() by name\\r\\n\\r\\n\",\"size\":4,\"title\":\"Alerts triggered by cluster\",\"queryType\":0,\"resourceType\":\"microsoft.operationalinsights/workspaces\",\"crossComponentResources\":[\"{workspaces}\"],\"visualization\":\"tiles\",\"tileSettings\":{\"showBorder\":false,\"titleContent\":{\"columnMatch\":\"name\",\"formatter\":1},\"leftContent\":{\"columnMatch\":\"count_\",\"formatter\":12,\"formatOptions\":{\"palette\":\"auto\"},\"numberFormat\":{\"unit\":17,\"options\":{\"maximumSignificantDigits\":3,\"maximumFractionDigits\":2}}}}},\"customWidth\":\"33\",\"name\":\"query - 12 - Copy - Copy\"},{\"type\":1,\"content\":{\"json\":\"### Seucirty alerts details\\r\\n\\r\\nTo filter, press on the severities below.\\r\\nYou can also filter based on a specific resource.\"},\"name\":\"text - 18\"},{\"type\":3,\"content\":{\"version\":\"KqlItem/1.0\",\"query\":\"SecurityAlert\\r\\n| where TimeGenerated {timeframe}\\r\\n| extend rg = extract(@\\\"/resourcegroups/([^/]+)\\\", 1, tolower(ResourceId))\\r\\n| extend sub = extract(@\\\"/subscriptions/([^/]+)\\\", 1, tolower(ResourceId))\\r\\n| extend nodesDetailsArr = todynamic('{nodeRgDetails}')\\r\\n| mv-expand singleNodeDetails = nodesDetailsArr\\r\\n| extend singleNodeArr = split(singleNodeDetails, \\\";\\\")\\r\\n| where tolower(singleNodeArr[0]) == rg and tolower(singleNodeArr[1]) == sub\\r\\n| project AlertSeverity\\r\\n| union\\r\\n(\\r\\nSecurityAlert \\r\\n| where tolower(ResourceId) in ({clustername})\\r\\n| where TimeGenerated {timeframe}\\r\\n| where AlertType startswith \\\"AKS_\\\"\\r\\n| project AlertSeverity\\r\\n)\\r\\n| summarize count() by AlertSeverity\",\"size\":0,\"title\":\"Alerts by severity\",\"exportMultipleValues\":true,\"exportedParameters\":[{\"fieldName\":\"AlertSeverity\",\"parameterName\":\"severity\",\"parameterType\":1,\"quote\":\"\"}],\"queryType\":0,\"resourceType\":\"microsoft.operationalinsights/workspaces\",\"crossComponentResources\":[\"{workspaces}\"],\"visualization\":\"tiles\",\"tileSettings\":{\"showBorder\":false,\"titleContent\":{\"columnMatch\":\"AlertSeverity\",\"formatter\":1},\"leftContent\":{\"columnMatch\":\"count_\",\"formatter\":12,\"formatOptions\":{\"palette\":\"auto\"},\"numberFormat\":{\"unit\":17,\"options\":{\"maximumSignificantDigits\":3,\"maximumFractionDigits\":2}}}}},\"customWidth\":\"11\",\"name\":\"Alerts by severity\"},{\"type\":3,\"content\":{\"version\":\"KqlItem/1.0\",\"query\":\"SecurityAlert\\r\\n| where TimeGenerated {timeframe}\\r\\n| where \\\"{severity}\\\" has AlertSeverity or isempty(\\\"{severity}\\\")\\r\\n| extend rg = extract(@\\\"/resourcegroups/([^/]+)\\\", 1, tolower(ResourceId))\\r\\n| extend sub = extract(@\\\"/subscriptions/([^/]+)\\\", 1, tolower(ResourceId))\\r\\n| extend nodesDetailsArr = todynamic('{nodeRgDetails}')\\r\\n| mv-expand singleNodeDetails = nodesDetailsArr\\r\\n| extend singleNodeArr = split(singleNodeDetails, \\\";\\\")\\r\\n| where tolower(singleNodeArr[0]) == rg and tolower(singleNodeArr[1]) == sub\\r\\n| project ResourceId\\r\\n| union\\r\\n(\\r\\nSecurityAlert \\r\\n| where tolower(ResourceId) in ({clustername})\\r\\n| where \\\"{severity}\\\" has AlertSeverity or isempty(\\\"{severity}\\\")\\r\\n| where TimeGenerated {timeframe}\\r\\n| where AlertType startswith \\\"AKS_\\\"\\r\\n| project ResourceId\\r\\n)\\r\\n| summarize Alerts = count() by ResourceId\\r\\n| order by Alerts desc\\r\\n| limit 10\",\"size\":0,\"title\":\"Resources with most alerts\",\"exportFieldName\":\"ResourceId\",\"exportParameterName\":\"selectedResource\",\"exportDefaultValue\":\"not_selected\",\"queryType\":0,\"resourceType\":\"microsoft.operationalinsights/workspaces\",\"crossComponentResources\":[\"{workspaces}\"],\"gridSettings\":{\"formatters\":[{\"columnMatch\":\"Alerts\",\"formatter\":4,\"formatOptions\":{\"palette\":\"red\"}}]}},\"customWidth\":\"22\",\"name\":\"Resources with most alerts\"},{\"type\":3,\"content\":{\"version\":\"KqlItem/1.0\",\"query\":\"\\r\\nSecurityAlert\\r\\n| where TimeGenerated {timeframe}\\r\\n| where \\\"{severity}\\\" has AlertSeverity or isempty(\\\"{severity}\\\")\\r\\n| extend rg = extract(@\\\"/resourcegroups/([^/]+)\\\", 1, tolower(ResourceId))\\r\\n| extend sub = extract(@\\\"/subscriptions/([^/]+)\\\", 1, tolower(ResourceId))\\r\\n| extend nodesDetailsArr = todynamic('{nodeRgDetails}')\\r\\n| mv-expand singleNodeDetails = nodesDetailsArr\\r\\n| extend singleNodeArr = split(singleNodeDetails, \\\";\\\")\\r\\n| where tolower(singleNodeArr[0]) == rg and tolower(singleNodeArr[1]) == sub\\r\\n| extend AlertResourceType = \\\"VM alerts\\\"\\r\\n| union\\r\\n(\\r\\nSecurityAlert \\r\\n| where tolower(ResourceId) in ({clustername})\\r\\n| where \\\"{severity}\\\" has AlertSeverity or isempty(\\\"{severity}\\\")\\r\\n| where TimeGenerated {timeframe}\\r\\n| where AlertType startswith \\\"AKS_\\\"\\r\\n| extend AlertResourceType = \\\"Cluster alerts\\\"\\r\\n)\\r\\n| summarize Alerts = count() by bin(TimeGenerated, {timeframe:grain}), AlertResourceType\",\"size\":0,\"title\":\"Alerts over time\",\"queryType\":0,\"resourceType\":\"microsoft.operationalinsights/workspaces\",\"crossComponentResources\":[\"{workspaces}\"],\"visualization\":\"timechart\"},\"customWidth\":\"66\",\"name\":\"Alerts over time\"},{\"type\":3,\"content\":{\"version\":\"KqlItem/1.0\",\"query\":\"SecurityAlert\\r\\n| where TimeGenerated {timeframe}\\r\\n| where \\\"{severity}\\\" has AlertSeverity or isempty(\\\"{severity}\\\")\\r\\n| extend rg = extract(@\\\"/resourcegroups/([^/]+)\\\", 1, tolower(ResourceId))\\r\\n| extend sub = extract(@\\\"/subscriptions/([^/]+)\\\", 1, tolower(ResourceId))\\r\\n| extend nodesDetailsArr = todynamic('{nodeRgDetails}')\\r\\n| mv-expand singleNodeDetails = nodesDetailsArr\\r\\n| extend singleNodeArr = split(singleNodeDetails, \\\";\\\")\\r\\n| where tolower(singleNodeArr[0]) == rg and tolower(singleNodeArr[1]) == sub\\r\\n| where tolower(ResourceId) == tolower(\\\"{selectedResource}\\\") or \\\"{selectedResource}\\\" == \\\"not_selected\\\"\\r\\n| project [\\\"Resource name\\\"] = ResourceId, TimeGenerated, AlertSeverity, [\\\"AKS cluster\\\"] = toupper(singleNodeArr[2]), DisplayName, AlertLink\\r\\n| union\\r\\n(\\r\\nSecurityAlert\\r\\n| where TimeGenerated {timeframe}\\r\\n| where tolower(ResourceId) in ({clustername})\\r\\n| where \\\"{severity}\\\" has AlertSeverity or isempty(\\\"{severity}\\\")\\r\\n| where AlertType startswith \\\"AKS_\\\"\\r\\n| where tolower(ResourceId) == tolower(\\\"{selectedResource}\\\") or \\\"{selectedResource}\\\" == \\\"not_selected\\\"\\r\\n| project [\\\"Resource name\\\"] = ResourceId, TimeGenerated, AlertSeverity, [\\\"AKS cluster\\\"] = ResourceId, DisplayName, AlertLink\\r\\n)\\r\\n| order by TimeGenerated asc\",\"size\":0,\"title\":\"Microsoft Defender alerts\",\"queryType\":0,\"resourceType\":\"microsoft.operationalinsights/workspaces\",\"crossComponentResources\":[\"{workspaces}\"],\"gridSettings\":{\"formatters\":[{\"columnMatch\":\"AlertLink\",\"formatter\":7,\"formatOptions\":{\"linkTarget\":\"Url\",\"linkLabel\":\"Go to alert \"}}],\"filter\":true},\"sortBy\":[]},\"name\":\"Microsoft Defender alerts\",\"styleSettings\":{\"showBorder\":true}}]},\"conditionalVisibility\":{\"parameterName\":\"mainTab\",\"comparison\":\"isEqualTo\",\"value\":\"alerts\"},\"name\":\"Defender Alerts\"},{\"type\":12,\"content\":{\"version\":\"NotebookGroup/1.0\",\"groupType\":\"editable\",\"items\":[{\"type\":1,\"content\":{\"json\":\"## Diagnostic logs coverage\"},\"name\":\"text - 15\"},{\"type\":3,\"content\":{\"version\":\"KqlItem/1.0\",\"query\":\"union withsource=_TableName *\\r\\n| where _TableName == \\\"AzureDiagnostics\\\" and Category == \\\"kube-audit\\\"\\r\\n| summarize count() by ResourceId = tolower(ResourceId)\\r\\n| summarize logsClusters = make_set(ResourceId)\\r\\n| extend selectedClusters = \\\"[{clustername}]\\\"\\r\\n| extend selectedClusters = replace(\\\"'\\\", '\\\"', selectedClusters)\\r\\n| extend selectedClusters = todynamic(selectedClusters)\\r\\n| mv-expand clusterId = selectedClusters\\r\\n| project clusterId = toupper(tostring(clusterId)), [\\\"Diagnostic logs\\\"] = (logsClusters has tostring(clusterId))\\r\\n| extend [\\\"Diagnostic settings\\\"] = iff([\\\"Diagnostic logs\\\"] == false, strcat(\\\"https://ms.portal.azure.com/#@microsoft.onmicrosoft.com/resource\\\", clusterId, \\\"/diagnostics\\\"), \\\"\\\")\\r\\n\",\"size\":0,\"timeContext\":{\"durationMs\":172800000},\"timeContextFromParameter\":\"timeframe\",\"queryType\":0,\"resourceType\":\"microsoft.operationalinsights/workspaces\",\"crossComponentResources\":[\"{workspaces}\"],\"gridSettings\":{\"formatters\":[{\"columnMatch\":\"Diagnostic logs\",\"formatter\":18,\"formatOptions\":{\"thresholdsOptions\":\"icons\",\"thresholdsGrid\":[{\"operator\":\"==\",\"thresholdValue\":\"false\",\"representation\":\"critical\",\"text\":\"\"},{\"operator\":\"Default\",\"thresholdValue\":null,\"representation\":\"success\",\"text\":\"\"}]}},{\"columnMatch\":\"Diagnostic settings\",\"formatter\":7,\"formatOptions\":{\"linkTarget\":\"Url\"}}],\"filter\":true,\"sortBy\":[{\"itemKey\":\"$gen_thresholds_Diagnostic logs_1\",\"sortOrder\":2}]},\"sortBy\":[{\"itemKey\":\"$gen_thresholds_Diagnostic logs_1\",\"sortOrder\":2}]},\"customWidth\":\"66\",\"name\":\"query - 14\"},{\"type\":3,\"content\":{\"version\":\"KqlItem/1.0\",\"query\":\"union withsource=_TableName *\\r\\n| where _TableName == \\\"AzureDiagnostics\\\" and Category == \\\"kube-audit\\\"\\r\\n| summarize count() by ResourceId = tolower(ResourceId)\\r\\n| summarize logsClusters = make_set(ResourceId)\\r\\n| extend selectedClusters = \\\"[{clustername}]\\\"\\r\\n| extend selectedClusters = replace(\\\"'\\\", '\\\"', selectedClusters)\\r\\n| extend selectedClusters = todynamic(selectedClusters)\\r\\n| mv-expand clusterId = selectedClusters\\r\\n| project clusterId = toupper(tostring(clusterId)), hasDiagnosticLogs = (logsClusters has tostring(clusterId))\\r\\n| summarize [\\\"number of clusters\\\"] = count() by hasDiagnosticLogs\\r\\n| extend hasDiagnosticLogs = iff(hasDiagnosticLogs == true, \\\"Clusters with Diagnostic logs\\\", \\\"Clusters without Diagnostic logs\\\")\\r\\n\",\"size\":0,\"timeContext\":{\"durationMs\":172800000},\"timeContextFromParameter\":\"timeframe\",\"queryType\":0,\"resourceType\":\"microsoft.operationalinsights/workspaces\",\"crossComponentResources\":[\"{workspaces}\"],\"visualization\":\"piechart\"},\"customWidth\":\"33\",\"name\":\"query - 17\"},{\"type\":12,\"content\":{\"version\":\"NotebookGroup/1.0\",\"groupType\":\"editable\",\"items\":[{\"type\":1,\"content\":{\"json\":\"## Cluster operations\"},\"name\":\"text - 16\"},{\"type\":11,\"content\":{\"version\":\"LinkItem/1.0\",\"style\":\"tabs\",\"links\":[{\"id\":\"3f616701-fd4b-482c-aff1-a85414daa05c\",\"cellValue\":\"dispalyedGraph\",\"linkTarget\":\"parameter\",\"linkLabel\":\"Masterclient operations\",\"subTarget\":\"masterclient\",\"preText\":\"\",\"style\":\"link\"},{\"id\":\"e6fa55f1-7d57-4f5e-8e83-429740853731\",\"cellValue\":\"dispalyedGraph\",\"linkTarget\":\"parameter\",\"linkLabel\":\"Pod creation operations\",\"subTarget\":\"podCreation\",\"style\":\"link\"},{\"id\":\"f4c46251-0090-4ca3-a81c-0686bff3ff35\",\"cellValue\":\"dispalyedGraph\",\"linkTarget\":\"parameter\",\"linkLabel\":\"Secret get\\\\list operations\",\"subTarget\":\"secretOperation\",\"style\":\"link\"}]},\"name\":\"links - 11\"},{\"type\":3,\"content\":{\"version\":\"KqlItem/1.0\",\"query\":\"AzureDiagnostics \\r\\n| where tolower(ResourceId) in ({clustername})\\r\\n| where TimeGenerated {timeframe}\\r\\n| where Category == \\\"kube-audit\\\"\\r\\n| where log_s has \\\"masterclient\\\"\\r\\n| project TimeGenerated, parse_json(log_s), ResourceId\\r\\n| project TimeGenerated, ResourceId, username = tostring(log_s[\\\"user\\\"].username)\\r\\n| where username == \\\"masterclient\\\"\\r\\n| extend name = extract(@\\\"/MICROSOFT.CONTAINERSERVICE/MANAGEDCLUSTERS/(.+)\\\", 1, ResourceId)\\r\\n| summarize count() by name, bin(TimeGenerated, 1h)\",\"size\":0,\"queryType\":0,\"resourceType\":\"microsoft.operationalinsights/workspaces\",\"crossComponentResources\":[\"{workspaces}\"],\"visualization\":\"timechart\",\"tileSettings\":{\"showBorder\":false,\"titleContent\":{\"columnMatch\":\"name\",\"formatter\":1},\"leftContent\":{\"columnMatch\":\"count_\",\"formatter\":12,\"formatOptions\":{\"palette\":\"auto\"},\"numberFormat\":{\"unit\":17,\"options\":{\"maximumSignificantDigits\":3,\"maximumFractionDigits\":2}}}},\"chartSettings\":{\"yAxis\":[\"count_\"]}},\"conditionalVisibility\":{\"parameterName\":\"dispalyedGraph\",\"comparison\":\"isEqualTo\",\"value\":\"masterclient\"},\"name\":\"Masterclient operations\",\"styleSettings\":{\"showBorder\":true}},{\"type\":3,\"content\":{\"version\":\"KqlItem/1.0\",\"query\":\"AzureDiagnostics\\r\\n| where Category == \\\"kube-audit\\\"\\r\\n| where tolower(ResourceId) in ({clustername})\\r\\n| where TimeGenerated {timeframe}\\r\\n| where log_s has \\\"pods\\\"\\r\\n| project TimeGenerated, parse_json(log_s), ResourceId\\r\\n| project AzureResourceId = ResourceId, TimeGenerated,\\r\\n RequestURI = tostring(log_s[\\\"requestURI\\\"]),\\r\\n Verb = tostring(log_s[\\\"verb\\\"]),\\r\\n ObjectRef = log_s[\\\"objectRef\\\"],\\r\\n ResponseStatus = log_s[\\\"responseStatus\\\"]\\r\\n//Main query\\r\\n| where ObjectRef.resource == \\\"pods\\\" and Verb == \\\"create\\\" and ResponseStatus.code startswith \\\"20\\\"\\r\\n and RequestURI endswith \\\"/pods\\\"\\r\\n| extend name = extract(@\\\"/MICROSOFT.CONTAINERSERVICE/MANAGEDCLUSTERS/(.+)\\\", 1, AzureResourceId)\\r\\n| summarize count() by name, bin(TimeGenerated, 1h)\",\"size\":0,\"queryType\":0,\"resourceType\":\"microsoft.operationalinsights/workspaces\",\"crossComponentResources\":[\"{workspaces}\"],\"visualization\":\"timechart\"},\"conditionalVisibility\":{\"parameterName\":\"dispalyedGraph\",\"comparison\":\"isEqualTo\",\"value\":\"podCreation\"},\"name\":\"pods creation\",\"styleSettings\":{\"showBorder\":true}},{\"type\":3,\"content\":{\"version\":\"KqlItem/1.0\",\"query\":\"AzureDiagnostics\\r\\n| where Category == \\\"kube-audit\\\"\\r\\n| where tolower(ResourceId) in ({clustername})\\r\\n| where TimeGenerated {timeframe}\\r\\n| where log_s has \\\"secrets\\\"\\r\\n| project TimeGenerated, parse_json(log_s), ResourceId\\r\\n| project AzureResourceId = ResourceId, TimeGenerated,\\r\\n RequestURI = tostring(log_s[\\\"requestURI\\\"]),\\r\\n Verb = tostring(log_s[\\\"verb\\\"]),\\r\\n ObjectRef = log_s[\\\"objectRef\\\"],\\r\\n ResponseStatus = log_s[\\\"responseStatus\\\"]\\r\\n//Main query\\r\\n| where ObjectRef.resource == \\\"secrets\\\" and (Verb == \\\"list\\\" or Verb == \\\"get\\\") and ResponseStatus.code startswith \\\"20\\\"\\r\\n| where ObjectRef.name != \\\"tunnelfront\\\" and ObjectRef.name != \\\"tunnelend\\\" and ObjectRef.name != \\\"kubernetes-dashboard-key-holder\\\"\\r\\n| extend name = extract(@\\\"/MICROSOFT.CONTAINERSERVICE/MANAGEDCLUSTERS/(.+)\\\", 1, AzureResourceId)\\r\\n| summarize count() by name, bin(TimeGenerated, 1h)\",\"size\":0,\"timeContext\":{\"durationMs\":172800000},\"timeContextFromParameter\":\"timeframe\",\"queryType\":0,\"resourceType\":\"microsoft.operationalinsights/workspaces\",\"crossComponentResources\":[\"{workspaces}\"],\"visualization\":\"timechart\",\"gridSettings\":{\"sortBy\":[{\"itemKey\":\"count_\",\"sortOrder\":2}]},\"sortBy\":[{\"itemKey\":\"count_\",\"sortOrder\":2}]},\"conditionalVisibility\":{\"parameterName\":\"dispalyedGraph\",\"comparison\":\"isEqualTo\",\"value\":\"secretOperation\"},\"name\":\"secrets operation\",\"styleSettings\":{\"showBorder\":true}},{\"type\":3,\"content\":{\"version\":\"KqlItem/1.0\",\"query\":\"let ascAlerts = \\nunion withsource=_TableName *\\n| where _TableName == \\\"SecurityAlert\\\"\\n| where tolower(ResourceId) in ({clustername})\\n| where TimeGenerated {timeframe}\\n| extend AlertType = column_ifexists(\\\"AlertType\\\", \\\"\\\")\\n| where AlertType == \\\"AKS_PrivilegedContainer\\\"\\n| extend ExtendedProperties = column_ifexists(\\\"ExtendedProperties\\\", todynamic(\\\"\\\"))\\n| extend ExtendedProperties = parse_json(ExtendedProperties)\\n| extend AlertLink = column_ifexists(\\\"AlertLink\\\", \\\"\\\")\\n| summarize arg_min(TimeGenerated, AlertLink) by AzureResourceId = ResourceId, name = tostring(ExtendedProperties[\\\"Pod name\\\"]), podNamespace = tostring(ExtendedProperties[\\\"Namespace\\\"])\\n;\\nlet podOperations = AzureDiagnostics\\n| where Category == \\\"kube-audit\\\"\\n| where tolower(ResourceId) in ({clustername})\\n| where TimeGenerated {timeframe}\\n| where log_s has \\\"privileged\\\"\\n| project TimeGenerated, parse_json(log_s), ResourceId\\n| project AzureResourceId = ResourceId, TimeGenerated,\\n RequestURI = tostring(log_s[\\\"requestURI\\\"]),\\n Verb = tostring(log_s[\\\"verb\\\"]),\\n ObjectRef = log_s[\\\"objectRef\\\"],\\n RequestObject = log_s[\\\"requestObject\\\"],\\n ResponseStatus = log_s[\\\"responseStatus\\\"],\\n ResponseObject = log_s[\\\"responseObject\\\"]\\n//Main query\\n| where ObjectRef.resource == \\\"pods\\\" and Verb == \\\"create\\\" and ResponseStatus.code startswith \\\"20\\\" and RequestObject has \\\"privileged\\\"\\n and RequestURI endswith \\\"/pods\\\"\\n| extend containers = RequestObject.spec.containers\\n| mvexpand containers\\n| where containers.securityContext.privileged == true\\n| summarize TimeGenerated = min(TimeGenerated) by\\n name = tostring(ResponseObject.metadata.name),\\n podNamespace = tostring(ResponseObject.metadata.namespace),\\n imageName = tostring(containers.image),\\n containerName = tostring(containers.name),\\n AzureResourceId\\n| extend id = strcat(name,\\\";\\\", AzureResourceId)\\n| extend parent = AzureResourceId\\n| join kind=leftouter (ascAlerts) on AzureResourceId, name, podNamespace\\n;\\nlet cached = materialize(podOperations)\\n;\\nlet clusters = cached | distinct AzureResourceId\\n;\\n// Main query\\ncached\\n| union\\n(\\nclusters\\n| project \\n name = AzureResourceId,\\n id = AzureResourceId,\\n parent = \\\"\\\" \\n)\\n| project-away name1, podNamespace1, TimeGenerated1\",\"size\":1,\"title\":\"Privileged containers creation\",\"queryType\":0,\"resourceType\":\"microsoft.operationalinsights/workspaces\",\"crossComponentResources\":[\"{workspaces}\"],\"visualization\":\"table\",\"gridSettings\":{\"formatters\":[{\"columnMatch\":\"name\",\"formatter\":13,\"formatOptions\":{\"linkTarget\":null,\"showIcon\":true}},{\"columnMatch\":\"AzureResourceId\",\"formatter\":5},{\"columnMatch\":\"id\",\"formatter\":5},{\"columnMatch\":\"parent\",\"formatter\":5},{\"columnMatch\":\"AlertLink\",\"formatter\":7,\"formatOptions\":{\"linkTarget\":\"Url\",\"linkLabel\":\"\"}}],\"hierarchySettings\":{\"idColumn\":\"id\",\"parentColumn\":\"parent\",\"treeType\":0,\"expanderColumn\":\"name\"}},\"sortBy\":[]},\"customWidth\":\"66\",\"name\":\"Privileged container\",\"styleSettings\":{\"showBorder\":true}},{\"type\":3,\"content\":{\"version\":\"KqlItem/1.0\",\"query\":\"SecurityAlert \\r\\n| where tolower(ResourceId) in ({clustername})\\r\\n| where TimeGenerated {timeframe}\\r\\n| where AlertType == \\\"AKS_PrivilegedContainer\\\"\\r\\n| project name = extract(@\\\"/MICROSOFT.CONTAINERSERVICE/MANAGEDCLUSTERS/(.+)\\\", 1, ResourceId)\\r\\n| summarize alert = count() by name\",\"size\":1,\"title\":\"AKS clusters with related Microsoft Defender alerts\",\"queryType\":0,\"resourceType\":\"microsoft.operationalinsights/workspaces\",\"crossComponentResources\":[\"{workspaces}\"],\"visualization\":\"piechart\",\"tileSettings\":{\"showBorder\":false,\"titleContent\":{\"columnMatch\":\"name\",\"formatter\":1},\"leftContent\":{\"columnMatch\":\"alert\",\"formatter\":12,\"formatOptions\":{\"palette\":\"auto\"},\"numberFormat\":{\"unit\":17,\"options\":{\"maximumSignificantDigits\":3,\"maximumFractionDigits\":2}}}},\"graphSettings\":{\"type\":0,\"topContent\":{\"columnMatch\":\"name\",\"formatter\":1},\"centerContent\":{\"columnMatch\":\"alert\",\"formatter\":1,\"numberFormat\":{\"unit\":17,\"options\":{\"maximumSignificantDigits\":3,\"maximumFractionDigits\":2}}}}},\"customWidth\":\"33\",\"name\":\"query - 7\",\"styleSettings\":{\"showBorder\":true}},{\"type\":3,\"content\":{\"version\":\"KqlItem/1.0\",\"query\":\"let baseQuery = AzureDiagnostics \\r\\n| where Category == \\\"kube-audit\\\"\\r\\n| where tolower(ResourceId) in ({clustername})\\r\\n| where TimeGenerated {timeframe}\\r\\n| where log_s has \\\"exec\\\"\\r\\n| project TimeGenerated, parse_json(log_s), ResourceId\\r\\n| project TimeGenerated,\\r\\n AzureResourceId = ResourceId,\\r\\n User = log_s[\\\"user\\\"],\\r\\n StageTimestamp = todatetime(log_s[\\\"stageTimestamp\\\"]),\\r\\n Timestamp = todatetime(log_s[\\\"timestamp\\\"]),\\r\\n Stage = tostring(log_s[\\\"stage\\\"]),\\r\\n RequestURI = tostring(log_s[\\\"requestURI\\\"]),\\r\\n UserAgent = tostring(log_s[\\\"userAgent\\\"]),\\r\\n Verb = tostring(log_s[\\\"verb\\\"]),\\r\\n ObjectRef = log_s[\\\"objectRef\\\"],\\r\\n ResponseStatus = log_s[\\\"responseStatus\\\"]\\r\\n| where ObjectRef.resource == \\\"pods\\\" and Verb == \\\"create\\\" and ResponseStatus.code == 101 and ObjectRef.subresource == \\\"exec\\\"\\r\\n| project operationTime = TimeGenerated,\\r\\n RequestURI,\\r\\n podName = tostring(ObjectRef.name),\\r\\n podNamespace = tostring(ObjectRef.namespace),\\r\\n username = tostring(User.username),\\r\\n AzureResourceId\\r\\n// Parse the exec command\\r\\n| extend commands = extractall(@\\\"command=([^\\\\&]*)\\\", RequestURI)\\r\\n| extend commandsStr = url_decode(strcat_array(commands, \\\" \\\"))\\r\\n| project-away ['commands'], RequestURI\\r\\n| where username != \\\"aksProblemDetector\\\"\\r\\n;\\r\\nlet cached = materialize(baseQuery);\\r\\nlet execOperations = \\r\\ncached\\r\\n| summarize operationTime = min(operationTime), numberOfPerations = count() by name = commandsStr, username, podNamespace, podName, AzureResourceId\\r\\n| extend id = name\\r\\n| extend parentId = podName\\r\\n| project id, parentId, name, operationTime, numberOfPerations, podNamespace, username, AzureResourceId\\r\\n;\\r\\nlet podOperations = \\r\\ncached\\r\\n| summarize operationTime = min(operationTime), numberOfPerations = count() by name = podName, podNamespace, AzureResourceId\\r\\n| extend id = name\\r\\n| extend parentId = AzureResourceId\\r\\n| project id, parentId, name, operationTime, numberOfPerations, podNamespace, username = \\\"\\\", AzureResourceId\\r\\n;\\r\\nlet clusterOperations = \\r\\ncached\\r\\n| summarize operationTime = min(operationTime), numberOfPerations = count() by name = AzureResourceId\\r\\n| extend id = name\\r\\n| extend parentId = \\\"\\\"\\r\\n| project id, parentId, name, operationTime, numberOfPerations, username = \\\"\\\", podNamespace = \\\"\\\", AzureResourceId = name\\r\\n;\\r\\nunion clusterOperations, podOperations, execOperations\",\"size\":1,\"title\":\"exec commands\",\"queryType\":0,\"resourceType\":\"microsoft.operationalinsights/workspaces\",\"crossComponentResources\":[\"{workspaces}\"],\"visualization\":\"table\",\"gridSettings\":{\"formatters\":[{\"columnMatch\":\"id\",\"formatter\":5},{\"columnMatch\":\"parentId\",\"formatter\":5},{\"columnMatch\":\"numberOfPerations\",\"formatter\":4,\"formatOptions\":{\"palette\":\"blue\",\"compositeBarSettings\":{\"labelText\":\"\",\"columnSettings\":[]}}},{\"columnMatch\":\"AzureResourceId\",\"formatter\":5}],\"hierarchySettings\":{\"idColumn\":\"id\",\"parentColumn\":\"parentId\",\"treeType\":0,\"expanderColumn\":\"name\",\"expandTopLevel\":false}}},\"customWidth\":\"33\",\"name\":\"exec commands\",\"styleSettings\":{\"showBorder\":true}},{\"type\":3,\"content\":{\"version\":\"KqlItem/1.0\",\"query\":\"SecurityAlert \\r\\n| where AlertType == \\\"AKS_MaliciousContainerExec\\\"\\r\\n| where tolower(ResourceId) in ({clustername})\\r\\n| where TimeGenerated {timeframe}\\r\\n| project TimeGenerated, ResourceId, ExtendedProperties = todynamic(ExtendedProperties)\\r\\n| project TimeGenerated, ResourceId, [\\\"Pod name\\\"] = ExtendedProperties[\\\"Pod name\\\"], Command = ExtendedProperties[\\\"Command\\\"]\",\"size\":1,\"title\":\"Related Microsoft Defender alerts details\",\"queryType\":0,\"resourceType\":\"microsoft.operationalinsights/workspaces\",\"crossComponentResources\":[\"{workspaces}\"],\"gridSettings\":{\"sortBy\":[{\"itemKey\":\"TimeGenerated\",\"sortOrder\":1}]},\"sortBy\":[{\"itemKey\":\"TimeGenerated\",\"sortOrder\":1}]},\"customWidth\":\"33\",\"name\":\"query - 9\",\"styleSettings\":{\"showBorder\":true}},{\"type\":3,\"content\":{\"version\":\"KqlItem/1.0\",\"query\":\"SecurityAlert \\r\\n| where AlertType == \\\"AKS_MaliciousContainerExec\\\"\\r\\n| where tolower(ResourceId) in ({clustername})\\r\\n| where TimeGenerated {timeframe}\\r\\n| project name = extract(@\\\"/MICROSOFT.CONTAINERSERVICE/MANAGEDCLUSTERS/(.+)\\\", 1, ResourceId)\\r\\n| summarize alert = count() by name\",\"size\":1,\"title\":\"AKS clusters with related Microsoft Defender alerts\",\"queryType\":0,\"resourceType\":\"microsoft.operationalinsights/workspaces\",\"crossComponentResources\":[\"{workspaces}\"],\"visualization\":\"piechart\"},\"customWidth\":\"33\",\"name\":\"query - 8\",\"styleSettings\":{\"showBorder\":true}},{\"type\":3,\"content\":{\"version\":\"KqlItem/1.0\",\"query\":\"let ascAlerts = \\r\\nunion withsource=_TableName *\\r\\n| where _TableName == \\\"SecurityAlert\\\"\\r\\n| where tolower(ResourceId) in ({clustername})\\r\\n| where TimeGenerated {timeframe}\\r\\n| extend AlertType = column_ifexists(\\\"AlertType\\\", \\\"\\\")\\r\\n| where AlertType == \\\"AKS_SensitiveMount\\\"\\r\\n| extend ExtendedProperties = column_ifexists(\\\"ExtendedProperties\\\", todynamic(\\\"\\\"))\\r\\n| extend ExtendedProperties = parse_json(ExtendedProperties)\\r\\n| extend AlertLink = column_ifexists(\\\"AlertLink\\\", \\\"\\\")\\r\\n| summarize arg_min(TimeGenerated, AlertLink) by AzureResourceId = ResourceId, containerName = tostring(ExtendedProperties[\\\"Container name\\\"]), mountPath = tostring(ExtendedProperties[\\\"Sensitive mount path\\\"])\\r\\n;\\r\\nlet podOperations = \\r\\nAzureDiagnostics \\r\\n| where tolower(ResourceId) in ({clustername})\\r\\n| where TimeGenerated {timeframe}\\r\\n| where Category == \\\"kube-audit\\\"\\r\\n| where log_s has \\\"hostPath\\\"\\r\\n| project TimeGenerated, parse_json(log_s), ResourceId\\r\\n//Parsing\\r\\n| project AzureResourceId = ResourceId, TimeGenerated,\\r\\n Verb = tostring(log_s[\\\"verb\\\"]),\\r\\n ObjectRef = log_s[\\\"objectRef\\\"],\\r\\n RequestObject = log_s[\\\"requestObject\\\"],\\r\\n ResponseStatus = log_s[\\\"responseStatus\\\"],\\r\\n ResponseObject = log_s[\\\"responseObject\\\"]\\r\\n//\\r\\n//Main query\\r\\n//\\r\\n| where ObjectRef.resource == \\\"pods\\\" and Verb == \\\"create\\\" and ResponseStatus.code startswith \\\"20\\\" and RequestObject has \\\"hostPath\\\"\\r\\n| extend volumes = RequestObject.spec.volumes\\r\\n| mvexpand volumes\\r\\n| extend mountPath = volumes.hostPath.path\\r\\n| where mountPath != \\\"\\\" \\r\\n| extend container = RequestObject.spec.containers\\r\\n| mvexpand container\\r\\n| extend detectionTime = TimeGenerated\\r\\n| project detectionTime,\\r\\n podName = ResponseObject.metadata.name,\\r\\n podNamespace = ResponseObject.metadata.namespace,\\r\\n containerName = container.name,\\r\\n containerImage = container.image,\\r\\n mountPath,\\r\\n mountName = volumes.name,\\r\\n AzureResourceId,\\r\\n container\\r\\n| extend volumeMounts = container.volumeMounts\\r\\n| mv-expand volumeMounts\\r\\n| where tostring(volumeMounts.name) == tostring(mountName)\\r\\n| summarize operationTime = min(detectionTime) by AzureResourceId, name = tostring(podName),tostring(podNamespace), tostring(containerName), tostring(containerImage), tostring(mountPath), tostring(mountName)\\r\\n| extend id = strcat(name, \\\";\\\", AzureResourceId)\\r\\n| extend parent = AzureResourceId\\r\\n| join kind=leftouter (ascAlerts) on AzureResourceId, containerName, mountPath\\r\\n;\\r\\nlet cached = materialize(podOperations)\\r\\n;\\r\\nlet clusters = cached | distinct AzureResourceId\\r\\n;\\r\\n// Main query\\r\\ncached\\r\\n| union\\r\\n(\\r\\nclusters\\r\\n| project \\r\\n name = toupper(AzureResourceId),\\r\\n id = AzureResourceId,\\r\\n parent = \\\"\\\" \\r\\n)\\r\\n| project-away containerName1, mountPath1, TimeGenerated\\r\\n\",\"size\":1,\"title\":\"hostPath mount\",\"queryType\":0,\"resourceType\":\"microsoft.operationalinsights/workspaces\",\"crossComponentResources\":[\"{workspaces}\"],\"gridSettings\":{\"formatters\":[{\"columnMatch\":\"AzureResourceId\",\"formatter\":5},{\"columnMatch\":\"name\",\"formatter\":13,\"formatOptions\":{\"linkTarget\":null,\"showIcon\":true}},{\"columnMatch\":\"id\",\"formatter\":5},{\"columnMatch\":\"parent\",\"formatter\":5},{\"columnMatch\":\"AzureResourceId1\",\"formatter\":5},{\"columnMatch\":\"AlertLink\",\"formatter\":7,\"formatOptions\":{\"linkTarget\":\"Url\"}}],\"hierarchySettings\":{\"idColumn\":\"id\",\"parentColumn\":\"parent\",\"treeType\":0,\"expanderColumn\":\"name\"}},\"sortBy\":[]},\"customWidth\":\"66\",\"name\":\"query - 10\",\"styleSettings\":{\"showBorder\":true}},{\"type\":3,\"content\":{\"version\":\"KqlItem/1.0\",\"query\":\"SecurityAlert \\r\\n| where AlertType == \\\"AKS_SensitiveMount\\\"\\r\\n| where tolower(ResourceId) in ({clustername})\\r\\n| where TimeGenerated {timeframe}\\r\\n| project name = extract(@\\\"/MICROSOFT.CONTAINERSERVICE/MANAGEDCLUSTERS/(.+)\\\", 1, ResourceId)\\r\\n| summarize alert = count() by name\",\"size\":1,\"title\":\"AKS clusters with related Microsoft Defender alerts\",\"queryType\":0,\"resourceType\":\"microsoft.operationalinsights/workspaces\",\"crossComponentResources\":[\"{workspaces}\"],\"visualization\":\"piechart\",\"sortBy\":[]},\"customWidth\":\"33\",\"name\":\"query - 10\",\"styleSettings\":{\"showBorder\":true}},{\"type\":3,\"content\":{\"version\":\"KqlItem/1.0\",\"query\":\"let bindingOper = AzureDiagnostics\\r\\n| where Category == \\\"kube-audit\\\"\\r\\n| where tolower(ResourceId) in ({clustername})\\r\\n| where TimeGenerated {timeframe}\\r\\n| where log_s has \\\"clusterrolebindings\\\"\\r\\n| project TimeGenerated, parse_json(log_s), ResourceId\\r\\n//Parsing\\r\\n| project AzureResourceId = ResourceId, TimeGenerated,\\r\\n RequestURI = tostring(log_s[\\\"requestURI\\\"]),\\r\\n User = log_s[\\\"user\\\"],\\r\\n Verb = tostring(log_s[\\\"verb\\\"]),\\r\\n ObjectRef = log_s[\\\"objectRef\\\"],\\r\\n RequestObject = log_s[\\\"requestObject\\\"],\\r\\n ResponseStatus = log_s[\\\"responseStatus\\\"]\\r\\n| where ObjectRef.resource == \\\"clusterrolebindings\\\" and Verb == \\\"create\\\" and ResponseStatus.code startswith \\\"20\\\" and RequestObject.roleRef.name == \\\"cluster-admin\\\" \\r\\n| extend subjects = RequestObject.subjects\\r\\n| mv-expand subjects\\r\\n| project AzureResourceId, TimeGenerated, subjectName = tostring(subjects.name), subjectKind = tostring(subjects[\\\"kind\\\"]), bindingName = tostring(ObjectRef.name)\\r\\n| summarize operationTime = min(TimeGenerated) by AzureResourceId, subjectName, subjectKind, bindingName\\r\\n| extend id = strcat(subjectName, \\\";\\\", AzureResourceId)\\r\\n| extend parent = AzureResourceId\\r\\n;\\r\\nlet cached = materialize(bindingOper)\\r\\n;\\r\\nlet clusters = cached | distinct AzureResourceId\\r\\n;\\r\\n// Main query\\r\\ncached\\r\\n| union\\r\\n(\\r\\nclusters\\r\\n| project \\r\\n name = AzureResourceId,\\r\\n id = AzureResourceId,\\r\\n parent = \\\"\\\" \\r\\n)\",\"size\":1,\"title\":\"Cluster-admin binding\",\"queryType\":0,\"resourceType\":\"microsoft.operationalinsights/workspaces\",\"crossComponentResources\":[\"{workspaces}\"],\"visualization\":\"table\",\"gridSettings\":{\"formatters\":[{\"columnMatch\":\"AzureResourceId\",\"formatter\":5},{\"columnMatch\":\"id\",\"formatter\":5},{\"columnMatch\":\"parent\",\"formatter\":5},{\"columnMatch\":\"name\",\"formatter\":13,\"formatOptions\":{\"linkTarget\":null,\"showIcon\":true}}],\"hierarchySettings\":{\"idColumn\":\"id\",\"parentColumn\":\"parent\",\"treeType\":0,\"expanderColumn\":\"name\"}}},\"customWidth\":\"66\",\"name\":\"query - 5\",\"styleSettings\":{\"showBorder\":true}},{\"type\":3,\"content\":{\"version\":\"KqlItem/1.0\",\"query\":\"SecurityAlert \\r\\n| where tolower(ResourceId) in ({clustername})\\r\\n| where TimeGenerated {timeframe}\\r\\n| where AlertType == \\\"AKS_ClusterAdminBinding\\\"\\r\\n| project name = extract(@\\\"/MICROSOFT.CONTAINERSERVICE/MANAGEDCLUSTERS/(.+)\\\", 1, ResourceId)\\r\\n| summarize count() by name\",\"size\":1,\"title\":\"AKS clusters with related Microsoft Defender alerts\",\"queryType\":0,\"resourceType\":\"microsoft.operationalinsights/workspaces\",\"crossComponentResources\":[\"{workspaces}\"],\"visualization\":\"piechart\"},\"customWidth\":\"33\",\"name\":\"query - 11\",\"styleSettings\":{\"showBorder\":true}},{\"type\":3,\"content\":{\"version\":\"KqlItem/1.0\",\"query\":\"AzureDiagnostics\\r\\n| where Category == \\\"kube-audit\\\"\\r\\n| where tolower(ResourceId) in ({clustername})\\r\\n| where TimeGenerated {timeframe}\\r\\n| where log_s has \\\"events\\\"\\r\\n| project TimeGenerated, parse_json(log_s), ResourceId\\r\\n//Parsing\\r\\n| project AzureResourceId = ResourceId, \\r\\n TimeGenerated,\\r\\n SourceIPs = tostring(log_s[\\\"sourceIPs\\\"][0]),\\r\\n User = log_s[\\\"user\\\"],\\r\\n Verb = tostring(log_s[\\\"verb\\\"]),\\r\\n ObjectRef = log_s[\\\"objectRef\\\"],\\r\\n ResponseStatus = log_s[\\\"responseStatus\\\"]\\r\\n| where ObjectRef.resource == \\\"events\\\" and Verb == \\\"delete\\\" and ResponseStatus.code == 200\\r\\n| project TimeGenerated, AzureResourceId, username = tostring(User.username), ipAddr = tostring(SourceIPs), \\r\\n eventName = tostring(ObjectRef.name), eventNamespace = tostring(ObjectRef.namespace), status = tostring(ResponseStatus.code)\\r\\n| summarize operationTime = min(TimeGenerated), eventNames = make_set(eventName, 10) by\\r\\n AzureResourceId, \\r\\n eventNamespace,\\r\\n username,\\r\\n ipAddr\\r\\n// Format the list of the event names\\r\\n| extend eventNames = substring(eventNames, 1 , strlen(eventNames) - 2)\\r\\n| extend eventNames = replace('\\\"', \\\"\\\", eventNames)\\r\\n| extend eventNames = replace(\\\",\\\", \\\", \\\", eventNames)\",\"size\":1,\"title\":\"Delete events\",\"queryType\":0,\"resourceType\":\"microsoft.operationalinsights/workspaces\",\"crossComponentResources\":[\"{workspaces}\"]},\"name\":\"query - 6\",\"styleSettings\":{\"showBorder\":true}}]},\"conditionalVisibility\":{\"parameterName\":\"diagnosticClusters\",\"comparison\":\"isEqualTo\",\"value\":\"yes\"},\"name\":\"diagnosticData\"},{\"type\":1,\"content\":{\"json\":\"No Diagnostic Logs data in the selected workspaces. \\r\\nTo enable Diagnostic Logs for your AKS cluster: Go to your AKS cluster --> Diagnostic settings --> Add diagnostic setting --> Select \\\"kube-audit\\\" and send the data to your workspace.\\r\\n\\r\\nGet more details here: https://learn.microsoft.com/azure/aks/view-master-logs\",\"style\":\"info\"},\"conditionalVisibility\":{\"parameterName\":\"diagnosticClusters\",\"comparison\":\"isEqualTo\",\"value\":\"no\"},\"name\":\"text - 4\"}]},\"conditionalVisibility\":{\"parameterName\":\"mainTab\",\"comparison\":\"isEqualTo\",\"value\":\"diagnostics\"},\"name\":\"diagnostics\"}],\"fromTemplateId\":\"sentinel-AksWorkbook\",\"$schema\":\"https://github.com/Microsoft/Application-Insights-Workbooks/blob/master/schema/workbook.json\"}" - }, - "resources": [ - { - "type": "Microsoft.ManagedIdentity/userAssignedIdentities", - "apiVersion": "2018-11-30", - "name": "[variables('clusterControlPlaneIdentityName')]", - "location": "[parameters('location')]", - "comments": "The control plane identity used by the cluster. Used for networking access (VNET joining and DNS updating)" - }, - { - "type": "Microsoft.ManagedIdentity/userAssignedIdentities", - "apiVersion": "2018-11-30", - "name": "mi-appgateway-frontend", - "location": "[parameters('location')]", - "comments": "User Managed Identity that App Gateway is assigned. Used for Azure Key Vault Access." - }, - { - "type": "Microsoft.Compute/virtualMachineScaleSets", - "apiVersion": "2020-12-01", - "name": "vmss-jumpboxes", - "comments": "Hosts the VMs used as AKS jumpboxes. Using VMSS as a way to manage spanning fault and update domains.", - "dependsOn": [ - "[resourceId('Microsoft.OperationalInsights/workspaces', variables('logAnalyticsWorkspaceName'))]", - "[resourceId('Microsoft.OperationsManagement/solutions', variables('vmInsightsSolutionName'))]" - ], - "location": "[parameters('location')]", - "zones": ["1", "2", "3"], - "sku": { - "name": "Standard_DS1_v2", - "tier": "Standard", - "capacity": 2 - }, - "properties": { - "additionalCapabilities": { - "ultraSSDEnabled": false - }, - "overprovision": false, - "singlePlacementGroup": true, - "upgradePolicy": { - "mode": "Automatic" - }, - "zoneBalance": false, - "virtualMachineProfile": { - "diagnosticsProfile": { - "bootDiagnostics": { - "enabled": true - } - }, - "osProfile": { - "computerNamePrefix": "aksjmp", - "linuxConfiguration": { - "disablePasswordAuthentication": true, - "provisionVMAgent": true, - "ssh": { - "publicKeys": [ - { - "path": "[concat('/home/', variables('jumpBoxDefaultAdminUserName'), '/.ssh/authorized_keys')]", - "keyData": "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQCcFvQl2lYPcK1tMB3Tx2R9n8a7w5MJCSef14x0ePRFr9XISWfCVCNKRLM3Al/JSlIoOVKoMsdw5farEgXkPDK5F+SKLss7whg2tohnQNQwQdXit1ZjgOXkis/uft98Cv8jDWPbhwYj+VH/Aif9rx8abfjbvwVWBGeA/OnvfVvXnr1EQfdLJgMTTh+hX/FCXCqsRkQcD91MbMCxpqk8nP6jmsxJBeLrgfOxjH8RHEdSp4fF76YsRFHCi7QOwTE/6U+DpssgQ8MTWRFRat97uTfcgzKe5MOfuZHZ++5WFBgaTr1vhmSbXteGiK7dQXOk2cLxSvKkzeaiju9Jy6hoSl5oMygUVd5fNPQ94QcqTkMxZ9tQ9vPWOHwbdLRD31Ses3IBtDV+S6ehraiXf/L/e0jRUYk8IL/J543gvhOZ0hj2sQqTj9XS2hZkstZtrB2ywrJzV5ByETUU/oF9OsysyFgnaQdyduVqEPHaqXqnJvBngqqas91plyT3tSLMez3iT0s= unused-generated-by-azure" - } - ] - } - }, - "customData": "[parameters('jumpBoxCloudInitAsBase64')]", - "adminUsername": "[variables('jumpBoxDefaultAdminUserName')]" - }, - "storageProfile": { - "osDisk": { - "createOption": "FromImage", - "caching": "ReadOnly", - "diffDiskSettings": { - "option": "Local" - }, - "osType": "Linux" - }, - "imageReference": { - "id": "[parameters('jumpBoxImageResourceId')]" - } - }, - "networkProfile": { - "networkInterfaceConfigurations": [ - { - "name": "vnet-spoke-BU0001A0005-01-nic01", - "properties": { - "primary": true, - "enableIPForwarding": false, - "enableAcceleratedNetworking": false, - "networkSecurityGroup": "[null()]", - "ipConfigurations": [ - { - "name": "default", - "properties": { - "primary": true, - "privateIPAddressVersion": "IPv4", - "publicIPAddressConfiguration": "[null()]", - "subnet": { - "id": "[variables('vnetManagementOpsSubnetResourceId')]" - } - } - } - ] - } - } - ] - } - } - }, - "resources": [ - { - "type": "extensions", - "apiVersion": "2019-12-01", - "name": "OMSExtension", - "location": "[parameters('location')]", - "dependsOn": [ - "[resourceId('Microsoft.Compute/virtualMachineScaleSets', 'vmss-jumpboxes')]" - ], - "properties": { - "publisher": "Microsoft.EnterpriseCloud.Monitoring", - "type": "OmsAgentForLinux", - "typeHandlerVersion": "1.13", - "autoUpgradeMinorVersion": true, - "settings": { - "stopOnMultipleConnections": true, - "azureResourceId": "[resourceId('Microsoft.Compute/virtualMachineScaleSets', 'vmss-jumpboxes')]", - "workspaceId": "[reference(resourceId('Microsoft.OperationalInsights/workspaces', variables('logAnalyticsWorkspaceName')), '2020-10-01').customerId]" - }, - "protectedSettings": { - "workspaceKey": "[listKeys(resourceId('Microsoft.OperationalInsights/workspaces', variables('logAnalyticsWorkspaceName')), '2020-10-01').primarySharedKey]" - } - } - }, - { - "type": "extensions", - "apiVersion": "2019-12-01", - "name": "DependencyAgentLinux", - "location": "[parameters('location')]", - "dependsOn": [ - "[resourceId('Microsoft.Compute/virtualMachineScaleSets', 'vmss-jumpboxes')]", - "[resourceId('Microsoft.Compute/virtualMachineScaleSets/extensions', 'vmss-jumpboxes', 'OMSExtension')]" - ], - "properties": { - "publisher": "Microsoft.Azure.Monitoring.DependencyAgent", - "type": "DependencyAgentLinux", - "typeHandlerVersion": "9.10", - "autoUpgradeMinorVersion": true - } - } - ] - }, - { - "type": "Microsoft.ManagedIdentity/userAssignedIdentities", - "apiVersion": "2018-11-30", - "name": "podmi-ingress-controller", - "location": "[parameters('location')]", - "comments": "User Managed Identity for the cluster's ingress controller pods. Used for Azure Key Vault Access.", - "resources": [ - { - "type": "providers/roleAssignments", - "apiVersion": "2020-04-01-preview", - "name": "[concat('Microsoft.Authorization/', guid(resourceGroup().id, variables('managedIdentityOperatorRole')))]", - "comments": "Grant the AKS cluster control plane with Managed Identity Operator role permissions over the this ingress controller pod identity so that it can be assigned to the relevant nodepools.", - "dependsOn": [ - "[resourceId('Microsoft.ManagedIdentity/userAssignedIdentities', variables('clusterControlPlaneIdentityName'))]", - "[resourceId('Microsoft.ManagedIdentity/userAssignedIdentities', 'podmi-ingress-controller')]" - - ], - "properties": { - "roleDefinitionId": "[variables('managedIdentityOperatorRole')]", - "principalId": "[reference(resourceId('Microsoft.ManagedIdentity/userAssignedIdentities', variables('clusterControlPlaneIdentityName'))).principalId]", - "principalType": "ServicePrincipal" - } - } - ] - }, - { - "type": "Microsoft.KeyVault/vaults", - "apiVersion": "2019-09-01", - "name": "[variables('keyVaultName')]", - "location": "[parameters('location')]", - "dependsOn": [ - "[resourceId('Microsoft.ManagedIdentity/userAssignedIdentities', 'mi-appgateway-frontend')]", - "[resourceId('Microsoft.ManagedIdentity/userAssignedIdentities', 'podmi-ingress-controller')]" - ], - "properties": { - "accessPolicies": [ - { - "tenantId": "[reference(resourceId('Microsoft.ManagedIdentity/userAssignedIdentities', 'mi-appgateway-frontend')).tenantId]", - "objectId": "[reference(resourceId('Microsoft.ManagedIdentity/userAssignedIdentities', 'mi-appgateway-frontend')).principalId]", - "permissions": { - "secrets": [ - "get" - ], - "certificates": [ - "get" - ], - "keys": [] - } - }, - { - "tenantId": "[reference(resourceId('Microsoft.ManagedIdentity/userAssignedIdentities', 'podmi-ingress-controller')).tenantId]", - "objectId": "[reference(resourceId('Microsoft.ManagedIdentity/userAssignedIdentities', 'podmi-ingress-controller')).principalId]", - "permissions": { - "secrets": [ - "get" - ], - "certificates": [ - "get" - ], - "keys": [] - } - } - ], - "sku": { - "family": "A", - "name": "standard" - }, - "tenantId": "[subscription().tenantId]", - "networkAcls": { - "bypass": "AzureServices", - "defaultAction": "Allow", - "ipRules": [], - "virtualNetworkRules": [] - }, - "enabledForDeployment": false, - "enabledForDiskEncryption": false, - "enabledForTemplateDeployment": false, - "enableSoftDelete": true - }, - "resources": [ - { - "type": "secrets", - "apiVersion": "2019-09-01", - "name": "sslcert", - "dependsOn": [ - "[resourceId('Microsoft.KeyVault/vaults', variables('keyVaultName') )]" - ], - "properties": { - "value": "[parameters('appGatewayListenerCertificate')]", - "recoveryLevel": "Purgeable" - } - }, - { - "type": "secrets", - "apiVersion": "2019-09-01", - "name": "appgw-ingress-internal-aks-ingress-contoso-com-tls", - "dependsOn": [ - "[resourceId('Microsoft.KeyVault/vaults', variables('keyVaultName'))]" - ], - "properties": { - "value": "[parameters('aksIngressControllerCertificate')]", - "recoveryLevel": "Purgeable" - } - }, - { - "type": "providers/diagnosticSettings", - "apiVersion": "2021-05-01-preview", - "name": "Microsoft.Insights/default", - "dependsOn": [ - "[resourceId('Microsoft.KeyVault/vaults', variables('keyVaultName'))]", - "[resourceId('Microsoft.OperationalInsights/workspaces', variables('logAnalyticsWorkspaceName'))]" - ], - "properties": { - "workspaceId": "[resourceId('Microsoft.OperationalInsights/workspaces', variables('logAnalyticsWorkspaceName'))]", - "logs": [ - { - "category": "AuditEvent", - "enabled": true - } - ], - "metrics": [ - { - "category": "AllMetrics", - "enabled": true - } - ] - } - } - ] - }, - { - "type": "Microsoft.Network/privateEndpoints", - "apiVersion": "2020-11-01", - "name": "[concat('pe-', variables('keyVaultName'))]", - "location": "[parameters('location')]", - "dependsOn": [ - "[resourceId('Microsoft.KeyVault/vaults', variables('keyVaultName'))]" - ], - "properties": { - "subnet": { - "id": "[variables('vnetPrivateLinkSubnetResourceId')]" - }, - "privateLinkServiceConnections": [ - { - "name": "[concat('to-', variables('vnetName'))]", - "properties": { - "privateLinkServiceId": "[resourceId('Microsoft.KeyVault/vaults', variables('keyVaultName'))]", - "groupIds": [ - "vault" - ] - } - } - ] - }, - "resources": [ - { - "type": "privateDnsZoneGroups", - "apiVersion": "2020-11-01", - "name": "[concat('for-', variables('keyVaultName'))]", - "location": "[parameters('location')]", - "dependsOn": [ - "[resourceId('Microsoft.Network/privateEndpoints', concat('pe-', variables('keyVaultName')))]" - ], - "properties": { - "privateDnsZoneConfigs": [ - { - "name": "privatelink-akv-net", - "properties": { - "privateDnsZoneId": "[resourceId(variables('vNetResourceGroup'), 'Microsoft.Network/privateDnsZones', 'privatelink.vaultcore.azure.net')]" - } - } - ] - } - } - ] - }, - { - "type": "Microsoft.Network/privateEndpoints", - "apiVersion": "2020-11-01", - "name": "[concat('pe-', variables('defaultAcrName'))]", - "location": "[parameters('location')]", - "dependsOn": [ - "[resourceId('Microsoft.ContainerRegistry/registries/replications', variables('defaultAcrName'), parameters('geoRedundancyLocation'))]" - ], - "properties": { - "subnet": { - "id": "[variables('vnetPrivateLinkSubnetResourceId')]" - }, - "privateLinkServiceConnections": [ - { - "name": "[concat('to-', variables('vnetName'))]", - "properties": { - "privateLinkServiceId": "[resourceId('Microsoft.ContainerRegistry/registries', variables('defaultAcrName'))]", - "groupIds": [ - "registry" - ] - } - } - ] - }, - "resources": [ - { - "type": "privateDnsZoneGroups", - "apiVersion": "2020-11-01", - "name": "[concat('for-', variables('defaultAcrName'))]", - "location": "[parameters('location')]", - "dependsOn": [ - "[resourceId('Microsoft.Network/privateEndpoints', concat('pe-', variables('defaultAcrName')))]" - ], - "properties": { - "privateDnsZoneConfigs": [ - { - "name": "privatelink-azurecr-io", - "properties": { - "privateDnsZoneId": "[resourceId(variables('vNetResourceGroup'), 'Microsoft.Network/privateDnsZones', 'privatelink.azurecr.io')]" - } - } - ] - } - } - ] - }, - { - "type": "Microsoft.Network/applicationGateways", - "apiVersion": "2020-11-01", - "name": "[variables('agwName')]", - "location": "[parameters('location')]", - "dependsOn": [ - "[resourceId('Microsoft.KeyVault/vaults', variables('keyVaultName') )]" - ], - "identity": { - "type": "UserAssigned", - "userAssignedIdentities": { - "[resourceId('Microsoft.ManagedIdentity/userAssignedIdentities', 'mi-appgateway-frontend')]": {} - } - }, - "zones": [ - "1", - "2", - "3" - ], - "properties": { - "sku": { - "name": "WAF_v2", - "tier": "WAF_v2" - }, - "sslPolicy": { - "policyType": "Custom", - "cipherSuites": [ - "TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384", - "TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256" - ], - "minProtocolVersion": "TLSv1_2" - }, - "trustedRootCertificates": [ - { - "name": "root-cert-wildcard-aks-ingress-contoso", - "properties": { - "keyVaultSecretId": "[concat(reference(variables('keyVaultName')).vaultUri,'secrets/appgw-ingress-internal-aks-ingress-contoso-com-tls')]" - } - } - ], - "gatewayIPConfigurations": [ - { - "name": "apw-ip-configuration", - "properties": { - "subnet": { - "id": "[concat(parameters('targetVnetResourceId'), '/subnets/snet-applicationgateway')]" - } - } - } - ], - "frontendIPConfigurations": [ - { - "name": "apw-frontend-ip-configuration", - "properties": { - "publicIPAddress": { - "id": "[resourceId(subscription().subscriptionId, variables('vNetResourceGroup'), 'Microsoft.Network/publicIpAddresses', 'pip-BU0001A0005-00')]" - } - } - } - ], - "frontendPorts": [ - { - "name": "apw-frontend-ports", - "properties": { - "port": 443 - } - } - ], - "autoscaleConfiguration": { - "minCapacity": 0, - "maxCapacity": 10 - }, - "webApplicationFirewallConfiguration": { - "enabled": true, - "firewallMode": "Prevention", - "ruleSetType": "OWASP", - "ruleSetVersion": "3.2", - "disabledRuleGroups": [], - "requestBodyCheck": true, - "maxRequestBodySizeInKb": 128, - "fileUploadLimitInMb": 100 - }, - "enableHttp2": false, - "sslCertificates": [ - { - "name": "[concat(variables('agwName'), '-ssl-certificate')]", - "properties": { - "keyVaultSecretId": "[concat(reference(variables('keyVaultName')).vaultUri,'secrets/sslcert')]" - } - } - ], - "probes": [ - { - "name": "probe-bu0001a0005-00.aks-ingress.contoso.com", - "properties": { - "protocol": "Https", - "path": "/favicon.ico", - "interval": 30, - "timeout": 30, - "unhealthyThreshold": 3, - "pickHostNameFromBackendHttpSettings": true, - "minServers": 0, - "match": {} - } - }, - { - "name": "ingress-controller", - "properties": { - "protocol": "Https", - "path": "/healthz", - "interval": 30, - "timeout": 30, - "unhealthyThreshold": 3, - "pickHostNameFromBackendHttpSettings": true, - "minServers": 0, - "match": {} - } - } - ], - "backendAddressPools": [ - { - "name": "bu0001a0005-00.aks-ingress.contoso.com", - "properties": { - "backendAddresses": [ - { - /* This is the IP address that our ingress controller will request */ - "ipAddress": "10.240.4.4" - } - ] - } - } - ], - "backendHttpSettingsCollection": [ - { - "name": "aks-ingress-contoso-backendpool-httpsettings", - "properties": { - "port": 443, - "protocol": "Https", - "cookieBasedAffinity": "Disabled", - "hostName": "bu0001a0005-00.aks-ingress.contoso.com", - "pickHostNameFromBackendAddress": false, - "requestTimeout": 20, - "probe": { - "id": "[concat(resourceId('Microsoft.Network/applicationGateways', variables('agwName')),'/probes/probe-bu0001a0005-00.aks-ingress.contoso.com')]" - }, - "trustedRootCertificates": [ - { - "id": "[concat(resourceId('Microsoft.Network/applicationGateways', variables('agwName')), '/trustedRootCertificates/root-cert-wildcard-aks-ingress-contoso')]" - } - ] - } - } - ], - "httpListeners": [ - { - "name": "listener-https", - "properties": { - "frontendIPConfiguration": { - "id": "[concat(variables('apwResourceId'), '/frontendIPConfigurations/apw-frontend-ip-configuration')]" - }, - "frontendPort": { - "id": "[concat(variables('apwResourceId'), '/frontendPorts/apw-frontend-ports')]" - }, - "protocol": "Https", - "sslCertificate": { - "id": "[concat(variables('apwResourceId'), '/sslCertificates/', variables('agwName'), '-ssl-certificate')]" - }, - "hostName": "bicycle.contoso.com", - "hostNames": [], - "requireServerNameIndication": true - } - } - ], - "requestRoutingRules": [ - { - "name": "apw-routing-rules", - "properties": { - "ruleType": "Basic", - "httpListener": { - "id": "[concat(variables('apwResourceId'), '/httpListeners/listener-https')]" - }, - "backendAddressPool": { - "id": "[concat(variables('apwResourceId'), '/backendAddressPools/bu0001a0005-00.aks-ingress.contoso.com')]" - }, - "backendHttpSettings": { - "id": "[concat(variables('apwResourceId'), '/backendHttpSettingsCollection/aks-ingress-contoso-backendpool-httpsettings')]" - } - } - } - ] - }, - "resources": [ - { - "type": "/providers/diagnosticSettings", - "apiVersion": "2017-05-01-preview", - "name": "Microsoft.Insights/default", - "dependsOn": [ - "[resourceId('Microsoft.Network/applicationGateways', variables('agwName'))]", - "[resourceId('Microsoft.OperationalInsights/workspaces', variables('logAnalyticsWorkspaceName'))]" - ], - "properties": { - "workspaceId": "[resourceId('Microsoft.OperationalInsights/workspaces', variables('logAnalyticsWorkspaceName'))]", - "logs": [ - { - "category": "ApplicationGatewayAccessLog", - "enabled": true - }, - { - "category": "ApplicationGatewayPerformanceLog", - "enabled": true - }, - { - "category": "ApplicationGatewayFirewallLog", - "enabled": true - } - ] - } - } - ] - }, - { - "type": "Microsoft.Resources/deployments", - "apiVersion": "2020-06-01", - "name": "EnsureClusterIdentityHasRbacToSelfManagedResources", - "dependsOn": [ - "[resourceId('Microsoft.ManagedIdentity/userAssignedIdentities', variables('clusterControlPlaneIdentityName'))]" - ], - "resourceGroup": "[variables('vNetResourceGroup')]", - "properties": { - "mode": "Incremental", - "template": { - "$schema": "https://schema.management.azure.com/schemas/2019-04-01/deploymentTemplate.json#", - "contentVersion": "1.0.0.0", - "resources": [ - { - "type": "Microsoft.Authorization/roleAssignments", - "apiVersion": "2020-04-01-preview", - "name": "[guid(parameters('targetVnetResourceId'), variables('dnsZoneContributorRole'), variables('clusterControlPlaneIdentityName'))]", - "scope": "[concat('Microsoft.Network/virtualNetworks/', variables('vnetName'))]", - "properties": { - "roleDefinitionId": "[variables('dnsZoneContributorRole')]", - "description": "Allows cluster identity to attach custom DNS zone with Private Link information to this virtual network.", - "principalId": "[reference(resourceId('Microsoft.ManagedIdentity/userAssignedIdentities', variables('clusterControlPlaneIdentityName'))).principalId]", - "principalType": "ServicePrincipal" - } - }, - { - "type": "Microsoft.Authorization/roleAssignments", - "apiVersion": "2020-04-01-preview", - "name": "[guid(variables('vnetSystemNodePoolSubnetResourceId'), variables('networkContributorRole'), variables('clusterControlPlaneIdentityName'))]", - "scope": "[concat('Microsoft.Network/virtualNetworks/', variables('vnetName'), '/subnets/', 'snet-cluster-systemnodepool')]", - "properties": { - "roleDefinitionId": "[variables('networkContributorRole')]", - "description": "Allows cluster identity to join the nodepool vmss resources to this subnet.", - "principalId": "[reference(resourceId('Microsoft.ManagedIdentity/userAssignedIdentities', variables('clusterControlPlaneIdentityName'))).principalId]", - "principalType": "ServicePrincipal" - } - }, - { - "type": "Microsoft.Authorization/roleAssignments", - "apiVersion": "2020-04-01-preview", - "name": "[guid(variables('vnetInScopeNodePoolSubnetResourceId'), variables('networkContributorRole'), variables('clusterControlPlaneIdentityName'))]", - "scope": "[concat('Microsoft.Network/virtualNetworks/', variables('vnetName'), '/subnets/', 'snet-cluster-inscopenodepools')]", - "properties": { - "roleDefinitionId": "[variables('networkContributorRole')]", - "description": "Allows cluster identity to join the nodepool vmss resources to this subnet.", - "principalId": "[reference(resourceId('Microsoft.ManagedIdentity/userAssignedIdentities', variables('clusterControlPlaneIdentityName'))).principalId]", - "principalType": "ServicePrincipal" - } - }, - { - "type": "Microsoft.Authorization/roleAssignments", - "apiVersion": "2020-04-01-preview", - "name": "[guid(variables('vnetOutOfScopeNodePoolSubnetResourceId'), variables('networkContributorRole'), variables('clusterControlPlaneIdentityName'))]", - "scope": "[concat('Microsoft.Network/virtualNetworks/', variables('vnetName'), '/subnets/', 'snet-cluster-outofscopenodepools')]", - "properties": { - "roleDefinitionId": "[variables('networkContributorRole')]", - "description": "Allows cluster identity to join the nodepool vmss resources to this subnet.", - "principalId": "[reference(resourceId('Microsoft.ManagedIdentity/userAssignedIdentities', variables('clusterControlPlaneIdentityName'))).principalId]", - "principalType": "ServicePrincipal" - } - }, - { - "type": "Microsoft.Authorization/roleAssignments", - "apiVersion": "2020-04-01-preview", - "name": "[guid(variables('vnetIngressServicesSubnetResourceId'), variables('networkContributorRole'), variables('clusterControlPlaneIdentityName'))]", - "scope": "[concat('Microsoft.Network/virtualNetworks/', variables('vnetName'), '/subnets/', 'snet-cluster-ingressservices')]", - "properties": { - "roleDefinitionId": "[variables('networkContributorRole')]", - "description": "Allows cluster identity to join load balancers (ingress resources) to this subnet.", - "principalId": "[reference(resourceId('Microsoft.ManagedIdentity/userAssignedIdentities', variables('clusterControlPlaneIdentityName'))).principalId]", - "principalType": "ServicePrincipal" - } - }, - { - "type": "Microsoft.Authorization/roleAssignments", - "apiVersion": "2020-04-01-preview", - "name": "[guid(resourceId('Microsoft.Network/privateDnsZones', concat('privatelink.', parameters('location'), '.azmk8s.io')), variables('dnsZoneContributorRole'), variables('clusterControlPlaneIdentityName'))]", - "scope": "[concat('Microsoft.Network/privateDnsZones/', concat('privatelink.', parameters('location'), '.azmk8s.io'))]", - "properties": { - "roleDefinitionId": "[variables('dnsZoneContributorRole')]", - "description": "Allows cluster identity to manage zone Entries for cluster's Private Link configuration.", - "principalId": "[reference(resourceId('Microsoft.ManagedIdentity/userAssignedIdentities', variables('clusterControlPlaneIdentityName'))).principalId]", - "principalType": "ServicePrincipal" - } - } - ] - } - } - }, - { - "type": "Microsoft.OperationalInsights/workspaces", - "apiVersion": "2020-08-01", - "name": "[variables('logAnalyticsWorkspaceName')]", - "location": "[parameters('location')]", - "properties": { - "sku": { - "name": "PerGB2018" - }, - "retentionInDays": 90, - "publicNetworkAccessForIngestion": "Enabled", - "publicNetworkAccessForQuery": "Enabled" - }, - "resources": [ - { - "type": "savedSearches", - "apiVersion": "2020-08-01", - "name": "AllPrometheus", - "dependsOn": [ - "[concat('Microsoft.OperationalInsights/workspaces/', variables('logAnalyticsWorkspaceName'))]" - ], - "properties": { - "eTag": "*", - "category": "Prometheus", - "displayName": "All collected Prometheus information", - "query": "InsightsMetrics | where Namespace == \"prometheus\"", - "version": 1 - } - }, - { - "type": "savedSearches", - "apiVersion": "2020-08-01", - "name": "ForbiddenReponsesOnIngress", - "dependsOn": [ - "[concat('Microsoft.OperationalInsights/workspaces/', variables('logAnalyticsWorkspaceName'))]" - ], - "properties": { - "eTag": "*", - "category": "Prometheus", - "displayName": "Increase number of forbidden response on the Ingress Controller", - "query": "let value = toscalar(InsightsMetrics | where Namespace == \"prometheus\" and Name == \"nginx_ingress_controller_requests\" | where parse_json(Tags).status == 403 | summarize Value = avg(Val) by bin(TimeGenerated, 5m) | summarize min = min(Value)); InsightsMetrics | where Namespace == \"prometheus\" and Name == \"nginx_ingress_controller_requests\" | where parse_json(Tags).status == 403 | summarize AggregatedValue = avg(Val)-value by bin(TimeGenerated, 5m) | order by TimeGenerated | render barchart", - "version": 1 - } - }, - { - "type": "savedSearches", - "apiVersion": "2020-08-01", - "name": "NodeRebootRequested", - "dependsOn": [ - "[concat('Microsoft.OperationalInsights/workspaces/', variables('logAnalyticsWorkspaceName'))]" - ], - "properties": { - "eTag": "*", - "category": "Prometheus", - "displayName": "Nodes reboot required by kured", - "query": "InsightsMetrics | where Namespace == \"prometheus\" and Name == \"kured_reboot_required\" | where Val > 0", - "version": 1 - } - } - ] - }, - { - "type": "Microsoft.Insights/scheduledQueryRules", - "apiVersion": "2021-08-01", - "name": "Image Imported into ACR from source other than approved Quarantine", - "location": "[parameters('location')]", - "dependsOn": [ - "[resourceId('Microsoft.OperationalInsights/workspaces', variables('logAnalyticsWorkspaceName'))]", - "[resourceId('Microsoft.ContainerRegistry/registries', variables('defaultAcrName'))]" - ], - "properties": { - "description":"The only images we want in live/ are those that came from this ACR instance, but from the quarantine/ repository.", - "actions": { - "actionGroups": [] - }, - "criteria": { - "allOf": [ - { - "operator": "GreaterThan", - "query": "ContainerRegistryRepositoryEvents\r\n| where OperationName == \"importImage\" and Repository startswith \"live/\" and MediaType !startswith strcat(_ResourceId, \"/quarantine\")", - "threshold": 0, - "timeAggregation": "Count", - "dimensions": [], - "failingPeriods": { - "minFailingPeriodsToAlert": 1, - "numberOfEvaluationPeriods": 1 - }, - "resourceIdColumn": "" - } - ] - }, - "enabled": true, - "evaluationFrequency": "PT10M", - "scopes": [ - "[resourceId('Microsoft.ContainerRegistry/registries', variables('defaultAcrName'))]" - ], - "severity": 3, - "windowSize": "PT10M", - "muteActionsDuration": null, - "overrideQueryTimeRange": null - } - }, - { - "type": "Microsoft.Insights/scheduledQueryRules", - "apiVersion": "2021-08-01", - "name": "PodFailedScheduledQuery", - "location": "[parameters('location')]", - "dependsOn": [ - "[resourceId('Microsoft.OperationalInsights/workspaces', variables('logAnalyticsWorkspaceName'))]", - "[resourceId('Microsoft.ContainerService/managedClusters', variables('clusterName'))]" - ], - "properties": { - "description":"Example from: https://learn.microsoft.com/azure/azure-monitor/insights/container-insights-alerts", - "actions": { - "actionGroups": [] - }, - "criteria": { - "allOf": [ - { - "metricMeasureColumn": "FailedCount", - "operator": "GreaterThan", - "query": "let trendBinSize = 1m;\r\nKubePodInventory\r\n| distinct ClusterName, TimeGenerated\r\n| summarize ClusterSnapshotCount = count() by bin(TimeGenerated, trendBinSize), ClusterName\r\n| join hint.strategy=broadcast (\r\n KubePodInventory\r\n | distinct ClusterName, Computer, PodUid, TimeGenerated, PodStatus\r\n | summarize TotalCount = count(),\r\n PendingCount = sumif(1, PodStatus =~ \"Pending\"),\r\n RunningCount = sumif(1, PodStatus =~ \"Running\"),\r\n SucceededCount = sumif(1, PodStatus =~ \"Succeeded\"),\r\n FailedCount = sumif(1, PodStatus =~ \"Failed\")\r\n by ClusterName, bin(TimeGenerated, trendBinSize)\r\n )\r\n on ClusterName, TimeGenerated \r\n| extend UnknownCount = TotalCount - PendingCount - RunningCount - SucceededCount - FailedCount\r\n| project TimeGenerated,\r\n ClusterName,\r\n TotalCount = todouble(TotalCount) / ClusterSnapshotCount,\r\n PendingCount = todouble(PendingCount) / ClusterSnapshotCount,\r\n RunningCount = todouble(RunningCount) / ClusterSnapshotCount,\r\n SucceededCount = todouble(SucceededCount) / ClusterSnapshotCount,\r\n FailedCount = todouble(FailedCount) / ClusterSnapshotCount,\r\n UnknownCount = todouble(UnknownCount) / ClusterSnapshotCount\r\n", - "threshold": 3, - "timeAggregation": "Average", - "dimensions": [], - "failingPeriods": { - "minFailingPeriodsToAlert": 1, - "numberOfEvaluationPeriods": 1 - }, - "resourceIdColumn": "" - } - ] - }, - "enabled": true, - "evaluationFrequency": "PT5M", - "scopes": [ - "[resourceId('Microsoft.ContainerService/managedClusters', variables('clusterName'))]" - ], - "severity": 3, - "windowSize": "PT5M", - "muteActionsDuration": null, - "overrideQueryTimeRange": "P2D" - } - }, - { - "type": "Microsoft.Insights/activityLogAlerts", - "apiVersion": "2017-04-01", - "name": "AllAzureAdvisorAlert", - "location": "Global", - "properties": { - "scopes": [ - "[resourceGroup().id]" - ], - "condition": { - "allOf": [ - { - "field": "category", - "equals": "Recommendation" - }, - { - "field": "operationName", - "equals": "Microsoft.Advisor/recommendations/available/action" - } - ] - }, - "actions": { - "actionGroups": [ - ] - }, - "enabled": true, - "description": "All azure advisor alerts" - } - }, - { - "type": "Microsoft.OperationsManagement/solutions", - "apiVersion": "2015-11-01-preview", - "name": "[variables('containerInsightsSolutionName')]", - "location": "[parameters('location')]", - "dependsOn": [ - "[resourceId('Microsoft.OperationalInsights/workspaces', variables('logAnalyticsWorkspaceName'))]" - ], - "properties": { - "workspaceResourceId": "[resourceId('Microsoft.OperationalInsights/workspaces', variables('logAnalyticsWorkspaceName'))]" - }, - "plan": { - "name": "[variables('containerInsightsSolutionName')]", - "product": "OMSGallery/ContainerInsights", - "promotionCode": "", - "publisher": "Microsoft" - } - }, - { - "type": "Microsoft.OperationsManagement/solutions", - "apiVersion": "2015-11-01-preview", - "name": "[variables('vmInsightsSolutionName')]", - "location": "[parameters('location')]", - "dependsOn": [ - "[resourceId('Microsoft.OperationalInsights/workspaces', variables('logAnalyticsWorkspaceName'))]" - ], - "properties": { - "workspaceResourceId": "[resourceId('Microsoft.OperationalInsights/workspaces', variables('logAnalyticsWorkspaceName'))]" - }, - "plan": { - "name": "[variables('vmInsightsSolutionName')]", - "product": "OMSGallery/VMInsights", - "promotionCode": "", - "publisher": "Microsoft" - } - }, - { - "type": "Microsoft.OperationsManagement/solutions", - "apiVersion": "2015-11-01-preview", - "name": "[variables('securityCenterSolutionName')]", - "location": "[parameters('location')]", - "comments": "Enables Azure Sentinal on this workspace.", - "dependsOn": [ - "[resourceId('Microsoft.OperationalInsights/workspaces', variables('logAnalyticsWorkspaceName'))]" - ], - "properties": { - "workspaceResourceId": "[resourceId('Microsoft.OperationalInsights/workspaces', variables('logAnalyticsWorkspaceName'))]" - }, - "plan": { - "name": "[variables('securityCenterSolutionName')]", - "product": "OMSGallery/SecurityInsights", - "promotionCode": "", - "publisher": "Microsoft" - } - }, - { - "type": "Microsoft.Insights/workbooks", - "apiVersion": "2020-10-20", - "name": "[guid(variables('securityCenterSolutionName'))]", - "location": "[parameters('location')]", - "comments": "Add the AKS Microsoft Defender for Cloud AKS workbook - https://securityinsights.hosting.portal.azure.net/securityinsights/Content/1.0.01480.0001-210119-121529/Workbooks/AksSecurity.json", - "tags": { - "hidden-title": "[concat('Azure Kubernetes Service (AKS) Security - ', variables('logAnalyticsWorkspaceName'))]" - }, - "dependsOn": [ - "[resourceId('Microsoft.OperationsManagement/solutions', variables('securityCenterSolutionName'))]" - ], - "kind": "shared", - "properties": { - "displayName": "[concat('Azure Kubernetes Service (AKS) Security - ', variables('logAnalyticsWorkspaceName'))]", - "serializedData": "[variables('aksSecurityCenterWorkbookData')]", - "version": "1.0", - "category": "sentinel", - "sourceId": "[resourceId('Microsoft.OperationalInsights/workspaces', variables('logAnalyticsWorkspaceName'))]", - "tags": [ - "AksSecurityWorkbook", - "1.2" - ] - } - }, - { - "type": "Microsoft.OperationsManagement/solutions", - "apiVersion": "2015-11-01-preview", - "name": "[concat('KeyVaultAnalytics(', variables('logAnalyticsWorkspaceName'),')')]", - "location": "[parameters('location')]", - "dependsOn": [ - "[resourceId('Microsoft.OperationalInsights/workspaces', variables('logAnalyticsWorkspaceName'))]" - ], - "properties": { - "workspaceResourceId": "[resourceId('Microsoft.OperationalInsights/workspaces', variables('logAnalyticsWorkspaceName'))]" - }, - "plan": { - "name": "[concat('KeyVaultAnalytics(', variables('logAnalyticsWorkspaceName'),')')]", - "product": "OMSGallery/KeyVaultAnalytics", - "promotionCode": "", - "publisher": "Microsoft" - } - }, - { - "type": "Microsoft.ContainerRegistry/registries", - "apiVersion": "2020-11-01-preview", - "name": "[variables('defaultAcrName')]", - "location": "[parameters('location')]", - "sku": { - "name": "Premium" - }, - "properties": { - "adminUserEnabled": false, - "networkRuleSet": { - "defaultAction": "Deny", - "virtualNetworkRules": [], - "ipRules": [] - }, - "policies": { - "quarantinePolicy": { - "status": "disabled" - }, - "trustPolicy": { - "type": "Notary", - "status": "enabled" - }, - "retentionPolicy": { - "days": 15, - "status": "enabled" - } - }, - "publicNetworkAccess": "Disabled", - "encryption": { - "status": "disabled" - }, - "dataEndpointEnabled": true, - "networkRuleBypassOptions": "AzureServices", - "zoneRedundancy": "Disabled" // This Preview feature only supports three regions at this time, and eastus2's paired region (centralus), does not support this. So disabling for now. - }, - "resources": [ - { - "type": "providers/roleAssignments", - "apiVersion": "2020-04-01-preview", - "name": "[concat('Microsoft.Authorization/', guid(resourceId('Microsoft.ContainerService/managedClusters', variables('clusterName')), variables('acrPullRole')))]", - "dependsOn": [ - "[resourceId('Microsoft.ContainerRegistry/registries', variables('defaultAcrName'))]", - "[resourceId('Microsoft.ContainerService/managedClusters', variables('clusterName'))]" - ], - "properties": { - "roleDefinitionId": "[variables('acrPullRole')]", - "description": "Allows the AKS Cluster's kubelet managed identity to pull images from this container registry.", - "principalId": "[reference(resourceId('Microsoft.ContainerService/managedClusters', variables('clusterName')), '2020-12-01').identityProfile.kubeletidentity.objectId]", - "principalType": "ServicePrincipal" - } - }, - { - "type": "replications", - "apiVersion": "2019-05-01", - "name": "[parameters('geoRedundancyLocation')]", - "location": "[parameters('geoRedundancyLocation')]", - "dependsOn": [ - "[resourceId('Microsoft.ContainerRegistry/registries', variables('defaultAcrName'))]" - ], - "properties": {} - }, - { - "type": "agentPools", - "apiVersion": "2019-06-01-preview", - "name": "acragent", - "location": "[parameters('location')]", - "comments": "ACR Build Agent pool", - "dependsOn": [ - "[resourceId('Microsoft.ContainerRegistry/registries', variables('defaultAcrName'))]" - ], - "properties": { - "count": 1, - "os": "Linux", - "tier": "S1", - "virtualNetworkSubnetResourceId": "[variables('vnetAcrBuildSubnetResourceId')]" - } - }, - { - "type": "providers/diagnosticSettings", - "apiVersion": "2021-05-01-preview", - "name": "Microsoft.Insights/default", - "dependsOn": [ - "[resourceId('Microsoft.ContainerRegistry/registries', variables('defaultAcrName'))]", - "[resourceId('Microsoft.OperationalInsights/workspaces', variables('logAnalyticsWorkspaceName'))]" - ], - "properties": { - "workspaceId": "[resourceId('Microsoft.OperationalInsights/workspaces', variables('logAnalyticsWorkspaceName'))]", - "metrics": [ - { - "timeGrain": "PT1M", - "category": "AllMetrics", - "enabled": true - } - ], - "logs": [ - { - "category": "ContainerRegistryRepositoryEvents", - "enabled": true - }, - { - "category": "ContainerRegistryLoginEvents", - "enabled": true - } - ] - } - } - ] - }, - { - "type": "Microsoft.ContainerService/managedClusters", - "apiVersion": "2021-05-01", - "name": "[variables('clusterName')]", - "location": "[parameters('location')]", - "tags": { - "Data classification": "Confidential", - "Business unit": "BU0001", - "Business criticality": "Business unit-critical" - }, - "dependsOn": [ - "[resourceId('Microsoft.OperationsManagement/solutions', variables('containerInsightsSolutionName'))]", - "[resourceId(variables('vNetResourceGroup'), 'Microsoft.Resources/deployments', 'EnsureClusterIdentityHasRbacToSelfManagedResources')]", - "[resourceId('Microsoft.ManagedIdentity/userAssignedIdentities', variables('clusterControlPlaneIdentityName'))]", - // You want policies created before cluster because they take some time to be made available and we want them - // to apply to your cluster as soon as possible. Nothing in this cluster "technically" depends on these existing, - // just trying to get coverage as soon as possible. - "[resourceId('Microsoft.Authorization/policyAssignments', variables('policyAssignmentNameAKSLinuxRestrictive'))]", - "[resourceId('Microsoft.Authorization/policyAssignments', variables('policyAssignmentNameEnforceHttpsIngress'))]", - "[resourceId('Microsoft.Authorization/policyAssignments', variables('policyAssignmentNameEnforceInternalLoadBalancers'))]", - "[resourceId('Microsoft.Authorization/policyAssignments', variables('policyAssignmentNameMustNotAutomountApiCreds'))]", - "[resourceId('Microsoft.Authorization/policyAssignments', variables('policyAssignmentNameMustUseSpecifiedLabels'))]", - "[resourceId('Microsoft.Authorization/policyAssignments', variables('policyAssignmentNameMustUseTheseExternalIps'))]", - "[resourceId('Microsoft.Authorization/policyAssignments', variables('policyAssignmentNameApprovedContainerPortsOnly'))]", - "[resourceId('Microsoft.Authorization/policyAssignments', variables('policyAssignmentNameApprovedServicePortsOnly'))]", - "[resourceId('Microsoft.Authorization/policyAssignments', variables('policyAssignmentNameRoRootFilesystem'))]", - "[resourceId('Microsoft.Authorization/policyAssignments', variables('policyAssignmentNameBlockDefaultNamespace'))]", - "[resourceId('Microsoft.Authorization/policyAssignments', variables('policyAssignmentNameEnforceResourceLimits'))]", - "[resourceId('Microsoft.Authorization/policyAssignments', variables('policyAssignmentNameEnforceImageSource'))]", - // Ensure jumboxes are available to use as soon as possible, don't wait until cluster is created. - "[resourceId('Microsoft.Compute/virtualMachineScaleSets', 'vmss-jumpboxes')]", - "[resourceId('Microsoft.ManagedIdentity/userAssignedIdentities', 'podmi-ingress-controller')]" - ], - "properties": { - "kubernetesVersion": "[variables('kubernetesVersion')]", - "dnsPrefix": "[uniqueString(subscription().subscriptionId, resourceGroup().id, variables('clusterName'))]", - "agentPoolProfiles": [ - { - "name": "npsystem", - "count": 3, - "vmSize": "Standard_DS2_v2", - "osDiskSizeGB": 80, - "osDiskType": "Ephemeral", - "osType": "Linux", - "minCount": 3, - "maxCount": 4, - "vnetSubnetID": "[variables('vnetSystemNodePoolSubnetResourceId')]", - "enableAutoScaling": true, - "type": "VirtualMachineScaleSets", - "mode": "System", - "scaleSetPriority": "Regular", - "scaleSetEvictionPolicy": "Delete", - "orchestratorVersion": "[variables('kubernetesVersion')]", - "enableNodePublicIP": false, - "maxPods": 30, - "availabilityZones": [ - "1", - "2", - "3" - ], - "upgradeSettings": { - "maxSurge": "33%" - }, - "tags": { - "pci-scope": "out-of-scope", - "Data classification": "Confidential", - "Business unit": "BU0001", - "Business criticality": "Business unit-critical" - }/* This can be used to prevent unexpected workloads from landing on system node pool. All add-ons support this taint., - "nodeTaints": [ - "CriticalAddonsOnly=true:NoSchedule" - ]*/ - }, - { - "name": "npinscope01", - "count": 2, - "vmSize": "Standard_DS3_v2", - "osDiskSizeGB": 120, - "osDiskType": "Ephemeral", - "osType": "Linux", - "minCount": 2, - "maxCount": 5, - "vnetSubnetID": "[variables('vnetInScopeNodePoolSubnetResourceId')]", - "enableAutoScaling": true, - "type": "VirtualMachineScaleSets", - "mode": "User", - "scaleSetPriority": "Regular", - "scaleSetEvictionPolicy": "Delete", - "orchestratorVersion": "[variables('kubernetesVersion')]", - "enableNodePublicIP": false, - "maxPods": 30, - "availabilityZones": [ - "1", - "2", - "3" - ], - "upgradeSettings": { - "maxSurge": "33%" - }, - "nodeLabels": { - "pci-scope": "in-scope" - }, - "tags": { - "pci-scope": "in-scope", - "Data classification": "Confidential", - "Business unit": "BU0001", - "Business criticality": "Business unit-critical" - } - }, - { - "name": "npooscope01", - "count": 2, - "vmSize": "Standard_DS3_v2", - "osDiskSizeGB": 120, - "osDiskType": "Ephemeral", - "osType": "Linux", - "minCount": 2, - "maxCount": 5, - "vnetSubnetID": "[variables('vnetOutOfScopeNodePoolSubnetResourceId')]", - "enableAutoScaling": true, - "type": "VirtualMachineScaleSets", - "mode": "User", - "scaleSetPriority": "Regular", - "scaleSetEvictionPolicy": "Delete", - "orchestratorVersion": "[variables('kubernetesVersion')]", - "enableNodePublicIP": false, - "maxPods": 30, - "availabilityZones": [ - "1", - "2", - "3" - ], - "upgradeSettings": { - "maxSurge": "33%" - }, - "nodeLabels": { - "pci-scope": "out-of-scope" - }, - "tags": { - "pci-scope": "out-of-scope", - "Data classification": "Confidential", - "Business unit": "BU0001", - "Business criticality": "Business unit-critical" - } - } - ], - "servicePrincipalProfile": { - "clientId": "msi" - }, - "addonProfiles": { - "httpApplicationRouting": { - "enabled": false - }, - "omsagent": { - "enabled": true, - "config": { - "logAnalyticsWorkspaceResourceId": "[resourceId('Microsoft.OperationalInsights/workspaces', variables('logAnalyticsWorkspaceName'))]" - } - }, - "aciConnectorLinux": { - "enabled": false - }, - "azurepolicy": { - "enabled": true, - "config": { - "version": "v2" - } - }, - "openServiceMesh": { - "enabled": true, - "config": {} - }, - "azureKeyvaultSecretsProvider": { - "enabled": true, - "config": { - "enableSecretRotation": "false" - } - } - }, - "nodeResourceGroup": "[variables('nodeResourceGroupName')]", - "enableRBAC": true, - "enablePodSecurityPolicy": false, - "maxAgentPools": 3, - "networkProfile": { - "networkPlugin": "azure", - "networkPolicy": "azure", - "outboundType": "userDefinedRouting", - "loadBalancerSku": "standard", - "loadBalancerProfile": "[json('null')]", - "serviceCidr": "172.16.0.0/16", - "dnsServiceIP": "172.16.0.10", - "dockerBridgeCidr": "172.18.0.1/16" - }, - "aadProfile": { - "managed": true, - "enableAzureRBAC": false, - "adminGroupObjectIDs": [ - "[parameters('clusterAdminAadGroupObjectId')]" - ], - "tenantID": "[parameters('k8sControlPlaneAuthorizationTenantId')]" - }, - "autoScalerProfile": { - "balance-similar-node-groups": "false", - "expander": "random", - "max-empty-bulk-delete": "10", - "max-graceful-termination-sec": "600", - "max-node-provision-time": "15m", - "max-total-unready-percentage": "45", - "new-pod-scale-up-delay": "0s", - "ok-total-unready-count": "3", - "scale-down-delay-after-add": "10m", - "scale-down-delay-after-delete": "20s", - "scale-down-delay-after-failure": "3m", - "scale-down-unneeded-time": "10m", - "scale-down-unready-time": "20m", - "scale-down-utilization-threshold": "0.5", - "scan-interval": "10s", - "skip-nodes-with-local-storage": "true", - "skip-nodes-with-system-pods": "true" - }, - /*"autoUpgradeProfile": { - "upgradeChannel": "none" - },*/ - "apiServerAccessProfile": { - "enablePrivateCluster": true, - "privateDNSZone": "[resourceId(variables('vNetResourceGroup'), 'Microsoft.Network/privateDnsZones', concat('privatelink.', parameters('location') ,'.azmk8s.io'))]", - "enablePrivateClusterPublicFQDN": false - }, - "podIdentityProfile": { - "enabled": true - }, - "disableLocalAccounts": true, - "securityProfile": { - "azureDefender": { - "enabled": true, - "logAnalyticsWorkspaceResourceId": "[resourceId('Microsoft.OperationalInsights/workspaces', variables('logAnalyticsWorkspaceName'))]" - } - } - }, - "identity": { - "type": "UserAssigned", - "userAssignedIdentities": { - "[resourceId('Microsoft.ManagedIdentity/userAssignedIdentities', variables('clusterControlPlaneIdentityName'))]": {} - } - }, - "sku": { - "name": "Basic", - "tier": "Paid" - }, - "resources": [ - { - "type": "providers/roleAssignments", - "apiVersion": "2020-04-01-preview", - "name": "[concat('Microsoft.Authorization/', guid(resourceId('Microsoft.ContainerService/managedClusters', variables('clusterName')), 'omsagent', variables('monitoringMetricsPublisherRole')))]", - "dependsOn": [ - "[resourceId('Microsoft.ContainerService/managedClusters', variables('clusterName'))]" - ], - "properties": { - "roleDefinitionId": "[variables('monitoringMetricsPublisherRole')]", - "description": "Grant the OMS Agent's Managed Identity the metrics publisher role to push alerts.", - "principalId": "[reference(resourceId('Microsoft.ContainerService/managedClusters', variables('clusterName')), '2020-12-01').addonProfiles.omsagent.identity.objectId]", - "principalType": "ServicePrincipal" - } - }, - { - "type": "/providers/diagnosticSettings", - "apiVersion": "2017-05-01-preview", - "name": "Microsoft.Insights/default", - "dependsOn": [ - "[resourceId('Microsoft.ContainerService/managedClusters', variables('clusterName'))]", - "[resourceId('Microsoft.OperationalInsights/workspaces', variables('logAnalyticsWorkspaceName'))]" - ], - "properties": { - "workspaceId": "[resourceId('Microsoft.OperationalInsights/workspaces', variables('logAnalyticsWorkspaceName'))]", - "logs": [ - { - "category": "cluster-autoscaler", - "enabled": true - }, - { - "category": "kube-controller-manager", - "enabled": true - }, - { - "category": "kube-audit-admin", - "enabled": true - }, - { - "category": "guard", - "enabled": true - } - ] - } - } - ] - }, - { - "type": "Microsoft.Insights/metricAlerts", - "apiVersion": "2018-03-01", - "name": "[concat('Node CPU utilization high for ', variables('clusterName'), ' CI-1')]", - "location": "global", - "dependsOn": [ - "[resourceId('Microsoft.ContainerService/managedClusters', variables('clusterName'))]", - "[resourceId('Microsoft.OperationsManagement/solutions',variables('containerInsightsSolutionName'))]" - ], - "properties": { - "actions": [], - "criteria": { - "allOf": [ - { - "criterionType": "StaticThresholdCriterion", - "dimensions": [ - { - "name": "host", - "operator": "Include", - "values": [ - "*" - ] - } - ], - "metricName": "cpuUsagePercentage", - "metricNamespace": "Insights.Container/nodes", - "name": "Metric1", - "operator": "GreaterThan", - "threshold": 80.0, - "timeAggregation": "Average", - "skipMetricValidation": true - } - ], - "odata.type": "Microsoft.Azure.Monitor.SingleResourceMultipleMetricCriteria" - }, - "description": "Node CPU utilization across the cluster.", - "enabled": true, - "evaluationFrequency": "PT1M", - "scopes": [ - "[resourceId('Microsoft.ContainerService/managedClusters', variables('clusterName'))]" - ], - "severity": 3, - "targetResourceType": "microsoft.containerservice/managedclusters", - "windowSize": "PT5M" - } - }, - { - "type": "Microsoft.Insights/metricAlerts", - "apiVersion": "2018-03-01", - "name": "[concat('Node working set memory utilization high for ', variables('clusterName'), ' CI-2')]", - "location": "global", - "dependsOn": [ - "[resourceId('Microsoft.ContainerService/managedClusters', variables('clusterName'))]", - "[resourceId('Microsoft.OperationsManagement/solutions',variables('containerInsightsSolutionName'))]" - ], - "properties": { - "actions": [], - "criteria": { - "allOf": [ - { - "criterionType": "StaticThresholdCriterion", - "dimensions": [ - { - "name": "host", - "operator": "Include", - "values": [ - "*" - ] - } - ], - "metricName": "memoryWorkingSetPercentage", - "metricNamespace": "Insights.Container/nodes", - "name": "Metric1", - "operator": "GreaterThan", - "threshold": 80.0, - "timeAggregation": "Average", - "skipMetricValidation": true - } - ], - "odata.type": "Microsoft.Azure.Monitor.SingleResourceMultipleMetricCriteria" - }, - "description": "Node working set memory utilization across the cluster.", - "enabled": true, - "evaluationFrequency": "PT1M", - "scopes": [ - "[resourceId('Microsoft.ContainerService/managedClusters', variables('clusterName'))]" - ], - "severity": 3, - "targetResourceType": "microsoft.containerservice/managedclusters", - "windowSize": "PT5M" - } - }, - { - "type": "Microsoft.Insights/metricAlerts", - "apiVersion": "2018-03-01", - "name": "[concat('Jobs completed more than 6 hours ago for ', variables('clusterName'), ' CI-11')]", - "location": "global", - "dependsOn": [ - "[resourceId('Microsoft.ContainerService/managedClusters', variables('clusterName'))]", - "[resourceId('Microsoft.OperationsManagement/solutions',variables('containerInsightsSolutionName'))]" - ], - "properties": { - "actions": [], - "criteria": { - "allOf": [ - { - "criterionType": "StaticThresholdCriterion", - "dimensions": [ - { - "name": "controllerName", - "operator": "Include", - "values": [ - "*" - ] - }, - { - "name": "kubernetes namespace", - "operator": "Include", - "values": [ - "*" - ] - } - ], - "metricName": "completedJobsCount", - "metricNamespace": "Insights.Container/pods", - "name": "Metric1", - "operator": "GreaterThan", - "threshold": 0.0, - "timeAggregation": "Average", - "skipMetricValidation": true - } - ], - "odata.type": "Microsoft.Azure.Monitor.SingleResourceMultipleMetricCriteria" - }, - "description": "This alert monitors completed jobs (more than 6 hours ago).", - "enabled": true, - "evaluationFrequency": "PT1M", - "scopes": [ - "[resourceId('Microsoft.ContainerService/managedClusters', variables('clusterName'))]" - ], - "severity": 3, - "targetResourceType": "microsoft.containerservice/managedclusters", - "windowSize": "PT1M" - } - }, - { - "type": "Microsoft.Insights/metricAlerts", - "apiVersion": "2018-03-01", - "name": "[concat('Container CPU usage high for ', variables('clusterName'), ' CI-9')]", - "location": "global", - "dependsOn": [ - "[resourceId('Microsoft.ContainerService/managedClusters', variables('clusterName'))]", - "[resourceId('Microsoft.OperationsManagement/solutions',variables('containerInsightsSolutionName'))]" - ], - "properties": { - "actions": [], - "criteria": { - "allOf": [ - { - "criterionType": "StaticThresholdCriterion", - "dimensions": [ - { - "name": "controllerName", - "operator": "Include", - "values": [ - "*" - ] - }, - { - "name": "kubernetes namespace", - "operator": "Include", - "values": [ - "*" - ] - } - ], - "metricName": "cpuExceededPercentage", - "metricNamespace": "Insights.Container/containers", - "name": "Metric1", - "operator": "GreaterThan", - "threshold": 90.0, - "timeAggregation": "Average", - "skipMetricValidation": true - } - ], - "odata.type": "Microsoft.Azure.Monitor.SingleResourceMultipleMetricCriteria" - }, - "description": "This alert monitors container CPU utilization.", - "enabled": true, - "evaluationFrequency": "PT1M", - "scopes": [ - "[resourceId('Microsoft.ContainerService/managedClusters', variables('clusterName'))]" - ], - "severity": 3, - "targetResourceType": "microsoft.containerservice/managedclusters", - "windowSize": "PT5M" - } - }, - { - "type": "Microsoft.Insights/metricAlerts", - "apiVersion": "2018-03-01", - "name": "[concat('Container working set memory usage high for ', variables('clusterName'), ' CI-10')]", - "location": "global", - "dependsOn": [ - "[resourceId('Microsoft.ContainerService/managedClusters', variables('clusterName'))]", - "[resourceId('Microsoft.OperationsManagement/solutions',variables('containerInsightsSolutionName'))]" - ], - "properties": { - "actions": [], - "criteria": { - "allOf": [ - { - "criterionType": "StaticThresholdCriterion", - "dimensions": [ - { - "name": "controllerName", - "operator": "Include", - "values": [ - "*" - ] - }, - { - "name": "kubernetes namespace", - "operator": "Include", - "values": [ - "*" - ] - } - ], - "metricName": "memoryWorkingSetExceededPercentage", - "metricNamespace": "Insights.Container/containers", - "name": "Metric1", - "operator": "GreaterThan", - "threshold": 90.0, - "timeAggregation": "Average", - "skipMetricValidation": true - } - ], - "odata.type": "Microsoft.Azure.Monitor.SingleResourceMultipleMetricCriteria" - }, - "description": "This alert monitors container working set memory utilization.", - "enabled": true, - "evaluationFrequency": "PT1M", - "scopes": [ - "[resourceId('Microsoft.ContainerService/managedClusters', variables('clusterName'))]" - ], - "severity": 3, - "targetResourceType": "microsoft.containerservice/managedclusters", - "windowSize": "PT5M" - } - }, - { - "type": "Microsoft.Insights/metricAlerts", - "apiVersion": "2018-03-01", - "name": "[concat('Pods in failed state for ', variables('clusterName'), ' CI-4')]", - "location": "global", - "dependsOn": [ - "[resourceId('Microsoft.ContainerService/managedClusters', variables('clusterName'))]", - "[resourceId('Microsoft.OperationsManagement/solutions',variables('containerInsightsSolutionName'))]" - ], - "properties": { - "actions": [], - "criteria": { - "allOf": [ - { - "criterionType": "StaticThresholdCriterion", - "dimensions": [ - { - "name": "phase", - "operator": "Include", - "values": [ - "Failed" - ] - } - ], - "metricName": "podCount", - "metricNamespace": "Insights.Container/pods", - "name": "Metric1", - "operator": "GreaterThan", - "threshold": 0.0, - "timeAggregation": "Average", - "skipMetricValidation": true - } - ], - "odata.type": "Microsoft.Azure.Monitor.SingleResourceMultipleMetricCriteria" - }, - "description": "Pod status monitoring.", - "enabled": true, - "evaluationFrequency": "PT1M", - "scopes": [ - "[resourceId('Microsoft.ContainerService/managedClusters', variables('clusterName'))]" - ], - "severity": 3, - "targetResourceType": "microsoft.containerservice/managedclusters", - "windowSize": "PT5M" - } - }, - { - "type": "Microsoft.Insights/metricAlerts", - "apiVersion": "2018-03-01", - "name": "[concat('Disk usage high for ', variables('clusterName'), ' CI-5')]", - "location": "global", - "dependsOn": [ - "[resourceId('Microsoft.ContainerService/managedClusters', variables('clusterName'))]", - "[resourceId('Microsoft.OperationsManagement/solutions',variables('containerInsightsSolutionName'))]" - ], - "properties": { - "actions": [], - "criteria": { - "allOf": [ - { - "criterionType": "StaticThresholdCriterion", - "dimensions": [ - { - "name": "host", - "operator": "Include", - "values": [ - "*" - ] - }, - { - "name": "device", - "operator": "Include", - "values": [ - "*" - ] - } - ], - "metricName": "DiskUsedPercentage", - "metricNamespace": "Insights.Container/nodes", - "name": "Metric1", - "operator": "GreaterThan", - "threshold": 80.0, - "timeAggregation": "Average", - "skipMetricValidation": true - } - ], - "odata.type": "Microsoft.Azure.Monitor.SingleResourceMultipleMetricCriteria" - }, - "description": "This alert monitors disk usage for all nodes and storage devices.", - "enabled": true, - "evaluationFrequency": "PT1M", - "scopes": [ - "[resourceId('Microsoft.ContainerService/managedClusters', variables('clusterName'))]" - ], - "severity": 3, - "targetResourceType": "microsoft.containerservice/managedclusters", - "windowSize": "PT5M" - } - }, - { - "type": "Microsoft.Insights/metricAlerts", - "apiVersion": "2018-03-01", - "name": "[concat('Nodes in not ready status for ', variables('clusterName'), ' CI-3')]", - "location": "global", - "dependsOn": [ - "[resourceId('Microsoft.ContainerService/managedClusters', variables('clusterName'))]", - "[resourceId('Microsoft.OperationsManagement/solutions',variables('containerInsightsSolutionName'))]" - ], - "properties": { - "actions": [], - "criteria": { - "allOf": [ - { - "criterionType": "StaticThresholdCriterion", - "dimensions": [ - { - "name": "status", - "operator": "Include", - "values": [ - "NotReady" - ] - } - ], - "metricName": "nodesCount", - "metricNamespace": "Insights.Container/nodes", - "name": "Metric1", - "operator": "GreaterThan", - "threshold": 0.0, - "timeAggregation": "Average", - "skipMetricValidation": true - } - ], - "odata.type": "Microsoft.Azure.Monitor.SingleResourceMultipleMetricCriteria" - }, - "description": "Node status monitoring.", - "enabled": true, - "evaluationFrequency": "PT1M", - "scopes": [ - "[resourceId('Microsoft.ContainerService/managedClusters', variables('clusterName'))]" - ], - "severity": 3, - "targetResourceType": "microsoft.containerservice/managedclusters", - "windowSize": "PT5M" - } - }, - { - "type": "Microsoft.Insights/metricAlerts", - "apiVersion": "2018-03-01", - "name": "[concat('Containers getting OOM killed for ', variables('clusterName'), ' CI-6')]", - "location": "global", - "dependsOn": [ - "[resourceId('Microsoft.ContainerService/managedClusters', variables('clusterName'))]", - "[resourceId('Microsoft.OperationsManagement/solutions',variables('containerInsightsSolutionName'))]" - ], - "properties": { - "actions": [], - "criteria": { - "allOf": [ - { - "criterionType": "StaticThresholdCriterion", - "dimensions": [ - { - "name": "kubernetes namespace", - "operator": "Include", - "values": [ - "*" - ] - }, - { - "name": "controllerName", - "operator": "Include", - "values": [ - "*" - ] - } - ], - "metricName": "oomKilledContainerCount", - "metricNamespace": "Insights.Container/pods", - "name": "Metric1", - "operator": "GreaterThan", - "threshold": 0.0, - "timeAggregation": "Average", - "skipMetricValidation": true - } - ], - "odata.type": "Microsoft.Azure.Monitor.SingleResourceMultipleMetricCriteria" - }, - "description": "This alert monitors number of containers killed due to out of memory (OOM) error.", - "enabled": true, - "evaluationFrequency": "PT1M", - "scopes": [ - "[resourceId('Microsoft.ContainerService/managedClusters', variables('clusterName'))]" - ], - "severity": 3, - "targetResourceType": "microsoft.containerservice/managedclusters", - "windowSize": "PT1M" - } - }, - { - "type": "Microsoft.Insights/metricAlerts", - "apiVersion": "2018-03-01", - "name": "[concat('Persistent volume usage high for ', variables('clusterName'), ' CI-18')]", - "location": "global", - "dependsOn": [ - "[resourceId('Microsoft.ContainerService/managedClusters', variables('clusterName'))]", - "[resourceId('Microsoft.OperationsManagement/solutions',variables('containerInsightsSolutionName'))]" - ], - "properties": { - "actions": [], - "criteria": { - "allOf": [ - { - "criterionType": "StaticThresholdCriterion", - "dimensions": [ - { - "name": "podName", - "operator": "Include", - "values": [ - "*" - ] - }, - { - "name": "kubernetesNamespace", - "operator": "Include", - "values": [ - "*" - ] - } - ], - "metricName": "pvUsageExceededPercentage", - "metricNamespace": "Insights.Container/persistentvolumes", - "name": "Metric1", - "operator": "GreaterThan", - "threshold": 80.0, - "timeAggregation": "Average", - "skipMetricValidation": true - } - ], - "odata.type": "Microsoft.Azure.Monitor.SingleResourceMultipleMetricCriteria" - }, - "description": "This alert monitors persistent volume utilization.", - "enabled": false, - "evaluationFrequency": "PT1M", - "scopes": [ - "[resourceId('Microsoft.ContainerService/managedClusters', variables('clusterName'))]" - ], - "severity": 3, - "targetResourceType": "microsoft.containerservice/managedclusters", - "windowSize": "PT5M" - } - }, - { - "type": "Microsoft.Insights/metricAlerts", - "apiVersion": "2018-03-01", - "name": "[concat('Pods not in ready state for ', variables('clusterName'), ' CI-8')]", - "location": "global", - "dependsOn": [ - "[resourceId('Microsoft.ContainerService/managedClusters', variables('clusterName'))]", - "[resourceId('Microsoft.OperationsManagement/solutions',variables('containerInsightsSolutionName'))]" - ], - "properties": { - "actions": [], - "criteria": { - "allOf": [ - { - "criterionType": "StaticThresholdCriterion", - "dimensions": [ - { - "name": "controllerName", - "operator": "Include", - "values": [ - "*" - ] - }, - { - "name": "kubernetes namespace", - "operator": "Include", - "values": [ - "*" - ] - } - ], - "metricName": "PodReadyPercentage", - "metricNamespace": "Insights.Container/pods", - "name": "Metric1", - "operator": "LessThan", - "threshold": 80.0, - "timeAggregation": "Average", - "skipMetricValidation": true - } - ], - "odata.type": "Microsoft.Azure.Monitor.SingleResourceMultipleMetricCriteria" - }, - "description": "This alert monitors for excessive pods not in the ready state.", - "enabled": true, - "evaluationFrequency": "PT1M", - "scopes": [ - "[resourceId('Microsoft.ContainerService/managedClusters', variables('clusterName'))]" - ], - "severity": 3, - "targetResourceType": "microsoft.containerservice/managedclusters", - "windowSize": "PT5M" - } - }, - { - "type": "Microsoft.Insights/metricAlerts", - "apiVersion": "2018-03-01", - "name": "[concat('Restarting container count for ', variables('clusterName'), ' CI-7')]", - "location": "global", - "dependsOn": [ - "[resourceId('Microsoft.ContainerService/managedClusters', variables('clusterName'))]", - "[resourceId('Microsoft.OperationsManagement/solutions',variables('containerInsightsSolutionName'))]" - ], - "properties": { - "actions": [], - "criteria": { - "allOf": [ - { - "criterionType": "StaticThresholdCriterion", - "dimensions": [ - { - "name": "kubernetes namespace", - "operator": "Include", - "values": [ - "*" - ] - }, - { - "name": "controllerName", - "operator": "Include", - "values": [ - "*" - ] - } - ], - "metricName": "restartingContainerCount", - "metricNamespace": "Insights.Container/pods", - "name": "Metric1", - "operator": "GreaterThan", - "threshold": 0.0, - "timeAggregation": "Average", - "skipMetricValidation": true - } - ], - "odata.type": "Microsoft.Azure.Monitor.SingleResourceMultipleMetricCriteria" - }, - "description": "This alert monitors number of containers restarting across the cluster.", - "enabled": true, - "evaluationFrequency": "PT1M", - "scopes": [ - "[resourceId('Microsoft.ContainerService/managedClusters', variables('clusterName'))]" - ], - "severity": 3, - "targetResourceType": "Microsoft.ContainerService/managedClusters", - "windowSize": "PT1M" - } - }, - { - "type": "Microsoft.Authorization/policyAssignments", - "apiVersion": "2020-09-01", - "name": "[variables('policyAssignmentNameAKSLinuxRestrictive')]", - "comments": "Applying the 'AKS Linux Restrictive' policy to the resource group", - "properties": { - "displayName": "[trim(take(concat('[', variables('clusterName'),'] ', reference(variables('policyResourceIdAKSLinuxRestrictive'), '2020-09-01').displayName), 125))]", - "scope": "[subscriptionResourceId('Microsoft.Resources/resourceGroups', resourceGroup().name)]", - "policyDefinitionId": "[variables('policyResourceIdAKSLinuxRestrictive')]", - "parameters": { - "effect": { - "value": "audit" - }, - "excludedNamespaces": { - "value": [ - "kube-system", - "gatekeeper-system" - ] - } - } - } - }, - { - "type": "Microsoft.Authorization/policyAssignments", - "apiVersion": "2020-09-01", - "name": "[variables('policyAssignmentNameEnforceHttpsIngress')]", - "comments": "Applying the 'Enforce HTTPS ingress in Kubernetes cluster' policy to the resource group.", - "properties": { - "displayName": "[trim(take(concat('[', variables('clusterName'),'] ', reference(variables('policyResourceIdEnforceHttpsIngress'), '2020-09-01').displayName), 125))]", - "scope": "[subscriptionResourceId('Microsoft.Resources/resourceGroups', resourceGroup().name)]", - "policyDefinitionId": "[variables('policyResourceIdEnforceHttpsIngress')]", - "parameters": { - "effect": { - "value": "deny" - }, - "excludedNamespaces": { - "value": [] - } - } - } - }, - { - "type": "Microsoft.Authorization/policyAssignments", - "apiVersion": "2020-09-01", - "name": "[variables('policyAssignmentNameEnforceInternalLoadBalancers')]", - "comments": "Applying the 'Enforce internal load balancers in Kubernetes cluster' policy to the resource group.", - "properties": { - "displayName": "[trim(take(concat('[', variables('clusterName'),'] ', reference(variables('policyResourceIdEnforceInternalLoadBalancers'), '2020-09-01').displayName), 125))]", - "scope": "[subscriptionResourceId('Microsoft.Resources/resourceGroups', resourceGroup().name)]", - "policyDefinitionId": "[variables('policyResourceIdEnforceInternalLoadBalancers')]", - "parameters": { - "effect": { - "value": "deny" - }, - "excludedNamespaces": { - "value": [] - } - } - } - }, - { - "type": "Microsoft.Authorization/policyAssignments", - "apiVersion": "2020-09-01", - "name": "[variables('policyAssignmentNameMustNotAutomountApiCreds')]", - "comments": "Applying the 'No Automount of API credentials' policy to the resource group.", - "properties": { - "displayName": "[trim(take(concat('[', variables('clusterName'),'] ', reference(variables('policyResourceIdMustNotAutomountApiCreds'), '2020-09-01').displayName), 125))]", - "scope": "[subscriptionResourceId('Microsoft.Resources/resourceGroups', resourceGroup().name)]", - "policyDefinitionId": "[variables('policyResourceIdMustNotAutomountApiCreds')]", - "parameters": { - "effect": { - "value": "deny" - }, - "excludedNamespaces": { - "value": [ - "kube-system", - "gatekeeper-system", - "flux-system", // Required by Flux - "falco-system", // Required by Falco - "osm-system", // Required by OSM - "ingress-nginx", // Required by NGINX - "cluster-baseline-settings" // Required by KeyVault CSI & Kured - ] - } - } - } - }, - { - "type": "Microsoft.Authorization/policyAssignments", - "apiVersion": "2020-09-01", - "name": "[variables('policyAssignmentNameMustUseSpecifiedLabels')]", - "comments": "Applying the 'Must use specific labels' policy to the resource group.", - "properties": { - "displayName": "[trim(take(concat('[', variables('clusterName'),'] ', reference(variables('policyResourceIdMustUseSpecifiedLabels'), '2020-09-01').displayName), 125))]", - "scope": "[subscriptionResourceId('Microsoft.Resources/resourceGroups', resourceGroup().name)]", - "policyDefinitionId": "[variables('policyResourceIdMustUseSpecifiedLabels')]", - "parameters": { - "effect": { - "value": "audit" - }, - "labelsList": { - "value": [ - "pci-scope" - ] - } - } - } - }, - { - "type": "Microsoft.Authorization/policyAssignments", - "apiVersion": "2020-09-01", - "name": "[variables('policyAssignmentNameMustUseTheseExternalIps')]", - "comments": "Applying the 'Approved External IP Addresses' policy to the resource group.", - "properties": { - "displayName": "[trim(take(concat('[', variables('clusterName'),'] ', reference(variables('policyResourceIdMustUseTheseExternalIps'), '2020-09-01').displayName), 125))]", - "scope": "[subscriptionResourceId('Microsoft.Resources/resourceGroups', resourceGroup().name)]", - "policyDefinitionId": "[variables('policyResourceIdMustUseTheseExternalIps')]", - "parameters": { - "effect": { - "value": "deny" - }, - "excludedNamespaces": { - "value": [ - "kube-system", - "gatekeeper-system" - ] - }, - "allowedExternalIPs": { - "value": [] // No external IPs allowed (LoadBalancer Service types do not apply to this policy) - } - } - } - }, - { - "type": "Microsoft.Authorization/policyAssignments", - "apiVersion": "2020-09-01", - "name": "[variables('policyAssignmentNameApprovedContainerPortsOnly')]", - "comments": "Applying the 'Approved Container Ports Only' policy to for the workload living in a0005-i and a0005-o namespaces.", - "properties": { - "displayName": "[trim(take(concat('[', variables('clusterName'),'-a0005] ', reference(variables('policyResourceIdApprovedContainerPortsOnly'), '2020-09-01').displayName), 125))]", - "scope": "[subscriptionResourceId('Microsoft.Resources/resourceGroups', resourceGroup().name)]", - "policyDefinitionId": "[variables('policyResourceIdApprovedContainerPortsOnly')]", - "parameters": { - "effect": { - "value": "audit" - }, - "excludedNamespaces": { - "value": [] - }, - "namespaces": { - "value": [ - "a0005-i", - "a0005-o" - ] - }, - "allowedContainerPortsList": { - "value": [ - "8080", // ASP.net service listens on this - "15000", // envoy proxy for service mesh - "15003", // envoy proxy for service mesh - "15010" // envoy proxy for service mesh - ] - } - } - } - }, - { - "type": "Microsoft.Authorization/policyAssignments", - "apiVersion": "2020-09-01", - "name": "[variables('policyAssignmentNameApprovedServicePortsOnly')]", - "comments": "Applying the 'Approved Service Ports Only' policy to the resource group.", - "properties": { - "displayName": "[trim(take(concat('[', variables('clusterName'),'] ', reference(variables('policyResourceIdApprovedServicePortsOnly'), '2020-09-01').displayName), 125))]", - "scope": "[subscriptionResourceId('Microsoft.Resources/resourceGroups', resourceGroup().name)]", - "policyDefinitionId": "[variables('policyResourceIdApprovedServicePortsOnly')]", - "parameters": { - "effect": { - "value": "audit" - }, - "excludedNamespaces": { - "value": [ - "kube-system", - "gatekeeper-system", - "osm-system" - ] - }, - "allowedServicePortsList": { - "value": [ - "443", // ingress-controller - "80", // flux source-controller and microservice workload - "8080" // web-frontend workload - ] - } - } - } - }, - { - "type": "Microsoft.Authorization/policyAssignments", - "apiVersion": "2020-09-01", - "name": "[variables('policyAssignmentNameRoRootFilesystem')]", - "comments": "Applying the 'Kubernetes cluster containers should run with a read only root file system' policy to the resource group.", - "properties": { - "displayName": "[trim(take(concat('[', variables('clusterName'),'] ', reference(variables('policyResourceIdRoRootFilesystem'), '2020-09-01').displayName), 125))]", - "scope": "[subscriptionResourceId('Microsoft.Resources/resourceGroups', resourceGroup().name)]", - "policyDefinitionId": "[variables('policyResourceIdRoRootFilesystem')]", - "parameters": { - "effect": { - "value": "audit" - }, - // Not all workloads support this. E.g. ASP.NET requires a non-readonly root file system to handle request buffering when there is memory pressure. - "excludedNamespaces": { - "value": [ - "kube-system", - "gatekeeper-system" - ] - } - } - } - }, - { - "type": "Microsoft.Authorization/policyAssignments", - "apiVersion": "2020-09-01", - "name": "[variables('policyAssignmentNameBlockDefaultNamespace')]", - "comments": "Applying the 'Block Default Namespace' policy at the resource group level.", - "properties": { - "displayName": "[trim(take(concat('[', variables('clusterName'),'] ', reference(variables('policyResourceIdBlockDefaultNamespace'), '2020-09-01').displayName), 125))]", - "scope": "[subscriptionResourceId('Microsoft.Resources/resourceGroups', resourceGroup().name)]", - "policyDefinitionId": "[variables('policyResourceIdBlockDefaultNamespace')]", - "parameters": { - "effect": { - "value": "deny" - }, - "excludedNamespaces": { - "value": [] - } - } - } - }, - { - "type": "Microsoft.Authorization/policyAssignments", - "apiVersion": "2020-09-01", - "name": "[variables('policyAssignmentNameEnforceResourceLimits')]", - "comments": "Applying the 'Container Images Resource Limits' policy at the resource group level.", - "properties": { - "displayName": "[trim(take(concat('[', variables('clusterName'),'] ', reference(variables('policyResourceIdEnforceResourceLimits'), '2020-09-01').displayName), 125))]", - "scope": "[subscriptionResourceId('Microsoft.Resources/resourceGroups', resourceGroup().name)]", - "policyDefinitionId": "[variables('policyResourceIdEnforceResourceLimits')]", - "parameters": { - "effect": { - "value": "audit" - }, - "cpuLimit": { - "value": "1500m" - }, - "memoryLimit": { - "value": "2Gi" - }, - "excludedNamespaces": { - "value": [ - "kube-system", - "gatekeeper-system" - ] - } - } - } - }, - { - "type": "Microsoft.Authorization/policyAssignments", - "apiVersion": "2020-03-01", - "name": "[variables('policyAssignmentNameEnforceImageSource')]", - "comments": "Applying the 'Allowed Container Images' regex policy at the resource group level.", - "properties": { - "displayName": "[trim(take(concat('[', variables('clusterName'),'] ', reference(variables('policyResourceIdEnforceImageSource'), '2020-09-01').displayName), 125))]", - "scope": "[subscriptionResourceId('Microsoft.Resources/resourceGroups', resourceGroup().name)]", - "policyDefinitionId": "[variables('policyResourceIdEnforceImageSource')]", - "parameters": { - "allowedContainerImagesRegex": { - "value": "[concat(variables('defaultAcrName'), '\\.azurecr\\.io\\/live\\/.+$|mcr\\.microsoft\\.com\\/oss\\/(openservicemesh\\/init:|envoyproxy\\/envoy:).+$')]" - }, - "effect": { - "value": "deny" - }, - "excludedNamespaces": { - "value": [ - "kube-system", - "gatekeeper-system" - ] - } - } - } - } - ], - "outputs": { - "aksClusterName": { - "type": "string", - "value": "[variables('clusterName')]" - }, - "agwName": { - "type": "string", - "value": "[variables('agwName')]" - }, - "keyVaultName": { - "type": "string", - "value": "[variables('keyVaultName')]" - }, - "containerRegistryName": { - "type": "string", - "value": "[variables('defaultAcrName')]" - }, - "quarantineContainerRegistryName": { - "type": "string", - "value": "[variables('defaultAcrName')]" - } - } -} From 87a222b7f08d1c43fb7decc84a157b1ff11c45b1 Mon Sep 17 00:00:00 2001 From: Fernando Antivero Date: Thu, 13 Oct 2022 15:55:40 +0000 Subject: [PATCH 079/115] use bicep in readme file --- docs/deploy/09-aks-cluster.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/docs/deploy/09-aks-cluster.md b/docs/deploy/09-aks-cluster.md index 32160abc..7e915517 100644 --- a/docs/deploy/09-aks-cluster.md +++ b/docs/deploy/09-aks-cluster.md @@ -45,17 +45,17 @@ Now that the [hub-spoke network is provisioned](./08-cluster-networking.md), the ```bash # [This takes about 20 minutes to run.] - az deployment group create -g rg-bu0001a0005 -f cluster-stamp.json -p targetVnetResourceId=${RESOURCEID_VNET_CLUSTERSPOKE} clusterAdminAadGroupObjectId=${AADOBJECTID_GROUP_CLUSTERADMIN} k8sControlPlaneAuthorizationTenantId=${TENANTID_K8SRBAC} appGatewayListenerCertificate=${APP_GATEWAY_LISTENER_CERTIFICATE_BASE64} aksIngressControllerCertificate=${INGRESS_CONTROLLER_CERTIFICATE_BASE64} jumpBoxImageResourceId=${RESOURCEID_IMAGE_JUMPBOX} jumpBoxCloudInitAsBase64=${CLOUDINIT_BASE64} + az deployment group create -g rg-bu0001a0005 -f cluster-stamp.bicep -p targetVnetResourceId=${RESOURCEID_VNET_CLUSTERSPOKE} clusterAdminAadGroupObjectId=${AADOBJECTID_GROUP_CLUSTERADMIN} k8sControlPlaneAuthorizationTenantId=${TENANTID_K8SRBAC} appGatewayListenerCertificate=${APP_GATEWAY_LISTENER_CERTIFICATE_BASE64} aksIngressControllerCertificate=${INGRESS_CONTROLLER_CERTIFICATE_BASE64} jumpBoxImageResourceId=${RESOURCEID_IMAGE_JUMPBOX} jumpBoxCloudInitAsBase64=${CLOUDINIT_BASE64} # Or if you updated and wish to use the parameters file … - #az deployment group create -g rg-bu0001a0005 -f cluster-stamp.json -p "@azuredeploy.parameters.prod.json" + #az deployment group create -g rg-bu0001a0005 -f cluster-stamp.bicep -p "@azuredeploy.parameters.prod.json" ``` 1. Update cluster deployment with managed identity assignments. - **cluster-stamp.v2.json** is a _tiny_ evolution of the **cluster-stamp.json** ARM template you literally just deployed in the step above. Because we are using Azure AD Pod Identity v1 as a Microsoft-managed add-on, the mechanism to associate identities with the cluster is via ARM template instead of via Kubernetes manifest deployments (as you would do with the vanilla open source solution). However, due to a current limitation of the add-on, managed identities for Pod Managed Identities CANNOT be associated to the cluster when the cluster is first being created, only as an update to an existing cluster. So this deployment will re-deploy with the Pod Managed Identity association as the _only change_. Pod Managed Identity v2 is in development and it will support assignment at cluster-creation time. This implementation will evolve to use Azure AD Pod Identity v2 when available and we'll remove this step and add the assignment directly in `cluster-stamp.json`. + **cluster-stamp.v2.json** is a _tiny_ evolution of the **cluster-stamp.bicep** ARM template you literally just deployed in the step above. Because we are using Azure AD Pod Identity v1 as a Microsoft-managed add-on, the mechanism to associate identities with the cluster is via ARM template instead of via Kubernetes manifest deployments (as you would do with the vanilla open source solution). However, due to a current limitation of the add-on, managed identities for Pod Managed Identities CANNOT be associated to the cluster when the cluster is first being created, only as an update to an existing cluster. So this deployment will re-deploy with the Pod Managed Identity association as the _only change_. Pod Managed Identity v2 is in development and it will support assignment at cluster-creation time. This implementation will evolve to use Azure AD Pod Identity v2 when available and we'll remove this step and add the assignment directly in `cluster-stamp.bicep`. - > :eyes: If you're curious to see what changed in the cluster stamp, [view the diff](https://diffviewer.azureedge.net/?l=https://raw.githubusercontent.com/mspnp/aks-baseline-regulated/main/cluster-stamp.json&r=https://raw.githubusercontent.com/mspnp/aks-baseline-regulated/main/cluster-stamp.v2.json). + > :eyes: If you're curious to see what changed in the cluster stamp, [view the diff](https://diffviewer.azureedge.net/?l=https://raw.githubusercontent.com/mspnp/aks-baseline-regulated/main/cluster-stamp.bicep&r=https://raw.githubusercontent.com/mspnp/aks-baseline-regulated/main/cluster-stamp.v2.json). ```bash # [This takes about five minutes to run.] From 660a2823b9a20b1dab403a3533417d111f9e1d37 Mon Sep 17 00:00:00 2001 From: Fernando Antivero Date: Thu, 13 Oct 2022 16:12:04 +0000 Subject: [PATCH 080/115] big fix: replace hardcoded zones for pickup zone function --- cluster-stamp.bicep | 18 +++--------------- 1 file changed, 3 insertions(+), 15 deletions(-) diff --git a/cluster-stamp.bicep b/cluster-stamp.bicep index 624237b9..f6406cd6 100644 --- a/cluster-stamp.bicep +++ b/cluster-stamp.bicep @@ -1342,11 +1342,7 @@ resource mc 'Microsoft.ContainerService/managedClusters@2021-05-01' = { orchestratorVersion: kubernetesVersion enableNodePublicIP: false maxPods: 30 - availabilityZones: [ - '1' - '2' - '3' - ] + availabilityZones: pickZones('Microsoft.ContainerService', 'managedClusters', location, 3) upgradeSettings: { maxSurge: '33%' } @@ -1375,11 +1371,7 @@ resource mc 'Microsoft.ContainerService/managedClusters@2021-05-01' = { orchestratorVersion: kubernetesVersion enableNodePublicIP: false maxPods: 30 - availabilityZones: [ - '1' - '2' - '3' - ] + availabilityZones: pickZones('Microsoft.ContainerService', 'managedClusters', location, 3) upgradeSettings: { maxSurge: '33%' } @@ -1411,11 +1403,7 @@ resource mc 'Microsoft.ContainerService/managedClusters@2021-05-01' = { orchestratorVersion: kubernetesVersion enableNodePublicIP: false maxPods: 30 - availabilityZones: [ - '1' - '2' - '3' - ] + availabilityZones: pickZones('Microsoft.ContainerService', 'managedClusters', location, 3) upgradeSettings: { maxSurge: '33%' } From 4beb2efff98c10f2d76673e3b44c277f37f25d49 Mon Sep 17 00:00:00 2001 From: Fernando Antivero Date: Fri, 14 Oct 2022 13:20:48 -0300 Subject: [PATCH 081/115] Apply PR Feedback: new code comments, typos, proper capitalization and more Co-authored-by: Chad Kittel --- cluster-stamp.bicep | 59 ++++++++++++++++++++++++--------------------- 1 file changed, 32 insertions(+), 27 deletions(-) diff --git a/cluster-stamp.bicep b/cluster-stamp.bicep index f6406cd6..8ff0421f 100644 --- a/cluster-stamp.bicep +++ b/cluster-stamp.bicep @@ -223,19 +223,19 @@ resource miClusterControlPlane 'Microsoft.ManagedIdentity/userAssignedIdentities location: location } -@description('The in-cluster ingress controller identity used by pod identity agent to acquire access tokens to read ssl certs from Azure KeyVault.') +@description('The in-cluster ingress controller identity used by the pod identity agent to acquire access tokens to read SSL certs from Azure Key Vault.') resource miIngressController 'Microsoft.ManagedIdentity/userAssignedIdentities@2022-01-31-preview' = { name: 'mi-${clusterName}-ingresscontroller' location: location } -@description('The regional load balancer identity used by your Application Gateway instance to acquire access tokens to read ssl certs and secrets from Azure KeyVault.') +@description('The regional load balancer identity used by your Application Gateway instance to acquire access tokens to read certs and secrets from Azure Key Vault.') resource miAppGateway 'Microsoft.ManagedIdentity/userAssignedIdentities@2022-01-31-preview' = { name: 'mi-appgateway' location: location } -@description('Grant the cluster control plane managed identity with managed identity operator role permissions; this allows to assign compute with the ingress controller managed identity; this is required for Azure Pod Idenity.') +@description('Grant the cluster control plane managed identity with managed identity operator role permissions; this allows to assign compute with the ingress controller managed identity; this is required for Azure Pod Identity.') resource icMiClusterControlPlaneManagedIdentityOperatorRole_roleAssignment 'Microsoft.Authorization/roleAssignments@2022-04-01' = { scope: miIngressController name: guid(resourceGroup().id, miClusterControlPlane.name, managedIdentityOperatorRole.id) @@ -246,7 +246,7 @@ resource icMiClusterControlPlaneManagedIdentityOperatorRole_roleAssignment 'Micr } } -@description('The secret storage management resource for the aks regulated cluster.') +@description('The secret storage management resource for the AKS regulated cluster.') resource kv 'Microsoft.KeyVault/vaults@2022-07-01' = { name: 'kv-${clusterName}' location: location @@ -296,7 +296,7 @@ resource kv 'Microsoft.KeyVault/vaults@2022-07-01' = { enableSoftDelete: true } - // The internet facing Tls certificate to establish https connections between your clients and your regional load balancer + // The internet facing TLS certificate to establish HTTPS connections between your clients and your regional load balancer resource kvsGatewaySslCert 'secrets' = { name: 'sslcert' properties: { @@ -304,7 +304,7 @@ resource kv 'Microsoft.KeyVault/vaults@2022-07-01' = { } } - // The aks regulated in-cluster Tls certificate to establish https connections between your regional load balancer and your ingress controller enabling e2e tls connections + // The in-cluster TLS certificate to establish HTTPS connections between your regional load balancer and your ingress controller, enabling end-to-end TLS connections. resource kvsAppGwIngressInternalAksIngressTls 'secrets' = { name: 'agw-ingress-incluster-aks-ingress-contoso-com-tls' properties: { @@ -313,7 +313,7 @@ resource kv 'Microsoft.KeyVault/vaults@2022-07-01' = { } } -@description('Grant the Azure Application Gateway managed identity with key vault secrets user role permissions; this allows pulling secrets from key vault.') +@description('Grant the Azure Application Gateway managed identity with Key Vault secrets user role permissions; this allows pulling secrets from Key Vault.') resource kvMiAppGatewaySecretsUserRole_roleAssignment 'Microsoft.Authorization/roleAssignments@2020-10-01-preview' = { scope: kv name: guid(resourceGroup().id, 'mi-appgateway', keyVaultSecretsUserRole.id) @@ -324,7 +324,7 @@ resource kvMiAppGatewaySecretsUserRole_roleAssignment 'Microsoft.Authorization/r } } -@description('Grant the Azure Application Gateway managed identity with key vault reader role permissions; this allows pulling frontend and backend certificates.') +@description('Grant the Azure Application Gateway managed identity with Key Vault reader role permissions; this allows pulling frontend and backend certificates.') resource kvMiAppGatewayKeyVaultReader_roleAssignment 'Microsoft.Authorization/roleAssignments@2020-10-01-preview' = { scope: kv name: guid(resourceGroup().id, 'mi-appgateway', keyVaultReaderRole.id) @@ -335,7 +335,7 @@ resource kvMiAppGatewayKeyVaultReader_roleAssignment 'Microsoft.Authorization/ro } } -@description('The aks regulated cluster log analytics workspace.') +@description('The AKS cluster and related resources log analytics workspace.') resource law 'Microsoft.OperationalInsights/workspaces@2021-12-01-preview' = { name: 'law-${clusterName}' location: location @@ -481,7 +481,7 @@ resource kv_diagnosticSettings 'Microsoft.Insights/diagnosticSettings@2021-05-01 } } -@description('The network interface in the spoke vnet that enables connecting privately the aks regulated cluster with kv.') +@description('The network interface in the spoke vnet that enables privately connecting the AKS cluster with Key Vault.') resource peKv 'Microsoft.Network/privateEndpoints@2022-01-01' = { name: 'pe-${kv.name}' location: location @@ -637,7 +637,7 @@ resource agw 'Microsoft.Network/applicationGateways@2022-01-01' = { properties: { backendAddresses: [ { - ipAddress: '10.240.4.4' + ipAddress: '10.240.4.4' // This is the IP address that our ingress controller will request } ] } @@ -930,7 +930,7 @@ resource cr_diagnosticSettings 'Microsoft.Insights/diagnosticSettings@2021-05-01 } } -@description('The network interface in the spoke vnet that enables connecting privately the aks regulated cluster with cr.') +@description('The network interface in the spoke vnet that enables privately connecting the AKS cluster with Container Registry.') resource peCr 'Microsoft.Network/privateEndpoints@2022-05-01' = { name: 'pe-${cr.name}' location: location @@ -1077,11 +1077,11 @@ resource paMustNotAutomountApiCreds 'Microsoft.Authorization/policyAssignments@2 value: [ 'kube-system' 'gatekeeper-system' - 'flux-system' - 'falco-system' - 'osm-system' - 'ingress-nginx' - 'cluster-baseline-settings' + 'flux-system' // Required by Flux + 'falco-system' // Required by Falco + 'osm-system' // Required by OSM + 'ingress-nginx' // Required by NGINX + 'cluster-baseline-settings' // Required by Key Vault CSI & Kured ] } } @@ -1124,7 +1124,7 @@ resource paMustUseTheseExternalIps 'Microsoft.Authorization/policyAssignments@20 ] } allowedExternalIPs: { - value: [] + value: [] // No external IPs allowed (LoadBalancer Service types do not apply to this policy) } } } @@ -1151,10 +1151,10 @@ resource paApprovedContainerPortsOnly 'Microsoft.Authorization/policyAssignments } allowedContainerPortsList: { value: [ - '8080' - '15000' - '15003' - '15010' + '8080' // ASP.net service listens on this + '15000' // envoy proxy for service mesh + '15003' // envoy proxy for service mesh + '15010' // envoy proxy for service mesh ] } } @@ -1180,15 +1180,16 @@ resource paApprovedServicePortsOnly 'Microsoft.Authorization/policyAssignments@2 } allowedServicePortsList: { value: [ - '443' - '80' - '8080' + '443' // ingress-controller + '80' // flux source-controller and microservice workload + '8080' // web-frontend workload ] } } } } +@decscription('Applying the 'Kubernetes cluster containers should run with a read only root file system' policy to the resource group.') resource paRoRootFilesystem 'Microsoft.Authorization/policyAssignments@2020-09-01' = { name: guid(pdRoRootFilesystemId, resourceGroup().name, clusterName) properties: { @@ -1199,6 +1200,7 @@ resource paRoRootFilesystem 'Microsoft.Authorization/policyAssignments@2020-09-0 effect: { value: 'audit' } + // Not all workloads support this. E.g. ASP.NET requires a non-readonly root file system to handle request buffering when there is memory pressure. excludedNamespaces: { value: [ 'kube-system' @@ -1524,6 +1526,9 @@ resource mc 'Microsoft.ContainerService/managedClusters@2021-05-01' = { omsContainerInsights ensureClusterIdentityHasRbacToSelfManagedResources + // You want policies created before cluster because they take some time to be made available and we want them + // to apply to your cluster as soon as possible. Nothing in this cluster "technically" depends on these existing, + // just trying to get coverage as soon as possible. paAksLinuxRestrictive paEnforceHttpsIngress paEnforceInternalLoadBalancers @@ -1536,7 +1541,7 @@ resource mc 'Microsoft.ContainerService/managedClusters@2021-05-01' = { paBlockDefaultNamespace paEnforceResourceLimits paEnforceImageSource - vmssJumpboxes + vmssJumpboxes // Ensure jumboxes are available to use as soon as possible, don't wait until cluster is created. miIngressController ] } @@ -1580,7 +1585,7 @@ resource crMiKubeletContainerRegistryPullRole_roleAssignment 'Microsoft.Authoriz } } -@description('Grant Oms agent managed identity with publisher metrics role permissions; this allows the Oms agent\'s identity to publish metrics in container insights.') +@description('Grant OMS agent managed identity with publisher metrics role permissions; this allows the OMS agent\'s identity to publish metrics in Container Insights.') resource sMiOmsMonitoringMetricPublisherRoleRole_roleAssignment 'Microsoft.Authorization/roleAssignments@2022-04-01' = { name: guid(mc.id, monitoringMetricsPublisherRole.id) properties: { From e34258a54b24423c642af20efab5465cc0536a0d Mon Sep 17 00:00:00 2001 From: Fernando Antivero Date: Fri, 14 Oct 2022 16:22:50 +0000 Subject: [PATCH 082/115] Apply PR Feeback: restore location default value --- cluster-stamp.bicep | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cluster-stamp.bicep b/cluster-stamp.bicep index 8ff0421f..f34953fd 100644 --- a/cluster-stamp.bicep +++ b/cluster-stamp.bicep @@ -37,7 +37,7 @@ param aksIngressControllerCertificate string ]) @description('AKS Service, Node Pools, and supporting services (KeyVault, App Gateway, etc) region. This needs to be the same region as the vnet provided in these parameters.') @minLength(4) -param location string +param location string = 'eastus2' @allowed([ 'australiasoutheast' From e368763695495ed298ff82ae81033c3e3d1a40f6 Mon Sep 17 00:00:00 2001 From: Fernando Antivero Date: Fri, 14 Oct 2022 16:23:45 +0000 Subject: [PATCH 083/115] Apply PR Feeback: restore geo redundancy location default value --- cluster-stamp.bicep | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cluster-stamp.bicep b/cluster-stamp.bicep index f34953fd..0b457ecd 100644 --- a/cluster-stamp.bicep +++ b/cluster-stamp.bicep @@ -64,7 +64,7 @@ param location string = 'eastus2' ]) @description('For Azure resources that support native geo-redunancy, provide the location the redundant service will have its secondary. Should be different than the location parameter and ideally should be a paired region - https://learn.microsoft.com/azure/best-practices-availability-paired-regions. This region does not need to support availability zones.') @minLength(4) -param geoRedundancyLocation string +param geoRedundancyLocation string = 'centralus' @description('The Azure resource ID of a VM image that will be used for the jump box.') @minLength(70) From a45abb238a6d92178e139dc1e04e567ef0a70185 Mon Sep 17 00:00:00 2001 From: Fernando Antivero Date: Fri, 14 Oct 2022 16:24:44 +0000 Subject: [PATCH 084/115] Apply PR Feeback: bug fix - remove wrong assigned default value --- cluster-stamp.bicep | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cluster-stamp.bicep b/cluster-stamp.bicep index 0b457ecd..420b1c16 100644 --- a/cluster-stamp.bicep +++ b/cluster-stamp.bicep @@ -72,7 +72,7 @@ param jumpBoxImageResourceId string @description('A cloud init file (starting with #cloud-config) as a base 64 encoded string used to perform image customization on the jump box VMs. Used for user-management in this context.') @minLength(100) -param jumpBoxCloudInitAsBase64 string = '10.200.0.0/26' +param jumpBoxCloudInitAsBase64 string /*** VARIABLES ***/ From 58838dcb92479bdf693311099c9e33a915c495ca Mon Sep 17 00:00:00 2001 From: Fernando Antivero Date: Fri, 14 Oct 2022 16:32:02 +0000 Subject: [PATCH 085/115] Apply PR Feedback: consider just calling this spokeResourceGroup that'll be more clear than targetResourceGroup --- cluster-stamp.bicep | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/cluster-stamp.bicep b/cluster-stamp.bicep index 420b1c16..e2d79663 100644 --- a/cluster-stamp.bicep +++ b/cluster-stamp.bicep @@ -124,14 +124,14 @@ var pdEnforceImageSourceId = tenantResourceId('Microsoft.Authorization/policyDef /*** EXISTING RESOURCE GROUP RESOURCES ***/ @description('Spoke resource group') -resource targetResourceGroup 'Microsoft.Resources/resourceGroups@2021-04-01' existing = { +resource spokeResourceGroup 'Microsoft.Resources/resourceGroups@2021-04-01' existing = { scope: subscription() name: '${split(targetVnetResourceId,'/')[4]}' } @description('The Spoke virtual network') resource targetVirtualNetwork 'Microsoft.Network/virtualNetworks@2022-01-01' existing = { - scope: targetResourceGroup + scope: spokeResourceGroup name: '${last(split(targetVnetResourceId,'/'))}' // Spoke virutual network's subnet for application gateway @@ -565,7 +565,7 @@ resource agw 'Microsoft.Network/applicationGateways@2022-01-01' = { name: 'agw-frontend-ip-configuration' properties: { publicIPAddress: { - id: resourceId(subscription().subscriptionId, targetResourceGroup.name, 'Microsoft.Network/publicIpAddresses', 'pip-BU0001A0005-00') + id: resourceId(subscription().subscriptionId, spokeResourceGroup.name, 'Microsoft.Network/publicIpAddresses', 'pip-BU0001A0005-00') } } } @@ -1306,7 +1306,7 @@ resource alaAllAzureAdvisorAlert 'Microsoft.Insights/activityLogAlerts@2020-10-0 module ensureClusterIdentityHasRbacToSelfManagedResources 'modules/ensureClusterIdentityHasRbacToSelfManagedResources.bicep' = { name: 'ensureClusterIdentityHasRbacToSelfManagedResources' - scope: targetResourceGroup + scope: spokeResourceGroup params: { miClusterControlPlanePrincipalId: miClusterControlPlane.properties.principalId clusterControlPlaneIdentityName: miClusterControlPlane.name From e64c830f2931cff605043b69d945d1fabda046cf Mon Sep 17 00:00:00 2001 From: Fernando Antivero Date: Fri, 14 Oct 2022 16:35:07 +0000 Subject: [PATCH 086/115] Proposal based on PR feedback: consider just calling this vnetSpoke that'll be more clear than targetVirtualNetwork --- cluster-stamp.bicep | 24 ++++++++--------- ...dentityHasRbacToSelfManagedResources.bicep | 26 +++++++++---------- 2 files changed, 25 insertions(+), 25 deletions(-) diff --git a/cluster-stamp.bicep b/cluster-stamp.bicep index e2d79663..773804c6 100644 --- a/cluster-stamp.bicep +++ b/cluster-stamp.bicep @@ -130,7 +130,7 @@ resource spokeResourceGroup 'Microsoft.Resources/resourceGroups@2021-04-01' exis } @description('The Spoke virtual network') -resource targetVirtualNetwork 'Microsoft.Network/virtualNetworks@2022-01-01' existing = { +resource vnetSpoke 'Microsoft.Network/virtualNetworks@2022-01-01' existing = { scope: spokeResourceGroup name: '${last(split(targetVnetResourceId,'/'))}' @@ -487,11 +487,11 @@ resource peKv 'Microsoft.Network/privateEndpoints@2022-01-01' = { location: location properties: { subnet: { - id: targetVirtualNetwork::snetPrivatelinkendpoints.id + id: vnetSpoke::snetPrivatelinkendpoints.id } privateLinkServiceConnections: [ { - name: 'to-${targetVirtualNetwork.name}' + name: 'to-${vnetSpoke.name}' properties: { privateLinkServiceId: kv.id groupIds: [ @@ -555,7 +555,7 @@ resource agw 'Microsoft.Network/applicationGateways@2022-01-01' = { name: 'agw-ip-configuration' properties: { subnet: { - id: targetVirtualNetwork::snetApplicationGateway.id + id: vnetSpoke::snetApplicationGateway.id } } } @@ -805,7 +805,7 @@ resource vmssJumpboxes 'Microsoft.Compute/virtualMachineScaleSets@2020-12-01' = privateIPAddressVersion: 'IPv4' publicIPAddressConfiguration: null subnet: { - id: targetVirtualNetwork::snetManagmentOps.id + id: vnetSpoke::snetManagmentOps.id } } } @@ -900,7 +900,7 @@ resource cr 'Microsoft.ContainerRegistry/registries@2022-02-01-preview' = { count: 1 os: 'Linux' tier: 'S1' - virtualNetworkSubnetResourceId: targetVirtualNetwork::snetManagmentCrAgents.id + virtualNetworkSubnetResourceId: vnetSpoke::snetManagmentCrAgents.id } } } @@ -936,11 +936,11 @@ resource peCr 'Microsoft.Network/privateEndpoints@2022-05-01' = { location: location properties: { subnet: { - id: targetVirtualNetwork::snetPrivatelinkendpoints.id + id: vnetSpoke::snetPrivatelinkendpoints.id } privateLinkServiceConnections: [ { - name: 'to-${targetVirtualNetwork.name}' + name: 'to-${vnetSpoke.name}' properties: { privateLinkServiceId: cr.id groupIds: [ @@ -1310,7 +1310,7 @@ module ensureClusterIdentityHasRbacToSelfManagedResources 'modules/ensureCluster params: { miClusterControlPlanePrincipalId: miClusterControlPlane.properties.principalId clusterControlPlaneIdentityName: miClusterControlPlane.name - targetVirtualNetworkName: targetVirtualNetwork.name + vnetSpokeName: vnetSpoke.name } } @@ -1335,7 +1335,7 @@ resource mc 'Microsoft.ContainerService/managedClusters@2021-05-01' = { osType: 'Linux' minCount: 3 maxCount: 4 - vnetSubnetID: targetVirtualNetwork::snetClusterSystemNodePools.id + vnetSubnetID: vnetSpoke::snetClusterSystemNodePools.id enableAutoScaling: true type: 'VirtualMachineScaleSets' mode: 'System' @@ -1364,7 +1364,7 @@ resource mc 'Microsoft.ContainerService/managedClusters@2021-05-01' = { osType: 'Linux' minCount: 2 maxCount: 5 - vnetSubnetID: targetVirtualNetwork::snetClusterInScopeNodePools.id + vnetSubnetID: vnetSpoke::snetClusterInScopeNodePools.id enableAutoScaling: true type: 'VirtualMachineScaleSets' mode: 'User' @@ -1396,7 +1396,7 @@ resource mc 'Microsoft.ContainerService/managedClusters@2021-05-01' = { osType: 'Linux' minCount: 2 maxCount: 5 - vnetSubnetID: targetVirtualNetwork::snetClusterOutScopeNodePools.id + vnetSubnetID: vnetSpoke::snetClusterOutScopeNodePools.id enableAutoScaling: true type: 'VirtualMachineScaleSets' mode: 'User' diff --git a/modules/ensureClusterIdentityHasRbacToSelfManagedResources.bicep b/modules/ensureClusterIdentityHasRbacToSelfManagedResources.bicep index 07f02824..11688342 100644 --- a/modules/ensureClusterIdentityHasRbacToSelfManagedResources.bicep +++ b/modules/ensureClusterIdentityHasRbacToSelfManagedResources.bicep @@ -14,7 +14,7 @@ param clusterControlPlaneIdentityName string @description('The regional network spoke VNet Resource name that the cluster is being joined to, so it can be used to discover subnets during role assignments.') @minLength(1) -param targetVirtualNetworkName string +param vnetSpokeName string /*** EXISTING SUBSCRIPTION RESOURCES ***/ @@ -34,8 +34,8 @@ resource pdzCr 'Microsoft.Network/privateDnsZones@2020-06-01' existing = { name: 'privatelink.azurecr.io' } -resource targetVirtualNetwork 'Microsoft.Network/virtualNetworks@2022-01-01' existing = { - name: targetVirtualNetworkName +resource vnetSpoke 'Microsoft.Network/virtualNetworks@2022-01-01' existing = { + name: vnetSpokeName resource snetClusterSystemNodePools 'subnets' existing = { name: 'snet-cluster-systemnodepool' @@ -57,8 +57,8 @@ resource targetVirtualNetwork 'Microsoft.Network/virtualNetworks@2022-01-01' exi /*** RESOURCES ***/ resource vnetMiClusterControlPlaneDnsZoneContributorRole_roleAssignment 'Microsoft.Authorization/roleAssignments@2020-04-01-preview' = { - scope: targetVirtualNetwork - name: guid(targetVirtualNetwork.id, dnsZoneContributorRole.id, clusterControlPlaneIdentityName) + scope: vnetSpoke + name: guid(vnetSpoke.id, dnsZoneContributorRole.id, clusterControlPlaneIdentityName) properties: { roleDefinitionId: dnsZoneContributorRole.id description: 'Allows cluster identity to attach custom DNS zone with Private Link information to this virtual network.' @@ -68,8 +68,8 @@ resource vnetMiClusterControlPlaneDnsZoneContributorRole_roleAssignment 'Microso } resource snetSystemNodePoolSubnetMiClusterControlPlaneNetworkContributorRole_roleAssignment 'Microsoft.Authorization/roleAssignments@2020-04-01-preview' = { - scope: targetVirtualNetwork::snetClusterSystemNodePools - name: guid(targetVirtualNetwork::snetClusterSystemNodePools.id, networkContributorRole.id, clusterControlPlaneIdentityName) + scope: vnetSpoke::snetClusterSystemNodePools + name: guid(vnetSpoke::snetClusterSystemNodePools.id, networkContributorRole.id, clusterControlPlaneIdentityName) properties: { roleDefinitionId: networkContributorRole.id description: 'Allows cluster identity to join the nodepool vmss resources to this subnet.' @@ -79,8 +79,8 @@ resource snetSystemNodePoolSubnetMiClusterControlPlaneNetworkContributorRole_rol } resource snetInScopeNodePoolSubnetsnetSystemNodePoolSubnetMiClusterControlPlaneNetworkContributorRole_roleAssignment 'Microsoft.Authorization/roleAssignments@2020-04-01-preview' = { - scope: targetVirtualNetwork::snetClusterInScopeNodePools - name: guid(targetVirtualNetwork::snetClusterInScopeNodePools.id, networkContributorRole.id, clusterControlPlaneIdentityName) + scope: vnetSpoke::snetClusterInScopeNodePools + name: guid(vnetSpoke::snetClusterInScopeNodePools.id, networkContributorRole.id, clusterControlPlaneIdentityName) properties: { roleDefinitionId: networkContributorRole.id description: 'Allows cluster identity to join the nodepool vmss resources to this subnet.' @@ -90,8 +90,8 @@ resource snetInScopeNodePoolSubnetsnetSystemNodePoolSubnetMiClusterControlPlaneN } resource snetOutOfScopeNodePoolSubnetMiClusterControlPlaneNetworkContributorRole_roleAssignment 'Microsoft.Authorization/roleAssignments@2020-04-01-preview' = { - scope: targetVirtualNetwork::snetClusterOutofScopeNodePools - name: guid(targetVirtualNetwork::snetClusterOutofScopeNodePools.id, networkContributorRole.id, clusterControlPlaneIdentityName) + scope: vnetSpoke::snetClusterOutofScopeNodePools + name: guid(vnetSpoke::snetClusterOutofScopeNodePools.id, networkContributorRole.id, clusterControlPlaneIdentityName) properties: { roleDefinitionId: networkContributorRole.id description: 'Allows cluster identity to join the nodepool vmss resources to this subnet.' @@ -101,8 +101,8 @@ resource snetOutOfScopeNodePoolSubnetMiClusterControlPlaneNetworkContributorRole } resource snetIngressServicesSubnetMiClusterControlPlaneNetworkContributorRole_roleAssignment 'Microsoft.Authorization/roleAssignments@2020-04-01-preview' = { - scope: targetVirtualNetwork::snetClusterIngressServices - name: guid(targetVirtualNetwork::snetClusterIngressServices.id, networkContributorRole.id, clusterControlPlaneIdentityName) + scope: vnetSpoke::snetClusterIngressServices + name: guid(vnetSpoke::snetClusterIngressServices.id, networkContributorRole.id, clusterControlPlaneIdentityName) properties: { roleDefinitionId: networkContributorRole.id description: 'Allows cluster identity to join load balancers (ingress resources) to this subnet.' From 16195350143b5d9a6de0bca50bf3388a3513d9f1 Mon Sep 17 00:00:00 2001 From: Fernando Antivero Date: Fri, 14 Oct 2022 16:37:23 +0000 Subject: [PATCH 087/115] Apply PR Feedback: bug fix - add missing rg scope --- cluster-stamp.bicep | 3 +++ 1 file changed, 3 insertions(+) diff --git a/cluster-stamp.bicep b/cluster-stamp.bicep index 773804c6..ecf8ee7a 100644 --- a/cluster-stamp.bicep +++ b/cluster-stamp.bicep @@ -172,14 +172,17 @@ resource vnetSpoke 'Microsoft.Network/virtualNetworks@2022-01-01' existing = { } resource pdzKv 'Microsoft.Network/privateDnsZones@2020-06-01' existing = { + scope: spokeResourceGroup name: 'privatelink.vaultcore.azure.net' } resource pdzCr 'Microsoft.Network/privateDnsZones@2020-06-01' existing = { + scope: spokeResourceGroup name: 'privatelink.azurecr.io' } resource pdzMc 'Microsoft.Network/privateDnsZones@2020-06-01' existing = { + scope: spokeResourceGroup name: 'privatelink.${location}.azmk8s.io' } From b3f417e45993174b3a4dd1a22e427956956fde7c Mon Sep 17 00:00:00 2001 From: Fernando Antivero Date: Fri, 14 Oct 2022 16:38:36 +0000 Subject: [PATCH 088/115] fix indentation --- cluster-stamp.bicep | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/cluster-stamp.bicep b/cluster-stamp.bicep index ecf8ee7a..c877f3a3 100644 --- a/cluster-stamp.bicep +++ b/cluster-stamp.bicep @@ -164,10 +164,10 @@ resource vnetSpoke 'Microsoft.Network/virtualNetworks@2022-01-01' existing = { name: 'snet-cluster-inscopenodepools' } - // spoke virtual network's subnet for cluster out-scope node pools - resource snetClusterOutScopeNodePools 'subnets' existing = { - name: 'snet-cluster-outofscopenodepools' - } + // spoke virtual network's subnet for cluster out-scope node pools + resource snetClusterOutScopeNodePools 'subnets' existing = { + name: 'snet-cluster-outofscopenodepools' + } } From c0f0d0e3c82de612752ff47ed855855e99219576 Mon Sep 17 00:00:00 2001 From: Fernando Antivero Date: Fri, 14 Oct 2022 16:39:03 +0000 Subject: [PATCH 089/115] Apply PR Feedback: bug fix - add partially missing code comments --- cluster-stamp.bicep | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cluster-stamp.bicep b/cluster-stamp.bicep index c877f3a3..88f868b8 100644 --- a/cluster-stamp.bicep +++ b/cluster-stamp.bicep @@ -220,7 +220,7 @@ resource monitoringMetricsPublisherRole 'Microsoft.Authorization/roleDefinitions /*** RESOURCES ***/ -@description('The control plane identity used by the cluster.') +@description('The control plane identity used by the cluster. Used for networking access (VNET joining and DNS updating)') resource miClusterControlPlane 'Microsoft.ManagedIdentity/userAssignedIdentities@2022-01-31-preview' = { name: 'mi-${clusterName}-controlplane' location: location From cc3369232c11c193021af4f780bd04f4d587fca3 Mon Sep 17 00:00:00 2001 From: Fernando Antivero Date: Fri, 14 Oct 2022 16:56:20 +0000 Subject: [PATCH 090/115] Apply PR Feedback: add existing a remove resourceId function call --- cluster-stamp.bicep | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/cluster-stamp.bicep b/cluster-stamp.bicep index 88f868b8..a8c5b868 100644 --- a/cluster-stamp.bicep +++ b/cluster-stamp.bicep @@ -186,6 +186,12 @@ resource pdzMc 'Microsoft.Network/privateDnsZones@2020-06-01' existing = { name: 'privatelink.${location}.azmk8s.io' } +@description('Used as primary entry point for workload. Expected to be assigned to an Azure Application Gateway.') +resource pipPrimaryCluster 'Microsoft.Network/publicIPAddresses@2022-05-01' existing = { + scope: spokeResourceGroup + name: 'pip-BU0001A0005-00' +} + /*** EXISTING SUBSCRIPTION RESOURCES ***/ @description('Built-in Azure RBAC role that must be applied to the kublet Managed Identity allowing it to further assign adding managed identities to the cluster\'s underlying VMSS.') @@ -568,7 +574,7 @@ resource agw 'Microsoft.Network/applicationGateways@2022-01-01' = { name: 'agw-frontend-ip-configuration' properties: { publicIPAddress: { - id: resourceId(subscription().subscriptionId, spokeResourceGroup.name, 'Microsoft.Network/publicIpAddresses', 'pip-BU0001A0005-00') + id: pipPrimaryCluster.id } } } From d03bb31aff1eacd8f2280c939ff00da149cddb4f Mon Sep 17 00:00:00 2001 From: Fernando Antivero Date: Fri, 14 Oct 2022 16:59:46 +0000 Subject: [PATCH 091/115] Apply PR Feedback: restore missing commented out code --- cluster-stamp.bicep | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/cluster-stamp.bicep b/cluster-stamp.bicep index a8c5b868..6f69b653 100644 --- a/cluster-stamp.bicep +++ b/cluster-stamp.bicep @@ -1357,6 +1357,11 @@ resource mc 'Microsoft.ContainerService/managedClusters@2021-05-01' = { upgradeSettings: { maxSurge: '33%' } + /* This can be used to prevent unexpected workloads from landing on system node pool. All add-ons support this taint., + "nodeTaints": [ + "CriticalAddonsOnly=true:NoSchedule" + ] + */ tags: { 'pci-scope': 'out-of-scope' 'Data classification': 'Confidential' From 1ee80e1c665df1a766fcd8aac0f0879866b39a6d Mon Sep 17 00:00:00 2001 From: Fernando Antivero Date: Fri, 14 Oct 2022 18:54:04 +0000 Subject: [PATCH 092/115] Apply PR Feeback: restore la- convention for log analytics resource type --- cluster-stamp.bicep | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cluster-stamp.bicep b/cluster-stamp.bicep index 6f69b653..1bfc4e3a 100644 --- a/cluster-stamp.bicep +++ b/cluster-stamp.bicep @@ -346,7 +346,7 @@ resource kvMiAppGatewayKeyVaultReader_roleAssignment 'Microsoft.Authorization/ro @description('The AKS cluster and related resources log analytics workspace.') resource law 'Microsoft.OperationalInsights/workspaces@2021-12-01-preview' = { - name: 'law-${clusterName}' + name: 'la-${clusterName}' location: location properties: { sku: { From 7f8872da61d245cc034d8ff310a257e5c2f181f8 Mon Sep 17 00:00:00 2001 From: Fernando Antivero Date: Fri, 14 Oct 2022 20:01:43 +0000 Subject: [PATCH 093/115] Apply PR Feedback: restore secret name to its preview value keeping the new prefix --- cluster-stamp.bicep | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cluster-stamp.bicep b/cluster-stamp.bicep index 1bfc4e3a..8a3b972d 100644 --- a/cluster-stamp.bicep +++ b/cluster-stamp.bicep @@ -315,7 +315,7 @@ resource kv 'Microsoft.KeyVault/vaults@2022-07-01' = { // The in-cluster TLS certificate to establish HTTPS connections between your regional load balancer and your ingress controller, enabling end-to-end TLS connections. resource kvsAppGwIngressInternalAksIngressTls 'secrets' = { - name: 'agw-ingress-incluster-aks-ingress-contoso-com-tls' + name: 'agw-ingress-internal-aks-ingress-contoso-com-tls' properties: { value: aksIngressControllerCertificate } From 300ae00bc8029a353683fa1061f337f31204bee7 Mon Sep 17 00:00:00 2001 From: Fernando Antivero Date: Fri, 14 Oct 2022 20:07:05 +0000 Subject: [PATCH 094/115] Apply PR Feeback: keep it acr instead of cr --- cluster-stamp.bicep | 28 ++++++++++++++-------------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/cluster-stamp.bicep b/cluster-stamp.bicep index 8a3b972d..db56778e 100644 --- a/cluster-stamp.bicep +++ b/cluster-stamp.bicep @@ -81,7 +81,7 @@ var kubernetesVersion = '1.23.3' var subRgUniqueString = uniqueString('aks', subscription().subscriptionId, resourceGroup().id) var clusterName = 'aks-${subRgUniqueString}' var jumpBoxDefaultAdminUserName = uniqueString(clusterName, resourceGroup().id) -var crName = 'craks${subRgUniqueString}' +var acrName = 'acraks${subRgUniqueString}' /*** EXISTING TENANT RESOURCES ***/ @@ -149,7 +149,7 @@ resource vnetSpoke 'Microsoft.Network/virtualNetworks@2022-01-01' existing = { name: 'snet-management-ops' } - // spoke virtual network's subnet for managment cr agent pools + // spoke virtual network's subnet for managment acr agent pools resource snetManagmentCrAgents 'subnets' existing = { name: 'snet-management-acragents' } @@ -862,8 +862,8 @@ resource vmssJumpboxes 'Microsoft.Compute/virtualMachineScaleSets@2020-12-01' = } @description('The private container registry for the aks regulated cluster.') -resource cr 'Microsoft.ContainerRegistry/registries@2022-02-01-preview' = { - name: crName +resource acr 'Microsoft.ContainerRegistry/registries@2022-02-01-preview' = { + name: acrName location: location sku: { name: 'Premium' @@ -915,7 +915,7 @@ resource cr 'Microsoft.ContainerRegistry/registries@2022-02-01-preview' = { } resource cr_diagnosticSettings 'Microsoft.Insights/diagnosticSettings@2021-05-01-preview' = { - scope: cr + scope: acr name: 'default' properties: { workspaceId: law.id @@ -941,7 +941,7 @@ resource cr_diagnosticSettings 'Microsoft.Insights/diagnosticSettings@2021-05-01 @description('The network interface in the spoke vnet that enables privately connecting the AKS cluster with Container Registry.') resource peCr 'Microsoft.Network/privateEndpoints@2022-05-01' = { - name: 'pe-${cr.name}' + name: 'pe-${acr.name}' location: location properties: { subnet: { @@ -951,7 +951,7 @@ resource peCr 'Microsoft.Network/privateEndpoints@2022-05-01' = { { name: 'to-${vnetSpoke.name}' properties: { - privateLinkServiceId: cr.id + privateLinkServiceId: acr.id groupIds: [ 'registry' ] @@ -960,11 +960,11 @@ resource peCr 'Microsoft.Network/privateEndpoints@2022-05-01' = { ] } dependsOn: [ - cr::grl + acr::grl ] resource pdzg 'privateDnsZoneGroups' = { - name: 'for-${cr.name}' + name: 'for-${acr.name}' properties: { privateDnsZoneConfigs: [ { @@ -980,7 +980,7 @@ resource peCr 'Microsoft.Network/privateEndpoints@2022-05-01' = { @description('The scheduled query that returns images being imported from repositories different than quarantine/') resource sqrNonQuarantineImportedImgesToCr 'Microsoft.Insights/scheduledQueryRules@2022-06-15' = { - name: 'Image Imported into ACR from ${cr.name} source other than approved Quarantine' + name: 'Image Imported into ACR from ${acr.name} source other than approved Quarantine' location: location properties: { description: 'The only images we want in live/ are those that came from this ACR instance, but from the quarantine/ repository.' @@ -1006,7 +1006,7 @@ resource sqrNonQuarantineImportedImgesToCr 'Microsoft.Insights/scheduledQueryRul enabled: true evaluationFrequency: 'PT10M' scopes: [ - cr.id + acr.id ] severity: 3 windowSize: 'PT10M' @@ -1271,7 +1271,7 @@ resource paEnforceImageSource 'Microsoft.Authorization/policyAssignments@2020-03 policyDefinitionId: pdEnforceImageSourceId parameters: { allowedContainerImagesRegex: { - value: '${crName}\\.azurecr\\.io\\/live\\/.+$|mcr\\.microsoft\\.com\\/oss\\/(openservicemesh\\/init:|envoyproxy\\/envoy:).+$' + value: '${acrName}\\.azurecr\\.io\\/live\\/.+$|mcr\\.microsoft\\.com\\/oss\\/(openservicemesh\\/init:|envoyproxy\\/envoy:).+$' } effect: { value: 'deny' @@ -1590,7 +1590,7 @@ resource mc_diagnosticSettings 'Microsoft.Insights/diagnosticSettings@2021-05-01 @description('Grant kubelet managed identity with container registry pull role permissions; this allows the AKS Cluster\'s kubelet managed identity to pull images from this container registry.') resource crMiKubeletContainerRegistryPullRole_roleAssignment 'Microsoft.Authorization/roleAssignments@2022-04-01' = { - scope: cr + scope: acr name: guid(resourceGroup().id, mc.id, containerRegistryPullRole.id) properties: { roleDefinitionId: containerRegistryPullRole.id @@ -2237,4 +2237,4 @@ resource maRestartingContainerCountCI7 'Microsoft.Insights/metricAlerts@2018-03- output agwName string = agw.name output keyVaultName string = kv.name -output quarantineContainerRegistryName string = cr.name +output quarantineContainerRegistryName string = acr.name From 780d3736bd7dd602309cdd49c91d46ac16393083 Mon Sep 17 00:00:00 2001 From: Fernando Antivero Date: Fri, 14 Oct 2022 17:10:27 -0300 Subject: [PATCH 095/115] Apply PR Feedback: use resourceGroup func Co-authored-by: Chad Kittel --- cluster-stamp.bicep | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cluster-stamp.bicep b/cluster-stamp.bicep index db56778e..1c87c8f4 100644 --- a/cluster-stamp.bicep +++ b/cluster-stamp.bicep @@ -1022,7 +1022,7 @@ resource paAksLinuxRestrictive 'Microsoft.Authorization/policyAssignments@2021-0 name: guid(psdAKSLinuxRestrictiveId, resourceGroup().name, clusterName) properties: { displayName: trim(take('[${clusterName}] ${reference(psdAKSLinuxRestrictiveId, '2020-09-01').displayName}', 125)) - scope: subscriptionResourceId('Microsoft.Resources/resourceGroups', resourceGroup().name) + scope: resourceGroup() policyDefinitionId: psdAKSLinuxRestrictiveId parameters: { effect: { From cdf8658e6b8614ca6e7693dae2506cb9f27af8d5 Mon Sep 17 00:00:00 2001 From: Fernando Antivero Date: Fri, 14 Oct 2022 20:15:36 +0000 Subject: [PATCH 096/115] propagate resource name to instance object name to keep it consistent --- cluster-stamp.bicep | 56 ++++++++++++++++++++++----------------------- 1 file changed, 28 insertions(+), 28 deletions(-) diff --git a/cluster-stamp.bicep b/cluster-stamp.bicep index 1c87c8f4..79316769 100644 --- a/cluster-stamp.bicep +++ b/cluster-stamp.bicep @@ -345,7 +345,7 @@ resource kvMiAppGatewayKeyVaultReader_roleAssignment 'Microsoft.Authorization/ro } @description('The AKS cluster and related resources log analytics workspace.') -resource law 'Microsoft.OperationalInsights/workspaces@2021-12-01-preview' = { +resource la 'Microsoft.OperationalInsights/workspaces@2021-12-01-preview' = { name: 'la-${clusterName}' location: location properties: { @@ -359,7 +359,7 @@ resource law 'Microsoft.OperationalInsights/workspaces@2021-12-01-preview' = { } resource lawAllPrometheus 'Microsoft.OperationalInsights/workspaces/savedSearches@2020-08-01' = { - parent: law + parent: la name: 'AllPrometheus' properties: { eTag: '*' @@ -371,7 +371,7 @@ resource lawAllPrometheus 'Microsoft.OperationalInsights/workspaces/savedSearche } resource lawForbiddenReponsesOnIngress 'Microsoft.OperationalInsights/workspaces/savedSearches@2020-08-01' = { - parent: law + parent: la name: 'ForbiddenReponsesOnIngress' properties: { eTag: '*' @@ -383,7 +383,7 @@ resource lawForbiddenReponsesOnIngress 'Microsoft.OperationalInsights/workspaces } resource lawNodeRebootRequested 'Microsoft.OperationalInsights/workspaces/savedSearches@2020-08-01' = { - parent: law + parent: la name: 'NodeRebootRequested' properties: { eTag: '*' @@ -395,13 +395,13 @@ resource lawNodeRebootRequested 'Microsoft.OperationalInsights/workspaces/savedS } resource omsContainerInsights 'Microsoft.OperationsManagement/solutions@2015-11-01-preview' = { - name: 'ContainerInsights(${law.name})' + name: 'ContainerInsights(${la.name})' location: location properties: { - workspaceResourceId: law.id + workspaceResourceId: la.id } plan: { - name: 'ContainerInsights(${law.name})' + name: 'ContainerInsights(${la.name})' product: 'OMSGallery/ContainerInsights' promotionCode: '' publisher: 'Microsoft' @@ -409,13 +409,13 @@ resource omsContainerInsights 'Microsoft.OperationsManagement/solutions@2015-11- } resource omsVmInsights 'Microsoft.OperationsManagement/solutions@2015-11-01-preview' = { - name: 'VMInsights(${law.name})' + name: 'VMInsights(${la.name})' location: location properties: { - workspaceResourceId: law.id + workspaceResourceId: la.id } plan: { - name: 'VMInsights(${law.name})' + name: 'VMInsights(${la.name})' product: 'OMSGallery/VMInsights' promotionCode: '' publisher: 'Microsoft' @@ -423,13 +423,13 @@ resource omsVmInsights 'Microsoft.OperationsManagement/solutions@2015-11-01-prev } resource omsSecurityInsights 'Microsoft.OperationsManagement/solutions@2015-11-01-preview' = { - name: 'SecurityInsights(${law.name})' + name: 'SecurityInsights(${la.name})' location: location properties: { - workspaceResourceId: law.id + workspaceResourceId: la.id } plan: { - name: 'SecurityInsights(${law.name})' + name: 'SecurityInsights(${la.name})' product: 'OMSGallery/SecurityInsights' promotionCode: '' publisher: 'Microsoft' @@ -440,15 +440,15 @@ resource miwSecurityInsights 'Microsoft.Insights/workbooks@2022-04-01' = { name: guid(omsSecurityInsights.name) location: location tags: { - 'hidden-title': 'Azure Kubernetes Service (AKS) Security - ${law.name}' + 'hidden-title': 'Azure Kubernetes Service (AKS) Security - ${la.name}' } kind: 'shared' properties: { - displayName: 'Azure Kubernetes Service (AKS) Security - ${law.name}' + displayName: 'Azure Kubernetes Service (AKS) Security - ${la.name}' serializedData: '{"version":"Notebook/1.0","items":[{"type":1,"content":{"json":"## AKS Security\\n"},"name":"text - 2"},{"type":9,"content":{"version":"KqlParameterItem/1.0","crossComponentResources":["{workspaces}"],"parameters":[{"id":"311d3728-7f8a-4b16-8a34-097d099323d5","version":"KqlParameterItem/1.0","name":"subscription","label":"Subscription","type":6,"isRequired":true,"multiSelect":true,"quote":"\'","delimiter":",","value":[],"typeSettings":{"additionalResourceOptions":[],"includeAll":false,"showDefault":false}},{"id":"3a56d260-4fb9-46d6-b121-cea854104c91","version":"KqlParameterItem/1.0","name":"workspaces","label":"Workspaces","type":5,"isRequired":true,"multiSelect":true,"quote":"\'","delimiter":",","query":"where type =~ \'microsoft.operationalinsights/workspaces\'\\r\\n| where strcat(\'/subscriptions/\',subscriptionId) in ({subscription})\\r\\n| project id","crossComponentResources":["{subscription}"],"typeSettings":{"additionalResourceOptions":["value::all"]},"queryType":1,"resourceType":"microsoft.resourcegraph/resources","value":["value::all"]},{"id":"9615cea6-c661-470a-b4ae-1aab8ae6f448","version":"KqlParameterItem/1.0","name":"clustername","label":"Cluster name","type":5,"isRequired":true,"multiSelect":true,"quote":"\'","delimiter":",","query":"where type == \\"microsoft.containerservice/managedclusters\\"\\r\\n| where strcat(\'/subscriptions/\',subscriptionId) in ({subscription})\\r\\n| distinct tolower(id)","crossComponentResources":["{subscription}"],"value":["value::all"],"typeSettings":{"resourceTypeFilter":{"microsoft.containerservice/managedclusters":true},"additionalResourceOptions":["value::all"],"showDefault":false},"timeContext":{"durationMs":86400000},"queryType":1,"resourceType":"microsoft.resourcegraph/resources"},{"id":"236c00ec-1493-4e60-927a-a18b8b120cd5","version":"KqlParameterItem/1.0","name":"timeframe","label":"Time range","type":4,"description":"Time","isRequired":true,"value":{"durationMs":172800000},"typeSettings":{"selectableValues":[{"durationMs":300000},{"durationMs":900000},{"durationMs":1800000},{"durationMs":3600000},{"durationMs":14400000},{"durationMs":43200000},{"durationMs":86400000},{"durationMs":172800000},{"durationMs":259200000},{"durationMs":604800000},{"durationMs":1209600000},{"durationMs":2419200000},{"durationMs":2592000000},{"durationMs":5184000000},{"durationMs":7776000000}],"allowCustom":true},"timeContext":{"durationMs":86400000}},{"id":"bf0a3e4f-fff9-450c-b9d3-c8c1dded9787","version":"KqlParameterItem/1.0","name":"nodeRgDetails","type":1,"query":"where type == \\"microsoft.containerservice/managedclusters\\"\\r\\n| where tolower(id) in ({clustername})\\r\\n| project nodeRG = properties.nodeResourceGroup, subscriptionId, id = toupper(id)\\r\\n| project nodeRgDetails = strcat(\'\\"\', nodeRG, \\";\\", subscriptionId, \\";\\", id, \'\\"\')","crossComponentResources":["value::all"],"isHiddenWhenLocked":true,"timeContext":{"durationMs":86400000},"queryType":1,"resourceType":"microsoft.resourcegraph/resources"},{"id":"df53126c-c40f-43d5-b99f-97ee3785c086","version":"KqlParameterItem/1.0","name":"diagnosticClusters","type":1,"query":"union withsource=_TableName *\\r\\n| where _TableName == \\"AzureDiagnostics\\" and Category == \\"kube-audit\\"\\r\\n| summarize diagnosticClusters = dcount(ResourceId)\\r\\n| project isDiagnosticCluster = iff(diagnosticClusters > 0, \\"yes\\", \\"no\\")","crossComponentResources":["{workspaces}"],"isHiddenWhenLocked":true,"timeContext":{"durationMs":172800000},"timeContextFromParameter":"timeframe","queryType":0,"resourceType":"microsoft.operationalinsights/workspaces"}],"style":"pills","queryType":0,"resourceType":"microsoft.operationalinsights/workspaces"},"name":"parameters - 3"},{"type":11,"content":{"version":"LinkItem/1.0","style":"tabs","links":[{"id":"07cf87dc-8234-47db-850d-ec41b2687b2a","cellValue":"mainTab","linkTarget":"parameter","linkLabel":"Microsoft Defender for Kubernetes","subTarget":"alerts","preText":"","style":"link"},{"id":"44033ee6-d83e-4253-a732-c258ef1da545","cellValue":"mainTab","linkTarget":"parameter","linkLabel":"Analytics over Diagnostic logs","subTarget":"diagnostics","style":"link"}]},"name":"links - 22"},{"type":12,"content":{"version":"NotebookGroup/1.0","groupType":"editable","items":[{"type":1,"content":{"json":"## Microsoft Defender for AKS coverage"},"name":"text - 10"},{"type":3,"content":{"version":"KqlItem/1.0","query":"datatable (Event:string)\\r\\n [\\"AKS Workbook\\"]\\r\\n| extend cluster = (strcat(\\"[\\", \\"{clustername}\\", \\"]\\"))\\r\\n| extend cluster = todynamic(replace(\\"\'\\", \'\\"\', cluster))\\r\\n| mvexpand cluster\\r\\n| extend subscriptionId = extract(@\\"/subscriptions/([^/]+)\\", 1, tostring(cluster))\\r\\n| summarize AksClusters = count() by subscriptionId, DefenderForAks = 0\\r\\n| union\\r\\n(\\r\\nsecurityresources\\r\\n| where type =~ \\"microsoft.security/pricings\\"\\r\\n| where name == \\"KubernetesService\\"\\r\\n| project DefenderForAks = iif(properties.pricingTier == \'Standard\', 1, 0), AksClusters = 0, subscriptionId\\r\\n)\\r\\n| summarize AksClusters = sum(AksClusters), DefenderForAks = sum(DefenderForAks) by subscriptionId\\r\\n| project Subscription = strcat(\'/subscriptions/\', subscriptionId), [\\"AKS clusters\\"] = AksClusters, [\'Defender for AKS\'] = iif(DefenderForAks > 0,\'yes\',\'no\'), [\'Onboard Microsoft Defender\'] = iif(DefenderForAks > 0, \'\', \'https://ms.portal.azure.com/#blade/Microsoft_Azure_Security/SecurityMenuBlade/26\')\\r\\n| order by [\'Defender for AKS\'] asc","size":0,"queryType":1,"resourceType":"microsoft.resourcegraph/resources","crossComponentResources":["{subscription}"],"gridSettings":{"formatters":[{"columnMatch":"Defender for AKS","formatter":18,"formatOptions":{"thresholdsOptions":"icons","thresholdsGrid":[{"operator":"==","thresholdValue":"no","representation":"4","text":""},{"operator":"Default","thresholdValue":null,"representation":"success","text":""}]}},{"columnMatch":"Onboard Microsoft Defender","formatter":7,"formatOptions":{"linkTarget":"Url","linkLabel":""}}]}},"customWidth":"66","name":"query - 9"},{"type":3,"content":{"version":"KqlItem/1.0","query":"datatable (Event:string)\\r\\n [\\"AKS Workbook\\"]\\r\\n| extend cluster = (strcat(\\"[\\", \\"{clustername}\\", \\"]\\"))\\r\\n| extend cluster = todynamic(replace(\\"\'\\", \'\\"\', cluster))\\r\\n| mvexpand cluster\\r\\n| extend subscriptionId = extract(@\\"/subscriptions/([^/]+)\\", 1, tostring(cluster))\\r\\n| summarize AksClusters = count() by subscriptionId, DefenderForAks = 0\\r\\n| union\\r\\n(\\r\\nsecurityresources\\r\\n| where type =~ \\"microsoft.security/pricings\\"\\r\\n| where name == \\"KubernetesService\\"\\r\\n| project DefenderForAks = iif(properties.pricingTier == \'Standard\', 1, 0), AksClusters = 0, subscriptionId\\r\\n)\\r\\n| summarize AksClusters = sum(AksClusters), DefenderForAks = sum(DefenderForAks) by subscriptionId\\r\\n| project Subscription = 1, [\'Defender for AKS\'] = iif(DefenderForAks > 0,\'Protected by Microsoft Defender\',\'Not protected by Microsoft Defender\')","size":0,"queryType":1,"resourceType":"microsoft.resourcegraph/resources","crossComponentResources":["{subscription}"],"visualization":"piechart"},"customWidth":"33","name":"query - 11"},{"type":1,"content":{"json":"### AKS alerts overview"},"name":"text - 21"},{"type":3,"content":{"version":"KqlItem/1.0","query":"SecurityAlert \\r\\n| where tolower(ResourceId) in ({clustername})\\r\\n| where TimeGenerated {timeframe}\\r\\n| where AlertType startswith \\"AKS_\\"\\r\\n| project image = tostring(todynamic(ExtendedProperties)[\\"Container image\\"]), AlertType\\r\\n| where image != \\"\\"\\r\\n| summarize AlertTypes = dcount(AlertType) by image\\r\\n| where AlertTypes > 1\\r\\n//| render piechart \\r\\n","size":4,"title":"Images with multiple alerts","queryType":0,"resourceType":"microsoft.operationalinsights/workspaces","crossComponentResources":["{workspaces}"],"visualization":"tiles","tileSettings":{"showBorder":false,"titleContent":{"columnMatch":"image","formatter":1},"leftContent":{"columnMatch":"AlertTypes","formatter":12,"formatOptions":{"palette":"auto"},"numberFormat":{"unit":17,"options":{"maximumSignificantDigits":3,"maximumFractionDigits":2}}}}},"customWidth":"33","name":"query - 12"},{"type":3,"content":{"version":"KqlItem/1.0","query":"SecurityAlert \\r\\n| where tolower(ResourceId) in ({clustername})\\r\\n| where TimeGenerated {timeframe}\\r\\n| where AlertType startswith \\"AKS_\\"\\r\\n| project AlertType, name = extract(@\\"/MICROSOFT.CONTAINERSERVICE/MANAGEDCLUSTERS/(.+)\\", 1, ResourceId)\\r\\n| summarize AlertTypes = dcount(AlertType) by name\\r\\n| where AlertTypes > 1\\r\\n","size":4,"title":"Clusters with multiple alert types","queryType":0,"resourceType":"microsoft.operationalinsights/workspaces","crossComponentResources":["{workspaces}"],"visualization":"tiles","tileSettings":{"showBorder":false,"titleContent":{"columnMatch":"name","formatter":1},"leftContent":{"columnMatch":"AlertTypes","formatter":12,"formatOptions":{"palette":"auto"},"numberFormat":{"unit":17,"options":{"maximumSignificantDigits":3,"maximumFractionDigits":2}}}}},"customWidth":"33","name":"query - 12 - Copy"},{"type":3,"content":{"version":"KqlItem/1.0","query":"SecurityAlert \\r\\n| where tolower(ResourceId) in ({clustername})\\r\\n| where TimeGenerated {timeframe}\\r\\n| where AlertType startswith \\"AKS_\\"\\r\\n| project AlertType, name = extract(@\\"/MICROSOFT.CONTAINERSERVICE/MANAGEDCLUSTERS/(.+)\\", 1, ResourceId)\\r\\n| summarize count() by name\\r\\n\\r\\n","size":4,"title":"Alerts triggered by cluster","queryType":0,"resourceType":"microsoft.operationalinsights/workspaces","crossComponentResources":["{workspaces}"],"visualization":"tiles","tileSettings":{"showBorder":false,"titleContent":{"columnMatch":"name","formatter":1},"leftContent":{"columnMatch":"count_","formatter":12,"formatOptions":{"palette":"auto"},"numberFormat":{"unit":17,"options":{"maximumSignificantDigits":3,"maximumFractionDigits":2}}}}},"customWidth":"33","name":"query - 12 - Copy - Copy"},{"type":1,"content":{"json":"### Seucirty alerts details\\r\\n\\r\\nTo filter, press on the severities below.\\r\\nYou can also filter based on a specific resource."},"name":"text - 18"},{"type":3,"content":{"version":"KqlItem/1.0","query":"SecurityAlert\\r\\n| where TimeGenerated {timeframe}\\r\\n| extend rg = extract(@\\"/resourcegroups/([^/]+)\\", 1, tolower(ResourceId))\\r\\n| extend sub = extract(@\\"/subscriptions/([^/]+)\\", 1, tolower(ResourceId))\\r\\n| extend nodesDetailsArr = todynamic(\'{nodeRgDetails}\')\\r\\n| mv-expand singleNodeDetails = nodesDetailsArr\\r\\n| extend singleNodeArr = split(singleNodeDetails, \\";\\")\\r\\n| where tolower(singleNodeArr[0]) == rg and tolower(singleNodeArr[1]) == sub\\r\\n| project AlertSeverity\\r\\n| union\\r\\n(\\r\\nSecurityAlert \\r\\n| where tolower(ResourceId) in ({clustername})\\r\\n| where TimeGenerated {timeframe}\\r\\n| where AlertType startswith \\"AKS_\\"\\r\\n| project AlertSeverity\\r\\n)\\r\\n| summarize count() by AlertSeverity","size":0,"title":"Alerts by severity","exportMultipleValues":true,"exportedParameters":[{"fieldName":"AlertSeverity","parameterName":"severity","parameterType":1,"quote":""}],"queryType":0,"resourceType":"microsoft.operationalinsights/workspaces","crossComponentResources":["{workspaces}"],"visualization":"tiles","tileSettings":{"showBorder":false,"titleContent":{"columnMatch":"AlertSeverity","formatter":1},"leftContent":{"columnMatch":"count_","formatter":12,"formatOptions":{"palette":"auto"},"numberFormat":{"unit":17,"options":{"maximumSignificantDigits":3,"maximumFractionDigits":2}}}}},"customWidth":"11","name":"Alerts by severity"},{"type":3,"content":{"version":"KqlItem/1.0","query":"SecurityAlert\\r\\n| where TimeGenerated {timeframe}\\r\\n| where \\"{severity}\\" has AlertSeverity or isempty(\\"{severity}\\")\\r\\n| extend rg = extract(@\\"/resourcegroups/([^/]+)\\", 1, tolower(ResourceId))\\r\\n| extend sub = extract(@\\"/subscriptions/([^/]+)\\", 1, tolower(ResourceId))\\r\\n| extend nodesDetailsArr = todynamic(\'{nodeRgDetails}\')\\r\\n| mv-expand singleNodeDetails = nodesDetailsArr\\r\\n| extend singleNodeArr = split(singleNodeDetails, \\";\\")\\r\\n| where tolower(singleNodeArr[0]) == rg and tolower(singleNodeArr[1]) == sub\\r\\n| project ResourceId\\r\\n| union\\r\\n(\\r\\nSecurityAlert \\r\\n| where tolower(ResourceId) in ({clustername})\\r\\n| where \\"{severity}\\" has AlertSeverity or isempty(\\"{severity}\\")\\r\\n| where TimeGenerated {timeframe}\\r\\n| where AlertType startswith \\"AKS_\\"\\r\\n| project ResourceId\\r\\n)\\r\\n| summarize Alerts = count() by ResourceId\\r\\n| order by Alerts desc\\r\\n| limit 10","size":0,"title":"Resources with most alerts","exportFieldName":"ResourceId","exportParameterName":"selectedResource","exportDefaultValue":"not_selected","queryType":0,"resourceType":"microsoft.operationalinsights/workspaces","crossComponentResources":["{workspaces}"],"gridSettings":{"formatters":[{"columnMatch":"Alerts","formatter":4,"formatOptions":{"palette":"red"}}]}},"customWidth":"22","name":"Resources with most alerts"},{"type":3,"content":{"version":"KqlItem/1.0","query":"\\r\\nSecurityAlert\\r\\n| where TimeGenerated {timeframe}\\r\\n| where \\"{severity}\\" has AlertSeverity or isempty(\\"{severity}\\")\\r\\n| extend rg = extract(@\\"/resourcegroups/([^/]+)\\", 1, tolower(ResourceId))\\r\\n| extend sub = extract(@\\"/subscriptions/([^/]+)\\", 1, tolower(ResourceId))\\r\\n| extend nodesDetailsArr = todynamic(\'{nodeRgDetails}\')\\r\\n| mv-expand singleNodeDetails = nodesDetailsArr\\r\\n| extend singleNodeArr = split(singleNodeDetails, \\";\\")\\r\\n| where tolower(singleNodeArr[0]) == rg and tolower(singleNodeArr[1]) == sub\\r\\n| extend AlertResourceType = \\"VM alerts\\"\\r\\n| union\\r\\n(\\r\\nSecurityAlert \\r\\n| where tolower(ResourceId) in ({clustername})\\r\\n| where \\"{severity}\\" has AlertSeverity or isempty(\\"{severity}\\")\\r\\n| where TimeGenerated {timeframe}\\r\\n| where AlertType startswith \\"AKS_\\"\\r\\n| extend AlertResourceType = \\"Cluster alerts\\"\\r\\n)\\r\\n| summarize Alerts = count() by bin(TimeGenerated, {timeframe:grain}), AlertResourceType","size":0,"title":"Alerts over time","queryType":0,"resourceType":"microsoft.operationalinsights/workspaces","crossComponentResources":["{workspaces}"],"visualization":"timechart"},"customWidth":"66","name":"Alerts over time"},{"type":3,"content":{"version":"KqlItem/1.0","query":"SecurityAlert\\r\\n| where TimeGenerated {timeframe}\\r\\n| where \\"{severity}\\" has AlertSeverity or isempty(\\"{severity}\\")\\r\\n| extend rg = extract(@\\"/resourcegroups/([^/]+)\\", 1, tolower(ResourceId))\\r\\n| extend sub = extract(@\\"/subscriptions/([^/]+)\\", 1, tolower(ResourceId))\\r\\n| extend nodesDetailsArr = todynamic(\'{nodeRgDetails}\')\\r\\n| mv-expand singleNodeDetails = nodesDetailsArr\\r\\n| extend singleNodeArr = split(singleNodeDetails, \\";\\")\\r\\n| where tolower(singleNodeArr[0]) == rg and tolower(singleNodeArr[1]) == sub\\r\\n| where tolower(ResourceId) == tolower(\\"{selectedResource}\\") or \\"{selectedResource}\\" == \\"not_selected\\"\\r\\n| project [\\"Resource name\\"] = ResourceId, TimeGenerated, AlertSeverity, [\\"AKS cluster\\"] = toupper(singleNodeArr[2]), DisplayName, AlertLink\\r\\n| union\\r\\n(\\r\\nSecurityAlert\\r\\n| where TimeGenerated {timeframe}\\r\\n| where tolower(ResourceId) in ({clustername})\\r\\n| where \\"{severity}\\" has AlertSeverity or isempty(\\"{severity}\\")\\r\\n| where AlertType startswith \\"AKS_\\"\\r\\n| where tolower(ResourceId) == tolower(\\"{selectedResource}\\") or \\"{selectedResource}\\" == \\"not_selected\\"\\r\\n| project [\\"Resource name\\"] = ResourceId, TimeGenerated, AlertSeverity, [\\"AKS cluster\\"] = ResourceId, DisplayName, AlertLink\\r\\n)\\r\\n| order by TimeGenerated asc","size":0,"title":"Microsoft Defender alerts","queryType":0,"resourceType":"microsoft.operationalinsights/workspaces","crossComponentResources":["{workspaces}"],"gridSettings":{"formatters":[{"columnMatch":"AlertLink","formatter":7,"formatOptions":{"linkTarget":"Url","linkLabel":"Go to alert "}}],"filter":true},"sortBy":[]},"name":"Microsoft Defender alerts","styleSettings":{"showBorder":true}}]},"conditionalVisibility":{"parameterName":"mainTab","comparison":"isEqualTo","value":"alerts"},"name":"Defender Alerts"},{"type":12,"content":{"version":"NotebookGroup/1.0","groupType":"editable","items":[{"type":1,"content":{"json":"## Diagnostic logs coverage"},"name":"text - 15"},{"type":3,"content":{"version":"KqlItem/1.0","query":"union withsource=_TableName *\\r\\n| where _TableName == \\"AzureDiagnostics\\" and Category == \\"kube-audit\\"\\r\\n| summarize count() by ResourceId = tolower(ResourceId)\\r\\n| summarize logsClusters = make_set(ResourceId)\\r\\n| extend selectedClusters = \\"[{clustername}]\\"\\r\\n| extend selectedClusters = replace(\\"\'\\", \'\\"\', selectedClusters)\\r\\n| extend selectedClusters = todynamic(selectedClusters)\\r\\n| mv-expand clusterId = selectedClusters\\r\\n| project clusterId = toupper(tostring(clusterId)), [\\"Diagnostic logs\\"] = (logsClusters has tostring(clusterId))\\r\\n| extend [\\"Diagnostic settings\\"] = iff([\\"Diagnostic logs\\"] == false, strcat(\\"https://ms.portal.azure.com/#@microsoft.onmicrosoft.com/resource\\", clusterId, \\"/diagnostics\\"), \\"\\")\\r\\n","size":0,"timeContext":{"durationMs":172800000},"timeContextFromParameter":"timeframe","queryType":0,"resourceType":"microsoft.operationalinsights/workspaces","crossComponentResources":["{workspaces}"],"gridSettings":{"formatters":[{"columnMatch":"Diagnostic logs","formatter":18,"formatOptions":{"thresholdsOptions":"icons","thresholdsGrid":[{"operator":"==","thresholdValue":"false","representation":"critical","text":""},{"operator":"Default","thresholdValue":null,"representation":"success","text":""}]}},{"columnMatch":"Diagnostic settings","formatter":7,"formatOptions":{"linkTarget":"Url"}}],"filter":true,"sortBy":[{"itemKey":"$gen_thresholds_Diagnostic logs_1","sortOrder":2}]},"sortBy":[{"itemKey":"$gen_thresholds_Diagnostic logs_1","sortOrder":2}]},"customWidth":"66","name":"query - 14"},{"type":3,"content":{"version":"KqlItem/1.0","query":"union withsource=_TableName *\\r\\n| where _TableName == \\"AzureDiagnostics\\" and Category == \\"kube-audit\\"\\r\\n| summarize count() by ResourceId = tolower(ResourceId)\\r\\n| summarize logsClusters = make_set(ResourceId)\\r\\n| extend selectedClusters = \\"[{clustername}]\\"\\r\\n| extend selectedClusters = replace(\\"\'\\", \'\\"\', selectedClusters)\\r\\n| extend selectedClusters = todynamic(selectedClusters)\\r\\n| mv-expand clusterId = selectedClusters\\r\\n| project clusterId = toupper(tostring(clusterId)), hasDiagnosticLogs = (logsClusters has tostring(clusterId))\\r\\n| summarize [\\"number of clusters\\"] = count() by hasDiagnosticLogs\\r\\n| extend hasDiagnosticLogs = iff(hasDiagnosticLogs == true, \\"Clusters with Diagnostic logs\\", \\"Clusters without Diagnostic logs\\")\\r\\n","size":0,"timeContext":{"durationMs":172800000},"timeContextFromParameter":"timeframe","queryType":0,"resourceType":"microsoft.operationalinsights/workspaces","crossComponentResources":["{workspaces}"],"visualization":"piechart"},"customWidth":"33","name":"query - 17"},{"type":12,"content":{"version":"NotebookGroup/1.0","groupType":"editable","items":[{"type":1,"content":{"json":"## Cluster operations"},"name":"text - 16"},{"type":11,"content":{"version":"LinkItem/1.0","style":"tabs","links":[{"id":"3f616701-fd4b-482c-aff1-a85414daa05c","cellValue":"dispalyedGraph","linkTarget":"parameter","linkLabel":"Masterclient operations","subTarget":"masterclient","preText":"","style":"link"},{"id":"e6fa55f1-7d57-4f5e-8e83-429740853731","cellValue":"dispalyedGraph","linkTarget":"parameter","linkLabel":"Pod creation operations","subTarget":"podCreation","style":"link"},{"id":"f4c46251-0090-4ca3-a81c-0686bff3ff35","cellValue":"dispalyedGraph","linkTarget":"parameter","linkLabel":"Secret get\\\\list operations","subTarget":"secretOperation","style":"link"}]},"name":"links - 11"},{"type":3,"content":{"version":"KqlItem/1.0","query":"AzureDiagnostics \\r\\n| where tolower(ResourceId) in ({clustername})\\r\\n| where TimeGenerated {timeframe}\\r\\n| where Category == \\"kube-audit\\"\\r\\n| where log_s has \\"masterclient\\"\\r\\n| project TimeGenerated, parse_json(log_s), ResourceId\\r\\n| project TimeGenerated, ResourceId, username = tostring(log_s[\\"user\\"].username)\\r\\n| where username == \\"masterclient\\"\\r\\n| extend name = extract(@\\"/MICROSOFT.CONTAINERSERVICE/MANAGEDCLUSTERS/(.+)\\", 1, ResourceId)\\r\\n| summarize count() by name, bin(TimeGenerated, 1h)","size":0,"queryType":0,"resourceType":"microsoft.operationalinsights/workspaces","crossComponentResources":["{workspaces}"],"visualization":"timechart","tileSettings":{"showBorder":false,"titleContent":{"columnMatch":"name","formatter":1},"leftContent":{"columnMatch":"count_","formatter":12,"formatOptions":{"palette":"auto"},"numberFormat":{"unit":17,"options":{"maximumSignificantDigits":3,"maximumFractionDigits":2}}}},"chartSettings":{"yAxis":["count_"]}},"conditionalVisibility":{"parameterName":"dispalyedGraph","comparison":"isEqualTo","value":"masterclient"},"name":"Masterclient operations","styleSettings":{"showBorder":true}},{"type":3,"content":{"version":"KqlItem/1.0","query":"AzureDiagnostics\\r\\n| where Category == \\"kube-audit\\"\\r\\n| where tolower(ResourceId) in ({clustername})\\r\\n| where TimeGenerated {timeframe}\\r\\n| where log_s has \\"pods\\"\\r\\n| project TimeGenerated, parse_json(log_s), ResourceId\\r\\n| project AzureResourceId = ResourceId, TimeGenerated,\\r\\n RequestURI = tostring(log_s[\\"requestURI\\"]),\\r\\n Verb = tostring(log_s[\\"verb\\"]),\\r\\n ObjectRef = log_s[\\"objectRef\\"],\\r\\n ResponseStatus = log_s[\\"responseStatus\\"]\\r\\n//Main query\\r\\n| where ObjectRef.resource == \\"pods\\" and Verb == \\"create\\" and ResponseStatus.code startswith \\"20\\"\\r\\n and RequestURI endswith \\"/pods\\"\\r\\n| extend name = extract(@\\"/MICROSOFT.CONTAINERSERVICE/MANAGEDCLUSTERS/(.+)\\", 1, AzureResourceId)\\r\\n| summarize count() by name, bin(TimeGenerated, 1h)","size":0,"queryType":0,"resourceType":"microsoft.operationalinsights/workspaces","crossComponentResources":["{workspaces}"],"visualization":"timechart"},"conditionalVisibility":{"parameterName":"dispalyedGraph","comparison":"isEqualTo","value":"podCreation"},"name":"pods creation","styleSettings":{"showBorder":true}},{"type":3,"content":{"version":"KqlItem/1.0","query":"AzureDiagnostics\\r\\n| where Category == \\"kube-audit\\"\\r\\n| where tolower(ResourceId) in ({clustername})\\r\\n| where TimeGenerated {timeframe}\\r\\n| where log_s has \\"secrets\\"\\r\\n| project TimeGenerated, parse_json(log_s), ResourceId\\r\\n| project AzureResourceId = ResourceId, TimeGenerated,\\r\\n RequestURI = tostring(log_s[\\"requestURI\\"]),\\r\\n Verb = tostring(log_s[\\"verb\\"]),\\r\\n ObjectRef = log_s[\\"objectRef\\"],\\r\\n ResponseStatus = log_s[\\"responseStatus\\"]\\r\\n//Main query\\r\\n| where ObjectRef.resource == \\"secrets\\" and (Verb == \\"list\\" or Verb == \\"get\\") and ResponseStatus.code startswith \\"20\\"\\r\\n| where ObjectRef.name != \\"tunnelfront\\" and ObjectRef.name != \\"tunnelend\\" and ObjectRef.name != \\"kubernetes-dashboard-key-holder\\"\\r\\n| extend name = extract(@\\"/MICROSOFT.CONTAINERSERVICE/MANAGEDCLUSTERS/(.+)\\", 1, AzureResourceId)\\r\\n| summarize count() by name, bin(TimeGenerated, 1h)","size":0,"timeContext":{"durationMs":172800000},"timeContextFromParameter":"timeframe","queryType":0,"resourceType":"microsoft.operationalinsights/workspaces","crossComponentResources":["{workspaces}"],"visualization":"timechart","gridSettings":{"sortBy":[{"itemKey":"count_","sortOrder":2}]},"sortBy":[{"itemKey":"count_","sortOrder":2}]},"conditionalVisibility":{"parameterName":"dispalyedGraph","comparison":"isEqualTo","value":"secretOperation"},"name":"secrets operation","styleSettings":{"showBorder":true}},{"type":3,"content":{"version":"KqlItem/1.0","query":"let ascAlerts = \\nunion withsource=_TableName *\\n| where _TableName == \\"SecurityAlert\\"\\n| where tolower(ResourceId) in ({clustername})\\n| where TimeGenerated {timeframe}\\n| extend AlertType = column_ifexists(\\"AlertType\\", \\"\\")\\n| where AlertType == \\"AKS_PrivilegedContainer\\"\\n| extend ExtendedProperties = column_ifexists(\\"ExtendedProperties\\", todynamic(\\"\\"))\\n| extend ExtendedProperties = parse_json(ExtendedProperties)\\n| extend AlertLink = column_ifexists(\\"AlertLink\\", \\"\\")\\n| summarize arg_min(TimeGenerated, AlertLink) by AzureResourceId = ResourceId, name = tostring(ExtendedProperties[\\"Pod name\\"]), podNamespace = tostring(ExtendedProperties[\\"Namespace\\"])\\n;\\nlet podOperations = AzureDiagnostics\\n| where Category == \\"kube-audit\\"\\n| where tolower(ResourceId) in ({clustername})\\n| where TimeGenerated {timeframe}\\n| where log_s has \\"privileged\\"\\n| project TimeGenerated, parse_json(log_s), ResourceId\\n| project AzureResourceId = ResourceId, TimeGenerated,\\n RequestURI = tostring(log_s[\\"requestURI\\"]),\\n Verb = tostring(log_s[\\"verb\\"]),\\n ObjectRef = log_s[\\"objectRef\\"],\\n RequestObject = log_s[\\"requestObject\\"],\\n ResponseStatus = log_s[\\"responseStatus\\"],\\n ResponseObject = log_s[\\"responseObject\\"]\\n//Main query\\n| where ObjectRef.resource == \\"pods\\" and Verb == \\"create\\" and ResponseStatus.code startswith \\"20\\" and RequestObject has \\"privileged\\"\\n and RequestURI endswith \\"/pods\\"\\n| extend containers = RequestObject.spec.containers\\n| mvexpand containers\\n| where containers.securityContext.privileged == true\\n| summarize TimeGenerated = min(TimeGenerated) by\\n name = tostring(ResponseObject.metadata.name),\\n podNamespace = tostring(ResponseObject.metadata.namespace),\\n imageName = tostring(containers.image),\\n containerName = tostring(containers.name),\\n AzureResourceId\\n| extend id = strcat(name,\\";\\", AzureResourceId)\\n| extend parent = AzureResourceId\\n| join kind=leftouter (ascAlerts) on AzureResourceId, name, podNamespace\\n;\\nlet cached = materialize(podOperations)\\n;\\nlet clusters = cached | distinct AzureResourceId\\n;\\n// Main query\\ncached\\n| union\\n(\\nclusters\\n| project \\n name = AzureResourceId,\\n id = AzureResourceId,\\n parent = \\"\\" \\n)\\n| project-away name1, podNamespace1, TimeGenerated1","size":1,"title":"Privileged containers creation","queryType":0,"resourceType":"microsoft.operationalinsights/workspaces","crossComponentResources":["{workspaces}"],"visualization":"table","gridSettings":{"formatters":[{"columnMatch":"name","formatter":13,"formatOptions":{"linkTarget":null,"showIcon":true}},{"columnMatch":"AzureResourceId","formatter":5},{"columnMatch":"id","formatter":5},{"columnMatch":"parent","formatter":5},{"columnMatch":"AlertLink","formatter":7,"formatOptions":{"linkTarget":"Url","linkLabel":""}}],"hierarchySettings":{"idColumn":"id","parentColumn":"parent","treeType":0,"expanderColumn":"name"}},"sortBy":[]},"customWidth":"66","name":"Privileged container","styleSettings":{"showBorder":true}},{"type":3,"content":{"version":"KqlItem/1.0","query":"SecurityAlert \\r\\n| where tolower(ResourceId) in ({clustername})\\r\\n| where TimeGenerated {timeframe}\\r\\n| where AlertType == \\"AKS_PrivilegedContainer\\"\\r\\n| project name = extract(@\\"/MICROSOFT.CONTAINERSERVICE/MANAGEDCLUSTERS/(.+)\\", 1, ResourceId)\\r\\n| summarize alert = count() by name","size":1,"title":"AKS clusters with related Microsoft Defender alerts","queryType":0,"resourceType":"microsoft.operationalinsights/workspaces","crossComponentResources":["{workspaces}"],"visualization":"piechart","tileSettings":{"showBorder":false,"titleContent":{"columnMatch":"name","formatter":1},"leftContent":{"columnMatch":"alert","formatter":12,"formatOptions":{"palette":"auto"},"numberFormat":{"unit":17,"options":{"maximumSignificantDigits":3,"maximumFractionDigits":2}}}},"graphSettings":{"type":0,"topContent":{"columnMatch":"name","formatter":1},"centerContent":{"columnMatch":"alert","formatter":1,"numberFormat":{"unit":17,"options":{"maximumSignificantDigits":3,"maximumFractionDigits":2}}}}},"customWidth":"33","name":"query - 7","styleSettings":{"showBorder":true}},{"type":3,"content":{"version":"KqlItem/1.0","query":"let baseQuery = AzureDiagnostics \\r\\n| where Category == \\"kube-audit\\"\\r\\n| where tolower(ResourceId) in ({clustername})\\r\\n| where TimeGenerated {timeframe}\\r\\n| where log_s has \\"exec\\"\\r\\n| project TimeGenerated, parse_json(log_s), ResourceId\\r\\n| project TimeGenerated,\\r\\n AzureResourceId = ResourceId,\\r\\n User = log_s[\\"user\\"],\\r\\n StageTimestamp = todatetime(log_s[\\"stageTimestamp\\"]),\\r\\n Timestamp = todatetime(log_s[\\"timestamp\\"]),\\r\\n Stage = tostring(log_s[\\"stage\\"]),\\r\\n RequestURI = tostring(log_s[\\"requestURI\\"]),\\r\\n UserAgent = tostring(log_s[\\"userAgent\\"]),\\r\\n Verb = tostring(log_s[\\"verb\\"]),\\r\\n ObjectRef = log_s[\\"objectRef\\"],\\r\\n ResponseStatus = log_s[\\"responseStatus\\"]\\r\\n| where ObjectRef.resource == \\"pods\\" and Verb == \\"create\\" and ResponseStatus.code == 101 and ObjectRef.subresource == \\"exec\\"\\r\\n| project operationTime = TimeGenerated,\\r\\n RequestURI,\\r\\n podName = tostring(ObjectRef.name),\\r\\n podNamespace = tostring(ObjectRef.namespace),\\r\\n username = tostring(User.username),\\r\\n AzureResourceId\\r\\n// Parse the exec command\\r\\n| extend commands = extractall(@\\"command=([^\\\\&]*)\\", RequestURI)\\r\\n| extend commandsStr = url_decode(strcat_array(commands, \\" \\"))\\r\\n| project-away [\'commands\'], RequestURI\\r\\n| where username != \\"aksProblemDetector\\"\\r\\n;\\r\\nlet cached = materialize(baseQuery);\\r\\nlet execOperations = \\r\\ncached\\r\\n| summarize operationTime = min(operationTime), numberOfPerations = count() by name = commandsStr, username, podNamespace, podName, AzureResourceId\\r\\n| extend id = name\\r\\n| extend parentId = podName\\r\\n| project id, parentId, name, operationTime, numberOfPerations, podNamespace, username, AzureResourceId\\r\\n;\\r\\nlet podOperations = \\r\\ncached\\r\\n| summarize operationTime = min(operationTime), numberOfPerations = count() by name = podName, podNamespace, AzureResourceId\\r\\n| extend id = name\\r\\n| extend parentId = AzureResourceId\\r\\n| project id, parentId, name, operationTime, numberOfPerations, podNamespace, username = \\"\\", AzureResourceId\\r\\n;\\r\\nlet clusterOperations = \\r\\ncached\\r\\n| summarize operationTime = min(operationTime), numberOfPerations = count() by name = AzureResourceId\\r\\n| extend id = name\\r\\n| extend parentId = \\"\\"\\r\\n| project id, parentId, name, operationTime, numberOfPerations, username = \\"\\", podNamespace = \\"\\", AzureResourceId = name\\r\\n;\\r\\nunion clusterOperations, podOperations, execOperations","size":1,"title":"exec commands","queryType":0,"resourceType":"microsoft.operationalinsights/workspaces","crossComponentResources":["{workspaces}"],"visualization":"table","gridSettings":{"formatters":[{"columnMatch":"id","formatter":5},{"columnMatch":"parentId","formatter":5},{"columnMatch":"numberOfPerations","formatter":4,"formatOptions":{"palette":"blue","compositeBarSettings":{"labelText":"","columnSettings":[]}}},{"columnMatch":"AzureResourceId","formatter":5}],"hierarchySettings":{"idColumn":"id","parentColumn":"parentId","treeType":0,"expanderColumn":"name","expandTopLevel":false}}},"customWidth":"33","name":"exec commands","styleSettings":{"showBorder":true}},{"type":3,"content":{"version":"KqlItem/1.0","query":"SecurityAlert \\r\\n| where AlertType == \\"AKS_MaliciousContainerExec\\"\\r\\n| where tolower(ResourceId) in ({clustername})\\r\\n| where TimeGenerated {timeframe}\\r\\n| project TimeGenerated, ResourceId, ExtendedProperties = todynamic(ExtendedProperties)\\r\\n| project TimeGenerated, ResourceId, [\\"Pod name\\"] = ExtendedProperties[\\"Pod name\\"], Command = ExtendedProperties[\\"Command\\"]","size":1,"title":"Related Microsoft Defender alerts details","queryType":0,"resourceType":"microsoft.operationalinsights/workspaces","crossComponentResources":["{workspaces}"],"gridSettings":{"sortBy":[{"itemKey":"TimeGenerated","sortOrder":1}]},"sortBy":[{"itemKey":"TimeGenerated","sortOrder":1}]},"customWidth":"33","name":"query - 9","styleSettings":{"showBorder":true}},{"type":3,"content":{"version":"KqlItem/1.0","query":"SecurityAlert \\r\\n| where AlertType == \\"AKS_MaliciousContainerExec\\"\\r\\n| where tolower(ResourceId) in ({clustername})\\r\\n| where TimeGenerated {timeframe}\\r\\n| project name = extract(@\\"/MICROSOFT.CONTAINERSERVICE/MANAGEDCLUSTERS/(.+)\\", 1, ResourceId)\\r\\n| summarize alert = count() by name","size":1,"title":"AKS clusters with related Microsoft Defender alerts","queryType":0,"resourceType":"microsoft.operationalinsights/workspaces","crossComponentResources":["{workspaces}"],"visualization":"piechart"},"customWidth":"33","name":"query - 8","styleSettings":{"showBorder":true}},{"type":3,"content":{"version":"KqlItem/1.0","query":"let ascAlerts = \\r\\nunion withsource=_TableName *\\r\\n| where _TableName == \\"SecurityAlert\\"\\r\\n| where tolower(ResourceId) in ({clustername})\\r\\n| where TimeGenerated {timeframe}\\r\\n| extend AlertType = column_ifexists(\\"AlertType\\", \\"\\")\\r\\n| where AlertType == \\"AKS_SensitiveMount\\"\\r\\n| extend ExtendedProperties = column_ifexists(\\"ExtendedProperties\\", todynamic(\\"\\"))\\r\\n| extend ExtendedProperties = parse_json(ExtendedProperties)\\r\\n| extend AlertLink = column_ifexists(\\"AlertLink\\", \\"\\")\\r\\n| summarize arg_min(TimeGenerated, AlertLink) by AzureResourceId = ResourceId, containerName = tostring(ExtendedProperties[\\"Container name\\"]), mountPath = tostring(ExtendedProperties[\\"Sensitive mount path\\"])\\r\\n;\\r\\nlet podOperations = \\r\\nAzureDiagnostics \\r\\n| where tolower(ResourceId) in ({clustername})\\r\\n| where TimeGenerated {timeframe}\\r\\n| where Category == \\"kube-audit\\"\\r\\n| where log_s has \\"hostPath\\"\\r\\n| project TimeGenerated, parse_json(log_s), ResourceId\\r\\n//Parsing\\r\\n| project AzureResourceId = ResourceId, TimeGenerated,\\r\\n Verb = tostring(log_s[\\"verb\\"]),\\r\\n ObjectRef = log_s[\\"objectRef\\"],\\r\\n RequestObject = log_s[\\"requestObject\\"],\\r\\n ResponseStatus = log_s[\\"responseStatus\\"],\\r\\n ResponseObject = log_s[\\"responseObject\\"]\\r\\n//\\r\\n//Main query\\r\\n//\\r\\n| where ObjectRef.resource == \\"pods\\" and Verb == \\"create\\" and ResponseStatus.code startswith \\"20\\" and RequestObject has \\"hostPath\\"\\r\\n| extend volumes = RequestObject.spec.volumes\\r\\n| mvexpand volumes\\r\\n| extend mountPath = volumes.hostPath.path\\r\\n| where mountPath != \\"\\" \\r\\n| extend container = RequestObject.spec.containers\\r\\n| mvexpand container\\r\\n| extend detectionTime = TimeGenerated\\r\\n| project detectionTime,\\r\\n podName = ResponseObject.metadata.name,\\r\\n podNamespace = ResponseObject.metadata.namespace,\\r\\n containerName = container.name,\\r\\n containerImage = container.image,\\r\\n mountPath,\\r\\n mountName = volumes.name,\\r\\n AzureResourceId,\\r\\n container\\r\\n| extend volumeMounts = container.volumeMounts\\r\\n| mv-expand volumeMounts\\r\\n| where tostring(volumeMounts.name) == tostring(mountName)\\r\\n| summarize operationTime = min(detectionTime) by AzureResourceId, name = tostring(podName),tostring(podNamespace), tostring(containerName), tostring(containerImage), tostring(mountPath), tostring(mountName)\\r\\n| extend id = strcat(name, \\";\\", AzureResourceId)\\r\\n| extend parent = AzureResourceId\\r\\n| join kind=leftouter (ascAlerts) on AzureResourceId, containerName, mountPath\\r\\n;\\r\\nlet cached = materialize(podOperations)\\r\\n;\\r\\nlet clusters = cached | distinct AzureResourceId\\r\\n;\\r\\n// Main query\\r\\ncached\\r\\n| union\\r\\n(\\r\\nclusters\\r\\n| project \\r\\n name = toupper(AzureResourceId),\\r\\n id = AzureResourceId,\\r\\n parent = \\"\\" \\r\\n)\\r\\n| project-away containerName1, mountPath1, TimeGenerated\\r\\n","size":1,"title":"hostPath mount","queryType":0,"resourceType":"microsoft.operationalinsights/workspaces","crossComponentResources":["{workspaces}"],"gridSettings":{"formatters":[{"columnMatch":"AzureResourceId","formatter":5},{"columnMatch":"name","formatter":13,"formatOptions":{"linkTarget":null,"showIcon":true}},{"columnMatch":"id","formatter":5},{"columnMatch":"parent","formatter":5},{"columnMatch":"AzureResourceId1","formatter":5},{"columnMatch":"AlertLink","formatter":7,"formatOptions":{"linkTarget":"Url"}}],"hierarchySettings":{"idColumn":"id","parentColumn":"parent","treeType":0,"expanderColumn":"name"}},"sortBy":[]},"customWidth":"66","name":"query - 10","styleSettings":{"showBorder":true}},{"type":3,"content":{"version":"KqlItem/1.0","query":"SecurityAlert \\r\\n| where AlertType == \\"AKS_SensitiveMount\\"\\r\\n| where tolower(ResourceId) in ({clustername})\\r\\n| where TimeGenerated {timeframe}\\r\\n| project name = extract(@\\"/MICROSOFT.CONTAINERSERVICE/MANAGEDCLUSTERS/(.+)\\", 1, ResourceId)\\r\\n| summarize alert = count() by name","size":1,"title":"AKS clusters with related Microsoft Defender alerts","queryType":0,"resourceType":"microsoft.operationalinsights/workspaces","crossComponentResources":["{workspaces}"],"visualization":"piechart","sortBy":[]},"customWidth":"33","name":"query - 10","styleSettings":{"showBorder":true}},{"type":3,"content":{"version":"KqlItem/1.0","query":"let bindingOper = AzureDiagnostics\\r\\n| where Category == \\"kube-audit\\"\\r\\n| where tolower(ResourceId) in ({clustername})\\r\\n| where TimeGenerated {timeframe}\\r\\n| where log_s has \\"clusterrolebindings\\"\\r\\n| project TimeGenerated, parse_json(log_s), ResourceId\\r\\n//Parsing\\r\\n| project AzureResourceId = ResourceId, TimeGenerated,\\r\\n RequestURI = tostring(log_s[\\"requestURI\\"]),\\r\\n User = log_s[\\"user\\"],\\r\\n Verb = tostring(log_s[\\"verb\\"]),\\r\\n ObjectRef = log_s[\\"objectRef\\"],\\r\\n RequestObject = log_s[\\"requestObject\\"],\\r\\n ResponseStatus = log_s[\\"responseStatus\\"]\\r\\n| where ObjectRef.resource == \\"clusterrolebindings\\" and Verb == \\"create\\" and ResponseStatus.code startswith \\"20\\" and RequestObject.roleRef.name == \\"cluster-admin\\" \\r\\n| extend subjects = RequestObject.subjects\\r\\n| mv-expand subjects\\r\\n| project AzureResourceId, TimeGenerated, subjectName = tostring(subjects.name), subjectKind = tostring(subjects[\\"kind\\"]), bindingName = tostring(ObjectRef.name)\\r\\n| summarize operationTime = min(TimeGenerated) by AzureResourceId, subjectName, subjectKind, bindingName\\r\\n| extend id = strcat(subjectName, \\";\\", AzureResourceId)\\r\\n| extend parent = AzureResourceId\\r\\n;\\r\\nlet cached = materialize(bindingOper)\\r\\n;\\r\\nlet clusters = cached | distinct AzureResourceId\\r\\n;\\r\\n// Main query\\r\\ncached\\r\\n| union\\r\\n(\\r\\nclusters\\r\\n| project \\r\\n name = AzureResourceId,\\r\\n id = AzureResourceId,\\r\\n parent = \\"\\" \\r\\n)","size":1,"title":"Cluster-admin binding","queryType":0,"resourceType":"microsoft.operationalinsights/workspaces","crossComponentResources":["{workspaces}"],"visualization":"table","gridSettings":{"formatters":[{"columnMatch":"AzureResourceId","formatter":5},{"columnMatch":"id","formatter":5},{"columnMatch":"parent","formatter":5},{"columnMatch":"name","formatter":13,"formatOptions":{"linkTarget":null,"showIcon":true}}],"hierarchySettings":{"idColumn":"id","parentColumn":"parent","treeType":0,"expanderColumn":"name"}}},"customWidth":"66","name":"query - 5","styleSettings":{"showBorder":true}},{"type":3,"content":{"version":"KqlItem/1.0","query":"SecurityAlert \\r\\n| where tolower(ResourceId) in ({clustername})\\r\\n| where TimeGenerated {timeframe}\\r\\n| where AlertType == \\"AKS_ClusterAdminBinding\\"\\r\\n| project name = extract(@\\"/MICROSOFT.CONTAINERSERVICE/MANAGEDCLUSTERS/(.+)\\", 1, ResourceId)\\r\\n| summarize count() by name","size":1,"title":"AKS clusters with related Microsoft Defender alerts","queryType":0,"resourceType":"microsoft.operationalinsights/workspaces","crossComponentResources":["{workspaces}"],"visualization":"piechart"},"customWidth":"33","name":"query - 11","styleSettings":{"showBorder":true}},{"type":3,"content":{"version":"KqlItem/1.0","query":"AzureDiagnostics\\r\\n| where Category == \\"kube-audit\\"\\r\\n| where tolower(ResourceId) in ({clustername})\\r\\n| where TimeGenerated {timeframe}\\r\\n| where log_s has \\"events\\"\\r\\n| project TimeGenerated, parse_json(log_s), ResourceId\\r\\n//Parsing\\r\\n| project AzureResourceId = ResourceId, \\r\\n TimeGenerated,\\r\\n SourceIPs = tostring(log_s[\\"sourceIPs\\"][0]),\\r\\n User = log_s[\\"user\\"],\\r\\n Verb = tostring(log_s[\\"verb\\"]),\\r\\n ObjectRef = log_s[\\"objectRef\\"],\\r\\n ResponseStatus = log_s[\\"responseStatus\\"]\\r\\n| where ObjectRef.resource == \\"events\\" and Verb == \\"delete\\" and ResponseStatus.code == 200\\r\\n| project TimeGenerated, AzureResourceId, username = tostring(User.username), ipAddr = tostring(SourceIPs), \\r\\n eventName = tostring(ObjectRef.name), eventNamespace = tostring(ObjectRef.namespace), status = tostring(ResponseStatus.code)\\r\\n| summarize operationTime = min(TimeGenerated), eventNames = make_set(eventName, 10) by\\r\\n AzureResourceId, \\r\\n eventNamespace,\\r\\n username,\\r\\n ipAddr\\r\\n// Format the list of the event names\\r\\n| extend eventNames = substring(eventNames, 1 , strlen(eventNames) - 2)\\r\\n| extend eventNames = replace(\'\\"\', \\"\\", eventNames)\\r\\n| extend eventNames = replace(\\",\\", \\", \\", eventNames)","size":1,"title":"Delete events","queryType":0,"resourceType":"microsoft.operationalinsights/workspaces","crossComponentResources":["{workspaces}"]},"name":"query - 6","styleSettings":{"showBorder":true}}]},"conditionalVisibility":{"parameterName":"diagnosticClusters","comparison":"isEqualTo","value":"yes"},"name":"diagnosticData"},{"type":1,"content":{"json":"No Diagnostic Logs data in the selected workspaces. \\r\\nTo enable Diagnostic Logs for your AKS cluster: Go to your AKS cluster --> Diagnostic settings --> Add diagnostic setting --> Select \\"kube-audit\\" and send the data to your workspace.\\r\\n\\r\\nGet more details here: https://learn.microsoft.com/azure/aks/view-master-logs","style":"info"},"conditionalVisibility":{"parameterName":"diagnosticClusters","comparison":"isEqualTo","value":"no"},"name":"text - 4"}]},"conditionalVisibility":{"parameterName":"mainTab","comparison":"isEqualTo","value":"diagnostics"},"name":"diagnostics"}],"fromTemplateId":"sentinel-AksWorkbook","$schema":"https://github.com/Microsoft/Application-Insights-Workbooks/blob/master/schema/workbook.json"}' version: '1.0' category: 'sentinel' - sourceId: law.id + sourceId: la.id tags: [ 'AksSecurityWorkbook' '1.2' @@ -457,13 +457,13 @@ resource miwSecurityInsights 'Microsoft.Insights/workbooks@2022-04-01' = { } resource omsKeyVaultAnalytics 'Microsoft.OperationsManagement/solutions@2015-11-01-preview' = { - name: 'KeyVaultAnalytics(${law.name})' + name: 'KeyVaultAnalytics(${la.name})' location: location properties: { - workspaceResourceId: law.id + workspaceResourceId: la.id } plan: { - name: 'KeyVaultAnalytics(${law.name})' + name: 'KeyVaultAnalytics(${la.name})' product: 'OMSGallery/KeyVaultAnalytics' promotionCode: '' publisher: 'Microsoft' @@ -474,7 +474,7 @@ resource kv_diagnosticSettings 'Microsoft.Insights/diagnosticSettings@2021-05-01 scope: kv name: 'default' properties: { - workspaceId: law.id + workspaceId: la.id logs: [ { category: 'AuditEvent' @@ -723,7 +723,7 @@ resource agw_diagnosticSettings 'Microsoft.Insights/diagnosticSettings@2021-05-0 scope: agw name: 'default' properties: { - workspaceId: law.id + workspaceId: la.id logs: [ { category: 'ApplicationGatewayAccessLog' @@ -839,10 +839,10 @@ resource vmssJumpboxes 'Microsoft.Compute/virtualMachineScaleSets@2020-12-01' = settings: { stopOnMultipleConnections: true azureResourceId: vmssJumpboxes.id - workspaceId: reference(law.id, '2020-10-01').customerId + workspaceId: reference(la.id, '2020-10-01').customerId } protectedSettings: { - workspaceKey: listKeys(law.id, '2020-10-01').primarySharedKey + workspaceKey: listKeys(la.id, '2020-10-01').primarySharedKey } } } @@ -918,7 +918,7 @@ resource cr_diagnosticSettings 'Microsoft.Insights/diagnosticSettings@2021-05-01 scope: acr name: 'default' properties: { - workspaceId: law.id + workspaceId: la.id metrics: [ { timeGrain: 'PT1M' @@ -1444,7 +1444,7 @@ resource mc 'Microsoft.ContainerService/managedClusters@2021-05-01' = { omsagent: { enabled: true config: { - logAnalyticsWorkspaceResourceId: law.id + logAnalyticsWorkspaceResourceId: la.id } } aciConnectorLinux: { @@ -1521,7 +1521,7 @@ resource mc 'Microsoft.ContainerService/managedClusters@2021-05-01' = { securityProfile: { azureDefender: { enabled: true - logAnalyticsWorkspaceResourceId: law.id + logAnalyticsWorkspaceResourceId: la.id } } } @@ -1563,7 +1563,7 @@ resource mc 'Microsoft.ContainerService/managedClusters@2021-05-01' = { resource mc_diagnosticSettings 'Microsoft.Insights/diagnosticSettings@2021-05-01-preview' = { name: 'default' properties: { - workspaceId: law.id + workspaceId: la.id logs: [ { category: 'cluster-autoscaler' @@ -1645,7 +1645,7 @@ resource sqrPodFailed 'Microsoft.Insights/scheduledQueryRules@2021-08-01' = { overrideQueryTimeRange: 'P2D' } dependsOn: [ - law + la ] } From c8782f5faae65ac6e1f80143e44a987223909159 Mon Sep 17 00:00:00 2001 From: Fernando Antivero Date: Fri, 14 Oct 2022 20:18:26 +0000 Subject: [PATCH 097/115] Apply PR Feedback: change to resourceGroup func in all ocurrences --- cluster-stamp.bicep | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/cluster-stamp.bicep b/cluster-stamp.bicep index 79316769..8d59ee4f 100644 --- a/cluster-stamp.bicep +++ b/cluster-stamp.bicep @@ -1042,7 +1042,7 @@ resource paEnforceHttpsIngress 'Microsoft.Authorization/policyAssignments@2020-0 name: guid(pdEnforceHttpsIngressId, resourceGroup().name, clusterName) properties: { displayName: trim(take('[${clusterName}] ${reference(pdEnforceHttpsIngressId, '2020-09-01').displayName}', 125)) - scope: subscriptionResourceId('Microsoft.Resources/resourceGroups', resourceGroup().name) + scope: resourceGroup() policyDefinitionId: pdEnforceHttpsIngressId parameters: { effect: { @@ -1059,7 +1059,7 @@ resource paEnforceInternalLoadBalancers 'Microsoft.Authorization/policyAssignmen name: guid(pdEnforceInternalLoadBalancersId, resourceGroup().name, clusterName) properties: { displayName: trim(take('[${clusterName}] ${reference(pdEnforceInternalLoadBalancersId, '2020-09-01').displayName}', 125)) - scope: subscriptionResourceId('Microsoft.Resources/resourceGroups', resourceGroup().name) + scope: resourceGroup() policyDefinitionId: pdEnforceInternalLoadBalancersId parameters: { effect: { @@ -1076,7 +1076,7 @@ resource paMustNotAutomountApiCreds 'Microsoft.Authorization/policyAssignments@2 name: guid(pdMustNotAutomountApiCreds, resourceGroup().name, clusterName) properties: { displayName: trim(take('[${clusterName}] ${reference(pdMustNotAutomountApiCreds, '2020-09-01').displayName}', 125)) - scope: subscriptionResourceId('Microsoft.Resources/resourceGroups', resourceGroup().name) + scope: resourceGroup() policyDefinitionId: pdMustNotAutomountApiCreds parameters: { effect: { @@ -1101,7 +1101,7 @@ resource paMustUseSpecifiedLabels 'Microsoft.Authorization/policyAssignments@202 name: guid(pdMustUseSpecifiedLabels, resourceGroup().name, clusterName) properties: { displayName: trim(take('[${clusterName}] ${reference(pdMustUseSpecifiedLabels, '2020-09-01').displayName}', 125)) - scope: subscriptionResourceId('Microsoft.Resources/resourceGroups', resourceGroup().name) + scope: resourceGroup() policyDefinitionId: pdMustUseSpecifiedLabels parameters: { effect: { @@ -1120,7 +1120,7 @@ resource paMustUseTheseExternalIps 'Microsoft.Authorization/policyAssignments@20 name: guid(pdAllowedExternalIPsId, resourceGroup().name, clusterName) properties: { displayName: trim(take('[${clusterName}] ${reference(pdAllowedExternalIPsId, '2020-09-01').displayName}', 125)) - scope: subscriptionResourceId('Microsoft.Resources/resourceGroups', resourceGroup().name) + scope: resourceGroup() policyDefinitionId: pdAllowedExternalIPsId parameters: { effect: { @@ -1143,7 +1143,7 @@ resource paApprovedContainerPortsOnly 'Microsoft.Authorization/policyAssignments name: guid(pdApprovedContainerPortsOnly, resourceGroup().name, clusterName) properties: { displayName: trim(take('[${clusterName}-a0005] ${reference(pdApprovedContainerPortsOnly, '2020-09-01').displayName}', 125)) - scope: subscriptionResourceId('Microsoft.Resources/resourceGroups', resourceGroup().name) + scope: resourceGroup() policyDefinitionId: pdApprovedContainerPortsOnly parameters: { effect: { @@ -1174,7 +1174,7 @@ resource paApprovedServicePortsOnly 'Microsoft.Authorization/policyAssignments@2 name: guid(pdApprovedServicePortsOnly, resourceGroup().name, clusterName) properties: { displayName: trim(take('[${clusterName}] ${reference(pdApprovedServicePortsOnly, '2020-09-01').displayName}', 125)) - scope: subscriptionResourceId('Microsoft.Resources/resourceGroups', resourceGroup().name) + scope: resourceGroup() policyDefinitionId: pdApprovedServicePortsOnly parameters: { effect: { @@ -1203,7 +1203,7 @@ resource paRoRootFilesystem 'Microsoft.Authorization/policyAssignments@2020-09-0 name: guid(pdRoRootFilesystemId, resourceGroup().name, clusterName) properties: { displayName: trim(take('[${clusterName}] ${reference(pdRoRootFilesystemId, '2020-09-01').displayName}', 125)) - scope: subscriptionResourceId('Microsoft.Resources/resourceGroups', resourceGroup().name) + scope: resourceGroup() policyDefinitionId: pdRoRootFilesystemId parameters: { effect: { @@ -1224,7 +1224,7 @@ resource paBlockDefaultNamespace 'Microsoft.Authorization/policyAssignments@2020 name: guid(pdDisallowNamespaceUsageId, resourceGroup().name, clusterName) properties: { displayName: trim(take('[${clusterName}] ${reference(pdDisallowNamespaceUsageId, '2020-09-01').displayName}', 125)) - scope: subscriptionResourceId('Microsoft.Resources/resourceGroups', resourceGroup().name) + scope: resourceGroup() policyDefinitionId: pdDisallowNamespaceUsageId parameters: { effect: { @@ -1241,7 +1241,7 @@ resource paEnforceResourceLimits 'Microsoft.Authorization/policyAssignments@2020 name: guid(pdEnforceResourceLimitsId, resourceGroup().name, clusterName) properties: { displayName: trim(take('[${clusterName}] ${reference(pdEnforceResourceLimitsId, '2020-09-01').displayName}', 125)) - scope: subscriptionResourceId('Microsoft.Resources/resourceGroups', resourceGroup().name) + scope: resourceGroup() policyDefinitionId: pdEnforceResourceLimitsId parameters: { effect: { @@ -1267,7 +1267,7 @@ resource paEnforceImageSource 'Microsoft.Authorization/policyAssignments@2020-03 name: guid(pdEnforceImageSourceId, resourceGroup().name, clusterName) properties: { displayName: trim(take('[${clusterName}] ${reference(pdEnforceImageSourceId, '2020-09-01').displayName}', 125)) - scope: subscriptionResourceId('Microsoft.Resources/resourceGroups', resourceGroup().name) + scope: resourceGroup() policyDefinitionId: pdEnforceImageSourceId parameters: { allowedContainerImagesRegex: { From 4333e6c177819965e405336f278f2df28f4bc3ac Mon Sep 17 00:00:00 2001 From: Fernando Antivero Date: Fri, 14 Oct 2022 20:20:54 +0000 Subject: [PATCH 098/115] bug fix description --- cluster-stamp.bicep | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cluster-stamp.bicep b/cluster-stamp.bicep index 8d59ee4f..9e369d47 100644 --- a/cluster-stamp.bicep +++ b/cluster-stamp.bicep @@ -1198,7 +1198,7 @@ resource paApprovedServicePortsOnly 'Microsoft.Authorization/policyAssignments@2 } } -@decscription('Applying the 'Kubernetes cluster containers should run with a read only root file system' policy to the resource group.') +@description('Applying the \'Kubernetes cluster containers should run with a read only root file system\' policy to the resource group.') resource paRoRootFilesystem 'Microsoft.Authorization/policyAssignments@2020-09-01' = { name: guid(pdRoRootFilesystemId, resourceGroup().name, clusterName) properties: { From 54df66b9fdac83488ef48e10df91d2fb0e415ca2 Mon Sep 17 00:00:00 2001 From: Fernando Antivero Date: Fri, 14 Oct 2022 20:27:39 +0000 Subject: [PATCH 099/115] Apply PR Feedback: go full rbac for accessing cert and secrets in this baseline --- cluster-stamp.bicep | 59 ++++++++++++++++++++++----------------------- 1 file changed, 29 insertions(+), 30 deletions(-) diff --git a/cluster-stamp.bicep b/cluster-stamp.bicep index 9e369d47..3106ce58 100644 --- a/cluster-stamp.bicep +++ b/cluster-stamp.bicep @@ -260,34 +260,7 @@ resource kv 'Microsoft.KeyVault/vaults@2022-07-01' = { name: 'kv-${clusterName}' location: location properties: { - accessPolicies: [ - { - tenantId: miAppGateway.properties.tenantId - objectId: miAppGateway.properties.principalId - permissions: { - secrets: [ - 'get' - ] - certificates: [ - 'get' - ] - keys: [] - } - } - { - tenantId: miIngressController.properties.tenantId - objectId: miIngressController.properties.principalId - permissions: { - secrets: [ - 'get' - ] - certificates: [ - 'get' - ] - keys: [] - } - } - ] + accessPolicies: [] sku: { family: 'A' name: 'standard' @@ -325,7 +298,7 @@ resource kv 'Microsoft.KeyVault/vaults@2022-07-01' = { @description('Grant the Azure Application Gateway managed identity with Key Vault secrets user role permissions; this allows pulling secrets from Key Vault.') resource kvMiAppGatewaySecretsUserRole_roleAssignment 'Microsoft.Authorization/roleAssignments@2020-10-01-preview' = { scope: kv - name: guid(resourceGroup().id, 'mi-appgateway', keyVaultSecretsUserRole.id) + name: guid(resourceGroup().id, miAppGateway.name, keyVaultSecretsUserRole.id) properties: { roleDefinitionId: keyVaultSecretsUserRole.id principalId: miAppGateway.properties.principalId @@ -336,7 +309,7 @@ resource kvMiAppGatewaySecretsUserRole_roleAssignment 'Microsoft.Authorization/r @description('Grant the Azure Application Gateway managed identity with Key Vault reader role permissions; this allows pulling frontend and backend certificates.') resource kvMiAppGatewayKeyVaultReader_roleAssignment 'Microsoft.Authorization/roleAssignments@2020-10-01-preview' = { scope: kv - name: guid(resourceGroup().id, 'mi-appgateway', keyVaultReaderRole.id) + name: guid(resourceGroup().id, miAppGateway.name, keyVaultReaderRole.id) properties: { roleDefinitionId: keyVaultReaderRole.id principalId: miAppGateway.properties.principalId @@ -344,6 +317,28 @@ resource kvMiAppGatewayKeyVaultReader_roleAssignment 'Microsoft.Authorization/ro } } +@description('Grant the Ingress Controller managed identity with Key Vault secrets user role permissions; this allows pulling secrets from Key Vault.') +resource kvMiIngressControllerSecretsUserRole_roleAssignment 'Microsoft.Authorization/roleAssignments@2020-10-01-preview' = { + scope: kv + name: guid(resourceGroup().id, miIngressController.name, keyVaultSecretsUserRole.id) + properties: { + roleDefinitionId: keyVaultSecretsUserRole.id + principalId: miIngressController.properties.principalId + principalType: 'ServicePrincipal' + } +} + +@description('Grant the Ingress Controller managed identity with Key Vault reader role permissions; this allows pulling frontend and backend certificates.') +resource kvMiIngressControllerKeyVaultReader_roleAssignment 'Microsoft.Authorization/roleAssignments@2020-10-01-preview' = { + scope: kv + name: guid(resourceGroup().id, miIngressController.name, keyVaultReaderRole.id) + properties: { + roleDefinitionId: keyVaultReaderRole.id + principalId: miIngressController.properties.principalId + principalType: 'ServicePrincipal' + } +} + @description('The AKS cluster and related resources log analytics workspace.') resource la 'Microsoft.OperationalInsights/workspaces@2021-12-01-preview' = { name: 'la-${clusterName}' @@ -1555,8 +1550,12 @@ resource mc 'Microsoft.ContainerService/managedClusters@2021-05-01' = { paBlockDefaultNamespace paEnforceResourceLimits paEnforceImageSource + vmssJumpboxes // Ensure jumboxes are available to use as soon as possible, don't wait until cluster is created. + miIngressController + kvMiIngressControllerSecretsUserRole_roleAssignment + kvMiIngressControllerKeyVaultReader_roleAssignment ] } From 75fb99b08c84c407f63572e784836bca95b1fbcb Mon Sep 17 00:00:00 2001 From: Fernando Antivero Date: Tue, 18 Oct 2022 18:50:54 +0000 Subject: [PATCH 100/115] upgrade cluster version --- cluster-stamp.bicep | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cluster-stamp.bicep b/cluster-stamp.bicep index 3106ce58..7cd6fe72 100644 --- a/cluster-stamp.bicep +++ b/cluster-stamp.bicep @@ -76,7 +76,7 @@ param jumpBoxCloudInitAsBase64 string /*** VARIABLES ***/ -var kubernetesVersion = '1.23.3' +var kubernetesVersion = '1.23.12' var subRgUniqueString = uniqueString('aks', subscription().subscriptionId, resourceGroup().id) var clusterName = 'aks-${subRgUniqueString}' From 75019244fbf930e667574d2c5ebcf27ac907e608 Mon Sep 17 00:00:00 2001 From: Fernando Antivero Date: Tue, 18 Oct 2022 18:51:30 +0000 Subject: [PATCH 101/115] bug fix: defender - use valid fields --- cluster-stamp.bicep | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/cluster-stamp.bicep b/cluster-stamp.bicep index 7cd6fe72..9313cd87 100644 --- a/cluster-stamp.bicep +++ b/cluster-stamp.bicep @@ -1514,9 +1514,11 @@ resource mc 'Microsoft.ContainerService/managedClusters@2021-05-01' = { } disableLocalAccounts: true securityProfile: { - azureDefender: { - enabled: true + defender: { logAnalyticsWorkspaceResourceId: la.id + securityMonitoring: { + enabled: true + } } } } From d0974f01c73a4bdf2a08c3cf6ebc7f3c9eef2bf5 Mon Sep 17 00:00:00 2001 From: Fernando Antivero Date: Tue, 18 Oct 2022 20:35:45 +0000 Subject: [PATCH 102/115] bug fix: kv rbac migration --- cluster-stamp.bicep | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/cluster-stamp.bicep b/cluster-stamp.bicep index 9313cd87..17af5108 100644 --- a/cluster-stamp.bicep +++ b/cluster-stamp.bicep @@ -260,23 +260,30 @@ resource kv 'Microsoft.KeyVault/vaults@2022-07-01' = { name: 'kv-${clusterName}' location: location properties: { - accessPolicies: [] + accessPolicies: [] // Azure RBAC is used instead sku: { family: 'A' name: 'standard' } tenantId: subscription().tenantId networkAcls: { - bypass: 'AzureServices' - defaultAction: 'Allow' + bypass: 'AzureServices' // Required for AppGW communication + defaultAction: 'Deny' ipRules: [] virtualNetworkRules: [] } + enableRbacAuthorization: true enabledForDeployment: false enabledForDiskEncryption: false enabledForTemplateDeployment: false enableSoftDelete: true + softDeleteRetentionInDays: 7 + createMode: 'default' } + dependsOn: [ + miAppGateway + miIngressController + ] // The internet facing TLS certificate to establish HTTPS connections between your clients and your regional load balancer resource kvsGatewaySslCert 'secrets' = { From 9bd31f5b9c785508be5d816c76f4e592dfb4b0ab Mon Sep 17 00:00:00 2001 From: Fernando Antivero Date: Tue, 18 Oct 2022 20:36:11 +0000 Subject: [PATCH 103/115] bug fix: update role assignments to the latest api version --- cluster-stamp.bicep | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/cluster-stamp.bicep b/cluster-stamp.bicep index 17af5108..973bd8e0 100644 --- a/cluster-stamp.bicep +++ b/cluster-stamp.bicep @@ -303,7 +303,7 @@ resource kv 'Microsoft.KeyVault/vaults@2022-07-01' = { } @description('Grant the Azure Application Gateway managed identity with Key Vault secrets user role permissions; this allows pulling secrets from Key Vault.') -resource kvMiAppGatewaySecretsUserRole_roleAssignment 'Microsoft.Authorization/roleAssignments@2020-10-01-preview' = { +resource kvMiAppGatewaySecretsUserRole_roleAssignment 'Microsoft.Authorization/roleAssignments@2022-04-01' = { scope: kv name: guid(resourceGroup().id, miAppGateway.name, keyVaultSecretsUserRole.id) properties: { @@ -314,7 +314,7 @@ resource kvMiAppGatewaySecretsUserRole_roleAssignment 'Microsoft.Authorization/r } @description('Grant the Azure Application Gateway managed identity with Key Vault reader role permissions; this allows pulling frontend and backend certificates.') -resource kvMiAppGatewayKeyVaultReader_roleAssignment 'Microsoft.Authorization/roleAssignments@2020-10-01-preview' = { +resource kvMiAppGatewayKeyVaultReader_roleAssignment 'Microsoft.Authorization/roleAssignments@2022-04-01' = { scope: kv name: guid(resourceGroup().id, miAppGateway.name, keyVaultReaderRole.id) properties: { @@ -325,7 +325,7 @@ resource kvMiAppGatewayKeyVaultReader_roleAssignment 'Microsoft.Authorization/ro } @description('Grant the Ingress Controller managed identity with Key Vault secrets user role permissions; this allows pulling secrets from Key Vault.') -resource kvMiIngressControllerSecretsUserRole_roleAssignment 'Microsoft.Authorization/roleAssignments@2020-10-01-preview' = { +resource kvMiIngressControllerSecretsUserRole_roleAssignment 'Microsoft.Authorization/roleAssignments@2022-04-01' = { scope: kv name: guid(resourceGroup().id, miIngressController.name, keyVaultSecretsUserRole.id) properties: { @@ -336,7 +336,7 @@ resource kvMiIngressControllerSecretsUserRole_roleAssignment 'Microsoft.Authoriz } @description('Grant the Ingress Controller managed identity with Key Vault reader role permissions; this allows pulling frontend and backend certificates.') -resource kvMiIngressControllerKeyVaultReader_roleAssignment 'Microsoft.Authorization/roleAssignments@2020-10-01-preview' = { +resource kvMiIngressControllerKeyVaultReader_roleAssignment 'Microsoft.Authorization/roleAssignments@2022-04-01' = { scope: kv name: guid(resourceGroup().id, miIngressController.name, keyVaultReaderRole.id) properties: { From dcb3a78a69347ed51dc98794c4b6f4f6ad1d3bcd Mon Sep 17 00:00:00 2001 From: Fernando Antivero Date: Tue, 18 Oct 2022 20:36:53 +0000 Subject: [PATCH 104/115] bug fix: add new priority required field in app gw routing rules --- cluster-stamp.bicep | 1 + 1 file changed, 1 insertion(+) diff --git a/cluster-stamp.bicep b/cluster-stamp.bicep index 973bd8e0..ed9ed0ef 100644 --- a/cluster-stamp.bicep +++ b/cluster-stamp.bicep @@ -699,6 +699,7 @@ resource agw 'Microsoft.Network/applicationGateways@2022-01-01' = { { name: 'agw-routing-rules' properties: { + priority: 1 ruleType: 'Basic' httpListener: { id: resourceId('Microsoft.Network/applicationGateways/httpListeners', 'agw-${clusterName}','listener-https') From ca9a7f409a5ea52b54bb73086bfa29b88a798550 Mon Sep 17 00:00:00 2001 From: Fernando Antivero Date: Tue, 18 Oct 2022 20:37:20 +0000 Subject: [PATCH 105/115] change reference to use object instance instead --- cluster-stamp.bicep | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cluster-stamp.bicep b/cluster-stamp.bicep index ed9ed0ef..6debdd66 100644 --- a/cluster-stamp.bicep +++ b/cluster-stamp.bicep @@ -842,10 +842,10 @@ resource vmssJumpboxes 'Microsoft.Compute/virtualMachineScaleSets@2020-12-01' = settings: { stopOnMultipleConnections: true azureResourceId: vmssJumpboxes.id - workspaceId: reference(la.id, '2020-10-01').customerId + workspaceId: la.properties.customerId } protectedSettings: { - workspaceKey: listKeys(la.id, '2020-10-01').primarySharedKey + workspaceKey: la.listKeys().primarySharedKey } } } From ca1e1848c2b09042140742a680de82efac15348f Mon Sep 17 00:00:00 2001 From: Fernando Antivero Date: Tue, 18 Oct 2022 20:38:09 +0000 Subject: [PATCH 106/115] bug fix: remove readonly scope policy assigment field --- cluster-stamp.bicep | 12 ------------ 1 file changed, 12 deletions(-) diff --git a/cluster-stamp.bicep b/cluster-stamp.bicep index 6debdd66..5123659e 100644 --- a/cluster-stamp.bicep +++ b/cluster-stamp.bicep @@ -1025,7 +1025,6 @@ resource paAksLinuxRestrictive 'Microsoft.Authorization/policyAssignments@2021-0 name: guid(psdAKSLinuxRestrictiveId, resourceGroup().name, clusterName) properties: { displayName: trim(take('[${clusterName}] ${reference(psdAKSLinuxRestrictiveId, '2020-09-01').displayName}', 125)) - scope: resourceGroup() policyDefinitionId: psdAKSLinuxRestrictiveId parameters: { effect: { @@ -1045,7 +1044,6 @@ resource paEnforceHttpsIngress 'Microsoft.Authorization/policyAssignments@2020-0 name: guid(pdEnforceHttpsIngressId, resourceGroup().name, clusterName) properties: { displayName: trim(take('[${clusterName}] ${reference(pdEnforceHttpsIngressId, '2020-09-01').displayName}', 125)) - scope: resourceGroup() policyDefinitionId: pdEnforceHttpsIngressId parameters: { effect: { @@ -1062,7 +1060,6 @@ resource paEnforceInternalLoadBalancers 'Microsoft.Authorization/policyAssignmen name: guid(pdEnforceInternalLoadBalancersId, resourceGroup().name, clusterName) properties: { displayName: trim(take('[${clusterName}] ${reference(pdEnforceInternalLoadBalancersId, '2020-09-01').displayName}', 125)) - scope: resourceGroup() policyDefinitionId: pdEnforceInternalLoadBalancersId parameters: { effect: { @@ -1079,7 +1076,6 @@ resource paMustNotAutomountApiCreds 'Microsoft.Authorization/policyAssignments@2 name: guid(pdMustNotAutomountApiCreds, resourceGroup().name, clusterName) properties: { displayName: trim(take('[${clusterName}] ${reference(pdMustNotAutomountApiCreds, '2020-09-01').displayName}', 125)) - scope: resourceGroup() policyDefinitionId: pdMustNotAutomountApiCreds parameters: { effect: { @@ -1104,7 +1100,6 @@ resource paMustUseSpecifiedLabels 'Microsoft.Authorization/policyAssignments@202 name: guid(pdMustUseSpecifiedLabels, resourceGroup().name, clusterName) properties: { displayName: trim(take('[${clusterName}] ${reference(pdMustUseSpecifiedLabels, '2020-09-01').displayName}', 125)) - scope: resourceGroup() policyDefinitionId: pdMustUseSpecifiedLabels parameters: { effect: { @@ -1123,7 +1118,6 @@ resource paMustUseTheseExternalIps 'Microsoft.Authorization/policyAssignments@20 name: guid(pdAllowedExternalIPsId, resourceGroup().name, clusterName) properties: { displayName: trim(take('[${clusterName}] ${reference(pdAllowedExternalIPsId, '2020-09-01').displayName}', 125)) - scope: resourceGroup() policyDefinitionId: pdAllowedExternalIPsId parameters: { effect: { @@ -1146,7 +1140,6 @@ resource paApprovedContainerPortsOnly 'Microsoft.Authorization/policyAssignments name: guid(pdApprovedContainerPortsOnly, resourceGroup().name, clusterName) properties: { displayName: trim(take('[${clusterName}-a0005] ${reference(pdApprovedContainerPortsOnly, '2020-09-01').displayName}', 125)) - scope: resourceGroup() policyDefinitionId: pdApprovedContainerPortsOnly parameters: { effect: { @@ -1177,7 +1170,6 @@ resource paApprovedServicePortsOnly 'Microsoft.Authorization/policyAssignments@2 name: guid(pdApprovedServicePortsOnly, resourceGroup().name, clusterName) properties: { displayName: trim(take('[${clusterName}] ${reference(pdApprovedServicePortsOnly, '2020-09-01').displayName}', 125)) - scope: resourceGroup() policyDefinitionId: pdApprovedServicePortsOnly parameters: { effect: { @@ -1206,7 +1198,6 @@ resource paRoRootFilesystem 'Microsoft.Authorization/policyAssignments@2020-09-0 name: guid(pdRoRootFilesystemId, resourceGroup().name, clusterName) properties: { displayName: trim(take('[${clusterName}] ${reference(pdRoRootFilesystemId, '2020-09-01').displayName}', 125)) - scope: resourceGroup() policyDefinitionId: pdRoRootFilesystemId parameters: { effect: { @@ -1227,7 +1218,6 @@ resource paBlockDefaultNamespace 'Microsoft.Authorization/policyAssignments@2020 name: guid(pdDisallowNamespaceUsageId, resourceGroup().name, clusterName) properties: { displayName: trim(take('[${clusterName}] ${reference(pdDisallowNamespaceUsageId, '2020-09-01').displayName}', 125)) - scope: resourceGroup() policyDefinitionId: pdDisallowNamespaceUsageId parameters: { effect: { @@ -1244,7 +1234,6 @@ resource paEnforceResourceLimits 'Microsoft.Authorization/policyAssignments@2020 name: guid(pdEnforceResourceLimitsId, resourceGroup().name, clusterName) properties: { displayName: trim(take('[${clusterName}] ${reference(pdEnforceResourceLimitsId, '2020-09-01').displayName}', 125)) - scope: resourceGroup() policyDefinitionId: pdEnforceResourceLimitsId parameters: { effect: { @@ -1270,7 +1259,6 @@ resource paEnforceImageSource 'Microsoft.Authorization/policyAssignments@2020-03 name: guid(pdEnforceImageSourceId, resourceGroup().name, clusterName) properties: { displayName: trim(take('[${clusterName}] ${reference(pdEnforceImageSourceId, '2020-09-01').displayName}', 125)) - scope: resourceGroup() policyDefinitionId: pdEnforceImageSourceId parameters: { allowedContainerImagesRegex: { From 8fe2a491f2958a06a216a65caf399f3b69f3e147 Mon Sep 17 00:00:00 2001 From: Fernando Antivero Date: Tue, 18 Oct 2022 20:38:59 +0000 Subject: [PATCH 107/115] bug fix: add managed cluster scope to diagnostic settings --- cluster-stamp.bicep | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/cluster-stamp.bicep b/cluster-stamp.bicep index 5123659e..5afaa95d 100644 --- a/cluster-stamp.bicep +++ b/cluster-stamp.bicep @@ -1558,6 +1558,7 @@ resource mc 'Microsoft.ContainerService/managedClusters@2021-05-01' = { } resource mc_diagnosticSettings 'Microsoft.Insights/diagnosticSettings@2021-05-01-preview' = { + scope: mc name: 'default' properties: { workspaceId: la.id @@ -1580,9 +1581,6 @@ resource mc_diagnosticSettings 'Microsoft.Insights/diagnosticSettings@2021-05-01 } ] } - dependsOn: [ - mc - ] } @description('Grant kubelet managed identity with container registry pull role permissions; this allows the AKS Cluster\'s kubelet managed identity to pull images from this container registry.') From c8bc649a57da5a5c0d031e3c2e86f2efe33ef9f1 Mon Sep 17 00:00:00 2001 From: Fernando Antivero Date: Tue, 18 Oct 2022 21:47:24 +0000 Subject: [PATCH 108/115] bug fix: role assign the proper resource --- cluster-stamp.bicep | 1 + ...dentityHasRbacToSelfManagedResources.bicep | 31 ++++++++++++++++--- 2 files changed, 27 insertions(+), 5 deletions(-) diff --git a/cluster-stamp.bicep b/cluster-stamp.bicep index 5afaa95d..6c35f12d 100644 --- a/cluster-stamp.bicep +++ b/cluster-stamp.bicep @@ -1311,6 +1311,7 @@ module ensureClusterIdentityHasRbacToSelfManagedResources 'modules/ensureCluster miClusterControlPlanePrincipalId: miClusterControlPlane.properties.principalId clusterControlPlaneIdentityName: miClusterControlPlane.name vnetSpokeName: vnetSpoke.name + location: location } } diff --git a/modules/ensureClusterIdentityHasRbacToSelfManagedResources.bicep b/modules/ensureClusterIdentityHasRbacToSelfManagedResources.bicep index 11688342..0b0c6307 100644 --- a/modules/ensureClusterIdentityHasRbacToSelfManagedResources.bicep +++ b/modules/ensureClusterIdentityHasRbacToSelfManagedResources.bicep @@ -16,6 +16,27 @@ param clusterControlPlaneIdentityName string @minLength(1) param vnetSpokeName string +@allowed([ + 'australiaeast' + 'canadacentral' + 'centralus' + 'eastus' + 'eastus2' + 'westus2' + 'francecentral' + 'germanywestcentral' + 'northeurope' + 'southafricanorth' + 'southcentralus' + 'uksouth' + 'westeurope' + 'japaneast' + 'southeastasia' +]) +@description('AKS Service, Node Pools, and supporting services (KeyVault, App Gateway, etc) region. This needs to be the same region as the vnet provided in these parameters.') +@minLength(4) +param location string + /*** EXISTING SUBSCRIPTION RESOURCES ***/ resource networkContributorRole 'Microsoft.Authorization/roleDefinitions@2022-04-01' existing = { @@ -30,8 +51,8 @@ resource dnsZoneContributorRole 'Microsoft.Authorization/roleDefinitions@2022-04 /*** EXISTING SPOKE RESOURCES ***/ -resource pdzCr 'Microsoft.Network/privateDnsZones@2020-06-01' existing = { - name: 'privatelink.azurecr.io' +resource pdzMc 'Microsoft.Network/privateDnsZones@2020-06-01' existing = { + name: 'privatelink.${location}.azmk8s.io' } resource vnetSpoke 'Microsoft.Network/virtualNetworks@2022-01-01' existing = { @@ -111,9 +132,9 @@ resource snetIngressServicesSubnetMiClusterControlPlaneNetworkContributorRole_ro } } -resource pdzCrPrivatelinkAzmk8sIoMiClusterControlPlaneDnsZoneContributorRole_roleAssignment 'Microsoft.Authorization/roleAssignments@2020-04-01-preview' = { - scope: pdzCr - name: guid(pdzCr.id, dnsZoneContributorRole.id, clusterControlPlaneIdentityName) +resource pdzMcPrivatelinkAzmk8sIoMiClusterControlPlaneDnsZoneContributorRole_roleAssignment 'Microsoft.Authorization/roleAssignments@2020-04-01-preview' = { + scope: pdzMc + name: guid(pdzMc.id, dnsZoneContributorRole.id, clusterControlPlaneIdentityName) properties: { roleDefinitionId: dnsZoneContributorRole.id description: 'Allows cluster identity to manage zone Entries for cluster\'s Private Link configuration.' From 6b6c06d62946b910097d63f83a21f72193502157 Mon Sep 17 00:00:00 2001 From: Fernando Antivero Date: Wed, 19 Oct 2022 18:07:39 +0000 Subject: [PATCH 109/115] bug fix: target the proper ipgroup when applying fw app rules --- networking/hub-region.v2.bicep | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/networking/hub-region.v2.bicep b/networking/hub-region.v2.bicep index ba7676cf..cec37e79 100644 --- a/networking/hub-region.v2.bicep +++ b/networking/hub-region.v2.bicep @@ -892,7 +892,7 @@ resource hubFirewall 'Microsoft.Network/azureFirewalls@2021-05-01' = { name: 'az-login' description: 'Allow jumpboxes to perform az login.' sourceIpGroups: [ - aks_ipgroup.id + aksJumpbox_ipgroup.id ] protocols: [ { @@ -909,7 +909,7 @@ resource hubFirewall 'Microsoft.Network/azureFirewalls@2021-05-01' = { name: 'az-management-api' description: 'Allow jumpboxes to communicate with Azure management APIs.' sourceIpGroups: [ - aks_ipgroup.id + aksJumpbox_ipgroup.id ] protocols: [ { @@ -926,7 +926,7 @@ resource hubFirewall 'Microsoft.Network/azureFirewalls@2021-05-01' = { name: 'az-cli-extensions' description: 'Allow jumpboxes query az cli status and download extensions' sourceIpGroups: [ - aks_ipgroup.id + aksJumpbox_ipgroup.id ] protocols: [ { @@ -946,7 +946,7 @@ resource hubFirewall 'Microsoft.Network/azureFirewalls@2021-05-01' = { name: 'github' description: 'Allow pulling things down from GitHub. [Only a requirement of this walkthrough because we deploy some manifests that you clone from your repo.]' sourceIpGroups: [ - aks_ipgroup.id + aksJumpbox_ipgroup.id ] protocols: [ { @@ -964,7 +964,7 @@ resource hubFirewall 'Microsoft.Network/azureFirewalls@2021-05-01' = { name: 'azure-monitor-addon' description: 'Required for Azure Monitor Extension on Jumpbox.' sourceIpGroups: [ - aks_ipgroup.id + aksJumpbox_ipgroup.id ] protocols: [ { From 6f45320304d72351fa4b811907def2d6d1fb068c Mon Sep 17 00:00:00 2001 From: Fernando Antivero Date: Wed, 19 Oct 2022 20:26:22 +0000 Subject: [PATCH 110/115] bug fix: add missing commented out code --- cluster-stamp.bicep | 3 +++ 1 file changed, 3 insertions(+) diff --git a/cluster-stamp.bicep b/cluster-stamp.bicep index 6c35f12d..e7934bcd 100644 --- a/cluster-stamp.bicep +++ b/cluster-stamp.bicep @@ -1501,6 +1501,9 @@ resource mc 'Microsoft.ContainerService/managedClusters@2021-05-01' = { 'skip-nodes-with-local-storage': 'true' 'skip-nodes-with-system-pods': 'true' } + // autoUpgradeProfile: { + // upgradeChannel: 'none' + // } apiServerAccessProfile: { enablePrivateCluster: true privateDNSZone: pdzMc.id From 6fb607f96d9cb03f25a20836b05c15f332a80bdd Mon Sep 17 00:00:00 2001 From: Fernando Antivero Date: Thu, 20 Oct 2022 20:16:27 +0000 Subject: [PATCH 111/115] bug fix: virtual network link for dns resolution must target hub vnet for fw --- networking/spoke-BU0001A0005-01.bicep | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/networking/spoke-BU0001A0005-01.bicep b/networking/spoke-BU0001A0005-01.bicep index 44386bcf..7220a3f8 100644 --- a/networking/spoke-BU0001A0005-01.bicep +++ b/networking/spoke-BU0001A0005-01.bicep @@ -862,7 +862,7 @@ resource aksPrivateDnsZones_virtualNetworkLink_toClusterVNet 'Microsoft.Network/ location: 'global' properties: { virtualNetwork: { - id: clusterVNet.id + id: hubVnetResourceId } registrationEnabled: false } From b275ce5b4b3ae322cf8e8bd318aacafbc5bf4f74 Mon Sep 17 00:00:00 2001 From: Fernando Antivero Date: Thu, 20 Oct 2022 21:41:44 +0000 Subject: [PATCH 112/115] migrate stamp v2 to bicep lang --- cluster-stamp.v2.bicep | 2249 ++++++++++++++++++++++++++++++ cluster-stamp.v2.json | 2458 --------------------------------- docs/deploy/09-aks-cluster.md | 4 +- 3 files changed, 2251 insertions(+), 2460 deletions(-) create mode 100644 cluster-stamp.v2.bicep delete mode 100644 cluster-stamp.v2.json diff --git a/cluster-stamp.v2.bicep b/cluster-stamp.v2.bicep new file mode 100644 index 00000000..8b8d0f26 --- /dev/null +++ b/cluster-stamp.v2.bicep @@ -0,0 +1,2249 @@ +targetScope = 'resourceGroup' + +/*** PARAMETERS ***/ + +@description('The regional network spoke VNet Resource ID that the cluster will be joined to') +@minLength(79) +param targetVnetResourceId string + +@description('Azure AD Group in the identified tenant that will be granted the highly privileged cluster-admin role.') +param clusterAdminAadGroupObjectId string + +@description('Your AKS control plane Cluster API authentication tenant') +param k8sControlPlaneAuthorizationTenantId string + +@description('The certificate data for app gateway TLS termination. It is base64') +param appGatewayListenerCertificate string + +@description('The base 64 encoded AKS Ingress Controller public certificate (as .crt or .cer) to be stored in Azure Key Vault as secret and referenced by Azure Application Gateway as a trusted root certificate.') +param aksIngressControllerCertificate string + +@allowed([ + 'australiaeast' + 'canadacentral' + 'centralus' + 'eastus' + 'eastus2' + 'westus2' + 'francecentral' + 'germanywestcentral' + 'northeurope' + 'southafricanorth' + 'southcentralus' + 'uksouth' + 'westeurope' + 'japaneast' + 'southeastasia' +]) +@description('AKS Service, Node Pools, and supporting services (KeyVault, App Gateway, etc) region. This needs to be the same region as the vnet provided in these parameters.') +@minLength(4) +param location string = 'eastus2' + +@allowed([ + 'australiasoutheast' + 'canadaeast' + 'eastus2' + 'westus' + 'centralus' + 'westcentralus' + 'francesouth' + 'germanynorth' + 'westeurope' + 'ukwest' + 'northeurope' + 'japanwest' + 'southafricawest' + 'northcentralus' + 'eastasia' + 'eastus' + 'westus2' + 'francecentral' + 'uksouth' + 'japaneast' + 'southeastasia' +]) +@description('For Azure resources that support native geo-redunancy, provide the location the redundant service will have its secondary. Should be different than the location parameter and ideally should be a paired region - https://learn.microsoft.com/azure/best-practices-availability-paired-regions. This region does not need to support availability zones.') +@minLength(4) +param geoRedundancyLocation string = 'centralus' + +@description('The Azure resource ID of a VM image that will be used for the jump box.') +@minLength(70) +param jumpBoxImageResourceId string + +@description('A cloud init file (starting with #cloud-config) as a base 64 encoded string used to perform image customization on the jump box VMs. Used for user-management in this context.') +@minLength(100) +param jumpBoxCloudInitAsBase64 string + +/*** VARIABLES ***/ + +var kubernetesVersion = '1.23.12' + +var subRgUniqueString = uniqueString('aks', subscription().subscriptionId, resourceGroup().id) +var clusterName = 'aks-${subRgUniqueString}' +var jumpBoxDefaultAdminUserName = uniqueString(clusterName, resourceGroup().id) +var acrName = 'acraks${subRgUniqueString}' + +/*** EXISTING TENANT RESOURCES ***/ + +@description('Built-in \'Kubernetes cluster pod security restricted standards for Linux-based workloads\' Azure Policy for Kubernetes initiative definition') +var psdAKSLinuxRestrictiveId = tenantResourceId('Microsoft.Authorization/policySetDefinitions', '42b8ef37-b724-4e24-bbc8-7a7708edfe00') + +@description('Built-in \'Kubernetes clusters should be accessible only over HTTPS\' Azure Policy for Kubernetes policy definition') +var pdEnforceHttpsIngressId = tenantResourceId('Microsoft.Authorization/policyDefinitions', '1a5b4dca-0b6f-4cf5-907c-56316bc1bf3d') + +@description('Built-in \'Kubernetes clusters should use internal load balancers\' Azure Policy for Kubernetes policy definition') +var pdEnforceInternalLoadBalancersId = tenantResourceId('Microsoft.Authorization/policyDefinitions', '3fc4dc25-5baf-40d8-9b05-7fe74c1bc64e') + +@description('Built-in \'Kubernetes cluster services should only use allowed external IPs\' Azure Policy for Kubernetes policy definition') +var pdAllowedExternalIPsId = tenantResourceId('Microsoft.Authorization/policyDefinitions', 'd46c275d-1680-448d-b2ec-e495a3b6cc89') + +@description('Built-in \'[Deprecated]: Kubernetes cluster containers should only listen on allowed ports\' Azure Policy policy definition') +var pdApprovedContainerPortsOnly = tenantResourceId('Microsoft.Authorization/policyDefinitions', '440b515e-a580-421e-abeb-b159a61ddcbc') + +@description('Built-in \'Kubernetes cluster services should listen only on allowed ports\' Azure Policy policy definition') +var pdApprovedServicePortsOnly = tenantResourceId('Microsoft.Authorization/policyDefinitions', '233a2a17-77ca-4fb1-9b6b-69223d272a44') + +@description('Built-in \'Kubernetes cluster pods should use specified labels\' Azure Policy policy definition') +var pdMustUseSpecifiedLabels = tenantResourceId('Microsoft.Authorization/policyDefinitions', '46592696-4c7b-4bf3-9e45-6c2763bdc0a6') + +@description('Built-in \'Kubernetes clusters should disable automounting API credentials\' Azure Policy policy definition') +var pdMustNotAutomountApiCreds = tenantResourceId('Microsoft.Authorization/policyDefinitions','423dd1ba-798e-40e4-9c4d-b6902674b423') + +@description('Built-in \'Kubernetes cluster containers should run with a read only root file systemv\' Azure Policy for Kubernetes policy definition') +var pdRoRootFilesystemId = tenantResourceId('Microsoft.Authorization/policyDefinitions', 'df49d893-a74c-421d-bc95-c663042e5b80') + +@description('Built-in \'Kubernetes clusters should not use the default namespace\' Azure Policy for Kubernetes policy definition') +var pdDisallowNamespaceUsageId = tenantResourceId('Microsoft.Authorization/policyDefinitions', '9f061a12-e40d-4183-a00e-171812443373') + +@description('Built-in \'AKS container CPU and memory resource limits should not exceed the specified limits\' Azure Policy for Kubernetes policy definition') +var pdEnforceResourceLimitsId = tenantResourceId('Microsoft.Authorization/policyDefinitions', 'e345eecc-fa47-480f-9e88-67dcc122b164') + +@description('Built-in \'AKS containers should only use allowed images\' Azure Policy for Kubernetes policy definition') +var pdEnforceImageSourceId = tenantResourceId('Microsoft.Authorization/policyDefinitions', 'febd0533-8e55-448f-b837-bd0e06f16469') + +/*** EXISTING RESOURCE GROUP RESOURCES ***/ + +@description('Spoke resource group') +resource spokeResourceGroup 'Microsoft.Resources/resourceGroups@2021-04-01' existing = { + scope: subscription() + name: '${split(targetVnetResourceId,'/')[4]}' +} + +@description('The Spoke virtual network') +resource vnetSpoke 'Microsoft.Network/virtualNetworks@2022-01-01' existing = { + scope: spokeResourceGroup + name: '${last(split(targetVnetResourceId,'/'))}' + + // Spoke virutual network's subnet for application gateway + resource snetApplicationGateway 'subnets' existing = { + name: 'snet-applicationgateway' + } + + // Spoke virutual network's subnet for all private endpoints + resource snetPrivatelinkendpoints 'subnets' existing = { + name: 'snet-privatelinkendpoints' + } + + // spoke virtual network's subnet for managment ops + resource snetManagmentOps 'subnets' existing = { + name: 'snet-management-ops' + } + + // spoke virtual network's subnet for managment acr agent pools + resource snetManagmentCrAgents 'subnets' existing = { + name: 'snet-management-acragents' + } + + // spoke virtual network's subnet for cluster system node pools + resource snetClusterSystemNodePools 'subnets' existing = { + name: 'snet-cluster-systemnodepool' + } + + // spoke virtual network's subnet for cluster in-scope node pools + resource snetClusterInScopeNodePools 'subnets' existing = { + name: 'snet-cluster-inscopenodepools' + } + + // spoke virtual network's subnet for cluster out-scope node pools + resource snetClusterOutScopeNodePools 'subnets' existing = { + name: 'snet-cluster-outofscopenodepools' + } + +} + +resource pdzKv 'Microsoft.Network/privateDnsZones@2020-06-01' existing = { + scope: spokeResourceGroup + name: 'privatelink.vaultcore.azure.net' +} + +resource pdzCr 'Microsoft.Network/privateDnsZones@2020-06-01' existing = { + scope: spokeResourceGroup + name: 'privatelink.azurecr.io' +} + +resource pdzMc 'Microsoft.Network/privateDnsZones@2020-06-01' existing = { + scope: spokeResourceGroup + name: 'privatelink.${location}.azmk8s.io' +} + +@description('Used as primary entry point for workload. Expected to be assigned to an Azure Application Gateway.') +resource pipPrimaryCluster 'Microsoft.Network/publicIPAddresses@2022-05-01' existing = { + scope: spokeResourceGroup + name: 'pip-BU0001A0005-00' +} + +/*** EXISTING SUBSCRIPTION RESOURCES ***/ + +@description('Built-in Azure RBAC role that must be applied to the kublet Managed Identity allowing it to further assign adding managed identities to the cluster\'s underlying VMSS.') +resource managedIdentityOperatorRole 'Microsoft.Authorization/roleDefinitions@2022-04-01' existing = { + name: 'f1a07417-d97a-45cb-824c-7a7467783830' + scope: subscription() +} + +@description('Built-in Azure RBAC role that is applied a Key Vault to grant with metadata, certificates, keys and secrets read privileges. Granted to App Gateway\'s managed identity.') +resource keyVaultReaderRole 'Microsoft.Authorization/roleDefinitions@2022-04-01' existing = { + name: '21090545-7ca7-4776-b22c-e363652d74d2' + scope: subscription() +} + +@description('Built-in Azure RBAC role that is applied to a Key Vault to grant with secrets content read privileges. Granted to both Key Vault and our workload\'s identity.') +resource keyVaultSecretsUserRole 'Microsoft.Authorization/roleDefinitions@2022-04-01' existing = { + name: '4633458b-17de-408a-b874-0445c86b69e6' + scope: subscription() +} + +@description('Built-in Azure RBAC role that is applied to a Container Registry to grant with pull privileges. Granted to AKS kubelet cluster\'s identity.') +resource containerRegistryPullRole 'Microsoft.Authorization/roleDefinitions@2022-04-01' existing = { + name: '7f951dda-4ed3-4680-a7ca-43fe172d538d' + scope: subscription() +} + +@description('Built-in Azure RBAC role that is applied to a Subscription to grant with publishing metrics. Granted to in-cluster agent\'s identity.') +resource monitoringMetricsPublisherRole 'Microsoft.Authorization/roleDefinitions@2022-04-01' existing = { + name: '3913510d-42f4-4e42-8a64-420c390055eb' + scope: subscription() +} + +/*** RESOURCES ***/ + +@description('The control plane identity used by the cluster. Used for networking access (VNET joining and DNS updating)') +resource miClusterControlPlane 'Microsoft.ManagedIdentity/userAssignedIdentities@2022-01-31-preview' = { + name: 'mi-${clusterName}-controlplane' + location: location +} + +@description('The in-cluster ingress controller identity used by the pod identity agent to acquire access tokens to read SSL certs from Azure Key Vault.') +resource miIngressController 'Microsoft.ManagedIdentity/userAssignedIdentities@2022-01-31-preview' = { + name: 'mi-${clusterName}-ingresscontroller' + location: location +} + +@description('The regional load balancer identity used by your Application Gateway instance to acquire access tokens to read certs and secrets from Azure Key Vault.') +resource miAppGateway 'Microsoft.ManagedIdentity/userAssignedIdentities@2022-01-31-preview' = { + name: 'mi-appgateway' + location: location +} + +@description('Grant the cluster control plane managed identity with managed identity operator role permissions; this allows to assign compute with the ingress controller managed identity; this is required for Azure Pod Identity.') +resource icMiClusterControlPlaneManagedIdentityOperatorRole_roleAssignment 'Microsoft.Authorization/roleAssignments@2022-04-01' = { + scope: miIngressController + name: guid(resourceGroup().id, miClusterControlPlane.name, managedIdentityOperatorRole.id) + properties: { + roleDefinitionId: managedIdentityOperatorRole.id + principalId: miClusterControlPlane.properties.principalId + principalType: 'ServicePrincipal' + } +} + +@description('The secret storage management resource for the AKS regulated cluster.') +resource kv 'Microsoft.KeyVault/vaults@2022-07-01' = { + name: 'kv-${clusterName}' + location: location + properties: { + accessPolicies: [] // Azure RBAC is used instead + sku: { + family: 'A' + name: 'standard' + } + tenantId: subscription().tenantId + networkAcls: { + bypass: 'AzureServices' // Required for AppGW communication + defaultAction: 'Deny' + ipRules: [] + virtualNetworkRules: [] + } + enableRbacAuthorization: true + enabledForDeployment: false + enabledForDiskEncryption: false + enabledForTemplateDeployment: false + enableSoftDelete: true + softDeleteRetentionInDays: 7 + createMode: 'default' + } + dependsOn: [ + miAppGateway + miIngressController + ] + + // The internet facing TLS certificate to establish HTTPS connections between your clients and your regional load balancer + resource kvsGatewaySslCert 'secrets' = { + name: 'sslcert' + properties: { + value: appGatewayListenerCertificate + } + } + + // The in-cluster TLS certificate to establish HTTPS connections between your regional load balancer and your ingress controller, enabling end-to-end TLS connections. + resource kvsAppGwIngressInternalAksIngressTls 'secrets' = { + name: 'agw-ingress-internal-aks-ingress-contoso-com-tls' + properties: { + value: aksIngressControllerCertificate + } + } +} + +@description('Grant the Azure Application Gateway managed identity with Key Vault secrets user role permissions; this allows pulling secrets from Key Vault.') +resource kvMiAppGatewaySecretsUserRole_roleAssignment 'Microsoft.Authorization/roleAssignments@2022-04-01' = { + scope: kv + name: guid(resourceGroup().id, miAppGateway.name, keyVaultSecretsUserRole.id) + properties: { + roleDefinitionId: keyVaultSecretsUserRole.id + principalId: miAppGateway.properties.principalId + principalType: 'ServicePrincipal' + } +} + +@description('Grant the Azure Application Gateway managed identity with Key Vault reader role permissions; this allows pulling frontend and backend certificates.') +resource kvMiAppGatewayKeyVaultReader_roleAssignment 'Microsoft.Authorization/roleAssignments@2022-04-01' = { + scope: kv + name: guid(resourceGroup().id, miAppGateway.name, keyVaultReaderRole.id) + properties: { + roleDefinitionId: keyVaultReaderRole.id + principalId: miAppGateway.properties.principalId + principalType: 'ServicePrincipal' + } +} + +@description('Grant the Ingress Controller managed identity with Key Vault secrets user role permissions; this allows pulling secrets from Key Vault.') +resource kvMiIngressControllerSecretsUserRole_roleAssignment 'Microsoft.Authorization/roleAssignments@2022-04-01' = { + scope: kv + name: guid(resourceGroup().id, miIngressController.name, keyVaultSecretsUserRole.id) + properties: { + roleDefinitionId: keyVaultSecretsUserRole.id + principalId: miIngressController.properties.principalId + principalType: 'ServicePrincipal' + } +} + +@description('Grant the Ingress Controller managed identity with Key Vault reader role permissions; this allows pulling frontend and backend certificates.') +resource kvMiIngressControllerKeyVaultReader_roleAssignment 'Microsoft.Authorization/roleAssignments@2022-04-01' = { + scope: kv + name: guid(resourceGroup().id, miIngressController.name, keyVaultReaderRole.id) + properties: { + roleDefinitionId: keyVaultReaderRole.id + principalId: miIngressController.properties.principalId + principalType: 'ServicePrincipal' + } +} + +@description('The AKS cluster and related resources log analytics workspace.') +resource la 'Microsoft.OperationalInsights/workspaces@2021-12-01-preview' = { + name: 'la-${clusterName}' + location: location + properties: { + sku: { + name: 'PerGB2018' + } + retentionInDays: 90 + publicNetworkAccessForIngestion: 'Enabled' + publicNetworkAccessForQuery: 'Enabled' + } +} + +resource lawAllPrometheus 'Microsoft.OperationalInsights/workspaces/savedSearches@2020-08-01' = { + parent: la + name: 'AllPrometheus' + properties: { + eTag: '*' + category: 'Prometheus' + displayName: 'All collected Prometheus information' + query: 'InsightsMetrics | where Namespace == "prometheus"' + version: 1 + } +} + +resource lawForbiddenReponsesOnIngress 'Microsoft.OperationalInsights/workspaces/savedSearches@2020-08-01' = { + parent: la + name: 'ForbiddenReponsesOnIngress' + properties: { + eTag: '*' + category: 'Prometheus' + displayName: 'Increase number of forbidden response on the Ingress Controller' + query: 'let value = toscalar(InsightsMetrics | where Namespace == "prometheus" and Name == "nginx_ingress_controller_requests" | where parse_json(Tags).status == 403 | summarize Value = avg(Val) by bin(TimeGenerated, 5m) | summarize min = min(Value)); InsightsMetrics | where Namespace == "prometheus" and Name == "nginx_ingress_controller_requests" | where parse_json(Tags).status == 403 | summarize AggregatedValue = avg(Val)-value by bin(TimeGenerated, 5m) | order by TimeGenerated | render barchart' + version: 1 + } +} + +resource lawNodeRebootRequested 'Microsoft.OperationalInsights/workspaces/savedSearches@2020-08-01' = { + parent: la + name: 'NodeRebootRequested' + properties: { + eTag: '*' + category: 'Prometheus' + displayName: 'Nodes reboot required by kured' + query: 'InsightsMetrics | where Namespace == "prometheus" and Name == "kured_reboot_required" | where Val > 0' + version: 1 + } +} + +resource omsContainerInsights 'Microsoft.OperationsManagement/solutions@2015-11-01-preview' = { + name: 'ContainerInsights(${la.name})' + location: location + properties: { + workspaceResourceId: la.id + } + plan: { + name: 'ContainerInsights(${la.name})' + product: 'OMSGallery/ContainerInsights' + promotionCode: '' + publisher: 'Microsoft' + } +} + +resource omsVmInsights 'Microsoft.OperationsManagement/solutions@2015-11-01-preview' = { + name: 'VMInsights(${la.name})' + location: location + properties: { + workspaceResourceId: la.id + } + plan: { + name: 'VMInsights(${la.name})' + product: 'OMSGallery/VMInsights' + promotionCode: '' + publisher: 'Microsoft' + } +} + +resource omsSecurityInsights 'Microsoft.OperationsManagement/solutions@2015-11-01-preview' = { + name: 'SecurityInsights(${la.name})' + location: location + properties: { + workspaceResourceId: la.id + } + plan: { + name: 'SecurityInsights(${la.name})' + product: 'OMSGallery/SecurityInsights' + promotionCode: '' + publisher: 'Microsoft' + } +} + +resource miwSecurityInsights 'Microsoft.Insights/workbooks@2022-04-01' = { + name: guid(omsSecurityInsights.name) + location: location + tags: { + 'hidden-title': 'Azure Kubernetes Service (AKS) Security - ${la.name}' + } + kind: 'shared' + properties: { + displayName: 'Azure Kubernetes Service (AKS) Security - ${la.name}' + serializedData: '{"version":"Notebook/1.0","items":[{"type":1,"content":{"json":"## AKS Security\\n"},"name":"text - 2"},{"type":9,"content":{"version":"KqlParameterItem/1.0","crossComponentResources":["{workspaces}"],"parameters":[{"id":"311d3728-7f8a-4b16-8a34-097d099323d5","version":"KqlParameterItem/1.0","name":"subscription","label":"Subscription","type":6,"isRequired":true,"multiSelect":true,"quote":"\'","delimiter":",","value":[],"typeSettings":{"additionalResourceOptions":[],"includeAll":false,"showDefault":false}},{"id":"3a56d260-4fb9-46d6-b121-cea854104c91","version":"KqlParameterItem/1.0","name":"workspaces","label":"Workspaces","type":5,"isRequired":true,"multiSelect":true,"quote":"\'","delimiter":",","query":"where type =~ \'microsoft.operationalinsights/workspaces\'\\r\\n| where strcat(\'/subscriptions/\',subscriptionId) in ({subscription})\\r\\n| project id","crossComponentResources":["{subscription}"],"typeSettings":{"additionalResourceOptions":["value::all"]},"queryType":1,"resourceType":"microsoft.resourcegraph/resources","value":["value::all"]},{"id":"9615cea6-c661-470a-b4ae-1aab8ae6f448","version":"KqlParameterItem/1.0","name":"clustername","label":"Cluster name","type":5,"isRequired":true,"multiSelect":true,"quote":"\'","delimiter":",","query":"where type == \\"microsoft.containerservice/managedclusters\\"\\r\\n| where strcat(\'/subscriptions/\',subscriptionId) in ({subscription})\\r\\n| distinct tolower(id)","crossComponentResources":["{subscription}"],"value":["value::all"],"typeSettings":{"resourceTypeFilter":{"microsoft.containerservice/managedclusters":true},"additionalResourceOptions":["value::all"],"showDefault":false},"timeContext":{"durationMs":86400000},"queryType":1,"resourceType":"microsoft.resourcegraph/resources"},{"id":"236c00ec-1493-4e60-927a-a18b8b120cd5","version":"KqlParameterItem/1.0","name":"timeframe","label":"Time range","type":4,"description":"Time","isRequired":true,"value":{"durationMs":172800000},"typeSettings":{"selectableValues":[{"durationMs":300000},{"durationMs":900000},{"durationMs":1800000},{"durationMs":3600000},{"durationMs":14400000},{"durationMs":43200000},{"durationMs":86400000},{"durationMs":172800000},{"durationMs":259200000},{"durationMs":604800000},{"durationMs":1209600000},{"durationMs":2419200000},{"durationMs":2592000000},{"durationMs":5184000000},{"durationMs":7776000000}],"allowCustom":true},"timeContext":{"durationMs":86400000}},{"id":"bf0a3e4f-fff9-450c-b9d3-c8c1dded9787","version":"KqlParameterItem/1.0","name":"nodeRgDetails","type":1,"query":"where type == \\"microsoft.containerservice/managedclusters\\"\\r\\n| where tolower(id) in ({clustername})\\r\\n| project nodeRG = properties.nodeResourceGroup, subscriptionId, id = toupper(id)\\r\\n| project nodeRgDetails = strcat(\'\\"\', nodeRG, \\";\\", subscriptionId, \\";\\", id, \'\\"\')","crossComponentResources":["value::all"],"isHiddenWhenLocked":true,"timeContext":{"durationMs":86400000},"queryType":1,"resourceType":"microsoft.resourcegraph/resources"},{"id":"df53126c-c40f-43d5-b99f-97ee3785c086","version":"KqlParameterItem/1.0","name":"diagnosticClusters","type":1,"query":"union withsource=_TableName *\\r\\n| where _TableName == \\"AzureDiagnostics\\" and Category == \\"kube-audit\\"\\r\\n| summarize diagnosticClusters = dcount(ResourceId)\\r\\n| project isDiagnosticCluster = iff(diagnosticClusters > 0, \\"yes\\", \\"no\\")","crossComponentResources":["{workspaces}"],"isHiddenWhenLocked":true,"timeContext":{"durationMs":172800000},"timeContextFromParameter":"timeframe","queryType":0,"resourceType":"microsoft.operationalinsights/workspaces"}],"style":"pills","queryType":0,"resourceType":"microsoft.operationalinsights/workspaces"},"name":"parameters - 3"},{"type":11,"content":{"version":"LinkItem/1.0","style":"tabs","links":[{"id":"07cf87dc-8234-47db-850d-ec41b2687b2a","cellValue":"mainTab","linkTarget":"parameter","linkLabel":"Microsoft Defender for Kubernetes","subTarget":"alerts","preText":"","style":"link"},{"id":"44033ee6-d83e-4253-a732-c258ef1da545","cellValue":"mainTab","linkTarget":"parameter","linkLabel":"Analytics over Diagnostic logs","subTarget":"diagnostics","style":"link"}]},"name":"links - 22"},{"type":12,"content":{"version":"NotebookGroup/1.0","groupType":"editable","items":[{"type":1,"content":{"json":"## Microsoft Defender for AKS coverage"},"name":"text - 10"},{"type":3,"content":{"version":"KqlItem/1.0","query":"datatable (Event:string)\\r\\n [\\"AKS Workbook\\"]\\r\\n| extend cluster = (strcat(\\"[\\", \\"{clustername}\\", \\"]\\"))\\r\\n| extend cluster = todynamic(replace(\\"\'\\", \'\\"\', cluster))\\r\\n| mvexpand cluster\\r\\n| extend subscriptionId = extract(@\\"/subscriptions/([^/]+)\\", 1, tostring(cluster))\\r\\n| summarize AksClusters = count() by subscriptionId, DefenderForAks = 0\\r\\n| union\\r\\n(\\r\\nsecurityresources\\r\\n| where type =~ \\"microsoft.security/pricings\\"\\r\\n| where name == \\"KubernetesService\\"\\r\\n| project DefenderForAks = iif(properties.pricingTier == \'Standard\', 1, 0), AksClusters = 0, subscriptionId\\r\\n)\\r\\n| summarize AksClusters = sum(AksClusters), DefenderForAks = sum(DefenderForAks) by subscriptionId\\r\\n| project Subscription = strcat(\'/subscriptions/\', subscriptionId), [\\"AKS clusters\\"] = AksClusters, [\'Defender for AKS\'] = iif(DefenderForAks > 0,\'yes\',\'no\'), [\'Onboard Microsoft Defender\'] = iif(DefenderForAks > 0, \'\', \'https://ms.portal.azure.com/#blade/Microsoft_Azure_Security/SecurityMenuBlade/26\')\\r\\n| order by [\'Defender for AKS\'] asc","size":0,"queryType":1,"resourceType":"microsoft.resourcegraph/resources","crossComponentResources":["{subscription}"],"gridSettings":{"formatters":[{"columnMatch":"Defender for AKS","formatter":18,"formatOptions":{"thresholdsOptions":"icons","thresholdsGrid":[{"operator":"==","thresholdValue":"no","representation":"4","text":""},{"operator":"Default","thresholdValue":null,"representation":"success","text":""}]}},{"columnMatch":"Onboard Microsoft Defender","formatter":7,"formatOptions":{"linkTarget":"Url","linkLabel":""}}]}},"customWidth":"66","name":"query - 9"},{"type":3,"content":{"version":"KqlItem/1.0","query":"datatable (Event:string)\\r\\n [\\"AKS Workbook\\"]\\r\\n| extend cluster = (strcat(\\"[\\", \\"{clustername}\\", \\"]\\"))\\r\\n| extend cluster = todynamic(replace(\\"\'\\", \'\\"\', cluster))\\r\\n| mvexpand cluster\\r\\n| extend subscriptionId = extract(@\\"/subscriptions/([^/]+)\\", 1, tostring(cluster))\\r\\n| summarize AksClusters = count() by subscriptionId, DefenderForAks = 0\\r\\n| union\\r\\n(\\r\\nsecurityresources\\r\\n| where type =~ \\"microsoft.security/pricings\\"\\r\\n| where name == \\"KubernetesService\\"\\r\\n| project DefenderForAks = iif(properties.pricingTier == \'Standard\', 1, 0), AksClusters = 0, subscriptionId\\r\\n)\\r\\n| summarize AksClusters = sum(AksClusters), DefenderForAks = sum(DefenderForAks) by subscriptionId\\r\\n| project Subscription = 1, [\'Defender for AKS\'] = iif(DefenderForAks > 0,\'Protected by Microsoft Defender\',\'Not protected by Microsoft Defender\')","size":0,"queryType":1,"resourceType":"microsoft.resourcegraph/resources","crossComponentResources":["{subscription}"],"visualization":"piechart"},"customWidth":"33","name":"query - 11"},{"type":1,"content":{"json":"### AKS alerts overview"},"name":"text - 21"},{"type":3,"content":{"version":"KqlItem/1.0","query":"SecurityAlert \\r\\n| where tolower(ResourceId) in ({clustername})\\r\\n| where TimeGenerated {timeframe}\\r\\n| where AlertType startswith \\"AKS_\\"\\r\\n| project image = tostring(todynamic(ExtendedProperties)[\\"Container image\\"]), AlertType\\r\\n| where image != \\"\\"\\r\\n| summarize AlertTypes = dcount(AlertType) by image\\r\\n| where AlertTypes > 1\\r\\n//| render piechart \\r\\n","size":4,"title":"Images with multiple alerts","queryType":0,"resourceType":"microsoft.operationalinsights/workspaces","crossComponentResources":["{workspaces}"],"visualization":"tiles","tileSettings":{"showBorder":false,"titleContent":{"columnMatch":"image","formatter":1},"leftContent":{"columnMatch":"AlertTypes","formatter":12,"formatOptions":{"palette":"auto"},"numberFormat":{"unit":17,"options":{"maximumSignificantDigits":3,"maximumFractionDigits":2}}}}},"customWidth":"33","name":"query - 12"},{"type":3,"content":{"version":"KqlItem/1.0","query":"SecurityAlert \\r\\n| where tolower(ResourceId) in ({clustername})\\r\\n| where TimeGenerated {timeframe}\\r\\n| where AlertType startswith \\"AKS_\\"\\r\\n| project AlertType, name = extract(@\\"/MICROSOFT.CONTAINERSERVICE/MANAGEDCLUSTERS/(.+)\\", 1, ResourceId)\\r\\n| summarize AlertTypes = dcount(AlertType) by name\\r\\n| where AlertTypes > 1\\r\\n","size":4,"title":"Clusters with multiple alert types","queryType":0,"resourceType":"microsoft.operationalinsights/workspaces","crossComponentResources":["{workspaces}"],"visualization":"tiles","tileSettings":{"showBorder":false,"titleContent":{"columnMatch":"name","formatter":1},"leftContent":{"columnMatch":"AlertTypes","formatter":12,"formatOptions":{"palette":"auto"},"numberFormat":{"unit":17,"options":{"maximumSignificantDigits":3,"maximumFractionDigits":2}}}}},"customWidth":"33","name":"query - 12 - Copy"},{"type":3,"content":{"version":"KqlItem/1.0","query":"SecurityAlert \\r\\n| where tolower(ResourceId) in ({clustername})\\r\\n| where TimeGenerated {timeframe}\\r\\n| where AlertType startswith \\"AKS_\\"\\r\\n| project AlertType, name = extract(@\\"/MICROSOFT.CONTAINERSERVICE/MANAGEDCLUSTERS/(.+)\\", 1, ResourceId)\\r\\n| summarize count() by name\\r\\n\\r\\n","size":4,"title":"Alerts triggered by cluster","queryType":0,"resourceType":"microsoft.operationalinsights/workspaces","crossComponentResources":["{workspaces}"],"visualization":"tiles","tileSettings":{"showBorder":false,"titleContent":{"columnMatch":"name","formatter":1},"leftContent":{"columnMatch":"count_","formatter":12,"formatOptions":{"palette":"auto"},"numberFormat":{"unit":17,"options":{"maximumSignificantDigits":3,"maximumFractionDigits":2}}}}},"customWidth":"33","name":"query - 12 - Copy - Copy"},{"type":1,"content":{"json":"### Seucirty alerts details\\r\\n\\r\\nTo filter, press on the severities below.\\r\\nYou can also filter based on a specific resource."},"name":"text - 18"},{"type":3,"content":{"version":"KqlItem/1.0","query":"SecurityAlert\\r\\n| where TimeGenerated {timeframe}\\r\\n| extend rg = extract(@\\"/resourcegroups/([^/]+)\\", 1, tolower(ResourceId))\\r\\n| extend sub = extract(@\\"/subscriptions/([^/]+)\\", 1, tolower(ResourceId))\\r\\n| extend nodesDetailsArr = todynamic(\'{nodeRgDetails}\')\\r\\n| mv-expand singleNodeDetails = nodesDetailsArr\\r\\n| extend singleNodeArr = split(singleNodeDetails, \\";\\")\\r\\n| where tolower(singleNodeArr[0]) == rg and tolower(singleNodeArr[1]) == sub\\r\\n| project AlertSeverity\\r\\n| union\\r\\n(\\r\\nSecurityAlert \\r\\n| where tolower(ResourceId) in ({clustername})\\r\\n| where TimeGenerated {timeframe}\\r\\n| where AlertType startswith \\"AKS_\\"\\r\\n| project AlertSeverity\\r\\n)\\r\\n| summarize count() by AlertSeverity","size":0,"title":"Alerts by severity","exportMultipleValues":true,"exportedParameters":[{"fieldName":"AlertSeverity","parameterName":"severity","parameterType":1,"quote":""}],"queryType":0,"resourceType":"microsoft.operationalinsights/workspaces","crossComponentResources":["{workspaces}"],"visualization":"tiles","tileSettings":{"showBorder":false,"titleContent":{"columnMatch":"AlertSeverity","formatter":1},"leftContent":{"columnMatch":"count_","formatter":12,"formatOptions":{"palette":"auto"},"numberFormat":{"unit":17,"options":{"maximumSignificantDigits":3,"maximumFractionDigits":2}}}}},"customWidth":"11","name":"Alerts by severity"},{"type":3,"content":{"version":"KqlItem/1.0","query":"SecurityAlert\\r\\n| where TimeGenerated {timeframe}\\r\\n| where \\"{severity}\\" has AlertSeverity or isempty(\\"{severity}\\")\\r\\n| extend rg = extract(@\\"/resourcegroups/([^/]+)\\", 1, tolower(ResourceId))\\r\\n| extend sub = extract(@\\"/subscriptions/([^/]+)\\", 1, tolower(ResourceId))\\r\\n| extend nodesDetailsArr = todynamic(\'{nodeRgDetails}\')\\r\\n| mv-expand singleNodeDetails = nodesDetailsArr\\r\\n| extend singleNodeArr = split(singleNodeDetails, \\";\\")\\r\\n| where tolower(singleNodeArr[0]) == rg and tolower(singleNodeArr[1]) == sub\\r\\n| project ResourceId\\r\\n| union\\r\\n(\\r\\nSecurityAlert \\r\\n| where tolower(ResourceId) in ({clustername})\\r\\n| where \\"{severity}\\" has AlertSeverity or isempty(\\"{severity}\\")\\r\\n| where TimeGenerated {timeframe}\\r\\n| where AlertType startswith \\"AKS_\\"\\r\\n| project ResourceId\\r\\n)\\r\\n| summarize Alerts = count() by ResourceId\\r\\n| order by Alerts desc\\r\\n| limit 10","size":0,"title":"Resources with most alerts","exportFieldName":"ResourceId","exportParameterName":"selectedResource","exportDefaultValue":"not_selected","queryType":0,"resourceType":"microsoft.operationalinsights/workspaces","crossComponentResources":["{workspaces}"],"gridSettings":{"formatters":[{"columnMatch":"Alerts","formatter":4,"formatOptions":{"palette":"red"}}]}},"customWidth":"22","name":"Resources with most alerts"},{"type":3,"content":{"version":"KqlItem/1.0","query":"\\r\\nSecurityAlert\\r\\n| where TimeGenerated {timeframe}\\r\\n| where \\"{severity}\\" has AlertSeverity or isempty(\\"{severity}\\")\\r\\n| extend rg = extract(@\\"/resourcegroups/([^/]+)\\", 1, tolower(ResourceId))\\r\\n| extend sub = extract(@\\"/subscriptions/([^/]+)\\", 1, tolower(ResourceId))\\r\\n| extend nodesDetailsArr = todynamic(\'{nodeRgDetails}\')\\r\\n| mv-expand singleNodeDetails = nodesDetailsArr\\r\\n| extend singleNodeArr = split(singleNodeDetails, \\";\\")\\r\\n| where tolower(singleNodeArr[0]) == rg and tolower(singleNodeArr[1]) == sub\\r\\n| extend AlertResourceType = \\"VM alerts\\"\\r\\n| union\\r\\n(\\r\\nSecurityAlert \\r\\n| where tolower(ResourceId) in ({clustername})\\r\\n| where \\"{severity}\\" has AlertSeverity or isempty(\\"{severity}\\")\\r\\n| where TimeGenerated {timeframe}\\r\\n| where AlertType startswith \\"AKS_\\"\\r\\n| extend AlertResourceType = \\"Cluster alerts\\"\\r\\n)\\r\\n| summarize Alerts = count() by bin(TimeGenerated, {timeframe:grain}), AlertResourceType","size":0,"title":"Alerts over time","queryType":0,"resourceType":"microsoft.operationalinsights/workspaces","crossComponentResources":["{workspaces}"],"visualization":"timechart"},"customWidth":"66","name":"Alerts over time"},{"type":3,"content":{"version":"KqlItem/1.0","query":"SecurityAlert\\r\\n| where TimeGenerated {timeframe}\\r\\n| where \\"{severity}\\" has AlertSeverity or isempty(\\"{severity}\\")\\r\\n| extend rg = extract(@\\"/resourcegroups/([^/]+)\\", 1, tolower(ResourceId))\\r\\n| extend sub = extract(@\\"/subscriptions/([^/]+)\\", 1, tolower(ResourceId))\\r\\n| extend nodesDetailsArr = todynamic(\'{nodeRgDetails}\')\\r\\n| mv-expand singleNodeDetails = nodesDetailsArr\\r\\n| extend singleNodeArr = split(singleNodeDetails, \\";\\")\\r\\n| where tolower(singleNodeArr[0]) == rg and tolower(singleNodeArr[1]) == sub\\r\\n| where tolower(ResourceId) == tolower(\\"{selectedResource}\\") or \\"{selectedResource}\\" == \\"not_selected\\"\\r\\n| project [\\"Resource name\\"] = ResourceId, TimeGenerated, AlertSeverity, [\\"AKS cluster\\"] = toupper(singleNodeArr[2]), DisplayName, AlertLink\\r\\n| union\\r\\n(\\r\\nSecurityAlert\\r\\n| where TimeGenerated {timeframe}\\r\\n| where tolower(ResourceId) in ({clustername})\\r\\n| where \\"{severity}\\" has AlertSeverity or isempty(\\"{severity}\\")\\r\\n| where AlertType startswith \\"AKS_\\"\\r\\n| where tolower(ResourceId) == tolower(\\"{selectedResource}\\") or \\"{selectedResource}\\" == \\"not_selected\\"\\r\\n| project [\\"Resource name\\"] = ResourceId, TimeGenerated, AlertSeverity, [\\"AKS cluster\\"] = ResourceId, DisplayName, AlertLink\\r\\n)\\r\\n| order by TimeGenerated asc","size":0,"title":"Microsoft Defender alerts","queryType":0,"resourceType":"microsoft.operationalinsights/workspaces","crossComponentResources":["{workspaces}"],"gridSettings":{"formatters":[{"columnMatch":"AlertLink","formatter":7,"formatOptions":{"linkTarget":"Url","linkLabel":"Go to alert "}}],"filter":true},"sortBy":[]},"name":"Microsoft Defender alerts","styleSettings":{"showBorder":true}}]},"conditionalVisibility":{"parameterName":"mainTab","comparison":"isEqualTo","value":"alerts"},"name":"Defender Alerts"},{"type":12,"content":{"version":"NotebookGroup/1.0","groupType":"editable","items":[{"type":1,"content":{"json":"## Diagnostic logs coverage"},"name":"text - 15"},{"type":3,"content":{"version":"KqlItem/1.0","query":"union withsource=_TableName *\\r\\n| where _TableName == \\"AzureDiagnostics\\" and Category == \\"kube-audit\\"\\r\\n| summarize count() by ResourceId = tolower(ResourceId)\\r\\n| summarize logsClusters = make_set(ResourceId)\\r\\n| extend selectedClusters = \\"[{clustername}]\\"\\r\\n| extend selectedClusters = replace(\\"\'\\", \'\\"\', selectedClusters)\\r\\n| extend selectedClusters = todynamic(selectedClusters)\\r\\n| mv-expand clusterId = selectedClusters\\r\\n| project clusterId = toupper(tostring(clusterId)), [\\"Diagnostic logs\\"] = (logsClusters has tostring(clusterId))\\r\\n| extend [\\"Diagnostic settings\\"] = iff([\\"Diagnostic logs\\"] == false, strcat(\\"https://ms.portal.azure.com/#@microsoft.onmicrosoft.com/resource\\", clusterId, \\"/diagnostics\\"), \\"\\")\\r\\n","size":0,"timeContext":{"durationMs":172800000},"timeContextFromParameter":"timeframe","queryType":0,"resourceType":"microsoft.operationalinsights/workspaces","crossComponentResources":["{workspaces}"],"gridSettings":{"formatters":[{"columnMatch":"Diagnostic logs","formatter":18,"formatOptions":{"thresholdsOptions":"icons","thresholdsGrid":[{"operator":"==","thresholdValue":"false","representation":"critical","text":""},{"operator":"Default","thresholdValue":null,"representation":"success","text":""}]}},{"columnMatch":"Diagnostic settings","formatter":7,"formatOptions":{"linkTarget":"Url"}}],"filter":true,"sortBy":[{"itemKey":"$gen_thresholds_Diagnostic logs_1","sortOrder":2}]},"sortBy":[{"itemKey":"$gen_thresholds_Diagnostic logs_1","sortOrder":2}]},"customWidth":"66","name":"query - 14"},{"type":3,"content":{"version":"KqlItem/1.0","query":"union withsource=_TableName *\\r\\n| where _TableName == \\"AzureDiagnostics\\" and Category == \\"kube-audit\\"\\r\\n| summarize count() by ResourceId = tolower(ResourceId)\\r\\n| summarize logsClusters = make_set(ResourceId)\\r\\n| extend selectedClusters = \\"[{clustername}]\\"\\r\\n| extend selectedClusters = replace(\\"\'\\", \'\\"\', selectedClusters)\\r\\n| extend selectedClusters = todynamic(selectedClusters)\\r\\n| mv-expand clusterId = selectedClusters\\r\\n| project clusterId = toupper(tostring(clusterId)), hasDiagnosticLogs = (logsClusters has tostring(clusterId))\\r\\n| summarize [\\"number of clusters\\"] = count() by hasDiagnosticLogs\\r\\n| extend hasDiagnosticLogs = iff(hasDiagnosticLogs == true, \\"Clusters with Diagnostic logs\\", \\"Clusters without Diagnostic logs\\")\\r\\n","size":0,"timeContext":{"durationMs":172800000},"timeContextFromParameter":"timeframe","queryType":0,"resourceType":"microsoft.operationalinsights/workspaces","crossComponentResources":["{workspaces}"],"visualization":"piechart"},"customWidth":"33","name":"query - 17"},{"type":12,"content":{"version":"NotebookGroup/1.0","groupType":"editable","items":[{"type":1,"content":{"json":"## Cluster operations"},"name":"text - 16"},{"type":11,"content":{"version":"LinkItem/1.0","style":"tabs","links":[{"id":"3f616701-fd4b-482c-aff1-a85414daa05c","cellValue":"dispalyedGraph","linkTarget":"parameter","linkLabel":"Masterclient operations","subTarget":"masterclient","preText":"","style":"link"},{"id":"e6fa55f1-7d57-4f5e-8e83-429740853731","cellValue":"dispalyedGraph","linkTarget":"parameter","linkLabel":"Pod creation operations","subTarget":"podCreation","style":"link"},{"id":"f4c46251-0090-4ca3-a81c-0686bff3ff35","cellValue":"dispalyedGraph","linkTarget":"parameter","linkLabel":"Secret get\\\\list operations","subTarget":"secretOperation","style":"link"}]},"name":"links - 11"},{"type":3,"content":{"version":"KqlItem/1.0","query":"AzureDiagnostics \\r\\n| where tolower(ResourceId) in ({clustername})\\r\\n| where TimeGenerated {timeframe}\\r\\n| where Category == \\"kube-audit\\"\\r\\n| where log_s has \\"masterclient\\"\\r\\n| project TimeGenerated, parse_json(log_s), ResourceId\\r\\n| project TimeGenerated, ResourceId, username = tostring(log_s[\\"user\\"].username)\\r\\n| where username == \\"masterclient\\"\\r\\n| extend name = extract(@\\"/MICROSOFT.CONTAINERSERVICE/MANAGEDCLUSTERS/(.+)\\", 1, ResourceId)\\r\\n| summarize count() by name, bin(TimeGenerated, 1h)","size":0,"queryType":0,"resourceType":"microsoft.operationalinsights/workspaces","crossComponentResources":["{workspaces}"],"visualization":"timechart","tileSettings":{"showBorder":false,"titleContent":{"columnMatch":"name","formatter":1},"leftContent":{"columnMatch":"count_","formatter":12,"formatOptions":{"palette":"auto"},"numberFormat":{"unit":17,"options":{"maximumSignificantDigits":3,"maximumFractionDigits":2}}}},"chartSettings":{"yAxis":["count_"]}},"conditionalVisibility":{"parameterName":"dispalyedGraph","comparison":"isEqualTo","value":"masterclient"},"name":"Masterclient operations","styleSettings":{"showBorder":true}},{"type":3,"content":{"version":"KqlItem/1.0","query":"AzureDiagnostics\\r\\n| where Category == \\"kube-audit\\"\\r\\n| where tolower(ResourceId) in ({clustername})\\r\\n| where TimeGenerated {timeframe}\\r\\n| where log_s has \\"pods\\"\\r\\n| project TimeGenerated, parse_json(log_s), ResourceId\\r\\n| project AzureResourceId = ResourceId, TimeGenerated,\\r\\n RequestURI = tostring(log_s[\\"requestURI\\"]),\\r\\n Verb = tostring(log_s[\\"verb\\"]),\\r\\n ObjectRef = log_s[\\"objectRef\\"],\\r\\n ResponseStatus = log_s[\\"responseStatus\\"]\\r\\n//Main query\\r\\n| where ObjectRef.resource == \\"pods\\" and Verb == \\"create\\" and ResponseStatus.code startswith \\"20\\"\\r\\n and RequestURI endswith \\"/pods\\"\\r\\n| extend name = extract(@\\"/MICROSOFT.CONTAINERSERVICE/MANAGEDCLUSTERS/(.+)\\", 1, AzureResourceId)\\r\\n| summarize count() by name, bin(TimeGenerated, 1h)","size":0,"queryType":0,"resourceType":"microsoft.operationalinsights/workspaces","crossComponentResources":["{workspaces}"],"visualization":"timechart"},"conditionalVisibility":{"parameterName":"dispalyedGraph","comparison":"isEqualTo","value":"podCreation"},"name":"pods creation","styleSettings":{"showBorder":true}},{"type":3,"content":{"version":"KqlItem/1.0","query":"AzureDiagnostics\\r\\n| where Category == \\"kube-audit\\"\\r\\n| where tolower(ResourceId) in ({clustername})\\r\\n| where TimeGenerated {timeframe}\\r\\n| where log_s has \\"secrets\\"\\r\\n| project TimeGenerated, parse_json(log_s), ResourceId\\r\\n| project AzureResourceId = ResourceId, TimeGenerated,\\r\\n RequestURI = tostring(log_s[\\"requestURI\\"]),\\r\\n Verb = tostring(log_s[\\"verb\\"]),\\r\\n ObjectRef = log_s[\\"objectRef\\"],\\r\\n ResponseStatus = log_s[\\"responseStatus\\"]\\r\\n//Main query\\r\\n| where ObjectRef.resource == \\"secrets\\" and (Verb == \\"list\\" or Verb == \\"get\\") and ResponseStatus.code startswith \\"20\\"\\r\\n| where ObjectRef.name != \\"tunnelfront\\" and ObjectRef.name != \\"tunnelend\\" and ObjectRef.name != \\"kubernetes-dashboard-key-holder\\"\\r\\n| extend name = extract(@\\"/MICROSOFT.CONTAINERSERVICE/MANAGEDCLUSTERS/(.+)\\", 1, AzureResourceId)\\r\\n| summarize count() by name, bin(TimeGenerated, 1h)","size":0,"timeContext":{"durationMs":172800000},"timeContextFromParameter":"timeframe","queryType":0,"resourceType":"microsoft.operationalinsights/workspaces","crossComponentResources":["{workspaces}"],"visualization":"timechart","gridSettings":{"sortBy":[{"itemKey":"count_","sortOrder":2}]},"sortBy":[{"itemKey":"count_","sortOrder":2}]},"conditionalVisibility":{"parameterName":"dispalyedGraph","comparison":"isEqualTo","value":"secretOperation"},"name":"secrets operation","styleSettings":{"showBorder":true}},{"type":3,"content":{"version":"KqlItem/1.0","query":"let ascAlerts = \\nunion withsource=_TableName *\\n| where _TableName == \\"SecurityAlert\\"\\n| where tolower(ResourceId) in ({clustername})\\n| where TimeGenerated {timeframe}\\n| extend AlertType = column_ifexists(\\"AlertType\\", \\"\\")\\n| where AlertType == \\"AKS_PrivilegedContainer\\"\\n| extend ExtendedProperties = column_ifexists(\\"ExtendedProperties\\", todynamic(\\"\\"))\\n| extend ExtendedProperties = parse_json(ExtendedProperties)\\n| extend AlertLink = column_ifexists(\\"AlertLink\\", \\"\\")\\n| summarize arg_min(TimeGenerated, AlertLink) by AzureResourceId = ResourceId, name = tostring(ExtendedProperties[\\"Pod name\\"]), podNamespace = tostring(ExtendedProperties[\\"Namespace\\"])\\n;\\nlet podOperations = AzureDiagnostics\\n| where Category == \\"kube-audit\\"\\n| where tolower(ResourceId) in ({clustername})\\n| where TimeGenerated {timeframe}\\n| where log_s has \\"privileged\\"\\n| project TimeGenerated, parse_json(log_s), ResourceId\\n| project AzureResourceId = ResourceId, TimeGenerated,\\n RequestURI = tostring(log_s[\\"requestURI\\"]),\\n Verb = tostring(log_s[\\"verb\\"]),\\n ObjectRef = log_s[\\"objectRef\\"],\\n RequestObject = log_s[\\"requestObject\\"],\\n ResponseStatus = log_s[\\"responseStatus\\"],\\n ResponseObject = log_s[\\"responseObject\\"]\\n//Main query\\n| where ObjectRef.resource == \\"pods\\" and Verb == \\"create\\" and ResponseStatus.code startswith \\"20\\" and RequestObject has \\"privileged\\"\\n and RequestURI endswith \\"/pods\\"\\n| extend containers = RequestObject.spec.containers\\n| mvexpand containers\\n| where containers.securityContext.privileged == true\\n| summarize TimeGenerated = min(TimeGenerated) by\\n name = tostring(ResponseObject.metadata.name),\\n podNamespace = tostring(ResponseObject.metadata.namespace),\\n imageName = tostring(containers.image),\\n containerName = tostring(containers.name),\\n AzureResourceId\\n| extend id = strcat(name,\\";\\", AzureResourceId)\\n| extend parent = AzureResourceId\\n| join kind=leftouter (ascAlerts) on AzureResourceId, name, podNamespace\\n;\\nlet cached = materialize(podOperations)\\n;\\nlet clusters = cached | distinct AzureResourceId\\n;\\n// Main query\\ncached\\n| union\\n(\\nclusters\\n| project \\n name = AzureResourceId,\\n id = AzureResourceId,\\n parent = \\"\\" \\n)\\n| project-away name1, podNamespace1, TimeGenerated1","size":1,"title":"Privileged containers creation","queryType":0,"resourceType":"microsoft.operationalinsights/workspaces","crossComponentResources":["{workspaces}"],"visualization":"table","gridSettings":{"formatters":[{"columnMatch":"name","formatter":13,"formatOptions":{"linkTarget":null,"showIcon":true}},{"columnMatch":"AzureResourceId","formatter":5},{"columnMatch":"id","formatter":5},{"columnMatch":"parent","formatter":5},{"columnMatch":"AlertLink","formatter":7,"formatOptions":{"linkTarget":"Url","linkLabel":""}}],"hierarchySettings":{"idColumn":"id","parentColumn":"parent","treeType":0,"expanderColumn":"name"}},"sortBy":[]},"customWidth":"66","name":"Privileged container","styleSettings":{"showBorder":true}},{"type":3,"content":{"version":"KqlItem/1.0","query":"SecurityAlert \\r\\n| where tolower(ResourceId) in ({clustername})\\r\\n| where TimeGenerated {timeframe}\\r\\n| where AlertType == \\"AKS_PrivilegedContainer\\"\\r\\n| project name = extract(@\\"/MICROSOFT.CONTAINERSERVICE/MANAGEDCLUSTERS/(.+)\\", 1, ResourceId)\\r\\n| summarize alert = count() by name","size":1,"title":"AKS clusters with related Microsoft Defender alerts","queryType":0,"resourceType":"microsoft.operationalinsights/workspaces","crossComponentResources":["{workspaces}"],"visualization":"piechart","tileSettings":{"showBorder":false,"titleContent":{"columnMatch":"name","formatter":1},"leftContent":{"columnMatch":"alert","formatter":12,"formatOptions":{"palette":"auto"},"numberFormat":{"unit":17,"options":{"maximumSignificantDigits":3,"maximumFractionDigits":2}}}},"graphSettings":{"type":0,"topContent":{"columnMatch":"name","formatter":1},"centerContent":{"columnMatch":"alert","formatter":1,"numberFormat":{"unit":17,"options":{"maximumSignificantDigits":3,"maximumFractionDigits":2}}}}},"customWidth":"33","name":"query - 7","styleSettings":{"showBorder":true}},{"type":3,"content":{"version":"KqlItem/1.0","query":"let baseQuery = AzureDiagnostics \\r\\n| where Category == \\"kube-audit\\"\\r\\n| where tolower(ResourceId) in ({clustername})\\r\\n| where TimeGenerated {timeframe}\\r\\n| where log_s has \\"exec\\"\\r\\n| project TimeGenerated, parse_json(log_s), ResourceId\\r\\n| project TimeGenerated,\\r\\n AzureResourceId = ResourceId,\\r\\n User = log_s[\\"user\\"],\\r\\n StageTimestamp = todatetime(log_s[\\"stageTimestamp\\"]),\\r\\n Timestamp = todatetime(log_s[\\"timestamp\\"]),\\r\\n Stage = tostring(log_s[\\"stage\\"]),\\r\\n RequestURI = tostring(log_s[\\"requestURI\\"]),\\r\\n UserAgent = tostring(log_s[\\"userAgent\\"]),\\r\\n Verb = tostring(log_s[\\"verb\\"]),\\r\\n ObjectRef = log_s[\\"objectRef\\"],\\r\\n ResponseStatus = log_s[\\"responseStatus\\"]\\r\\n| where ObjectRef.resource == \\"pods\\" and Verb == \\"create\\" and ResponseStatus.code == 101 and ObjectRef.subresource == \\"exec\\"\\r\\n| project operationTime = TimeGenerated,\\r\\n RequestURI,\\r\\n podName = tostring(ObjectRef.name),\\r\\n podNamespace = tostring(ObjectRef.namespace),\\r\\n username = tostring(User.username),\\r\\n AzureResourceId\\r\\n// Parse the exec command\\r\\n| extend commands = extractall(@\\"command=([^\\\\&]*)\\", RequestURI)\\r\\n| extend commandsStr = url_decode(strcat_array(commands, \\" \\"))\\r\\n| project-away [\'commands\'], RequestURI\\r\\n| where username != \\"aksProblemDetector\\"\\r\\n;\\r\\nlet cached = materialize(baseQuery);\\r\\nlet execOperations = \\r\\ncached\\r\\n| summarize operationTime = min(operationTime), numberOfPerations = count() by name = commandsStr, username, podNamespace, podName, AzureResourceId\\r\\n| extend id = name\\r\\n| extend parentId = podName\\r\\n| project id, parentId, name, operationTime, numberOfPerations, podNamespace, username, AzureResourceId\\r\\n;\\r\\nlet podOperations = \\r\\ncached\\r\\n| summarize operationTime = min(operationTime), numberOfPerations = count() by name = podName, podNamespace, AzureResourceId\\r\\n| extend id = name\\r\\n| extend parentId = AzureResourceId\\r\\n| project id, parentId, name, operationTime, numberOfPerations, podNamespace, username = \\"\\", AzureResourceId\\r\\n;\\r\\nlet clusterOperations = \\r\\ncached\\r\\n| summarize operationTime = min(operationTime), numberOfPerations = count() by name = AzureResourceId\\r\\n| extend id = name\\r\\n| extend parentId = \\"\\"\\r\\n| project id, parentId, name, operationTime, numberOfPerations, username = \\"\\", podNamespace = \\"\\", AzureResourceId = name\\r\\n;\\r\\nunion clusterOperations, podOperations, execOperations","size":1,"title":"exec commands","queryType":0,"resourceType":"microsoft.operationalinsights/workspaces","crossComponentResources":["{workspaces}"],"visualization":"table","gridSettings":{"formatters":[{"columnMatch":"id","formatter":5},{"columnMatch":"parentId","formatter":5},{"columnMatch":"numberOfPerations","formatter":4,"formatOptions":{"palette":"blue","compositeBarSettings":{"labelText":"","columnSettings":[]}}},{"columnMatch":"AzureResourceId","formatter":5}],"hierarchySettings":{"idColumn":"id","parentColumn":"parentId","treeType":0,"expanderColumn":"name","expandTopLevel":false}}},"customWidth":"33","name":"exec commands","styleSettings":{"showBorder":true}},{"type":3,"content":{"version":"KqlItem/1.0","query":"SecurityAlert \\r\\n| where AlertType == \\"AKS_MaliciousContainerExec\\"\\r\\n| where tolower(ResourceId) in ({clustername})\\r\\n| where TimeGenerated {timeframe}\\r\\n| project TimeGenerated, ResourceId, ExtendedProperties = todynamic(ExtendedProperties)\\r\\n| project TimeGenerated, ResourceId, [\\"Pod name\\"] = ExtendedProperties[\\"Pod name\\"], Command = ExtendedProperties[\\"Command\\"]","size":1,"title":"Related Microsoft Defender alerts details","queryType":0,"resourceType":"microsoft.operationalinsights/workspaces","crossComponentResources":["{workspaces}"],"gridSettings":{"sortBy":[{"itemKey":"TimeGenerated","sortOrder":1}]},"sortBy":[{"itemKey":"TimeGenerated","sortOrder":1}]},"customWidth":"33","name":"query - 9","styleSettings":{"showBorder":true}},{"type":3,"content":{"version":"KqlItem/1.0","query":"SecurityAlert \\r\\n| where AlertType == \\"AKS_MaliciousContainerExec\\"\\r\\n| where tolower(ResourceId) in ({clustername})\\r\\n| where TimeGenerated {timeframe}\\r\\n| project name = extract(@\\"/MICROSOFT.CONTAINERSERVICE/MANAGEDCLUSTERS/(.+)\\", 1, ResourceId)\\r\\n| summarize alert = count() by name","size":1,"title":"AKS clusters with related Microsoft Defender alerts","queryType":0,"resourceType":"microsoft.operationalinsights/workspaces","crossComponentResources":["{workspaces}"],"visualization":"piechart"},"customWidth":"33","name":"query - 8","styleSettings":{"showBorder":true}},{"type":3,"content":{"version":"KqlItem/1.0","query":"let ascAlerts = \\r\\nunion withsource=_TableName *\\r\\n| where _TableName == \\"SecurityAlert\\"\\r\\n| where tolower(ResourceId) in ({clustername})\\r\\n| where TimeGenerated {timeframe}\\r\\n| extend AlertType = column_ifexists(\\"AlertType\\", \\"\\")\\r\\n| where AlertType == \\"AKS_SensitiveMount\\"\\r\\n| extend ExtendedProperties = column_ifexists(\\"ExtendedProperties\\", todynamic(\\"\\"))\\r\\n| extend ExtendedProperties = parse_json(ExtendedProperties)\\r\\n| extend AlertLink = column_ifexists(\\"AlertLink\\", \\"\\")\\r\\n| summarize arg_min(TimeGenerated, AlertLink) by AzureResourceId = ResourceId, containerName = tostring(ExtendedProperties[\\"Container name\\"]), mountPath = tostring(ExtendedProperties[\\"Sensitive mount path\\"])\\r\\n;\\r\\nlet podOperations = \\r\\nAzureDiagnostics \\r\\n| where tolower(ResourceId) in ({clustername})\\r\\n| where TimeGenerated {timeframe}\\r\\n| where Category == \\"kube-audit\\"\\r\\n| where log_s has \\"hostPath\\"\\r\\n| project TimeGenerated, parse_json(log_s), ResourceId\\r\\n//Parsing\\r\\n| project AzureResourceId = ResourceId, TimeGenerated,\\r\\n Verb = tostring(log_s[\\"verb\\"]),\\r\\n ObjectRef = log_s[\\"objectRef\\"],\\r\\n RequestObject = log_s[\\"requestObject\\"],\\r\\n ResponseStatus = log_s[\\"responseStatus\\"],\\r\\n ResponseObject = log_s[\\"responseObject\\"]\\r\\n//\\r\\n//Main query\\r\\n//\\r\\n| where ObjectRef.resource == \\"pods\\" and Verb == \\"create\\" and ResponseStatus.code startswith \\"20\\" and RequestObject has \\"hostPath\\"\\r\\n| extend volumes = RequestObject.spec.volumes\\r\\n| mvexpand volumes\\r\\n| extend mountPath = volumes.hostPath.path\\r\\n| where mountPath != \\"\\" \\r\\n| extend container = RequestObject.spec.containers\\r\\n| mvexpand container\\r\\n| extend detectionTime = TimeGenerated\\r\\n| project detectionTime,\\r\\n podName = ResponseObject.metadata.name,\\r\\n podNamespace = ResponseObject.metadata.namespace,\\r\\n containerName = container.name,\\r\\n containerImage = container.image,\\r\\n mountPath,\\r\\n mountName = volumes.name,\\r\\n AzureResourceId,\\r\\n container\\r\\n| extend volumeMounts = container.volumeMounts\\r\\n| mv-expand volumeMounts\\r\\n| where tostring(volumeMounts.name) == tostring(mountName)\\r\\n| summarize operationTime = min(detectionTime) by AzureResourceId, name = tostring(podName),tostring(podNamespace), tostring(containerName), tostring(containerImage), tostring(mountPath), tostring(mountName)\\r\\n| extend id = strcat(name, \\";\\", AzureResourceId)\\r\\n| extend parent = AzureResourceId\\r\\n| join kind=leftouter (ascAlerts) on AzureResourceId, containerName, mountPath\\r\\n;\\r\\nlet cached = materialize(podOperations)\\r\\n;\\r\\nlet clusters = cached | distinct AzureResourceId\\r\\n;\\r\\n// Main query\\r\\ncached\\r\\n| union\\r\\n(\\r\\nclusters\\r\\n| project \\r\\n name = toupper(AzureResourceId),\\r\\n id = AzureResourceId,\\r\\n parent = \\"\\" \\r\\n)\\r\\n| project-away containerName1, mountPath1, TimeGenerated\\r\\n","size":1,"title":"hostPath mount","queryType":0,"resourceType":"microsoft.operationalinsights/workspaces","crossComponentResources":["{workspaces}"],"gridSettings":{"formatters":[{"columnMatch":"AzureResourceId","formatter":5},{"columnMatch":"name","formatter":13,"formatOptions":{"linkTarget":null,"showIcon":true}},{"columnMatch":"id","formatter":5},{"columnMatch":"parent","formatter":5},{"columnMatch":"AzureResourceId1","formatter":5},{"columnMatch":"AlertLink","formatter":7,"formatOptions":{"linkTarget":"Url"}}],"hierarchySettings":{"idColumn":"id","parentColumn":"parent","treeType":0,"expanderColumn":"name"}},"sortBy":[]},"customWidth":"66","name":"query - 10","styleSettings":{"showBorder":true}},{"type":3,"content":{"version":"KqlItem/1.0","query":"SecurityAlert \\r\\n| where AlertType == \\"AKS_SensitiveMount\\"\\r\\n| where tolower(ResourceId) in ({clustername})\\r\\n| where TimeGenerated {timeframe}\\r\\n| project name = extract(@\\"/MICROSOFT.CONTAINERSERVICE/MANAGEDCLUSTERS/(.+)\\", 1, ResourceId)\\r\\n| summarize alert = count() by name","size":1,"title":"AKS clusters with related Microsoft Defender alerts","queryType":0,"resourceType":"microsoft.operationalinsights/workspaces","crossComponentResources":["{workspaces}"],"visualization":"piechart","sortBy":[]},"customWidth":"33","name":"query - 10","styleSettings":{"showBorder":true}},{"type":3,"content":{"version":"KqlItem/1.0","query":"let bindingOper = AzureDiagnostics\\r\\n| where Category == \\"kube-audit\\"\\r\\n| where tolower(ResourceId) in ({clustername})\\r\\n| where TimeGenerated {timeframe}\\r\\n| where log_s has \\"clusterrolebindings\\"\\r\\n| project TimeGenerated, parse_json(log_s), ResourceId\\r\\n//Parsing\\r\\n| project AzureResourceId = ResourceId, TimeGenerated,\\r\\n RequestURI = tostring(log_s[\\"requestURI\\"]),\\r\\n User = log_s[\\"user\\"],\\r\\n Verb = tostring(log_s[\\"verb\\"]),\\r\\n ObjectRef = log_s[\\"objectRef\\"],\\r\\n RequestObject = log_s[\\"requestObject\\"],\\r\\n ResponseStatus = log_s[\\"responseStatus\\"]\\r\\n| where ObjectRef.resource == \\"clusterrolebindings\\" and Verb == \\"create\\" and ResponseStatus.code startswith \\"20\\" and RequestObject.roleRef.name == \\"cluster-admin\\" \\r\\n| extend subjects = RequestObject.subjects\\r\\n| mv-expand subjects\\r\\n| project AzureResourceId, TimeGenerated, subjectName = tostring(subjects.name), subjectKind = tostring(subjects[\\"kind\\"]), bindingName = tostring(ObjectRef.name)\\r\\n| summarize operationTime = min(TimeGenerated) by AzureResourceId, subjectName, subjectKind, bindingName\\r\\n| extend id = strcat(subjectName, \\";\\", AzureResourceId)\\r\\n| extend parent = AzureResourceId\\r\\n;\\r\\nlet cached = materialize(bindingOper)\\r\\n;\\r\\nlet clusters = cached | distinct AzureResourceId\\r\\n;\\r\\n// Main query\\r\\ncached\\r\\n| union\\r\\n(\\r\\nclusters\\r\\n| project \\r\\n name = AzureResourceId,\\r\\n id = AzureResourceId,\\r\\n parent = \\"\\" \\r\\n)","size":1,"title":"Cluster-admin binding","queryType":0,"resourceType":"microsoft.operationalinsights/workspaces","crossComponentResources":["{workspaces}"],"visualization":"table","gridSettings":{"formatters":[{"columnMatch":"AzureResourceId","formatter":5},{"columnMatch":"id","formatter":5},{"columnMatch":"parent","formatter":5},{"columnMatch":"name","formatter":13,"formatOptions":{"linkTarget":null,"showIcon":true}}],"hierarchySettings":{"idColumn":"id","parentColumn":"parent","treeType":0,"expanderColumn":"name"}}},"customWidth":"66","name":"query - 5","styleSettings":{"showBorder":true}},{"type":3,"content":{"version":"KqlItem/1.0","query":"SecurityAlert \\r\\n| where tolower(ResourceId) in ({clustername})\\r\\n| where TimeGenerated {timeframe}\\r\\n| where AlertType == \\"AKS_ClusterAdminBinding\\"\\r\\n| project name = extract(@\\"/MICROSOFT.CONTAINERSERVICE/MANAGEDCLUSTERS/(.+)\\", 1, ResourceId)\\r\\n| summarize count() by name","size":1,"title":"AKS clusters with related Microsoft Defender alerts","queryType":0,"resourceType":"microsoft.operationalinsights/workspaces","crossComponentResources":["{workspaces}"],"visualization":"piechart"},"customWidth":"33","name":"query - 11","styleSettings":{"showBorder":true}},{"type":3,"content":{"version":"KqlItem/1.0","query":"AzureDiagnostics\\r\\n| where Category == \\"kube-audit\\"\\r\\n| where tolower(ResourceId) in ({clustername})\\r\\n| where TimeGenerated {timeframe}\\r\\n| where log_s has \\"events\\"\\r\\n| project TimeGenerated, parse_json(log_s), ResourceId\\r\\n//Parsing\\r\\n| project AzureResourceId = ResourceId, \\r\\n TimeGenerated,\\r\\n SourceIPs = tostring(log_s[\\"sourceIPs\\"][0]),\\r\\n User = log_s[\\"user\\"],\\r\\n Verb = tostring(log_s[\\"verb\\"]),\\r\\n ObjectRef = log_s[\\"objectRef\\"],\\r\\n ResponseStatus = log_s[\\"responseStatus\\"]\\r\\n| where ObjectRef.resource == \\"events\\" and Verb == \\"delete\\" and ResponseStatus.code == 200\\r\\n| project TimeGenerated, AzureResourceId, username = tostring(User.username), ipAddr = tostring(SourceIPs), \\r\\n eventName = tostring(ObjectRef.name), eventNamespace = tostring(ObjectRef.namespace), status = tostring(ResponseStatus.code)\\r\\n| summarize operationTime = min(TimeGenerated), eventNames = make_set(eventName, 10) by\\r\\n AzureResourceId, \\r\\n eventNamespace,\\r\\n username,\\r\\n ipAddr\\r\\n// Format the list of the event names\\r\\n| extend eventNames = substring(eventNames, 1 , strlen(eventNames) - 2)\\r\\n| extend eventNames = replace(\'\\"\', \\"\\", eventNames)\\r\\n| extend eventNames = replace(\\",\\", \\", \\", eventNames)","size":1,"title":"Delete events","queryType":0,"resourceType":"microsoft.operationalinsights/workspaces","crossComponentResources":["{workspaces}"]},"name":"query - 6","styleSettings":{"showBorder":true}}]},"conditionalVisibility":{"parameterName":"diagnosticClusters","comparison":"isEqualTo","value":"yes"},"name":"diagnosticData"},{"type":1,"content":{"json":"No Diagnostic Logs data in the selected workspaces. \\r\\nTo enable Diagnostic Logs for your AKS cluster: Go to your AKS cluster --> Diagnostic settings --> Add diagnostic setting --> Select \\"kube-audit\\" and send the data to your workspace.\\r\\n\\r\\nGet more details here: https://learn.microsoft.com/azure/aks/view-master-logs","style":"info"},"conditionalVisibility":{"parameterName":"diagnosticClusters","comparison":"isEqualTo","value":"no"},"name":"text - 4"}]},"conditionalVisibility":{"parameterName":"mainTab","comparison":"isEqualTo","value":"diagnostics"},"name":"diagnostics"}],"fromTemplateId":"sentinel-AksWorkbook","$schema":"https://github.com/Microsoft/Application-Insights-Workbooks/blob/master/schema/workbook.json"}' + version: '1.0' + category: 'sentinel' + sourceId: la.id + tags: [ + 'AksSecurityWorkbook' + '1.2' + ] + } +} + +resource omsKeyVaultAnalytics 'Microsoft.OperationsManagement/solutions@2015-11-01-preview' = { + name: 'KeyVaultAnalytics(${la.name})' + location: location + properties: { + workspaceResourceId: la.id + } + plan: { + name: 'KeyVaultAnalytics(${la.name})' + product: 'OMSGallery/KeyVaultAnalytics' + promotionCode: '' + publisher: 'Microsoft' + } +} + +resource kv_diagnosticSettings 'Microsoft.Insights/diagnosticSettings@2021-05-01-preview' = { + scope: kv + name: 'default' + properties: { + workspaceId: la.id + logs: [ + { + category: 'AuditEvent' + enabled: true + } + ] + metrics: [ + { + category: 'AllMetrics' + enabled: true + } + ] + } +} + +@description('The network interface in the spoke vnet that enables privately connecting the AKS cluster with Key Vault.') +resource peKv 'Microsoft.Network/privateEndpoints@2022-01-01' = { + name: 'pe-${kv.name}' + location: location + properties: { + subnet: { + id: vnetSpoke::snetPrivatelinkendpoints.id + } + privateLinkServiceConnections: [ + { + name: 'to-${vnetSpoke.name}' + properties: { + privateLinkServiceId: kv.id + groupIds: [ + 'vault' + ] + } + } + ] + } + + resource pdzg 'privateDnsZoneGroups' = { + name: 'for-${kv.name}' + properties: { + privateDnsZoneConfigs: [ + { + name: 'privatelink-akv-net' + properties: { + privateDnsZoneId: pdzKv.id + } + } + ] + } + } +} + +@description('The regional load balancer resource that ingests all the client requests and forward them back to the aks regulated cluster after passing the configured WAF rules.') +resource agw 'Microsoft.Network/applicationGateways@2022-01-01' = { + name: 'agw-${clusterName}' + location: location + identity: { + type: 'UserAssigned' + userAssignedIdentities: { + '${miAppGateway.id}': { + } + } + } + zones: pickZones('Microsoft.Network', 'applicationGateways', location, 3) + properties: { + sku: { + name: 'WAF_v2' + tier: 'WAF_v2' + } + sslPolicy: { + policyType: 'Custom' + cipherSuites: [ + 'TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384' + 'TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256' + ] + minProtocolVersion: 'TLSv1_2' + } + trustedRootCertificates: [ + { + name: 'root-cert-wildcard-aks-ingress-contoso' + properties: { + keyVaultSecretId: kv::kvsAppGwIngressInternalAksIngressTls.properties.secretUri + } + } + ] + gatewayIPConfigurations: [ + { + name: 'agw-ip-configuration' + properties: { + subnet: { + id: vnetSpoke::snetApplicationGateway.id + } + } + } + ] + frontendIPConfigurations: [ + { + name: 'agw-frontend-ip-configuration' + properties: { + publicIPAddress: { + id: pipPrimaryCluster.id + } + } + } + ] + frontendPorts: [ + { + name: 'agw-frontend-ports' + properties: { + port: 443 + } + } + ] + autoscaleConfiguration: { + minCapacity: 0 + maxCapacity: 10 + } + webApplicationFirewallConfiguration: { + enabled: true + firewallMode: 'Prevention' + ruleSetType: 'OWASP' + ruleSetVersion: '3.2' + disabledRuleGroups: [] + requestBodyCheck: true + maxRequestBodySizeInKb: 128 + fileUploadLimitInMb: 100 + } + enableHttp2: false + sslCertificates: [ + { + name: 'agw-${clusterName}-ssl-certificate' + properties: { + keyVaultSecretId: kv::kvsGatewaySslCert.properties.secretUri + } + } + ] + probes: [ + { + name: 'probe-bu0001a0005-00.aks-ingress.contoso.com' + properties: { + protocol: 'Https' + path: '/favicon.ico' + interval: 30 + timeout: 30 + unhealthyThreshold: 3 + pickHostNameFromBackendHttpSettings: true + minServers: 0 + match: { + } + } + } + { + name: 'ingress-controller' + properties: { + protocol: 'Https' + path: '/healthz' + interval: 30 + timeout: 30 + unhealthyThreshold: 3 + pickHostNameFromBackendHttpSettings: true + minServers: 0 + match: { + } + } + } + ] + backendAddressPools: [ + { + name: 'bu0001a0005-00.aks-ingress.contoso.com' + properties: { + backendAddresses: [ + { + ipAddress: '10.240.4.4' // This is the IP address that our ingress controller will request + } + ] + } + } + ] + backendHttpSettingsCollection: [ + { + name: 'aks-ingress-contoso-backendpool-httpsettings' + properties: { + port: 443 + protocol: 'Https' + cookieBasedAffinity: 'Disabled' + hostName: 'bu0001a0005-00.aks-ingress.contoso.com' + pickHostNameFromBackendAddress: false + requestTimeout: 20 + probe: { + id: resourceId('Microsoft.Network/applicationGateways/probes', 'agw-${clusterName}','probe-bu0001a0005-00.aks-ingress.contoso.com') + } + trustedRootCertificates: [ + { + id: resourceId('Microsoft.Network/applicationGateways/trustedRootCertificates', 'agw-${clusterName}','root-cert-wildcard-aks-ingress-contoso') + } + ] + } + } + ] + httpListeners: [ + { + name: 'listener-https' + properties: { + frontendIPConfiguration: { + id: resourceId('Microsoft.Network/applicationGateways/frontendIPConfigurations', 'agw-${clusterName}','agw-frontend-ip-configuration') + } + frontendPort: { + id: resourceId('Microsoft.Network/applicationGateways/frontendPorts', 'agw-${clusterName}','agw-frontend-ports') + } + protocol: 'Https' + sslCertificate: { + id: resourceId('Microsoft.Network/applicationGateways/sslCertificates', 'agw-${clusterName}','agw-${clusterName}-ssl-certificate') + } + hostName: 'bicycle.contoso.com' + hostNames: [] + requireServerNameIndication: true + } + } + ] + requestRoutingRules: [ + { + name: 'agw-routing-rules' + properties: { + priority: 1 + ruleType: 'Basic' + httpListener: { + id: resourceId('Microsoft.Network/applicationGateways/httpListeners', 'agw-${clusterName}','listener-https') + } + backendAddressPool: { + id: resourceId('Microsoft.Network/applicationGateways/backendAddressPools', 'agw-${clusterName}','bu0001a0005-00.aks-ingress.contoso.com') + } + backendHttpSettings: { + id: resourceId('Microsoft.Network/applicationGateways/backendHttpSettingsCollection', 'agw-${clusterName}','aks-ingress-contoso-backendpool-httpsettings') + } + } + } + ] + } + dependsOn: [ + peKv + kvMiAppGatewayKeyVaultReader_roleAssignment + kvMiAppGatewaySecretsUserRole_roleAssignment + ] +} + +@description('The diagnostic settings configuration for the aks regulated cluster regional load balancer.') +resource agw_diagnosticSettings 'Microsoft.Insights/diagnosticSettings@2021-05-01-preview' = { + scope: agw + name: 'default' + properties: { + workspaceId: la.id + logs: [ + { + category: 'ApplicationGatewayAccessLog' + enabled: true + } + { + category: 'ApplicationGatewayPerformanceLog' + enabled: true + } + { + category: 'ApplicationGatewayFirewallLog' + enabled: true + } + ] + } +} + +@description('The compute for operations jumpboxes; these machines are assigned to cluster operator users') +resource vmssJumpboxes 'Microsoft.Compute/virtualMachineScaleSets@2020-12-01' = { + name: 'vmss-jumpboxes' + location: location + zones: pickZones('Microsoft.Compute', 'virtualMachineScaleSets', location, 3) + sku: { + name: 'Standard_DS1_v2' + tier: 'Standard' + capacity: 2 + } + properties: { + additionalCapabilities: { + ultraSSDEnabled: false + } + overprovision: false + singlePlacementGroup: true + upgradePolicy: { + mode: 'Automatic' + } + zoneBalance: false + virtualMachineProfile: { + diagnosticsProfile: { + bootDiagnostics: { + enabled: true + } + } + osProfile: { + computerNamePrefix: 'aksjmp' + linuxConfiguration: { + disablePasswordAuthentication: true + provisionVMAgent: true + ssh: { + publicKeys: [ + { + path: '/home/${jumpBoxDefaultAdminUserName}/.ssh/authorized_keys' + keyData: 'ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQCcFvQl2lYPcK1tMB3Tx2R9n8a7w5MJCSef14x0ePRFr9XISWfCVCNKRLM3Al/JSlIoOVKoMsdw5farEgXkPDK5F+SKLss7whg2tohnQNQwQdXit1ZjgOXkis/uft98Cv8jDWPbhwYj+VH/Aif9rx8abfjbvwVWBGeA/OnvfVvXnr1EQfdLJgMTTh+hX/FCXCqsRkQcD91MbMCxpqk8nP6jmsxJBeLrgfOxjH8RHEdSp4fF76YsRFHCi7QOwTE/6U+DpssgQ8MTWRFRat97uTfcgzKe5MOfuZHZ++5WFBgaTr1vhmSbXteGiK7dQXOk2cLxSvKkzeaiju9Jy6hoSl5oMygUVd5fNPQ94QcqTkMxZ9tQ9vPWOHwbdLRD31Ses3IBtDV+S6ehraiXf/L/e0jRUYk8IL/J543gvhOZ0hj2sQqTj9XS2hZkstZtrB2ywrJzV5ByETUU/oF9OsysyFgnaQdyduVqEPHaqXqnJvBngqqas91plyT3tSLMez3iT0s= unused-generated-by-azure' + } + ] + } + } + customData: jumpBoxCloudInitAsBase64 + adminUsername: jumpBoxDefaultAdminUserName + } + storageProfile: { + osDisk: { + createOption: 'FromImage' + caching: 'ReadOnly' + diffDiskSettings: { + option: 'Local' + } + osType: 'Linux' + } + imageReference: { + id: jumpBoxImageResourceId + } + } + networkProfile: { + networkInterfaceConfigurations: [ + { + name: 'vnet-spoke-BU0001A0005-01-nic01' + properties: { + primary: true + enableIPForwarding: false + enableAcceleratedNetworking: false + networkSecurityGroup: null + ipConfigurations: [ + { + name: 'default' + properties: { + primary: true + privateIPAddressVersion: 'IPv4' + publicIPAddressConfiguration: null + subnet: { + id: vnetSpoke::snetManagmentOps.id + } + } + } + ] + } + } + ] + } + } + } + dependsOn: [ + omsVmInsights + ] + + resource extOmsAgentForLinux 'extensions' = { + name: 'OMSExtension' + properties: { + publisher: 'Microsoft.EnterpriseCloud.Monitoring' + type: 'OmsAgentForLinux' + typeHandlerVersion: '1.13' + autoUpgradeMinorVersion: true + settings: { + stopOnMultipleConnections: true + azureResourceId: vmssJumpboxes.id + workspaceId: la.properties.customerId + } + protectedSettings: { + workspaceKey: la.listKeys().primarySharedKey + } + } + } + + resource extDependencyAgentLinux 'extensions' = { + name: 'DependencyAgentLinux' + properties: { + publisher: 'Microsoft.Azure.Monitoring.DependencyAgent' + type: 'DependencyAgentLinux' + typeHandlerVersion: '9.10' + autoUpgradeMinorVersion: true + } + dependsOn: [ + extOmsAgentForLinux + ] + } +} + +@description('The private container registry for the aks regulated cluster.') +resource acr 'Microsoft.ContainerRegistry/registries@2022-02-01-preview' = { + name: acrName + location: location + sku: { + name: 'Premium' + } + properties: { + adminUserEnabled: false + networkRuleSet: { + defaultAction: 'Deny' + virtualNetworkRules: [] + ipRules: [] + } + policies: { + quarantinePolicy: { + status: 'disabled' + } + trustPolicy: { + type: 'Notary' + status: 'enabled' + } + retentionPolicy: { + days: 15 + status: 'enabled' + } + } + publicNetworkAccess: 'Disabled' + encryption: { + status: 'disabled' + } + dataEndpointEnabled: true + networkRuleBypassOptions: 'AzureServices' + zoneRedundancy: 'Disabled' + } + + resource grl 'replications' = { + name: geoRedundancyLocation + location: geoRedundancyLocation + } + + resource ap 'agentPools@2019-06-01-preview' = { + name: 'acragent' + location: location + properties: { + count: 1 + os: 'Linux' + tier: 'S1' + virtualNetworkSubnetResourceId: vnetSpoke::snetManagmentCrAgents.id + } + } +} + +resource cr_diagnosticSettings 'Microsoft.Insights/diagnosticSettings@2021-05-01-preview' = { + scope: acr + name: 'default' + properties: { + workspaceId: la.id + metrics: [ + { + timeGrain: 'PT1M' + category: 'AllMetrics' + enabled: true + } + ] + logs: [ + { + category: 'ContainerRegistryRepositoryEvents' + enabled: true + } + { + category: 'ContainerRegistryLoginEvents' + enabled: true + } + ] + } +} + +@description('The network interface in the spoke vnet that enables privately connecting the AKS cluster with Container Registry.') +resource peCr 'Microsoft.Network/privateEndpoints@2022-05-01' = { + name: 'pe-${acr.name}' + location: location + properties: { + subnet: { + id: vnetSpoke::snetPrivatelinkendpoints.id + } + privateLinkServiceConnections: [ + { + name: 'to-${vnetSpoke.name}' + properties: { + privateLinkServiceId: acr.id + groupIds: [ + 'registry' + ] + } + } + ] + } + dependsOn: [ + acr::grl + ] + + resource pdzg 'privateDnsZoneGroups' = { + name: 'for-${acr.name}' + properties: { + privateDnsZoneConfigs: [ + { + name: 'privatelink-azurecr-io' + properties: { + privateDnsZoneId: pdzCr.id + } + } + ] + } + } +} + +@description('The scheduled query that returns images being imported from repositories different than quarantine/') +resource sqrNonQuarantineImportedImgesToCr 'Microsoft.Insights/scheduledQueryRules@2022-06-15' = { + name: 'Image Imported into ACR from ${acr.name} source other than approved Quarantine' + location: location + properties: { + description: 'The only images we want in live/ are those that came from this ACR instance, but from the quarantine/ repository.' + actions: { + actionGroups: [] + } + criteria: { + allOf: [ + { + operator: 'GreaterThan' + query: 'ContainerRegistryRepositoryEvents\r\n| where OperationName == "importImage" and Repository startswith "live/" and MediaType !startswith strcat(_ResourceId, "/quarantine")' + threshold: 0 + timeAggregation: 'Count' + dimensions: [] + failingPeriods: { + minFailingPeriodsToAlert: 1 + numberOfEvaluationPeriods: 1 + } + resourceIdColumn: '' + } + ] + } + enabled: true + evaluationFrequency: 'PT10M' + scopes: [ + acr.id + ] + severity: 3 + windowSize: 'PT10M' + muteActionsDuration: null + overrideQueryTimeRange: null + } + dependsOn: [ + cr_diagnosticSettings + ] +} + +resource paAksLinuxRestrictive 'Microsoft.Authorization/policyAssignments@2021-06-01' = { + name: guid(psdAKSLinuxRestrictiveId, resourceGroup().name, clusterName) + properties: { + displayName: trim(take('[${clusterName}] ${reference(psdAKSLinuxRestrictiveId, '2020-09-01').displayName}', 125)) + policyDefinitionId: psdAKSLinuxRestrictiveId + parameters: { + effect: { + value: 'audit' + } + excludedNamespaces: { + value: [ + 'kube-system' + 'gatekeeper-system' + ] + } + } + } +} + +resource paEnforceHttpsIngress 'Microsoft.Authorization/policyAssignments@2020-09-01' = { + name: guid(pdEnforceHttpsIngressId, resourceGroup().name, clusterName) + properties: { + displayName: trim(take('[${clusterName}] ${reference(pdEnforceHttpsIngressId, '2020-09-01').displayName}', 125)) + policyDefinitionId: pdEnforceHttpsIngressId + parameters: { + effect: { + value: 'deny' + } + excludedNamespaces: { + value: [] + } + } + } +} + +resource paEnforceInternalLoadBalancers 'Microsoft.Authorization/policyAssignments@2020-09-01' = { + name: guid(pdEnforceInternalLoadBalancersId, resourceGroup().name, clusterName) + properties: { + displayName: trim(take('[${clusterName}] ${reference(pdEnforceInternalLoadBalancersId, '2020-09-01').displayName}', 125)) + policyDefinitionId: pdEnforceInternalLoadBalancersId + parameters: { + effect: { + value: 'deny' + } + excludedNamespaces: { + value: [] + } + } + } +} + +resource paMustNotAutomountApiCreds 'Microsoft.Authorization/policyAssignments@2020-09-01' = { + name: guid(pdMustNotAutomountApiCreds, resourceGroup().name, clusterName) + properties: { + displayName: trim(take('[${clusterName}] ${reference(pdMustNotAutomountApiCreds, '2020-09-01').displayName}', 125)) + policyDefinitionId: pdMustNotAutomountApiCreds + parameters: { + effect: { + value: 'deny' + } + excludedNamespaces: { + value: [ + 'kube-system' + 'gatekeeper-system' + 'flux-system' // Required by Flux + 'falco-system' // Required by Falco + 'osm-system' // Required by OSM + 'ingress-nginx' // Required by NGINX + 'cluster-baseline-settings' // Required by Key Vault CSI & Kured + ] + } + } + } +} + +resource paMustUseSpecifiedLabels 'Microsoft.Authorization/policyAssignments@2020-09-01' = { + name: guid(pdMustUseSpecifiedLabels, resourceGroup().name, clusterName) + properties: { + displayName: trim(take('[${clusterName}] ${reference(pdMustUseSpecifiedLabels, '2020-09-01').displayName}', 125)) + policyDefinitionId: pdMustUseSpecifiedLabels + parameters: { + effect: { + value: 'audit' + } + labelsList: { + value: [ + 'pci-scope' + ] + } + } + } +} + +resource paMustUseTheseExternalIps 'Microsoft.Authorization/policyAssignments@2020-09-01' = { + name: guid(pdAllowedExternalIPsId, resourceGroup().name, clusterName) + properties: { + displayName: trim(take('[${clusterName}] ${reference(pdAllowedExternalIPsId, '2020-09-01').displayName}', 125)) + policyDefinitionId: pdAllowedExternalIPsId + parameters: { + effect: { + value: 'deny' + } + excludedNamespaces: { + value: [ + 'kube-system' + 'gatekeeper-system' + ] + } + allowedExternalIPs: { + value: [] // No external IPs allowed (LoadBalancer Service types do not apply to this policy) + } + } + } +} + +resource paApprovedContainerPortsOnly 'Microsoft.Authorization/policyAssignments@2020-09-01' = { + name: guid(pdApprovedContainerPortsOnly, resourceGroup().name, clusterName) + properties: { + displayName: trim(take('[${clusterName}-a0005] ${reference(pdApprovedContainerPortsOnly, '2020-09-01').displayName}', 125)) + policyDefinitionId: pdApprovedContainerPortsOnly + parameters: { + effect: { + value: 'audit' + } + excludedNamespaces: { + value: [] + } + namespaces: { + value: [ + 'a0005-i' + 'a0005-o' + ] + } + allowedContainerPortsList: { + value: [ + '8080' // ASP.net service listens on this + '15000' // envoy proxy for service mesh + '15003' // envoy proxy for service mesh + '15010' // envoy proxy for service mesh + ] + } + } + } +} + +resource paApprovedServicePortsOnly 'Microsoft.Authorization/policyAssignments@2020-09-01' = { + name: guid(pdApprovedServicePortsOnly, resourceGroup().name, clusterName) + properties: { + displayName: trim(take('[${clusterName}] ${reference(pdApprovedServicePortsOnly, '2020-09-01').displayName}', 125)) + policyDefinitionId: pdApprovedServicePortsOnly + parameters: { + effect: { + value: 'audit' + } + excludedNamespaces: { + value: [ + 'kube-system' + 'gatekeeper-system' + 'osm-system' + ] + } + allowedServicePortsList: { + value: [ + '443' // ingress-controller + '80' // flux source-controller and microservice workload + '8080' // web-frontend workload + ] + } + } + } +} + +@description('Applying the \'Kubernetes cluster containers should run with a read only root file system\' policy to the resource group.') +resource paRoRootFilesystem 'Microsoft.Authorization/policyAssignments@2020-09-01' = { + name: guid(pdRoRootFilesystemId, resourceGroup().name, clusterName) + properties: { + displayName: trim(take('[${clusterName}] ${reference(pdRoRootFilesystemId, '2020-09-01').displayName}', 125)) + policyDefinitionId: pdRoRootFilesystemId + parameters: { + effect: { + value: 'audit' + } + // Not all workloads support this. E.g. ASP.NET requires a non-readonly root file system to handle request buffering when there is memory pressure. + excludedNamespaces: { + value: [ + 'kube-system' + 'gatekeeper-system' + ] + } + } + } +} + +resource paBlockDefaultNamespace 'Microsoft.Authorization/policyAssignments@2020-09-01' = { + name: guid(pdDisallowNamespaceUsageId, resourceGroup().name, clusterName) + properties: { + displayName: trim(take('[${clusterName}] ${reference(pdDisallowNamespaceUsageId, '2020-09-01').displayName}', 125)) + policyDefinitionId: pdDisallowNamespaceUsageId + parameters: { + effect: { + value: 'deny' + } + excludedNamespaces: { + value: [] + } + } + } +} + +resource paEnforceResourceLimits 'Microsoft.Authorization/policyAssignments@2020-09-01' = { + name: guid(pdEnforceResourceLimitsId, resourceGroup().name, clusterName) + properties: { + displayName: trim(take('[${clusterName}] ${reference(pdEnforceResourceLimitsId, '2020-09-01').displayName}', 125)) + policyDefinitionId: pdEnforceResourceLimitsId + parameters: { + effect: { + value: 'audit' + } + cpuLimit: { + value: '1500m' + } + memoryLimit: { + value: '2Gi' + } + excludedNamespaces: { + value: [ + 'kube-system' + 'gatekeeper-system' + ] + } + } + } +} + +resource paEnforceImageSource 'Microsoft.Authorization/policyAssignments@2020-03-01' = { + name: guid(pdEnforceImageSourceId, resourceGroup().name, clusterName) + properties: { + displayName: trim(take('[${clusterName}] ${reference(pdEnforceImageSourceId, '2020-09-01').displayName}', 125)) + policyDefinitionId: pdEnforceImageSourceId + parameters: { + allowedContainerImagesRegex: { + value: '${acrName}\\.azurecr\\.io\\/live\\/.+$|mcr\\.microsoft\\.com\\/oss\\/(openservicemesh\\/init:|envoyproxy\\/envoy:).+$' + } + effect: { + value: 'deny' + } + excludedNamespaces: { + value: [ + 'kube-system' + 'gatekeeper-system' + ] + } + } + } +} + +resource alaAllAzureAdvisorAlert 'Microsoft.Insights/activityLogAlerts@2020-10-01' = { + name: 'AllAzureAdvisorAlert' + location: 'Global' + properties: { + scopes: [ + resourceGroup().id + ] + condition: { + allOf: [ + { + field: 'category' + equals: 'Recommendation' + } + { + field: 'operationName' + equals: 'Microsoft.Advisor/recommendations/available/action' + } + ] + } + actions: { + actionGroups: [] + } + enabled: true + description: 'All azure advisor alerts' + } +} + +module ensureClusterIdentityHasRbacToSelfManagedResources 'modules/ensureClusterIdentityHasRbacToSelfManagedResources.bicep' = { + name: 'ensureClusterIdentityHasRbacToSelfManagedResources' + scope: spokeResourceGroup + params: { + miClusterControlPlanePrincipalId: miClusterControlPlane.properties.principalId + clusterControlPlaneIdentityName: miClusterControlPlane.name + vnetSpokeName: vnetSpoke.name + location: location + } +} + +resource mc 'Microsoft.ContainerService/managedClusters@2021-05-01' = { + name: clusterName + location: location + tags: { + 'Data classification': 'Confidential' + 'Business unit': 'BU0001' + 'Business criticality': 'Business unit-critical' + } + properties: { + kubernetesVersion: kubernetesVersion + dnsPrefix: uniqueString(subscription().subscriptionId, resourceGroup().id, clusterName) + agentPoolProfiles: [ + { + name: 'npsystem' + count: 3 + vmSize: 'Standard_DS2_v2' + osDiskSizeGB: 80 + osDiskType: 'Ephemeral' + osType: 'Linux' + minCount: 3 + maxCount: 4 + vnetSubnetID: vnetSpoke::snetClusterSystemNodePools.id + enableAutoScaling: true + type: 'VirtualMachineScaleSets' + mode: 'System' + scaleSetPriority: 'Regular' + scaleSetEvictionPolicy: 'Delete' + orchestratorVersion: kubernetesVersion + enableNodePublicIP: false + maxPods: 30 + availabilityZones: pickZones('Microsoft.ContainerService', 'managedClusters', location, 3) + upgradeSettings: { + maxSurge: '33%' + } + /* This can be used to prevent unexpected workloads from landing on system node pool. All add-ons support this taint., + "nodeTaints": [ + "CriticalAddonsOnly=true:NoSchedule" + ] + */ + tags: { + 'pci-scope': 'out-of-scope' + 'Data classification': 'Confidential' + 'Business unit': 'BU0001' + 'Business criticality': 'Business unit-critical' + } + } + { + name: 'npinscope01' + count: 2 + vmSize: 'Standard_DS3_v2' + osDiskSizeGB: 120 + osDiskType: 'Ephemeral' + osType: 'Linux' + minCount: 2 + maxCount: 5 + vnetSubnetID: vnetSpoke::snetClusterInScopeNodePools.id + enableAutoScaling: true + type: 'VirtualMachineScaleSets' + mode: 'User' + scaleSetPriority: 'Regular' + scaleSetEvictionPolicy: 'Delete' + orchestratorVersion: kubernetesVersion + enableNodePublicIP: false + maxPods: 30 + availabilityZones: pickZones('Microsoft.ContainerService', 'managedClusters', location, 3) + upgradeSettings: { + maxSurge: '33%' + } + nodeLabels: { + 'pci-scope': 'in-scope' + } + tags: { + 'pci-scope': 'in-scope' + 'Data classification': 'Confidential' + 'Business unit': 'BU0001' + 'Business criticality': 'Business unit-critical' + } + } + { + name: 'npooscope01' + count: 2 + vmSize: 'Standard_DS3_v2' + osDiskSizeGB: 120 + osDiskType: 'Ephemeral' + osType: 'Linux' + minCount: 2 + maxCount: 5 + vnetSubnetID: vnetSpoke::snetClusterOutScopeNodePools.id + enableAutoScaling: true + type: 'VirtualMachineScaleSets' + mode: 'User' + scaleSetPriority: 'Regular' + scaleSetEvictionPolicy: 'Delete' + orchestratorVersion: kubernetesVersion + enableNodePublicIP: false + maxPods: 30 + availabilityZones: pickZones('Microsoft.ContainerService', 'managedClusters', location, 3) + upgradeSettings: { + maxSurge: '33%' + } + nodeLabels: { + 'pci-scope': 'out-of-scope' + } + tags: { + 'pci-scope': 'out-of-scope' + 'Data classification': 'Confidential' + 'Business unit': 'BU0001' + 'Business criticality': 'Business unit-critical' + } + } + ] + servicePrincipalProfile: { + clientId: 'msi' + } + addonProfiles: { + httpApplicationRouting: { + enabled: false + } + omsagent: { + enabled: true + config: { + logAnalyticsWorkspaceResourceId: la.id + } + } + aciConnectorLinux: { + enabled: false + } + azurepolicy: { + enabled: true + config: { + version: 'v2' + } + } + openServiceMesh: { + enabled: true + config: { + } + } + azureKeyvaultSecretsProvider: { + enabled: true + config: { + enableSecretRotation: 'false' + } + } + } + nodeResourceGroup: 'rg-${clusterName}-nodepools' + enableRBAC: true + enablePodSecurityPolicy: false + maxAgentPools: 3 + networkProfile: { + networkPlugin: 'azure' + networkPolicy: 'azure' + outboundType: 'userDefinedRouting' + loadBalancerSku: 'standard' + loadBalancerProfile: json('null') + serviceCidr: '172.16.0.0/16' + dnsServiceIP: '172.16.0.10' + dockerBridgeCidr: '172.18.0.1/16' + } + aadProfile: { + managed: true + enableAzureRBAC: false + adminGroupObjectIDs: [ + clusterAdminAadGroupObjectId + ] + tenantID: k8sControlPlaneAuthorizationTenantId + } + autoScalerProfile: { + 'balance-similar-node-groups': 'false' + expander: 'random' + 'max-empty-bulk-delete': '10' + 'max-graceful-termination-sec': '600' + 'max-node-provision-time': '15m' + 'max-total-unready-percentage': '45' + 'new-pod-scale-up-delay': '0s' + 'ok-total-unready-count': '3' + 'scale-down-delay-after-add': '10m' + 'scale-down-delay-after-delete': '20s' + 'scale-down-delay-after-failure': '3m' + 'scale-down-unneeded-time': '10m' + 'scale-down-unready-time': '20m' + 'scale-down-utilization-threshold': '0.5' + 'scan-interval': '10s' + 'skip-nodes-with-local-storage': 'true' + 'skip-nodes-with-system-pods': 'true' + } + // autoUpgradeProfile: { + // upgradeChannel: 'none' + // } + apiServerAccessProfile: { + enablePrivateCluster: true + privateDNSZone: pdzMc.id + enablePrivateClusterPublicFQDN: false + } + podIdentityProfile: { + enabled: true + userAssignedIdentities: [ + { + name: 'podmi-ingress-controller' + namespace: 'ingress-nginx' + identity: { + resourceId: miIngressController.id + clientId: miIngressController.properties.clientId + objectId: miIngressController.properties.principalId + } + } + ] + } + disableLocalAccounts: true + securityProfile: { + defender: { + logAnalyticsWorkspaceResourceId: la.id + securityMonitoring: { + enabled: true + } + } + } + } + identity: { + type: 'UserAssigned' + userAssignedIdentities: { + '${miClusterControlPlane.id}': { + } + } + } + sku: { + name: 'Basic' + tier: 'Paid' + } + dependsOn: [ + omsContainerInsights + ensureClusterIdentityHasRbacToSelfManagedResources + + // You want policies created before cluster because they take some time to be made available and we want them + // to apply to your cluster as soon as possible. Nothing in this cluster "technically" depends on these existing, + // just trying to get coverage as soon as possible. + paAksLinuxRestrictive + paEnforceHttpsIngress + paEnforceInternalLoadBalancers + paMustNotAutomountApiCreds + paMustUseSpecifiedLabels + paMustUseTheseExternalIps + paApprovedContainerPortsOnly + paApprovedServicePortsOnly + paRoRootFilesystem + paBlockDefaultNamespace + paEnforceResourceLimits + paEnforceImageSource + + vmssJumpboxes // Ensure jumboxes are available to use as soon as possible, don't wait until cluster is created. + + kvMiIngressControllerSecretsUserRole_roleAssignment + kvMiIngressControllerKeyVaultReader_roleAssignment + ] +} + +resource mc_diagnosticSettings 'Microsoft.Insights/diagnosticSettings@2021-05-01-preview' = { + scope: mc + name: 'default' + properties: { + workspaceId: la.id + logs: [ + { + category: 'cluster-autoscaler' + enabled: true + } + { + category: 'kube-controller-manager' + enabled: true + } + { + category: 'kube-audit-admin' + enabled: true + } + { + category: 'guard' + enabled: true + } + ] + } +} + +@description('Grant kubelet managed identity with container registry pull role permissions; this allows the AKS Cluster\'s kubelet managed identity to pull images from this container registry.') +resource crMiKubeletContainerRegistryPullRole_roleAssignment 'Microsoft.Authorization/roleAssignments@2022-04-01' = { + scope: acr + name: guid(resourceGroup().id, mc.id, containerRegistryPullRole.id) + properties: { + roleDefinitionId: containerRegistryPullRole.id + principalId: mc.properties.identityProfile.kubeletidentity.objectId + principalType: 'ServicePrincipal' + } +} + +@description('Grant OMS agent managed identity with publisher metrics role permissions; this allows the OMS agent\'s identity to publish metrics in Container Insights.') +resource sMiOmsMonitoringMetricPublisherRoleRole_roleAssignment 'Microsoft.Authorization/roleAssignments@2022-04-01' = { + name: guid(mc.id, monitoringMetricsPublisherRole.id) + properties: { + roleDefinitionId: monitoringMetricsPublisherRole.id + principalId: mc.properties.addonProfiles.omsagent.identity.objectId + principalType: 'ServicePrincipal' + } +} + +resource sqrPodFailed 'Microsoft.Insights/scheduledQueryRules@2021-08-01' = { + name: 'PodFailedScheduledQuery' + location: location + properties: { + description: 'Example from: https://learn.microsoft.com/azure/azure-monitor/insights/container-insights-alerts' + actions: { + actionGroups: [] + } + criteria: { + allOf: [ + { + metricMeasureColumn: 'FailedCount' + operator: 'GreaterThan' + query: 'let trendBinSize = 1m;\r\nKubePodInventory\r\n| distinct ClusterName, TimeGenerated\r\n| summarize ClusterSnapshotCount = count() by bin(TimeGenerated, trendBinSize), ClusterName\r\n| join hint.strategy=broadcast (\r\n KubePodInventory\r\n | distinct ClusterName, Computer, PodUid, TimeGenerated, PodStatus\r\n | summarize TotalCount = count(),\r\n PendingCount = sumif(1, PodStatus =~ "Pending"),\r\n RunningCount = sumif(1, PodStatus =~ "Running"),\r\n SucceededCount = sumif(1, PodStatus =~ "Succeeded"),\r\n FailedCount = sumif(1, PodStatus =~ "Failed")\r\n by ClusterName, bin(TimeGenerated, trendBinSize)\r\n )\r\n on ClusterName, TimeGenerated \r\n| extend UnknownCount = TotalCount - PendingCount - RunningCount - SucceededCount - FailedCount\r\n| project TimeGenerated,\r\n ClusterName,\r\n TotalCount = todouble(TotalCount) / ClusterSnapshotCount,\r\n PendingCount = todouble(PendingCount) / ClusterSnapshotCount,\r\n RunningCount = todouble(RunningCount) / ClusterSnapshotCount,\r\n SucceededCount = todouble(SucceededCount) / ClusterSnapshotCount,\r\n FailedCount = todouble(FailedCount) / ClusterSnapshotCount,\r\n UnknownCount = todouble(UnknownCount) / ClusterSnapshotCount\r\n' + threshold: 3 + timeAggregation: 'Average' + dimensions: [] + failingPeriods: { + minFailingPeriodsToAlert: 1 + numberOfEvaluationPeriods: 1 + } + resourceIdColumn: '' + } + ] + } + enabled: true + evaluationFrequency: 'PT5M' + scopes: [ + mc.id + ] + severity: 3 + windowSize: 'PT5M' + muteActionsDuration: null + overrideQueryTimeRange: 'P2D' + } + dependsOn: [ + la + ] +} + +resource maNodeCpuUtilizationHighCI1 'Microsoft.Insights/metricAlerts@2018-03-01' = { + name: 'Node CPU utilization high for ${clusterName} CI-1' + location: 'global' + properties: { + actions: [] + criteria: { + allOf: [ + { + criterionType: 'StaticThresholdCriterion' + dimensions: [ + { + name: 'host' + operator: 'Include' + values: [ + '*' + ] + } + ] + metricName: 'cpuUsagePercentage' + metricNamespace: 'Insights.Container/nodes' + name: 'Metric1' + operator: 'GreaterThan' + threshold: 80 + timeAggregation: 'Average' + skipMetricValidation: true + } + ] + 'odata.type': 'Microsoft.Azure.Monitor.SingleResourceMultipleMetricCriteria' + } + description: 'Node CPU utilization across the cluster.' + enabled: true + evaluationFrequency: 'PT1M' + scopes: [ + mc.id + ] + severity: 3 + targetResourceType: 'microsoft.containerservice/managedclusters' + windowSize: 'PT5M' + } + dependsOn: [ + omsContainerInsights + ] +} + +resource maNodeCpuUtilizationHighCI2 'Microsoft.Insights/metricAlerts@2018-03-01' = { + name: 'Node working set memory utilization high for ${clusterName} CI-2' + location: 'global' + properties: { + actions: [] + criteria: { + allOf: [ + { + criterionType: 'StaticThresholdCriterion' + dimensions: [ + { + name: 'host' + operator: 'Include' + values: [ + '*' + ] + } + ] + metricName: 'memoryWorkingSetPercentage' + metricNamespace: 'Insights.Container/nodes' + name: 'Metric1' + operator: 'GreaterThan' + threshold: 80 + timeAggregation: 'Average' + skipMetricValidation: true + } + ] + 'odata.type': 'Microsoft.Azure.Monitor.SingleResourceMultipleMetricCriteria' + } + description: 'Node working set memory utilization across the cluster.' + enabled: true + evaluationFrequency: 'PT1M' + scopes: [ + mc.id + ] + severity: 3 + targetResourceType: 'microsoft.containerservice/managedclusters' + windowSize: 'PT5M' + } + dependsOn: [ + omsContainerInsights + ] +} + +resource maJobsCompletedMoreThan6hAgoCI11 'Microsoft.Insights/metricAlerts@2018-03-01' = { + name: 'Jobs completed more than 6 hours ago for ${clusterName} CI-11' + location: 'global' + properties: { + actions: [] + criteria: { + allOf: [ + { + criterionType: 'StaticThresholdCriterion' + dimensions: [ + { + name: 'controllerName' + operator: 'Include' + values: [ + '*' + ] + } + { + name: 'kubernetes namespace' + operator: 'Include' + values: [ + '*' + ] + } + ] + metricName: 'completedJobsCount' + metricNamespace: 'Insights.Container/pods' + name: 'Metric1' + operator: 'GreaterThan' + threshold: 0 + timeAggregation: 'Average' + skipMetricValidation: true + } + ] + 'odata.type': 'Microsoft.Azure.Monitor.SingleResourceMultipleMetricCriteria' + } + description: 'This alert monitors completed jobs (more than 6 hours ago).' + enabled: true + evaluationFrequency: 'PT1M' + scopes: [ + mc.id + ] + severity: 3 + targetResourceType: 'microsoft.containerservice/managedclusters' + windowSize: 'PT1M' + } + dependsOn: [ + omsContainerInsights + ] +} + +resource maContainerCpuUtilizationHighCI9 'Microsoft.Insights/metricAlerts@2018-03-01' = { + name: 'Container CPU usage high for ${clusterName} CI-9' + location: 'global' + properties: { + actions: [] + criteria: { + allOf: [ + { + criterionType: 'StaticThresholdCriterion' + dimensions: [ + { + name: 'controllerName' + operator: 'Include' + values: [ + '*' + ] + } + { + name: 'kubernetes namespace' + operator: 'Include' + values: [ + '*' + ] + } + ] + metricName: 'cpuExceededPercentage' + metricNamespace: 'Insights.Container/containers' + name: 'Metric1' + operator: 'GreaterThan' + threshold: 90 + timeAggregation: 'Average' + skipMetricValidation: true + } + ] + 'odata.type': 'Microsoft.Azure.Monitor.SingleResourceMultipleMetricCriteria' + } + description: 'This alert monitors container CPU utilization.' + enabled: true + evaluationFrequency: 'PT1M' + scopes: [ + mc.id + ] + severity: 3 + targetResourceType: 'microsoft.containerservice/managedclusters' + windowSize: 'PT5M' + } + dependsOn: [ + omsContainerInsights + ] +} + +resource maContainerWorkingSetMemoryUsageHighCI10 'Microsoft.Insights/metricAlerts@2018-03-01' = { + name: 'Container working set memory usage high for ${clusterName} CI-10' + location: 'global' + properties: { + actions: [] + criteria: { + allOf: [ + { + criterionType: 'StaticThresholdCriterion' + dimensions: [ + { + name: 'controllerName' + operator: 'Include' + values: [ + '*' + ] + } + { + name: 'kubernetes namespace' + operator: 'Include' + values: [ + '*' + ] + } + ] + metricName: 'memoryWorkingSetExceededPercentage' + metricNamespace: 'Insights.Container/containers' + name: 'Metric1' + operator: 'GreaterThan' + threshold: 90 + timeAggregation: 'Average' + skipMetricValidation: true + } + ] + 'odata.type': 'Microsoft.Azure.Monitor.SingleResourceMultipleMetricCriteria' + } + description: 'This alert monitors container working set memory utilization.' + enabled: true + evaluationFrequency: 'PT1M' + scopes: [ + mc.id + ] + severity: 3 + targetResourceType: 'microsoft.containerservice/managedclusters' + windowSize: 'PT5M' + } + dependsOn: [ + omsContainerInsights + ] +} + +resource maPodsInFailedStateCI4 'Microsoft.Insights/metricAlerts@2018-03-01' = { + name: 'Pods in failed state for ${clusterName} CI-4' + location: 'global' + properties: { + actions: [] + criteria: { + allOf: [ + { + criterionType: 'StaticThresholdCriterion' + dimensions: [ + { + name: 'phase' + operator: 'Include' + values: [ + 'Failed' + ] + } + ] + metricName: 'podCount' + metricNamespace: 'Insights.Container/pods' + name: 'Metric1' + operator: 'GreaterThan' + threshold: 0 + timeAggregation: 'Average' + skipMetricValidation: true + } + ] + 'odata.type': 'Microsoft.Azure.Monitor.SingleResourceMultipleMetricCriteria' + } + description: 'Pod status monitoring.' + enabled: true + evaluationFrequency: 'PT1M' + scopes: [ + mc.id + ] + severity: 3 + targetResourceType: 'microsoft.containerservice/managedclusters' + windowSize: 'PT5M' + } + dependsOn: [ + omsContainerInsights + ] +} + +resource maDiskUsageHighCI5 'Microsoft.Insights/metricAlerts@2018-03-01' = { + name: 'Disk usage high for ${clusterName} CI-5' + location: 'global' + properties: { + actions: [] + criteria: { + allOf: [ + { + criterionType: 'StaticThresholdCriterion' + dimensions: [ + { + name: 'host' + operator: 'Include' + values: [ + '*' + ] + } + { + name: 'device' + operator: 'Include' + values: [ + '*' + ] + } + ] + metricName: 'DiskUsedPercentage' + metricNamespace: 'Insights.Container/nodes' + name: 'Metric1' + operator: 'GreaterThan' + threshold: 80 + timeAggregation: 'Average' + skipMetricValidation: true + } + ] + 'odata.type': 'Microsoft.Azure.Monitor.SingleResourceMultipleMetricCriteria' + } + description: 'This alert monitors disk usage for all nodes and storage devices.' + enabled: true + evaluationFrequency: 'PT1M' + scopes: [ + mc.id + ] + severity: 3 + targetResourceType: 'microsoft.containerservice/managedclusters' + windowSize: 'PT5M' + } + dependsOn: [ + omsContainerInsights + ] +} + +resource maNodesInNotReadyStatusCI3 'Microsoft.Insights/metricAlerts@2018-03-01' = { + name: 'Nodes in not ready status for ${clusterName} CI-3' + location: 'global' + properties: { + actions: [] + criteria: { + allOf: [ + { + criterionType: 'StaticThresholdCriterion' + dimensions: [ + { + name: 'status' + operator: 'Include' + values: [ + 'NotReady' + ] + } + ] + metricName: 'nodesCount' + metricNamespace: 'Insights.Container/nodes' + name: 'Metric1' + operator: 'GreaterThan' + threshold: 0 + timeAggregation: 'Average' + skipMetricValidation: true + } + ] + 'odata.type': 'Microsoft.Azure.Monitor.SingleResourceMultipleMetricCriteria' + } + description: 'Node status monitoring.' + enabled: true + evaluationFrequency: 'PT1M' + scopes: [ + mc.id + ] + severity: 3 + targetResourceType: 'microsoft.containerservice/managedclusters' + windowSize: 'PT5M' + } + dependsOn: [ + omsContainerInsights + ] +} + +resource maContainersGettingOomKilledCI6 'Microsoft.Insights/metricAlerts@2018-03-01' = { + name: 'Containers getting OOM killed for ${clusterName} CI-6' + location: 'global' + properties: { + actions: [] + criteria: { + allOf: [ + { + criterionType: 'StaticThresholdCriterion' + dimensions: [ + { + name: 'kubernetes namespace' + operator: 'Include' + values: [ + '*' + ] + } + { + name: 'controllerName' + operator: 'Include' + values: [ + '*' + ] + } + ] + metricName: 'oomKilledContainerCount' + metricNamespace: 'Insights.Container/pods' + name: 'Metric1' + operator: 'GreaterThan' + threshold: 0 + timeAggregation: 'Average' + skipMetricValidation: true + } + ] + 'odata.type': 'Microsoft.Azure.Monitor.SingleResourceMultipleMetricCriteria' + } + description: 'This alert monitors number of containers killed due to out of memory (OOM) error.' + enabled: true + evaluationFrequency: 'PT1M' + scopes: [ + mc.id + ] + severity: 3 + targetResourceType: 'microsoft.containerservice/managedclusters' + windowSize: 'PT1M' + } + dependsOn: [ + omsContainerInsights + ] +} + +resource maPersistentVolumeUsageHighCI18 'Microsoft.Insights/metricAlerts@2018-03-01' = { + name: 'Persistent volume usage high for ${clusterName} CI-18' + location: 'global' + properties: { + actions: [] + criteria: { + allOf: [ + { + criterionType: 'StaticThresholdCriterion' + dimensions: [ + { + name: 'podName' + operator: 'Include' + values: [ + '*' + ] + } + { + name: 'kubernetesNamespace' + operator: 'Include' + values: [ + '*' + ] + } + ] + metricName: 'pvUsageExceededPercentage' + metricNamespace: 'Insights.Container/persistentvolumes' + name: 'Metric1' + operator: 'GreaterThan' + threshold: 80 + timeAggregation: 'Average' + skipMetricValidation: true + } + ] + 'odata.type': 'Microsoft.Azure.Monitor.SingleResourceMultipleMetricCriteria' + } + description: 'This alert monitors persistent volume utilization.' + enabled: false + evaluationFrequency: 'PT1M' + scopes: [ + mc.id + ] + severity: 3 + targetResourceType: 'microsoft.containerservice/managedclusters' + windowSize: 'PT5M' + } + dependsOn: [ + omsContainerInsights + ] +} + +resource maPodsNotInReadyStateCI8 'Microsoft.Insights/metricAlerts@2018-03-01' = { + name: 'Pods not in ready state for ${clusterName} CI-8' + location: 'global' + properties: { + actions: [] + criteria: { + allOf: [ + { + criterionType: 'StaticThresholdCriterion' + dimensions: [ + { + name: 'controllerName' + operator: 'Include' + values: [ + '*' + ] + } + { + name: 'kubernetes namespace' + operator: 'Include' + values: [ + '*' + ] + } + ] + metricName: 'PodReadyPercentage' + metricNamespace: 'Insights.Container/pods' + name: 'Metric1' + operator: 'LessThan' + threshold: 80 + timeAggregation: 'Average' + skipMetricValidation: true + } + ] + 'odata.type': 'Microsoft.Azure.Monitor.SingleResourceMultipleMetricCriteria' + } + description: 'This alert monitors for excessive pods not in the ready state.' + enabled: true + evaluationFrequency: 'PT1M' + scopes: [ + mc.id + ] + severity: 3 + targetResourceType: 'microsoft.containerservice/managedclusters' + windowSize: 'PT5M' + } + dependsOn: [ + omsContainerInsights + ] +} + +resource maRestartingContainerCountCI7 'Microsoft.Insights/metricAlerts@2018-03-01' = { + name: 'Restarting container count for ${clusterName} CI-7' + location: 'global' + properties: { + actions: [] + criteria: { + allOf: [ + { + criterionType: 'StaticThresholdCriterion' + dimensions: [ + { + name: 'kubernetes namespace' + operator: 'Include' + values: [ + '*' + ] + } + { + name: 'controllerName' + operator: 'Include' + values: [ + '*' + ] + } + ] + metricName: 'restartingContainerCount' + metricNamespace: 'Insights.Container/pods' + name: 'Metric1' + operator: 'GreaterThan' + threshold: 0 + timeAggregation: 'Average' + skipMetricValidation: true + } + ] + 'odata.type': 'Microsoft.Azure.Monitor.SingleResourceMultipleMetricCriteria' + } + description: 'This alert monitors number of containers restarting across the cluster.' + enabled: true + evaluationFrequency: 'PT1M' + scopes: [ + mc.id + ] + severity: 3 + targetResourceType: 'Microsoft.ContainerService/managedClusters' + windowSize: 'PT1M' + } + dependsOn: [ + omsContainerInsights + ] +} + +/*** OUTPUTS ***/ + +output agwName string = agw.name +output keyVaultName string = kv.name +output quarantineContainerRegistryName string = acr.name diff --git a/cluster-stamp.v2.json b/cluster-stamp.v2.json deleted file mode 100644 index de9ba71c..00000000 --- a/cluster-stamp.v2.json +++ /dev/null @@ -1,2458 +0,0 @@ -{ - "$schema": "https://schema.management.azure.com/schemas/2019-04-01/deploymentTemplate.json#", - "contentVersion": "1.0.2.0", - "parameters": { - "targetVnetResourceId": { - "type": "string", - "minLength": 79, - "metadata": { - "description": "The regional network spoke VNet Resource ID that the cluster will be joined to" - } - }, - "clusterAdminAadGroupObjectId": { - "type": "string", - "metadata": { - "description": "Azure AD Group in the identified tenant that will be granted the highly privileged cluster-admin role." - } - }, - "k8sControlPlaneAuthorizationTenantId": { - "type": "string", - "metadata": { - "description": "Your AKS control plane Cluster API authentication tenant" - } - }, - "appGatewayListenerCertificate": { - "type": "string", - "metadata": { - "description": "The certificate data for app gateway TLS termination. It is base64" - } - }, - "aksIngressControllerCertificate": { - "type": "string", - "metadata": { - "description": "The base 64 encoded AKS Ingress Controller public certificate (as .crt or .cer) to be stored in Azure Key Vault as secret and referenced by Azure Application Gateway as a trusted root certificate." - } - }, - "location": { - "defaultValue": "eastus2", - "type": "string", - "allowedValues": [ - "australiaeast", - "canadacentral", - "centralus", - "eastus", - "eastus2", - "westus2", - "francecentral", - "germanywestcentral", - "northeurope", - "southafricanorth", - "southcentralus", - "uksouth", - "westeurope", - "japaneast", - "southeastasia" - ], - "metadata": { - "description": "AKS Service, Node Pools, and supporting services (KeyVault, App Gateway, etc) region. This needs to be the same region as the vnet provided in these parameters." - } - }, - "geoRedundancyLocation": { - "defaultValue": "centralus", - "type": "string", - "allowedValues": [ - "australiasoutheast", - "canadaeast", - "eastus2", - "westus", - "centralus", - "westcentralus", - "francesouth", - "germanynorth", - "westeurope", - "ukwest", - "northeurope", - "japanwest", - "southafricawest", - "northcentralus", - "eastasia", - "eastus", - "westus2", - "francecentral", - "uksouth", - "japaneast", - "southeastasia" - ], - "metadata": { - "description": "For Azure resources that support native geo-redunancy, provide the location the redundant service will have its secondary. Should be different than the location parameter and ideally should be a paired region - https://learn.microsoft.com/azure/best-practices-availability-paired-regions. This region does not need to support availability zones." - } - }, - "jumpBoxImageResourceId": { - "type": "string", - "minLength": 70, - "metadata": { - "description": "The Azure resource ID of a VM image that will be used for the jump box." - } - }, - "jumpBoxCloudInitAsBase64": { - "type": "string", - "minLength": 100, - "metadata": { - "description": "A cloud init file (starting with #cloud-config) as a base 64 encoded string used to perform image customization on the jump box VMs. Used for user-management in this context." - } - } - }, - "variables": { - "kubernetesVersion": "1.23.3", - - "networkContributorRole": "[concat(subscription().Id, '/providers/Microsoft.Authorization/roleDefinitions/4d97b98b-1d4f-4787-a291-c67834d212e7')]", - "monitoringMetricsPublisherRole": "[concat(subscription().Id, '/providers/Microsoft.Authorization/roleDefinitions/3913510d-42f4-4e42-8a64-420c390055eb')]", - "acrPullRole": "[concat(subscription().Id, '/providers/Microsoft.Authorization/roleDefinitions/7f951dda-4ed3-4680-a7ca-43fe172d538d')]", - "dnsZoneContributorRole": "[concat(subscription().id, '/providers/Microsoft.Authorization/roleDefinitions/b12aa53e-6015-4669-85d0-8515ebb3ae7f')]", - "managedIdentityOperatorRole": "[concat(subscription().Id, '/providers/Microsoft.Authorization/roleDefinitions/f1a07417-d97a-45cb-824c-7a7467783830')]", - - "subRgUniqueString": "[uniqueString('aks', subscription().subscriptionId, resourceGroup().id)]", - - "nodeResourceGroupName": "[concat('rg-', variables('clusterName'), '-nodepools')]", - "clusterName": "[concat('aks-', variables('subRgUniqueString'))]", - "logAnalyticsWorkspaceName": "[concat('la-', variables('clusterName'))]", - "containerInsightsSolutionName": "[concat('ContainerInsights(', variables('logAnalyticsWorkspaceName'),')')]", - "vmInsightsSolutionName": "[concat('VMInsights(', variables('logAnalyticsWorkspaceName'),')')]", - "securityCenterSolutionName": "[concat('SecurityInsights(', variables('logAnalyticsWorkspaceName'),')')]", - "defaultAcrName": "[concat('acraks', variables('subRgUniqueString'))]", - - "vNetResourceGroup": "[split(parameters('targetVnetResourceId'),'/')[4]]", - "vnetName": "[split(parameters('targetVnetResourceId'),'/')[8]]", - "vnetSystemNodePoolSubnetResourceId": "[concat(parameters('targetVnetResourceId'), '/subnets/snet-cluster-systemnodepool')]", - "vnetInScopeNodePoolSubnetResourceId": "[concat(parameters('targetVnetResourceId'), '/subnets/snet-cluster-inscopenodepools')]", - "vnetPrivateLinkSubnetResourceId": "[concat(parameters('targetVnetResourceId'), '/subnets/snet-privatelinkendpoints')]", - "vnetAcrBuildSubnetResourceId": "[concat(parameters('targetVnetResourceId'), '/subnets/snet-management-acragents')]", - "vnetOutOfScopeNodePoolSubnetResourceId": "[concat(parameters('targetVnetResourceId'), '/subnets/snet-cluster-outofscopenodepools')]", - "vnetIngressServicesSubnetResourceId": "[concat(parameters('targetVnetResourceId'), '/subnets/snet-cluster-ingressservices')]", - "vnetManagementOpsSubnetResourceId": "[concat(parameters('targetVnetResourceId'), '/subnets/snet-management-ops')]", - - "agwName": "[concat('apw-', variables('clusterName'))]", - "apwResourceId": "[resourceId('Microsoft.Network/applicationGateways', variables('agwName'))]", - - "jumpBoxDefaultAdminUserName": "[uniqueString(variables('clusterName'), resourceGroup().id)]", - - "clusterControlPlaneIdentityName": "[concat('mi-', variables('clusterName'), '-controlplane')]", - - "keyVaultName": "[concat('kv-', variables('clusterName'))]", - - "policyResourceIdAKSLinuxRestrictive": "/providers/Microsoft.Authorization/policySetDefinitions/42b8ef37-b724-4e24-bbc8-7a7708edfe00", - "policyResourceIdEnforceHttpsIngress": "/providers/Microsoft.Authorization/policyDefinitions/1a5b4dca-0b6f-4cf5-907c-56316bc1bf3d", - "policyResourceIdEnforceInternalLoadBalancers": "/providers/Microsoft.Authorization/policyDefinitions/3fc4dc25-5baf-40d8-9b05-7fe74c1bc64e", - "policyResourceIdRoRootFilesystem": "/providers/Microsoft.Authorization/policyDefinitions/df49d893-a74c-421d-bc95-c663042e5b80", - "policyResourceIdEnforceResourceLimits": "/providers/Microsoft.Authorization/policyDefinitions/e345eecc-fa47-480f-9e88-67dcc122b164", - "policyResourceIdEnforceImageSource": "/providers/Microsoft.Authorization/policyDefinitions/febd0533-8e55-448f-b837-bd0e06f16469", - "policyResourceIdBlockDefaultNamespace": "/providers/Microsoft.Authorization/policyDefinitions/9f061a12-e40d-4183-a00e-171812443373", - "policyResourceIdApprovedContainerPortsOnly": "/providers/Microsoft.Authorization/policyDefinitions/440b515e-a580-421e-abeb-b159a61ddcbc", - "policyResourceIdApprovedServicePortsOnly": "/providers/Microsoft.Authorization/policyDefinitions/233a2a17-77ca-4fb1-9b6b-69223d272a44", - "policyResourceIdMustUseSpecifiedLabels": "/providers/Microsoft.Authorization/policyDefinitions/46592696-4c7b-4bf3-9e45-6c2763bdc0a6", - "policyResourceIdMustUseTheseExternalIps": "/providers/Microsoft.Authorization/policyDefinitions/d46c275d-1680-448d-b2ec-e495a3b6cc89", - "policyResourceIdMustNotAutomountApiCreds": "/providers/Microsoft.Authorization/policyDefinitions/423dd1ba-798e-40e4-9c4d-b6902674b423", - "policyAssignmentNameAKSLinuxRestrictive": "[guid(variables('policyResourceIdAKSLinuxRestrictive'), resourceGroup().name, variables('clusterName'))]", - "policyAssignmentNameEnforceHttpsIngress": "[guid(variables('policyResourceIdEnforceHttpsIngress'), resourceGroup().name, variables('clusterName'))]", - "policyAssignmentNameEnforceInternalLoadBalancers": "[guid(variables('policyResourceIdEnforceInternalLoadBalancers'), resourceGroup().name, variables('clusterName'))]", - "policyAssignmentNameRoRootFilesystem": "[guid(variables('policyResourceIdRoRootFilesystem'), resourceGroup().name, variables('clusterName'))]", - "policyAssignmentNameEnforceResourceLimits": "[guid(variables('policyResourceIdEnforceResourceLimits'), resourceGroup().name, variables('clusterName'))]", - "policyAssignmentNameEnforceImageSource": "[guid(variables('policyResourceIdEnforceImageSource'), resourceGroup().name, variables('clusterName'))]", - "policyAssignmentNameBlockDefaultNamespace": "[guid(variables('policyResourceIdBlockDefaultNamespace'), resourceGroup().name, variables('clusterName'))]", - "policyAssignmentNameApprovedContainerPortsOnly": "[guid(variables('policyResourceIdApprovedContainerPortsOnly'), resourceGroup().name, variables('clusterName'))]", - "policyAssignmentNameApprovedServicePortsOnly": "[guid(variables('policyResourceIdApprovedServicePortsOnly'), resourceGroup().name, variables('clusterName'))]", - "policyAssignmentNameMustUseSpecifiedLabels": "[guid(variables('policyResourceIdMustUseSpecifiedLabels'), resourceGroup().name, variables('clusterName'))]", - "policyAssignmentNameMustUseTheseExternalIps": "[guid(variables('policyResourceIdMustUseTheseExternalIps'), resourceGroup().name, variables('clusterName'))]", - "policyAssignmentNameMustNotAutomountApiCreds": "[guid(variables('policyResourceIdMustNotAutomountApiCreds'), resourceGroup().name, variables('clusterName'))]", - - "aksSecurityCenterWorkbookData": "{\"version\":\"Notebook/1.0\",\"items\":[{\"type\":1,\"content\":{\"json\":\"## AKS Security\\n\"},\"name\":\"text - 2\"},{\"type\":9,\"content\":{\"version\":\"KqlParameterItem/1.0\",\"crossComponentResources\":[\"{workspaces}\"],\"parameters\":[{\"id\":\"311d3728-7f8a-4b16-8a34-097d099323d5\",\"version\":\"KqlParameterItem/1.0\",\"name\":\"subscription\",\"label\":\"Subscription\",\"type\":6,\"isRequired\":true,\"multiSelect\":true,\"quote\":\"'\",\"delimiter\":\",\",\"value\":[],\"typeSettings\":{\"additionalResourceOptions\":[],\"includeAll\":false,\"showDefault\":false}},{\"id\":\"3a56d260-4fb9-46d6-b121-cea854104c91\",\"version\":\"KqlParameterItem/1.0\",\"name\":\"workspaces\",\"label\":\"Workspaces\",\"type\":5,\"isRequired\":true,\"multiSelect\":true,\"quote\":\"'\",\"delimiter\":\",\",\"query\":\"where type =~ 'microsoft.operationalinsights/workspaces'\\r\\n| where strcat('/subscriptions/',subscriptionId) in ({subscription})\\r\\n| project id\",\"crossComponentResources\":[\"{subscription}\"],\"typeSettings\":{\"additionalResourceOptions\":[\"value::all\"]},\"queryType\":1,\"resourceType\":\"microsoft.resourcegraph/resources\",\"value\":[\"value::all\"]},{\"id\":\"9615cea6-c661-470a-b4ae-1aab8ae6f448\",\"version\":\"KqlParameterItem/1.0\",\"name\":\"clustername\",\"label\":\"Cluster name\",\"type\":5,\"isRequired\":true,\"multiSelect\":true,\"quote\":\"'\",\"delimiter\":\",\",\"query\":\"where type == \\\"microsoft.containerservice/managedclusters\\\"\\r\\n| where strcat('/subscriptions/',subscriptionId) in ({subscription})\\r\\n| distinct tolower(id)\",\"crossComponentResources\":[\"{subscription}\"],\"value\":[\"value::all\"],\"typeSettings\":{\"resourceTypeFilter\":{\"microsoft.containerservice/managedclusters\":true},\"additionalResourceOptions\":[\"value::all\"],\"showDefault\":false},\"timeContext\":{\"durationMs\":86400000},\"queryType\":1,\"resourceType\":\"microsoft.resourcegraph/resources\"},{\"id\":\"236c00ec-1493-4e60-927a-a18b8b120cd5\",\"version\":\"KqlParameterItem/1.0\",\"name\":\"timeframe\",\"label\":\"Time range\",\"type\":4,\"description\":\"Time\",\"isRequired\":true,\"value\":{\"durationMs\":172800000},\"typeSettings\":{\"selectableValues\":[{\"durationMs\":300000},{\"durationMs\":900000},{\"durationMs\":1800000},{\"durationMs\":3600000},{\"durationMs\":14400000},{\"durationMs\":43200000},{\"durationMs\":86400000},{\"durationMs\":172800000},{\"durationMs\":259200000},{\"durationMs\":604800000},{\"durationMs\":1209600000},{\"durationMs\":2419200000},{\"durationMs\":2592000000},{\"durationMs\":5184000000},{\"durationMs\":7776000000}],\"allowCustom\":true},\"timeContext\":{\"durationMs\":86400000}},{\"id\":\"bf0a3e4f-fff9-450c-b9d3-c8c1dded9787\",\"version\":\"KqlParameterItem/1.0\",\"name\":\"nodeRgDetails\",\"type\":1,\"query\":\"where type == \\\"microsoft.containerservice/managedclusters\\\"\\r\\n| where tolower(id) in ({clustername})\\r\\n| project nodeRG = properties.nodeResourceGroup, subscriptionId, id = toupper(id)\\r\\n| project nodeRgDetails = strcat('\\\"', nodeRG, \\\";\\\", subscriptionId, \\\";\\\", id, '\\\"')\",\"crossComponentResources\":[\"value::all\"],\"isHiddenWhenLocked\":true,\"timeContext\":{\"durationMs\":86400000},\"queryType\":1,\"resourceType\":\"microsoft.resourcegraph/resources\"},{\"id\":\"df53126c-c40f-43d5-b99f-97ee3785c086\",\"version\":\"KqlParameterItem/1.0\",\"name\":\"diagnosticClusters\",\"type\":1,\"query\":\"union withsource=_TableName *\\r\\n| where _TableName == \\\"AzureDiagnostics\\\" and Category == \\\"kube-audit\\\"\\r\\n| summarize diagnosticClusters = dcount(ResourceId)\\r\\n| project isDiagnosticCluster = iff(diagnosticClusters > 0, \\\"yes\\\", \\\"no\\\")\",\"crossComponentResources\":[\"{workspaces}\"],\"isHiddenWhenLocked\":true,\"timeContext\":{\"durationMs\":172800000},\"timeContextFromParameter\":\"timeframe\",\"queryType\":0,\"resourceType\":\"microsoft.operationalinsights/workspaces\"}],\"style\":\"pills\",\"queryType\":0,\"resourceType\":\"microsoft.operationalinsights/workspaces\"},\"name\":\"parameters - 3\"},{\"type\":11,\"content\":{\"version\":\"LinkItem/1.0\",\"style\":\"tabs\",\"links\":[{\"id\":\"07cf87dc-8234-47db-850d-ec41b2687b2a\",\"cellValue\":\"mainTab\",\"linkTarget\":\"parameter\",\"linkLabel\":\"Microsoft Defender for Kubernetes\",\"subTarget\":\"alerts\",\"preText\":\"\",\"style\":\"link\"},{\"id\":\"44033ee6-d83e-4253-a732-c258ef1da545\",\"cellValue\":\"mainTab\",\"linkTarget\":\"parameter\",\"linkLabel\":\"Analytics over Diagnostic logs\",\"subTarget\":\"diagnostics\",\"style\":\"link\"}]},\"name\":\"links - 22\"},{\"type\":12,\"content\":{\"version\":\"NotebookGroup/1.0\",\"groupType\":\"editable\",\"items\":[{\"type\":1,\"content\":{\"json\":\"## Microsoft Defender for AKS coverage\"},\"name\":\"text - 10\"},{\"type\":3,\"content\":{\"version\":\"KqlItem/1.0\",\"query\":\"datatable (Event:string)\\r\\n [\\\"AKS Workbook\\\"]\\r\\n| extend cluster = (strcat(\\\"[\\\", \\\"{clustername}\\\", \\\"]\\\"))\\r\\n| extend cluster = todynamic(replace(\\\"'\\\", '\\\"', cluster))\\r\\n| mvexpand cluster\\r\\n| extend subscriptionId = extract(@\\\"/subscriptions/([^/]+)\\\", 1, tostring(cluster))\\r\\n| summarize AksClusters = count() by subscriptionId, DefenderForAks = 0\\r\\n| union\\r\\n(\\r\\nsecurityresources\\r\\n| where type =~ \\\"microsoft.security/pricings\\\"\\r\\n| where name == \\\"KubernetesService\\\"\\r\\n| project DefenderForAks = iif(properties.pricingTier == 'Standard', 1, 0), AksClusters = 0, subscriptionId\\r\\n)\\r\\n| summarize AksClusters = sum(AksClusters), DefenderForAks = sum(DefenderForAks) by subscriptionId\\r\\n| project Subscription = strcat('/subscriptions/', subscriptionId), [\\\"AKS clusters\\\"] = AksClusters, ['Defender for AKS'] = iif(DefenderForAks > 0,'yes','no'), ['Microsoft Microsoft Defender'] = iif(DefenderForAks > 0, '', 'https://ms.portal.azure.com/#blade/Microsoft_Azure_Security/SecurityMenuBlade/26')\\r\\n| order by ['Defender for AKS'] asc\",\"size\":0,\"queryType\":1,\"resourceType\":\"microsoft.resourcegraph/resources\",\"crossComponentResources\":[\"{subscription}\"],\"gridSettings\":{\"formatters\":[{\"columnMatch\":\"Defender for AKS\",\"formatter\":18,\"formatOptions\":{\"thresholdsOptions\":\"icons\",\"thresholdsGrid\":[{\"operator\":\"==\",\"thresholdValue\":\"no\",\"representation\":\"4\",\"text\":\"\"},{\"operator\":\"Default\",\"thresholdValue\":null,\"representation\":\"success\",\"text\":\"\"}]}},{\"columnMatch\":\"Onboard Microsoft Defender\",\"formatter\":7,\"formatOptions\":{\"linkTarget\":\"Url\",\"linkLabel\":\"\"}}]}},\"customWidth\":\"66\",\"name\":\"query - 9\"},{\"type\":3,\"content\":{\"version\":\"KqlItem/1.0\",\"query\":\"datatable (Event:string)\\r\\n [\\\"AKS Workbook\\\"]\\r\\n| extend cluster = (strcat(\\\"[\\\", \\\"{clustername}\\\", \\\"]\\\"))\\r\\n| extend cluster = todynamic(replace(\\\"'\\\", '\\\"', cluster))\\r\\n| mvexpand cluster\\r\\n| extend subscriptionId = extract(@\\\"/subscriptions/([^/]+)\\\", 1, tostring(cluster))\\r\\n| summarize AksClusters = count() by subscriptionId, DefenderForAks = 0\\r\\n| union\\r\\n(\\r\\nsecurityresources\\r\\n| where type =~ \\\"microsoft.security/pricings\\\"\\r\\n| where name == \\\"KubernetesService\\\"\\r\\n| project DefenderForAks = iif(properties.pricingTier == 'Standard', 1, 0), AksClusters = 0, subscriptionId\\r\\n)\\r\\n| summarize AksClusters = sum(AksClusters), DefenderForAks = sum(DefenderForAks) by subscriptionId\\r\\n| project Subscription = 1, ['Defender for AKS'] = iif(DefenderForAks > 0,'Protected by Microsoft Defender','Not protected by Microsoft Defender')\",\"size\":0,\"queryType\":1,\"resourceType\":\"microsoft.resourcegraph/resources\",\"crossComponentResources\":[\"{subscription}\"],\"visualization\":\"piechart\"},\"customWidth\":\"33\",\"name\":\"query - 11\"},{\"type\":1,\"content\":{\"json\":\"### AKS alerts overview\"},\"name\":\"text - 21\"},{\"type\":3,\"content\":{\"version\":\"KqlItem/1.0\",\"query\":\"SecurityAlert \\r\\n| where tolower(ResourceId) in ({clustername})\\r\\n| where TimeGenerated {timeframe}\\r\\n| where AlertType startswith \\\"AKS_\\\"\\r\\n| project image = tostring(todynamic(ExtendedProperties)[\\\"Container image\\\"]), AlertType\\r\\n| where image != \\\"\\\"\\r\\n| summarize AlertTypes = dcount(AlertType) by image\\r\\n| where AlertTypes > 1\\r\\n//| render piechart \\r\\n\",\"size\":4,\"title\":\"Images with multiple alerts\",\"queryType\":0,\"resourceType\":\"microsoft.operationalinsights/workspaces\",\"crossComponentResources\":[\"{workspaces}\"],\"visualization\":\"tiles\",\"tileSettings\":{\"showBorder\":false,\"titleContent\":{\"columnMatch\":\"image\",\"formatter\":1},\"leftContent\":{\"columnMatch\":\"AlertTypes\",\"formatter\":12,\"formatOptions\":{\"palette\":\"auto\"},\"numberFormat\":{\"unit\":17,\"options\":{\"maximumSignificantDigits\":3,\"maximumFractionDigits\":2}}}}},\"customWidth\":\"33\",\"name\":\"query - 12\"},{\"type\":3,\"content\":{\"version\":\"KqlItem/1.0\",\"query\":\"SecurityAlert \\r\\n| where tolower(ResourceId) in ({clustername})\\r\\n| where TimeGenerated {timeframe}\\r\\n| where AlertType startswith \\\"AKS_\\\"\\r\\n| project AlertType, name = extract(@\\\"/MICROSOFT.CONTAINERSERVICE/MANAGEDCLUSTERS/(.+)\\\", 1, ResourceId)\\r\\n| summarize AlertTypes = dcount(AlertType) by name\\r\\n| where AlertTypes > 1\\r\\n\",\"size\":4,\"title\":\"Clusters with multiple alert types\",\"queryType\":0,\"resourceType\":\"microsoft.operationalinsights/workspaces\",\"crossComponentResources\":[\"{workspaces}\"],\"visualization\":\"tiles\",\"tileSettings\":{\"showBorder\":false,\"titleContent\":{\"columnMatch\":\"name\",\"formatter\":1},\"leftContent\":{\"columnMatch\":\"AlertTypes\",\"formatter\":12,\"formatOptions\":{\"palette\":\"auto\"},\"numberFormat\":{\"unit\":17,\"options\":{\"maximumSignificantDigits\":3,\"maximumFractionDigits\":2}}}}},\"customWidth\":\"33\",\"name\":\"query - 12 - Copy\"},{\"type\":3,\"content\":{\"version\":\"KqlItem/1.0\",\"query\":\"SecurityAlert \\r\\n| where tolower(ResourceId) in ({clustername})\\r\\n| where TimeGenerated {timeframe}\\r\\n| where AlertType startswith \\\"AKS_\\\"\\r\\n| project AlertType, name = extract(@\\\"/MICROSOFT.CONTAINERSERVICE/MANAGEDCLUSTERS/(.+)\\\", 1, ResourceId)\\r\\n| summarize count() by name\\r\\n\\r\\n\",\"size\":4,\"title\":\"Alerts triggered by cluster\",\"queryType\":0,\"resourceType\":\"microsoft.operationalinsights/workspaces\",\"crossComponentResources\":[\"{workspaces}\"],\"visualization\":\"tiles\",\"tileSettings\":{\"showBorder\":false,\"titleContent\":{\"columnMatch\":\"name\",\"formatter\":1},\"leftContent\":{\"columnMatch\":\"count_\",\"formatter\":12,\"formatOptions\":{\"palette\":\"auto\"},\"numberFormat\":{\"unit\":17,\"options\":{\"maximumSignificantDigits\":3,\"maximumFractionDigits\":2}}}}},\"customWidth\":\"33\",\"name\":\"query - 12 - Copy - Copy\"},{\"type\":1,\"content\":{\"json\":\"### Seucirty alerts details\\r\\n\\r\\nTo filter, press on the severities below.\\r\\nYou can also filter based on a specific resource.\"},\"name\":\"text - 18\"},{\"type\":3,\"content\":{\"version\":\"KqlItem/1.0\",\"query\":\"SecurityAlert\\r\\n| where TimeGenerated {timeframe}\\r\\n| extend rg = extract(@\\\"/resourcegroups/([^/]+)\\\", 1, tolower(ResourceId))\\r\\n| extend sub = extract(@\\\"/subscriptions/([^/]+)\\\", 1, tolower(ResourceId))\\r\\n| extend nodesDetailsArr = todynamic('{nodeRgDetails}')\\r\\n| mv-expand singleNodeDetails = nodesDetailsArr\\r\\n| extend singleNodeArr = split(singleNodeDetails, \\\";\\\")\\r\\n| where tolower(singleNodeArr[0]) == rg and tolower(singleNodeArr[1]) == sub\\r\\n| project AlertSeverity\\r\\n| union\\r\\n(\\r\\nSecurityAlert \\r\\n| where tolower(ResourceId) in ({clustername})\\r\\n| where TimeGenerated {timeframe}\\r\\n| where AlertType startswith \\\"AKS_\\\"\\r\\n| project AlertSeverity\\r\\n)\\r\\n| summarize count() by AlertSeverity\",\"size\":0,\"title\":\"Alerts by severity\",\"exportMultipleValues\":true,\"exportedParameters\":[{\"fieldName\":\"AlertSeverity\",\"parameterName\":\"severity\",\"parameterType\":1,\"quote\":\"\"}],\"queryType\":0,\"resourceType\":\"microsoft.operationalinsights/workspaces\",\"crossComponentResources\":[\"{workspaces}\"],\"visualization\":\"tiles\",\"tileSettings\":{\"showBorder\":false,\"titleContent\":{\"columnMatch\":\"AlertSeverity\",\"formatter\":1},\"leftContent\":{\"columnMatch\":\"count_\",\"formatter\":12,\"formatOptions\":{\"palette\":\"auto\"},\"numberFormat\":{\"unit\":17,\"options\":{\"maximumSignificantDigits\":3,\"maximumFractionDigits\":2}}}}},\"customWidth\":\"11\",\"name\":\"Alerts by severity\"},{\"type\":3,\"content\":{\"version\":\"KqlItem/1.0\",\"query\":\"SecurityAlert\\r\\n| where TimeGenerated {timeframe}\\r\\n| where \\\"{severity}\\\" has AlertSeverity or isempty(\\\"{severity}\\\")\\r\\n| extend rg = extract(@\\\"/resourcegroups/([^/]+)\\\", 1, tolower(ResourceId))\\r\\n| extend sub = extract(@\\\"/subscriptions/([^/]+)\\\", 1, tolower(ResourceId))\\r\\n| extend nodesDetailsArr = todynamic('{nodeRgDetails}')\\r\\n| mv-expand singleNodeDetails = nodesDetailsArr\\r\\n| extend singleNodeArr = split(singleNodeDetails, \\\";\\\")\\r\\n| where tolower(singleNodeArr[0]) == rg and tolower(singleNodeArr[1]) == sub\\r\\n| project ResourceId\\r\\n| union\\r\\n(\\r\\nSecurityAlert \\r\\n| where tolower(ResourceId) in ({clustername})\\r\\n| where \\\"{severity}\\\" has AlertSeverity or isempty(\\\"{severity}\\\")\\r\\n| where TimeGenerated {timeframe}\\r\\n| where AlertType startswith \\\"AKS_\\\"\\r\\n| project ResourceId\\r\\n)\\r\\n| summarize Alerts = count() by ResourceId\\r\\n| order by Alerts desc\\r\\n| limit 10\",\"size\":0,\"title\":\"Resources with most alerts\",\"exportFieldName\":\"ResourceId\",\"exportParameterName\":\"selectedResource\",\"exportDefaultValue\":\"not_selected\",\"queryType\":0,\"resourceType\":\"microsoft.operationalinsights/workspaces\",\"crossComponentResources\":[\"{workspaces}\"],\"gridSettings\":{\"formatters\":[{\"columnMatch\":\"Alerts\",\"formatter\":4,\"formatOptions\":{\"palette\":\"red\"}}]}},\"customWidth\":\"22\",\"name\":\"Resources with most alerts\"},{\"type\":3,\"content\":{\"version\":\"KqlItem/1.0\",\"query\":\"\\r\\nSecurityAlert\\r\\n| where TimeGenerated {timeframe}\\r\\n| where \\\"{severity}\\\" has AlertSeverity or isempty(\\\"{severity}\\\")\\r\\n| extend rg = extract(@\\\"/resourcegroups/([^/]+)\\\", 1, tolower(ResourceId))\\r\\n| extend sub = extract(@\\\"/subscriptions/([^/]+)\\\", 1, tolower(ResourceId))\\r\\n| extend nodesDetailsArr = todynamic('{nodeRgDetails}')\\r\\n| mv-expand singleNodeDetails = nodesDetailsArr\\r\\n| extend singleNodeArr = split(singleNodeDetails, \\\";\\\")\\r\\n| where tolower(singleNodeArr[0]) == rg and tolower(singleNodeArr[1]) == sub\\r\\n| extend AlertResourceType = \\\"VM alerts\\\"\\r\\n| union\\r\\n(\\r\\nSecurityAlert \\r\\n| where tolower(ResourceId) in ({clustername})\\r\\n| where \\\"{severity}\\\" has AlertSeverity or isempty(\\\"{severity}\\\")\\r\\n| where TimeGenerated {timeframe}\\r\\n| where AlertType startswith \\\"AKS_\\\"\\r\\n| extend AlertResourceType = \\\"Cluster alerts\\\"\\r\\n)\\r\\n| summarize Alerts = count() by bin(TimeGenerated, {timeframe:grain}), AlertResourceType\",\"size\":0,\"title\":\"Alerts over time\",\"queryType\":0,\"resourceType\":\"microsoft.operationalinsights/workspaces\",\"crossComponentResources\":[\"{workspaces}\"],\"visualization\":\"timechart\"},\"customWidth\":\"66\",\"name\":\"Alerts over time\"},{\"type\":3,\"content\":{\"version\":\"KqlItem/1.0\",\"query\":\"SecurityAlert\\r\\n| where TimeGenerated {timeframe}\\r\\n| where \\\"{severity}\\\" has AlertSeverity or isempty(\\\"{severity}\\\")\\r\\n| extend rg = extract(@\\\"/resourcegroups/([^/]+)\\\", 1, tolower(ResourceId))\\r\\n| extend sub = extract(@\\\"/subscriptions/([^/]+)\\\", 1, tolower(ResourceId))\\r\\n| extend nodesDetailsArr = todynamic('{nodeRgDetails}')\\r\\n| mv-expand singleNodeDetails = nodesDetailsArr\\r\\n| extend singleNodeArr = split(singleNodeDetails, \\\";\\\")\\r\\n| where tolower(singleNodeArr[0]) == rg and tolower(singleNodeArr[1]) == sub\\r\\n| where tolower(ResourceId) == tolower(\\\"{selectedResource}\\\") or \\\"{selectedResource}\\\" == \\\"not_selected\\\"\\r\\n| project [\\\"Resource name\\\"] = ResourceId, TimeGenerated, AlertSeverity, [\\\"AKS cluster\\\"] = toupper(singleNodeArr[2]), DisplayName, AlertLink\\r\\n| union\\r\\n(\\r\\nSecurityAlert\\r\\n| where TimeGenerated {timeframe}\\r\\n| where tolower(ResourceId) in ({clustername})\\r\\n| where \\\"{severity}\\\" has AlertSeverity or isempty(\\\"{severity}\\\")\\r\\n| where AlertType startswith \\\"AKS_\\\"\\r\\n| where tolower(ResourceId) == tolower(\\\"{selectedResource}\\\") or \\\"{selectedResource}\\\" == \\\"not_selected\\\"\\r\\n| project [\\\"Resource name\\\"] = ResourceId, TimeGenerated, AlertSeverity, [\\\"AKS cluster\\\"] = ResourceId, DisplayName, AlertLink\\r\\n)\\r\\n| order by TimeGenerated asc\",\"size\":0,\"title\":\"Microsoft Defender alerts\",\"queryType\":0,\"resourceType\":\"microsoft.operationalinsights/workspaces\",\"crossComponentResources\":[\"{workspaces}\"],\"gridSettings\":{\"formatters\":[{\"columnMatch\":\"AlertLink\",\"formatter\":7,\"formatOptions\":{\"linkTarget\":\"Url\",\"linkLabel\":\"Go to alert \"}}],\"filter\":true},\"sortBy\":[]},\"name\":\"Microsoft Defender alerts\",\"styleSettings\":{\"showBorder\":true}}]},\"conditionalVisibility\":{\"parameterName\":\"mainTab\",\"comparison\":\"isEqualTo\",\"value\":\"alerts\"},\"name\":\"Defender Alerts\"},{\"type\":12,\"content\":{\"version\":\"NotebookGroup/1.0\",\"groupType\":\"editable\",\"items\":[{\"type\":1,\"content\":{\"json\":\"## Diagnostic logs coverage\"},\"name\":\"text - 15\"},{\"type\":3,\"content\":{\"version\":\"KqlItem/1.0\",\"query\":\"union withsource=_TableName *\\r\\n| where _TableName == \\\"AzureDiagnostics\\\" and Category == \\\"kube-audit\\\"\\r\\n| summarize count() by ResourceId = tolower(ResourceId)\\r\\n| summarize logsClusters = make_set(ResourceId)\\r\\n| extend selectedClusters = \\\"[{clustername}]\\\"\\r\\n| extend selectedClusters = replace(\\\"'\\\", '\\\"', selectedClusters)\\r\\n| extend selectedClusters = todynamic(selectedClusters)\\r\\n| mv-expand clusterId = selectedClusters\\r\\n| project clusterId = toupper(tostring(clusterId)), [\\\"Diagnostic logs\\\"] = (logsClusters has tostring(clusterId))\\r\\n| extend [\\\"Diagnostic settings\\\"] = iff([\\\"Diagnostic logs\\\"] == false, strcat(\\\"https://ms.portal.azure.com/#@microsoft.onmicrosoft.com/resource\\\", clusterId, \\\"/diagnostics\\\"), \\\"\\\")\\r\\n\",\"size\":0,\"timeContext\":{\"durationMs\":172800000},\"timeContextFromParameter\":\"timeframe\",\"queryType\":0,\"resourceType\":\"microsoft.operationalinsights/workspaces\",\"crossComponentResources\":[\"{workspaces}\"],\"gridSettings\":{\"formatters\":[{\"columnMatch\":\"Diagnostic logs\",\"formatter\":18,\"formatOptions\":{\"thresholdsOptions\":\"icons\",\"thresholdsGrid\":[{\"operator\":\"==\",\"thresholdValue\":\"false\",\"representation\":\"critical\",\"text\":\"\"},{\"operator\":\"Default\",\"thresholdValue\":null,\"representation\":\"success\",\"text\":\"\"}]}},{\"columnMatch\":\"Diagnostic settings\",\"formatter\":7,\"formatOptions\":{\"linkTarget\":\"Url\"}}],\"filter\":true,\"sortBy\":[{\"itemKey\":\"$gen_thresholds_Diagnostic logs_1\",\"sortOrder\":2}]},\"sortBy\":[{\"itemKey\":\"$gen_thresholds_Diagnostic logs_1\",\"sortOrder\":2}]},\"customWidth\":\"66\",\"name\":\"query - 14\"},{\"type\":3,\"content\":{\"version\":\"KqlItem/1.0\",\"query\":\"union withsource=_TableName *\\r\\n| where _TableName == \\\"AzureDiagnostics\\\" and Category == \\\"kube-audit\\\"\\r\\n| summarize count() by ResourceId = tolower(ResourceId)\\r\\n| summarize logsClusters = make_set(ResourceId)\\r\\n| extend selectedClusters = \\\"[{clustername}]\\\"\\r\\n| extend selectedClusters = replace(\\\"'\\\", '\\\"', selectedClusters)\\r\\n| extend selectedClusters = todynamic(selectedClusters)\\r\\n| mv-expand clusterId = selectedClusters\\r\\n| project clusterId = toupper(tostring(clusterId)), hasDiagnosticLogs = (logsClusters has tostring(clusterId))\\r\\n| summarize [\\\"number of clusters\\\"] = count() by hasDiagnosticLogs\\r\\n| extend hasDiagnosticLogs = iff(hasDiagnosticLogs == true, \\\"Clusters with Diagnostic logs\\\", \\\"Clusters without Diagnostic logs\\\")\\r\\n\",\"size\":0,\"timeContext\":{\"durationMs\":172800000},\"timeContextFromParameter\":\"timeframe\",\"queryType\":0,\"resourceType\":\"microsoft.operationalinsights/workspaces\",\"crossComponentResources\":[\"{workspaces}\"],\"visualization\":\"piechart\"},\"customWidth\":\"33\",\"name\":\"query - 17\"},{\"type\":12,\"content\":{\"version\":\"NotebookGroup/1.0\",\"groupType\":\"editable\",\"items\":[{\"type\":1,\"content\":{\"json\":\"## Cluster operations\"},\"name\":\"text - 16\"},{\"type\":11,\"content\":{\"version\":\"LinkItem/1.0\",\"style\":\"tabs\",\"links\":[{\"id\":\"3f616701-fd4b-482c-aff1-a85414daa05c\",\"cellValue\":\"dispalyedGraph\",\"linkTarget\":\"parameter\",\"linkLabel\":\"Masterclient operations\",\"subTarget\":\"masterclient\",\"preText\":\"\",\"style\":\"link\"},{\"id\":\"e6fa55f1-7d57-4f5e-8e83-429740853731\",\"cellValue\":\"dispalyedGraph\",\"linkTarget\":\"parameter\",\"linkLabel\":\"Pod creation operations\",\"subTarget\":\"podCreation\",\"style\":\"link\"},{\"id\":\"f4c46251-0090-4ca3-a81c-0686bff3ff35\",\"cellValue\":\"dispalyedGraph\",\"linkTarget\":\"parameter\",\"linkLabel\":\"Secret get\\\\list operations\",\"subTarget\":\"secretOperation\",\"style\":\"link\"}]},\"name\":\"links - 11\"},{\"type\":3,\"content\":{\"version\":\"KqlItem/1.0\",\"query\":\"AzureDiagnostics \\r\\n| where tolower(ResourceId) in ({clustername})\\r\\n| where TimeGenerated {timeframe}\\r\\n| where Category == \\\"kube-audit\\\"\\r\\n| where log_s has \\\"masterclient\\\"\\r\\n| project TimeGenerated, parse_json(log_s), ResourceId\\r\\n| project TimeGenerated, ResourceId, username = tostring(log_s[\\\"user\\\"].username)\\r\\n| where username == \\\"masterclient\\\"\\r\\n| extend name = extract(@\\\"/MICROSOFT.CONTAINERSERVICE/MANAGEDCLUSTERS/(.+)\\\", 1, ResourceId)\\r\\n| summarize count() by name, bin(TimeGenerated, 1h)\",\"size\":0,\"queryType\":0,\"resourceType\":\"microsoft.operationalinsights/workspaces\",\"crossComponentResources\":[\"{workspaces}\"],\"visualization\":\"timechart\",\"tileSettings\":{\"showBorder\":false,\"titleContent\":{\"columnMatch\":\"name\",\"formatter\":1},\"leftContent\":{\"columnMatch\":\"count_\",\"formatter\":12,\"formatOptions\":{\"palette\":\"auto\"},\"numberFormat\":{\"unit\":17,\"options\":{\"maximumSignificantDigits\":3,\"maximumFractionDigits\":2}}}},\"chartSettings\":{\"yAxis\":[\"count_\"]}},\"conditionalVisibility\":{\"parameterName\":\"dispalyedGraph\",\"comparison\":\"isEqualTo\",\"value\":\"masterclient\"},\"name\":\"Masterclient operations\",\"styleSettings\":{\"showBorder\":true}},{\"type\":3,\"content\":{\"version\":\"KqlItem/1.0\",\"query\":\"AzureDiagnostics\\r\\n| where Category == \\\"kube-audit\\\"\\r\\n| where tolower(ResourceId) in ({clustername})\\r\\n| where TimeGenerated {timeframe}\\r\\n| where log_s has \\\"pods\\\"\\r\\n| project TimeGenerated, parse_json(log_s), ResourceId\\r\\n| project AzureResourceId = ResourceId, TimeGenerated,\\r\\n RequestURI = tostring(log_s[\\\"requestURI\\\"]),\\r\\n Verb = tostring(log_s[\\\"verb\\\"]),\\r\\n ObjectRef = log_s[\\\"objectRef\\\"],\\r\\n ResponseStatus = log_s[\\\"responseStatus\\\"]\\r\\n//Main query\\r\\n| where ObjectRef.resource == \\\"pods\\\" and Verb == \\\"create\\\" and ResponseStatus.code startswith \\\"20\\\"\\r\\n and RequestURI endswith \\\"/pods\\\"\\r\\n| extend name = extract(@\\\"/MICROSOFT.CONTAINERSERVICE/MANAGEDCLUSTERS/(.+)\\\", 1, AzureResourceId)\\r\\n| summarize count() by name, bin(TimeGenerated, 1h)\",\"size\":0,\"queryType\":0,\"resourceType\":\"microsoft.operationalinsights/workspaces\",\"crossComponentResources\":[\"{workspaces}\"],\"visualization\":\"timechart\"},\"conditionalVisibility\":{\"parameterName\":\"dispalyedGraph\",\"comparison\":\"isEqualTo\",\"value\":\"podCreation\"},\"name\":\"pods creation\",\"styleSettings\":{\"showBorder\":true}},{\"type\":3,\"content\":{\"version\":\"KqlItem/1.0\",\"query\":\"AzureDiagnostics\\r\\n| where Category == \\\"kube-audit\\\"\\r\\n| where tolower(ResourceId) in ({clustername})\\r\\n| where TimeGenerated {timeframe}\\r\\n| where log_s has \\\"secrets\\\"\\r\\n| project TimeGenerated, parse_json(log_s), ResourceId\\r\\n| project AzureResourceId = ResourceId, TimeGenerated,\\r\\n RequestURI = tostring(log_s[\\\"requestURI\\\"]),\\r\\n Verb = tostring(log_s[\\\"verb\\\"]),\\r\\n ObjectRef = log_s[\\\"objectRef\\\"],\\r\\n ResponseStatus = log_s[\\\"responseStatus\\\"]\\r\\n//Main query\\r\\n| where ObjectRef.resource == \\\"secrets\\\" and (Verb == \\\"list\\\" or Verb == \\\"get\\\") and ResponseStatus.code startswith \\\"20\\\"\\r\\n| where ObjectRef.name != \\\"tunnelfront\\\" and ObjectRef.name != \\\"tunnelend\\\" and ObjectRef.name != \\\"kubernetes-dashboard-key-holder\\\"\\r\\n| extend name = extract(@\\\"/MICROSOFT.CONTAINERSERVICE/MANAGEDCLUSTERS/(.+)\\\", 1, AzureResourceId)\\r\\n| summarize count() by name, bin(TimeGenerated, 1h)\",\"size\":0,\"timeContext\":{\"durationMs\":172800000},\"timeContextFromParameter\":\"timeframe\",\"queryType\":0,\"resourceType\":\"microsoft.operationalinsights/workspaces\",\"crossComponentResources\":[\"{workspaces}\"],\"visualization\":\"timechart\",\"gridSettings\":{\"sortBy\":[{\"itemKey\":\"count_\",\"sortOrder\":2}]},\"sortBy\":[{\"itemKey\":\"count_\",\"sortOrder\":2}]},\"conditionalVisibility\":{\"parameterName\":\"dispalyedGraph\",\"comparison\":\"isEqualTo\",\"value\":\"secretOperation\"},\"name\":\"secrets operation\",\"styleSettings\":{\"showBorder\":true}},{\"type\":3,\"content\":{\"version\":\"KqlItem/1.0\",\"query\":\"let ascAlerts = \\nunion withsource=_TableName *\\n| where _TableName == \\\"SecurityAlert\\\"\\n| where tolower(ResourceId) in ({clustername})\\n| where TimeGenerated {timeframe}\\n| extend AlertType = column_ifexists(\\\"AlertType\\\", \\\"\\\")\\n| where AlertType == \\\"AKS_PrivilegedContainer\\\"\\n| extend ExtendedProperties = column_ifexists(\\\"ExtendedProperties\\\", todynamic(\\\"\\\"))\\n| extend ExtendedProperties = parse_json(ExtendedProperties)\\n| extend AlertLink = column_ifexists(\\\"AlertLink\\\", \\\"\\\")\\n| summarize arg_min(TimeGenerated, AlertLink) by AzureResourceId = ResourceId, name = tostring(ExtendedProperties[\\\"Pod name\\\"]), podNamespace = tostring(ExtendedProperties[\\\"Namespace\\\"])\\n;\\nlet podOperations = AzureDiagnostics\\n| where Category == \\\"kube-audit\\\"\\n| where tolower(ResourceId) in ({clustername})\\n| where TimeGenerated {timeframe}\\n| where log_s has \\\"privileged\\\"\\n| project TimeGenerated, parse_json(log_s), ResourceId\\n| project AzureResourceId = ResourceId, TimeGenerated,\\n RequestURI = tostring(log_s[\\\"requestURI\\\"]),\\n Verb = tostring(log_s[\\\"verb\\\"]),\\n ObjectRef = log_s[\\\"objectRef\\\"],\\n RequestObject = log_s[\\\"requestObject\\\"],\\n ResponseStatus = log_s[\\\"responseStatus\\\"],\\n ResponseObject = log_s[\\\"responseObject\\\"]\\n//Main query\\n| where ObjectRef.resource == \\\"pods\\\" and Verb == \\\"create\\\" and ResponseStatus.code startswith \\\"20\\\" and RequestObject has \\\"privileged\\\"\\n and RequestURI endswith \\\"/pods\\\"\\n| extend containers = RequestObject.spec.containers\\n| mvexpand containers\\n| where containers.securityContext.privileged == true\\n| summarize TimeGenerated = min(TimeGenerated) by\\n name = tostring(ResponseObject.metadata.name),\\n podNamespace = tostring(ResponseObject.metadata.namespace),\\n imageName = tostring(containers.image),\\n containerName = tostring(containers.name),\\n AzureResourceId\\n| extend id = strcat(name,\\\";\\\", AzureResourceId)\\n| extend parent = AzureResourceId\\n| join kind=leftouter (ascAlerts) on AzureResourceId, name, podNamespace\\n;\\nlet cached = materialize(podOperations)\\n;\\nlet clusters = cached | distinct AzureResourceId\\n;\\n// Main query\\ncached\\n| union\\n(\\nclusters\\n| project \\n name = AzureResourceId,\\n id = AzureResourceId,\\n parent = \\\"\\\" \\n)\\n| project-away name1, podNamespace1, TimeGenerated1\",\"size\":1,\"title\":\"Privileged containers creation\",\"queryType\":0,\"resourceType\":\"microsoft.operationalinsights/workspaces\",\"crossComponentResources\":[\"{workspaces}\"],\"visualization\":\"table\",\"gridSettings\":{\"formatters\":[{\"columnMatch\":\"name\",\"formatter\":13,\"formatOptions\":{\"linkTarget\":null,\"showIcon\":true}},{\"columnMatch\":\"AzureResourceId\",\"formatter\":5},{\"columnMatch\":\"id\",\"formatter\":5},{\"columnMatch\":\"parent\",\"formatter\":5},{\"columnMatch\":\"AlertLink\",\"formatter\":7,\"formatOptions\":{\"linkTarget\":\"Url\",\"linkLabel\":\"\"}}],\"hierarchySettings\":{\"idColumn\":\"id\",\"parentColumn\":\"parent\",\"treeType\":0,\"expanderColumn\":\"name\"}},\"sortBy\":[]},\"customWidth\":\"66\",\"name\":\"Privileged container\",\"styleSettings\":{\"showBorder\":true}},{\"type\":3,\"content\":{\"version\":\"KqlItem/1.0\",\"query\":\"SecurityAlert \\r\\n| where tolower(ResourceId) in ({clustername})\\r\\n| where TimeGenerated {timeframe}\\r\\n| where AlertType == \\\"AKS_PrivilegedContainer\\\"\\r\\n| project name = extract(@\\\"/MICROSOFT.CONTAINERSERVICE/MANAGEDCLUSTERS/(.+)\\\", 1, ResourceId)\\r\\n| summarize alert = count() by name\",\"size\":1,\"title\":\"AKS clusters with related Microsoft Defender alerts\",\"queryType\":0,\"resourceType\":\"microsoft.operationalinsights/workspaces\",\"crossComponentResources\":[\"{workspaces}\"],\"visualization\":\"piechart\",\"tileSettings\":{\"showBorder\":false,\"titleContent\":{\"columnMatch\":\"name\",\"formatter\":1},\"leftContent\":{\"columnMatch\":\"alert\",\"formatter\":12,\"formatOptions\":{\"palette\":\"auto\"},\"numberFormat\":{\"unit\":17,\"options\":{\"maximumSignificantDigits\":3,\"maximumFractionDigits\":2}}}},\"graphSettings\":{\"type\":0,\"topContent\":{\"columnMatch\":\"name\",\"formatter\":1},\"centerContent\":{\"columnMatch\":\"alert\",\"formatter\":1,\"numberFormat\":{\"unit\":17,\"options\":{\"maximumSignificantDigits\":3,\"maximumFractionDigits\":2}}}}},\"customWidth\":\"33\",\"name\":\"query - 7\",\"styleSettings\":{\"showBorder\":true}},{\"type\":3,\"content\":{\"version\":\"KqlItem/1.0\",\"query\":\"let baseQuery = AzureDiagnostics \\r\\n| where Category == \\\"kube-audit\\\"\\r\\n| where tolower(ResourceId) in ({clustername})\\r\\n| where TimeGenerated {timeframe}\\r\\n| where log_s has \\\"exec\\\"\\r\\n| project TimeGenerated, parse_json(log_s), ResourceId\\r\\n| project TimeGenerated,\\r\\n AzureResourceId = ResourceId,\\r\\n User = log_s[\\\"user\\\"],\\r\\n StageTimestamp = todatetime(log_s[\\\"stageTimestamp\\\"]),\\r\\n Timestamp = todatetime(log_s[\\\"timestamp\\\"]),\\r\\n Stage = tostring(log_s[\\\"stage\\\"]),\\r\\n RequestURI = tostring(log_s[\\\"requestURI\\\"]),\\r\\n UserAgent = tostring(log_s[\\\"userAgent\\\"]),\\r\\n Verb = tostring(log_s[\\\"verb\\\"]),\\r\\n ObjectRef = log_s[\\\"objectRef\\\"],\\r\\n ResponseStatus = log_s[\\\"responseStatus\\\"]\\r\\n| where ObjectRef.resource == \\\"pods\\\" and Verb == \\\"create\\\" and ResponseStatus.code == 101 and ObjectRef.subresource == \\\"exec\\\"\\r\\n| project operationTime = TimeGenerated,\\r\\n RequestURI,\\r\\n podName = tostring(ObjectRef.name),\\r\\n podNamespace = tostring(ObjectRef.namespace),\\r\\n username = tostring(User.username),\\r\\n AzureResourceId\\r\\n// Parse the exec command\\r\\n| extend commands = extractall(@\\\"command=([^\\\\&]*)\\\", RequestURI)\\r\\n| extend commandsStr = url_decode(strcat_array(commands, \\\" \\\"))\\r\\n| project-away ['commands'], RequestURI\\r\\n| where username != \\\"aksProblemDetector\\\"\\r\\n;\\r\\nlet cached = materialize(baseQuery);\\r\\nlet execOperations = \\r\\ncached\\r\\n| summarize operationTime = min(operationTime), numberOfPerations = count() by name = commandsStr, username, podNamespace, podName, AzureResourceId\\r\\n| extend id = name\\r\\n| extend parentId = podName\\r\\n| project id, parentId, name, operationTime, numberOfPerations, podNamespace, username, AzureResourceId\\r\\n;\\r\\nlet podOperations = \\r\\ncached\\r\\n| summarize operationTime = min(operationTime), numberOfPerations = count() by name = podName, podNamespace, AzureResourceId\\r\\n| extend id = name\\r\\n| extend parentId = AzureResourceId\\r\\n| project id, parentId, name, operationTime, numberOfPerations, podNamespace, username = \\\"\\\", AzureResourceId\\r\\n;\\r\\nlet clusterOperations = \\r\\ncached\\r\\n| summarize operationTime = min(operationTime), numberOfPerations = count() by name = AzureResourceId\\r\\n| extend id = name\\r\\n| extend parentId = \\\"\\\"\\r\\n| project id, parentId, name, operationTime, numberOfPerations, username = \\\"\\\", podNamespace = \\\"\\\", AzureResourceId = name\\r\\n;\\r\\nunion clusterOperations, podOperations, execOperations\",\"size\":1,\"title\":\"exec commands\",\"queryType\":0,\"resourceType\":\"microsoft.operationalinsights/workspaces\",\"crossComponentResources\":[\"{workspaces}\"],\"visualization\":\"table\",\"gridSettings\":{\"formatters\":[{\"columnMatch\":\"id\",\"formatter\":5},{\"columnMatch\":\"parentId\",\"formatter\":5},{\"columnMatch\":\"numberOfPerations\",\"formatter\":4,\"formatOptions\":{\"palette\":\"blue\",\"compositeBarSettings\":{\"labelText\":\"\",\"columnSettings\":[]}}},{\"columnMatch\":\"AzureResourceId\",\"formatter\":5}],\"hierarchySettings\":{\"idColumn\":\"id\",\"parentColumn\":\"parentId\",\"treeType\":0,\"expanderColumn\":\"name\",\"expandTopLevel\":false}}},\"customWidth\":\"33\",\"name\":\"exec commands\",\"styleSettings\":{\"showBorder\":true}},{\"type\":3,\"content\":{\"version\":\"KqlItem/1.0\",\"query\":\"SecurityAlert \\r\\n| where AlertType == \\\"AKS_MaliciousContainerExec\\\"\\r\\n| where tolower(ResourceId) in ({clustername})\\r\\n| where TimeGenerated {timeframe}\\r\\n| project TimeGenerated, ResourceId, ExtendedProperties = todynamic(ExtendedProperties)\\r\\n| project TimeGenerated, ResourceId, [\\\"Pod name\\\"] = ExtendedProperties[\\\"Pod name\\\"], Command = ExtendedProperties[\\\"Command\\\"]\",\"size\":1,\"title\":\"Related Microsoft Defender alerts details\",\"queryType\":0,\"resourceType\":\"microsoft.operationalinsights/workspaces\",\"crossComponentResources\":[\"{workspaces}\"],\"gridSettings\":{\"sortBy\":[{\"itemKey\":\"TimeGenerated\",\"sortOrder\":1}]},\"sortBy\":[{\"itemKey\":\"TimeGenerated\",\"sortOrder\":1}]},\"customWidth\":\"33\",\"name\":\"query - 9\",\"styleSettings\":{\"showBorder\":true}},{\"type\":3,\"content\":{\"version\":\"KqlItem/1.0\",\"query\":\"SecurityAlert \\r\\n| where AlertType == \\\"AKS_MaliciousContainerExec\\\"\\r\\n| where tolower(ResourceId) in ({clustername})\\r\\n| where TimeGenerated {timeframe}\\r\\n| project name = extract(@\\\"/MICROSOFT.CONTAINERSERVICE/MANAGEDCLUSTERS/(.+)\\\", 1, ResourceId)\\r\\n| summarize alert = count() by name\",\"size\":1,\"title\":\"AKS clusters with related Microsoft Defender alerts\",\"queryType\":0,\"resourceType\":\"microsoft.operationalinsights/workspaces\",\"crossComponentResources\":[\"{workspaces}\"],\"visualization\":\"piechart\"},\"customWidth\":\"33\",\"name\":\"query - 8\",\"styleSettings\":{\"showBorder\":true}},{\"type\":3,\"content\":{\"version\":\"KqlItem/1.0\",\"query\":\"let ascAlerts = \\r\\nunion withsource=_TableName *\\r\\n| where _TableName == \\\"SecurityAlert\\\"\\r\\n| where tolower(ResourceId) in ({clustername})\\r\\n| where TimeGenerated {timeframe}\\r\\n| extend AlertType = column_ifexists(\\\"AlertType\\\", \\\"\\\")\\r\\n| where AlertType == \\\"AKS_SensitiveMount\\\"\\r\\n| extend ExtendedProperties = column_ifexists(\\\"ExtendedProperties\\\", todynamic(\\\"\\\"))\\r\\n| extend ExtendedProperties = parse_json(ExtendedProperties)\\r\\n| extend AlertLink = column_ifexists(\\\"AlertLink\\\", \\\"\\\")\\r\\n| summarize arg_min(TimeGenerated, AlertLink) by AzureResourceId = ResourceId, containerName = tostring(ExtendedProperties[\\\"Container name\\\"]), mountPath = tostring(ExtendedProperties[\\\"Sensitive mount path\\\"])\\r\\n;\\r\\nlet podOperations = \\r\\nAzureDiagnostics \\r\\n| where tolower(ResourceId) in ({clustername})\\r\\n| where TimeGenerated {timeframe}\\r\\n| where Category == \\\"kube-audit\\\"\\r\\n| where log_s has \\\"hostPath\\\"\\r\\n| project TimeGenerated, parse_json(log_s), ResourceId\\r\\n//Parsing\\r\\n| project AzureResourceId = ResourceId, TimeGenerated,\\r\\n Verb = tostring(log_s[\\\"verb\\\"]),\\r\\n ObjectRef = log_s[\\\"objectRef\\\"],\\r\\n RequestObject = log_s[\\\"requestObject\\\"],\\r\\n ResponseStatus = log_s[\\\"responseStatus\\\"],\\r\\n ResponseObject = log_s[\\\"responseObject\\\"]\\r\\n//\\r\\n//Main query\\r\\n//\\r\\n| where ObjectRef.resource == \\\"pods\\\" and Verb == \\\"create\\\" and ResponseStatus.code startswith \\\"20\\\" and RequestObject has \\\"hostPath\\\"\\r\\n| extend volumes = RequestObject.spec.volumes\\r\\n| mvexpand volumes\\r\\n| extend mountPath = volumes.hostPath.path\\r\\n| where mountPath != \\\"\\\" \\r\\n| extend container = RequestObject.spec.containers\\r\\n| mvexpand container\\r\\n| extend detectionTime = TimeGenerated\\r\\n| project detectionTime,\\r\\n podName = ResponseObject.metadata.name,\\r\\n podNamespace = ResponseObject.metadata.namespace,\\r\\n containerName = container.name,\\r\\n containerImage = container.image,\\r\\n mountPath,\\r\\n mountName = volumes.name,\\r\\n AzureResourceId,\\r\\n container\\r\\n| extend volumeMounts = container.volumeMounts\\r\\n| mv-expand volumeMounts\\r\\n| where tostring(volumeMounts.name) == tostring(mountName)\\r\\n| summarize operationTime = min(detectionTime) by AzureResourceId, name = tostring(podName),tostring(podNamespace), tostring(containerName), tostring(containerImage), tostring(mountPath), tostring(mountName)\\r\\n| extend id = strcat(name, \\\";\\\", AzureResourceId)\\r\\n| extend parent = AzureResourceId\\r\\n| join kind=leftouter (ascAlerts) on AzureResourceId, containerName, mountPath\\r\\n;\\r\\nlet cached = materialize(podOperations)\\r\\n;\\r\\nlet clusters = cached | distinct AzureResourceId\\r\\n;\\r\\n// Main query\\r\\ncached\\r\\n| union\\r\\n(\\r\\nclusters\\r\\n| project \\r\\n name = toupper(AzureResourceId),\\r\\n id = AzureResourceId,\\r\\n parent = \\\"\\\" \\r\\n)\\r\\n| project-away containerName1, mountPath1, TimeGenerated\\r\\n\",\"size\":1,\"title\":\"hostPath mount\",\"queryType\":0,\"resourceType\":\"microsoft.operationalinsights/workspaces\",\"crossComponentResources\":[\"{workspaces}\"],\"gridSettings\":{\"formatters\":[{\"columnMatch\":\"AzureResourceId\",\"formatter\":5},{\"columnMatch\":\"name\",\"formatter\":13,\"formatOptions\":{\"linkTarget\":null,\"showIcon\":true}},{\"columnMatch\":\"id\",\"formatter\":5},{\"columnMatch\":\"parent\",\"formatter\":5},{\"columnMatch\":\"AzureResourceId1\",\"formatter\":5},{\"columnMatch\":\"AlertLink\",\"formatter\":7,\"formatOptions\":{\"linkTarget\":\"Url\"}}],\"hierarchySettings\":{\"idColumn\":\"id\",\"parentColumn\":\"parent\",\"treeType\":0,\"expanderColumn\":\"name\"}},\"sortBy\":[]},\"customWidth\":\"66\",\"name\":\"query - 10\",\"styleSettings\":{\"showBorder\":true}},{\"type\":3,\"content\":{\"version\":\"KqlItem/1.0\",\"query\":\"SecurityAlert \\r\\n| where AlertType == \\\"AKS_SensitiveMount\\\"\\r\\n| where tolower(ResourceId) in ({clustername})\\r\\n| where TimeGenerated {timeframe}\\r\\n| project name = extract(@\\\"/MICROSOFT.CONTAINERSERVICE/MANAGEDCLUSTERS/(.+)\\\", 1, ResourceId)\\r\\n| summarize alert = count() by name\",\"size\":1,\"title\":\"AKS clusters with related Microsoft Defender alerts\",\"queryType\":0,\"resourceType\":\"microsoft.operationalinsights/workspaces\",\"crossComponentResources\":[\"{workspaces}\"],\"visualization\":\"piechart\",\"sortBy\":[]},\"customWidth\":\"33\",\"name\":\"query - 10\",\"styleSettings\":{\"showBorder\":true}},{\"type\":3,\"content\":{\"version\":\"KqlItem/1.0\",\"query\":\"let bindingOper = AzureDiagnostics\\r\\n| where Category == \\\"kube-audit\\\"\\r\\n| where tolower(ResourceId) in ({clustername})\\r\\n| where TimeGenerated {timeframe}\\r\\n| where log_s has \\\"clusterrolebindings\\\"\\r\\n| project TimeGenerated, parse_json(log_s), ResourceId\\r\\n//Parsing\\r\\n| project AzureResourceId = ResourceId, TimeGenerated,\\r\\n RequestURI = tostring(log_s[\\\"requestURI\\\"]),\\r\\n User = log_s[\\\"user\\\"],\\r\\n Verb = tostring(log_s[\\\"verb\\\"]),\\r\\n ObjectRef = log_s[\\\"objectRef\\\"],\\r\\n RequestObject = log_s[\\\"requestObject\\\"],\\r\\n ResponseStatus = log_s[\\\"responseStatus\\\"]\\r\\n| where ObjectRef.resource == \\\"clusterrolebindings\\\" and Verb == \\\"create\\\" and ResponseStatus.code startswith \\\"20\\\" and RequestObject.roleRef.name == \\\"cluster-admin\\\" \\r\\n| extend subjects = RequestObject.subjects\\r\\n| mv-expand subjects\\r\\n| project AzureResourceId, TimeGenerated, subjectName = tostring(subjects.name), subjectKind = tostring(subjects[\\\"kind\\\"]), bindingName = tostring(ObjectRef.name)\\r\\n| summarize operationTime = min(TimeGenerated) by AzureResourceId, subjectName, subjectKind, bindingName\\r\\n| extend id = strcat(subjectName, \\\";\\\", AzureResourceId)\\r\\n| extend parent = AzureResourceId\\r\\n;\\r\\nlet cached = materialize(bindingOper)\\r\\n;\\r\\nlet clusters = cached | distinct AzureResourceId\\r\\n;\\r\\n// Main query\\r\\ncached\\r\\n| union\\r\\n(\\r\\nclusters\\r\\n| project \\r\\n name = AzureResourceId,\\r\\n id = AzureResourceId,\\r\\n parent = \\\"\\\" \\r\\n)\",\"size\":1,\"title\":\"Cluster-admin binding\",\"queryType\":0,\"resourceType\":\"microsoft.operationalinsights/workspaces\",\"crossComponentResources\":[\"{workspaces}\"],\"visualization\":\"table\",\"gridSettings\":{\"formatters\":[{\"columnMatch\":\"AzureResourceId\",\"formatter\":5},{\"columnMatch\":\"id\",\"formatter\":5},{\"columnMatch\":\"parent\",\"formatter\":5},{\"columnMatch\":\"name\",\"formatter\":13,\"formatOptions\":{\"linkTarget\":null,\"showIcon\":true}}],\"hierarchySettings\":{\"idColumn\":\"id\",\"parentColumn\":\"parent\",\"treeType\":0,\"expanderColumn\":\"name\"}}},\"customWidth\":\"66\",\"name\":\"query - 5\",\"styleSettings\":{\"showBorder\":true}},{\"type\":3,\"content\":{\"version\":\"KqlItem/1.0\",\"query\":\"SecurityAlert \\r\\n| where tolower(ResourceId) in ({clustername})\\r\\n| where TimeGenerated {timeframe}\\r\\n| where AlertType == \\\"AKS_ClusterAdminBinding\\\"\\r\\n| project name = extract(@\\\"/MICROSOFT.CONTAINERSERVICE/MANAGEDCLUSTERS/(.+)\\\", 1, ResourceId)\\r\\n| summarize count() by name\",\"size\":1,\"title\":\"AKS clusters with related Microsoft Defender alerts\",\"queryType\":0,\"resourceType\":\"microsoft.operationalinsights/workspaces\",\"crossComponentResources\":[\"{workspaces}\"],\"visualization\":\"piechart\"},\"customWidth\":\"33\",\"name\":\"query - 11\",\"styleSettings\":{\"showBorder\":true}},{\"type\":3,\"content\":{\"version\":\"KqlItem/1.0\",\"query\":\"AzureDiagnostics\\r\\n| where Category == \\\"kube-audit\\\"\\r\\n| where tolower(ResourceId) in ({clustername})\\r\\n| where TimeGenerated {timeframe}\\r\\n| where log_s has \\\"events\\\"\\r\\n| project TimeGenerated, parse_json(log_s), ResourceId\\r\\n//Parsing\\r\\n| project AzureResourceId = ResourceId, \\r\\n TimeGenerated,\\r\\n SourceIPs = tostring(log_s[\\\"sourceIPs\\\"][0]),\\r\\n User = log_s[\\\"user\\\"],\\r\\n Verb = tostring(log_s[\\\"verb\\\"]),\\r\\n ObjectRef = log_s[\\\"objectRef\\\"],\\r\\n ResponseStatus = log_s[\\\"responseStatus\\\"]\\r\\n| where ObjectRef.resource == \\\"events\\\" and Verb == \\\"delete\\\" and ResponseStatus.code == 200\\r\\n| project TimeGenerated, AzureResourceId, username = tostring(User.username), ipAddr = tostring(SourceIPs), \\r\\n eventName = tostring(ObjectRef.name), eventNamespace = tostring(ObjectRef.namespace), status = tostring(ResponseStatus.code)\\r\\n| summarize operationTime = min(TimeGenerated), eventNames = make_set(eventName, 10) by\\r\\n AzureResourceId, \\r\\n eventNamespace,\\r\\n username,\\r\\n ipAddr\\r\\n// Format the list of the event names\\r\\n| extend eventNames = substring(eventNames, 1 , strlen(eventNames) - 2)\\r\\n| extend eventNames = replace('\\\"', \\\"\\\", eventNames)\\r\\n| extend eventNames = replace(\\\",\\\", \\\", \\\", eventNames)\",\"size\":1,\"title\":\"Delete events\",\"queryType\":0,\"resourceType\":\"microsoft.operationalinsights/workspaces\",\"crossComponentResources\":[\"{workspaces}\"]},\"name\":\"query - 6\",\"styleSettings\":{\"showBorder\":true}}]},\"conditionalVisibility\":{\"parameterName\":\"diagnosticClusters\",\"comparison\":\"isEqualTo\",\"value\":\"yes\"},\"name\":\"diagnosticData\"},{\"type\":1,\"content\":{\"json\":\"No Diagnostic Logs data in the selected workspaces. \\r\\nTo enable Diagnostic Logs for your AKS cluster: Go to your AKS cluster --> Diagnostic settings --> Add diagnostic setting --> Select \\\"kube-audit\\\" and send the data to your workspace.\\r\\n\\r\\nGet more details here: https://learn.microsoft.com/azure/aks/view-master-logs\",\"style\":\"info\"},\"conditionalVisibility\":{\"parameterName\":\"diagnosticClusters\",\"comparison\":\"isEqualTo\",\"value\":\"no\"},\"name\":\"text - 4\"}]},\"conditionalVisibility\":{\"parameterName\":\"mainTab\",\"comparison\":\"isEqualTo\",\"value\":\"diagnostics\"},\"name\":\"diagnostics\"}],\"fromTemplateId\":\"sentinel-AksWorkbook\",\"$schema\":\"https://github.com/Microsoft/Application-Insights-Workbooks/blob/master/schema/workbook.json\"}" - }, - "resources": [ - { - "type": "Microsoft.ManagedIdentity/userAssignedIdentities", - "apiVersion": "2018-11-30", - "name": "[variables('clusterControlPlaneIdentityName')]", - "location": "[parameters('location')]", - "comments": "The control plane identity used by the cluster. Used for networking access (VNET joining and DNS updating)" - }, - { - "type": "Microsoft.ManagedIdentity/userAssignedIdentities", - "apiVersion": "2018-11-30", - "name": "mi-appgateway-frontend", - "location": "[parameters('location')]", - "comments": "User Managed Identity that App Gateway is assigned. Used for Azure Key Vault Access." - }, - { - "type": "Microsoft.Compute/virtualMachineScaleSets", - "apiVersion": "2020-12-01", - "name": "vmss-jumpboxes", - "comments": "Hosts the VMs used as AKS jumpboxes. Using VMSS as a way to manage spanning fault and update domains.", - "dependsOn": [ - "[resourceId('Microsoft.OperationalInsights/workspaces', variables('logAnalyticsWorkspaceName'))]", - "[resourceId('Microsoft.OperationsManagement/solutions', variables('vmInsightsSolutionName'))]" - ], - "location": "[parameters('location')]", - "zones": ["1", "2", "3"], - "sku": { - "name": "Standard_DS1_v2", - "tier": "Standard", - "capacity": 2 - }, - "properties": { - "additionalCapabilities": { - "ultraSSDEnabled": false - }, - "overprovision": false, - "singlePlacementGroup": true, - "upgradePolicy": { - "mode": "Automatic" - }, - "zoneBalance": false, - "virtualMachineProfile": { - "diagnosticsProfile": { - "bootDiagnostics": { - "enabled": true - } - }, - "osProfile": { - "computerNamePrefix": "aksjmp", - "linuxConfiguration": { - "disablePasswordAuthentication": true, - "provisionVMAgent": true, - "ssh": { - "publicKeys": [ - { - "path": "[concat('/home/', variables('jumpBoxDefaultAdminUserName'), '/.ssh/authorized_keys')]", - "keyData": "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQCcFvQl2lYPcK1tMB3Tx2R9n8a7w5MJCSef14x0ePRFr9XISWfCVCNKRLM3Al/JSlIoOVKoMsdw5farEgXkPDK5F+SKLss7whg2tohnQNQwQdXit1ZjgOXkis/uft98Cv8jDWPbhwYj+VH/Aif9rx8abfjbvwVWBGeA/OnvfVvXnr1EQfdLJgMTTh+hX/FCXCqsRkQcD91MbMCxpqk8nP6jmsxJBeLrgfOxjH8RHEdSp4fF76YsRFHCi7QOwTE/6U+DpssgQ8MTWRFRat97uTfcgzKe5MOfuZHZ++5WFBgaTr1vhmSbXteGiK7dQXOk2cLxSvKkzeaiju9Jy6hoSl5oMygUVd5fNPQ94QcqTkMxZ9tQ9vPWOHwbdLRD31Ses3IBtDV+S6ehraiXf/L/e0jRUYk8IL/J543gvhOZ0hj2sQqTj9XS2hZkstZtrB2ywrJzV5ByETUU/oF9OsysyFgnaQdyduVqEPHaqXqnJvBngqqas91plyT3tSLMez3iT0s= unused-generated-by-azure" - } - ] - } - }, - "customData": "[parameters('jumpBoxCloudInitAsBase64')]", - "adminUsername": "[variables('jumpBoxDefaultAdminUserName')]" - }, - "storageProfile": { - "osDisk": { - "createOption": "FromImage", - "caching": "ReadOnly", - "diffDiskSettings": { - "option": "Local" - }, - "osType": "Linux" - }, - "imageReference": { - "id": "[parameters('jumpBoxImageResourceId')]" - } - }, - "networkProfile": { - "networkInterfaceConfigurations": [ - { - "name": "vnet-spoke-BU0001A0005-01-nic01", - "properties": { - "primary": true, - "enableIPForwarding": false, - "enableAcceleratedNetworking": false, - "networkSecurityGroup": "[null()]", - "ipConfigurations": [ - { - "name": "default", - "properties": { - "primary": true, - "privateIPAddressVersion": "IPv4", - "publicIPAddressConfiguration": "[null()]", - "subnet": { - "id": "[variables('vnetManagementOpsSubnetResourceId')]" - } - } - } - ] - } - } - ] - } - } - }, - "resources": [ - { - "type": "extensions", - "apiVersion": "2019-12-01", - "name": "OMSExtension", - "location": "[parameters('location')]", - "dependsOn": [ - "[resourceId('Microsoft.Compute/virtualMachineScaleSets', 'vmss-jumpboxes')]" - ], - "properties": { - "publisher": "Microsoft.EnterpriseCloud.Monitoring", - "type": "OmsAgentForLinux", - "typeHandlerVersion": "1.13", - "autoUpgradeMinorVersion": true, - "settings": { - "stopOnMultipleConnections": true, - "azureResourceId": "[resourceId('Microsoft.Compute/virtualMachineScaleSets', 'vmss-jumpboxes')]", - "workspaceId": "[reference(resourceId('Microsoft.OperationalInsights/workspaces', variables('logAnalyticsWorkspaceName')), '2020-10-01').customerId]" - }, - "protectedSettings": { - "workspaceKey": "[listKeys(resourceId('Microsoft.OperationalInsights/workspaces', variables('logAnalyticsWorkspaceName')), '2020-10-01').primarySharedKey]" - } - } - }, - { - "type": "extensions", - "apiVersion": "2019-12-01", - "name": "DependencyAgentLinux", - "location": "[parameters('location')]", - "dependsOn": [ - "[resourceId('Microsoft.Compute/virtualMachineScaleSets', 'vmss-jumpboxes')]", - "[resourceId('Microsoft.Compute/virtualMachineScaleSets/extensions', 'vmss-jumpboxes', 'OMSExtension')]" - ], - "properties": { - "publisher": "Microsoft.Azure.Monitoring.DependencyAgent", - "type": "DependencyAgentLinux", - "typeHandlerVersion": "9.10", - "autoUpgradeMinorVersion": true - } - } - ] - }, - { - "type": "Microsoft.ManagedIdentity/userAssignedIdentities", - "apiVersion": "2018-11-30", - "name": "podmi-ingress-controller", - "location": "[parameters('location')]", - "comments": "User Managed Identity for the cluster's ingress controller pods. Used for Azure Key Vault Access.", - "resources": [ - { - "type": "providers/roleAssignments", - "apiVersion": "2020-04-01-preview", - "name": "[concat('Microsoft.Authorization/', guid(resourceGroup().id, variables('managedIdentityOperatorRole')))]", - "comments": "Grant the AKS cluster control plane with Managed Identity Operator role permissions over the this ingress controller pod identity so that it can be assigned to the relevant nodepools.", - "dependsOn": [ - "[resourceId('Microsoft.ManagedIdentity/userAssignedIdentities', variables('clusterControlPlaneIdentityName'))]", - "[resourceId('Microsoft.ManagedIdentity/userAssignedIdentities', 'podmi-ingress-controller')]" - - ], - "properties": { - "roleDefinitionId": "[variables('managedIdentityOperatorRole')]", - "principalId": "[reference(resourceId('Microsoft.ManagedIdentity/userAssignedIdentities', variables('clusterControlPlaneIdentityName'))).principalId]", - "principalType": "ServicePrincipal" - } - } - ] - }, - { - "type": "Microsoft.KeyVault/vaults", - "apiVersion": "2019-09-01", - "name": "[variables('keyVaultName')]", - "location": "[parameters('location')]", - "dependsOn": [ - "[resourceId('Microsoft.ManagedIdentity/userAssignedIdentities', 'mi-appgateway-frontend')]", - "[resourceId('Microsoft.ManagedIdentity/userAssignedIdentities', 'podmi-ingress-controller')]" - ], - "properties": { - "accessPolicies": [ - { - "tenantId": "[reference(resourceId('Microsoft.ManagedIdentity/userAssignedIdentities', 'mi-appgateway-frontend')).tenantId]", - "objectId": "[reference(resourceId('Microsoft.ManagedIdentity/userAssignedIdentities', 'mi-appgateway-frontend')).principalId]", - "permissions": { - "secrets": [ - "get" - ], - "certificates": [ - "get" - ], - "keys": [] - } - }, - { - "tenantId": "[reference(resourceId('Microsoft.ManagedIdentity/userAssignedIdentities', 'podmi-ingress-controller')).tenantId]", - "objectId": "[reference(resourceId('Microsoft.ManagedIdentity/userAssignedIdentities', 'podmi-ingress-controller')).principalId]", - "permissions": { - "secrets": [ - "get" - ], - "certificates": [ - "get" - ], - "keys": [] - } - } - ], - "sku": { - "family": "A", - "name": "standard" - }, - "tenantId": "[subscription().tenantId]", - "networkAcls": { - "bypass": "AzureServices", - "defaultAction": "Allow", - "ipRules": [], - "virtualNetworkRules": [] - }, - "enabledForDeployment": false, - "enabledForDiskEncryption": false, - "enabledForTemplateDeployment": false, - "enableSoftDelete": true - }, - "resources": [ - { - "type": "secrets", - "apiVersion": "2019-09-01", - "name": "sslcert", - "dependsOn": [ - "[resourceId('Microsoft.KeyVault/vaults', variables('keyVaultName') )]" - ], - "properties": { - "value": "[parameters('appGatewayListenerCertificate')]", - "recoveryLevel": "Purgeable" - } - }, - { - "type": "secrets", - "apiVersion": "2019-09-01", - "name": "appgw-ingress-internal-aks-ingress-contoso-com-tls", - "dependsOn": [ - "[resourceId('Microsoft.KeyVault/vaults', variables('keyVaultName'))]" - ], - "properties": { - "value": "[parameters('aksIngressControllerCertificate')]", - "recoveryLevel": "Purgeable" - } - }, - { - "type": "providers/diagnosticSettings", - "apiVersion": "2021-05-01-preview", - "name": "Microsoft.Insights/default", - "dependsOn": [ - "[resourceId('Microsoft.KeyVault/vaults', variables('keyVaultName'))]", - "[resourceId('Microsoft.OperationalInsights/workspaces', variables('logAnalyticsWorkspaceName'))]" - ], - "properties": { - "workspaceId": "[resourceId('Microsoft.OperationalInsights/workspaces', variables('logAnalyticsWorkspaceName'))]", - "logs": [ - { - "category": "AuditEvent", - "enabled": true - } - ], - "metrics": [ - { - "category": "AllMetrics", - "enabled": true - } - ] - } - } - ] - }, - { - "type": "Microsoft.Network/privateEndpoints", - "apiVersion": "2020-11-01", - "name": "[concat('pe-', variables('keyVaultName'))]", - "location": "[parameters('location')]", - "dependsOn": [ - "[resourceId('Microsoft.KeyVault/vaults', variables('keyVaultName'))]" - ], - "properties": { - "subnet": { - "id": "[variables('vnetPrivateLinkSubnetResourceId')]" - }, - "privateLinkServiceConnections": [ - { - "name": "[concat('to-', variables('vnetName'))]", - "properties": { - "privateLinkServiceId": "[resourceId('Microsoft.KeyVault/vaults', variables('keyVaultName'))]", - "groupIds": [ - "vault" - ] - } - } - ] - }, - "resources": [ - { - "type": "privateDnsZoneGroups", - "apiVersion": "2020-11-01", - "name": "[concat('for-', variables('keyVaultName'))]", - "location": "[parameters('location')]", - "dependsOn": [ - "[resourceId('Microsoft.Network/privateEndpoints', concat('pe-', variables('keyVaultName')))]" - ], - "properties": { - "privateDnsZoneConfigs": [ - { - "name": "privatelink-akv-net", - "properties": { - "privateDnsZoneId": "[resourceId(variables('vNetResourceGroup'), 'Microsoft.Network/privateDnsZones', 'privatelink.vaultcore.azure.net')]" - } - } - ] - } - } - ] - }, - { - "type": "Microsoft.Network/privateEndpoints", - "apiVersion": "2020-11-01", - "name": "[concat('pe-', variables('defaultAcrName'))]", - "location": "[parameters('location')]", - "dependsOn": [ - "[resourceId('Microsoft.ContainerRegistry/registries/replications', variables('defaultAcrName'), parameters('geoRedundancyLocation'))]" - ], - "properties": { - "subnet": { - "id": "[variables('vnetPrivateLinkSubnetResourceId')]" - }, - "privateLinkServiceConnections": [ - { - "name": "[concat('to-', variables('vnetName'))]", - "properties": { - "privateLinkServiceId": "[resourceId('Microsoft.ContainerRegistry/registries', variables('defaultAcrName'))]", - "groupIds": [ - "registry" - ] - } - } - ] - }, - "resources": [ - { - "type": "privateDnsZoneGroups", - "apiVersion": "2020-11-01", - "name": "[concat('for-', variables('defaultAcrName'))]", - "location": "[parameters('location')]", - "dependsOn": [ - "[resourceId('Microsoft.Network/privateEndpoints', concat('pe-', variables('defaultAcrName')))]" - ], - "properties": { - "privateDnsZoneConfigs": [ - { - "name": "privatelink-azurecr-io", - "properties": { - "privateDnsZoneId": "[resourceId(variables('vNetResourceGroup'), 'Microsoft.Network/privateDnsZones', 'privatelink.azurecr.io')]" - } - } - ] - } - } - ] - }, - { - "type": "Microsoft.Network/applicationGateways", - "apiVersion": "2020-11-01", - "name": "[variables('agwName')]", - "location": "[parameters('location')]", - "dependsOn": [ - "[resourceId('Microsoft.KeyVault/vaults', variables('keyVaultName') )]" - ], - "identity": { - "type": "UserAssigned", - "userAssignedIdentities": { - "[resourceId('Microsoft.ManagedIdentity/userAssignedIdentities', 'mi-appgateway-frontend')]": {} - } - }, - "zones": [ - "1", - "2", - "3" - ], - "properties": { - "sku": { - "name": "WAF_v2", - "tier": "WAF_v2" - }, - "sslPolicy": { - "policyType": "Custom", - "cipherSuites": [ - "TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384", - "TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256" - ], - "minProtocolVersion": "TLSv1_2" - }, - "trustedRootCertificates": [ - { - "name": "root-cert-wildcard-aks-ingress-contoso", - "properties": { - "keyVaultSecretId": "[concat(reference(variables('keyVaultName')).vaultUri,'secrets/appgw-ingress-internal-aks-ingress-contoso-com-tls')]" - } - } - ], - "gatewayIPConfigurations": [ - { - "name": "apw-ip-configuration", - "properties": { - "subnet": { - "id": "[concat(parameters('targetVnetResourceId'), '/subnets/snet-applicationgateway')]" - } - } - } - ], - "frontendIPConfigurations": [ - { - "name": "apw-frontend-ip-configuration", - "properties": { - "publicIPAddress": { - "id": "[resourceId(subscription().subscriptionId, variables('vNetResourceGroup'), 'Microsoft.Network/publicIpAddresses', 'pip-BU0001A0005-00')]" - } - } - } - ], - "frontendPorts": [ - { - "name": "apw-frontend-ports", - "properties": { - "port": 443 - } - } - ], - "autoscaleConfiguration": { - "minCapacity": 0, - "maxCapacity": 10 - }, - "webApplicationFirewallConfiguration": { - "enabled": true, - "firewallMode": "Prevention", - "ruleSetType": "OWASP", - "ruleSetVersion": "3.2", - "disabledRuleGroups": [], - "requestBodyCheck": true, - "maxRequestBodySizeInKb": 128, - "fileUploadLimitInMb": 100 - }, - "enableHttp2": false, - "sslCertificates": [ - { - "name": "[concat(variables('agwName'), '-ssl-certificate')]", - "properties": { - "keyVaultSecretId": "[concat(reference(variables('keyVaultName')).vaultUri,'secrets/sslcert')]" - } - } - ], - "probes": [ - { - "name": "probe-bu0001a0005-00.aks-ingress.contoso.com", - "properties": { - "protocol": "Https", - "path": "/favicon.ico", - "interval": 30, - "timeout": 30, - "unhealthyThreshold": 3, - "pickHostNameFromBackendHttpSettings": true, - "minServers": 0, - "match": {} - } - }, - { - "name": "ingress-controller", - "properties": { - "protocol": "Https", - "path": "/healthz", - "interval": 30, - "timeout": 30, - "unhealthyThreshold": 3, - "pickHostNameFromBackendHttpSettings": true, - "minServers": 0, - "match": {} - } - } - ], - "backendAddressPools": [ - { - "name": "bu0001a0005-00.aks-ingress.contoso.com", - "properties": { - "backendAddresses": [ - { - /* This is the IP address that our ingress controller will request */ - "ipAddress": "10.240.4.4" - } - ] - } - } - ], - "backendHttpSettingsCollection": [ - { - "name": "aks-ingress-contoso-backendpool-httpsettings", - "properties": { - "port": 443, - "protocol": "Https", - "cookieBasedAffinity": "Disabled", - "hostName": "bu0001a0005-00.aks-ingress.contoso.com", - "pickHostNameFromBackendAddress": false, - "requestTimeout": 20, - "probe": { - "id": "[concat(resourceId('Microsoft.Network/applicationGateways', variables('agwName')),'/probes/probe-bu0001a0005-00.aks-ingress.contoso.com')]" - }, - "trustedRootCertificates": [ - { - "id": "[concat(resourceId('Microsoft.Network/applicationGateways', variables('agwName')), '/trustedRootCertificates/root-cert-wildcard-aks-ingress-contoso')]" - } - ] - } - } - ], - "httpListeners": [ - { - "name": "listener-https", - "properties": { - "frontendIPConfiguration": { - "id": "[concat(variables('apwResourceId'), '/frontendIPConfigurations/apw-frontend-ip-configuration')]" - }, - "frontendPort": { - "id": "[concat(variables('apwResourceId'), '/frontendPorts/apw-frontend-ports')]" - }, - "protocol": "Https", - "sslCertificate": { - "id": "[concat(variables('apwResourceId'), '/sslCertificates/', variables('agwName'), '-ssl-certificate')]" - }, - "hostName": "bicycle.contoso.com", - "hostNames": [], - "requireServerNameIndication": true - } - } - ], - "requestRoutingRules": [ - { - "name": "apw-routing-rules", - "properties": { - "ruleType": "Basic", - "httpListener": { - "id": "[concat(variables('apwResourceId'), '/httpListeners/listener-https')]" - }, - "backendAddressPool": { - "id": "[concat(variables('apwResourceId'), '/backendAddressPools/bu0001a0005-00.aks-ingress.contoso.com')]" - }, - "backendHttpSettings": { - "id": "[concat(variables('apwResourceId'), '/backendHttpSettingsCollection/aks-ingress-contoso-backendpool-httpsettings')]" - } - } - } - ] - }, - "resources": [ - { - "type": "/providers/diagnosticSettings", - "apiVersion": "2017-05-01-preview", - "name": "Microsoft.Insights/default", - "dependsOn": [ - "[resourceId('Microsoft.Network/applicationGateways', variables('agwName'))]", - "[resourceId('Microsoft.OperationalInsights/workspaces', variables('logAnalyticsWorkspaceName'))]" - ], - "properties": { - "workspaceId": "[resourceId('Microsoft.OperationalInsights/workspaces', variables('logAnalyticsWorkspaceName'))]", - "logs": [ - { - "category": "ApplicationGatewayAccessLog", - "enabled": true - }, - { - "category": "ApplicationGatewayPerformanceLog", - "enabled": true - }, - { - "category": "ApplicationGatewayFirewallLog", - "enabled": true - } - ] - } - } - ] - }, - { - "type": "Microsoft.Resources/deployments", - "apiVersion": "2020-06-01", - "name": "EnsureClusterIdentityHasRbacToSelfManagedResources", - "dependsOn": [ - "[resourceId('Microsoft.ManagedIdentity/userAssignedIdentities', variables('clusterControlPlaneIdentityName'))]" - ], - "resourceGroup": "[variables('vNetResourceGroup')]", - "properties": { - "mode": "Incremental", - "template": { - "$schema": "https://schema.management.azure.com/schemas/2019-04-01/deploymentTemplate.json#", - "contentVersion": "1.0.0.0", - "resources": [ - { - "type": "Microsoft.Authorization/roleAssignments", - "apiVersion": "2020-04-01-preview", - "name": "[guid(parameters('targetVnetResourceId'), variables('dnsZoneContributorRole'), variables('clusterControlPlaneIdentityName'))]", - "scope": "[concat('Microsoft.Network/virtualNetworks/', variables('vnetName'))]", - "properties": { - "roleDefinitionId": "[variables('dnsZoneContributorRole')]", - "description": "Allows cluster identity to attach custom DNS zone with Private Link information to this virtual network.", - "principalId": "[reference(resourceId('Microsoft.ManagedIdentity/userAssignedIdentities', variables('clusterControlPlaneIdentityName'))).principalId]", - "principalType": "ServicePrincipal" - } - }, - { - "type": "Microsoft.Authorization/roleAssignments", - "apiVersion": "2020-04-01-preview", - "name": "[guid(variables('vnetSystemNodePoolSubnetResourceId'), variables('networkContributorRole'), variables('clusterControlPlaneIdentityName'))]", - "scope": "[concat('Microsoft.Network/virtualNetworks/', variables('vnetName'), '/subnets/', 'snet-cluster-systemnodepool')]", - "properties": { - "roleDefinitionId": "[variables('networkContributorRole')]", - "description": "Allows cluster identity to join the nodepool vmss resources to this subnet.", - "principalId": "[reference(resourceId('Microsoft.ManagedIdentity/userAssignedIdentities', variables('clusterControlPlaneIdentityName'))).principalId]", - "principalType": "ServicePrincipal" - } - }, - { - "type": "Microsoft.Authorization/roleAssignments", - "apiVersion": "2020-04-01-preview", - "name": "[guid(variables('vnetInScopeNodePoolSubnetResourceId'), variables('networkContributorRole'), variables('clusterControlPlaneIdentityName'))]", - "scope": "[concat('Microsoft.Network/virtualNetworks/', variables('vnetName'), '/subnets/', 'snet-cluster-inscopenodepools')]", - "properties": { - "roleDefinitionId": "[variables('networkContributorRole')]", - "description": "Allows cluster identity to join the nodepool vmss resources to this subnet.", - "principalId": "[reference(resourceId('Microsoft.ManagedIdentity/userAssignedIdentities', variables('clusterControlPlaneIdentityName'))).principalId]", - "principalType": "ServicePrincipal" - } - }, - { - "type": "Microsoft.Authorization/roleAssignments", - "apiVersion": "2020-04-01-preview", - "name": "[guid(variables('vnetOutOfScopeNodePoolSubnetResourceId'), variables('networkContributorRole'), variables('clusterControlPlaneIdentityName'))]", - "scope": "[concat('Microsoft.Network/virtualNetworks/', variables('vnetName'), '/subnets/', 'snet-cluster-outofscopenodepools')]", - "properties": { - "roleDefinitionId": "[variables('networkContributorRole')]", - "description": "Allows cluster identity to join the nodepool vmss resources to this subnet.", - "principalId": "[reference(resourceId('Microsoft.ManagedIdentity/userAssignedIdentities', variables('clusterControlPlaneIdentityName'))).principalId]", - "principalType": "ServicePrincipal" - } - }, - { - "type": "Microsoft.Authorization/roleAssignments", - "apiVersion": "2020-04-01-preview", - "name": "[guid(variables('vnetIngressServicesSubnetResourceId'), variables('networkContributorRole'), variables('clusterControlPlaneIdentityName'))]", - "scope": "[concat('Microsoft.Network/virtualNetworks/', variables('vnetName'), '/subnets/', 'snet-cluster-ingressservices')]", - "properties": { - "roleDefinitionId": "[variables('networkContributorRole')]", - "description": "Allows cluster identity to join load balancers (ingress resources) to this subnet.", - "principalId": "[reference(resourceId('Microsoft.ManagedIdentity/userAssignedIdentities', variables('clusterControlPlaneIdentityName'))).principalId]", - "principalType": "ServicePrincipal" - } - }, - { - "type": "Microsoft.Authorization/roleAssignments", - "apiVersion": "2020-04-01-preview", - "name": "[guid(resourceId('Microsoft.Network/privateDnsZones', concat('privatelink.', parameters('location'), '.azmk8s.io')), variables('dnsZoneContributorRole'), variables('clusterControlPlaneIdentityName'))]", - "scope": "[concat('Microsoft.Network/privateDnsZones/', concat('privatelink.', parameters('location'), '.azmk8s.io'))]", - "properties": { - "roleDefinitionId": "[variables('dnsZoneContributorRole')]", - "description": "Allows cluster identity to manage zone Entries for cluster's Private Link configuration.", - "principalId": "[reference(resourceId('Microsoft.ManagedIdentity/userAssignedIdentities', variables('clusterControlPlaneIdentityName'))).principalId]", - "principalType": "ServicePrincipal" - } - } - ] - } - } - }, - { - "type": "Microsoft.OperationalInsights/workspaces", - "apiVersion": "2020-08-01", - "name": "[variables('logAnalyticsWorkspaceName')]", - "location": "[parameters('location')]", - "properties": { - "sku": { - "name": "PerGB2018" - }, - "retentionInDays": 90, - "publicNetworkAccessForIngestion": "Enabled", - "publicNetworkAccessForQuery": "Enabled" - }, - "resources": [ - { - "type": "savedSearches", - "apiVersion": "2020-08-01", - "name": "AllPrometheus", - "dependsOn": [ - "[concat('Microsoft.OperationalInsights/workspaces/', variables('logAnalyticsWorkspaceName'))]" - ], - "properties": { - "eTag": "*", - "category": "Prometheus", - "displayName": "All collected Prometheus information", - "query": "InsightsMetrics | where Namespace == \"prometheus\"", - "version": 1 - } - }, - { - "type": "savedSearches", - "apiVersion": "2020-08-01", - "name": "ForbiddenReponsesOnIngress", - "dependsOn": [ - "[concat('Microsoft.OperationalInsights/workspaces/', variables('logAnalyticsWorkspaceName'))]" - ], - "properties": { - "eTag": "*", - "category": "Prometheus", - "displayName": "Increase number of forbidden response on the Ingress Controller", - "query": "let value = toscalar(InsightsMetrics | where Namespace == \"prometheus\" and Name == \"nginx_ingress_controller_requests\" | where parse_json(Tags).status == 403 | summarize Value = avg(Val) by bin(TimeGenerated, 5m) | summarize min = min(Value)); InsightsMetrics | where Namespace == \"prometheus\" and Name == \"nginx_ingress_controller_requests\" | where parse_json(Tags).status == 403 | summarize AggregatedValue = avg(Val)-value by bin(TimeGenerated, 5m) | order by TimeGenerated | render barchart", - "version": 1 - } - }, - { - "type": "savedSearches", - "apiVersion": "2020-08-01", - "name": "NodeRebootRequested", - "dependsOn": [ - "[concat('Microsoft.OperationalInsights/workspaces/', variables('logAnalyticsWorkspaceName'))]" - ], - "properties": { - "eTag": "*", - "category": "Prometheus", - "displayName": "Nodes reboot required by kured", - "query": "InsightsMetrics | where Namespace == \"prometheus\" and Name == \"kured_reboot_required\" | where Val > 0", - "version": 1 - } - } - ] - }, - { - "type": "Microsoft.Insights/scheduledQueryRules", - "apiVersion": "2021-08-01", - "name": "Image Imported into ACR from source other than approved Quarantine", - "location": "[parameters('location')]", - "dependsOn": [ - "[resourceId('Microsoft.OperationalInsights/workspaces', variables('logAnalyticsWorkspaceName'))]", - "[resourceId('Microsoft.ContainerRegistry/registries', variables('defaultAcrName'))]" - ], - "properties": { - "description":"The only images we want in live/ are those that came from this ACR instance, but from the quarantine/ repository.", - "actions": { - "actionGroups": [] - }, - "criteria": { - "allOf": [ - { - "operator": "GreaterThan", - "query": "ContainerRegistryRepositoryEvents\r\n| where OperationName == \"importImage\" and Repository startswith \"live/\" and MediaType !startswith strcat(_ResourceId, \"/quarantine\")", - "threshold": 0, - "timeAggregation": "Count", - "dimensions": [], - "failingPeriods": { - "minFailingPeriodsToAlert": 1, - "numberOfEvaluationPeriods": 1 - }, - "resourceIdColumn": "" - } - ] - }, - "enabled": true, - "evaluationFrequency": "PT10M", - "scopes": [ - "[resourceId('Microsoft.ContainerRegistry/registries', variables('defaultAcrName'))]" - ], - "severity": 3, - "windowSize": "PT10M", - "muteActionsDuration": null, - "overrideQueryTimeRange": null - } - }, - { - "type": "Microsoft.Insights/scheduledQueryRules", - "apiVersion": "2021-08-01", - "name": "PodFailedScheduledQuery", - "location": "[parameters('location')]", - "dependsOn": [ - "[resourceId('Microsoft.OperationalInsights/workspaces', variables('logAnalyticsWorkspaceName'))]", - "[resourceId('Microsoft.ContainerService/managedClusters', variables('clusterName'))]" - ], - "properties": { - "description":"Example from: https://learn.microsoft.com/azure/azure-monitor/insights/container-insights-alerts", - "actions": { - "actionGroups": [] - }, - "criteria": { - "allOf": [ - { - "metricMeasureColumn": "FailedCount", - "operator": "GreaterThan", - "query": "let trendBinSize = 1m;\r\nKubePodInventory\r\n| distinct ClusterName, TimeGenerated\r\n| summarize ClusterSnapshotCount = count() by bin(TimeGenerated, trendBinSize), ClusterName\r\n| join hint.strategy=broadcast (\r\n KubePodInventory\r\n | distinct ClusterName, Computer, PodUid, TimeGenerated, PodStatus\r\n | summarize TotalCount = count(),\r\n PendingCount = sumif(1, PodStatus =~ \"Pending\"),\r\n RunningCount = sumif(1, PodStatus =~ \"Running\"),\r\n SucceededCount = sumif(1, PodStatus =~ \"Succeeded\"),\r\n FailedCount = sumif(1, PodStatus =~ \"Failed\")\r\n by ClusterName, bin(TimeGenerated, trendBinSize)\r\n )\r\n on ClusterName, TimeGenerated \r\n| extend UnknownCount = TotalCount - PendingCount - RunningCount - SucceededCount - FailedCount\r\n| project TimeGenerated,\r\n ClusterName,\r\n TotalCount = todouble(TotalCount) / ClusterSnapshotCount,\r\n PendingCount = todouble(PendingCount) / ClusterSnapshotCount,\r\n RunningCount = todouble(RunningCount) / ClusterSnapshotCount,\r\n SucceededCount = todouble(SucceededCount) / ClusterSnapshotCount,\r\n FailedCount = todouble(FailedCount) / ClusterSnapshotCount,\r\n UnknownCount = todouble(UnknownCount) / ClusterSnapshotCount\r\n", - "threshold": 3, - "timeAggregation": "Average", - "dimensions": [], - "failingPeriods": { - "minFailingPeriodsToAlert": 1, - "numberOfEvaluationPeriods": 1 - }, - "resourceIdColumn": "" - } - ] - }, - "enabled": true, - "evaluationFrequency": "PT5M", - "scopes": [ - "[resourceId('Microsoft.ContainerService/managedClusters', variables('clusterName'))]" - ], - "severity": 3, - "windowSize": "PT5M", - "muteActionsDuration": null, - "overrideQueryTimeRange": "P2D" - } - }, - { - "type": "Microsoft.Insights/activityLogAlerts", - "apiVersion": "2017-04-01", - "name": "AllAzureAdvisorAlert", - "location": "Global", - "properties": { - "scopes": [ - "[resourceGroup().id]" - ], - "condition": { - "allOf": [ - { - "field": "category", - "equals": "Recommendation" - }, - { - "field": "operationName", - "equals": "Microsoft.Advisor/recommendations/available/action" - } - ] - }, - "actions": { - "actionGroups": [ - ] - }, - "enabled": true, - "description": "All azure advisor alerts" - } - }, - { - "type": "Microsoft.OperationsManagement/solutions", - "apiVersion": "2015-11-01-preview", - "name": "[variables('containerInsightsSolutionName')]", - "location": "[parameters('location')]", - "dependsOn": [ - "[resourceId('Microsoft.OperationalInsights/workspaces', variables('logAnalyticsWorkspaceName'))]" - ], - "properties": { - "workspaceResourceId": "[resourceId('Microsoft.OperationalInsights/workspaces', variables('logAnalyticsWorkspaceName'))]" - }, - "plan": { - "name": "[variables('containerInsightsSolutionName')]", - "product": "OMSGallery/ContainerInsights", - "promotionCode": "", - "publisher": "Microsoft" - } - }, - { - "type": "Microsoft.OperationsManagement/solutions", - "apiVersion": "2015-11-01-preview", - "name": "[variables('vmInsightsSolutionName')]", - "location": "[parameters('location')]", - "dependsOn": [ - "[resourceId('Microsoft.OperationalInsights/workspaces', variables('logAnalyticsWorkspaceName'))]" - ], - "properties": { - "workspaceResourceId": "[resourceId('Microsoft.OperationalInsights/workspaces', variables('logAnalyticsWorkspaceName'))]" - }, - "plan": { - "name": "[variables('vmInsightsSolutionName')]", - "product": "OMSGallery/VMInsights", - "promotionCode": "", - "publisher": "Microsoft" - } - }, - { - "type": "Microsoft.OperationsManagement/solutions", - "apiVersion": "2015-11-01-preview", - "name": "[variables('securityCenterSolutionName')]", - "location": "[parameters('location')]", - "comments": "Enables Azure Sentinal on this workspace.", - "dependsOn": [ - "[resourceId('Microsoft.OperationalInsights/workspaces', variables('logAnalyticsWorkspaceName'))]" - ], - "properties": { - "workspaceResourceId": "[resourceId('Microsoft.OperationalInsights/workspaces', variables('logAnalyticsWorkspaceName'))]" - }, - "plan": { - "name": "[variables('securityCenterSolutionName')]", - "product": "OMSGallery/SecurityInsights", - "promotionCode": "", - "publisher": "Microsoft" - } - }, - { - "type": "Microsoft.Insights/workbooks", - "apiVersion": "2020-10-20", - "name": "[guid(variables('securityCenterSolutionName'))]", - "location": "[parameters('location')]", - "comments": "Add the AKS Microsoft Defender for Cloud AKS workbook - https://securityinsights.hosting.portal.azure.net/securityinsights/Content/1.0.01480.0001-210119-121529/Workbooks/AksSecurity.json", - "tags": { - "hidden-title": "[concat('Azure Kubernetes Service (AKS) Security - ', variables('logAnalyticsWorkspaceName'))]" - }, - "dependsOn": [ - "[resourceId('Microsoft.OperationsManagement/solutions', variables('securityCenterSolutionName'))]" - ], - "kind": "shared", - "properties": { - "displayName": "[concat('Azure Kubernetes Service (AKS) Security - ', variables('logAnalyticsWorkspaceName'))]", - "serializedData": "[variables('aksSecurityCenterWorkbookData')]", - "version": "1.0", - "category": "sentinel", - "sourceId": "[resourceId('Microsoft.OperationalInsights/workspaces', variables('logAnalyticsWorkspaceName'))]", - "tags": [ - "AksSecurityWorkbook", - "1.2" - ] - } - }, - { - "type": "Microsoft.OperationsManagement/solutions", - "apiVersion": "2015-11-01-preview", - "name": "[concat('KeyVaultAnalytics(', variables('logAnalyticsWorkspaceName'),')')]", - "location": "[parameters('location')]", - "dependsOn": [ - "[resourceId('Microsoft.OperationalInsights/workspaces', variables('logAnalyticsWorkspaceName'))]" - ], - "properties": { - "workspaceResourceId": "[resourceId('Microsoft.OperationalInsights/workspaces', variables('logAnalyticsWorkspaceName'))]" - }, - "plan": { - "name": "[concat('KeyVaultAnalytics(', variables('logAnalyticsWorkspaceName'),')')]", - "product": "OMSGallery/KeyVaultAnalytics", - "promotionCode": "", - "publisher": "Microsoft" - } - }, - { - "type": "Microsoft.ContainerRegistry/registries", - "apiVersion": "2020-11-01-preview", - "name": "[variables('defaultAcrName')]", - "location": "[parameters('location')]", - "sku": { - "name": "Premium" - }, - "properties": { - "adminUserEnabled": false, - "networkRuleSet": { - "defaultAction": "Deny", - "virtualNetworkRules": [], - "ipRules": [] - }, - "policies": { - "quarantinePolicy": { - "status": "disabled" - }, - "trustPolicy": { - "type": "Notary", - "status": "enabled" - }, - "retentionPolicy": { - "days": 15, - "status": "enabled" - } - }, - "publicNetworkAccess": "Disabled", - "encryption": { - "status": "disabled" - }, - "dataEndpointEnabled": true, - "networkRuleBypassOptions": "AzureServices", - "zoneRedundancy": "Disabled" // This Preview feature only supports three regions at this time, and eastus2's paired region (centralus), does not support this. So disabling for now. - }, - "resources": [ - { - "type": "providers/roleAssignments", - "apiVersion": "2020-04-01-preview", - "name": "[concat('Microsoft.Authorization/', guid(resourceId('Microsoft.ContainerService/managedClusters', variables('clusterName')), variables('acrPullRole')))]", - "dependsOn": [ - "[resourceId('Microsoft.ContainerRegistry/registries', variables('defaultAcrName'))]", - "[resourceId('Microsoft.ContainerService/managedClusters', variables('clusterName'))]" - ], - "properties": { - "roleDefinitionId": "[variables('acrPullRole')]", - "description": "Allows the AKS Cluster's kubelet managed identity to pull images from this container registry.", - "principalId": "[reference(resourceId('Microsoft.ContainerService/managedClusters', variables('clusterName')), '2020-12-01').identityProfile.kubeletidentity.objectId]", - "principalType": "ServicePrincipal" - } - }, - { - "type": "replications", - "apiVersion": "2019-05-01", - "name": "[parameters('geoRedundancyLocation')]", - "location": "[parameters('geoRedundancyLocation')]", - "dependsOn": [ - "[resourceId('Microsoft.ContainerRegistry/registries', variables('defaultAcrName'))]" - ], - "properties": {} - }, - { - "type": "agentPools", - "apiVersion": "2019-06-01-preview", - "name": "acragent", - "location": "[parameters('location')]", - "comments": "ACR Build Agent pool", - "dependsOn": [ - "[resourceId('Microsoft.ContainerRegistry/registries', variables('defaultAcrName'))]" - ], - "properties": { - "count": 1, - "os": "Linux", - "tier": "S1", - "virtualNetworkSubnetResourceId": "[variables('vnetAcrBuildSubnetResourceId')]" - } - }, - { - "type": "providers/diagnosticSettings", - "apiVersion": "2021-05-01-preview", - "name": "Microsoft.Insights/default", - "dependsOn": [ - "[resourceId('Microsoft.ContainerRegistry/registries', variables('defaultAcrName'))]", - "[resourceId('Microsoft.OperationalInsights/workspaces', variables('logAnalyticsWorkspaceName'))]" - ], - "properties": { - "workspaceId": "[resourceId('Microsoft.OperationalInsights/workspaces', variables('logAnalyticsWorkspaceName'))]", - "metrics": [ - { - "timeGrain": "PT1M", - "category": "AllMetrics", - "enabled": true - } - ], - "logs": [ - { - "category": "ContainerRegistryRepositoryEvents", - "enabled": true - }, - { - "category": "ContainerRegistryLoginEvents", - "enabled": true - } - ] - } - } - ] - }, - { - "type": "Microsoft.ContainerService/managedClusters", - "apiVersion": "2021-05-01", - "name": "[variables('clusterName')]", - "location": "[parameters('location')]", - "tags": { - "Data classification": "Confidential", - "Business unit": "BU0001", - "Business criticality": "Business unit-critical" - }, - "dependsOn": [ - "[resourceId('Microsoft.OperationsManagement/solutions', variables('containerInsightsSolutionName'))]", - "[resourceId(variables('vNetResourceGroup'), 'Microsoft.Resources/deployments', 'EnsureClusterIdentityHasRbacToSelfManagedResources')]", - "[resourceId('Microsoft.ManagedIdentity/userAssignedIdentities', variables('clusterControlPlaneIdentityName'))]", - // You want policies created before cluster because they take some time to be made available and we want them - // to apply to your cluster as soon as possible. Nothing in this cluster "technically" depends on these existing, - // just trying to get coverage as soon as possible. - "[resourceId('Microsoft.Authorization/policyAssignments', variables('policyAssignmentNameAKSLinuxRestrictive'))]", - "[resourceId('Microsoft.Authorization/policyAssignments', variables('policyAssignmentNameEnforceHttpsIngress'))]", - "[resourceId('Microsoft.Authorization/policyAssignments', variables('policyAssignmentNameEnforceInternalLoadBalancers'))]", - "[resourceId('Microsoft.Authorization/policyAssignments', variables('policyAssignmentNameMustNotAutomountApiCreds'))]", - "[resourceId('Microsoft.Authorization/policyAssignments', variables('policyAssignmentNameMustUseSpecifiedLabels'))]", - "[resourceId('Microsoft.Authorization/policyAssignments', variables('policyAssignmentNameMustUseTheseExternalIps'))]", - "[resourceId('Microsoft.Authorization/policyAssignments', variables('policyAssignmentNameApprovedContainerPortsOnly'))]", - "[resourceId('Microsoft.Authorization/policyAssignments', variables('policyAssignmentNameApprovedServicePortsOnly'))]", - "[resourceId('Microsoft.Authorization/policyAssignments', variables('policyAssignmentNameRoRootFilesystem'))]", - "[resourceId('Microsoft.Authorization/policyAssignments', variables('policyAssignmentNameBlockDefaultNamespace'))]", - "[resourceId('Microsoft.Authorization/policyAssignments', variables('policyAssignmentNameEnforceResourceLimits'))]", - "[resourceId('Microsoft.Authorization/policyAssignments', variables('policyAssignmentNameEnforceImageSource'))]", - // Ensure jumboxes are available to use as soon as possible, don't wait until cluster is created. - "[resourceId('Microsoft.Compute/virtualMachineScaleSets', 'vmss-jumpboxes')]", - "[resourceId('Microsoft.ManagedIdentity/userAssignedIdentities', 'podmi-ingress-controller')]" - ], - "properties": { - "kubernetesVersion": "[variables('kubernetesVersion')]", - "dnsPrefix": "[uniqueString(subscription().subscriptionId, resourceGroup().id, variables('clusterName'))]", - "agentPoolProfiles": [ - { - "name": "npsystem", - "count": 3, - "vmSize": "Standard_DS2_v2", - "osDiskSizeGB": 80, - "osDiskType": "Ephemeral", - "osType": "Linux", - "minCount": 3, - "maxCount": 4, - "vnetSubnetID": "[variables('vnetSystemNodePoolSubnetResourceId')]", - "enableAutoScaling": true, - "type": "VirtualMachineScaleSets", - "mode": "System", - "scaleSetPriority": "Regular", - "scaleSetEvictionPolicy": "Delete", - "orchestratorVersion": "[variables('kubernetesVersion')]", - "enableNodePublicIP": false, - "maxPods": 30, - "availabilityZones": [ - "1", - "2", - "3" - ], - "upgradeSettings": { - "maxSurge": "33%" - }, - "tags": { - "pci-scope": "out-of-scope", - "Data classification": "Confidential", - "Business unit": "BU0001", - "Business criticality": "Business unit-critical" - }/* This can be used to prevent unexpected workloads from landing on system node pool. All add-ons support this taint., - "nodeTaints": [ - "CriticalAddonsOnly=true:NoSchedule" - ]*/ - }, - { - "name": "npinscope01", - "count": 2, - "vmSize": "Standard_DS3_v2", - "osDiskSizeGB": 120, - "osDiskType": "Ephemeral", - "osType": "Linux", - "minCount": 2, - "maxCount": 5, - "vnetSubnetID": "[variables('vnetInScopeNodePoolSubnetResourceId')]", - "enableAutoScaling": true, - "type": "VirtualMachineScaleSets", - "mode": "User", - "scaleSetPriority": "Regular", - "scaleSetEvictionPolicy": "Delete", - "orchestratorVersion": "[variables('kubernetesVersion')]", - "enableNodePublicIP": false, - "maxPods": 30, - "availabilityZones": [ - "1", - "2", - "3" - ], - "upgradeSettings": { - "maxSurge": "33%" - }, - "nodeLabels": { - "pci-scope": "in-scope" - }, - "tags": { - "pci-scope": "in-scope", - "Data classification": "Confidential", - "Business unit": "BU0001", - "Business criticality": "Business unit-critical" - } - }, - { - "name": "npooscope01", - "count": 2, - "vmSize": "Standard_DS3_v2", - "osDiskSizeGB": 120, - "osDiskType": "Ephemeral", - "osType": "Linux", - "minCount": 2, - "maxCount": 5, - "vnetSubnetID": "[variables('vnetOutOfScopeNodePoolSubnetResourceId')]", - "enableAutoScaling": true, - "type": "VirtualMachineScaleSets", - "mode": "User", - "scaleSetPriority": "Regular", - "scaleSetEvictionPolicy": "Delete", - "orchestratorVersion": "[variables('kubernetesVersion')]", - "enableNodePublicIP": false, - "maxPods": 30, - "availabilityZones": [ - "1", - "2", - "3" - ], - "upgradeSettings": { - "maxSurge": "33%" - }, - "nodeLabels": { - "pci-scope": "out-of-scope" - }, - "tags": { - "pci-scope": "out-of-scope", - "Data classification": "Confidential", - "Business unit": "BU0001", - "Business criticality": "Business unit-critical" - } - } - ], - "servicePrincipalProfile": { - "clientId": "msi" - }, - "addonProfiles": { - "httpApplicationRouting": { - "enabled": false - }, - "omsagent": { - "enabled": true, - "config": { - "logAnalyticsWorkspaceResourceId": "[resourceId('Microsoft.OperationalInsights/workspaces', variables('logAnalyticsWorkspaceName'))]" - } - }, - "aciConnectorLinux": { - "enabled": false - }, - "azurepolicy": { - "enabled": true, - "config": { - "version": "v2" - } - }, - "openServiceMesh": { - "enabled": true, - "config": {} - }, - "azureKeyvaultSecretsProvider": { - "enabled": true, - "config": { - "enableSecretRotation": "false" - } - } - }, - "nodeResourceGroup": "[variables('nodeResourceGroupName')]", - "enableRBAC": true, - "enablePodSecurityPolicy": false, - "maxAgentPools": 3, - "networkProfile": { - "networkPlugin": "azure", - "networkPolicy": "azure", - "outboundType": "userDefinedRouting", - "loadBalancerSku": "standard", - "loadBalancerProfile": "[json('null')]", - "serviceCidr": "172.16.0.0/16", - "dnsServiceIP": "172.16.0.10", - "dockerBridgeCidr": "172.18.0.1/16" - }, - "aadProfile": { - "managed": true, - "enableAzureRBAC": false, - "adminGroupObjectIDs": [ - "[parameters('clusterAdminAadGroupObjectId')]" - ], - "tenantID": "[parameters('k8sControlPlaneAuthorizationTenantId')]" - }, - "autoScalerProfile": { - "balance-similar-node-groups": "false", - "expander": "random", - "max-empty-bulk-delete": "10", - "max-graceful-termination-sec": "600", - "max-node-provision-time": "15m", - "max-total-unready-percentage": "45", - "new-pod-scale-up-delay": "0s", - "ok-total-unready-count": "3", - "scale-down-delay-after-add": "10m", - "scale-down-delay-after-delete": "20s", - "scale-down-delay-after-failure": "3m", - "scale-down-unneeded-time": "10m", - "scale-down-unready-time": "20m", - "scale-down-utilization-threshold": "0.5", - "scan-interval": "10s", - "skip-nodes-with-local-storage": "true", - "skip-nodes-with-system-pods": "true" - }, - /*"autoUpgradeProfile": { - "upgradeChannel": "none" - },*/ - "apiServerAccessProfile": { - "enablePrivateCluster": true, - "privateDNSZone": "[resourceId(variables('vNetResourceGroup'), 'Microsoft.Network/privateDnsZones', concat('privatelink.', parameters('location') ,'.azmk8s.io'))]", - "enablePrivateClusterPublicFQDN": false - }, - "podIdentityProfile": { - "enabled": true, - "userAssignedIdentities": [ - { - "name": "podmi-ingress-controller", - "namespace": "ingress-nginx", - "identity": { - "resourceId": "[resourceId('Microsoft.ManagedIdentity/userAssignedIdentities', 'podmi-ingress-controller')]", - "clientId": "[reference(resourceId('Microsoft.ManagedIdentity/userAssignedIdentities', 'podmi-ingress-controller')).clientId]", - "objectId": "[reference(resourceId('Microsoft.ManagedIdentity/userAssignedIdentities', 'podmi-ingress-controller')).principalId]" - } - } - ] - }, - "disableLocalAccounts": true, - "securityProfile": { - "azureDefender": { - "enabled": true, - "logAnalyticsWorkspaceResourceId": "[resourceId('Microsoft.OperationalInsights/workspaces', variables('logAnalyticsWorkspaceName'))]" - } - } - }, - "identity": { - "type": "UserAssigned", - "userAssignedIdentities": { - "[resourceId('Microsoft.ManagedIdentity/userAssignedIdentities', variables('clusterControlPlaneIdentityName'))]": {} - } - }, - "sku": { - "name": "Basic", - "tier": "Paid" - }, - "resources": [ - { - "type": "providers/roleAssignments", - "apiVersion": "2020-04-01-preview", - "name": "[concat('Microsoft.Authorization/', guid(resourceId('Microsoft.ContainerService/managedClusters', variables('clusterName')), 'omsagent', variables('monitoringMetricsPublisherRole')))]", - "dependsOn": [ - "[resourceId('Microsoft.ContainerService/managedClusters', variables('clusterName'))]" - ], - "properties": { - "roleDefinitionId": "[variables('monitoringMetricsPublisherRole')]", - "description": "Grant the OMS Agent's Managed Identity the metrics publisher role to push alerts.", - "principalId": "[reference(resourceId('Microsoft.ContainerService/managedClusters', variables('clusterName')), '2020-12-01').addonProfiles.omsagent.identity.objectId]", - "principalType": "ServicePrincipal" - } - }, - { - "type": "/providers/diagnosticSettings", - "apiVersion": "2017-05-01-preview", - "name": "Microsoft.Insights/default", - "dependsOn": [ - "[resourceId('Microsoft.ContainerService/managedClusters', variables('clusterName'))]", - "[resourceId('Microsoft.OperationalInsights/workspaces', variables('logAnalyticsWorkspaceName'))]" - ], - "properties": { - "workspaceId": "[resourceId('Microsoft.OperationalInsights/workspaces', variables('logAnalyticsWorkspaceName'))]", - "logs": [ - { - "category": "cluster-autoscaler", - "enabled": true - }, - { - "category": "kube-controller-manager", - "enabled": true - }, - { - "category": "kube-audit-admin", - "enabled": true - }, - { - "category": "guard", - "enabled": true - } - ] - } - } - ] - }, - { - "type": "Microsoft.Insights/metricAlerts", - "apiVersion": "2018-03-01", - "name": "[concat('Node CPU utilization high for ', variables('clusterName'), ' CI-1')]", - "location": "global", - "dependsOn": [ - "[resourceId('Microsoft.ContainerService/managedClusters', variables('clusterName'))]", - "[resourceId('Microsoft.OperationsManagement/solutions',variables('containerInsightsSolutionName'))]" - ], - "properties": { - "actions": [], - "criteria": { - "allOf": [ - { - "criterionType": "StaticThresholdCriterion", - "dimensions": [ - { - "name": "host", - "operator": "Include", - "values": [ - "*" - ] - } - ], - "metricName": "cpuUsagePercentage", - "metricNamespace": "Insights.Container/nodes", - "name": "Metric1", - "operator": "GreaterThan", - "threshold": 80.0, - "timeAggregation": "Average", - "skipMetricValidation": true - } - ], - "odata.type": "Microsoft.Azure.Monitor.SingleResourceMultipleMetricCriteria" - }, - "description": "Node CPU utilization across the cluster.", - "enabled": true, - "evaluationFrequency": "PT1M", - "scopes": [ - "[resourceId('Microsoft.ContainerService/managedClusters', variables('clusterName'))]" - ], - "severity": 3, - "targetResourceType": "microsoft.containerservice/managedclusters", - "windowSize": "PT5M" - } - }, - { - "type": "Microsoft.Insights/metricAlerts", - "apiVersion": "2018-03-01", - "name": "[concat('Node working set memory utilization high for ', variables('clusterName'), ' CI-2')]", - "location": "global", - "dependsOn": [ - "[resourceId('Microsoft.ContainerService/managedClusters', variables('clusterName'))]", - "[resourceId('Microsoft.OperationsManagement/solutions',variables('containerInsightsSolutionName'))]" - ], - "properties": { - "actions": [], - "criteria": { - "allOf": [ - { - "criterionType": "StaticThresholdCriterion", - "dimensions": [ - { - "name": "host", - "operator": "Include", - "values": [ - "*" - ] - } - ], - "metricName": "memoryWorkingSetPercentage", - "metricNamespace": "Insights.Container/nodes", - "name": "Metric1", - "operator": "GreaterThan", - "threshold": 80.0, - "timeAggregation": "Average", - "skipMetricValidation": true - } - ], - "odata.type": "Microsoft.Azure.Monitor.SingleResourceMultipleMetricCriteria" - }, - "description": "Node working set memory utilization across the cluster.", - "enabled": true, - "evaluationFrequency": "PT1M", - "scopes": [ - "[resourceId('Microsoft.ContainerService/managedClusters', variables('clusterName'))]" - ], - "severity": 3, - "targetResourceType": "microsoft.containerservice/managedclusters", - "windowSize": "PT5M" - } - }, - { - "type": "Microsoft.Insights/metricAlerts", - "apiVersion": "2018-03-01", - "name": "[concat('Jobs completed more than 6 hours ago for ', variables('clusterName'), ' CI-11')]", - "location": "global", - "dependsOn": [ - "[resourceId('Microsoft.ContainerService/managedClusters', variables('clusterName'))]", - "[resourceId('Microsoft.OperationsManagement/solutions',variables('containerInsightsSolutionName'))]" - ], - "properties": { - "actions": [], - "criteria": { - "allOf": [ - { - "criterionType": "StaticThresholdCriterion", - "dimensions": [ - { - "name": "controllerName", - "operator": "Include", - "values": [ - "*" - ] - }, - { - "name": "kubernetes namespace", - "operator": "Include", - "values": [ - "*" - ] - } - ], - "metricName": "completedJobsCount", - "metricNamespace": "Insights.Container/pods", - "name": "Metric1", - "operator": "GreaterThan", - "threshold": 0.0, - "timeAggregation": "Average", - "skipMetricValidation": true - } - ], - "odata.type": "Microsoft.Azure.Monitor.SingleResourceMultipleMetricCriteria" - }, - "description": "This alert monitors completed jobs (more than 6 hours ago).", - "enabled": true, - "evaluationFrequency": "PT1M", - "scopes": [ - "[resourceId('Microsoft.ContainerService/managedClusters', variables('clusterName'))]" - ], - "severity": 3, - "targetResourceType": "microsoft.containerservice/managedclusters", - "windowSize": "PT1M" - } - }, - { - "type": "Microsoft.Insights/metricAlerts", - "apiVersion": "2018-03-01", - "name": "[concat('Container CPU usage high for ', variables('clusterName'), ' CI-9')]", - "location": "global", - "dependsOn": [ - "[resourceId('Microsoft.ContainerService/managedClusters', variables('clusterName'))]", - "[resourceId('Microsoft.OperationsManagement/solutions',variables('containerInsightsSolutionName'))]" - ], - "properties": { - "actions": [], - "criteria": { - "allOf": [ - { - "criterionType": "StaticThresholdCriterion", - "dimensions": [ - { - "name": "controllerName", - "operator": "Include", - "values": [ - "*" - ] - }, - { - "name": "kubernetes namespace", - "operator": "Include", - "values": [ - "*" - ] - } - ], - "metricName": "cpuExceededPercentage", - "metricNamespace": "Insights.Container/containers", - "name": "Metric1", - "operator": "GreaterThan", - "threshold": 90.0, - "timeAggregation": "Average", - "skipMetricValidation": true - } - ], - "odata.type": "Microsoft.Azure.Monitor.SingleResourceMultipleMetricCriteria" - }, - "description": "This alert monitors container CPU utilization.", - "enabled": true, - "evaluationFrequency": "PT1M", - "scopes": [ - "[resourceId('Microsoft.ContainerService/managedClusters', variables('clusterName'))]" - ], - "severity": 3, - "targetResourceType": "microsoft.containerservice/managedclusters", - "windowSize": "PT5M" - } - }, - { - "type": "Microsoft.Insights/metricAlerts", - "apiVersion": "2018-03-01", - "name": "[concat('Container working set memory usage high for ', variables('clusterName'), ' CI-10')]", - "location": "global", - "dependsOn": [ - "[resourceId('Microsoft.ContainerService/managedClusters', variables('clusterName'))]", - "[resourceId('Microsoft.OperationsManagement/solutions',variables('containerInsightsSolutionName'))]" - ], - "properties": { - "actions": [], - "criteria": { - "allOf": [ - { - "criterionType": "StaticThresholdCriterion", - "dimensions": [ - { - "name": "controllerName", - "operator": "Include", - "values": [ - "*" - ] - }, - { - "name": "kubernetes namespace", - "operator": "Include", - "values": [ - "*" - ] - } - ], - "metricName": "memoryWorkingSetExceededPercentage", - "metricNamespace": "Insights.Container/containers", - "name": "Metric1", - "operator": "GreaterThan", - "threshold": 90.0, - "timeAggregation": "Average", - "skipMetricValidation": true - } - ], - "odata.type": "Microsoft.Azure.Monitor.SingleResourceMultipleMetricCriteria" - }, - "description": "This alert monitors container working set memory utilization.", - "enabled": true, - "evaluationFrequency": "PT1M", - "scopes": [ - "[resourceId('Microsoft.ContainerService/managedClusters', variables('clusterName'))]" - ], - "severity": 3, - "targetResourceType": "microsoft.containerservice/managedclusters", - "windowSize": "PT5M" - } - }, - { - "type": "Microsoft.Insights/metricAlerts", - "apiVersion": "2018-03-01", - "name": "[concat('Pods in failed state for ', variables('clusterName'), ' CI-4')]", - "location": "global", - "dependsOn": [ - "[resourceId('Microsoft.ContainerService/managedClusters', variables('clusterName'))]", - "[resourceId('Microsoft.OperationsManagement/solutions',variables('containerInsightsSolutionName'))]" - ], - "properties": { - "actions": [], - "criteria": { - "allOf": [ - { - "criterionType": "StaticThresholdCriterion", - "dimensions": [ - { - "name": "phase", - "operator": "Include", - "values": [ - "Failed" - ] - } - ], - "metricName": "podCount", - "metricNamespace": "Insights.Container/pods", - "name": "Metric1", - "operator": "GreaterThan", - "threshold": 0.0, - "timeAggregation": "Average", - "skipMetricValidation": true - } - ], - "odata.type": "Microsoft.Azure.Monitor.SingleResourceMultipleMetricCriteria" - }, - "description": "Pod status monitoring.", - "enabled": true, - "evaluationFrequency": "PT1M", - "scopes": [ - "[resourceId('Microsoft.ContainerService/managedClusters', variables('clusterName'))]" - ], - "severity": 3, - "targetResourceType": "microsoft.containerservice/managedclusters", - "windowSize": "PT5M" - } - }, - { - "type": "Microsoft.Insights/metricAlerts", - "apiVersion": "2018-03-01", - "name": "[concat('Disk usage high for ', variables('clusterName'), ' CI-5')]", - "location": "global", - "dependsOn": [ - "[resourceId('Microsoft.ContainerService/managedClusters', variables('clusterName'))]", - "[resourceId('Microsoft.OperationsManagement/solutions',variables('containerInsightsSolutionName'))]" - ], - "properties": { - "actions": [], - "criteria": { - "allOf": [ - { - "criterionType": "StaticThresholdCriterion", - "dimensions": [ - { - "name": "host", - "operator": "Include", - "values": [ - "*" - ] - }, - { - "name": "device", - "operator": "Include", - "values": [ - "*" - ] - } - ], - "metricName": "DiskUsedPercentage", - "metricNamespace": "Insights.Container/nodes", - "name": "Metric1", - "operator": "GreaterThan", - "threshold": 80.0, - "timeAggregation": "Average", - "skipMetricValidation": true - } - ], - "odata.type": "Microsoft.Azure.Monitor.SingleResourceMultipleMetricCriteria" - }, - "description": "This alert monitors disk usage for all nodes and storage devices.", - "enabled": true, - "evaluationFrequency": "PT1M", - "scopes": [ - "[resourceId('Microsoft.ContainerService/managedClusters', variables('clusterName'))]" - ], - "severity": 3, - "targetResourceType": "microsoft.containerservice/managedclusters", - "windowSize": "PT5M" - } - }, - { - "type": "Microsoft.Insights/metricAlerts", - "apiVersion": "2018-03-01", - "name": "[concat('Nodes in not ready status for ', variables('clusterName'), ' CI-3')]", - "location": "global", - "dependsOn": [ - "[resourceId('Microsoft.ContainerService/managedClusters', variables('clusterName'))]", - "[resourceId('Microsoft.OperationsManagement/solutions',variables('containerInsightsSolutionName'))]" - ], - "properties": { - "actions": [], - "criteria": { - "allOf": [ - { - "criterionType": "StaticThresholdCriterion", - "dimensions": [ - { - "name": "status", - "operator": "Include", - "values": [ - "NotReady" - ] - } - ], - "metricName": "nodesCount", - "metricNamespace": "Insights.Container/nodes", - "name": "Metric1", - "operator": "GreaterThan", - "threshold": 0.0, - "timeAggregation": "Average", - "skipMetricValidation": true - } - ], - "odata.type": "Microsoft.Azure.Monitor.SingleResourceMultipleMetricCriteria" - }, - "description": "Node status monitoring.", - "enabled": true, - "evaluationFrequency": "PT1M", - "scopes": [ - "[resourceId('Microsoft.ContainerService/managedClusters', variables('clusterName'))]" - ], - "severity": 3, - "targetResourceType": "microsoft.containerservice/managedclusters", - "windowSize": "PT5M" - } - }, - { - "type": "Microsoft.Insights/metricAlerts", - "apiVersion": "2018-03-01", - "name": "[concat('Containers getting OOM killed for ', variables('clusterName'), ' CI-6')]", - "location": "global", - "dependsOn": [ - "[resourceId('Microsoft.ContainerService/managedClusters', variables('clusterName'))]", - "[resourceId('Microsoft.OperationsManagement/solutions',variables('containerInsightsSolutionName'))]" - ], - "properties": { - "actions": [], - "criteria": { - "allOf": [ - { - "criterionType": "StaticThresholdCriterion", - "dimensions": [ - { - "name": "kubernetes namespace", - "operator": "Include", - "values": [ - "*" - ] - }, - { - "name": "controllerName", - "operator": "Include", - "values": [ - "*" - ] - } - ], - "metricName": "oomKilledContainerCount", - "metricNamespace": "Insights.Container/pods", - "name": "Metric1", - "operator": "GreaterThan", - "threshold": 0.0, - "timeAggregation": "Average", - "skipMetricValidation": true - } - ], - "odata.type": "Microsoft.Azure.Monitor.SingleResourceMultipleMetricCriteria" - }, - "description": "This alert monitors number of containers killed due to out of memory (OOM) error.", - "enabled": true, - "evaluationFrequency": "PT1M", - "scopes": [ - "[resourceId('Microsoft.ContainerService/managedClusters', variables('clusterName'))]" - ], - "severity": 3, - "targetResourceType": "microsoft.containerservice/managedclusters", - "windowSize": "PT1M" - } - }, - { - "type": "Microsoft.Insights/metricAlerts", - "apiVersion": "2018-03-01", - "name": "[concat('Persistent volume usage high for ', variables('clusterName'), ' CI-18')]", - "location": "global", - "dependsOn": [ - "[resourceId('Microsoft.ContainerService/managedClusters', variables('clusterName'))]", - "[resourceId('Microsoft.OperationsManagement/solutions',variables('containerInsightsSolutionName'))]" - ], - "properties": { - "actions": [], - "criteria": { - "allOf": [ - { - "criterionType": "StaticThresholdCriterion", - "dimensions": [ - { - "name": "podName", - "operator": "Include", - "values": [ - "*" - ] - }, - { - "name": "kubernetesNamespace", - "operator": "Include", - "values": [ - "*" - ] - } - ], - "metricName": "pvUsageExceededPercentage", - "metricNamespace": "Insights.Container/persistentvolumes", - "name": "Metric1", - "operator": "GreaterThan", - "threshold": 80.0, - "timeAggregation": "Average", - "skipMetricValidation": true - } - ], - "odata.type": "Microsoft.Azure.Monitor.SingleResourceMultipleMetricCriteria" - }, - "description": "This alert monitors persistent volume utilization.", - "enabled": false, - "evaluationFrequency": "PT1M", - "scopes": [ - "[resourceId('Microsoft.ContainerService/managedClusters', variables('clusterName'))]" - ], - "severity": 3, - "targetResourceType": "microsoft.containerservice/managedclusters", - "windowSize": "PT5M" - } - }, - { - "type": "Microsoft.Insights/metricAlerts", - "apiVersion": "2018-03-01", - "name": "[concat('Pods not in ready state for ', variables('clusterName'), ' CI-8')]", - "location": "global", - "dependsOn": [ - "[resourceId('Microsoft.ContainerService/managedClusters', variables('clusterName'))]", - "[resourceId('Microsoft.OperationsManagement/solutions',variables('containerInsightsSolutionName'))]" - ], - "properties": { - "actions": [], - "criteria": { - "allOf": [ - { - "criterionType": "StaticThresholdCriterion", - "dimensions": [ - { - "name": "controllerName", - "operator": "Include", - "values": [ - "*" - ] - }, - { - "name": "kubernetes namespace", - "operator": "Include", - "values": [ - "*" - ] - } - ], - "metricName": "PodReadyPercentage", - "metricNamespace": "Insights.Container/pods", - "name": "Metric1", - "operator": "LessThan", - "threshold": 80.0, - "timeAggregation": "Average", - "skipMetricValidation": true - } - ], - "odata.type": "Microsoft.Azure.Monitor.SingleResourceMultipleMetricCriteria" - }, - "description": "This alert monitors for excessive pods not in the ready state.", - "enabled": true, - "evaluationFrequency": "PT1M", - "scopes": [ - "[resourceId('Microsoft.ContainerService/managedClusters', variables('clusterName'))]" - ], - "severity": 3, - "targetResourceType": "microsoft.containerservice/managedclusters", - "windowSize": "PT5M" - } - }, - { - "type": "Microsoft.Insights/metricAlerts", - "apiVersion": "2018-03-01", - "name": "[concat('Restarting container count for ', variables('clusterName'), ' CI-7')]", - "location": "global", - "dependsOn": [ - "[resourceId('Microsoft.ContainerService/managedClusters', variables('clusterName'))]", - "[resourceId('Microsoft.OperationsManagement/solutions',variables('containerInsightsSolutionName'))]" - ], - "properties": { - "actions": [], - "criteria": { - "allOf": [ - { - "criterionType": "StaticThresholdCriterion", - "dimensions": [ - { - "name": "kubernetes namespace", - "operator": "Include", - "values": [ - "*" - ] - }, - { - "name": "controllerName", - "operator": "Include", - "values": [ - "*" - ] - } - ], - "metricName": "restartingContainerCount", - "metricNamespace": "Insights.Container/pods", - "name": "Metric1", - "operator": "GreaterThan", - "threshold": 0.0, - "timeAggregation": "Average", - "skipMetricValidation": true - } - ], - "odata.type": "Microsoft.Azure.Monitor.SingleResourceMultipleMetricCriteria" - }, - "description": "This alert monitors number of containers restarting across the cluster.", - "enabled": true, - "evaluationFrequency": "PT1M", - "scopes": [ - "[resourceId('Microsoft.ContainerService/managedClusters', variables('clusterName'))]" - ], - "severity": 3, - "targetResourceType": "Microsoft.ContainerService/managedClusters", - "windowSize": "PT1M" - } - }, - { - "type": "Microsoft.Authorization/policyAssignments", - "apiVersion": "2020-09-01", - "name": "[variables('policyAssignmentNameAKSLinuxRestrictive')]", - "comments": "Applying the 'AKS Linux Restrictive' policy to the resource group", - "properties": { - "displayName": "[trim(take(concat('[', variables('clusterName'),'] ', reference(variables('policyResourceIdAKSLinuxRestrictive'), '2020-09-01').displayName), 125))]", - "scope": "[subscriptionResourceId('Microsoft.Resources/resourceGroups', resourceGroup().name)]", - "policyDefinitionId": "[variables('policyResourceIdAKSLinuxRestrictive')]", - "parameters": { - "effect": { - "value": "audit" - }, - "excludedNamespaces": { - "value": [ - "kube-system", - "gatekeeper-system" - ] - } - } - } - }, - { - "type": "Microsoft.Authorization/policyAssignments", - "apiVersion": "2020-09-01", - "name": "[variables('policyAssignmentNameEnforceHttpsIngress')]", - "comments": "Applying the 'Enforce HTTPS ingress in Kubernetes cluster' policy to the resource group.", - "properties": { - "displayName": "[trim(take(concat('[', variables('clusterName'),'] ', reference(variables('policyResourceIdEnforceHttpsIngress'), '2020-09-01').displayName), 125))]", - "scope": "[subscriptionResourceId('Microsoft.Resources/resourceGroups', resourceGroup().name)]", - "policyDefinitionId": "[variables('policyResourceIdEnforceHttpsIngress')]", - "parameters": { - "effect": { - "value": "deny" - }, - "excludedNamespaces": { - "value": [] - } - } - } - }, - { - "type": "Microsoft.Authorization/policyAssignments", - "apiVersion": "2020-09-01", - "name": "[variables('policyAssignmentNameEnforceInternalLoadBalancers')]", - "comments": "Applying the 'Enforce internal load balancers in Kubernetes cluster' policy to the resource group.", - "properties": { - "displayName": "[trim(take(concat('[', variables('clusterName'),'] ', reference(variables('policyResourceIdEnforceInternalLoadBalancers'), '2020-09-01').displayName), 125))]", - "scope": "[subscriptionResourceId('Microsoft.Resources/resourceGroups', resourceGroup().name)]", - "policyDefinitionId": "[variables('policyResourceIdEnforceInternalLoadBalancers')]", - "parameters": { - "effect": { - "value": "deny" - }, - "excludedNamespaces": { - "value": [] - } - } - } - }, - { - "type": "Microsoft.Authorization/policyAssignments", - "apiVersion": "2020-09-01", - "name": "[variables('policyAssignmentNameMustNotAutomountApiCreds')]", - "comments": "Applying the 'No Automount of API credentials' policy to the resource group.", - "properties": { - "displayName": "[trim(take(concat('[', variables('clusterName'),'] ', reference(variables('policyResourceIdMustNotAutomountApiCreds'), '2020-09-01').displayName), 125))]", - "scope": "[subscriptionResourceId('Microsoft.Resources/resourceGroups', resourceGroup().name)]", - "policyDefinitionId": "[variables('policyResourceIdMustNotAutomountApiCreds')]", - "parameters": { - "effect": { - "value": "deny" - }, - "excludedNamespaces": { - "value": [ - "kube-system", - "gatekeeper-system", - "flux-system", // Required by Flux - "falco-system", // Required by Falco - "osm-system", // Required by OSM - "ingress-nginx", // Required by NGINX - "cluster-baseline-settings" // Required by KeyVault CSI & Kured - ] - } - } - } - }, - { - "type": "Microsoft.Authorization/policyAssignments", - "apiVersion": "2020-09-01", - "name": "[variables('policyAssignmentNameMustUseSpecifiedLabels')]", - "comments": "Applying the 'Must use specific labels' policy to the resource group.", - "properties": { - "displayName": "[trim(take(concat('[', variables('clusterName'),'] ', reference(variables('policyResourceIdMustUseSpecifiedLabels'), '2020-09-01').displayName), 125))]", - "scope": "[subscriptionResourceId('Microsoft.Resources/resourceGroups', resourceGroup().name)]", - "policyDefinitionId": "[variables('policyResourceIdMustUseSpecifiedLabels')]", - "parameters": { - "effect": { - "value": "audit" - }, - "labelsList": { - "value": [ - "pci-scope" - ] - } - } - } - }, - { - "type": "Microsoft.Authorization/policyAssignments", - "apiVersion": "2020-09-01", - "name": "[variables('policyAssignmentNameMustUseTheseExternalIps')]", - "comments": "Applying the 'Approved External IP Addresses' policy to the resource group.", - "properties": { - "displayName": "[trim(take(concat('[', variables('clusterName'),'] ', reference(variables('policyResourceIdMustUseTheseExternalIps'), '2020-09-01').displayName), 125))]", - "scope": "[subscriptionResourceId('Microsoft.Resources/resourceGroups', resourceGroup().name)]", - "policyDefinitionId": "[variables('policyResourceIdMustUseTheseExternalIps')]", - "parameters": { - "effect": { - "value": "deny" - }, - "excludedNamespaces": { - "value": [ - "kube-system", - "gatekeeper-system" - ] - }, - "allowedExternalIPs": { - "value": [] // No external IPs allowed (LoadBalancer Service types do not apply to this policy) - } - } - } - }, - { - "type": "Microsoft.Authorization/policyAssignments", - "apiVersion": "2020-09-01", - "name": "[variables('policyAssignmentNameApprovedContainerPortsOnly')]", - "comments": "Applying the 'Approved Container Ports Only' policy to for the workload living in a0005-i and a0005-o namespaces.", - "properties": { - "displayName": "[trim(take(concat('[', variables('clusterName'),'-a0005] ', reference(variables('policyResourceIdApprovedContainerPortsOnly'), '2020-09-01').displayName), 125))]", - "scope": "[subscriptionResourceId('Microsoft.Resources/resourceGroups', resourceGroup().name)]", - "policyDefinitionId": "[variables('policyResourceIdApprovedContainerPortsOnly')]", - "parameters": { - "effect": { - "value": "audit" - }, - "excludedNamespaces": { - "value": [] - }, - "namespaces": { - "value": [ - "a0005-i", - "a0005-o" - ] - }, - "allowedContainerPortsList": { - "value": [ - "8080", // ASP.net service listens on this - "15000", // envoy proxy for service mesh - "15003", // envoy proxy for service mesh - "15010" // envoy proxy for service mesh - ] - } - } - } - }, - { - "type": "Microsoft.Authorization/policyAssignments", - "apiVersion": "2020-09-01", - "name": "[variables('policyAssignmentNameApprovedServicePortsOnly')]", - "comments": "Applying the 'Approved Service Ports Only' policy to the resource group.", - "properties": { - "displayName": "[trim(take(concat('[', variables('clusterName'),'] ', reference(variables('policyResourceIdApprovedServicePortsOnly'), '2020-09-01').displayName), 125))]", - "scope": "[subscriptionResourceId('Microsoft.Resources/resourceGroups', resourceGroup().name)]", - "policyDefinitionId": "[variables('policyResourceIdApprovedServicePortsOnly')]", - "parameters": { - "effect": { - "value": "audit" - }, - "excludedNamespaces": { - "value": [ - "kube-system", - "gatekeeper-system", - "osm-system" - ] - }, - "allowedServicePortsList": { - "value": [ - "443", // ingress-controller - "80", // flux source-controller and microservice workload - "8080" // web-frontend workload - ] - } - } - } - }, - { - "type": "Microsoft.Authorization/policyAssignments", - "apiVersion": "2020-09-01", - "name": "[variables('policyAssignmentNameRoRootFilesystem')]", - "comments": "Applying the 'Kubernetes cluster containers should run with a read only root file system' policy to the resource group.", - "properties": { - "displayName": "[trim(take(concat('[', variables('clusterName'),'] ', reference(variables('policyResourceIdRoRootFilesystem'), '2020-09-01').displayName), 125))]", - "scope": "[subscriptionResourceId('Microsoft.Resources/resourceGroups', resourceGroup().name)]", - "policyDefinitionId": "[variables('policyResourceIdRoRootFilesystem')]", - "parameters": { - "effect": { - "value": "audit" - }, - // Not all workloads support this. E.g. ASP.NET requires a non-readonly root file system to handle request buffering when there is memory pressure. - "excludedNamespaces": { - "value": [ - "kube-system", - "gatekeeper-system" - ] - } - } - } - }, - { - "type": "Microsoft.Authorization/policyAssignments", - "apiVersion": "2020-09-01", - "name": "[variables('policyAssignmentNameBlockDefaultNamespace')]", - "comments": "Applying the 'Block Default Namespace' policy at the resource group level.", - "properties": { - "displayName": "[trim(take(concat('[', variables('clusterName'),'] ', reference(variables('policyResourceIdBlockDefaultNamespace'), '2020-09-01').displayName), 125))]", - "scope": "[subscriptionResourceId('Microsoft.Resources/resourceGroups', resourceGroup().name)]", - "policyDefinitionId": "[variables('policyResourceIdBlockDefaultNamespace')]", - "parameters": { - "effect": { - "value": "deny" - }, - "excludedNamespaces": { - "value": [] - } - } - } - }, - { - "type": "Microsoft.Authorization/policyAssignments", - "apiVersion": "2020-09-01", - "name": "[variables('policyAssignmentNameEnforceResourceLimits')]", - "comments": "Applying the 'Container Images Resource Limits' policy at the resource group level.", - "properties": { - "displayName": "[trim(take(concat('[', variables('clusterName'),'] ', reference(variables('policyResourceIdEnforceResourceLimits'), '2020-09-01').displayName), 125))]", - "scope": "[subscriptionResourceId('Microsoft.Resources/resourceGroups', resourceGroup().name)]", - "policyDefinitionId": "[variables('policyResourceIdEnforceResourceLimits')]", - "parameters": { - "effect": { - "value": "audit" - }, - "cpuLimit": { - "value": "2000m" - }, - "memoryLimit": { - "value": "1Gi" - }, - "excludedNamespaces": { - "value": [ - "kube-system", - "gatekeeper-system" - ] - } - } - } - }, - { - "type": "Microsoft.Authorization/policyAssignments", - "apiVersion": "2020-03-01", - "name": "[variables('policyAssignmentNameEnforceImageSource')]", - "comments": "Applying the 'Allowed Container Images' regex policy at the resource group level.", - "properties": { - "displayName": "[trim(take(concat('[', variables('clusterName'),'] ', reference(variables('policyResourceIdEnforceImageSource'), '2020-09-01').displayName), 125))]", - "scope": "[subscriptionResourceId('Microsoft.Resources/resourceGroups', resourceGroup().name)]", - "policyDefinitionId": "[variables('policyResourceIdEnforceImageSource')]", - "parameters": { - "allowedContainerImagesRegex": { - "value": "[concat(variables('defaultAcrName'), '\\.azurecr\\.io\\/live\\/.+$|mcr\\.microsoft\\.com\\/oss\\/(openservicemesh\\/init:|envoyproxy\\/envoy:).+$')]" - }, - "effect": { - "value": "deny" - }, - "excludedNamespaces": { - "value": [ - "kube-system", - "gatekeeper-system" - ] - } - } - } - } - ], - "outputs": { - "aksClusterName": { - "type": "string", - "value": "[variables('clusterName')]" - }, - "agwName": { - "type": "string", - "value": "[variables('agwName')]" - }, - "keyVaultName": { - "type": "string", - "value": "[variables('keyVaultName')]" - }, - "containerRegistryName": { - "type": "string", - "value": "[variables('defaultAcrName')]" - }, - "quarantineContainerRegistryName": { - "type": "string", - "value": "[variables('defaultAcrName')]" - } - } -} diff --git a/docs/deploy/09-aks-cluster.md b/docs/deploy/09-aks-cluster.md index 7e915517..d165f72b 100644 --- a/docs/deploy/09-aks-cluster.md +++ b/docs/deploy/09-aks-cluster.md @@ -55,11 +55,11 @@ Now that the [hub-spoke network is provisioned](./08-cluster-networking.md), the **cluster-stamp.v2.json** is a _tiny_ evolution of the **cluster-stamp.bicep** ARM template you literally just deployed in the step above. Because we are using Azure AD Pod Identity v1 as a Microsoft-managed add-on, the mechanism to associate identities with the cluster is via ARM template instead of via Kubernetes manifest deployments (as you would do with the vanilla open source solution). However, due to a current limitation of the add-on, managed identities for Pod Managed Identities CANNOT be associated to the cluster when the cluster is first being created, only as an update to an existing cluster. So this deployment will re-deploy with the Pod Managed Identity association as the _only change_. Pod Managed Identity v2 is in development and it will support assignment at cluster-creation time. This implementation will evolve to use Azure AD Pod Identity v2 when available and we'll remove this step and add the assignment directly in `cluster-stamp.bicep`. - > :eyes: If you're curious to see what changed in the cluster stamp, [view the diff](https://diffviewer.azureedge.net/?l=https://raw.githubusercontent.com/mspnp/aks-baseline-regulated/main/cluster-stamp.bicep&r=https://raw.githubusercontent.com/mspnp/aks-baseline-regulated/main/cluster-stamp.v2.json). + > :eyes: If you're curious to see what changed in the cluster stamp, [view the diff](https://diffviewer.azureedge.net/?l=https://raw.githubusercontent.com/mspnp/aks-baseline-regulated/main/cluster-stamp.bicep&r=https://raw.githubusercontent.com/mspnp/aks-baseline-regulated/main/cluster-stamp.v2.bicep). ```bash # [This takes about five minutes to run.] - az deployment group create -g rg-bu0001a0005 -f cluster-stamp.v2.json -p targetVnetResourceId=${RESOURCEID_VNET_CLUSTERSPOKE} clusterAdminAadGroupObjectId=${AADOBJECTID_GROUP_CLUSTERADMIN} k8sControlPlaneAuthorizationTenantId=${TENANTID_K8SRBAC} appGatewayListenerCertificate=${APP_GATEWAY_LISTENER_CERTIFICATE_BASE64} aksIngressControllerCertificate=${AKS_INGRESS_CONTROLLER_CERTIFICATE_BASE64} jumpBoxImageResourceId=${RESOURCEID_IMAGE_JUMPBOX} jumpBoxCloudInitAsBase64=${CLOUDINIT_BASE64} + az deployment group create -g rg-bu0001a0005 -f cluster-stamp.v2.bicep -p targetVnetResourceId=${RESOURCEID_VNET_CLUSTERSPOKE} clusterAdminAadGroupObjectId=${AADOBJECTID_GROUP_CLUSTERADMIN} k8sControlPlaneAuthorizationTenantId=${TENANTID_K8SRBAC} appGatewayListenerCertificate=${APP_GATEWAY_LISTENER_CERTIFICATE_BASE64} aksIngressControllerCertificate=${AKS_INGRESS_CONTROLLER_CERTIFICATE_BASE64} jumpBoxImageResourceId=${RESOURCEID_IMAGE_JUMPBOX} jumpBoxCloudInitAsBase64=${CLOUDINIT_BASE64} # Or if you used the parameters file … #az deployment group create -g rg-bu0001a0005 -f cluster-stamp.v2.json -p "@azuredeploy.parameters.prod.json" From 39c735f6596aea54c0378110d03fd6880ac3af8c Mon Sep 17 00:00:00 2001 From: Fernando Antivero Date: Thu, 20 Oct 2022 22:09:28 +0000 Subject: [PATCH 113/115] Address PR Feedback: bug fix - managedClusters doesn't support availability zones --- cluster-stamp.bicep | 6 +++--- cluster-stamp.v2.bicep | 6 +++--- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/cluster-stamp.bicep b/cluster-stamp.bicep index e7934bcd..fa5d268b 100644 --- a/cluster-stamp.bicep +++ b/cluster-stamp.bicep @@ -1345,7 +1345,7 @@ resource mc 'Microsoft.ContainerService/managedClusters@2021-05-01' = { orchestratorVersion: kubernetesVersion enableNodePublicIP: false maxPods: 30 - availabilityZones: pickZones('Microsoft.ContainerService', 'managedClusters', location, 3) + availabilityZones: pickZones('Microsoft.Compute', 'virtualMachineScaleSets', location, 3) upgradeSettings: { maxSurge: '33%' } @@ -1379,7 +1379,7 @@ resource mc 'Microsoft.ContainerService/managedClusters@2021-05-01' = { orchestratorVersion: kubernetesVersion enableNodePublicIP: false maxPods: 30 - availabilityZones: pickZones('Microsoft.ContainerService', 'managedClusters', location, 3) + availabilityZones: pickZones('Microsoft.Compute', 'virtualMachineScaleSets', location, 3) upgradeSettings: { maxSurge: '33%' } @@ -1411,7 +1411,7 @@ resource mc 'Microsoft.ContainerService/managedClusters@2021-05-01' = { orchestratorVersion: kubernetesVersion enableNodePublicIP: false maxPods: 30 - availabilityZones: pickZones('Microsoft.ContainerService', 'managedClusters', location, 3) + availabilityZones: pickZones('Microsoft.Compute', 'virtualMachineScaleSets', location, 3) upgradeSettings: { maxSurge: '33%' } diff --git a/cluster-stamp.v2.bicep b/cluster-stamp.v2.bicep index 8b8d0f26..627db48c 100644 --- a/cluster-stamp.v2.bicep +++ b/cluster-stamp.v2.bicep @@ -1345,7 +1345,7 @@ resource mc 'Microsoft.ContainerService/managedClusters@2021-05-01' = { orchestratorVersion: kubernetesVersion enableNodePublicIP: false maxPods: 30 - availabilityZones: pickZones('Microsoft.ContainerService', 'managedClusters', location, 3) + availabilityZones: pickZones('Microsoft.Compute', 'virtualMachineScaleSets', location, 3) upgradeSettings: { maxSurge: '33%' } @@ -1379,7 +1379,7 @@ resource mc 'Microsoft.ContainerService/managedClusters@2021-05-01' = { orchestratorVersion: kubernetesVersion enableNodePublicIP: false maxPods: 30 - availabilityZones: pickZones('Microsoft.ContainerService', 'managedClusters', location, 3) + availabilityZones: pickZones('Microsoft.Compute', 'virtualMachineScaleSets', location, 3) upgradeSettings: { maxSurge: '33%' } @@ -1411,7 +1411,7 @@ resource mc 'Microsoft.ContainerService/managedClusters@2021-05-01' = { orchestratorVersion: kubernetesVersion enableNodePublicIP: false maxPods: 30 - availabilityZones: pickZones('Microsoft.ContainerService', 'managedClusters', location, 3) + availabilityZones: pickZones('Microsoft.Compute', 'virtualMachineScaleSets', location, 3) upgradeSettings: { maxSurge: '33%' } From ae8ae34d2cd4f3b52791c56c2c3a7858c1bf255a Mon Sep 17 00:00:00 2001 From: Fernando Antivero Date: Fri, 21 Oct 2022 19:28:54 +0000 Subject: [PATCH 114/115] Apply PR Feedback: as part of this migration enable the acr zone redundancy --- cluster-stamp.bicep | 2 +- cluster-stamp.v2.bicep | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/cluster-stamp.bicep b/cluster-stamp.bicep index fa5d268b..7fc6abb5 100644 --- a/cluster-stamp.bicep +++ b/cluster-stamp.bicep @@ -897,7 +897,7 @@ resource acr 'Microsoft.ContainerRegistry/registries@2022-02-01-preview' = { } dataEndpointEnabled: true networkRuleBypassOptions: 'AzureServices' - zoneRedundancy: 'Disabled' + zoneRedundancy: 'Enabled' } resource grl 'replications' = { diff --git a/cluster-stamp.v2.bicep b/cluster-stamp.v2.bicep index 627db48c..2f782774 100644 --- a/cluster-stamp.v2.bicep +++ b/cluster-stamp.v2.bicep @@ -897,7 +897,7 @@ resource acr 'Microsoft.ContainerRegistry/registries@2022-02-01-preview' = { } dataEndpointEnabled: true networkRuleBypassOptions: 'AzureServices' - zoneRedundancy: 'Disabled' + zoneRedundancy: 'Enabled' } resource grl 'replications' = { From 4af8db0872c0e7de022146e5406cccdc22782144 Mon Sep 17 00:00:00 2001 From: Fernando Antivero Date: Fri, 21 Oct 2022 20:02:31 +0000 Subject: [PATCH 115/115] bug fix: fix commented out nodeTaints --- cluster-stamp.bicep | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/cluster-stamp.bicep b/cluster-stamp.bicep index 7fc6abb5..ae6a8b5a 100644 --- a/cluster-stamp.bicep +++ b/cluster-stamp.bicep @@ -1349,11 +1349,10 @@ resource mc 'Microsoft.ContainerService/managedClusters@2021-05-01' = { upgradeSettings: { maxSurge: '33%' } - /* This can be used to prevent unexpected workloads from landing on system node pool. All add-ons support this taint., - "nodeTaints": [ - "CriticalAddonsOnly=true:NoSchedule" - ] - */ + // This can be used to prevent unexpected workloads from landing on system node pool. All add-ons support this taint. + // nodeTaints: [ + // "CriticalAddonsOnly=true:NoSchedule" + // ] tags: { 'pci-scope': 'out-of-scope' 'Data classification': 'Confidential'