Gangams/arc k8s aad msi auth

charts/azuremonitor-containers/templates/omsagent-arc-k8s-crd.yaml

@@ -19,8 +19,16 @@ metadata:
   name: container-insights-clusteridentityrequest
   namespace: azure-arc
 spec:
-  audience: https://monitoring.azure.com/
+  {{- if eq (.Values.Azure.Cluster.Cloud | lower) "azurepubliccloud" }}
+  audience: https://monitor.azure.com/
+  {{- else if eq (.Values.Azure.Cluster.Cloud | lower) "azurechinacloud" }}
+  audience: https://monitor.azure.cn/
+  {{- else if eq (.Values.Azure.Cluster.Cloud | lower) "azureusgovernmentcloud" }}
+  audience: https://monitor.azure.us/
+  {{- else }}
+  audience: https://monitor.azure.com/
+  {{- end }}
   {{- if not (empty .Values.Azure.Extension.Name) }}
-  resourceId: {{ .Values.Azure.Extension.Name }} 
+  resourceId: {{ .Values.Azure.Extension.Name }}
   {{- end }}
 {{- end }}

charts/azuremonitor-containers/templates/omsagent-daemonset-windows.yaml

@@ -69,6 +69,8 @@ spec:
        {{- else if ne .Values.Azure.Cluster.ResourceId "<your_cluster_id>" }}
        - name: AKS_RESOURCE_ID
          value: {{ .Values.Azure.Cluster.ResourceId | quote }}
+       - name: USING_AAD_MSI_AUTH
+         value: {{ .Values.omsagent.useAADAuth | quote }}
        {{- if ne .Values.Azure.Cluster.Region "<your_cluster_region>" }}
        - name: AKS_REGION
          value: {{ .Values.Azure.Cluster.Region | quote }}

charts/azuremonitor-containers/templates/omsagent-daemonset.yaml

@@ -37,6 +37,16 @@ spec:
    serviceAccountName: omsagent
    {{- end }}
    containers:
+{{- if and (ne .Values.Azure.Cluster.ResourceId "<your_cluster_id>") (.Values.omsagent.useAADAuth) }}
+     - name: addon-token-adapter
+       imagePullPolicy: IfNotPresent
+       env:
+       - name: AZMON_COLLECT_ENV
+         value: "false"
+       - name: TOKEN_NAMESPACE
+         value: "azure-arc"
+{{-  .Values.Azure.Identity.MSIAdapterYaml | nindent 7 }}
+{{- end }}
      - name: omsagent
        {{- if eq (.Values.omsagent.domain | lower) "opinsights.azure.cn" }}
        image: "mcr.azk8s.cn/azuremonitor/containerinsights/ciprod:{{ .Values.omsagent.image.tag }}"
@@ -57,6 +67,8 @@ spec:
        {{- else if ne .Values.Azure.Cluster.ResourceId "<your_cluster_id>" }}
        - name: AKS_RESOURCE_ID
          value: {{ .Values.Azure.Cluster.ResourceId | quote }}
+       - name: USING_AAD_MSI_AUTH
+         value: {{ .Values.omsagent.useAADAuth | quote }}
        {{- if ne .Values.Azure.Cluster.Region "<your_cluster_region>" }}
        - name: AKS_REGION
          value: {{ .Values.Azure.Cluster.Region | quote }}
@@ -159,6 +171,8 @@ spec:
        {{- else if ne .Values.Azure.Cluster.ResourceId "<your_cluster_id>" }}
        - name: AKS_RESOURCE_ID
          value: {{ .Values.Azure.Cluster.ResourceId | quote }}
+       - name: USING_AAD_MSI_AUTH
+         value: {{ .Values.omsagent.useAADAuth | quote }}
        {{- if ne .Values.Azure.Cluster.Region "<your_cluster_region>" }}
        - name: AKS_REGION
          value: {{ .Values.Azure.Cluster.Region | quote }}

charts/azuremonitor-containers/templates/omsagent-deployment.yaml

@@ -33,6 +33,16 @@ spec:
    serviceAccountName: omsagent
   {{- end }}
    containers:
+{{- if and (ne .Values.Azure.Cluster.ResourceId "<your_cluster_id>") (.Values.omsagent.useAADAuth) }}
+     - name: addon-token-adapter
+       imagePullPolicy: IfNotPresent
+       env:
+       - name: AZMON_COLLECT_ENV
+         value: "false"
+       - name: TOKEN_NAMESPACE
+         value: "azure-arc"
+{{-  .Values.Azure.Identity.MSIAdapterYaml | nindent 7 }}
+{{- end }}
      - name: omsagent
        {{- if  eq (.Values.omsagent.domain | lower) "opinsights.azure.cn" }}
        image: "mcr.azk8s.cn/azuremonitor/containerinsights/ciprod:{{ .Values.omsagent.image.tag }}"
@@ -53,6 +63,8 @@ spec:
        {{- else if ne .Values.Azure.Cluster.ResourceId "<your_cluster_id>" }}
        - name: AKS_RESOURCE_ID
          value: {{ .Values.Azure.Cluster.ResourceId | quote }}
+       - name: USING_AAD_MSI_AUTH
+         value: {{ .Values.omsagent.useAADAuth | quote }}
        {{- if ne .Values.Azure.Cluster.Region "<your_cluster_region>" }}
        - name: AKS_REGION
          value: {{ .Values.Azure.Cluster.Region | quote }}

charts/azuremonitor-containers/templates/omsagent-rbac.yaml

@@ -30,18 +30,9 @@ rules:
   verbs: ["list"]
 - apiGroups: ["clusterconfig.azure.com"]
   resources: ["azureclusteridentityrequests", "azureclusteridentityrequests/status"]
-  resourceNames: ["container-insights-clusteridentityrequest"]
-  verbs: ["get", "create", "patch"]
+  verbs: ["get", "create", "patch", "list", "update", "delete"]
 - nonResourceURLs: ["/metrics"]
   verbs: ["get"]
-#arc k8s extension model grants access as part of the extension msi
-#remove this explicit permission once the extension available in public preview
-{{- if (empty .Values.Azure.Extension.Name) }}
-- apiGroups: [""]
-  resources: ["secrets"]
-  resourceNames: ["container-insights-clusteridentityrequest-token"]
-  verbs: ["get"]
-{{- end }}
 ---
 kind: ClusterRoleBinding
 {{- if .Capabilities.APIVersions.Has "rbac.authorization.k8s.io/v1" }}

charts/azuremonitor-containers/values.yaml

@@ -7,6 +7,7 @@
 ## Values of under Azure are being populated by Azure Arc K8s RP during the installation of the extension
 Azure:
   Cluster:
+    Cloud: <your_cluster_cloud>
     Region: <your_cluster_region>
     ResourceId: <your_cluster_id>
   Extension:
@@ -45,6 +46,9 @@ omsagent:
   # if set to true additional agent workflow logs will be emitted which are used for e2e and arc k8s conformance testing
   ISTEST: false
 
+  # This flag used to determine whether to use AAD MSI auth or not for Arc K8s cluster
+  useAADAuth: false
+
   ## To get your workspace id and key do the following
   ## You can create a Azure Loganalytics workspace from portal.azure.com and get its ID & PRIMARY KEY from 'Advanced Settings' tab in the Ux.
 

kubernetes/windows/main.ps1

@@ -302,10 +302,19 @@ function Set-EnvironmentVariables {
         Write-Host "Failed to set environment variable AGENT_VERSION for target 'machine' since it is either null or empty"
     }
 
+    $kubernetesPort = [System.Environment]::GetEnvironmentVariable("KUBERNETES_PORT_443_TCP_PORT", "process")
+    if (![string]::IsNullOrEmpty($kubernetesPort)) {
+        [System.Environment]::SetEnvironmentVariable("KUBERNETES_PORT_443_TCP_PORT", $kubernetesPort, "machine")
+        Write-Host "Successfully set environment variable KUBERNETES_PORT_443_TCP_PORT - $($kubernetesPort) for target 'machine'..."
+    }
+    else {
+        Write-Host "Failed to set environment variable KUBERNETES_PORT_443_TCP_PORT for target 'machine' since it is either null or empty"
+    }
+
     # run config parser
     ruby /opt/omsagentwindows/scripts/ruby/tomlparser.rb
     .\setenv.ps1
-    
+
     #Parse the configmap to set the right environment variables for agent config.
     ruby /opt/omsagentwindows/scripts/ruby/tomlparser-agent-config.rb
     .\setagentenv.ps1

scripts/onboarding/aks/onboarding-using-msi-auth/existingClusterOnboarding.json

@@ -14,10 +14,10 @@
               "description": "Location of the AKS resource e.g. \"East US\""
             }
         },
-        "aksResourceTagValues": {
+        "resourceTagValues": {
             "type": "object",
             "metadata": {
-              "description": "Existing all tags on AKS Cluster Resource"
+              "description": "Existing or new tags to use on AKS, ContainerInsights and DataCollectionRule Resources"
             }
         },
         "workspaceLocation": {
@@ -31,12 +31,6 @@
             "metadata": {
                 "description": "Full Resource ID of the log analitycs workspace that will be used for data destination. For example /subscriptions/00000000-0000-0000-0000-0000-00000000/resourceGroups/ResourceGroupName/providers/Microsoft.operationalinsights/workspaces/ws_xyz"
             }
-        },
-        "dcrResourceTagValues": {
-            "type": "object",
-            "metadata": {
-              "description": "Existing or new tags on DCR Resource"
-            }
         }
     },
     "variables": {
@@ -70,7 +64,7 @@
                         "apiVersion": "2019-11-01-preview",
                         "name": "[variables('dcrName')]",
                         "location": "[parameters('workspaceLocation')]",
-                        "tags": "[parameters('dcrResourceTagValues')]",
+                        "tags": "[parameters('resourceTagValues')]",
                         "kind": "Linux",
                         "properties": {
                             "dataSources": {
@@ -184,7 +178,7 @@
                         "name": "[variables('clusterName')]",
                         "type": "Microsoft.ContainerService/managedClusters",
                         "location": "[parameters('aksResourceLocation')]",
-                        "tags": "[parameters('aksResourceTagValues')]",
+                        "tags": "[parameters('resourceTagValues')]",
                         "apiVersion": "2018-03-31",
                         "properties": {
                           "mode": "Incremental",

scripts/onboarding/aks/onboarding-using-msi-auth/existingClusterParam.json

@@ -8,20 +8,13 @@
       "aksResourceLocation": {
         "value": "<aksClusterLocation>"
       },
-      "aksResourceTagValues": {
-        "value": {
-          "<existingOrnew-tag-name1>": "<existingOrnew-tag-value1>",
-          "<existingOrnew-tag-name2>": "<existingOrnew-tag-value2>",
-          "<existingOrnew-tag-nameN>": "<existingOrnew-tag-valueN>"
-        }
-      },
       "workspaceResourceId": {
         "value": "/subscriptions/<SubscriptionId>/resourceGroups/<ResourceGroup>/providers/Microsoft.OperationalInsights/workspaces/<workspaceName>"
       },
       "workspaceLocation": {
         "value": "<workspaceLocation>"
       },
-      "dcrResourceTagValues": {
+      "resourceTagValues": {
         "value": {
           "<existingOrnew-tag-name1>": "<existingOrnew-tag-value1>",
           "<existingOrnew-tag-name2>": "<existingOrnew-tag-value2>",