Gangams/add support for extension msi for arc k8s cluster (#495)

* wip

* add env var for the arc k8s extension name

* chart update

* extension msi updates

* fix bug

* revert chart and image to prod version

* minor text changes

* image tag to prod

* wip

* wip

* wip

* wip

* final updates

* fix whitespaces

* simplify crd yaml

charts/azuremonitor-containers/templates/omsagent-arc-k8s-crd.yaml

@@ -1,9 +1,26 @@
 {{- if or ( contains "microsoft.kubernetes/connectedclusters" (.Values.Azure.Cluster.ResourceId | lower) ) ( contains "microsoft.kubernetes/connectedclusters" (.Values.omsagent.env.clusterId | lower)) }}
+#extension model
+{{- if not (empty .Values.Azure.Extension.Name) }}
+apiVersion: clusterconfig.azure.com/v1beta1
+kind: AzureExtensionIdentity
+metadata:
+  name: {{ .Values.Azure.Extension.Name }}
+  namespace: azure-arc
+spec:
+  serviceAccounts:
+    - name: omsagent
+      namespace: kube-system
+  tokenNamespace: azure-arc
+---
+{{- end }}
 apiVersion:  clusterconfig.azure.com/v1beta1
 kind: AzureClusterIdentityRequest
 metadata:
   name: container-insights-clusteridentityrequest
   namespace: azure-arc
 spec:
   audience: https://monitoring.azure.com/
+  {{- if not (empty .Values.Azure.Extension.Name) }}
+  resourceId: {{ .Values.Azure.Extension.Name }} 
+  {{- end }}
 {{- end }}

charts/azuremonitor-containers/templates/omsagent-daemonset.yaml

@@ -70,8 +70,12 @@ spec:
          valueFrom:
             fieldRef:
               fieldPath: status.hostIP
+       {{- if not (empty .Values.Azure.Extension.Name) }}        
+       - name: ARC_K8S_EXTENSION_NAME
+         value: {{ .Values.Azure.Extension.Name | quote }} 
+       {{- end }}                        
        - name: USER_ASSIGNED_IDENTITY_CLIENT_ID
-         value: ""
+         value: ""    
        {{- if .Values.omsagent.logsettings.logflushintervalsecs }}
        - name: FBIT_SERVICE_FLUSH_INTERVAL
          value: {{ .Values.omsagent.logsettings.logflushintervalsecs | quote }}

charts/azuremonitor-containers/templates/omsagent-deployment.yaml

@@ -67,8 +67,12 @@ spec:
          valueFrom:
             fieldRef:
               fieldPath: status.hostIP
+       {{- if not (empty .Values.Azure.Extension.Name) }}
+       - name: ARC_K8S_EXTENSION_NAME
+         value: {{ .Values.Azure.Extension.Name | quote }} 
+       {{- end }}                       
        - name: USER_ASSIGNED_IDENTITY_CLIENT_ID
-         value: ""
+         value: ""   
        securityContext:
          privileged: true
        ports:

charts/azuremonitor-containers/templates/omsagent-rbac.yaml

@@ -33,10 +33,14 @@ rules:
   verbs: ["get", "create", "patch"]
 - nonResourceURLs: ["/metrics"]
   verbs: ["get"]
+#arc k8s extension model grants access as part of the extension msi 
+#remove this explicit permission once the extension available in public preview
+{{- if (empty .Values.Azure.Extension.Name) }}
 - apiGroups: [""]
   resources: ["secrets"]
   resourceNames: ["container-insights-clusteridentityrequest-token"]
   verbs: ["get"]
+{{- end }}
 ---
 kind: ClusterRoleBinding
 apiVersion: rbac.authorization.k8s.io/v1beta1

charts/azuremonitor-containers/values.yaml

@@ -4,11 +4,14 @@
 
 ## Microsoft OMS Agent image for kubernetes cluster monitoring
 ## ref: https://github.com/microsoft/Docker-Provider/tree/ci_prod
-## Values of ResourceId and Region under Azure->Cluster being populated by Azure Arc K8s RP during the installation of the extension
+## Values of under Azure are being populated by Azure Arc K8s RP during the installation of the extension
 Azure:
   Cluster:
     Region: <your_cluster_region>
-    ResourceId: <your_cluster_id>
+    ResourceId: <your_cluster_id> 
+  Extension:
+    Name: ""    
+    ResourceId: ""
 omsagent:
   image:
     repo: "mcr.microsoft.com/azuremonitor/containerinsights/ciprod"

source/plugins/ruby/arc_k8s_cluster_identity.rb

@@ -18,7 +18,7 @@ class ArcK8sClusterIdentity
   @@crd_resource_uri_template = "%{kube_api_server_url}/apis/%{cluster_config_crd_api_version}/namespaces/%{cluster_identity_resource_namespace}/azureclusteridentityrequests/%{cluster_identity_resource_name}"
   @@secret_resource_uri_template = "%{kube_api_server_url}/api/v1/namespaces/%{cluster_identity_token_secret_namespace}/secrets/%{token_secret_name}"
   @@azure_monitor_custom_metrics_audience = "https://monitoring.azure.com/"
-  @@cluster_identity_request_kind = "AzureClusterIdentityRequest"
+  @@cluster_identity_request_kind = "AzureClusterIdentityRequest" 
 
   def initialize
     @LogPath = "/var/opt/microsoft/docker-cimprov/log/arc_k8s_cluster_identity.log"
@@ -33,7 +33,9 @@ def initialize
       @log.warn "got api server url nil from KubernetesApiClient.getKubeAPIServerUrl @ #{Time.now.utc.iso8601}"
     end
     @http_client = get_http_client
-    @service_account_token = get_service_account_token
+    @service_account_token = get_service_account_token 
+    @extensionName = ENV["ARC_K8S_EXTENSION_NAME"]   
+    @log.info "extension name:#{@extensionName} @ #{Time.now.utc.iso8601}"
     @log.info "initialize complete @ #{Time.now.utc.iso8601}"
   end
 
@@ -148,7 +150,7 @@ def renew_near_expiry_token()
       update_response = @http_client.request(update_request)
       @log.info "Got response of #{update_response.code} for PATCH #{crd_request_uri} @ #{Time.now.utc.iso8601}"
       if update_response.code.to_i == 404
-        @log.info "since crd resource doesnt exist since creating crd resource : #{@@cluster_identity_resource_name} @ #{Time.now.utc.iso8601}"
+        @log.info "since crd resource doesnt exist hence creating crd resource : #{@@cluster_identity_resource_name} @ #{Time.now.utc.iso8601}"
         create_request = Net::HTTP::Post.new(crd_request_uri)
         create_request["Content-Type"] = "application/json"
         create_request["Authorization"] = "Bearer #{@service_account_token}"
@@ -211,6 +213,9 @@ def get_crd_request_body
     body["metadata"]["namespace"] = @@cluster_identity_resource_namespace
     body["spec"] = {}
     body["spec"]["audience"] = @@azure_monitor_custom_metrics_audience
+    if !@extensionName.nil? && !@extensionName.empty? 
+        body["spec"]["resourceId"] = @extensionName      
+    end 
     return body
   end
 end