Autofill ACI Connector credentials and enable logs

examples/addons/aci-connector/README.md

@@ -3,7 +3,7 @@
 
 This is the ACI Connector add-on.  Add this add-on to your json file as shown below to automatically enable ACI Connector in your new Kubernetes cluster.
 
-```
+```json
 {
     "apiVersion": "vlabs",
     "properties": {
@@ -15,25 +15,11 @@ This is the ACI Connector add-on.  Add this add-on to your json file as shown be
               "name": "aci-connector",
               "enabled" : true,
               "config": {
-                  "clientId": "",
-                  "clientKey": "",
-                  "tenantId": "",
-                  "subscriptionId": "",
-                  "resourceGroup": "",
                   "region": "",
                   "nodeName": "",
                   "os": "",
                   "taint": ""
-              },
-              "containers": [
-                {
-                  "name": "aci-connector",
-                  "cpuRequests": "50m",
-                  "memoryRequests": "150Mi",
-                  "cpuLimits": "50m",
-                  "memoryLimits": "150Mi"
-                }
-              ]
+              }
             }
           ]
         }
@@ -48,7 +34,7 @@ This is the ACI Connector add-on.  Add this add-on to your json file as shown be
           "name": "agentpool",
           "count": 3,
           "vmSize": "Standard_DS2_v2",
-          "availabilityProfile": "AvailabilitySet"
+          "availabilityProfile": "VirtualMachineScaleSets"
         }
       ],
       "linuxProfile": {
@@ -72,32 +58,24 @@ This is the ACI Connector add-on.  Add this add-on to your json file as shown be
 
 You can validate that the add-on is running as expected with the following commands:
 
-Make sure to create resource group:
-```
-az group create \
-    --name "[resource group name]" \
-    --location "[location]"
-```
+You should see ACI Connector as `Running` after executing:
 
-You should see ACI Connector as running after running:
-```
-$ kubectl get pods -n kube-system
+```bash
+kubectl get pods -n kube-system
 ```
 
-You should see ACI Connector node after running:
-```
-$ kubectl get nodes
+You should see ACI Connector node after executing:
+
+```bash
+kubectl get nodes
 ```
 
 Follow the README at https://github.com/virtual-kubelet/virtual-kubelet for examples.
 
-# Configuration
+## Configuration
+
 |Name|Required|Description|Default Value|
 |---|---|---|---|
-|clientId|yes|your client id||
-|clientKey|yes|your client key||
-|tenantId|yes|your tenant id||
-|resourceGroup|yes|your resource group||
 |region|no|Azure region|"westus"|
 |nodeName|no|node name|"aci-connector"|
 |os|no|operating system (Linux/Windows)|"Linux"|
@@ -109,6 +87,6 @@ Follow the README at https://github.com/virtual-kubelet/virtual-kubelet for exam
 |cpuLimits|no|cpu limits for the container|"50m"|
 |memoryLimits|no|memory limits for the container|"150Mi"|
 
+## Supported Orchestrators
 
-# Supported Orchestrators
 Kubernetes

examples/addons/aci-connector/kubernetes-aci-connector.json

@@ -1,61 +1,43 @@
 {
-    "apiVersion": "vlabs",
-    "properties": {
-      "orchestratorProfile": {
-        "orchestratorType": "Kubernetes",
-        "kubernetesConfig": {
-          "addons": [
-            {
-              "name": "aci-connector",
-              "enabled" : true,
-              "config": {
-                  "clientId": "",
-                  "clientKey": "",
-                  "tenantId": "",
-                  "subscriptionId": "",
-                  "resourceGroup": "",
-                  "region": "eastus"
-              },
-              "containers": [
-                {
-                  "name": "aci-connector",
-                  "image": "microsoft/aci-connector-k8s:latest",
-                  "cpuRequests": "50m",
-                  "memoryRequests": "150Mi",
-                  "cpuLimits": "50m",
-                  "memoryLimits": "150Mi"
-                }
-              ]
-            }
-          ]
-        }
-      },
-      "masterProfile": {
-        "count": 1,
-        "dnsPrefix": "",
-        "vmSize": "Standard_DS2_v2"
-      },
-      "agentPoolProfiles": [
-        {
-          "name": "agentpool",
-          "count": 3,
-          "vmSize": "Standard_DS2_v2",
-          "availabilityProfile": "AvailabilitySet"
-        }
-      ],
-      "linuxProfile": {
-        "adminUsername": "azureuser",
-        "ssh": {
-          "publicKeys": [
-            {
-              "keyData": ""
-            }
-          ]
-        }
-      },
-      "servicePrincipalProfile": {
-        "clientId": "",
-        "secret": ""
+  "apiVersion": "vlabs",
+  "properties": {
+    "orchestratorProfile": {
+      "orchestratorType": "Kubernetes",
+      "kubernetesConfig": {
+        "addons": [
+          {
+            "name": "aci-connector",
+            "enabled": true
+          }
+        ]
       }
+    },
+    "masterProfile": {
+      "count": 1,
+      "dnsPrefix": "",
+      "vmSize": "Standard_DS2_v2"
+    },
+    "agentPoolProfiles": [
+      {
+        "name": "agentpool",
+        "count": 3,
+        "vmSize": "Standard_DS2_v2",
+        "availabilityProfile": "VirtualMachineScaleSets"
+      }
+    ],
+    "linuxProfile": {
+      "adminUsername": "azureuser",
+      "ssh": {
+        "publicKeys": [
+          {
+            "keyData": ""
+          }
+        ]
+      }
+    },
+    "servicePrincipalProfile": {
+      "clientId": "",
+      "secret": ""
     }
   }
+}

parts/k8s/addons/kubernetesmasteraddons-aci-connector-deployment.yaml

@@ -58,6 +58,8 @@ metadata:
 type: Opaque
 data:
   credentials.json: <kubernetesACIConnectorCredentials>
+  cert.pem: <kubernetesACIConnectorCert>
+  key.pem: <kubernetesACIConnectorKey>
 ---
 apiVersion: extensions/v1beta1
 kind: Deployment
@@ -77,17 +79,29 @@ spec:
         app: aci-connector
     spec:
       serviceAccountName: aci-connector
+      nodeSelector:
+        beta.kubernetes.io/os: linux
       containers:
       - name: aci-connector
         image: <kubernetesACIConnectorSpec>
         imagePullPolicy: Always
         env:
+        - name: KUBELET_PORT
+          value: "10250"
         - name: AZURE_AUTH_LOCATION
           value: /etc/virtual-kubelet/credentials.json
         - name: ACI_RESOURCE_GROUP
           value: <kubernetesACIConnectorResourceGroup>
         - name: ACI_REGION
           value: <kubernetesACIConnectorRegion>
+        - name: APISERVER_CERT_LOCATION
+          value: /etc/virtual-kubelet/cert.pem
+        - name: APISERVER_KEY_LOCATION
+          value: /etc/virtual-kubelet/key.pem
+        - name: VKUBELET_POD_IP
+          valueFrom:
+            fieldRef:
+              fieldPath: status.podIP
         resources:
           requests:
             cpu: <kubernetesACIConnectorCPURequests>

parts/k8s/kubernetescustomscript.sh

@@ -432,10 +432,27 @@ function configClusterAutoscalerAddon() {
     sed -i "s|<kubernetesClusterAutoscalerVMSSName>|$(echo $PRIMARY_SCALE_SET)|g" "/etc/kubernetes/addons/cluster-autoscaler-deployment.yaml"
 }
 
-function configAddons() {
+configACIConnectorAddon() {
+    ACI_CONNECTOR_CREDENTIALS=$(printf "{\"clientId\": \"$(echo $SERVICE_PRINCIPAL_CLIENT_ID)\", \"clientSecret\": \"$(echo $SERVICE_PRINCIPAL_CLIENT_SECRET)\", \"tenantId\": \"$(echo $TENANT_ID)\", \"subscriptionId\": \"$(echo $SUBSCRIPTION_ID)\", \"activeDirectoryEndpointUrl\": \"https://login.microsoftonline.com\",\"resourceManagerEndpointUrl\": \"https://management.azure.com/\", \"activeDirectoryGraphResourceId\": \"https://graph.windows.net/\", \"sqlManagementEndpointUrl\": \"https://management.core.windows.net:8443/\", \"galleryEndpointUrl\": \"https://gallery.azure.com/\", \"managementEndpointUrl\": \"https://management.core.windows.net/\"}" | base64 -w 0)
+
+    openssl req -newkey rsa:4096 -new -nodes -x509 -days 3650 -keyout /etc/kubernetes/certs/aci-connector-key.pem -out /etc/kubernetes/certs/aci-connector-cert.pem -subj "/C=US/ST=CA/L=virtualkubelet/O=virtualkubelet/OU=virtualkubelet/CN=virtualkubelet"
+    ACI_CONNECTOR_KEY=$(base64 /etc/kubernetes/certs/aci-connector-key.pem -w0)
+    ACI_CONNECTOR_CERT=$(base64 /etc/kubernetes/certs/aci-connector-cert.pem -w0)
+
+    sed -i "s|<kubernetesACIConnectorCredentials>|$ACI_CONNECTOR_CREDENTIALS|g" "/etc/kubernetes/addons/aci-connector-deployment.yaml"
+    sed -i "s|<kubernetesACIConnectorResourceGroup>|$(echo $RESOURCE_GROUP)|g" "/etc/kubernetes/addons/aci-connector-deployment.yaml"
+    sed -i "s|<kubernetesACIConnectorCert>|$(echo $ACI_CONNECTOR_CERT)|g" "/etc/kubernetes/addons/aci-connector-deployment.yaml"
+    sed -i "s|<kubernetesACIConnectorKey>|$(echo $ACI_CONNECTOR_KEY)|g" "/etc/kubernetes/addons/aci-connector-deployment.yaml"
+}
+
+configAddons() {
     if [[ "${CLUSTER_AUTOSCALER_ADDON}" = True ]]; then
         configClusterAutoscalerAddon
     fi
+
+    if [[ "${ACI_CONNECTOR_ADDON}" = True ]]; then
+        configACIConnectorAddon
+    fi
 }
 
 if [[ $OS == $UBUNTU_OS_NAME ]]; then
@@ -444,7 +461,7 @@ if [[ $OS == $UBUNTU_OS_NAME ]]; then
     echo `date`,`hostname`, apt-get_update_end>>/opt/m
 	# make sure walinuxagent doesn't get updated in the middle of running this script
 	retrycmd_if_failure 20 5 30 apt-mark hold walinuxagent || exit $ERR_HOLD_WALINUXAGENT
-    
+
 fi
 
 waitForCloudInit

parts/k8s/kubernetesmastercustomdata.yml

@@ -237,10 +237,7 @@ MASTER_ARTIFACTS_CONFIG_PLACEHOLDER
 {{end}}
 
 {{if .OrchestratorProfile.KubernetesConfig.IsACIConnectorEnabled}}
-    ACI_CONNECTOR_CREDENTIALS=$(printf "{\"clientId\": \"{{WrapAsVariable "kubernetesACIConnectorClientId"}}\", \"clientSecret\": \"{{WrapAsVariable "kubernetesACIConnectorClientKey"}}\", \"tenantId\": \"{{WrapAsVariable "kubernetesACIConnectorTenantId"}}\", \"subscriptionId\": \"{{WrapAsVariable "kubernetesACIConnectorSubscriptionId"}}\", \"activeDirectoryEndpointUrl\": \"https://login.microsoftonline.com\",\"resourceManagerEndpointUrl\": \"https://management.azure.com/\", \"activeDirectoryGraphResourceId\": \"https://graph.windows.net/\", \"sqlManagementEndpointUrl\": \"https://management.core.windows.net:8443/\", \"galleryEndpointUrl\": \"https://gallery.azure.com/\", \"managementEndpointUrl\": \"https://management.core.windows.net/\"}" | base64 -w 0)
     sed -i "s|<kubernetesACIConnectorSpec>|{{WrapAsVariable "kubernetesACIConnectorSpec"}}|g" "/etc/kubernetes/addons/aci-connector-deployment.yaml"
-    sed -i "s|<kubernetesACIConnectorCredentials>|$ACI_CONNECTOR_CREDENTIALS|g" "/etc/kubernetes/addons/aci-connector-deployment.yaml"
-    sed -i "s|<kubernetesACIConnectorResourceGroup>|{{WrapAsVariable "kubernetesACIConnectorResourceGroup"}}|g" "/etc/kubernetes/addons/aci-connector-deployment.yaml"
     sed -i "s|<kubernetesACIConnectorNodeName>|{{WrapAsVariable "kubernetesACIConnectorNodeName"}}|g" "/etc/kubernetes/addons/aci-connector-deployment.yaml"
     sed -i "s|<kubernetesACIConnectorOS>|{{WrapAsVariable "kubernetesACIConnectorOS"}}|g" "/etc/kubernetes/addons/aci-connector-deployment.yaml"
     sed -i "s|<kubernetesACIConnectorTaint>|{{WrapAsVariable "kubernetesACIConnectorTaint"}}|g" "/etc/kubernetes/addons/aci-connector-deployment.yaml"

parts/k8s/kubernetesmastervars.t

@@ -95,12 +95,8 @@
     "kubernetesTillerCPULimit": "[parameters('kubernetesTillerCPULimit')]",
     "kubernetesTillerMemoryLimit": "[parameters('kubernetesTillerMemoryLimit')]",
     "kubernetesTillerMaxHistory": "[parameters('kubernetesTillerMaxHistory')]",
+    "kubernetesACIConnectorEnabled": "[parameters('kubernetesACIConnectorEnabled')]",
     "kubernetesACIConnectorSpec": "[parameters('kubernetesACIConnectorSpec')]",
-    "kubernetesACIConnectorClientId": "[parameters('kubernetesACIConnectorClientId')]",
-    "kubernetesACIConnectorClientKey": "[parameters('kubernetesACIConnectorClientKey')]",
-    "kubernetesACIConnectorTenantId": "[parameters('kubernetesACIConnectorTenantId')]",
-    "kubernetesACIConnectorSubscriptionId": "[parameters('kubernetesACIConnectorSubscriptionId')]",
-    "kubernetesACIConnectorResourceGroup": "[parameters('kubernetesACIConnectorResourceGroup')]",
     "kubernetesACIConnectorNodeName": "[parameters('kubernetesACIConnectorNodeName')]",
     "kubernetesACIConnectorOS": "[parameters('kubernetesACIConnectorOS')]",
     "kubernetesACIConnectorTaint": "[parameters('kubernetesACIConnectorTaint')]",
@@ -237,7 +233,7 @@
     "mountetcdScript": "{{GetKubernetesB64Mountetcd}}",
 {{if not IsOpenShift}}
 {{if not IsHostedMaster}}
-    "provisionScriptParametersMaster": "[concat('MASTER_VM_NAME=',variables('masterVMNames')[variables('masterOffset')],' ETCD_PEER_URL=',variables('masterEtcdPeerURLs')[variables('masterOffset')],' ETCD_CLIENT_URL=',variables('masterEtcdClientURLs')[variables('masterOffset')],' MASTER_NODE=true CLUSTER_AUTOSCALER_ADDON=',variables('kubernetesClusterAutoscalerEnabled'),' APISERVER_PRIVATE_KEY=',variables('apiServerPrivateKey'),' CA_CERTIFICATE=',variables('caCertificate'),' CA_PRIVATE_KEY=',variables('caPrivateKey'),' MASTER_FQDN=',variables('masterFqdnPrefix'),' KUBECONFIG_CERTIFICATE=',variables('kubeConfigCertificate'),' KUBECONFIG_KEY=',variables('kubeConfigPrivateKey'),' ETCD_SERVER_CERTIFICATE=',variables('etcdServerCertificate'),' ETCD_CLIENT_CERTIFICATE=',variables('etcdClientCertificate'),' ETCD_SERVER_PRIVATE_KEY=',variables('etcdServerPrivateKey'),' ETCD_CLIENT_PRIVATE_KEY=',variables('etcdClientPrivateKey'),' ETCD_PEER_CERTIFICATES=',string(variables('etcdPeerCertificates')),' ETCD_PEER_PRIVATE_KEYS=',string(variables('etcdPeerPrivateKeys')))]",
+    "provisionScriptParametersMaster": "[concat('MASTER_VM_NAME=',variables('masterVMNames')[variables('masterOffset')],' ETCD_PEER_URL=',variables('masterEtcdPeerURLs')[variables('masterOffset')],' ETCD_CLIENT_URL=',variables('masterEtcdClientURLs')[variables('masterOffset')],' MASTER_NODE=true CLUSTER_AUTOSCALER_ADDON=',variables('kubernetesClusterAutoscalerEnabled'),' ACI_CONNECTOR_ADDON=',variables('kubernetesACIConnectorEnabled'),' APISERVER_PRIVATE_KEY=',variables('apiServerPrivateKey'),' CA_CERTIFICATE=',variables('caCertificate'),' CA_PRIVATE_KEY=',variables('caPrivateKey'),' MASTER_FQDN=',variables('masterFqdnPrefix'),' KUBECONFIG_CERTIFICATE=',variables('kubeConfigCertificate'),' KUBECONFIG_KEY=',variables('kubeConfigPrivateKey'),' ETCD_SERVER_CERTIFICATE=',variables('etcdServerCertificate'),' ETCD_CLIENT_CERTIFICATE=',variables('etcdClientCertificate'),' ETCD_SERVER_PRIVATE_KEY=',variables('etcdServerPrivateKey'),' ETCD_CLIENT_PRIVATE_KEY=',variables('etcdClientPrivateKey'),' ETCD_PEER_CERTIFICATES=',string(variables('etcdPeerCertificates')),' ETCD_PEER_PRIVATE_KEYS=',string(variables('etcdPeerPrivateKeys')))]",
   {{if EnableEncryptionWithExternalKms}}
     {{ if not UseManagedIdentity}}
     "servicePrincipalObjectId": "[parameters('servicePrincipalObjectId')]",

parts/k8s/kubernetesparams.t

@@ -346,45 +346,17 @@
       },
       "type": "string"
     },
-    "kubernetesACIConnectorSpec": {
-      {{PopulateClassicModeDefaultValue "kubernetesACIConnectorSpec"}}
-      "metadata": {
-        "description": "The container spec for ACI Connector."
-      },
-      "type": "string"
-    },
-    "kubernetesACIConnectorClientId": {
-      {{PopulateClassicModeDefaultValue "kubernetesACIConnectorClientId"}}
-      "metadata": {
-        "description": "Client id for ACI Connector."
-      },
-      "type": "string"
-    },
-    "kubernetesACIConnectorClientKey": {
-      {{PopulateClassicModeDefaultValue "kubernetesACIConnectorClientKey"}}
-      "metadata": {
-        "description": "Client key for ACI Connector."
-      },
-      "type": "string"
-    },
-    "kubernetesACIConnectorTenantId": {
-      {{PopulateClassicModeDefaultValue "kubernetesACIConnectorTenantId"}}
-      "metadata": {
-        "description": "Tenant id for ACI Connector."
-      },
-      "type": "string"
-    },
-    "kubernetesACIConnectorSubscriptionId": {
-      {{PopulateClassicModeDefaultValue "kubernetesACIConnectorSubscriptionId"}}
+    "kubernetesACIConnectorEnabled": {
+      "defaultValue": false,
       "metadata": {
-        "description": "Subscription id for ACI Connector."
+        "description": "ACI Connector Status"
       },
-      "type": "string"
+      "type": "bool"
     },
-    "kubernetesACIConnectorResourceGroup": {
-      {{PopulateClassicModeDefaultValue "kubernetesACIConnectorResourceGroup"}}
+    "kubernetesACIConnectorSpec": {
+      {{PopulateClassicModeDefaultValue "kubernetesACIConnectorSpec"}}
       "metadata": {
-        "description": "Resource group for ACI Connector."
+        "description": "The container spec for ACI Connector."
       },
       "type": "string"
     },