Make deployment parameters robust for various acs-engine versions

cluster-autoscaler/cloudprovider/azure/README.md

@@ -36,7 +36,7 @@ Fill the values of cluster-autoscaler-azure secret in [cluster-autoscaler-vmss.y
 - SubscriptionID: `<base64-encode-subscription-id>`
 - TenantID: `<base64-encoded-tenant-id>`
 
-Note that all data should be encoded with base64.
+> Note that all data above should be encoded with base64.
 
 And fill the node groups in container command by `--nodes`, e.g.
 
@@ -68,12 +68,8 @@ kubectl create -f cluster-autoscaler-vmss-master.yaml
 Pre-requirements:
 
 - Get credentials from above `permissions` step.
-- Get the required parameters from acs-engine deployments (usually under directory `_output/<master-dns-prefix>` after running `acs-engine deploy` command)
-  - Get `APIServerPrivateKey`, `CAPrivateKey`, `ClientPrivateKey` and `KubeConfigPrivateKey` from `azuredeploy.parameters.json`
-  - Get `EtcdClientPrivateKey` and `EtcdServerPrivateKey` if the cluster is deployed by acs-engine >= v0.12.0
-  - If windows nodes are included, also get `WindowsAdminPassword` from acs-engine deployment manifests
-  - Get the initial Azure deployment name from azure portal. If you have multiple deployments (e.g. have run `acs-engine scale` command), make sure to get the first one
-  - Get a node pool name for nodes scaling from acs-engine deployment manifests
+- Get the initial Azure deployment name from azure portal. If you have multiple deployments (e.g. have run `acs-engine scale` command), make sure to get the first one.
+- Get a node pool name for nodes scaling from acs-engine deployment manifests
 - Encode each data with base64.
 
 Fill the values of cluster-autoscaler-azure secret in [cluster-autoscaler-standard-master.yaml](cluster-autoscaler-standard-master.yaml), including
@@ -83,17 +79,9 @@ Fill the values of cluster-autoscaler-azure secret in [cluster-autoscaler-standa
 - ResourceGroup: `<base64-encoded-resource-group>`
 - SubscriptionID: `<base64-encode-subscription-id>`
 - TenantID: `<base64-encoded-tenant-id>`
-- NodeGroup: `<base64-encoded-node-pool-name>`
 - Deployment: `<base64-encoded-azure-initial-deploy-name>`
-- APIServerPrivateKey: `<base64-encoded-apiserver-private-key>`
-- CAPrivateKey: `<base64-encoded-ca-private-key>`
-- ClientPrivateKey: `<base64-encoded-client-private-key>`
-- KubeConfigPrivateKey: `<base64-encoded-kubeconfig-private-key>`
-- WindowsAdminPassword: `<base64-encoded-windows-admin-password>` (set `""` if no windows nodes in the cluster)
-- EtcdClientPrivateKey: `<base64-encoded-etcd-client-private-key>` (set `""` for acs-engine < v0.12.0)
-- EtcdServerPrivateKey: `<base64-encoded-etcd-server-private-key>` (set to `""` for acs-engine < v0.12.0)
 
-Note that all data should be encoded with base64.
+> Note that all data above should be encoded with base64.
 
 And fill the node groups in container command by `--nodes`, e.g.
 
@@ -108,6 +96,12 @@ or multiple node groups:
         - --nodes=1:10:agentpool2
 ```
 
+Create Azure deploy parameters secret `cluster-autoscaler-azure-deploy-parameters` by running
+
+```sh
+kubectl -n kube-system create secret generic cluster-autoscaler-azure-deploy-parameters --from-file=deploy-parameters=./_output/<your-output-path>/azuredeploy.parameters.json
+```
+
 Then deploy cluster-autoscaler by running
 
 ```sh

cluster-autoscaler/cloudprovider/azure/azure_agent_pool.go

@@ -71,53 +71,17 @@ func NewAgentPool(spec *dynamic.NodeGroupSpec, az *AzureManager) (*AgentPool, er
 }
 
 func (as *AgentPool) initialize() error {
-	deploy, err := as.manager.azClient.deploymentsClient.Get(as.manager.config.ResourceGroup, as.manager.config.Deployment)
-	if err != nil {
-		glog.Errorf("deploymentsClient.Get(%s, %s) failed: %v", as.manager.config.ResourceGroup, as.manager.config.Deployment, err)
-		return err
-	}
-
 	template, err := as.manager.azClient.deploymentsClient.ExportTemplate(as.manager.config.ResourceGroup, as.manager.config.Deployment)
 	if err != nil {
 		glog.Errorf("deploymentsClient.ExportTemplate(%s, %s) failed: %v", as.manager.config.ResourceGroup, as.manager.config.Deployment, err)
 		return err
 	}
 
-	as.parameters = *deploy.Properties.Parameters
-	as.preprocessParameters()
-
 	as.template = *template.Template
+	as.parameters = as.manager.config.DeploymentParameters
 	return normalizeForK8sVMASScalingUp(as.template)
 }
 
-func (as *AgentPool) preprocessParameters() {
-	// Delete type key from parameters.
-	for k := range as.parameters {
-		if v, ok := as.parameters[k].(map[string]interface{}); ok {
-			delete(v, "type")
-		}
-	}
-
-	// fulfill secure parameters.
-	as.parameters["apiServerPrivateKey"] = map[string]string{"value": as.manager.config.APIServerPrivateKey}
-	as.parameters["caPrivateKey"] = map[string]string{"value": as.manager.config.CAPrivateKey}
-	as.parameters["clientPrivateKey"] = map[string]string{"value": as.manager.config.ClientPrivateKey}
-	as.parameters["kubeConfigPrivateKey"] = map[string]string{"value": as.manager.config.KubeConfigPrivateKey}
-	as.parameters["servicePrincipalClientId"] = map[string]string{"value": as.manager.config.AADClientID}
-	as.parameters["servicePrincipalClientSecret"] = map[string]string{"value": as.manager.config.AADClientSecret}
-	if as.manager.config.WindowsAdminPassword != "" {
-		as.parameters["windowsAdminPassword"] = map[string]string{"value": as.manager.config.WindowsAdminPassword}
-	}
-
-	// etcd TLS parameters (for acs-engine >= v0.12.0).
-	if as.manager.config.EtcdClientPrivateKey != "" {
-		as.parameters["etcdClientPrivateKey"] = map[string]string{"value": as.manager.config.EtcdClientPrivateKey}
-	}
-	if as.manager.config.EtcdServerPrivateKey != "" {
-		as.parameters["etcdServerPrivateKey"] = map[string]string{"value": as.manager.config.EtcdServerPrivateKey}
-	}
-}
-
 // MinSize returns minimum size of the node group.
 func (as *AgentPool) MinSize() int {
 	return as.minSize

cluster-autoscaler/cloudprovider/azure/azure_manager.go

@@ -38,6 +38,9 @@ const (
 
 	scaleToZeroSupported = false
 	refreshInterval      = 1 * time.Minute
+
+	// The path of deployment parameters for standard vm.
+	deploymentParametersPath = "/var/lib/azure/azuredeploy.parameters.json"
 )
 
 // AzureManager handles Azure communication and data caching.
@@ -67,15 +70,8 @@ type Config struct {
 	UseManagedIdentityExtension bool   `json:"useManagedIdentityExtension" yaml:"useManagedIdentityExtension"`
 
 	// Configs only for standard vmType (agent pools).
-	Deployment           string `json:"deployment" yaml:"deployment"`
-	APIServerPrivateKey  string `json:"apiServerPrivateKey" yaml:"apiServerPrivateKey"`
-	CAPrivateKey         string `json:"caPrivateKey" yaml:"caPrivateKey"`
-	ClientPrivateKey     string `json:"clientPrivateKey" yaml:"clientPrivateKey"`
-	KubeConfigPrivateKey string `json:"kubeConfigPrivateKey" yaml:"kubeConfigPrivateKey"`
-	WindowsAdminPassword string `json:"windowsAdminPassword" yaml:"windowsAdminPassword"`
-	// etcd TLS parameters (for acs-engine >= v0.12.0).
-	EtcdClientPrivateKey string `json:"etcdClientPrivateKey" yaml:"etcdClientPrivateKey"`
-	EtcdServerPrivateKey string `json:"etcdServerPrivateKey" yaml:"etcdServerPrivateKey"`
+	Deployment           string                 `json:"deployment" yaml:"deployment"`
+	DeploymentParameters map[string]interface{} `json:"deploymentParameters" yaml:"deploymentParameters"`
 }
 
 // TrimSpace removes all leading and trailing white spaces.
@@ -90,13 +86,6 @@ func (c *Config) TrimSpace() {
 	c.AADClientCertPath = strings.TrimSpace(c.AADClientCertPath)
 	c.AADClientCertPassword = strings.TrimSpace(c.AADClientCertPassword)
 	c.Deployment = strings.TrimSpace(c.Deployment)
-	c.APIServerPrivateKey = strings.TrimSpace(c.APIServerPrivateKey)
-	c.CAPrivateKey = strings.TrimSpace(c.CAPrivateKey)
-	c.ClientPrivateKey = strings.TrimSpace(c.ClientPrivateKey)
-	c.KubeConfigPrivateKey = strings.TrimSpace(c.KubeConfigPrivateKey)
-	c.WindowsAdminPassword = strings.TrimSpace(c.WindowsAdminPassword)
-	c.EtcdClientPrivateKey = strings.TrimSpace(c.EtcdClientPrivateKey)
-	c.EtcdServerPrivateKey = strings.TrimSpace(c.EtcdServerPrivateKey)
 }
 
 // CreateAzureManager creates Azure Manager object to work with Azure.
@@ -120,13 +109,6 @@ func CreateAzureManager(configReader io.Reader, discoveryOpts cloudprovider.Node
 		cfg.AADClientCertPath = os.Getenv("ARM_CLIENT_CERT_PATH")
 		cfg.AADClientCertPassword = os.Getenv("ARM_CLIENT_CERT_PASSWORD")
 		cfg.Deployment = os.Getenv("ARM_DEPLOYMENT")
-		cfg.APIServerPrivateKey = os.Getenv("ARM_APISEVER_PRIVATE_KEY")
-		cfg.CAPrivateKey = os.Getenv("ARM_CA_PRIVATE_KEY")
-		cfg.ClientPrivateKey = os.Getenv("ARM_CLIENT_PRIVATE_KEY")
-		cfg.KubeConfigPrivateKey = os.Getenv("ARM_KUBECONFIG_PRIVATE_KEY")
-		cfg.WindowsAdminPassword = os.Getenv("ARM_WINDOWS_ADMIN_PASSWORD")
-		cfg.EtcdClientPrivateKey = os.Getenv("ARM_ETCD_CLIENT_RPIVATE_KEY")
-		cfg.EtcdServerPrivateKey = os.Getenv("ARM_ETCD_SERVER_PRIVATE_KEY")
 
 		useManagedIdentityExtensionFromEnv := os.Getenv("ARM_USE_MANAGED_IDENTITY_EXTENSION")
 		if len(useManagedIdentityExtensionFromEnv) > 0 {
@@ -143,6 +125,17 @@ func CreateAzureManager(configReader io.Reader, discoveryOpts cloudprovider.Node
 		cfg.VMType = vmTypeVMSS
 	}
 
+	// Read parameters from deploymentParametersPath if it is not set.
+	if cfg.VMType == vmTypeStandard && len(cfg.DeploymentParameters) == 0 {
+		parameters, err := readDeploymentParameters(deploymentParametersPath)
+		if err != nil {
+			glog.Errorf("readDeploymentParameters failed with error: %v", err)
+			return nil, err
+		}
+
+		cfg.DeploymentParameters = parameters
+	}
+
 	// Defaulting env to Azure Public Cloud.
 	env := azure.PublicCloud
 	if cfg.Cloud != "" {

cluster-autoscaler/cloudprovider/azure/azure_util.go

@@ -19,7 +19,9 @@ package azure
 import (
 	"crypto/rsa"
 	"crypto/x509"
+	"encoding/json"
 	"fmt"
+	"io/ioutil"
 	"net/url"
 	"regexp"
 	"sort"
@@ -398,20 +400,8 @@ func validateConfig(cfg *Config) error {
 			return fmt.Errorf("deployment not set")
 		}
 
-		if cfg.APIServerPrivateKey == "" {
-			return fmt.Errorf("apiServerPrivateKey not set")
-		}
-
-		if cfg.CAPrivateKey == "" {
-			return fmt.Errorf("caPrivateKey not set")
-		}
-
-		if cfg.ClientPrivateKey == "" {
-			return fmt.Errorf("clientPrivateKey not set")
-		}
-
-		if cfg.KubeConfigPrivateKey == "" {
-			return fmt.Errorf("kubeConfigPrivateKey not set")
+		if len(cfg.DeploymentParameters) == 0 {
+			return fmt.Errorf("deploymentParameters not set")
 		}
 	}
 
@@ -428,3 +418,24 @@ func getLastSegment(ID string) (string, error) {
 
 	return name, nil
 }
+
+// readDeploymentParameters gets deployment parameters from paramFilePath.
+func readDeploymentParameters(paramFilePath string) (map[string]interface{}, error) {
+	contents, err := ioutil.ReadFile(paramFilePath)
+	if err != nil {
+		glog.Errorf("Failed to read deployment parameters from file %q: %v", paramFilePath, err)
+		return nil, err
+	}
+
+	deploymentParameters := make(map[string]interface{})
+	if err := json.Unmarshal(contents, &deploymentParameters); err != nil {
+		glog.Errorf("Failed to unmarshal deployment parameters from file %q: %v", paramFilePath, err)
+		return nil, err
+	}
+
+	if v, ok := deploymentParameters["parameters"]; ok {
+		return v.(map[string]interface{}), nil
+	}
+
+	return nil, fmt.Errorf("failed to get deployment parameters from file %s", paramFilePath)
+}

cluster-autoscaler/cloudprovider/azure/cluster-autoscaler-standard-master.yaml

@@ -110,13 +110,6 @@ data:
   SubscriptionID: <base64-encode-subscription-id>
   TenantID: <base64-encoded-tenant-id>
   Deployment: <base64-encoded-azure-initial-deploy-name>
-  APIServerPrivateKey: <base64-encoded-apiserver-private-key>
-  CAPrivateKey: <base64-encoded-ca-private-key>
-  ClientPrivateKey: <base64-encoded-client-private-key>
-  KubeConfigPrivateKey: <base64-encoded-kubeconfig-private-key>
-  WindowsAdminPassword: <base64-encoded-windows-admin-password>
-  EtcdClientPrivateKey: <base64-encoded-etcd-client-private-key>
-  EtcdServerPrivateKey: <base64-encoded-etcd-server-private-key>
   VMType: c3RhbmRhcmQ=
 kind: Secret
 metadata:
@@ -193,41 +186,6 @@ spec:
             secretKeyRef:
               key: Deployment
               name: cluster-autoscaler-azure
-        - name: ARM_APISEVER_PRIVATE_KEY
-          valueFrom:
-            secretKeyRef:
-              key: APIServerPrivateKey
-              name: cluster-autoscaler-azure
-        - name: ARM_CA_PRIVATE_KEY
-          valueFrom:
-            secretKeyRef:
-              key: CAPrivateKey
-              name: cluster-autoscaler-azure
-        - name: ARM_CLIENT_PRIVATE_KEY
-          valueFrom:
-            secretKeyRef:
-              key: ClientPrivateKey
-              name: cluster-autoscaler-azure
-        - name: ARM_KUBECONFIG_PRIVATE_KEY
-          valueFrom:
-            secretKeyRef:
-              key: KubeConfigPrivateKey
-              name: cluster-autoscaler-azure
-        - name: ARM_WINDOWS_ADMIN_PASSWORD
-          valueFrom:
-            secretKeyRef:
-              key: WindowsAdminPassword
-              name: cluster-autoscaler-azure
-        - name: ARM_ETCD_CLIENT_RPIVATE_KEY
-          valueFrom:
-            secretKeyRef:
-              key: EtcdClientPrivateKey
-              name: cluster-autoscaler-azure
-        - name: ARM_ETCD_SERVER_PRIVATE_KEY
-          valueFrom:
-            secretKeyRef:
-              key: EtcdServerPrivateKey
-              name: cluster-autoscaler-azure
         image: gcr.io/google_containers/cluster-autoscaler:{{ ca_version }}
         imagePullPolicy: Always
         name: cluster-autoscaler
@@ -242,10 +200,19 @@ spec:
         - mountPath: /etc/ssl/certs/ca-certificates.crt
           name: ssl-certs
           readOnly: true
+        - mountPath: /var/lib/azure/
+          name: deploy-parameters
+          readOnly: true
       dnsPolicy: ClusterFirst
       restartPolicy: Always
       volumes:
       - hostPath:
           path: /etc/ssl/certs/ca-certificates.crt
           type: ""
-        name: ssl-certs
+        name: ssl-certs
+      - name: deploy-parameters
+        secret:
+          secretName: cluster-autoscaler-azure-deploy-parameters
+          items:
+          - key: deploy-parameters
+            path: azuredeploy.parameters.json