From 9b50b39c29c3a6aa7b954538191bf1a8b0858bfe Mon Sep 17 00:00:00 2001 From: Pengfei Ni Date: Sat, 24 Feb 2018 15:52:48 +0800 Subject: [PATCH 1/2] Read deployment parameters from files instead of environment variables --- .../cloudprovider/azure/azure_agent_pool.go | 38 +----------------- .../cloudprovider/azure/azure_manager.go | 39 ++++++++----------- .../cloudprovider/azure/azure_util.go | 39 ++++++++++++------- 3 files changed, 42 insertions(+), 74 deletions(-) diff --git a/cluster-autoscaler/cloudprovider/azure/azure_agent_pool.go b/cluster-autoscaler/cloudprovider/azure/azure_agent_pool.go index 928d33ae9f25..e053315382a2 100644 --- a/cluster-autoscaler/cloudprovider/azure/azure_agent_pool.go +++ b/cluster-autoscaler/cloudprovider/azure/azure_agent_pool.go @@ -71,53 +71,17 @@ func NewAgentPool(spec *dynamic.NodeGroupSpec, az *AzureManager) (*AgentPool, er } func (as *AgentPool) initialize() error { - deploy, err := as.manager.azClient.deploymentsClient.Get(as.manager.config.ResourceGroup, as.manager.config.Deployment) - if err != nil { - glog.Errorf("deploymentsClient.Get(%s, %s) failed: %v", as.manager.config.ResourceGroup, as.manager.config.Deployment, err) - return err - } - template, err := as.manager.azClient.deploymentsClient.ExportTemplate(as.manager.config.ResourceGroup, as.manager.config.Deployment) if err != nil { glog.Errorf("deploymentsClient.ExportTemplate(%s, %s) failed: %v", as.manager.config.ResourceGroup, as.manager.config.Deployment, err) return err } - as.parameters = *deploy.Properties.Parameters - as.preprocessParameters() - as.template = *template.Template + as.parameters = as.manager.config.DeploymentParameters return normalizeForK8sVMASScalingUp(as.template) } -func (as *AgentPool) preprocessParameters() { - // Delete type key from parameters. - for k := range as.parameters { - if v, ok := as.parameters[k].(map[string]interface{}); ok { - delete(v, "type") - } - } - - // fulfill secure parameters. - as.parameters["apiServerPrivateKey"] = map[string]string{"value": as.manager.config.APIServerPrivateKey} - as.parameters["caPrivateKey"] = map[string]string{"value": as.manager.config.CAPrivateKey} - as.parameters["clientPrivateKey"] = map[string]string{"value": as.manager.config.ClientPrivateKey} - as.parameters["kubeConfigPrivateKey"] = map[string]string{"value": as.manager.config.KubeConfigPrivateKey} - as.parameters["servicePrincipalClientId"] = map[string]string{"value": as.manager.config.AADClientID} - as.parameters["servicePrincipalClientSecret"] = map[string]string{"value": as.manager.config.AADClientSecret} - if as.manager.config.WindowsAdminPassword != "" { - as.parameters["windowsAdminPassword"] = map[string]string{"value": as.manager.config.WindowsAdminPassword} - } - - // etcd TLS parameters (for acs-engine >= v0.12.0). - if as.manager.config.EtcdClientPrivateKey != "" { - as.parameters["etcdClientPrivateKey"] = map[string]string{"value": as.manager.config.EtcdClientPrivateKey} - } - if as.manager.config.EtcdServerPrivateKey != "" { - as.parameters["etcdServerPrivateKey"] = map[string]string{"value": as.manager.config.EtcdServerPrivateKey} - } -} - // MinSize returns minimum size of the node group. func (as *AgentPool) MinSize() int { return as.minSize diff --git a/cluster-autoscaler/cloudprovider/azure/azure_manager.go b/cluster-autoscaler/cloudprovider/azure/azure_manager.go index 715ecb308aaa..a11c7275a67a 100644 --- a/cluster-autoscaler/cloudprovider/azure/azure_manager.go +++ b/cluster-autoscaler/cloudprovider/azure/azure_manager.go @@ -38,6 +38,9 @@ const ( scaleToZeroSupported = false refreshInterval = 1 * time.Minute + + // The path of deployment parameters for standard vm. + deploymentParametersPath = "/var/lib/azure/azuredeploy.parameters.json" ) // AzureManager handles Azure communication and data caching. @@ -67,15 +70,8 @@ type Config struct { UseManagedIdentityExtension bool `json:"useManagedIdentityExtension" yaml:"useManagedIdentityExtension"` // Configs only for standard vmType (agent pools). - Deployment string `json:"deployment" yaml:"deployment"` - APIServerPrivateKey string `json:"apiServerPrivateKey" yaml:"apiServerPrivateKey"` - CAPrivateKey string `json:"caPrivateKey" yaml:"caPrivateKey"` - ClientPrivateKey string `json:"clientPrivateKey" yaml:"clientPrivateKey"` - KubeConfigPrivateKey string `json:"kubeConfigPrivateKey" yaml:"kubeConfigPrivateKey"` - WindowsAdminPassword string `json:"windowsAdminPassword" yaml:"windowsAdminPassword"` - // etcd TLS parameters (for acs-engine >= v0.12.0). - EtcdClientPrivateKey string `json:"etcdClientPrivateKey" yaml:"etcdClientPrivateKey"` - EtcdServerPrivateKey string `json:"etcdServerPrivateKey" yaml:"etcdServerPrivateKey"` + Deployment string `json:"deployment" yaml:"deployment"` + DeploymentParameters map[string]interface{} `json:"deploymentParameters" yaml:"deploymentParameters"` } // TrimSpace removes all leading and trailing white spaces. @@ -90,13 +86,6 @@ func (c *Config) TrimSpace() { c.AADClientCertPath = strings.TrimSpace(c.AADClientCertPath) c.AADClientCertPassword = strings.TrimSpace(c.AADClientCertPassword) c.Deployment = strings.TrimSpace(c.Deployment) - c.APIServerPrivateKey = strings.TrimSpace(c.APIServerPrivateKey) - c.CAPrivateKey = strings.TrimSpace(c.CAPrivateKey) - c.ClientPrivateKey = strings.TrimSpace(c.ClientPrivateKey) - c.KubeConfigPrivateKey = strings.TrimSpace(c.KubeConfigPrivateKey) - c.WindowsAdminPassword = strings.TrimSpace(c.WindowsAdminPassword) - c.EtcdClientPrivateKey = strings.TrimSpace(c.EtcdClientPrivateKey) - c.EtcdServerPrivateKey = strings.TrimSpace(c.EtcdServerPrivateKey) } // CreateAzureManager creates Azure Manager object to work with Azure. @@ -120,13 +109,6 @@ func CreateAzureManager(configReader io.Reader, discoveryOpts cloudprovider.Node cfg.AADClientCertPath = os.Getenv("ARM_CLIENT_CERT_PATH") cfg.AADClientCertPassword = os.Getenv("ARM_CLIENT_CERT_PASSWORD") cfg.Deployment = os.Getenv("ARM_DEPLOYMENT") - cfg.APIServerPrivateKey = os.Getenv("ARM_APISEVER_PRIVATE_KEY") - cfg.CAPrivateKey = os.Getenv("ARM_CA_PRIVATE_KEY") - cfg.ClientPrivateKey = os.Getenv("ARM_CLIENT_PRIVATE_KEY") - cfg.KubeConfigPrivateKey = os.Getenv("ARM_KUBECONFIG_PRIVATE_KEY") - cfg.WindowsAdminPassword = os.Getenv("ARM_WINDOWS_ADMIN_PASSWORD") - cfg.EtcdClientPrivateKey = os.Getenv("ARM_ETCD_CLIENT_RPIVATE_KEY") - cfg.EtcdServerPrivateKey = os.Getenv("ARM_ETCD_SERVER_PRIVATE_KEY") useManagedIdentityExtensionFromEnv := os.Getenv("ARM_USE_MANAGED_IDENTITY_EXTENSION") if len(useManagedIdentityExtensionFromEnv) > 0 { @@ -143,6 +125,17 @@ func CreateAzureManager(configReader io.Reader, discoveryOpts cloudprovider.Node cfg.VMType = vmTypeVMSS } + // Read parameters from deploymentParametersPath if it is not set. + if cfg.VMType == vmTypeStandard && len(cfg.DeploymentParameters) == 0 { + parameters, err := readDeploymentParameters(deploymentParametersPath) + if err != nil { + glog.Errorf("readDeploymentParameters failed with error: %v", err) + return nil, err + } + + cfg.DeploymentParameters = parameters + } + // Defaulting env to Azure Public Cloud. env := azure.PublicCloud if cfg.Cloud != "" { diff --git a/cluster-autoscaler/cloudprovider/azure/azure_util.go b/cluster-autoscaler/cloudprovider/azure/azure_util.go index 38827a834f79..2f2ff5e41a74 100644 --- a/cluster-autoscaler/cloudprovider/azure/azure_util.go +++ b/cluster-autoscaler/cloudprovider/azure/azure_util.go @@ -19,7 +19,9 @@ package azure import ( "crypto/rsa" "crypto/x509" + "encoding/json" "fmt" + "io/ioutil" "net/url" "regexp" "sort" @@ -398,20 +400,8 @@ func validateConfig(cfg *Config) error { return fmt.Errorf("deployment not set") } - if cfg.APIServerPrivateKey == "" { - return fmt.Errorf("apiServerPrivateKey not set") - } - - if cfg.CAPrivateKey == "" { - return fmt.Errorf("caPrivateKey not set") - } - - if cfg.ClientPrivateKey == "" { - return fmt.Errorf("clientPrivateKey not set") - } - - if cfg.KubeConfigPrivateKey == "" { - return fmt.Errorf("kubeConfigPrivateKey not set") + if len(cfg.DeploymentParameters) == 0 { + return fmt.Errorf("deploymentParameters not set") } } @@ -428,3 +418,24 @@ func getLastSegment(ID string) (string, error) { return name, nil } + +// readDeploymentParameters gets deployment parameters from paramFilePath. +func readDeploymentParameters(paramFilePath string) (map[string]interface{}, error) { + contents, err := ioutil.ReadFile(paramFilePath) + if err != nil { + glog.Errorf("Failed to read deployment parameters from file %q: %v", paramFilePath, err) + return nil, err + } + + deploymentParameters := make(map[string]interface{}) + if err := json.Unmarshal(contents, &deploymentParameters); err != nil { + glog.Errorf("Failed to unmarshal deployment parameters from file %q: %v", paramFilePath, err) + return nil, err + } + + if v, ok := deploymentParameters["parameters"]; ok { + return v.(map[string]interface{}), nil + } + + return nil, fmt.Errorf("failed to get deployment parameters from file %s", paramFilePath) +} From 91f6bfdd79fcd4f781a4bf155849de7bc9112e3a Mon Sep 17 00:00:00 2001 From: Pengfei Ni Date: Sat, 24 Feb 2018 15:53:29 +0800 Subject: [PATCH 2/2] Update documentation and manifests --- .../cloudprovider/azure/README.md | 26 ++++----- .../cluster-autoscaler-standard-master.yaml | 53 ++++--------------- .../azure/cluster-autoscaler-standard.yaml | 53 ++++--------------- 3 files changed, 30 insertions(+), 102 deletions(-) diff --git a/cluster-autoscaler/cloudprovider/azure/README.md b/cluster-autoscaler/cloudprovider/azure/README.md index 300dc519d967..0e3f6690fed2 100644 --- a/cluster-autoscaler/cloudprovider/azure/README.md +++ b/cluster-autoscaler/cloudprovider/azure/README.md @@ -36,7 +36,7 @@ Fill the values of cluster-autoscaler-azure secret in [cluster-autoscaler-vmss.y - SubscriptionID: `` - TenantID: `` -Note that all data should be encoded with base64. +> Note that all data above should be encoded with base64. And fill the node groups in container command by `--nodes`, e.g. @@ -68,12 +68,8 @@ kubectl create -f cluster-autoscaler-vmss-master.yaml Pre-requirements: - Get credentials from above `permissions` step. -- Get the required parameters from acs-engine deployments (usually under directory `_output/` after running `acs-engine deploy` command) - - Get `APIServerPrivateKey`, `CAPrivateKey`, `ClientPrivateKey` and `KubeConfigPrivateKey` from `azuredeploy.parameters.json` - - Get `EtcdClientPrivateKey` and `EtcdServerPrivateKey` if the cluster is deployed by acs-engine >= v0.12.0 - - If windows nodes are included, also get `WindowsAdminPassword` from acs-engine deployment manifests - - Get the initial Azure deployment name from azure portal. If you have multiple deployments (e.g. have run `acs-engine scale` command), make sure to get the first one - - Get a node pool name for nodes scaling from acs-engine deployment manifests +- Get the initial Azure deployment name from azure portal. If you have multiple deployments (e.g. have run `acs-engine scale` command), make sure to get the first one. +- Get a node pool name for nodes scaling from acs-engine deployment manifests - Encode each data with base64. Fill the values of cluster-autoscaler-azure secret in [cluster-autoscaler-standard-master.yaml](cluster-autoscaler-standard-master.yaml), including @@ -83,17 +79,9 @@ Fill the values of cluster-autoscaler-azure secret in [cluster-autoscaler-standa - ResourceGroup: `` - SubscriptionID: `` - TenantID: `` -- NodeGroup: `` - Deployment: `` -- APIServerPrivateKey: `` -- CAPrivateKey: `` -- ClientPrivateKey: `` -- KubeConfigPrivateKey: `` -- WindowsAdminPassword: `` (set `""` if no windows nodes in the cluster) -- EtcdClientPrivateKey: `` (set `""` for acs-engine < v0.12.0) -- EtcdServerPrivateKey: `` (set to `""` for acs-engine < v0.12.0) -Note that all data should be encoded with base64. +> Note that all data above should be encoded with base64. And fill the node groups in container command by `--nodes`, e.g. @@ -108,6 +96,12 @@ or multiple node groups: - --nodes=1:10:agentpool2 ``` +Create Azure deploy parameters secret `cluster-autoscaler-azure-deploy-parameters` by running + +```sh +kubectl -n kube-system create secret generic cluster-autoscaler-azure-deploy-parameters --from-file=deploy-parameters=./_output//azuredeploy.parameters.json +``` + Then deploy cluster-autoscaler by running ```sh diff --git a/cluster-autoscaler/cloudprovider/azure/cluster-autoscaler-standard-master.yaml b/cluster-autoscaler/cloudprovider/azure/cluster-autoscaler-standard-master.yaml index 38c1dae7706c..48d5d1983a59 100644 --- a/cluster-autoscaler/cloudprovider/azure/cluster-autoscaler-standard-master.yaml +++ b/cluster-autoscaler/cloudprovider/azure/cluster-autoscaler-standard-master.yaml @@ -110,13 +110,6 @@ data: SubscriptionID: TenantID: Deployment: - APIServerPrivateKey: - CAPrivateKey: - ClientPrivateKey: - KubeConfigPrivateKey: - WindowsAdminPassword: - EtcdClientPrivateKey: - EtcdServerPrivateKey: VMType: c3RhbmRhcmQ= kind: Secret metadata: @@ -193,41 +186,6 @@ spec: secretKeyRef: key: Deployment name: cluster-autoscaler-azure - - name: ARM_APISEVER_PRIVATE_KEY - valueFrom: - secretKeyRef: - key: APIServerPrivateKey - name: cluster-autoscaler-azure - - name: ARM_CA_PRIVATE_KEY - valueFrom: - secretKeyRef: - key: CAPrivateKey - name: cluster-autoscaler-azure - - name: ARM_CLIENT_PRIVATE_KEY - valueFrom: - secretKeyRef: - key: ClientPrivateKey - name: cluster-autoscaler-azure - - name: ARM_KUBECONFIG_PRIVATE_KEY - valueFrom: - secretKeyRef: - key: KubeConfigPrivateKey - name: cluster-autoscaler-azure - - name: ARM_WINDOWS_ADMIN_PASSWORD - valueFrom: - secretKeyRef: - key: WindowsAdminPassword - name: cluster-autoscaler-azure - - name: ARM_ETCD_CLIENT_RPIVATE_KEY - valueFrom: - secretKeyRef: - key: EtcdClientPrivateKey - name: cluster-autoscaler-azure - - name: ARM_ETCD_SERVER_PRIVATE_KEY - valueFrom: - secretKeyRef: - key: EtcdServerPrivateKey - name: cluster-autoscaler-azure image: gcr.io/google_containers/cluster-autoscaler:{{ ca_version }} imagePullPolicy: Always name: cluster-autoscaler @@ -242,10 +200,19 @@ spec: - mountPath: /etc/ssl/certs/ca-certificates.crt name: ssl-certs readOnly: true + - mountPath: /var/lib/azure/ + name: deploy-parameters + readOnly: true dnsPolicy: ClusterFirst restartPolicy: Always volumes: - hostPath: path: /etc/ssl/certs/ca-certificates.crt type: "" - name: ssl-certs \ No newline at end of file + name: ssl-certs + - name: deploy-parameters + secret: + secretName: cluster-autoscaler-azure-deploy-parameters + items: + - key: deploy-parameters + path: azuredeploy.parameters.json \ No newline at end of file diff --git a/cluster-autoscaler/cloudprovider/azure/cluster-autoscaler-standard.yaml b/cluster-autoscaler/cloudprovider/azure/cluster-autoscaler-standard.yaml index 9c3a9f39c3cd..5d30cd89f8ed 100644 --- a/cluster-autoscaler/cloudprovider/azure/cluster-autoscaler-standard.yaml +++ b/cluster-autoscaler/cloudprovider/azure/cluster-autoscaler-standard.yaml @@ -110,13 +110,6 @@ data: SubscriptionID: TenantID: Deployment: - APIServerPrivateKey: - CAPrivateKey: - ClientPrivateKey: - KubeConfigPrivateKey: - WindowsAdminPassword: - EtcdClientPrivateKey: - EtcdServerPrivateKey: VMType: c3RhbmRhcmQ= kind: Secret metadata: @@ -186,41 +179,6 @@ spec: secretKeyRef: key: Deployment name: cluster-autoscaler-azure - - name: ARM_APISEVER_PRIVATE_KEY - valueFrom: - secretKeyRef: - key: APIServerPrivateKey - name: cluster-autoscaler-azure - - name: ARM_CA_PRIVATE_KEY - valueFrom: - secretKeyRef: - key: CAPrivateKey - name: cluster-autoscaler-azure - - name: ARM_CLIENT_PRIVATE_KEY - valueFrom: - secretKeyRef: - key: ClientPrivateKey - name: cluster-autoscaler-azure - - name: ARM_KUBECONFIG_PRIVATE_KEY - valueFrom: - secretKeyRef: - key: KubeConfigPrivateKey - name: cluster-autoscaler-azure - - name: ARM_WINDOWS_ADMIN_PASSWORD - valueFrom: - secretKeyRef: - key: WindowsAdminPassword - name: cluster-autoscaler-azure - - name: ARM_ETCD_CLIENT_RPIVATE_KEY - valueFrom: - secretKeyRef: - key: EtcdClientPrivateKey - name: cluster-autoscaler-azure - - name: ARM_ETCD_SERVER_PRIVATE_KEY - valueFrom: - secretKeyRef: - key: EtcdServerPrivateKey - name: cluster-autoscaler-azure image: gcr.io/google_containers/cluster-autoscaler:{{ ca_version }} imagePullPolicy: Always name: cluster-autoscaler @@ -235,10 +193,19 @@ spec: - mountPath: /etc/ssl/certs/ca-certificates.crt name: ssl-certs readOnly: true + - mountPath: /var/lib/azure/ + name: deploy-parameters + readOnly: true dnsPolicy: ClusterFirst restartPolicy: Always volumes: - hostPath: path: /etc/ssl/certs/ca-certificates.crt type: "" - name: ssl-certs \ No newline at end of file + name: ssl-certs + - name: deploy-parameters + secret: + secretName: cluster-autoscaler-azure-deploy-parameters + items: + - key: deploy-parameters + path: azuredeploy.parameters.json \ No newline at end of file