From 60376f970e2b7508f24b8e4cf95ee491e5724ef1 Mon Sep 17 00:00:00 2001 From: Kubernetes Prow Robot Date: Mon, 21 Sep 2020 08:22:31 -0700 Subject: [PATCH] Merge pull request #3203 from detiber/configSplit2 [cluster-autoscaler] Support using --cloud-config for clusterapi provider --- .../cloudprovider/clusterapi/README.md | 45 ++++++-- .../clusterapi/clusterapi_provider.go | 24 +++- .../config/autoscaling_options.go | 3 + cluster-autoscaler/main.go | 106 +++++++++--------- 4 files changed, 112 insertions(+), 66 deletions(-) diff --git a/cluster-autoscaler/cloudprovider/clusterapi/README.md b/cluster-autoscaler/cloudprovider/clusterapi/README.md index 3f12c397fef4..ef0277f00228 100644 --- a/cluster-autoscaler/cloudprovider/clusterapi/README.md +++ b/cluster-autoscaler/cloudprovider/clusterapi/README.md @@ -10,14 +10,6 @@ cluster. The cluster-api provider requires Kubernetes v1.16 or greater to run the v1alpha3 version of the API. -## Cluster API Prerequisites - -Please be aware that currently the cluster autoscaler only supports CAPI -clusters that have joined their management and workload clusters into a single -cluster. For more information about this please see the -[Cluster API Concepts documentations](https://cluster-api.sigs.k8s.io/user/concepts.html) -and the [`clusterctl move` command documentation](https://cluster-api.sigs.k8s.io/user/concepts.html). - ## Starting the Autoscaler To enable the Cluster API provider, you must first specify it in the command @@ -62,6 +54,43 @@ in the staging namespace, belonging to the purple cluster, with the label owner= --node-group-auto-discovery=clusterapi:namespace=staging,clusterName=purple,owner=jim ``` +## Connecting cluster-autoscaler to Cluster API management and workload Clusters + +You will also need to provide the path to the kubeconfig(s) for the management +and workload cluster you wish cluster-autoscaler to run against. To specify the +kubeconfig path for the workload cluster to monitor, use the `--kubeconfig` +option and supply the path to the kubeconfig. If the `--kubeconfig` option is +not specified, cluster-autoscaler will attempt to use an in-cluster configuration. +To specify the kubeconfig path for the management cluster to monitor, use the +`--cloud-config` option and supply the path to the kubeconfig. If the +`--cloud-config` option is not specified it will fall back to using the kubeconfig +that was provided with the `--kubeconfig` option. + +Use in-cluster config for both management and workload cluster: +``` +cluster-autoscaler --cloud-provider=clusterapi +``` + +Use in-cluster config for workload cluster, specify kubeconfig for management cluster: +``` +cluster-autoscaler --cloud-provider=clusterapi --cloud-config=/mnt/kubeconfig +``` + +Use in-cluster config for management cluster, specify kubeconfig for workload cluster: +``` +cluster-autoscaler --cloud-provider=clusterapi --kubeconfig=/mnt/kubeconfig --clusterapi-cloud-config-authoritative +``` + +Use separate kubeconfigs for both management and workload cluster: +``` +cluster-autoscaler --cloud-provider=clusterapi --kubeconfig=/mnt/workload.kubeconfig --cloud-config=/mnt/management.kubeconfig +``` + +Use a single provided kubeconfig for both management and workload cluster: +``` +cluster-autoscaler --cloud-provider=clusterapi --kubeconfig=/mnt/workload.kubeconfig +``` + ## Enabling Autoscaling To enable the automatic scaling of components in your cluster-api managed diff --git a/cluster-autoscaler/cloudprovider/clusterapi/clusterapi_provider.go b/cluster-autoscaler/cloudprovider/clusterapi/clusterapi_provider.go index fbac1fd89f32..35a046e64e47 100644 --- a/cluster-autoscaler/cloudprovider/clusterapi/clusterapi_provider.go +++ b/cluster-autoscaler/cloudprovider/clusterapi/clusterapi_provider.go @@ -140,30 +140,42 @@ func newProvider( // BuildClusterAPI builds CloudProvider implementation for machine api. func BuildClusterAPI(opts config.AutoscalingOptions, do cloudprovider.NodeGroupDiscoveryOptions, rl *cloudprovider.ResourceLimiter) cloudprovider.CloudProvider { - externalConfig, err := clientcmd.BuildConfigFromFlags("", opts.KubeConfigPath) + managementKubeconfig := opts.CloudConfig + if managementKubeconfig == "" && !opts.ClusterAPICloudConfigAuthoritative { + managementKubeconfig = opts.KubeConfigPath + } + + managementConfig, err := clientcmd.BuildConfigFromFlags("", managementKubeconfig) + if err != nil { + klog.Fatalf("cannot build management cluster config: %v", err) + } + + workloadKubeconfig := opts.KubeConfigPath + + workloadConfig, err := clientcmd.BuildConfigFromFlags("", workloadKubeconfig) if err != nil { - klog.Fatalf("cannot build config: %v", err) + klog.Fatalf("cannot build workload cluster config: %v", err) } // Grab a dynamic interface that we can create informers from - managementClient, err := dynamic.NewForConfig(externalConfig) + managementClient, err := dynamic.NewForConfig(managementConfig) if err != nil { klog.Fatalf("could not generate dynamic client for config") } - workloadClient, err := kubernetes.NewForConfig(externalConfig) + workloadClient, err := kubernetes.NewForConfig(workloadConfig) if err != nil { klog.Fatalf("create kube clientset failed: %v", err) } - managementDiscoveryClient, err := discovery.NewDiscoveryClientForConfig(externalConfig) + managementDiscoveryClient, err := discovery.NewDiscoveryClientForConfig(managementConfig) if err != nil { klog.Fatalf("create discovery client failed: %v", err) } cachedDiscovery := memory.NewMemCacheClient(managementDiscoveryClient) managementScaleClient, err := scale.NewForConfig( - externalConfig, + managementConfig, restmapper.NewDeferredDiscoveryRESTMapper(cachedDiscovery), dynamic.LegacyAPIPathResolverFunc, scale.NewDiscoveryScaleKindResolver(managementDiscoveryClient)) diff --git a/cluster-autoscaler/config/autoscaling_options.go b/cluster-autoscaler/config/autoscaling_options.go index b7c5c4c6e939..7e0e85c7dffc 100644 --- a/cluster-autoscaler/config/autoscaling_options.go +++ b/cluster-autoscaler/config/autoscaling_options.go @@ -142,4 +142,7 @@ type AutoscalingOptions struct { AWSUseStaticInstanceList bool // Path to kube configuration if available KubeConfigPath string + // ClusterAPICloudConfigAuthoritative tells the Cluster API provider to treat the CloudConfig option as authoritative and + // not use KubeConfigPath as a fallback when it is not provided. + ClusterAPICloudConfigAuthoritative bool } diff --git a/cluster-autoscaler/main.go b/cluster-autoscaler/main.go index f40501e22ab3..723b73b7463a 100644 --- a/cluster-autoscaler/main.go +++ b/cluster-autoscaler/main.go @@ -169,10 +169,11 @@ var ( regional = flag.Bool("regional", false, "Cluster is regional.") newPodScaleUpDelay = flag.Duration("new-pod-scale-up-delay", 0*time.Second, "Pods less than this old will not be considered for scale-up.") - ignoreTaintsFlag = multiStringFlag("ignore-taint", "Specifies a taint to ignore in node templates when considering to scale a node group") - balancingIgnoreLabelsFlag = multiStringFlag("balancing-ignore-label", "Specifies a label to ignore in addition to the basic and cloud-provider set of labels when comparing if two node groups are similar") - awsUseStaticInstanceList = flag.Bool("aws-use-static-instance-list", false, "Should CA fetch instance types in runtime or use a static list. AWS only") - enableProfiling = flag.Bool("profiling", false, "Is debug/pprof endpoint enabled") + ignoreTaintsFlag = multiStringFlag("ignore-taint", "Specifies a taint to ignore in node templates when considering to scale a node group") + balancingIgnoreLabelsFlag = multiStringFlag("balancing-ignore-label", "Specifies a label to ignore in addition to the basic and cloud-provider set of labels when comparing if two node groups are similar") + awsUseStaticInstanceList = flag.Bool("aws-use-static-instance-list", false, "Should CA fetch instance types in runtime or use a static list. AWS only") + enableProfiling = flag.Bool("profiling", false, "Is debug/pprof endpoint enabled") + clusterAPICloudConfigAuthoritative = flag.Bool("clusterapi-cloud-config-authoritative", false, "Treat the cloud-config flag authoritatively (do not fallback to using kubeconfig flag). ClusterAPI only") ) func createAutoscalingOptions() config.AutoscalingOptions { @@ -193,54 +194,55 @@ func createAutoscalingOptions() config.AutoscalingOptions { klog.Fatalf("Failed to parse flags: %v", err) } return config.AutoscalingOptions{ - CloudConfig: *cloudConfig, - CloudProviderName: *cloudProviderFlag, - NodeGroupAutoDiscovery: *nodeGroupAutoDiscoveryFlag, - MaxTotalUnreadyPercentage: *maxTotalUnreadyPercentage, - OkTotalUnreadyCount: *okTotalUnreadyCount, - ScaleUpFromZero: *scaleUpFromZero, - EstimatorName: *estimatorFlag, - ExpanderName: *expanderFlag, - IgnoreDaemonSetsUtilization: *ignoreDaemonSetsUtilization, - IgnoreMirrorPodsUtilization: *ignoreMirrorPodsUtilization, - MaxBulkSoftTaintCount: *maxBulkSoftTaintCount, - MaxBulkSoftTaintTime: *maxBulkSoftTaintTime, - MaxEmptyBulkDelete: *maxEmptyBulkDeleteFlag, - MaxGracefulTerminationSec: *maxGracefulTerminationFlag, - MaxNodeProvisionTime: *maxNodeProvisionTime, - MaxNodesTotal: *maxNodesTotal, - MaxCoresTotal: maxCoresTotal, - MinCoresTotal: minCoresTotal, - MaxMemoryTotal: maxMemoryTotal, - MinMemoryTotal: minMemoryTotal, - GpuTotal: parsedGpuTotal, - NodeGroups: *nodeGroupsFlag, - ScaleDownDelayAfterAdd: *scaleDownDelayAfterAdd, - ScaleDownDelayAfterDelete: *scaleDownDelayAfterDelete, - ScaleDownDelayAfterFailure: *scaleDownDelayAfterFailure, - ScaleDownEnabled: *scaleDownEnabled, - ScaleDownUnneededTime: *scaleDownUnneededTime, - ScaleDownUnreadyTime: *scaleDownUnreadyTime, - ScaleDownUtilizationThreshold: *scaleDownUtilizationThreshold, - ScaleDownGpuUtilizationThreshold: *scaleDownGpuUtilizationThreshold, - ScaleDownNonEmptyCandidatesCount: *scaleDownNonEmptyCandidatesCount, - ScaleDownCandidatesPoolRatio: *scaleDownCandidatesPoolRatio, - ScaleDownCandidatesPoolMinCount: *scaleDownCandidatesPoolMinCount, - WriteStatusConfigMap: *writeStatusConfigMapFlag, - BalanceSimilarNodeGroups: *balanceSimilarNodeGroupsFlag, - ConfigNamespace: *namespace, - ClusterName: *clusterName, - NodeAutoprovisioningEnabled: *nodeAutoprovisioningEnabled, - MaxAutoprovisionedNodeGroupCount: *maxAutoprovisionedNodeGroupCount, - UnremovableNodeRecheckTimeout: *unremovableNodeRecheckTimeout, - ExpendablePodsPriorityCutoff: *expendablePodsPriorityCutoff, - Regional: *regional, - NewPodScaleUpDelay: *newPodScaleUpDelay, - IgnoredTaints: *ignoreTaintsFlag, - BalancingExtraIgnoredLabels: *balancingIgnoreLabelsFlag, - KubeConfigPath: *kubeConfigFile, - NodeDeletionDelayTimeout: *nodeDeletionDelayTimeout, - AWSUseStaticInstanceList: *awsUseStaticInstanceList, + CloudConfig: *cloudConfig, + CloudProviderName: *cloudProviderFlag, + NodeGroupAutoDiscovery: *nodeGroupAutoDiscoveryFlag, + MaxTotalUnreadyPercentage: *maxTotalUnreadyPercentage, + OkTotalUnreadyCount: *okTotalUnreadyCount, + ScaleUpFromZero: *scaleUpFromZero, + EstimatorName: *estimatorFlag, + ExpanderName: *expanderFlag, + IgnoreDaemonSetsUtilization: *ignoreDaemonSetsUtilization, + IgnoreMirrorPodsUtilization: *ignoreMirrorPodsUtilization, + MaxBulkSoftTaintCount: *maxBulkSoftTaintCount, + MaxBulkSoftTaintTime: *maxBulkSoftTaintTime, + MaxEmptyBulkDelete: *maxEmptyBulkDeleteFlag, + MaxGracefulTerminationSec: *maxGracefulTerminationFlag, + MaxNodeProvisionTime: *maxNodeProvisionTime, + MaxNodesTotal: *maxNodesTotal, + MaxCoresTotal: maxCoresTotal, + MinCoresTotal: minCoresTotal, + MaxMemoryTotal: maxMemoryTotal, + MinMemoryTotal: minMemoryTotal, + GpuTotal: parsedGpuTotal, + NodeGroups: *nodeGroupsFlag, + ScaleDownDelayAfterAdd: *scaleDownDelayAfterAdd, + ScaleDownDelayAfterDelete: *scaleDownDelayAfterDelete, + ScaleDownDelayAfterFailure: *scaleDownDelayAfterFailure, + ScaleDownEnabled: *scaleDownEnabled, + ScaleDownUnneededTime: *scaleDownUnneededTime, + ScaleDownUnreadyTime: *scaleDownUnreadyTime, + ScaleDownUtilizationThreshold: *scaleDownUtilizationThreshold, + ScaleDownGpuUtilizationThreshold: *scaleDownGpuUtilizationThreshold, + ScaleDownNonEmptyCandidatesCount: *scaleDownNonEmptyCandidatesCount, + ScaleDownCandidatesPoolRatio: *scaleDownCandidatesPoolRatio, + ScaleDownCandidatesPoolMinCount: *scaleDownCandidatesPoolMinCount, + WriteStatusConfigMap: *writeStatusConfigMapFlag, + BalanceSimilarNodeGroups: *balanceSimilarNodeGroupsFlag, + ConfigNamespace: *namespace, + ClusterName: *clusterName, + NodeAutoprovisioningEnabled: *nodeAutoprovisioningEnabled, + MaxAutoprovisionedNodeGroupCount: *maxAutoprovisionedNodeGroupCount, + UnremovableNodeRecheckTimeout: *unremovableNodeRecheckTimeout, + ExpendablePodsPriorityCutoff: *expendablePodsPriorityCutoff, + Regional: *regional, + NewPodScaleUpDelay: *newPodScaleUpDelay, + IgnoredTaints: *ignoreTaintsFlag, + BalancingExtraIgnoredLabels: *balancingIgnoreLabelsFlag, + KubeConfigPath: *kubeConfigFile, + NodeDeletionDelayTimeout: *nodeDeletionDelayTimeout, + AWSUseStaticInstanceList: *awsUseStaticInstanceList, + ClusterAPICloudConfigAuthoritative: *clusterAPICloudConfigAuthoritative, } }