Merge pull request #1465 from DualSpark/k-c-m-attachedetachflag

Updates for new k-c-m flag

pkg/apis/kops/componentconfig.go

@@ -16,6 +16,8 @@ limitations under the License.
 
 package kops
 
+import metav1 "k8s.io/kubernetes/pkg/apis/meta/v1"
+
 type KubeletConfigSpec struct {
 	APIServers string `json:"apiServers,omitempty" flag:"api-servers"`
 
@@ -533,6 +535,10 @@ type KubeControllerManagerConfig struct {
 	//// corresponding flag of the kube-apiserver. WARNING: the generic garbage
 	//// collector is an alpha feature.
 	//EnableGarbageCollector bool `json:"enableGarbageCollector"`
+
+	// ReconcilerSyncLoopPeriod is the amount of time the reconciler sync states loop
+	// wait between successive executions. Is set to 1 min by kops by default
+	AttachDetachReconcileSyncPeriod *metav1.Duration `json:"attachDetachReconcileSyncPeriod,omitempty" flag:"attach-detach-reconcile-sync-period"`
 }
 
 type KubeSchedulerConfig struct {

pkg/apis/kops/v1alpha1/componentconfig.go

@@ -16,6 +16,8 @@ limitations under the License.
 
 package v1alpha1
 
+import metav1 "k8s.io/kubernetes/pkg/apis/meta/v1"
+
 type KubeletConfigSpec struct {
 	APIServers string `json:"apiServers,omitempty" flag:"api-servers"`
 
@@ -530,6 +532,10 @@ type KubeControllerManagerConfig struct {
 	//// corresponding flag of the kube-apiserver. WARNING: the generic garbage
 	//// collector is an alpha feature.
 	//EnableGarbageCollector bool `json:"enableGarbageCollector"`
+
+	// ReconcilerSyncLoopPeriod is the amount of time the reconciler sync states loop
+	// wait between successive executions. Is set to 1 min by kops by default
+	AttachDetachReconcileSyncPeriod *metav1.Duration `json:"attachDetachReconcileSyncPeriod,omitempty" flag:"attach-detach-reconcile-sync-period"`
 }
 
 type KubeSchedulerConfig struct {

pkg/apis/kops/v1alpha1/zz_generated.conversion.go

@@ -1013,6 +1013,7 @@ func autoConvert_v1alpha1_KubeControllerManagerConfig_To_kops_KubeControllerMana
 	} else {
 		out.LeaderElection = nil
 	}
+	out.AttachDetachReconcileSyncPeriod = in.AttachDetachReconcileSyncPeriod
 	return nil
 }
 
@@ -1041,6 +1042,7 @@ func autoConvert_kops_KubeControllerManagerConfig_To_v1alpha1_KubeControllerMana
 	} else {
 		out.LeaderElection = nil
 	}
+	out.AttachDetachReconcileSyncPeriod = in.AttachDetachReconcileSyncPeriod
 	return nil
 }
 

pkg/apis/kops/v1alpha2/componentconfig.go

@@ -16,6 +16,8 @@ limitations under the License.
 
 package v1alpha2
 
+import metav1 "k8s.io/kubernetes/pkg/apis/meta/v1"
+
 type KubeletConfigSpec struct {
 	APIServers string `json:"apiServers,omitempty" flag:"api-servers"`
 
@@ -170,6 +172,9 @@ type KubeControllerManagerConfig struct {
 	RootCAFile string `json:"rootCAFile,omitempty" flag:"root-ca-file"`
 	// leaderElection defines the configuration of leader election client.
 	LeaderElection *LeaderElectionConfiguration `json:"leaderElection,omitempty"`
+	// ReconcilerSyncLoopPeriod is the amount of time the reconciler sync states loop
+	// wait between successive executions. Is set to 1 min by kops by default
+	AttachDetachReconcileSyncPeriod *metav1.Duration `json:"attachDetachReconcileSyncPeriod,omitempty" flag:"attach-detach-reconcile-sync-period"`
 }
 
 type KubeSchedulerConfig struct {

pkg/flagbuilder/build_flags.go

@@ -18,11 +18,13 @@ package flagbuilder
 
 import (
 	"fmt"
-	"github.com/golang/glog"
-	"k8s.io/kops/upup/pkg/fi/utils"
 	"reflect"
 	"sort"
 	"strings"
+
+	"github.com/golang/glog"
+	"k8s.io/kops/upup/pkg/fi/utils"
+	metav1 "k8s.io/kubernetes/pkg/apis/meta/v1"
 )
 
 // BuildFlags builds flag arguments based on "flag" tags on the structure
@@ -104,6 +106,9 @@ func BuildFlags(options interface{}) (string, error) {
 		case bool, int, int32, int64, float32, float64:
 			vString := fmt.Sprintf("%v", v)
 			flag = fmt.Sprintf("--%s=%s", flagName, vString)
+		case metav1.Duration:
+			vString := v.Duration.String()
+			flag = fmt.Sprintf("--%s=%s", flagName, vString)
 
 		default:
 			return fmt.Errorf("BuildFlags of value type not handled: %T %s=%v", v, path, v)

pkg/model/components/kubecontrollermanager.go

@@ -0,0 +1,104 @@
+/*
+Copyright 2017 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package components
+
+import (
+	"fmt"
+	"time"
+
+	"github.com/golang/glog"
+	"k8s.io/kops/pkg/apis/kops"
+	"k8s.io/kops/upup/pkg/fi/loader"
+	metav1 "k8s.io/kubernetes/pkg/apis/meta/v1"
+)
+
+const (
+	defaultAttachDetachReconcileSyncPeriod = time.Minute
+)
+
+// KubeControllerManagerOptionsBuilder adds options for the kubernetes controller manager to the model.
+type KubeControllerManagerOptionsBuilder struct {
+	Context *OptionsContext
+}
+
+var _ loader.OptionsBuilder = &KubeControllerManagerOptionsBuilder{}
+
+// BuildOptions generates the configurations used to create kubernetes controller manager manifest
+func (b *KubeControllerManagerOptionsBuilder) BuildOptions(o interface{}) error {
+
+	options := o.(*kops.ClusterSpec)
+
+	if options.KubeControllerManager == nil {
+		options.KubeControllerManager = &kops.KubeControllerManagerConfig{}
+	}
+
+	k8sv148, err := kops.ParseKubernetesVersion("v1.4.8")
+
+	if err != nil {
+		return fmt.Errorf("Unable to parse kubernetesVersion %s", err)
+	}
+
+	k8sv152, err := kops.ParseKubernetesVersion("v1.5.2")
+
+	if err != nil {
+		return fmt.Errorf("Unable to parse kubernetesVersion %s", err)
+	}
+
+	if err != nil {
+		return fmt.Errorf("Unable to parse kubernetesVersion %s", err)
+	}
+
+	kubernetesVersion, err := b.Context.KubernetesVersion()
+	if err != nil {
+		return fmt.Errorf("Unable to parse kubernetesVersion %s", err)
+	}
+
+	// In 1.4.8+ and 1.5.2+ k8s added the capability to tune the duration upon which the volume attach detach
+	// component is called.
+	// See https://github.com/kubernetes/kubernetes/pull/39551
+	// TLDR; set this too low, and have a few EBS Volumes, and you will spam AWS api
+
+	// if 1.4.8+ and 1.5.2+
+	if kubernetesVersion.GTE(*k8sv148) || kubernetesVersion.GTE(*k8sv152) {
+
+		glog.V(4).Infof("Kubernetes version %q supports AttachDetachReconcileSyncPeriod; will configure", kubernetesVersion)
+		// If not set ... or set to 0s ... which is stupid
+		if options.KubeControllerManager.AttachDetachReconcileSyncPeriod == nil ||
+			options.KubeControllerManager.AttachDetachReconcileSyncPeriod.Duration.String() == "0s" {
+
+			glog.V(8).Infof("AttachDetachReconcileSyncPeriod is not set; will set to default %v", defaultAttachDetachReconcileSyncPeriod)
+			options.KubeControllerManager.AttachDetachReconcileSyncPeriod = &metav1.Duration{Duration: defaultAttachDetachReconcileSyncPeriod}
+
+			// If less than 1 min and greater than 1 sec ... you get a warning
+		} else if options.KubeControllerManager.AttachDetachReconcileSyncPeriod.Duration < defaultAttachDetachReconcileSyncPeriod &&
+			options.KubeControllerManager.AttachDetachReconcileSyncPeriod.Duration > time.Second {
+
+			glog.Infof("KubeControllerManager AttachDetachReconcileSyncPeriod is set lower than recommended: %s", defaultAttachDetachReconcileSyncPeriod)
+
+			// If less than 1sec you get an error.  Controller is coded to not allow configuration
+			// less than one second.
+		} else if options.KubeControllerManager.AttachDetachReconcileSyncPeriod.Duration < time.Second {
+			return fmt.Errorf("AttachDetachReconcileSyncPeriod cannot be set to less than 1 second")
+		}
+	} else {
+
+		glog.V(4).Infof("not setting AttachDetachReconcileSyncPeriod, k8s version is too low")
+		options.KubeControllerManager.AttachDetachReconcileSyncPeriod = nil
+	}
+
+	return nil
+}

pkg/model/components/kubecontrollermanager_test.go

@@ -0,0 +1,85 @@
+/*
+Copyright 2017 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package components
+
+import (
+	"testing"
+	"time"
+
+	api "k8s.io/kops/pkg/apis/kops"
+)
+
+type ClusterParams struct {
+	CloudProvider     string
+	KubernetesVersion string
+	UpdatePolicy      string
+}
+
+func buildCluster() *api.Cluster {
+
+	return &api.Cluster{
+		Spec: api.ClusterSpec{
+			CloudProvider:     "aws",
+			KubernetesVersion: "v1.4.0",
+		},
+	}
+}
+
+func Test_Build_KCM_Builder_Lower_Version(t *testing.T) {
+	c := buildCluster()
+
+	kcm := &KubeControllerManagerOptionsBuilder{
+		Context: &OptionsContext{
+			Cluster: c,
+		},
+	}
+
+	spec := c.Spec
+	err := kcm.BuildOptions(&spec)
+
+	if err != nil {
+		t.Fatalf("unexpected error from BuildOptions: %v", err)
+	}
+
+	if spec.KubeControllerManager.AttachDetachReconcileSyncPeriod != nil {
+		t.Fatalf("AttachDetachReconcileSyncPeriod should not be set for old kubernetes version %s", spec.KubernetesVersion)
+	}
+
+}
+
+func Test_Build_KCM_Builder_High_Enough_Version(t *testing.T) {
+	c := buildCluster()
+	c.Spec.KubernetesVersion = "1.4.8"
+
+	kcm := &KubeControllerManagerOptionsBuilder{
+		Context: &OptionsContext{
+			Cluster: c,
+		},
+	}
+
+	spec := c.Spec
+	err := kcm.BuildOptions(&spec)
+
+	if err != nil {
+		t.Fatalf("unexpected error from BuildOptions %s", err)
+	}
+
+	if spec.KubeControllerManager.AttachDetachReconcileSyncPeriod.Duration != time.Minute {
+		t.Fatalf("AttachDetachReconcileSyncPeriod should be set to 1m - %s", spec.KubeControllerManager.AttachDetachReconcileSyncPeriod.Duration.String())
+	}
+
+}

upup/pkg/fi/cloudup/populate_cluster_spec.go

@@ -266,6 +266,7 @@ func (c *populateClusterSpec) run() error {
 			codeModels = append(codeModels, &components.DockerOptionsBuilder{Context: optionsContext})
 			codeModels = append(codeModels, &components.NetworkingOptionsBuilder{Context: optionsContext})
 			codeModels = append(codeModels, &components.KubeletOptionsBuilder{Context: optionsContext})
+			codeModels = append(codeModels, &components.KubeControllerManagerOptionsBuilder{Context: optionsContext})
 			fileModels = append(fileModels, m)
 
 		default:

upup/pkg/fi/cloudup/populatecluster_test.go

@@ -450,3 +450,46 @@ func TestPopulateCluster_DockerVersion(t *testing.T) {
 		}
 	}
 }
+
+func TestPopulateCluster_KubeController_High_Enough_Version(t *testing.T) {
+	c := buildMinimalCluster()
+	c.Spec.KubernetesVersion = "v1.5.2"
+
+	err := PerformAssignments(c)
+	if err != nil {
+		t.Fatalf("error from PerformAssignments: %v", err)
+	}
+
+	addEtcdClusters(c)
+
+	full, err := PopulateClusterSpec(c)
+	if err != nil {
+		t.Fatalf("Unexpected error from PopulateCluster: %v", err)
+	}
+
+	if full.Spec.KubeControllerManager.AttachDetachReconcileSyncPeriod == nil {
+		t.Fatalf("AttachDetachReconcileSyncPeriod not set correctly")
+	}
+
+}
+
+func TestPopulateCluster_KubeController_Fail(t *testing.T) {
+	c := buildMinimalCluster()
+	c.Spec.KubernetesVersion = "1.4.7"
+
+	err := PerformAssignments(c)
+	if err != nil {
+		t.Fatalf("error from PerformAssignments: %v", err)
+	}
+
+	addEtcdClusters(c)
+
+	full, err := PopulateClusterSpec(c)
+	if err != nil {
+		t.Fatalf("Unexpected error from PopulateCluster: %v", err)
+	}
+
+	if full.Spec.KubeControllerManager.AttachDetachReconcileSyncPeriod != nil {
+		t.Fatalf("AttachDetachReconcileSyncPeriodh is not supported in 1.4.7")
+	}
+}