Merge pull request #10661 from hakman/etcd-manager-defaults

Update AWS etcd-manager volumes defaults
kubernetes · Jan 29, 2021 · 3d39be7 · 3d39be7
2 parents 6fbc742 + 4c5d7dd
commit 3d39be7
Show file tree

Hide file tree

Showing 62 changed files with 486 additions and 182 deletions.
diff --git a/cmd/kops/create_cluster.go b/cmd/kops/create_cluster.go
@@ -167,6 +167,7 @@ func NewCmdCreateCluster(f *util.Factory, out io.Writer) *cobra.Command {
 
 	sshPublicKey := ""
 	associatePublicIP := false
+	encryptEtcdStorage := false
 
 	cmd := &cobra.Command{
 		Use:     "cluster",
@@ -180,6 +181,10 @@ func NewCmdCreateCluster(f *util.Factory, out io.Writer) *cobra.Command {
 				options.AssociatePublicIP = &associatePublicIP
 			}
 
+			if cmd.Flag("encrypt-etcd-storage").Changed {
+				options.EncryptEtcdStorage = &encryptEtcdStorage
+			}
+
 			err := rootCommand.ProcessArgs(args)
 			if err != nil {
 				exitWithError(err)
@@ -244,7 +249,7 @@ func NewCmdCreateCluster(f *util.Factory, out io.Writer) *cobra.Command {
 	cmd.Flags().StringVar(&options.NetworkCIDR, "network-cidr", options.NetworkCIDR, "Set to override the default network CIDR")
 	cmd.Flags().BoolVar(&options.DisableSubnetTags, "disable-subnet-tags", options.DisableSubnetTags, "Set to disable automatic subnet tagging")
 
-	cmd.Flags().BoolVar(&options.EncryptEtcdStorage, "encrypt-etcd-storage", options.EncryptEtcdStorage, "Generate key in aws kms and use it for encrypt etcd volumes")
+	cmd.Flags().BoolVar(&encryptEtcdStorage, "encrypt-etcd-storage", false, "Generate key in aws kms and use it for encrypt etcd volumes")
 	cmd.Flags().StringVar(&options.EtcdStorageType, "etcd-storage-type", options.EtcdStorageType, "The default storage type for etc members")
 
 	cmd.Flags().StringVar(&options.Networking, "networking", options.Networking, "Networking mode to use.  kubenet, external, weave, flannel-vxlan (or flannel), flannel-udp, calico, canal, kube-router, amazonvpc, cilium, cilium-etcd, cni, lyftvpc.")

diff --git a/pkg/model/master_volumes.go b/pkg/model/master_volumes.go
@@ -40,7 +40,7 @@ import (
 
 const (
 	DefaultEtcdVolumeSize             = 20
-	DefaultAWSEtcdVolumeType          = "gp2"
+	DefaultAWSEtcdVolumeType          = "gp3"
 	DefaultAWSEtcdVolumeIonIops       = 100
 	DefaultAWSEtcdVolumeGp3Iops       = 3000
 	DefaultAWSEtcdVolumeGp3Throughput = 125
@@ -123,6 +123,9 @@ func (b *MasterVolumeBuilder) Build(c *fi.ModelBuilderContext) error {
 
 func (b *MasterVolumeBuilder) addAWSVolume(c *fi.ModelBuilderContext, name string, volumeSize int32, zone string, etcd kops.EtcdClusterSpec, m kops.EtcdMemberSpec, allMembers []string) error {
 	volumeType := fi.StringValue(m.VolumeType)
+	if volumeType == "" {
+		volumeType = DefaultAWSEtcdVolumeType
+	}
 	volumeIops := fi.Int32Value(m.VolumeIops)
 	volumeThroughput := fi.Int32Value(m.VolumeThroughput)
 	switch volumeType {
@@ -138,7 +141,7 @@ func (b *MasterVolumeBuilder) addAWSVolume(c *fi.ModelBuilderContext, name strin
 			volumeThroughput = DefaultAWSEtcdVolumeGp3Throughput
 		}
 	default:
-		volumeType = DefaultAWSEtcdVolumeType
+		return fmt.Errorf("unknown volume type %q", volumeType)
 	}
 
 	// The tags are how protokube knows to mount the volume and use it for etcd

diff --git a/tests/integration/create_cluster/complex/expected-v1alpha2.yaml b/tests/integration/create_cluster/complex/expected-v1alpha2.yaml
@@ -15,13 +15,15 @@ spec:
   etcdClusters:
   - cpuRequest: 200m
     etcdMembers:
-    - instanceGroup: master-us-test-1a
+    - encryptedVolume: true
+      instanceGroup: master-us-test-1a
       name: a
     memoryRequest: 100Mi
     name: main
   - cpuRequest: 100m
     etcdMembers:
-    - instanceGroup: master-us-test-1a
+    - encryptedVolume: true
+      instanceGroup: master-us-test-1a
       name: a
     memoryRequest: 100Mi
     name: events

diff --git a/tests/integration/create_cluster/ha/expected-v1alpha2.yaml b/tests/integration/create_cluster/ha/expected-v1alpha2.yaml
@@ -15,21 +15,27 @@ spec:
   etcdClusters:
   - cpuRequest: 200m
     etcdMembers:
-    - instanceGroup: master-us-test-1a
+    - encryptedVolume: true
+      instanceGroup: master-us-test-1a
       name: a
-    - instanceGroup: master-us-test-1b
+    - encryptedVolume: true
+      instanceGroup: master-us-test-1b
       name: b
-    - instanceGroup: master-us-test-1c
+    - encryptedVolume: true
+      instanceGroup: master-us-test-1c
       name: c
     memoryRequest: 100Mi
     name: main
   - cpuRequest: 100m
     etcdMembers:
-    - instanceGroup: master-us-test-1a
+    - encryptedVolume: true
+      instanceGroup: master-us-test-1a
       name: a
-    - instanceGroup: master-us-test-1b
+    - encryptedVolume: true
+      instanceGroup: master-us-test-1b
       name: b
-    - instanceGroup: master-us-test-1c
+    - encryptedVolume: true
+      instanceGroup: master-us-test-1c
       name: c
     memoryRequest: 100Mi
     name: events

diff --git a/tests/integration/create_cluster/ha_shared_zone/expected-v1alpha2.yaml b/tests/integration/create_cluster/ha_shared_zone/expected-v1alpha2.yaml
@@ -15,21 +15,27 @@ spec:
   etcdClusters:
   - cpuRequest: 200m
     etcdMembers:
-    - instanceGroup: master-us-test-1a-1
+    - encryptedVolume: true
+      instanceGroup: master-us-test-1a-1
       name: etcd-1
-    - instanceGroup: master-us-test-1a-2
+    - encryptedVolume: true
+      instanceGroup: master-us-test-1a-2
       name: etcd-2
-    - instanceGroup: master-us-test-1a-3
+    - encryptedVolume: true
+      instanceGroup: master-us-test-1a-3
       name: etcd-3
     memoryRequest: 100Mi
     name: main
   - cpuRequest: 100m
     etcdMembers:
-    - instanceGroup: master-us-test-1a-1
+    - encryptedVolume: true
+      instanceGroup: master-us-test-1a-1
       name: etcd-1
-    - instanceGroup: master-us-test-1a-2
+    - encryptedVolume: true
+      instanceGroup: master-us-test-1a-2
       name: etcd-2
-    - instanceGroup: master-us-test-1a-3
+    - encryptedVolume: true
+      instanceGroup: master-us-test-1a-3
       name: etcd-3
     memoryRequest: 100Mi
     name: events

diff --git a/tests/integration/create_cluster/ha_shared_zones/expected-v1alpha2.yaml b/tests/integration/create_cluster/ha_shared_zones/expected-v1alpha2.yaml
@@ -15,29 +15,39 @@ spec:
   etcdClusters:
   - cpuRequest: 200m
     etcdMembers:
-    - instanceGroup: master-us-test-1a-1
+    - encryptedVolume: true
+      instanceGroup: master-us-test-1a-1
       name: a-1
-    - instanceGroup: master-us-test-1b-1
+    - encryptedVolume: true
+      instanceGroup: master-us-test-1b-1
       name: b-1
-    - instanceGroup: master-us-test-1a-2
+    - encryptedVolume: true
+      instanceGroup: master-us-test-1a-2
       name: a-2
-    - instanceGroup: master-us-test-1b-2
+    - encryptedVolume: true
+      instanceGroup: master-us-test-1b-2
       name: b-2
-    - instanceGroup: master-us-test-1a-3
+    - encryptedVolume: true
+      instanceGroup: master-us-test-1a-3
       name: a-3
     memoryRequest: 100Mi
     name: main
   - cpuRequest: 100m
     etcdMembers:
-    - instanceGroup: master-us-test-1a-1
+    - encryptedVolume: true
+      instanceGroup: master-us-test-1a-1
       name: a-1
-    - instanceGroup: master-us-test-1b-1
+    - encryptedVolume: true
+      instanceGroup: master-us-test-1b-1
       name: b-1
-    - instanceGroup: master-us-test-1a-2
+    - encryptedVolume: true
+      instanceGroup: master-us-test-1a-2
       name: a-2
-    - instanceGroup: master-us-test-1b-2
+    - encryptedVolume: true
+      instanceGroup: master-us-test-1b-2
       name: b-2
-    - instanceGroup: master-us-test-1a-3
+    - encryptedVolume: true
+      instanceGroup: master-us-test-1a-3
       name: a-3
     memoryRequest: 100Mi
     name: events

diff --git a/tests/integration/create_cluster/ingwspecified/expected-v1alpha2.yaml b/tests/integration/create_cluster/ingwspecified/expected-v1alpha2.yaml
@@ -17,13 +17,15 @@ spec:
   etcdClusters:
   - cpuRequest: 200m
     etcdMembers:
-    - instanceGroup: master-us-test-1a
+    - encryptedVolume: true
+      instanceGroup: master-us-test-1a
       name: a
     memoryRequest: 100Mi
     name: main
   - cpuRequest: 100m
     etcdMembers:
-    - instanceGroup: master-us-test-1a
+    - encryptedVolume: true
+      instanceGroup: master-us-test-1a
       name: a
     memoryRequest: 100Mi
     name: events

diff --git a/tests/integration/create_cluster/minimal/expected-v1alpha2.yaml b/tests/integration/create_cluster/minimal/expected-v1alpha2.yaml
@@ -15,13 +15,15 @@ spec:
   etcdClusters:
   - cpuRequest: 200m
     etcdMembers:
-    - instanceGroup: master-us-test-1a
+    - encryptedVolume: true
+      instanceGroup: master-us-test-1a
       name: a
     memoryRequest: 100Mi
     name: main
   - cpuRequest: 100m
     etcdMembers:
-    - instanceGroup: master-us-test-1a
+    - encryptedVolume: true
+      instanceGroup: master-us-test-1a
       name: a
     memoryRequest: 100Mi
     name: events

diff --git a/tests/integration/create_cluster/ngwspecified/expected-v1alpha2.yaml b/tests/integration/create_cluster/ngwspecified/expected-v1alpha2.yaml
@@ -17,13 +17,15 @@ spec:
   etcdClusters:
   - cpuRequest: 200m
     etcdMembers:
-    - instanceGroup: master-us-test-1a
+    - encryptedVolume: true
+      instanceGroup: master-us-test-1a
       name: a
     memoryRequest: 100Mi
     name: main
   - cpuRequest: 100m
     etcdMembers:
-    - instanceGroup: master-us-test-1a
+    - encryptedVolume: true
+      instanceGroup: master-us-test-1a
       name: a
     memoryRequest: 100Mi
     name: events

diff --git a/tests/integration/create_cluster/overrides/expected-v1alpha2.yaml b/tests/integration/create_cluster/overrides/expected-v1alpha2.yaml
@@ -15,13 +15,15 @@ spec:
   etcdClusters:
   - cpuRequest: 200m
     etcdMembers:
-    - instanceGroup: master-us-test-1a
+    - encryptedVolume: true
+      instanceGroup: master-us-test-1a
       name: a
     memoryRequest: 100Mi
     name: main
   - cpuRequest: 100m
     etcdMembers:
-    - instanceGroup: master-us-test-1a
+    - encryptedVolume: true
+      instanceGroup: master-us-test-1a
       name: a
     memoryRequest: 100Mi
     name: events

diff --git a/tests/integration/create_cluster/private/expected-v1alpha2.yaml b/tests/integration/create_cluster/private/expected-v1alpha2.yaml
@@ -21,13 +21,15 @@ spec:
   etcdClusters:
   - cpuRequest: 200m
     etcdMembers:
-    - instanceGroup: master-us-test-1a
+    - encryptedVolume: true
+      instanceGroup: master-us-test-1a
       name: a
     memoryRequest: 100Mi
     name: main
   - cpuRequest: 100m
     etcdMembers:
-    - instanceGroup: master-us-test-1a
+    - encryptedVolume: true
+      instanceGroup: master-us-test-1a
       name: a
     memoryRequest: 100Mi
     name: events

diff --git a/tests/integration/create_cluster/private_shared_subnets/expected-v1alpha2.yaml b/tests/integration/create_cluster/private_shared_subnets/expected-v1alpha2.yaml
@@ -17,13 +17,15 @@ spec:
   etcdClusters:
   - cpuRequest: 200m
     etcdMembers:
-    - instanceGroup: master-us-test-1a
+    - encryptedVolume: true
+      instanceGroup: master-us-test-1a
       name: a
     memoryRequest: 100Mi
     name: main
   - cpuRequest: 100m
     etcdMembers:
-    - instanceGroup: master-us-test-1a
+    - encryptedVolume: true
+      instanceGroup: master-us-test-1a
       name: a
     memoryRequest: 100Mi
     name: events

diff --git a/tests/integration/create_cluster/shared_subnets/expected-v1alpha2.yaml b/tests/integration/create_cluster/shared_subnets/expected-v1alpha2.yaml
@@ -15,13 +15,15 @@ spec:
   etcdClusters:
   - cpuRequest: 200m
     etcdMembers:
-    - instanceGroup: master-us-test-1a
+    - encryptedVolume: true
+      instanceGroup: master-us-test-1a
       name: a
     memoryRequest: 100Mi
     name: main
   - cpuRequest: 100m
     etcdMembers:
-    - instanceGroup: master-us-test-1a
+    - encryptedVolume: true
+      instanceGroup: master-us-test-1a
       name: a
     memoryRequest: 100Mi
     name: events

diff --git a/tests/integration/create_cluster/shared_subnets_vpc_lookup/expected-v1alpha2.yaml b/tests/integration/create_cluster/shared_subnets_vpc_lookup/expected-v1alpha2.yaml
@@ -15,13 +15,15 @@ spec:
   etcdClusters:
   - cpuRequest: 200m
     etcdMembers:
-    - instanceGroup: master-us-test-1a
+    - encryptedVolume: true
+      instanceGroup: master-us-test-1a
       name: a
     memoryRequest: 100Mi
     name: main
   - cpuRequest: 100m
     etcdMembers:
-    - instanceGroup: master-us-test-1a
+    - encryptedVolume: true
+      instanceGroup: master-us-test-1a
       name: a
     memoryRequest: 100Mi
     name: events

diff --git a/tests/integration/create_cluster/shared_vpc/expected-v1alpha2.yaml b/tests/integration/create_cluster/shared_vpc/expected-v1alpha2.yaml
@@ -15,13 +15,15 @@ spec:
   etcdClusters:
   - cpuRequest: 200m
     etcdMembers:
-    - instanceGroup: master-us-test-1a
+    - encryptedVolume: true
+      instanceGroup: master-us-test-1a
       name: a
     memoryRequest: 100Mi
     name: main
   - cpuRequest: 100m
     etcdMembers:
-    - instanceGroup: master-us-test-1a
+    - encryptedVolume: true
+      instanceGroup: master-us-test-1a
       name: a
     memoryRequest: 100Mi
     name: events

diff --git a/tests/integration/update_cluster/bastionadditional_user-data/kubernetes.tf b/tests/integration/update_cluster/bastionadditional_user-data/kubernetes.tf
@@ -264,6 +264,7 @@ resource "aws_autoscaling_group" "nodes-bastionuserdata-example-com" {
 resource "aws_ebs_volume" "us-test-1a-etcd-events-bastionuserdata-example-com" {
   availability_zone = "us-test-1a"
   encrypted         = false
+  iops              = 3000
   size              = 20
   tags = {
     "KubernetesCluster"                                 = "bastionuserdata.example.com"
@@ -272,12 +273,14 @@ resource "aws_ebs_volume" "us-test-1a-etcd-events-bastionuserdata-example-com" {
     "k8s.io/role/master"                                = "1"
     "kubernetes.io/cluster/bastionuserdata.example.com" = "owned"
   }
-  type = "gp2"
+  throughput = 125
+  type       = "gp3"
 }
 
 resource "aws_ebs_volume" "us-test-1a-etcd-main-bastionuserdata-example-com" {
   availability_zone = "us-test-1a"
   encrypted         = false
+  iops              = 3000
   size              = 20
   tags = {
     "KubernetesCluster"                                 = "bastionuserdata.example.com"
@@ -286,7 +289,8 @@ resource "aws_ebs_volume" "us-test-1a-etcd-main-bastionuserdata-example-com" {
     "k8s.io/role/master"                                = "1"
     "kubernetes.io/cluster/bastionuserdata.example.com" = "owned"
   }
-  type = "gp2"
+  throughput = 125
+  type       = "gp3"
 }
 
 resource "aws_eip" "us-test-1a-bastionuserdata-example-com" {