From 1316922d757ec671d811ede3366d0271e5ad019a Mon Sep 17 00:00:00 2001 From: mikesplain Date: Fri, 21 Jun 2019 09:21:30 -0400 Subject: [PATCH 1/2] First pass at instance protection --- pkg/apis/kops/instancegroup.go | 2 ++ pkg/apis/kops/v1alpha1/instancegroup.go | 2 ++ .../kops/v1alpha1/zz_generated.conversion.go | 2 ++ .../kops/v1alpha1/zz_generated.deepcopy.go | 5 +++++ pkg/apis/kops/v1alpha2/instancegroup.go | 2 ++ .../kops/v1alpha2/zz_generated.conversion.go | 2 ++ .../kops/v1alpha2/zz_generated.deepcopy.go | 5 +++++ pkg/apis/kops/zz_generated.deepcopy.go | 5 +++++ pkg/model/awsmodel/autoscalinggroup.go | 2 ++ .../fi/cloudup/awstasks/autoscalinggroup.go | 18 ++++++++++++++++++ 10 files changed, 45 insertions(+) diff --git a/pkg/apis/kops/instancegroup.go b/pkg/apis/kops/instancegroup.go index 2c13350943580..3b1518d249f21 100644 --- a/pkg/apis/kops/instancegroup.go +++ b/pkg/apis/kops/instancegroup.go @@ -149,6 +149,8 @@ type InstanceGroupSpec struct { IAM *IAMProfileSpec `json:"iam,omitempty"` // SecurityGroupOverride overrides the default security group created by Kops for this IG (AWS only). SecurityGroupOverride *string `json:"securityGroupOverride,omitempty"` + // InstanceProtection makes new instances in an autoscaling group protected from scale in + InstanceProtection *bool `json:"instanceProtection,omitempty"` } const ( diff --git a/pkg/apis/kops/v1alpha1/instancegroup.go b/pkg/apis/kops/v1alpha1/instancegroup.go index ef900d9e54237..32c5daa9c3b3c 100644 --- a/pkg/apis/kops/v1alpha1/instancegroup.go +++ b/pkg/apis/kops/v1alpha1/instancegroup.go @@ -136,6 +136,8 @@ type InstanceGroupSpec struct { IAM *IAMProfileSpec `json:"iam,omitempty"` // SecurityGroupOverride overrides the default security group created by Kops for this IG (AWS only). SecurityGroupOverride *string `json:"securityGroupOverride,omitempty"` + // InstanceProtection makes new instances in an autoscaling group protected from scale in + InstanceProtection *bool `json:"instanceProtection,omitempty"` } const ( diff --git a/pkg/apis/kops/v1alpha1/zz_generated.conversion.go b/pkg/apis/kops/v1alpha1/zz_generated.conversion.go index 666eaeef0053b..c30a6889285b7 100644 --- a/pkg/apis/kops/v1alpha1/zz_generated.conversion.go +++ b/pkg/apis/kops/v1alpha1/zz_generated.conversion.go @@ -2751,6 +2751,7 @@ func autoConvert_v1alpha1_InstanceGroupSpec_To_kops_InstanceGroupSpec(in *Instan out.IAM = nil } out.SecurityGroupOverride = in.SecurityGroupOverride + out.InstanceProtection = in.InstanceProtection return nil } @@ -2871,6 +2872,7 @@ func autoConvert_kops_InstanceGroupSpec_To_v1alpha1_InstanceGroupSpec(in *kops.I out.IAM = nil } out.SecurityGroupOverride = in.SecurityGroupOverride + out.InstanceProtection = in.InstanceProtection return nil } diff --git a/pkg/apis/kops/v1alpha1/zz_generated.deepcopy.go b/pkg/apis/kops/v1alpha1/zz_generated.deepcopy.go index 6d27934413a3a..7d0156490e59b 100644 --- a/pkg/apis/kops/v1alpha1/zz_generated.deepcopy.go +++ b/pkg/apis/kops/v1alpha1/zz_generated.deepcopy.go @@ -1529,6 +1529,11 @@ func (in *InstanceGroupSpec) DeepCopyInto(out *InstanceGroupSpec) { *out = new(string) **out = **in } + if in.InstanceProtection != nil { + in, out := &in.InstanceProtection, &out.InstanceProtection + *out = new(bool) + **out = **in + } return } diff --git a/pkg/apis/kops/v1alpha2/instancegroup.go b/pkg/apis/kops/v1alpha2/instancegroup.go index 44ef670283890..4adee1ffcb591 100644 --- a/pkg/apis/kops/v1alpha2/instancegroup.go +++ b/pkg/apis/kops/v1alpha2/instancegroup.go @@ -138,6 +138,8 @@ type InstanceGroupSpec struct { IAM *IAMProfileSpec `json:"iam,omitempty"` // SecurityGroupOverride overrides the default security group created by Kops for this IG (AWS only). SecurityGroupOverride *string `json:"securityGroupOverride,omitempty"` + // InstanceProtection makes new instances in an autoscaling group protected from scale in + InstanceProtection *bool `json:"instanceProtection,omitempty"` } const ( diff --git a/pkg/apis/kops/v1alpha2/zz_generated.conversion.go b/pkg/apis/kops/v1alpha2/zz_generated.conversion.go index ea124cdcb9270..efb85d57c9de1 100644 --- a/pkg/apis/kops/v1alpha2/zz_generated.conversion.go +++ b/pkg/apis/kops/v1alpha2/zz_generated.conversion.go @@ -2869,6 +2869,7 @@ func autoConvert_v1alpha2_InstanceGroupSpec_To_kops_InstanceGroupSpec(in *Instan out.IAM = nil } out.SecurityGroupOverride = in.SecurityGroupOverride + out.InstanceProtection = in.InstanceProtection return nil } @@ -2994,6 +2995,7 @@ func autoConvert_kops_InstanceGroupSpec_To_v1alpha2_InstanceGroupSpec(in *kops.I out.IAM = nil } out.SecurityGroupOverride = in.SecurityGroupOverride + out.InstanceProtection = in.InstanceProtection return nil } diff --git a/pkg/apis/kops/v1alpha2/zz_generated.deepcopy.go b/pkg/apis/kops/v1alpha2/zz_generated.deepcopy.go index 78f7605bc5970..8a434c9bebfa1 100644 --- a/pkg/apis/kops/v1alpha2/zz_generated.deepcopy.go +++ b/pkg/apis/kops/v1alpha2/zz_generated.deepcopy.go @@ -1491,6 +1491,11 @@ func (in *InstanceGroupSpec) DeepCopyInto(out *InstanceGroupSpec) { *out = new(string) **out = **in } + if in.InstanceProtection != nil { + in, out := &in.InstanceProtection, &out.InstanceProtection + *out = new(bool) + **out = **in + } return } diff --git a/pkg/apis/kops/zz_generated.deepcopy.go b/pkg/apis/kops/zz_generated.deepcopy.go index 0f46defb06328..a72c087c4c646 100644 --- a/pkg/apis/kops/zz_generated.deepcopy.go +++ b/pkg/apis/kops/zz_generated.deepcopy.go @@ -1657,6 +1657,11 @@ func (in *InstanceGroupSpec) DeepCopyInto(out *InstanceGroupSpec) { *out = new(string) **out = **in } + if in.InstanceProtection != nil { + in, out := &in.InstanceProtection, &out.InstanceProtection + *out = new(bool) + **out = **in + } return } diff --git a/pkg/model/awsmodel/autoscalinggroup.go b/pkg/model/awsmodel/autoscalinggroup.go index 74e0daefb0eb5..0519fa8334a2b 100644 --- a/pkg/model/awsmodel/autoscalinggroup.go +++ b/pkg/model/awsmodel/autoscalinggroup.go @@ -315,6 +315,8 @@ func (b *AutoscalingGroupModelBuilder) buildAutoScalingGroupTask(c *fi.ModelBuil } t.SuspendProcesses = &processes + t.InstanceProtection = ig.Spec.InstanceProtection + // @step: are we using a mixed instance policy if ig.Spec.MixedInstancesPolicy != nil { spec := ig.Spec.MixedInstancesPolicy diff --git a/upup/pkg/fi/cloudup/awstasks/autoscalinggroup.go b/upup/pkg/fi/cloudup/awstasks/autoscalinggroup.go index a23ebf9e5e8b2..7c2aba8d89722 100644 --- a/upup/pkg/fi/cloudup/awstasks/autoscalinggroup.go +++ b/upup/pkg/fi/cloudup/awstasks/autoscalinggroup.go @@ -45,6 +45,8 @@ type AutoscalingGroup struct { // Granularity specifys the granularity of the metrics Granularity *string + // InstanceProtection makes new instances in an autoscaling group protected from scale in + InstanceProtection *bool // LaunchConfiguration is the launch configuration for the autoscaling group LaunchConfiguration *LaunchConfiguration // LaunchTemplate is the launch template for the asg @@ -171,6 +173,10 @@ func (e *AutoscalingGroup) Find(c *fi.Context) (*AutoscalingGroup, error) { // Avoid spurious changes actual.Lifecycle = e.Lifecycle + if g.NewInstancesProtectedFromScaleIn != nil { + actual.InstanceProtection = g.NewInstancesProtectedFromScaleIn + } + return actual, nil } @@ -315,6 +321,11 @@ func (v *AutoscalingGroup) RenderAWS(t *awsup.AWSAPITarget, a, e, changes *Autos return fmt.Errorf("error suspending processes: %v", err) } } + + if e.InstanceProtection != nil { + request.NewInstancesProtectedFromScaleIn = e.InstanceProtection + } + } else { // @logic: else we have found a autoscaling group and we need to evaluate the difference request := &autoscaling.UpdateAutoScalingGroupInput{ @@ -451,6 +462,11 @@ func (v *AutoscalingGroup) RenderAWS(t *awsup.AWSAPITarget, a, e, changes *Autos changes.SuspendProcesses = nil } + if changes.InstanceProtection != nil { + request.NewInstancesProtectedFromScaleIn = e.InstanceProtection + changes.InstanceProtection = nil + } + empty := &AutoscalingGroup{} if !reflect.DeepEqual(empty, changes) { klog.Warningf("cannot apply changes to AutoScalingGroup: %v", changes) @@ -628,6 +644,7 @@ type terraformAutoscalingGroup struct { MetricsGranularity *string `json:"metrics_granularity,omitempty"` EnabledMetrics []*string `json:"enabled_metrics,omitempty"` SuspendedProcesses []*string `json:"suspended_processes,omitempty"` + InstanceProtection *bool `json:"protect_from_scale_in,omitempty"` } // RenderTerraform is responsible for rendering the terraform codebase @@ -638,6 +655,7 @@ func (_ *AutoscalingGroup) RenderTerraform(t *terraform.TerraformTarget, a, e, c MaxSize: e.MaxSize, MetricsGranularity: e.Granularity, EnabledMetrics: aws.StringSlice(e.Metrics), + InstanceProtection: e.InstanceProtection, } for _, s := range e.Subnets { From a279a29cda9fa66f5479a10c58b511f9a25cbf6b Mon Sep 17 00:00:00 2001 From: mikesplain Date: Fri, 21 Jun 2019 14:00:30 -0400 Subject: [PATCH 2/2] Add docs and test --- docs/instance_groups.md | 21 +++++++++++++++++++ .../mixed_instances/in-v1alpha2.yaml | 1 + .../mixed_instances/kubernetes.tf | 5 +++-- 3 files changed, 25 insertions(+), 2 deletions(-) diff --git a/docs/instance_groups.md b/docs/instance_groups.md index aeb2db2f6e8e8..95926a3a7fc71 100644 --- a/docs/instance_groups.md +++ b/docs/instance_groups.md @@ -474,6 +474,27 @@ spec: - AZRebalance ``` +## Protect new instances from scale in + +Autoscaling groups may scale up or down automatically to balance types of instances, regions, etc. +[Instance protection](https://docs.aws.amazon.com/autoscaling/ec2/userguide/as-instance-termination.html#instance-protection) prevents the ASG from being scaled in. + +``` +# Example for nodes +apiVersion: kops.k8s.io/v1alpha2 +kind: InstanceGroup +metadata: + labels: + kops.k8s.io/cluster: k8s.dev.local + name: nodes +spec: + machineType: m4.xlarge + maxSize: 20 + minSize: 2 + role: Node + instanceProtection: true +``` + ## Attaching existing Load Balancers to Instance Groups Instance groups can be linked to up to 10 load balancers. When attached, any instance launched will diff --git a/tests/integration/update_cluster/mixed_instances/in-v1alpha2.yaml b/tests/integration/update_cluster/mixed_instances/in-v1alpha2.yaml index dd863e09399f4..590c3b4127200 100644 --- a/tests/integration/update_cluster/mixed_instances/in-v1alpha2.yaml +++ b/tests/integration/update_cluster/mixed_instances/in-v1alpha2.yaml @@ -68,6 +68,7 @@ spec: maxSize: 2 minSize: 2 role: Node + instanceProtection: true subnets: - us-test-1b mixedInstancesPolicy: diff --git a/tests/integration/update_cluster/mixed_instances/kubernetes.tf b/tests/integration/update_cluster/mixed_instances/kubernetes.tf index 33ee2cba72b5a..c2e536eee832e 100644 --- a/tests/integration/update_cluster/mixed_instances/kubernetes.tf +++ b/tests/integration/update_cluster/mixed_instances/kubernetes.tf @@ -228,8 +228,9 @@ resource "aws_autoscaling_group" "nodes-mixedinstances-example-com" { propagate_at_launch = true } - metrics_granularity = "1Minute" - enabled_metrics = ["GroupDesiredCapacity", "GroupInServiceInstances", "GroupMaxSize", "GroupMinSize", "GroupPendingInstances", "GroupStandbyInstances", "GroupTerminatingInstances", "GroupTotalInstances"] + metrics_granularity = "1Minute" + enabled_metrics = ["GroupDesiredCapacity", "GroupInServiceInstances", "GroupMaxSize", "GroupMinSize", "GroupPendingInstances", "GroupStandbyInstances", "GroupTerminatingInstances", "GroupTotalInstances"] + protect_from_scale_in = true } resource "aws_ebs_volume" "us-test-1a-etcd-events-mixedinstances-example-com" {