Skip to content

Commit

Permalink
Added Propagation of ManagedNodeGroup Tags to their corresponding Aut…
Browse files Browse the repository at this point in the history 
…oScalingGroups (#5002)

* Added Propagation of ManagedNodeGroup Tags to their AutoScalingGroups

Changes	to ensure that AutoScalingGroups Tags are the same as their
 ManagedNodeGroup.

All tags are copied from the ManagedNodeGroup to the AutoScalingGroup.
If the tags already exists, it is overridden.

This is the default behaviour (as it is for Unmanaged NodeGroup) and
can be enabled using propagateASGTags boolean configuration.

Issue #1571

* Update generated files

* fix existing unit tests

* update documentation

* Update generated files

* consider new way of configuring tags propagation after the rebase

* re-add disableASGTagPropagation doc for managed nodegroup

* consider tags limits, move logic to a more appropriated place, re-add test for managernodegroup

* fix usage of aws-sdk-go-v2, improve task parallelisation for managednodegroup creation

* fix test linting

* add unit tests

* improve unit test structure for PropagateManagedNodeGroupTagsToASG, rename astypes package to asTypes

* use const for auto-scaling-group string

* improve some code structure
  • Loading branch information
@SlevinWasAlreadyTaken
SlevinWasAlreadyTaken authored Apr 25, 2022
1 parent 227ab2c commit 0284bc3
Show file tree
Hide file tree
Showing 15 changed files with 392 additions and 33 deletions.
21 changes: 14 additions & 7 deletions pkg/apis/eksctl.io/v1alpha5/assets/schema.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1076,6 +1076,11 @@
"x-intellij-html-description": "Enable <a href=\"/usage/vpc-networking/#use-private-subnets-for-initial-nodegroup\">private networking</a> for nodegroup",
"default": "false"
},
"propagateASGTags": {
"type": "boolean",
"description": "Propagate all taints and labels to the ASG automatically.",
"x-intellij-html-description": "Propagate all taints and labels to the ASG automatically."
},
"releaseVersion": {
"type": "string",
"description": "the AMI version of the EKS optimized AMI to use",
Expand Down Expand Up @@ -1108,8 +1113,8 @@
"type": "string"
},
"type": "object",
"description": "Applied to the Autoscaling Group and to the EC2 instances (unmanaged), Applied to the EKS Nodegroup resource and to the EC2 instances (managed)",
"x-intellij-html-description": "Applied to the Autoscaling Group and to the EC2 instances (unmanaged), Applied to the EKS Nodegroup resource and to the EC2 instances (managed)",
"description": "Applied to the Autoscaling Group and to the EC2 instances (unmanaged), Applied to the Autoscaling Group, the EKS Nodegroup resource and to the EC2 instances (managed)",
"x-intellij-html-description": "Applied to the Autoscaling Group and to the EC2 instances (unmanaged), Applied to the Autoscaling Group, the EKS Nodegroup resource and to the EC2 instances (managed)",
"default": "{}"
},
"taints": {
Expand Down Expand Up @@ -1191,6 +1196,7 @@
"additionalVolumes",
"preBootstrapCommands",
"overrideBootstrapCommand",
"propagateASGTags",
"disableIMDSv1",
"disablePodIMDS",
"placement",
Expand Down Expand Up @@ -1321,8 +1327,9 @@
},
"disableASGTagPropagation": {
"type": "boolean",
"description": "disable the tag propagation in case desired capacity is 0.",
"x-intellij-html-description": "disable the tag propagation in case desired capacity is 0."
"description": "disables the tag propagation to ASG in case desired capacity is 0.",
"x-intellij-html-description": "disables the tag propagation to ASG in case desired capacity is 0.",
"default": false
},
"disableIMDSv1": {
"type": "boolean",
Expand Down Expand Up @@ -1450,8 +1457,8 @@
"type": "string"
},
"type": "object",
"description": "Applied to the Autoscaling Group and to the EC2 instances (unmanaged), Applied to the EKS Nodegroup resource and to the EC2 instances (managed)",
"x-intellij-html-description": "Applied to the Autoscaling Group and to the EC2 instances (unmanaged), Applied to the EKS Nodegroup resource and to the EC2 instances (managed)",
"description": "Applied to the Autoscaling Group and to the EC2 instances (unmanaged), Applied to the Autoscaling Group, the EKS Nodegroup resource and to the EC2 instances (managed)",
"x-intellij-html-description": "Applied to the Autoscaling Group and to the EC2 instances (unmanaged), Applied to the Autoscaling Group, the EKS Nodegroup resource and to the EC2 instances (managed)",
"default": "{}"
},
"taints": {
Expand Down Expand Up @@ -1538,6 +1545,7 @@
"additionalVolumes",
"preBootstrapCommands",
"overrideBootstrapCommand",
"propagateASGTags",
"disableIMDSv1",
"disablePodIMDS",
"placement",
Expand All @@ -1555,7 +1563,6 @@
"clusterDNS",
"kubeletExtraConfig",
"containerRuntime",
"propagateASGTags",
"disableASGTagPropagation",
"maxInstanceLifetime"
],
Expand Down
13 changes: 7 additions & 6 deletions pkg/apis/eksctl.io/v1alpha5/types.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -997,11 +997,8 @@ type NodeGroup struct {
// +optional
ContainerRuntime *string `json:"containerRuntime,omitempty"`

// Propagate all taints and labels to the ASG automatically.
// +optional
PropagateASGTags *bool `json:"propagateASGTags,omitempty"`

// DisableASGTagPropagation disable the tag propagation in case desired capacity is 0.
// DisableASGTagPropagation disables the tag propagation to ASG in case desired capacity is 0.
// Defaults to `false`
// +optional
DisableASGTagPropagation *bool `json:"disableASGTagPropagation,omitempty"`

Expand Down Expand Up @@ -1315,7 +1312,7 @@ type NodeGroupBase struct {
// +optional
PrivateNetworking bool `json:"privateNetworking"`
// Applied to the Autoscaling Group and to the EC2 instances (unmanaged),
// Applied to the EKS Nodegroup resource and to the EC2 instances (managed)
// Applied to the Autoscaling Group, the EKS Nodegroup resource and to the EC2 instances (managed)
// +optional
Tags map[string]string `json:"tags,omitempty"`
// +optional
Expand Down Expand Up @@ -1368,6 +1365,10 @@ type NodeGroupBase struct {
// +optional
OverrideBootstrapCommand *string `json:"overrideBootstrapCommand,omitempty"`

// Propagate all taints and labels to the ASG automatically.
// +optional
PropagateASGTags *bool `json:"propagateASGTags,omitempty"`

// DisableIMDSv1 requires requests to the metadata service to use IMDSv2 tokens
// Defaults to `false`
// +optional
Expand Down
10 changes: 5 additions & 5 deletions pkg/apis/eksctl.io/v1alpha5/zz_generated.deepcopy.go
View file

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

1 change: 1 addition & 0 deletions pkg/cfn/builder/nodegroup.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,7 @@ import (

// MaximumTagNumber for ASGs as described here https://docs.aws.amazon.com/autoscaling/ec2/userguide/autoscaling-tagging.html
const MaximumTagNumber = 50
const MaximumCreatedTagNumberPerCall = 25

// NodeGroupResourceSet stores the resource information of the nodegroup
type NodeGroupResourceSet struct {
Expand Down
99 changes: 94 additions & 5 deletions pkg/cfn/manager/api.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,8 @@ import (
"strings"
"time"

"github.com/aws/aws-sdk-go-v2/service/autoscaling"
asTypes "github.com/aws/aws-sdk-go-v2/service/autoscaling/types"
"github.com/aws/aws-sdk-go-v2/service/cloudformation"
"github.com/aws/aws-sdk-go-v2/service/cloudformation/types"
"github.com/aws/aws-sdk-go-v2/service/cloudtrail"
Expand All @@ -26,11 +28,12 @@ import (
)

const (
resourcesRootPath = "Resources"
outputsRootPath = "Outputs"
mappingsRootPath = "Mappings"
ourStackRegexFmt = "^(eksctl|EKS)-%s-((cluster|nodegroup-.+|addon-.+|fargate|karpenter)|(VPC|ServiceRole|ControlPlane|DefaultNodeGroup))$"
clusterStackRegex = "eksctl-.*-cluster"
resourcesRootPath = "Resources"
resourceTypeAutoScalingGroup = "auto-scaling-group"
outputsRootPath = "Outputs"
mappingsRootPath = "Mappings"
ourStackRegexFmt = "^(eksctl|EKS)-%s-((cluster|nodegroup-.+|addon-.+|fargate|karpenter)|(VPC|ServiceRole|ControlPlane|DefaultNodeGroup))$"
clusterStackRegex = "eksctl-.*-cluster"
)

var (
Expand Down Expand Up @@ -238,6 +241,92 @@ func (c *StackCollection) createStackRequest(ctx context.Context, stackName stri
return stack, nil
}

func (c *StackCollection) PropagateManagedNodeGroupTagsToASG(ngName string, ngTags map[string]string, asgNames []string, errCh chan error) error {
go func() {
defer close(errCh)
// build the input tags for all ASGs attached to the managed nodegroup
asgTags := []asTypes.Tag{}

for _, asgName := range asgNames {
// skip directly if not tags are required to be created
if len(ngTags) == 0 {
continue
}

// check if the number of tags on the ASG would go over the defined limit
if err := c.checkASGTagsNumber(ngName, asgName, ngTags); err != nil {
errCh <- err
return
}
// build the list of tags to attach to the ASG
for ngTagKey, ngTagValue := range ngTags {
asgTag := asTypes.Tag{
ResourceId: aws.String(asgName),
ResourceType: aws.String(resourceTypeAutoScalingGroup),
Key: aws.String(ngTagKey),
Value: aws.String(ngTagValue),
PropagateAtLaunch: aws.Bool(false),
}
asgTags = append(asgTags, asgTag)
}
}

// consider the maximum number of tags we can create at once...
var chunkedASGTags [][]asTypes.Tag
chunkSize := builder.MaximumCreatedTagNumberPerCall
for start := 0; start < len(asgTags); start += chunkSize {
end := start + chunkSize
if end > len(asgTags) {
end = len(asgTags)
}
chunkedASGTags = append(chunkedASGTags, asgTags[start:end])
}
// ...then create all of them in a loop
for _, asgTags := range chunkedASGTags {
input := &autoscaling.CreateOrUpdateTagsInput{Tags: asgTags}
if _, err := c.asgAPI.CreateOrUpdateTags(context.Background(), input); err != nil {
errCh <- errors.Wrapf(err, "creating or updating asg tags for managed nodegroup %q", ngName)
return
}
}
errCh <- nil
}()
return nil
}

// checkASGTagsNumber limit considering the new propagated tags
func (c *StackCollection) checkASGTagsNumber(ngName, asgName string, propagatedTags map[string]string) error {
tagsFilter := &autoscaling.DescribeTagsInput{
Filters: []asTypes.Filter{
{
Name: aws.String(resourceTypeAutoScalingGroup),
Values: []string{asgName},
},
},
}
output, err := c.asgAPI.DescribeTags(context.Background(), tagsFilter)
if err != nil {
return errors.Wrapf(err, "describing asg %q tags for managed nodegroup %q", asgName, ngName)
}
asgTags := output.Tags
// intersection of key tags to consider the number of tags going
// to be attached to the ASG
uniqueTagKeyCount := len(asgTags) + len(propagatedTags)
for ngTagKey := range propagatedTags {
for _, asgTag := range asgTags {
// decrease the unique tag key count if there is a match
if aws.StringValue(asgTag.Key) == ngTagKey {
uniqueTagKeyCount--
break
}
}
}
if uniqueTagKeyCount > builder.MaximumTagNumber {
return fmt.Errorf("number of tags is exceeding the maximum amount for asg %d, was: %d", builder.MaximumTagNumber, uniqueTagKeyCount)
}
return nil
}

// UpdateStack will update a CloudFormation stack by creating and executing a ChangeSet
func (c *StackCollection) UpdateStack(ctx context.Context, options UpdateStackOptions) error {
logger.Info(options.Description)
Expand Down
111 changes: 111 additions & 0 deletions pkg/cfn/manager/api_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,21 +5,132 @@ import (
"errors"
"fmt"

"github.com/aws/aws-sdk-go-v2/service/autoscaling"
asTypes "github.com/aws/aws-sdk-go-v2/service/autoscaling/types"
cfn "github.com/aws/aws-sdk-go-v2/service/cloudformation"
"github.com/aws/aws-sdk-go-v2/service/cloudformation/types"
"github.com/aws/aws-sdk-go/aws"
"github.com/aws/aws-sdk-go/aws/request"
"github.com/aws/aws-sdk-go/awstesting"

. "github.com/onsi/ginkgo"
. "github.com/onsi/ginkgo/extensions/table"
. "github.com/onsi/gomega"
"github.com/stretchr/testify/mock"

api "github.com/weaveworks/eksctl/pkg/apis/eksctl.io/v1alpha5"
"github.com/weaveworks/eksctl/pkg/cfn/builder"
"github.com/weaveworks/eksctl/pkg/testutils/mockprovider"
)

var _ = Describe("StackCollection", func() {
Context("PropagateManagedNodeGroupTagsToASG", func() {
var (
asgName string
ngName string
ngTags map[string]string
errCh chan error
p *mockprovider.MockProvider
)
BeforeEach(func() {
asgName = "asg-test-name"
ngName = "ng-test-name"
ngTags = map[string]string{
"tag_key_1": "tag_value_1",
}
errCh = make(chan error)
p = mockprovider.NewMockProvider()
})

It("can create propagate tag", func() {
// DescribeTags classic mock
describeTagsInput := &autoscaling.DescribeTagsInput{
Filters: []asTypes.Filter{{Name: aws.String(resourceTypeAutoScalingGroup), Values: []string{asgName}}},
}
p.MockASG().On("DescribeTags", mock.Anything, describeTagsInput).Return(&autoscaling.DescribeTagsOutput{}, nil)

// CreateOrUpdateTags classic mock
createOrUpdateTagsInput := &autoscaling.CreateOrUpdateTagsInput{
Tags: []asTypes.Tag{
{
ResourceId: aws.String(asgName),
ResourceType: aws.String(resourceTypeAutoScalingGroup),
Key: aws.String("tag_key_1"),
Value: aws.String("tag_value_1"),
PropagateAtLaunch: aws.Bool(false),
},
},
}
p.MockASG().On("CreateOrUpdateTags", mock.Anything, createOrUpdateTagsInput).Return(&autoscaling.CreateOrUpdateTagsOutput{}, nil)

sm := NewStackCollection(p, api.NewClusterConfig())
err := sm.PropagateManagedNodeGroupTagsToASG(ngName, ngTags, []string{asgName}, errCh)
Expect(err).NotTo(HaveOccurred())
err = <-errCh
Expect(err).NotTo(HaveOccurred())
})
It("cannot propagate tags in chunks of 25", func() {
// populate the createOrUpdateTagsSliceInput for easier generation of chunks
createOrUpdateTagsSliceInput := []asTypes.Tag{}
for i := 0; i < 30; i++ {
tagKey, tagValue := fmt.Sprintf("tag_key_%d", i), fmt.Sprintf("tag_value_%d", i)
ngTags[tagKey] = tagValue
createOrUpdateTagsSliceInput = append(createOrUpdateTagsSliceInput, asTypes.Tag{
ResourceId: aws.String(asgName),
ResourceType: aws.String(resourceTypeAutoScalingGroup),
Key: aws.String(tagKey),
Value: aws.String(tagValue),
PropagateAtLaunch: aws.Bool(false),
})
}

// DescribeTags classic mock
describeTagsInput := &autoscaling.DescribeTagsInput{
Filters: []asTypes.Filter{{Name: aws.String(resourceTypeAutoScalingGroup), Values: []string{asgName}}},
}
p.MockASG().On("DescribeTags", mock.Anything, describeTagsInput).Return(&autoscaling.DescribeTagsOutput{}, nil)

// CreateOrUpdateTags chunked mock
// generate the expected chunk of tags
chunkSize := builder.MaximumCreatedTagNumberPerCall
firstchunkLenMatcher := func(input *autoscaling.CreateOrUpdateTagsInput) bool {
return len(input.Tags) == len(createOrUpdateTagsSliceInput[:chunkSize])
}
secondChunkLenMatcher := func(input *autoscaling.CreateOrUpdateTagsInput) bool {
return len(input.Tags) == len(createOrUpdateTagsSliceInput[chunkSize:])
}

// setup the call verification of the two chunks
// NOTE: because of the use of map (unordered processing), we just verify size of chunk
p.MockASG().On("CreateOrUpdateTags", mock.Anything, mock.MatchedBy(firstchunkLenMatcher)).Return(&autoscaling.CreateOrUpdateTagsOutput{}, nil)
p.MockASG().On("CreateOrUpdateTags", mock.Anything, mock.MatchedBy(secondChunkLenMatcher)).Return(&autoscaling.CreateOrUpdateTagsOutput{}, nil)

sm := NewStackCollection(p, api.NewClusterConfig())
err := sm.PropagateManagedNodeGroupTagsToASG(ngName, ngTags, []string{asgName}, errCh)
Expect(err).NotTo(HaveOccurred())
err = <-errCh
Expect(err).NotTo(HaveOccurred())
})
It("cannot propagate if too many tags", func() {
// fill parameters
for i := 0; i < builder.MaximumTagNumber+1; i++ {
ngTags[fmt.Sprintf("tag_key_%d", i)] = fmt.Sprintf("tag_value_%d", i)
}

// DescribeTags classic mock
describeTagsInput := &autoscaling.DescribeTagsInput{
Filters: []asTypes.Filter{{Name: aws.String(resourceTypeAutoScalingGroup), Values: []string{asgName}}},
}
p.MockASG().On("DescribeTags", mock.Anything, describeTagsInput).Return(&autoscaling.DescribeTagsOutput{}, nil)

sm := NewStackCollection(p, api.NewClusterConfig())
err := sm.PropagateManagedNodeGroupTagsToASG(ngName, ngTags, []string{asgName}, errCh)
Expect(err).NotTo(HaveOccurred())
err = <-errCh
Expect(err).To(MatchError(ContainSubstring("maximum amount for asg")))
})
})

Context("UpdateStack", func() {
It("succeeds if no changes required", func() {
// Order of AWS SDK invocation
Expand Down
Loading

0 comments on commit 0284bc3

Please sign in to comment.